elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14 import static org.opendaylight.infrautils.utils.concurrent.LoggingFutures.addErrorLogging;
  15 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  16
  17 import com.google.common.base.Optional;
  18 import com.google.common.base.Preconditions;
  19 import com.google.common.collect.Lists;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import java.math.BigInteger;
  22 import java.util.ArrayList;
  23 import java.util.Collections;
  24 import java.util.HashSet;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.Objects;
  28 import java.util.Queue;
  29 import java.util.Set;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentLinkedQueue;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.locks.ReentrantLock;
  34 import javax.annotation.PostConstruct;
  35 import javax.inject.Inject;
  36 import javax.inject.Singleton;
  37 import org.apache.commons.lang3.StringUtils;
  38 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  41 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  42 import org.opendaylight.genius.infra.Datastore.Configuration;
  43 import org.opendaylight.genius.infra.Datastore.Operational;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  46 import org.opendaylight.genius.infra.TransactionAdapter;
  47 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  48 import org.opendaylight.genius.infra.TypedWriteTransaction;
  49 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  50 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  51 import org.opendaylight.genius.itm.globals.ITMConstants;
  52 import org.opendaylight.genius.mdsalutil.FlowEntity;
  53 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  58 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  59 import org.opendaylight.genius.mdsalutil.NwConstants;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  65 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  66 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  67 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  68 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  71 import org.opendaylight.genius.utils.JvmGlobalLocks;
  72 import org.opendaylight.genius.utils.ServiceIndex;
  73 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  74 import org.opendaylight.infrautils.utils.concurrent.LoggingFutures;
  75 import org.opendaylight.infrautils.utils.concurrent.NamedSimpleReentrantLock.Acquired;
  76 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  77 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  78 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  79 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  80 import org.opendaylight.netvirt.elan.recovery.impl.ElanServiceRecoveryHandler;
  81 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  82 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  83 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  84 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  85 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  86 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  87 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  88 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  89 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  90 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  91 import org.opendaylight.serviceutils.srm.RecoverableListener;
  92 import org.opendaylight.serviceutils.srm.ServiceRecoveryRegistry;
  93 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  94 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  95 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupId;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.GroupKey;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 140 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 141 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 142 import org.opendaylight.yangtools.yang.common.Uint32;
 143 import org.opendaylight.yangtools.yang.common.Uint64;
 144 import org.slf4j.Logger;
 145 import org.slf4j.LoggerFactory;
 146
 147 /**
 148  * Class in charge of handling creations, modifications and removals of
 149  * ElanInterfaces.
 150  *
 151  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 152  */
 153 @Singleton
 154 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager>
 155         implements RecoverableListener {
 156     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 157     private static final Logger EVENT_LOGGER = LoggerFactory.getLogger("NetvirtEventLogger");
 158     public static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 159     private static final boolean SH_FLAG_SET = true;
 160     private static final boolean SH_FLAG_UNSET = false;
 161
 162     private final DataBroker broker;
 163     private final ManagedNewTransactionRunner txRunner;
 164     private final IMdsalApiManager mdsalManager;
 165     private final IInterfaceManager interfaceManager;
 166     private final IdManagerService idManager;
 167     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 168     private final INeutronVpnManager neutronVpnManager;
 169     private final ElanItmUtils elanItmUtils;
 170     private final ElanEtreeUtils elanEtreeUtils;
 171     private final ElanL2GatewayUtils elanL2GatewayUtils;
 172     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 173     private final ElanUtils elanUtils;
 174     private final JobCoordinator jobCoordinator;
 175     private final ElanInstanceCache elanInstanceCache;
 176     private final ElanInterfaceCache elanInterfaceCache;
 177     private final ElanGroupCache elanGroupCache;
 178
 179     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 180         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 181
 182     @Inject
 183     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 184                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 185                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 186                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 187                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 188                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 189                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 190                                 final ElanInstanceCache elanInstanceCache,
 191                                 final ElanInterfaceCache elanInterfaceCache,
 192                                 final ElanServiceRecoveryHandler elanServiceRecoveryHandler,
 193                                 ElanGroupCache elanGroupCache,
 194                                 final ServiceRecoveryRegistry serviceRecoveryRegistry) {
 195         super(ElanInterface.class, ElanInterfaceManager.class);
 196         this.broker = dataBroker;
 197         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 198         this.idManager = managerService;
 199         this.mdsalManager = mdsalApiManager;
 200         this.interfaceManager = interfaceManager;
 201         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 202         this.neutronVpnManager = neutronVpnManager;
 203         this.elanItmUtils = elanItmUtils;
 204         this.elanEtreeUtils = elanEtreeUtils;
 205         this.elanL2GatewayUtils = elanL2GatewayUtils;
 206         this.elanUtils = elanUtils;
 207         this.jobCoordinator = jobCoordinator;
 208         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 209         this.elanInstanceCache = elanInstanceCache;
 210         this.elanInterfaceCache = elanInterfaceCache;
 211         this.elanGroupCache = elanGroupCache;
 212         serviceRecoveryRegistry.addRecoverableListener(elanServiceRecoveryHandler.buildServiceRegistryKey(), this);
 213     }
 214
 215     @Override
 216     @PostConstruct
 217     public void init() {
 218         registerListener();
 219     }
 220
 221     @Override
 222     public void registerListener() {
 223         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 224     }
 225
 226     @Override
 227     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 228         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 229     }
 230
 231     @Override
 232     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 233         String interfaceName = del.getName();
 234         String elanInstanceName = del.getElanInstanceName();
 235         EVENT_LOGGER.debug("ELAN-Interface, REMOVE {} Instance {}", interfaceName, elanInstanceName);
 236         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 237         if (elanInterfaces != null && elanInterfaces.contains(del)) {
 238             elanInterfaces.remove(del);
 239             if (elanInterfaces.isEmpty()) {
 240                 unProcessedElanInterfaces.remove(elanInstanceName);
 241             }
 242         }
 243         ElanInstance elanInfo = elanInstanceCache.get(elanInstanceName).orNull();
 244         /*
 245          * Handling in case the elan instance is deleted.If the Elan instance is
 246          * deleted, there is no need to explicitly delete the elan interfaces
 247          */
 248         if (elanInfo == null) {
 249             return;
 250         }
 251         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 252         if (interfaceInfo == null && elanInfo.isExternal()) {
 253             // In deleting external network, the underlying ietf Inteface might have been removed
 254             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 255             // from Operational DS instead
 256             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 257         }
 258         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 259                 interfaceName, interfaceInfo, this);
 260         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 261     }
 262
 263     private static class RemoveElanInterfaceHolder {
 264         boolean isLastElanInterface = false;
 265         Uint64 dpId = null;
 266     }
 267
 268     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 269     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 270             InterfaceInfo interfaceInfo) {
 271         String elanName = elanInfo.getElanInstanceName();
 272         EVENT_LOGGER.debug("ELAN-InterfaceState, REMOVE {} Instance {}", interfaceName, elanName);
 273         Uint32 elanTag = elanInfo.getElanTag();
 274         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 275         List<ListenableFuture<Void>> futures = new ArrayList<>();
 276         RemoveElanInterfaceHolder holder = new RemoveElanInterfaceHolder();
 277         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 278             Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 279             if (elanState == null) {
 280                 return;
 281             }
 282             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 283                 List<String> elanInterfaces = elanState.getElanInterfaces();
 284                 if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 285                     holder.isLastElanInterface = true;
 286                 }
 287                 if (interfaceInfo != null) {
 288                     holder.dpId = interfaceInfo.getDpId();
 289                     DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, holder.dpId,
 290                         interfaceName, elanTag, interfaceTx);
 291                     /*
 292                      * If there are not elan ports, remove the unknown dmac, terminating
 293                      * service table flows, remote/local bc group
 294                      */
 295                     if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 296                         || dpnInterfaces.getInterfaces().isEmpty()) {
 297                         // No more Elan Interfaces in this DPN
 298                         EVENT_LOGGER.debug("ELAN-Flows, REMOVE {} Instance {}", interfaceName, elanName);
 299                         LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(),
 300                             holder.dpId);
 301                         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 302                             removeDefaultTermFlow(holder.dpId, elanInfo.getElanTag().toJava());
 303                         }
 304                         removeUnknownDmacFlow(holder.dpId, elanInfo, flowTx, elanInfo.getElanTag().toJava());
 305                         removeEtreeUnknownDmacFlow(holder.dpId, elanInfo, flowTx);
 306                         removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 307                         removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 308                         removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 309                         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 310                             if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 311                                 elanUtils.removeTerminatingServiceAction(holder.dpId,
 312                                     ElanUtils.getVxlanSegmentationId(elanInfo).intValue());
 313                             }
 314                             unsetExternalTunnelTable(holder.dpId, elanInfo, flowTx);
 315                         }
 316                     } else {
 317                         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, flowTx);
 318                     }
 319                 }
 320             }));
 321         }));
 322         futures.forEach(ElanUtils::waitForTransactionToComplete);
 323
 324         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 325                 interfaceName, elanInfo, interfaceInfo, this, holder.isLastElanInterface);
 326         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 327                 ElanConstants.JOB_MAX_RETRIES);
 328
 329         return futures;
 330     }
 331
 332     private void removeEtreeUnknownDmacFlow(Uint64 dpId, ElanInstance elanInfo,
 333             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 334             throws ExecutionException, InterruptedException {
 335         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag().toJava());
 336         if (etreeLeafTag != null) {
 337             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue().toJava();
 338             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 339         }
 340     }
 341
 342     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 343             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 344             throws ExecutionException, InterruptedException {
 345         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 346         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 347     }
 348
 349     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 350             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 351             throws ExecutionException, InterruptedException {
 352         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 353         if (etreeInstance != null) {
 354             Uint64 dpnId = interfaceInfo.getDpId();
 355             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue().toJava());
 356             LOG.trace("deleted the localBroadCast Group:{}", groupId);
 357             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 358         }
 359     }
 360
 361     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 362             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 363             throws ExecutionException, InterruptedException {
 364         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 365         if (etreeInstance != null) {
 366             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue().toJava();
 367             Uint64 dpnId = interfaceInfo.getDpId();
 368             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 369             LOG.trace("deleting the remoteBroadCast group:{}", groupId);
 370             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 371         }
 372     }
 373
 374     private static Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName,
 375             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
 376         String elanName = elanInfo.getElanInstanceName();
 377         Elan elanState = ElanUtils.getElanByName(tx, elanName);
 378         if (elanState == null) {
 379             return elanState;
 380         }
 381         List<String> existingElanInterfaces = elanState.getElanInterfaces();
 382         List<String> elanInterfaces = new ArrayList<>();
 383         if (existingElanInterfaces != null) {
 384             elanInterfaces.addAll(existingElanInterfaces);
 385         }
 386         boolean isRemoved = elanInterfaces.remove(interfaceName);
 387         if (!isRemoved) {
 388             return elanState;
 389         }
 390
 391         if (elanInterfaces.isEmpty()) {
 392             tx.delete(ElanUtils.getElanInstanceOperationalDataPath(elanName));
 393             tx.delete(ElanUtils.getElanMacTableOperationalDataPath(elanName));
 394             tx.delete(ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 395             tx.delete(ElanUtils.getElanDpnOperationDataPath(elanName));
 396         } else {
 397             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 398                     .withKey(new ElanKey(elanName)).build();
 399             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanName), updateElanState);
 400         }
 401         return elanState;
 402     }
 403
 404     private void deleteElanInterfaceFromConfigDS(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
 405             throws ReadFailedException {
 406         // removing the ElanInterface from the config data_store if interface is
 407         // not present in Interface config DS
 408         if (interfaceManager.getInterfaceInfoFromConfigDataStore(TransactionAdapter.toReadWriteTransaction(tx),
 409             interfaceName) == null
 410                 && elanInterfaceCache.get(interfaceName).isPresent()) {
 411             tx.delete(ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 412         }
 413     }
 414
 415     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 416             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 417         String elanName = elanInfo.getElanInstanceName();
 418         EVENT_LOGGER.debug("ELAN-InterfaceEntries, REMOVE {} Instance {}", interfaceName, elanName);
 419         List<ListenableFuture<Void>> futures = new ArrayList<>();
 420         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 421             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 422                 InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 423                         .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 424                 Optional<ElanInterfaceMac> existingElanInterfaceMac = interfaceTx.read(elanInterfaceId).get();
 425                 LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 426                 if (interfaceInfo != null) {
 427                     if (existingElanInterfaceMac.isPresent()) {
 428                         List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 429                         if (existingMacEntries != null) {
 430                             List<PhysAddress> macAddresses = new ArrayList<>();
 431                             for (MacEntry macEntry : existingMacEntries) {
 432                                 PhysAddress macAddress = macEntry.getMacAddress();
 433                                 LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 434                                         macAddress.getValue(), interfaceName);
 435                                 Optional<MacEntry> macEntryOptional =
 436                                         elanUtils.getMacEntryForElanInstance(interfaceTx, elanName, macAddress);
 437                                 if (!isLastElanInterface && macEntryOptional.isPresent()) {
 438                                     interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 439                                 }
 440                                 EVENT_LOGGER.debug("ELAN-MacFlows, REMOVE {} Instance {} Mac {}",
 441                                         interfaceName, elanName, macAddress);
 442                                 elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 443                                 macAddresses.add(macAddress);
 444                             }
 445
 446                             // Removing all those MACs from External Devices belonging
 447                             // to this ELAN
 448                             if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 449                                 elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 450                             }
 451                         }
 452                     }
 453                     removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 454                     removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 455                 } else if (existingElanInterfaceMac.isPresent()) {
 456                     // Interface does not exist in ConfigDS, so lets remove everything
 457                     // about that interface related to Elan
 458                     List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 459                     if (macEntries != null) {
 460                         for (MacEntry macEntry : macEntries) {
 461                             PhysAddress macAddress = macEntry.getMacAddress();
 462                             if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 463                                 interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 464                             }
 465                         }
 466                     }
 467                 }
 468                 if (existingElanInterfaceMac.isPresent()) {
 469                     interfaceTx.delete(elanInterfaceId);
 470                 }
 471                 unbindService(interfaceName, flowTx);
 472                 deleteElanInterfaceFromConfigDS(interfaceName, flowTx);
 473             }));
 474         }));
 475         return futures;
 476     }
 477
 478     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, Uint64 dpId,
 479                                                                          String interfaceName, Uint32 elanTag,
 480                                                                          TypedReadWriteTransaction<Operational> tx)
 481             throws ExecutionException, InterruptedException {
 482         // FIXME: pass in and use ElanInstanceKey instead?
 483         final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanName);
 484         lock.lock();
 485         try {
 486             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 487             if (dpnInterfaces != null) {
 488                 List<String> interfaceLists = null;
 489                 if (dpnInterfaces.getInterfaces() != null) {
 490                     interfaceLists = new ArrayList<>(dpnInterfaces.getInterfaces());
 491                 }
 492                 if (interfaceLists != null) {
 493                     interfaceLists.remove(interfaceName);
 494                 }
 495
 496                 if (interfaceLists == null || interfaceLists.isEmpty()) {
 497                     deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 498                     deleteElanDpnInterface(elanName, dpId, tx);
 499                 } else {
 500                     dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 501                 }
 502             }
 503             return dpnInterfaces;
 504         } finally {
 505             lock.unlock();
 506         }
 507     }
 508
 509     private void deleteAllRemoteMacsInADpn(String elanName, Uint64 dpId, Uint32 elanTag) {
 510         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 511         addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 512             for (DpnInterfaces dpnInterface : dpnInterfaces) {
 513                 Uint64 currentDpId = dpnInterface.getDpId();
 514                 if (!currentDpId.equals(dpId) && dpnInterface.getInterfaces() != null) {
 515                     for (String elanInterface : dpnInterface.getInterfaces()) {
 516                         ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 517                         if (macs == null || macs.getMacEntry() == null) {
 518                             continue;
 519                         }
 520                         for (MacEntry mac : macs.getMacEntry()) {
 521                             removeTheMacFlowInTheDPN(dpId, elanTag, mac, confTx);
 522                             removeEtreeMacFlowInTheDPN(dpId, elanTag, mac, confTx);
 523                         }
 524                     }
 525                 }
 526             }
 527         }), LOG, "Error deleting remote MACs in DPN {}", dpId);
 528     }
 529
 530     private void removeEtreeMacFlowInTheDPN(Uint64 dpId, Uint32 elanTag, MacEntry mac,
 531             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 532         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag.longValue());
 533         if (etreeLeafTag != null) {
 534             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), mac, confTx);
 535         }
 536     }
 537
 538     private void removeTheMacFlowInTheDPN(Uint64 dpId, Uint32 elanTag, MacEntry mac,
 539             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 540         mdsalManager
 541                 .removeFlow(confTx, dpId,
 542                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 543                                 ElanUtils.getKnownDynamicmacFlowRef(elanTag, mac.getMacAddress().getValue())));
 544     }
 545
 546     /*
 547     * Possible Scenarios for update
 548     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 549         then all {1,2,3,4} should be removed
 550
 551         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 552         then all {1,2,3,4} should be added
 553
 554         c. if orig = {1,2,3,4} updated={2,3,4}
 555         then 1 should be removed
 556
 557         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 558         then we should just add 5
 559
 560         e. if orig = {1,2,3,4} updated={2,3,4,5}
 561         then 1 should be removed , 5 should be added
 562     * */
 563     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 564     @Override
 565     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 566         // updating the static-Mac Entries for the existing elanInterface
 567         String elanName = update.getElanInstanceName();
 568         String interfaceName = update.getName();
 569         LOG.info("Update static mac entries for elan interface {} in elan instance {}", interfaceName, elanName);
 570         EVENT_LOGGER.debug("ELAN-Interface, UPDATE {} Instance {}", original.getName(), elanName);
 571
 572         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 573         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 574         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 575         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 576
 577         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 578                 deletedEntry.getMacAddress()));
 579
 580         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 581         * Hence add the macentries for the same.*/
 582         for (StaticMacEntries staticMacEntry : updatedEntries) {
 583             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 584                     staticMacEntry.getMacAddress());
 585             addErrorLogging(ElanUtils.waitForTransactionToComplete(
 586                 txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 587                     Optional<MacEntry> existingMacEntry = tx.read(macEntryIdentifier).get();
 588                     if (existingMacEntry.isPresent()) {
 589                         LOG.debug("updating elan interface forwarding table for mac entry {} elan instance {}",
 590                                 existingMacEntry.get(), elanName);
 591                         elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 592                             elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 593                             tx);
 594                     } else {
 595                         LOG.info("Adding elan interface forwarding table for mac entry {} elan interface"
 596                                 + " {} elan instance {}.", staticMacEntry.getMacAddress(), interfaceName, elanName);
 597                         elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 598                             elanName, interfaceName, staticMacEntry, tx);
 599                     }
 600                 })), LOG, "Error in update: identifier={}, original={}, update={}", identifier, original, update);
 601         }
 602     }
 603
 604     @Override
 605     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 606         LOG.info("Init for ELAN interface Add {}", elanInterfaceAdded);
 607         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
 608             String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 609             String interfaceName = elanInterfaceAdded.getName();
 610             EVENT_LOGGER.debug("ELAN-Interface, ADD {} Instance {}", interfaceName, elanInstanceName);
 611             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 612             if (interfaceInfo == null) {
 613                 LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 614                 return;
 615             }
 616             ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orNull();
 617
 618             if (elanInstance == null) {
 619                 // Add the ElanInstance in the Configuration data-store
 620                 List<String> elanInterfaces = new ArrayList<>();
 621                 elanInterfaces.add(interfaceName);
 622                 elanInstance = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 623                     confTx -> ElanUtils.updateOperationalDataStore(idManager,
 624                         new ElanInstanceBuilder().setElanInstanceName(elanInstanceName).setDescription(
 625                             elanInterfaceAdded.getDescription()).build(), elanInterfaces, confTx, operTx)).get();
 626             }
 627
 628             Uint32 elanTag = elanInstance.getElanTag();
 629             // If elan tag is not updated, then put the elan interface into
 630             // unprocessed entry map and entry. Let entries
 631             // in this map get processed during ELAN update DCN.
 632             if (elanTag == null || elanTag.longValue() == 0L) {
 633                 ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 634                 if (elanInterfaces == null) {
 635                     elanInterfaces = new ConcurrentLinkedQueue<>();
 636                 }
 637                 if (!elanInterfaces.contains(elanInterfaceAdded)) {
 638                     elanInterfaces.add(elanInterfaceAdded);
 639                 }
 640                 LOG.error("ELAN tag for instance {} is not created. Adding it to unprocessed list."
 641                         + " Recreate the network if this message is seen multiple times", elanInstanceName);
 642                 unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 643                 return;
 644             }
 645             InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 646                     interfaceInfo, elanInstance, this);
 647             jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 648         }), LOG, "Error processing added ELAN interface");
 649     }
 650
 651     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) {
 652         LOG.trace("Handling unprocessed elan interfaces for elan instance {}", elanInstance.getElanInstanceName());
 653         List<ListenableFuture<Void>> futures = new ArrayList<>();
 654         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 655         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 656             return futures;
 657         }
 658         for (ElanInterface elanInterface : elanInterfaces) {
 659             String interfaceName = elanInterface.getName();
 660             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 661             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 662         }
 663         unProcessedElanInterfaces.remove(elanInstance.getElanInstanceName());
 664         LOG.info("Unprocessed elan interfaces for elan instance {} has been removed.",
 665                 elanInstance.getElanInstanceName());
 666         return futures;
 667     }
 668
 669     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 670             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 671         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 672                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 673         List<DpnInterfaces> dpnInterfaceLists = null;
 674         if (elanDpnInterfacesList != null) {
 675             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 676         }
 677         if (dpnInterfaceLists != null && !dpnInterfaceLists.isEmpty()) {
 678             Uint64 dstDpId = interfaceInfo.getDpId();
 679             processRemoteDmacFlowForInterface(dstDpId, elanInstance, dpnInterfaceLists, writeFlowGroupTx);
 680         }
 681     }
 682
 683     private void processRemoteDmacFlowForInterface(Uint64 dstDpId, ElanInstance elanInstance,
 684              List<DpnInterfaces> dpnInterfaceLists, TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 685         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 686             if (Objects.equals(dpnInterfaces.getDpId(), dstDpId)) {
 687                 continue;
 688             }
 689             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 690             if (remoteElanInterfaces == null || remoteElanInterfaces.isEmpty()) {
 691                 continue;
 692             }
 693             for (String remoteIf : remoteElanInterfaces) {
 694                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 695                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 696                 if (elanIfMac == null || remoteInterface == null) {
 697                     continue;
 698                 }
 699                 List<MacEntry> remoteMacEntries = elanIfMac.nonnullMacEntry();
 700                 for (MacEntry macEntry : remoteMacEntries) {
 701                     String macAddress = macEntry.getMacAddress().getValue();
 702                     LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 703                             dstDpId, elanInstance.getElanInstanceName());
 704                     elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 705                             remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 706                             elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 707                 }
 708             }
 709         }
 710     }
 711
 712     private static class AddElanInterfaceHolder {
 713         private DpnInterfaces dpnInterfaces = null;
 714         private boolean isFirstInterfaceInDpn = false;
 715         private Uint64 dpId;
 716     }
 717
 718     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 719     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 720             InterfaceInfo interfaceInfo, ElanInstance elanInstance) {
 721         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 722         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 723         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 724
 725         String interfaceName = elanInterface.getName();
 726         String elanInstanceName = elanInterface.getElanInstanceName();
 727         LOG.trace("Adding elan interface: interface name {} , instance name {}", interfaceName, elanInstanceName);
 728         EVENT_LOGGER.debug("ELAN-InterfaceState, ADD {} Instance {}", interfaceName, elanInstanceName);
 729
 730         List<ListenableFuture<Void>> futures = new ArrayList<>();
 731         AddElanInterfaceHolder holder = new AddElanInterfaceHolder();
 732         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 733             Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 734             if (elanInfo == null) {
 735                 LOG.trace("elanInfo is null for elan instance: {}", elanInstanceName);
 736                 List<String> elanInterfaces = new ArrayList<>();
 737                 elanInterfaces.add(interfaceName);
 738                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 739                     confTx -> ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, confTx,
 740                         operTx)));
 741             } else {
 742                 createElanStateList(elanInstanceName, interfaceName, operTx);
 743             }
 744             // Specific actions to the DPN where the ElanInterface has been added,
 745             // for example, programming the
 746             // External tunnel table if needed or adding the ElanInterface to the
 747             // DpnInterfaces in the operational DS.
 748             holder.dpId = interfaceInfo.getDpId();
 749             if (holder.dpId != null && !holder.dpId.equals(ElanConstants.INVALID_DPN)) {
 750                 // FIXME: use elanInstaince.key() instead?
 751                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanInstanceName);
 752                 lock.lock();
 753                 try {
 754                     InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 755                         .getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId);
 756                     Optional<DpnInterfaces> existingElanDpnInterfaces = operTx.read(elanDpnInterfaces).get();
 757                     if (ElanUtils.isVlan(elanInstance)) {
 758                         holder.isFirstInterfaceInDpn =  checkIfFirstInterface(interfaceName,
 759                             elanInstanceName, existingElanDpnInterfaces);
 760                     } else {
 761                         holder.isFirstInterfaceInDpn = !existingElanDpnInterfaces.isPresent();
 762                     }
 763                     if (holder.isFirstInterfaceInDpn) {
 764                         // ELAN's 1st ElanInterface added to this DPN
 765                         if (!existingElanDpnInterfaces.isPresent()) {
 766                             holder.dpnInterfaces =
 767                                 createElanInterfacesList(elanInstanceName, interfaceName, holder.dpId, operTx);
 768                         } else {
 769                             List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 770                             List<String> elanInterfaces =
 771                                 existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 772                             elanInterfaces.add(interfaceName);
 773                             holder.dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, holder.dpId,
 774                                 elanInterfaces, operTx);
 775                         }
 776                         LOG.debug("1st interface {} for elan {} is added to dpn {}",
 777                                 interfaceName, elanInstanceName, holder.dpId);
 778                         // The 1st ElanInterface in a DPN must program the Ext Tunnel
 779                         // table, but only if Elan has VNI
 780                         if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 781                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 782                                 confTx -> setExternalTunnelTable(holder.dpId, elanInstance, confTx)));
 783                         }
 784                         elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 785                             interfaceName);
 786                     } else {
 787                         List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 788                         List<String> elanInterfaces =
 789                             existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 790                         elanInterfaces.add(interfaceName);
 791                         if (elanInterfaces.size() == 1) { // 1st dpn interface
 792                             elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 793                                 interfaceName);
 794                         }
 795                         holder.dpnInterfaces =
 796                             updateElanDpnInterfacesList(elanInstanceName, holder.dpId, elanInterfaces, operTx);
 797                         LOG.debug("Interface {} for elan {} is added to dpn {}",
 798                                 interfaceName, elanInstanceName, holder.dpId);
 799                     }
 800                 } finally {
 801                     lock.unlock();
 802                 }
 803             }
 804
 805             // add code to install Local/Remote BC group, unknow DMAC entry,
 806             // terminating service table flow entry
 807             // call bindservice of interfacemanager to create ingress table flow
 808             // enty.
 809         }));
 810         futures.forEach(ElanUtils::waitForTransactionToComplete);
 811         futures.add(
 812             ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 813                 confTx -> installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, holder.dpnInterfaces,
 814                     holder.isFirstInterfaceInDpn, confTx))));
 815
 816         // add the vlan provider interface to remote BC group for the elan
 817         // for internal vlan networks
 818         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 819             if (interfaceManager.isExternalInterface(interfaceName)) {
 820                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 821                 handleExternalInterfaceEvent(elanInstance, holder.dpnInterfaces, holder.dpId);
 822             }
 823         }
 824         if (holder.isFirstInterfaceInDpn) {
 825             // ELAN's 1st ElanInterface added to this DPN
 826             LOG.debug("Adding dpn into operational dpn list {}", holder.dpId);
 827             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 828                 operTx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId),
 829                         holder.dpnInterfaces, CREATE_MISSING_PARENTS);
 830             }));
 831         } else {
 832             LOG.debug("Updated dpn into operational dpn list {}", holder.dpId);
 833         }
 834
 835         scheduleElanInterfaceWorkerAfterRemoteBcGroup(elanInstance, interfaceInfo, holder.dpnInterfaces,
 836                 holder.isFirstInterfaceInDpn, elanInterface);
 837         return futures;
 838     }
 839
 840     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 841     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 842             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn) {
 843         String elanInstanceName = elanInstance.getElanInstanceName();
 844         String interfaceName = elanInterface.getName();
 845         List<ListenableFuture<Void>> futures = new ArrayList<>();
 846         Uint64 dpId = interfaceInfo.getDpId();
 847         boolean isInterfaceOperational = isOperational(interfaceInfo);
 848         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 849             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 850                 installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 851                     isFirstInterfaceInDpn, confTx);
 852
 853                 List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 854                 List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 855
 856                 if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 857                     for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 858                         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 859                             staticMacEntry.getMacAddress());
 860                         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 861                             LogicalDatastoreType.OPERATIONAL, macId);
 862                         if (existingMacEntry.isPresent()) {
 863                             elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 864                                 elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 865                                 existingMacEntry.get(), operTx);
 866                         } else {
 867                             elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(elanInstanceName,
 868                                 interfaceName, staticMacEntry, operTx);
 869                         }
 870
 871                         if (isInterfaceOperational) {
 872                             // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 873                             // DPNs
 874                             String macAddress = staticMacEntry.getMacAddress().getValue();
 875                             LOG.info(
 876                                 "programming smac and dmacs for {} on source and other DPNs for elan {} and interface"
 877                                     + " {}",
 878                                 macAddress, elanInstanceName, interfaceName);
 879                             EVENT_LOGGER.debug("ELAN-MacFlows, ADD {} Instance {} Mac {}",
 880                                     interfaceName, elanInstanceName, macAddress);
 881                             elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 882                                 staticMacEntry.getMacAddress().getValue(), true, confTx);
 883                         }
 884                     }
 885
 886                     if (isInterfaceOperational) {
 887                         // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 888                         // on purpose.
 889                         for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 890                             staticMacAddresses.add(staticMacEntry.getMacAddress());
 891                         }
 892                         elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 893                             staticMacAddresses);
 894                     }
 895                 }
 896             }));
 897         }));
 898         futures.forEach(ElanUtils::waitForTransactionToComplete);
 899         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 900             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 901             try {
 902                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 903                 if (neutronPort != null) {
 904                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 905                 }
 906             } catch (IllegalArgumentException ex) {
 907                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 908             }
 909         }
 910         return futures;
 911     }
 912
 913     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 914             PhysAddress physAddress) {
 915         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 916         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 917         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 918                 LogicalDatastoreType.OPERATIONAL, macId);
 919
 920         if (!existingMacEntry.isPresent()) {
 921             return;
 922         }
 923
 924         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 925                 .withKey(new MacEntryKey(physAddress)).build();
 926         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 927                 elanInstanceCache.get(elanInstanceName).orNull(), interfaceInfo, macEntry);
 928     }
 929
 930     private boolean checkIfFirstInterface(String elanInterface, String elanInstanceName,
 931             Optional<DpnInterfaces> existingElanDpnInterfaces) {
 932         String routerPortUuid = ElanUtils.getRouterPordIdFromElanInstance(broker, elanInstanceName);
 933         if (!existingElanDpnInterfaces.isPresent()) {
 934             return true;
 935         }
 936         if (elanInterface.equals(elanInstanceName) || elanInterface.equals(routerPortUuid)) {
 937             return false;
 938         }
 939         DpnInterfaces dpnInterfaces = existingElanDpnInterfaces.get();
 940         int dummyInterfaceCount =  0;
 941         List<String> interfaces = dpnInterfaces.getInterfaces();
 942         if (interfaces == null) {
 943             return true;
 944         }
 945         if (interfaces.contains(routerPortUuid)) {
 946             dummyInterfaceCount++;
 947         }
 948         if (interfaces.contains(elanInstanceName)) {
 949             dummyInterfaceCount++;
 950         }
 951         return interfaces.size() == dummyInterfaceCount;
 952     }
 953
 954     private static InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName,
 955             PhysAddress physAddress) {
 956         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 957                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 958     }
 959
 960     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 961             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 962         if (!isOperational(interfaceInfo)) {
 963             LOG.warn("Interface {} is not operational", elanInterface.getName());
 964             EVENT_LOGGER.debug("ELAN-InterfaceEntries, ADD {} Instance {} Interface Status {}, returning",
 965                     elanInterface.getName(), elanInstance.getElanInstanceName(), interfaceInfo.getOpState());
 966             return;
 967         }
 968         Uint64 dpId = interfaceInfo.getDpId();
 969         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 970             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, confTx);
 971         }
 972         setupFilterEqualsTable(elanInstance, interfaceInfo, confTx);
 973         if (isFirstInterfaceInDpn) {
 974             // Terminating Service , UnknownDMAC Table.
 975             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 976             // for ELAN Instance is VxLAN
 977             if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 978                 setupTerminateServiceTable(elanInstance, dpId, confTx);
 979             }
 980             setupUnknownDMacTable(elanInstance, dpId, confTx);
 981             /*
 982              * Install remote DMAC flow. This is required since this DPN is
 983              * added later to the elan instance and remote DMACs of other
 984              * interfaces in this elan instance are not present in the current
 985              * dpn.
 986              */
 987             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 988                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 989                         elanInstance.getElanInstanceName());
 990                 programRemoteDmacFlow(elanInstance, interfaceInfo, confTx);
 991             }
 992         }
 993         // bind the Elan service to the Interface
 994         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), confTx);
 995     }
 996
 997     private void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 998             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 999         if (!isOperational(interfaceInfo)) {
1000             LOG.warn("Entries for interface on dpn {} not installed since interface {} is not operational",
1001                     dpnInterfaces.getDpId(), interfaceInfo.getInterfaceName());
1002             EVENT_LOGGER.debug("ELAN-1stInterfaceEntries, ADD {} Instance {} Interface Status {}, returning",
1003                     interfaceInfo.getInterfaceName(), elanInfo.getElanInstanceName(), interfaceInfo.getOpState());
1004             return;
1005         }
1006         EVENT_LOGGER.debug("ELAN-1stInterfaceEntries, ADD {} Instance {}",
1007                 interfaceInfo.getInterfaceName(), elanInfo.getElanInstanceName());
1008         // LocalBroadcast Group creation with elan-Interfaces
1009         LOG.info("Installing entries for interface {} on dpn {}", interfaceInfo.getInterfaceName(),
1010                 dpnInterfaces.getDpId());
1011         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, confTx);
1012         if (isFirstInterfaceInDpn) {
1013             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
1014             try {
1015                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1016             } catch (InterruptedException e1) {
1017                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
1018             }
1019         }
1020     }
1021
1022     public InstanceIdentifier<Group> getGroupIid(ElanInstance elanInfo, Uint64 dpnId) {
1023         long remoteBcGroupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag().toJava());
1024         return InstanceIdentifier.builder(Nodes.class)
1025                 .child(Node.class, new NodeKey(new org.opendaylight.yang.gen.v1.urn.opendaylight
1026                         .inventory.rev130819.NodeId("openflow:" + dpnId.toString())))
1027                 .augmentation(FlowCapableNode.class)
1028                 .child(Group.class, new GroupKey(new GroupId(remoteBcGroupId))).build();
1029     }
1030
1031     public void scheduleElanInterfaceWorkerAfterRemoteBcGroup(ElanInstance elanInfo,
1032                                                               InterfaceInfo interfaceInfo,
1033                                                               DpnInterfaces dpnInterfaces,
1034                                                               boolean isFirstInterfaceInDpn,
1035                                                               ElanInterface elanInterface) {
1036         if (!isOperational(interfaceInfo)) {
1037             LOG.debug("Interface {} is not operational", elanInterface.getName());
1038             return;
1039         }
1040         String elanInterfaceJobKey = ElanUtils.getElanInterfaceJobKey(interfaceInfo.getInterfaceName());
1041         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(elanInterfaceJobKey,
1042                 elanInterface, interfaceInfo, elanInfo, isFirstInterfaceInDpn, this);
1043         InstanceIdentifier<Group> groupInstanceId = getGroupIid(elanInfo, dpnInterfaces.getDpId());
1044         elanGroupCache.addJobToWaitList(groupInstanceId, () -> {
1045             jobCoordinator.enqueueJob(elanInterfaceJobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
1046         });
1047     }
1048
1049     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1050             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1051         int ifTag = interfaceInfo.getInterfaceTag();
1052         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1053                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 11, elanInfo.getElanInstanceName(),
1054                 0, 0, Uint64.valueOf(ElanConstants.COOKIE_ELAN_FILTER_EQUALS.toJava().add(BigInteger.valueOf(ifTag))),
1055                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
1056                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
1057
1058         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flow);
1059         LOG.trace("Filter equals table(55) flow entry created on dpn: {} for interface port: {}",
1060                 interfaceInfo.getDpId(), interfaceInfo.getPortName());
1061
1062         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1063                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 12, elanInfo.getElanInstanceName(), 0,
1064                 0, Uint64.valueOf(ElanConstants.COOKIE_ELAN_FILTER_EQUALS.toJava().add(BigInteger.valueOf(ifTag))),
1065                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
1066
1067         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flowEntry);
1068         LOG.trace("Filter equals table(55) drop flow entry created on dpn: {} for interface port: {}",
1069                 interfaceInfo.getDpId(), interfaceInfo.getPortName());
1070     }
1071
1072     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1073             TypedReadWriteTransaction<Configuration> flowTx) throws ExecutionException, InterruptedException {
1074         int ifTag = interfaceInfo.getInterfaceTag();
1075         Flow flow = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1076             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"));
1077
1078         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flow);
1079
1080         Flow flowEntity = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1081             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"));
1082
1083         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flowEntity);
1084     }
1085
1086     private static List<MatchInfo> buildMatchesForVni(Uint64 vni) {
1087         List<MatchInfo> mkMatches = new ArrayList<>();
1088         MatchInfo match = new MatchTunnelId(vni);
1089         mkMatches.add(match);
1090         return mkMatches;
1091     }
1092
1093     private static List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1094         List<InstructionInfo> mkInstructions = new ArrayList<>();
1095         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1096         return mkInstructions;
1097     }
1098
1099     private static List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1100         List<MatchInfo> mkMatches = new ArrayList<>();
1101         // Matching metadata
1102         mkMatches.add(new MatchMetadata(
1103                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1104         return mkMatches;
1105     }
1106
1107     /**
1108      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1109      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1110      *
1111      * @param elanTag
1112      *            elanTag to be written in metadata when flow is selected
1113      * @return the instructions ready to be installed in a flow
1114      */
1115     private static List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Uint32 elanTag) {
1116         List<InstructionInfo> mkInstructions = new ArrayList<>();
1117         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag.longValue()),
1118                 ElanHelper.getElanMetadataMask()));
1119         /* applicable for EXTERNAL_TUNNEL_TABLE only
1120         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1121         */
1122         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1123         return mkInstructions;
1124     }
1125
1126     // Install DMAC entry on dst DPN
1127     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1128             Uint64 dstDpId) {
1129         String interfaceName = interfaceInfo.getInterfaceName();
1130         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1131         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1132             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1133             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(
1134                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1135                     for (MacEntry macEntry : macEntries) {
1136                         String macAddress = macEntry.getMacAddress().getValue();
1137                         LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress,
1138                             interfaceName);
1139                         try (Acquired lock = ElanUtils.lockElanMacDPN(elanInfo.getElanTag().toJava(), macAddress,
1140                             interfaceInfo.getDpId())) {
1141                             LOG.info("Acquired lock for mac : {}, proceeding with remote dmac install operation",
1142                                 macAddress);
1143                             elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress, tx);
1144                         }
1145                     }
1146                 })));
1147         }
1148         return emptyList();
1149     }
1150
1151     private static void createDropBucket(List<Bucket> listBucket) {
1152         List<Action> actionsInfos = new ArrayList<>();
1153         actionsInfos.add(new ActionDrop().buildAction());
1154         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1155                 MDSALUtil.WATCH_GROUP);
1156         listBucket.add(dropBucket);
1157     }
1158
1159     private void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1160             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1161         if (!isOperational(interfaceInfo)) {
1162             EVENT_LOGGER.debug("ELAN-LBG, ADD {} Instance {} Interface Status {}, returning",
1163                     interfaceInfo.getInterfaceName(), elanInfo.getElanInstanceName(), interfaceInfo.getOpState());
1164             return;
1165         }
1166         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1167         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1168     }
1169
1170     private void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1171             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1172         List<Bucket> listBucket = new ArrayList<>();
1173         int bucketId = 0;
1174         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag().toJava());
1175
1176         List<String> interfaces = new ArrayList<>();
1177         if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1178             interfaces = newDpnInterface.getInterfaces();
1179         }
1180         for (String ifName : interfaces) {
1181             // In case if there is a InterfacePort in the cache which is not in
1182             // operational state, skip processing it
1183             InterfaceInfo ifInfo = interfaceManager
1184                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1185             if (!isOperational(ifInfo)) {
1186                 continue;
1187             }
1188
1189             if (!interfaceManager.isExternalInterface(ifName)) {
1190                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1191                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1192                 bucketId++;
1193             }
1194         }
1195
1196         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1197                 MDSALUtil.buildBucketLists(listBucket));
1198         LOG.trace("installing the localBroadCast Group:{}", group);
1199         mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1200     }
1201
1202     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1203             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1204         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1205         if (etreeInstance != null) {
1206             List<Bucket> listBucket = new ArrayList<>();
1207             int bucketId = 0;
1208
1209             List<String> interfaces = new ArrayList<>();
1210             if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1211                 interfaces = newDpnInterface.getInterfaces();
1212             }
1213             for (String ifName : interfaces) {
1214                 // In case if there is a InterfacePort in the cache which is not
1215                 // in
1216                 // operational state, skip processing it
1217                 InterfaceInfo ifInfo = interfaceManager
1218                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1219                 if (!isOperational(ifInfo)) {
1220                     continue;
1221                 }
1222
1223                 if (!interfaceManager.isExternalInterface(ifName)) {
1224                     // only add root interfaces
1225                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1226                 }
1227             }
1228
1229             if (listBucket.isEmpty()) { // No Buckets
1230                 createDropBucket(listBucket);
1231             }
1232
1233             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().toJava();
1234             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1235             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1236                     MDSALUtil.buildBucketLists(listBucket));
1237             LOG.trace("installing the localBroadCast Group:{}", group);
1238             mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1239         }
1240     }
1241
1242     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1243             InterfaceInfo ifInfo) {
1244         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1245         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1246             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1247                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1248             bucketId++;
1249         }
1250         return bucketId;
1251     }
1252
1253     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1254             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1255             throws ExecutionException, InterruptedException {
1256         Uint64 dpnId = interfaceInfo.getDpId();
1257         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag().toJava());
1258         LOG.trace("deleted the localBroadCast Group:{}", groupId);
1259         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1260     }
1261
1262     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1263             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1264             throws ExecutionException, InterruptedException {
1265         Uint64 dpnId = interfaceInfo.getDpId();
1266         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag().toJava());
1267         LOG.trace("deleting the remoteBroadCast group:{}", groupId);
1268         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1269     }
1270
1271     /**
1272      * Installs a flow in the External Tunnel table consisting in translating
1273      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1274      * elanTag that will be used later in the ELANs pipeline.
1275      * @param dpnId the dpn id
1276      */
1277     private void setExternalTunnelTable(Uint64 dpnId, ElanInstance elanInfo,
1278             TypedWriteTransaction<Configuration> confTx) {
1279         Uint32 elanTag = elanInfo.getElanTag();
1280         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1281                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag.longValue()), 5, // prio
1282                 elanInfo.getElanInstanceName(), // flowName
1283                 0, // idleTimeout
1284                 0, // hardTimeout
1285                 Uint64.valueOf(ITMConstants.COOKIE_ITM_EXTERNAL.toJava().add(BigInteger.valueOf(elanTag.longValue()))),
1286                 buildMatchesForVni(Uint64.valueOf(ElanUtils.getVxlanSegmentationId(elanInfo).longValue())),
1287                 getInstructionsIntOrExtTunnelTable(elanTag));
1288
1289         mdsalManager.addFlow(confTx, flowEntity);
1290     }
1291
1292     /**
1293      * Removes, from External Tunnel table, the flow that translates from VNI to
1294      * elanTag. Important: ensure this method is only called whenever there is
1295      * no other ElanInterface in the specified DPN
1296      *
1297      * @param dpnId DPN whose Ext Tunnel table is going to be modified
1298      */
1299     private void unsetExternalTunnelTable(Uint64 dpnId, ElanInstance elanInfo,
1300             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
1301         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1302         // last ElanInstance plus
1303         // adding a new one does (almost at the same time) are executed in that
1304         // exact order
1305
1306         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag().toJava());
1307         FlowEntity flowEntity = new FlowEntityBuilder()
1308             .setDpnId(dpnId)
1309             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1310             .setFlowId(flowId)
1311             .build();
1312         mdsalManager.removeFlow(confTx, flowEntity);
1313     }
1314
1315     public void setupTerminateServiceTable(ElanInstance elanInfo, Uint64 dpId,
1316             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1317         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1318         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1319     }
1320
1321     public void setupTerminateServiceTable(ElanInstance elanInfo, Uint64 dpId, Uint32 elanTag,
1322             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1323         List<? extends MatchInfoBase> listMatchInfoBase;
1324         List<InstructionInfo> instructionInfos;
1325         Uint32 serviceId;
1326         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1327             serviceId = elanTag;
1328             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId(elanTag);
1329             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag.longValue()));
1330         } else {
1331             serviceId = ElanUtils.getVxlanSegmentationId(elanInfo);
1332             listMatchInfoBase = buildMatchesForVni(Uint64.valueOf(serviceId));
1333             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1334         }
1335         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1336                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId.longValue()), 5,
1337                 String.format("%s:%s", "ITM Flow Entry ", elanTag.toString()), 0, 0,
1338                 Uint64.valueOf(ITMConstants.COOKIE_ITM.toJava().add(BigInteger.valueOf(elanTag.longValue()))),
1339                 listMatchInfoBase, instructionInfos);
1340         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1341         LOG.info("Installed internal tunnel table (36) flow entry on dpn: {} for elan: {}, tag: {}", dpId,
1342                 elanInfo.getElanInstanceName(), elanTag);
1343     }
1344
1345     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, Uint64 dpId,
1346             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1347         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1348         if (etreeInstance != null) {
1349             setupTerminateServiceTable(elanInfo, dpId,
1350                 etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1351         }
1352     }
1353
1354     public void setupUnknownDMacTable(ElanInstance elanInfo, Uint64 dpId,
1355             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1356         long elanTag = elanInfo.getElanTag().toJava();
1357         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1358         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1359         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1360     }
1361
1362     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, Uint64 dpId, long elanTag,
1363             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1364         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1365         if (etreeLeafTag != null) {
1366             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue().toJava();
1367             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1368             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1369         }
1370     }
1371
1372     private void installLocalUnknownFlow(ElanInstance elanInfo, Uint64 dpId, long elanTag,
1373             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1374         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1375                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1376                 5, elanInfo.getElanInstanceName(), 0, 0,
1377                 Uint64.valueOf(ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.toJava().add(BigInteger.valueOf(elanTag))),
1378                 getMatchesForElanTag(elanTag, /* SH flag */false),
1379                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1380
1381         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1382         LOG.trace("Installed unknown dmac table (53) flow entry on dpn: {} for elan: {}, tag: {}",
1383                 dpId, elanInfo.getElanInstanceName(), elanTag);
1384     }
1385
1386     private void installRemoteUnknownFlow(ElanInstance elanInfo, Uint64 dpId, long elanTag,
1387             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1388         // only if ELAN can connect to external network, perform the following
1389
1390         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1391             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1392                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1393                     5, elanInfo.getElanInstanceName(), 0, 0,
1394                     Uint64.valueOf(ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.toJava().add(BigInteger.valueOf(elanTag))),
1395                     getMatchesForElanTag(elanTag, /* SH flag */true),
1396                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1397             mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1398             LOG.trace("Installed unknown dmac table (53) flow entry on dpn: {} for elan connected to "
1399                     + "external network: {}, tag: {}", dpId, elanInfo.getElanInstanceName(), elanTag);
1400         }
1401     }
1402
1403
1404     private void removeUnknownDmacFlow(Uint64 dpId, ElanInstance elanInfo,
1405             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx, long elanTag)
1406             throws ExecutionException, InterruptedException {
1407         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1408                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1409         mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow);
1410
1411         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1412             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1413                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1414                     .build();
1415             mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow2);
1416         }
1417     }
1418
1419     private void removeDefaultTermFlow(Uint64 dpId, long elanTag) {
1420         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1421     }
1422
1423     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1424             TypedWriteTransaction<Configuration> tx) {
1425         if (isStandardElanService(elanInterface)) {
1426             bindElanService(elanInfo.getElanTag().toJava(), elanInfo.getElanInstanceName(),
1427                     elanInterface.getName(), lportTag, tx);
1428         } else { // Etree service
1429             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1430         }
1431     }
1432
1433     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1434             TypedWriteTransaction<Configuration> tx) {
1435         int instructionKey = 0;
1436         List<Instruction> instructions = new ArrayList<>();
1437         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1438                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1439
1440         List<Action> actions = new ArrayList<>();
1441         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1442                 lportTag).buildAction());
1443         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1444                 elanTag).buildAction());
1445         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1446
1447         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1448                 ++instructionKey));
1449
1450         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1451         BoundServices serviceInfo = ElanUtils.getBoundServices(
1452                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1453                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1454         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1455         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1456                 bindServiceId);
1457         if (!existingElanService.isPresent()) {
1458             tx.put(bindServiceId, serviceInfo, CREATE_MISSING_PARENTS);
1459             LOG.trace("Done binding elan service for elan: {} for interface: {}", elanInstanceName, interfaceName);
1460         }
1461     }
1462
1463     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1464             TypedWriteTransaction<Configuration> tx) {
1465         if (elanInterface.augmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1466             bindElanService(elanInfo.getElanTag().toJava(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1467                     lportTag, tx);
1468         } else {
1469             EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1470             if (etreeInstance == null) {
1471                 LOG.error("EtreeInterface {} is associated with a non EtreeInstance: {}",
1472                         elanInterface.getName(), elanInfo.getElanInstanceName());
1473             } else {
1474                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue().toJava(), elanInfo.getElanInstanceName(),
1475                         elanInterface.getName(), lportTag, tx);
1476             }
1477         }
1478     }
1479
1480     private static boolean isStandardElanService(ElanInterface elanInterface) {
1481         return elanInterface.augmentation(EtreeInterface.class) == null;
1482     }
1483
1484     protected void unbindService(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
1485             throws ExecutionException, InterruptedException {
1486         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1487         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1488         if (tx.read(bindServiceId).get().isPresent()) {
1489             tx.delete(bindServiceId);
1490         }
1491     }
1492
1493     private static String getFlowRef(long tableId, long elanTag) {
1494         return String.valueOf(tableId) + elanTag;
1495     }
1496
1497     private static String getFlowRef(long tableId, long elanTag, String flowName) {
1498         return new StringBuilder().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1499                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1500     }
1501
1502     private static String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1503         return String.valueOf(tableId) + elanTag + shFlag;
1504     }
1505
1506     private static List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1507         List<Action> listAction = new ArrayList<>();
1508         int actionKey = 0;
1509         listAction.add(
1510             new ActionSetFieldTunnelId(Uint64.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1511         actionKey++;
1512         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1513         return listAction;
1514     }
1515
1516     private static DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, Uint64 dpId,
1517             List<String> interfaceNames, TypedWriteTransaction<Operational> tx) {
1518         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1519                 .withKey(new DpnInterfacesKey(dpId)).build();
1520         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1521                 CREATE_MISSING_PARENTS);
1522         LOG.trace("Updated operational dpn interfaces for elan: {} with interfaces: {}", elanInstanceName,
1523                 interfaceNames);
1524         return dpnInterface;
1525     }
1526
1527     /**
1528      * Delete elan dpn interface from operational DS.
1529      *
1530      * @param elanInstanceName
1531      *            the elan instance name
1532      * @param dpId
1533      *            the dp id
1534      */
1535     private static void deleteElanDpnInterface(String elanInstanceName, Uint64 dpId,
1536             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1537         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1538                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1539         Optional<DpnInterfaces> dpnInterfaces = tx.read(dpnInterfacesId).get();
1540         if (dpnInterfaces.isPresent()) {
1541             tx.delete(dpnInterfacesId);
1542         }
1543     }
1544
1545     private static DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName,
1546             Uint64 dpId, TypedWriteTransaction<Operational> tx) {
1547         List<String> interfaceNames = new ArrayList<>();
1548         interfaceNames.add(interfaceName);
1549         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1550                 .withKey(new DpnInterfacesKey(dpId)).build();
1551         return dpnInterface;
1552     }
1553
1554     private static void createElanStateList(String elanInstanceName, String interfaceName,
1555             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1556         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1557         Optional<Elan> elanInterfaceLists = tx.read(elanInstance).get();
1558         // Adding new Elan Interface Port to the operational DataStore without
1559         // Static-Mac Entries..
1560         if (elanInterfaceLists.isPresent()) {
1561             List<String> interfaceLists = new ArrayList<>();
1562             interfaceLists.add(interfaceName);
1563             List<String> existingInterfaceLists = elanInterfaceLists.get().getElanInterfaces();
1564             if (existingInterfaceLists != null && !existingInterfaceLists.isEmpty()) {
1565                 existingInterfaceLists.forEach(iface -> interfaceLists.add(iface));
1566             }
1567
1568             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1569                     .withKey(new ElanKey(elanInstanceName)).build();
1570             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName), elanState, CREATE_MISSING_PARENTS);
1571             LOG.trace("Updated operational elan state for elan: {} with interfaces: {}", elanInstanceName,
1572                     interfaceLists);
1573         }
1574     }
1575
1576     private static boolean isOperational(InterfaceInfo interfaceInfo) {
1577         return interfaceInfo != null && interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1578     }
1579
1580     @SuppressWarnings("checkstyle:IllegalCatch")
1581     public void handleInternalTunnelStateEvent(Uint64 srcDpId, Uint64 dstDpId) {
1582         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1583         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1584                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1585         if (dpnInterfaceLists == null) {
1586             return;
1587         }
1588         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1589         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1590             int cnt = 0;
1591             String elanName = elanDpns.getElanInstanceName();
1592             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1593             if (elanInfo == null) {
1594                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1595                         + "skipping this ELAN for tunnel handling", elanName);
1596                 continue;
1597             }
1598             if (!isVxlanNetworkOrVxlanSegment(elanInfo)) {
1599                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1600                 continue;
1601             }
1602             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1603             if (dpnInterfaces == null) {
1604                 continue;
1605             }
1606             DpnInterfaces dstDpnIf = null;
1607             for (DpnInterfaces dpnIf : dpnInterfaces) {
1608                 Uint64 dpnIfDpId = dpnIf.getDpId();
1609                 if (Objects.equals(dpnIfDpId, srcDpId)) {
1610                     cnt++;
1611                 } else if (Objects.equals(dpnIfDpId, dstDpId)) {
1612                     cnt++;
1613                     dstDpnIf = dpnIf;
1614                 }
1615             }
1616             if (cnt == 2) {
1617                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1618                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1619                 jobCoordinator.enqueueJob(elanName, () -> {
1620                     // update Remote BC Group
1621                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1622                     try {
1623                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1624                             confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId,
1625                                 confTx)).get();
1626                     } catch (RuntimeException e) {
1627                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1628                     }
1629                     Set<String> interfaceLists = new HashSet<>();
1630                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1631                     for (String ifName : interfaceLists) {
1632                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1633                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1634                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1635                             if (isOperational(interfaceInfo)) {
1636                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1637                             }
1638                             return emptyList();
1639                         }, ElanConstants.JOB_MAX_RETRIES);
1640                     }
1641                     return emptyList();
1642                 }, ElanConstants.JOB_MAX_RETRIES);
1643             }
1644
1645         }
1646     }
1647
1648     /**
1649      * Handle external tunnel state event.
1650      *
1651      * @param externalTunnel
1652      *            the external tunnel
1653      * @param intrf
1654      *            the interface
1655      */
1656     void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1657         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1658             return;
1659         }
1660         // dpId/externalNodeId will be available either in source or destination
1661         // based on the tunnel end point
1662         Uint64 dpId = null;
1663         NodeId externalNodeId = null;
1664         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1665             dpId = Uint64.valueOf(externalTunnel.getSourceDevice());
1666             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1667         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1668             dpId = Uint64.valueOf(externalTunnel.getDestinationDevice());
1669             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1670         }
1671         if (dpId == null || externalNodeId == null) {
1672             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1673             return;
1674         }
1675
1676         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1677         if (dpnInterfaceLists == null) {
1678             return;
1679         }
1680         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1681         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1682             String elanName = elanDpns.getElanInstanceName();
1683             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1684
1685             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1686             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1687                     || dpnInterfaces.getInterfaces().isEmpty()) {
1688                 continue;
1689             }
1690             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1691
1692             final Uint64 finalDpId = dpId;
1693             LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1694                 confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, finalDpId, confTx)), LOG,
1695                 "Error setting up ELAN BGs");
1696             // install L2gwDevices local macs in dpn.
1697             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1698             // Install dpn macs on external device
1699             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1700                     externalNodeId);
1701         }
1702         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1703     }
1704
1705     /**
1706      * Installs dpn macs in external device. first it checks if the physical
1707      * locator towards this dpn tep is present or not if the physical locator is
1708      * present go ahead and add the ucast macs otherwise update the mcast mac
1709      * entry to include this dpn tep ip and schedule the job to put ucast macs
1710      * once the physical locator is programmed in device
1711      *
1712      * @param elanName
1713      *            the elan name
1714      * @param lstElanInterfaceNames
1715      *            the lst Elan interface names
1716      * @param dpnId
1717      *            the dpn id
1718      * @param externalNodeId
1719      *            the external node id
1720      */
1721     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, Uint64 dpnId,
1722             NodeId externalNodeId) {
1723         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1724                 externalNodeId.getValue());
1725         if (elanL2GwDevice == null) {
1726             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1727             return;
1728         }
1729         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1730         if (dpnTepIp == null) {
1731             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1732                     externalNodeId);
1733             return;
1734         }
1735
1736         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1737         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1738                 LogicalDatastoreType.OPERATIONAL);
1739         boolean phyLocAlreadyExists =
1740                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1741                 dpnTepIp);
1742         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1743                 phyLocAlreadyExists, dpnTepIp.stringValue(), elanName, externalNodeId.getValue());
1744         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1745
1746         if (phyLocAlreadyExists) {
1747             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1748             return;
1749         }
1750         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1751         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1752     }
1753
1754     /**
1755      * Validate external tunnel state event.
1756      *
1757      * @param externalTunnel
1758      *            the external tunnel
1759      * @param intrf
1760      *            the intrf
1761      * @return true, if successful
1762      */
1763     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1764         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1765             String srcDevice = externalTunnel.getDestinationDevice();
1766             String destDevice = externalTunnel.getSourceDevice();
1767             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1768                     LogicalDatastoreType.CONFIGURATION);
1769             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1770                     otherEndPointExtTunnel);
1771             if (otherEndPointExtTunnel != null) {
1772                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1773                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1774                 if (otherEndPointInterfaceOperational) {
1775                     return true;
1776                 } else {
1777                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1778                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1779                 }
1780             }
1781         }
1782         return false;
1783     }
1784
1785     private static List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1786         List<MatchInfo> mkMatches = new ArrayList<>();
1787         // Matching metadata
1788         mkMatches.add(
1789                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1790         mkMatches.add(new MatchTunnelId(Uint64.valueOf(lportTag)));
1791         return mkMatches;
1792     }
1793
1794     @Override
1795     protected ElanInterfaceManager getDataTreeChangeListener() {
1796         return this;
1797     }
1798
1799     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1800                                              Uint64 dpId) {
1801         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1802         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1803             confTx -> elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId,
1804                 confTx)), LOG, "Error setting up remote BC group for ELAN {}", elanInstance.getPhysicalNetworkName());
1805         try {
1806             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1807         } catch (InterruptedException e) {
1808             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1809         }
1810     }
1811 }