vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnManagerImpl.java

   1 /*
   2  * Copyright (c) 2015 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.util.concurrent.ListenableFuture;
  12 import java.math.BigInteger;
  13 import java.util.ArrayList;
  14 import java.util.Collection;
  15 import java.util.Collections;
  16 import java.util.List;
  17 import java.util.concurrent.ExecutionException;
  18 import java.util.concurrent.Future;
  19 import javax.annotation.PostConstruct;
  20 import javax.inject.Inject;
  21 import javax.inject.Singleton;
  22 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  23 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  24 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  25 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  26 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  27 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  28 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  29 import org.opendaylight.genius.mdsalutil.FlowEntity;
  30 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  31 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  32 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  33 import org.opendaylight.genius.mdsalutil.UpgradeState;
  34 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  35 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  36 import org.opendaylight.genius.mdsalutil.nxmatches.NxMatchRegister;
  37 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  38 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  39 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput;
  40 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput.ArpReponderInputBuilder;
  41 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderUtil;
  42 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  43 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  44 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  45 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  46 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  47 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  48 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  49 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  50 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  51 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  52 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  53 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
  54 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
  55 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  56 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  70 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  71 import org.opendaylight.yangtools.yang.common.RpcError;
  72 import org.opendaylight.yangtools.yang.common.RpcResult;
  73 import org.slf4j.Logger;
  74 import org.slf4j.LoggerFactory;
  75
  76 @Singleton
  77 public class VpnManagerImpl implements IVpnManager {
  78
  79     private static final Logger LOG = LoggerFactory.getLogger(VpnManagerImpl.class);
  80     private final DataBroker dataBroker;
  81     private final ManagedNewTransactionRunner txRunner;
  82     private final IdManagerService idManager;
  83     private final IMdsalApiManager mdsalManager;
  84     private final IElanService elanService;
  85     private final IInterfaceManager interfaceManager;
  86     private final VpnSubnetRouteHandler vpnSubnetRouteHandler;
  87     private final OdlInterfaceRpcService ifaceMgrRpcService;
  88     private final IVpnLinkService ivpnLinkService;
  89     private final IFibManager fibManager;
  90     private final IBgpManager bgpManager;
  91     private final InterVpnLinkCache interVpnLinkCache;
  92     private final DataTreeEventCallbackRegistrar eventCallbacks;
  93     private final UpgradeState upgradeState;
  94
  95     @Inject
  96     public VpnManagerImpl(final DataBroker dataBroker,
  97                           final IdManagerService idManagerService,
  98                           final IMdsalApiManager mdsalManager,
  99                           final IElanService elanService,
 100                           final IInterfaceManager interfaceManager,
 101                           final VpnSubnetRouteHandler vpnSubnetRouteHandler,
 102                           final OdlInterfaceRpcService ifaceMgrRpcService,
 103                           final IVpnLinkService ivpnLinkService,
 104                           final IFibManager fibManager,
 105                           final IBgpManager bgpManager,
 106                           final InterVpnLinkCache interVpnLinkCache,
 107                           final DataTreeEventCallbackRegistrar dataTreeEventCallbackRegistrar,
 108                           final UpgradeState upgradeState) {
 109         this.dataBroker = dataBroker;
 110         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 111         this.idManager = idManagerService;
 112         this.mdsalManager = mdsalManager;
 113         this.elanService = elanService;
 114         this.interfaceManager = interfaceManager;
 115         this.vpnSubnetRouteHandler = vpnSubnetRouteHandler;
 116         this.ifaceMgrRpcService = ifaceMgrRpcService;
 117         this.ivpnLinkService = ivpnLinkService;
 118         this.fibManager = fibManager;
 119         this.bgpManager = bgpManager;
 120         this.interVpnLinkCache = interVpnLinkCache;
 121         this.eventCallbacks = dataTreeEventCallbackRegistrar;
 122         this.upgradeState = upgradeState;
 123     }
 124
 125     @PostConstruct
 126     public void start() {
 127         LOG.info("{} start", getClass().getSimpleName());
 128         createIdPool();
 129     }
 130
 131     private void createIdPool() {
 132         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 133             .setPoolName(VpnConstants.VPN_IDPOOL_NAME)
 134             .setLow(VpnConstants.VPN_IDPOOL_LOW)
 135             .setHigh(VpnConstants.VPN_IDPOOL_HIGH)
 136             .build();
 137         try {
 138             Future<RpcResult<Void>> result = idManager.createIdPool(createPool);
 139             if (result != null && result.get().isSuccessful()) {
 140                 LOG.info("Created IdPool for VPN Service");
 141             }
 142         } catch (InterruptedException | ExecutionException e) {
 143             LOG.error("Failed to create idPool for VPN Service", e);
 144         }
 145
 146         // Now an IdPool for InterVpnLink endpoint's pseudo ports
 147         CreateIdPoolInput createPseudoLporTagPool =
 148             new CreateIdPoolInputBuilder().setPoolName(VpnConstants.PSEUDO_LPORT_TAG_ID_POOL_NAME)
 149                 .setLow(VpnConstants.LOWER_PSEUDO_LPORT_TAG)
 150                 .setHigh(VpnConstants.UPPER_PSEUDO_LPORT_TAG)
 151                 .build();
 152         try {
 153             Future<RpcResult<Void>> result = idManager.createIdPool(createPseudoLporTagPool);
 154             if (result.get().isSuccessful()) {
 155                 LOG.debug("Created IdPool for Pseudo Port tags");
 156             } else {
 157                 Collection<RpcError> errors = result.get().getErrors();
 158                 StringBuilder errMsg = new StringBuilder();
 159                 for (RpcError err : errors) {
 160                     errMsg.append(err.getMessage()).append("\n");
 161                 }
 162                 LOG.error("IdPool creation for PseudoPort tags failed. Reasons: {}", errMsg);
 163             }
 164         } catch (InterruptedException | ExecutionException e) {
 165             LOG.error("Failed to create idPool for Pseudo Port tags", e);
 166         }
 167     }
 168
 169     @Override
 170     public void addExtraRoute(String vpnName, String destination, String nextHop, String rd, String routerID,
 171         int label,RouteOrigin origin) {
 172         LOG.info("Adding extra route with destination {}, nextHop {}, label{} and origin {}",
 173             destination, nextHop, label, origin);
 174         VpnInstanceOpDataEntry vpnOpEntry = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
 175         Boolean isVxlan = VpnUtil.isL3VpnOverVxLan(vpnOpEntry.getL3vni());
 176         VrfEntry.EncapType encapType = VpnUtil.getEncapType(isVxlan);
 177         addExtraRoute(vpnName, destination, nextHop, rd, routerID, vpnOpEntry.getL3vni(),
 178                 origin,/*intfName*/ null, null /*Adjacency*/, encapType, null);
 179     }
 180
 181     @Override
 182     public void addExtraRoute(String vpnName, String destination, String nextHop, String rd, String routerID,
 183             Long l3vni, RouteOrigin origin, String intfName, Adjacency operationalAdj,
 184             VrfEntry.EncapType encapType, WriteTransaction writeConfigTxn) {
 185
 186         Boolean writeConfigTxnPresent = true;
 187         if (writeConfigTxn == null) {
 188             writeConfigTxnPresent = false;
 189             writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 190         }
 191
 192         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
 193         VpnUtil.syncUpdate(
 194                 dataBroker,
 195                 LogicalDatastoreType.OPERATIONAL,
 196                 VpnExtraRouteHelper.getVpnToExtrarouteVrfIdIdentifier(vpnName, rd != null ? rd : routerID,
 197                         destination),
 198                 VpnUtil.getVpnToExtraroute(destination, Collections.singletonList(nextHop)));
 199
 200         BigInteger dpnId = null;
 201         if (intfName != null && !intfName.isEmpty()) {
 202             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
 203             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 204             if (nextHopIp == null || nextHopIp.isEmpty()) {
 205                 LOG.error("addExtraRoute: NextHop for interface {} is null / empty."
 206                         + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
 207                         dpnId);
 208                 return;
 209             }
 210             nextHop = nextHopIp;
 211         }
 212
 213         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 214
 215         // TODO: This is a limitation to be stated in docs. When configuring static route to go to
 216         // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
 217         // first place.
 218         Optional<InterVpnLinkDataComposite> optVpnLink = interVpnLinkCache.getInterVpnLinkByEndpoint(nextHop);
 219         if (optVpnLink.isPresent() && optVpnLink.get().isActive()) {
 220             InterVpnLinkDataComposite interVpnLink = optVpnLink.get();
 221             // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
 222             // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
 223             // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
 224             // This is like leaking one of the Vpn2 routes towards Vpn1
 225             String srcVpnUuid = interVpnLink.getVpnNameByIpAddress(nextHop);
 226             String dstVpnUuid = interVpnLink.getOtherVpnNameByIpAddress(nextHop);
 227             String dstVpnRd = VpnUtil.getVpnRd(dataBroker, dstVpnUuid);
 228             long newLabel = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 229                     VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
 230             if (newLabel == 0) {
 231                 LOG.error("addExtraRoute: Unable to fetch label from Id Manager. Bailing out of adding intervpnlink"
 232                         + " route for destination {}", destination);
 233                 return;
 234             }
 235             ivpnLinkService.leakRoute(interVpnLink, srcVpnUuid, dstVpnUuid, destination, newLabel, RouteOrigin.STATIC);
 236         } else {
 237             Optional<Routes> optVpnExtraRoutes = VpnExtraRouteHelper
 238                     .getVpnExtraroutes(dataBroker, vpnName, rd != null ? rd : routerID, destination);
 239             if (optVpnExtraRoutes.isPresent()) {
 240                 List<String> nhList = optVpnExtraRoutes.get().getNexthopIpList();
 241                 if (nhList != null && nhList.size() > 1) {
 242                     // If nhList is greater than one for vpnextraroute, a call to populatefib doesn't update vrfentry.
 243                     fibManager.refreshVrfEntry(primaryRd, destination);
 244                 } else {
 245                     L3vpnInput input = new L3vpnInput().setNextHop(operationalAdj).setNextHopIp(nextHop).setL3vni(l3vni)
 246                             .setPrimaryRd(primaryRd).setVpnName(vpnName).setDpnId(dpnId)
 247                             .setEncapType(encapType).setRd(rd).setRouteOrigin(origin);
 248                     L3vpnRegistry.getRegisteredPopulator(encapType).populateFib(input, writeConfigTxn);
 249                 }
 250             }
 251         }
 252
 253         if (!writeConfigTxnPresent) {
 254             writeConfigTxn.submit();
 255         }
 256     }
 257
 258     @Override
 259     public void delExtraRoute(String vpnName, String destination, String nextHop, String rd, String routerID) {
 260         LOG.info("Deleting extra route with destination {} and nextHop {}", destination, nextHop);
 261         delExtraRoute(vpnName, destination, nextHop, rd, routerID, null, null);
 262     }
 263
 264     @Override
 265     public void delExtraRoute(String vpnName, String destination, String nextHop, String rd, String routerID,
 266             String intfName, WriteTransaction writeConfigTxn) {
 267         Boolean writeConfigTxnPresent = true;
 268         BigInteger dpnId = null;
 269         if (writeConfigTxn == null) {
 270             writeConfigTxnPresent = false;
 271             writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 272         }
 273         String tunnelIp = nextHop;
 274         if (intfName != null && !intfName.isEmpty()) {
 275             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
 276             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 277             if (nextHopIp == null || nextHopIp.isEmpty()) {
 278                 LOG.error("delExtraRoute: NextHop for interface {} is null / empty."
 279                         + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
 280                         dpnId);
 281             }
 282             tunnelIp = nextHopIp;
 283         }
 284         if (rd != null) {
 285             String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
 286             removePrefixFromBGP(primaryRd, rd, vpnName, destination, nextHop, tunnelIp, dpnId, writeConfigTxn);
 287             LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName, rd);
 288         } else {
 289             // add FIB route directly
 290             fibManager.removeOrUpdateFibEntry(routerID, destination, tunnelIp, writeConfigTxn);
 291             LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName,
 292                     routerID);
 293         }
 294         if (!writeConfigTxnPresent) {
 295             writeConfigTxn.submit();
 296         }
 297     }
 298
 299  // TODO Clean up the exception handling
 300     @Override
 301     @SuppressWarnings("checkstyle:IllegalCatch")
 302     public void removePrefixFromBGP(String primaryRd, String rd, String vpnName, String prefix, String nextHop,
 303                                      String tunnelIp, BigInteger dpnId, WriteTransaction writeConfigTxn) {
 304         try {
 305             LOG.info("removePrefixFromBGP: VPN WITHDRAW: Removing Fib Entry rd {} prefix {} nexthop {}", rd, prefix,
 306                     nextHop);
 307             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 308             synchronized (vpnNamePrefixKey.intern()) {
 309                 Optional<Routes> optVpnExtraRoutes = VpnExtraRouteHelper
 310                         .getVpnExtraroutes(dataBroker, vpnName, rd, prefix);
 311                 if (optVpnExtraRoutes.isPresent()) {
 312                     List<String> nhList = optVpnExtraRoutes.get().getNexthopIpList();
 313                     if (nhList != null && nhList.size() > 1) {
 314                         // If nhList is more than 1, just update vpntoextraroute and prefixtointerface DS
 315                         // For other cases, remove the corresponding tep ip from fibentry and withdraw prefix
 316                         nhList.remove(nextHop);
 317                         VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 318                                 VpnExtraRouteHelper.getVpnToExtrarouteVrfIdIdentifier(vpnName, rd, prefix),
 319                                 VpnUtil.getVpnToExtraroute(prefix, nhList));
 320                         MDSALUtil.syncDelete(dataBroker,
 321                                 LogicalDatastoreType.CONFIGURATION, VpnExtraRouteHelper.getUsedRdsIdentifier(
 322                                 VpnUtil.getVpnId(dataBroker, vpnName), prefix, nextHop));
 323                         LOG.debug("removePrefixFromBGP: Removed vpn-to-extraroute with rd {} prefix {} nexthop {}",
 324                                 rd, prefix, nextHop);
 325                         fibManager.refreshVrfEntry(primaryRd, prefix);
 326                         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 327                         Optional<Prefixes> prefixToInterface = VpnUtil.getPrefixToInterface(dataBroker, vpnId, nextHop);
 328                         if (prefixToInterface.isPresent()) {
 329                             writeConfigTxn.delete(LogicalDatastoreType.OPERATIONAL,
 330                                     VpnUtil.getAdjacencyIdentifier(prefixToInterface.get().getVpnInterfaceName(),
 331                                             prefix));
 332                         }
 333                         LOG.info("VPN WITHDRAW: removePrefixFromBGP: Removed Fib Entry rd {} prefix {} nexthop {}",
 334                                 rd, prefix, tunnelIp);
 335                         return;
 336                     }
 337                 }
 338                 fibManager.removeOrUpdateFibEntry(primaryRd, prefix, tunnelIp, writeConfigTxn);
 339                 if (VpnUtil.isEligibleForBgp(rd, vpnName, dpnId, null /*networkName*/)) {
 340                     // TODO: Might be needed to include nextHop here
 341                     bgpManager.withdrawPrefix(rd, prefix);
 342                 }
 343             }
 344             LOG.info("removePrefixFromBGP: VPN WITHDRAW: Removed Fib Entry rd {} prefix {} nexthop {}", rd, prefix,
 345                     nextHop);
 346         } catch (RuntimeException e) {
 347             LOG.error("removePrefixFromBGP: Delete prefix {} rd {} nextHop {} failed", prefix, rd, nextHop);
 348         }
 349     }
 350
 351     @Override
 352     public boolean isVPNConfigured() {
 353         InstanceIdentifier<VpnInstances> vpnsIdentifier = InstanceIdentifier.builder(VpnInstances.class).build();
 354         Optional<VpnInstances> optionalVpns = TransactionUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION,
 355             vpnsIdentifier);
 356         if (!optionalVpns.isPresent()
 357             || optionalVpns.get().getVpnInstance() == null
 358             || optionalVpns.get().getVpnInstance().isEmpty()) {
 359             LOG.trace("isVPNConfigured: No VPNs configured.");
 360             return false;
 361         }
 362         LOG.trace("isVPNConfigured: VPNs are configured on the system.");
 363         return true;
 364     }
 365
 366     @Override
 367     public List<BigInteger> getDpnsOnVpn(String vpnInstanceName) {
 368         return VpnUtil.getDpnsOnVpn(dataBroker, vpnInstanceName);
 369     }
 370
 371     @Override
 372     public boolean existsVpn(String vpnName) {
 373         return VpnUtil.getVpnInstance(dataBroker, vpnName) != null;
 374     }
 375
 376     @Override
 377     public void addSubnetMacIntoVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 378             BigInteger dpnId, WriteTransaction tx) {
 379         setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
 380             (vpnId, dpId, subnetVpnId) -> addGwMac(srcMacAddress, tx, vpnId, dpId, subnetVpnId));
 381     }
 382
 383     @Override
 384     public void removeSubnetMacFromVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 385             BigInteger dpnId, WriteTransaction tx) {
 386         setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
 387             (vpnId, dpId, subnetVpnId) -> removeGwMac(srcMacAddress, tx, vpnId, dpId, subnetVpnId));
 388     }
 389
 390     @FunctionalInterface
 391     private interface VpnInstanceSubnetMacSetupMethod {
 392         void process(long vpnId, BigInteger dpId, long subnetVpnId);
 393     }
 394
 395     private void setupSubnetMacInVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 396             BigInteger dpnId, VpnInstanceSubnetMacSetupMethod consumer) {
 397         if (vpnName == null) {
 398             LOG.warn("Cannot setup subnet MAC {} on DPN {}, null vpnName", srcMacAddress, dpnId);
 399             return;
 400         }
 401
 402         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 403         long subnetVpnId = VpnUtil.getVpnId(dataBroker, subnetVpnName);
 404         if (dpnId.equals(BigInteger.ZERO)) {
 405             /* Apply the MAC on all DPNs in a VPN */
 406             for (BigInteger dpId : VpnUtil.getDpnsOnVpn(dataBroker, vpnName)) {
 407                 consumer.process(vpnId, dpId, subnetVpnId);
 408             }
 409         } else {
 410             consumer.process(vpnId, dpnId, subnetVpnId);
 411         }
 412     }
 413
 414     private void addGwMac(String srcMacAddress, WriteTransaction tx, long vpnId, BigInteger dpId, long subnetVpnId) {
 415         FlowEntity flowEntity = VpnUtil.buildL3vpnGatewayFlow(dpId, srcMacAddress, vpnId, subnetVpnId);
 416         mdsalManager.addFlowToTx(flowEntity, tx);
 417     }
 418
 419     private void removeGwMac(String srcMacAddress, WriteTransaction tx, long vpnId, BigInteger dpId, long subnetVpnId) {
 420         FlowEntity flowEntity = VpnUtil.buildL3vpnGatewayFlow(dpId, srcMacAddress, vpnId, subnetVpnId);
 421         mdsalManager.removeFlowToTx(flowEntity, tx);
 422     }
 423
 424     @Override
 425     public void addRouterGwMacFlow(String routerName, String routerGwMac, BigInteger dpnId, Uuid extNetworkId,
 426             String subnetVpnName, WriteTransaction writeTx) {
 427         setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId, writeTx,
 428             (vpnId, tx) -> addSubnetMacIntoVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, tx), "Installing");
 429     }
 430
 431     @Override
 432     public void removeRouterGwMacFlow(String routerName, String routerGwMac, BigInteger dpnId, Uuid extNetworkId,
 433             String subnetVpnName, WriteTransaction writeTx) {
 434         setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId, writeTx,
 435             (vpnId, tx) -> removeSubnetMacFromVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, tx), "Removing");
 436     }
 437
 438     @FunctionalInterface
 439     private interface RouterGwMacFlowSetupMethod {
 440         void process(String vpnId, WriteTransaction tx);
 441     }
 442
 443     private void setupRouterGwMacFlow(String routerName, String routerGwMac, BigInteger dpnId, Uuid extNetworkId,
 444             WriteTransaction writeTx, RouterGwMacFlowSetupMethod consumer, String operation) {
 445         if (routerGwMac == null) {
 446             LOG.warn("Failed to handle router GW flow in GW-MAC table. MAC address is missing for router-id {}",
 447                     routerName);
 448             return;
 449         }
 450
 451         if (dpnId == null || BigInteger.ZERO.equals(dpnId)) {
 452             LOG.info("setupRouterGwMacFlow: DPN id is missing for router-id {}",
 453                     routerName);
 454             return;
 455         }
 456
 457         Uuid vpnId = VpnUtil.getExternalNetworkVpnId(dataBroker, extNetworkId);
 458         if (vpnId == null) {
 459             LOG.warn("Network {} is not associated with VPN", extNetworkId.getValue());
 460             return;
 461         }
 462
 463         LOG.info("{} router GW MAC flow for router-id {} on switch {}", operation, routerName, dpnId);
 464         if (writeTx == null) {
 465             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 466                 tx -> consumer.process(vpnId.getValue(), tx)), LOG, "Commit transaction");
 467         } else {
 468             consumer.process(vpnId.getValue(), writeTx);
 469         }
 470     }
 471
 472     @Override
 473     public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 474             BigInteger dpnId, Uuid extNetworkId, WriteTransaction writeTx) {
 475
 476         if (dpnId == null || BigInteger.ZERO.equals(dpnId)) {
 477             LOG.warn("Failed to install arp responder flows for router {}. DPN id is missing.", id);
 478             return;
 479         }
 480
 481         String extInterfaceName = elanService.getExternalElanInterface(extNetworkId.getValue(), dpnId);
 482         if (extInterfaceName != null) {
 483             doAddArpResponderFlowsToExternalNetworkIps(
 484                     id, fixedIps, macAddress, dpnId, extNetworkId, writeTx, extInterfaceName);
 485             return;
 486         }
 487
 488         LOG.warn("Failed to install responder flows for {}. No external interface found for DPN id {}", id, dpnId);
 489
 490         if (!upgradeState.isUpgradeInProgress()) {
 491             return;
 492         }
 493
 494         // The following through the end of the function deals with an upgrade scenario where the neutron configuration
 495         // is restored before the OVS switches reconnect. In such a case, the elan-dpn-interfaces entries will be
 496         // missing from the operational data store. In order to mitigate this we use DataTreeEventCallbackRegistrar
 497         // to wait for the exact operational md-sal object we need to contain the external interface we need.
 498
 499         LOG.info("Upgrade in process, waiting for an external interface to appear on dpn {} for elan {}",
 500                 dpnId, extNetworkId.getValue());
 501
 502         InstanceIdentifier<DpnInterfaces> dpnInterfacesIid =
 503                             elanService.getElanDpnInterfaceOperationalDataPath(extNetworkId.getValue(), dpnId);
 504
 505         eventCallbacks.onAddOrUpdate(LogicalDatastoreType.OPERATIONAL, dpnInterfacesIid, (unused, alsoUnused) -> {
 506             LOG.info("Reattempting write of arp responder for external interfaces for external network {}",
 507                     extNetworkId);
 508             DpnInterfaces dpnInterfaces = elanService.getElanInterfaceInfoByElanDpn(extNetworkId.getValue(), dpnId);
 509             if (dpnInterfaces == null) {
 510                 LOG.error("Could not retrieve DpnInterfaces for {}, {}", extNetworkId.getValue(), dpnId);
 511                 return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 512             }
 513
 514             String extIfc = null;
 515             for (String dpnInterface : dpnInterfaces.getInterfaces()) {
 516                 if (interfaceManager.isExternalInterface(dpnInterface)) {
 517                     extIfc = dpnInterface;
 518                     break;
 519                 }
 520             }
 521
 522             if (extIfc == null) {
 523                 if (upgradeState.isUpgradeInProgress()) {
 524                     LOG.info("External interface not found yet in elan {} on dpn {}, keep waiting",
 525                             extNetworkId.getValue(), dpnInterfaces);
 526                     return DataTreeEventCallbackRegistrar.NextAction.CALL_AGAIN;
 527                 } else {
 528                     return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 529                 }
 530             }
 531
 532             final String extIfcFinal = extIfc;
 533             ListenableFuture<Void> listenableFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 534                 doAddArpResponderFlowsToExternalNetworkIps(
 535                         id, fixedIps, macAddress, dpnId, extNetworkId, tx, extIfcFinal);
 536             });
 537             ListenableFutures.addErrorLogging(listenableFuture, LOG,
 538                     "Error while configuring arp responder for ext. interface");
 539
 540             return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER; });
 541
 542     }
 543
 544     @Override
 545     public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 546             BigInteger dpnId, String extInterfaceName, int lportTag, WriteTransaction writeTx) {
 547         if (fixedIps == null || fixedIps.isEmpty()) {
 548             LOG.debug("No external IPs defined for {}", id);
 549             return;
 550         }
 551
 552         LOG.info("Installing ARP responder flows for {} fixed-ips {} on switch {}", id, fixedIps, dpnId);
 553
 554         if (writeTx == null) {
 555             ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 556                 tx -> {
 557                     for (String fixedIp : fixedIps) {
 558                         installArpResponderFlowsToExternalNetworkIp(macAddress, dpnId, extInterfaceName, lportTag,
 559                                 fixedIp);
 560                     }
 561                 });
 562             ListenableFutures.addErrorLogging(future, LOG, "Commit transaction");
 563         } else {
 564             for (String fixedIp : fixedIps) {
 565                 installArpResponderFlowsToExternalNetworkIp(macAddress, dpnId, extInterfaceName, lportTag,
 566                         fixedIp);
 567             }
 568         }
 569     }
 570
 571     private void doAddArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 572                             BigInteger dpnId, Uuid extNetworkId, WriteTransaction writeTx, String extInterfaceName) {
 573         Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
 574         if (extInterfaceState == null) {
 575             LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
 576                     id);
 577             return;
 578         }
 579
 580         Integer lportTag = extInterfaceState.getIfIndex();
 581         if (lportTag == null) {
 582             LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
 583             return;
 584         }
 585
 586         if (macAddress == null) {
 587
 588             LOG.debug("Failed to install arp responder flows for router-gateway-ip {} for router {}."
 589                     + "External Gw MacAddress is missing.", fixedIps,  id);
 590             return;
 591         }
 592
 593         addArpResponderFlowsToExternalNetworkIps(id, fixedIps, macAddress, dpnId, extInterfaceName, lportTag,
 594                 writeTx);
 595     }
 596
 597     @Override
 598     public void removeArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 599             BigInteger dpnId, Uuid extNetworkId) {
 600
 601         if (dpnId == null || BigInteger.ZERO.equals(dpnId)) {
 602             LOG.warn("Failed to remove arp responder flows for router {}. DPN id is missing.", id);
 603             return;
 604         }
 605
 606         String extInterfaceName = elanService.getExternalElanInterface(extNetworkId.getValue(), dpnId);
 607         if (extInterfaceName == null) {
 608             LOG.warn("Failed to remove responder flows for {}. No external interface found for DPN id {}", id, dpnId);
 609             return;
 610         }
 611
 612         Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
 613         if (extInterfaceState == null) {
 614             LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
 615                     id);
 616             return;
 617         }
 618
 619         Integer lportTag = extInterfaceState.getIfIndex();
 620         if (lportTag == null) {
 621             LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
 622             return;
 623         }
 624
 625         if (macAddress == null) {
 626
 627             LOG.debug("Failed to remove arp responder flows for router-gateway-ip {} for router {}."
 628                     + "External Gw MacAddress is missing.", fixedIps,  id);
 629             return;
 630         }
 631
 632         removeArpResponderFlowsToExternalNetworkIps(id, fixedIps, dpnId,
 633                 extInterfaceName, lportTag);
 634     }
 635
 636     @Override
 637     public void removeArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps,
 638             BigInteger dpnId, String extInterfaceName, int lportTag) {
 639         if (fixedIps == null || fixedIps.isEmpty()) {
 640             LOG.debug("No external IPs defined for {}", id);
 641             return;
 642         }
 643
 644         LOG.info("Removing ARP responder flows for {} fixed-ips {} on switch {}", id, fixedIps, dpnId);
 645
 646         for (String fixedIp : fixedIps) {
 647             removeArpResponderFlowsToExternalNetworkIp(dpnId, lportTag, fixedIp, extInterfaceName);
 648         }
 649     }
 650
 651     @Override
 652     public String getPrimaryRdFromVpnInstance(VpnInstance vpnInstance) {
 653         return VpnUtil.getPrimaryRd(vpnInstance);
 654     }
 655
 656     @Override
 657     public List<MatchInfoBase> getEgressMatchesForVpn(String vpnName) {
 658         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 659         if (vpnId == VpnConstants.INVALID_ID) {
 660             LOG.warn("No VPN id found for {}", vpnName);
 661             return Collections.emptyList();
 662         }
 663
 664         return Collections
 665                 .singletonList(new NxMatchRegister(VpnConstants.VPN_REG_ID, vpnId, MetaDataUtil.getVpnIdMaskForReg()));
 666     }
 667
 668     private void installArpResponderFlowsToExternalNetworkIp(String macAddress, BigInteger dpnId,
 669             String extInterfaceName, int lportTag, String fixedIp) {
 670         // reset the split-horizon bit to allow traffic to be sent back to the
 671         // provider port
 672         List<Instruction> instructions = new ArrayList<>();
 673         instructions.add(
 674                 new InstructionWriteMetadata(BigInteger.ZERO, MetaDataUtil.METADATA_MASK_SH_FLAG).buildInstruction(1));
 675         instructions.addAll(
 676                 ArpResponderUtil.getExtInterfaceInstructions(interfaceManager, extInterfaceName, fixedIp, macAddress));
 677         ArpReponderInputBuilder builder = new ArpReponderInputBuilder().setDpId(dpnId)
 678                 .setInterfaceName(extInterfaceName).setSpa(fixedIp).setSha(macAddress).setLportTag(lportTag);
 679         builder.setInstructions(instructions);
 680         elanService.addArpResponderFlow(builder.buildForInstallFlow());
 681     }
 682
 683     private void removeArpResponderFlowsToExternalNetworkIp(BigInteger dpnId, Integer lportTag, String fixedIp,
 684             String extInterfaceName) {
 685         ArpResponderInput arpInput = new ArpReponderInputBuilder().setDpId(dpnId).setInterfaceName(extInterfaceName)
 686                 .setSpa(fixedIp).setLportTag(lportTag).buildForRemoveFlow();
 687         elanService.removeArpResponderFlow(arpInput);
 688     }
 689
 690     @Override
 691     public void onSubnetAddedToVpn(Subnetmap subnetmap, boolean isBgpVpn, Long elanTag) {
 692         vpnSubnetRouteHandler.onSubnetAddedToVpn(subnetmap, isBgpVpn, elanTag);
 693     }
 694
 695     @Override
 696     public void onSubnetDeletedFromVpn(Subnetmap subnetmap, boolean isBgpVpn) {
 697         vpnSubnetRouteHandler.onSubnetDeletedFromVpn(subnetmap, isBgpVpn);
 698     }
 699
 700     @Override
 701     public VpnInstance getVpnInstance(DataBroker broker, String vpnInstanceName) {
 702         return VpnUtil.getVpnInstance(broker, vpnInstanceName);
 703     }
 704
 705     @Override
 706     public String getVpnRd(DataBroker broker, String vpnName) {
 707         return VpnUtil.getVpnRd(broker, vpnName);
 708     }
 709
 710     @Override
 711     public VpnPortipToPort getNeutronPortFromVpnPortFixedIp(DataBroker broker, String vpnName, String fixedIp) {
 712         return VpnUtil.getNeutronPortFromVpnPortFixedIp(broker, vpnName, fixedIp);
 713     }
 714 }