vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnManagerImpl.java

   1 /*
   2  * Copyright (c) 2015 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.util.concurrent.ListenableFuture;
  12 import java.math.BigInteger;
  13 import java.util.ArrayList;
  14 import java.util.Collection;
  15 import java.util.Collections;
  16 import java.util.List;
  17 import java.util.concurrent.ExecutionException;
  18 import java.util.concurrent.Future;
  19 import javax.annotation.PostConstruct;
  20 import javax.inject.Inject;
  21 import javax.inject.Singleton;
  22 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  23 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  24 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  25 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  26 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  27 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  28 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  29 import org.opendaylight.genius.mdsalutil.FlowEntity;
  30 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  31 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  32 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  33 import org.opendaylight.genius.mdsalutil.NwConstants;
  34 import org.opendaylight.genius.mdsalutil.UpgradeState;
  35 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  36 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  37 import org.opendaylight.genius.mdsalutil.nxmatches.NxMatchRegister;
  38 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  39 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  40 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput;
  41 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput.ArpReponderInputBuilder;
  42 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderUtil;
  43 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  44 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  45 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  46 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  47 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  48 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  49 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  50 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  51 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  52 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  53 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  54 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
  55 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
  56 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  57 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  71 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  72 import org.opendaylight.yangtools.yang.common.RpcError;
  73 import org.opendaylight.yangtools.yang.common.RpcResult;
  74 import org.slf4j.Logger;
  75 import org.slf4j.LoggerFactory;
  76
  77 @Singleton
  78 public class VpnManagerImpl implements IVpnManager {
  79
  80     private static final Logger LOG = LoggerFactory.getLogger(VpnManagerImpl.class);
  81     private final DataBroker dataBroker;
  82     private final ManagedNewTransactionRunner txRunner;
  83     private final IdManagerService idManager;
  84     private final IMdsalApiManager mdsalManager;
  85     private final IElanService elanService;
  86     private final IInterfaceManager interfaceManager;
  87     private final VpnSubnetRouteHandler vpnSubnetRouteHandler;
  88     private final OdlInterfaceRpcService ifaceMgrRpcService;
  89     private final IVpnLinkService ivpnLinkService;
  90     private final IFibManager fibManager;
  91     private final IBgpManager bgpManager;
  92     private final InterVpnLinkCache interVpnLinkCache;
  93     private final DataTreeEventCallbackRegistrar eventCallbacks;
  94     private final UpgradeState upgradeState;
  95
  96     @Inject
  97     public VpnManagerImpl(final DataBroker dataBroker,
  98                           final IdManagerService idManagerService,
  99                           final IMdsalApiManager mdsalManager,
 100                           final VpnFootprintService vpnFootprintService,
 101                           final IElanService elanService,
 102                           final IInterfaceManager interfaceManager,
 103                           final VpnSubnetRouteHandler vpnSubnetRouteHandler,
 104                           final OdlInterfaceRpcService ifaceMgrRpcService,
 105                           final IVpnLinkService ivpnLinkService,
 106                           final IFibManager fibManager,
 107                           final IBgpManager bgpManager,
 108                           final InterVpnLinkCache interVpnLinkCache,
 109                           final DataTreeEventCallbackRegistrar dataTreeEventCallbackRegistrar,
 110                           final UpgradeState upgradeState) {
 111         this.dataBroker = dataBroker;
 112         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 113         this.idManager = idManagerService;
 114         this.mdsalManager = mdsalManager;
 115         this.elanService = elanService;
 116         this.interfaceManager = interfaceManager;
 117         this.vpnSubnetRouteHandler = vpnSubnetRouteHandler;
 118         this.ifaceMgrRpcService = ifaceMgrRpcService;
 119         this.ivpnLinkService = ivpnLinkService;
 120         this.fibManager = fibManager;
 121         this.bgpManager = bgpManager;
 122         this.interVpnLinkCache = interVpnLinkCache;
 123         this.eventCallbacks = dataTreeEventCallbackRegistrar;
 124         this.upgradeState = upgradeState;
 125     }
 126
 127     @PostConstruct
 128     public void start() {
 129         LOG.info("{} start", getClass().getSimpleName());
 130         createIdPool();
 131     }
 132
 133     private void createIdPool() {
 134         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 135             .setPoolName(VpnConstants.VPN_IDPOOL_NAME)
 136             .setLow(VpnConstants.VPN_IDPOOL_LOW)
 137             .setHigh(VpnConstants.VPN_IDPOOL_HIGH)
 138             .build();
 139         try {
 140             Future<RpcResult<Void>> result = idManager.createIdPool(createPool);
 141             if (result != null && result.get().isSuccessful()) {
 142                 LOG.info("Created IdPool for VPN Service");
 143             }
 144         } catch (InterruptedException | ExecutionException e) {
 145             LOG.error("Failed to create idPool for VPN Service", e);
 146         }
 147
 148         // Now an IdPool for InterVpnLink endpoint's pseudo ports
 149         CreateIdPoolInput createPseudoLporTagPool =
 150             new CreateIdPoolInputBuilder().setPoolName(VpnConstants.PSEUDO_LPORT_TAG_ID_POOL_NAME)
 151                 .setLow(VpnConstants.LOWER_PSEUDO_LPORT_TAG)
 152                 .setHigh(VpnConstants.UPPER_PSEUDO_LPORT_TAG)
 153                 .build();
 154         try {
 155             Future<RpcResult<Void>> result = idManager.createIdPool(createPseudoLporTagPool);
 156             if (result.get().isSuccessful()) {
 157                 LOG.debug("Created IdPool for Pseudo Port tags");
 158             } else {
 159                 Collection<RpcError> errors = result.get().getErrors();
 160                 StringBuilder errMsg = new StringBuilder();
 161                 for (RpcError err : errors) {
 162                     errMsg.append(err.getMessage()).append("\n");
 163                 }
 164                 LOG.error("IdPool creation for PseudoPort tags failed. Reasons: {}", errMsg);
 165             }
 166         } catch (InterruptedException | ExecutionException e) {
 167             LOG.error("Failed to create idPool for Pseudo Port tags", e);
 168         }
 169     }
 170
 171     @Override
 172     public void addExtraRoute(String vpnName, String destination, String nextHop, String rd, String routerID,
 173         int label,RouteOrigin origin) {
 174         LOG.info("Adding extra route with destination {}, nextHop {}, label{} and origin {}",
 175             destination, nextHop, label, origin);
 176         VpnInstanceOpDataEntry vpnOpEntry = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
 177         Boolean isVxlan = VpnUtil.isL3VpnOverVxLan(vpnOpEntry.getL3vni());
 178         VrfEntry.EncapType encapType = VpnUtil.getEncapType(isVxlan);
 179         addExtraRoute(vpnName, destination, nextHop, rd, routerID, label, vpnOpEntry.getL3vni(),
 180                 origin,/*intfName*/ null, null /*Adjacency*/, encapType, null);
 181     }
 182
 183     @Override
 184     public void addExtraRoute(String vpnName, String destination, String nextHop, String rd, String routerID,
 185             int label, Long l3vni, RouteOrigin origin, String intfName, Adjacency operationalAdj,
 186             VrfEntry.EncapType encapType, WriteTransaction writeConfigTxn) {
 187
 188         Boolean writeConfigTxnPresent = true;
 189         if (writeConfigTxn == null) {
 190             writeConfigTxnPresent = false;
 191             writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 192         }
 193
 194         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
 195         VpnUtil.syncUpdate(
 196                 dataBroker,
 197                 LogicalDatastoreType.OPERATIONAL,
 198                 VpnExtraRouteHelper.getVpnToExtrarouteVrfIdIdentifier(vpnName, rd != null ? rd : routerID,
 199                         destination),
 200                 VpnUtil.getVpnToExtraroute(destination, Collections.singletonList(nextHop)));
 201
 202         BigInteger dpnId = null;
 203         if (intfName != null && !intfName.isEmpty()) {
 204             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
 205             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 206             if (nextHopIp == null || nextHopIp.isEmpty()) {
 207                 LOG.error("addExtraRoute: NextHop for interface {} is null / empty."
 208                         + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
 209                         dpnId);
 210                 return;
 211             }
 212             nextHop = nextHopIp;
 213         }
 214
 215         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 216
 217         // TODO: This is a limitation to be stated in docs. When configuring static route to go to
 218         // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
 219         // first place.
 220         Optional<InterVpnLinkDataComposite> optVpnLink = interVpnLinkCache.getInterVpnLinkByEndpoint(nextHop);
 221         if (optVpnLink.isPresent() && optVpnLink.get().isActive()) {
 222             InterVpnLinkDataComposite interVpnLink = optVpnLink.get();
 223             // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
 224             // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
 225             // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
 226             // This is like leaking one of the Vpn2 routes towards Vpn1
 227             String srcVpnUuid = interVpnLink.getVpnNameByIpAddress(nextHop);
 228             String dstVpnUuid = interVpnLink.getOtherVpnNameByIpAddress(nextHop);
 229             String dstVpnRd = VpnUtil.getVpnRd(dataBroker, dstVpnUuid);
 230             long newLabel = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 231                     VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
 232             if (newLabel == 0) {
 233                 LOG.error("addExtraRoute: Unable to fetch label from Id Manager. Bailing out of adding intervpnlink"
 234                         + " route for destination {}", destination);
 235                 return;
 236             }
 237             ivpnLinkService.leakRoute(interVpnLink, srcVpnUuid, dstVpnUuid, destination, newLabel, RouteOrigin.STATIC,
 238                     NwConstants.ADD_FLOW);
 239         } else {
 240             Optional<Routes> optVpnExtraRoutes = VpnExtraRouteHelper
 241                     .getVpnExtraroutes(dataBroker, vpnName, rd != null ? rd : routerID, destination);
 242             if (optVpnExtraRoutes.isPresent()) {
 243                 List<String> nhList = optVpnExtraRoutes.get().getNexthopIpList();
 244                 if (nhList != null && nhList.size() > 1) {
 245                     // If nhList is greater than one for vpnextraroute, a call to populatefib doesn't update vrfentry.
 246                     fibManager.refreshVrfEntry(primaryRd, destination);
 247                 } else {
 248                     L3vpnInput input = new L3vpnInput().setNextHop(operationalAdj).setNextHopIp(nextHop).setL3vni(l3vni)
 249                             .setPrimaryRd(primaryRd).setVpnName(vpnName).setDpnId(dpnId)
 250                             .setEncapType(encapType).setRd(rd).setRouteOrigin(origin);
 251                     L3vpnRegistry.getRegisteredPopulator(encapType).populateFib(input, writeConfigTxn, null);
 252                 }
 253             }
 254         }
 255
 256         if (!writeConfigTxnPresent) {
 257             writeConfigTxn.submit();
 258         }
 259     }
 260
 261     @Override
 262     public void delExtraRoute(String vpnName, String destination, String nextHop, String rd, String routerID) {
 263         LOG.info("Deleting extra route with destination {} and nextHop {}", destination, nextHop);
 264         delExtraRoute(vpnName, destination, nextHop, rd, routerID, null, null);
 265     }
 266
 267     @Override
 268     public void delExtraRoute(String vpnName, String destination, String nextHop, String rd, String routerID,
 269             String intfName, WriteTransaction writeConfigTxn) {
 270         Boolean writeConfigTxnPresent = true;
 271         BigInteger dpnId = null;
 272         if (writeConfigTxn == null) {
 273             writeConfigTxnPresent = false;
 274             writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 275         }
 276         String tunnelIp = nextHop;
 277         if (intfName != null && !intfName.isEmpty()) {
 278             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
 279             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 280             if (nextHopIp == null || nextHopIp.isEmpty()) {
 281                 LOG.error("delExtraRoute: NextHop for interface {} is null / empty."
 282                         + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
 283                         dpnId);
 284             }
 285             tunnelIp = nextHopIp;
 286         }
 287         if (rd != null) {
 288             String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
 289             removePrefixFromBGP(primaryRd, rd, vpnName, destination, nextHop, tunnelIp, dpnId, writeConfigTxn);
 290             LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName, rd);
 291         } else {
 292             // add FIB route directly
 293             fibManager.removeOrUpdateFibEntry(dataBroker, routerID, destination, tunnelIp, writeConfigTxn);
 294             LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName,
 295                     routerID);
 296         }
 297         if (!writeConfigTxnPresent) {
 298             writeConfigTxn.submit();
 299         }
 300     }
 301
 302  // TODO Clean up the exception handling
 303     @Override
 304     @SuppressWarnings("checkstyle:IllegalCatch")
 305     public void removePrefixFromBGP(String primaryRd, String rd, String vpnName, String prefix, String nextHop,
 306                                      String tunnelIp, BigInteger dpnId, WriteTransaction writeConfigTxn) {
 307         try {
 308             LOG.info("removePrefixFromBGP: VPN WITHDRAW: Removing Fib Entry rd {} prefix {} nexthop {}", rd, prefix,
 309                     nextHop);
 310             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 311             synchronized (vpnNamePrefixKey.intern()) {
 312                 Optional<Routes> optVpnExtraRoutes = VpnExtraRouteHelper
 313                         .getVpnExtraroutes(dataBroker, vpnName, rd, prefix);
 314                 if (optVpnExtraRoutes.isPresent()) {
 315                     List<String> nhList = optVpnExtraRoutes.get().getNexthopIpList();
 316                     if (nhList != null && nhList.size() > 1) {
 317                         // If nhList is more than 1, just update vpntoextraroute and prefixtointerface DS
 318                         // For other cases, remove the corresponding tep ip from fibentry and withdraw prefix
 319                         nhList.remove(nextHop);
 320                         VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 321                                 VpnExtraRouteHelper.getVpnToExtrarouteVrfIdIdentifier(vpnName, rd, prefix),
 322                                 VpnUtil.getVpnToExtraroute(prefix, nhList));
 323                         MDSALUtil.syncDelete(dataBroker,
 324                                 LogicalDatastoreType.CONFIGURATION, VpnExtraRouteHelper.getUsedRdsIdentifier(
 325                                 VpnUtil.getVpnId(dataBroker, vpnName), prefix, nextHop));
 326                         LOG.debug("removePrefixFromBGP: Removed vpn-to-extraroute with rd {} prefix {} nexthop {}",
 327                                 rd, prefix, nextHop);
 328                         fibManager.refreshVrfEntry(primaryRd, prefix);
 329                         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 330                         Optional<Prefixes> prefixToInterface = VpnUtil.getPrefixToInterface(dataBroker, vpnId, nextHop);
 331                         if (prefixToInterface.isPresent()) {
 332                             writeConfigTxn.delete(LogicalDatastoreType.OPERATIONAL,
 333                                     VpnUtil.getAdjacencyIdentifier(prefixToInterface.get().getVpnInterfaceName(),
 334                                             prefix));
 335                         }
 336                         LOG.info("VPN WITHDRAW: removePrefixFromBGP: Removed Fib Entry rd {} prefix {} nexthop {}",
 337                                 rd, prefix, tunnelIp);
 338                         return;
 339                     }
 340                 }
 341                 fibManager.removeOrUpdateFibEntry(dataBroker, primaryRd, prefix, tunnelIp, writeConfigTxn);
 342                 if (VpnUtil.isEligibleForBgp(rd, vpnName, dpnId, null /*networkName*/)) {
 343                     // TODO: Might be needed to include nextHop here
 344                     bgpManager.withdrawPrefix(rd, prefix);
 345                 }
 346             }
 347             LOG.info("removePrefixFromBGP: VPN WITHDRAW: Removed Fib Entry rd {} prefix {} nexthop {}", rd, prefix,
 348                     nextHop);
 349         } catch (RuntimeException e) {
 350             LOG.error("removePrefixFromBGP: Delete prefix {} rd {} nextHop {} failed", prefix, rd, nextHop);
 351         }
 352     }
 353
 354     @Override
 355     public boolean isVPNConfigured() {
 356         InstanceIdentifier<VpnInstances> vpnsIdentifier = InstanceIdentifier.builder(VpnInstances.class).build();
 357         Optional<VpnInstances> optionalVpns = TransactionUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION,
 358             vpnsIdentifier);
 359         if (!optionalVpns.isPresent()
 360             || optionalVpns.get().getVpnInstance() == null
 361             || optionalVpns.get().getVpnInstance().isEmpty()) {
 362             LOG.trace("isVPNConfigured: No VPNs configured.");
 363             return false;
 364         }
 365         LOG.trace("isVPNConfigured: VPNs are configured on the system.");
 366         return true;
 367     }
 368
 369     @Override
 370     public List<BigInteger> getDpnsOnVpn(String vpnInstanceName) {
 371         return VpnUtil.getDpnsOnVpn(dataBroker, vpnInstanceName);
 372     }
 373
 374     @Override
 375     public boolean existsVpn(String vpnName) {
 376         return VpnUtil.getVpnInstance(dataBroker, vpnName) != null;
 377     }
 378
 379     @Override
 380     public void addSubnetMacIntoVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 381             BigInteger dpnId, WriteTransaction tx) {
 382         setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
 383             (vpnId, dpId, subnetVpnId) -> addGwMac(srcMacAddress, tx, vpnId, dpId, subnetVpnId));
 384     }
 385
 386     @Override
 387     public void removeSubnetMacFromVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 388             BigInteger dpnId, WriteTransaction tx) {
 389         setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
 390             (vpnId, dpId, subnetVpnId) -> removeGwMac(srcMacAddress, tx, vpnId, dpId, subnetVpnId));
 391     }
 392
 393     @FunctionalInterface
 394     private interface VpnInstanceSubnetMacSetupMethod {
 395         void process(long vpnId, BigInteger dpId, long subnetVpnId);
 396     }
 397
 398     private void setupSubnetMacInVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 399             BigInteger dpnId, VpnInstanceSubnetMacSetupMethod consumer) {
 400         if (vpnName == null) {
 401             LOG.warn("Cannot setup subnet MAC {} on DPN {}, null vpnName", srcMacAddress, dpnId);
 402             return;
 403         }
 404
 405         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 406         long subnetVpnId = VpnUtil.getVpnId(dataBroker, subnetVpnName);
 407         if (dpnId.equals(BigInteger.ZERO)) {
 408             /* Apply the MAC on all DPNs in a VPN */
 409             for (BigInteger dpId : VpnUtil.getDpnsOnVpn(dataBroker, vpnName)) {
 410                 consumer.process(vpnId, dpId, subnetVpnId);
 411             }
 412         } else {
 413             consumer.process(vpnId, dpnId, subnetVpnId);
 414         }
 415     }
 416
 417     private void addGwMac(String srcMacAddress, WriteTransaction tx, long vpnId, BigInteger dpId, long subnetVpnId) {
 418         FlowEntity flowEntity = VpnUtil.buildL3vpnGatewayFlow(dpId, srcMacAddress, vpnId, subnetVpnId);
 419         mdsalManager.addFlowToTx(flowEntity, tx);
 420     }
 421
 422     private void removeGwMac(String srcMacAddress, WriteTransaction tx, long vpnId, BigInteger dpId, long subnetVpnId) {
 423         FlowEntity flowEntity = VpnUtil.buildL3vpnGatewayFlow(dpId, srcMacAddress, vpnId, subnetVpnId);
 424         mdsalManager.removeFlowToTx(flowEntity, tx);
 425     }
 426
 427     @Override
 428     public void addRouterGwMacFlow(String routerName, String routerGwMac, BigInteger dpnId, Uuid extNetworkId,
 429             String subnetVpnName, WriteTransaction writeTx) {
 430         setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId, writeTx,
 431             (vpnId, tx) -> addSubnetMacIntoVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, tx), "Installing");
 432     }
 433
 434     @Override
 435     public void removeRouterGwMacFlow(String routerName, String routerGwMac, BigInteger dpnId, Uuid extNetworkId,
 436             String subnetVpnName, WriteTransaction writeTx) {
 437         setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId, writeTx,
 438             (vpnId, tx) -> removeSubnetMacFromVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, tx), "Removing");
 439     }
 440
 441     @FunctionalInterface
 442     private interface RouterGwMacFlowSetupMethod {
 443         void process(String vpnId, WriteTransaction tx);
 444     }
 445
 446     private void setupRouterGwMacFlow(String routerName, String routerGwMac, BigInteger dpnId, Uuid extNetworkId,
 447             WriteTransaction writeTx, RouterGwMacFlowSetupMethod consumer, String operation) {
 448         if (routerGwMac == null) {
 449             LOG.warn("Failed to handle router GW flow in GW-MAC table. MAC address is missing for router-id {}",
 450                     routerName);
 451             return;
 452         }
 453
 454         if (dpnId == null || BigInteger.ZERO.equals(dpnId)) {
 455             LOG.info("setupRouterGwMacFlow: DPN id is missing for router-id {}",
 456                     routerName);
 457             return;
 458         }
 459
 460         Uuid vpnId = VpnUtil.getExternalNetworkVpnId(dataBroker, extNetworkId);
 461         if (vpnId == null) {
 462             LOG.warn("Network {} is not associated with VPN", extNetworkId.getValue());
 463             return;
 464         }
 465
 466         LOG.info("{} router GW MAC flow for router-id {} on switch {}", operation, routerName, dpnId);
 467         if (writeTx == null) {
 468             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 469                 tx -> consumer.process(vpnId.getValue(), tx)), LOG, "Commit transaction");
 470         } else {
 471             consumer.process(vpnId.getValue(), writeTx);
 472         }
 473     }
 474
 475     @Override
 476     public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 477             BigInteger dpnId, Uuid extNetworkId, WriteTransaction writeTx) {
 478
 479         if (dpnId == null || BigInteger.ZERO.equals(dpnId)) {
 480             LOG.warn("Failed to install arp responder flows for router {}. DPN id is missing.", id);
 481             return;
 482         }
 483
 484         String extInterfaceName = elanService.getExternalElanInterface(extNetworkId.getValue(), dpnId);
 485         if (extInterfaceName != null) {
 486             doAddArpResponderFlowsToExternalNetworkIps(
 487                     id, fixedIps, macAddress, dpnId, extNetworkId, writeTx, extInterfaceName);
 488             return;
 489         }
 490
 491         LOG.warn("Failed to install responder flows for {}. No external interface found for DPN id {}", id, dpnId);
 492
 493         if (!upgradeState.isUpgradeInProgress()) {
 494             return;
 495         }
 496
 497         // The following through the end of the function deals with an upgrade scenario where the neutron configuration
 498         // is restored before the OVS switches reconnect. In such a case, the elan-dpn-interfaces entries will be
 499         // missing from the operational data store. In order to mitigate this we use DataTreeEventCallbackRegistrar
 500         // to wait for the exact operational md-sal object we need to contain the external interface we need.
 501
 502         LOG.info("Upgrade in process, waiting for an external interface to appear on dpn {} for elan {}",
 503                 dpnId, extNetworkId.getValue());
 504
 505         InstanceIdentifier<DpnInterfaces> dpnInterfacesIid =
 506                             elanService.getElanDpnInterfaceOperationalDataPath(extNetworkId.getValue(), dpnId);
 507
 508         eventCallbacks.onAddOrUpdate(LogicalDatastoreType.OPERATIONAL, dpnInterfacesIid, (unused, alsoUnused) -> {
 509             LOG.info("Reattempting write of arp responder for external interfaces for external network {}",
 510                     extNetworkId);
 511             DpnInterfaces dpnInterfaces = elanService.getElanInterfaceInfoByElanDpn(extNetworkId.getValue(), dpnId);
 512             if (dpnInterfaces == null) {
 513                 LOG.error("Could not retrieve DpnInterfaces for {}, {}", extNetworkId.getValue(), dpnId);
 514                 return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 515             }
 516
 517             String extIfc = null;
 518             for (String dpnInterface : dpnInterfaces.getInterfaces()) {
 519                 if (interfaceManager.isExternalInterface(dpnInterface)) {
 520                     extIfc = dpnInterface;
 521                     break;
 522                 }
 523             }
 524
 525             if (extIfc == null) {
 526                 if (upgradeState.isUpgradeInProgress()) {
 527                     LOG.info("External interface not found yet in elan {} on dpn {}, keep waiting",
 528                             extNetworkId.getValue(), dpnInterfaces);
 529                     return DataTreeEventCallbackRegistrar.NextAction.CALL_AGAIN;
 530                 } else {
 531                     return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 532                 }
 533             }
 534
 535             final String extIfcFinal = extIfc;
 536             ListenableFuture<Void> listenableFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 537                 doAddArpResponderFlowsToExternalNetworkIps(
 538                         id, fixedIps, macAddress, dpnId, extNetworkId, tx, extIfcFinal);
 539             });
 540             ListenableFutures.addErrorLogging(listenableFuture, LOG,
 541                     "Error while configuring arp responder for ext. interface");
 542
 543             return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER; });
 544
 545     }
 546
 547     @Override
 548     public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 549                      BigInteger dpnId, long vpnId, String extInterfaceName, int lportTag, WriteTransaction writeTx) {
 550         if (fixedIps == null || fixedIps.isEmpty()) {
 551             LOG.debug("No external IPs defined for {}", id);
 552             return;
 553         }
 554
 555         LOG.info("Installing ARP responder flows for {} fixed-ips {} on switch {}", id, fixedIps, dpnId);
 556
 557         if (writeTx == null) {
 558             ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 559                 tx -> {
 560                     for (String fixedIp : fixedIps) {
 561                         installArpResponderFlowsToExternalNetworkIp(macAddress, dpnId, extInterfaceName, lportTag,
 562                                 vpnId,fixedIp, tx);
 563                     }
 564                 });
 565             ListenableFutures.addErrorLogging(future, LOG, "Commit transaction");
 566         } else {
 567             for (String fixedIp : fixedIps) {
 568                 installArpResponderFlowsToExternalNetworkIp(macAddress, dpnId, extInterfaceName, lportTag, vpnId,
 569                         fixedIp, writeTx);
 570             }
 571         }
 572     }
 573
 574     private void doAddArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 575                             BigInteger dpnId, Uuid extNetworkId, WriteTransaction writeTx, String extInterfaceName) {
 576         Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
 577         if (extInterfaceState == null) {
 578             LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
 579                     id);
 580             return;
 581         }
 582
 583         Integer lportTag = extInterfaceState.getIfIndex();
 584         if (lportTag == null) {
 585             LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
 586             return;
 587         }
 588
 589         if (macAddress == null) {
 590
 591             LOG.debug("Failed to install arp responder flows for router-gateway-ip {} for router {}."
 592                     + "External Gw MacAddress is missing.", fixedIps,  id);
 593             return;
 594         }
 595
 596         long vpnId = getVpnIdFromExtNetworkId(extNetworkId);
 597         addArpResponderFlowsToExternalNetworkIps(id, fixedIps, macAddress, dpnId, vpnId, extInterfaceName, lportTag,
 598                 writeTx);
 599     }
 600
 601     @Override
 602     public void removeArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 603             BigInteger dpnId, Uuid extNetworkId, WriteTransaction writeTx) {
 604
 605         if (dpnId == null || BigInteger.ZERO.equals(dpnId)) {
 606             LOG.warn("Failed to remove arp responder flows for router {}. DPN id is missing.", id);
 607             return;
 608         }
 609
 610         String extInterfaceName = elanService.getExternalElanInterface(extNetworkId.getValue(), dpnId);
 611         if (extInterfaceName == null) {
 612             LOG.warn("Failed to remove responder flows for {}. No external interface found for DPN id {}", id, dpnId);
 613             return;
 614         }
 615
 616         Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
 617         if (extInterfaceState == null) {
 618             LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
 619                     id);
 620             return;
 621         }
 622
 623         Integer lportTag = extInterfaceState.getIfIndex();
 624         if (lportTag == null) {
 625             LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
 626             return;
 627         }
 628
 629         if (macAddress == null) {
 630
 631             LOG.debug("Failed to remove arp responder flows for router-gateway-ip {} for router {}."
 632                     + "External Gw MacAddress is missing.", fixedIps,  id);
 633             return;
 634         }
 635
 636         removeArpResponderFlowsToExternalNetworkIps(id, fixedIps, dpnId,
 637                 extInterfaceName, lportTag, writeTx);
 638     }
 639
 640     @Override
 641     public void removeArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps,
 642             BigInteger dpnId, String extInterfaceName, int lportTag, WriteTransaction writeTx) {
 643         if (fixedIps == null || fixedIps.isEmpty()) {
 644             LOG.debug("No external IPs defined for {}", id);
 645             return;
 646         }
 647
 648         LOG.info("Removing ARP responder flows for {} fixed-ips {} on switch {}", id, fixedIps, dpnId);
 649
 650         for (String fixedIp : fixedIps) {
 651             removeArpResponderFlowsToExternalNetworkIp(dpnId, lportTag, fixedIp, extInterfaceName);
 652         }
 653     }
 654
 655     @Override
 656     public String getPrimaryRdFromVpnInstance(VpnInstance vpnInstance) {
 657         return VpnUtil.getPrimaryRd(vpnInstance);
 658     }
 659
 660     @Override
 661     public List<MatchInfoBase> getEgressMatchesForVpn(String vpnName) {
 662         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 663         if (vpnId == VpnConstants.INVALID_ID) {
 664             LOG.warn("No VPN id found for {}", vpnName);
 665             return Collections.emptyList();
 666         }
 667
 668         return Collections
 669                 .singletonList(new NxMatchRegister(VpnConstants.VPN_REG_ID, vpnId, MetaDataUtil.getVpnIdMaskForReg()));
 670     }
 671
 672     private void installArpResponderFlowsToExternalNetworkIp(String macAddress, BigInteger dpnId,
 673             String extInterfaceName, int lportTag, long vpnId, String fixedIp, WriteTransaction writeTx) {
 674         // reset the split-horizon bit to allow traffic to be sent back to the
 675         // provider port
 676         List<Instruction> instructions = new ArrayList<>();
 677         instructions.add(
 678                 new InstructionWriteMetadata(BigInteger.ZERO, MetaDataUtil.METADATA_MASK_SH_FLAG).buildInstruction(1));
 679         instructions.addAll(
 680                 ArpResponderUtil.getExtInterfaceInstructions(interfaceManager, extInterfaceName, fixedIp, macAddress));
 681         ArpReponderInputBuilder builder = new ArpReponderInputBuilder().setDpId(dpnId)
 682                 .setInterfaceName(extInterfaceName).setSpa(fixedIp).setSha(macAddress).setLportTag(lportTag);
 683         builder.setInstructions(instructions);
 684         elanService.addArpResponderFlow(builder.buildForInstallFlow());
 685     }
 686
 687     private void removeArpResponderFlowsToExternalNetworkIp(BigInteger dpnId, Integer lportTag, String fixedIp,
 688             String extInterfaceName) {
 689         ArpResponderInput arpInput = new ArpReponderInputBuilder().setDpId(dpnId).setInterfaceName(extInterfaceName)
 690                 .setSpa(fixedIp).setLportTag(lportTag).buildForRemoveFlow();
 691         elanService.removeArpResponderFlow(arpInput);
 692     }
 693
 694     private long getVpnIdFromExtNetworkId(Uuid extNetworkId) {
 695         Uuid vpnInstanceId = VpnUtil.getExternalNetworkVpnId(dataBroker, extNetworkId);
 696         if (vpnInstanceId == null) {
 697             LOG.debug("Network {} is not associated with VPN", extNetworkId.getValue());
 698             return VpnConstants.INVALID_ID;
 699         }
 700
 701         return VpnUtil.getVpnId(dataBroker, vpnInstanceId.getValue());
 702     }
 703
 704     @Override
 705     public void onSubnetAddedToVpn(Subnetmap subnetmap, boolean isBgpVpn, Long elanTag) {
 706         vpnSubnetRouteHandler.onSubnetAddedToVpn(subnetmap, isBgpVpn, elanTag);
 707     }
 708
 709     @Override
 710     public void onSubnetDeletedFromVpn(Subnetmap subnetmap, boolean isBgpVpn) {
 711         vpnSubnetRouteHandler.onSubnetDeletedFromVpn(subnetmap, isBgpVpn);
 712     }
 713
 714     @Override
 715     public VpnInstance getVpnInstance(DataBroker broker, String vpnInstanceName) {
 716         return VpnUtil.getVpnInstance(broker, vpnInstanceName);
 717     }
 718
 719     @Override
 720     public String getVpnRd(DataBroker broker, String vpnName) {
 721         return VpnUtil.getVpnRd(broker, vpnName);
 722     }
 723
 724     @Override
 725     public VpnPortipToPort getNeutronPortFromVpnPortFixedIp(DataBroker broker, String vpnName, String fixedIp) {
 726         return VpnUtil.getNeutronPortFromVpnPortFixedIp(broker, vpnName, fixedIp);
 727     }
 728 }