vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnManagerImpl.java

   1 /*
   2  * Copyright (c) 2015 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.util.concurrent.ListenableFuture;
  12 import java.math.BigInteger;
  13 import java.util.ArrayList;
  14 import java.util.Collection;
  15 import java.util.Collections;
  16 import java.util.List;
  17 import java.util.concurrent.ExecutionException;
  18 import java.util.concurrent.Future;
  19 import javax.annotation.PostConstruct;
  20 import javax.inject.Inject;
  21 import javax.inject.Singleton;
  22 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  23 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  24 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  25 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  26 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  27 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  28 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  29 import org.opendaylight.genius.mdsalutil.FlowEntity;
  30 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  31 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  32 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  33 import org.opendaylight.genius.mdsalutil.UpgradeState;
  34 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  35 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  36 import org.opendaylight.genius.mdsalutil.nxmatches.NxMatchRegister;
  37 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  38 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  39 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput;
  40 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput.ArpReponderInputBuilder;
  41 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderUtil;
  42 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  43 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  44 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  45 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  46 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  47 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  48 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  49 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  50 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  51 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  52 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  53 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
  54 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
  55 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  56 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutput;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  72 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  73 import org.opendaylight.yangtools.yang.common.RpcError;
  74 import org.opendaylight.yangtools.yang.common.RpcResult;
  75 import org.slf4j.Logger;
  76 import org.slf4j.LoggerFactory;
  77
  78 @Singleton
  79 public class VpnManagerImpl implements IVpnManager {
  80
  81     private static final Logger LOG = LoggerFactory.getLogger(VpnManagerImpl.class);
  82     private final DataBroker dataBroker;
  83     private final ManagedNewTransactionRunner txRunner;
  84     private final IdManagerService idManager;
  85     private final IMdsalApiManager mdsalManager;
  86     private final IElanService elanService;
  87     private final IInterfaceManager interfaceManager;
  88     private final VpnSubnetRouteHandler vpnSubnetRouteHandler;
  89     private final OdlInterfaceRpcService ifaceMgrRpcService;
  90     private final IVpnLinkService ivpnLinkService;
  91     private final IFibManager fibManager;
  92     private final IBgpManager bgpManager;
  93     private final InterVpnLinkCache interVpnLinkCache;
  94     private final DataTreeEventCallbackRegistrar eventCallbacks;
  95     private final UpgradeState upgradeState;
  96     private final ItmRpcService itmRpcService;
  97
  98     @Inject
  99     public VpnManagerImpl(final DataBroker dataBroker,
 100                           final IdManagerService idManagerService,
 101                           final IMdsalApiManager mdsalManager,
 102                           final IElanService elanService,
 103                           final IInterfaceManager interfaceManager,
 104                           final VpnSubnetRouteHandler vpnSubnetRouteHandler,
 105                           final OdlInterfaceRpcService ifaceMgrRpcService,
 106                           final IVpnLinkService ivpnLinkService,
 107                           final IFibManager fibManager,
 108                           final IBgpManager bgpManager,
 109                           final InterVpnLinkCache interVpnLinkCache,
 110                           final DataTreeEventCallbackRegistrar dataTreeEventCallbackRegistrar,
 111                           final UpgradeState upgradeState,
 112                           final ItmRpcService itmRpcService) {
 113         this.dataBroker = dataBroker;
 114         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 115         this.idManager = idManagerService;
 116         this.mdsalManager = mdsalManager;
 117         this.elanService = elanService;
 118         this.interfaceManager = interfaceManager;
 119         this.vpnSubnetRouteHandler = vpnSubnetRouteHandler;
 120         this.ifaceMgrRpcService = ifaceMgrRpcService;
 121         this.ivpnLinkService = ivpnLinkService;
 122         this.fibManager = fibManager;
 123         this.bgpManager = bgpManager;
 124         this.interVpnLinkCache = interVpnLinkCache;
 125         this.eventCallbacks = dataTreeEventCallbackRegistrar;
 126         this.upgradeState = upgradeState;
 127         this.itmRpcService = itmRpcService;
 128     }
 129
 130     @PostConstruct
 131     public void start() {
 132         LOG.info("{} start", getClass().getSimpleName());
 133         createIdPool();
 134     }
 135
 136     private void createIdPool() {
 137         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 138             .setPoolName(VpnConstants.VPN_IDPOOL_NAME)
 139             .setLow(VpnConstants.VPN_IDPOOL_LOW)
 140             .setHigh(VpnConstants.VPN_IDPOOL_HIGH)
 141             .build();
 142         try {
 143             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPool);
 144             if (result != null && result.get().isSuccessful()) {
 145                 LOG.info("Created IdPool for VPN Service");
 146             }
 147         } catch (InterruptedException | ExecutionException e) {
 148             LOG.error("Failed to create idPool for VPN Service", e);
 149         }
 150
 151         // Now an IdPool for InterVpnLink endpoint's pseudo ports
 152         CreateIdPoolInput createPseudoLporTagPool =
 153             new CreateIdPoolInputBuilder().setPoolName(VpnConstants.PSEUDO_LPORT_TAG_ID_POOL_NAME)
 154                 .setLow(VpnConstants.LOWER_PSEUDO_LPORT_TAG)
 155                 .setHigh(VpnConstants.UPPER_PSEUDO_LPORT_TAG)
 156                 .build();
 157         try {
 158             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPseudoLporTagPool);
 159             if (result.get().isSuccessful()) {
 160                 LOG.debug("Created IdPool for Pseudo Port tags");
 161             } else {
 162                 Collection<RpcError> errors = result.get().getErrors();
 163                 StringBuilder errMsg = new StringBuilder();
 164                 for (RpcError err : errors) {
 165                     errMsg.append(err.getMessage()).append("\n");
 166                 }
 167                 LOG.error("IdPool creation for PseudoPort tags failed. Reasons: {}", errMsg);
 168             }
 169         } catch (InterruptedException | ExecutionException e) {
 170             LOG.error("Failed to create idPool for Pseudo Port tags", e);
 171         }
 172     }
 173
 174     @Override
 175     public void addExtraRoute(String vpnName, String destination, String nextHop, String rd, String routerID,
 176         int label,RouteOrigin origin) {
 177         LOG.info("Adding extra route with destination {}, nextHop {}, label{} and origin {}",
 178             destination, nextHop, label, origin);
 179         VpnInstanceOpDataEntry vpnOpEntry = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
 180         Boolean isVxlan = VpnUtil.isL3VpnOverVxLan(vpnOpEntry.getL3vni());
 181         VrfEntry.EncapType encapType = VpnUtil.getEncapType(isVxlan);
 182         addExtraRoute(vpnName, destination, nextHop, rd, routerID, vpnOpEntry.getL3vni(),
 183                 origin,/*intfName*/ null, null /*Adjacency*/, encapType, null);
 184     }
 185
 186     @Override
 187     public void addExtraRoute(String vpnName, String destination, String nextHop, String rd, String routerID,
 188             Long l3vni, RouteOrigin origin, String intfName, Adjacency operationalAdj,
 189             VrfEntry.EncapType encapType, WriteTransaction writeConfigTxn) {
 190
 191         Boolean writeConfigTxnPresent = true;
 192         if (writeConfigTxn == null) {
 193             writeConfigTxnPresent = false;
 194             writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 195         }
 196
 197         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
 198         VpnUtil.syncUpdate(
 199                 dataBroker,
 200                 LogicalDatastoreType.OPERATIONAL,
 201                 VpnExtraRouteHelper.getVpnToExtrarouteVrfIdIdentifier(vpnName, rd != null ? rd : routerID,
 202                         destination),
 203                 VpnUtil.getVpnToExtraroute(destination, Collections.singletonList(nextHop)));
 204
 205         BigInteger dpnId = null;
 206         if (intfName != null && !intfName.isEmpty()) {
 207             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
 208             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 209             if (nextHopIp == null || nextHopIp.isEmpty()) {
 210                 LOG.error("addExtraRoute: NextHop for interface {} is null / empty."
 211                         + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
 212                         dpnId);
 213                 return;
 214             }
 215             nextHop = nextHopIp;
 216         }
 217
 218         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 219
 220         // TODO: This is a limitation to be stated in docs. When configuring static route to go to
 221         // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
 222         // first place.
 223         Optional<InterVpnLinkDataComposite> optVpnLink = interVpnLinkCache.getInterVpnLinkByEndpoint(nextHop);
 224         if (optVpnLink.isPresent() && optVpnLink.get().isActive()) {
 225             InterVpnLinkDataComposite interVpnLink = optVpnLink.get();
 226             // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
 227             // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
 228             // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
 229             // This is like leaking one of the Vpn2 routes towards Vpn1
 230             String srcVpnUuid = interVpnLink.getVpnNameByIpAddress(nextHop);
 231             String dstVpnUuid = interVpnLink.getOtherVpnNameByIpAddress(nextHop);
 232             String dstVpnRd = VpnUtil.getVpnRd(dataBroker, dstVpnUuid);
 233             long newLabel = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 234                     VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
 235             if (newLabel == 0) {
 236                 LOG.error("addExtraRoute: Unable to fetch label from Id Manager. Bailing out of adding intervpnlink"
 237                         + " route for destination {}", destination);
 238                 return;
 239             }
 240             ivpnLinkService.leakRoute(interVpnLink, srcVpnUuid, dstVpnUuid, destination, newLabel, RouteOrigin.STATIC);
 241         } else {
 242             Optional<Routes> optVpnExtraRoutes = VpnExtraRouteHelper
 243                     .getVpnExtraroutes(dataBroker, vpnName, rd != null ? rd : routerID, destination);
 244             if (optVpnExtraRoutes.isPresent()) {
 245                 List<String> nhList = optVpnExtraRoutes.get().getNexthopIpList();
 246                 if (nhList != null && nhList.size() > 1) {
 247                     // If nhList is greater than one for vpnextraroute, a call to populatefib doesn't update vrfentry.
 248                     fibManager.refreshVrfEntry(primaryRd, destination);
 249                 } else {
 250                     L3vpnInput input = new L3vpnInput().setNextHop(operationalAdj).setNextHopIp(nextHop).setL3vni(l3vni)
 251                             .setPrimaryRd(primaryRd).setVpnName(vpnName).setDpnId(dpnId)
 252                             .setEncapType(encapType).setRd(rd).setRouteOrigin(origin);
 253                     L3vpnRegistry.getRegisteredPopulator(encapType).populateFib(input, writeConfigTxn);
 254                 }
 255             }
 256         }
 257
 258         if (!writeConfigTxnPresent) {
 259             writeConfigTxn.submit();
 260         }
 261     }
 262
 263     @Override
 264     public void delExtraRoute(String vpnName, String destination, String nextHop, String rd, String routerID) {
 265         LOG.info("Deleting extra route with destination {} and nextHop {}", destination, nextHop);
 266         delExtraRoute(vpnName, destination, nextHop, rd, routerID, null, null);
 267     }
 268
 269     @Override
 270     public void delExtraRoute(String vpnName, String destination, String nextHop, String rd, String routerID,
 271             String intfName, WriteTransaction writeConfigTxn) {
 272         Boolean writeConfigTxnPresent = true;
 273         BigInteger dpnId = null;
 274         if (writeConfigTxn == null) {
 275             writeConfigTxnPresent = false;
 276             writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 277         }
 278         String tunnelIp = nextHop;
 279         if (intfName != null && !intfName.isEmpty()) {
 280             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
 281             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 282             if (nextHopIp == null || nextHopIp.isEmpty()) {
 283                 LOG.error("delExtraRoute: NextHop for interface {} is null / empty."
 284                         + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
 285                         dpnId);
 286             }
 287             tunnelIp = nextHopIp;
 288         }
 289         if (rd != null) {
 290             String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
 291             removePrefixFromBGP(primaryRd, rd, vpnName, destination, nextHop, tunnelIp, dpnId, writeConfigTxn);
 292             LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName, rd);
 293         } else {
 294             // add FIB route directly
 295             fibManager.removeOrUpdateFibEntry(routerID, destination, tunnelIp, writeConfigTxn);
 296             LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName,
 297                     routerID);
 298         }
 299         if (!writeConfigTxnPresent) {
 300             writeConfigTxn.submit();
 301         }
 302     }
 303
 304  // TODO Clean up the exception handling
 305     @Override
 306     @SuppressWarnings("checkstyle:IllegalCatch")
 307     public void removePrefixFromBGP(String primaryRd, String rd, String vpnName, String prefix, String nextHop,
 308                                      String tunnelIp, BigInteger dpnId, WriteTransaction writeConfigTxn) {
 309         try {
 310             LOG.info("removePrefixFromBGP: VPN WITHDRAW: Removing Fib Entry rd {} prefix {} nexthop {}", rd, prefix,
 311                     nextHop);
 312             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 313             synchronized (vpnNamePrefixKey.intern()) {
 314                 Optional<Routes> optVpnExtraRoutes = VpnExtraRouteHelper
 315                         .getVpnExtraroutes(dataBroker, vpnName, rd, prefix);
 316                 if (optVpnExtraRoutes.isPresent()) {
 317                     List<String> nhList = optVpnExtraRoutes.get().getNexthopIpList();
 318                     if (nhList != null && nhList.size() > 1) {
 319                         // If nhList is more than 1, just update vpntoextraroute and prefixtointerface DS
 320                         // For other cases, remove the corresponding tep ip from fibentry and withdraw prefix
 321                         nhList.remove(nextHop);
 322                         VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 323                                 VpnExtraRouteHelper.getVpnToExtrarouteVrfIdIdentifier(vpnName, rd, prefix),
 324                                 VpnUtil.getVpnToExtraroute(prefix, nhList));
 325                         MDSALUtil.syncDelete(dataBroker,
 326                                 LogicalDatastoreType.CONFIGURATION, VpnExtraRouteHelper.getUsedRdsIdentifier(
 327                                 VpnUtil.getVpnId(dataBroker, vpnName), prefix, nextHop));
 328                         LOG.debug("removePrefixFromBGP: Removed vpn-to-extraroute with rd {} prefix {} nexthop {}",
 329                                 rd, prefix, nextHop);
 330                         fibManager.refreshVrfEntry(primaryRd, prefix);
 331                         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 332                         Optional<Prefixes> prefixToInterface = VpnUtil.getPrefixToInterface(dataBroker, vpnId, nextHop);
 333                         if (prefixToInterface.isPresent()) {
 334                             writeConfigTxn.delete(LogicalDatastoreType.OPERATIONAL,
 335                                     VpnUtil.getAdjacencyIdentifier(prefixToInterface.get().getVpnInterfaceName(),
 336                                             prefix));
 337                         }
 338                         LOG.info("VPN WITHDRAW: removePrefixFromBGP: Removed Fib Entry rd {} prefix {} nexthop {}",
 339                                 rd, prefix, tunnelIp);
 340                         return;
 341                     }
 342                 }
 343                 fibManager.removeOrUpdateFibEntry(primaryRd, prefix, tunnelIp, writeConfigTxn);
 344                 if (VpnUtil.isEligibleForBgp(primaryRd, vpnName, dpnId, null /*networkName*/)) {
 345                     // TODO: Might be needed to include nextHop here
 346                     bgpManager.withdrawPrefix(rd, prefix);
 347                 }
 348             }
 349             LOG.info("removePrefixFromBGP: VPN WITHDRAW: Removed Fib Entry rd {} prefix {} nexthop {}", rd, prefix,
 350                     nextHop);
 351         } catch (RuntimeException e) {
 352             LOG.error("removePrefixFromBGP: Delete prefix {} rd {} nextHop {} failed", prefix, rd, nextHop);
 353         }
 354     }
 355
 356     @Override
 357     public boolean isVPNConfigured() {
 358         InstanceIdentifier<VpnInstances> vpnsIdentifier = InstanceIdentifier.builder(VpnInstances.class).build();
 359         Optional<VpnInstances> optionalVpns = TransactionUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION,
 360             vpnsIdentifier);
 361         if (!optionalVpns.isPresent()
 362             || optionalVpns.get().getVpnInstance() == null
 363             || optionalVpns.get().getVpnInstance().isEmpty()) {
 364             LOG.trace("isVPNConfigured: No VPNs configured.");
 365             return false;
 366         }
 367         LOG.trace("isVPNConfigured: VPNs are configured on the system.");
 368         return true;
 369     }
 370
 371     @Override
 372     public List<BigInteger> getDpnsOnVpn(String vpnInstanceName) {
 373         return VpnUtil.getDpnsOnVpn(dataBroker, vpnInstanceName);
 374     }
 375
 376     @Override
 377     public boolean existsVpn(String vpnName) {
 378         return VpnUtil.getVpnInstance(dataBroker, vpnName) != null;
 379     }
 380
 381     @Override
 382     public void addSubnetMacIntoVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 383             BigInteger dpnId, WriteTransaction tx) {
 384         setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
 385             (vpnId, dpId, subnetVpnId) -> addGwMac(srcMacAddress, tx, vpnId, dpId, subnetVpnId));
 386     }
 387
 388     @Override
 389     public void removeSubnetMacFromVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 390             BigInteger dpnId, WriteTransaction tx) {
 391         setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
 392             (vpnId, dpId, subnetVpnId) -> removeGwMac(srcMacAddress, tx, vpnId, dpId, subnetVpnId));
 393     }
 394
 395     @FunctionalInterface
 396     private interface VpnInstanceSubnetMacSetupMethod {
 397         void process(long vpnId, BigInteger dpId, long subnetVpnId);
 398     }
 399
 400     private void setupSubnetMacInVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 401             BigInteger dpnId, VpnInstanceSubnetMacSetupMethod consumer) {
 402         if (vpnName == null) {
 403             LOG.warn("Cannot setup subnet MAC {} on DPN {}, null vpnName", srcMacAddress, dpnId);
 404             return;
 405         }
 406
 407         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 408         long subnetVpnId = VpnUtil.getVpnId(dataBroker, subnetVpnName);
 409         if (dpnId.equals(BigInteger.ZERO)) {
 410             /* Apply the MAC on all DPNs in a VPN */
 411             for (BigInteger dpId : VpnUtil.getDpnsOnVpn(dataBroker, vpnName)) {
 412                 consumer.process(vpnId, dpId, subnetVpnId);
 413             }
 414         } else {
 415             consumer.process(vpnId, dpnId, subnetVpnId);
 416         }
 417     }
 418
 419     private void addGwMac(String srcMacAddress, WriteTransaction tx, long vpnId, BigInteger dpId, long subnetVpnId) {
 420         FlowEntity flowEntity = VpnUtil.buildL3vpnGatewayFlow(dpId, srcMacAddress, vpnId, subnetVpnId);
 421         mdsalManager.addFlowToTx(flowEntity, tx);
 422     }
 423
 424     private void removeGwMac(String srcMacAddress, WriteTransaction tx, long vpnId, BigInteger dpId, long subnetVpnId) {
 425         FlowEntity flowEntity = VpnUtil.buildL3vpnGatewayFlow(dpId, srcMacAddress, vpnId, subnetVpnId);
 426         mdsalManager.removeFlowToTx(flowEntity, tx);
 427     }
 428
 429     @Override
 430     public void addRouterGwMacFlow(String routerName, String routerGwMac, BigInteger dpnId, Uuid extNetworkId,
 431             String subnetVpnName, WriteTransaction writeTx) {
 432         setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId, writeTx,
 433             (vpnId, tx) -> addSubnetMacIntoVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, tx), "Installing");
 434     }
 435
 436     @Override
 437     public void removeRouterGwMacFlow(String routerName, String routerGwMac, BigInteger dpnId, Uuid extNetworkId,
 438             String subnetVpnName, WriteTransaction writeTx) {
 439         setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId, writeTx,
 440             (vpnId, tx) -> removeSubnetMacFromVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, tx), "Removing");
 441     }
 442
 443     @FunctionalInterface
 444     private interface RouterGwMacFlowSetupMethod {
 445         void process(String vpnId, WriteTransaction tx);
 446     }
 447
 448     private void setupRouterGwMacFlow(String routerName, String routerGwMac, BigInteger dpnId, Uuid extNetworkId,
 449             WriteTransaction writeTx, RouterGwMacFlowSetupMethod consumer, String operation) {
 450         if (routerGwMac == null) {
 451             LOG.warn("Failed to handle router GW flow in GW-MAC table. MAC address is missing for router-id {}",
 452                     routerName);
 453             return;
 454         }
 455
 456         if (dpnId == null || BigInteger.ZERO.equals(dpnId)) {
 457             LOG.info("setupRouterGwMacFlow: DPN id is missing for router-id {}",
 458                     routerName);
 459             return;
 460         }
 461
 462         Uuid vpnId = VpnUtil.getExternalNetworkVpnId(dataBroker, extNetworkId);
 463         if (vpnId == null) {
 464             LOG.warn("Network {} is not associated with VPN", extNetworkId.getValue());
 465             return;
 466         }
 467
 468         LOG.info("{} router GW MAC flow for router-id {} on switch {}", operation, routerName, dpnId);
 469         if (writeTx == null) {
 470             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 471                 tx -> consumer.process(vpnId.getValue(), tx)), LOG, "Commit transaction");
 472         } else {
 473             consumer.process(vpnId.getValue(), writeTx);
 474         }
 475     }
 476
 477     @Override
 478     public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 479             BigInteger dpnId, Uuid extNetworkId, WriteTransaction writeTx) {
 480
 481         if (dpnId == null || BigInteger.ZERO.equals(dpnId)) {
 482             LOG.warn("Failed to install arp responder flows for router {}. DPN id is missing.", id);
 483             return;
 484         }
 485
 486         String extInterfaceName = elanService.getExternalElanInterface(extNetworkId.getValue(), dpnId);
 487         if (extInterfaceName != null) {
 488             doAddArpResponderFlowsToExternalNetworkIps(
 489                     id, fixedIps, macAddress, dpnId, extNetworkId, writeTx, extInterfaceName);
 490             return;
 491         }
 492
 493         LOG.warn("Failed to install responder flows for {}. No external interface found for DPN id {}", id, dpnId);
 494
 495         if (!upgradeState.isUpgradeInProgress()) {
 496             return;
 497         }
 498
 499         // The following through the end of the function deals with an upgrade scenario where the neutron configuration
 500         // is restored before the OVS switches reconnect. In such a case, the elan-dpn-interfaces entries will be
 501         // missing from the operational data store. In order to mitigate this we use DataTreeEventCallbackRegistrar
 502         // to wait for the exact operational md-sal object we need to contain the external interface we need.
 503
 504         LOG.info("Upgrade in process, waiting for an external interface to appear on dpn {} for elan {}",
 505                 dpnId, extNetworkId.getValue());
 506
 507         InstanceIdentifier<DpnInterfaces> dpnInterfacesIid =
 508                             elanService.getElanDpnInterfaceOperationalDataPath(extNetworkId.getValue(), dpnId);
 509
 510         eventCallbacks.onAddOrUpdate(LogicalDatastoreType.OPERATIONAL, dpnInterfacesIid, (unused, alsoUnused) -> {
 511             LOG.info("Reattempting write of arp responder for external interfaces for external network {}",
 512                     extNetworkId);
 513             DpnInterfaces dpnInterfaces = elanService.getElanInterfaceInfoByElanDpn(extNetworkId.getValue(), dpnId);
 514             if (dpnInterfaces == null) {
 515                 LOG.error("Could not retrieve DpnInterfaces for {}, {}", extNetworkId.getValue(), dpnId);
 516                 return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 517             }
 518
 519             String extIfc = null;
 520             for (String dpnInterface : dpnInterfaces.getInterfaces()) {
 521                 if (interfaceManager.isExternalInterface(dpnInterface)) {
 522                     extIfc = dpnInterface;
 523                     break;
 524                 }
 525             }
 526
 527             if (extIfc == null) {
 528                 if (upgradeState.isUpgradeInProgress()) {
 529                     LOG.info("External interface not found yet in elan {} on dpn {}, keep waiting",
 530                             extNetworkId.getValue(), dpnInterfaces);
 531                     return DataTreeEventCallbackRegistrar.NextAction.CALL_AGAIN;
 532                 } else {
 533                     return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 534                 }
 535             }
 536
 537             final String extIfcFinal = extIfc;
 538             ListenableFuture<Void> listenableFuture =
 539                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> doAddArpResponderFlowsToExternalNetworkIps(
 540                     id, fixedIps, macAddress, dpnId, extNetworkId, tx, extIfcFinal));
 541             ListenableFutures.addErrorLogging(listenableFuture, LOG,
 542                     "Error while configuring arp responder for ext. interface");
 543
 544             return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 545         });
 546
 547     }
 548
 549     @Override
 550     public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 551             BigInteger dpnId, String extInterfaceName, int lportTag, WriteTransaction writeTx) {
 552         if (fixedIps == null || fixedIps.isEmpty()) {
 553             LOG.debug("No external IPs defined for {}", id);
 554             return;
 555         }
 556
 557         LOG.info("Installing ARP responder flows for {} fixed-ips {} on switch {}", id, fixedIps, dpnId);
 558
 559         if (writeTx == null) {
 560             ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 561                 tx -> {
 562                     for (String fixedIp : fixedIps) {
 563                         installArpResponderFlowsToExternalNetworkIp(macAddress, dpnId, extInterfaceName, lportTag,
 564                                 fixedIp);
 565                     }
 566                 });
 567             ListenableFutures.addErrorLogging(future, LOG, "Commit transaction");
 568         } else {
 569             for (String fixedIp : fixedIps) {
 570                 installArpResponderFlowsToExternalNetworkIp(macAddress, dpnId, extInterfaceName, lportTag,
 571                         fixedIp);
 572             }
 573         }
 574     }
 575
 576     private void doAddArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 577                             BigInteger dpnId, Uuid extNetworkId, WriteTransaction writeTx, String extInterfaceName) {
 578         Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
 579         if (extInterfaceState == null) {
 580             LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
 581                     id);
 582             return;
 583         }
 584
 585         Integer lportTag = extInterfaceState.getIfIndex();
 586         if (lportTag == null) {
 587             LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
 588             return;
 589         }
 590
 591         if (macAddress == null) {
 592
 593             LOG.debug("Failed to install arp responder flows for router-gateway-ip {} for router {}."
 594                     + "External Gw MacAddress is missing.", fixedIps,  id);
 595             return;
 596         }
 597
 598         addArpResponderFlowsToExternalNetworkIps(id, fixedIps, macAddress, dpnId, extInterfaceName, lportTag,
 599                 writeTx);
 600     }
 601
 602     @Override
 603     public void removeArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 604             BigInteger dpnId, Uuid extNetworkId) {
 605
 606         if (dpnId == null || BigInteger.ZERO.equals(dpnId)) {
 607             LOG.warn("Failed to remove arp responder flows for router {}. DPN id is missing.", id);
 608             return;
 609         }
 610
 611         String extInterfaceName = elanService.getExternalElanInterface(extNetworkId.getValue(), dpnId);
 612         if (extInterfaceName == null) {
 613             LOG.warn("Failed to remove responder flows for {}. No external interface found for DPN id {}", id, dpnId);
 614             return;
 615         }
 616
 617         Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
 618         if (extInterfaceState == null) {
 619             LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
 620                     id);
 621             return;
 622         }
 623
 624         Integer lportTag = extInterfaceState.getIfIndex();
 625         if (lportTag == null) {
 626             LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
 627             return;
 628         }
 629
 630         if (macAddress == null) {
 631
 632             LOG.debug("Failed to remove arp responder flows for router-gateway-ip {} for router {}."
 633                     + "External Gw MacAddress is missing.", fixedIps,  id);
 634             return;
 635         }
 636
 637         removeArpResponderFlowsToExternalNetworkIps(id, fixedIps, dpnId,
 638                 extInterfaceName, lportTag);
 639     }
 640
 641     @Override
 642     public void removeArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps,
 643             BigInteger dpnId, String extInterfaceName, int lportTag) {
 644         if (fixedIps == null || fixedIps.isEmpty()) {
 645             LOG.debug("No external IPs defined for {}", id);
 646             return;
 647         }
 648
 649         LOG.info("Removing ARP responder flows for {} fixed-ips {} on switch {}", id, fixedIps, dpnId);
 650
 651         for (String fixedIp : fixedIps) {
 652             removeArpResponderFlowsToExternalNetworkIp(dpnId, lportTag, fixedIp, extInterfaceName);
 653         }
 654     }
 655
 656     @Override
 657     public String getPrimaryRdFromVpnInstance(VpnInstance vpnInstance) {
 658         return VpnUtil.getPrimaryRd(vpnInstance);
 659     }
 660
 661     @Override
 662     public List<MatchInfoBase> getEgressMatchesForVpn(String vpnName) {
 663         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 664         if (vpnId == VpnConstants.INVALID_ID) {
 665             LOG.warn("No VPN id found for {}", vpnName);
 666             return Collections.emptyList();
 667         }
 668
 669         return Collections
 670                 .singletonList(new NxMatchRegister(VpnConstants.VPN_REG_ID, vpnId, MetaDataUtil.getVpnIdMaskForReg()));
 671     }
 672
 673     private void installArpResponderFlowsToExternalNetworkIp(String macAddress, BigInteger dpnId,
 674             String extInterfaceName, int lportTag, String fixedIp) {
 675         // reset the split-horizon bit to allow traffic to be sent back to the
 676         // provider port
 677         List<Instruction> instructions = new ArrayList<>();
 678         instructions.add(
 679                 new InstructionWriteMetadata(BigInteger.ZERO, MetaDataUtil.METADATA_MASK_SH_FLAG).buildInstruction(1));
 680         instructions.addAll(
 681                 ArpResponderUtil.getExtInterfaceInstructions(interfaceManager, itmRpcService, extInterfaceName,
 682                         fixedIp, macAddress));
 683         ArpReponderInputBuilder builder = new ArpReponderInputBuilder().setDpId(dpnId)
 684                 .setInterfaceName(extInterfaceName).setSpa(fixedIp).setSha(macAddress).setLportTag(lportTag);
 685         builder.setInstructions(instructions);
 686         elanService.addArpResponderFlow(builder.buildForInstallFlow());
 687     }
 688
 689     private void removeArpResponderFlowsToExternalNetworkIp(BigInteger dpnId, Integer lportTag, String fixedIp,
 690             String extInterfaceName) {
 691         ArpResponderInput arpInput = new ArpReponderInputBuilder().setDpId(dpnId).setInterfaceName(extInterfaceName)
 692                 .setSpa(fixedIp).setLportTag(lportTag).buildForRemoveFlow();
 693         elanService.removeArpResponderFlow(arpInput);
 694     }
 695
 696     @Override
 697     public void onSubnetAddedToVpn(Subnetmap subnetmap, boolean isBgpVpn, Long elanTag) {
 698         vpnSubnetRouteHandler.onSubnetAddedToVpn(subnetmap, isBgpVpn, elanTag);
 699     }
 700
 701     @Override
 702     public void onSubnetDeletedFromVpn(Subnetmap subnetmap, boolean isBgpVpn) {
 703         vpnSubnetRouteHandler.onSubnetDeletedFromVpn(subnetmap, isBgpVpn);
 704     }
 705
 706     @Override
 707     public VpnInstance getVpnInstance(DataBroker broker, String vpnInstanceName) {
 708         return VpnUtil.getVpnInstance(broker, vpnInstanceName);
 709     }
 710
 711     @Override
 712     public String getVpnRd(DataBroker broker, String vpnName) {
 713         return VpnUtil.getVpnRd(broker, vpnName);
 714     }
 715
 716     @Override
 717     public VpnPortipToPort getNeutronPortFromVpnPortFixedIp(DataBroker broker, String vpnName, String fixedIp) {
 718         return VpnUtil.getNeutronPortFromVpnPortFixedIp(broker, vpnName, fixedIp);
 719     }
 720 }