vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/intervpnlink/InterVpnLinkListener.java

   1 /*
   2  * Copyright (c) 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager.intervpnlink;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.base.Preconditions;
  15 import com.google.common.util.concurrent.FutureCallback;
  16 import com.google.common.util.concurrent.Futures;
  17 import com.google.common.util.concurrent.ListenableFuture;
  18 import com.google.common.util.concurrent.MoreExecutors;
  19 import java.util.Collections;
  20 import java.util.List;
  21 import java.util.concurrent.ExecutionException;
  22 import java.util.concurrent.Future;
  23 import javax.annotation.PostConstruct;
  24 import javax.inject.Inject;
  25 import javax.inject.Singleton;
  26 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  27 import org.opendaylight.controller.md.sal.binding.api.NotificationPublishService;
  28 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  29 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  30 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  31 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  32 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  33 import org.opendaylight.genius.mdsalutil.NwConstants;
  34 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  35 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  36 import org.opendaylight.infrautils.utils.concurrent.JdkFutures;
  37 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  38 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  39 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  40 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  41 import org.opendaylight.netvirt.vpnmanager.VpnConstants;
  42 import org.opendaylight.netvirt.vpnmanager.VpnFootprintService;
  43 import org.opendaylight.netvirt.vpnmanager.VpnOpDataSyncer;
  44 import org.opendaylight.netvirt.vpnmanager.VpnUtil;
  45 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  46 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  47 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  48 import org.opendaylight.netvirt.vpnmanager.intervpnlink.tasks.InterVpnLinkCleanedCheckerTask;
  49 import org.opendaylight.netvirt.vpnmanager.intervpnlink.tasks.InterVpnLinkCreatorTask;
  50 import org.opendaylight.netvirt.vpnmanager.intervpnlink.tasks.InterVpnLinkRemoverTask;
  51 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInput;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInputBuilder;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.InterVpnLinkCreationError;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.InterVpnLinkCreationErrorBuilder;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.InterVpnLinks;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.creation.error.InterVpnLinkCreationErrorMessage;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.creation.error.InterVpnLinkCreationErrorMessageBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkStateBuilder;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.inter.vpn.link.state.FirstEndpointState;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.inter.vpn.link.state.FirstEndpointStateBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.inter.vpn.link.state.SecondEndpointState;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.inter.vpn.link.state.SecondEndpointStateBuilder;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLinkKey;
  76 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  77 import org.opendaylight.yangtools.yang.common.RpcResult;
  78 import org.opendaylight.yangtools.yang.common.Uint32;
  79 import org.opendaylight.yangtools.yang.common.Uint64;
  80 import org.slf4j.Logger;
  81 import org.slf4j.LoggerFactory;
  82
  83 @Singleton
  84 public class InterVpnLinkListener extends AsyncDataTreeChangeListenerBase<InterVpnLink, InterVpnLinkListener> {
  85
  86     private static final Logger LOG = LoggerFactory.getLogger(InterVpnLinkListener.class);
  87
  88     private static final long INVALID_ID = 0;
  89
  90     private final DataBroker dataBroker;
  91     private final ManagedNewTransactionRunner txRunner;
  92     private final IMdsalApiManager mdsalManager;
  93     private final IdManagerService idManager;
  94     private final IBgpManager bgpManager;
  95     private final IFibManager fibManager;
  96     private final NotificationPublishService notificationsService;
  97     private final IVpnLinkService ivpnLinkService;
  98     private final InterVpnLinkLocator ivpnLinkLocator;
  99     private final VpnFootprintService vpnFootprintService;
 100     private final VpnOpDataSyncer vpnOpDataSyncer;
 101     private final JobCoordinator jobCoordinator;
 102     private final InterVpnLinkCache interVpnLinkCache;
 103     private final VpnUtil vpnUtil;
 104     private final InterVpnLinkUtil interVpnLinkUtil;
 105
 106     @Inject
 107     public InterVpnLinkListener(final DataBroker dataBroker, final IdManagerService idManager,
 108                                 final IMdsalApiManager mdsalManager, final IBgpManager bgpManager,
 109                                 final IFibManager fibManager, final NotificationPublishService notifService,
 110                                 final IVpnLinkService interVpnLinkService,
 111                                 final InterVpnLinkLocator interVpnLinkLocator,
 112                                 final VpnFootprintService vpnFootprintService,
 113                                 final VpnOpDataSyncer vpnOpDataSyncer,
 114                                 final JobCoordinator jobCoordinator,
 115                                 final InterVpnLinkCache interVpnLinkCache, final VpnUtil vpnUtil,
 116                                 final InterVpnLinkUtil interVpnLinkUtil) {
 117         this.dataBroker = dataBroker;
 118         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 119         this.idManager = idManager;
 120         this.mdsalManager = mdsalManager;
 121         this.bgpManager = bgpManager;
 122         this.fibManager = fibManager;
 123         this.notificationsService = notifService;
 124         this.ivpnLinkService = interVpnLinkService;
 125         this.ivpnLinkLocator = interVpnLinkLocator;
 126         this.vpnFootprintService = vpnFootprintService;
 127         this.vpnOpDataSyncer = vpnOpDataSyncer;
 128         this.jobCoordinator = jobCoordinator;
 129         this.interVpnLinkCache = interVpnLinkCache;
 130         this.vpnUtil = vpnUtil;
 131         this.interVpnLinkUtil = interVpnLinkUtil;
 132     }
 133
 134     @PostConstruct
 135     public void start() {
 136         LOG.info("{} start", getClass().getSimpleName());
 137         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 138     }
 139
 140     @Override
 141     protected InstanceIdentifier<InterVpnLink> getWildCardPath() {
 142         return InstanceIdentifier.create(InterVpnLinks.class).child(InterVpnLink.class);
 143     }
 144
 145     @Override
 146     protected InterVpnLinkListener getDataTreeChangeListener() {
 147         return InterVpnLinkListener.this;
 148     }
 149
 150     @Override
 151     protected void add(InstanceIdentifier<InterVpnLink> identifier, InterVpnLink add) {
 152
 153         String ivpnLinkName = add.getName();
 154         LOG.debug("Reacting to IVpnLink {} creation. Vpn1=[name={}  EndpointIp={}]  Vpn2=[name={} endpointIP={}]",
 155                   ivpnLinkName, add.getFirstEndpoint().getVpnUuid(), add.getFirstEndpoint().getIpAddress(),
 156                   add.getSecondEndpoint().getVpnUuid(), add.getSecondEndpoint().getIpAddress());
 157
 158         // Create VpnLink state
 159         InstanceIdentifier<InterVpnLinkState> vpnLinkStateIid = InterVpnLinkUtil.getInterVpnLinkStateIid(ivpnLinkName);
 160         InterVpnLinkState vpnLinkState = new InterVpnLinkStateBuilder().setInterVpnLinkName(ivpnLinkName).build();
 161         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, vpnLinkStateIid, vpnLinkState);
 162
 163         InterVpnLinkKey key = add.key();
 164         Uuid vpn1Uuid = add.getFirstEndpoint().getVpnUuid();
 165         String vpn1Name = vpn1Uuid.getValue();
 166         Uuid vpn2Uuid = add.getSecondEndpoint().getVpnUuid();
 167         String vpn2Name = vpn2Uuid.getValue();
 168         String vpn1PrimaryRd = vpnUtil.getPrimaryRd(vpn1Name);
 169         String vpn2PrimaryRd = vpnUtil.getPrimaryRd(vpn1Name);
 170         if (!vpnUtil.isVpnPendingDelete(vpn1PrimaryRd)
 171                 && !vpnUtil.isVpnPendingDelete(vpn2PrimaryRd)) {
 172             if (vpnUtil.getVpnInstance(vpn1Name) == null) {
 173                 String errMsg = "InterVpnLink " + ivpnLinkName + " creation error: could not find 1st endpoint Vpn "
 174                         + vpn1Name;
 175                 setInError(vpnLinkStateIid, vpnLinkState, errMsg);
 176                 return;
 177             }
 178             if (!checkVpnAvailability(key, vpn1Name)) {
 179                 String errMsg = "InterVpnLink " + ivpnLinkName + " creation error: Vpn " + vpn1Name
 180                         + " is already associated to an inter-vpn-link ";
 181                 setInError(vpnLinkStateIid, vpnLinkState, errMsg);
 182                 return;
 183             }
 184
 185             // Second VPN
 186             if (vpnUtil.getVpnInstance(vpn2Name) == null) {
 187                 String errMsg = "InterVpnLink " + ivpnLinkName + " creation error: could not find 2nd endpoint Vpn "
 188                         + vpn2Name;
 189                 setInError(vpnLinkStateIid, vpnLinkState, errMsg);
 190                 return;
 191             }
 192             if (!checkVpnAvailability(key, vpn2Name)) {
 193                 String errMsg = "InterVpnLink " + ivpnLinkName + " creation error: Vpn " + vpn2Name
 194                         + " is already associated with an inter-vpn-link";
 195                 setInError(vpnLinkStateIid, vpnLinkState, errMsg);
 196                 return;
 197             }
 198
 199             interVpnLinkCache.addInterVpnLinkToCaches(add);
 200
 201             // Wait for VPN Operational data ready
 202             Uint32 vpn1Id = vpnUtil.getVpnId(vpn1Name);
 203             if (VpnConstants.INVALID_ID.equals(vpn1Id)) {
 204                 boolean vpn1Ready =
 205                         vpnOpDataSyncer.waitForVpnDataReady(VpnOpDataSyncer.VpnOpDataType.vpnInstanceToId, vpn1Name,
 206                                 VpnConstants.PER_VPN_INSTANCE_MAX_WAIT_TIME_IN_MILLISECONDS);
 207                 if (!vpn1Ready) {
 208                     String errMsg =
 209                             "InterVpnLink " + ivpnLinkName + " creation error: Operational Data for VPN " + vpn1Name
 210                                     + " not ready after " + VpnConstants.PER_INTERFACE_MAX_WAIT_TIME_IN_MILLISECONDS
 211                                     + " milliseconds";
 212                     setInError(vpnLinkStateIid, vpnLinkState, errMsg);
 213                     return;
 214                 }
 215             }
 216             Uint32 vpn2Id = vpnUtil.getVpnId(vpn2Name);
 217             if (VpnConstants.INVALID_ID.equals(vpn2Id)) {
 218                 boolean vpn1Ready =
 219                         vpnOpDataSyncer.waitForVpnDataReady(VpnOpDataSyncer.VpnOpDataType.vpnInstanceToId,vpn2Name,
 220                                 VpnConstants.PER_VPN_INSTANCE_MAX_WAIT_TIME_IN_MILLISECONDS);
 221                 if (!vpn1Ready) {
 222                     String errMsg =
 223                             "InterVpnLink " + ivpnLinkName + " creation error: Operational Data for VPN " + vpn2Name
 224                                     + " not ready after " + VpnConstants.PER_INTERFACE_MAX_WAIT_TIME_IN_MILLISECONDS
 225                                     + " milliseconds";
 226                     setInError(vpnLinkStateIid, vpnLinkState, errMsg);
 227                     return;
 228                 }
 229             }
 230
 231             List<Uint64> firstDpnList = ivpnLinkLocator.selectSuitableDpns(add);
 232             if (firstDpnList != null && !firstDpnList.isEmpty()) {
 233                 List<Uint64> secondDpnList = firstDpnList;
 234
 235                 Long firstVpnLportTag = allocateVpnLinkLportTag(key.getName() + vpn1Name);
 236                 Long secondVpnLportTag = allocateVpnLinkLportTag(key.getName() + vpn2Name);
 237                 FirstEndpointState firstEndPointState =
 238                         new FirstEndpointStateBuilder().setVpnUuid(vpn1Uuid).setDpId(firstDpnList)
 239                                 .setLportTag(firstVpnLportTag).build();
 240                 SecondEndpointState secondEndPointState =
 241                         new SecondEndpointStateBuilder().setVpnUuid(vpn2Uuid).setDpId(secondDpnList)
 242                                 .setLportTag(secondVpnLportTag).build();
 243
 244                 interVpnLinkUtil.updateInterVpnLinkState(ivpnLinkName, InterVpnLinkState.State.Active,
 245                         firstEndPointState, secondEndPointState, interVpnLinkCache);
 246
 247                 // Note that in the DPN of the firstEndpoint we install the lportTag of the secondEndpoint and viceversa
 248                 interVpnLinkUtil.installLPortDispatcherTableFlow(ivpnLinkName, firstDpnList, vpn2Name,
 249                         secondVpnLportTag);
 250                 interVpnLinkUtil.installLPortDispatcherTableFlow(ivpnLinkName, secondDpnList, vpn1Name,
 251                         firstVpnLportTag);
 252                 // Update the VPN -> DPNs Map.
 253                 // Note: when a set of DPNs is calculated for Vpn1, these DPNs are added to the VpnToDpn map of Vpn2.
 254                 // Why? because we do the handover from Vpn1 to Vpn2 in those DPNs, so in those DPNs we must know how
 255                 // to reach to Vpn2 targets. If new Vpn2 targets are added later, the Fib will be maintained in these
 256                 // DPNs even if Vpn2 is not physically present there.
 257                 interVpnLinkUtil.updateVpnFootprint(vpn2Name, vpn2PrimaryRd, firstDpnList);
 258                 interVpnLinkUtil.updateVpnFootprint(vpn1Name, vpn1PrimaryRd, secondDpnList);
 259
 260                 // Program static routes if needed
 261                 Optional<InterVpnLinkDataComposite> interVpnLink =
 262                         interVpnLinkCache.getInterVpnLinkByName(ivpnLinkName);
 263                 ivpnLinkService.handleStaticRoutes(interVpnLink.get());
 264
 265                 // Now, if the corresponding flags are activated, there will be some routes exchange
 266                 ivpnLinkService.exchangeRoutes(interVpnLink.get());
 267             } else {
 268                 // If there is no connection to DPNs, the InterVpnLink is created and the InterVpnLinkState is also
 269                 // created with the corresponding LPortTags but no DPN is assigned since there is no DPN operative.
 270                 Long firstVpnLportTag = allocateVpnLinkLportTag(key.getName() + vpn1Name);
 271                 Long secondVpnLportTag = allocateVpnLinkLportTag(key.getName() + vpn2Name);
 272                 FirstEndpointState firstEndPointState =
 273                         new FirstEndpointStateBuilder().setVpnUuid(vpn1Uuid).setLportTag(firstVpnLportTag)
 274                                 .setDpId(Collections.emptyList()).build();
 275                 SecondEndpointState secondEndPointState =
 276                         new SecondEndpointStateBuilder().setVpnUuid(vpn2Uuid).setLportTag(secondVpnLportTag)
 277                                 .setDpId(Collections.emptyList()).build();
 278                 interVpnLinkUtil.updateInterVpnLinkState(ivpnLinkName, InterVpnLinkState.State.Error,
 279                         firstEndPointState, secondEndPointState, interVpnLinkCache);
 280             }
 281         }
 282     }
 283
 284     private boolean checkVpnAvailability(InterVpnLinkKey key, String vpnName) {
 285         Preconditions.checkNotNull(vpnName);
 286
 287         List<InterVpnLinkDataComposite> allInterVpnLinks = interVpnLinkCache.getAllInterVpnLinks();
 288         if (allInterVpnLinks.isEmpty()) {
 289             return true;
 290         }
 291
 292         return allInterVpnLinks.stream().noneMatch(ivl -> !ivl.getInterVpnLinkName().equals(key.getName())
 293                                                           && ivl.isVpnLinked(vpnName));
 294     }
 295
 296     @Override
 297     protected void remove(InstanceIdentifier<InterVpnLink> identifier, InterVpnLink del) {
 298
 299         LOG.debug("Reacting to InterVpnLink {} removal", del.getName());
 300
 301         // Remove learnt routes
 302         // Remove entries in the LPortDispatcher table
 303         // Remove the corresponding entries in InterVpnLinkState
 304
 305         // For each endpoint, remove all routes that have been learnt by intervpnLink
 306         String vpn1Uuid = del.getFirstEndpoint().getVpnUuid().getValue();
 307         String rd1 = vpnUtil.getVpnRd(vpn1Uuid);
 308         LOG.debug("Removing leaked routes in VPN {}  rd={}", vpn1Uuid, rd1);
 309         vpnUtil.removeVrfEntriesByOrigin(rd1, RouteOrigin.INTERVPN);
 310         List<VrfEntry> vrfEntriesSecondEndpoint =
 311             vpnUtil.findVrfEntriesByNexthop(rd1, del.getSecondEndpoint().getIpAddress().getValue());
 312
 313         String vpn2Uuid = del.getSecondEndpoint().getVpnUuid().getValue();
 314         String rd2 = vpnUtil.getVpnRd(vpn2Uuid);
 315         LOG.debug("Removing leaked routes in VPN {}  rd={}", vpn2Uuid, rd2);
 316         vpnUtil.removeVrfEntriesByOrigin(rd2, RouteOrigin.INTERVPN);
 317         List<VrfEntry> vrfEntriesFirstEndpoint =
 318             vpnUtil.findVrfEntriesByNexthop(rd2, del.getFirstEndpoint().getIpAddress().getValue());
 319
 320         Optional<InterVpnLinkState> optIVpnLinkState = interVpnLinkUtil.getInterVpnLinkState(del.getName());
 321         if (optIVpnLinkState.isPresent()) {
 322             InterVpnLinkState interVpnLinkState = optIVpnLinkState.get();
 323             boolean isVpnFirstEndPoint = true;
 324             if (interVpnLinkState.getFirstEndpointState() != null) {
 325                 Long firstEndpointLportTag = interVpnLinkState.getFirstEndpointState().getLportTag().toJava();
 326                 removeVpnLinkEndpointFlows(del, vpn2Uuid, rd1,
 327                     interVpnLinkState.getSecondEndpointState().getDpId(),
 328                     firstEndpointLportTag.intValue(),
 329                     del.getFirstEndpoint().getIpAddress().getValue(),
 330                     vrfEntriesSecondEndpoint, isVpnFirstEndPoint);
 331             } else {
 332                 LOG.info("Could not get first endpoint state attributes for InterVpnLink {}", del.getName());
 333             }
 334             isVpnFirstEndPoint = false;
 335             if (interVpnLinkState.getSecondEndpointState() != null) {
 336                 Long secondEndpointLportTag = interVpnLinkState.getSecondEndpointState().getLportTag().toJava();
 337                 removeVpnLinkEndpointFlows(del, vpn1Uuid, rd2,
 338                                            interVpnLinkState.getFirstEndpointState().getDpId(),
 339                                            secondEndpointLportTag.intValue(),
 340                                            del.getSecondEndpoint().getIpAddress().getValue(),
 341                                            vrfEntriesFirstEndpoint, isVpnFirstEndPoint);
 342             } else {
 343                 LOG.info("Could not get second endpoint state attributes for InterVpnLink {}", del.getName());
 344             }
 345         }
 346
 347         vpnUtil.removeVrfEntries(rd1, vrfEntriesSecondEndpoint);
 348         vpnUtil.removeVrfEntries(rd2, vrfEntriesFirstEndpoint);
 349         vpnUtil.withdrawRoutes(rd1, vrfEntriesSecondEndpoint);
 350         vpnUtil.withdrawRoutes(rd2, vrfEntriesFirstEndpoint);
 351
 352         // Release idManager with LPortTag associated to endpoints
 353         LOG.debug("Releasing InterVpnLink {} endpoints LportTags", del.getName());
 354         InterVpnLinkKey key = del.key();
 355         Uuid firstEndpointVpnUuid = del.getFirstEndpoint().getVpnUuid();
 356         Uuid secondEndpointVpnUuid = del.getSecondEndpoint().getVpnUuid();
 357         releaseVpnLinkLPortTag(key.getName() + firstEndpointVpnUuid.getValue());
 358         releaseVpnLinkLPortTag(key.getName() + secondEndpointVpnUuid.getValue());
 359
 360         // Routes with nextHop pointing to an end-point of the inter-vpn-link are populated into FIB table.
 361         // The action in that case is a nx_resubmit to LPortDispatcher table. This is done in FibManager.
 362         // At this point. we need to check if is there any entry in FIB table pointing to LPortDispatcher table.
 363         // Remove it in that case.
 364
 365         // Removing the InterVpnLinkState
 366         InstanceIdentifier<InterVpnLinkState> interVpnLinkStateIid =
 367             InterVpnLinkUtil.getInterVpnLinkStateIid(del.getName());
 368         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx ->
 369             tx.delete(interVpnLinkStateIid)), LOG, "Error deleting inter-VPN link state {}", interVpnLinkStateIid);
 370     }
 371
 372     // We're catching Exception here to continue deleting as much as possible
 373     // TODO Rework this so it's done in one transaction
 374     @SuppressWarnings("checkstyle:IllegalCatch")
 375     private void removeVpnLinkEndpointFlows(InterVpnLink del, String vpnUuid, String rd, List<Uint64> dpns,
 376                                             int otherEndpointLportTag, String otherEndpointIpAddr,
 377                                             List<VrfEntry> vrfEntries, final boolean isVpnFirstEndPoint) {
 378
 379         String interVpnLinkName = del.getName();
 380         LOG.debug("Removing endpoint flows for vpn {}.  InterVpnLink={}.  OtherEndpointLportTag={}",
 381             vpnUuid, interVpnLinkName, otherEndpointLportTag);
 382         if (dpns == null) {
 383             LOG.debug("VPN {} endpoint is not instantiated in any DPN for InterVpnLink {}",
 384                 vpnUuid, interVpnLinkName);
 385             return;
 386         }
 387
 388         for (Uint64 dpnId : dpns) {
 389             try {
 390                 // Removing flow from LportDispatcher table
 391                 String flowRef = InterVpnLinkUtil.getLportDispatcherFlowRef(interVpnLinkName, otherEndpointLportTag);
 392                 FlowKey flowKey = new FlowKey(new FlowId(flowRef));
 393                 Flow flow = new FlowBuilder().withKey(flowKey).setId(new FlowId(flowRef))
 394                     .setTableId(NwConstants.LPORT_DISPATCHER_TABLE).setFlowName(flowRef)
 395                     .build();
 396                 mdsalManager.removeFlow(dpnId, flow);
 397
 398                 // Also remove the 'fake' iface from the VpnToDpn map
 399                 interVpnLinkUtil.removeIVpnLinkIfaceFromVpnFootprint(vpnUuid, rd, dpnId);
 400
 401             } catch (Exception e) {
 402                 // Whatever happens it should not stop it from trying to remove as much as possible
 403                 LOG.warn("Error while removing InterVpnLink {} Endpoint flows on dpn {}. Reason: ",
 404                     interVpnLinkName, dpnId, e);
 405             }
 406         }
 407         // Removing flow from FIB and LFIB tables
 408         LOG.trace("Removing flow in FIB and LFIB tables for vpn {} interVpnLink {} otherEndpointIpAddr {}",
 409             vpnUuid, interVpnLinkName, otherEndpointIpAddr);
 410         cleanUpInterVPNRoutes(interVpnLinkName, vrfEntries, isVpnFirstEndPoint);
 411     }
 412
 413
 414     private void releaseVpnLinkLPortTag(String idKey) {
 415         ReleaseIdInput releaseIdInput =
 416             new ReleaseIdInputBuilder().setPoolName(VpnConstants.PSEUDO_LPORT_TAG_ID_POOL_NAME).setIdKey(idKey).build();
 417
 418         JdkFutures.addErrorLogging(idManager.releaseId(releaseIdInput), LOG, "Release Id");
 419     }
 420
 421     @Override
 422     protected void update(InstanceIdentifier<InterVpnLink> identifier, InterVpnLink original, InterVpnLink update) {
 423
 424         LOG.debug("Update InterVpnLink {}. "
 425                 + " original=[1stEndpoint=[vpn=<{}> ipAddr=<{}>] 2ndEndpoint=[vpn=<{}> ipAddr=<{}>]]"
 426                 + " update=[1stEndpoint=[vpn=<{}> ipAddr=<{}>] 2ndEndpoint=[vpn=<{}> ipAddr=<{}>]]",
 427             original.getName(),
 428             original.getFirstEndpoint().getVpnUuid(), original.getFirstEndpoint().getIpAddress(),
 429             original.getSecondEndpoint().getVpnUuid(), original.getSecondEndpoint().getIpAddress(),
 430             update.getFirstEndpoint().getVpnUuid(), update.getFirstEndpoint().getIpAddress(),
 431             update.getSecondEndpoint().getVpnUuid(), update.getSecondEndpoint().getIpAddress());
 432
 433         String specificJobKey = "InterVpnLink.update." + original.getName();
 434         jobCoordinator.enqueueJob(specificJobKey, new InterVpnLinkRemoverTask(dataBroker, identifier));
 435         jobCoordinator.enqueueJob(specificJobKey, new InterVpnLinkCleanedCheckerTask(dataBroker, original,
 436                 interVpnLinkUtil, vpnUtil));
 437         jobCoordinator.enqueueJob(specificJobKey, new InterVpnLinkCreatorTask(dataBroker, update));
 438     }
 439
 440     private Long allocateVpnLinkLportTag(String idKey) {
 441         AllocateIdInput getIdInput =
 442             new AllocateIdInputBuilder().setPoolName(VpnConstants.PSEUDO_LPORT_TAG_ID_POOL_NAME)
 443                 .setIdKey(idKey)
 444                 .build();
 445         try {
 446             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 447             RpcResult<AllocateIdOutput> rpcResult = result.get();
 448             if (rpcResult.isSuccessful()) {
 449                 return rpcResult.getResult().getIdValue().toJava();
 450             } else {
 451                 LOG.warn("RPC Call to Get Unique Id returned with Errors {}", rpcResult.getErrors());
 452             }
 453         } catch (InterruptedException | ExecutionException e) {
 454             LOG.warn("Exception when getting Unique Id", e);
 455         }
 456         return INVALID_ID;
 457     }
 458
 459     protected void setInError(final InstanceIdentifier<InterVpnLinkState> vpnLinkStateIid,
 460         final InterVpnLinkState vpnLinkState,
 461         String errorMsg) {
 462         LOG.error("Setting InterVpnLink {} in error. Reason: {}", vpnLinkState.getInterVpnLinkName(), errorMsg);
 463
 464         // Setting InterVPNLink in error state in MDSAL
 465         InterVpnLinkState vpnLinkErrorState =
 466             new InterVpnLinkStateBuilder(vpnLinkState).setState(InterVpnLinkState.State.Error)
 467                 .setErrorDescription(errorMsg)
 468                 .build();
 469         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx ->
 470             tx.put(vpnLinkStateIid, vpnLinkErrorState, CREATE_MISSING_PARENTS)),
 471                 LOG, "Error storing the VPN link error state for {}, {}", vpnLinkStateIid, vpnLinkErrorState);
 472
 473         // Sending out an error Notification
 474         InterVpnLinkCreationErrorMessage errMsg =
 475             new InterVpnLinkCreationErrorMessageBuilder().setErrorMessage(errorMsg).build();
 476         InterVpnLinkCreationError notif =
 477             new InterVpnLinkCreationErrorBuilder().setInterVpnLinkCreationErrorMessage(errMsg).build();
 478         final ListenableFuture<?> eventFuture = this.notificationsService.offerNotification(notif);
 479         Futures.addCallback(eventFuture, new FutureCallback<Object>() {
 480             @Override
 481             public void onFailure(Throwable error) {
 482                 LOG.warn("Error when sending notification about InterVpnLink creation issue. InterVpnLink name={} "
 483                                 + "state={}.", vpnLinkState.getInterVpnLinkName(), vpnLinkState, error);
 484             }
 485
 486             @Override
 487             public void onSuccess(Object arg) {
 488                 LOG.trace("Error notification for InterVpnLink successfully sent. VpnLinkName={} state={}",
 489                     vpnLinkState.getInterVpnLinkName(), vpnLinkState);
 490             }
 491         }, MoreExecutors.directExecutor());
 492     }
 493
 494     /**
 495      * Removes all the flows from FIB/LFIB matching an endpoint from the intervpnlink.
 496      *
 497      * @param interVpnLinkName name of the intervpnlink
 498      * @param vrfEntries list of vrfs matching the first/second intervpnlink endpoints
 499      * @param isVpnFirstEndPoint indicates whether vrfEntries belong to the vpn of the first endpoint
 500      */
 501     private void cleanUpInterVPNRoutes(final String interVpnLinkName,
 502         List<VrfEntry> vrfEntries,
 503         final boolean isVpnFirstEndPoint) {
 504
 505         for (VrfEntry vrfEntry : vrfEntries) {
 506             fibManager.removeInterVPNLinkRouteFlows(interVpnLinkName, isVpnFirstEndPoint, vrfEntry);
 507         }
 508     }
 509 }