mappingservice/implementation/src/main/java/org/opendaylight/lispflowmapping/implementation/lisp/MapServer.java

   1 /*
   2  * Copyright (c) 2014, 2017 Contextream, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.lispflowmapping.implementation.lisp;
  10
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.collect.Maps;
  13 import com.google.common.util.concurrent.ThreadFactoryBuilder;
  14 import java.net.InetAddress;
  15 import java.net.NetworkInterface;
  16 import java.net.SocketException;
  17 import java.util.ArrayList;
  18 import java.util.Arrays;
  19 import java.util.Enumeration;
  20 import java.util.Iterator;
  21 import java.util.LinkedHashSet;
  22 import java.util.List;
  23 import java.util.Map;
  24 import java.util.Set;
  25 import java.util.concurrent.Executors;
  26 import java.util.concurrent.ScheduledExecutorService;
  27 import java.util.concurrent.ScheduledFuture;
  28 import java.util.concurrent.ThreadFactory;
  29 import java.util.concurrent.TimeUnit;
  30 import org.apache.commons.lang3.BooleanUtils;
  31 import org.opendaylight.lispflowmapping.config.ConfigIni;
  32 import org.opendaylight.lispflowmapping.implementation.util.LoggingUtil;
  33 import org.opendaylight.lispflowmapping.implementation.util.MSNotificationInputUtil;
  34 import org.opendaylight.lispflowmapping.interfaces.dao.SubKeys;
  35 import org.opendaylight.lispflowmapping.interfaces.dao.Subscriber;
  36 import org.opendaylight.lispflowmapping.interfaces.lisp.IMapNotifyHandler;
  37 import org.opendaylight.lispflowmapping.interfaces.lisp.IMapServerAsync;
  38 import org.opendaylight.lispflowmapping.interfaces.lisp.ISmrNotificationListener;
  39 import org.opendaylight.lispflowmapping.interfaces.lisp.SmrEvent;
  40 import org.opendaylight.lispflowmapping.interfaces.mappingservice.IMappingService;
  41 import org.opendaylight.lispflowmapping.lisp.authentication.LispAuthenticationUtil;
  42 import org.opendaylight.lispflowmapping.lisp.type.LispMessage;
  43 import org.opendaylight.lispflowmapping.lisp.type.MappingData;
  44 import org.opendaylight.lispflowmapping.lisp.util.LispAddressStringifier;
  45 import org.opendaylight.lispflowmapping.lisp.util.LispAddressUtil;
  46 import org.opendaylight.lispflowmapping.lisp.util.MapNotifyBuilderHelper;
  47 import org.opendaylight.lispflowmapping.lisp.util.MapRequestUtil;
  48 import org.opendaylight.lispflowmapping.lisp.util.MappingRecordUtil;
  49 import org.opendaylight.lispflowmapping.lisp.util.MaskUtil;
  50 import org.opendaylight.lispflowmapping.lisp.util.SourceDestKeyHelper;
  51 import org.opendaylight.mdsal.binding.api.NotificationService;
  52 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.PortNumber;
  53 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.lisp.address.types.rev151105.lisp.address.Address;
  54 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.lisp.address.types.rev151105.lisp.address.address.SourceDestKey;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.inet.binary.types.rev160303.IpAddressBinary;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.binary.address.types.rev160504.augmented.lisp.address.address.Ipv4PrefixBinary;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.binary.address.types.rev160504.augmented.lisp.address.address.Ipv4PrefixBinaryBuilder;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.binary.address.types.rev160504.augmented.lisp.address.address.Ipv6PrefixBinary;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.binary.address.types.rev160504.augmented.lisp.address.address.Ipv6PrefixBinaryBuilder;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.MapRegister;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.SiteId;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.eid.container.Eid;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.eid.container.EidBuilder;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.eid.list.EidItem;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.eid.list.EidItemBuilder;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.mapnotifymessage.MapNotifyBuilder;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.mapping.authkey.container.MappingAuthkey;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.mapping.record.container.MappingRecord;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.mapping.record.list.MappingRecordItem;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.mapping.record.list.MappingRecordItemBuilder;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.maprequestnotification.MapRequestBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.transport.address.TransportAddress;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.transport.address.TransportAddressBuilder;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.mappingservice.rev150906.MappingChanged;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.mappingservice.rev150906.MappingOrigin;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.mappingservice.rev150906.OdlMappingserviceListener;
  77 import org.opendaylight.yangtools.concepts.ListenerRegistration;
  78 import org.slf4j.Logger;
  79 import org.slf4j.LoggerFactory;
  80
  81 public class MapServer implements IMapServerAsync, OdlMappingserviceListener, ISmrNotificationListener {
  82
  83     private static final Logger LOG = LoggerFactory.getLogger(MapServer.class);
  84     private static final byte[] ALL_ZEROES_XTR_ID = new byte[] {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ,0};
  85     private final IMappingService mapService;
  86     private boolean subscriptionService;
  87     private final IMapNotifyHandler notifyHandler;
  88     private final NotificationService notificationService;
  89     private ListenerRegistration<MapServer> mapServerListenerRegistration;
  90     private final SmrScheduler scheduler;
  91
  92     public MapServer(IMappingService mapService, boolean subscriptionService,
  93                      IMapNotifyHandler notifyHandler, NotificationService notificationService) {
  94         Preconditions.checkNotNull(mapService);
  95         this.mapService = mapService;
  96         this.subscriptionService = subscriptionService;
  97         this.notifyHandler = notifyHandler;
  98         this.notificationService = notificationService;
  99         if (notificationService != null) {
 100             notificationService.registerNotificationListener(this);
 101         }
 102         scheduler = new SmrScheduler();
 103     }
 104
 105     @Override
 106     public void setSubscriptionService(boolean subscriptionService) {
 107         this.subscriptionService = subscriptionService;
 108     }
 109
 110     @Override
 111     @SuppressWarnings("unchecked")
 112     public void handleMapRegister(MapRegister mapRegister) {
 113         boolean mappingUpdated = false;
 114         boolean merge = ConfigIni.getInstance().mappingMergeIsSet() && mapRegister.isMergeEnabled();
 115         MappingRecord oldMapping;
 116
 117         if (merge) {
 118             if (!mapRegister.isXtrSiteIdPresent() || mapRegister.getXtrId() == null) {
 119                 LOG.error("Merge bit is set in Map-Register, but xTR-ID is not present. Will not merge.");
 120                 merge = false;
 121             } else if (Arrays.equals(mapRegister.getXtrId().getValue(), ALL_ZEROES_XTR_ID)) {
 122                 LOG.warn("Merge bit is set in Map-Register, but xTR-ID is all zeroes.");
 123             }
 124         }
 125
 126         for (MappingRecordItem record : mapRegister.getMappingRecordItem()) {
 127             MappingRecord mapping = record.getMappingRecord();
 128             Eid eid = mapping.getEid();
 129             MappingData mappingData = new MappingData(mapping, System.currentTimeMillis());
 130             mappingData.setMergeEnabled(merge);
 131             mappingData.setXtrId(mapRegister.getXtrId());
 132
 133             oldMapping = getMappingRecord(mapService.getMapping(MappingOrigin.Southbound, eid));
 134             mapService.addMapping(MappingOrigin.Southbound, eid, getSiteId(mapRegister), mappingData);
 135             if (merge) {
 136                 MappingRecord newMapping = getMappingRecord(mapService.getMapping(MappingOrigin.Southbound, eid));
 137                 if (MappingRecordUtil.mappingChanged(oldMapping, newMapping)) {
 138                     // If there is a SB mapping change with merge on, Map-Notify will be sent to ALL xTRs, not jus the
 139                     // one registering (merging is done in the MappingSystem code)
 140                     mappingUpdated = true;
 141                 }
 142             }
 143         }
 144         if (BooleanUtils.isTrue(mapRegister.isWantMapNotify())) {
 145             LOG.trace("MapRegister wants MapNotify");
 146             MapNotifyBuilder builder = new MapNotifyBuilder();
 147             List<TransportAddress> rlocs = null;
 148             if (merge) {
 149                 Set<IpAddressBinary> notifyRlocs = new LinkedHashSet<>();
 150                 List<MappingRecordItem> mergedMappings = new ArrayList<>();
 151                 for (MappingRecordItem record : mapRegister.getMappingRecordItem()) {
 152                     MappingRecord mapping = record.getMappingRecord();
 153                     MappingRecord currentRecord = getMappingRecord(mapService.getMapping(MappingOrigin.Southbound,
 154                             mapping.getEid()));
 155                     mergedMappings.add(new MappingRecordItemBuilder().setMappingRecord(currentRecord).build());
 156                     Set<IpAddressBinary> sourceRlocs = (Set<IpAddressBinary>) mapService.getData(
 157                             MappingOrigin.Southbound, mapping.getEid(), SubKeys.SRC_RLOCS);
 158                     if (sourceRlocs != null) {
 159                         notifyRlocs.addAll(sourceRlocs);
 160                     }
 161                 }
 162                 MapNotifyBuilderHelper.setFromMapRegisterAndMappingRecordItems(builder, mapRegister, mergedMappings);
 163                 // send map-notify to merge group only when mapping record is changed
 164                 if (mappingUpdated) {
 165                     rlocs = getTransportAddresses(notifyRlocs);
 166                 }
 167             } else {
 168                 MapNotifyBuilderHelper.setFromMapRegister(builder, mapRegister);
 169             }
 170             List<MappingRecordItem> mappings = builder.getMappingRecordItem();
 171             if (mappings != null && mappings.get(0) != null && mappings.get(0).getMappingRecord() != null
 172                     && mappings.get(0).getMappingRecord().getEid() != null) {
 173                 MappingAuthkey authkey = mapService.getAuthenticationKey(mappings.get(0).getMappingRecord().getEid());
 174                 if (authkey != null) {
 175                     builder.setAuthenticationData(LispAuthenticationUtil.createAuthenticationData(builder.build(),
 176                             authkey.getKeyString()));
 177                 }
 178             }
 179             notifyHandler.handleMapNotify(builder.build(), rlocs);
 180         }
 181     }
 182
 183     private static List<TransportAddress> getTransportAddresses(Set<IpAddressBinary> addresses) {
 184         List<TransportAddress> rlocs = new ArrayList<>();
 185         for (IpAddressBinary address : addresses) {
 186             TransportAddressBuilder tab = new TransportAddressBuilder();
 187             tab.setIpAddress(address);
 188             tab.setPort(new PortNumber(LispMessage.PORT_NUM));
 189             rlocs.add(tab.build());
 190         }
 191         return rlocs;
 192     }
 193
 194     private static SiteId getSiteId(MapRegister mapRegister) {
 195         return mapRegister.getSiteId() != null ? new SiteId(mapRegister.getSiteId()) : null;
 196     }
 197
 198     private static MappingRecord getMappingRecord(MappingData mappingData) {
 199         return mappingData != null ? mappingData.getRecord() : null;
 200     }
 201
 202     @Override
 203     public void onMappingChanged(MappingChanged notification) {
 204         if (subscriptionService) {
 205             Eid eid = notification.getEid();
 206             if (eid == null) {
 207                 eid = notification.getMappingRecord().getEid();
 208             }
 209             LOG.trace("MappingChanged event for {} of type: `{}'", LispAddressStringifier.getString(eid),
 210                     notification.getChangeType());
 211             Set<Subscriber> subscribers = MSNotificationInputUtil.toSubscriberSet(notification.getSubscriberItem());
 212             LoggingUtil.logSubscribers(LOG, eid, subscribers);
 213             if (mapService.isMaster()) {
 214                 sendSmrs(eid, subscribers);
 215                 if (eid.getAddress() instanceof SourceDestKey) {
 216                     Set<Subscriber> dstSubscribers = MSNotificationInputUtil.toSubscriberSetFromDst(
 217                             notification.getDstSubscriberItem());
 218                     LoggingUtil.logSubscribers(LOG, SourceDestKeyHelper.getDstBinary(eid), dstSubscribers);
 219                     sendSmrs(SourceDestKeyHelper.getDstBinary(eid), dstSubscribers);
 220                 }
 221             }
 222         }
 223     }
 224
 225     private void handleSmr(Eid eid, Set<Subscriber> subscribers) {
 226         sendSmrs(eid, subscribers);
 227
 228         // For SrcDst LCAF also send SMRs to Dst prefix
 229         if (eid.getAddress() instanceof SourceDestKey) {
 230             Eid dstAddr = SourceDestKeyHelper.getDstBinary(eid);
 231             Set<Subscriber> dstSubs = mapService.getSubscribers(dstAddr);
 232             sendSmrs(dstAddr, dstSubs);
 233         }
 234     }
 235
 236     private void sendSmrs(Eid eid, Set<Subscriber> subscribers) {
 237         if (subscribers == null) {
 238             return;
 239         }
 240         final MapRequestBuilder mrb = MapRequestUtil.prepareSMR(eid, LispAddressUtil.toRloc(getLocalAddress()));
 241         LOG.trace("Built SMR packet template (EID field will be set later): " + mrb.build().toString());
 242
 243         scheduler.scheduleSmrs(mrb, subscribers.iterator());
 244     }
 245
 246     private static InetAddress getLocalAddress() {
 247         try {
 248             Enumeration<NetworkInterface> interfaces = NetworkInterface.getNetworkInterfaces();
 249             while (interfaces.hasMoreElements()) {
 250                 NetworkInterface current = interfaces.nextElement();
 251                 LOG.trace("Interface " + current.toString());
 252                 if (!current.isUp() || current.isLoopback() || current.isVirtual()) {
 253                     continue;
 254                 }
 255                 Enumeration<InetAddress> addresses = current.getInetAddresses();
 256                 while (addresses.hasMoreElements()) {
 257                     InetAddress currentAddr = addresses.nextElement();
 258                     // Skip loopback and link local addresses
 259                     if (currentAddr.isLoopbackAddress() || currentAddr.isLinkLocalAddress()) {
 260                         continue;
 261                     }
 262                     LOG.debug(currentAddr.getHostAddress());
 263                     return currentAddr;
 264                 }
 265             }
 266         } catch (SocketException se) {
 267             LOG.debug("Caught socket exception", se);
 268         }
 269         return null;
 270     }
 271
 272     @Override
 273     public void onSmrInvokedReceived(SmrEvent event) {
 274         scheduler.smrReceived(event);
 275     }
 276
 277     /**
 278      * Task scheduler is responsible for resending SMR messages to a subscriber (xTR)
 279      * {@value ConfigIni#LISP_SMR_RETRY_COUNT} times, or until {@link ISmrNotificationListener#onSmrInvokedReceived}
 280      * is triggered.
 281      */
 282     private class SmrScheduler {
 283         final int cpuCores = Runtime.getRuntime().availableProcessors();
 284         private final ThreadFactory threadFactory = new ThreadFactoryBuilder()
 285                 .setNameFormat("smr-executor-%d").build();
 286         private final ScheduledExecutorService executor = Executors.newScheduledThreadPool(cpuCores * 2, threadFactory);
 287         private final Map<Eid, Map<Subscriber, ScheduledFuture<?>>> eidFutureMap = Maps.newConcurrentMap();
 288
 289         void scheduleSmrs(MapRequestBuilder mrb, Iterator<Subscriber> subscribers) {
 290             final Eid srcEid = fixSrcEidMask(mrb.getSourceEid().getEid());
 291             cancelExistingFuturesForEid(srcEid);
 292
 293             final Map<Subscriber, ScheduledFuture<?>> subscriberFutureMap = Maps.newConcurrentMap();
 294
 295             // Using Iterator ensures that we don't get a ConcurrentModificationException when removing a Subscriber
 296             // from a Set.
 297             while (subscribers.hasNext()) {
 298                 Subscriber subscriber = subscribers.next();
 299                 if (subscriber.timedOut()) {
 300                     LOG.debug("Lazy removing expired subscriber entry " + subscriber.getString());
 301                     subscribers.remove();
 302                 } else {
 303                     final ScheduledFuture<?> future = executor.scheduleAtFixedRate(new CancellableRunnable(
 304                             mrb, subscriber), 0L, ConfigIni.getInstance().getSmrTimeout(), TimeUnit.MILLISECONDS);
 305                     subscriberFutureMap.put(subscriber, future);
 306                 }
 307             }
 308
 309             if (subscriberFutureMap.isEmpty()) {
 310                 return;
 311             }
 312             eidFutureMap.put(srcEid, subscriberFutureMap);
 313         }
 314
 315         void smrReceived(SmrEvent event) {
 316             final List<Subscriber> subscriberList = event.getSubscriberList();
 317             for (Subscriber subscriber : subscriberList) {
 318                 if (LOG.isTraceEnabled()) {
 319                     LOG.trace("SMR-invoked event, EID {}, subscriber {}",
 320                             LispAddressStringifier.getString(event.getEid()),
 321                             subscriber.getString());
 322                     LOG.trace("eidFutureMap: {}", eidFutureMap);
 323                 }
 324                 final Map<Subscriber, ScheduledFuture<?>> subscriberFutureMap = eidFutureMap.get(event.getEid());
 325                 if (subscriberFutureMap != null) {
 326                     final ScheduledFuture<?> future = subscriberFutureMap.get(subscriber);
 327                     if (future != null && !future.isCancelled()) {
 328                         future.cancel(true);
 329                         if (LOG.isDebugEnabled()) {
 330                             LOG.debug("SMR-invoked MapRequest received, scheduled task for subscriber {}, EID {} with"
 331                                     + " nonce {} has been cancelled", subscriber.getString(),
 332                                     LispAddressStringifier.getString(event.getEid()), event.getNonce());
 333                         }
 334                         subscriberFutureMap.remove(subscriber);
 335                     } else {
 336                         if (future == null) {
 337                             LOG.trace("No outstanding SMR tasks for EID {}, subscriber {}",
 338                                     LispAddressStringifier.getString(event.getEid()), subscriber.getString());
 339                         } else {
 340                             LOG.trace("Future {} is cancelled", future);
 341                         }
 342                     }
 343                     if (subscriberFutureMap.isEmpty()) {
 344                         eidFutureMap.remove(event.getEid());
 345                     }
 346                 } else {
 347                     if (LOG.isTraceEnabled()) {
 348                         LOG.trace("No outstanding SMR tasks for EID {}",
 349                                 LispAddressStringifier.getString(event.getEid()));
 350                     }
 351                 }
 352             }
 353         }
 354
 355         private void cancelExistingFuturesForEid(Eid eid) {
 356             synchronized (eidFutureMap) {
 357                 if (eidFutureMap.containsKey(eid)) {
 358                     final Map<Subscriber, ScheduledFuture<?>> subscriberFutureMap = eidFutureMap.get(eid);
 359                     Iterator<Subscriber> oldSubscribers = subscriberFutureMap.keySet().iterator();
 360                     while (oldSubscribers.hasNext()) {
 361                         Subscriber subscriber = oldSubscribers.next();
 362                         ScheduledFuture<?> subscriberFuture = subscriberFutureMap.get(subscriber);
 363                         subscriberFuture.cancel(true);
 364                     }
 365                     eidFutureMap.remove(eid);
 366                 }
 367             }
 368         }
 369
 370         /*
 371          * See https://bugs.opendaylight.org/show_bug.cgi?id=8469#c1 why this is necessary.
 372          *
 373          * TL;DR  The sourceEid field in the MapRequestBuilder object will be serialized to a packet on the wire, and
 374          * a Map-Request can't store the prefix length in the source EID.
 375          *
 376          * Since we store all prefixes as binary internally, we only care about and fix those address types.
 377          */
 378         private Eid fixSrcEidMask(Eid eid) {
 379             Address address = eid.getAddress();
 380             if (address instanceof Ipv4PrefixBinary) {
 381                 return new EidBuilder(eid).setAddress(new Ipv4PrefixBinaryBuilder((Ipv4PrefixBinary) address)
 382                         .setIpv4MaskLength(MaskUtil.IPV4_MAX_MASK).build()).build();
 383             } else if (address instanceof Ipv6PrefixBinary) {
 384                 return new EidBuilder(eid).setAddress(new Ipv6PrefixBinaryBuilder((Ipv6PrefixBinary) address)
 385                         .setIpv6MaskLength(MaskUtil.IPV6_MAX_MASK).build()).build();
 386             }
 387             return eid;
 388         }
 389
 390         private final class CancellableRunnable implements Runnable {
 391             private final MapRequestBuilder mrb;
 392             private final Subscriber subscriber;
 393             private int executionCount = 1;
 394
 395             CancellableRunnable(MapRequestBuilder mrb, Subscriber subscriber) {
 396                 this.mrb = mrb;
 397                 this.subscriber = subscriber;
 398             }
 399
 400             @SuppressWarnings("checkstyle:IllegalCatch")
 401             @Override
 402             public void run() {
 403                 final Eid srcEid = mrb.getSourceEid().getEid();
 404
 405                 try {
 406                     // The address stored in the SMR's EID record is used as Source EID in the SMR-invoked
 407                     // Map-Request. To ensure consistent behavior it is set to the value used to originally request
 408                     // a given mapping.
 409                     if (executionCount <= ConfigIni.getInstance().getSmrRetryCount()) {
 410                         synchronized (mrb) {
 411                             mrb.setEidItem(new ArrayList<EidItem>());
 412                             mrb.getEidItem().add(new EidItemBuilder().setEid(subscriber.getSrcEid()).build());
 413                             notifyHandler.handleSMR(mrb.build(), subscriber.getSrcRloc());
 414                             if (LOG.isTraceEnabled()) {
 415                                 LOG.trace("Attempt #{} to send SMR to subscriber {} for EID {}",
 416                                         executionCount,
 417                                         subscriber.getString(),
 418                                         LispAddressStringifier.getString(mrb.getSourceEid().getEid()));
 419                             }
 420                         }
 421                     } else {
 422                         LOG.trace("Cancelling execution of a SMR Map-Request after {} failed attempts.",
 423                                 executionCount - 1);
 424                         cancelAndRemove(subscriber, srcEid);
 425                         return;
 426                     }
 427                 } catch (Exception e) {
 428                     LOG.error("Errors encountered while handling SMR:", e);
 429                     cancelAndRemove(subscriber, srcEid);
 430                     return;
 431                 }
 432                 executionCount++;
 433             }
 434
 435             private void cancelAndRemove(Subscriber sub, Eid eid) {
 436                 final Map<Subscriber, ScheduledFuture<?>> subscriberFutureMap = eidFutureMap.get(eid);
 437                 if (subscriberFutureMap == null) {
 438                     LOG.warn("Couldn't find subscriber {} in SMR scheduler internal list", sub);
 439                     return;
 440                 }
 441
 442                 if (subscriberFutureMap.containsKey(sub)) {
 443                     ScheduledFuture<?> eidFuture = subscriberFutureMap.get(sub);
 444                     subscriberFutureMap.remove(sub);
 445                     eidFuture.cancel(false);
 446                 }
 447                 if (subscriberFutureMap.isEmpty()) {
 448                     eidFutureMap.remove(eid);
 449                 }
 450             }
 451         }
 452     }
 453 }