integrationtest/src/test/java/org/opendaylight/lispflowmapping/integrationtest/MultiSiteScenario.java

   1 /**
   2  * Copyright (c) 2016 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.lispflowmapping.integrationtest;
   9
  10 import static org.junit.Assert.assertEquals;
  11 import static org.junit.Assert.assertNotNull;
  12 import static org.junit.Assert.assertTrue;
  13 import static org.junit.Assert.fail;
  14 import static org.opendaylight.lispflowmapping.integrationtest.MappingServiceIntegrationTest.ourAddress;
  15 import static org.opendaylight.lispflowmapping.integrationtest.MultiSiteScenarioUtil.SITE_A;
  16 import static org.opendaylight.lispflowmapping.integrationtest.MultiSiteScenarioUtil.SITE_D5;
  17
  18 import java.util.ArrayList;
  19 import java.util.Collections;
  20 import java.util.List;
  21 import org.apache.commons.lang3.ArrayUtils;
  22 import org.opendaylight.lispflowmapping.integrationtest.MultiSiteScenarioUtil.Site;
  23 import org.opendaylight.lispflowmapping.interfaces.lisp.IFlowMapping;
  24 import org.opendaylight.lispflowmapping.interfaces.mappingservice.IMappingService;
  25 import org.opendaylight.lispflowmapping.lisp.util.LispAddressStringifier;
  26 import org.opendaylight.lispflowmapping.lisp.util.LispAddressUtil;
  27 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
  28 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Prefix;
  29 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.lisp.address.types.rev151105.InstanceIdType;
  30 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.lisp.address.types.rev151105.lisp.address.Address;
  31 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.lisp.address.types.rev151105.lisp.address.address.Ipv4;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.MapReply;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.XtrId;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.eid.container.Eid;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.eid.list.EidItem;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.eid.list.EidItemBuilder;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.locatorrecords.LocatorRecord;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.locatorrecords.LocatorRecordBuilder;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.mapping.record.container.MappingRecord;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.mapping.record.container.MappingRecordBuilder;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.mapping.record.list.MappingRecordItem;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.mapping.record.list.MappingRecordItemBuilder;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.mapregisternotification.MapRegisterBuilder;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.maprequest.ItrRloc;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.maprequest.ItrRlocBuilder;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.maprequest.SourceEidBuilder;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.maprequestnotification.MapRequestBuilder;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.rloc.container.Rloc;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.mappingservice.rev150906.MappingOrigin;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.mappingservice.rev150906.mapping.authkey.container.MappingAuthkey;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.mappingservice.rev150906.mapping.authkey.container.MappingAuthkeyBuilder;
  52 import org.slf4j.Logger;
  53 import org.slf4j.LoggerFactory;
  54
  55
  56 /**
  57  * Contains methods for:
  58  * - checking sites visibility (availability) - simulation of ping command
  59  * - for preparing and processing data structures necessary for creating map request, map register LISP messages.
  60  *
  61  */
  62 class MultiSiteScenario {
  63
  64     private final int DEFAULT_NETWORK_MASK = 24;
  65     private final int IP_MASK = 32;
  66
  67     private final Integer TTL = 1440;
  68     private final String MAP_RECORD_A = "MAP_RECORD_1";
  69
  70     private final Short DEFAULT_MULTICAST_PRIORITY = 255;
  71     private final Short DEFAULT_MULTICAST_WEIGHT = 0;
  72     private final Boolean DEFAULT_LOCAL_LOCATOR = true;
  73     private final Boolean DEFAULT_RLOC_PROBED = false;
  74     private final Boolean DEFAULT_ROUTED = true;
  75
  76     private final MappingAuthkey NULL_AUTH_KEY = new MappingAuthkeyBuilder().setKeyType(0).build();
  77     private final IMappingService mapService;
  78     private final IFlowMapping lms;
  79
  80     private static final Logger LOG = LoggerFactory.getLogger(MultiSiteScenario.class);
  81
  82     enum EidType {
  83         EID_SRC_DST, EID_WITH_PREFIX
  84     }
  85
  86     MultiSiteScenario(final IMappingService mapService, final IFlowMapping lms) {
  87         this.mapService = mapService;
  88         this.lms = lms;
  89     }
  90
  91     private Ipv4Address verifyIpv4Address(final LocatorRecord locatorRecord) {
  92         assertNotNull(locatorRecord);
  93         final Rloc rloc = locatorRecord.getRloc();
  94         assertNotNull(rloc);
  95         final Address address = rloc.getAddress();
  96         assertTrue(address instanceof Ipv4);
  97         return ((Ipv4) address).getIpv4();
  98     }
  99
 100     private List<LocatorRecord> verifyLocatorRecordExists(final MappingRecord mappingRecord) {
 101         final List<LocatorRecord> locatorRecords = mappingRecord.getLocatorRecord();
 102         assertNotNull(locatorRecords);
 103         return locatorRecords;
 104     }
 105
 106     private MappingRecord verifyMappingRecord(MapReply mapReply) {
 107         assertNotNull(mapReply);
 108         final List<MappingRecordItem> mappingRecordItems = mapReply.getMappingRecordItem();
 109         assertNotNull(mappingRecordItems);
 110         assertEquals(1, mappingRecordItems.size());
 111         final MappingRecordItem mappingRecordItem = mappingRecordItems.get(0);
 112         assertNotNull(mappingRecordItem);
 113         final MappingRecord mappingRecord = mappingRecordItem.getMappingRecord();
 114         assertNotNull(mappingRecord);
 115         return mappingRecord;
 116     }
 117
 118     private void emitMapRegisterMessage(final Site dstSite) {
 119         final MapRegisterBuilder mapRegisterBuilder = new MapRegisterBuilder();
 120         final MappingRecordItemBuilder mappingRecordItemBuilder = new MappingRecordItemBuilder();
 121         mappingRecordItemBuilder.setMappingRecordItemId(MAP_RECORD_A);
 122
 123         final MappingRecordBuilder mrb = prepareMappingRecord(EidType.EID_WITH_PREFIX, null, dstSite);
 124         mappingRecordItemBuilder.setMappingRecord(mrb.build());
 125         mapRegisterBuilder.setMappingRecordItem(Collections.singletonList(mappingRecordItemBuilder.build()));
 126
 127         lms.handleMapRegister(mapRegisterBuilder.build());
 128     }
 129
 130     void setCommonAuthentication() {
 131         Eid eid = LispAddressUtil.toEid(new Ipv4Prefix("0.0.0.0/0"), SITE_A.getVNI());
 132         mapService.addAuthenticationKey(eid, NULL_AUTH_KEY);
 133
 134         eid = LispAddressUtil.toEid(new Ipv4Prefix("0.0.0.0/0"), SITE_D5.getVNI());
 135         mapService.addAuthenticationKey(eid, NULL_AUTH_KEY);
 136         sleepForSeconds(1);
 137     }
 138
 139     private Eid toEid(final String destSiteEidPrefix, final InstanceIdType vniValue, final int mask) {
 140         return LispAddressUtil.toEid(new Ipv4Prefix(destSiteEidPrefix + "/" + mask), vniValue);
 141
 142     }
 143
 144     private MapReply emitMapRequestMessage(final String siteFromEidPrefix, final String siteToEidPrefix, final
 145     InstanceIdType vniValue) {
 146         final MapRequestBuilder mapRequestBuilder = new MapRequestBuilder();
 147         final EidItemBuilder eidItemBuilder = new EidItemBuilder();
 148         Eid dstEid = toEid(siteToEidPrefix, vniValue, IP_MASK);
 149
 150         eidItemBuilder.setEid(dstEid);
 151         eidItemBuilder.setEidItemId(siteFromEidPrefix + siteToEidPrefix);
 152         final List<EidItem> eidItem = Collections.singletonList(eidItemBuilder.build());
 153         final Eid srcEid = toEid(siteFromEidPrefix, vniValue, IP_MASK);
 154         mapRequestBuilder.setSourceEid(new SourceEidBuilder().setEid(srcEid).build());
 155         mapRequestBuilder.setEidItem(eidItem);
 156         mapRequestBuilder.setItrRloc(prepareDummyItrRloc());
 157         return lms.handleMapRequest(mapRequestBuilder.build());
 158     }
 159
 160     private List<ItrRloc> prepareDummyItrRloc() {
 161         List<ItrRloc> itrRlocs = new ArrayList<>();
 162         final ItrRlocBuilder itrRlocBuilder = new ItrRlocBuilder();
 163         itrRlocBuilder.setItrRlocId(ourAddress);
 164         itrRlocBuilder.setRloc(LispAddressUtil.asIpv4Rloc(ourAddress));
 165         itrRlocs.add(itrRlocBuilder.build());
 166         return itrRlocs;
 167     }
 168
 169     void storeNorthMappingNegative(final Site dstSite, final MappingRecord.Action action) {
 170         final Ipv4Prefix ipv4Prefix = new Ipv4Prefix(dstSite.getEidPrefix() + "/" + DEFAULT_NETWORK_MASK);
 171         final Eid eidAsIpv4Prefix = LispAddressUtil.toEid(ipv4Prefix, dstSite.getVNI());
 172
 173         final MappingRecordBuilder mrbNegative = prepareMappingRecord(EidType.EID_WITH_PREFIX, null, dstSite);
 174         mrbNegative.setEid(eidAsIpv4Prefix);
 175         mrbNegative.setAction(action);
 176
 177         mapService.addMapping(MappingOrigin.Northbound, eidAsIpv4Prefix, dstSite.getSiteId(), mrbNegative.build());
 178     }
 179
 180     void deleteNorthMappingNegative(final Site dstSite) {
 181         final Ipv4Prefix ipv4Prefix = new Ipv4Prefix(dstSite.getEidPrefix() + "/" + DEFAULT_NETWORK_MASK);
 182         final Eid eidAsIpv4Prefix = LispAddressUtil.toEid(ipv4Prefix, dstSite.getVNI());
 183
 184         mapService.removeMapping(MappingOrigin.Northbound, eidAsIpv4Prefix);
 185     }
 186
 187     void storeNorthMappingSrcDst(final Site srcSite, final Site ... dstSite) {
 188         final MappingRecordBuilder mrb = prepareMappingRecord(EidType.EID_SRC_DST, srcSite,
 189                 dstSite);
 190         mapService.addMapping(MappingOrigin.Northbound, mrb.getEid(), dstSite[0].getSiteId(), mrb.build());
 191     }
 192
 193     void storeNorthMappingIpPrefix(final Site... dstSite) {
 194         final MappingRecordBuilder mrb = prepareMappingRecord(EidType.EID_WITH_PREFIX, null, dstSite);
 195         mapService.addMapping(MappingOrigin.Northbound, mrb.getEid(), dstSite[0].getSiteId(), mrb.build());
 196     }
 197
 198     private void storeDestinationSiteMappingViaSouthbound(final Site dstSite) {
 199         emitMapRegisterMessage(dstSite);
 200     }
 201
 202     private MappingRecordBuilder prepareMappingRecordGeneral(final EidType eidType,
 203                                                              final Site srcSite, final Site dstSite) {
 204         final MappingRecordBuilder mrb = provideCommonMapRecordBuilder();
 205         mrb.setXtrId(new XtrId(ArrayUtils.addAll(dstSite.getSiteId().getValue(), dstSite.getSiteId().getValue())));
 206
 207         Eid eid = null;
 208         if (EidType.EID_SRC_DST.equals(eidType)) {
 209             if (srcSite != null && dstSite != null && srcSite.getEidPrefix() != null && dstSite.getEidPrefix() !=
 210                     null) {
 211                 eid = LispAddressUtil.asSrcDstEid(srcSite.getEidPrefix(), dstSite.getEidPrefix(), DEFAULT_NETWORK_MASK,
 212                         DEFAULT_NETWORK_MASK, dstSite.getVNI().getValue().intValue());
 213             }
 214         }
 215
 216         mrb.setEid(eid == null ? toEid(dstSite.getEidPrefix(), dstSite.getVNI(), DEFAULT_NETWORK_MASK) : eid);
 217         return mrb;
 218     }
 219
 220     private MappingRecordBuilder prepareMappingRecord(final EidType eidType, final Site srcSite, final Site...
 221             dstSites) {
 222         assertTrue(isEidEquals(dstSites));
 223         final MappingRecordBuilder mrb = prepareMappingRecordGeneral(eidType, srcSite, dstSites[0]);
 224         final List<LocatorRecord> locatorRecords = new ArrayList<>();
 225         if (!dstSites[0].isForDeletion) {
 226             for (Site dstSite : dstSites) {
 227                 if (dstSite.getRloc() != null) {
 228                     locatorRecords.add(provideLocatorRecord(LispAddressUtil.toRloc(new Ipv4Address(dstSite.getRloc())),
 229                             dstSite.getRloc(), dstSite.getWeight(), dstSite.getPriority()));
 230                 }
 231             }
 232         }
 233         mrb.setLocatorRecord(locatorRecords);
 234
 235         return mrb;
 236     }
 237
 238     private boolean isEidEquals(final Site... sites) {
 239         String commonEid = null;
 240         for (Site site : sites) {
 241             final String concreteEidPrefix = site.getEidPrefix();
 242             if (!concreteEidPrefix.equals(commonEid)) {
 243                 if (commonEid == null) {
 244                     commonEid = concreteEidPrefix;
 245                 } else {
 246                     return false;
 247                 }
 248             }
 249         }
 250         return true;
 251     }
 252
 253     private LocatorRecord provideLocatorRecord(final Rloc rloc, final String rlocStr, final short weight, final short
 254             priority) {
 255         final LocatorRecordBuilder locatorRecordBuilder = new LocatorRecordBuilder();
 256         locatorRecordBuilder.setRloc(rloc);
 257         locatorRecordBuilder.setLocatorId(rlocStr);
 258         locatorRecordBuilder.setPriority(priority);
 259         locatorRecordBuilder.setWeight(weight);
 260         locatorRecordBuilder.setMulticastPriority(DEFAULT_MULTICAST_PRIORITY);
 261         locatorRecordBuilder.setMulticastWeight(DEFAULT_MULTICAST_WEIGHT);
 262         locatorRecordBuilder.setLocalLocator(DEFAULT_LOCAL_LOCATOR);
 263         locatorRecordBuilder.setRlocProbed(DEFAULT_RLOC_PROBED);
 264         locatorRecordBuilder.setRouted(DEFAULT_ROUTED);
 265         return locatorRecordBuilder.build();
 266     }
 267
 268     private MappingRecordBuilder provideCommonMapRecordBuilder() {
 269         final MappingRecordBuilder mappingRecordBuilder = new MappingRecordBuilder();
 270         mappingRecordBuilder.setRecordTtl(TTL);
 271         mappingRecordBuilder.setAction(MappingRecord.Action.NoAction);
 272         mappingRecordBuilder.setAuthoritative(true);
 273         mappingRecordBuilder.setTimestamp(System.currentTimeMillis());
 274         return mappingRecordBuilder;
 275     }
 276
 277     void deleteNorthMapingSrcDst(final Site srcSite, final Site dstSite) {
 278         final Eid eid = LispAddressUtil.asSrcDstEid(srcSite.getEidPrefix(), dstSite.getEidPrefix(),
 279                 DEFAULT_NETWORK_MASK, DEFAULT_NETWORK_MASK, dstSite.getVNI().getValue().intValue());
 280         mapService.removeMapping(MappingOrigin.Northbound, eid);
 281     }
 282
 283     void deleteSouthboundMappings(final Site dstSite) {
 284         emitMapRegisterMessage(dstSite);
 285
 286     }
 287
 288     void storeSouthboundMappings(final Site ... sites) {
 289         for (Site site : sites) {
 290             storeDestinationSiteMappingViaSouthbound(site);
 291         }
 292     }
 293
 294     boolean isPossibleAssertPingResultImmediately(final boolean expectedPingWorks, final boolean isPartialyWorking,
 295                                                   final String  msg) {
 296         //ping fail is unwanted. ping definitely failed
 297         if (expectedPingWorks && !isPartialyWorking) {
 298             fail(msg);
 299         }
 300
 301         //ping fail is wanted. still can fail later
 302         if (!expectedPingWorks && isPartialyWorking) {
 303             return false;
 304         }
 305
 306         //ping fail is unwanted. still can fail later
 307         if (expectedPingWorks && isPartialyWorking) {
 308             return false;
 309         }
 310
 311         //ping fail is wanted. ping definitely failed.
 312         if (!expectedPingWorks && !isPartialyWorking) {
 313             return true;
 314         }
 315         return false;
 316     }
 317
 318     boolean checkActionAndRloc(final Site dstSite, boolean expectedPingWorks, MapReply mapReplyFromSrcToDst, final
 319                                 Site  ... additionalSitesFromMapping) {
 320         final MappingRecord mappingRecord = verifyMappingRecord(mapReplyFromSrcToDst);
 321         final boolean isNotDroppendSrcDst = !MappingRecord.Action.Drop.equals(mappingRecord.getAction());
 322
 323         if (isPossibleAssertPingResultImmediately(expectedPingWorks, isNotDroppendSrcDst, "Drop action has appeared " +
 324                 "during ping")) {
 325             return true;
 326         }
 327
 328         final List<LocatorRecord> locatorRecords = verifyLocatorRecordExists(mappingRecord);
 329         for (Site expectedTargetSite : concatenateSites(dstSite, additionalSitesFromMapping)) {
 330             boolean expectedTargetFound = false;
 331             for (LocatorRecord locatorRecord : locatorRecords) {
 332                 if (expectedTargetSite.getRloc().equals(rlocToString(locatorRecord))) {
 333                     final Ipv4Address ipv4AddressSrcDst = verifyIpv4Address(locatorRecord);
 334                     final boolean isRlocSrcDstEqual = ipv4AddressSrcDst.getValue().equals(expectedTargetSite.getRloc());
 335                     if (isPossibleAssertPingResultImmediately(expectedPingWorks, isRlocSrcDstEqual, "Unexpected RLOC." +
 336                             "Expected value " + dstSite.getRloc() + ". Real value " + ipv4AddressSrcDst.getValue() +
 337                             ".")) {
 338                         return true;
 339                     }
 340
 341                     final boolean isWeightEquals = expectedTargetSite.getWeight() == locatorRecord.getWeight();
 342                     if (isPossibleAssertPingResultImmediately(expectedPingWorks, isWeightEquals, "Weight isn't equal." +
 343                             "Expected value " + expectedTargetSite.getWeight() + ". Value from mapping" +
 344                             locatorRecord.getWeight() + ".")) {
 345                         return true;
 346                     }
 347
 348                     final boolean isPriorityEquals = expectedTargetSite.getPriority() == locatorRecord.getPriority();
 349                     if (isPossibleAssertPingResultImmediately(expectedPingWorks, isPriorityEquals, "Priority isn't " +
 350                             "equal. Expected value " + expectedTargetSite.getPriority() + ". Value from mapping" +
 351                             locatorRecord.getPriority() + ".")) {
 352                         return true;
 353                     }
 354
 355                     expectedTargetFound = true;
 356                     break;
 357                 }
 358             }
 359             if (isPossibleAssertPingResultImmediately(expectedPingWorks, expectedTargetFound, "Mapping for " +
 360                 expectedTargetSite.getRloc() + " was expected but wasn't returned from mapping service." +
 361                 expectedTargetFound)) {
 362                 return true;
 363             }
 364
 365         }
 366
 367         return false;
 368     }
 369
 370     private String rlocToString(final LocatorRecord locatorRecord) {
 371         return LispAddressStringifier.getString(locatorRecord.getRloc());
 372     }
 373
 374     private Iterable<Site> concatenateSites(final Site dstSite, final Site... additionalSitesFromMapping) {
 375         final List<Site> sites = new ArrayList<>();
 376         sites.add(dstSite);
 377         for (Site additionalSite : additionalSitesFromMapping) {
 378             sites.add(additionalSite);
 379         }
 380         return sites;
 381     }
 382
 383     private void assertPing(final Site srcSite, final int srcHostIndex, final Site dstSite, final int dstHostIndex,
 384                          boolean expectedPingWorks, final Site ... additionalSitesFromMapping) {
 385         final MapReply mapReplyFromSrcToDst = emitMapRequestMessage(srcSite.getHost(srcHostIndex), dstSite.getHost
 386                 (dstHostIndex), dstSite.getVNI());
 387         if (checkActionAndRloc(dstSite, expectedPingWorks, mapReplyFromSrcToDst,
 388                 additionalSitesFromMapping)) {
 389             return;
 390         }
 391
 392         final MapReply mapReplyFromDstToSrc = emitMapRequestMessage(dstSite.getHost(dstHostIndex), srcSite.getHost
 393                 (srcHostIndex), srcSite.getVNI());
 394         if (checkActionAndRloc(srcSite, expectedPingWorks, mapReplyFromDstToSrc)) {
 395             return;
 396         }
 397
 398         final InstanceIdType iidDst = mapReplyFromSrcToDst.getMappingRecordItem().get(0).getMappingRecord().getEid().
 399                 getVirtualNetworkId();
 400         final InstanceIdType iidSrc = mapReplyFromDstToSrc.getMappingRecordItem().get(0).getMappingRecord().getEid().
 401                 getVirtualNetworkId();
 402
 403         final boolean isIIDEqual = iidDst.equals(iidSrc);
 404
 405         if (expectedPingWorks != isIIDEqual) {
 406             fail("IID problem. Dst value " + iidDst.getValue() + ". Src value " + iidSrc.getValue() + ".");
 407         }
 408     }
 409
 410     void assertPingWorks(final Site srcSite, final int srcHostIndex, final Site dstSite, final int dstHostIndex,
 411                          final Site ... additionalSitesFromMapping) {
 412         assertPing(srcSite, srcHostIndex, dstSite, dstHostIndex, true, additionalSitesFromMapping);
 413     }
 414
 415     void assertPingFails(final Site srcSite, final int srcHostIndex, final Site dstSite, final int dstHostIndex) {
 416         assertPing(srcSite, srcHostIndex, dstSite, dstHostIndex, false);
 417     }
 418
 419     private void sleepForSeconds(int seconds) {
 420         try {
 421             Thread.sleep(seconds * 1000);
 422         } catch (InterruptedException e) {
 423             LOG.trace("Interrupted while sleeping", e);
 424         }
 425     }
 426
 427 }