2 * Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.lispflowmapping.integrationtest;
10 import static org.junit.Assert.assertEquals;
11 import static org.junit.Assert.assertNotNull;
12 import static org.junit.Assert.assertTrue;
13 import static org.junit.Assert.fail;
14 import static org.opendaylight.lispflowmapping.integrationtest.MappingServiceIntegrationTest.ourAddress;
15 import static org.opendaylight.lispflowmapping.integrationtest.MultiSiteScenarioUtil.SITE_A;
16 import static org.opendaylight.lispflowmapping.integrationtest.MultiSiteScenarioUtil.SITE_D5;
18 import java.nio.ByteBuffer;
19 import java.util.ArrayList;
20 import java.util.Arrays;
21 import java.util.Collections;
22 import java.util.HashSet;
23 import java.util.List;
26 import org.apache.commons.lang3.ArrayUtils;
27 import org.opendaylight.lispflowmapping.integrationtest.MultiSiteScenarioUtil.Site;
28 import org.opendaylight.lispflowmapping.interfaces.lisp.IFlowMapping;
29 import org.opendaylight.lispflowmapping.interfaces.mappingservice.IMappingService;
30 import org.opendaylight.lispflowmapping.lisp.serializer.MapRequestSerializer;
31 import org.opendaylight.lispflowmapping.lisp.util.LispAddressStringifier;
32 import org.opendaylight.lispflowmapping.lisp.util.LispAddressUtil;
33 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.lisp.address.types.rev151105.InstanceIdType;
34 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.lisp.address.types.rev151105.lisp.address.Address;
35 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.inet.binary.types.rev160303.Ipv4AddressBinary;
36 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.binary.address.types.rev160504.augmented.lisp.address.address.Ipv4Binary;
37 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.MapReply;
38 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.MapRequest;
39 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.XtrId;
40 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.eid.container.Eid;
41 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.eid.list.EidItem;
42 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.eid.list.EidItemBuilder;
43 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.locatorrecords.LocatorRecord;
44 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.locatorrecords.LocatorRecordBuilder;
45 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.mapping.record.container.MappingRecord;
46 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.mapping.record.container.MappingRecordBuilder;
47 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.mapping.record.list.MappingRecordItem;
48 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.mapping.record.list.MappingRecordItemBuilder;
49 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.mapregisternotification.MapRegisterBuilder;
50 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.maprequest.ItrRloc;
51 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.maprequest.ItrRlocBuilder;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.maprequest.SourceEid;
53 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.maprequest.SourceEidBuilder;
54 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.maprequestnotification.MapRequestBuilder;
55 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.lisp.proto.rev151105.rloc.container.Rloc;
56 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.mappingservice.rev150906.MappingOrigin;
57 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.mappingservice.rev150906.mapping.authkey.container.MappingAuthkey;
58 import org.opendaylight.yang.gen.v1.urn.opendaylight.lfm.mappingservice.rev150906.mapping.authkey.container.MappingAuthkeyBuilder;
59 import org.slf4j.Logger;
60 import org.slf4j.LoggerFactory;
62 import com.google.common.net.InetAddresses;
65 * Contains methods for:
66 * - checking sites visibility (availability) - simulation of ping command
67 * - for preparing and processing data structures necessary for creating map request, map register LISP messages.
70 class MultiSiteScenario {
72 private final int DEFAULT_NETWORK_MASK = 24;
73 private final int IP_MASK = 32;
75 private final Integer TTL = 1440;
76 private final String MAP_RECORD_A = "MAP_RECORD_1";
78 private final Short DEFAULT_MULTICAST_PRIORITY = 255;
79 private final Short DEFAULT_MULTICAST_WEIGHT = 0;
80 private final Boolean DEFAULT_LOCAL_LOCATOR = true;
81 private final Boolean DEFAULT_RLOC_PROBED = false;
82 private final Boolean DEFAULT_ROUTED = true;
84 private final MappingAuthkey NULL_AUTH_KEY = new MappingAuthkeyBuilder().setKeyType(0).build();
85 private final IMappingService mapService;
86 private final IFlowMapping lms;
88 private static final Logger LOG = LoggerFactory.getLogger(MultiSiteScenario.class);
91 EID_SRC_DST, EID_WITH_PREFIX
94 MultiSiteScenario(final IMappingService mapService, final IFlowMapping lms) {
95 this.mapService = mapService;
99 private Ipv4AddressBinary verifyIpv4Address(final LocatorRecord locatorRecord) {
100 assertNotNull(locatorRecord);
101 final Rloc rloc = locatorRecord.getRloc();
103 final Address address = rloc.getAddress();
104 assertTrue(address instanceof Ipv4Binary);
105 return ((Ipv4Binary) address).getIpv4Binary();
108 private List<LocatorRecord> verifyLocatorRecordExists(final MappingRecord mappingRecord) {
109 final List<LocatorRecord> locatorRecords = mappingRecord.getLocatorRecord();
110 assertNotNull(locatorRecords);
111 return locatorRecords;
114 private MappingRecord verifyMappingRecord(MapReply mapReply) {
115 assertNotNull(mapReply);
116 final List<MappingRecordItem> mappingRecordItems = mapReply.getMappingRecordItem();
117 assertNotNull(mappingRecordItems);
118 assertEquals(1, mappingRecordItems.size());
119 final MappingRecordItem mappingRecordItem = mappingRecordItems.get(0);
120 assertNotNull(mappingRecordItem);
121 final MappingRecord mappingRecord = mappingRecordItem.getMappingRecord();
122 assertNotNull(mappingRecord);
123 return mappingRecord;
126 private void emitMapRegisterMessage(final Site dstSite, final boolean merge) {
127 final MapRegisterBuilder mapRegisterBuilder = new MapRegisterBuilder();
128 mapRegisterBuilder.setMergeEnabled(merge);
129 final MappingRecordItemBuilder mappingRecordItemBuilder = new MappingRecordItemBuilder();
130 mappingRecordItemBuilder.setMappingRecordItemId(MAP_RECORD_A);
132 final MappingRecordBuilder mrb = prepareMappingRecord(EidType.EID_WITH_PREFIX, null, dstSite);
133 mappingRecordItemBuilder.setMappingRecord(mrb.build());
134 mapRegisterBuilder.setMappingRecordItem(Collections.singletonList(mappingRecordItemBuilder.build()));
136 lms.handleMapRegister(mapRegisterBuilder.build());
139 void setCommonAuthentication() {
140 Eid eid = LispAddressUtil.asIpv4PrefixBinaryEid("0.0.0.0/0", SITE_A.getVNI());
141 mapService.addAuthenticationKey(eid, NULL_AUTH_KEY);
143 eid = LispAddressUtil.asIpv4PrefixBinaryEid("0.0.0.0/0", SITE_D5.getVNI());
144 mapService.addAuthenticationKey(eid, NULL_AUTH_KEY);
148 private Eid toEid(final String destSiteEidPrefix, final InstanceIdType vniValue, final int mask) {
149 return LispAddressUtil.asIpv4PrefixBinaryEid(destSiteEidPrefix + "/" + mask, vniValue);
153 private MapReply emitMapRequestMessage(final String siteFromEidPrefix, final String siteToEidPrefix, final
154 InstanceIdType vniValue) {
155 final MapRequestBuilder mapRequestBuilder = new MapRequestBuilder();
156 final EidItemBuilder eidItemBuilder = new EidItemBuilder();
157 Eid dstEid = toEid(siteToEidPrefix, vniValue, IP_MASK);
159 eidItemBuilder.setEid(dstEid);
160 eidItemBuilder.setEidItemId(siteFromEidPrefix + siteToEidPrefix);
161 final List<EidItem> eidItem = Collections.singletonList(eidItemBuilder.build());
162 final Eid srcEid = toEid(siteFromEidPrefix, vniValue, IP_MASK);
163 mapRequestBuilder.setSourceEid(new SourceEidBuilder().setEid(srcEid).build());
164 mapRequestBuilder.setEidItem(eidItem);
165 mapRequestBuilder.setItrRloc(prepareDummyItrRloc());
166 return lms.handleMapRequest(mapRequestBuilder.build());
169 private List<ItrRloc> prepareDummyItrRloc() {
170 List<ItrRloc> itrRlocs = new ArrayList<>();
171 final ItrRlocBuilder itrRlocBuilder = new ItrRlocBuilder();
172 itrRlocBuilder.setItrRlocId(ourAddress);
173 itrRlocBuilder.setRloc(LispAddressUtil.asIpv4Rloc(ourAddress));
174 itrRlocs.add(itrRlocBuilder.build());
178 void storeNorthMappingNegative(final Site dstSite, final MappingRecord.Action action) {
179 final Eid eidAsIpv4Prefix = LispAddressUtil.asIpv4PrefixBinaryEid(
180 dstSite.getEidPrefix() + "/" + DEFAULT_NETWORK_MASK, dstSite.getVNI());
182 final MappingRecordBuilder mrbNegative = prepareMappingRecord(EidType.EID_WITH_PREFIX, null, dstSite);
183 mrbNegative.setEid(eidAsIpv4Prefix);
184 mrbNegative.setAction(action);
186 mapService.addMapping(MappingOrigin.Northbound, eidAsIpv4Prefix, dstSite.getSiteId(), mrbNegative.build(),
190 void deleteNorthMappingNegative(final Site dstSite) {
191 final Eid eidAsIpv4Prefix = LispAddressUtil.asIpv4PrefixBinaryEid(
192 dstSite.getEidPrefix() + "/" + DEFAULT_NETWORK_MASK, dstSite.getVNI());
194 mapService.removeMapping(MappingOrigin.Northbound, eidAsIpv4Prefix);
197 void storeNorthMappingSrcDst(final Site srcSite, final Site ... dstSite) {
198 final MappingRecordBuilder mrb = prepareMappingRecord(EidType.EID_SRC_DST, srcSite,
200 mapService.addMapping(MappingOrigin.Northbound, mrb.getEid(), dstSite[0].getSiteId(), mrb.build(), false);
203 void storeNorthMappingIpPrefix(final Site... dstSite) {
204 final MappingRecordBuilder mrb = prepareMappingRecord(EidType.EID_WITH_PREFIX, null, dstSite);
205 mapService.addMapping(MappingOrigin.Northbound, mrb.getEid(), dstSite[0].getSiteId(), mrb.build(), false);
208 private void storeDestinationSiteMappingViaSouthbound(final Site dstSite, final boolean merge) {
209 emitMapRegisterMessage(dstSite, merge);
212 private MappingRecordBuilder prepareMappingRecordGeneral(final EidType eidType,
213 final Site srcSite, final Site dstSite) {
214 final MappingRecordBuilder mrb = provideCommonMapRecordBuilder();
215 mrb.setXtrId(new XtrId(ArrayUtils.addAll(dstSite.getSiteId().getValue(), dstSite.getSiteId().getValue())));
218 if (EidType.EID_SRC_DST.equals(eidType)) {
219 if (srcSite != null && dstSite != null && srcSite.getEidPrefix() != null && dstSite.getEidPrefix() !=
221 eid = LispAddressUtil.asSrcDstEid(srcSite.getEidPrefix(), dstSite.getEidPrefix(), DEFAULT_NETWORK_MASK,
222 DEFAULT_NETWORK_MASK, dstSite.getVNI().getValue().intValue());
226 mrb.setEid(eid == null ? toEid(dstSite.getEidPrefix(), dstSite.getVNI(), DEFAULT_NETWORK_MASK) : eid);
230 private MappingRecordBuilder prepareMappingRecord(final EidType eidType, final Site srcSite, final Site...
232 assertTrue(isEidEquals(dstSites));
233 final MappingRecordBuilder mrb = prepareMappingRecordGeneral(eidType, srcSite, dstSites[0]);
234 final List<LocatorRecord> locatorRecords = new ArrayList<>();
235 if (!dstSites[0].isForDeletion) {
236 for (Site dstSite : dstSites) {
237 if (dstSite.getRloc() != null) {
238 locatorRecords.add(provideLocatorRecord(LispAddressUtil.asIpv4Rloc(dstSite.getRloc()),
239 dstSite.getRloc(), dstSite.getWeight(), dstSite.getPriority()));
243 mrb.setLocatorRecord(locatorRecords);
248 private boolean isEidEquals(final Site... sites) {
249 String commonEid = null;
250 for (Site site : sites) {
251 final String concreteEidPrefix = site.getEidPrefix();
252 if (!concreteEidPrefix.equals(commonEid)) {
253 if (commonEid == null) {
254 commonEid = concreteEidPrefix;
263 private LocatorRecord provideLocatorRecord(final Rloc rloc, final String rlocStr, final short weight, final short
265 final LocatorRecordBuilder locatorRecordBuilder = new LocatorRecordBuilder();
266 locatorRecordBuilder.setRloc(rloc);
267 locatorRecordBuilder.setLocatorId(rlocStr);
268 locatorRecordBuilder.setPriority(priority);
269 locatorRecordBuilder.setWeight(weight);
270 locatorRecordBuilder.setMulticastPriority(DEFAULT_MULTICAST_PRIORITY);
271 locatorRecordBuilder.setMulticastWeight(DEFAULT_MULTICAST_WEIGHT);
272 locatorRecordBuilder.setLocalLocator(DEFAULT_LOCAL_LOCATOR);
273 locatorRecordBuilder.setRlocProbed(DEFAULT_RLOC_PROBED);
274 locatorRecordBuilder.setRouted(DEFAULT_ROUTED);
275 return locatorRecordBuilder.build();
278 private MappingRecordBuilder provideCommonMapRecordBuilder() {
279 final MappingRecordBuilder mappingRecordBuilder = new MappingRecordBuilder();
280 mappingRecordBuilder.setRecordTtl(TTL);
281 mappingRecordBuilder.setAction(MappingRecord.Action.NoAction);
282 mappingRecordBuilder.setAuthoritative(true);
283 mappingRecordBuilder.setTimestamp(System.currentTimeMillis());
284 return mappingRecordBuilder;
287 void deleteNorthMapingSrcDst(final Site srcSite, final Site dstSite) {
288 final Eid eid = LispAddressUtil.asSrcDstEid(srcSite.getEidPrefix(), dstSite.getEidPrefix(),
289 DEFAULT_NETWORK_MASK, DEFAULT_NETWORK_MASK, dstSite.getVNI().getValue().intValue());
290 mapService.removeMapping(MappingOrigin.Northbound, eid);
293 void deleteSouthboundMappings(final Site dstSite) {
294 emitMapRegisterMessage(dstSite, false);
298 void storeSouthboundMappings(final boolean merge, final Site ... sites) {
299 for (Site site : sites) {
300 storeDestinationSiteMappingViaSouthbound(site, merge);
304 boolean isPossibleAssertPingResultImmediately(final boolean expectedPingWorks, final boolean isPartialyWorking,
306 //ping fail is unwanted. ping definitely failed
307 if (expectedPingWorks && !isPartialyWorking) {
311 //ping fail is wanted. still can fail later
312 if (!expectedPingWorks && isPartialyWorking) {
316 //ping fail is unwanted. still can fail later
317 if (expectedPingWorks && isPartialyWorking) {
321 //ping fail is wanted. ping definitely failed.
322 if (!expectedPingWorks && !isPartialyWorking) {
328 boolean checkActionAndRloc(final Site dstSite, boolean expectedPingWorks, MapReply mapReplyFromSrcToDst, final
329 Site ... additionalSitesFromMapping) {
330 final MappingRecord mappingRecord = verifyMappingRecord(mapReplyFromSrcToDst);
331 final boolean isNotDroppendSrcDst = !MappingRecord.Action.Drop.equals(mappingRecord.getAction());
333 if (isPossibleAssertPingResultImmediately(expectedPingWorks, isNotDroppendSrcDst, "Drop action has appeared " +
338 final List<LocatorRecord> locatorRecords = verifyLocatorRecordExists(mappingRecord);
339 for (Site expectedTargetSite : concatenateSites(dstSite, additionalSitesFromMapping)) {
340 boolean expectedTargetFound = false;
341 for (LocatorRecord locatorRecord : locatorRecords) {
342 if (expectedTargetSite.getRloc().equals(rlocToString(locatorRecord))) {
343 final byte[] ipv4AddressSrcDst = verifyIpv4Address(locatorRecord).getValue();
344 final byte[] rloc = InetAddresses.forString((expectedTargetSite.getRloc())).getAddress();
345 final boolean isRlocSrcDstEqual = Arrays.equals(ipv4AddressSrcDst, rloc);
346 if (isPossibleAssertPingResultImmediately(expectedPingWorks, isRlocSrcDstEqual, "Unexpected RLOC." +
347 "Expected value " + rloc + ". Real value " + ipv4AddressSrcDst +
352 final boolean isWeightEquals = expectedTargetSite.getWeight() == locatorRecord.getWeight();
353 if (isPossibleAssertPingResultImmediately(expectedPingWorks, isWeightEquals, "Weight isn't equal." +
354 "Expected value " + expectedTargetSite.getWeight() + ". Value from mapping" +
355 locatorRecord.getWeight() + ".")) {
359 final boolean isPriorityEquals = expectedTargetSite.getPriority() == locatorRecord.getPriority();
360 if (isPossibleAssertPingResultImmediately(expectedPingWorks, isPriorityEquals, "Priority isn't " +
361 "equal. Expected value " + expectedTargetSite.getPriority() + ". Value from mapping" +
362 locatorRecord.getPriority() + ".")) {
366 expectedTargetFound = true;
370 if (isPossibleAssertPingResultImmediately(expectedPingWorks, expectedTargetFound, "Mapping for " +
371 expectedTargetSite.getRloc() + " was expected but wasn't returned from mapping service." +
372 expectedTargetFound)) {
381 private String rlocToString(final LocatorRecord locatorRecord) {
382 return LispAddressStringifier.getString(locatorRecord.getRloc());
385 private Iterable<Site> concatenateSites(final Site dstSite, final Site... additionalSitesFromMapping) {
386 final List<Site> sites = new ArrayList<>();
388 for (Site additionalSite : additionalSitesFromMapping) {
389 sites.add(additionalSite);
394 private void assertPing(final Site srcSite, final int srcHostIndex, final Site dstSite, final int dstHostIndex,
395 boolean expectedPingWorks, final Site ... additionalSitesFromMapping) {
396 final MapReply mapReplyFromSrcToDst = emitMapRequestMessage(srcSite.getHost(srcHostIndex), dstSite.getHost
397 (dstHostIndex), dstSite.getVNI());
398 if (checkActionAndRloc(dstSite, expectedPingWorks, mapReplyFromSrcToDst, additionalSitesFromMapping)) {
402 final MapReply mapReplyFromDstToSrc = emitMapRequestMessage(dstSite.getHost(dstHostIndex), srcSite.getHost
403 (srcHostIndex), srcSite.getVNI());
404 if (checkActionAndRloc(srcSite, expectedPingWorks, mapReplyFromDstToSrc)) {
408 final InstanceIdType iidDst = mapReplyFromSrcToDst.getMappingRecordItem().get(0).getMappingRecord().getEid().
409 getVirtualNetworkId();
410 final InstanceIdType iidSrc = mapReplyFromDstToSrc.getMappingRecordItem().get(0).getMappingRecord().getEid().
411 getVirtualNetworkId();
413 final boolean isIIDEqual = iidDst.equals(iidSrc);
415 if (expectedPingWorks != isIIDEqual) {
416 fail("IID problem. Dst value " + iidDst.getValue() + ". Src value " + iidSrc.getValue() + ".");
420 void assertPingWorks(final Site srcSite, final int srcHostIndex, final Site dstSite, final int dstHostIndex,
421 final Site ... additionalSitesFromMapping) {
422 assertPing(srcSite, srcHostIndex, dstSite, dstHostIndex, true, additionalSitesFromMapping);
425 void assertPingFails(final Site srcSite, final int srcHostIndex, final Site dstSite, final int dstHostIndex) {
426 assertPing(srcSite, srcHostIndex, dstSite, dstHostIndex, false);
429 private void sleepForSeconds(int seconds) {
431 Thread.sleep(seconds * 1000);
432 } catch (InterruptedException e) {
433 LOG.trace("Interrupted while sleeping", e);
438 private List<MapRequest> translateBuffersToMapRequest(byte[][] buffers) {
439 final List<MapRequest> mapRequests = new ArrayList<>();
440 for (byte[] buffer : buffers) {
441 final MapRequest mapRequest = MapRequestSerializer.getInstance().deserialize(ByteBuffer.wrap(buffer));
442 assertNotNull(mapRequest);
443 mapRequests.add(mapRequest);
448 private Set<Eid> prepareExpectedEid(final String ... hosts) {
449 final Set<Eid> eids = new HashSet<>();
450 for (String host : hosts) {
451 eids.add(LispAddressUtil.asIpv4PrefixBinaryEid(host + "/" + IP_MASK,
452 new InstanceIdType(MultiSiteScenarioUtil.VNI2)));
457 private SourceEid prepareSourceEid(final String eidPrefix) {
458 final SourceEidBuilder sourceEidBuilder = new SourceEidBuilder();
459 final Eid eid = LispAddressUtil.asIpv4Eid(eidPrefix, MultiSiteScenarioUtil.VNI2);
460 return sourceEidBuilder.setEid(eid).build();
463 void checkSMR(final SocketReader socketReader, final String site, final String ... hosts) {
464 List<MapRequest> mapRequests = translateBuffersToMapRequest(socketReader.getBuffers(hosts.length));
465 final Set<Eid> eids = prepareExpectedEid(hosts);
466 final SourceEid expectedSourceEid = prepareSourceEid(site);
467 for(MapRequest mapRequest : mapRequests) {
468 assertTrue(mapRequest.isSmr());
469 final SourceEid receivedSourceEid = mapRequest.getSourceEid();
470 assertEquals(expectedSourceEid, receivedSourceEid);
471 final List<EidItem> currentEidItems = mapRequest.getEidItem();
472 assertNotNull(currentEidItems);
473 assertTrue(SMRContainsExpectedEid(eids, currentEidItems));
475 //all expected eids should be after looping via mapRequests matched.
476 assertTrue("Expected eids wasn't/weren't found " + eids, eids.isEmpty());
479 private boolean SMRContainsExpectedEid(Set<Eid> eids, List<EidItem> currentEidItems) {
480 for (EidItem eidItem : currentEidItems) {
481 //if eid from map request is matched then it is removed from set of expected eids
482 if (!eids.remove(eidItem.getEid())) {
483 fail("SMR contained " + eidItem.getEid() + " which wasn't expected.");