BUG-7594: Rework sal-remoterpc-connector messages
[controller.git] / opendaylight / md-sal / sal-remoterpc-connector / src / test / java / org / opendaylight / controller / remote / rpc / registry / RpcRegistryTest.java
index ab609413dd82731466495365d0c46f28731fcdc0..84062735c22ddf5a3cbd6a40e74e64466d745e5d 100644 (file)
+/*
+ * Copyright (c) 2014, 2015 Cisco Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+
 package org.opendaylight.controller.remote.rpc.registry;
 
-import akka.actor.ActorPath;
+import static org.junit.Assert.fail;
+import static org.opendaylight.controller.remote.rpc.registry.gossip.BucketStoreAccess.Singletons.GET_ALL_BUCKETS;
+import static org.opendaylight.controller.remote.rpc.registry.gossip.BucketStoreAccess.Singletons.GET_BUCKET_VERSIONS;
+
 import akka.actor.ActorRef;
-import akka.actor.ActorSelection;
 import akka.actor.ActorSystem;
-import akka.actor.ChildActorPath;
-import akka.actor.Props;
-import akka.japi.Pair;
+import akka.actor.Address;
+import akka.cluster.Cluster;
+import akka.cluster.ClusterEvent.CurrentClusterState;
+import akka.cluster.Member;
+import akka.cluster.MemberStatus;
+import akka.cluster.UniqueAddress;
 import akka.testkit.JavaTestKit;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.Uninterruptibles;
 import com.typesafe.config.ConfigFactory;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
-import org.opendaylight.controller.remote.rpc.RouteIdentifierImpl;
-import org.opendaylight.controller.sal.connector.api.RpcRouter;
+import org.opendaylight.controller.cluster.common.actor.AkkaConfigurationReader;
+import org.opendaylight.controller.md.sal.dom.api.DOMRpcIdentifier;
+import org.opendaylight.controller.remote.rpc.RemoteRpcProviderConfig;
+import org.opendaylight.controller.remote.rpc.registry.RpcRegistry.Messages.AddOrUpdateRoutes;
+import org.opendaylight.controller.remote.rpc.registry.RpcRegistry.Messages.RemoveRoutes;
+import org.opendaylight.controller.remote.rpc.registry.RpcRegistry.Messages.UpdateRemoteEndpoints;
+import org.opendaylight.controller.remote.rpc.registry.RpcRegistry.RemoteRpcEndpoint;
+import org.opendaylight.controller.remote.rpc.registry.gossip.Bucket;
 import org.opendaylight.yangtools.yang.common.QName;
-import scala.concurrent.Await;
-import scala.concurrent.Future;
+import org.opendaylight.yangtools.yang.model.api.SchemaPath;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import scala.concurrent.duration.Duration;
 import scala.concurrent.duration.FiniteDuration;
 
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import static org.opendaylight.controller.remote.rpc.registry.RpcRegistry.Messages.AddOrUpdateRoute;
-import static org.opendaylight.controller.remote.rpc.registry.RpcRegistry.Messages.FindRouters;
-import static org.opendaylight.controller.remote.rpc.registry.RpcRegistry.Messages.FindRoutersReply;
-import static org.opendaylight.controller.remote.rpc.registry.RpcRegistry.Messages.SetLocalRouter;
-
 public class RpcRegistryTest {
+    private static final Logger LOG = LoggerFactory.getLogger(RpcRegistryTest.class);
 
     private static ActorSystem node1;
     private static ActorSystem node2;
     private static ActorSystem node3;
 
+    private JavaTestKit invoker1;
+    private JavaTestKit invoker2;
+    private JavaTestKit invoker3;
+    private JavaTestKit registrar1;
+    private JavaTestKit registrar2;
+    private JavaTestKit registrar3;
     private ActorRef registry1;
     private ActorRef registry2;
     private ActorRef registry3;
 
+    private int routeIdCounter = 1;
+
     @BeforeClass
-    public static void setup() throws InterruptedException {
-        Thread.sleep(1000); //give some time for previous test to close netty ports
-        node1 = ActorSystem.create("opendaylight-rpc", ConfigFactory.load().getConfig("memberA"));
-        node2 = ActorSystem.create("opendaylight-rpc", ConfigFactory.load().getConfig("memberB"));
-        node3 = ActorSystem.create("opendaylight-rpc", ConfigFactory.load().getConfig("memberC"));
+    public static void staticSetup() throws InterruptedException {
+        AkkaConfigurationReader reader = ConfigFactory::load;
+
+        RemoteRpcProviderConfig config1 = new RemoteRpcProviderConfig.Builder("memberA").gossipTickInterval("200ms")
+                .withConfigReader(reader).build();
+        RemoteRpcProviderConfig config2 = new RemoteRpcProviderConfig.Builder("memberB").gossipTickInterval("200ms")
+                .withConfigReader(reader).build();
+        RemoteRpcProviderConfig config3 = new RemoteRpcProviderConfig.Builder("memberC").gossipTickInterval("200ms")
+                .withConfigReader(reader).build();
+        node1 = ActorSystem.create("opendaylight-rpc", config1.get());
+        node2 = ActorSystem.create("opendaylight-rpc", config2.get());
+        node3 = ActorSystem.create("opendaylight-rpc", config3.get());
+
+        waitForMembersUp(node1, Cluster.get(node2).selfUniqueAddress(), Cluster.get(node3).selfUniqueAddress());
+        waitForMembersUp(node2, Cluster.get(node1).selfUniqueAddress(), Cluster.get(node3).selfUniqueAddress());
+    }
+
+    static void waitForMembersUp(final ActorSystem node, final UniqueAddress... addresses) {
+        Set<UniqueAddress> otherMembersSet = Sets.newHashSet(addresses);
+        Stopwatch sw = Stopwatch.createStarted();
+        while (sw.elapsed(TimeUnit.SECONDS) <= 10) {
+            CurrentClusterState state = Cluster.get(node).state();
+            for (Member m : state.getMembers()) {
+                if (m.status() == MemberStatus.up() && otherMembersSet.remove(m.uniqueAddress())
+                        && otherMembersSet.isEmpty()) {
+                    return;
+                }
+            }
+
+            Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
+        }
+
+        fail("Member(s) " + otherMembersSet + " are not Up");
     }
 
     @AfterClass
-    public static void teardown(){
+    public static void staticTeardown() {
         JavaTestKit.shutdownActorSystem(node1);
         JavaTestKit.shutdownActorSystem(node2);
         JavaTestKit.shutdownActorSystem(node3);
-        if (node1 != null)
-            node1.shutdown();
-        if (node2 != null)
-            node2.shutdown();
-        if (node3 != null)
-            node3.shutdown();
-
     }
 
     @Before
-    public void createRpcRegistry() throws InterruptedException {
-        registry1 = node1.actorOf(Props.create(RpcRegistry.class));
-        registry2 = node2.actorOf(Props.create(RpcRegistry.class));
-        registry3 = node3.actorOf(Props.create(RpcRegistry.class));
+    public void setup() {
+        invoker1 = new JavaTestKit(node1);
+        registrar1 = new JavaTestKit(node1);
+        registry1 = node1.actorOf(RpcRegistry.props(config(node1), invoker1.getRef(), registrar1.getRef()));
+        invoker2 = new JavaTestKit(node2);
+        registrar2 = new JavaTestKit(node2);
+        registry2 = node2.actorOf(RpcRegistry.props(config(node2), invoker2.getRef(), registrar2.getRef()));
+        invoker3 = new JavaTestKit(node3);
+        registrar3 = new JavaTestKit(node3);
+        registry3 = node3.actorOf(RpcRegistry.props(config(node3), invoker3.getRef(), registrar3.getRef()));
+    }
+
+    private static RemoteRpcProviderConfig config(final ActorSystem node) {
+        return new RemoteRpcProviderConfig(node.settings().config());
     }
 
     @After
-    public void stopRpcRegistry() throws InterruptedException {
-        if (registry1 != null)
+    public void teardown() {
+        if (registry1 != null) {
             node1.stop(registry1);
-        if (registry2 != null)
+        }
+        if (registry2 != null) {
             node2.stop(registry2);
-        if (registry3 != null)
+        }
+        if (registry3 != null) {
             node3.stop(registry3);
+        }
+
+        if (invoker1 != null) {
+            node1.stop(invoker1.getRef());
+        }
+        if (invoker2 != null) {
+            node2.stop(invoker2.getRef());
+        }
+        if (invoker3 != null) {
+            node3.stop(invoker3.getRef());
+        }
+
+        if (registrar1 != null) {
+            node1.stop(registrar1.getRef());
+        }
+        if (registrar2 != null) {
+            node2.stop(registrar2.getRef());
+        }
+        if (registrar3 != null) {
+            node3.stop(registrar3.getRef());
+        }
     }
 
     /**
-     * One node cluster.
-     * Register rpc. Ensure router can be found
-     *
-     * @throws URISyntaxException
-     * @throws InterruptedException
+     * One node cluster. 1. Register rpc, ensure router can be found 2. Then remove rpc, ensure its
+     * deleted
      */
     @Test
-    public void testWhenRpcAddedOneNodeShouldAppearOnSameNode() throws URISyntaxException, InterruptedException {
+    public void testAddRemoveRpcOnSameNode() throws Exception {
+        LOG.info("testAddRemoveRpcOnSameNode starting");
 
-        final JavaTestKit mockBroker = new JavaTestKit(node1);
+        Address nodeAddress = node1.provider().getDefaultAddress();
 
-        //Add rpc on node 1
-        registry1.tell(new SetLocalRouter(mockBroker.getRef()), mockBroker.getRef());
-        registry1.tell(getAddRouteMessage(), mockBroker.getRef());
+        // Add rpc on node 1
 
-        Thread.sleep(1000);//
+        List<DOMRpcIdentifier> addedRouteIds = createRouteIds();
 
-        //find the route on node 1's registry
-        registry1.tell(new FindRouters(createRouteId()), mockBroker.getRef());
-        FindRoutersReply message = mockBroker.expectMsgClass(JavaTestKit.duration("10 second"), FindRoutersReply.class);
-        List<Pair<ActorRef, Long>> pairs = message.getRouterWithUpdateTime();
+        registry1.tell(new AddOrUpdateRoutes(addedRouteIds), ActorRef.noSender());
+
+        // Bucket store should get an update bucket message. Updated bucket contains added rpc.
+        final JavaTestKit testKit = new JavaTestKit(node1);
+
+        Map<Address, Bucket<RoutingTable>> buckets = retrieveBuckets(registry1, testKit, nodeAddress);
+        verifyBucket(buckets.get(nodeAddress), addedRouteIds);
+
+        Map<Address, Long> versions = retrieveVersions(registry1, testKit);
+        Assert.assertEquals("Version for bucket " + nodeAddress, (Long) buckets.get(nodeAddress).getVersion(),
+                versions.get(nodeAddress));
+
+        // Now remove rpc
+        registry1.tell(new RemoveRoutes(addedRouteIds), ActorRef.noSender());
+
+        // Bucket store should get an update bucket message. Rpc is removed in the updated bucket
+
+        verifyEmptyBucket(testKit, registry1, nodeAddress);
+
+        LOG.info("testAddRemoveRpcOnSameNode ending");
 
-        validateRouterReceived(pairs, mockBroker.getRef());
     }
 
     /**
-     * Three node cluster.
-     * Register rpc on 1 node. Ensure its router can be found on other 2.
-     *
-     * @throws URISyntaxException
-     * @throws InterruptedException
+     * Three node cluster. 1. Register rpc on 1 node, ensure 2nd node gets updated 2. Remove rpc on
+     * 1 node, ensure 2nd node gets updated
      */
     @Test
-    public void testWhenRpcAddedOneNodeShouldAppearOnAnother() throws URISyntaxException, InterruptedException {
+    public void testRpcAddRemoveInCluster() throws Exception {
 
-        validateSystemStartup();
+        LOG.info("testRpcAddRemoveInCluster starting");
 
-        final JavaTestKit mockBroker1 = new JavaTestKit(node1);
-        final JavaTestKit mockBroker2 = new JavaTestKit(node2);
-        final JavaTestKit mockBroker3 = new JavaTestKit(node3);
+        List<DOMRpcIdentifier> addedRouteIds = createRouteIds();
 
-        //Add rpc on node 1
-        registry1.tell(new SetLocalRouter(mockBroker1.getRef()), mockBroker1.getRef());
-        registry1.tell(getAddRouteMessage(), mockBroker1.getRef());
+        Address node1Address = node1.provider().getDefaultAddress();
 
-        Thread.sleep(5000);// give some time for bucket store data sync
+        // Add rpc on node 1
+        registry1.tell(new AddOrUpdateRoutes(addedRouteIds), ActorRef.noSender());
 
-        //find the route in node 2's registry
-        registry2.tell(new FindRouters(createRouteId()), mockBroker2.getRef());
-        FindRoutersReply message = mockBroker2.expectMsgClass(JavaTestKit.duration("10 second"), FindRoutersReply.class);
-        List<Pair<ActorRef, Long>> pairs = message.getRouterWithUpdateTime();
+        // Bucket store on node2 should get a message to update its local copy of remote buckets
+        final JavaTestKit testKit = new JavaTestKit(node2);
 
-        validateRouterReceived(pairs, mockBroker1.getRef());
+        Map<Address, Bucket<RoutingTable>> buckets = retrieveBuckets(registry2, testKit, node1Address);
+        verifyBucket(buckets.get(node1Address), addedRouteIds);
 
-        //find the route in node 3's registry
-        registry3.tell(new FindRouters(createRouteId()), mockBroker3.getRef());
-        message = mockBroker3.expectMsgClass(JavaTestKit.duration("10 second"), FindRoutersReply.class);
-        pairs = message.getRouterWithUpdateTime();
+        // Now remove
+        registry1.tell(new RemoveRoutes(addedRouteIds), ActorRef.noSender());
 
-        validateRouterReceived(pairs, mockBroker1.getRef());
+        // Bucket store on node2 should get a message to update its local copy of remote buckets.
+        // Wait for the bucket for node1 to be empty.
 
+        verifyEmptyBucket(testKit, registry2, node1Address);
+
+        LOG.info("testRpcAddRemoveInCluster ending");
+    }
+
+    private void verifyEmptyBucket(final JavaTestKit testKit, final ActorRef registry, final Address address)
+            throws AssertionError {
+        Map<Address, Bucket<RoutingTable>> buckets;
+        int numTries = 0;
+        while (true) {
+            buckets = retrieveBuckets(registry1, testKit, address);
+
+            try {
+                verifyBucket(buckets.get(address), Collections.emptyList());
+                break;
+            } catch (AssertionError e) {
+                if (++numTries >= 50) {
+                    throw e;
+                }
+            }
+
+            Uninterruptibles.sleepUninterruptibly(200, TimeUnit.MILLISECONDS);
+        }
     }
 
     /**
-     * Three node cluster.
-     * Register rpc on 2 nodes. Ensure 2 routers are found on 3rd.
-     *
-     * @throws Exception
+     * Three node cluster. Register rpc on 2 nodes. Ensure 3rd gets updated.
      */
     @Test
-    public void testAnRpcAddedOnMultiNodesShouldReturnMultiRouter() throws Exception {
+    public void testRpcAddedOnMultiNodes() throws Exception {
+        final JavaTestKit testKit = new JavaTestKit(node3);
 
-        validateSystemStartup();
+        // Add rpc on node 1
+        List<DOMRpcIdentifier> addedRouteIds1 = createRouteIds();
+        registry1.tell(new AddOrUpdateRoutes(addedRouteIds1), ActorRef.noSender());
 
-        final JavaTestKit mockBroker1 = new JavaTestKit(node1);
-        final JavaTestKit mockBroker2 = new JavaTestKit(node2);
-        final JavaTestKit mockBroker3 = new JavaTestKit(node3);
+        final UpdateRemoteEndpoints req1 = registrar3.expectMsgClass(Duration.create(3, TimeUnit.SECONDS),
+            UpdateRemoteEndpoints.class);
 
-        //Thread.sleep(5000);//let system come up
+        // Add rpc on node 2
+        List<DOMRpcIdentifier> addedRouteIds2 = createRouteIds();
+        registry2.tell(new AddOrUpdateRoutes(addedRouteIds2), ActorRef.noSender());
 
-        //Add rpc on node 1
-        registry1.tell(new SetLocalRouter(mockBroker1.getRef()), mockBroker1.getRef());
-        registry1.tell(getAddRouteMessage(), mockBroker1.getRef());
+        final UpdateRemoteEndpoints req2 = registrar3.expectMsgClass(Duration.create(3, TimeUnit.SECONDS),
+            UpdateRemoteEndpoints.class);
+        Address node2Address = node2.provider().getDefaultAddress();
+        Address node1Address = node1.provider().getDefaultAddress();
 
-        //Add same rpc on node 2
-        registry2.tell(new SetLocalRouter(mockBroker2.getRef()), mockBroker2.getRef());
-        registry2.tell(getAddRouteMessage(), mockBroker2.getRef());
+        Map<Address, Bucket<RoutingTable>> buckets = retrieveBuckets(registry3, testKit, node1Address,
+                node2Address);
 
-        registry3.tell(new SetLocalRouter(mockBroker3.getRef()), mockBroker3.getRef());
-        Thread.sleep(5000);// give some time for bucket store data sync
+        verifyBucket(buckets.get(node1Address), addedRouteIds1);
+        verifyBucket(buckets.get(node2Address), addedRouteIds2);
 
-        //find the route in node 3's registry
-        registry3.tell(new FindRouters(createRouteId()), mockBroker3.getRef());
-        FindRoutersReply message = mockBroker3.expectMsgClass(JavaTestKit.duration("10 second"), FindRoutersReply.class);
-        List<Pair<ActorRef, Long>> pairs = message.getRouterWithUpdateTime();
+        Map<Address, Long> versions = retrieveVersions(registry3, testKit);
+        Assert.assertEquals("Version for bucket " + node1Address, (Long) buckets.get(node1Address).getVersion(),
+                versions.get(node1Address));
+        Assert.assertEquals("Version for bucket " + node2Address, (Long) buckets.get(node2Address).getVersion(),
+                versions.get(node2Address));
 
-        validateMultiRouterReceived(pairs, mockBroker1.getRef(), mockBroker2.getRef());
+        assertEndpoints(req1, node1Address, invoker1);
+        assertEndpoints(req2, node2Address, invoker2);
 
     }
 
-    private void validateMultiRouterReceived(List<Pair<ActorRef, Long>> actual, ActorRef... expected) {
-        Assert.assertTrue(actual != null);
-        Assert.assertTrue(actual.size() == expected.length);
-    }
+    private static void assertEndpoints(final UpdateRemoteEndpoints msg, final Address address,
+            final JavaTestKit invoker) {
+        final Map<Address, Optional<RemoteRpcEndpoint>> endpoints = msg.getEndpoints();
+        Assert.assertEquals(1, endpoints.size());
 
-    private void validateRouterReceived(List<Pair<ActorRef, Long>> actual, ActorRef expected){
-        Assert.assertTrue(actual != null);
-        Assert.assertTrue(actual.size() == 1);
+        final Optional<RemoteRpcEndpoint> maybeEndpoint = endpoints.get(address);
+        Assert.assertNotNull(maybeEndpoint);
+        Assert.assertTrue(maybeEndpoint.isPresent());
 
-        for (Pair<ActorRef, Long> pair : actual){
-            Assert.assertTrue(expected.path().uid() == pair.first().path().uid());
-        }
+        final RemoteRpcEndpoint endpoint = maybeEndpoint.get();
+        final ActorRef router = endpoint.getRouter();
+        Assert.assertNotNull(router);
+
+        router.tell("hello", ActorRef.noSender());
+        final String s = invoker.expectMsgClass(Duration.create(3, TimeUnit.SECONDS), String.class);
+        Assert.assertEquals("hello", s);
     }
 
-    private void validateSystemStartup() throws InterruptedException {
+    private static Map<Address, Long> retrieveVersions(final ActorRef bucketStore, final JavaTestKit testKit) {
+        bucketStore.tell(GET_BUCKET_VERSIONS, testKit.getRef());
+        @SuppressWarnings("unchecked")
+        final Map<Address, Long> reply = testKit.expectMsgClass(Duration.create(3, TimeUnit.SECONDS), Map.class);
+        return reply;
+    }
 
-        Thread.sleep(5000);
-        ActorPath gossiper1Path = new ChildActorPath(new ChildActorPath(registry1.path(), "store"), "gossiper");
-        ActorPath gossiper2Path = new ChildActorPath(new ChildActorPath(registry2.path(), "store"), "gossiper");
-        ActorPath gossiper3Path = new ChildActorPath(new ChildActorPath(registry3.path(), "store"), "gossiper");
+    private static void verifyBucket(final Bucket<RoutingTable> bucket, final List<DOMRpcIdentifier> expRouteIds) {
+        RoutingTable table = bucket.getData();
+        Assert.assertNotNull("Bucket RoutingTable is null", table);
+        for (DOMRpcIdentifier r : expRouteIds) {
+            if (!table.contains(r)) {
+                Assert.fail("RoutingTable does not contain " + r + ". Actual: " + table);
+            }
+        }
 
-        ActorSelection gossiper1 = node1.actorSelection(gossiper1Path);
-        ActorSelection gossiper2 = node2.actorSelection(gossiper2Path);
-        ActorSelection gossiper3 = node3.actorSelection(gossiper3Path);
+        Assert.assertEquals("RoutingTable size", expRouteIds.size(), table.size());
+    }
 
+    private static Map<Address, Bucket<RoutingTable>> retrieveBuckets(final ActorRef bucketStore,
+            final JavaTestKit testKit, final Address... addresses) {
+        int numTries = 0;
+        while (true) {
+            bucketStore.tell(GET_ALL_BUCKETS, testKit.getRef());
+            @SuppressWarnings("unchecked")
+            Map<Address, Bucket<RoutingTable>> buckets = testKit.expectMsgClass(Duration.create(3, TimeUnit.SECONDS),
+                    Map.class);
+
+            boolean foundAll = true;
+            for (Address addr : addresses) {
+                Bucket<RoutingTable> bucket = buckets.get(addr);
+                if (bucket == null) {
+                    foundAll = false;
+                    break;
+                }
+            }
 
-        if (!resolveReference(gossiper1, gossiper2, gossiper3))
-            Assert.fail("Could not find gossipers");
-    }
+            if (foundAll) {
+                return buckets;
+            }
 
-    private Boolean resolveReference(ActorSelection... gossipers) throws InterruptedException {
+            if (++numTries >= 50) {
+                Assert.fail("Missing expected buckets for addresses: " + Arrays.toString(addresses)
+                        + ", Actual: " + buckets);
+            }
 
-        Boolean resolved = true;
+            Uninterruptibles.sleepUninterruptibly(200, TimeUnit.MILLISECONDS);
+        }
+    }
 
-        for (int i=0; i< 5; i++) {
-            Thread.sleep(1000);
-            for (ActorSelection gossiper : gossipers) {
-                Future<ActorRef> future = gossiper.resolveOne(new FiniteDuration(5000, TimeUnit.MILLISECONDS));
+    @Test
+    public void testAddRoutesConcurrency() throws Exception {
+        final JavaTestKit testKit = new JavaTestKit(node1);
+
+        final int nRoutes = 500;
+        final Collection<DOMRpcIdentifier> added = new ArrayList<>(nRoutes);
+        for (int i = 0; i < nRoutes; i++) {
+            final DOMRpcIdentifier routeId = DOMRpcIdentifier.create(SchemaPath.create(true,
+                    new QName(new URI("/mockrpc"), "type" + i)));
+            added.add(routeId);
+
+            //Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
+            registry1.tell(new AddOrUpdateRoutes(Arrays.asList(routeId)),
+                    ActorRef.noSender());
+        }
 
-                ActorRef ref = null;
-                try {
-                    ref = Await.result(future, new FiniteDuration(10000, TimeUnit.MILLISECONDS));
-                } catch (Exception e) {
-                    e.printStackTrace();
+        FiniteDuration duration = Duration.create(3, TimeUnit.SECONDS);
+        int numTries = 0;
+        while (true) {
+            registry1.tell(GET_ALL_BUCKETS, testKit.getRef());
+            @SuppressWarnings("unchecked")
+            Map<Address, Bucket<RoutingTable>> buckets = testKit.expectMsgClass(duration, Map.class);
+
+            Bucket<RoutingTable> localBucket = buckets.values().iterator().next();
+            RoutingTable table = localBucket.getData();
+            if (table != null && table.size() == nRoutes) {
+                for (DOMRpcIdentifier r : added) {
+                    Assert.assertTrue("RoutingTable contains " + r, table.contains(r));
                 }
 
-                if (ref == null)
-                    resolved = false;
+                break;
             }
 
-            if (resolved) break;
-        }
-        return resolved;
-    }
+            if (++numTries >= 50) {
+                Assert.fail("Expected # routes: " + nRoutes + ", Actual: " + table.size());
+            }
 
-    private AddOrUpdateRoute getAddRouteMessage() throws URISyntaxException {
-        return new AddOrUpdateRoute(createRouteId());
+            Uninterruptibles.sleepUninterruptibly(200, TimeUnit.MILLISECONDS);
+        }
     }
 
-    private RpcRouter.RouteIdentifier<?,?,?> createRouteId() throws URISyntaxException {
-        QName type = new QName(new URI("/mockrpc"), "mockrpc");
-        return new RouteIdentifierImpl(null, type, null);
+    private List<DOMRpcIdentifier> createRouteIds() throws URISyntaxException {
+        QName type = new QName(new URI("/mockrpc"), "mockrpc" + routeIdCounter++);
+        List<DOMRpcIdentifier> routeIds = new ArrayList<>(1);
+        routeIds.add(DOMRpcIdentifier.create(SchemaPath.create(true, type)));
+        return routeIds;
     }
-}
\ No newline at end of file
+}