+ // Bucket store should get an update bucket message. Updated bucket contains added rpc.
+ probe1.expectMsgClass(FiniteDuration.apply(10, TimeUnit.SECONDS),
+ Messages.BucketStoreMessages.UpdateBucket.class);
+
+ // Now remove rpc
+ registry1.tell(getRemoveRouteMessage(), mockBroker.getRef());
+
+ // Bucket store should get an update bucket message. Rpc is removed in the updated bucket
+ probe1.expectMsgClass(FiniteDuration.apply(10, TimeUnit.SECONDS),
+ Messages.BucketStoreMessages.UpdateBucket.class);
+
+ System.out.println("testAddRemoveRpcOnSameNode ending");
+
+ }
+
+ /**
+ * Three node cluster. 1. Register rpc on 1 node, ensure 2nd node gets updated 2. Remove rpc on
+ * 1 node, ensure 2nd node gets updated
+ *
+ * @throws URISyntaxException
+ * @throws InterruptedException
+ */
+ @Test
+ public void testRpcAddRemoveInCluster() throws Exception {
+
+ System.out.println("testRpcAddRemoveInCluster starting");
+
+ final JavaTestKit mockBroker1 = new JavaTestKit(node1);
+
+ // install probe on node2's bucket store
+ final ActorPath bucketStorePath = new ChildActorPath(registry2.path(), "store");
+ final JavaTestKit probe2 = createProbeForMessage(node2, bucketStorePath,
+ Messages.BucketStoreMessages.UpdateRemoteBuckets.class);
+
+ // Add rpc on node 1
+ registry1.tell(new SetLocalRouter(mockBroker1.getRef()), mockBroker1.getRef());
+ registry1.tell(getAddRouteMessage(), mockBroker1.getRef());
+
+ // Bucket store on node2 should get a message to update its local copy of remote buckets
+ probe2.expectMsgClass(FiniteDuration.apply(10, TimeUnit.SECONDS),
+ Messages.BucketStoreMessages.UpdateRemoteBuckets.class);
+
+ // Now remove
+ registry1.tell(getRemoveRouteMessage(), mockBroker1.getRef());
+
+ // Bucket store on node2 should get a message to update its local copy of remote buckets
+ probe2.expectMsgClass(FiniteDuration.apply(10, TimeUnit.SECONDS),
+ Messages.BucketStoreMessages.UpdateRemoteBuckets.class);
+
+ System.out.println("testRpcAddRemoveInCluster ending");
+ }
+
+ /**
+ * Three node cluster. Register rpc on 2 nodes. Ensure 3rd gets updated.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testRpcAddedOnMultiNodes() throws Exception {
+
+ final JavaTestKit mockBroker1 = new JavaTestKit(node1);
+ final JavaTestKit mockBroker2 = new JavaTestKit(node2);
+ final JavaTestKit mockBroker3 = new JavaTestKit(node3);
+
+ registry3.tell(new SetLocalRouter(mockBroker3.getRef()), mockBroker3.getRef());
+
+ // install probe on node 3
+ final ActorPath bucketStorePath = new ChildActorPath(registry3.path(), "store");
+ final JavaTestKit probe3 = createProbeForMessage(node3, bucketStorePath,
+ Messages.BucketStoreMessages.UpdateRemoteBuckets.class);
+
+ // Add rpc on node 1
+ registry1.tell(new SetLocalRouter(mockBroker1.getRef()), mockBroker1.getRef());
+ registry1.tell(getAddRouteMessage(), mockBroker1.getRef());
+
+ probe3.expectMsgClass(FiniteDuration.apply(10, TimeUnit.SECONDS),
+ Messages.BucketStoreMessages.UpdateRemoteBuckets.class);
+
+ // Add same rpc on node 2
+ registry2.tell(new SetLocalRouter(mockBroker2.getRef()), mockBroker2.getRef());
+ registry2.tell(getAddRouteMessage(), mockBroker2.getRef());
+
+ probe3.expectMsgClass(FiniteDuration.apply(10, TimeUnit.SECONDS),
+ Messages.BucketStoreMessages.UpdateRemoteBuckets.class);
+ }
+
+ private JavaTestKit createProbeForMessage(ActorSystem node, ActorPath subjectPath, final Class<?> clazz)
+ throws Exception {
+ final JavaTestKit probe = new JavaTestKit(node);
+
+ ConditionalProbe conditionalProbe = new ConditionalProbe(probe.getRef(), new Predicate<Object>() {
+ @Override
+ public boolean apply(@Nullable Object input) {
+ if (input != null) {
+ return clazz.equals(input.getClass());
+ } else {
+ return false;
+ }
+ }
+ });
+
+ FiniteDuration duration = Duration.create(3, TimeUnit.SECONDS);
+ Timeout timeout = new Timeout(duration);
+ int maxTries = 30;
+ int i = 0;
+ while(true) {
+ ActorSelection subject = node.actorSelection(subjectPath);
+ Future<Object> future = Patterns.ask(subject, conditionalProbe, timeout);
+
+ try {
+ Await.ready(future, duration);
+ break;
+ } catch (TimeoutException | InterruptedException e) {
+ if(++i > maxTries) {
+ throw e;
+ }
+ }