Merge "Try to eliminate random failures in RaftActorTest"
authorTom Pantelis <tpanteli@brocade.com>
Fri, 13 Feb 2015 02:40:13 +0000 (02:40 +0000)
committerGerrit Code Review <gerrit@opendaylight.org>
Fri, 13 Feb 2015 02:40:14 +0000 (02:40 +0000)
24 files changed:
opendaylight/config/config-parent/pom.xml
opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/Leader.java
opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/LeaderElectionScenariosTest.java [new file with mode: 0644]
opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/LeaderTest.java
opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/utils/MessageCollectorActor.java
opendaylight/md-sal/sal-akka-raft/src/test/resources/simplelogger.properties [new file with mode: 0644]
opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/akka.conf
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardTransaction.java
opendaylight/md-sal/sal-distributed-datastore/src/test/resources/simplelogger.properties [new file with mode: 0644]
opendaylight/networkconfiguration/neutron/northbound/src/main/java/org/opendaylight/controller/networkconfig/neutron/northbound/NeutronFirewallNorthbound.java
opendaylight/networkconfiguration/neutron/northbound/src/main/java/org/opendaylight/controller/networkconfig/neutron/northbound/NeutronFirewallPolicyNorthbound.java
opendaylight/networkconfiguration/neutron/northbound/src/main/java/org/opendaylight/controller/networkconfig/neutron/northbound/NeutronFirewallRulesNorthbound.java
opendaylight/networkconfiguration/neutron/northbound/src/main/java/org/opendaylight/controller/networkconfig/neutron/northbound/NeutronFloatingIPsNorthbound.java
opendaylight/networkconfiguration/neutron/northbound/src/main/java/org/opendaylight/controller/networkconfig/neutron/northbound/NeutronLoadBalancerHealthMonitorNorthbound.java
opendaylight/networkconfiguration/neutron/northbound/src/main/java/org/opendaylight/controller/networkconfig/neutron/northbound/NeutronLoadBalancerListenerNorthbound.java
opendaylight/networkconfiguration/neutron/northbound/src/main/java/org/opendaylight/controller/networkconfig/neutron/northbound/NeutronLoadBalancerNorthbound.java
opendaylight/networkconfiguration/neutron/northbound/src/main/java/org/opendaylight/controller/networkconfig/neutron/northbound/NeutronLoadBalancerPoolMembersNorthbound.java
opendaylight/networkconfiguration/neutron/northbound/src/main/java/org/opendaylight/controller/networkconfig/neutron/northbound/NeutronLoadBalancerPoolNorthbound.java
opendaylight/networkconfiguration/neutron/northbound/src/main/java/org/opendaylight/controller/networkconfig/neutron/northbound/NeutronNetworksNorthbound.java
opendaylight/networkconfiguration/neutron/northbound/src/main/java/org/opendaylight/controller/networkconfig/neutron/northbound/NeutronPortsNorthbound.java
opendaylight/networkconfiguration/neutron/northbound/src/main/java/org/opendaylight/controller/networkconfig/neutron/northbound/NeutronRoutersNorthbound.java
opendaylight/networkconfiguration/neutron/northbound/src/main/java/org/opendaylight/controller/networkconfig/neutron/northbound/NeutronSecurityGroupsNorthbound.java
opendaylight/networkconfiguration/neutron/northbound/src/main/java/org/opendaylight/controller/networkconfig/neutron/northbound/NeutronSecurityRulesNorthbound.java
opendaylight/networkconfiguration/neutron/northbound/src/main/java/org/opendaylight/controller/networkconfig/neutron/northbound/NeutronSubnetsNorthbound.java

index af39b63447cdf6429371a40a8155de53272cba1d..e6e2bb8478bcda319b31d070ea6ca294c1c21e7a 100644 (file)
@@ -123,6 +123,25 @@ and is available at http://www.eclipse.org/legal/epl-v10.html
             </filesets>
           </configuration>
         </plugin>
+        <plugin>
+          <groupId>org.codehaus.mojo</groupId>
+          <artifactId>build-helper-maven-plugin</artifactId>
+          <executions>
+            <execution>
+              <id>add-yang-sources</id>
+              <phase>generate-sources</phase>
+              <goals>
+                <goal>add-source</goal>
+              </goals>
+              <configuration>
+                <sources>
+                  <source>${jmxGeneratorPath}</source>
+                  <source>${salGeneratorPath}</source>
+                </sources>
+              </configuration>
+            </execution>
+          </executions>
+        </plugin>
       </plugins>
     </pluginManagement>
   </build>
index fcfaee36033f3eba278f1f351d3c8cb3e974feb1..7a94c0c15849038f35105789856f98cb74580d51 100644 (file)
@@ -12,7 +12,6 @@ import akka.actor.Cancellable;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import org.opendaylight.controller.cluster.raft.RaftActorContext;
-import org.opendaylight.controller.cluster.raft.base.messages.InitiateInstallSnapshot;
 import org.opendaylight.controller.cluster.raft.base.messages.IsolatedLeaderCheck;
 import scala.concurrent.duration.FiniteDuration;
 
@@ -45,8 +44,6 @@ public class Leader extends AbstractLeader {
     public Leader(RaftActorContext context) {
         super(context);
 
-        scheduleInstallSnapshotCheck(context.getConfigParams().getIsolatedCheckInterval());
-
         scheduleIsolatedLeaderCheck(
             new FiniteDuration(context.getConfigParams().getHeartBeatInterval().length() * 10,
                 context.getConfigParams().getHeartBeatInterval().unit()));
@@ -66,30 +63,6 @@ public class Leader extends AbstractLeader {
         return super.handleMessage(sender, originalMessage);
     }
 
-    protected void stopInstallSnapshotSchedule() {
-        if (installSnapshotSchedule != null && !installSnapshotSchedule.isCancelled()) {
-            installSnapshotSchedule.cancel();
-        }
-    }
-
-    protected void scheduleInstallSnapshotCheck(FiniteDuration interval) {
-        if (getFollowerIds().isEmpty()) {
-            // Optimization - do not bother scheduling a heartbeat as there are
-            // no followers
-            return;
-        }
-
-        stopInstallSnapshotSchedule();
-
-        // Schedule a message to send append entries to followers that can
-        // accept an append entries with some data in it
-        installSnapshotSchedule =
-            context.getActorSystem().scheduler().scheduleOnce(
-                interval,
-                context.getActor(), new InitiateInstallSnapshot(),
-                context.getActorSystem().dispatcher(), context.getActor());
-    }
-
     protected void stopIsolatedLeaderCheckSchedule() {
         if (isolatedLeaderCheckSchedule != null && !isolatedLeaderCheckSchedule.isCancelled()) {
             isolatedLeaderCheckSchedule.cancel();
@@ -104,7 +77,6 @@ public class Leader extends AbstractLeader {
 
     @Override
     public void close() throws Exception {
-        stopInstallSnapshotSchedule();
         stopIsolatedLeaderCheckSchedule();
         super.close();
     }
diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/LeaderElectionScenariosTest.java b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/LeaderElectionScenariosTest.java
new file mode 100644 (file)
index 0000000..3aac005
--- /dev/null
@@ -0,0 +1,795 @@
+/*
+ * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.controller.cluster.raft.behaviors;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import akka.actor.ActorRef;
+import akka.actor.ActorSystem;
+import akka.actor.Props;
+import akka.dispatch.Dispatchers;
+import akka.testkit.JavaTestKit;
+import akka.testkit.TestActorRef;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.util.concurrent.Uninterruptibles;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.junit.After;
+import org.junit.Test;
+import org.opendaylight.controller.cluster.raft.DefaultConfigParamsImpl;
+import org.opendaylight.controller.cluster.raft.MockRaftActorContext;
+import org.opendaylight.controller.cluster.raft.MockRaftActorContext.MockPayload;
+import org.opendaylight.controller.cluster.raft.MockRaftActorContext.MockReplicatedLogEntry;
+import org.opendaylight.controller.cluster.raft.MockRaftActorContext.SimpleReplicatedLog;
+import org.opendaylight.controller.cluster.raft.RaftActorContext;
+import org.opendaylight.controller.cluster.raft.RaftState;
+import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout;
+import org.opendaylight.controller.cluster.raft.base.messages.SendHeartBeat;
+import org.opendaylight.controller.cluster.raft.messages.AppendEntries;
+import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply;
+import org.opendaylight.controller.cluster.raft.messages.RequestVote;
+import org.opendaylight.controller.cluster.raft.messages.RequestVoteReply;
+import org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.impl.SimpleLogger;
+import scala.concurrent.duration.FiniteDuration;
+
+/**
+ * Tests various leader election scenarios.
+ *
+ * @author Thomas Pantelis
+ */
+public class LeaderElectionScenariosTest {
+
+    private static final int HEARTBEAT_INTERVAL = 50;
+
+    public static class MemberActor extends MessageCollectorActor {
+
+        volatile RaftActorBehavior behavior;
+        Map<Class<?>, CountDownLatch> messagesReceivedLatches = new ConcurrentHashMap<>();
+        Map<Class<?>, Boolean> dropMessagesToBehavior = new ConcurrentHashMap<>();
+        CountDownLatch behaviorStateChangeLatch;
+
+        public static Props props() {
+            return Props.create(MemberActor.class).withDispatcher(Dispatchers.DefaultDispatcherId());
+        }
+
+        @Override
+        public void onReceive(Object message) throws Exception {
+            // Ignore scheduled SendHeartBeat messages.
+            if(message instanceof SendHeartBeat) {
+                return;
+            }
+
+            try {
+                if(behavior != null && !dropMessagesToBehavior.containsKey(message.getClass())) {
+                    RaftActorBehavior oldBehavior = behavior;
+                    behavior = behavior.handleMessage(getSender(), message);
+                    if(behavior != oldBehavior && behaviorStateChangeLatch != null) {
+                        behaviorStateChangeLatch.countDown();
+                    }
+                }
+            } finally {
+                super.onReceive(message);
+
+                CountDownLatch latch = messagesReceivedLatches.get(message.getClass());
+                if(latch != null) {
+                    latch.countDown();
+                }
+            }
+        }
+
+        void expectBehaviorStateChange() {
+            behaviorStateChangeLatch = new CountDownLatch(1);
+        }
+
+        void waitForBehaviorStateChange() {
+            assertTrue("Expected behavior state change",
+                    Uninterruptibles.awaitUninterruptibly(behaviorStateChangeLatch, 5, TimeUnit.SECONDS));
+        }
+
+        void expectMessageClass(Class<?> expClass, int expCount) {
+            messagesReceivedLatches.put(expClass, new CountDownLatch(expCount));
+        }
+
+        void waitForExpectedMessages(Class<?> expClass) {
+            CountDownLatch latch = messagesReceivedLatches.get(expClass);
+            assertNotNull("No messages received for " + expClass, latch);
+            assertTrue("Missing messages of type " + expClass,
+                    Uninterruptibles.awaitUninterruptibly(latch, 5, TimeUnit.SECONDS));
+        }
+
+        void dropMessagesToBehavior(Class<?> msgClass) {
+            dropMessagesToBehavior(msgClass, 1);
+        }
+
+        void dropMessagesToBehavior(Class<?> msgClass, int expCount) {
+            expectMessageClass(msgClass, expCount);
+            dropMessagesToBehavior.put(msgClass, Boolean.TRUE);
+        }
+
+        void clearDropMessagesToBehavior() {
+            dropMessagesToBehavior.clear();
+        }
+
+        @Override
+        public void clear() {
+            behaviorStateChangeLatch = null;
+            clearDropMessagesToBehavior();
+            messagesReceivedLatches.clear();
+            super.clear();
+        }
+
+        void forwardCapturedMessageToBehavior(Class<?> msgClass, ActorRef sender) throws Exception {
+            Object message = getFirstMatching(getSelf(), msgClass);
+            assertNotNull("Message of type " + msgClass + " not received", message);
+            getSelf().tell(message, sender);
+        }
+
+        void forwardCapturedMessagesToBehavior(Class<?> msgClass, ActorRef sender) throws Exception {
+            for(Object m: getAllMatching(getSelf(), msgClass)) {
+                getSelf().tell(m, sender);
+            }
+        }
+
+        <T> T getCapturedMessage(Class<T> msgClass) throws Exception {
+            Object message = getFirstMatching(getSelf(), msgClass);
+            assertNotNull("Message of type " + msgClass + " not received", message);
+            return (T) message;
+        }
+    }
+
+    static {
+        System.setProperty(SimpleLogger.LOG_KEY_PREFIX + MockRaftActorContext.class.getName(), "trace");
+    }
+
+    private final Logger testLog = LoggerFactory.getLogger(MockRaftActorContext.class);
+    private final ActorSystem system = ActorSystem.create("test");
+
+    @After
+    public void tearDown() {
+        JavaTestKit.shutdownActorSystem(system);
+    }
+
+    private DefaultConfigParamsImpl newConfigParams() {
+        DefaultConfigParamsImpl configParams = new DefaultConfigParamsImpl();
+        configParams.setHeartBeatInterval(new FiniteDuration(HEARTBEAT_INTERVAL, TimeUnit.MILLISECONDS));
+        configParams.setElectionTimeoutFactor(100000);
+        configParams.setIsolatedLeaderCheckInterval(new FiniteDuration(1, TimeUnit.DAYS));
+        return configParams;
+    }
+
+    private MockRaftActorContext newRaftActorContext(String id, ActorRef actor,
+            Map<String, String> peerAddresses) {
+        MockRaftActorContext context = new MockRaftActorContext(id, system, actor);
+        context.setPeerAddresses(peerAddresses);
+        context.getTermInformation().updateAndPersist(1, "");
+        return context;
+    }
+
+    private void verifyBehaviorState(String name, TestActorRef<MemberActor> actor, RaftState expState) {
+        assertEquals(name + " behavior state", expState, actor.underlyingActor().behavior.state());
+    }
+
+    private void initializeLeaderBehavior(TestActorRef<MemberActor> actor, RaftActorContext context,
+            int numActiveFollowers) throws Exception {
+        // Leader sends immediate heartbeats - we don't care about it so ignore it.
+
+        actor.underlyingActor().expectMessageClass(AppendEntriesReply.class, numActiveFollowers);
+        Leader leader = new Leader(context);
+        actor.underlyingActor().waitForExpectedMessages(AppendEntriesReply.class);
+        actor.underlyingActor().behavior = leader;
+
+        actor.underlyingActor().forwardCapturedMessagesToBehavior(AppendEntriesReply.class, ActorRef.noSender());
+        actor.underlyingActor().clear();
+    }
+
+    private TestActorRef<MemberActor> newMemberActor(String name) throws Exception {
+        TestActorRef<MemberActor> actor = TestActorRef.create(system, MemberActor.props(), name);
+        MessageCollectorActor.waitUntilReady(actor);
+        return actor;
+    }
+
+    private void sendHeartbeat(TestActorRef<MemberActor> leaderActor) {
+        Uninterruptibles.sleepUninterruptibly(HEARTBEAT_INTERVAL, TimeUnit.MILLISECONDS);
+        leaderActor.underlyingActor().behavior.handleMessage(leaderActor, new SendHeartBeat());
+    }
+
+    @Test
+    public void testDelayedMessagesScenario() throws Exception {
+        testLog.info("Starting testDelayedMessagesScenario");
+
+        TestActorRef<MemberActor> member1Actor = newMemberActor("member1");
+        TestActorRef<MemberActor> member2Actor = newMemberActor("member2");
+        TestActorRef<MemberActor> member3Actor = newMemberActor("member3");
+
+        // Create member 2's behavior initially as Follower
+
+        MockRaftActorContext member2Context = newRaftActorContext("member2", member2Actor,
+                ImmutableMap.<String,String>builder().
+                    put("member1", member1Actor.path().toString()).
+                    put("member3", member3Actor.path().toString()).build());
+
+        DefaultConfigParamsImpl member2ConfigParams = newConfigParams();
+        member2Context.setConfigParams(member2ConfigParams);
+
+        Follower member2Behavior = new Follower(member2Context);
+        member2Actor.underlyingActor().behavior = member2Behavior;
+
+        // Create member 3's behavior initially as Follower
+
+        MockRaftActorContext member3Context = newRaftActorContext("member3", member3Actor,
+                ImmutableMap.<String,String>builder().
+                    put("member1", member1Actor.path().toString()).
+                    put("member2", member2Actor.path().toString()).build());
+
+        DefaultConfigParamsImpl member3ConfigParams = newConfigParams();
+        member3Context.setConfigParams(member3ConfigParams);
+
+        Follower member3Behavior = new Follower(member3Context);
+        member3Actor.underlyingActor().behavior = member3Behavior;
+
+        // Create member 1's behavior initially as Leader
+
+        MockRaftActorContext member1Context = newRaftActorContext("member1", member1Actor,
+                ImmutableMap.<String,String>builder().
+                    put("member2", member2Actor.path().toString()).
+                    put("member3", member3Actor.path().toString()).build());
+
+        DefaultConfigParamsImpl member1ConfigParams = newConfigParams();
+        member1Context.setConfigParams(member1ConfigParams);
+
+        initializeLeaderBehavior(member1Actor, member1Context, 2);
+
+        member2Actor.underlyingActor().clear();
+        member3Actor.underlyingActor().clear();
+
+        // Send ElectionTimeout to member 2 to simulate missing heartbeat from the Leader. member 2
+        // should switch to Candidate and send out RequestVote messages. Set member 1 and 3 actors
+        // to capture RequestVote but not to forward to the behavior just yet as we want to
+        // control the order of RequestVote messages to member 1 and 3.
+
+        member1Actor.underlyingActor().dropMessagesToBehavior(RequestVote.class);
+
+        member2Actor.underlyingActor().expectBehaviorStateChange();
+
+        member3Actor.underlyingActor().dropMessagesToBehavior(RequestVote.class);
+
+        member2Actor.tell(new ElectionTimeout(), ActorRef.noSender());
+
+        member1Actor.underlyingActor().waitForExpectedMessages(RequestVote.class);
+        member3Actor.underlyingActor().waitForExpectedMessages(RequestVote.class);
+
+        member2Actor.underlyingActor().waitForBehaviorStateChange();
+        verifyBehaviorState("member 2", member2Actor, RaftState.Candidate);
+
+        assertEquals("member 1 election term", 1, member1Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 3 election term", 1, member3Context.getTermInformation().getCurrentTerm());
+
+        // At this point member 1 and 3 actors have captured the RequestVote messages. First
+        // forward the RequestVote message to member 1's behavior. Since the RequestVote term
+        // is greater than member 1's term, member 1 should switch to Follower without replying
+        // to RequestVote and update its term to 2.
+
+        member1Actor.underlyingActor().clearDropMessagesToBehavior();
+        member1Actor.underlyingActor().expectBehaviorStateChange();
+        member1Actor.underlyingActor().forwardCapturedMessageToBehavior(RequestVote.class, member2Actor);
+        member1Actor.underlyingActor().waitForExpectedMessages(RequestVote.class);
+
+        member1Actor.underlyingActor().waitForBehaviorStateChange();
+        verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
+
+        // Now forward member 3's captured RequestVote message to its behavior. Since member 3 is
+        // already a Follower, it should update its term to 2 and send a RequestVoteReply back to
+        // member 2 granting the vote b/c the RequestVote's term, lastLogTerm, and lastLogIndex
+        // should satisfy the criteria for granting the vote. However, we'll delay sending the
+        // RequestVoteReply to member 2's behavior to simulate network latency.
+
+        member2Actor.underlyingActor().dropMessagesToBehavior(RequestVoteReply.class);
+
+        member3Actor.underlyingActor().clearDropMessagesToBehavior();
+        member3Actor.underlyingActor().expectMessageClass(RequestVote.class, 1);
+        member3Actor.underlyingActor().forwardCapturedMessageToBehavior(RequestVote.class, member2Actor);
+        member3Actor.underlyingActor().waitForExpectedMessages(RequestVote.class);
+        verifyBehaviorState("member 3", member3Actor, RaftState.Follower);
+
+        assertEquals("member 1 election term", 2, member1Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 3 election term", 2, member3Context.getTermInformation().getCurrentTerm());
+
+        // Send ElectionTimeout to member 3 to simulate missing heartbeat from a Leader. member 3
+        // should switch to Candidate and send out RequestVote messages. member 1 should grant the
+        // vote and send a reply. After receiving the RequestVoteReply, member 3 should switch to leader.
+
+        member2Actor.underlyingActor().expectBehaviorStateChange();
+        member3Actor.underlyingActor().clear();
+        member3Actor.underlyingActor().expectMessageClass(RequestVoteReply.class, 1);
+        member3Actor.underlyingActor().expectMessageClass(AppendEntriesReply.class, 2);
+
+        member3Actor.tell(new ElectionTimeout(), ActorRef.noSender());
+
+        member3Actor.underlyingActor().waitForExpectedMessages(RequestVoteReply.class);
+
+        RequestVoteReply requestVoteReply = member3Actor.underlyingActor().getCapturedMessage(RequestVoteReply.class);
+        assertEquals("getTerm", member3Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm());
+        assertEquals("isVoteGranted", true, requestVoteReply.isVoteGranted());
+
+        verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
+
+        // member 2 should've switched to Follower as member 3's RequestVote term (3) was greater
+        // than member 2's term (2).
+
+        member2Actor.underlyingActor().waitForBehaviorStateChange();
+        verifyBehaviorState("member 2", member2Actor, RaftState.Follower);
+
+        // The switch to leader should cause an immediate AppendEntries heartbeat from member 3.
+
+        member3Actor.underlyingActor().waitForExpectedMessages(AppendEntriesReply.class);
+
+        assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
+
+        // Now forward the original delayed RequestVoteReply from member 3 to member 2 that granted
+        // the vote. Since member 2 is now a Follower, the RequestVoteReply should be ignored.
+
+        member2Actor.underlyingActor().clearDropMessagesToBehavior();
+        member2Actor.underlyingActor().forwardCapturedMessageToBehavior(RequestVoteReply.class, member3Actor);
+
+        member2Actor.underlyingActor().waitForExpectedMessages(RequestVoteReply.class);
+
+        verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
+        verifyBehaviorState("member 2", member2Actor, RaftState.Follower);
+        verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
+
+        assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
+
+        testLog.info("testDelayedMessagesScenario done");
+    }
+
+    @Test
+    public void testPartitionedLeadersScenario() throws Exception {
+        testLog.info("Starting testPartitionedLeadersScenario");
+
+        TestActorRef<MemberActor> member1Actor = newMemberActor("member1");
+        TestActorRef<MemberActor> member2Actor = newMemberActor("member2");
+        TestActorRef<MemberActor> member3Actor = newMemberActor("member3");
+
+        // Create member 2's behavior initially as Follower
+
+        MockRaftActorContext member2Context = newRaftActorContext("member2", member2Actor,
+                ImmutableMap.<String,String>builder().
+                    put("member1", member1Actor.path().toString()).
+                    put("member3", member3Actor.path().toString()).build());
+
+        DefaultConfigParamsImpl member2ConfigParams = newConfigParams();
+        member2Context.setConfigParams(member2ConfigParams);
+
+        Follower member2Behavior = new Follower(member2Context);
+        member2Actor.underlyingActor().behavior = member2Behavior;
+
+        // Create member 3's behavior initially as Follower
+
+        MockRaftActorContext member3Context = newRaftActorContext("member3", member3Actor,
+                ImmutableMap.<String,String>builder().
+                    put("member1", member1Actor.path().toString()).
+                    put("member2", member2Actor.path().toString()).build());
+
+        DefaultConfigParamsImpl member3ConfigParams = newConfigParams();
+        member3Context.setConfigParams(member3ConfigParams);
+
+        Follower member3Behavior = new Follower(member3Context);
+        member3Actor.underlyingActor().behavior = member3Behavior;
+
+        // Create member 1's behavior initially as Leader
+
+        MockRaftActorContext member1Context = newRaftActorContext("member1", member1Actor,
+                ImmutableMap.<String,String>builder().
+                    put("member2", member2Actor.path().toString()).
+                    put("member3", member3Actor.path().toString()).build());
+
+        DefaultConfigParamsImpl member1ConfigParams = newConfigParams();
+        member1Context.setConfigParams(member1ConfigParams);
+
+        initializeLeaderBehavior(member1Actor, member1Context, 2);
+
+        member2Actor.underlyingActor().clear();
+        member3Actor.underlyingActor().clear();
+
+        // Send ElectionTimeout to member 2 to simulate no heartbeat from the Leader (member 1).
+        // member 2 should switch to Candidate, start new term 2 and send out RequestVote messages.
+        // member 1 will switch to Follower b/c its term is less than the RequestVote term, also it
+        // won't send back a reply. member 3 will drop the message (ie won't forward it to its behavior) to
+        // simulate loss of network connectivity between member 2 and 3.
+
+        member1Actor.underlyingActor().expectMessageClass(RequestVote.class, 1);
+
+        member2Actor.underlyingActor().expectBehaviorStateChange();
+
+        member3Actor.underlyingActor().dropMessagesToBehavior(RequestVote.class);
+
+        member2Actor.tell(new ElectionTimeout(), ActorRef.noSender());
+
+        member1Actor.underlyingActor().waitForExpectedMessages(RequestVote.class);
+        member3Actor.underlyingActor().waitForExpectedMessages(RequestVote.class);
+
+        // member 1 should switch to Follower as the RequestVote term is greater than its term. It
+        // won't send back a RequestVoteReply in this case.
+
+        verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
+
+        // member 2 should switch to Candidate since member 1 didn't reply.
+
+        member2Actor.underlyingActor().waitForBehaviorStateChange();
+        verifyBehaviorState("member 2", member2Actor, RaftState.Candidate);
+
+        assertEquals("member 1 election term", 2, member1Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 3 election term", 1, member3Context.getTermInformation().getCurrentTerm());
+
+        // Send ElectionTimeout to member 3 to simulate no heartbeat from the Leader (member 1).
+        // member 2 should switch to Candidate and send out RequestVote messages. member 1 will reply and
+        // grant the vote but member 2 will drop the message to simulate loss of network connectivity.
+
+        member1Actor.underlyingActor().clear();
+        member1Actor.underlyingActor().expectMessageClass(RequestVote.class, 1);
+        member1Actor.underlyingActor().expectMessageClass(AppendEntries.class, 1);
+
+        member2Actor.underlyingActor().clear();
+        member2Actor.underlyingActor().dropMessagesToBehavior(RequestVote.class);
+        member2Actor.underlyingActor().dropMessagesToBehavior(AppendEntries.class);
+
+        member3Actor.underlyingActor().clear();
+        member3Actor.underlyingActor().expectMessageClass(RequestVoteReply.class, 1);
+        member3Actor.underlyingActor().expectMessageClass(AppendEntriesReply.class, 1);
+
+        member3Actor.tell(new ElectionTimeout(), ActorRef.noSender());
+
+        member1Actor.underlyingActor().waitForExpectedMessages(RequestVote.class);
+        member2Actor.underlyingActor().waitForExpectedMessages(RequestVote.class);
+        member3Actor.underlyingActor().waitForExpectedMessages(RequestVoteReply.class);
+
+        RequestVoteReply requestVoteReply = member3Actor.underlyingActor().getCapturedMessage(RequestVoteReply.class);
+        assertEquals("getTerm", member3Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm());
+        assertEquals("isVoteGranted", true, requestVoteReply.isVoteGranted());
+
+        // when member 3 switches to Leader it will immediately send out heartbeat AppendEntries to
+        // the followers. Wait for AppendEntries to member 1 and its AppendEntriesReply. The
+        // AppendEntries message to member 2 is dropped.
+
+        member1Actor.underlyingActor().waitForExpectedMessages(AppendEntries.class);
+        member2Actor.underlyingActor().waitForExpectedMessages(AppendEntries.class);
+        member3Actor.underlyingActor().waitForExpectedMessages(AppendEntriesReply.class);
+
+        verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
+        verifyBehaviorState("member 2", member2Actor, RaftState.Candidate);
+        verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
+
+        assertEquals("member 1 election term", 2, member1Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 3 election term", 2, member3Context.getTermInformation().getCurrentTerm());
+
+        // member 2 is partitioned from the Leader (member 3) and hasn't received any messages. It
+        // would get another ElectionTimeout so simulate that. member 1 should send back a reply
+        // granting the vote. Messages (RequestVote and AppendEntries) from member 2 to member 3
+        // are dropped to simulate loss of network connectivity. Note member 2 will increment its
+        // election term to 3.
+
+        member1Actor.underlyingActor().clear();
+        member1Actor.underlyingActor().expectMessageClass(AppendEntries.class, 1);
+
+        member2Actor.underlyingActor().clear();
+        member2Actor.underlyingActor().expectMessageClass(RequestVoteReply.class, 1);
+        member2Actor.underlyingActor().expectMessageClass(AppendEntriesReply.class, 1);
+
+        member3Actor.underlyingActor().clear();
+        member3Actor.underlyingActor().dropMessagesToBehavior(AppendEntries.class);
+        member3Actor.underlyingActor().dropMessagesToBehavior(RequestVote.class);
+
+        member2Actor.tell(new ElectionTimeout(), ActorRef.noSender());
+
+        member2Actor.underlyingActor().waitForExpectedMessages(RequestVoteReply.class);
+
+        requestVoteReply = member2Actor.underlyingActor().getCapturedMessage(RequestVoteReply.class);
+        assertEquals("getTerm", member2Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm());
+        assertEquals("isVoteGranted", true, requestVoteReply.isVoteGranted());
+
+        member3Actor.underlyingActor().waitForExpectedMessages(RequestVote.class);
+
+        member1Actor.underlyingActor().waitForExpectedMessages(AppendEntries.class);
+        member3Actor.underlyingActor().waitForExpectedMessages(AppendEntries.class);
+        member2Actor.underlyingActor().waitForExpectedMessages(AppendEntriesReply.class);
+
+        // We end up with 2 partitioned leaders both leading member 1. The term for member 1 and 3
+        // is 3 and member 3's term is 2.
+
+        verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
+        verifyBehaviorState("member 2", member2Actor, RaftState.Leader);
+        verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
+
+        assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 3 election term", 2, member3Context.getTermInformation().getCurrentTerm());
+
+        // Re-establish connectivity between member 2 and 3, ie stop dropping messages between
+        // the 2. Send heartbeats (AppendEntries) from member 3. Both member 1 and 2 should send back
+        // an unsuccessful AppendEntriesReply b/c their term (3) is greater than member 3's term (2).
+        // This should cause member 3 to switch to Follower.
+
+        RaftActorBehavior savedMember1Behavior = member1Actor.underlyingActor().behavior;
+        RaftActorBehavior savedMember2Behavior = member2Actor.underlyingActor().behavior;
+        RaftActorBehavior savedMember3Behavior = member3Actor.underlyingActor().behavior;
+        long savedMember3Term = member3Context.getTermInformation().getCurrentTerm();
+        String savedMember3VoterFor = member3Context.getTermInformation().getVotedFor();
+
+        member1Actor.underlyingActor().clear();
+        member1Actor.underlyingActor().expectMessageClass(AppendEntries.class, 1);
+
+        member2Actor.underlyingActor().clear();
+        member2Actor.underlyingActor().expectMessageClass(AppendEntries.class, 1);
+
+        member3Actor.underlyingActor().clear();
+        member3Actor.underlyingActor().expectMessageClass(AppendEntriesReply.class, 1);
+
+        sendHeartbeat(member3Actor);
+
+        member3Actor.underlyingActor().waitForExpectedMessages(AppendEntriesReply.class);
+
+        AppendEntriesReply appendEntriesReply = member3Actor.underlyingActor().
+                getCapturedMessage(AppendEntriesReply.class);
+        assertEquals("isSuccess", false, appendEntriesReply.isSuccess());
+        assertEquals("getTerm", 3, appendEntriesReply.getTerm());
+
+        verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
+        verifyBehaviorState("member 2", member2Actor, RaftState.Leader);
+        verifyBehaviorState("member 3", member3Actor, RaftState.Follower);
+
+        assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
+
+        // Revert back to the partitioned leaders state to test the other sequence where member 2
+        // sends heartbeats first before member 3. member 1 should return a successful
+        // AppendEntriesReply b/c his term matches member 2's. member 3 should switch to Follower
+        // as his term is less than member 2's.
+
+        member1Actor.underlyingActor().behavior = savedMember1Behavior;
+        member2Actor.underlyingActor().behavior = savedMember2Behavior;
+        member3Actor.underlyingActor().behavior = savedMember3Behavior;
+
+        member3Context.getTermInformation().update(savedMember3Term, savedMember3VoterFor);
+
+        member1Actor.underlyingActor().clear();
+        member1Actor.underlyingActor().expectMessageClass(AppendEntries.class, 1);
+
+        member2Actor.underlyingActor().clear();
+        member2Actor.underlyingActor().expectMessageClass(AppendEntriesReply.class, 1);
+
+        member3Actor.underlyingActor().clear();
+        member3Actor.underlyingActor().expectMessageClass(AppendEntries.class, 1);
+
+        sendHeartbeat(member2Actor);
+
+        member1Actor.underlyingActor().waitForExpectedMessages(AppendEntries.class);
+        member3Actor.underlyingActor().waitForExpectedMessages(AppendEntries.class);
+
+        member2Actor.underlyingActor().waitForExpectedMessages(AppendEntriesReply.class);
+
+        verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
+        verifyBehaviorState("member 2", member2Actor, RaftState.Leader);
+        verifyBehaviorState("member 3", member3Actor, RaftState.Follower);
+
+        assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
+
+        testLog.info("testPartitionedLeadersScenario done");
+    }
+
+    @Test
+    public void testPartitionedCandidateOnStartupScenario() throws Exception {
+        testLog.info("Starting testPartitionedCandidateOnStartupScenario");
+
+        TestActorRef<MemberActor> member1Actor = newMemberActor("member1") ;
+        TestActorRef<MemberActor> member2Actor = newMemberActor("member2");
+        TestActorRef<MemberActor> member3Actor = newMemberActor("member3");
+
+        // Create member 2's behavior as Follower.
+
+        MockRaftActorContext member2Context = newRaftActorContext("member2", member2Actor,
+                ImmutableMap.<String,String>builder().
+                    put("member1", member1Actor.path().toString()).
+                    put("member3", member3Actor.path().toString()).build());
+
+        DefaultConfigParamsImpl member2ConfigParams = newConfigParams();
+        member2Context.setConfigParams(member2ConfigParams);
+
+        Follower member2Behavior = new Follower(member2Context);
+        member2Actor.underlyingActor().behavior = member2Behavior;
+
+        // Create member 1's behavior as Leader.
+
+        MockRaftActorContext member1Context = newRaftActorContext("member1", member1Actor,
+                ImmutableMap.<String,String>builder().
+                    put("member2", member2Actor.path().toString()).
+                    put("member3", member3Actor.path().toString()).build());
+
+        DefaultConfigParamsImpl member1ConfigParams = newConfigParams();
+        member1Context.setConfigParams(member1ConfigParams);
+
+        initializeLeaderBehavior(member1Actor, member1Context, 1);
+
+        member2Actor.underlyingActor().clear();
+        member3Actor.underlyingActor().clear();
+
+        // Initialize the ReplicatedLog and election term info for member 1 and 2. The current term
+        // will be 3 and the last term will be 2.
+
+        SimpleReplicatedLog replicatedLog = new SimpleReplicatedLog();
+        replicatedLog.append(new MockReplicatedLogEntry(2, 1, new MockPayload("")));
+        replicatedLog.append(new MockReplicatedLogEntry(3, 1, new MockPayload("")));
+
+        member1Context.setReplicatedLog(replicatedLog);
+        member1Context.getTermInformation().update(3, "");
+
+        member2Context.setReplicatedLog(replicatedLog);
+        member2Context.getTermInformation().update(3, member1Context.getId());
+
+        // Create member 3's behavior initially as a Candidate.
+
+        MockRaftActorContext member3Context = newRaftActorContext("member3", member3Actor,
+                ImmutableMap.<String,String>builder().
+                    put("member1", member1Actor.path().toString()).
+                    put("member2", member2Actor.path().toString()).build());
+
+        DefaultConfigParamsImpl member3ConfigParams = newConfigParams();
+        member3Context.setConfigParams(member3ConfigParams);
+
+        // Initialize the ReplicatedLog and election term info for Candidate member 3. The current term
+        // will be 2 and the last term will be 1 so it is behind the leader's log.
+
+        SimpleReplicatedLog candidateReplicatedLog = new SimpleReplicatedLog();
+        candidateReplicatedLog.append(new MockReplicatedLogEntry(1, 1, new MockPayload("")));
+        candidateReplicatedLog.append(new MockReplicatedLogEntry(2, 1, new MockPayload("")));
+
+        member3Context.setReplicatedLog(candidateReplicatedLog);
+        member3Context.getTermInformation().update(2, member1Context.getId());
+
+        // The member 3 Candidate will start a new term and send RequestVotes. However it will be
+        // partitioned from the cluster by having member 1 and 2 drop its RequestVote messages.
+
+        int numCandidateElections = 5;
+        long candidateElectionTerm = member3Context.getTermInformation().getCurrentTerm() + numCandidateElections;
+
+        member1Actor.underlyingActor().dropMessagesToBehavior(RequestVote.class, numCandidateElections);
+
+        member2Actor.underlyingActor().dropMessagesToBehavior(RequestVote.class, numCandidateElections);
+
+        Candidate member3Behavior = new Candidate(member3Context);
+        member3Actor.underlyingActor().behavior = member3Behavior;
+
+        // Send several additional ElectionTimeouts to Candidate member 3. Each ElectionTimeout will
+        // start a new term so Candidate member 3's current term will be greater than the leader's
+        // current term.
+
+        for(int i = 0; i < numCandidateElections - 1; i++) {
+            member3Actor.tell(new ElectionTimeout(), ActorRef.noSender());
+        }
+
+        member1Actor.underlyingActor().waitForExpectedMessages(RequestVote.class);
+        member2Actor.underlyingActor().waitForExpectedMessages(RequestVote.class);
+
+        verifyBehaviorState("member 1", member1Actor, RaftState.Leader);
+        verifyBehaviorState("member 2", member2Actor, RaftState.Follower);
+        verifyBehaviorState("member 3", member3Actor, RaftState.Candidate);
+
+        assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 3 election term", candidateElectionTerm,
+                member3Context.getTermInformation().getCurrentTerm());
+
+        // Now send a couple more ElectionTimeouts to Candidate member 3 with the partition resolved.
+        //
+        // On the first RequestVote, Leader member 1 should switch to Follower as its term (s) is less than
+        // the RequestVote's term (8) from member 3. No RequestVoteReply should be sent by member 1.
+        // Follower member 2 should update its term since it less than the RequestVote's term and
+        // should return a RequestVoteReply but should not grant the vote as its last term and index
+        // is greater than the RequestVote's lastLogTerm and lastLogIndex, ie member 2's log is later
+        // or more up to date than member 3's.
+        //
+        // On the second RequestVote, both member 1 and 2 are followers so they should update their
+        // term and return a RequestVoteReply but should not grant the vote.
+
+        candidateElectionTerm += 2;
+        for(int i = 0; i < 2; i++) {
+            member1Actor.underlyingActor().clear();
+            member1Actor.underlyingActor().expectMessageClass(RequestVote.class, 1);
+            member2Actor.underlyingActor().clear();
+            member2Actor.underlyingActor().expectMessageClass(RequestVote.class, 1);
+            member3Actor.underlyingActor().clear();
+            member3Actor.underlyingActor().expectMessageClass(RequestVoteReply.class, 1);
+
+            member3Actor.tell(new ElectionTimeout(), ActorRef.noSender());
+
+            member1Actor.underlyingActor().waitForExpectedMessages(RequestVote.class);
+            member2Actor.underlyingActor().waitForExpectedMessages(RequestVote.class);
+
+            member3Actor.underlyingActor().waitForExpectedMessages(RequestVoteReply.class);
+
+            RequestVoteReply requestVoteReply = member3Actor.underlyingActor().getCapturedMessage(RequestVoteReply.class);
+            assertEquals("getTerm", member3Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm());
+            assertEquals("isVoteGranted", false, requestVoteReply.isVoteGranted());
+        }
+
+        verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
+        verifyBehaviorState("member 2", member2Actor, RaftState.Follower);
+        verifyBehaviorState("member 3", member3Actor, RaftState.Candidate);
+
+        // Even though member 3 didn't get voted for, member 1 and 2 should have updated their term
+        // to member 3's.
+
+        assertEquals("member 1 election term", candidateElectionTerm,
+                member1Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 2 election term", candidateElectionTerm,
+                member2Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 3 election term", candidateElectionTerm,
+                member3Context.getTermInformation().getCurrentTerm());
+
+        // At this point we have no leader. Candidate member 3 would continue to start new elections
+        // but wouldn't be granted a vote. One of the 2 followers would eventually time out from
+        // not having received a heartbeat from a leader and switch to candidate and start a new
+        // election. We'll simulate that here by sending an ElectionTimeout to member 1.
+
+        member1Actor.underlyingActor().clear();
+        member1Actor.underlyingActor().expectMessageClass(RequestVoteReply.class, 1);
+        member2Actor.underlyingActor().clear();
+        member2Actor.underlyingActor().expectMessageClass(RequestVote.class, 1);
+        member3Actor.underlyingActor().clear();
+        member3Actor.underlyingActor().expectMessageClass(RequestVote.class, 1);
+        member3Actor.underlyingActor().expectBehaviorStateChange();
+
+        member1Actor.tell(new ElectionTimeout(), ActorRef.noSender());
+
+        member2Actor.underlyingActor().waitForExpectedMessages(RequestVote.class);
+        member3Actor.underlyingActor().waitForExpectedMessages(RequestVote.class);
+
+        // The RequestVoteReply should come from Follower member 2 and the vote should be granted
+        // since member 2's last term and index matches member 1's.
+
+        member1Actor.underlyingActor().waitForExpectedMessages(RequestVoteReply.class);
+
+        RequestVoteReply requestVoteReply = member1Actor.underlyingActor().getCapturedMessage(RequestVoteReply.class);
+        assertEquals("getTerm", member1Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm());
+        assertEquals("isVoteGranted", true, requestVoteReply.isVoteGranted());
+
+        // Candidate member 3 should change to follower as its term should be less than the
+        // RequestVote term (member 1 started a new term higher than the other member's terms).
+
+        member3Actor.underlyingActor().waitForBehaviorStateChange();
+
+        verifyBehaviorState("member 1", member1Actor, RaftState.Leader);
+        verifyBehaviorState("member 2", member2Actor, RaftState.Follower);
+        verifyBehaviorState("member 3", member3Actor, RaftState.Follower);
+
+        // newTerm should be 10.
+
+        long newTerm = candidateElectionTerm + 1;
+        assertEquals("member 1 election term", newTerm, member1Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 2 election term", newTerm, member2Context.getTermInformation().getCurrentTerm());
+        assertEquals("member 3 election term", newTerm, member3Context.getTermInformation().getCurrentTerm());
+
+        testLog.info("testPartitionedCandidateOnStartupScenario done");
+    }
+}
index 3f551b3a30efbf4c9b9e47108c35a75f06a57d0e..8251c6b265bd5ca85e247119104b4e6b268be9f6 100644 (file)
@@ -44,16 +44,10 @@ import org.opendaylight.controller.cluster.raft.messages.RequestVoteReply;
 import org.opendaylight.controller.cluster.raft.utils.DoNothingActor;
 import org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor;
 import org.opendaylight.controller.protobuff.messages.cluster.raft.InstallSnapshotMessages;
-import org.slf4j.impl.SimpleLogger;
 import scala.concurrent.duration.FiniteDuration;
 
 public class LeaderTest extends AbstractRaftActorBehaviorTest {
 
-    static {
-        // This enables trace logging for the tests.
-        System.setProperty(SimpleLogger.LOG_KEY_PREFIX + MockRaftActorContext.class.getName(), "trace");
-    }
-
     private final ActorRef leaderActor =
         getSystem().actorOf(Props.create(DoNothingActor.class));
     private final ActorRef senderActor =
index c5acb1f2a401dfc6935bc30deb1ba993048cfb91..79c90cf051cc928ac50e563a136c62030809bbc2 100644 (file)
@@ -17,6 +17,7 @@ import com.google.common.util.concurrent.Uninterruptibles;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import scala.concurrent.Await;
 import scala.concurrent.Future;
 import scala.concurrent.duration.Duration;
@@ -24,14 +25,21 @@ import scala.concurrent.duration.FiniteDuration;
 
 
 public class MessageCollectorActor extends UntypedActor {
+    private static final String ARE_YOU_READY = "ARE_YOU_READY";
+
     private final List<Object> messages = new ArrayList<>();
 
     @Override public void onReceive(Object message) throws Exception {
+        if(message.equals(ARE_YOU_READY)) {
+            getSender().tell("yes", getSelf());
+            return;
+        }
+
         if(message instanceof String){
             if("get-all-messages".equals(message)){
-                getSender().tell(new ArrayList(messages), getSelf());
+                getSender().tell(new ArrayList<>(messages), getSelf());
             }
-        } else {
+        } else if(message != null) {
             messages.add(message);
         }
     }
@@ -45,11 +53,7 @@ public class MessageCollectorActor extends UntypedActor {
         Timeout operationTimeout = new Timeout(operationDuration);
         Future<Object> future = Patterns.ask(actor, "get-all-messages", operationTimeout);
 
-        try {
-            return (List<Object>) Await.result(future, operationDuration);
-        } catch (Exception e) {
-            throw e;
-        }
+        return (List<Object>) Await.result(future, operationDuration);
     }
 
     /**
@@ -88,4 +92,17 @@ public class MessageCollectorActor extends UntypedActor {
         return output;
     }
 
+    public static void waitUntilReady(ActorRef actor) throws Exception {
+        long timeout = 500;
+        FiniteDuration duration = Duration.create(timeout, TimeUnit.MILLISECONDS);
+        for(int i = 0; i < 10; i++) {
+            try {
+                Await.ready(Patterns.ask(actor, ARE_YOU_READY, timeout), duration);
+                return;
+            } catch (TimeoutException e) {
+            }
+        }
+
+        throw new TimeoutException("Actor not ready in time.");
+    }
 }
diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/resources/simplelogger.properties b/opendaylight/md-sal/sal-akka-raft/src/test/resources/simplelogger.properties
new file mode 100644 (file)
index 0000000..4e79807
--- /dev/null
@@ -0,0 +1,6 @@
+org.slf4j.simpleLogger.showDateTime=true
+org.slf4j.simpleLogger.dateTimeFormat=hh:mm:ss,S a
+org.slf4j.simpleLogger.logFile=System.out
+org.slf4j.simpleLogger.showShortLogName=true
+org.slf4j.simpleLogger.levelInBrackets=true
+org.slf4j.simpleLogger.org.opendaylight.controller.cluster.raft=trace
\ No newline at end of file
index f196ad1644791b92e5ec32a43ebeb36cb7411734..10f6a4209c95b7af8db7680ce8a28b1f62378e19 100644 (file)
@@ -39,7 +39,7 @@ odl-cluster-data {
     cluster {
       seed-nodes = ["akka.tcp://opendaylight-cluster-data@127.0.0.1:2550"]
 
-      auto-down-unreachable-after = 10s
+      auto-down-unreachable-after = 300s
 
       roles = [
         "member-1"
@@ -77,7 +77,7 @@ odl-cluster-rpc {
     cluster {
       seed-nodes = ["akka.tcp://odl-cluster-rpc@127.0.0.1:2551"]
 
-      auto-down-unreachable-after = 10s
+      auto-down-unreachable-after = 300s
     }
   }
 }
index 8a37dfee4d8ccc698a5bb9096d4ecdf9ed228771..1e2386ae1bb8ea4ae6075266bbf12f56600c0516 100644 (file)
@@ -124,29 +124,20 @@ public abstract class ShardTransaction extends AbstractUntypedActorWithMetering
 
     protected void readData(DOMStoreReadTransaction transaction, ReadData message,
             final boolean returnSerialized) {
-        final ActorRef sender = getSender();
-        final ActorRef self = getSelf();
-        final YangInstanceIdentifier path = message.getPath();
-        final CheckedFuture<Optional<NormalizedNode<?, ?>>, ReadFailedException> future =
-                transaction.read(path);
-
-        future.addListener(new Runnable() {
-            @Override
-            public void run() {
-                try {
-                    Optional<NormalizedNode<?, ?>> optional = future.checkedGet();
-                    ReadDataReply readDataReply = new ReadDataReply(optional.orNull());
 
-                    sender.tell((returnSerialized ? readDataReply.toSerializable(clientTxVersion):
-                        readDataReply), self);
+        final YangInstanceIdentifier path = message.getPath();
+        try {
+            final CheckedFuture<Optional<NormalizedNode<?, ?>>, ReadFailedException> future = transaction.read(path);
+            Optional<NormalizedNode<?, ?>> optional = future.checkedGet();
+            ReadDataReply readDataReply = new ReadDataReply(optional.orNull());
 
-                } catch (Exception e) {
-                    shardStats.incrementFailedReadTransactionsCount();
-                    sender.tell(new akka.actor.Status.Failure(e), self);
-                }
+            sender().tell((returnSerialized ? readDataReply.toSerializable(clientTxVersion): readDataReply), self());
 
-            }
-        }, getContext().dispatcher());
+        } catch (Exception e) {
+            LOG.error(String.format("Unexpected error reading path %s", path), e);
+            shardStats.incrementFailedReadTransactionsCount();
+            sender().tell(new akka.actor.Status.Failure(e), self());
+        }
     }
 
     protected void dataExists(DOMStoreReadTransaction transaction, DataExists message,
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/resources/simplelogger.properties b/opendaylight/md-sal/sal-distributed-datastore/src/test/resources/simplelogger.properties
new file mode 100644 (file)
index 0000000..9ed3d27
--- /dev/null
@@ -0,0 +1,6 @@
+org.slf4j.simpleLogger.showDateTime=true
+org.slf4j.simpleLogger.dateTimeFormat=hh:mm:ss,S a
+org.slf4j.simpleLogger.logFile=System.out
+org.slf4j.simpleLogger.showShortLogName=true
+org.slf4j.simpleLogger.levelInBrackets=true
+org.slf4j.simpleLogger.log.org.opendaylight.controller.cluster.datastore=trace
\ No newline at end of file
index 33043d4389b2b736a4db4066c170b748c0fb258a..b97a554f60b72feb84a4f73e26768722e34c0c17 100644 (file)
@@ -187,13 +187,19 @@ public class NeutronFirewallNorthbound {
             firewallInterface.addNeutronFirewall(singleton);
             Object[] instances = NeutronUtil.getInstances(INeutronFirewallAware.class, this);
             if (instances != null) {
-                for (Object instance : instances) {
-                    INeutronFirewallAware service = (INeutronFirewallAware) instance;
-                    int status = service.canCreateNeutronFirewall(singleton);
-                    if (status < 200 || status > 299) {
-                        return Response.status(status).build();
+                if (instances.length > 0) {
+                    for (Object instance : instances) {
+                        INeutronFirewallAware service = (INeutronFirewallAware) instance;
+                        int status = service.canCreateNeutronFirewall(singleton);
+                        if (status < 200 || status > 299) {
+                            return Response.status(status).build();
+                        }
                     }
+                } else {
+                    throw new ServiceUnavailableException("No providers registered.  Please try again later");
                 }
+            } else {
+                throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
             }
             firewallInterface.addNeutronFirewall(singleton);
             if (instances != null) {
@@ -220,13 +226,19 @@ public class NeutronFirewallNorthbound {
                     throw new BadRequestException("Firewall UUID already exists");
                 }
                 if (instances != null) {
-                    for (Object instance : instances) {
-                        INeutronFirewallAware service = (INeutronFirewallAware) instance;
-                        int status = service.canCreateNeutronFirewall(test);
-                        if (status < 200 || status > 299) {
-                            return Response.status(status).build();
+                    if (instances.length > 0) {
+                        for (Object instance : instances) {
+                            INeutronFirewallAware service = (INeutronFirewallAware) instance;
+                            int status = service.canCreateNeutronFirewall(test);
+                            if (status < 200 || status > 299) {
+                                return Response.status(status).build();
+                            }
                         }
+                    } else {
+                        throw new ServiceUnavailableException("No providers registered.  Please try again later");
                     }
+                } else {
+                    throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
                 }
             }
 
@@ -298,13 +310,19 @@ public class NeutronFirewallNorthbound {
 
         Object[] instances = NeutronUtil.getInstances(INeutronFirewallAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronFirewallAware service = (INeutronFirewallAware) instance;
-                int status = service.canUpdateNeutronFirewall(delta, original);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronFirewallAware service = (INeutronFirewallAware) instance;
+                    int status = service.canUpdateNeutronFirewall(delta, original);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
         /*
@@ -352,13 +370,19 @@ public class NeutronFirewallNorthbound {
         NeutronFirewall singleton = firewallInterface.getNeutronFirewall(firewallUUID);
         Object[] instances = NeutronUtil.getInstances(INeutronFirewallAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronFirewallAware service = (INeutronFirewallAware) instance;
-                int status = service.canDeleteNeutronFirewall(singleton);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronFirewallAware service = (INeutronFirewallAware) instance;
+                    int status = service.canDeleteNeutronFirewall(singleton);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
         /*
index 08e563138cdc99f632a5c28d8637096bd5aebc58..e9b813d73129f0592bcfd5ffc3fcd793560f10cb 100644 (file)
@@ -184,13 +184,19 @@ public class NeutronFirewallPolicyNorthbound {
 
             Object[] instances = NeutronUtil.getInstances(INeutronFirewallPolicyAware.class, this);
             if (instances != null) {
-                for (Object instance : instances) {
-                    INeutronFirewallPolicyAware service = (INeutronFirewallPolicyAware) instance;
-                    int status = service.canCreateNeutronFirewallPolicy(singleton);
-                    if (status < 200 || status > 299) {
-                        return Response.status(status).build();
+                if (instances.length > 0) {
+                    for (Object instance : instances) {
+                        INeutronFirewallPolicyAware service = (INeutronFirewallPolicyAware) instance;
+                        int status = service.canCreateNeutronFirewallPolicy(singleton);
+                        if (status < 200 || status > 299) {
+                            return Response.status(status).build();
+                        }
                     }
+                } else {
+                    throw new ServiceUnavailableException("No providers registered.  Please try again later");
                 }
+            } else {
+                throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
             }
             firewallPolicyInterface.addNeutronFirewallPolicy(singleton);
             if (instances != null) {
@@ -218,13 +224,19 @@ public class NeutronFirewallPolicyNorthbound {
                     throw new BadRequestException("Firewall Policy UUID already exists");
                 }
                 if (instances != null) {
-                    for (Object instance : instances) {
-                        INeutronFirewallPolicyAware service = (INeutronFirewallPolicyAware) instance;
-                        int status = service.canCreateNeutronFirewallPolicy(test);
-                        if (status < 200 || status > 299) {
-                            return Response.status(status).build();
+                    if (instances.length > 0) {
+                        for (Object instance : instances) {
+                            INeutronFirewallPolicyAware service = (INeutronFirewallPolicyAware) instance;
+                            int status = service.canCreateNeutronFirewallPolicy(test);
+                            if (status < 200 || status > 299) {
+                                return Response.status(status).build();
+                            }
                         }
+                    } else {
+                        throw new ServiceUnavailableException("No providers registered.  Please try again later");
                     }
+                } else {
+                    throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
                 }
             }
             /*
@@ -295,13 +307,19 @@ public class NeutronFirewallPolicyNorthbound {
 
         Object[] instances = NeutronUtil.getInstances(INeutronFirewallPolicyAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronFirewallPolicyAware service = (INeutronFirewallPolicyAware) instance;
-                int status = service.canUpdateNeutronFirewallPolicy(delta, original);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronFirewallPolicyAware service = (INeutronFirewallPolicyAware) instance;
+                    int status = service.canUpdateNeutronFirewallPolicy(delta, original);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
         /*
@@ -349,13 +367,19 @@ public class NeutronFirewallPolicyNorthbound {
         NeutronFirewallPolicy singleton = firewallPolicyInterface.getNeutronFirewallPolicy(firewallPolicyUUID);
         Object[] instances = NeutronUtil.getInstances(INeutronFirewallPolicyAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronFirewallPolicyAware service = (INeutronFirewallPolicyAware) instance;
-                int status = service.canDeleteNeutronFirewallPolicy(singleton);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronFirewallPolicyAware service = (INeutronFirewallPolicyAware) instance;
+                    int status = service.canDeleteNeutronFirewallPolicy(singleton);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
         firewallPolicyInterface.removeNeutronFirewallPolicy(firewallPolicyUUID);
index 5e51711e5b9d21f8f548e02eff67e7230140dfcb..40b830da559f41a0711423bb4481116f668d1cca 100644 (file)
@@ -220,13 +220,19 @@ public class NeutronFirewallRulesNorthbound {
             firewallRuleInterface.addNeutronFirewallRule(singleton);
             Object[] instances = NeutronUtil.getInstances(INeutronFirewallRuleAware.class, this);
             if (instances != null) {
-                for (Object instance : instances) {
-                    INeutronFirewallRuleAware service = (INeutronFirewallRuleAware) instance;
-                    int status = service.canCreateNeutronFirewallRule(singleton);
-                    if (status < 200 || status > 299) {
-                        return Response.status(status).build();
+                if (instances.length > 0) {
+                    for (Object instance : instances) {
+                        INeutronFirewallRuleAware service = (INeutronFirewallRuleAware) instance;
+                        int status = service.canCreateNeutronFirewallRule(singleton);
+                        if (status < 200 || status > 299) {
+                            return Response.status(status).build();
+                        }
                     }
+                } else {
+                    throw new ServiceUnavailableException("No providers registered.  Please try again later");
                 }
+            } else {
+                throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
             }
             // add rule to cache
             singleton.initDefaults();
@@ -256,13 +262,19 @@ public class NeutronFirewallRulesNorthbound {
                     throw new BadRequestException("Firewall Rule UUID already exists");
                 }
                 if (instances != null) {
-                    for (Object instance : instances) {
-                        INeutronFirewallRuleAware service = (INeutronFirewallRuleAware) instance;
-                        int status = service.canCreateNeutronFirewallRule(test);
-                        if (status < 200 || status > 299) {
-                            return Response.status(status).build();
+                    if (instances.length > 0) {
+                        for (Object instance : instances) {
+                            INeutronFirewallRuleAware service = (INeutronFirewallRuleAware) instance;
+                            int status = service.canCreateNeutronFirewallRule(test);
+                            if (status < 200 || status > 299) {
+                                return Response.status(status).build();
+                            }
                         }
+                    } else {
+                        throw new ServiceUnavailableException("No providers registered.  Please try again later");
                     }
+                } else {
+                    throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
                 }
             }
             /*
@@ -342,13 +354,19 @@ public class NeutronFirewallRulesNorthbound {
 
         Object[] instances = NeutronUtil.getInstances(INeutronFirewallRuleAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronFirewallRuleAware service = (INeutronFirewallRuleAware) instance;
-                int status = service.canUpdateNeutronFirewallRule(delta, original);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronFirewallRuleAware service = (INeutronFirewallRuleAware) instance;
+                    int status = service.canUpdateNeutronFirewallRule(delta, original);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
         /*
@@ -399,13 +417,19 @@ public class NeutronFirewallRulesNorthbound {
         NeutronFirewallRule singleton = firewallRuleInterface.getNeutronFirewallRule(firewallRuleUUID);
         Object[] instances = NeutronUtil.getInstances(INeutronFirewallRuleAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronFirewallRuleAware service = (INeutronFirewallRuleAware) instance;
-                int status = service.canDeleteNeutronFirewallRule(singleton);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronFirewallRuleAware service = (INeutronFirewallRuleAware) instance;
+                    int status = service.canDeleteNeutronFirewallRule(singleton);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
         /*
index 812a09a0a6951aac5414a69d45b82db209ba94f1..3e6c2a4febbec6870737258314029af201ba4ca4 100644 (file)
@@ -242,12 +242,18 @@ public class NeutronFloatingIPsNorthbound {
             }
             Object[] instances = NeutronUtil.getInstances(INeutronFloatingIPAware.class, this);
             if (instances != null) {
-                for (Object instance : instances) {
-                    INeutronFloatingIPAware service = (INeutronFloatingIPAware) instance;
-                    int status = service.canCreateFloatingIP(singleton);
-                    if (status < 200 || status > 299)
-                        return Response.status(status).build();
+                if (instances.length > 0) {
+                    for (Object instance : instances) {
+                        INeutronFloatingIPAware service = (INeutronFloatingIPAware) instance;
+                        int status = service.canCreateFloatingIP(singleton);
+                        if (status < 200 || status > 299)
+                            return Response.status(status).build();
+                    }
+                } else {
+                    throw new ServiceUnavailableException("No providers registered.  Please try again later");
                 }
+            } else {
+                throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
             }
             floatingIPInterface.addFloatingIP(singleton);
             if (instances != null) {
@@ -363,12 +369,18 @@ public class NeutronFloatingIPsNorthbound {
         NeutronFloatingIP target = floatingIPInterface.getFloatingIP(floatingipUUID);
         Object[] instances = NeutronUtil.getInstances(INeutronFloatingIPAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronFloatingIPAware service = (INeutronFloatingIPAware) instance;
-                int status = service.canUpdateFloatingIP(singleton, target);
-                if (status < 200 || status > 299)
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronFloatingIPAware service = (INeutronFloatingIPAware) instance;
+                    int status = service.canUpdateFloatingIP(singleton, target);
+                    if (status < 200 || status > 299)
+                        return Response.status(status).build();
+                }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
         floatingIPInterface.updateFloatingIP(floatingipUUID, singleton);
         target = floatingIPInterface.getFloatingIP(floatingipUUID);
@@ -406,12 +418,18 @@ public class NeutronFloatingIPsNorthbound {
         NeutronFloatingIP singleton = floatingIPInterface.getFloatingIP(floatingipUUID);
         Object[] instances = NeutronUtil.getInstances(INeutronFloatingIPAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronFloatingIPAware service = (INeutronFloatingIPAware) instance;
-                int status = service.canDeleteFloatingIP(singleton);
-                if (status < 200 || status > 299)
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronFloatingIPAware service = (INeutronFloatingIPAware) instance;
+                    int status = service.canDeleteFloatingIP(singleton);
+                    if (status < 200 || status > 299)
+                        return Response.status(status).build();
+                }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
         floatingIPInterface.removeFloatingIP(floatingipUUID);
         if (instances != null) {
index 85aba5d69080d29e1d596303af0b909a2a751ea4..aa30e948a2edbc716a35dfa5d04de55079ffc979 100644 (file)
@@ -210,13 +210,19 @@ public class NeutronLoadBalancerHealthMonitorNorthbound {
 
             Object[] instances = NeutronUtil.getInstances(INeutronLoadBalancerHealthMonitorAware.class, this);
             if (instances != null) {
-                for (Object instance : instances) {
-                    INeutronLoadBalancerHealthMonitorAware service = (INeutronLoadBalancerHealthMonitorAware) instance;
-                    int status = service.canCreateNeutronLoadBalancerHealthMonitor(singleton);
-                    if (status < 200 || status > 299) {
-                        return Response.status(status).build();
+                if (instances.length > 0) {
+                    for (Object instance : instances) {
+                        INeutronLoadBalancerHealthMonitorAware service = (INeutronLoadBalancerHealthMonitorAware) instance;
+                        int status = service.canCreateNeutronLoadBalancerHealthMonitor(singleton);
+                        if (status < 200 || status > 299) {
+                            return Response.status(status).build();
+                        }
                     }
+                } else {
+                    throw new ServiceUnavailableException("No providers registered.  Please try again later");
                 }
+            } else {
+                throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
             }
             loadBalancerHealthMonitorInterface.addNeutronLoadBalancerHealthMonitor(singleton);
             if (instances != null) {
@@ -245,13 +251,19 @@ public class NeutronLoadBalancerHealthMonitorNorthbound {
                     throw new BadRequestException("LoadBalancerHealthMonitor UUID already exists");
                 }
                 if (instances != null) {
-                    for (Object instance : instances) {
-                        INeutronLoadBalancerHealthMonitorAware service = (INeutronLoadBalancerHealthMonitorAware) instance;
-                        int status = service.canCreateNeutronLoadBalancerHealthMonitor(test);
-                        if (status < 200 || status > 299) {
-                            return Response.status(status).build();
+                    if (instances.length > 0) {
+                        for (Object instance : instances) {
+                            INeutronLoadBalancerHealthMonitorAware service = (INeutronLoadBalancerHealthMonitorAware) instance;
+                            int status = service.canCreateNeutronLoadBalancerHealthMonitor(test);
+                            if (status < 200 || status > 299) {
+                                return Response.status(status).build();
+                            }
                         }
+                    } else {
+                        throw new ServiceUnavailableException("No providers registered.  Please try again later");
                     }
+                } else {
+                    throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
                 }
             }
             /*
@@ -328,13 +340,19 @@ public class NeutronLoadBalancerHealthMonitorNorthbound {
 
         Object[] instances = NeutronUtil.getInstances(INeutronLoadBalancerHealthMonitorAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronLoadBalancerHealthMonitorAware service = (INeutronLoadBalancerHealthMonitorAware) instance;
-                int status = service.canUpdateNeutronLoadBalancerHealthMonitor(delta, original);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronLoadBalancerHealthMonitorAware service = (INeutronLoadBalancerHealthMonitorAware) instance;
+                    int status = service.canUpdateNeutronLoadBalancerHealthMonitor(delta, original);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
         /*
@@ -386,13 +404,19 @@ public class NeutronLoadBalancerHealthMonitorNorthbound {
         NeutronLoadBalancerHealthMonitor singleton = loadBalancerHealthMonitorInterface.getNeutronLoadBalancerHealthMonitor(loadBalancerHealthMonitorID);
         Object[] instances = NeutronUtil.getInstances(INeutronLoadBalancerHealthMonitorAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronLoadBalancerHealthMonitorAware service = (INeutronLoadBalancerHealthMonitorAware) instance;
-                int status = service.canDeleteNeutronLoadBalancerHealthMonitor(singleton);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronLoadBalancerHealthMonitorAware service = (INeutronLoadBalancerHealthMonitorAware) instance;
+                    int status = service.canDeleteNeutronLoadBalancerHealthMonitor(singleton);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
         loadBalancerHealthMonitorInterface.removeNeutronLoadBalancerHealthMonitor(loadBalancerHealthMonitorID);
         if (instances != null) {
index 345c7ee0f3ee522ba5e7bc40281b4275050a067f..5d877c59f490a75af689d55d1961523e42d877da 100644 (file)
@@ -198,13 +198,19 @@ public class NeutronLoadBalancerListenerNorthbound {
 
             Object[] instances = NeutronUtil.getInstances(INeutronLoadBalancerListenerAware.class, this);
             if (instances != null) {
-                for (Object instance : instances) {
-                    INeutronLoadBalancerListenerAware service = (INeutronLoadBalancerListenerAware) instance;
-                    int status = service.canCreateNeutronLoadBalancerListener(singleton);
-                    if (status < 200 || status > 299) {
-                        return Response.status(status).build();
+                if (instances.length > 0) {
+                    for (Object instance : instances) {
+                        INeutronLoadBalancerListenerAware service = (INeutronLoadBalancerListenerAware) instance;
+                        int status = service.canCreateNeutronLoadBalancerListener(singleton);
+                        if (status < 200 || status > 299) {
+                            return Response.status(status).build();
+                        }
                     }
+                } else {
+                    throw new ServiceUnavailableException("No providers registered.  Please try again later");
                 }
+            } else {
+                throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
             }
             loadBalancerListenerInterface.addNeutronLoadBalancerListener(singleton);
             if (instances != null) {
@@ -232,13 +238,19 @@ public class NeutronLoadBalancerListenerNorthbound {
                     throw new BadRequestException("LoadBalancerListener UUID already exists");
                 }
                 if (instances != null) {
-                    for (Object instance : instances) {
-                        INeutronLoadBalancerListenerAware service = (INeutronLoadBalancerListenerAware) instance;
-                        int status = service.canCreateNeutronLoadBalancerListener(test);
-                        if (status < 200 || status > 299) {
-                            return Response.status(status).build();
+                    if (instances.length > 0) {
+                        for (Object instance : instances) {
+                            INeutronLoadBalancerListenerAware service = (INeutronLoadBalancerListenerAware) instance;
+                            int status = service.canCreateNeutronLoadBalancerListener(test);
+                            if (status < 200 || status > 299) {
+                                return Response.status(status).build();
+                            }
                         }
+                    } else {
+                        throw new ServiceUnavailableException("No providers registered.  Please try again later");
                     }
+                } else {
+                    throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
                 }
             }
             /*
@@ -312,13 +324,19 @@ public class NeutronLoadBalancerListenerNorthbound {
 
         Object[] instances = NeutronUtil.getInstances(INeutronLoadBalancerListenerAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronLoadBalancerListenerAware service = (INeutronLoadBalancerListenerAware) instance;
-                int status = service.canUpdateNeutronLoadBalancerListener(delta, original);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronLoadBalancerListenerAware service = (INeutronLoadBalancerListenerAware) instance;
+                    int status = service.canUpdateNeutronLoadBalancerListener(delta, original);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
         /*
@@ -366,13 +384,19 @@ public class NeutronLoadBalancerListenerNorthbound {
         NeutronLoadBalancerListener singleton = loadBalancerListenerInterface.getNeutronLoadBalancerListener(loadBalancerListenerID);
         Object[] instances = NeutronUtil.getInstances(INeutronLoadBalancerListenerAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronLoadBalancerListenerAware service = (INeutronLoadBalancerListenerAware) instance;
-                int status = service.canDeleteNeutronLoadBalancerListener(singleton);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronLoadBalancerListenerAware service = (INeutronLoadBalancerListenerAware) instance;
+                    int status = service.canDeleteNeutronLoadBalancerListener(singleton);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
         loadBalancerListenerInterface.removeNeutronLoadBalancerListener(loadBalancerListenerID);
index 308041fe599336cd4daee3266345472f9dbad5bf..67557ce41f7cf2816dcd611a042cf7b2466cc4a9 100644 (file)
@@ -186,14 +186,21 @@ public class NeutronLoadBalancerNorthbound {
             }
             Object[] instances = NeutronUtil.getInstances(INeutronLoadBalancerAware.class, this);
             if (instances != null) {
-                for (Object instance : instances) {
-                    INeutronLoadBalancerAware service = (INeutronLoadBalancerAware) instance;
-                    int status = service.canCreateNeutronLoadBalancer(singleton);
-                    if (status < 200 || status > 299) {
-                        return Response.status(status).build();
+                if (instances.length > 0) {
+                    for (Object instance : instances) {
+                        INeutronLoadBalancerAware service = (INeutronLoadBalancerAware) instance;
+                        int status = service.canCreateNeutronLoadBalancer(singleton);
+                        if (status < 200 || status > 299) {
+                            return Response.status(status).build();
+                        }
                     }
+                } else {
+                    throw new ServiceUnavailableException("No providers registered.  Please try again later");
                 }
+            } else {
+                throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
             }
+
             loadBalancerInterface.addNeutronLoadBalancer(singleton);
             if (instances != null) {
                 for (Object instance : instances) {
@@ -220,13 +227,19 @@ public class NeutronLoadBalancerNorthbound {
                     throw new BadRequestException("Load Balancer Pool UUID already exists");
                 }
                 if (instances != null) {
-                    for (Object instance : instances) {
-                        INeutronLoadBalancerAware service = (INeutronLoadBalancerAware) instance;
-                        int status = service.canCreateNeutronLoadBalancer(test);
-                        if (status < 200 || status > 299) {
-                            return Response.status(status).build();
+                    if (instances.length > 0) {
+                        for (Object instance : instances) {
+                            INeutronLoadBalancerAware service = (INeutronLoadBalancerAware) instance;
+                            int status = service.canCreateNeutronLoadBalancer(test);
+                            if (status < 200 || status > 299) {
+                                return Response.status(status).build();
+                            }
                         }
+                    } else {
+                        throw new ServiceUnavailableException("No providers registered.  Please try again later");
                     }
+                } else {
+                    throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
                 }
             }
             /*
@@ -298,13 +311,19 @@ public class NeutronLoadBalancerNorthbound {
 
         Object[] instances = NeutronUtil.getInstances(INeutronLoadBalancerAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronLoadBalancerAware service = (INeutronLoadBalancerAware) instance;
-                int status = service.canUpdateNeutronLoadBalancer(delta, original);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronLoadBalancerAware service = (INeutronLoadBalancerAware) instance;
+                    int status = service.canUpdateNeutronLoadBalancer(delta, original);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
         /*
@@ -355,15 +374,22 @@ public class NeutronLoadBalancerNorthbound {
         NeutronLoadBalancer singleton = loadBalancerInterface.getNeutronLoadBalancer(loadBalancerID);
         Object[] instances = NeutronUtil.getInstances(INeutronLoadBalancerAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronLoadBalancerAware service = (INeutronLoadBalancerAware) instance;
-                int status = service.canDeleteNeutronLoadBalancer(singleton);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronLoadBalancerAware service = (INeutronLoadBalancerAware) instance;
+                    int status = service.canDeleteNeutronLoadBalancer(singleton);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
+
         loadBalancerInterface.removeNeutronLoadBalancer(loadBalancerID);
         if (instances != null) {
             for (Object instance : instances) {
index f9540679cdfbab5b42ea564ba2d2c38aeb6d45d8..22d118ae79f6030706c558ddb0d6c7519adf1b64 100644 (file)
@@ -200,14 +200,21 @@ public Response createLoadBalancerPoolMember(
 
         Object[] instances = NeutronUtil.getInstances(INeutronLoadBalancerPoolMemberAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronLoadBalancerPoolMemberAware service = (INeutronLoadBalancerPoolMemberAware) instance;
-                int status = service.canCreateNeutronLoadBalancerPoolMember(singleton);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronLoadBalancerPoolMemberAware service = (INeutronLoadBalancerPoolMemberAware) instance;
+                    int status = service.canCreateNeutronLoadBalancerPoolMember(singleton);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
+
         if (instances != null) {
             for (Object instance : instances) {
                 INeutronLoadBalancerPoolMemberAware service = (INeutronLoadBalancerPoolMemberAware) instance;
@@ -242,13 +249,19 @@ public Response createLoadBalancerPoolMember(
                 throw new BadRequestException("Load Balancer PoolMember UUID already exists");
             }
             if (instances != null) {
-                for (Object instance : instances) {
-                    INeutronLoadBalancerPoolMemberAware service = (INeutronLoadBalancerPoolMemberAware) instance;
-                    int status = service.canCreateNeutronLoadBalancerPoolMember(test);
-                    if (status < 200 || status > 299) {
-                        return Response.status(status).build();
+                if (instances.length > 0) {
+                    for (Object instance : instances) {
+                        INeutronLoadBalancerPoolMemberAware service = (INeutronLoadBalancerPoolMemberAware) instance;
+                        int status = service.canCreateNeutronLoadBalancerPoolMember(test);
+                        if (status < 200 || status > 299) {
+                            return Response.status(status).build();
+                        }
                     }
+                } else {
+                    throw new ServiceUnavailableException("No providers registered.  Please try again later");
                 }
+            } else {
+                throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
             }
         }
         /*
@@ -334,13 +347,19 @@ public Response deleteLoadBalancerPoolMember(
 
     Object[] instances = NeutronUtil.getInstances(INeutronLoadBalancerPoolMemberAware.class, this);
     if (instances != null) {
-        for (Object instance : instances) {
-            INeutronLoadBalancerPoolMemberAware service = (INeutronLoadBalancerPoolMemberAware) instance;
-            int status = service.canDeleteNeutronLoadBalancerPoolMember(singleton);
-            if (status < 200 || status > 299) {
-                return Response.status(status).build();
+        if (instances.length > 0) {
+            for (Object instance : instances) {
+                INeutronLoadBalancerPoolMemberAware service = (INeutronLoadBalancerPoolMemberAware) instance;
+                int status = service.canDeleteNeutronLoadBalancerPoolMember(singleton);
+                if (status < 200 || status > 299) {
+                    return Response.status(status).build();
+                }
             }
+        } else {
+            throw new ServiceUnavailableException("No providers registered.  Please try again later");
         }
+    } else {
+        throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
     }
 
     if (instances != null) {
index 77bb525c84e2d65b40225edcbbd12177113318ce..ea4e2d1c9a181b1c54059bd3b10fd1e43917d938 100644 (file)
@@ -198,13 +198,19 @@ public class NeutronLoadBalancerPoolNorthbound {
 
             Object[] instances = NeutronUtil.getInstances(INeutronLoadBalancerPoolAware.class, this);
             if (instances != null) {
-                for (Object instance : instances) {
-                    INeutronLoadBalancerPoolAware service = (INeutronLoadBalancerPoolAware) instance;
-                    int status = service.canCreateNeutronLoadBalancerPool(singleton);
-                    if (status < 200 || status > 299) {
-                        return Response.status(status).build();
+                if (instances.length > 0) {
+                    for (Object instance : instances) {
+                        INeutronLoadBalancerPoolAware service = (INeutronLoadBalancerPoolAware) instance;
+                        int status = service.canCreateNeutronLoadBalancerPool(singleton);
+                        if (status < 200 || status > 299) {
+                            return Response.status(status).build();
+                        }
                     }
+                } else {
+                    throw new ServiceUnavailableException("No providers registered.  Please try again later");
                 }
+            } else {
+                throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
             }
             loadBalancerPoolInterface.addNeutronLoadBalancerPool(singleton);
             if (instances != null) {
@@ -232,13 +238,19 @@ public class NeutronLoadBalancerPoolNorthbound {
                     throw new BadRequestException("Load Balancer Pool UUID already exists");
                 }
                 if (instances != null) {
-                    for (Object instance : instances) {
-                        INeutronLoadBalancerPoolAware service = (INeutronLoadBalancerPoolAware) instance;
-                        int status = service.canCreateNeutronLoadBalancerPool(test);
-                        if (status < 200 || status > 299) {
-                            return Response.status(status).build();
+                    if (instances.length > 0) {
+                        for (Object instance : instances) {
+                            INeutronLoadBalancerPoolAware service = (INeutronLoadBalancerPoolAware) instance;
+                            int status = service.canCreateNeutronLoadBalancerPool(test);
+                            if (status < 200 || status > 299) {
+                                return Response.status(status).build();
+                            }
                         }
+                    } else {
+                        throw new ServiceUnavailableException("No providers registered.  Please try again later");
                     }
+                } else {
+                    throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
                 }
             }
             /*
@@ -311,13 +323,19 @@ public class NeutronLoadBalancerPoolNorthbound {
 
         Object[] instances = NeutronUtil.getInstances(INeutronLoadBalancerPoolAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronLoadBalancerPoolAware service = (INeutronLoadBalancerPoolAware) instance;
-                int status = service.canUpdateNeutronLoadBalancerPool(delta, original);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronLoadBalancerPoolAware service = (INeutronLoadBalancerPoolAware) instance;
+                    int status = service.canUpdateNeutronLoadBalancerPool(delta, original);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
         /*
@@ -366,13 +384,19 @@ public class NeutronLoadBalancerPoolNorthbound {
         NeutronLoadBalancerPool singleton = loadBalancerPoolInterface.getNeutronLoadBalancerPool(loadBalancerPoolUUID);
         Object[] instances = NeutronUtil.getInstances(INeutronLoadBalancerPoolAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronLoadBalancerPoolAware service = (INeutronLoadBalancerPoolAware) instance;
-                int status = service.canDeleteNeutronLoadBalancerPool(singleton);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronLoadBalancerPoolAware service = (INeutronLoadBalancerPoolAware) instance;
+                    int status = service.canDeleteNeutronLoadBalancerPool(singleton);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
         /*
index 9c99f346e87eb715c3b1d12380f382467a3f4f70..3a3c657956133f11db8087037492cb4616865d08 100644 (file)
@@ -204,13 +204,19 @@ public class NeutronNetworksNorthbound {
 
             Object[] instances = NeutronUtil.getInstances(INeutronNetworkAware.class, this);
             if (instances != null) {
-                for (Object instance : instances) {
-                    INeutronNetworkAware service = (INeutronNetworkAware) instance;
-                    int status = service.canCreateNetwork(singleton);
-                    if (status < 200 || status > 299) {
-                        return Response.status(status).build();
+                if (instances.length > 0) {
+                    for (Object instance : instances) {
+                        INeutronNetworkAware service = (INeutronNetworkAware) instance;
+                        int status = service.canCreateNetwork(singleton);
+                        if (status < 200 || status > 299) {
+                            return Response.status(status).build();
+                        }
                     }
+                } else {
+                    throw new ServiceUnavailableException("No providers registered.  Please try again later");
                 }
+            } else {
+                throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
             }
 
             // add network to cache
@@ -242,13 +248,19 @@ public class NeutronNetworksNorthbound {
                     throw new BadRequestException("network UUID already exists");
                 }
                 if (instances != null) {
-                    for (Object instance: instances) {
-                        INeutronNetworkAware service = (INeutronNetworkAware) instance;
-                        int status = service.canCreateNetwork(test);
-                        if (status < 200 || status > 299) {
-                            return Response.status(status).build();
+                    if (instances.length > 0) {
+                        for (Object instance: instances) {
+                            INeutronNetworkAware service = (INeutronNetworkAware) instance;
+                            int status = service.canCreateNetwork(test);
+                            if (status < 200 || status > 299) {
+                                return Response.status(status).build();
+                            }
                         }
+                    } else {
+                        throw new ServiceUnavailableException("No providers registered.  Please try again later");
                     }
+                } else {
+                    throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
                 }
                 testMap.put(test.getID(),test);
             }
@@ -312,14 +324,20 @@ public class NeutronNetworksNorthbound {
 
         Object[] instances = NeutronUtil.getInstances(INeutronNetworkAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronNetworkAware service = (INeutronNetworkAware) instance;
-                NeutronNetwork original = networkInterface.getNetwork(netUUID);
-                int status = service.canUpdateNetwork(delta, original);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronNetworkAware service = (INeutronNetworkAware) instance;
+                    NeutronNetwork original = networkInterface.getNetwork(netUUID);
+                    int status = service.canUpdateNetwork(delta, original);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
         // update network object and return the modified object
@@ -366,14 +384,21 @@ public class NeutronNetworksNorthbound {
         NeutronNetwork singleton = networkInterface.getNetwork(netUUID);
         Object[] instances = NeutronUtil.getInstances(INeutronNetworkAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronNetworkAware service = (INeutronNetworkAware) instance;
-                int status = service.canDeleteNetwork(singleton);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronNetworkAware service = (INeutronNetworkAware) instance;
+                    int status = service.canDeleteNetwork(singleton);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
+
         networkInterface.removeNetwork(netUUID);
         if (instances != null) {
             for (Object instance : instances) {
index 23f4979f2385cf2464e262683765521e3a1c4ecf..5ff3de58d6b7139ad3792e78ccad686dcccb9c4b 100644 (file)
@@ -259,16 +259,21 @@ public class NeutronPortsNorthbound {
 
             Object[] instances = NeutronUtil.getInstances(INeutronPortAware.class, this);
             if (instances != null) {
-                for (Object instance : instances) {
-                    INeutronPortAware service = (INeutronPortAware) instance;
-                    int status = service.canCreatePort(singleton);
-                    if (status < 200 || status > 299) {
-                        return Response.status(status).build();
+                if (instances.length > 0) {
+                    for (Object instance : instances) {
+                        INeutronPortAware service = (INeutronPortAware) instance;
+                        int status = service.canCreatePort(singleton);
+                        if (status < 200 || status > 299) {
+                            return Response.status(status).build();
+                        }
                     }
+                } else {
+                    throw new ServiceUnavailableException("No providers registered.  Please try again later");
                 }
+            } else {
+                throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
             }
 
-
             // add the port to the cache
             portInterface.addPort(singleton);
             if (instances != null) {
@@ -353,13 +358,19 @@ public class NeutronPortsNorthbound {
                     }
                 }
                 if (instances != null) {
-                    for (Object instance : instances) {
-                        INeutronPortAware service = (INeutronPortAware) instance;
-                        int status = service.canCreatePort(test);
-                        if (status < 200 || status > 299) {
-                            return Response.status(status).build();
+                    if (instances.length > 0) {
+                        for (Object instance : instances) {
+                            INeutronPortAware service = (INeutronPortAware) instance;
+                            int status = service.canCreatePort(test);
+                            if (status < 200 || status > 299) {
+                                return Response.status(status).build();
+                            }
                         }
+                    } else {
+                        throw new ServiceUnavailableException("No providers registered.  Please try again later");
                     }
+                } else {
+                    throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
                 }
             }
 
@@ -429,13 +440,19 @@ public class NeutronPortsNorthbound {
 
         Object[] instances = NeutronUtil.getInstances(INeutronPortAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronPortAware service = (INeutronPortAware) instance;
-                int status = service.canUpdatePort(singleton, original);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronPortAware service = (INeutronPortAware) instance;
+                    int status = service.canUpdatePort(singleton, original);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
         // Verify the new fixed ips are valid
@@ -511,13 +528,19 @@ public class NeutronPortsNorthbound {
         NeutronPort singleton = portInterface.getPort(portUUID);
         Object[] instances = NeutronUtil.getInstances(INeutronPortAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronPortAware service = (INeutronPortAware) instance;
-                int status = service.canDeletePort(singleton);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronPortAware service = (INeutronPortAware) instance;
+                    int status = service.canDeletePort(singleton);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
         portInterface.removePort(portUUID);
         if (instances != null) {
index 45e84f4d153834ba10ff0b20846fe230f395ce3d..ccf5ddd1a4e2d8370caa25083081457152f8ddc6 100644 (file)
@@ -194,12 +194,18 @@ public class NeutronRoutersNorthbound {
             }
             Object[] instances = NeutronUtil.getInstances(INeutronRouterAware.class, this);
             if (instances != null) {
-                for (Object instance : instances) {
-                    INeutronRouterAware service = (INeutronRouterAware) instance;
-                    int status = service.canCreateRouter(singleton);
-                    if (status < 200 || status > 299)
-                        return Response.status(status).build();
+                if (instances.length > 0) {
+                    for (Object instance : instances) {
+                        INeutronRouterAware service = (INeutronRouterAware) instance;
+                        int status = service.canCreateRouter(singleton);
+                        if (status < 200 || status > 299)
+                            return Response.status(status).build();
+                    }
+                } else {
+                    throw new ServiceUnavailableException("No providers registered.  Please try again later");
                 }
+            } else {
+                throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
             }
 
             /*
@@ -270,12 +276,18 @@ public class NeutronRoutersNorthbound {
 
         Object[] instances = NeutronUtil.getInstances(INeutronRouterAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronRouterAware service = (INeutronRouterAware) instance;
-                int status = service.canUpdateRouter(singleton, original);
-                if (status < 200 || status > 299)
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronRouterAware service = (INeutronRouterAware) instance;
+                    int status = service.canUpdateRouter(singleton, original);
+                    if (status < 200 || status > 299)
+                        return Response.status(status).build();
+                }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
         /*
          * if the external gateway info is being changed, verify that the new network
@@ -335,12 +347,18 @@ public class NeutronRoutersNorthbound {
         NeutronRouter singleton = routerInterface.getRouter(routerUUID);
         Object[] instances = NeutronUtil.getInstances(INeutronRouterAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronRouterAware service = (INeutronRouterAware) instance;
-                int status = service.canDeleteRouter(singleton);
-                if (status < 200 || status > 299)
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronRouterAware service = (INeutronRouterAware) instance;
+                    int status = service.canDeleteRouter(singleton);
+                    if (status < 200 || status > 299)
+                        return Response.status(status).build();
+                }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
         routerInterface.removeRouter(routerUUID);
         if (instances != null) {
@@ -415,12 +433,18 @@ public class NeutronRoutersNorthbound {
             throw new ResourceConflictException("Target Port already allocated");
         Object[] instances = NeutronUtil.getInstances(INeutronRouterAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronRouterAware service = (INeutronRouterAware) instance;
-                int status = service.canAttachInterface(target, input);
-                if (status < 200 || status > 299)
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronRouterAware service = (INeutronRouterAware) instance;
+                    int status = service.canAttachInterface(target, input);
+                    if (status < 200 || status > 299)
+                        return Response.status(status).build();
+                }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
         //mark the port device id and device owner fields
@@ -493,12 +517,18 @@ public class NeutronRoutersNorthbound {
 
             Object[] instances = NeutronUtil.getInstances(INeutronRouterAware.class, this);
             if (instances != null) {
-                for (Object instance : instances) {
-                    INeutronRouterAware service = (INeutronRouterAware) instance;
-                    int status = service.canDetachInterface(target, input);
-                    if (status < 200 || status > 299)
-                        return Response.status(status).build();
+                if (instances.length > 0) {
+                    for (Object instance : instances) {
+                        INeutronRouterAware service = (INeutronRouterAware) instance;
+                        int status = service.canDetachInterface(target, input);
+                        if (status < 200 || status > 299)
+                            return Response.status(status).build();
+                    }
+                } else {
+                    throw new ServiceUnavailableException("No providers registered.  Please try again later");
                 }
+            } else {
+                throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
             }
 
             // reset the port ownership
@@ -528,11 +558,25 @@ public class NeutronRoutersNorthbound {
             input.setSubnetUUID(targetInterface.getSubnetUUID());
             input.setID(target.getID());
             input.setTenantID(target.getTenantID());
+            Object[] instances = NeutronUtil.getInstances(INeutronRouterAware.class, this);
+            if (instances != null) {
+                if (instances.length > 0) {
+                    for (Object instance : instances) {
+                        INeutronRouterAware service = (INeutronRouterAware) instance;
+                        int status = service.canDetachInterface(target, input);
+                        if (status < 200 || status > 299)
+                            return Response.status(status).build();
+                    }
+                } else {
+                    throw new ServiceUnavailableException("No providers registered.  Please try again later");
+                }
+            } else {
+                throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
+            }
             NeutronPort port = portInterface.getPort(input.getPortUUID());
             port.setDeviceID(null);
             port.setDeviceOwner(null);
             target.removeInterface(input.getPortUUID());
-            Object[] instances = NeutronUtil.getInstances(INeutronRouterAware.class, this);
             for (Object instance : instances) {
                 INeutronRouterAware service = (INeutronRouterAware) instance;
                 service.neutronRouterInterfaceDetached(target, input);
@@ -560,19 +604,32 @@ public class NeutronRoutersNorthbound {
             }
             if (!subnet.isValidIP(port.getFixedIPs().get(0).getIpAddress()))
                 throw new ResourceConflictException("Target Port IP not in Target Subnet");
-            input.setID(target.getID());
-            input.setTenantID(target.getTenantID());
             Object[] instances = NeutronUtil.getInstances(INeutronRouterAware.class, this);
             if (instances != null) {
-                for (Object instance : instances) {
-                    INeutronRouterAware service = (INeutronRouterAware) instance;
-                    service.canDetachInterface(target, input);
+                if (instances.length > 0) {
+                    for (Object instance : instances) {
+                        INeutronRouterAware service = (INeutronRouterAware) instance;
+                        int status = service.canDetachInterface(target, input);
+                        if (status < 200 || status > 299)
+                            return Response.status(status).build();
+                    }
+                } else {
+                    throw new ServiceUnavailableException("No providers registered.  Please try again later");
                 }
+            } else {
+                throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
             }
+            input.setID(target.getID());
+            input.setTenantID(target.getTenantID());
             port.setDeviceID(null);
             port.setDeviceOwner(null);
             target.removeInterface(input.getPortUUID());
-            for (Object instance : instances) {
+            if (instances != null) {
+                for (Object instance : instances) {
+                    INeutronRouterAware service = (INeutronRouterAware) instance;
+                    service.canDetachInterface(target, input);
+                }
+            }            for (Object instance : instances) {
                 INeutronRouterAware service = (INeutronRouterAware) instance;
                 service.neutronRouterInterfaceDetached(target, input);
             }
index bce4de7cc2b91887607ef720fcea62d93a044e8e..d9ca6d4512af618a53b1353f73a82f72ac9b16c9 100644 (file)
@@ -178,13 +178,19 @@ public class NeutronSecurityGroupsNorthbound {
 
             Object[] instances = NeutronUtil.getInstances(INeutronSecurityGroupAware.class, this);
             if (instances != null) {
-                for (Object instance : instances) {
-                    INeutronSecurityGroupAware service = (INeutronSecurityGroupAware) instance;
-                    int status = service.canCreateNeutronSecurityGroup(singleton);
-                    if (status < 200 || status > 299) {
-                        return Response.status(status).build();
+                if (instances.length > 0) {
+                    for (Object instance : instances) {
+                        INeutronSecurityGroupAware service = (INeutronSecurityGroupAware) instance;
+                        int status = service.canCreateNeutronSecurityGroup(singleton);
+                        if (status < 200 || status > 299) {
+                            return Response.status(status).build();
+                        }
                     }
+                } else {
+                    throw new ServiceUnavailableException("No providers registered.  Please try again later");
                 }
+            } else {
+                throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
             }
             // Add to Neutron cache
             securityGroupInterface.addNeutronSecurityGroup(singleton);
@@ -209,10 +215,18 @@ public class NeutronSecurityGroupsNorthbound {
                 if (securityGroupInterface.neutronSecurityGroupExists(test.getSecurityGroupUUID())) {
                     throw new BadRequestException("Security Group UUID already is already created");
                 }
-                if (instances != null) for (Object instance : instances) {
-                    INeutronSecurityGroupAware service = (INeutronSecurityGroupAware) instance;
-                    int status = service.canCreateNeutronSecurityGroup(test);
-                    if ((status < 200) || (status > 299)) return Response.status(status).build();
+                if (instances != null) {
+                    if (instances.length > 0) {
+                        for (Object instance : instances) {
+                            INeutronSecurityGroupAware service = (INeutronSecurityGroupAware) instance;
+                            int status = service.canCreateNeutronSecurityGroup(test);
+                            if ((status < 200) || (status > 299)) return Response.status(status).build();
+                        }
+                    } else {
+                        throw new BadRequestException("No providers registered.  Please try again later");
+                    }
+                } else {
+                    throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
                 }
             }
 
@@ -278,13 +292,19 @@ public class NeutronSecurityGroupsNorthbound {
 
         Object[] instances =  NeutronUtil.getInstances(INeutronSecurityGroupAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronSecurityGroupAware service = (INeutronSecurityGroupAware) instance;
-                int status = service.canUpdateNeutronSecurityGroup(delta, original);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronSecurityGroupAware service = (INeutronSecurityGroupAware) instance;
+                    int status = service.canUpdateNeutronSecurityGroup(delta, original);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
         /*
@@ -333,13 +353,19 @@ public class NeutronSecurityGroupsNorthbound {
         NeutronSecurityGroup singleton = securityGroupInterface.getNeutronSecurityGroup(securityGroupUUID);
         Object[] instances = NeutronUtil.getInstances(INeutronSecurityGroupAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronSecurityGroupAware service = (INeutronSecurityGroupAware) instance;
-                int status = service.canDeleteNeutronSecurityGroup(singleton);
-                if ((status < 200) || (status > 299)) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronSecurityGroupAware service = (INeutronSecurityGroupAware) instance;
+                    int status = service.canDeleteNeutronSecurityGroup(singleton);
+                    if ((status < 200) || (status > 299)) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
         /*
@@ -354,4 +380,4 @@ public class NeutronSecurityGroupsNorthbound {
         }
         return Response.status(204).build();
     }
-}
\ No newline at end of file
+}
index 762317aaf871888ede9ff3f94186f05d42e848e2..9ce98e2b4f343400c8f794e009decebbcd049d76 100644 (file)
@@ -201,13 +201,19 @@ public class NeutronSecurityRulesNorthbound {
             }
             Object[] instances = NeutronUtil.getInstances(INeutronSecurityRuleAware.class, this);
             if (instances != null) {
-                for (Object instance : instances) {
-                    INeutronSecurityRuleAware service = (INeutronSecurityRuleAware) instance;
-                    int status = service.canCreateNeutronSecurityRule(singleton);
-                    if ((status < 200) || (status > 299)) {
-                        return Response.status(status).build();
+                if (instances.length > 0) {
+                    for (Object instance : instances) {
+                        INeutronSecurityRuleAware service = (INeutronSecurityRuleAware) instance;
+                        int status = service.canCreateNeutronSecurityRule(singleton);
+                        if ((status < 200) || (status > 299)) {
+                            return Response.status(status).build();
+                        }
                     }
+                } else {
+                    throw new ServiceUnavailableException("No providers registered.  Please try again later");
                 }
+            } else {
+                throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
             }
 
             // add rule to cache
@@ -246,13 +252,19 @@ public class NeutronSecurityRulesNorthbound {
                     throw new BadRequestException("Security Rule UUID already exists");
                 }
                 if (instances != null) {
-                    for (Object instance : instances) {
-                        INeutronSecurityRuleAware service = (INeutronSecurityRuleAware) instance;
-                        int status = service.canCreateNeutronSecurityRule(test);
-                        if ((status < 200) || (status > 299)) {
-                            return Response.status(status).build();
+                    if (instances.length > 0) {
+                        for (Object instance : instances) {
+                            INeutronSecurityRuleAware service = (INeutronSecurityRuleAware) instance;
+                            int status = service.canCreateNeutronSecurityRule(test);
+                            if ((status < 200) || (status > 299)) {
+                                return Response.status(status).build();
+                            }
                         }
+                    } else {
+                        throw new ServiceUnavailableException("No providers registered.  Please try again later");
                     }
+                } else {
+                    throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
                 }
             }
 
@@ -328,13 +340,19 @@ public class NeutronSecurityRulesNorthbound {
 
         Object[] instances = NeutronUtil.getInstances(INeutronSecurityRuleAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronSecurityRuleAware service = (INeutronSecurityRuleAware) instance;
-                int status = service.canUpdateNeutronSecurityRule(delta, original);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronSecurityRuleAware service = (INeutronSecurityRuleAware) instance;
+                    int status = service.canUpdateNeutronSecurityRule(delta, original);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
         /*
@@ -383,15 +401,22 @@ public class NeutronSecurityRulesNorthbound {
         NeutronSecurityRule singleton = securityRuleInterface.getNeutronSecurityRule(securityRuleUUID);
         Object[] instances = NeutronUtil.getInstances(INeutronSecurityRuleAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronSecurityRuleAware service = (INeutronSecurityRuleAware) instance;
-                int status = service.canDeleteNeutronSecurityRule(singleton);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronSecurityRuleAware service = (INeutronSecurityRuleAware) instance;
+                    int status = service.canDeleteNeutronSecurityRule(singleton);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
+
         /*
          * remove it and return 204 status
          */
@@ -404,4 +429,4 @@ public class NeutronSecurityRulesNorthbound {
         }
         return Response.status(204).build();
     }
-}
\ No newline at end of file
+}
index 142b09f8e4f7ddc9559695f17cf28b755a1e2ee4..fa66501d0ef02183a2089ca9ddc82274e1552d60 100644 (file)
@@ -219,13 +219,19 @@ public class NeutronSubnetsNorthbound {
             }
             Object[] instances = NeutronUtil.getInstances(INeutronSubnetAware.class, this);
             if (instances != null) {
-                for (Object instance : instances) {
-                    INeutronSubnetAware service = (INeutronSubnetAware) instance;
-                    int status = service.canCreateSubnet(singleton);
-                    if (status < 200 || status > 299) {
-                        return Response.status(status).build();
+                if (instances.length > 0) {
+                    for (Object instance : instances) {
+                        INeutronSubnetAware service = (INeutronSubnetAware) instance;
+                        int status = service.canCreateSubnet(singleton);
+                        if (status < 200 || status > 299) {
+                            return Response.status(status).build();
+                        }
                     }
+                } else {
+                    throw new ServiceUnavailableException("No providers registered.  Please try again later");
                 }
+            } else {
+                throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
             }
             subnetInterface.addSubnet(singleton);
             if (instances != null) {
@@ -269,13 +275,19 @@ public class NeutronSubnetsNorthbound {
                     throw new ResourceConflictException("IP pool overlaps with gateway");
                 }
                 if (instances != null) {
-                    for (Object instance : instances) {
-                        INeutronSubnetAware service = (INeutronSubnetAware) instance;
-                        int status = service.canCreateSubnet(test);
-                        if (status < 200 || status > 299) {
-                            return Response.status(status).build();
+                    if (instances.length > 0) {
+                        for (Object instance : instances) {
+                            INeutronSubnetAware service = (INeutronSubnetAware) instance;
+                            int status = service.canCreateSubnet(test);
+                            if (status < 200 || status > 299) {
+                                return Response.status(status).build();
+                            }
                         }
+                    } else {
+                        throw new ServiceUnavailableException("No providers registered.  Please try again later");
                     }
+                } else {
+                    throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
                 }
             }
 
@@ -344,13 +356,19 @@ public class NeutronSubnetsNorthbound {
 
         Object[] instances = NeutronUtil.getInstances(INeutronSubnetAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronSubnetAware service = (INeutronSubnetAware) instance;
-                int status = service.canUpdateSubnet(delta, original);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronSubnetAware service = (INeutronSubnetAware) instance;
+                    int status = service.canUpdateSubnet(delta, original);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
         /*
@@ -399,13 +417,19 @@ public class NeutronSubnetsNorthbound {
         NeutronSubnet singleton = subnetInterface.getSubnet(subnetUUID);
         Object[] instances = NeutronUtil.getInstances(INeutronSubnetAware.class, this);
         if (instances != null) {
-            for (Object instance : instances) {
-                INeutronSubnetAware service = (INeutronSubnetAware) instance;
-                int status = service.canDeleteSubnet(singleton);
-                if (status < 200 || status > 299) {
-                    return Response.status(status).build();
+            if (instances.length > 0) {
+                for (Object instance : instances) {
+                    INeutronSubnetAware service = (INeutronSubnetAware) instance;
+                    int status = service.canDeleteSubnet(singleton);
+                    if (status < 200 || status > 299) {
+                        return Response.status(status).build();
+                    }
                 }
+            } else {
+                throw new ServiceUnavailableException("No providers registered.  Please try again later");
             }
+        } else {
+            throw new ServiceUnavailableException("Couldn't get providers list.  Please try again later");
         }
 
         /*