Added unit tests for flow mod tasks. 07/2007/1
authorTakayuki Kawagishi <kawagishi-takayuki@mxj.nes.nec.co.jp>
Fri, 18 Oct 2013 10:22:36 +0000 (19:22 +0900)
committerTakayuki Kawagishi <kawagishi-takayuki@mxj.nes.nec.co.jp>
Fri, 18 Oct 2013 10:22:36 +0000 (19:22 +0900)
Change-Id: Ib19343fd3c00d6e48a682432ced1d23ea5173b5b
Signed-off-by: Takayuki Kawagishi <kawagishi-takayuki@mxj.nes.nec.co.jp>
manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/ClusterFlowTaskTest.java [new file with mode: 0644]
manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/FlowModTaskTestBase.java [new file with mode: 0644]
manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/FlowTaskTest.java [new file with mode: 0644]
manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/LocalFlowTaskTest.java [new file with mode: 0644]
manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/RemoteFlowRequestTest.java [new file with mode: 0644]
manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/TestBase.java
manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/TestStub.java
manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/VTNFlowDatabaseTest.java [new file with mode: 0644]
manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/VTNManagerImplTestCommon.java
manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/VTNManagerImplWithNodesTest.java

diff --git a/manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/ClusterFlowTaskTest.java b/manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/ClusterFlowTaskTest.java
new file mode 100644 (file)
index 0000000..fac4575
--- /dev/null
@@ -0,0 +1,237 @@
+/*
+ * Copyright (c) 2013 NEC Corporation
+ * All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this
+ * distribution, and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.vtn.manager.internal;
+
+import static org.junit.Assert.*;
+
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import org.junit.Test;
+import org.opendaylight.controller.forwardingrulesmanager.FlowEntry;
+import org.opendaylight.controller.sal.core.Node;
+import org.opendaylight.controller.sal.core.NodeConnector;
+import org.opendaylight.controller.sal.match.Match;
+import org.opendaylight.controller.sal.match.MatchType;
+import org.opendaylight.controller.sal.utils.NodeConnectorCreator;
+import org.opendaylight.controller.sal.utils.NodeCreator;
+import org.opendaylight.vtn.manager.internal.cluster.ClusterEvent;
+import org.opendaylight.vtn.manager.internal.cluster.FlowModResult;
+import org.opendaylight.vtn.manager.internal.cluster.VTNFlow;
+
+/**
+ * test for {@link ClusterAddTask} and {@link ClusterRemoveTask}.
+ */
+public class ClusterFlowTaskTest extends FlowModTaskTestBase {
+
+    /**
+     * Test method for
+     * {@link ClusterFlowModTask#ClusterFlowModTask(VTNManagerImpl, FlowEntry)},
+     * {@link ClusterFlowAddTask#ClusterFlowAddTask(VTNManagerImpl, FlowEntry)},
+     * {@link ClusterFlowAddTask#ClusterFlowRemoveTask(VTNManagerImpl, FlowEntry)},
+     */
+    @Test
+    public void testClusterFlowTask() {
+        long timeout = vtnMgr.getVTNConfig().getFlowModTimeout();
+
+        VTNFlowDatabase fdb = new VTNFlowDatabase("test");
+        VTNFlow flow = fdb.create(vtnMgr);
+
+        Node node0 = NodeCreator.createOFNode(Long.valueOf(0L));
+        Node node1 = NodeCreator.createOFNode(Long.valueOf(1L));
+
+        // ingress only.
+        NodeConnector innc
+            = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("10"),
+                                                         node0);
+        NodeConnector outnc
+            = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                         node0);
+        Match match = new Match();
+        match.setField(MatchType.IN_PORT, innc);
+        match.setField(MatchType.DL_VLAN, (short) 1);
+        ActionList actions = new ActionList(outnc.getNode());
+        actions.addOutput(outnc);
+        int pri = 1;
+        flow.addFlow(vtnMgr, match, actions, pri);
+
+        // test ClusterFlowAddTask
+        int i = 0;
+        for (FlowEntry ent : flow.getFlowEntries()) {
+            ClusterFlowAddTask task = new ClusterFlowAddTask(vtnMgr, ent);
+            assertEquals(ent, task.getFlowEntry());
+            assertSame(vtnMgr, task.getVTNManager());
+
+            task.run();
+            assertEquals(FlowModResult.SUCCEEDED, task.getResult(timeout));
+            assertEquals(++i, stubObj.getFlowEntries().size());
+            assertTrue(stubObj.getFlowEntries().contains(ent));
+
+            Set<ClusterEvent> events = getPostedClusterEvent();
+            assertEquals(1, events.size());
+            clearPostedClusterEvent();
+        }
+
+        // add conflict flow entry.
+        FlowEntry entConflict = flow.getFlowEntries().iterator().next();
+        ClusterFlowAddTask taskConflict = new  ClusterFlowAddTask(vtnMgr, entConflict);
+        taskConflict.run();
+        assertEquals(FlowModResult.FAILED, taskConflict.getResult(timeout));
+        assertEquals(i, stubObj.getFlowEntries().size());
+        assertTrue(stubObj.getFlowEntries().contains(entConflict));
+
+        Set<ClusterEvent> events = getPostedClusterEvent();
+        assertEquals(1, events.size());
+        clearPostedClusterEvent();
+
+        // test ClusterFlowRemoveTask
+        for (FlowEntry ent : flow.getFlowEntries()) {
+            ClusterFlowRemoveTask task = new ClusterFlowRemoveTask(vtnMgr, ent);
+            assertEquals(ent, task.getFlowEntry());
+            assertSame(vtnMgr, task.getVTNManager());
+
+            task.run();
+            assertEquals(FlowModResult.SUCCEEDED, task.getResult(timeout));
+            assertEquals(--i, stubObj.getFlowEntries().size());
+            assertFalse(stubObj.getFlowEntries().contains(ent));
+
+            events = getPostedClusterEvent();
+            assertEquals(1, events.size());
+            clearPostedClusterEvent();
+        }
+
+        // remove not existing entry.
+        FlowEntry entNoexist = flow.getFlowEntries().iterator().next();
+        ClusterFlowRemoveTask taskNoexist
+            = new  ClusterFlowRemoveTask(vtnMgr, entNoexist);
+        taskNoexist.run();
+        assertEquals(FlowModResult.FAILED, taskNoexist.getResult(timeout));
+        assertEquals(0, stubObj.getFlowEntries().size());
+
+        events = getPostedClusterEvent();
+        assertEquals(1, events.size());
+        clearPostedClusterEvent();
+
+        // timed out
+        ForwardingRulesManagerStub frm = new ForwardingRulesManagerStub();
+        vtnMgr.setForwardingRuleManager(frm);
+
+        FlowEntry entTimeout = flow.getFlowEntries().iterator().next();
+        ClusterFlowAddTask taskTimeoutAdd = new  ClusterFlowAddTask(vtnMgr, entTimeout);
+        taskTimeoutAdd.run();
+        assertEquals(FlowModResult.FAILED, taskTimeoutAdd.getResult(timeout));
+        assertEquals(0, stubObj.getFlowEntries().size());
+
+        events = getPostedClusterEvent();
+        assertEquals(1, events.size());
+        clearPostedClusterEvent();
+
+        ClusterFlowRemoveTask taskTimeoutRemove
+            = new ClusterFlowRemoveTask(vtnMgr, entTimeout);
+        taskTimeoutRemove.run();
+        assertEquals(FlowModResult.FAILED, taskTimeoutRemove.getResult(timeout));
+        assertEquals(0, stubObj.getFlowEntries().size());
+
+        events = getPostedClusterEvent();
+        assertEquals(1, events.size());
+        clearPostedClusterEvent();
+
+        // not local entry.
+        flow = fdb.create(vtnMgr);
+        innc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("10"),
+                                                          node1);
+        outnc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                           node1);
+        match = new Match();
+        match.setField(MatchType.IN_PORT, innc);
+        match.setField(MatchType.DL_VLAN, (short) 1);
+        actions = new ActionList(outnc.getNode());
+        actions.addOutput(outnc);
+        flow.addFlow(vtnMgr, match, actions, pri);
+        ClusterFlowAddTask addTask = new ClusterFlowAddTask(vtnMgr, flow.getFlowEntries().get(0));
+        addTask.run();
+        // when ignored, getResult() return SUCCEEDED
+        assertEquals(FlowModResult.SUCCEEDED, addTask.getResult(timeout));
+        assertEquals(0, stubObj.getFlowEntries().size());
+
+        events = getPostedClusterEvent();
+        assertEquals(1, events.size());
+        clearPostedClusterEvent();
+
+        ClusterFlowRemoveTask removeTask = new ClusterFlowRemoveTask(vtnMgr, flow.getFlowEntries().get(0));
+        removeTask.run();
+        // when ignored, getResult() return SUCCEEDED
+        assertEquals(FlowModResult.SUCCEEDED, removeTask.getResult(timeout));
+        assertEquals(0, stubObj.getFlowEntries().size());
+
+        events = getPostedClusterEvent();
+        assertEquals(1, events.size());
+        clearPostedClusterEvent();
+    }
+
+    /**
+     * Test method for
+     * {@link FlowModTask#getResult(long)},
+     * {@link FlowModTask#getResultAbs(long)},
+     * {@link FlowModTask#setResult(boolean)}.
+     */
+    @Test
+    public void testGetResult() {
+        VTNFlowDatabase fdb = new VTNFlowDatabase("test");
+        VTNFlow vflow = fdb.create(vtnMgr);
+        long timeout = vtnMgr.getVTNConfig().getFlowModTimeout();
+
+        // getResult() timeout
+        FlowEntry flow = null;
+        ClusterFlowAddTask addTask = new ClusterFlowAddTask(vtnMgr, flow);
+        long start = System.currentTimeMillis();
+        assertEquals(FlowModResult.TIMEDOUT, addTask.getResult(timeout));
+        long end = System.currentTimeMillis();
+        assertTrue(timeout <= (end - start));
+
+        long timeoutRemote = vtnMgr.getVTNConfig().getRemoteFlowModTimeout();
+        ClusterFlowRemoveTask rmTask = new ClusterFlowRemoveTask(vtnMgr, flow);
+        start = System.currentTimeMillis();
+        assertEquals(FlowModResult.TIMEDOUT, rmTask.getResult(timeoutRemote));
+        end = System.currentTimeMillis();
+        assertTrue(timeoutRemote <= (end - start));
+
+        // succeeded
+        addTask = new ClusterFlowAddTask(vtnMgr, flow);
+        addTask.setResult(true);
+        assertEquals(FlowModResult.SUCCEEDED, addTask.getResult(timeout));
+
+        rmTask = new ClusterFlowRemoveTask(vtnMgr, flow);
+        rmTask.setResult(true);
+        assertEquals(FlowModResult.SUCCEEDED, rmTask.getResult(timeoutRemote));
+
+        // failed
+        addTask = new ClusterFlowAddTask(vtnMgr, flow);
+        addTask.setResult(false);
+        assertEquals(FlowModResult.FAILED, addTask.getResult(timeout));
+
+        rmTask = new ClusterFlowRemoveTask(vtnMgr, flow);
+        rmTask.setResult(false);
+        assertEquals(FlowModResult.FAILED, rmTask.getResult(timeoutRemote));
+
+        // getResult() interrupted
+        TimerTask timerTask = new InterruptTask(Thread.currentThread());
+        Timer timer = new Timer();
+
+        addTask = new ClusterFlowAddTask(vtnMgr, flow);
+        timer.schedule(timerTask, timeout / 2);
+        assertEquals(FlowModResult.INTERRUPTED, addTask.getResult(timeout));
+
+        timerTask = new InterruptTask(Thread.currentThread());
+        rmTask = new ClusterFlowRemoveTask(vtnMgr, flow);
+        timer.schedule(timerTask, timeoutRemote / 2);
+        assertEquals(FlowModResult.INTERRUPTED, rmTask.getResult(timeoutRemote));
+    }
+}
\ No newline at end of file
diff --git a/manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/FlowModTaskTestBase.java b/manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/FlowModTaskTestBase.java
new file mode 100644 (file)
index 0000000..fd3b89f
--- /dev/null
@@ -0,0 +1,589 @@
+/*
+ * Copyright (c) 2013 NEC Corporation
+ * All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this
+ * distribution, and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.vtn.manager.internal;
+
+import static org.junit.Assert.*;
+
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Hashtable;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimerTask;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.felix.dm.impl.ComponentImpl;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.opendaylight.controller.clustering.services.IClusterGlobalServices;
+import org.opendaylight.controller.connectionmanager.ConnectionMgmtScheme;
+import org.opendaylight.controller.connectionmanager.IConnectionManager;
+import org.opendaylight.controller.forwardingrulesmanager.FlowConfig;
+import org.opendaylight.controller.forwardingrulesmanager.FlowEntry;
+import org.opendaylight.controller.forwardingrulesmanager.IForwardingRulesManager;
+import org.opendaylight.controller.forwardingrulesmanager.PortGroupConfig;
+import org.opendaylight.controller.forwardingrulesmanager.PortGroupProvider;
+import org.opendaylight.controller.sal.connection.ConnectionConstants;
+import org.opendaylight.controller.sal.connection.ConnectionLocality;
+import org.opendaylight.controller.sal.core.Node;
+import org.opendaylight.controller.sal.core.NodeConnector;
+import org.opendaylight.controller.sal.flowprogrammer.Flow;
+import org.opendaylight.controller.sal.flowprogrammer.IFlowProgrammerListener;
+import org.opendaylight.controller.sal.match.Match;
+import org.opendaylight.controller.sal.match.MatchType;
+import org.opendaylight.controller.sal.utils.NodeConnectorCreator;
+import org.opendaylight.controller.sal.utils.NodeCreator;
+import org.opendaylight.controller.sal.utils.Status;
+import org.opendaylight.controller.sal.utils.StatusCode;
+import org.opendaylight.vtn.manager.internal.VTNManagerImplTestCommon.NopFlowTask;
+import org.opendaylight.vtn.manager.internal.cluster.ClusterEvent;
+import org.opendaylight.vtn.manager.internal.cluster.FlowGroupId;
+import org.opendaylight.vtn.manager.internal.cluster.FlowModResult;
+import org.opendaylight.vtn.manager.internal.cluster.FlowModResultEvent;
+import org.opendaylight.vtn.manager.internal.cluster.VTNFlow;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Common class for tests of FlowModTask.
+ */
+public class FlowModTaskTestBase extends TestBase {
+    protected VTNManagerImpl vtnMgr = null;
+    protected GlobalResourceManager resMgr;
+    protected TestStub stubObj = null;
+    protected static int stubMode = 0;
+    protected static int clusterMode = 0;
+
+    protected class ConnectionManagerStub implements IConnectionManager {
+
+        @Override
+        public Node connect(String arg0, Map<ConnectionConstants, String> arg1) {
+            return null;
+        }
+
+        @Override
+        public Node connect(String arg0, String arg1,
+                Map<ConnectionConstants, String> arg2) {
+            return null;
+        }
+
+        @Override
+        public Status disconnect(Node arg0) {
+            return null;
+        }
+
+        @Override
+        public ConnectionMgmtScheme getActiveScheme() {
+            return null;
+        }
+
+        @Override
+        public Set<Node> getLocalNodes() {
+            return null;
+        }
+
+        @Override
+        public ConnectionLocality getLocalityStatus(Node arg0) {
+            if (arg0.getID().equals(Long.valueOf("0"))) {
+                return ConnectionLocality.LOCAL;
+            } else if (arg0.getID().equals(Long.valueOf("1"))) {
+                return ConnectionLocality.NOT_LOCAL;
+            } else {
+                return ConnectionLocality.NOT_CONNECTED;
+            }
+        }
+
+        @Override
+        public Set<Node> getNodes(InetAddress arg0) {
+            return null;
+        }
+
+        @Override
+        public boolean isLocal(Node arg0) {
+            return false;
+        }
+    }
+
+    protected class ForwardingRulesManagerStub implements
+            IForwardingRulesManager {
+        /**
+         * frmMode
+         *  0 : installLocal() and uninstallLocal don't return.
+         *  1 : installLocal() and uninstallLocal always return
+         *      set code.
+         *
+         */
+        private int frmMode = 0;
+
+        private StatusCode returnStatusCode = null;
+
+        /**
+         * constructor
+         */
+        ForwardingRulesManagerStub () {
+
+        }
+
+        /**
+         * constructor
+         */
+        ForwardingRulesManagerStub(int mode, StatusCode code) {
+            frmMode = mode;
+            returnStatusCode = code;
+        }
+
+        @Override
+        public void addOutputPort(Node arg0, String arg1,
+                List<NodeConnector> arg2) {
+        }
+
+        @Override
+        public boolean addPortGroupConfig(String arg0, String arg1, boolean arg2) {
+            return false;
+        }
+
+        @Override
+        public Status addStaticFlow(FlowConfig arg0) {
+            return null;
+        }
+
+        @Override
+        public boolean checkFlowEntryConflict(FlowEntry arg0) {
+            return false;
+        }
+
+        @Override
+        public boolean delPortGroupConfig(String arg0) {
+            return false;
+        }
+
+        @Override
+        public List<FlowEntry> getFlowEntriesForGroup(String arg0) {
+            return null;
+        }
+
+        @Override
+        public List<FlowEntry> getInstalledFlowEntriesForGroup(String arg0) {
+            return null;
+        }
+
+        @Override
+        public List<Node> getListNodeWithConfiguredFlows() {
+            return null;
+        }
+
+        @Override
+        public NodeConnector getOutputPort(Node arg0, String arg1) {
+            return null;
+        }
+
+        @Override
+        public Map<String, PortGroupConfig> getPortGroupConfigs() {
+            return null;
+        }
+
+        @Override
+        public PortGroupProvider getPortGroupProvider() {
+            return null;
+        }
+
+        @Override
+        public FlowConfig getStaticFlow(String arg0, Node arg1) {
+            return null;
+        }
+
+        @Override
+        public List<String> getStaticFlowNamesForNode(Node arg0) {
+            return null;
+        }
+
+        @Override
+        public List<FlowConfig> getStaticFlows() {
+            return null;
+        }
+
+        @Override
+        public List<FlowConfig> getStaticFlows(Node arg0) {
+            return null;
+        }
+
+        @Override
+        public Map<String, Object> getTSPolicyData() {
+            return null;
+        }
+
+        @Override
+        public Object getTSPolicyData(String arg0) {
+            return null;
+        }
+
+        @Override
+        public Status installFlowEntry(FlowEntry arg0) {
+            if (frmMode == 1) {
+                return new Status(returnStatusCode, null);
+            }
+
+            long timeout = vtnMgr.getVTNConfig().getFlowModTimeout();
+            try {
+                Thread.sleep(timeout * 2);
+            } catch (InterruptedException e) {
+                return new Status(StatusCode.UNDEFINED);
+            }
+            return new Status(StatusCode.SUCCESS);
+        }
+
+        @Override
+        public Status installFlowEntryAsync(FlowEntry arg0) {
+            return null;
+        }
+
+        @Override
+        public Status modifyFlowEntry(FlowEntry arg0, FlowEntry arg1) {
+            return null;
+        }
+
+        @Override
+        public Status modifyFlowEntryAsync(FlowEntry arg0, FlowEntry arg1) {
+            return null;
+        }
+
+        @Override
+        public Status modifyOrAddFlowEntry(FlowEntry arg0) {
+            return null;
+        }
+
+        @Override
+        public Status modifyOrAddFlowEntryAsync(FlowEntry arg0) {
+            return null;
+        }
+
+        @Override
+        public Status modifyStaticFlow(FlowConfig arg0) {
+            return null;
+        }
+
+        @Override
+        public void removeOutputPort(Node arg0, String arg1,
+                List<NodeConnector> arg2) {
+        }
+
+        @Override
+        public Status removeStaticFlow(FlowConfig arg0) {
+            return null;
+        }
+
+        @Override
+        public Status removeStaticFlow(String arg0, Node arg1) {
+            return null;
+        }
+
+        @Override
+        public void replaceOutputPort(Node arg0, String arg1, NodeConnector arg2) {
+        }
+
+        @Override
+        public Status saveConfig() {
+            return null;
+        }
+
+        @Override
+        public void setTSPolicyData(String arg0, Object arg1, boolean arg2) {
+        }
+
+        @Override
+        public Status solicitStatusResponse(Node arg0, boolean arg1) {
+            return null;
+        }
+
+        @Override
+        public Status toggleStaticFlowStatus(FlowConfig arg0) {
+            return null;
+        }
+
+        @Override
+        public Status toggleStaticFlowStatus(String arg0, Node arg1) {
+            return null;
+        }
+
+        @Override
+        public Status uninstallFlowEntry(FlowEntry arg0) {
+            if (frmMode == 1) {
+                return new Status(returnStatusCode, null);
+            }
+
+            long timeout = vtnMgr.getVTNConfig().getFlowModTimeout();
+            try {
+                Thread.sleep(timeout * 2);
+            } catch (InterruptedException e) {
+                return new Status(StatusCode.UNDEFINED);
+            }
+            return new Status(StatusCode.SUCCESS);
+        }
+
+        @Override
+        public Status uninstallFlowEntryAsync(FlowEntry arg0) {
+            return null;
+        }
+
+        @Override
+        public Status uninstallFlowEntryGroup(String arg0) {
+            return null;
+        }
+
+        @Override
+        public Status uninstallFlowEntryGroupAsync(String arg0) {
+            return null;
+        }
+    }
+
+    /**
+     * Stub of VTNManagerImpl.
+     */
+    protected class VTNManagerImplStub extends VTNManagerImpl {
+        Set<ClusterEvent> eventSet = new HashSet<ClusterEvent>();
+
+        // Override postEvent() for tests.
+        @Override
+        public void postEvent(ClusterEvent cev) {
+           eventSet.add(cev);
+        }
+
+        public Set<ClusterEvent> getClusterEventSetStub() {
+            return eventSet;
+        }
+
+        public void clearClusterEventSetStub() {
+            eventSet.clear();
+        }
+    }
+
+    protected Set<ClusterEvent> getPostedClusterEvent() {
+        Set<ClusterEvent> sets
+            = ((VTNManagerImplStub) vtnMgr).getClusterEventSetStub();
+        return new HashSet<ClusterEvent>(sets);
+    }
+
+    protected void clearPostedClusterEvent() {
+        ((VTNManagerImplStub) vtnMgr).clearClusterEventSetStub();
+    }
+
+
+    /**
+     * A timer task used to emulate remote event.
+     */
+    protected class ResultTimerTask extends TimerTask {
+        private VTNManagerImpl vtnManager = null;
+        private FlowEntry flowEntry = null;
+        private FlowModResult result = null;
+        private boolean isLocal = false;
+
+        public ResultTimerTask(VTNManagerImpl mgr, FlowEntry ent, FlowModResult res,
+                               boolean local) {
+            vtnManager = mgr;
+            flowEntry = ent;
+            result = res;
+            isLocal = local;
+        }
+
+        @Override
+        public void run() {
+            FlowModResultEvent re
+                = new FlowModResultEvent(flowEntry.getFlowName(), result);
+            re.received(vtnManager, isLocal);
+        }
+    }
+
+    /**
+     * A timer task used to interrupt a thread.
+     */
+    protected class InterruptTask extends TimerTask {
+        private Thread targetThread = null;
+
+        InterruptTask(Thread th) {
+            targetThread = th;
+        }
+
+        @Override
+        public void run() {
+            targetThread.interrupt();
+        }
+    }
+
+
+    @BeforeClass
+    static public void beforeClass() {
+        stubMode = 2;
+    }
+
+    @Before
+    public void before() {
+        setupStartupDir();
+
+        vtnMgr = new VTNManagerImplStub();
+        resMgr = new GlobalResourceManager();
+        ComponentImpl c = new ComponentImpl(null, null, null);
+        stubObj = new TestStub(stubMode);
+        ConnectionManagerStub cm = new ConnectionManagerStub();
+
+        Hashtable<String, String> properties = new Hashtable<String, String>();
+        properties.put("containerName", "default");
+        c.setServiceProperties(properties);
+
+        resMgr.setClusterGlobalService(stubObj);
+        resMgr.init(c);
+        vtnMgr.setResourceManager(resMgr);
+        vtnMgr.setClusterContainerService(stubObj);
+        vtnMgr.setSwitchManager(stubObj);
+        vtnMgr.setTopologyManager(stubObj);
+        vtnMgr.setDataPacketService(stubObj);
+        vtnMgr.setRouting(stubObj);
+        vtnMgr.setHostTracker(stubObj);
+        vtnMgr.setForwardingRuleManager(stubObj);
+        vtnMgr.setConnectionManager(cm);
+        startVTNManager(c);
+    }
+
+    @After
+    public void after() {
+        stopVTNManager(true);
+        resMgr.destroy();
+
+        cleanupStartupDir();
+    }
+
+    /**
+     * startup VTNManager
+     */
+    protected void startVTNManager(ComponentImpl c) {
+        vtnMgr.init(c);
+        vtnMgr.clearDisabledNode();
+    }
+
+    /**
+     * stop VTNManager
+     * @param clearCache    if true clear cache maintained in VTNManager.
+     */
+    protected void stopVTNManager(boolean clearCache) {
+        vtnMgr.stopping();
+        if (clearCache) {
+            vtnMgr.containerDestroy();
+        }
+        vtnMgr.stop();
+        vtnMgr.destroy();
+    }
+
+    /**
+     *
+     */
+    protected void setupResourceManager(ComponentImpl cmp,
+                                        IClusterGlobalServices cls) {
+        resMgr.setClusterGlobalService(cls);
+        resMgr.init(cmp);
+    }
+
+    /**
+     * Flush all pending tasks on the VTN flow thread.
+     */
+    protected void flushFlowTasks() {
+        NopFlowTask task = new NopFlowTask(vtnMgr);
+        vtnMgr.postFlowTask(task);
+        assertTrue(task.await(10L, TimeUnit.SECONDS));
+    }
+
+    /**
+     * Flush all pending tasks on the VTN flow thread.
+     */
+    protected void flushFlowTasks(long wait) {
+        NopFlowTask task = new NopFlowTask(vtnMgr);
+        vtnMgr.postFlowTask(task);
+        assertTrue(task.await(wait, TimeUnit.SECONDS));
+    }
+
+    /**
+     *  A dummy flow task to flush pending tasks.
+     */
+    protected class NopFlowTask extends FlowModTask {
+        /**
+         * A latch to wait for completion.
+         */
+        private final CountDownLatch  latch = new CountDownLatch(1);
+
+        protected NopFlowTask(VTNManagerImpl mgr) {
+            super(mgr);
+        }
+
+        /**
+         * Wake up all threads waiting for this task.
+         *
+         * @return  {@code true} is always returned.
+         */
+        @Override
+        protected boolean execute() {
+            latch.countDown();
+            return true;
+        }
+
+        /**
+         * Return a logger object for this class.
+         *
+         * @return  A logger object.
+         */
+        @Override
+        protected Logger getLogger() {
+            return LoggerFactory.getLogger(getClass());
+        }
+
+        /**
+         * Wait for completion of this task.
+         *
+         * @param timeout  The maximum time to wait.
+         * @param unit     The time unit of the {@code timeout} argument.
+         * @return  {@code true} is returned if this task completed.
+         *          Otherwise {@code false} is returned.
+         */
+        private boolean await(long timeout, TimeUnit unit) {
+            try {
+                return latch.await(timeout, unit);
+            } catch (InterruptedException e) {
+                return false;
+            }
+        }
+    }
+
+    /**
+     * Add FlowEntry to VTNFlow.
+     *
+     * @param   flow        A {@link VTNFlow}.
+     * @param   inPort      A ingress {@link NodeConector}.
+     * @param   inVlan      A incoming VLAN ID.
+     * @param   outPort     A outgoing {@link NodeConnector}.
+     * @param   priority    A priority of FlowEntry.
+     * @return {@link VTNFlow}.
+     */
+    protected VTNFlow addFlowEntry(VTNManagerImpl mgr, VTNFlow flow,
+            NodeConnector inPort, short inVlan, NodeConnector outPort,
+            int priority) {
+        Match match = new Match();
+        match.setField(MatchType.IN_PORT, inPort);
+        match.setField(MatchType.DL_VLAN, inVlan);
+        ActionList actions = new ActionList(outPort.getNode());
+        actions.addOutput(outPort);
+        flow.addFlow(mgr, match, actions, priority);
+
+        return flow;
+    }
+
+}
diff --git a/manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/FlowTaskTest.java b/manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/FlowTaskTest.java
new file mode 100644 (file)
index 0000000..c43c7e5
--- /dev/null
@@ -0,0 +1,675 @@
+/*
+ * Copyright (c) 2013 NEC Corporation
+ * All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this
+ * distribution, and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.vtn.manager.internal;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Hashtable;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.felix.dm.impl.ComponentImpl;
+import org.junit.Test;
+import org.opendaylight.controller.forwardingrulesmanager.FlowEntry;
+import org.opendaylight.controller.sal.core.Node;
+import org.opendaylight.controller.sal.core.NodeConnector;
+import org.opendaylight.controller.sal.match.Match;
+import org.opendaylight.controller.sal.match.MatchType;
+import org.opendaylight.controller.sal.utils.NodeConnectorCreator;
+import org.opendaylight.controller.sal.utils.NodeCreator;
+import org.opendaylight.controller.sal.utils.StatusCode;
+import org.opendaylight.vtn.manager.internal.cluster.ClusterEvent;
+import org.opendaylight.vtn.manager.internal.cluster.FlowGroupId;
+import org.opendaylight.vtn.manager.internal.cluster.FlowModResult;
+import org.opendaylight.vtn.manager.internal.cluster.VTNFlow;
+
+/**
+ * test for {@link FlowAddTaskTest} and {@link FlowRemoveTaskTest}.
+ */
+public class FlowTaskTest extends FlowModTaskTestBase {
+
+    /**
+     * Test method for
+     * {@link FlowAddTask#execute()},
+     * {@link FlowRemoveTask#execute()},
+     * {@link FlowModTask#run()}.
+     */
+    @Test
+    public void testFlowTaskTest() {
+        long timeout = vtnMgr.getVTNConfig().getFlowModTimeout();
+        long remoteTimeout = vtnMgr.getVTNConfig().getRemoteFlowModTimeout();
+        VTNFlowDatabase fdb = new VTNFlowDatabase("test");
+        VTNFlow flow = fdb.create(vtnMgr);
+
+        Node node0 = NodeCreator.createOFNode(Long.valueOf(0L));
+        Node node1 = NodeCreator.createOFNode(Long.valueOf(1L));
+        Node node2 = NodeCreator.createOFNode(Long.valueOf(2L));
+
+        // in case ingress only.
+        NodeConnector inncIngress
+            = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("10"),
+                                                         node0);
+        NodeConnector outncIngress
+            = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                         node0);
+        int pri = 1;
+
+        flow = addFlowEntry(vtnMgr, flow, inncIngress, (short) 1,
+                            outncIngress, pri);
+        FlowAddTask task = new FlowAddTask(vtnMgr, flow);
+        task.run();
+        assertEquals(FlowModResult.SUCCEEDED, task.getResult(timeout));
+        checkRegsiterdFlowEntry(vtnMgr, 1, flow, flow, 1);
+
+        Iterator<FlowEntry> it = flow.getFlowEntries().iterator();
+        FlowEntry ingress = it.next();
+        FlowRemoveTask rtask = new FlowRemoveTask(vtnMgr, flow.getGroupId(),
+                                                  ingress, it);
+        rtask.run();
+        assertEquals(FlowModResult.SUCCEEDED, rtask.getResult());
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        task.run();
+        assertEquals(FlowModResult.SUCCEEDED, task.getResult(timeout));
+        checkRegsiterdFlowEntry(vtnMgr, 1, flow, flow, 1);
+
+        fdb.createIndex(vtnMgr, flow);
+        fdb.clear(vtnMgr);
+        flushFlowTasks(remoteTimeout);
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        // in case ingress + local.
+        NodeConnector innc
+            = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("12"),
+                                                         node0);
+        NodeConnector outnc
+            = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                         node0);
+        flow = addFlowEntry(vtnMgr, flow, innc, (short) 1, outnc, pri);
+        task = new FlowAddTask(vtnMgr, flow);
+        task.run();
+        assertEquals(FlowModResult.SUCCEEDED, task.getResult(timeout));
+        checkRegsiterdFlowEntry(vtnMgr, 1, flow, flow, 2);
+
+        it = flow.getFlowEntries().iterator();
+        ingress = it.next();
+        rtask = new FlowRemoveTask(vtnMgr, flow.getGroupId(), ingress, it);
+        rtask.run();
+        assertEquals(FlowModResult.SUCCEEDED, rtask.getResult());
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        task.run();
+        assertEquals(FlowModResult.SUCCEEDED, task.getResult(timeout));
+        checkRegsiterdFlowEntry(vtnMgr, 1, flow, flow, 2);
+
+        // in case uninstallLocal() fails.
+        ForwardingRulesManagerStub stub
+            = new ForwardingRulesManagerStub(1, StatusCode.BADREQUEST);
+        vtnMgr.setForwardingRuleManager(stub);
+        it = flow.getFlowEntries().iterator();
+        ingress = it.next();
+        rtask = new FlowRemoveTask(vtnMgr, flow.getGroupId(), ingress, it);
+        rtask.run();
+        assertEquals(FlowModResult.FAILED, rtask.getResult());
+
+        vtnMgr.setForwardingRuleManager(stubObj);
+        fdb.createIndex(vtnMgr, flow);
+        fdb.clear(vtnMgr);
+        flushFlowTasks(remoteTimeout);
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        // in case ingress + local + remote.
+        // in this case failed to add remote flow entry.
+        innc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("10"),
+                                                          node1);
+        outnc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                           node1);
+        flow = addFlowEntry(vtnMgr, flow, innc, (short) 1, outnc, pri);
+        task = new FlowAddTask(vtnMgr, flow);
+        task.run();
+        assertEquals(FlowModResult.FAILED, task.getResult(remoteTimeout));
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        it = flow.getFlowEntries().iterator();
+        ingress = it.next();
+        rtask = new FlowRemoveTask(vtnMgr, flow.getGroupId(), ingress, it);
+        rtask.run();
+        assertEquals(FlowModResult.FAILED, rtask.getResult());
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        fdb.clear(vtnMgr);
+        flushFlowTasks(remoteTimeout);
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        // ingress + local + remote + not connected node.
+        innc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("10"),
+                                                          node2);
+        outnc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                           node2);
+        flow = addFlowEntry(vtnMgr, flow, innc, (short) 1, outnc, pri);
+        task = new FlowAddTask(vtnMgr, flow);
+        task.run();
+        assertEquals(FlowModResult.FAILED, task.getResult(remoteTimeout));
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        it = flow.getFlowEntries().iterator();
+        ingress = it.next();
+        rtask = new FlowRemoveTask(vtnMgr, flow.getGroupId(), ingress, it);
+        rtask.run();
+        assertEquals(FlowModResult.FAILED, rtask.getResult());
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        fdb.clear(vtnMgr);
+        flushFlowTasks(remoteTimeout);
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        // ingress + remote
+        // in this case fail to add remote flow entry.
+        VTNFlow flowRemote = fdb.create(vtnMgr);
+        flowRemote = addFlowEntry(vtnMgr, flowRemote, inncIngress, (short) 1,
+                                  outncIngress, pri);
+
+        innc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("10"),
+                                                          node1);
+        outnc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                           node1);
+        flowRemote = addFlowEntry(vtnMgr, flowRemote, innc, (short) 1, outnc, pri);
+
+        task = new FlowAddTask(vtnMgr, flowRemote);
+        task.run();
+        assertEquals(FlowModResult.FAILED, task.getResult(remoteTimeout));
+        checkRegsiterdFlowEntry(vtnMgr, 0, flowRemote, null, 0);
+
+        // in case ingress is remote node.
+        flow = fdb.create(vtnMgr);
+        innc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("10"),
+                                                          node1);
+        outnc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                           node1);
+        flow = addFlowEntry(vtnMgr, flow, innc, (short) 1, outnc, pri);
+        task = new FlowAddTask(vtnMgr, flow);
+        task.run();
+        assertEquals(FlowModResult.FAILED, task.getResult(remoteTimeout));
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        // in case ingress is not connected node.
+        flow = fdb.create(vtnMgr);
+        innc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("10"),
+                                                          node2);
+        outnc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                           node2);
+        flow = addFlowEntry(vtnMgr, flow, innc, (short) 1, outnc, pri);
+
+        task = new FlowAddTask(vtnMgr, flow);
+        task.run();
+        assertEquals(FlowModResult.FAILED, task.getResult(remoteTimeout));
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        // empty flow
+        flow = fdb.create(vtnMgr);
+        task = new FlowAddTask(vtnMgr, flow);
+        task.run();
+        assertEquals(FlowModResult.FAILED, task.getResult(remoteTimeout));
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+    }
+
+    /**
+     * Test method for
+     * {@link FlowAddTask#execute()},
+     * {@link FlowRemoveTask#execute()},
+     * {@link FlowModTask#run()}.
+     *
+     * In this case {@link installLocal()} and
+     * {@link uninstallLocal()} timeout.
+     */
+    @Test
+    public void testFlowTaskTestLocalTimeout() {
+        long timeout = vtnMgr.getVTNConfig().getFlowModTimeout();
+
+        VTNFlowDatabase fdb = new VTNFlowDatabase("test");
+        VTNFlow flow = fdb.create(vtnMgr);
+
+        Node node0 = NodeCreator.createOFNode(Long.valueOf(0L));
+
+        // replace forwardingRulesManager to stub.
+        // installLocal() and uninstallLocal() implemented in it
+        // wait until timeout.
+        ForwardingRulesManagerStub frm = new ForwardingRulesManagerStub();
+        vtnMgr.setForwardingRuleManager(frm);
+
+        // in case ingress only.
+        NodeConnector innc
+            = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("12"),
+                                                         node0);
+        NodeConnector outnc
+            = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                         node0);
+        int pri = 1;
+        flow = addFlowEntry(vtnMgr, flow, innc, (short) 1, outnc, pri);
+
+        fdb.createIndex(vtnMgr, flow);
+        FlowAddTask task = new FlowAddTask(vtnMgr, flow);
+        task.run();
+        assertEquals(FlowModResult.FAILED, task.getResult(timeout));
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        // in case ingress + local
+        innc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("12"),
+                                                          node0);
+        outnc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                           node0);
+        flow = addFlowEntry(vtnMgr, flow, innc, (short) 1, outnc, pri);
+        task = new FlowAddTask(vtnMgr, flow);
+        task.run();
+        assertEquals(FlowModResult.FAILED, task.getResult(timeout));
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        // in case thread is interrupted.
+        TimerTask timerTask = new InterruptTask(Thread.currentThread());
+        Timer timer = new Timer();
+
+        task = new FlowAddTask(vtnMgr, flow);
+        timer.schedule(timerTask, timeout / 2);
+        task.run();
+        assertEquals(FlowModResult.FAILED, task.getResult(timeout));
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+    }
+
+    /**
+     * Test method for
+     * {@link FlowAddTask#execute()},
+     * {@link FlowRemoveTask#execute()},
+     * {@link FlowModTask#run()}.
+     *
+     * <p>
+     *  in case system have Remote cluster nodes.
+     * </p>
+     */
+    @Test
+    public void testFlowTaskTestHavingRemoteNodes() {
+        // set IClusterGlobalService to stub which work
+        // as have multiple cluster nodes.
+        TestStub stubNew = new TestStub(2, 2);
+        ComponentImpl c = new ComponentImpl(null, null, null);
+        Hashtable<String, String> properties = new Hashtable<String, String>();
+        properties.put("containerName", "default");
+        c.setServiceProperties(properties);
+        resMgr.setClusterGlobalService(stubNew);
+        resMgr.init(c);
+        stopVTNManager(true);
+        startVTNManager(c);
+
+        long timeout = vtnMgr.getVTNConfig().getFlowModTimeout();
+        long remoteTimeout = vtnMgr.getVTNConfig().getRemoteFlowModTimeout();
+        VTNFlowDatabase fdb = new VTNFlowDatabase("test");
+        VTNFlow flow = fdb.create(vtnMgr);
+
+        Node node0 = NodeCreator.createOFNode(Long.valueOf(0L));
+        Node node1 = NodeCreator.createOFNode(Long.valueOf(1L));
+        Node node2 = NodeCreator.createOFNode(Long.valueOf(2L));
+
+        // ingress
+        NodeConnector innc
+            = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("10"),
+                                                         node0);
+        NodeConnector outnc
+            = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                         node0);
+        Match match = new Match();
+        match.setField(MatchType.IN_PORT, innc);
+        match.setField(MatchType.DL_VLAN, (short) 1);
+        ActionList actions = new ActionList(outnc.getNode());
+        actions.addOutput(outnc);
+        int pri = 1;
+        flow.addFlow(vtnMgr, match, actions, pri);
+
+        // + local entry.
+        innc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("12"),
+                                                          node0);
+        outnc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                           node0);
+        flow = addFlowEntry(vtnMgr, flow, innc, (short) 1, outnc, pri);
+
+        // + remote entry.
+        innc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("10"),
+                                                          node1);
+        outnc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                           node1);
+        flow = addFlowEntry(vtnMgr, flow, innc, (short) 1, outnc, pri);
+        FlowAddTask task = new FlowAddTask(vtnMgr, flow);
+        task.run();
+        assertEquals(FlowModResult.FAILED, task.getResult(remoteTimeout));
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        Set<ClusterEvent> events = getPostedClusterEvent();
+        assertEquals(2, events.size());
+        clearPostedClusterEvent();
+
+        Iterator<FlowEntry> it = flow.getFlowEntries().iterator();
+        FlowEntry ingress = it.next();
+        FlowRemoveTask rtask = new FlowRemoveTask(vtnMgr, flow.getGroupId(), ingress, it);
+        rtask.run();
+        assertEquals(FlowModResult.FAILED, rtask.getResult(remoteTimeout));
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        events = getPostedClusterEvent();
+        assertEquals(1, events.size());
+        clearPostedClusterEvent();
+
+        fdb.clear(vtnMgr);
+        flushFlowTasks(remoteTimeout);
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        // ingress + local + remote.
+        // in this case succeed to add and remove remote flow entry.
+        FlowEntry rent = null;
+        it = flow.getFlowEntries().iterator();
+        while (it.hasNext()) {
+            rent = it.next();
+        }
+        TimerTask timerTask = new ResultTimerTask(vtnMgr, rent,
+                                                  FlowModResult.SUCCEEDED, false);
+        Timer timer = new Timer();
+        timer.schedule(timerTask, remoteTimeout / 2);
+
+        task = new FlowAddTask(vtnMgr, flow);
+        task.run();
+        assertEquals(FlowModResult.SUCCEEDED, task.getResult(remoteTimeout));
+        // entry managed by local is installed.
+        checkRegsiterdFlowEntry(vtnMgr, 1, flow, flow, 2);
+
+        events = getPostedClusterEvent();
+        assertEquals(1, events.size());
+        clearPostedClusterEvent();
+
+        it = flow.getFlowEntries().iterator();
+        ingress = it.next();
+        timerTask = new ResultTimerTask(vtnMgr, rent, FlowModResult.SUCCEEDED,
+                                        false);
+        timer.schedule(timerTask, remoteTimeout / 2);
+        rtask = new FlowRemoveTask(vtnMgr, flow.getGroupId(), ingress, it);
+        rtask.run();
+        assertEquals(FlowModResult.SUCCEEDED, rtask.getResult(remoteTimeout));
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        events = getPostedClusterEvent();
+        assertEquals(1, events.size());
+        clearPostedClusterEvent();
+
+        fdb.clear(vtnMgr);
+        flushFlowTasks(remoteTimeout);
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        // ingress + local + remote + not connected node.
+        // in this case succeed to add and remove remote flow entry.
+        innc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("10"),
+                                                          node2);
+        outnc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                           node2);
+        flow = addFlowEntry(vtnMgr, flow, innc, (short) 1, outnc, pri);
+
+        rent = null;
+        it = flow.getFlowEntries().iterator();
+        while (it.hasNext()) {
+            rent = it.next();
+        }
+        timerTask = new ResultTimerTask(vtnMgr, rent, FlowModResult.SUCCEEDED,
+                                        false);
+        timer.schedule(timerTask, remoteTimeout / 2);
+
+        task = new FlowAddTask(vtnMgr, flow);
+        task.run();
+        assertEquals(FlowModResult.FAILED, task.getResult(remoteTimeout));
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        events = getPostedClusterEvent();
+        // when include not connected node,
+        // don't execute install of remote entry.
+        assertEquals(0, events.size());
+        clearPostedClusterEvent();
+
+        fdb.clear(vtnMgr);
+        flushFlowTasks(remoteTimeout);
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        // in case installLocal() timeout.
+        ForwardingRulesManagerStub frm = new ForwardingRulesManagerStub();
+        vtnMgr.setForwardingRuleManager(frm);
+
+        innc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("12"),
+                                                          node0);
+        outnc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                           node0);
+        flow = addFlowEntry(vtnMgr, flow, innc, (short) 1, outnc, pri);
+
+        fdb.createIndex(vtnMgr, flow);
+        task = new FlowAddTask(vtnMgr, flow);
+        task.run();
+        assertEquals(FlowModResult.FAILED, task.getResult(timeout));
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+    }
+
+    /**
+     * Test method for
+     * {@link FlowRemoveTask}.
+     */
+    @Test
+    public void testFlowRemoveTask() {
+        long timeout = vtnMgr.getVTNConfig().getFlowModTimeout();
+
+        Set<FlowGroupId> gidset = new HashSet<FlowGroupId>();
+        List<FlowEntry> ingress = new ArrayList<FlowEntry>();
+        List<FlowEntry> entries = new ArrayList<FlowEntry>();
+
+        VTNFlowDatabase fdb = new VTNFlowDatabase("test");
+        VTNFlow flow = fdb.create(vtnMgr);
+
+        Node node0 = NodeCreator.createOFNode(Long.valueOf(0L));
+
+        // add a VTNFlow.
+        NodeConnector innc
+            = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("10"),
+                                                         node0);
+        NodeConnector outnc
+            = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                         node0);
+        Match match = new Match();
+        match.setField(MatchType.IN_PORT, innc);
+        match.setField(MatchType.DL_VLAN, (short) 1);
+
+        ActionList actions = new ActionList(outnc.getNode());
+        actions.addOutput(outnc);
+        int pri = 1;
+        flow.addFlow(vtnMgr, match, actions, pri);
+
+        innc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("12"),
+                                                          node0);
+        outnc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                           node0);
+        flow = addFlowEntry(vtnMgr, flow, innc, (short) 1, outnc, pri);
+
+        fdb.createIndex(vtnMgr, flow);
+        FlowAddTask task = new FlowAddTask(vtnMgr, flow);
+        task.run();
+        assertEquals(FlowModResult.SUCCEEDED, task.getResult(timeout));
+        checkRegsiterdFlowEntry(vtnMgr, 1, flow, flow, 2);
+
+        gidset.add(flow.getGroupId());
+        Iterator<FlowEntry> it = flow.getFlowEntries().iterator();
+        ingress.add(it.next());
+        while (it.hasNext()) {
+            entries.add(it.next());
+        }
+
+        // add a second VTNFlow.
+        flow = fdb.create(vtnMgr);
+        innc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                          node0);
+        outnc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("12"),
+                                                           node0);
+        flow = addFlowEntry(vtnMgr, flow, innc, (short) 1, outnc, pri);
+
+
+        innc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("13"),
+                                                          node0);
+        outnc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("14"),
+                                                           node0);
+        flow = addFlowEntry(vtnMgr, flow, innc, (short) 1, outnc, pri);
+
+        fdb.createIndex(vtnMgr, flow);
+        task = new FlowAddTask(vtnMgr, flow);
+        task.run();
+        assertEquals(FlowModResult.SUCCEEDED, task.getResult(timeout));
+        checkRegsiterdFlowEntry(vtnMgr, 2, flow, flow, 4);
+
+        gidset.add(flow.getGroupId());
+        it = flow.getFlowEntries().iterator();
+        ingress.add(it.next());
+        while (it.hasNext()) {
+            entries.add(it.next());
+        }
+
+        // test FlowRemoveTask
+        FlowRemoveTask rtask = new FlowRemoveTask(vtnMgr, gidset, ingress,
+                                                  entries);
+        rtask.run();
+        assertEquals(FlowModResult.SUCCEEDED, rtask.getResult());
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+
+        // remove with no entry existing.
+        gidset.clear();
+        ingress.clear();
+        entries.clear();
+        gidset.add(flow.getGroupId());
+        it = flow.getFlowEntries().iterator();
+        ingress.add(it.next());
+        while (it.hasNext()) {
+            entries.add(it.next());
+        }
+
+        rtask = new FlowRemoveTask(vtnMgr, gidset, ingress, entries);
+        rtask.run();
+        assertEquals(FlowModResult.FAILED, rtask.getResult(timeout));
+        checkRegsiterdFlowEntry(vtnMgr, 0, flow, null, 0);
+    }
+
+    /**
+     * Test method for
+     * {@link FlowModTask#getVTNManager()}.
+     */
+    @Test
+    public void testGetVTNManager() {
+        VTNFlowDatabase fdb = new VTNFlowDatabase("test");
+        VTNFlow flow = fdb.create(vtnMgr);
+
+        FlowAddTask addTask = new FlowAddTask(vtnMgr, flow);
+        assertSame(vtnMgr, addTask.getVTNManager());
+
+        Iterator<FlowEntry> it = flow.getFlowEntries().iterator();
+        FlowRemoveTask removeTask = new FlowRemoveTask(vtnMgr,
+                                                       flow.getGroupId(),
+                                                       null, it);
+        assertSame(vtnMgr, removeTask.getVTNManager());
+    }
+
+    /**
+     * Test method for
+     * {@link FlowModTask#getResult(long)},
+     * {@link FlowModTask#getResultAbs(long)},
+     * {@link FlowModTask#setResult(boolean)}.
+     */
+    @Test
+    public void testGetResult() {
+        VTNFlowDatabase fdb = new VTNFlowDatabase("test");
+        VTNFlow flow = fdb.create(vtnMgr);
+        long timeout = vtnMgr.getVTNConfig().getFlowModTimeout();
+
+        // getResult() timeout
+        FlowAddTask addTask = new FlowAddTask(vtnMgr, flow);
+        long start = System.currentTimeMillis();
+        assertEquals(FlowModResult.TIMEDOUT, addTask.getResult(timeout));
+        long end = System.currentTimeMillis();
+        assertTrue(timeout <= (end - start));
+
+        long timeoutRemote = vtnMgr.getVTNConfig().getRemoteFlowModTimeout();
+        FlowEntry ingress = null;
+        Iterator<FlowEntry> it = flow.getFlowEntries().iterator();
+        FlowRemoveTask rmTask = new FlowRemoveTask(vtnMgr, flow.getGroupId(), ingress, it);
+        start = System.currentTimeMillis();
+        assertEquals(FlowModResult.TIMEDOUT, rmTask.getResult());
+        end = System.currentTimeMillis();
+        assertTrue(timeoutRemote <= (end - start));
+
+        long timeoutRemoteBulk = vtnMgr.getVTNConfig().getRemoteBulkFlowModTimeout();
+        List<FlowEntry> ingresss = new ArrayList<FlowEntry>();
+        List<FlowEntry> entries = new ArrayList<FlowEntry>();
+        Set<FlowGroupId> gidset = new HashSet<FlowGroupId>();
+        rmTask = new FlowRemoveTask(vtnMgr, gidset, ingresss, entries);
+        start = System.currentTimeMillis();
+        assertEquals(FlowModResult.TIMEDOUT, rmTask.getResult());
+        end = System.currentTimeMillis();
+        assertTrue(timeoutRemoteBulk <= (end - start));
+
+        // succeeded
+        addTask = new FlowAddTask(vtnMgr, flow);
+        addTask.setResult(true);
+        assertEquals(FlowModResult.SUCCEEDED, addTask.getResult(timeout));
+
+        rmTask = new FlowRemoveTask(vtnMgr, flow.getGroupId(), ingress, it);
+        rmTask.setResult(true);
+        assertEquals(FlowModResult.SUCCEEDED, rmTask.getResult(timeoutRemote));
+
+        // failed
+        addTask = new FlowAddTask(vtnMgr, flow);
+        addTask.setResult(false);
+        assertEquals(FlowModResult.FAILED, addTask.getResult(timeout));
+
+        rmTask = new FlowRemoveTask(vtnMgr, flow.getGroupId(), ingress, it);
+        rmTask.setResult(false);
+        assertEquals(FlowModResult.FAILED, rmTask.getResult(timeoutRemote));
+
+        // getResult() interrupted
+        TimerTask timerTask = new InterruptTask(Thread.currentThread());
+        Timer timer = new Timer();
+
+        addTask = new FlowAddTask(vtnMgr, flow);
+        timer.schedule(timerTask, timeout / 2);
+        assertEquals(FlowModResult.INTERRUPTED, addTask.getResult(timeout));
+
+        timerTask = new InterruptTask(Thread.currentThread());
+        rmTask = new FlowRemoveTask(vtnMgr, flow.getGroupId(), ingress, it);
+        timer.schedule(timerTask, timeoutRemote / 2);
+        assertEquals(FlowModResult.INTERRUPTED, rmTask.getResult(timeoutRemote));
+    }
+
+
+    // private methods
+
+    /**
+     * check specified Flow Entry is registerd correctly.
+     *
+     * @param numFlows          the number of Flows.
+     * @param registerdFlow     VTNFlow which is registerd.
+     * @param numFlowEntries    the number of Flow Entries.
+     */
+    private void checkRegsiterdFlowEntry(VTNManagerImpl mgr, int numFlows,
+                                         VTNFlow registerdFlow, VTNFlow expectedFlow,
+                                         int numFlowEntries) {
+        ConcurrentMap<FlowGroupId, VTNFlow> db = mgr.getFlowDB();
+        assertEquals(numFlows, db.size());
+        assertEquals(expectedFlow, db.get(registerdFlow.getGroupId()));
+        assertEquals(numFlowEntries, stubObj.getFlowEntries().size());
+    }
+}
diff --git a/manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/LocalFlowTaskTest.java b/manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/LocalFlowTaskTest.java
new file mode 100644 (file)
index 0000000..23bb748
--- /dev/null
@@ -0,0 +1,187 @@
+/*
+ * Copyright (c) 2013 NEC Corporation
+ * All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this
+ * distribution, and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.vtn.manager.internal;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import org.junit.Test;
+import org.opendaylight.controller.forwardingrulesmanager.FlowEntry;
+import org.opendaylight.controller.sal.core.Node;
+import org.opendaylight.controller.sal.core.NodeConnector;
+import org.opendaylight.controller.sal.match.Match;
+import org.opendaylight.controller.sal.match.MatchType;
+import org.opendaylight.controller.sal.utils.NodeConnectorCreator;
+import org.opendaylight.controller.sal.utils.NodeCreator;
+import org.opendaylight.vtn.manager.internal.cluster.ClusterEvent;
+import org.opendaylight.vtn.manager.internal.cluster.FlowGroupId;
+import org.opendaylight.vtn.manager.internal.cluster.FlowModResult;
+import org.opendaylight.vtn.manager.internal.cluster.VTNFlow;
+
+/**
+ * test for {@link LocalFlowTask}.
+ */
+public class LocalFlowTaskTest extends FlowModTaskTestBase {
+
+    /**
+     * Test method for
+     * {@link LocalFlowAddTask} and {@link LocalFlowRemoveTask}.
+     */
+    @Test
+    public void testLocalFlowTask() {
+        long timeout = vtnMgr.getVTNConfig().getFlowModTimeout();
+
+        VTNFlowDatabase fdb = new VTNFlowDatabase("test");
+        VTNFlow flow = fdb.create(vtnMgr);
+
+        Node node0 = NodeCreator.createOFNode(Long.valueOf(0L));
+
+        // ingress only.
+        NodeConnector innc
+            = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("10"),
+                                                         node0);
+        NodeConnector outnc
+            = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                         node0);
+        Match match = new Match();
+        match.setField(MatchType.IN_PORT, innc);
+        match.setField(MatchType.DL_VLAN, (short) 1);
+
+        ActionList actions = new ActionList(outnc.getNode());
+        actions.addOutput(outnc);
+        int pri = 1;
+        flow.addFlow(vtnMgr, match, actions, pri);
+
+        // test LocalFlowAddTask
+        int i = 0;
+        for (FlowEntry ent : flow.getFlowEntries()) {
+            LocalFlowAddTask task = new LocalFlowAddTask(vtnMgr, ent);
+            assertEquals(ent, task.getFlowEntry());
+            assertSame(vtnMgr, task.getVTNManager());
+
+            task.run();
+            assertEquals(FlowModResult.SUCCEEDED, task.getResult(timeout));
+            assertEquals(++i, stubObj.getFlowEntries().size());
+            assertTrue(stubObj.getFlowEntries().contains(ent));
+        }
+
+        // add conflict flow entry.
+        FlowEntry entConflict = flow.getFlowEntries().iterator().next();
+        LocalFlowAddTask taskConflict = new  LocalFlowAddTask(vtnMgr, entConflict);
+        taskConflict.run();
+        assertEquals(FlowModResult.FAILED, taskConflict.getResult(timeout));
+        assertEquals(i, stubObj.getFlowEntries().size());
+        assertTrue(stubObj.getFlowEntries().contains(entConflict));
+
+        // test LocalFlowRemoveTask
+        for (FlowEntry ent : flow.getFlowEntries()) {
+            LocalFlowRemoveTask task = new LocalFlowRemoveTask(vtnMgr, ent);
+            assertEquals(ent, task.getFlowEntry());
+            assertSame(vtnMgr, task.getVTNManager());
+
+            task.run();
+            assertEquals(FlowModResult.SUCCEEDED, task.getResult(timeout));
+            assertEquals(--i, stubObj.getFlowEntries().size());
+            assertFalse(stubObj.getFlowEntries().contains(ent));
+        }
+
+        // add conflict flow entry.
+        FlowEntry entNoexist = flow.getFlowEntries().iterator().next();
+        LocalFlowRemoveTask taskNoexist
+            = new  LocalFlowRemoveTask(vtnMgr, entNoexist);
+        taskNoexist.run();
+        assertEquals(FlowModResult.FAILED, taskNoexist.getResult(timeout));
+        assertEquals(0, stubObj.getFlowEntries().size());
+
+        // timed out
+        ForwardingRulesManagerStub frm = new ForwardingRulesManagerStub();
+        vtnMgr.setForwardingRuleManager(frm);
+
+        FlowEntry entTimeout = flow.getFlowEntries().iterator().next();
+        LocalFlowAddTask taskTimeoutAdd = new  LocalFlowAddTask(vtnMgr, entTimeout);
+        taskTimeoutAdd.run();
+        assertEquals(FlowModResult.FAILED, taskTimeoutAdd.getResult(timeout));
+        assertEquals(0, stubObj.getFlowEntries().size());
+
+        LocalFlowRemoveTask taskTimeoutRemove
+            = new LocalFlowRemoveTask(vtnMgr, entTimeout);
+        taskTimeoutRemove.run();
+        assertEquals(FlowModResult.FAILED, taskTimeoutRemove.getResult(timeout));
+        assertEquals(0, stubObj.getFlowEntries().size());
+
+        Set<ClusterEvent> sets = getPostedClusterEvent();
+        assertEquals(0, sets.size());
+    }
+
+    /**
+     * Test method for
+     * {@link FlowModTask#getResult(long)},
+     * {@link FlowModTask#getResultAbs(long)},
+     * {@link FlowModTask#setResult(boolean)}.
+     */
+    @Test
+    public void testGetResult() {
+        VTNFlowDatabase fdb = new VTNFlowDatabase("test");
+        VTNFlow vflow = fdb.create(vtnMgr);
+        long timeout = vtnMgr.getVTNConfig().getFlowModTimeout();
+
+        // getResult() timeout
+        FlowEntry flow = null;
+        LocalFlowAddTask addTask = new LocalFlowAddTask(vtnMgr, flow);
+        long start = System.currentTimeMillis();
+        assertEquals(FlowModResult.TIMEDOUT, addTask.getResult(timeout));
+        long end = System.currentTimeMillis();
+        assertTrue(timeout <= (end - start));
+
+        long timeoutRemote = vtnMgr.getVTNConfig().getRemoteFlowModTimeout();
+        LocalFlowRemoveTask rmTask = new LocalFlowRemoveTask(vtnMgr, flow);
+        start = System.currentTimeMillis();
+        assertEquals(FlowModResult.TIMEDOUT, rmTask.getResult(timeoutRemote));
+        end = System.currentTimeMillis();
+        assertTrue(timeoutRemote <= (end - start));
+
+        // succeeded
+        addTask = new LocalFlowAddTask(vtnMgr, flow);
+        addTask.setResult(true);
+        assertEquals(FlowModResult.SUCCEEDED, addTask.getResult(timeout));
+
+        rmTask = new LocalFlowRemoveTask(vtnMgr, flow);
+        rmTask.setResult(true);
+        assertEquals(FlowModResult.SUCCEEDED, rmTask.getResult(timeoutRemote));
+
+        // failed
+        addTask = new LocalFlowAddTask(vtnMgr, flow);
+        addTask.setResult(false);
+        assertEquals(FlowModResult.FAILED, addTask.getResult(timeout));
+
+        rmTask = new LocalFlowRemoveTask(vtnMgr, flow);
+        rmTask.setResult(false);
+        assertEquals(FlowModResult.FAILED, rmTask.getResult(timeoutRemote));
+
+        // getResult() interrupted
+        TimerTask timerTask = new InterruptTask(Thread.currentThread());
+        Timer timer = new Timer();
+
+        addTask = new LocalFlowAddTask(vtnMgr, flow);
+        timer.schedule(timerTask, timeout / 2);
+        assertEquals(FlowModResult.INTERRUPTED, addTask.getResult(timeout));
+
+        timerTask = new InterruptTask(Thread.currentThread());
+        rmTask = new LocalFlowRemoveTask(vtnMgr, flow);
+        timer.schedule(timerTask, timeoutRemote / 2);
+        assertEquals(FlowModResult.INTERRUPTED, rmTask.getResult(timeoutRemote));
+    }
+}
\ No newline at end of file
diff --git a/manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/RemoteFlowRequestTest.java b/manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/RemoteFlowRequestTest.java
new file mode 100644 (file)
index 0000000..9267b0e
--- /dev/null
@@ -0,0 +1,273 @@
+/*
+ * Copyright (c) 2013 NEC Corporation
+ * All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this
+ * distribution, and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.vtn.manager.internal;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.opendaylight.controller.forwardingrulesmanager.FlowEntry;
+import org.opendaylight.controller.sal.core.Node;
+import org.opendaylight.controller.sal.core.NodeConnector;
+import org.opendaylight.controller.sal.match.Match;
+import org.opendaylight.controller.sal.match.MatchType;
+import org.opendaylight.controller.sal.utils.NodeConnectorCreator;
+import org.opendaylight.controller.sal.utils.NodeCreator;
+import org.opendaylight.vtn.manager.internal.cluster.FlowModResult;
+import org.opendaylight.vtn.manager.internal.cluster.VTNFlow;
+
+/**
+ * test for {@link RemoteFlowRequest}.
+ */
+public class RemoteFlowRequestTest extends VTNManagerImplTestCommon {
+
+
+    @BeforeClass
+    static public void beforeClass() {
+        stubMode = 2;
+    }
+
+    /**
+     * Test method for
+     * {@link RemoteFlowRequest#RemoteFlowRequest(List)},
+     * {@link RemoteFlowRequest#setResult(String, FlowModResult, int)},
+     * {@link RemoteFlowRequest#getResultAbs(long, boolean)}.
+     */
+    @Test
+    public void testRemoteFlowRequest() {
+        long timeout = vtnMgr.getVTNConfig().getRemoteFlowModTimeout();
+
+        VTNFlowDatabase fdb = new VTNFlowDatabase("test");
+        VTNFlow flow = fdb.create(vtnMgr);
+
+        // create flow entries.
+        Node node0 = NodeCreator.createOFNode(Long.valueOf(0L));
+        NodeConnector innc
+            = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("10"),
+                                                         node0);
+        NodeConnector outnc
+            = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                         node0);
+        Match match = new Match();
+        match.setField(MatchType.IN_PORT, innc);
+        match.setField(MatchType.DL_VLAN, (short) 1);
+
+        ActionList actions = new ActionList(outnc.getNode());
+        actions.addOutput(outnc);
+        int pri = 1;
+        flow.addFlow(vtnMgr, match, actions, pri);
+
+        innc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("12"),
+                                                          node0);
+        outnc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf("11"),
+                                                           node0);
+        match = new Match();
+        match.setField(MatchType.IN_PORT, innc);
+        match.setField(MatchType.DL_VLAN, (short) 1);
+        actions = new ActionList(outnc.getNode());
+        actions.addOutput(outnc);
+        flow.addFlow(vtnMgr, match, actions, pri);
+
+        List<FlowEntry> entries = flow.getFlowEntries();
+
+        Set<FlowModResult> resultSet = new HashSet<FlowModResult>();
+        resultSet.add(FlowModResult.SUCCEEDED);
+        resultSet.add(FlowModResult.FAILED);
+        resultSet.add(FlowModResult.IGNORED);
+
+        // removeNode == 1
+        for (FlowModResult result0 : resultSet) {
+            for (FlowModResult result1 : resultSet) {
+                for (Boolean all : createBooleans(false)) {
+                    for (int c = 0; c < 3; c++) {
+                        RemoteFlowRequest req = new RemoteFlowRequest(entries);
+                        if (c < 2) {
+                            req.setResult(entries.get(0).getFlowName(),
+                                          result0, c);
+                            req.setResult(entries.get(1).getFlowName(),
+                                          result1, c);
+                        } else {
+                            // in case c == 2
+                            req.setResult(entries.get(0).getFlowName(),
+                                          FlowModResult.IGNORED, c);
+                            req.setResult(entries.get(0).getFlowName(),
+                                          result0, c);
+                            req.setResult(entries.get(1).getFlowName(),
+                                          result1, c);
+                            req.setResult(entries.get(1).getFlowName(),
+                                          FlowModResult.IGNORED, c);
+                        }
+
+                        if (result0 == FlowModResult.SUCCEEDED
+                                && result1 == FlowModResult.SUCCEEDED) {
+                            assertTrue(req.getResultAbs(0L, all.booleanValue()));
+                            // call again
+                            assertTrue(req.getResultAbs(0L, all.booleanValue()));
+                        } else {
+                            assertFalse(req.getResultAbs(0L, all.booleanValue()));
+                            // call again
+                            assertFalse(req.getResultAbs(0L, all.booleanValue()));
+                        }
+                    }
+                }
+            }
+        }
+
+        // one is not set.
+        for (FlowModResult result0 : resultSet) {
+            for (Boolean all : createBooleans(false)) {
+                RemoteFlowRequest req = new RemoteFlowRequest(entries);
+                req.setResult(entries.get(0).getFlowName(), result0, 1);
+                assertFalse(req.getResultAbs(0L, all.booleanValue()));
+            }
+        }
+
+        // set result which have invalid flow entry name.
+        for (Boolean all : createBooleans(false)) {
+            RemoteFlowRequest req = new RemoteFlowRequest(entries);
+            req.setResult(entries.get(0).getFlowName(),
+                          FlowModResult.SUCCEEDED, 1);
+            req.setResult(entries.get(1).getFlowName(),
+                          FlowModResult.SUCCEEDED, 1);
+            req.setResult("not-match-name", FlowModResult.FAILED, 1);
+            assertTrue(req.getResultAbs(0L, all.booleanValue()));
+        }
+
+        // in case time out
+        for (Boolean all : createBooleans(false)) {
+            RemoteFlowRequest req = new RemoteFlowRequest(entries);
+            assertFalse(req.getResultAbs(1L, all.booleanValue()));
+        }
+
+        // remoteNode == 2
+        for (FlowModResult result0 : resultSet) {
+            for (FlowModResult result1 : resultSet) {
+                for (Boolean all : createBooleans(false)) {
+                    RemoteFlowRequest req = new RemoteFlowRequest(entries);
+                    req.setResult(entries.get(0).getFlowName(),
+                                  FlowModResult.IGNORED, 2);
+                    req.setResult(entries.get(0).getFlowName(), result0, 2);
+                    req.setResult(entries.get(1).getFlowName(), result1, 2);
+                    req.setResult(entries.get(1).getFlowName(),
+                                  FlowModResult.IGNORED, 2);
+
+                    if (result0 == FlowModResult.SUCCEEDED
+                            && result1 == FlowModResult.SUCCEEDED) {
+                        assertTrue(req.getResultAbs(0L, all.booleanValue()));
+                        // call again
+                        assertTrue(req.getResultAbs(0L, all.booleanValue()));
+                    } else {
+                        assertFalse(req.getResultAbs(0L, all.booleanValue()));
+                        // call again
+                        assertFalse(req.getResultAbs(0L, all.booleanValue()));
+                    }
+                }
+            }
+        }
+
+        // in case when getResult is waited setResult is called.
+        class ResultTimerTask extends TimerTask {
+            private RemoteFlowRequest request = null;
+            private String flowName = null;
+            private FlowModResult result = null;
+            private int remoteNodes = 0;
+
+            ResultTimerTask(RemoteFlowRequest req, String name, FlowModResult res,
+                            int nodes) {
+                request = req;
+                flowName = name;
+                result = res;
+                remoteNodes = nodes;
+            }
+
+            @Override
+            public void run() {
+                request.setResult(flowName, result, remoteNodes);
+            }
+        }
+
+        for (FlowModResult result0 : resultSet) {
+            for (FlowModResult result1 : resultSet) {
+                for (Boolean all : createBooleans(false)) {
+
+                    RemoteFlowRequest req = new RemoteFlowRequest(entries);
+                    req.setResult(entries.get(0).getFlowName(), result0, 1);
+
+                    TimerTask task = new ResultTimerTask(req,
+                            entries.get(1).getFlowName(), result1, 1);
+                    Timer timer = new Timer();
+                    timer.schedule(task, 10L);
+
+                    long resTimeout = 1000L + System.currentTimeMillis();
+
+                    boolean res = req.getResultAbs(resTimeout,
+                                                   all.booleanValue());
+                    long afterTime = System.currentTimeMillis();
+
+                    if (result0 == FlowModResult.SUCCEEDED
+                            && result1 == FlowModResult.SUCCEEDED) {
+                        assertTrue(res);
+                        assertTrue(resTimeout > afterTime);
+                    } else if (result0 == FlowModResult.IGNORED
+                            && all == Boolean.TRUE) {
+                        assertFalse(res);
+                        assertTrue(resTimeout <= afterTime);
+                    } else if (result1 == FlowModResult.IGNORED
+                            && all == Boolean.TRUE) {
+                        // this case time out
+                        assertFalse(res);
+                        assertTrue(resTimeout <= afterTime);
+                    } else if ((result0 != FlowModResult.SUCCEEDED
+                            || result1 != FlowModResult.SUCCEEDED)
+                            && all == Boolean.TRUE) {
+                        // TODO: need to check
+
+                    } else {
+                        assertFalse(res);
+                        assertTrue(resTimeout > afterTime);
+                    }
+                }
+            }
+        }
+
+        class InterruptTask extends TimerTask {
+            private Thread targetThread = null;
+
+            InterruptTask(Thread th) {
+                targetThread = th;
+            }
+
+            @Override
+            public void run() {
+                targetThread.interrupt();
+            }
+        }
+
+        for (Boolean all : createBooleans(false)) {
+            RemoteFlowRequest req = new RemoteFlowRequest(entries);
+            req.setResult(entries.get(0).getFlowName(), FlowModResult.SUCCEEDED, 1);
+
+            TimerTask task = new InterruptTask(Thread.currentThread());
+            Timer timer = new Timer();
+            timer.schedule(task, 500L);
+
+            long resTimeout = 1000L + System.currentTimeMillis();
+
+            boolean res = req.getResultAbs(resTimeout, all.booleanValue());
+            assertFalse(res);
+        }
+    }
+}
index b5ce69b9ae86311394c184a83fdfd9a3785139b5..857ea4776c09105de7fda642815cd177a970aa55 100644 (file)
@@ -224,6 +224,43 @@ public abstract class TestBase extends Assert {
         return list;
     }
 
+    /**
+     * Create a list of Short objects and a {@code null}.
+     *
+     * @param start    The first value.
+     * @param count    The number of Shorts to be created.
+     * @return A list of {@link Short}.
+     */
+    protected static List<Short> createShorts(short start, short count) {
+        return createShorts(start, count, true);
+    }
+
+    /**
+     * Create a list of Short objects.
+     *
+     * @param start    The first value.
+     * @param count    The number of Shorts to be created.
+     * @param setNull  Set {@code null} to returned list if {@code true}.
+     * @return A list of {@link Shorts}.
+     */
+    protected static List<Short> createShorts(short start, short count,
+                                              boolean setNull) {
+        List<Short> list = new ArrayList<Short>();
+        if (count > 0) {
+            if (setNull) {
+                list.add(null);
+                count--;
+            }
+
+            for (short i = 0; i < count; i++, start++) {
+                list.add(Short.valueOf(start));
+            }
+        }
+
+        return list;
+    }
+
+
     /**
      * Create a list of integer objects and a {@code null}.
      *
index 49a279be39913b948606d83a570a57296c80508c..581eac16378edb99ee35591d3b8ae4bad82619ac 100644 (file)
@@ -10,6 +10,7 @@
 package org.opendaylight.vtn.manager.internal;
 
 import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -88,7 +89,7 @@ import org.opendaylight.vtn.manager.SwitchPort;
  *   Note that stubMode can be set to 0 or 2 or 3 only. (other is not implemented yet.)
  * </p>
  */
-class TestStub implements IClusterGlobalServices, IClusterContainerServices,
+public class TestStub implements IClusterGlobalServices, IClusterContainerServices,
     ISwitchManager, ITopologyManager, IDataPacketService, IRouting,
     IForwardingRulesManager, IfIptoHost, IConnectionManager {
 
@@ -105,6 +106,11 @@ class TestStub implements IClusterGlobalServices, IClusterContainerServices,
      */
     private int stubMode = 0;
 
+    /**
+     * Number of Cluster nodes.
+     */
+    private int clusterMode = 0;
+
     /**
      * Set of existing node.
      */
@@ -151,12 +157,18 @@ class TestStub implements IClusterGlobalServices, IClusterContainerServices,
     private SubnetConfig savedSubnetConfig = null;
 
     /**
-     * Installad flow entries.
+     * Installed flow entries.
      */
     private HashSet<FlowEntry>  flowEntries;
 
     /**
-     * Constractor of TestStub
+     * List of InetAddress of cluster nodes
+     */
+    List<InetAddress> nodeInetAddresses = null;
+
+
+    /**
+     * Constructor of TestStub
      */
     public TestStub() {
         stubMode = 0;
@@ -171,6 +183,12 @@ class TestStub implements IClusterGlobalServices, IClusterContainerServices,
         setup();
     }
 
+    public TestStub(int stub, int cluster) {
+        stubMode = stub;
+        clusterMode = cluster;
+        setup();
+    }
+
     /**
      * setup datas.
      */
@@ -334,6 +352,21 @@ class TestStub implements IClusterGlobalServices, IClusterContainerServices,
                 }
             }
         }
+
+        if (clusterMode > 0) {
+            nodeInetAddresses = new ArrayList<InetAddress>();
+
+            for (byte i = 0; i < clusterMode; i++) {
+                InetAddress ia = null;
+                try {
+                    ia = InetAddress.getByAddress(
+                            new byte[] {0x00, 0x00, 0x00, 0x00});
+                } catch (UnknownHostException e) {
+                    Assert.fail("failed to create InetAddress.");
+                }
+                nodeInetAddresses.add(ia);
+            }
+        }
     }
 
     private Set<Node> getNodeSet(Set<Node> nodeSet) {
@@ -886,6 +919,9 @@ class TestStub implements IClusterGlobalServices, IClusterContainerServices,
 
     @Override
     public Status installFlowEntry(FlowEntry flow) {
+        if (flow == null) {
+            return new Status(StatusCode.NOTACCEPTABLE, null);
+        }
         if (flowEntries.add(flow)) {
             return new Status(StatusCode.SUCCESS, null);
         }
@@ -894,6 +930,9 @@ class TestStub implements IClusterGlobalServices, IClusterContainerServices,
 
     @Override
     public Status uninstallFlowEntry(FlowEntry flow) {
+        if (flow == null) {
+            return new Status(StatusCode.NOTACCEPTABLE, null);
+        }
         if (flowEntries.remove(flow)) {
             return new Status(StatusCode.SUCCESS, null);
         }
diff --git a/manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/VTNFlowDatabaseTest.java b/manager/implementation/src/test/java/org/opendaylight/vtn/manager/internal/VTNFlowDatabaseTest.java
new file mode 100644 (file)
index 0000000..62a88c5
--- /dev/null
@@ -0,0 +1,980 @@
+/*
+ * Copyright (c) 2013 NEC Corporation
+ * All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this
+ * distribution, and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.vtn.manager.internal;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.opendaylight.controller.forwardingrulesmanager.FlowEntry;
+import org.opendaylight.controller.sal.core.Node;
+import org.opendaylight.controller.sal.core.NodeConnector;
+import org.opendaylight.controller.sal.match.Match;
+import org.opendaylight.controller.sal.match.MatchType;
+import org.opendaylight.controller.sal.utils.NodeConnectorCreator;
+import org.opendaylight.controller.sal.utils.NodeCreator;
+import org.opendaylight.vtn.manager.VBridgeIfPath;
+import org.opendaylight.vtn.manager.VBridgePath;
+import org.opendaylight.vtn.manager.VTenantPath;
+import org.opendaylight.vtn.manager.internal.cluster.FlowGroupId;
+import org.opendaylight.vtn.manager.internal.cluster.MacVlan;
+import org.opendaylight.vtn.manager.internal.cluster.VTNFlow;
+
+/**
+ * JUnit test for {@link VTNFlowDatabaseTest}
+ */
+public class VTNFlowDatabaseTest extends VTNManagerImplTestCommon {
+
+    @BeforeClass
+    public static void beforeClass() {
+        stubMode = 2;
+    }
+
+    /**
+     * Test method for
+     * {@link VTNFlowDatabase#create(VTNManagerImpl)},
+     * {@link VTNFlowDatabase#VTNFlowDatabase(java.lang.String)}.
+     */
+    @Test
+    public void testCreate() {
+        String tname = "testcreate";
+        String tnameOther = "other";
+        VTNFlowDatabase fdb = new VTNFlowDatabase("testcreate");
+        VTNFlowDatabase fdbOther = new VTNFlowDatabase("other");
+        VTNFlow testFlow0 = fdb.create(vtnMgr);
+        VTNFlow testFlow1 = fdb.create(vtnMgr);
+        VTNFlow otherFlow0 = fdbOther.create(vtnMgr);
+
+        assertNotNull(testFlow0);
+        assertNotNull(testFlow1);
+        assertNotNull(otherFlow0);
+
+        assertEquals(tname, testFlow0.getGroupId().getTenantName());
+        assertEquals(tname, testFlow1.getGroupId().getTenantName());
+        assertEquals(tnameOther, otherFlow0.getGroupId().getTenantName());
+
+        assertFalse(testFlow0.getGroupId().equals(testFlow1.getGroupId()));
+        assertFalse(testFlow0.getGroupId().equals(otherFlow0.getGroupId()));
+    }
+
+    /**
+     * Test method for
+     * {@link VTNFlowDatabase#install(VTNManagerImpl, VTNFlow)}.
+     */
+    @Test
+    public void testInstall() {
+        VTNFlowDatabase fdb = new VTNFlowDatabase("test");
+        VTNFlow flow = fdb.create(vtnMgr);
+
+        Node node0 = NodeCreator.createOFNode(Long.valueOf(0L));
+        Node node1 = NodeCreator.createOFNode(Long.valueOf(1L));
+        Set<Short> portIds = new HashSet<Short>(createShorts((short) 10, (short) 2,
+                                                             false));
+        Set<NodeConnector> ncset0
+            = NodeConnectorCreator.createOFNodeConnectorSet(portIds, node0);
+        Set<NodeConnector> ncset1
+            = NodeConnectorCreator.createOFNodeConnectorSet(portIds, node1);
+
+        int pri = 1;
+
+        // add one flowEntry.
+        Iterator<NodeConnector> it = ncset0.iterator();
+        NodeConnector innc = it.next();
+        NodeConnector outnc = it.next();
+        addFlowEntry(vtnMgr, flow, innc, (short) 0, outnc, pri);
+        fdb.install(vtnMgr, flow);
+        flushFlowTasks();
+
+        ConcurrentMap<FlowGroupId, VTNFlow> db = vtnMgr.getFlowDB();
+        VTNFlow rflow = db.get(flow.getGroupId());
+        assertEquals(flow, rflow);
+        assertEquals(1, stubObj.getFlowEntries().size());
+
+        fdb.clear(vtnMgr);
+        flushFlowTasks();
+        assertEquals(0, db.values().size());
+
+        // add 2 flowEntries.
+        it = ncset1.iterator();
+        outnc = it.next();
+        innc = it.next();
+        flow = addFlowEntry(vtnMgr, flow, innc, (short) 0, outnc, pri);
+
+        fdb.install(vtnMgr, flow);
+        flushFlowTasks();
+
+        rflow = db.get(flow.getGroupId());
+        assertEquals(flow, rflow);
+        assertEquals(2, stubObj.getFlowEntries().size());
+
+        // add Flow which have same groupID.
+        VTNFlow conflictFlow = new VTNFlow(rflow.getGroupId());
+        fdb.install(vtnMgr, conflictFlow);
+        flushFlowTasks();
+        VTNFlow flowAfter = db.get(rflow.getGroupId());
+        assertEquals(rflow, flowAfter);
+
+        // add Flow which have different groupID and same ingress flow.
+        conflictFlow = fdb.create(vtnMgr);
+        it = ncset0.iterator();
+        innc = it.next();
+        outnc = it.next();
+        addFlowEntry(vtnMgr, conflictFlow, innc, (short) 0, outnc, pri);
+
+        it = ncset1.iterator();
+        outnc = it.next();
+        innc = it.next();
+        Match match = new Match();
+        match.setField(MatchType.IN_PORT, innc);
+        match.setField(MatchType.DL_SRC,
+                       new byte[] {0x00, 0x00, 0x00, 0x00, 0x00, 0x00});
+        ActionList actions = new ActionList(innc.getNode());
+        actions.addOutput(outnc);
+        conflictFlow.addFlow(vtnMgr, match, actions, pri);
+
+        fdb.install(vtnMgr, conflictFlow);
+        flushFlowTasks();
+
+        flowAfter = db.get(conflictFlow.getGroupId());
+        assertNull(flowAfter);
+
+        // clear all entry.
+        fdb.clear(vtnMgr);
+        flushFlowTasks();
+        assertEquals(0, db.values().size());
+
+        // clear all entry (with no entry).
+        fdb.clear(vtnMgr);
+        flushFlowTasks();
+        assertEquals(0, db.values().size());
+
+        // in case VTNManager is not worked.
+        stopVTNManager(false);
+        fdb.install(vtnMgr, conflictFlow);
+        assertEquals(0, db.values().size());
+
+    }
+
+    /**
+     * Test method for
+     * {@link VTNFlowDatabase#flowRemoved(VTNManagerImpl, FlowEntry)},
+     * {@link VTNFlowDatabase#flowRemoved(VTNManagerImpl, FlowGroupId)}.
+     */
+    @Test
+    public void testFlowRemovedFlowEntryAndFlowGroupId() {
+        VTNFlowDatabase fdb = new VTNFlowDatabase("test");
+
+        Node node0 = NodeCreator.createOFNode(Long.valueOf(0L));
+        Node node1 = NodeCreator.createOFNode(Long.valueOf(1L));
+        Set<Short> portIds0 = new HashSet<Short>(createShorts((short) 10, (short) 4,
+                                                              false));
+        Set<NodeConnector> ncSet0
+            = NodeConnectorCreator.createOFNodeConnectorSet(portIds0, node0);
+
+        int pri = 1;
+
+        Set<VTNFlow> flows = new HashSet<VTNFlow>();
+        for (NodeConnector innc0 : ncSet0) {
+            VTNFlow flow = fdb.create(vtnMgr);
+            NodeConnector outnc
+                = NodeConnectorCreator.createOFNodeConnector(Short.valueOf((short) 15),
+                                                             innc0.getNode());
+            addFlowEntry(vtnMgr, flow, innc0, (short) 0, outnc, pri);
+
+            NodeConnector innc1
+                = NodeConnectorCreator.createOFNodeConnector((Short) innc0.getID(),
+                                                             node1);
+            outnc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf((short) 15),
+                                                               innc1.getNode());
+            addFlowEntry(vtnMgr, flow, innc1, (short) 0, outnc, pri);
+
+            fdb.install(vtnMgr, flow);
+            flushFlowTasks();
+
+            flows.add(flow);
+        }
+
+        int numFlows = flows.size();
+        ConcurrentMap<FlowGroupId, VTNFlow> db = vtnMgr.getFlowDB();
+        assertEquals(numFlows, db.values().size());
+        assertEquals(numFlows * 2, stubObj.getFlowEntries().size());
+
+        // test flowRemoved(VTNMangerImpl, FlowEntry)
+        for (VTNFlow flow : flows) {
+            int i = 0;
+            for (FlowEntry ent : flow.getFlowEntries()) {
+                fdb.flowRemoved(vtnMgr, ent);
+                flushFlowTasks();
+
+                if (i == 0) {
+                VTNFlow regFlow = db.get(flow.getGroupId());
+                assertNull(regFlow);
+                assertEquals(numFlows - 1, db.values().size());
+                assertEquals(numFlows * 2 - 1, stubObj.getFlowEntries().size());
+
+                // because florwRemoved(VTNManagerImpl, FlowEntry) is invoked
+                // when VTN flow was expired, this invoked
+                // after FlowEnry have already been removed.
+                // in this test case need to remove FlowEntry in DB of stub.
+                stubObj.uninstallFlowEntry(ent);
+
+                Set<VTNFlow> revert = new HashSet<VTNFlow>();
+                revert.add(flow);
+                revertFlowEntries(vtnMgr, fdb, revert, numFlows, numFlows * 2);
+                } else {
+                    VTNFlow regFlow = db.get(flow.getGroupId());
+                    assertNotNull(regFlow);
+                    assertEquals(numFlows, db.values().size());
+                    assertEquals(numFlows * 2, stubObj.getFlowEntries().size());
+                }
+
+                i++;
+            }
+        }
+
+        // specify null to FlowEntry.
+        FlowEntry fent = null;
+        fdb.flowRemoved(vtnMgr, fent);
+        flushFlowTasks();
+
+        assertEquals(numFlows, db.values().size());
+        assertEquals(numFlows * 2, stubObj.getFlowEntries().size());
+
+        // check flowRemoved(VTNMangerImpl, FlowGroupId)
+        for (VTNFlow flow : flows) {
+            fdb.flowRemoved(vtnMgr, flow.getGroupId());
+            flushFlowTasks();
+
+            // flowRemoved(VTNManagerImpl, FlowGroupID) is invoked
+            // from remove cluster node.
+            // this case flowEntry in stub is removed by remove cluster node.
+            // so that in this test case FlowEntry still exist in DB.
+            VTNFlow regFlow = db.get(flow.getGroupId());
+            assertNotNull(regFlow);
+            assertEquals(numFlows, db.values().size());
+            assertEquals(numFlows * 2, stubObj.getFlowEntries().size());
+
+            // need to remove FlowEntry in DB of stub before revert Flow.
+            for (FlowEntry ent : flow.getFlowEntries()) {
+                stubObj.uninstallFlowEntry(ent);
+            }
+
+            Set<VTNFlow> revert = new HashSet<VTNFlow>();
+            revert.add(flow);
+            revertFlowEntries(vtnMgr, fdb, revert, numFlows, numFlows * 2);
+        }
+
+        // specify null to FlowEntry.
+        FlowGroupId gid = null;
+        fdb.flowRemoved(vtnMgr, gid);
+        flushFlowTasks();
+
+        assertEquals(numFlows, db.values().size());
+        assertEquals(numFlows * 2, stubObj.getFlowEntries().size());
+
+        // after clear.
+        fdb.clear(vtnMgr);
+        flushFlowTasks();
+
+        VTNFlow flow = flows.iterator().next();
+        List<FlowEntry> flowEntries = flow.getFlowEntries();
+        fdb.flowRemoved(vtnMgr, flowEntries.get(0));
+        flushFlowTasks();
+        assertEquals(0, db.values().size());
+        assertEquals(0, stubObj.getFlowEntries().size());
+
+        fdb.flowRemoved(vtnMgr, flow.getGroupId());
+        flushFlowTasks();
+        assertEquals(0, db.values().size());
+        assertEquals(0, stubObj.getFlowEntries().size());
+    }
+
+    /**
+     * Test method for
+     * {@link VTNFlowDatabase#removeFlows(VTNManagerImpl, Node, boolean)},
+     * {@link VTNFlowDatabase#removeFlows(VTNManagerImpl, Node)}.
+     */
+    @Test
+    public void testRemoveFlowsNode() {
+        VTNFlowDatabase fdb = new VTNFlowDatabase("test");
+
+        Node node0 = NodeCreator.createOFNode(Long.valueOf(0L));
+        Node node1 = NodeCreator.createOFNode(Long.valueOf(1L));
+        Node node2 = NodeCreator.createOFNode(Long.valueOf(2L));
+        Set<Short> portIds0 = new HashSet<Short>(createShorts((short) 10, (short) 5,
+                                                              false));
+        Set<Short> portIds1 = new HashSet<Short>(createShorts((short) 10, (short) 4,
+                                                              false));
+        Set<Short> portIds2 = new HashSet<Short>(createShorts((short) 10, (short) 3,
+                                                              false));
+        Set<NodeConnector> ncset0
+            = NodeConnectorCreator.createOFNodeConnectorSet(portIds0, node0);
+        Set<NodeConnector> ncset1
+            = NodeConnectorCreator.createOFNodeConnectorSet(portIds1, node1);
+        Set<NodeConnector> ncset2
+            = NodeConnectorCreator.createOFNodeConnectorSet(portIds2, node2);
+
+        int pri = 1;
+
+        // add flowEntry as test parameter.
+        VTNFlow flow0 = fdb.create(vtnMgr);
+        NodeConnector outnc
+            = NodeConnectorCreator.createOFNodeConnector(Short.valueOf((short) 15),
+                                                         node0);
+        for (NodeConnector innc : ncset0) {
+            addFlowEntry(vtnMgr, flow0, innc, (short) 0, outnc, pri);
+        }
+        fdb.install(vtnMgr, flow0);
+        flushFlowTasks();
+
+        VTNFlow flow1 = fdb.create(vtnMgr);
+        outnc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf((short) 15),
+                                                           node1);
+        for (NodeConnector innc : ncset1) {
+            addFlowEntry(vtnMgr, flow1, innc, (short) 0, outnc, pri);
+        }
+        fdb.install(vtnMgr, flow1);
+        flushFlowTasks();
+
+        VTNFlow flow2 = fdb.create(vtnMgr);
+        outnc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf((short) 15),
+                                                           node2);
+        for (NodeConnector innc : ncset2) {
+           addFlowEntry(vtnMgr, flow2, innc, (short) 0, outnc, pri);
+        }
+        fdb.install(vtnMgr, flow2);
+        flushFlowTasks();
+
+        int numEntry = ncset0.size() + ncset1.size() + ncset2.size();
+        ConcurrentMap<FlowGroupId, VTNFlow> db = vtnMgr.getFlowDB();
+        assertEquals(3, db.values().size());
+        assertEquals(numEntry, stubObj.getFlowEntries().size());
+
+        // remove Flows related to node1.
+        // flow entries in switch are also removed.
+        FlowRemoveTask task = fdb.removeFlows(vtnMgr, node1, true);
+        assertNotNull(task);
+        flushFlowTasks();
+
+        assertEquals(ncset0.size() + ncset2.size(),
+                     stubObj.getFlowEntries().size());
+
+        VTNFlow rflow = db.get(flow0.getGroupId());
+        assertEquals(flow0, rflow);
+        rflow = db.get(flow1.getGroupId());
+        assertEquals(null, rflow);
+        rflow = db.get(flow2.getGroupId());
+        assertEquals(flow2, rflow);
+
+        // revert flow1
+        fdb.install(vtnMgr, flow1);
+        flushFlowTasks();
+        assertEquals(ncset0.size() + ncset1.size() + ncset2.size(),
+                     stubObj.getFlowEntries().size());
+
+        // remove Flows related to node2.
+        // flow entries in switch aren't removed.
+        task = fdb.removeFlows(vtnMgr, node2);
+        assertNotNull(task);
+        flushFlowTasks();
+
+        assertEquals(ncset0.size() + ncset1.size() + ncset2.size(),
+                     stubObj.getFlowEntries().size());
+
+        rflow = db.get(flow0.getGroupId());
+        assertEquals(flow0, rflow);
+        rflow = db.get(flow1.getGroupId());
+        assertEquals(flow1, rflow);
+        rflow = db.get(flow2.getGroupId());
+        assertEquals(null, rflow);
+
+        // specify node which already removed.
+        task = fdb.removeFlows(vtnMgr, node2);
+        flushFlowTasks();
+        assertNull(task);
+
+        // TODO: revisit
+        fdb.install(vtnMgr, flow2);
+        flushFlowTasks();
+//        assertEquals(ncset0.size() + ncset1.size() + ncset2.size(),
+//                stubObj.getFlowEntries().size());
+
+        // in case there are no flow entry.
+        fdb.clear(vtnMgr);
+        flushFlowTasks();
+
+        task = fdb.removeFlows(vtnMgr, node0);
+        assertNull(task);
+        flushFlowTasks();
+        assertEquals(0, stubObj.getFlowEntries().size());
+
+    }
+
+    /**
+     * Test method for
+     * {@link VTNFlowDatabase#removeFlows(VTNManagerImpl, NodeConnector)}.
+     */
+    @Test
+    public void testRemoveFlowsNodeConnector() {
+        VTNFlowDatabase fdb = new VTNFlowDatabase("test");
+
+        Node node0 = NodeCreator.createOFNode(Long.valueOf(0L));
+        Node node1 = NodeCreator.createOFNode(Long.valueOf(1L));
+        Node node2 = NodeCreator.createOFNode(Long.valueOf(2L));
+        Set<Short> portIds0 = new HashSet<Short>(createShorts((short) 10, (short) 5,
+                                                              false));
+        Set<Short> portIds1 = new HashSet<Short>(createShorts((short) 10, (short) 4,
+                                                              false));
+        Set<Short> portIds2 = new HashSet<Short>(createShorts((short) 10, (short) 3,
+                                                              false));
+        Set<NodeConnector> ncset0
+            = NodeConnectorCreator.createOFNodeConnectorSet(portIds0, node0);
+        Set<NodeConnector> ncset1
+            = NodeConnectorCreator.createOFNodeConnectorSet(portIds1, node1);
+        Set<NodeConnector> ncset2
+            = NodeConnectorCreator.createOFNodeConnectorSet(portIds2, node2);
+
+        int pri = 1;
+
+        // add flow entries.
+        Set<VTNFlow> flows = new HashSet<VTNFlow>();
+        int i = 0;
+        Iterator<NodeConnector> incset1 = ncset1.iterator();
+        Iterator<NodeConnector> incset2 = ncset2.iterator();
+        for (NodeConnector innc : ncset0) {
+            VTNFlow flow = fdb.create(vtnMgr);
+            NodeConnector outnc
+                = NodeConnectorCreator.createOFNodeConnector(Short.valueOf((short) 15),
+                                                             node0);
+            addFlowEntry(vtnMgr, flow, innc, (short) 0, outnc, pri);
+
+            if(i > 0) {
+                innc = incset1.next();
+                outnc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf((short) 15),
+                                                                   node1);
+                addFlowEntry(vtnMgr, flow, innc, (short) 0, outnc, pri);
+            }
+            if (i > 1) {
+                innc = incset2.next();
+                outnc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf((short) 15),
+                                                                   node2);
+                addFlowEntry(vtnMgr, flow, innc, (short) 0, outnc, pri);
+            }
+
+            fdb.install(vtnMgr, flow);
+            flushFlowTasks();
+            flows.add(flow);
+
+            if (++i > 2) {
+                break;
+            }
+        }
+        assertEquals(6, stubObj.getFlowEntries().size());
+
+        // remove by specifying node connector.
+        NodeConnector removePort
+            = NodeConnectorCreator.createOFNodeConnector(Short.valueOf((short) 15),
+                                                         node2);
+        FlowRemoveTask task = fdb.removeFlows(vtnMgr, removePort);
+        assertNotNull(task);
+        flushFlowTasks();
+
+        assertEquals(3, stubObj.getFlowEntries().size());
+        ConcurrentMap<FlowGroupId, VTNFlow> db = vtnMgr.getFlowDB();
+        for (VTNFlow flow : flows) {
+            VTNFlow rflow = db.get(flow.getGroupId());
+            if (flow.getFlowNodes().contains(node2)) {
+                rflow = db.get(flow.getGroupId());
+                assertEquals(null, rflow);
+            } else {
+                rflow = db.get(flow.getGroupId());
+                assertEquals(flow, rflow);
+            }
+        }
+
+        // specify port which was already removed.
+        task = fdb.removeFlows(vtnMgr, removePort);
+        assertNull(task);
+        flushFlowTasks();
+        assertEquals(3, stubObj.getFlowEntries().size());
+
+        // in case there are no flow entry.
+        fdb.clear(vtnMgr);
+        flushFlowTasks();
+
+        task = fdb.removeFlows(vtnMgr, removePort);
+        assertNull(task);
+        flushFlowTasks();
+        assertEquals(0, stubObj.getFlowEntries().size());
+    }
+
+    /**
+     * Test method for
+     * {@link VTNFlowDatabase#removeFlows(VTNManagerImpl, VTenantPath)},
+     * {@link VTNFlowDatabase#removeFlows(VTNManagerImpl, MacVlan)}.
+     */
+    @Test
+    public void testRemoveFlowsVTenantPathAndMacVlan() {
+        VTNFlowDatabase fdb = new VTNFlowDatabase("test");
+
+        Node node0 = NodeCreator.createOFNode(Long.valueOf(0L));
+        Set<Short> portIds0 = new HashSet<Short>(createShorts((short) 10, (short) 5,
+                                                              false));
+        Set<NodeConnector> ncset0
+            = NodeConnectorCreator.createOFNodeConnectorSet(portIds0, node0);
+
+        int pri = 1;
+
+        // add flow entries.
+        Set<VTNFlow> flows = new HashSet<VTNFlow>();
+        NodeConnector outnc
+            = NodeConnectorCreator.createOFNodeConnector(Short.valueOf((short) 15),
+                                                         node0);
+        String tname = "tenant";
+        String bname = "bridge";
+        String ifname = "interface";
+        VTenantPath tpath = new VTenantPath(tname);
+        VBridgePath bpath = new VBridgePath(tname, bname);
+        VBridgeIfPath bifpath = new VBridgeIfPath(tname, bname, ifname);
+        Set<VTenantPath> pathSet = new HashSet<VTenantPath>();
+        MacVlan[] macVlans = new MacVlan[] {
+                new MacVlan(new byte[] {0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, (short) 0),
+                new MacVlan(new byte[] {0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, (short) 4095),
+                new MacVlan(new byte[] {0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, (short) 0),
+        };
+
+        int numEntries = 0;
+        for (NodeConnector innc : ncset0) {
+            VTNFlow flow = fdb.create(vtnMgr);
+            Match match = new Match();
+            match.setField(MatchType.IN_PORT, innc);
+            ActionList actions = new ActionList(innc.getNode());
+            actions.addOutput(outnc);
+            flow.addFlow(vtnMgr, match, actions, pri);
+
+            MacVlan macVlan = null;
+            if (numEntries == 0) {
+                pathSet.add(tpath);
+                macVlan = macVlans[0];
+            } else if (numEntries == 1) {
+                pathSet.add(bpath);
+                macVlan = macVlans[1];
+            } else if (numEntries == 2) {
+                pathSet.add(bifpath);
+                macVlan = macVlans[2];
+            } else if (numEntries >= 3) {
+                // when numEntries >= 0, add empty pathSet and add null to macVlan.
+                pathSet.clear();
+            }
+            flow.addDependency(pathSet);
+            flow.addDependency(macVlan);
+
+            fdb.install(vtnMgr, flow);
+            flushFlowTasks();
+            flows.add(flow);
+            numEntries++;
+        }
+
+        ConcurrentMap<FlowGroupId, VTNFlow> db = vtnMgr.getFlowDB();
+        assertEquals(numEntries, db.values().size());
+        assertEquals(numEntries, stubObj.getFlowEntries().size());
+
+        // specify VBridgeIfPath
+        FlowRemoveTask task = fdb.removeFlows(vtnMgr, bifpath);
+        assertNotNull(task);
+        flushFlowTasks();
+        assertEquals(numEntries - 1, stubObj.getFlowEntries().size());
+        Set<VTNFlow> revertFlows = new HashSet<VTNFlow>();
+        revertFlows = checkFlowDBEntriesVTenantPath(vtnMgr, fdb, flows, bifpath);
+
+        revertFlowEntries(vtnMgr, fdb, revertFlows, numEntries, numEntries);
+
+        // specify VBridge
+        task = fdb.removeFlows(vtnMgr, bpath);
+        assertNotNull(task);
+        flushFlowTasks();
+        assertEquals(numEntries - 2, stubObj.getFlowEntries().size());
+        revertFlows.clear();
+        revertFlows = checkFlowDBEntriesVTenantPath(vtnMgr, fdb, flows, bpath);
+
+        revertFlowEntries(vtnMgr, fdb, revertFlows, numEntries, numEntries);
+
+        // specify VTenant
+        task = fdb.removeFlows(vtnMgr, tpath);
+        assertNotNull(task);
+        flushFlowTasks();
+        assertEquals(numEntries - 3, stubObj.getFlowEntries().size());
+        revertFlows.clear();
+        revertFlows = checkFlowDBEntriesVTenantPath(vtnMgr, fdb, flows, tpath);
+
+        task = fdb.removeFlows(vtnMgr, tpath);
+        assertNull(task);
+        flushFlowTasks();
+        assertEquals(numEntries - 3, stubObj.getFlowEntries().size());
+
+        revertFlowEntries(vtnMgr, fdb, revertFlows, numEntries, numEntries);
+
+        // not match tenant
+        VTenantPath notMatchPath = new VTenantPath("notmatch");
+        task = fdb.removeFlows(vtnMgr, notMatchPath);
+        assertNull(task);
+        flushFlowTasks();
+        assertEquals(numEntries, stubObj.getFlowEntries().size());
+        revertFlows.clear();
+        revertFlows = checkFlowDBEntriesVTenantPath(vtnMgr, fdb, flows, notMatchPath);
+
+        // for MacVlan
+        for (MacVlan macVlan : macVlans) {
+            task = fdb.removeFlows(vtnMgr, macVlan);
+            assertNotNull(task);
+            flushFlowTasks();
+            assertEquals(numEntries - 1, stubObj.getFlowEntries().size());
+            revertFlows.clear();
+            revertFlows = checkFlowDBEntriesMacVlan(vtnMgr, fdb, flows, macVlan);
+
+            revertFlowEntries(vtnMgr, fdb, revertFlows, numEntries, numEntries);
+        }
+
+        // not match MacVlan
+        MacVlan notMatchMv = new MacVlan(new byte[] {(byte) 0x00, (byte) 0x00, (byte) 0x00,
+                                                     (byte) 0xff, (byte) 0xff, (byte) 0xff},
+                                         (short) 1);
+        task = fdb.removeFlows(vtnMgr, notMatchMv);
+        assertNull(task);
+        flushFlowTasks();
+        assertEquals(numEntries, stubObj.getFlowEntries().size());
+        revertFlows.clear();
+        revertFlows = checkFlowDBEntriesMacVlan(vtnMgr, fdb, flows, notMatchMv);
+
+
+        // in case there are no flow entry.
+        fdb.clear(vtnMgr);
+        flushFlowTasks();
+
+        task = fdb.removeFlows(vtnMgr, tpath);
+        assertNull(task);
+        flushFlowTasks();
+        assertEquals(0, stubObj.getFlowEntries().size());
+
+        task = fdb.removeFlows(vtnMgr, macVlans[0]);
+        assertNull(task);
+        flushFlowTasks();
+        assertEquals(0, stubObj.getFlowEntries().size());
+    }
+
+    /**
+     * Check entries in flowDB
+     * after {@link removeFlows(VTNManager, VTenantPath)} was invoked.
+     *
+     * @param mgr   VTNManager service.
+     * @param fdb   VTNFlowDatabase.
+     * @param flows flows initially installed.
+     * @param rpath VTenantPath.
+     *              it is expected that a FlowEntry dependesOn this was removed.
+     * @return Set of VTNFlow which was removed.
+     */
+    private Set<VTNFlow> checkFlowDBEntriesVTenantPath (VTNManagerImpl mgr,
+            VTNFlowDatabase fdb, Set<VTNFlow> flows, VTenantPath rpath) {
+        Set<VTNFlow> revertFlows = new HashSet<VTNFlow>();
+        ConcurrentMap<FlowGroupId, VTNFlow> db = vtnMgr.getFlowDB();
+
+        for (VTNFlow flow : flows) {
+            VTNFlow rflow = db.get(flow.getGroupId());
+            if (flow.dependsOn(rpath)) {
+                rflow = db.get(flow.getGroupId());
+                assertEquals(null, rflow);
+                revertFlows.add(flow);
+            } else {
+                rflow = db.get(flow.getGroupId());
+                assertEquals(flow, rflow);
+            }
+        }
+
+        return revertFlows;
+    }
+
+    /**
+     * Check Flow entries in flowDB
+     * after {@link removeFlows(VTNManager, MacVlan)} was invoked.
+     *
+     * @param mgr       VTNManager service.
+     * @param fdb       VTNFlowDatabase.
+     * @param flows     flows initially installed.
+     * @param macVlan   MacVlan.
+     *                  it is expected that a FlowEntry dependesOn this was removed.
+     * @return Set of VTNFlow which was removed.
+     */
+    private Set<VTNFlow> checkFlowDBEntriesMacVlan (VTNManagerImpl mgr,
+            VTNFlowDatabase fdb, Set<VTNFlow> flows, MacVlan macVlan) {
+        Set<VTNFlow> revertFlows = new HashSet<VTNFlow>();
+        ConcurrentMap<FlowGroupId, VTNFlow> db = vtnMgr.getFlowDB();
+
+        for (VTNFlow flow : flows) {
+            VTNFlow rflow = db.get(flow.getGroupId());
+            if (flow.dependsOn(macVlan)) {
+                rflow = db.get(flow.getGroupId());
+                assertEquals(null, rflow);
+                revertFlows.add(flow);
+            } else {
+                rflow = db.get(flow.getGroupId());
+                assertEquals(flow, rflow);
+            }
+        }
+
+        return revertFlows;
+    }
+
+    /**
+     * Install FlowEntry which is specified.
+     *
+     * @param mgr           VTN Manager service
+     * @param fdb           VTNFlowDatabase.
+     * @param revertFlows   VTNFlows which is installed.
+     * @param numFlows      The expected number of VTNFlows after install.
+     * @param numEntries    The expected number of FlowEntries after install.
+     */
+    private void revertFlowEntries (VTNManagerImpl mgr, VTNFlowDatabase fdb,
+                                    Set<VTNFlow> revertFlows, int numFlows,
+                                    int numEntries) {
+        ConcurrentMap<FlowGroupId, VTNFlow> db = vtnMgr.getFlowDB();
+        for (VTNFlow flow : revertFlows) {
+            fdb.install(vtnMgr, flow);
+            flushFlowTasks();
+        }
+        assertEquals(numFlows, db.values().size());
+        assertEquals(numEntries, stubObj.getFlowEntries().size());
+
+    }
+
+    /**
+     * Test method for {@link VTNFlowDatabase#removeFlows(VTNManagerImpl, List)}.
+     */
+    @Test
+    public void testRemoveFlowsListOfVTNFlow() {
+        VTNFlowDatabase fdb = new VTNFlowDatabase("test");
+
+        Node node0 = NodeCreator.createOFNode(Long.valueOf(0L));
+        Set<Short> portIds0 = new HashSet<Short>(createShorts((short) 10, (short) 5,
+                                                              false));
+        Set<NodeConnector> ncset0
+            = NodeConnectorCreator.createOFNodeConnectorSet(portIds0, node0);
+
+        int pri = 1;
+
+        // add flowEntry as test parameter.
+        Set<VTNFlow> flows = new HashSet<VTNFlow>();
+        NodeConnector outnc
+            = NodeConnectorCreator.createOFNodeConnector(Short.valueOf((short) 15),
+                                                         node0);
+        for (NodeConnector innc : ncset0) {
+            VTNFlow flow = fdb.create(vtnMgr);
+            addFlowEntry(vtnMgr, flow, innc, (short) 0, outnc, pri);
+
+            fdb.install(vtnMgr, flow);
+            flushFlowTasks();
+
+            flows.add(flow);
+        }
+
+        ConcurrentMap<FlowGroupId, VTNFlow> db = vtnMgr.getFlowDB();
+        assertEquals(flows.size(), db.values().size());
+        assertEquals(flows.size(), stubObj.getFlowEntries().size());
+
+        List<VTNFlow> removeFlows = new ArrayList<VTNFlow>();
+        FlowRemoveTask task = fdb.removeFlows(vtnMgr, removeFlows);
+        assertNull(task);
+        assertEquals(flows.size(), db.values().size());
+        assertEquals(flows.size(), stubObj.getFlowEntries().size());
+
+        int numTotal = flows.size();
+        for (VTNFlow flow : flows) {
+            removeFlows.add(flow);
+            task = fdb.removeFlows(vtnMgr, removeFlows);
+            assertNotNull(task);
+            flushFlowTasks();
+            assertEquals("(numRemove)" + removeFlows.size() + ",(flow)" + flow.toString(),
+                         numTotal - removeFlows.size(), db.values().size());
+            assertEquals("(numRemove)" + removeFlows.size() + ",(flow)" + flow.toString(),
+                         numTotal - removeFlows.size(), stubObj.getFlowEntries().size());
+
+            checkFlowDBEntriesListOfVTNFlow(vtnMgr, fdb, flows, removeFlows);
+            Set<VTNFlow> removeSets = new HashSet<VTNFlow>(removeFlows);
+            revertFlowEntries(vtnMgr, fdb, removeSets, numTotal, numTotal);
+        }
+
+        // specify non match flow
+        VTNFlow flow = flows.iterator().next();
+        VTNFlow flowNew = new VTNFlow(flow.getGroupId());
+        NodeConnector innc
+            = NodeConnectorCreator.createOFNodeConnector(Short.valueOf((short)1),
+                                                         node0);
+
+        addFlowEntry(vtnMgr, flowNew, innc, (short) 0, outnc, pri);
+
+        List<VTNFlow> flowsNew = new ArrayList<VTNFlow>();
+        flowsNew.add(flowNew);
+
+        int preSize = db.values().size();
+        int preEntries = stubObj.getFlowEntries().size();
+
+        task = fdb.removeFlows(vtnMgr, flowsNew);
+        // TODO: is it correct?
+        assertNotNull(task);
+        flushFlowTasks();
+
+//        assertEquals(preSize, db.values().size());
+//        assertEquals(preEntries, stubObj.getFlowEntries().size());
+
+        // in case there are no flow entry.
+        fdb.clear(vtnMgr);
+        flushFlowTasks();
+
+        task = fdb.removeFlows(vtnMgr, removeFlows);
+        // TODO: need to fix ?
+        // assertNull(task);
+        flushFlowTasks();
+        assertEquals(0, db.values().size());
+        assertEquals(0, stubObj.getFlowEntries().size());
+
+    }
+
+    /**
+     * Check Flow entries in flowDB
+     * after {@link removeFlows(VTNManager, List<VTNFlow>)} was invoked.
+     *
+     * @param mgr           VTNManager service.
+     * @param fdb           VTNFlowDatabase.
+     * @param flows         flows initially installed.
+     * @param removeFlows   List of VTNFlows which was removed.
+     */
+    private void checkFlowDBEntriesListOfVTNFlow(VTNManagerImpl mgr,
+                                                 VTNFlowDatabase fdb,
+                                                 Set<VTNFlow> flows,
+                                                 List<VTNFlow> removeFlows) {
+        ConcurrentMap<FlowGroupId, VTNFlow> db = vtnMgr.getFlowDB();
+
+        for (VTNFlow flow : flows) {
+            VTNFlow rflow = db.get(flow.getGroupId());
+            if (removeFlows.contains(flow)) {
+                rflow = db.get(flow.getGroupId());
+                assertEquals(null, rflow);
+            } else {
+                rflow = db.get(flow.getGroupId());
+                assertEquals(flow, rflow);
+            }
+        }
+    }
+
+    /**
+     * Test method for
+     * {@link VTNFlowDatabase#containsIngressFlow(FlowEntry)}.
+     */
+    @Test
+    public void testContainsIngressFlow() {
+        VTNFlowDatabase fdb = new VTNFlowDatabase("test");
+
+        Node node0 = NodeCreator.createOFNode(Long.valueOf(0L));
+        Node node1 = NodeCreator.createOFNode(Long.valueOf(1L));
+        Set<Short> portIds0 = new HashSet<Short>(createShorts((short) 10, (short) 5,
+                                                              false));
+        Set<Short> portIds1 = new HashSet<Short>(createShorts((short) 10, (short) 5,
+                                                              false));
+        Set<NodeConnector> ncSet0
+            = NodeConnectorCreator.createOFNodeConnectorSet(portIds0, node0);
+        Set<NodeConnector> ncSet1
+            = NodeConnectorCreator.createOFNodeConnectorSet(portIds1, node1);
+        Set<Set<NodeConnector>> ncSets = new HashSet<Set<NodeConnector>>();
+        ncSets.add(ncSet0);
+        ncSets.add(ncSet1);
+
+        int pri = 1;
+
+        for (Set<NodeConnector> incSet : ncSets) {
+            Set<NodeConnector> encSet = null;
+            if (incSet.equals(ncSet0)) {
+                encSet = ncSet1;
+            } else {
+                encSet = ncSet0;
+            }
+
+            // add flowEntry as test parameter.
+            Set<VTNFlow> flows = new HashSet<VTNFlow>();
+            Iterator<NodeConnector> iNcset1 = encSet.iterator();
+            for (NodeConnector innc : incSet) {
+                VTNFlow flow = fdb.create(vtnMgr);
+                NodeConnector outnc
+                    = NodeConnectorCreator.createOFNodeConnector(Short.valueOf((short) 15),
+                                                                 innc.getNode());
+                addFlowEntry(vtnMgr, flow, innc, (short) 0, outnc, pri);
+
+                if (!iNcset1.hasNext()) {
+                    break;
+                }
+                NodeConnector innc1 = iNcset1.next();
+                outnc = NodeConnectorCreator.createOFNodeConnector(Short.valueOf((short) 15),
+                                                                   innc1.getNode());
+                addFlowEntry(vtnMgr, flow, innc1, (short) 0, outnc, pri);
+
+                fdb.install(vtnMgr, flow);
+                flushFlowTasks();
+
+                flows.add(flow);
+            }
+
+            for (VTNFlow flow : flows) {
+                for (FlowEntry ent : flow.getFlowEntries()) {
+                    if (ent.getNode().equals(incSet.iterator().next().getNode())) {
+                        assertTrue(ent.toString(), fdb.containsIngressFlow(ent));
+                    } else {
+                        assertFalse(ent.toString(), fdb.containsIngressFlow(ent));
+                    }
+                }
+            }
+
+            fdb.clear(vtnMgr);
+
+            // call after cleared.
+            FlowEntry ent = flows.iterator().next().getFlowEntries().get(0);
+            assertFalse(fdb.containsIngressFlow(ent));
+        }
+    }
+
+
+    // private methods
+
+    /**
+     * Add FlowEntry to VTNFlow.
+     *
+     * @param   flow        A {@link VTNFlow}.
+     * @param   inPort      A ingress {@link NodeConector}.
+     * @param   inVlan      A incoming VLAN ID.
+     * @param   outPort     A outgoing {@link NodeConnector}.
+     * @param   priority    A priority of FlowEntry.
+     * @return {@link VTNFlow}.
+     */
+    private VTNFlow addFlowEntry(VTNManagerImpl mgr, VTNFlow flow,
+            NodeConnector inPort, short inVlan, NodeConnector outPort,
+            int priority) {
+        Match match = new Match();
+        match.setField(MatchType.IN_PORT, inPort);
+        match.setField(MatchType.DL_VLAN, inVlan);
+        ActionList actions = new ActionList(outPort.getNode());
+        actions.addOutput(outPort);
+        flow.addFlow(mgr, match, actions, priority);
+
+        return flow;
+    }
+}
index 4e40119832f40a78b625da7ca1660b7297d33c7e..722c40e46bbb222dac1bcb879fcf8b3e06f3a29d 100644 (file)
@@ -58,6 +58,8 @@ import org.opendaylight.vtn.manager.VTenantConfig;
 import org.opendaylight.vtn.manager.VTenantPath;
 import org.opendaylight.vtn.manager.VlanMap;
 import org.opendaylight.vtn.manager.VlanMapConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Common class for tests of {@link VTNManagerImpl}.
@@ -67,6 +69,7 @@ public class VTNManagerImplTestCommon extends TestBase {
     protected GlobalResourceManager resMgr;
     protected TestStub stubObj = null;
     protected static int stubMode = 0;
+    protected static int clusterMode = 0;
 
     /**
      *  Mock-up of IfHostListener.
@@ -665,4 +668,71 @@ public class VTNManagerImplTestCommon extends TestBase {
             }
         }
     }
+
+    /**
+     * Flush all pending tasks on the VTN flow thread.
+     */
+    protected void flushFlowTasks() {
+        flushFlowTasks(10000L);
+    }
+
+    /**
+     * Flush all pending tasks on the VTN flow thread.
+     */
+    protected void flushFlowTasks(long wait) {
+        NopFlowTask task = new NopFlowTask(vtnMgr);
+        vtnMgr.postFlowTask(task);
+        assertTrue(task.await(wait/1000, TimeUnit.SECONDS));
+    }
+
+    /**
+     *  A dummy flow task to flush pending tasks.
+     */
+    protected class NopFlowTask extends FlowModTask {
+        /**
+         * A latch to wait for completion.
+         */
+        private final CountDownLatch  latch = new CountDownLatch(1);
+
+        protected NopFlowTask(VTNManagerImpl mgr) {
+            super(mgr);
+        }
+
+        /**
+         * Wake up all threads waiting for this task.
+         *
+         * @return  {@code true} is always returned.
+         */
+        @Override
+        protected boolean execute() {
+            latch.countDown();
+            return true;
+        }
+
+        /**
+         * Return a logger object for this class.
+         *
+         * @return  A logger object.
+         */
+        @Override
+        protected Logger getLogger() {
+            return LoggerFactory.getLogger(getClass());
+        }
+
+        /**
+         * Wait for completion of this task.
+         *
+         * @param timeout  The maximum time to wait.
+         * @param unit     The time unit of the {@code timeout} argument.
+         * @return  {@code true} is returned if this task completed.
+         *          Otherwise {@code false} is returned.
+         */
+        private boolean await(long timeout, TimeUnit unit) {
+            try {
+                return latch.await(timeout, unit);
+            } catch (InterruptedException e) {
+                return false;
+            }
+        }
+    }
 }
index 3aec1be358e35166e376b3bacb506bc744b93abd..7ad19ccace33b517037e1769a108908663ee7239 100644 (file)
@@ -2784,38 +2784,4 @@ public class VTNManagerImplWithNodesTest extends VTNManagerImplTestCommon {
             assertSame(0, stub.getFlowEntries().size());
         }
     }
-
-    /**
-     * A dummy flow task to flush pending tasks.
-     */
-    private class NopFlowTask extends FlowModTask {
-        /**
-         * Construct a new task.
-         *
-         * @param mgr  VTN Manager service.
-         */
-        private NopFlowTask(VTNManagerImpl mgr) {
-            super(mgr);
-        }
-
-        /**
-         * Execute this task.
-         *
-         * @return  {@code true} is always returned.
-         */
-        @Override
-        protected boolean execute() {
-            return true;
-        }
-
-        /**
-         * Return a logger object for this class.
-         *
-         * @return  A logger object.
-         */
-        @Override
-        protected Logger getLogger() {
-            return LoggerFactory.getLogger(getClass());
-        }
-    }
 }