Refactor tests launching procedure

The possibility to run test separately was lost during preliminary work
to prepare jobs parallelization.
This refactoring restores that feature but with a different approach.
For example, the equivalents of the old commands
    $ tox -e topoPortmapping
and
    $ tox -e portmapping,pce
are now
    $ tox -e tests121 topo_portmapping
and
    $ tox -e tests121 "portmapping pce"
(preceded by $ tox -e buildcontroller)

The possibility to choose the script launcher command from a global
variable have been added.
For example,
    $ export LAUNCHER='python3'
allows to replace 'nosetests --with-xunit' by a straight call to python3
to have more debug output during tests.

Tests order can also be changed by directly modifying script filenames
without touching tox.ini. Filenames have been modified here to add a
numbering and keep the tests current order.

JIRA: TRNSPRTPCE-482
Signed-off-by: Guillaume Lambert <guillaume.lambert@orange.com>
Change-Id: I725d1dfac42bd309d18fbf87ff1c85bc1bce5be9

diff --git a/tests/launch_tests.sh b/tests/launch_tests.sh
new file mode 100755 (executable)
index 0000000..c62307b
--- /dev/null
+++ b/tests/launch_tests.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+
+for arg in $@; do
+    if [ -z "$test_suite" ]; then
+        test_suite=$1
+    else
+# Tox reinterprets "olm" and "pce" by "../olm" "../pce" because of the
+# changedir directive and the presence of modules folders "olm" and pce"
+# at project root.
+        if [ "$arg" = "../olm" ]; then
+            arglist=$arglist" olm"
+        elif [ "$arg" = "../pce" ]; then
+            arglist=$arglist" pce"
+        else
+            arglist=$arglist" "$arg
+        fi
+    fi
+done
+
+scriptlist=""
+if [ -z "$arglist" ]; then
+    scriptlist="transportpce_tests/$test_suite/test[0-9][0-9]_*.py"
+else
+    for test in $arglist; do
+        scriptlist=$scriptlist" transportpce_tests/$test_suite/test[0-9][0-9]_$test.py"
+    done
+fi
+if [ -z "$LAUNCHER" ]; then
+    LAUNCHER="nosetests --with-xunit";
+fi
+
+for script in $scriptlist; do
+    echo $LAUNCHER $(ls $script)
+    $LAUNCHER $script || exit 1
+done

diff --git a/tests/transportpce_tests/1.2.1/test_portmapping.py b/tests/transportpce_tests/1.2.1/test01_portmapping.py
similarity index 100%
rename from tests/transportpce_tests/1.2.1/test_portmapping.py
rename to tests/transportpce_tests/1.2.1/test01_portmapping.py

diff --git a/tests/transportpce_tests/1.2.1/test_topo_portmapping.py b/tests/transportpce_tests/1.2.1/test02_topo_portmapping.py
similarity index 100%
rename from tests/transportpce_tests/1.2.1/test_topo_portmapping.py
rename to tests/transportpce_tests/1.2.1/test02_topo_portmapping.py

diff --git a/tests/transportpce_tests/1.2.1/test_topology.py b/tests/transportpce_tests/1.2.1/test03_topology.py
similarity index 100%
rename from tests/transportpce_tests/1.2.1/test_topology.py
rename to tests/transportpce_tests/1.2.1/test03_topology.py

diff --git a/tests/transportpce_tests/1.2.1/test_renderer_service_path_nominal.py b/tests/transportpce_tests/1.2.1/test04_renderer_service_path_nominal.py
similarity index 100%
rename from tests/transportpce_tests/1.2.1/test_renderer_service_path_nominal.py
rename to tests/transportpce_tests/1.2.1/test04_renderer_service_path_nominal.py

diff --git a/tests/transportpce_tests/1.2.1/test_olm.py b/tests/transportpce_tests/1.2.1/test05_olm.py
similarity index 100%
rename from tests/transportpce_tests/1.2.1/test_olm.py
rename to tests/transportpce_tests/1.2.1/test05_olm.py

diff --git a/tests/transportpce_tests/1.2.1/test_end2end.py b/tests/transportpce_tests/1.2.1/test06_end2end.py
similarity index 100%
rename from tests/transportpce_tests/1.2.1/test_end2end.py
rename to tests/transportpce_tests/1.2.1/test06_end2end.py

diff --git a/tests/transportpce_tests/2.2.1/test_portmapping.py b/tests/transportpce_tests/2.2.1/test01_portmapping.py
similarity index 100%
rename from tests/transportpce_tests/2.2.1/test_portmapping.py
rename to tests/transportpce_tests/2.2.1/test01_portmapping.py

diff --git a/tests/transportpce_tests/2.2.1/test_topology.py b/tests/transportpce_tests/2.2.1/test02_topology.py
similarity index 100%
rename from tests/transportpce_tests/2.2.1/test_topology.py
rename to tests/transportpce_tests/2.2.1/test02_topology.py

diff --git a/tests/transportpce_tests/2.2.1/test_otn_topology.py b/tests/transportpce_tests/2.2.1/test03_otn_topology.py
similarity index 100%
rename from tests/transportpce_tests/2.2.1/test_otn_topology.py
rename to tests/transportpce_tests/2.2.1/test03_otn_topology.py

diff --git a/tests/transportpce_tests/2.2.1/test_flex_grid.py b/tests/transportpce_tests/2.2.1/test04_flex_grid.py
similarity index 100%
rename from tests/transportpce_tests/2.2.1/test_flex_grid.py
rename to tests/transportpce_tests/2.2.1/test04_flex_grid.py

diff --git a/tests/transportpce_tests/2.2.1/test_renderer_service_path_nominal.py b/tests/transportpce_tests/2.2.1/test05_renderer_service_path_nominal.py
similarity index 100%
rename from tests/transportpce_tests/2.2.1/test_renderer_service_path_nominal.py
rename to tests/transportpce_tests/2.2.1/test05_renderer_service_path_nominal.py

diff --git a/tests/transportpce_tests/2.2.1/test_otn_renderer.py b/tests/transportpce_tests/2.2.1/test06_otn_renderer.py
similarity index 100%
rename from tests/transportpce_tests/2.2.1/test_otn_renderer.py
rename to tests/transportpce_tests/2.2.1/test06_otn_renderer.py

diff --git a/tests/transportpce_tests/2.2.1/test_otn_sh_renderer.py b/tests/transportpce_tests/2.2.1/test07_otn_sh_renderer.py
similarity index 100%
rename from tests/transportpce_tests/2.2.1/test_otn_sh_renderer.py
rename to tests/transportpce_tests/2.2.1/test07_otn_sh_renderer.py

diff --git a/tests/transportpce_tests/2.2.1/test_olm.py b/tests/transportpce_tests/2.2.1/test08_olm.py
similarity index 100%
rename from tests/transportpce_tests/2.2.1/test_olm.py
rename to tests/transportpce_tests/2.2.1/test08_olm.py

diff --git a/tests/transportpce_tests/2.2.1/test_tapi.py b/tests/transportpce_tests/2.2.1/test09_tapi.py
similarity index 100%
rename from tests/transportpce_tests/2.2.1/test_tapi.py
rename to tests/transportpce_tests/2.2.1/test09_tapi.py

diff --git a/tests/transportpce_tests/2.2.1/test_otn_end2end.py b/tests/transportpce_tests/2.2.1/test10_otn_end2end.py
similarity index 100%
rename from tests/transportpce_tests/2.2.1/test_otn_end2end.py
rename to tests/transportpce_tests/2.2.1/test10_otn_end2end.py

diff --git a/tests/transportpce_tests/2.2.1/test_end2end.py b/tests/transportpce_tests/2.2.1/test11_end2end.py
similarity index 100%
rename from tests/transportpce_tests/2.2.1/test_end2end.py
rename to tests/transportpce_tests/2.2.1/test11_end2end.py

diff --git a/tests/transportpce_tests/7.1/test_portmapping.py b/tests/transportpce_tests/7.1/test01_portmapping.py
similarity index 100%
rename from tests/transportpce_tests/7.1/test_portmapping.py
rename to tests/transportpce_tests/7.1/test01_portmapping.py

diff --git a/tests/transportpce_tests/pce/test_pce.py b/tests/transportpce_tests/pce/test01_pce.py
similarity index 100%
rename from tests/transportpce_tests/pce/test_pce.py
rename to tests/transportpce_tests/pce/test01_pce.py

diff --git a/tests/transportpce_tests/pce/test_pce_400G.py b/tests/transportpce_tests/pce/test02_pce_400G.py
similarity index 100%
rename from tests/transportpce_tests/pce/test_pce_400G.py
rename to tests/transportpce_tests/pce/test02_pce_400G.py

diff --git a/tox.ini b/tox.ini
index a9bb3f1956025734675f2cb996e284591ff1ee4a..281c2e12bde1e84ecd0282cf0a319022bd153a2f 100644 (file)
--- a/tox.ini
+++ b/tox.ini
@@ -66,42 +66,42 @@ commands =
 
 [testenv:testsPCE]
 depends = buildcontroller
+whitelist_externals = launch_tests.sh
+passenv = LAUNCHER
 commands =
-  nosetests --with-xunit transportpce_tests/pce/test_pce.py
-  nosetests --with-xunit transportpce_tests/pce/test_pce_400G.py
+  ./launch_tests.sh pce {posargs:}
 
 [testenv:tests121]
 depends = buildcontroller,sims121,testsPCE
 #the last dependency is to avoid temporarily concurrent ressources problem to fix in parallel mode
-commands =
-  nosetests --with-xunit transportpce_tests/1.2.1/test_portmapping.py
-  nosetests --with-xunit transportpce_tests/1.2.1/test_topo_portmapping.py
-  nosetests --with-xunit transportpce_tests/1.2.1/test_topology.py
-  nosetests --with-xunit transportpce_tests/1.2.1/test_renderer_service_path_nominal.py
-  nosetests --with-xunit transportpce_tests/1.2.1/test_olm.py
-  nosetests --with-xunit transportpce_tests/1.2.1/test_end2end.py
+whitelist_externals = launch_tests.sh
+passenv = LAUNCHER
+commands =
+  ./launch_tests.sh 1.2.1 {posargs:}
+# By default, all tests in the folder transportpce_tests/1.2.1/ are run.
+# Though, a restricted list of tests can be passed as argument to 'tox -e tests121'.
+# For example, 'tox -e tests121 portmapping' will run only the portmapping test.
+# and 'tox -e tests121 "portmapping pce"' will run the portmapping and pce tests.
+# Additionnally, a different launcher command than 'nosetests --with-xunit' can be
+# used by exporting the variable LAUNCHER. For example 'export LAUNCHER="python3"'
+# which is more verbose than nose or 'export LAUNCHER="ls -l"' to only list script
+# files attributes.
 
 [testenv:tests221]
 depends = buildcontroller,sims221,tests121
 #the last dependency is to avoid temporarily concurrent ressources problem to fix in parallel mode
+whitelist_externals = launch_tests.sh
+passenv = LAUNCHER
 commands =
-  nosetests --with-xunit transportpce_tests/2.2.1/test_portmapping.py
-  nosetests --with-xunit transportpce_tests/2.2.1/test_topology.py
-  nosetests --with-xunit transportpce_tests/2.2.1/test_otn_topology.py
-  nosetests --with-xunit transportpce_tests/2.2.1/test_flex_grid.py
-  nosetests --with-xunit transportpce_tests/2.2.1/test_renderer_service_path_nominal.py
-  nosetests --with-xunit transportpce_tests/2.2.1/test_otn_renderer.py
-  nosetests --with-xunit transportpce_tests/2.2.1/test_otn_sh_renderer.py
-  nosetests --with-xunit transportpce_tests/2.2.1/test_olm.py
-  nosetests --with-xunit transportpce_tests/2.2.1/test_tapi.py
-  nosetests --with-xunit transportpce_tests/2.2.1/test_otn_end2end.py
-  nosetests --with-xunit transportpce_tests/2.2.1/test_end2end.py
+  ./launch_tests.sh 2.2.1 {posargs:}
 
 [testenv:tests71]
 depends = buildcontroller,sims71,tests221
 #the last dependency is to avoid temporarily concurrent ressources problem to fix in parallel mode
+whitelist_externals = launch_tests.sh
+passenv = LAUNCHER
 commands =
-  nosetests --with-xunit transportpce_tests/7.1/test_portmapping.py
+  ./launch_tests.sh 7.1 {posargs:}
 
 [testenv:gnpy]
 depends = buildcontroller