From 1012f1723c3815f5c704fa9a908c3cfdce58ec31 Mon Sep 17 00:00:00 2001 From: Guillaume Lambert Date: Fri, 18 Jun 2021 22:18:39 +0200 Subject: [PATCH] Refactor tests launching procedure The possibility to run test separately was lost during preliminary work to prepare jobs parallelization. This refactoring restores that feature but with a different approach. For example, the equivalents of the old commands $ tox -e topoPortmapping and $ tox -e portmapping,pce are now $ tox -e tests121 topo_portmapping and $ tox -e tests121 "portmapping pce" (preceded by $ tox -e buildcontroller) The possibility to choose the script launcher command from a global variable have been added. For example, $ export LAUNCHER='python3' allows to replace 'nosetests --with-xunit' by a straight call to python3 to have more debug output during tests. Tests order can also be changed by directly modifying script filenames without touching tox.ini. Filenames have been modified here to add a numbering and keep the tests current order. JIRA: TRNSPRTPCE-482 Signed-off-by: Guillaume Lambert Change-Id: I725d1dfac42bd309d18fbf87ff1c85bc1bce5be9 --- tests/launch_tests.sh | 35 ++++++++++++++ ...t_portmapping.py => test01_portmapping.py} | 0 ...tmapping.py => test02_topo_portmapping.py} | 0 .../{test_topology.py => test03_topology.py} | 0 ...> test04_renderer_service_path_nominal.py} | 0 .../1.2.1/{test_olm.py => test05_olm.py} | 0 .../{test_end2end.py => test06_end2end.py} | 0 ...t_portmapping.py => test01_portmapping.py} | 0 .../{test_topology.py => test02_topology.py} | 0 ...otn_topology.py => test03_otn_topology.py} | 0 ...{test_flex_grid.py => test04_flex_grid.py} | 0 ...> test05_renderer_service_path_nominal.py} | 0 ...otn_renderer.py => test06_otn_renderer.py} | 0 ..._renderer.py => test07_otn_sh_renderer.py} | 0 .../2.2.1/{test_olm.py => test08_olm.py} | 0 .../2.2.1/{test_tapi.py => test09_tapi.py} | 0 ...t_otn_end2end.py => test10_otn_end2end.py} | 0 .../{test_end2end.py => test11_end2end.py} | 0 ...t_portmapping.py => test01_portmapping.py} | 0 ... => test01_device_change_notifications.py} | 0 .../pce/{test_pce.py => test01_pce.py} | 0 .../{test_pce_400G.py => test02_pce_400G.py} | 0 tox.ini | 47 ++++++++++--------- 23 files changed, 60 insertions(+), 22 deletions(-) create mode 100755 tests/launch_tests.sh rename tests/transportpce_tests/1.2.1/{test_portmapping.py => test01_portmapping.py} (100%) rename tests/transportpce_tests/1.2.1/{test_topo_portmapping.py => test02_topo_portmapping.py} (100%) rename tests/transportpce_tests/1.2.1/{test_topology.py => test03_topology.py} (100%) rename tests/transportpce_tests/1.2.1/{test_renderer_service_path_nominal.py => test04_renderer_service_path_nominal.py} (100%) rename tests/transportpce_tests/1.2.1/{test_olm.py => test05_olm.py} (100%) rename tests/transportpce_tests/1.2.1/{test_end2end.py => test06_end2end.py} (100%) rename tests/transportpce_tests/2.2.1/{test_portmapping.py => test01_portmapping.py} (100%) rename tests/transportpce_tests/2.2.1/{test_topology.py => test02_topology.py} (100%) rename tests/transportpce_tests/2.2.1/{test_otn_topology.py => test03_otn_topology.py} (100%) rename tests/transportpce_tests/2.2.1/{test_flex_grid.py => test04_flex_grid.py} (100%) rename tests/transportpce_tests/2.2.1/{test_renderer_service_path_nominal.py => test05_renderer_service_path_nominal.py} (100%) rename tests/transportpce_tests/2.2.1/{test_otn_renderer.py => test06_otn_renderer.py} (100%) rename tests/transportpce_tests/2.2.1/{test_otn_sh_renderer.py => test07_otn_sh_renderer.py} (100%) rename tests/transportpce_tests/2.2.1/{test_olm.py => test08_olm.py} (100%) rename tests/transportpce_tests/2.2.1/{test_tapi.py => test09_tapi.py} (100%) rename tests/transportpce_tests/2.2.1/{test_otn_end2end.py => test10_otn_end2end.py} (100%) rename tests/transportpce_tests/2.2.1/{test_end2end.py => test11_end2end.py} (100%) rename tests/transportpce_tests/7.1/{test_portmapping.py => test01_portmapping.py} (100%) rename tests/transportpce_tests/hybrid/{test_device_change_notifications.py => test01_device_change_notifications.py} (100%) rename tests/transportpce_tests/pce/{test_pce.py => test01_pce.py} (100%) rename tests/transportpce_tests/pce/{test_pce_400G.py => test02_pce_400G.py} (100%) diff --git a/tests/launch_tests.sh b/tests/launch_tests.sh new file mode 100755 index 000000000..c62307bd9 --- /dev/null +++ b/tests/launch_tests.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +for arg in $@; do + if [ -z "$test_suite" ]; then + test_suite=$1 + else +# Tox reinterprets "olm" and "pce" by "../olm" "../pce" because of the +# changedir directive and the presence of modules folders "olm" and pce" +# at project root. + if [ "$arg" = "../olm" ]; then + arglist=$arglist" olm" + elif [ "$arg" = "../pce" ]; then + arglist=$arglist" pce" + else + arglist=$arglist" "$arg + fi + fi +done + +scriptlist="" +if [ -z "$arglist" ]; then + scriptlist="transportpce_tests/$test_suite/test[0-9][0-9]_*.py" +else + for test in $arglist; do + scriptlist=$scriptlist" transportpce_tests/$test_suite/test[0-9][0-9]_$test.py" + done +fi +if [ -z "$LAUNCHER" ]; then + LAUNCHER="nosetests --with-xunit"; +fi + +for script in $scriptlist; do + echo $LAUNCHER $(ls $script) + $LAUNCHER $script || exit 1 +done diff --git a/tests/transportpce_tests/1.2.1/test_portmapping.py b/tests/transportpce_tests/1.2.1/test01_portmapping.py similarity index 100% rename from tests/transportpce_tests/1.2.1/test_portmapping.py rename to tests/transportpce_tests/1.2.1/test01_portmapping.py diff --git a/tests/transportpce_tests/1.2.1/test_topo_portmapping.py b/tests/transportpce_tests/1.2.1/test02_topo_portmapping.py similarity index 100% rename from tests/transportpce_tests/1.2.1/test_topo_portmapping.py rename to tests/transportpce_tests/1.2.1/test02_topo_portmapping.py diff --git a/tests/transportpce_tests/1.2.1/test_topology.py b/tests/transportpce_tests/1.2.1/test03_topology.py similarity index 100% rename from tests/transportpce_tests/1.2.1/test_topology.py rename to tests/transportpce_tests/1.2.1/test03_topology.py diff --git a/tests/transportpce_tests/1.2.1/test_renderer_service_path_nominal.py b/tests/transportpce_tests/1.2.1/test04_renderer_service_path_nominal.py similarity index 100% rename from tests/transportpce_tests/1.2.1/test_renderer_service_path_nominal.py rename to tests/transportpce_tests/1.2.1/test04_renderer_service_path_nominal.py diff --git a/tests/transportpce_tests/1.2.1/test_olm.py b/tests/transportpce_tests/1.2.1/test05_olm.py similarity index 100% rename from tests/transportpce_tests/1.2.1/test_olm.py rename to tests/transportpce_tests/1.2.1/test05_olm.py diff --git a/tests/transportpce_tests/1.2.1/test_end2end.py b/tests/transportpce_tests/1.2.1/test06_end2end.py similarity index 100% rename from tests/transportpce_tests/1.2.1/test_end2end.py rename to tests/transportpce_tests/1.2.1/test06_end2end.py diff --git a/tests/transportpce_tests/2.2.1/test_portmapping.py b/tests/transportpce_tests/2.2.1/test01_portmapping.py similarity index 100% rename from tests/transportpce_tests/2.2.1/test_portmapping.py rename to tests/transportpce_tests/2.2.1/test01_portmapping.py diff --git a/tests/transportpce_tests/2.2.1/test_topology.py b/tests/transportpce_tests/2.2.1/test02_topology.py similarity index 100% rename from tests/transportpce_tests/2.2.1/test_topology.py rename to tests/transportpce_tests/2.2.1/test02_topology.py diff --git a/tests/transportpce_tests/2.2.1/test_otn_topology.py b/tests/transportpce_tests/2.2.1/test03_otn_topology.py similarity index 100% rename from tests/transportpce_tests/2.2.1/test_otn_topology.py rename to tests/transportpce_tests/2.2.1/test03_otn_topology.py diff --git a/tests/transportpce_tests/2.2.1/test_flex_grid.py b/tests/transportpce_tests/2.2.1/test04_flex_grid.py similarity index 100% rename from tests/transportpce_tests/2.2.1/test_flex_grid.py rename to tests/transportpce_tests/2.2.1/test04_flex_grid.py diff --git a/tests/transportpce_tests/2.2.1/test_renderer_service_path_nominal.py b/tests/transportpce_tests/2.2.1/test05_renderer_service_path_nominal.py similarity index 100% rename from tests/transportpce_tests/2.2.1/test_renderer_service_path_nominal.py rename to tests/transportpce_tests/2.2.1/test05_renderer_service_path_nominal.py diff --git a/tests/transportpce_tests/2.2.1/test_otn_renderer.py b/tests/transportpce_tests/2.2.1/test06_otn_renderer.py similarity index 100% rename from tests/transportpce_tests/2.2.1/test_otn_renderer.py rename to tests/transportpce_tests/2.2.1/test06_otn_renderer.py diff --git a/tests/transportpce_tests/2.2.1/test_otn_sh_renderer.py b/tests/transportpce_tests/2.2.1/test07_otn_sh_renderer.py similarity index 100% rename from tests/transportpce_tests/2.2.1/test_otn_sh_renderer.py rename to tests/transportpce_tests/2.2.1/test07_otn_sh_renderer.py diff --git a/tests/transportpce_tests/2.2.1/test_olm.py b/tests/transportpce_tests/2.2.1/test08_olm.py similarity index 100% rename from tests/transportpce_tests/2.2.1/test_olm.py rename to tests/transportpce_tests/2.2.1/test08_olm.py diff --git a/tests/transportpce_tests/2.2.1/test_tapi.py b/tests/transportpce_tests/2.2.1/test09_tapi.py similarity index 100% rename from tests/transportpce_tests/2.2.1/test_tapi.py rename to tests/transportpce_tests/2.2.1/test09_tapi.py diff --git a/tests/transportpce_tests/2.2.1/test_otn_end2end.py b/tests/transportpce_tests/2.2.1/test10_otn_end2end.py similarity index 100% rename from tests/transportpce_tests/2.2.1/test_otn_end2end.py rename to tests/transportpce_tests/2.2.1/test10_otn_end2end.py diff --git a/tests/transportpce_tests/2.2.1/test_end2end.py b/tests/transportpce_tests/2.2.1/test11_end2end.py similarity index 100% rename from tests/transportpce_tests/2.2.1/test_end2end.py rename to tests/transportpce_tests/2.2.1/test11_end2end.py diff --git a/tests/transportpce_tests/7.1/test_portmapping.py b/tests/transportpce_tests/7.1/test01_portmapping.py similarity index 100% rename from tests/transportpce_tests/7.1/test_portmapping.py rename to tests/transportpce_tests/7.1/test01_portmapping.py diff --git a/tests/transportpce_tests/hybrid/test_device_change_notifications.py b/tests/transportpce_tests/hybrid/test01_device_change_notifications.py similarity index 100% rename from tests/transportpce_tests/hybrid/test_device_change_notifications.py rename to tests/transportpce_tests/hybrid/test01_device_change_notifications.py diff --git a/tests/transportpce_tests/pce/test_pce.py b/tests/transportpce_tests/pce/test01_pce.py similarity index 100% rename from tests/transportpce_tests/pce/test_pce.py rename to tests/transportpce_tests/pce/test01_pce.py diff --git a/tests/transportpce_tests/pce/test_pce_400G.py b/tests/transportpce_tests/pce/test02_pce_400G.py similarity index 100% rename from tests/transportpce_tests/pce/test_pce_400G.py rename to tests/transportpce_tests/pce/test02_pce_400G.py diff --git a/tox.ini b/tox.ini index 92b8b4fed..154f8a8ee 100644 --- a/tox.ini +++ b/tox.ini @@ -66,48 +66,51 @@ commands = [testenv:testsPCE] depends = buildcontroller +whitelist_externals = launch_tests.sh +passenv = LAUNCHER commands = - nosetests --with-xunit transportpce_tests/pce/test_pce.py - nosetests --with-xunit transportpce_tests/pce/test_pce_400G.py + ./launch_tests.sh pce {posargs:} [testenv:tests121] depends = buildcontroller,sims121,testsPCE #the last dependency is to avoid temporarily concurrent ressources problem to fix in parallel mode -commands = - nosetests --with-xunit transportpce_tests/1.2.1/test_portmapping.py - nosetests --with-xunit transportpce_tests/1.2.1/test_topo_portmapping.py - nosetests --with-xunit transportpce_tests/1.2.1/test_topology.py - nosetests --with-xunit transportpce_tests/1.2.1/test_renderer_service_path_nominal.py - nosetests --with-xunit transportpce_tests/1.2.1/test_olm.py - nosetests --with-xunit transportpce_tests/1.2.1/test_end2end.py +whitelist_externals = launch_tests.sh +passenv = LAUNCHER +commands = + ./launch_tests.sh 1.2.1 {posargs:} +# By default, all tests in the folder transportpce_tests/1.2.1/ are run. +# Though, a restricted list of tests can be passed as argument to 'tox -e tests121'. +# For example, 'tox -e tests121 portmapping' will run only the portmapping test. +# and 'tox -e tests121 "portmapping pce"' will run the portmapping and pce tests. +# Additionnally, a different launcher command than 'nosetests --with-xunit' can be +# used by exporting the variable LAUNCHER. For example 'export LAUNCHER="python3"' +# which is more verbose than nose or 'export LAUNCHER="ls -l"' to only list script +# files attributes. [testenv:tests221] depends = buildcontroller,sims221,tests121 #the last dependency is to avoid temporarily concurrent ressources problem to fix in parallel mode +whitelist_externals = launch_tests.sh +passenv = LAUNCHER commands = - nosetests --with-xunit transportpce_tests/2.2.1/test_portmapping.py - nosetests --with-xunit transportpce_tests/2.2.1/test_topology.py - nosetests --with-xunit transportpce_tests/2.2.1/test_otn_topology.py - nosetests --with-xunit transportpce_tests/2.2.1/test_flex_grid.py - nosetests --with-xunit transportpce_tests/2.2.1/test_renderer_service_path_nominal.py - nosetests --with-xunit transportpce_tests/2.2.1/test_otn_renderer.py - nosetests --with-xunit transportpce_tests/2.2.1/test_otn_sh_renderer.py - nosetests --with-xunit transportpce_tests/2.2.1/test_olm.py - nosetests --with-xunit transportpce_tests/2.2.1/test_tapi.py - nosetests --with-xunit transportpce_tests/2.2.1/test_otn_end2end.py - nosetests --with-xunit transportpce_tests/2.2.1/test_end2end.py + ./launch_tests.sh 2.2.1 {posargs:} [testenv:tests71] depends = buildcontroller,sims71,tests221 #the last dependency is to avoid temporarily concurrent ressources problem to fix in parallel mode +whitelist_externals = launch_tests.sh +passenv = LAUNCHER commands = - nosetests --with-xunit transportpce_tests/7.1/test_portmapping.py + ./launch_tests.sh 7.1 {posargs:} [testenv:tests_hybrid] depends = buildcontroller,sims121,sims221,sims71,tests71 #the last dependency is to avoid temporarily concurrent ressources problem to fix in parallel mode +whitelist_externals = launch_tests.sh +passenv = LAUNCHER commands = - nosetests --with-xunit transportpce_tests/hybrid/test_device_change_notifications.py +# nosetests --with-xunit transportpce_tests/hybrid/test01_device_change_notifications.py + ./launch_tests.sh hybrid {posargs:} [testenv:gnpy] depends = buildcontroller -- 2.36.6