Collect debugs before tempest teardown

- the tempest.api.network bucket is still run with ostestr so it
  can take advantage of the parallel execution and will not pause
  on teardown.

- all scenario tests are broken out to be run one at a time and
  will collect debugs on teardown.

Change-Id: I2e30d543c61ddbc726a7d409e2444dbd46ae488c
Signed-off-by: Jamo Luhrsen <jluhrsen@redhat.com>

diff --git a/csit/libraries/DevstackUtils.robot b/csit/libraries/DevstackUtils.robot
index e4ce884aedab2662d1f079b9968d45d69a722948..78b818e13c7ae0ca6ae14d7a48a41715f0197d02 100644 (file)
--- a/csit/libraries/DevstackUtils.robot
+++ b/csit/libraries/DevstackUtils.robot
@@ -20,7 +20,8 @@ ${blacklist_file}    /tmp/blacklist.txt
 @{stable/ocata_exclusion_regexes}    ${EMPTY}
 @{stable/pike_exclusion_regexes}    ${EMPTY}
 @{master_exclusion_regexes}    ${EMPTY}
-${tempest_config_file}    /opt/stack/tempest/etc/tempest.conf
+${tempest_dir}    /opt/stack/tempest
+${tempest_config_file}    ${tempest_dir}/etc/tempest.conf
 ${external_physical_network}    physnet1
 ${external_net_name}    external-net
 ${external_subnet_name}    external-subnet
@@ -28,36 +29,64 @@ ${external_subnet_name}    external-subnet
 ${external_gateway}    10.10.10.250
 ${external_subnet_allocation_pool}    start=10.10.10.2,end=10.10.10.249
 ${external_subnet}    10.10.10.0/24
+${default_timeout}    420s
 
 *** Keywords ***
 Run Tempest Tests
-    [Arguments]    ${tempest_regex}    ${exclusion_file}=/dev/null    ${tempest_conf}=""    ${tempest_directory}=/opt/stack/tempest    ${timeout}=420s
-    [Documentation]    Execute the tempest tests.
+    [Arguments]    ${tempest_regex}    ${timeout}=${default_timeout}    ${debug}=False
+    Run Keyword If    "${debug}"=="False"    Run Tempest Tests Without Debug    ${tempest_regex}    timeout=${timeout}
+    Run Keyword If    "${debug}"=="True"    Run Tempest Tests With Debug    ${tempest_regex}    timeout=${timeout}
+    Run Keyword If    "${debug}"!="True" and "${debug}"!="False"    Fail    debug argument must be True or False
+
+Run Tempest Tests Without Debug
+    [Arguments]    ${tempest_regex}    ${tempest_directory}=${tempest_dir}    ${timeout}=${default_timeout}
+    [Documentation]    Using ostestr will allow us to (by default) run tests in paralllel.
+    ...    Because of the parallel fashion, we must ensure there is no pause on teardown so that flag in tempest.conf is
+    ...    explicitly set to False.
+    Return From Keyword If    "skip_if_${OPENSTACK_BRANCH}" in @{TEST_TAGS}
+    Return From Keyword If    "skip_if_${SECURITY_GROUP_MODE}" in @{TEST_TAGS}
+    ${tempest_conn_id}=    SSHLibrary.Open Connection    ${OS_CONTROL_NODE_IP}    prompt=${DEFAULT_LINUX_PROMPT_STRICT}
+    SSHKeywords.Flexible SSH Login    ${OS_USER}    ${DEVSTACK_SYSTEM_PASSWORD}
+    Write Commands Until Prompt    source ${DEVSTACK_DEPLOY_PATH}/openrc admin admin
+    Write Commands Until Prompt    cd ${tempest_directory}
+    SSHLibrary.Read
+    Tempest Conf Modify Pause On Test Teardown    False
+    SSHLibrary.Set Client Configuration    timeout=${timeout}
+    ${output}=    Write Commands Until Prompt    ostestr --regex ${tempest_regex}    timeout=${timeout}
+    Log    ${output}
+    SSHLibrary.Close Connection
+    Should Contain    ${output}    Failed: 0
+
+Run Tempest Tests With Debug
+    [Arguments]    ${tempest_regex}    ${tempest_directory}=${tempest_dir}    ${timeout}=${default_timeout}
+    [Documentation]    After setting pause_teardown=True in tempest.conf, use the python -m testtools.run module to execute
+    ...    a single tempest test case. We need to run only one tempest test case at a time as there will
+    ...    be potentional for an unkown number of debug pdb() prompts to catch and continue if we are running multiple
+    ...    test cases with a single command. Essentially, this keyword only handles one breakpoint at a single teardown.
     Return From Keyword If    "skip_if_${OPENSTACK_BRANCH}" in @{TEST_TAGS}
     Return From Keyword If    "skip_if_${SECURITY_GROUP_MODE}" in @{TEST_TAGS}
-    ${devstack_conn_id}=    Get ControlNode Connection
-    Switch Connection    ${devstack_conn_id}
-    # There seems to be a bug in the mitaka version of os-testr that does not allow --regex to work in conjunction
-    # with a blacklist-file. Upgrading with pip should resolve this. This can probably go away once mitaka is no
-    # longer tested in this environment. But, while it's being tested the mitaka devstack setup will be bringing
-    # in this broken os-testr, so we manually upgrade here.
-    Write Commands Until Prompt    sudo pip install os-testr --upgrade    timeout=120s
+    ${tempest_conn_id}=    SSHLibrary.Open Connection    ${OS_CONTROL_NODE_IP}    prompt=${DEFAULT_LINUX_PROMPT_STRICT}
+    SSHKeywords.Flexible SSH Login    ${OS_USER}    ${DEVSTACK_SYSTEM_PASSWORD}
     Write Commands Until Prompt    source ${DEVSTACK_DEPLOY_PATH}/openrc admin admin
     Write Commands Until Prompt    cd ${tempest_directory}
-    # From Ocata and moving forward, we can replace 'ostestr' with 'tempest run'
-    ${results}=    Write Commands Until Prompt    ostestr --regex ${tempest_regex} -b ${exclusion_file}    timeout=${timeout}
-    # Save stdout to file
-    Create File    tempest_output_${tempest_regex}.log    data=${results}
-    # output tempest generated log file which may have different debug levels than what stdout would show
-    # FIXME: having the INFO level tempest logs is helpful as it gives details like the UUIDs of nouns used in the
-    # the tests which can sometimes be tracked in ODL and Openstack logs when debugging. However, this "cat" step
-    # does not even complete for the tempest.api.network tests in under 2min. We need a faster way to get this
-    # info. Probably pulling the log file and storing it on the log server is best. Hopefully someone can get
-    # to this. For now, commenting out this next debug step.
-    # ${output}=    Write Commands Until Prompt    cat ${tempest_directory}/tempest.log    timeout=120s
-    # Log    ${output}
-    Should Contain    ${results}    Failed: 0
-    # TODO: also need to verify some non-zero pass count as well as other results are ok (e.g. skipped, etc)
+    SSHLibrary.Read
+    Tempest Conf Modify Pause On Test Teardown    True
+    SSHLibrary.Set Client Configuration    timeout=${timeout}
+    SSHLibrary.Write    python -m testtools.run ${tempest_regex}
+    ${output}=    SSHLibrary.Read Until Regexp    ${DEFAULT_LINUX_PROMPT_STRICT}|pdb.set_trace()
+    Log    ${output}
+    Show Debugs
+    Get Test Teardown Debugs
+    SSHLibrary.Switch Connection    ${tempest_conn_id}
+    SSHLibrary.Write    continue
+    ${output}=    SSHLibrary.Read Until Regexp    ${DEFAULT_LINUX_PROMPT_STRICT}|pdb.set_trace()
+    Log    ${output}
+    SSHLibrary.Write    continue
+    ${output}=    SSHLibrary.Read Until Prompt
+    Log    ${output}
+    SSHLibrary.Close Connection
+    Should Contain    ${output}    OK
+    Should Not Contain    ${output}    FAILED
 
 Log In To Tempest Executor And Setup Test Environment
     [Documentation]    Initialize SetupUtils, open SSH connection to a devstack system and source the openstack
@@ -90,6 +119,12 @@ Tempest Conf Add External Network And Floating Network Name
     Write Commands Until Prompt    sudo cat ${tempest_config_file}
     Write Commands Until Prompt    sudo chmod 777 ${tempest_config_file}
 
+Tempest Conf Modify Pause On Test Teardown
+    [Arguments]    ${pause_flag}
+    [Documentation]    Sets the DEFAULT section flag for pausing the test teardown. If True the tempest test case
+    ...    being executed will break to a pdb() debug shell when it hits it's teardown() function.
+    Modify Config In File On Existing SSH Connection    ${tempest_config_file}    set    DEFAULT    pause_teardown    ${pause_flag}
+
 Modify Config In File On Existing SSH Connection
     [Arguments]    ${config_file}    ${modifier}    ${config_section}    ${config_key}    ${config_value}=${EMPTY}
     [Documentation]    uses crudini to populate oslo cofg file.

diff --git a/csit/libraries/OpenStackOperations.robot b/csit/libraries/OpenStackOperations.robot
index b937e6b0e81a42aa07030f518c03a653c40a6632..a7ccdef1a94fa4991d4742d42e4e4f2e471cbcb4 100644 (file)
--- a/csit/libraries/OpenStackOperations.robot
+++ b/csit/libraries/OpenStackOperations.robot
@@ -142,7 +142,7 @@ List Ports
 
 List Nova VMs
     [Documentation]    List VMs and return output with nova client.
-    ${rc}    ${output}=    Run And Return Rc And Output    openstack server list
+    ${rc}    ${output}=    Run And Return Rc And Output    openstack server list --all-projects
     Log    ${output}
     Should Not Be True    ${rc}
     [Return]    ${output}

diff --git a/csit/suites/openstack/tempest/tempest.robot b/csit/suites/openstack/tempest/tempest.robot
index 7500f13a0884906fa56b54cd973b53d0722a82f6..73b31a992edb7e23e9519874c4de27a9f43936d4 100644 (file)
--- a/csit/suites/openstack/tempest/tempest.robot
+++ b/csit/suites/openstack/tempest/tempest.robot
@@ -17,66 +17,79 @@ Resource          ../../../variables/netvirt/Variables.robot
 
 *** Test Cases ***
 tempest.api.network
-    ${TEST_NAME}    ${blacklist_file}    ${tempest_config_file}    timeout=900s
+    ${TEST_NAME}    debug=False    timeout=900s
 
 tempest.scenario.test_aggregates_basic_ops.TestAggregatesBasicOps.test_aggregate_basic_ops
-    ${TEST_NAME}    ${blacklist_file}    ${tempest_config_file}
+    ${TEST_NAME}    debug=True
 
 tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_pause_unpause
-    ${TEST_NAME}    ${blacklist_file}    ${tempest_config_file}
+    ${TEST_NAME}    debug=True
 
 tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_reboot
-    ${TEST_NAME}    ${blacklist_file}    ${tempest_config_file}
+    ${TEST_NAME}    debug=True
 
 tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_rebuild
-    ${TEST_NAME}    ${blacklist_file}    ${tempest_config_file}
+    ${TEST_NAME}    debug=True
 
 tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_stop_start
-    ${TEST_NAME}    ${blacklist_file}    ${tempest_config_file}
+    ${TEST_NAME}    debug=True
 
 tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_suspend_resume
-    ${TEST_NAME}    ${blacklist_file}    ${tempest_config_file}
+    ${TEST_NAME}    debug=True
 
 tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_connectivity_between_vms_on_different_networks
-    ${TEST_NAME}    ${blacklist_file}    ${tempest_config_file}
+    ${TEST_NAME}    debug=True
 
 tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_hotplug_nic
-    # Failing due to default security rules behavior missing in Mitaka
-    ${TEST_NAME}    ${blacklist_file}    ${tempest_config_file}
+    ${TEST_NAME}    debug=True
 
 tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_mtu_sized_frames
-    ${TEST_NAME}    ${blacklist_file}    ${tempest_config_file}
+    ${TEST_NAME}    debug=True
 
 tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_network_basic_ops
-    ${TEST_NAME}    ${blacklist_file}    ${tempest_config_file}
+    ${TEST_NAME}    debug=True
 
 tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_port_security_macspoofing_port
-    # Failing due to default security rules behavior missing in Mitaka, and also in all transparent runs
-    ${TEST_NAME}    ${blacklist_file}    ${tempest_config_file}
+    ${TEST_NAME}    debug=True
 
 tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_preserve_preexisting_port
-    ${TEST_NAME}    ${blacklist_file}    ${tempest_config_file}
+    ${TEST_NAME}    debug=True
 
 tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_router_rescheduling
-    ${TEST_NAME}    ${blacklist_file}    ${tempest_config_file}
+    ${TEST_NAME}    debug=True
 
 tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_subnet_details
-    ${TEST_NAME}    ${blacklist_file}    ${tempest_config_file}
+    ${TEST_NAME}    debug=True
 
 tempest.scenario.test_network_v6.TestGettingAddress.test_dhcp6_stateless_from_os
-    ${TEST_NAME}    ${blacklist_file}    ${tempest_config_file}
+    ${TEST_NAME}    debug=True
 
 tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_dhcp6_stateless_from_os
-    ${TEST_NAME}    ${blacklist_file}    ${tempest_config_file}
+    ${TEST_NAME}    debug=True
 
 tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_slaac_from_os
-    ${TEST_NAME}    ${blacklist_file}    ${tempest_config_file}
+    ${TEST_NAME}    debug=True
 
 tempest.scenario.test_network_v6.TestGettingAddress.test_slaac_from_os
-    ${TEST_NAME}    ${blacklist_file}    ${tempest_config_file}
+    ${TEST_NAME}    debug=True
 
-tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps
-    ${TEST_NAME}    ${blacklist_file}    ${tempest_config_file}    timeout=900s
+tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps.test_boot_into_disabled_port_security_network_without_secgroup
+    ${TEST_NAME}    debug=True
+
+tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps.test_cross_tenant_traffic
+    ${TEST_NAME}    debug=True
+
+tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps.test_in_tenant_traffic
+    ${TEST_NAME}    debug=True
+
+tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps.test_multiple_security_groups
+    ${TEST_NAME}    debug=True
+
+tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps.test_port_security_disable_security_group
+    ${TEST_NAME}    debug=True
+
+tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps.test_port_update_new_security_group
+    ${TEST_NAME}    debug=True
 
 tempest.scenario.test_server_basic_ops.TestServerBasicOps.test_server_basic_ops
-    ${TEST_NAME}    ${blacklist_file}    ${tempest_config_file}
+    ${TEST_NAME}    debug=True