Skip RoutersNegativeTest for mitaka

This is from the tempest.api.network suite. This is a newly (as
of Mar 2017) added test and fails in mitaka. The openstack fix
was made for newton+ but will not be backported to mitaka.

Change-Id: Ieb8062badd8b305da6d6b43d45b3ea34a88b579b
Signed-off-by: Jamo Luhrsen <jluhrsen@redhat.com>

diff --git a/csit/libraries/DevstackUtils.robot b/csit/libraries/DevstackUtils.robot
index d3bc916100c09be65e2cb73d559dee849dcff208..a81c95cc926e1030341e98686b5dc43d9e93915a 100644 (file)
--- a/csit/libraries/DevstackUtils.robot
+++ b/csit/libraries/DevstackUtils.robot
@@ -17,7 +17,7 @@ ${DEVSTACK_SYSTEM_PASSWORD}    \    # set to empty, but provide for others to ov
 
 *** Keywords ***
 Run Tempest Tests
-    [Arguments]    ${tempest_regex}    ${tempest_exclusion_regex}=""    ${tempest_conf}=""    ${tempest_directory}=/opt/stack/tempest    ${timeout}=600s
+    [Arguments]    ${tempest_regex}    ${tempest_exclusion_regex}=""    ${tempest_directory}=/opt/stack/tempest    ${timeout}=600s
     [Documentation]    Execute the tempest tests.
     Return From Keyword If    "skip_if_${OPENSTACK_BRANCH}" in @{TEST_TAGS}
     Return From Keyword If    "skip_if_${SECURITY_GROUP_MODE}" in @{TEST_TAGS}
@@ -25,10 +25,10 @@ Run Tempest Tests
     Switch Connection    ${devstack_conn_id}
     Write Commands Until Prompt    source ${DEVSTACK_DEPLOY_PATH}/openrc admin admin
     Write Commands Until Prompt    cd ${tempest_directory}
-    # From Ocata and moving forward, we can replace 'ostestr' with 'tempest run'
-    # Note: If black-regex cancels out the entire regex (white-regex), all tests are run
-    # --black-regex ${tempest_exclusion_regex} is removed for now since it only seems to work from newton
-    ${results}=    Write Commands Until Prompt    ostestr --regex ${tempest_regex}    timeout=${timeout}
+    # TODO: From Ocata and moving forward, we can replace 'ostestr' with 'tempest run'
+    # Note: --black-regex ${tempest_exclusion_regex} only seems to work from newton
+    ${cmd}=    Set Variable If    "${OPENSTACK_BRANCH}"=="stable/mitaka"    ostestr --regex ${tempest_regex}    ostestr --regex ${tempest_regex} --black-regex ${tempest_exclusion_regex}
+    ${results}=    Write Commands Until Prompt    ${cmd}    timeout=${timeout}
     Log    ${results}
     # Save stdout to file
     Create File    tempest_output_${tempest_regex}.log    data=${results}

diff --git a/csit/suites/openstack/tempest/tempest.robot b/csit/suites/openstack/tempest/tempest.robot
index 4fdd575cfec6a4653feb519bc1349cbdc3d14177..2db4c2e2535ce82d735fbe1ada7e55c8c1c2dac8 100644 (file)
--- a/csit/suites/openstack/tempest/tempest.robot
+++ b/csit/suites/openstack/tempest/tempest.robot
@@ -14,7 +14,7 @@ Resource          ../../../libraries/Utils.robot
 Variables         ../../../variables/Variables.py
 
 *** Variables ***
-${exclusion_regex}    'metering|test_l3_agent_scheduler.L3AgentSchedulerTestJSON|test_extensions.ExtensionsTestJSON.test_list_show_extensions|test_routers_dvr.RoutersTestDVR.test_centralized_router_update_to_dvr'
+${exclusion_regex}    'test_routers_negative.RoutersNegativeIpV6Test.test_router_set_gateway_used_ip_returns_409|test_routers_negative.RoutersNegativeTest.test_router_set_gateway_used_ip_returns_409'
 ${tempest_config_file}    /opt/stack/tempest/etc/tempest.conf
 ${external_net_name}    external-net
 ${external_subnet_name}    external-subnet
@@ -24,56 +24,70 @@ ${network_vlan_id}    167
 
 *** Test Cases ***
 tempest.api.network
-    ${TEST_NAME}    ${exclusion_regex}    ${tempest_config_file}
+    # mitaka release and earlier would fail on the RoutersNegative* tests, so skipping here and running
+    # explicitly in their own test cases which will skip_if_stable/mitaka is the openstack branch
+    ${TEST_NAME}    ${exclusion_regex}
+
+tempest.api.network.test_routers_negative.RoutersNegativeIpV6Test
+    [Tags]    skip_if_stable/mitaka
+    # OpenStack patch to fix this was merged in newton+ and will not be back-ported to mitaka
+    # https://review.openstack.org/#/c/219215
+    ${TEST_NAME}
+
+tempest.api.network.test_routers_negative.RoutersNegativeTest
+    [Tags]    skip_if_stable/mitaka
+    # OpenStack patch to fix this was merged in newton+ and will not be back-ported to mitaka
+    # https://review.openstack.org/#/c/219215
+    ${TEST_NAME}
 
 tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_connectivity_between_vms_on_different_networks
-    ${TEST_NAME}    ${exclusion_regex}    ${tempest_config_file}
+    ${TEST_NAME}
 
 tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_hotplug_nic
     [Tags]    skip_if_stable/mitaka
     # Failing due to default security rules behavior missing in Mitaka
-    ${TEST_NAME}    ${exclusion_regex}    ${tempest_config_file}
+    ${TEST_NAME}
 
 tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_mtu_sized_frames
-    ${TEST_NAME}    ${exclusion_regex}    ${tempest_config_file}
+    ${TEST_NAME}
 
 tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_network_basic_ops
-    ${TEST_NAME}    ${exclusion_regex}    ${tempest_config_file}
+    ${TEST_NAME}
 
 tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_preserve_preexisting_port
-    ${TEST_NAME}    ${exclusion_regex}    ${tempest_config_file}
+    ${TEST_NAME}
 
 tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_router_rescheduling
-    ${TEST_NAME}    ${exclusion_regex}    ${tempest_config_file}
+    ${TEST_NAME}
 
 tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_subnet_details
-    ${TEST_NAME}    ${exclusion_regex}    ${tempest_config_file}
+    ${TEST_NAME}
 
 tempest.scenario.test_aggregates_basic_ops.TestAggregatesBasicOps.test_aggregate_basic_ops
-    ${TEST_NAME}    ${exclusion_regex}    ${tempest_config_file}
+    ${TEST_NAME}
 
 tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_pause_unpause
-    ${TEST_NAME}    ${exclusion_regex}    ${tempest_config_file}
+    ${TEST_NAME}
 
 tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_stop_start
-    ${TEST_NAME}    ${exclusion_regex}    ${tempest_config_file}
+    ${TEST_NAME}
 
 tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_reboot
-    ${TEST_NAME}    ${exclusion_regex}    ${tempest_config_file}
+    ${TEST_NAME}
 
 tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_rebuild
-    ${TEST_NAME}    ${exclusion_regex}    ${tempest_config_file}
+    ${TEST_NAME}
 
 tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_suspend_resume
-    ${TEST_NAME}    ${exclusion_regex}    ${tempest_config_file}
+    ${TEST_NAME}
 
 tempest.scenario.test_server_basic_ops.TestServerBasicOps.test_server_basic_ops
-    ${TEST_NAME}    ${exclusion_regex}    ${tempest_config_file}
+    ${TEST_NAME}
 
 tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_port_security_macspoofing_port
     [Tags]    skip_if_transparent    skip_if_stable/mitaka
     # Failing due to default security rules behavior missing in Mitaka, and also in all transparent runs
-    ${TEST_NAME}    ${exclusion_regex}    ${tempest_config_file}
+    ${TEST_NAME}
 
 *** Keywords ***
 Log In To Tempest Executor And Setup Test Environment
@@ -98,9 +112,9 @@ Tempest Conf Add External Network
     [Arguments]    ${external_network_id}
     [Documentation]    Tempest will be run with a config file - this function will add the
     ...    given external network ID to the configuration file.
-    Modify Config In File On Existing SSH Connection    ${tempest_config_file}    set    network    public_network_id    ${external_network_id}
-    Modify Config In File On Existing SSH Connection    ${tempest_config_file}    set    DEFAULT    debug    False
-    Modify Config In File On Existing SSH Connection    ${tempest_config_file}    set    DEFAULT    log_level    INFO
+    Modify Config In File On Existing SSH Connection    set    network    public_network_id    ${external_network_id}
+    Modify Config In File On Existing SSH Connection    set    DEFAULT    debug    False
+    Modify Config In File On Existing SSH Connection    set    DEFAULT    log_level    INFO
     Write Commands Until Prompt    sudo cat ${tempest_config_file}
     Write Commands Until Prompt    sudo chmod 777 ${tempest_config_file}