Stabilize and improve cars-rest perf suite

- use a more strict prompt for checking when test tool is
  done. Some 500 error responses (which are ok when leaders
  are down) gave a > prompt which was being wrongly interpreted
  as the system prompt. now, making it look for ~]> instead.

- running the suites twice now, so we can see ask based and
  tell based protocols

- making the python client retry on 500 errors which can come
  when leaders are down and we have askTimeouts

- polling a little longer on some steps

Change-Id: I426f1b03f2fdd3258a39c03bd8e568f16beebf88
Signed-off-by: Jamo Luhrsen <jluhrsen@redhat.com>

diff --git a/csit/libraries/SSHKeywords.robot b/csit/libraries/SSHKeywords.robot
index c3f1078ed4016a2361acf5945dcbee3dcf3c8019..179933daaf3c8bb8afd66ae20b8a08ee40b482cd 100644 (file)
--- a/csit/libraries/SSHKeywords.robot
+++ b/csit/libraries/SSHKeywords.robot
@@ -30,9 +30,9 @@ Open_Connection_To_ODL_System
     [Return]    ${odl_connection}
 
 Open_Connection_To_Tools_System
-    [Arguments]    ${ip_address}=${TOOLS_SYSTEM_IP}    ${timeout}=10s
+    [Arguments]    ${ip_address}=${TOOLS_SYSTEM_IP}    ${timeout}=10s    ${prompt}=${TOOLS_SYSTEM_PROMPT}
     [Documentation]    Open a connection to the tools system at ${ip_address} and return its identifier.
-    ${tools_connection} =    SSHLibrary.Open_Connection    ${ip_address}    prompt=${TOOLS_SYSTEM_PROMPT}    timeout=${timeout}
+    ${tools_connection} =    SSHLibrary.Open_Connection    ${ip_address}    prompt=${prompt}    timeout=${timeout}
     Flexible_Mininet_Login
     [Return]    ${tools_connection}
 

diff --git a/csit/libraries/Utils.robot b/csit/libraries/Utils.robot
index 1e7baff52d6e6eb30b00dfd33d46fca98286b2f7..497dcb16d7de843edaa25a912971ee9d43cf200a 100644 (file)
--- a/csit/libraries/Utils.robot
+++ b/csit/libraries/Utils.robot
@@ -425,7 +425,9 @@ Get Log File Name
     ...    log files if they happen to run in one job.
     ${name}=    BuiltIn.Evaluate    """${SUITE_NAME}""".replace(" ","-").replace("/","-").replace(".","-")
     ${suffix}=    BuiltIn.Set_Variable_If    '${testcase}' != ''    --${testcase}    ${EMPTY}
-    [Return]    ${testtool}--${name}${suffix}.log
+    ${date} =    DateTime.Get Current Date
+    ${timestamp} =    DateTime.Convert Date    ${date}    epoch
+    [Return]    ${testtool}--${name}${suffix}.${timestamp}.log
 
 Set_User_Configurable_Variable_Default
     [Arguments]    ${name}    ${value}

diff --git a/csit/suites/controller/ThreeNodes_Datastore/puts_during_isolation.robot b/csit/suites/controller/ThreeNodes_Datastore/puts_during_isolation.robot
index e0cb183c84d89dc1be96421fbef8de83703a7d10..093a810445dae508290a5409649898a43da3f5c8 100644 (file)
--- a/csit/suites/controller/ThreeNodes_Datastore/puts_during_isolation.robot
+++ b/csit/suites/controller/ThreeNodes_Datastore/puts_during_isolation.robot
@@ -42,7 +42,7 @@ Start_Adding_Cars_To_Follower
     ${follower_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    member_index=${idx}
     Start Tool    ${ADDCMD}    --host ${follower_ip} ${TOOL_OPTIONS}
     ${session} =    Resolve_Http_Session_For_Member    member_index=${car_leader_index}
-    BuiltIn.Wait_Until_Keyword_Succeeds    5x    2s    Ensure_Cars_Being_Configured    ${session}
+    BuiltIn.Wait_Until_Keyword_Succeeds    10x    5s    Ensure_Cars_Being_Configured    ${session}
 
 Isolate_Current_Car_Leader
     [Documentation]    Isolating cluster node which is the car shard leader.
@@ -52,7 +52,7 @@ Isolate_Current_Car_Leader
 
 Verify_New_Car_Leader_Elected
     [Documentation]    Verify new owner of the car shard is elected.
-    BuiltIn.Wait_Until_Keyword_Succeeds    5x    2s    ClusterManagement.Verify_Shard_Leader_Elected    ${SHARD_NAME}    ${SHARD_TYPE}    ${True}
+    BuiltIn.Wait_Until_Keyword_Succeeds    10x    5s    ClusterManagement.Verify_Shard_Leader_Elected    ${SHARD_NAME}    ${SHARD_TYPE}    ${True}
     ...    ${old_car_leader}    member_index_list=${old_car_followers}
     CarPeople.Set_Tmp_Variables_For_Shard_For_Nodes    ${old_car_followers}    shard_name=${SHARD_NAME}    shard_type=${SHARD_TYPE}
 
@@ -79,7 +79,7 @@ Start Suite
     [Documentation]    Upload the script file and create a virtual env
     SetupUtils.Setup_Utils_For_Setup_And_Teardown
     SetupUtils.Setup_Logging_For_Debug_Purposes_On_List_Or_All    ${TEST_LOG_LEVEL}    ${TEST_LOG_COMPONENTS}
-    ${mininet_conn_id} =    SSHKeywords.Open_Connection_To_Tools_System
+    ${mininet_conn_id} =    SSHKeywords.Open_Connection_To_Tools_System    prompt=~]>
     Builtin.Set Suite Variable    ${mininet_conn_id}
     SSHLibrary.Put File    ${CURDIR}/../../../../tools/odl-mdsal-clustering-tests/scripts/${TOOL_NAME}    .
     ${stdout}    ${stderr}    ${rc}=    SSHLibrary.Execute Command    ls    return_stdout=True    return_stderr=True
@@ -93,6 +93,10 @@ Start Suite
 Stop Suite
     [Documentation]    Stop the tool, remove virtual env and close ssh connection towards tools vm.
     Stop_Tool
+    ${session} =    Resolve_Http_Session_For_Member    member_index=${new_leader_index}
+    # best effort to make sure cars are deleted in case more suites will run after this and the delete test case had trouble
+    ${rsp}=    RequestsLibrary.Delete Request    ${session}    ${CARURL}
+    BuiltIn.Log    ${rsp.status_code} : ${rsp.text}
     SSHKeywords.Virtual_Env_Delete
     Store_File_To_Workspace    ${out_file}    ${out_file}
     SSHLibrary.Close All Connections

diff --git a/csit/testplans/controller-rest-clust-cars-perf.txt b/csit/testplans/controller-rest-clust-cars-perf.txt
index a089fa35d26900f2f87333d4e131965ee3ca0d70..230db23b71136ec8874af6c87ee09194262eb96e 100644 (file)
--- a/csit/testplans/controller-rest-clust-cars-perf.txt
+++ b/csit/testplans/controller-rest-clust-cars-perf.txt
@@ -7,3 +7,6 @@
 # Place the suites in run order:
 integration/test/csit/suites/controller/ThreeNodes_Datastore/010_crud_mdsal_perf.robot
 integration/test/csit/suites/controller/ThreeNodes_Datastore/puts_during_isolation.robot
+integration/test/csit/suites/controller/dom_data_broker/restart_odl_with_tell_based_true.robot
+integration/test/csit/suites/controller/ThreeNodes_Datastore/010_crud_mdsal_perf.robot
+integration/test/csit/suites/controller/ThreeNodes_Datastore/puts_during_isolation.robot

diff --git a/tools/odl-mdsal-clustering-tests/scripts/cluster_rest_script.py b/tools/odl-mdsal-clustering-tests/scripts/cluster_rest_script.py
index 4136a0304f6538c8e2f95ba55288b7c08cd280f9..3731d0c516bb9ee0d12be815715d5be959d97297 100644 (file)
--- a/tools/odl-mdsal-clustering-tests/scripts/cluster_rest_script.py
+++ b/tools/odl-mdsal-clustering-tests/scripts/cluster_rest_script.py
@@ -589,7 +589,7 @@ def add_car_with_retries(odl_ip, port, thread_count, item_count, auth, items_per
 
     logger.info("Add %s car(s) to %s:%s (%s per request)",
                 item_count, odl_ip, port, items_per_request)
-    retry_rcs = [401, 404, 503]
+    retry_rcs = [401, 404, 500, 503]
     res = _task_executor(_prepare_add_car, odl_ip=odl_ip, port=port,
                          thread_count=thread_count, item_count=item_count,
                          items_per_request=items_per_request, auth=auth,