From 2b751d17ffc128da388de181b8f7a5748ccd5d54 Mon Sep 17 00:00:00 2001 From: Vratko Polak Date: Wed, 21 Sep 2016 13:46:10 +0200 Subject: [PATCH] Many library improvements. Major changes: + Introduced SSHKywords.Run_Keyword_Preserve_Connection. + Most ClusterManagement keywords should not affect active SSH connection anymore. Minor improvements: + CarPeople.Set_Variables_For_Shard also sets \${${shard_name}_first_follower_index}. + ClusterManagement variables sorted. + ${KARAF_HOME} constructed in ClusterManagement. + ClustwerManagement.Get_Owner_And_Candidates_For_Type_And_Id added. + Start_Members_From_List_Or_All checks sync only once per 10 seconds. + Renamed Run_Command_On_* to Run_Bash_Command_On_*. + Introduced Run_Karaf_Command_On_*. + Clean_Directories_* and Install_Feature_* addd to ClusterManagement. + (Unsafe) With_Ssh_To_List_Or_All_Run_Keyword and Safe_* added. + Restore_Current_Ssh_Connection_From_Index moved from KarafKeywords to SSHKeywords. + Introduced KarafKeywords.Safe_Issue_Command_On_Karaf_Console. + SSHKeywords.Open_Connection_To_* take ${ip_address} as an optional argument. + Karaf EOF workaround added to SetiupUtils.Setup_Utils_For_Setup_And_Teardown. Change-Id: Ia61abe2bdaa15f90e3042742275ddbd45b4fc72d Signed-off-by: Vratko Polak --- csit/libraries/CarPeople.robot | 2 + csit/libraries/ClusterManagement.robot | 143 +++++++++++++++++++++---- csit/libraries/KarafKeywords.robot | 44 +++----- csit/libraries/NexusKeywords.robot | 4 +- csit/libraries/SSHKeywords.robot | 53 +++++++-- csit/libraries/SetupUtils.robot | 3 +- csit/libraries/Utils.robot | 10 +- 7 files changed, 190 insertions(+), 69 deletions(-) diff --git a/csit/libraries/CarPeople.robot b/csit/libraries/CarPeople.robot index 1f326c416f..3ff015d21c 100644 --- a/csit/libraries/CarPeople.robot +++ b/csit/libraries/CarPeople.robot @@ -49,6 +49,8 @@ Set_Variables_For_Shard ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} BuiltIn.Set_Suite_Variable \${${shard_name}_leader_index} ${leader} BuiltIn.Set_Suite_Variable \${${shard_name}_follower_indices} ${follower_list} + ${first_follower_index} = Collections.Get_From_List ${follower_list} 0 + BuiltIn.Set_Suite_Variable \${${shard_name}_first_follower_index} ${first_follower_index} ${leader_session} = ClusterManagement.Resolve_Http_Session_For_Member member_index=${leader} BuiltIn.Set_Suite_Variable \${${shard_name}_leader_session} ${leader_session} ${sessions} = BuiltIn.Create_List diff --git a/csit/libraries/ClusterManagement.robot b/csit/libraries/ClusterManagement.robot index b54fd3f8a1..591cb6d5a5 100644 --- a/csit/libraries/ClusterManagement.robot +++ b/csit/libraries/ClusterManagement.robot @@ -32,15 +32,19 @@ Documentation Resource housing Keywords common to several suites for cluster Library RequestsLibrary # for Create_Session and To_Json Library Collections Resource ${CURDIR}/CompareStream.robot +Resource ${CURDIR}/KarafKeywords.robot +Resource ${CURDIR}/SSHKeywords.robot Resource ${CURDIR}/TemplatedRequests.robot # for Get_As_Json_From_Uri Resource ${CURDIR}/Utils.robot # for Run_Command_On_Controller *** Variables *** +${ENTITY_OWNER_URI} restconf/operational/entity-owners:entity-owners ${JAVA_HOME} ${EMPTY} # releng/builder scripts should provide correct value ${JOLOKIA_CONF_SHARD_MANAGER_URI} jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore ${JOLOKIA_OPER_SHARD_MANAGER_URI} jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-operational,type=DistributedOperationalDatastore ${JOLOKIA_READ_URI} jolokia/read/org.opendaylight.controller -${ENTITY_OWNER_URI} restconf/operational/entity-owners:entity-owners +${KARAF_HOME} ${WORKSPACE}${/}${BUNDLEFOLDER} # TODO: Migrate to Variables.robot +@{ODL_DEFAULT_DATA_PATHS} tmp/ data/ cache/ snapshots/ journal/ etc/opendaylight/current/ ${RESTCONF_MODULES_DIR} ${CURDIR}/../variables/restconf/modules *** Keywords *** @@ -169,6 +173,7 @@ Get_Owner_And_Candidates_For_Device ... The returned candidate list is sorted numerically. ... Note that "candidate list" definition currently differs between Beryllium and Boron. ... It is recommended to use Get_Owner_And_Successors_For_Device instead of this keyword, see documentation there. + BuiltIn.Comment TODO: Can this implementation be changed to call Get_Owner_And_Candidates_For_Type_And_Id? ${session} = Resolve_Http_Session_For_Member member_index=${member_index} ${data} = TemplatedRequests.Get_As_Json_From_Uri uri=${ENTITY_OWNER_URI} session=${session} ${candidate_list} = BuiltIn.Create_List @@ -196,6 +201,40 @@ Get_Owner_And_Candidates_For_Device Collections.Sort_List ${candidate_list} [Return] ${owner} ${candidate_list} +Get_Owner_And_Candidates_For_Type_And_Id + [Arguments] ${type} ${id} ${member_index} ${require_candidate_list}=${EMPTY} + [Documentation] Returns the owner and a list of candidates for entity specified by ${type} and ${id} + ... Request is sent to member ${member_index}. + ... Candidates are all members that register to own a device, so the list of candiates includes the owner. + ... Bear in mind that for Boron and beyond, candidates are not removed on node down or isolation. + ... If ${require_candidate_list} is not \${EMPTY}, check whether the actual list of candidates matches. + ... Note that differs from "given list" semantics used in other keywords, + ... namely you cannot use \${EMPTY} to stand for "full list" in this keyword. + BuiltIn.Comment TODO: Find a way to unify and deduplicate code blocks in Get_Owner_And_Candidates_* keywords. + ${session} = Resolve_Http_Session_For_Member member_index=${member_index} + ${data} = TemplatedRequests.Get_As_Json_From_Uri uri=${ENTITY_OWNER_URI} session=${session} + ${candidate_list} = BuiltIn.Create_List + ${json} = RequestsLibrary.To_Json ${data} + ${entity_type_list} = Collections.Get_From_Dictionary &{json}[entity-owners] entity-type + ${entity_type_index} = Utils.Get_Index_From_List_Of_Dictionaries ${entity_type_list} type ${type} + BuiltIn.Should_Not_Be_Equal ${entity_type_index} -1 No Entity Owner found for ${type} + ${entity_list} = Collections.Get_From_Dictionary @{entity_type_list}[${entity_type_index}] entity + ${entity_index} = Utils.Get_Index_From_List_Of_Dictionaries ${entity_list} id ${id} + BuiltIn.Should Not Be Equal ${entity_index} -1 Id ${id} not found in Entity Owner ${type} + ${entity_owner} = Collections.Get_From_Dictionary @{entity_list}[${entity_index}] owner + BuiltIn.Should_Not_Be_Empty ${entity_owner} No owner found for type=${type} id=${id} + ${owner} = String.Replace_String ${entity_owner} member- ${EMPTY} + ${owner} = BuiltIn.Convert_To_Integer ${owner} + ${entity_candidates_list} = Collections.Get_From_Dictionary @{entity_list}[${entity_index}] candidate + : FOR ${entity_candidate} IN @{entity_candidates_list} + \ ${candidate} = String.Replace_String &{entity_candidate}[name] member- ${EMPTY} + \ ${candidate} = BuiltIn.Convert_To_Integer ${candidate} + \ Collections.Append_To_List ${candidate_list} ${candidate} + Collections.Sort_List ${candidate_list} + BuiltIn.Comment TODO: Separate check lines into Verify_Owner_And_Candidates_For_Type_And_Id + BuiltIn.Run_Keyword_If """${require_candidate_list}""" BuiltIn.Should_Be_Equal ${require_candidate_list} ${candidate_list} Candidate list does not match: ${candidate_list} is not ${require_candidate_list} + [Return] ${owner} ${candidate_list} + Extract_Service_Entity_Type [Arguments] ${data} [Documentation] Remove superfluous device data from Entity Owner printout. @@ -239,7 +278,7 @@ Kill_Members_From_List_Or_All ${kill_index_list} = ClusterManagement__Given_Or_Internal_Index_List given_list=${member_index_list} ${index_list} = ClusterManagement__Given_Or_Internal_Index_List given_list=${original_index_list} ${command} = BuiltIn.Set_Variable ps axf | grep karaf | grep -v grep | awk '{print \"kill -9 \" $1}' | sh - Run_Command_On_List_Or_All command=${command} member_index_list=${member_index_list} + Run_Bash_Command_On_List_Or_All command=${command} member_index_list=${member_index_list} ${updated_index_list} = BuiltIn.Create_List @{index_list} Collections.Remove_Values_From_List ${updated_index_list} @{kill_index_list} BuiltIn.Return_From_Keyword_If not ${confirm} ${updated_index_list} @@ -256,24 +295,24 @@ Start_Single_Member Start_Members_From_List_Or_All ${index_list} ${wait_for_sync} ${timeout} Start_Members_From_List_Or_All - [Arguments] ${member_index_list}=${EMPTY} ${wait_for_sync}=True ${timeout}=300s ${karaf_home}=${WORKSPACE}${/}${BUNDLEFOLDER} ${export_java_home}=${JAVA_HOME} + [Arguments] ${member_index_list}=${EMPTY} ${wait_for_sync}=True ${timeout}=300s ${karaf_home}=${KARAF_HOME} ${export_java_home}=${JAVA_HOME} [Documentation] If the list is empty, start all cluster members. Otherwise, start members based on present indices. ... If ${wait_for_sync}, wait for cluster sync on listed members. ... Optionally karaf_home can be overriden. Optionally specific JAVA_HOME is used for starting. ${base_command} = BuiltIn.Set_Variable ${karaf_home}/bin/start ${command} = BuiltIn.Set_Variable_If "${export_java_home}" export JAVA_HOME="${export_java_home}"; ${base_command} ${base_command} - Run_Command_On_List_Or_All command=${command} member_index_list=${member_index_list} + Run_Bash_Command_On_List_Or_All command=${command} member_index_list=${member_index_list} BuiltIn.Return_From_Keyword_If not ${wait_for_sync} - BuiltIn.Wait_Until_Keyword_Succeeds ${timeout} 1s Check_Cluster_Is_In_Sync member_index_list=${member_index_list} + BuiltIn.Wait_Until_Keyword_Succeeds ${timeout} 10s Check_Cluster_Is_In_Sync member_index_list=${member_index_list} # TODO: Do we also want to check Shard Leaders here? Clean_Journals_And_Snapshots_On_List_Or_All - [Arguments] ${member_index_list}=${EMPTY} ${karaf_home}=${WORKSPACE}${/}${BUNDLEFOLDER} + [Arguments] ${member_index_list}=${EMPTY} ${karaf_home}=${KARAF_HOME} [Documentation] Delete journal and snapshots directories on every node listed (or all). ${index_list} = ClusterManagement__Given_Or_Internal_Index_List given_list=${member_index_list} ${command} = Set Variable rm -rf "${karaf_home}/journal" "${karaf_home}/snapshots" : FOR ${index} IN @{index_list} # usually: 1, 2, 3. - \ Run_Command_On_Member command=${command} member_index=${index} + \ Run_Bash_Command_On_Member command=${command} member_index=${index} Verify_Karaf_Is_Not_Running_On_Member [Arguments] ${member_index} @@ -291,7 +330,7 @@ Count_Running_Karafs_On_Member [Arguments] ${member_index} [Documentation] Remotely execute grep for karaf process, return count as string. ${command} = BuiltIn.Set_Variable ps axf | grep karaf | grep -v grep | wc -l - ${count} = Run_Command_On_Member command=${command} member_index=${member_index} + ${count} = Run_Bash_Command_On_Member command=${command} member_index=${member_index} [Return] ${count} Isolate_Member_From_List_Or_All @@ -303,9 +342,9 @@ Isolate_Member_From_List_Or_All : FOR ${index} IN @{index_list} \ ${destination} = Collections.Get_From_Dictionary ${ClusterManagement__index_to_ip_mapping} ${index} \ ${command} = BuiltIn.Set_Variable sudo /sbin/iptables -I OUTPUT -p all --source ${source} --destination ${destination} -j DROP - \ BuiltIn.Run_Keyword_If "${index}" != "${isolate_member_index}" Run_Command_On_Member command=${command} member_index=${isolate_member_index} + \ BuiltIn.Run_Keyword_If "${index}" != "${isolate_member_index}" Run_Bash_Command_On_Member command=${command} member_index=${isolate_member_index} ${command} = BuiltIn.Set_Variable sudo /sbin/iptables -L -n - ${output} = Run_Command_On_Member command=${command} member_index=${isolate_member_index} + ${output} = Run_Bash_Command_On_Member command=${command} member_index=${isolate_member_index} BuiltIn.Log ${output} ${updated_index_list} = BuiltIn.Create_List @{index_list} Collections.Remove_Values_From_List ${updated_index_list} ${isolate_member_index} @@ -319,39 +358,101 @@ Rejoin_Member_From_List_Or_All : FOR ${index} IN @{index_list} \ ${destination} = Collections.Get_From_Dictionary ${ClusterManagement__index_to_ip_mapping} ${index} \ ${command} = BuiltIn.Set_Variable sudo /sbin/iptables -D OUTPUT -p all --source ${source} --destination ${destination} -j DROP - \ BuiltIn.Run_Keyword_If "${index}" != "${rejoin_member_index}" Run_Command_On_Member command=${command} member_index=${rejoin_member_index} + \ BuiltIn.Run_Keyword_If "${index}" != "${rejoin_member_index}" Run_Bash_Command_On_Member command=${command} member_index=${rejoin_member_index} ${command} = BuiltIn.Set_Variable sudo /sbin/iptables -L -n - ${output} = Run_Command_On_Member command=${command} member_index=${rejoin_member_index} + ${output} = Run_Bash_Command_On_Member command=${command} member_index=${rejoin_member_index} BuiltIn.Log ${output} Flush_Iptables_From_List_Or_All [Arguments] ${member_index_list}=${EMPTY} [Documentation] If the list is empty, flush IPTables in all ODL instances. Otherwise, flush member based on present indices. ${command} = BuiltIn.Set_Variable sudo iptables -v -F - ${output} = Run_Command_On_List_Or_All command=${command} member_index_list=${member_index_list} + ${output} = Run_Bash_Command_On_List_Or_All command=${command} member_index_list=${member_index_list} -Run_Command_On_List_Or_All +Run_Bash_Command_On_List_Or_All [Arguments] ${command} ${member_index_list}=${EMPTY} [Documentation] Cycle through indices (or all), run command on each. ${index_list} = ClusterManagement__Given_Or_Internal_Index_List given_list=${member_index_list} : FOR ${index} IN @{index_list} - \ Run_Command_On_Member command=${command} member_index=${index} + \ Run_Bash_Command_On_Member command=${command} member_index=${index} + +Run_Bash_Command_On_Member + [Arguments] ${command} ${member_index} + [Documentation] Obtain IP, call Utils and return output. This does not preserve active ssh session. + # TODO: Rename these keyword to Run_Bash_Command_On_Member to distinguish from Karaf (or even Windows) commands. + ${member_ip} = Collections.Get_From_Dictionary dictionary=${ClusterManagement__index_to_ip_mapping} key=${member_index} + ${output} = SSHKeywords.Run_Keyword_Preserve_Connection Utils.Run_Command_On_Controller ${member_ip} ${command} + [Return] ${output} Run_Karaf_Command_On_List_Or_All - [Arguments] ${command} ${member_index_list}=${EMPTY} + [Arguments] ${command} ${member_index_list}=${EMPTY} ${timeout}=10s [Documentation] Cycle through indices (or all), run karaf command on each. ${index_list} = ClusterManagement__Given_Or_Internal_Index_List given_list=${member_index_list} : FOR ${index} IN @{index_list} \ ${member_ip} = Collections.Get_From_Dictionary dictionary=${ClusterManagement__index_to_ip_mapping} key=${index} - \ KarafKeywords.Issue Command On Karaf Console ${command} ${member_ip} + \ KarafKeywords.Safe_Issue_Command_On_Karaf_Console ${command} ${member_ip} timeout=${timeout} -Run_Command_On_Member - [Arguments] ${command} ${member_index} - [Documentation] Obtain IP, call Utils and return output. This does not preserve active ssh session. +Run_Karaf_Command_On_Member + [Arguments] ${command} ${member_index} ${timeout}=10s + [Documentation] Obtain IP address, call KarafKeywords and return output. This does not preserve active ssh session. + ... This keyword is not used by Run_Karaf_Command_On_List_Or_All, but returned output may be useful. ${member_ip} = Collections.Get_From_Dictionary dictionary=${ClusterManagement__index_to_ip_mapping} key=${member_index} - ${output} = Utils.Run_Command_On_Controller ${member_ip} ${command} + ${output} = KarafKeywords.Safe_Issue_Command_On_Karaf_Console ${command} controller=${member_ip} timeout=${timeout} + [Return] ${output} + +Install_Feature_On_List_Or_All + [Arguments] ${feature_name} ${member_index_list}=${EMPTY} ${timeout}=60s + [Documentation] Attempt installation on each member from list (or all). Then look for failures. + ${index_list} = ClusterManagement__Given_Or_Internal_Index_List given_list=${member_index_list} + ${status_list} = BuiltIn.Create_List + : FOR ${index} IN @{index_list} + \ ${status} ${text} = BuiltIn.Run_Keyword_And_Ignore_Error Install_Feature_On_Member feature_name=${feature_name} member_index=${index} + \ ... timeout=${timeout} + \ BuiltIn.Log ${text} + \ Collections.Append_To_List ${status_list} ${status} + : FOR ${status} IN @{status_list} + \ BuiltIn.Run_Keyword_If "${status}" != "PASS" BuiltIn.Fail ${feature_name} installation failed, see log. + +Install_Feature_On_Member + [Arguments] ${feature_name} ${member_index} ${timeout}=60s + [Documentation] Run feature:install karaf command, fail if installation was not successful. Return output. + ${status} ${output} = BuiltIn.Run_Keyword_And_Ignore_Error Run_Karaf_Command_On_Member command=feature:install ${feature_name} member_index=${member_index} timeout=${timeout} + BuiltIn.Run_Keyword_If "${status}" != "PASS" BuiltIn.Fail Failed to install ${feature_name}: ${output} + BuiltIn.Should_Not_Contain ${output} Can't install Failed to install ${feature_name}: ${output} [Return] ${output} +With_Ssh_To_List_Or_All_Run_Keyword + [Arguments] ${member_index_list} ${keyword_name} @{args} &{kwargs} + [Documentation] For each index in given list (or all): activate SSH connection, run given Keyword, close active connection. Return None. + ... Note that if the Keyword affects SSH connections, results are still deterministic, but perhaps undesirable. + ... Beware that in order to avoid "got positional argument after named arguments", first two arguments in the call should not be named. + BuiltIn.Comment This keyword is experimental and there is high risk of being replaced by another approach. + # TODO: For_Index_From_List_Or_All_Run_Keyword applied to With_Ssh_To_Member_Run_Keyword? + ${index_list} = ClusterManagement__Given_Or_Internal_Index_List given_list=${member_index_list} + : FOR ${member_index} IN @{index_list} + \ ${member_ip} = Resolve_IP_Address_For_Member ${member_index} + \ SSHKeywords.Open_Connection_To_Odl_System ip_address=${member_ip} + \ BuiltIn.Run_Keyword ${keyword_name} @{args} &{kwargs} + \ SSHLibrary.Close_Connection + +Safe_With_Ssh_To_List_Or_All_Run_Keyword + [Arguments] ${member_index_list} ${keyword_name} @{args} &{kwargs} + [Documentation] Remember active ssh connection index, call With_Ssh_To_List_Or_All_Run_Keyword, return None. Restore the conection index on teardown. + SSHKeywords.Run_Keyword_Preserve_Connection With_Ssh_To_List_Or_All_Run_Keyword ${member_index_list} ${keyword_name} @{args} &{kwargs} + +Clean_Directories_On_List_Or_All + [Arguments] ${member_index_list}=${EMPTY} ${directory_list}=${EMPTY} ${karaf_home}=${KARAF_HOME} + [Documentation] Clear @{directory_list} or @{ODL_DEFAULT_DATA_PATHS} for members in given list or all. Return None. + ... This is intended to return Karaf (offline) to the state it was upon the first boot. + ${path_list} = Builtin.Set Variable If "${directory_list}" == "${EMPTY}" ${ODL_DEFAULT_DATA_PATHS} ${directory_list} + Safe_With_Ssh_To_List_Or_All_Run_Keyword ${member_index_list} ClusterManagement__Clean_Directories ${path_list} ${karaf_home} + +ClusterManagement__Clean_Directories + [Arguments] ${relative_path_list} ${karaf_home} + [Documentation] For each relative path, remove files with respect to ${karaf_home}. Return None. + : FOR ${relative_path} IN @{relative_path_list} + \ SSHLibrary.Execute_Command rm -rf ${karaf_home}${/}${relative_path} + Put_As_Json_And_Check_Member_List_Or_All [Arguments] ${uri} ${data} ${member_index} ${member_index_list}=${EMPTY} [Documentation] Send a PUT with the supplied uri ${uri} and body ${data} to member ${member_index}. diff --git a/csit/libraries/KarafKeywords.robot b/csit/libraries/KarafKeywords.robot index d1d28eef16..dd79850fae 100644 --- a/csit/libraries/KarafKeywords.robot +++ b/csit/libraries/KarafKeywords.robot @@ -2,7 +2,8 @@ Documentation Karaf library. This library is useful to deal with controller Karaf console. Library SSHLibrary Library OperatingSystem -Variables ../variables/Variables.py +Resource ${CURDIR}/SSHKeywords.robot +Variables ${CURDIR}/../variables/Variables.py *** Variables *** ${WORKSPACE} /tmp @@ -18,7 +19,7 @@ Verify Feature Is Installed Issue Command On Karaf Console [Arguments] ${cmd} ${controller}=${ODL_SYSTEM_IP} ${karaf_port}=${KARAF_SHELL_PORT} ${timeout}=5 ${loglevel}=INFO - [Documentation] Will execute the given ${cmd} by ssh'ing to the karaf console running on ${ODL_SYSTEM_IP} + [Documentation] Will execute the given ${cmd} by ssh'ing to the karaf console running on ${controller} ... Note that this keyword will open&close new SSH connection, without switching back to previously current session. Open Connection ${controller} port=${karaf_port} prompt=${KARAF_PROMPT} timeout=${timeout} Login ${KARAF_USER} ${KARAF_PASSWORD} loglevel=${loglevel} @@ -28,6 +29,12 @@ Issue Command On Karaf Console Log ${output} [Return] ${output} +Safe_Issue_Command_On_Karaf_Console + [Arguments] ${cmd} ${controller}=${ODL_SYSTEM_IP} ${karaf_port}=${KARAF_SHELL_PORT} ${timeout}=5 ${loglevel}=INFO + [Documentation] Run Issue_Command_On_Karaf_Console but restore previous connection afterwards. + BuiltIn.Run_Keyword_And_Return SSHKeywords.Run_Keyword_Preserve_Connection Issue_Command_On_Karaf_Console ${cmd} ${controller} ${karaf_port} ${timeout} + ... ${loglevel} + Check For Elements On Karaf Command Output Message [Arguments] ${cmd} ${elements} ${controller}=${ODL_SYSTEM_IP} ${karaf_port}=${KARAF_SHELL_PORT} ${timeout}=5 [Documentation] Will execute the command using Issue Command On Karaf Console then check for the given elements @@ -80,33 +87,6 @@ Uninstall a Feature Log ${output} [Return] ${output} -Restore Current SSH Connection From Index - [Arguments] ${connection_index} - [Documentation] Restore active SSH connection in SSHLibrary to given index. - ... - ... Restore the currently active connection state in - ... SSHLibrary to match the state returned by "Switch - ... Connection" or "Get Connection". More specifically makes - ... sure that there will be no active connection when the - ... \${connection_index} reported by these means is None. - ... - ... There is a misfeature in SSHLibrary: Invoking "SSHLibrary.Switch_Connection" - ... and passing None as the "index_or_alias" argument to it has exactly the - ... same effect as invoking "Close Connection". - ... https://github.com/robotframework/SSHLibrary/blob/master/src/SSHLibrary/library.py#L560 - ... - ... We want to have Keyword which will "switch out" to previous - ... "no connection active" state without killing the background one. - ... - ... As some suites may hypothetically rely on non-writability of active connection, - ... workaround is applied by opening and closing temporary connection. - ... Unfortunately this will fail if run on Jython and there is no SSH server - ... running on localhost, port 22 but there is nothing easy that can be done about it. - BuiltIn.Run Keyword And Return If ${connection_index} is not None SSHLibrary.Switch Connection ${connection_index} - # The background connection is still current, bury it. - SSHLibrary.Open Connection 127.0.0.1 - SSHLibrary.Close Connection - Open Controller Karaf Console On Background [Documentation] Connect to the controller's karaf console, but do not switch to it. ${current_ssh_connection}= SSHLibrary.Get Connection @@ -114,7 +94,7 @@ Open Controller Karaf Console On Background ${karaf_connection}= SSHLibrary.Get Connection SSHLibrary.Login ${KARAF_USER} ${KARAF_PASSWORD} BuiltIn.Set Suite Variable ${KarafKeywords__karaf_connection_index} ${karaf_connection.index} - [Teardown] Restore Current SSH Connection From Index ${current_ssh_connection.index} + [Teardown] SSHKeywords.Restore Current SSH Connection From Index ${current_ssh_connection.index} Configure Timeout For Karaf Console [Arguments] ${timeout} @@ -122,7 +102,7 @@ Configure Timeout For Karaf Console BuiltIn.Run Keyword If ${KarafKeywords__karaf_connection_index} == -1 Fail Need to connect to a Karaf Console first ${current_connection_index}= SSHLibrary.Switch Connection ${KarafKeywords__karaf_connection_index} SSHLibrary.Set_Client_Configuration timeout=${timeout} - [Teardown] Restore Current SSH Connection From Index ${current_connection_index} + [Teardown] SshKeywords.Restore Current SSH Connection From Index ${current_connection_index} Execute Controller Karaf Command On Background [Arguments] ${command} @@ -135,7 +115,7 @@ Execute Controller Karaf Command On Background BuiltIn.Run Keyword If '${status_write}' != 'PASS' BuiltIn.Fail Failed to send the command: ${command} BuiltIn.Log ${message_wait} BuiltIn.Run Keyword If '${status_wait}' != 'PASS' BuiltIn.Fail Failed to see prompt after sending the command: ${command} - [Teardown] Restore Current SSH Connection From Index ${current_connection_index} + [Teardown] SshKeywords.Restore Current SSH Connection From Index ${current_connection_index} [Return] ${message_wait} Execute Controller Karaf Command With Retry On Background diff --git a/csit/libraries/NexusKeywords.robot b/csit/libraries/NexusKeywords.robot index 541619e5e9..62ec47273b 100644 --- a/csit/libraries/NexusKeywords.robot +++ b/csit/libraries/NexusKeywords.robot @@ -17,7 +17,7 @@ Documentation Nexus repository access keywords, and supporting Java and Mave Library OperatingSystem Library SSHLibrary Library String -Resource SSHKeywords.robot +Resource ${CURDIR}/SSHKeywords.robot *** Variables *** ${JDKVERSION} None @@ -71,7 +71,7 @@ NexusKeywords__Detect_Version_To_Pull SSHKeywords.Open_Connection_To_ODL_System ${version} ${result} = SSHLibrary.Execute_Command sh search.sh ${WORKSPACE}/${BUNDLEFOLDER}/system ${itemlist} return_rc=True SSHLibrary.Close_Connection - Restore Current SSH Connection From Index ${current_ssh_connection.index} + SSHKeywords.Restore Current SSH Connection From Index ${current_ssh_connection.index} BuiltIn.Log ${version} BuiltIn.Run_Keyword_If ${result}!=0 BuiltIn.Fail Component "${component}" not found, cannot locate test tool ${version} ${location} = String.Split_String ${version} max_split=1 diff --git a/csit/libraries/SSHKeywords.robot b/csit/libraries/SSHKeywords.robot index 83e551466e..edd72f770f 100644 --- a/csit/libraries/SSHKeywords.robot +++ b/csit/libraries/SSHKeywords.robot @@ -14,7 +14,7 @@ Documentation Resource enhancing SSHLibrary with Keywords used in multiple s ... you can place them here. ... ... TODO: Migrate Keywords related to handling SSH here. -... That may include Utils.Flexible_SSH_Login, KarafKeywords.Restore_Current_SSH_Connection_From_Index and similar. +... That may include Utils.Flexible_SSH_Login, and similar. Library SSHLibrary Resource ${CURDIR}/Utils.robot @@ -24,17 +24,54 @@ ${SSHKeywords__current_venv_path} /tmp/defaultvenv *** Keywords *** Open_Connection_To_ODL_System - [Documentation] Open a connection to the ODL system and return its identifier. - ... On clustered systems this opens the connection to the first node. - ${odl} = SSHLibrary.Open_Connection ${ODL_SYSTEM_IP} prompt=${ODL_SYSTEM_PROMPT} timeout=10s + [Arguments] ${ip_address}=${ODL_SYSTEM_IP} + [Documentation] Open a connection to the ODL system at ${ip_address} and return its identifier. + ${odl_connection} = SSHLibrary.Open_Connection ${ip_address} prompt=${ODL_SYSTEM_PROMPT} timeout=10s Utils.Flexible_Controller_Login - [Return] ${odl} + [Return] ${odl_connection} Open_Connection_To_Tools_System - [Documentation] Open a connection to the tools system and return its identifier. - ${tools} = SSHLibrary.Open_Connection ${TOOLS_SYSTEM_IP} prompt=${TOOLS_SYSTEM_PROMPT} + [Arguments] ${ip_address}=${TOOLS_SYSTEM_IP} + [Documentation] Open a connection to the tools system at ${ip_address} and return its identifier. + ${tools_connection} = SSHLibrary.Open_Connection ${ip_address} prompt=${TOOLS_SYSTEM_PROMPT} Utils.Flexible_Mininet_Login - [Return] ${tools} + [Return] ${tools_connection} + +Restore_Current_Ssh_Connection_From_Index + [Arguments] ${connection_index} + [Documentation] Restore active SSH connection in SSHLibrary to given index. + ... + ... Restore the currently active connection state in + ... SSHLibrary to match the state returned by "Switch + ... Connection" or "Get Connection". More specifically makes + ... sure that there will be no active connection when the + ... \${connection_index} reported by these means is None. + ... + ... There is a misfeature in SSHLibrary: Invoking "SSHLibrary.Switch_Connection" + ... and passing None as the "index_or_alias" argument to it has exactly the + ... same effect as invoking "Close Connection". + ... https://github.com/robotframework/SSHLibrary/blob/master/src/SSHLibrary/library.py#L560 + ... + ... We want to have Keyword which will "switch out" to previous + ... "no connection active" state without killing the background one. + ... + ... As some suites may hypothetically rely on non-writability of active connection, + ... workaround is applied by opening and closing temporary connection. + ... Unfortunately this will fail if run on Jython and there is no SSH server + ... running on localhost, port 22 but there is nothing easy that can be done about it. + BuiltIn.Run Keyword And Return If ${connection_index} is not None SSHLibrary.Switch Connection ${connection_index} + # The background connection is still current, bury it. + SSHLibrary.Open Connection 127.0.0.1 + SSHLibrary.Close Connection + +Run_Keyword_Preserve_Connection + [Arguments] ${keyword_name} @{args} &{kwargs} + [Documentation] Store current connection index, run keyword returning its result, restore connection in teardown. + ... Note that in order to avoid "got positional argument after named arguments", it is safer to use positional (not named) arguments on call. + ${current_connection}= SSHLibrary.Get_Connection + BuiltIn.Run_Keyword_And_Return ${keyword_name} @{args} &{kwargs} + # Resource name has to be prepended, as KarafKeywords still contains a redirect. + [Teardown] SSHKeywords.Restore_Current_SSH_Connection_From_Index ${current_connection.index} Log_Command_Results [Arguments] ${stdout} ${stderr} ${rc} diff --git a/csit/libraries/SetupUtils.robot b/csit/libraries/SetupUtils.robot index 9e1152ff37..766f5556d8 100644 --- a/csit/libraries/SetupUtils.robot +++ b/csit/libraries/SetupUtils.robot @@ -13,7 +13,8 @@ ${SetupUtils__Known_Bug_ID} ${EMPTY} Setup_Utils_For_Setup_And_Teardown [Documentation] Prepare both FailFast and karaf logging, to be used in suite setup. FailFast.Do_Not_Fail_Fast_From_Now_On - KarafKeywords.Open_Controller_Karaf_Console_On_Background + BuiltIn.Comment First connections to Karaf console may fail, so WUKS is used. TODO: Track as a Bug. + BuiltIn.Wait_Until_Keyword_Succeeds 3x 0.2s KarafKeywords.Open_Controller_Karaf_Console_On_Background BuiltIn.Run Keyword And Ignore Error KarafKeywords.Log_Test_Suite_Start_To_Controller_Karaf Setup_Test_With_Logging_And_Fast_Failing diff --git a/csit/libraries/Utils.robot b/csit/libraries/Utils.robot index 350c165bf7..61612a9013 100644 --- a/csit/libraries/Utils.robot +++ b/csit/libraries/Utils.robot @@ -6,10 +6,10 @@ Library DateTime Library Process Library Collections Library RequestsLibrary -Library ./UtilLibrary.py -Resource KarafKeywords.robot -Resource TemplatedRequests.robot -Variables ../variables/Variables.py +Library ${CURDIR}/UtilLibrary.py +Resource ${CURDIR}/SSHKeywords.robot +Resource ${CURDIR}/TemplatedRequests.robot +Variables ${CURDIR}/../variables/Variables.py *** Variables *** # TODO: Introduce ${tree_size} and use instead of 1 in the next line. @@ -241,7 +241,7 @@ Run Command On Remote System ${stdout} ${stderr} SSHLibrary.Execute Command ${cmd} return_stderr=True SSHLibrary.Close Connection Log ${stderr} - [Teardown] KarafKeywords.Restore_Current_SSH_Connection_From_Index ${current_ssh_connection.index} + [Teardown] SSHKeywords.Restore_Current_SSH_Connection_From_Index ${current_ssh_connection.index} [Return] ${stdout} Write_Bare_Ctrl_C -- 2.36.6