+ # TODO: Migrate callers to Check_Bash_Command_*
+ ${member_ip} = Collections.Get_From_Dictionary
+ ... dictionary=${ClusterManagement__index_to_ip_mapping}
+ ... key=${member_index}
+ ${output} = SSHKeywords.Run_Keyword_Preserve_Connection
+ ... Utils.Run_Command_On_Controller
+ ... ${member_ip}
+ ... ${command}
+ Log ${output}
+ RETURN ${output}
+
+Run_Karaf_Command_On_List_Or_All
+ [Documentation] Cycle through indices (or all), run karaf command on each.
+ [Arguments] ${command} ${member_index_list}=${EMPTY} ${timeout}=10s
+ ${index_list} = List_Indices_Or_All given_list=${member_index_list}
+ FOR ${index} IN @{index_list}
+ ${member_ip} = Collections.Get_From_Dictionary
+ ... dictionary=${ClusterManagement__index_to_ip_mapping}
+ ... key=${index}
+ KarafKeywords.Safe_Issue_Command_On_Karaf_Console ${command} ${member_ip} timeout=${timeout}
+ END
+
+Run_Karaf_Command_On_Member
+ [Documentation] Obtain IP address, call KarafKeywords and return output. This does not preserve active ssh session.
+ ... This keyword is not used by Run_Karaf_Command_On_List_Or_All, but returned output may be useful.
+ [Arguments] ${command} ${member_index} ${timeout}=10s
+ ${member_ip} = Collections.Get_From_Dictionary
+ ... dictionary=${ClusterManagement__index_to_ip_mapping}
+ ... key=${member_index}
+ ${output} = KarafKeywords.Safe_Issue_Command_On_Karaf_Console
+ ... ${command}
+ ... controller=${member_ip}
+ ... timeout=${timeout}
+ RETURN ${output}
+
+Install_Feature_On_List_Or_All
+ [Documentation] Attempt installation on each member from list (or all). Then look for failures.
+ [Arguments] ${feature_name} ${member_index_list}=${EMPTY} ${timeout}=60s
+ ${index_list} = List_Indices_Or_All given_list=${member_index_list}
+ ${status_list} = BuiltIn.Create_List
+ FOR ${index} IN @{index_list}
+ ${status} ${text} = BuiltIn.Run_Keyword_And_Ignore_Error
+ ... Install_Feature_On_Member
+ ... feature_name=${feature_name}
+ ... member_index=${index}
+ ... timeout=${timeout}
+ BuiltIn.Log ${text}
+ Collections.Append_To_List ${status_list} ${status}
+ END
+ FOR ${status} IN @{status_list}
+ IF "${status}" != "PASS"
+ BuiltIn.Fail ${feature_name} installation failed, see log.
+ END
+ END
+
+Install_Feature_On_Member
+ [Documentation] Run feature:install karaf command, fail if installation was not successful. Return output.
+ [Arguments] ${feature_name} ${member_index} ${timeout}=60s
+ ${status} ${output} = BuiltIn.Run_Keyword_And_Ignore_Error
+ ... Run_Karaf_Command_On_Member
+ ... command=feature:install ${feature_name}
+ ... member_index=${member_index}
+ ... timeout=${timeout}
+ IF "${status}" != "PASS"
+ BuiltIn.Fail Failed to install ${feature_name}: ${output}
+ END
+ BuiltIn.Should_Not_Contain ${output} Can't install Failed to install ${feature_name}: ${output}
+ RETURN ${output}
+
+With_Ssh_To_List_Or_All_Run_Keyword
+ [Documentation] For each index in given list (or all): activate SSH connection, run given Keyword, close active connection. Return None.
+ ... Beware that in order to avoid "got positional argument after named arguments", first two arguments in the call should not be named.
+ [Arguments] ${member_index_list} ${keyword_name} @{args} &{kwargs}
+ BuiltIn.Comment This keyword is experimental and there is high risk of being replaced by another approach.
+ # TODO: For_Index_From_List_Or_All_Run_Keyword applied to With_Ssh_To_Member_Run_Keyword?
+ # TODO: Imagine another keyword, using ScalarClosures and adding member index as first argument for each call. Worth it?
+ ${index_list} = List_Indices_Or_All given_list=${member_index_list}
+ FOR ${member_index} IN @{index_list}
+ ${member_ip} = Resolve_IP_Address_For_Member ${member_index}
+ SSHKeywords.Run_Unsafely_Keyword_Over_Temporary_Odl_Session
+ ... ${member_ip}
+ ... ${keyword_name}
+ ... @{args}
+ ... &{kwargs}
+ END
+
+Safe_With_Ssh_To_List_Or_All_Run_Keyword
+ [Documentation] Remember active ssh connection index, call With_Ssh_To_List_Or_All_Run_Keyword, return None. Restore the conection index on teardown.
+ [Arguments] ${member_index_list} ${keyword_name} @{args} &{kwargs}
+ SSHKeywords.Run_Keyword_Preserve_Connection
+ ... With_Ssh_To_List_Or_All_Run_Keyword
+ ... ${member_index_list}
+ ... ${keyword_name}
+ ... @{args}
+ ... &{kwargs}
+
+Clean_Directories_On_List_Or_All
+ [Documentation] Clear @{directory_list} or @{ODL_DEFAULT_DATA_PATHS} for members in given list or all. Return None.
+ ... If \${tmp_dir} is nonempty, use that location to preserve data/log/.
+ ... This is intended to return Karaf (offline) to the state it was upon the first boot.
+ [Arguments] ${member_index_list}=${EMPTY} ${directory_list}=${EMPTY} ${karaf_home}=${KARAF_HOME} ${tmp_dir}=${EMPTY}
+ ${path_list} = Builtin.Set Variable If
+ ... "${directory_list}" == "${EMPTY}"
+ ... ${ODL_DEFAULT_DATA_PATHS}
+ ... ${directory_list}
+ IF """${tmp_dir}""" != ""
+ Check_Bash_Command_On_List_Or_All
+ ... mkdir -p '${tmp_dir}' && rm -vrf '${tmp_dir}/log' && mv -vf '${karaf_home}/data/log' '${tmp_dir}/'
+ ... ${member_index_list}
+ END
+ Safe_With_Ssh_To_List_Or_All_Run_Keyword
+ ... ${member_index_list}
+ ... ClusterManagement__Clean_Directories
+ ... ${path_list}
+ ... ${karaf_home}
+ IF """${tmp_dir}""" != ""
+ Check_Bash_Command_On_List_Or_All
+ ... mkdir -p '${karaf_home}/data' && rm -vrf '${karaf_home}/log' && mv -vf '${tmp_dir}/log' '${karaf_home}/data/'
+ ... ${member_index_list}
+ END
+
+Store_Karaf_Log_On_List_Or_All
+ [Documentation] Saves karaf.log to the ${dst_dir} for members in given list or all. Return None.
+ [Arguments] ${member_index_list}=${EMPTY} ${dst_dir}=/tmp ${karaf_home}=${KARAF_HOME}
+ Safe_With_Ssh_To_List_Or_All_Run_Keyword
+ ... ${member_index_list}
+ ... SSHKeywords.Execute_Command_Should_Pass
+ ... cp ${karaf_home}/data/log/karaf.log ${dst_dir}
+
+Restore_Karaf_Log_On_List_Or_All
+ [Documentation] Places stored karaf.log to the ${karaf_home}/data/log for members in given list or all. Return None.
+ [Arguments] ${member_index_list}=${EMPTY} ${src_dir}=/tmp ${karaf_home}=${KARAF_HOME}
+ Safe_With_Ssh_To_List_Or_All_Run_Keyword
+ ... ${member_index_list}
+ ... SSHKeywords.Execute_Command_Should_Pass
+ ... cp ${src_dir}/karaf.log ${karaf_home}/data/log/
+
+ClusterManagement__Clean_Directories
+ [Documentation] For each relative path, remove files with respect to ${karaf_home}. Return None.
+ [Arguments] ${relative_path_list} ${karaf_home}
+ FOR ${relative_path} IN @{relative_path_list}
+ SSHLibrary.Execute_Command rm -rf ${karaf_home}${/}${relative_path}
+ END