X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=csit%2Flibraries%2FClusterManagement.robot;h=43136bf08dc75022b09dc5bafcacdf6de9a78614;hb=d429e3c8acaccd2e34163a45e28d85ecba58df16;hp=3a72b01fa1a33cc199b58ffe6a4004ff7f75caa4;hpb=9192bdd94707b997cf6464b1920a8e9eff44988f;p=integration%2Ftest.git

diff --git a/csit/libraries/ClusterManagement.robot b/csit/libraries/ClusterManagement.robot
index 3a72b01fa1..43136bf08d 100644
--- a/csit/libraries/ClusterManagement.robot
+++ b/csit/libraries/ClusterManagement.robot
@@ -58,7 +58,9 @@ ${NODE_UNFREEZE_COMMAND}    ps axf | grep org.apache.karaf | grep -v grep | awk
 
 *** Keywords ***
 ClusterManagement_Setup
+    [Arguments]    ${http_timeout}=1    ${http_retries}=0
     [Documentation]    Detect repeated call, or detect number of members and initialize derived suite variables.
+    ...    Http sessions are created with parameters to not waste time when ODL is no accepting connections properly.
     # Avoid multiple initialization by several downstream libraries.
     ${already_done} =    BuiltIn.Get_Variable_Value    \${ClusterManagement__has_setup_run}    False
     BuiltIn.Return_From_Keyword_If    ${already_done}
@@ -66,7 +68,7 @@ ClusterManagement_Setup
     ${cluster_size} =    BuiltIn.Get_Variable_Value    \${NUM_ODL_SYSTEM}    1
     ${status}    ${possibly_int_of_members} =    BuiltIn.Run_Keyword_And_Ignore_Error    BuiltIn.Convert_To_Integer    ${cluster_size}
     ${int_of_members} =    BuiltIn.Set_Variable_If    '${status}' != 'PASS'    ${1}    ${possibly_int_of_members}
-    ClusterManagement__Compute_Derived_Variables    int_of_members=${int_of_members}
+    ClusterManagement__Compute_Derived_Variables    int_of_members=${int_of_members}    http_timeout=${http_timeout}    http_retries=${http_retries}
 
 Check_Cluster_Is_In_Sync
     [Arguments]    ${member_index_list}=${EMPTY}
@@ -133,20 +135,20 @@ Get_Raft_State_Of_Shard_At_Member
     ...    Optionally, check restconf works.
     ${session} =    Resolve_Http_Session_For_Member    member_index=${member_index}
     # TODO: Does the used URI tend to generate large data which floods log.html?
-    BuiltIn.Run_Keyword_If    ${verify_restconf}    TemplatedRequests.Get_As_Json_Templated    session=${session}    folder=${RESTCONF_MODULES_DIR}    verify=False
+    BuiltIn.Run_Keyword_If    ${verify_restconf}    TemplatedRequests.Get_As_Json_Templated    session=${session}    folder=${RESTCONF_MODULES_DIR}    verify=False    http_timeout=5
     ${type_class} =    Resolve_Shard_Type_Class    shard_type=${shard_type}
     ${uri} =    BuiltIn.Set_Variable    ${JOLOKIA_READ_URI}:Category=Shards,name=member-${member_index}-shard-${shard_name}-${shard_type},type=${type_class}
-    ${data_text} =    TemplatedRequests.Get_As_Json_From_Uri    uri=${uri}    session=${session}
+    ${data_text} =    TemplatedRequests.Get_As_Json_From_Uri    uri=${uri}    session=${session}    http_timeout=5
     ${data_object} =    RequestsLibrary.To_Json    ${data_text}
     ${value} =    Collections.Get_From_Dictionary    ${data_object}    value
     ${raft_state} =    Collections.Get_From_Dictionary    ${value}    RaftState
     [Return]    ${raft_state}
 
 Verify_Shard_Leader_Elected
-    [Arguments]    ${shard_name}    ${shard_type}    ${new_elected}    ${old_leader}    ${member_index_list}=${EMPTY}
+    [Arguments]    ${shard_name}    ${shard_type}    ${new_elected}    ${old_leader}    ${member_index_list}=${EMPTY}    ${verify_restconf}=True
     [Documentation]    Verify new leader was elected or remained the same. Bool paramter ${new_elected} indicates if
     ...    new leader is elected or should remained the same as ${old_leader}
-    ${leader}    ${followers}=    Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}    member_index_list=${member_index_list}
+    ${leader}    ${followers}=    Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}    member_index_list=${member_index_list}    verify_restconf=${verify_restconf}
     BuiltIn.Run_Keyword_If    ${new_elected}    BuiltIn.Should_Not_Be_Equal_As_Numbers    ${old_leader}    ${leader}
     BuiltIn.Run_Keyword_Unless    ${new_elected}    BuiltIn.Should_Be_Equal_As_numbers    ${old_leader}    ${leader}
     BuiltIn.Return_From_Keyword    ${leader}    ${followers}
@@ -487,17 +489,39 @@ Flush_Iptables_From_List_Or_All
     ${command} =    BuiltIn.Set_Variable    sudo iptables -v -F
     ${output} =    Run_Bash_Command_On_List_Or_All    command=${command}    member_index_list=${member_index_list}
 
+Check_Bash_Command_On_List_Or_All
+    [Arguments]    ${command}    ${member_index_list}=${EMPTY}    ${return_success_only}=False    ${log_on_success}=True    ${log_on_failure}=True    ${stderr_must_be_empty}=True
+    [Documentation]    Cycle through indices (or all), run bash command on each, using temporary SSH session and restoring the previously active one.
+    ${index_list} =    List_Indices_Or_All    given_list=${member_index_list}
+    : FOR    ${index}    IN    @{index_list}
+    \    Check_Bash_Command_On_Member    command=${command}    member_index=${index}    return_success_only=${return_success_only}    log_on_success=${log_on_success}    log_on_failure=${log_on_failure}
+    \    ...    stderr_must_be_empty=${stderr_must_be_empty}
+
+Check_Bash_Command_On_Member
+    [Arguments]    ${command}    ${member_index}    ${return_success_only}=False    ${log_on_success}=True    ${log_on_failure}=True    ${stderr_must_be_empty}=True
+    [Documentation]    Open SSH session, call SSHKeywords.Execute_Command_Passes, close session, restore previously active session and return output.
+    BuiltIn.Run_Keyword_And_Return    SSHKeywords.Run_Keyword_Preserve_Connection    Check_Unsafely_Bash_Command_On_Member    ${command}    ${member_index}    return_success_only=${return_success_only}    log_on_success=${log_on_success}
+    ...    log_on_failure=${log_on_failure}    stderr_must_be_empty=${stderr_must_be_empty}
+
+Check_Unsafely_Bash_Command_On_Member
+    [Arguments]    ${command}    ${member_index}    ${return_success_only}=False    ${log_on_success}=True    ${log_on_failure}=True    ${stderr_must_be_empty}=True
+    [Documentation]    Obtain Ip address, open session, call SSHKeywords.Execute_Command_Passes, close session and return output. This affects which SSH session is active.
+    ${member_ip} =    Resolve_Ip_Address_For_Member    ${member_index}
+    BuiltIn.Run_Keyword_And_Return    SSHKeywords.Run_Unsafely_Keyword_Over_Temporary_Odl_Session    ${member_ip}    Execute_Command_Passes    ${command}    return_success_only=${return_success_only}    log_on_success=${log_on_success}
+    ...    log_on_failure=${log_on_failure}    stderr_must_be_empty=${stderr_must_be_empty}
+
 Run_Bash_Command_On_List_Or_All
     [Arguments]    ${command}    ${member_index_list}=${EMPTY}
     [Documentation]    Cycle through indices (or all), run command on each.
+    # TODO: Migrate callers to Check_Bash_Command_*
     ${index_list} =    List_Indices_Or_All    given_list=${member_index_list}
     : FOR    ${index}    IN    @{index_list}
     \    Run_Bash_Command_On_Member    command=${command}    member_index=${index}
 
 Run_Bash_Command_On_Member
     [Arguments]    ${command}    ${member_index}
-    [Documentation]    Obtain IP, call Utils and return output. This does not preserve active ssh session.
-    # TODO: Rename these keyword to Run_Bash_Command_On_Member to distinguish from Karaf (or even Windows) commands.
+    [Documentation]    Obtain IP, call Utils and return output. This keeps previous ssh session active.
+    # TODO: Migrate callers to Check_Bash_Command_*
     ${member_ip} =    Collections.Get_From_Dictionary    dictionary=${ClusterManagement__index_to_ip_mapping}    key=${member_index}
     ${output} =    SSHKeywords.Run_Keyword_Preserve_Connection    Utils.Run_Command_On_Controller    ${member_ip}    ${command}
     [Return]    ${output}
@@ -542,7 +566,6 @@ Install_Feature_On_Member
 With_Ssh_To_List_Or_All_Run_Keyword
     [Arguments]    ${member_index_list}    ${keyword_name}    @{args}    &{kwargs}
     [Documentation]    For each index in given list (or all): activate SSH connection, run given Keyword, close active connection. Return None.
-    ...    Note that if the Keyword affects SSH connections, results are still deterministic, but perhaps undesirable.
     ...    Beware that in order to avoid "got positional argument after named arguments", first two arguments in the call should not be named.
     BuiltIn.Comment    This keyword is experimental and there is high risk of being replaced by another approach.
     # TODO: For_Index_From_List_Or_All_Run_Keyword applied to With_Ssh_To_Member_Run_Keyword?
@@ -550,9 +573,7 @@ With_Ssh_To_List_Or_All_Run_Keyword
     ${index_list} =    List_Indices_Or_All    given_list=${member_index_list}
     : FOR    ${member_index}    IN    @{index_list}
     \    ${member_ip} =    Resolve_IP_Address_For_Member    ${member_index}
-    \    SSHKeywords.Open_Connection_To_Odl_System    ip_address=${member_ip}
-    \    BuiltIn.Run_Keyword    ${keyword_name}    @{args}    &{kwargs}
-    \    SSHLibrary.Close_Connection
+    \    SSHKeywords.Run_Unsafely_Keyword_Over_Temporary_Odl_Session    ${member_ip}    ${keyword_name}    @{args}    &{kwargs}
 
 Safe_With_Ssh_To_List_Or_All_Run_Keyword
     [Arguments]    ${member_index_list}    ${keyword_name}    @{args}    &{kwargs}
@@ -560,11 +581,14 @@ Safe_With_Ssh_To_List_Or_All_Run_Keyword
     SSHKeywords.Run_Keyword_Preserve_Connection    With_Ssh_To_List_Or_All_Run_Keyword    ${member_index_list}    ${keyword_name}    @{args}    &{kwargs}
 
 Clean_Directories_On_List_Or_All
-    [Arguments]    ${member_index_list}=${EMPTY}    ${directory_list}=${EMPTY}    ${karaf_home}=${KARAF_HOME}
+    [Arguments]    ${member_index_list}=${EMPTY}    ${directory_list}=${EMPTY}    ${karaf_home}=${KARAF_HOME}    ${tmp_dir}=${EMPTY}
     [Documentation]    Clear @{directory_list} or @{ODL_DEFAULT_DATA_PATHS} for members in given list or all. Return None.
+    ...    If \${tmp_dir} is nonempty, use that location to preserve data/log/.
     ...    This is intended to return Karaf (offline) to the state it was upon the first boot.
     ${path_list} =    Builtin.Set Variable If    "${directory_list}" == "${EMPTY}"    ${ODL_DEFAULT_DATA_PATHS}    ${directory_list}
+    BuiltIn.Run_Keyword_If    """${tmp_dir}""" != ""    Check_Bash_Command_On_List_Or_All    mkdir -p '${tmp_dir}' && rm -vrf '${tmp_dir}/log' && mv -vf '${karaf_home}/data/log' '${tmp_dir}/'    ${member_index_list}
     Safe_With_Ssh_To_List_Or_All_Run_Keyword    ${member_index_list}    ClusterManagement__Clean_Directories    ${path_list}    ${karaf_home}
+    BuiltIn.Run_Keyword_If    """${tmp_dir}""" != ""    Check_Bash_Command_On_List_Or_All    mkdir -p '${karaf_home}/data' && rm -vrf '${karaf_home}/log' && mv -vf '${tmp_dir}/log' '${karaf_home}/data/'    ${member_index_list}
 
 Store_Karaf_Log_On_List_Or_All
     [Arguments]    ${member_index_list}=${EMPTY}    ${dst_dir}=/tmp    ${karaf_home}=${KARAF_HOME}
@@ -659,6 +683,15 @@ Resolve_IP_Address_For_Member
     ${ip_address} =    Collections.Get From Dictionary    dictionary=${ClusterManagement__index_to_ip_mapping}    key=${member_index}
     [Return]    ${ip_address}
 
+Resolve_IP_Address_For_Members
+    [Arguments]    ${member_index_list}
+    [Documentation]    Return a list of IP address of given indexes.
+    ${member_ip_list} =    BuiltIn.Create_List
+    : FOR    ${index}    IN    @{member_index_list}
+    \    ${ip_address} =    Collections.Get From Dictionary    dictionary=${ClusterManagement__index_to_ip_mapping}    key=${index}
+    \    Collections.Append_To_List    ${member_ip_list}    ${ip_address}
+    [Return]    ${member_ip_list}
+
 Resolve_Http_Session_For_Member
     [Arguments]    ${member_index}
     [Documentation]    Return RequestsLibrary session alias pointing to node of given index.
@@ -709,19 +742,20 @@ List_Indices_Minus_Member
     [Return]    ${index_list}
 
 ClusterManagement__Compute_Derived_Variables
-    [Arguments]    ${int_of_members}
+    [Arguments]    ${int_of_members}    ${http_timeout}=1    ${http_retries}=0
     [Documentation]    Construct index list, session list and IP mapping, publish them as suite variables.
     @{member_index_list} =    BuiltIn.Create_List
     @{session_list} =    BuiltIn.Create_List
     &{index_to_ip_mapping} =    BuiltIn.Create_Dictionary
     : FOR    ${index}    IN RANGE    1    ${int_of_members+1}
-    \    ClusterManagement__Include_Member_Index    ${index}    ${member_index_list}    ${session_list}    ${index_to_ip_mapping}
+    \    ClusterManagement__Include_Member_Index    ${index}    ${member_index_list}    ${session_list}    ${index_to_ip_mapping}    http_timeout=${http_timeout}
+    \    ...    http_retries=${http_retries}
     BuiltIn.Set_Suite_Variable    \${ClusterManagement__member_index_list}    ${member_index_list}
     BuiltIn.Set_Suite_Variable    \${ClusterManagement__index_to_ip_mapping}    ${index_to_ip_mapping}
     BuiltIn.Set_Suite_Variable    \${ClusterManagement__session_list}    ${session_list}
 
 ClusterManagement__Include_Member_Index
-    [Arguments]    ${index}    ${member_index_list}    ${session_list}    ${index_to_ip_mapping}
+    [Arguments]    ${index}    ${member_index_list}    ${session_list}    ${index_to_ip_mapping}    ${http_timeout}=1    ${http_retries}=0
     [Documentation]    Add a corresponding item based on index into the last three arguments.
     ...    Create the Http session whose alias is added to list.
     Collections.Append_To_List    ${member_index_list}    ${index}
@@ -730,7 +764,7 @@ ClusterManagement__Include_Member_Index
     Collections.Set_To_Dictionary    ${index_to_ip_mapping}    ${index}    ${member_ip}
     # Http session, with ${AUTH}, without headers.
     ${session_alias} =    Resolve_Http_Session_For_Member    member_index=${index}
-    RequestsLibrary.Create_Session    ${session_alias}    http://${member_ip}:${RESTCONFPORT}    auth=${AUTH}    max_retries=0
+    RequestsLibrary.Create_Session    ${session_alias}    http://${member_ip}:${RESTCONFPORT}    auth=${AUTH}    timeout=${http_timeout}    max_retries=${http_retries}
     Collections.Append_To_List    ${session_list}    ${session_alias}
 
 Sync_Status_Should_Be_False