Enable check for status after member starts up

[integration/test.git] / csit / libraries / ClusterManagement.robot

diff --git a/csit/libraries/ClusterManagement.robot b/csit/libraries/ClusterManagement.robot
index 4b61a87055d02fc19fa2eec5b0f79f4142d5fe16..13e87b6cdef1b5d4ac28fbbedadfdb174f388294 100644 (file)
--- a/csit/libraries/ClusterManagement.robot
+++ b/csit/libraries/ClusterManagement.robot
@@ -432,7 +432,7 @@ Stop_Single_Member
     [Return]    ${updated_index_list}
 
 Stop_Members_From_List_Or_All
-    [Arguments]    ${member_index_list}=${EMPTY}    ${original_index_list}=${EMPTY}    ${confirm}=True    ${timeout}=120s
+    [Arguments]    ${member_index_list}=${EMPTY}    ${original_index_list}=${EMPTY}    ${confirm}=True    ${timeout}=240s
     [Documentation]    If the list is empty, stops all ODL instances. Otherwise stop members based on \${stop_index_list}
     ...    If \${confirm} is True, verify stopped instances are not there anymore.
     ...    The KW will return a list of available members: \${updated index_list}=\${original_index_list}-\${member_index_list}
@@ -448,17 +448,18 @@ Stop_Members_From_List_Or_All
     [Return]    ${updated_index_list}
 
 Start_Single_Member
-    [Arguments]    ${member}    ${wait_for_sync}=True    ${timeout}=300s    ${msg}=${EMPTY}    ${check_system_status}=False    ${service_list}=@{EMPTY}
+    [Arguments]    ${member}    ${wait_for_sync}=True    ${timeout}=300s    ${msg}=${EMPTY}    ${check_system_status}=True    ${verify_restconf}=True
+    ...    ${service_list}=${EMPTY_LIST}
     [Documentation]    Convenience keyword that starts the specified member of the cluster.
     ${index_list} =    ClusterManagement__Build_List    ${member}
     ${member_ip} =    Return_Member_IP    ${member}
     ${msg} =    Builtin.Set Variable If    "${msg}" == "${EMPTY}"    Starting ODL${member} ${member_ip}    Starting ODL${member} ${member_ip}, ${msg}
     KarafKeywords.Log_Message_To_Controller_Karaf    ${msg}
-    Start_Members_From_List_Or_All    ${index_list}    ${wait_for_sync}    ${timeout}    check_system_status=${check_system_status}    service_list=@{service_list}
+    Start_Members_From_List_Or_All    ${index_list}    ${wait_for_sync}    ${timeout}    check_system_status=${check_system_status}    verify_restconf=${verify_restconf}    service_list=${service_list}
 
 Start_Members_From_List_Or_All
     [Arguments]    ${member_index_list}=${EMPTY}    ${wait_for_sync}=True    ${timeout}=300s    ${karaf_home}=${EMPTY}    ${export_java_home}=${EMPTY}    ${gc_log_dir}=${EMPTY}
-    ...    ${check_system_status}=False    ${service_list}=@{EMPTY}
+    ...    ${check_system_status}=True    ${verify_restconf}=True    ${service_list}=${EMPTY_LIST}
     [Documentation]    If the list is empty, start all cluster members. Otherwise, start members based on present indices.
     ...    If ${wait_for_sync}, wait for cluster sync on listed members.
     ...    Optionally karaf_home can be overriden. Optionally specific JAVA_HOME is used for starting.
@@ -469,12 +470,29 @@ Start_Members_From_List_Or_All
     ${gc_filepath} =    BuiltIn.Set_Variable_If    """${karaf_home}""" != ""    ${karaf_home}/data/log/gc_${epoch}.log    ${GC_LOG_PATH}/gc_${epoch}.log
     ${gc_options} =    BuiltIn.Set_Variable_If    "docker" not in """${node_start_command}"""    -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:${gc_filepath}    ${EMPTY}
     Run_Bash_Command_On_List_Or_All    command=${command} ${gc_options}    member_index_list=${member_index_list}
-    BuiltIn.Return_From_Keyword_If    not ${wait_for_sync}
-    BuiltIn.Wait_Until_Keyword_Succeeds    ${timeout}    10s    Check_Cluster_Is_In_Sync    member_index_list=${member_index_list}
-    BuiltIn.Return_From_Keyword_If    not ${check_system_status}
-    CompareStream.Run_Keyword_If_At_Least_Oxygen    Wait Until Keyword Succeeds    60    2    ClusterManagement.Check Status Of Services Is OPERATIONAL    @{service_list}
+    BuiltIn.Wait_Until_Keyword_Succeeds    ${timeout}    10s    Verify_Members_Are_Ready    ${member_index_list}    ${wait_for_sync}    ${verify_restconf}
+    ...    ${check_system_status}    ${service_list}
     [Teardown]    Run_Bash_Command_On_List_Or_All    command=netstat -pnatu | grep 2550
 
+Verify_Members_Are_Ready
+    [Arguments]    ${member_index_list}    ${verify_cluster_sync}    ${verify_restconf}    ${verify_system_status}    ${service_list}
+    [Documentation]    Verifies the specified readiness conditions for the given listed members after startup.
+    ...    If ${verify_cluster_sync}, verifies the datastores have synced with the rest of the cluster.
+    ...    If ${verify_restconf}, verifies RESTCONF is available.
+    ...    If ${verify_system_status}, verifies the system services are OPERATIONAL.
+    BuiltIn.Run_Keyword_If    ${verify_cluster_sync}    Check_Cluster_Is_In_Sync    ${member_index_list}
+    BuiltIn.Run_Keyword_If    ${verify_restconf}    Verify_Restconf_Is_Available    ${member_index_list}
+    # for backward compatibility, some consumers might not be passing @{service_list}, but since we can't set a list to a default
+    # value, we need to check here if it's empty in order to skip the check which would throw an error
+    BuiltIn.Run_Keyword_If    ${verify_system_status} and ("${service_list}" != "[[]]")    ClusterManagement.Check Status Of Services Is OPERATIONAL    @{service_list}
+
+Verify_Restconf_Is_Available
+    [Arguments]    ${member_index_list}
+    ${index_list} =    List_Indices_Or_All    given_list=${member_index_list}
+    : FOR    ${index}    IN    @{index_list}
+    \    ${session} =    Resolve_Http_Session_For_Member    member_index=${index}
+    \    TemplatedRequests.Get_As_Json_Templated    session=${session}    folder=${RESTCONF_MODULES_DIR}    verify=False
+
 Freeze_Single_Member
     [Arguments]    ${member}
     [Documentation]    Convenience keyword that stops the specified member of the cluster by freezing the jvm.