X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=csit%2Flibraries%2FClusterManagement.robot;h=92c861e1f42cb5e3d4d6bdcc5338c6ff2e0bd2f3;hb=94651ca9711d8eb316721775561e815830bc6c7e;hp=0278e204919ca7810d0ff3a27867438f778df562;hpb=c97210d403b181f11b66fc1a840398e7a3e911dd;p=integration%2Ftest.git diff --git a/csit/libraries/ClusterManagement.robot b/csit/libraries/ClusterManagement.robot index 0278e20491..92c861e1f4 100644 --- a/csit/libraries/ClusterManagement.robot +++ b/csit/libraries/ClusterManagement.robot @@ -36,6 +36,7 @@ Resource ${CURDIR}/KarafKeywords.robot Resource ${CURDIR}/SSHKeywords.robot Resource ${CURDIR}/TemplatedRequests.robot # for Get_As_Json_From_Uri Resource ${CURDIR}/Utils.robot # for Run_Command_On_Controller +Resource ../variables/Variables.robot *** Variables *** ${ENTITY_OWNER_URI} restconf/operational/entity-owners:entity-owners @@ -43,17 +44,23 @@ ${GC_LOG_PATH} ${KARAF_HOME}/data/log ${JAVA_HOME} ${EMPTY} # releng/builder scripts should provide correct value ${JOLOKIA_CONF_SHARD_MANAGER_URI} jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore ${JOLOKIA_OPER_SHARD_MANAGER_URI} jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-operational,type=DistributedOperationalDatastore +${JOLOKIA_CONFIG_LOCAL_SHARDS_URI} jolokia/read/org.opendaylight.controller:type=DistributedConfigDatastore,Category=ShardManager,name=shard-manager-config/LocalShards +${JOLOKIA_OPER_LOCAL_SHARDS_URI} jolokia/read/org.opendaylight.controller:type=DistributedOperationalDatastore,Category=ShardManager,name=shard-manager-operational/LocalShards ${JOLOKIA_READ_URI} jolokia/read/org.opendaylight.controller -${KARAF_HOME} ${WORKSPACE}${/}${BUNDLEFOLDER} # TODO: Migrate to Variables.robot # Bug 9044 workaround: delete etc/host.key before restart. @{ODL_DEFAULT_DATA_PATHS} tmp/ data/ cache/ snapshots/ journal/ etc/opendaylight/current/ etc/host.key ${RESTCONF_MODULES_DIR} ${CURDIR}/../variables/restconf/modules -${SINGLETON_NETCONF_DEVICE_ID_PREFIX} /odl-general-entity:entity[odl-general-entity:name='KeyedInstanceIdentifier{targetType=interface org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node, path=[org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NetworkTopology, org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.Topology[key=TopologyKey [_topologyId=Uri [_value=topology-netconf]]], org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node[key=NodeKey [_nodeId=Uri [_value= -${SINGLETON_NETCONF_DEVICE_ID_SUFFIX} ]]]]}'] +${SINGLETON_NETCONF_DEVICE_ID_PREFIX_OLD} /odl-general-entity:entity[odl-general-entity:name='KeyedInstanceIdentifier{targetType=interface org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node, path=[org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NetworkTopology, org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.Topology[key=TopologyKey [_topologyId=Uri [_value=topology-netconf]]], org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node[key=NodeKey [_nodeId=Uri [_value= +${SINGLETON_NETCONF_DEVICE_ID_SUFFIX_OLD} ]]]]}'] +${SINGLETON_NETCONF_DEVICE_ID_PREFIX} /odl-general-entity:entity[odl-general-entity:name='KeyedInstanceIdentifier{targetType=interface org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node, path=[org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NetworkTopology, org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.Topology[key=TopologyKey{_topologyId=Uri{_value=topology-netconf}}], org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node[key=NodeKey{_nodeId=Uri{_value= +${SINGLETON_NETCONF_DEVICE_ID_SUFFIX} }}]]}'] ${SINGLETON_BGPCEP_DEVICE_ID_PREFIX} /odl-general-entity:entity[odl-general-entity:name=' ${SINGLETON_BGPCEP_DEVICE_ID_SUFFIX} -service-group'] +${SINGLETON_SXP_DEVICE_ID_PREFIX} /odl-general-entity:entity[odl-general-entity:name=' +${SINGLETON_SXP_DEVICE_ID_SUFFIX} '] ${SINGLETON_ELECTION_ENTITY_TYPE} org.opendaylight.mdsal.ServiceEntityType ${SINGLETON_CHANGE_OWNERSHIP_ENTITY_TYPE} org.opendaylight.mdsal.AsyncServiceCloseEntityType +${NODE_ROLE_INDEX_START} 1 ${NODE_START_COMMAND} ${KARAF_HOME}/bin/start ${NODE_STOP_COMMAND} ${KARAF_HOME}/bin/stop ${NODE_KARAF_COUNT_COMMAND} ps axf | grep org.apache.karaf | grep -v grep | wc -l @@ -79,10 +86,11 @@ Check_Cluster_Is_In_Sync [Arguments] ${member_index_list}=${EMPTY} [Documentation] Fail if no-sync is detected on a member from list (or any). ${index_list} = List_Indices_Or_All given_list=${member_index_list} - : FOR ${index} IN @{index_list} # usually: 1, 2, 3. - \ ${status} = Get_Sync_Status_Of_Member member_index=${index} - \ BuiltIn.Continue_For_Loop_If 'True' == '${status}' - \ BuiltIn.Fail Index ${index} has incorrect status: ${status} + FOR ${index} IN @{index_list} # usually: 1, 2, 3. + ${status} = Get_Sync_Status_Of_Member member_index=${index} + BuiltIn.Continue_For_Loop_If 'True' == '${status}' + BuiltIn.Fail Index ${index} has incorrect status: ${status} + END Get_Sync_Status_Of_Member [Arguments] ${member_index} @@ -99,8 +107,9 @@ Verify_Leader_Exists_For_Each_Shard [Arguments] ${shard_name_list} ${shard_type}=operational ${member_index_list}=${EMPTY} ${verify_restconf}=True [Documentation] For each shard name, call Get_Leader_And_Followers_For_Shard. ... Not much logic there, but single Keyword is useful when using BuiltIn.Wait_Until_Keyword_Succeeds. - : FOR ${shard_name} IN @{shard_name_list} - \ Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} validate=True member_index_list=${member_index_list} verify_restconf=${verify_restconf} + FOR ${shard_name} IN @{shard_name_list} + Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} validate=True member_index_list=${member_index_list} verify_restconf=${verify_restconf} + END Get_Leader_And_Followers_For_Shard [Arguments] ${shard_name}=default ${shard_type}=operational ${validate}=True ${member_index_list}=${EMPTY} ${verify_restconf}=True ${http_timeout}=${EMPTY} @@ -127,12 +136,13 @@ Get_State_Info_For_Shard ${ds_type} = BuiltIn.Set_Variable_If '${shard_type}' != 'config' operational config ${leader_list} = BuiltIn.Create_List ${follower_list} = BuiltIn.Create_List - : FOR ${index} IN @{index_list} # usually: 1, 2, 3. - \ ${raft_state} = Get_Raft_State_Of_Shard_At_Member shard_name=${shard_name} shard_type=${ds_type} member_index=${index} verify_restconf=${verify_restconf} - \ ... http_timeout=${http_timeout} - \ BuiltIn.Run_Keyword_If 'Follower' == '${raft_state}' Collections.Append_To_List ${follower_list} ${index} - \ ... ELSE IF 'Leader' == '${raft_state}' Collections.Append_To_List ${leader_list} ${index} - \ ... ELSE IF ${validate} BuiltIn.Fail Unrecognized Raft state: ${raft_state} + FOR ${index} IN @{index_list} # usually: 1, 2, 3. + ${raft_state} = Get_Raft_State_Of_Shard_At_Member shard_name=${shard_name} shard_type=${ds_type} member_index=${index} verify_restconf=${verify_restconf} + ... http_timeout=${http_timeout} + BuiltIn.Run_Keyword_If 'Follower' == '${raft_state}' Collections.Append_To_List ${follower_list} ${index} + ... ELSE IF 'Leader' == '${raft_state}' Collections.Append_To_List ${leader_list} ${index} + ... ELSE IF ${validate} BuiltIn.Fail Unrecognized Raft state: ${raft_state} + END [Return] ${leader_list} ${follower_list} Get_Raft_State_Of_Shard_At_Member @@ -151,7 +161,8 @@ Get_Raft_Property_From_Shard_Member # TODO: Does the used URI tend to generate large data which floods log.html? BuiltIn.Run_Keyword_If ${verify_restconf} TemplatedRequests.Get_As_Json_Templated session=${session} folder=${RESTCONF_MODULES_DIR} verify=False http_timeout=${http_timeout} ${type_class} = Resolve_Shard_Type_Class shard_type=${shard_type} - ${uri} = BuiltIn.Set_Variable ${JOLOKIA_READ_URI}:Category=Shards,name=member-${member_index}-shard-${shard_name}-${shard_type},type=${type_class} + ${cluster_index} = Evaluate ${member_index}+${NODE_ROLE_INDEX_START}-1 + ${uri} = BuiltIn.Set_Variable ${JOLOKIA_READ_URI}:Category=Shards,name=member-${cluster_index}-shard-${shard_name}-${shard_type},type=${type_class} ${data_text} = TemplatedRequests.Get_As_Json_From_Uri uri=${uri} session=${session} http_timeout=${http_timeout} ${data_object} = RequestsLibrary.To_Json ${data_text} ${value} = Collections.Get_From_Dictionary ${data_object} value @@ -230,10 +241,11 @@ Get_Owner_And_Candidates_For_Device_Old ${owner} = String.Replace_String ${entity_owner} member- ${EMPTY} ${owner} = BuiltIn.Convert_To_Integer ${owner} ${entity_candidates_list} = Collections.Get_From_Dictionary @{entity_list}[${entity_index}] candidate - : FOR ${entity_candidate} IN @{entity_candidates_list} - \ ${candidate} = String.Replace_String &{entity_candidate}[name] member- ${EMPTY} - \ ${candidate} = BuiltIn.Convert_To_Integer ${candidate} - \ Collections.Append_To_List ${candidate_list} ${candidate} + FOR ${entity_candidate} IN @{entity_candidates_list} + ${candidate} = String.Replace_String &{entity_candidate}[name] member- ${EMPTY} + ${candidate} = BuiltIn.Convert_To_Integer ${candidate} + Collections.Append_To_List ${candidate_list} ${candidate} + END Collections.Sort_List ${candidate_list} [Return] ${owner} ${candidate_list} @@ -251,11 +263,11 @@ Get_Owner_And_Candidates_For_Device_Singleton_Netconf ... Parsing method is set as netconf (using netconf device id prefix and suffix) # Get election entity type results ${type} = BuiltIn.Set_Variable ${SINGLETON_ELECTION_ENTITY_TYPE} - ${id} = BuiltIn.Set_Variable ${SINGLETON_NETCONF_DEVICE_ID_PREFIX}${device_name}${SINGLETON_NETCONF_DEVICE_ID_SUFFIX} + ${id} = CompareStream.Set_Variable_If_At_Least_Fluorine ${SINGLETON_NETCONF_DEVICE_ID_PREFIX}${device_name}${SINGLETON_NETCONF_DEVICE_ID_SUFFIX} ${SINGLETON_NETCONF_DEVICE_ID_PREFIX_OLD}${device_name}${SINGLETON_NETCONF_DEVICE_ID_SUFFIX_OLD} ${owner_1} ${candidate_list_1} = Get_Owner_And_Candidates_For_Type_And_Id ${type} ${id} ${member_index} http_timeout=${http_timeout} # Get change ownership entity type results ${type} = BuiltIn.Set_Variable ${SINGLETON_CHANGE_OWNERSHIP_ENTITY_TYPE} - ${id} = BuiltIn.Set_Variable ${SINGLETON_NETCONF_DEVICE_ID_PREFIX}${device_name}${SINGLETON_NETCONF_DEVICE_ID_SUFFIX} + ${id} = CompareStream.Set_Variable_If_At_Least_Fluorine ${SINGLETON_NETCONF_DEVICE_ID_PREFIX}${device_name}${SINGLETON_NETCONF_DEVICE_ID_SUFFIX} ${SINGLETON_NETCONF_DEVICE_ID_PREFIX_OLD}${device_name}${SINGLETON_NETCONF_DEVICE_ID_SUFFIX_OLD} ${owner_2} ${candidate_list_2} = Get_Owner_And_Candidates_For_Type_And_Id ${type} ${id} ${member_index} http_timeout=${http_timeout} # Owners must be same, if not, there is still some election or change ownership in progress BuiltIn.Should_Be_Equal_As_Integers ${owner_1} ${owner_2} Owners for device ${device_name} are not same @@ -276,6 +288,21 @@ Get_Owner_And_Candidates_For_Device_Singleton_Bgpcep BuiltIn.Should_Be_Equal_As_Integers ${owner_1} ${owner_2} Owners for device ${device_name} are not same [Return] ${owner_1} ${candidate_list_1} +Get_Owner_And_Candidates_For_Device_Singleton_Sxp + [Arguments] ${device_name} ${member_index} ${http_timeout}=${EMPTY} + [Documentation] Returns the owner and a list of candidates for the SB device ${device_name}. Request is sent to member ${member_index}. + # Get election entity type results + ${type} = BuiltIn.Set_Variable ${SINGLETON_ELECTION_ENTITY_TYPE} + ${id} = BuiltIn.Set_Variable ${SINGLETON_SXP_DEVICE_ID_PREFIX}${device_name}${SINGLETON_SXP_DEVICE_ID_SUFFIX} + ${owner_1} ${candidate_list_1} = Get_Owner_And_Candidates_For_Type_And_Id ${type} ${id} ${member_index} http_timeout=${http_timeout} + # Get change ownership entity type results + ${type} = BuiltIn.Set_Variable ${SINGLETON_CHANGE_OWNERSHIP_ENTITY_TYPE} + ${id} = BuiltIn.Set_Variable ${SINGLETON_SXP_DEVICE_ID_PREFIX}${device_name}${SINGLETON_SXP_DEVICE_ID_SUFFIX} + ${owner_2} ${candidate_list_2} = Get_Owner_And_Candidates_For_Type_And_Id ${type} ${id} ${member_index} http_timeout=${http_timeout} + # Owners must be same, if not, there is still some election or change ownership in progress + BuiltIn.Should_Be_Equal_As_Integers ${owner_1} ${owner_2} Owners for device ${device_name} are not same + [Return] ${owner_1} ${candidate_list_1} + Get_Owner_And_Candidates_For_Device [Arguments] ${device_name} ${device_type} ${member_index} ${http_timeout}=${EMPTY} [Documentation] Returns the owner and a list of candidates for the SB device ${device_name} of type ${device_type}. Request is sent to member ${member_index}. @@ -334,10 +361,11 @@ Get_Owner_And_Candidates_For_Type_And_Id ${owner} = String.Replace_String ${entity_owner} member- ${EMPTY} ${owner} = BuiltIn.Convert_To_Integer ${owner} ${entity_candidates_list} = Collections.Get_From_Dictionary @{entity_list}[${entity_index}] candidate - : FOR ${entity_candidate} IN @{entity_candidates_list} - \ ${candidate} = String.Replace_String &{entity_candidate}[name] member- ${EMPTY} - \ ${candidate} = BuiltIn.Convert_To_Integer ${candidate} - \ Collections.Append_To_List ${candidate_list} ${candidate} + FOR ${entity_candidate} IN @{entity_candidates_list} + ${candidate} = String.Replace_String &{entity_candidate}[name] member- ${EMPTY} + ${candidate} = BuiltIn.Convert_To_Integer ${candidate} + Collections.Append_To_List ${candidate_list} ${candidate} + END Collections.Sort_List ${candidate_list} BuiltIn.Comment TODO: Separate check lines into Verify_Owner_And_Candidates_For_Type_And_Id BuiltIn.Run_Keyword_If """${require_candidate_list}""" BuiltIn.Should_Be_Equal ${require_candidate_list} ${candidate_list} Candidate list does not match: ${candidate_list} is not ${require_candidate_list} @@ -392,21 +420,25 @@ Kill_Members_From_List_Or_All BuiltIn.Return_From_Keyword_If not ${confirm} ${updated_index_list} # TODO: Convert to WUKS with configurable timeout if it turns out 1 second is not enough. BuiltIn.Sleep 1s Kill -9 closes open files, which may take longer than ssh overhead, but not long enough to warrant WUKS. - : FOR ${index} IN @{kill_index_list} - \ Verify_Karaf_Is_Not_Running_On_Member member_index=${index} + FOR ${index} IN @{kill_index_list} + Verify_Karaf_Is_Not_Running_On_Member member_index=${index} + END Run_Bash_Command_On_List_Or_All command=netstat -pnatu | grep 2550 [Return] ${updated_index_list} Stop_Single_Member - [Arguments] ${member} ${original_index_list}=${EMPTY} ${confirm}=True + [Arguments] ${member} ${original_index_list}=${EMPTY} ${confirm}=True ${msg}=${EMPTY} [Documentation] Convenience keyword that stops the specified member of the cluster. ... The KW will return a list of available members: \${updated index_list}=\${original_index_list}-\${member} ${index_list} = ClusterManagement__Build_List ${member} + ${member_ip} = Return_Member_IP ${member} + ${msg} = Builtin.Set Variable If "${msg}" == "${EMPTY}" Stopping ODL${member} ${member_ip} Stopping ODL${member} ${member_ip}, ${msg} + KarafKeywords.Log_Message_To_Controller_Karaf ${msg} ${updated_index_list} = Stop_Members_From_List_Or_All ${index_list} ${original_index_list} ${confirm} [Return] ${updated_index_list} Stop_Members_From_List_Or_All - [Arguments] ${member_index_list}=${EMPTY} ${original_index_list}=${EMPTY} ${confirm}=True ${timeout}=120s + [Arguments] ${member_index_list}=${EMPTY} ${original_index_list}=${EMPTY} ${confirm}=True ${timeout}=240s [Documentation] If the list is empty, stops all ODL instances. Otherwise stop members based on \${stop_index_list} ... If \${confirm} is True, verify stopped instances are not there anymore. ... The KW will return a list of available members: \${updated index_list}=\${original_index_list}-\${member_index_list} @@ -416,19 +448,25 @@ Stop_Members_From_List_Or_All ${updated_index_list} = BuiltIn.Create_List @{index_list} Collections.Remove_Values_From_List ${updated_index_list} @{stop_index_list} BuiltIn.Return_From_Keyword_If not ${confirm} ${updated_index_list} - : FOR ${index} IN @{stop_index_list} - \ BuiltIn.Wait Until Keyword Succeeds ${timeout} 2s Verify_Karaf_Is_Not_Running_On_Member member_index=${index} + FOR ${index} IN @{stop_index_list} + BuiltIn.Wait Until Keyword Succeeds ${timeout} 2s Verify_Karaf_Is_Not_Running_On_Member member_index=${index} + END Run_Bash_Command_On_List_Or_All command=netstat -pnatu | grep 2550 [Return] ${updated_index_list} Start_Single_Member - [Arguments] ${member} ${wait_for_sync}=True ${timeout}=300s + [Arguments] ${member} ${wait_for_sync}=True ${timeout}=300s ${msg}=${EMPTY} ${check_system_status}=False ${verify_restconf}=True + ... ${service_list}=${EMPTY_LIST} [Documentation] Convenience keyword that starts the specified member of the cluster. ${index_list} = ClusterManagement__Build_List ${member} - Start_Members_From_List_Or_All ${index_list} ${wait_for_sync} ${timeout} + ${member_ip} = Return_Member_IP ${member} + ${msg} = Builtin.Set Variable If "${msg}" == "${EMPTY}" Starting ODL${member} ${member_ip} Starting ODL${member} ${member_ip}, ${msg} + KarafKeywords.Log_Message_To_Controller_Karaf ${msg} + Start_Members_From_List_Or_All ${index_list} ${wait_for_sync} ${timeout} check_system_status=${check_system_status} verify_restconf=${verify_restconf} service_list=${service_list} Start_Members_From_List_Or_All [Arguments] ${member_index_list}=${EMPTY} ${wait_for_sync}=True ${timeout}=300s ${karaf_home}=${EMPTY} ${export_java_home}=${EMPTY} ${gc_log_dir}=${EMPTY} + ... ${check_system_status}=False ${verify_restconf}=True ${service_list}=${EMPTY_LIST} [Documentation] If the list is empty, start all cluster members. Otherwise, start members based on present indices. ... If ${wait_for_sync}, wait for cluster sync on listed members. ... Optionally karaf_home can be overriden. Optionally specific JAVA_HOME is used for starting. @@ -439,11 +477,30 @@ Start_Members_From_List_Or_All ${gc_filepath} = BuiltIn.Set_Variable_If """${karaf_home}""" != "" ${karaf_home}/data/log/gc_${epoch}.log ${GC_LOG_PATH}/gc_${epoch}.log ${gc_options} = BuiltIn.Set_Variable_If "docker" not in """${node_start_command}""" -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:${gc_filepath} ${EMPTY} Run_Bash_Command_On_List_Or_All command=${command} ${gc_options} member_index_list=${member_index_list} - BuiltIn.Return_From_Keyword_If not ${wait_for_sync} - BuiltIn.Wait_Until_Keyword_Succeeds ${timeout} 10s Check_Cluster_Is_In_Sync member_index_list=${member_index_list} - # TODO: Do we also want to check Shard Leaders here? + BuiltIn.Wait_Until_Keyword_Succeeds ${timeout} 10s Verify_Members_Are_Ready ${member_index_list} ${wait_for_sync} ${verify_restconf} + ... ${check_system_status} ${service_list} [Teardown] Run_Bash_Command_On_List_Or_All command=netstat -pnatu | grep 2550 +Verify_Members_Are_Ready + [Arguments] ${member_index_list} ${verify_cluster_sync} ${verify_restconf} ${verify_system_status} ${service_list} + [Documentation] Verifies the specified readiness conditions for the given listed members after startup. + ... If ${verify_cluster_sync}, verifies the datastores have synced with the rest of the cluster. + ... If ${verify_restconf}, verifies RESTCONF is available. + ... If ${verify_system_status}, verifies the system services are OPERATIONAL. + BuiltIn.Run_Keyword_If ${verify_cluster_sync} Check_Cluster_Is_In_Sync ${member_index_list} + BuiltIn.Run_Keyword_If ${verify_restconf} Verify_Restconf_Is_Available ${member_index_list} + # for backward compatibility, some consumers might not be passing @{service_list}, but since we can't set a list to a default + # value, we need to check here if it's empty in order to skip the check which would throw an error + BuiltIn.Run_Keyword_If ${verify_system_status} and ("${service_list}" != "[[]]") ClusterManagement.Check Status Of Services Is OPERATIONAL @{service_list} + +Verify_Restconf_Is_Available + [Arguments] ${member_index_list} + ${index_list} = List_Indices_Or_All given_list=${member_index_list} + FOR ${index} IN @{index_list} + ${session} = Resolve_Http_Session_For_Member member_index=${index} + TemplatedRequests.Get_As_Json_Templated session=${session} folder=${RESTCONF_MODULES_DIR} verify=False + END + Freeze_Single_Member [Arguments] ${member} [Documentation] Convenience keyword that stops the specified member of the cluster by freezing the jvm. @@ -464,15 +521,16 @@ Freeze_Or_Unfreeze_Members_From_List_Or_All ${freeze_index_list} = List_Indices_Or_All given_list=${member_index_list} Run_Bash_Command_On_List_Or_All command=${command} member_index_list=${member_index_list} -Clean_Journals_And_Snapshots_On_List_Or_All +Clean_Journals_Data_And_Snapshots_On_List_Or_All [Arguments] ${member_index_list}=${EMPTY} ${karaf_home}=${KARAF_HOME} [Documentation] Delete journal and snapshots directories on every node listed (or all). ... BEWARE: If only a subset of members is cleaned, this causes RetiredGenerationException in Carbon after the affected node re-start. ... See https://bugs.opendaylight.org/show_bug.cgi?id=8138 ${index_list} = List_Indices_Or_All given_list=${member_index_list} - ${command} = Set Variable rm -rf "${karaf_home}/journal" "${karaf_home}/snapshots" - : FOR ${index} IN @{index_list} # usually: 1, 2, 3. - \ Run_Bash_Command_On_Member command=${command} member_index=${index} + ${command} = Set Variable rm -rf "${karaf_home}/journal" "${karaf_home}/snapshots" "${karaf_home}/data" + FOR ${index} IN @{index_list} # usually: 1, 2, 3. + Run_Bash_Command_On_Member command=${command} member_index=${index} + END Verify_Karaf_Is_Not_Running_On_Member [Arguments] ${member_index} @@ -500,10 +558,11 @@ Isolate_Member_From_List_Or_All ${index_list} = List_Indices_Or_All given_list=${member_index_list} ${source} = Collections.Get_From_Dictionary ${ClusterManagement__index_to_ip_mapping} ${isolate_member_index} ${dport} = BuiltIn.Set_Variable_If '${port}' != '${EMPTY}' --dport ${port} ${EMPTY} - : FOR ${index} IN @{index_list} - \ ${destination} = Collections.Get_From_Dictionary ${ClusterManagement__index_to_ip_mapping} ${index} - \ ${command} = BuiltIn.Set_Variable sudo /sbin/iptables -I OUTPUT -p ${protocol} ${dport} --source ${source} --destination ${destination} -j DROP - \ BuiltIn.Run_Keyword_If "${index}" != "${isolate_member_index}" Run_Bash_Command_On_Member command=${command} member_index=${isolate_member_index} + FOR ${index} IN @{index_list} + ${destination} = Collections.Get_From_Dictionary ${ClusterManagement__index_to_ip_mapping} ${index} + ${command} = BuiltIn.Set_Variable sudo /sbin/iptables -I OUTPUT -p ${protocol} ${dport} --source ${source} --destination ${destination} -j DROP + BuiltIn.Run_Keyword_If "${index}" != "${isolate_member_index}" Run_Bash_Command_On_Member command=${command} member_index=${isolate_member_index} + END ${command} = BuiltIn.Set_Variable sudo /sbin/iptables -L -n ${output} = Run_Bash_Command_On_Member command=${command} member_index=${isolate_member_index} BuiltIn.Log ${output} @@ -517,10 +576,11 @@ Rejoin_Member_From_List_Or_All ${index_list} = List_Indices_Or_All given_list=${member_index_list} ${source} = Collections.Get_From_Dictionary ${ClusterManagement__index_to_ip_mapping} ${rejoin_member_index} ${dport} = BuiltIn.Set_Variable_If '${port}' != '${EMPTY}' --dport ${port} ${EMPTY} - : FOR ${index} IN @{index_list} - \ ${destination} = Collections.Get_From_Dictionary ${ClusterManagement__index_to_ip_mapping} ${index} - \ ${command} = BuiltIn.Set_Variable sudo /sbin/iptables -D OUTPUT -p ${protocol} ${dport} --source ${source} --destination ${destination} -j DROP - \ BuiltIn.Run_Keyword_If "${index}" != "${rejoin_member_index}" Run_Bash_Command_On_Member command=${command} member_index=${rejoin_member_index} + FOR ${index} IN @{index_list} + ${destination} = Collections.Get_From_Dictionary ${ClusterManagement__index_to_ip_mapping} ${index} + ${command} = BuiltIn.Set_Variable sudo /sbin/iptables -D OUTPUT -p ${protocol} ${dport} --source ${source} --destination ${destination} -j DROP + BuiltIn.Run_Keyword_If "${index}" != "${rejoin_member_index}" Run_Bash_Command_On_Member command=${command} member_index=${rejoin_member_index} + END ${command} = BuiltIn.Set_Variable sudo /sbin/iptables -L -n ${output} = Run_Bash_Command_On_Member command=${command} member_index=${rejoin_member_index} BuiltIn.Log ${output} @@ -536,9 +596,10 @@ Check_Bash_Command_On_List_Or_All [Arguments] ${command} ${member_index_list}=${EMPTY} ${return_success_only}=False ${log_on_success}=True ${log_on_failure}=True ${stderr_must_be_empty}=True [Documentation] Cycle through indices (or all), run bash command on each, using temporary SSH session and restoring the previously active one. ${index_list} = List_Indices_Or_All given_list=${member_index_list} - : FOR ${index} IN @{index_list} - \ Check_Bash_Command_On_Member command=${command} member_index=${index} return_success_only=${return_success_only} log_on_success=${log_on_success} log_on_failure=${log_on_failure} - \ ... stderr_must_be_empty=${stderr_must_be_empty} + FOR ${index} IN @{index_list} + Check_Bash_Command_On_Member command=${command} member_index=${index} return_success_only=${return_success_only} log_on_success=${log_on_success} log_on_failure=${log_on_failure} + ... stderr_must_be_empty=${stderr_must_be_empty} + END Check_Bash_Command_On_Member [Arguments] ${command} ${member_index} ${return_success_only}=False ${log_on_success}=True ${log_on_failure}=True ${stderr_must_be_empty}=True @@ -558,8 +619,9 @@ Run_Bash_Command_On_List_Or_All [Documentation] Cycle through indices (or all), run command on each. # TODO: Migrate callers to Check_Bash_Command_* ${index_list} = List_Indices_Or_All given_list=${member_index_list} - : FOR ${index} IN @{index_list} - \ Run_Bash_Command_On_Member command=${command} member_index=${index} + FOR ${index} IN @{index_list} + Run_Bash_Command_On_Member command=${command} member_index=${index} + END Run_Bash_Command_On_Member [Arguments] ${command} ${member_index} @@ -574,9 +636,10 @@ Run_Karaf_Command_On_List_Or_All [Arguments] ${command} ${member_index_list}=${EMPTY} ${timeout}=10s [Documentation] Cycle through indices (or all), run karaf command on each. ${index_list} = List_Indices_Or_All given_list=${member_index_list} - : FOR ${index} IN @{index_list} - \ ${member_ip} = Collections.Get_From_Dictionary dictionary=${ClusterManagement__index_to_ip_mapping} key=${index} - \ KarafKeywords.Safe_Issue_Command_On_Karaf_Console ${command} ${member_ip} timeout=${timeout} + FOR ${index} IN @{index_list} + ${member_ip} = Collections.Get_From_Dictionary dictionary=${ClusterManagement__index_to_ip_mapping} key=${index} + KarafKeywords.Safe_Issue_Command_On_Karaf_Console ${command} ${member_ip} timeout=${timeout} + END Run_Karaf_Command_On_Member [Arguments] ${command} ${member_index} ${timeout}=10s @@ -591,13 +654,15 @@ Install_Feature_On_List_Or_All [Documentation] Attempt installation on each member from list (or all). Then look for failures. ${index_list} = List_Indices_Or_All given_list=${member_index_list} ${status_list} = BuiltIn.Create_List - : FOR ${index} IN @{index_list} - \ ${status} ${text} = BuiltIn.Run_Keyword_And_Ignore_Error Install_Feature_On_Member feature_name=${feature_name} member_index=${index} - \ ... timeout=${timeout} - \ BuiltIn.Log ${text} - \ Collections.Append_To_List ${status_list} ${status} - : FOR ${status} IN @{status_list} - \ BuiltIn.Run_Keyword_If "${status}" != "PASS" BuiltIn.Fail ${feature_name} installation failed, see log. + FOR ${index} IN @{index_list} + ${status} ${text} = BuiltIn.Run_Keyword_And_Ignore_Error Install_Feature_On_Member feature_name=${feature_name} member_index=${index} + ... timeout=${timeout} + BuiltIn.Log ${text} + Collections.Append_To_List ${status_list} ${status} + END + FOR ${status} IN @{status_list} + BuiltIn.Run_Keyword_If "${status}" != "PASS" BuiltIn.Fail ${feature_name} installation failed, see log. + END Install_Feature_On_Member [Arguments] ${feature_name} ${member_index} ${timeout}=60s @@ -615,9 +680,10 @@ With_Ssh_To_List_Or_All_Run_Keyword # TODO: For_Index_From_List_Or_All_Run_Keyword applied to With_Ssh_To_Member_Run_Keyword? # TODO: Imagine another keyword, using ScalarClosures and adding member index as first argument for each call. Worth it? ${index_list} = List_Indices_Or_All given_list=${member_index_list} - : FOR ${member_index} IN @{index_list} - \ ${member_ip} = Resolve_IP_Address_For_Member ${member_index} - \ SSHKeywords.Run_Unsafely_Keyword_Over_Temporary_Odl_Session ${member_ip} ${keyword_name} @{args} &{kwargs} + FOR ${member_index} IN @{index_list} + ${member_ip} = Resolve_IP_Address_For_Member ${member_index} + SSHKeywords.Run_Unsafely_Keyword_Over_Temporary_Odl_Session ${member_ip} ${keyword_name} @{args} &{kwargs} + END Safe_With_Ssh_To_List_Or_All_Run_Keyword [Arguments] ${member_index_list} ${keyword_name} @{args} &{kwargs} @@ -647,8 +713,9 @@ Restore_Karaf_Log_On_List_Or_All ClusterManagement__Clean_Directories [Arguments] ${relative_path_list} ${karaf_home} [Documentation] For each relative path, remove files with respect to ${karaf_home}. Return None. - : FOR ${relative_path} IN @{relative_path_list} - \ SSHLibrary.Execute_Command rm -rf ${karaf_home}${/}${relative_path} + FOR ${relative_path} IN @{relative_path_list} + SSHLibrary.Execute_Command rm -rf ${karaf_home}${/}${relative_path} + END Put_As_Json_And_Check_Member_List_Or_All [Arguments] ${uri} ${data} ${member_index} ${member_index_list}=${EMPTY} @@ -692,27 +759,30 @@ Check_Json_Member_List_Or_All [Documentation] Send a GET with the supplied uri to all or some members defined in ${member_index_list}. ... Then check received data is = ${expected data}. ${index_list} = List_Indices_Or_All given_list=${member_index_list} - : FOR ${index} IN @{index_list} - \ ${data} = Get_From_Member uri=${uri} member_index=${index} - \ TemplatedRequests.Normalize_Jsons_And_Compare ${expected_data} ${data} + FOR ${index} IN @{index_list} + ${data} = Get_From_Member uri=${uri} member_index=${index} + TemplatedRequests.Normalize_Jsons_And_Compare ${expected_data} ${data} + END Check_Item_Occurrence_Member_List_Or_All [Arguments] ${uri} ${dictionary} ${member_index_list}=${EMPTY} [Documentation] Send a GET with the supplied uri to all or some members defined in ${member_index_list}. ... Then check received for occurrences of items expressed in a dictionary ${dictionary}. ${index_list} = List_Indices_Or_All given_list=${member_index_list} - : FOR ${index} IN @{index_list} - \ ${data} = Get_From_Member uri=${uri} member_index=${index} - \ Utils.Check Item Occurrence ${data} ${dictionary} + FOR ${index} IN @{index_list} + ${data} = Get_From_Member uri=${uri} member_index=${index} + Utils.Check Item Occurrence ${data} ${dictionary} + END Check_No_Content_Member_List_Or_All [Arguments] ${uri} ${member_index_list}=${EMPTY} [Documentation] Send a GET with the supplied uri to all or some members defined in ${member_index_list}. ... Then check there is no content. ${index_list} = List_Indices_Or_All given_list=${member_index_list} - : FOR ${index} IN @{index_list} - \ ${session} = Resolve_Http_Session_For_Member member_index=${index} - \ Utils.No_Content_From_URI ${session} ${uri} + FOR ${index} IN @{index_list} + ${session} = Resolve_Http_Session_For_Member member_index=${index} + Utils.No_Content_From_URI ${session} ${uri} + END Get_From_Member [Arguments] ${uri} ${member_index} ${access}=${ACCEPT_EMPTY} @@ -731,9 +801,10 @@ Resolve_IP_Address_For_Members [Arguments] ${member_index_list} [Documentation] Return a list of IP address of given indexes. ${member_ip_list} = BuiltIn.Create_List - : FOR ${index} IN @{member_index_list} - \ ${ip_address} = Collections.Get From Dictionary dictionary=${ClusterManagement__index_to_ip_mapping} key=${index} - \ Collections.Append_To_List ${member_ip_list} ${ip_address} + FOR ${index} IN @{member_index_list} + ${ip_address} = Collections.Get From Dictionary dictionary=${ClusterManagement__index_to_ip_mapping} key=${index} + Collections.Append_To_List ${member_ip_list} ${ip_address} + END [Return] ${member_ip_list} Resolve_Http_Session_For_Member @@ -791,9 +862,10 @@ ClusterManagement__Compute_Derived_Variables @{member_index_list} = BuiltIn.Create_List @{session_list} = BuiltIn.Create_List &{index_to_ip_mapping} = BuiltIn.Create_Dictionary - : FOR ${index} IN RANGE 1 ${int_of_members+1} - \ ClusterManagement__Include_Member_Index ${index} ${member_index_list} ${session_list} ${index_to_ip_mapping} http_timeout=${http_timeout} - \ ... http_retries=${http_retries} + FOR ${index} IN RANGE 1 ${int_of_members+1} + ClusterManagement__Include_Member_Index ${index} ${member_index_list} ${session_list} ${index_to_ip_mapping} http_timeout=${http_timeout} + ... http_retries=${http_retries} + END BuiltIn.Set_Suite_Variable \${ClusterManagement__member_index_list} ${member_index_list} BuiltIn.Set_Suite_Variable \${ClusterManagement__index_to_ip_mapping} ${index_to_ip_mapping} BuiltIn.Set_Suite_Variable \${ClusterManagement__session_list} ${session_list} @@ -829,3 +901,20 @@ Return_Member_IP ${member_int} = BuiltIn.Convert_To_Integer ${member_index} ${member_ip} = Collections.Get_From_Dictionary dictionary=${ClusterManagement__index_to_ip_mapping} key=${member_int} [Return] ${member_ip} + +Check Service Status + [Arguments] ${odl_ip} ${system_ready_state} ${service_state} @{service_list} + [Documentation] Issues the karaf shell command showSvcStatus to verify the ready and service states are the same as the arguments passed + ${service_status_output} = BuiltIn.Run Keyword If ${NUM_ODL_SYSTEM} > 1 KarafKeywords.Issue_Command_On_Karaf_Console showSvcStatus -n ${odl_ip} ${odl_ip} ${KARAF_SHELL_PORT} + ... ELSE KarafKeywords.Issue_Command_On_Karaf_Console showSvcStatus ${odl_ip} ${KARAF_SHELL_PORT} + BuiltIn.Should Contain ${service_status_output} ${system_ready_state} + FOR ${service} IN @{service_list} + BuiltIn.Should Match Regexp ${service_status_output} ${service} +: ${service_state} + END + +Check Status Of Services Is OPERATIONAL + [Arguments] @{service_list} + [Documentation] This keyword will verify whether all the services are operational in all the ODL nodes + FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} + ClusterManagement.Check Service Status ${ODL_SYSTEM_${i+1}_IP} ACTIVE OPERATIONAL @{service_list} + END