... TODO: Unify capitalization of Leaders and Followers.
Library RequestsLibrary # for Create_Session and To_Json
Library Collections
+Library String
+Library ClusterEntities.py
Resource ${CURDIR}/CompareStream.robot
Resource ${CURDIR}/KarafKeywords.robot
Resource ${CURDIR}/SSHKeywords.robot
Resource ../variables/Variables.robot
*** Variables ***
-${ENTITY_OWNER_URI} restconf/operational/entity-owners:entity-owners
+${RESTCONF_URI} rests
${GC_LOG_PATH} ${KARAF_HOME}/data/log
${JAVA_HOME} ${EMPTY} # releng/builder scripts should provide correct value
${JOLOKIA_CONF_SHARD_MANAGER_URI} jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore
${JOLOKIA_OPER_LOCAL_SHARDS_URI} jolokia/read/org.opendaylight.controller:type=DistributedOperationalDatastore,Category=ShardManager,name=shard-manager-operational/LocalShards
${JOLOKIA_READ_URI} jolokia/read/org.opendaylight.controller
# Bug 9044 workaround: delete etc/host.key before restart.
-@{ODL_DEFAULT_DATA_PATHS} tmp/ data/ cache/ snapshots/ journal/ etc/opendaylight/current/ etc/host.key
+@{ODL_DEFAULT_DATA_PATHS} tmp/ data/ cache/ snapshots/ journal/ segmented-journal/ etc/opendaylight/current/ etc/host.key
${RESTCONF_MODULES_DIR} ${CURDIR}/../variables/restconf/modules
-${SINGLETON_NETCONF_DEVICE_ID_PREFIX} /odl-general-entity:entity[odl-general-entity:name='KeyedInstanceIdentifier{targetType=interface org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node, path=[org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NetworkTopology, org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.Topology[key=TopologyKey [_topologyId=Uri [_value=topology-netconf]]], org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node[key=NodeKey [_nodeId=Uri [_value=
-${SINGLETON_NETCONF_DEVICE_ID_SUFFIX} ]]]]}']
-${SINGLETON_BGPCEP_DEVICE_ID_PREFIX} /odl-general-entity:entity[odl-general-entity:name='
-${SINGLETON_BGPCEP_DEVICE_ID_SUFFIX} -service-group']
+${SINGLETON_NETCONF_DEVICE_ID_PREFIX} KeyedInstanceIdentifier{targetType=interface org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node, path=[org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NetworkTopology, org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.Topology[key=TopologyKey{_topologyId=Uri{_value=topology-netconf}}], org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node[key=NodeKey{_nodeId=Uri{_value=
+${SINGLETON_NETCONF_DEVICE_ID_SUFFIX} }}]]}
+${SINGLETON_BGPCEP_DEVICE_ID_SUFFIX} -service-group
+&{SINGLETON_DEVICE_ID_PREFIX} bgpcep=${EMPTY} netconf=${SINGLETON_NETCONF_DEVICE_ID_PREFIX} openflow=${EMPTY} sxp=${EMPTY}
+&{SINGLETON_DEVICE_ID_SUFFIX} bgpcep=${SINGLETON_BGPCEP_DEVICE_ID_SUFFIX} netconf=${SINGLETON_NETCONF_DEVICE_ID_SUFFIX} openflow=${EMPTY} sxp=${EMPTY}
${SINGLETON_ELECTION_ENTITY_TYPE} org.opendaylight.mdsal.ServiceEntityType
${SINGLETON_CHANGE_OWNERSHIP_ENTITY_TYPE} org.opendaylight.mdsal.AsyncServiceCloseEntityType
${NODE_ROLE_INDEX_START} 1
[Arguments] ${member_index_list}=${EMPTY}
[Documentation] Fail if no-sync is detected on a member from list (or any).
${index_list} = List_Indices_Or_All given_list=${member_index_list}
- : FOR ${index} IN @{index_list} # usually: 1, 2, 3.
- \ ${status} = Get_Sync_Status_Of_Member member_index=${index}
- \ BuiltIn.Continue_For_Loop_If 'True' == '${status}'
- \ BuiltIn.Fail Index ${index} has incorrect status: ${status}
+ FOR ${index} IN @{index_list} # usually: 1, 2, 3.
+ ${status} = Get_Sync_Status_Of_Member member_index=${index}
+ BuiltIn.Continue_For_Loop_If 'True' == '${status}'
+ BuiltIn.Fail Index ${index} has incorrect status: ${status}
+ END
Get_Sync_Status_Of_Member
[Arguments] ${member_index}
${oper_status} = ClusterManagement__Parse_Sync_Status shard_manager_text=${oper_text}
[Return] ${oper_status}
-Dump_Local_Shards_For_Each_Member
- [Arguments] ${member_index_list}=${EMPTY}
- [Documentation] Obtain IP, two GETs from jolokia URIs, return combined local shard list
- ${index_list} = List_Indices_Or_All given_list=${member_index_list}
- : FOR ${member_index} IN @{index_list} # usually: 1, 2, 3.
- \ ${session} = Resolve_Http_Session_For_Member member_index=${member_index}
- \ ${conf_shard_list} = Wait Until Keyword Succeeds 60 2 TemplatedRequests.Get_As_Json_From_Uri uri=${JOLOKIA_CONFIG_LOCAL_SHARDS_URI}
- \ ... session=${session}
- \ Log ${conf_shard_list}
- \ ${oper_shard_list} = Wait Until Keyword Succeeds 60 2 TemplatedRequests.Get_As_Json_From_Uri uri=${JOLOKIA_OPER_LOCAL_SHARDS_URI}
- \ ... session=${session}
- \ Log ${oper_shard_list}
-
Verify_Leader_Exists_For_Each_Shard
[Arguments] ${shard_name_list} ${shard_type}=operational ${member_index_list}=${EMPTY} ${verify_restconf}=True
[Documentation] For each shard name, call Get_Leader_And_Followers_For_Shard.
... Not much logic there, but single Keyword is useful when using BuiltIn.Wait_Until_Keyword_Succeeds.
- : FOR ${shard_name} IN @{shard_name_list}
- \ Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} validate=True member_index_list=${member_index_list} verify_restconf=${verify_restconf}
+ FOR ${shard_name} IN @{shard_name_list}
+ Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} validate=True member_index_list=${member_index_list} verify_restconf=${verify_restconf}
+ END
Get_Leader_And_Followers_For_Shard
[Arguments] ${shard_name}=default ${shard_type}=operational ${validate}=True ${member_index_list}=${EMPTY} ${verify_restconf}=True ${http_timeout}=${EMPTY}
${ds_type} = BuiltIn.Set_Variable_If '${shard_type}' != 'config' operational config
${leader_list} = BuiltIn.Create_List
${follower_list} = BuiltIn.Create_List
- : FOR ${index} IN @{index_list} # usually: 1, 2, 3.
- \ ${raft_state} = Get_Raft_State_Of_Shard_At_Member shard_name=${shard_name} shard_type=${ds_type} member_index=${index} verify_restconf=${verify_restconf}
- \ ... http_timeout=${http_timeout}
- \ BuiltIn.Run_Keyword_If 'Follower' == '${raft_state}' Collections.Append_To_List ${follower_list} ${index}
- \ ... ELSE IF 'Leader' == '${raft_state}' Collections.Append_To_List ${leader_list} ${index}
- \ ... ELSE IF ${validate} BuiltIn.Fail Unrecognized Raft state: ${raft_state}
+ FOR ${index} IN @{index_list} # usually: 1, 2, 3.
+ ${raft_state} = Get_Raft_State_Of_Shard_At_Member shard_name=${shard_name} shard_type=${ds_type} member_index=${index} verify_restconf=${verify_restconf}
+ ... http_timeout=${http_timeout}
+ BuiltIn.Run_Keyword_If 'Follower' == '${raft_state}' Collections.Append_To_List ${follower_list} ${index}
+ ... ELSE IF 'Leader' == '${raft_state}' Collections.Append_To_List ${leader_list} ${index}
+ ... ELSE IF ${validate} BuiltIn.Fail Unrecognized Raft state: ${raft_state}
+ END
[Return] ${leader_list} ${follower_list}
Get_Raft_State_Of_Shard_At_Member
... http_timeout=${http_timeout}
[Return] ${raft_state}
+Get_Raft_State_Of_Shard_Of_All_Member_Nodes
+ [Arguments] ${shard_name}=default ${shard_type}=config ${member_index_list}=${EMPTY}
+ [Documentation] Get raft state of shard of all member nodes
+ ${index_list} = List_Indices_Or_All given_list=${member_index_list}
+ Collections.Sort_List ${index_list}
+ FOR ${index} IN @{index_list}
+ ClusterManagement.Get Raft State Of Shard At Member shard_name=${shard_name} shard_type=${shard_type} member_index=${index}
+ END
+
Get_Raft_Property_From_Shard_Member
[Arguments] ${property} ${shard_name} ${shard_type} ${member_index} ${verify_restconf}=False ${http_timeout}=${EMPTY}
[Documentation] Send request to Jolokia on indexed member, return extracted Raft property.
Collections.Remove_Values_From_List ${successor_list} ${owner}
[Return] ${owner} ${successor_list}
-Get_Owner_And_Candidates_For_Device_Old
+Get_Owner_And_Candidates_For_Device_Rpc
[Arguments] ${device_name} ${device_type} ${member_index} ${http_timeout}=${EMPTY}
[Documentation] Returns the owner and a list of candidates for the SB device ${device_name} of type ${device_type}. Request is sent to member ${member_index}.
... Candidates are all members that register to own a device, so the list of candiates includes the owner.
... Note that "candidate list" definition currently differs between Beryllium and Boron.
... It is recommended to use Get_Owner_And_Successors_For_Device instead of this keyword, see documentation there.
BuiltIn.Comment TODO: Can this implementation be changed to call Get_Owner_And_Candidates_For_Type_And_Id?
- ${session} = Resolve_Http_Session_For_Member member_index=${member_index}
- ${data} = TemplatedRequests.Get_As_Json_From_Uri uri=${ENTITY_OWNER_URI} session=${session} http_timeout=${http_timeout}
- ${candidate_list} = BuiltIn.Create_List
+ ${index} = BuiltIn.Convert_To_Integer ${member_index}
+ ${ip} = Resolve_IP_Address_For_Member member_index=${index}
${entity_type} = BuiltIn.Set_Variable_If '${device_type}' == 'netconf' netconf-node/${device_name} ${device_type}
- ${clear_data} = BuiltIn.Run_Keyword_If '${device_type}' == 'openflow' or '${device_type}' == 'netconf' Extract_OpenFlow_Device_Data ${data}
- ... ELSE IF '${device_type}' == 'ovsdb' Extract_Ovsdb_Device_Data ${data}
- ... ELSE IF '${device_type}' == 'org.opendaylight.mdsal.ServiceEntityType' Extract_Service_Entity_Type ${data}
- ... ELSE Fail Not recognized device type: ${device_type}
- ${json} = RequestsLibrary.To_Json ${clear_data}
- ${entity_type_list} = Collections.Get_From_Dictionary &{json}[entity-owners] entity-type
- ${entity_type_index} = Utils.Get_Index_From_List_Of_Dictionaries ${entity_type_list} type ${entity_type}
- BuiltIn.Should_Not_Be_Equal_As_Integers ${entity_type_index} -1 No Entity Owner found for ${device_type}
- ${entity_list} = Collections.Get_From_Dictionary @{entity_type_list}[${entity_type_index}] entity
- ${entity_index} = Utils.Get_Index_From_List_Of_Dictionaries ${entity_list} id ${device_name}
- BuiltIn.Should_Not_Be_Equal_As_Integers ${entity_index} -1 Device ${device_name} not found in Entity Owner ${device_type}
- ${entity_owner} = Collections.Get_From_Dictionary @{entity_list}[${entity_index}] owner
+ ${url} = BuiltIn.Catenate SEPARATOR= http:// ${ip} :8181/ ${RESTCONF_URI}
+ ${entity_result} = ClusterEntities.get entity ${url} ${entity_type} ${device_name}
+ ${entity_candidates} = Collections.Get_From_Dictionary ${entity_result} candidates
+ ${entity_owner} = Collections.Get_From_Dictionary ${entity_result} owner
BuiltIn.Should_Not_Be_Empty ${entity_owner} No owner found for ${device_name}
${owner} = String.Replace_String ${entity_owner} member- ${EMPTY}
${owner} = BuiltIn.Convert_To_Integer ${owner}
- ${entity_candidates_list} = Collections.Get_From_Dictionary @{entity_list}[${entity_index}] candidate
- : FOR ${entity_candidate} IN @{entity_candidates_list}
- \ ${candidate} = String.Replace_String &{entity_candidate}[name] member- ${EMPTY}
- \ ${candidate} = BuiltIn.Convert_To_Integer ${candidate}
- \ Collections.Append_To_List ${candidate_list} ${candidate}
+ ${candidate_list} = BuiltIn.Create_List
+ FOR ${entity_candidate} IN @{entity_candidates}
+ ${candidate} = String.Replace_String ${entity_candidate} member- ${EMPTY}
+ ${candidate} = BuiltIn.Convert_To_Integer ${candidate}
+ Collections.Append_To_List ${candidate_list} ${candidate}
+ END
Collections.Sort_List ${candidate_list}
[Return] ${owner} ${candidate_list}
Get_Owner_And_Candidates_For_Device_Singleton
[Arguments] ${device_name} ${device_type} ${member_index} ${http_timeout}=${EMPTY}
[Documentation] Returns the owner and a list of candidates for the SB device ${device_name} of type ${device_type}. Request is sent to member ${member_index}.
- ... Parsing method is selected by device type
- ... Separate kw for every supported device type must be defined
- BuiltIn.Keyword_Should_Exist Get_Owner_And_Candidates_For_Device_Singleton_${device_type}
- BuiltIn.Run_Keyword_And_Return Get_Owner_And_Candidates_For_Device_Singleton_${device_type} ${device_name} ${member_index} http_timeout=${http_timeout}
-
-Get_Owner_And_Candidates_For_Device_Singleton_Netconf
- [Arguments] ${device_name} ${member_index} ${http_timeout}=${EMPTY}
- [Documentation] Returns the owner and a list of candidates for the SB device ${device_name} of type netconf. Request is sent to member ${member_index}.
- ... Parsing method is set as netconf (using netconf device id prefix and suffix)
- # Get election entity type results
- ${type} = BuiltIn.Set_Variable ${SINGLETON_ELECTION_ENTITY_TYPE}
- ${id} = BuiltIn.Set_Variable ${SINGLETON_NETCONF_DEVICE_ID_PREFIX}${device_name}${SINGLETON_NETCONF_DEVICE_ID_SUFFIX}
- ${owner_1} ${candidate_list_1} = Get_Owner_And_Candidates_For_Type_And_Id ${type} ${id} ${member_index} http_timeout=${http_timeout}
- # Get change ownership entity type results
- ${type} = BuiltIn.Set_Variable ${SINGLETON_CHANGE_OWNERSHIP_ENTITY_TYPE}
- ${id} = BuiltIn.Set_Variable ${SINGLETON_NETCONF_DEVICE_ID_PREFIX}${device_name}${SINGLETON_NETCONF_DEVICE_ID_SUFFIX}
- ${owner_2} ${candidate_list_2} = Get_Owner_And_Candidates_For_Type_And_Id ${type} ${id} ${member_index} http_timeout=${http_timeout}
- # Owners must be same, if not, there is still some election or change ownership in progress
- BuiltIn.Should_Be_Equal_As_Integers ${owner_1} ${owner_2} Owners for device ${device_name} are not same
- [Return] ${owner_1} ${candidate_list_1}
-
-Get_Owner_And_Candidates_For_Device_Singleton_Bgpcep
- [Arguments] ${device_name} ${member_index} ${http_timeout}=${EMPTY}
- [Documentation] Returns the owner and a list of candidates for the SB device ${device_name}. Request is sent to member ${member_index}.
+ # Normalize device type to the lowercase as in ${SINGLETON_DEVICE_ID_PREFIX} & ${SINGLETON_DEVICE_ID_SUFFIX}
+ ${device_type} = String.Convert To Lower Case ${device_type}
+ # Set device ID prefix
+ Collections.Dictionary Should Contain Key ${SINGLETON_DEVICE_ID_PREFIX} ${device_type}
+ ${device_id_prefix} = Collections.Get From Dictionary ${SINGLETON_DEVICE_ID_PREFIX} ${device_type}
+ Log ${device_id_prefix}
+ # Set device ID suffix
+ Collections.Dictionary Should Contain Key ${SINGLETON_DEVICE_ID_SUFFIX} ${device_type}
+ ${device_id_suffix} = Collections.Get From Dictionary ${SINGLETON_DEVICE_ID_SUFFIX} ${device_type}
+ Log ${device_id_suffix}
+ # Set device ID
+ ${id} = BuiltIn.Set_Variable ${device_id_prefix}${device_name}${device_id_suffix}
+ Log ${id}
# Get election entity type results
${type} = BuiltIn.Set_Variable ${SINGLETON_ELECTION_ENTITY_TYPE}
- ${id} = BuiltIn.Set_Variable ${SINGLETON_BGPCEP_DEVICE_ID_PREFIX}${device_name}${SINGLETON_BGPCEP_DEVICE_ID_SUFFIX}
${owner_1} ${candidate_list_1} = Get_Owner_And_Candidates_For_Type_And_Id ${type} ${id} ${member_index} http_timeout=${http_timeout}
# Get change ownership entity type results
${type} = BuiltIn.Set_Variable ${SINGLETON_CHANGE_OWNERSHIP_ENTITY_TYPE}
- ${id} = BuiltIn.Set_Variable ${SINGLETON_BGPCEP_DEVICE_ID_PREFIX}${device_name}${SINGLETON_BGPCEP_DEVICE_ID_SUFFIX}
${owner_2} ${candidate_list_2} = Get_Owner_And_Candidates_For_Type_And_Id ${type} ${id} ${member_index} http_timeout=${http_timeout}
# Owners must be same, if not, there is still some election or change ownership in progress
BuiltIn.Should_Be_Equal_As_Integers ${owner_1} ${owner_2} Owners for device ${device_name} are not same
... http_timeout=${http_timeout}
BuiltIn.Return_From_Keyword_If "${status}"=="PASS" ${results}
# If singleton failed, try parsing in old way
- ${status} ${results} = BuiltIn.Run_Keyword_And_Ignore_Error Get_Owner_And_Candidates_For_Device_Old device_name=${device_name} device_type=${device_type} member_index=${member_index}
+ ${status} ${results} = BuiltIn.Run_Keyword_And_Ignore_Error Get_Owner_And_Candidates_For_Device_Rpc device_name=${device_name} device_type=${device_type} member_index=${member_index}
... http_timeout=${http_timeout}
# previous 3 lines (BuilIn.Return.., # If singleton..., ${status}....) could be deleted when old way will not be supported anymore
BuiltIn.Run_Keyword_If '${status}'=='FAIL' BuiltIn.Fail Could not parse owner and candidates for device ${device_name}
... Note that differs from "given list" semantics used in other keywords,
... namely you cannot use \${EMPTY} to stand for "full list" in this keyword.
BuiltIn.Comment TODO: Find a way to unify and deduplicate code blocks in Get_Owner_And_Candidates_* keywords.
- ${session} = Resolve_Http_Session_For_Member member_index=${member_index}
- ${data} = TemplatedRequests.Get_As_Json_From_Uri uri=${ENTITY_OWNER_URI} session=${session} http_timeout=${http_timeout}
- ${candidate_list} = BuiltIn.Create_List
- ${json} = RequestsLibrary.To_Json ${data}
- ${entity_type_list} = Collections.Get_From_Dictionary &{json}[entity-owners] entity-type
- ${entity_type_index} = Utils.Get_Index_From_List_Of_Dictionaries ${entity_type_list} type ${type}
- BuiltIn.Should_Not_Be_Equal_As_Integers ${entity_type_index} -1 No Entity Owner found for ${type}
- ${entity_list} = Collections.Get_From_Dictionary @{entity_type_list}[${entity_type_index}] entity
- ${entity_index} = Utils.Get_Index_From_List_Of_Dictionaries ${entity_list} id ${id}
- BuiltIn.Should Not_Be_Equal_As_Integers ${entity_index} -1 Id ${id} not found in Entity Owner ${type}
- ${entity_owner} = Collections.Get_From_Dictionary @{entity_list}[${entity_index}] owner
- BuiltIn.Should_Not_Be_Empty ${entity_owner} No owner found for type=${type} id=${id}
- ${owner} = String.Replace_String ${entity_owner} member- ${EMPTY}
- ${owner} = BuiltIn.Convert_To_Integer ${owner}
- ${entity_candidates_list} = Collections.Get_From_Dictionary @{entity_list}[${entity_index}] candidate
- : FOR ${entity_candidate} IN @{entity_candidates_list}
- \ ${candidate} = String.Replace_String &{entity_candidate}[name] member- ${EMPTY}
- \ ${candidate} = BuiltIn.Convert_To_Integer ${candidate}
- \ Collections.Append_To_List ${candidate_list} ${candidate}
- Collections.Sort_List ${candidate_list}
- BuiltIn.Comment TODO: Separate check lines into Verify_Owner_And_Candidates_For_Type_And_Id
- BuiltIn.Run_Keyword_If """${require_candidate_list}""" BuiltIn.Should_Be_Equal ${require_candidate_list} ${candidate_list} Candidate list does not match: ${candidate_list} is not ${require_candidate_list}
- [Return] ${owner} ${candidate_list}
+ ${owner} ${candidates} = Get_Owner_And_Candidates_For_Device_Rpc ${id} ${type} ${member_index} http_timeout=${http_timeout}
+ [Return] ${owner} ${candidates}
Extract_Service_Entity_Type
[Arguments] ${data}
BuiltIn.Return_From_Keyword_If not ${confirm} ${updated_index_list}
# TODO: Convert to WUKS with configurable timeout if it turns out 1 second is not enough.
BuiltIn.Sleep 1s Kill -9 closes open files, which may take longer than ssh overhead, but not long enough to warrant WUKS.
- : FOR ${index} IN @{kill_index_list}
- \ Verify_Karaf_Is_Not_Running_On_Member member_index=${index}
+ FOR ${index} IN @{kill_index_list}
+ Verify_Karaf_Is_Not_Running_On_Member member_index=${index}
+ END
Run_Bash_Command_On_List_Or_All command=netstat -pnatu | grep 2550
[Return] ${updated_index_list}
Stop_Single_Member
- [Arguments] ${member} ${original_index_list}=${EMPTY} ${confirm}=True
+ [Arguments] ${member} ${original_index_list}=${EMPTY} ${confirm}=True ${msg}=${EMPTY}
[Documentation] Convenience keyword that stops the specified member of the cluster.
... The KW will return a list of available members: \${updated index_list}=\${original_index_list}-\${member}
${index_list} = ClusterManagement__Build_List ${member}
+ ${member_ip} = Return_Member_IP ${member}
+ ${msg} = Builtin.Set Variable If "${msg}" == "${EMPTY}" Stopping ODL${member} ${member_ip} Stopping ODL${member} ${member_ip}, ${msg}
+ KarafKeywords.Log_Message_To_Controller_Karaf ${msg}
${updated_index_list} = Stop_Members_From_List_Or_All ${index_list} ${original_index_list} ${confirm}
[Return] ${updated_index_list}
Stop_Members_From_List_Or_All
- [Arguments] ${member_index_list}=${EMPTY} ${original_index_list}=${EMPTY} ${confirm}=True ${timeout}=120s
+ [Arguments] ${member_index_list}=${EMPTY} ${original_index_list}=${EMPTY} ${confirm}=True ${timeout}=360s
[Documentation] If the list is empty, stops all ODL instances. Otherwise stop members based on \${stop_index_list}
... If \${confirm} is True, verify stopped instances are not there anymore.
... The KW will return a list of available members: \${updated index_list}=\${original_index_list}-\${member_index_list}
${updated_index_list} = BuiltIn.Create_List @{index_list}
Collections.Remove_Values_From_List ${updated_index_list} @{stop_index_list}
BuiltIn.Return_From_Keyword_If not ${confirm} ${updated_index_list}
- : FOR ${index} IN @{stop_index_list}
- \ BuiltIn.Wait Until Keyword Succeeds ${timeout} 2s Verify_Karaf_Is_Not_Running_On_Member member_index=${index}
+ FOR ${index} IN @{stop_index_list}
+ BuiltIn.Wait Until Keyword Succeeds ${timeout} 2s Verify_Karaf_Is_Not_Running_On_Member member_index=${index}
+ END
Run_Bash_Command_On_List_Or_All command=netstat -pnatu | grep 2550
[Return] ${updated_index_list}
Start_Single_Member
- [Arguments] ${member} ${wait_for_sync}=True ${timeout}=300s ${check_system_status}=False @{service_list}=@{EMPTY}
+ [Arguments] ${member} ${wait_for_sync}=True ${timeout}=300s ${msg}=${EMPTY} ${check_system_status}=False ${verify_restconf}=True
+ ... ${service_list}=${EMPTY_LIST}
[Documentation] Convenience keyword that starts the specified member of the cluster.
${index_list} = ClusterManagement__Build_List ${member}
- Start_Members_From_List_Or_All ${index_list} ${wait_for_sync} ${timeout} check_system_status=${check_system_status} service_list=@{service_list}
+ ${member_ip} = Return_Member_IP ${member}
+ ${msg} = Builtin.Set Variable If "${msg}" == "${EMPTY}" Starting ODL${member} ${member_ip} Starting ODL${member} ${member_ip}, ${msg}
+ KarafKeywords.Log_Message_To_Controller_Karaf ${msg}
+ Start_Members_From_List_Or_All ${index_list} ${wait_for_sync} ${timeout} check_system_status=${check_system_status} verify_restconf=${verify_restconf} service_list=${service_list}
Start_Members_From_List_Or_All
- [Arguments] ${member_index_list}=${EMPTY} ${wait_for_sync}=True ${timeout}=300s ${karaf_home}=${EMPTY} ${export_java_home}=${EMPTY} ${gc_log_dir}=${EMPTY}
- ... ${check_system_status}=False @{service_list}=@{EMPTY}
+ [Arguments] ${member_index_list}=${EMPTY} ${wait_for_sync}=True ${timeout}=360s ${karaf_home}=${EMPTY} ${export_java_home}=${EMPTY} ${gc_log_dir}=${EMPTY}
+ ... ${check_system_status}=False ${verify_restconf}=True ${service_list}=${EMPTY_LIST}
[Documentation] If the list is empty, start all cluster members. Otherwise, start members based on present indices.
... If ${wait_for_sync}, wait for cluster sync on listed members.
... Optionally karaf_home can be overriden. Optionally specific JAVA_HOME is used for starting.
${gc_filepath} = BuiltIn.Set_Variable_If """${karaf_home}""" != "" ${karaf_home}/data/log/gc_${epoch}.log ${GC_LOG_PATH}/gc_${epoch}.log
${gc_options} = BuiltIn.Set_Variable_If "docker" not in """${node_start_command}""" -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:${gc_filepath} ${EMPTY}
Run_Bash_Command_On_List_Or_All command=${command} ${gc_options} member_index_list=${member_index_list}
- BuiltIn.Return_From_Keyword_If not ${wait_for_sync}
- BuiltIn.Wait_Until_Keyword_Succeeds ${timeout} 10s Check_Cluster_Is_In_Sync member_index_list=${member_index_list}
- BuiltIn.Return_From_Keyword_If not ${check_system_status}
- CompareStream.Run_Keyword_If_At_Least_Oxygen Wait Until Keyword Succeeds 60 2 ClusterManagement.Check Status of Services @{service_list}
+ BuiltIn.Wait_Until_Keyword_Succeeds ${timeout} 10s Verify_Members_Are_Ready ${member_index_list} ${wait_for_sync} ${verify_restconf}
+ ... ${check_system_status} ${service_list}
[Teardown] Run_Bash_Command_On_List_Or_All command=netstat -pnatu | grep 2550
+Verify_Members_Are_Ready
+ [Arguments] ${member_index_list} ${verify_cluster_sync} ${verify_restconf} ${verify_system_status} ${service_list}
+ [Documentation] Verifies the specified readiness conditions for the given listed members after startup.
+ ... If ${verify_cluster_sync}, verifies the datastores have synced with the rest of the cluster.
+ ... If ${verify_restconf}, verifies RESTCONF is available.
+ ... If ${verify_system_status}, verifies the system services are OPERATIONAL.
+ BuiltIn.Run_Keyword_If ${verify_cluster_sync} Check_Cluster_Is_In_Sync ${member_index_list}
+ BuiltIn.Run_Keyword_If ${verify_restconf} Verify_Restconf_Is_Available ${member_index_list}
+ # for backward compatibility, some consumers might not be passing @{service_list}, but since we can't set a list to a default
+ # value, we need to check here if it's empty in order to skip the check which would throw an error
+ BuiltIn.Run_Keyword_If ${verify_system_status} and ("${service_list}" != "[[]]") ClusterManagement.Check Status Of Services Is OPERATIONAL @{service_list}
+
+Verify_Restconf_Is_Available
+ [Arguments] ${member_index_list}
+ ${index_list} = List_Indices_Or_All given_list=${member_index_list}
+ FOR ${index} IN @{index_list}
+ ${session} = Resolve_Http_Session_For_Member member_index=${index}
+ TemplatedRequests.Get_As_Json_Templated session=${session} folder=${RESTCONF_MODULES_DIR} verify=False
+ END
+
Freeze_Single_Member
[Arguments] ${member}
[Documentation] Convenience keyword that stops the specified member of the cluster by freezing the jvm.
... BEWARE: If only a subset of members is cleaned, this causes RetiredGenerationException in Carbon after the affected node re-start.
... See https://bugs.opendaylight.org/show_bug.cgi?id=8138
${index_list} = List_Indices_Or_All given_list=${member_index_list}
- ${command} = Set Variable rm -rf "${karaf_home}/journal" "${karaf_home}/snapshots" "${karaf_home}/data"
- : FOR ${index} IN @{index_list} # usually: 1, 2, 3.
- \ Run_Bash_Command_On_Member command=${command} member_index=${index}
+ ${command} = Set Variable rm -rf "${karaf_home}/"*journal "${karaf_home}/snapshots" "${karaf_home}/data"
+ FOR ${index} IN @{index_list} # usually: 1, 2, 3.
+ Run_Bash_Command_On_Member command=${command} member_index=${index}
+ END
Verify_Karaf_Is_Not_Running_On_Member
[Arguments] ${member_index}
${index_list} = List_Indices_Or_All given_list=${member_index_list}
${source} = Collections.Get_From_Dictionary ${ClusterManagement__index_to_ip_mapping} ${isolate_member_index}
${dport} = BuiltIn.Set_Variable_If '${port}' != '${EMPTY}' --dport ${port} ${EMPTY}
- : FOR ${index} IN @{index_list}
- \ ${destination} = Collections.Get_From_Dictionary ${ClusterManagement__index_to_ip_mapping} ${index}
- \ ${command} = BuiltIn.Set_Variable sudo /sbin/iptables -I OUTPUT -p ${protocol} ${dport} --source ${source} --destination ${destination} -j DROP
- \ BuiltIn.Run_Keyword_If "${index}" != "${isolate_member_index}" Run_Bash_Command_On_Member command=${command} member_index=${isolate_member_index}
+ FOR ${index} IN @{index_list}
+ ${destination} = Collections.Get_From_Dictionary ${ClusterManagement__index_to_ip_mapping} ${index}
+ ${command} = BuiltIn.Set_Variable sudo /sbin/iptables -I OUTPUT -p ${protocol} ${dport} --source ${source} --destination ${destination} -j DROP
+ BuiltIn.Run_Keyword_If "${index}" != "${isolate_member_index}" Run_Bash_Command_On_Member command=${command} member_index=${isolate_member_index}
+ END
${command} = BuiltIn.Set_Variable sudo /sbin/iptables -L -n
${output} = Run_Bash_Command_On_Member command=${command} member_index=${isolate_member_index}
BuiltIn.Log ${output}
${index_list} = List_Indices_Or_All given_list=${member_index_list}
${source} = Collections.Get_From_Dictionary ${ClusterManagement__index_to_ip_mapping} ${rejoin_member_index}
${dport} = BuiltIn.Set_Variable_If '${port}' != '${EMPTY}' --dport ${port} ${EMPTY}
- : FOR ${index} IN @{index_list}
- \ ${destination} = Collections.Get_From_Dictionary ${ClusterManagement__index_to_ip_mapping} ${index}
- \ ${command} = BuiltIn.Set_Variable sudo /sbin/iptables -D OUTPUT -p ${protocol} ${dport} --source ${source} --destination ${destination} -j DROP
- \ BuiltIn.Run_Keyword_If "${index}" != "${rejoin_member_index}" Run_Bash_Command_On_Member command=${command} member_index=${rejoin_member_index}
+ FOR ${index} IN @{index_list}
+ ${destination} = Collections.Get_From_Dictionary ${ClusterManagement__index_to_ip_mapping} ${index}
+ ${command} = BuiltIn.Set_Variable sudo /sbin/iptables -D OUTPUT -p ${protocol} ${dport} --source ${source} --destination ${destination} -j DROP
+ BuiltIn.Run_Keyword_If "${index}" != "${rejoin_member_index}" Run_Bash_Command_On_Member command=${command} member_index=${rejoin_member_index}
+ END
${command} = BuiltIn.Set_Variable sudo /sbin/iptables -L -n
${output} = Run_Bash_Command_On_Member command=${command} member_index=${rejoin_member_index}
BuiltIn.Log ${output}
[Arguments] ${command} ${member_index_list}=${EMPTY} ${return_success_only}=False ${log_on_success}=True ${log_on_failure}=True ${stderr_must_be_empty}=True
[Documentation] Cycle through indices (or all), run bash command on each, using temporary SSH session and restoring the previously active one.
${index_list} = List_Indices_Or_All given_list=${member_index_list}
- : FOR ${index} IN @{index_list}
- \ Check_Bash_Command_On_Member command=${command} member_index=${index} return_success_only=${return_success_only} log_on_success=${log_on_success} log_on_failure=${log_on_failure}
- \ ... stderr_must_be_empty=${stderr_must_be_empty}
+ FOR ${index} IN @{index_list}
+ Check_Bash_Command_On_Member command=${command} member_index=${index} return_success_only=${return_success_only} log_on_success=${log_on_success} log_on_failure=${log_on_failure}
+ ... stderr_must_be_empty=${stderr_must_be_empty}
+ END
Check_Bash_Command_On_Member
[Arguments] ${command} ${member_index} ${return_success_only}=False ${log_on_success}=True ${log_on_failure}=True ${stderr_must_be_empty}=True
[Documentation] Cycle through indices (or all), run command on each.
# TODO: Migrate callers to Check_Bash_Command_*
${index_list} = List_Indices_Or_All given_list=${member_index_list}
- : FOR ${index} IN @{index_list}
- \ Run_Bash_Command_On_Member command=${command} member_index=${index}
+ FOR ${index} IN @{index_list}
+ Run_Bash_Command_On_Member command=${command} member_index=${index}
+ END
Run_Bash_Command_On_Member
[Arguments] ${command} ${member_index}
[Arguments] ${command} ${member_index_list}=${EMPTY} ${timeout}=10s
[Documentation] Cycle through indices (or all), run karaf command on each.
${index_list} = List_Indices_Or_All given_list=${member_index_list}
- : FOR ${index} IN @{index_list}
- \ ${member_ip} = Collections.Get_From_Dictionary dictionary=${ClusterManagement__index_to_ip_mapping} key=${index}
- \ KarafKeywords.Safe_Issue_Command_On_Karaf_Console ${command} ${member_ip} timeout=${timeout}
+ FOR ${index} IN @{index_list}
+ ${member_ip} = Collections.Get_From_Dictionary dictionary=${ClusterManagement__index_to_ip_mapping} key=${index}
+ KarafKeywords.Safe_Issue_Command_On_Karaf_Console ${command} ${member_ip} timeout=${timeout}
+ END
Run_Karaf_Command_On_Member
[Arguments] ${command} ${member_index} ${timeout}=10s
[Documentation] Attempt installation on each member from list (or all). Then look for failures.
${index_list} = List_Indices_Or_All given_list=${member_index_list}
${status_list} = BuiltIn.Create_List
- : FOR ${index} IN @{index_list}
- \ ${status} ${text} = BuiltIn.Run_Keyword_And_Ignore_Error Install_Feature_On_Member feature_name=${feature_name} member_index=${index}
- \ ... timeout=${timeout}
- \ BuiltIn.Log ${text}
- \ Collections.Append_To_List ${status_list} ${status}
- : FOR ${status} IN @{status_list}
- \ BuiltIn.Run_Keyword_If "${status}" != "PASS" BuiltIn.Fail ${feature_name} installation failed, see log.
+ FOR ${index} IN @{index_list}
+ ${status} ${text} = BuiltIn.Run_Keyword_And_Ignore_Error Install_Feature_On_Member feature_name=${feature_name} member_index=${index}
+ ... timeout=${timeout}
+ BuiltIn.Log ${text}
+ Collections.Append_To_List ${status_list} ${status}
+ END
+ FOR ${status} IN @{status_list}
+ BuiltIn.Run_Keyword_If "${status}" != "PASS" BuiltIn.Fail ${feature_name} installation failed, see log.
+ END
Install_Feature_On_Member
[Arguments] ${feature_name} ${member_index} ${timeout}=60s
# TODO: For_Index_From_List_Or_All_Run_Keyword applied to With_Ssh_To_Member_Run_Keyword?
# TODO: Imagine another keyword, using ScalarClosures and adding member index as first argument for each call. Worth it?
${index_list} = List_Indices_Or_All given_list=${member_index_list}
- : FOR ${member_index} IN @{index_list}
- \ ${member_ip} = Resolve_IP_Address_For_Member ${member_index}
- \ SSHKeywords.Run_Unsafely_Keyword_Over_Temporary_Odl_Session ${member_ip} ${keyword_name} @{args} &{kwargs}
+ FOR ${member_index} IN @{index_list}
+ ${member_ip} = Resolve_IP_Address_For_Member ${member_index}
+ SSHKeywords.Run_Unsafely_Keyword_Over_Temporary_Odl_Session ${member_ip} ${keyword_name} @{args} &{kwargs}
+ END
Safe_With_Ssh_To_List_Or_All_Run_Keyword
[Arguments] ${member_index_list} ${keyword_name} @{args} &{kwargs}
ClusterManagement__Clean_Directories
[Arguments] ${relative_path_list} ${karaf_home}
[Documentation] For each relative path, remove files with respect to ${karaf_home}. Return None.
- : FOR ${relative_path} IN @{relative_path_list}
- \ SSHLibrary.Execute_Command rm -rf ${karaf_home}${/}${relative_path}
+ FOR ${relative_path} IN @{relative_path_list}
+ SSHLibrary.Execute_Command rm -rf ${karaf_home}${/}${relative_path}
+ END
Put_As_Json_And_Check_Member_List_Or_All
[Arguments] ${uri} ${data} ${member_index} ${member_index_list}=${EMPTY}
Wait Until Keyword Succeeds 5s 1s Check_Json_Member_List_Or_All uri=${uri} expected_data=${data} member_index_list=${member_index_list}
[Return] ${response_text}
+Put_As_Json_And_Check_Member_List_Or_All_RFC8040
+ # FIXME: merge this keyword with Bierman02 version above when OVSDB test is migrated to RFC8040
+ [Arguments] ${uri} ${data} ${member_index} ${member_index_list}=${EMPTY}
+ [Documentation] Send a PUT with the supplied uri ${uri} and body ${data} to member ${member_index}.
+ ... Then check data is replicated in all or some members defined in ${member_index_list}.
+ ${response_text} = Put_As_Json_To_Member uri=${uri} data=${data} member_index=${member_index}
+ Wait Until Keyword Succeeds 5s 1s Check_Json_Member_List_Or_All uri=${uri}?content=config expected_data=${data} member_index_list=${member_index_list}
+ [Return] ${response_text}
+
Put_As_Json_To_Member
[Arguments] ${uri} ${data} ${member_index}
[Documentation] Send a PUT with the supplied uri and data to member ${member_index}.
[Documentation] Send a GET with the supplied uri to all or some members defined in ${member_index_list}.
... Then check received data is = ${expected data}.
${index_list} = List_Indices_Or_All given_list=${member_index_list}
- : FOR ${index} IN @{index_list}
- \ ${data} = Get_From_Member uri=${uri} member_index=${index}
- \ TemplatedRequests.Normalize_Jsons_And_Compare ${expected_data} ${data}
+ FOR ${index} IN @{index_list}
+ ${data} = Get_From_Member uri=${uri} member_index=${index}
+ TemplatedRequests.Normalize_Jsons_And_Compare ${expected_data} ${data}
+ END
Check_Item_Occurrence_Member_List_Or_All
[Arguments] ${uri} ${dictionary} ${member_index_list}=${EMPTY}
[Documentation] Send a GET with the supplied uri to all or some members defined in ${member_index_list}.
... Then check received for occurrences of items expressed in a dictionary ${dictionary}.
${index_list} = List_Indices_Or_All given_list=${member_index_list}
- : FOR ${index} IN @{index_list}
- \ ${data} = Get_From_Member uri=${uri} member_index=${index}
- \ Utils.Check Item Occurrence ${data} ${dictionary}
+ FOR ${index} IN @{index_list}
+ ${data} = Get_From_Member uri=${uri} member_index=${index}
+ Utils.Check Item Occurrence ${data} ${dictionary}
+ END
Check_No_Content_Member_List_Or_All
[Arguments] ${uri} ${member_index_list}=${EMPTY}
[Documentation] Send a GET with the supplied uri to all or some members defined in ${member_index_list}.
... Then check there is no content.
${index_list} = List_Indices_Or_All given_list=${member_index_list}
- : FOR ${index} IN @{index_list}
- \ ${session} = Resolve_Http_Session_For_Member member_index=${index}
- \ Utils.No_Content_From_URI ${session} ${uri}
+ FOR ${index} IN @{index_list}
+ ${session} = Resolve_Http_Session_For_Member member_index=${index}
+ Utils.No_Content_From_URI ${session} ${uri}
+ END
Get_From_Member
[Arguments] ${uri} ${member_index} ${access}=${ACCEPT_EMPTY}
[Arguments] ${member_index_list}
[Documentation] Return a list of IP address of given indexes.
${member_ip_list} = BuiltIn.Create_List
- : FOR ${index} IN @{member_index_list}
- \ ${ip_address} = Collections.Get From Dictionary dictionary=${ClusterManagement__index_to_ip_mapping} key=${index}
- \ Collections.Append_To_List ${member_ip_list} ${ip_address}
+ FOR ${index} IN @{member_index_list}
+ ${ip_address} = Collections.Get From Dictionary dictionary=${ClusterManagement__index_to_ip_mapping} key=${index}
+ Collections.Append_To_List ${member_ip_list} ${ip_address}
+ END
[Return] ${member_ip_list}
Resolve_Http_Session_For_Member
@{member_index_list} = BuiltIn.Create_List
@{session_list} = BuiltIn.Create_List
&{index_to_ip_mapping} = BuiltIn.Create_Dictionary
- : FOR ${index} IN RANGE 1 ${int_of_members+1}
- \ ClusterManagement__Include_Member_Index ${index} ${member_index_list} ${session_list} ${index_to_ip_mapping} http_timeout=${http_timeout}
- \ ... http_retries=${http_retries}
+ FOR ${index} IN RANGE 1 ${int_of_members+1}
+ ClusterManagement__Include_Member_Index ${index} ${member_index_list} ${session_list} ${index_to_ip_mapping} http_timeout=${http_timeout}
+ ... http_retries=${http_retries}
+ END
BuiltIn.Set_Suite_Variable \${ClusterManagement__member_index_list} ${member_index_list}
BuiltIn.Set_Suite_Variable \${ClusterManagement__index_to_ip_mapping} ${index_to_ip_mapping}
BuiltIn.Set_Suite_Variable \${ClusterManagement__session_list} ${session_list}
${service_status_output} = BuiltIn.Run Keyword If ${NUM_ODL_SYSTEM} > 1 KarafKeywords.Issue_Command_On_Karaf_Console showSvcStatus -n ${odl_ip} ${odl_ip} ${KARAF_SHELL_PORT}
... ELSE KarafKeywords.Issue_Command_On_Karaf_Console showSvcStatus ${odl_ip} ${KARAF_SHELL_PORT}
BuiltIn.Should Contain ${service_status_output} ${system_ready_state}
- : FOR ${service} IN @{service_list}
- \ BuiltIn.Should Match Regexp ${service_status_output} ${service} +: ${service_state}
+ FOR ${service} IN @{service_list}
+ BuiltIn.Should Match Regexp ${service_status_output} ${service} +: ${service_state}
+ END
-Check Status of Services
+Check Status Of Services Is OPERATIONAL
[Arguments] @{service_list}
[Documentation] This keyword will verify whether all the services are operational in all the ODL nodes
- : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
- \ ClusterManagement.Check Service Status ${ODL_SYSTEM_${i+1}_IP} ACTIVE OPERATIONAL @{service_list}
+ FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
+ ClusterManagement.Check Service Status ${ODL_SYSTEM_${i+1}_IP} ACTIVE OPERATIONAL @{service_list}
+ END
Code Review / integration / test.git / blobdiff