*** Variables ***
${ENTITY_OWNER_URI} restconf/operational/entity-owners:entity-owners
+${GC_LOG_PATH} ${KARAF_HOME}/data/log
${JAVA_HOME} ${EMPTY} # releng/builder scripts should provide correct value
${JOLOKIA_CONF_SHARD_MANAGER_URI} jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore
${JOLOKIA_OPER_SHARD_MANAGER_URI} jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-operational,type=DistributedOperationalDatastore
${JOLOKIA_READ_URI} jolokia/read/org.opendaylight.controller
${KARAF_HOME} ${WORKSPACE}${/}${BUNDLEFOLDER} # TODO: Migrate to Variables.robot
-@{ODL_DEFAULT_DATA_PATHS} tmp/ data/ cache/ snapshots/ journal/ etc/opendaylight/current/
+# Bug 9044 workaround: delete etc/host.key before restart.
+@{ODL_DEFAULT_DATA_PATHS} tmp/ data/ cache/ snapshots/ journal/ etc/opendaylight/current/ etc/host.key
${RESTCONF_MODULES_DIR} ${CURDIR}/../variables/restconf/modules
${SINGLETON_NETCONF_DEVICE_ID_PREFIX} /odl-general-entity:entity[odl-general-entity:name='KeyedInstanceIdentifier{targetType=interface org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node, path=[org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NetworkTopology, org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.Topology[key=TopologyKey [_topologyId=Uri [_value=topology-netconf]]], org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node[key=NodeKey [_nodeId=Uri [_value=
${SINGLETON_NETCONF_DEVICE_ID_SUFFIX} ]]]]}']
\ Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} validate=True member_index_list=${member_index_list} verify_restconf=${verify_restconf}
Get_Leader_And_Followers_For_Shard
- [Arguments] ${shard_name}=default ${shard_type}=operational ${validate}=True ${member_index_list}=${EMPTY} ${verify_restconf}=True
+ [Arguments] ${shard_name}=default ${shard_type}=operational ${validate}=True ${member_index_list}=${EMPTY} ${verify_restconf}=True ${http_timeout}=5
[Documentation] Get role lists, validate there is one leader, return the leader and list of followers.
... Optionally, issue GET to a simple restconf URL to make sure subsequent operations will not encounter 503.
${leader_list} ${follower_list} = Get_State_Info_For_Shard shard_name=${shard_name} shard_type=${shard_type} validate=True member_index_list=${member_index_list}
- ... verify_restconf=${verify_restconf}
+ ... verify_restconf=${verify_restconf} http_timeout=${http_timeout}
${leader_count} = BuiltIn.Get_Length ${leader_list}
BuiltIn.Run_Keyword_If ${leader_count} < 1 BuiltIn.Fail No leader found.
BuiltIn.Length_Should_Be ${leader_list} ${1} Too many Leaders.
[Return] ${leader} ${follower_list}
Get_State_Info_For_Shard
- [Arguments] ${shard_name}=default ${shard_type}=operational ${validate}=False ${member_index_list}=${EMPTY} ${verify_restconf}=False
+ [Arguments] ${shard_name}=default ${shard_type}=operational ${validate}=False ${member_index_list}=${EMPTY} ${verify_restconf}=False ${http_timeout}=5
[Documentation] Return lists of Leader and Follower member indices from a given member index list
... (or from the full list if empty). If \${shard_type} is not 'config', 'operational' is assumed.
... If \${validate}, Fail if raft state is not Leader or Follower (for example on Candidate).
${follower_list} = BuiltIn.Create_List
: FOR ${index} IN @{index_list} # usually: 1, 2, 3.
\ ${raft_state} = Get_Raft_State_Of_Shard_At_Member shard_name=${shard_name} shard_type=${ds_type} member_index=${index} verify_restconf=${verify_restconf}
+ \ ... http_timeout=${http_timeout}
\ BuiltIn.Run_Keyword_If 'Follower' == '${raft_state}' Collections.Append_To_List ${follower_list} ${index}
\ ... ELSE IF 'Leader' == '${raft_state}' Collections.Append_To_List ${leader_list} ${index}
\ ... ELSE IF ${validate} BuiltIn.Fail Unrecognized Raft state: ${raft_state}
[Return] ${leader_list} ${follower_list}
Get_Raft_State_Of_Shard_At_Member
- [Arguments] ${shard_name} ${shard_type} ${member_index} ${verify_restconf}=False
+ [Arguments] ${shard_name} ${shard_type} ${member_index} ${verify_restconf}=False ${http_timeout}=5
[Documentation] Send request to Jolokia on indexed member, return extracted Raft status.
... Optionally, check restconf works.
+ ${raft_state} = Get_Raft_Property_From_Shard_Member RaftState ${shard_name} ${shard_type} ${member_index} verify_restconf=${verify_restconf}
+ ... http_timeout=${http_timeout}
+ [Return] ${raft_state}
+
+Get_Raft_Property_From_Shard_Member
+ [Arguments] ${property} ${shard_name} ${shard_type} ${member_index} ${verify_restconf}=False ${http_timeout}=5
+ [Documentation] Send request to Jolokia on indexed member, return extracted Raft property.
+ ... Optionally, check restconf works.
${session} = Resolve_Http_Session_For_Member member_index=${member_index}
# TODO: Does the used URI tend to generate large data which floods log.html?
- BuiltIn.Run_Keyword_If ${verify_restconf} TemplatedRequests.Get_As_Json_Templated session=${session} folder=${RESTCONF_MODULES_DIR} verify=False http_timeout=5
+ BuiltIn.Run_Keyword_If ${verify_restconf} TemplatedRequests.Get_As_Json_Templated session=${session} folder=${RESTCONF_MODULES_DIR} verify=False http_timeout=${http_timeout}
${type_class} = Resolve_Shard_Type_Class shard_type=${shard_type}
${uri} = BuiltIn.Set_Variable ${JOLOKIA_READ_URI}:Category=Shards,name=member-${member_index}-shard-${shard_name}-${shard_type},type=${type_class}
- ${data_text} = TemplatedRequests.Get_As_Json_From_Uri uri=${uri} session=${session} http_timeout=5
+ ${data_text} = TemplatedRequests.Get_As_Json_From_Uri uri=${uri} session=${session} http_timeout=${http_timeout}
${data_object} = RequestsLibrary.To_Json ${data_text}
${value} = Collections.Get_From_Dictionary ${data_object} value
- ${raft_state} = Collections.Get_From_Dictionary ${value} RaftState
- [Return] ${raft_state}
+ ${raft_property} = Collections.Get_From_Dictionary ${value} ${property}
+ [Return] ${raft_property}
Verify_Shard_Leader_Elected
- [Arguments] ${shard_name} ${shard_type} ${new_elected} ${old_leader} ${member_index_list}=${EMPTY}
+ [Arguments] ${shard_name} ${shard_type} ${new_elected} ${old_leader} ${member_index_list}=${EMPTY} ${verify_restconf}=True
[Documentation] Verify new leader was elected or remained the same. Bool paramter ${new_elected} indicates if
... new leader is elected or should remained the same as ${old_leader}
- ${leader} ${followers}= Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${member_index_list}
+ ${leader} ${followers}= Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${member_index_list} verify_restconf=${verify_restconf}
BuiltIn.Run_Keyword_If ${new_elected} BuiltIn.Should_Not_Be_Equal_As_Numbers ${old_leader} ${leader}
BuiltIn.Run_Keyword_Unless ${new_elected} BuiltIn.Should_Be_Equal_As_numbers ${old_leader} ${leader}
BuiltIn.Return_From_Keyword ${leader} ${followers}
${owner} ${successor_list} = Get_Owner_And_Successors_For_Device device_name=${device_name} device_type=${device_type} member_index=${member_index}
Collections.List_Should_Contain_Value ${index_list} ${owner} Owner ${owner} is not in candidate list ${index_list}
# In Beryllium or after stopping an instance, the removed instance does not show in the candidate list.
- ${expected_candidate_list_origin} = BuiltIn.Set_Variable_If '${ODL_STREAM}' == 'beryllium' or ${after_stop} ${index_list} ${ClusterManagement__member_index_list}
+ ${expected_candidate_list_origin} = BuiltIn.Set_Variable_If ${after_stop} ${index_list} ${ClusterManagement__member_index_list}
# We do not want to manipulate either origin list.
${expected_successor_list} = BuiltIn.Create_List @{expected_candidate_list_origin}
Collections.Remove_Values_From_List ${expected_successor_list} ${owner}
[Return] ${owner} ${index_list}
Get_Owner_And_Successors_For_Device
- [Arguments] ${device_name} ${device_type} ${member_index}
+ [Arguments] ${device_name} ${device_type} ${member_index} ${http_timeout}=${EMPTY}
[Documentation] Returns the owner and a list of successors for the SB device ${device_name} of type ${device_type}. Request is sent to member ${member_index}.
... Successors are those device candidates not elected as owner. The list of successors = (list of candidates) - (owner).
... The returned successor list is sorted numerically.
... Note that "candidate list" definition currently differs between Beryllium and Boron.
... Use Verify_Owner_And_Successors_For_Device if you want the older semantics (inaccessible nodes not present in the list).
- ${owner} ${candidate_list} = Get_Owner_And_Candidates_For_Device device_name=${device_name} device_type=${device_type} member_index=${member_index}
+ # TODO: somewhere to introduce ${DEFAULT_RESTCONF_DATASTORE_TIMEOUT}. Value may depend on protocol (ask vs tell) and perhaps stream.
+ ${owner} ${candidate_list} = Get_Owner_And_Candidates_For_Device device_name=${device_name} device_type=${device_type} member_index=${member_index} http_timeout=${http_timeout}
${successor_list} = BuiltIn.Create_List @{candidate_list} # Copy operation is not required, but new variable name requires a line anyway.
Collections.Remove_Values_From_List ${successor_list} ${owner}
[Return] ${owner} ${successor_list}
Get_Owner_And_Candidates_For_Device_Old
- [Arguments] ${device_name} ${device_type} ${member_index}
+ [Arguments] ${device_name} ${device_type} ${member_index} ${http_timeout}=${EMPTY}
[Documentation] Returns the owner and a list of candidates for the SB device ${device_name} of type ${device_type}. Request is sent to member ${member_index}.
... Candidates are all members that register to own a device, so the list of candiates includes the owner.
... The returned candidate list is sorted numerically.
... It is recommended to use Get_Owner_And_Successors_For_Device instead of this keyword, see documentation there.
BuiltIn.Comment TODO: Can this implementation be changed to call Get_Owner_And_Candidates_For_Type_And_Id?
${session} = Resolve_Http_Session_For_Member member_index=${member_index}
- ${data} = TemplatedRequests.Get_As_Json_From_Uri uri=${ENTITY_OWNER_URI} session=${session}
+ ${data} = TemplatedRequests.Get_As_Json_From_Uri uri=${ENTITY_OWNER_URI} session=${session} http_timeout=${http_timeout}
${candidate_list} = BuiltIn.Create_List
${entity_type} = BuiltIn.Set_Variable_If '${device_type}' == 'netconf' netconf-node/${device_name} ${device_type}
${clear_data} = BuiltIn.Run_Keyword_If '${device_type}' == 'openflow' or '${device_type}' == 'netconf' Extract_OpenFlow_Device_Data ${data}
[Return] ${owner} ${candidate_list}
Get_Owner_And_Candidates_For_Device_Singleton
- [Arguments] ${device_name} ${device_type} ${member_index}
+ [Arguments] ${device_name} ${device_type} ${member_index} ${http_timeout}=${EMPTY}
[Documentation] Returns the owner and a list of candidates for the SB device ${device_name} of type ${device_type}. Request is sent to member ${member_index}.
... Parsing method is selected by device type
... Separate kw for every supported device type must be defined
BuiltIn.Keyword_Should_Exist Get_Owner_And_Candidates_For_Device_Singleton_${device_type}
- BuiltIn.Run_Keyword_And_Return Get_Owner_And_Candidates_For_Device_Singleton_${device_type} ${device_name} ${member_index}
+ BuiltIn.Run_Keyword_And_Return Get_Owner_And_Candidates_For_Device_Singleton_${device_type} ${device_name} ${member_index} http_timeout=${http_timeout}
Get_Owner_And_Candidates_For_Device_Singleton_Netconf
- [Arguments] ${device_name} ${member_index}
+ [Arguments] ${device_name} ${member_index} ${http_timeout}=${EMPTY}
[Documentation] Returns the owner and a list of candidates for the SB device ${device_name} of type netconf. Request is sent to member ${member_index}.
... Parsing method is set as netconf (using netconf device id prefix and suffix)
# Get election entity type results
${type} = BuiltIn.Set_Variable ${SINGLETON_ELECTION_ENTITY_TYPE}
${id} = BuiltIn.Set_Variable ${SINGLETON_NETCONF_DEVICE_ID_PREFIX}${device_name}${SINGLETON_NETCONF_DEVICE_ID_SUFFIX}
- ${owner_1} ${candidate_list_1} = Get_Owner_And_Candidates_For_Type_And_Id ${type} ${id} ${member_index}
+ ${owner_1} ${candidate_list_1} = Get_Owner_And_Candidates_For_Type_And_Id ${type} ${id} ${member_index} http_timeout=${http_timeout}
# Get change ownership entity type results
${type} = BuiltIn.Set_Variable ${SINGLETON_CHANGE_OWNERSHIP_ENTITY_TYPE}
${id} = BuiltIn.Set_Variable ${SINGLETON_NETCONF_DEVICE_ID_PREFIX}${device_name}${SINGLETON_NETCONF_DEVICE_ID_SUFFIX}
- ${owner_2} ${candidate_list_2} = Get_Owner_And_Candidates_For_Type_And_Id ${type} ${id} ${member_index}
+ ${owner_2} ${candidate_list_2} = Get_Owner_And_Candidates_For_Type_And_Id ${type} ${id} ${member_index} http_timeout=${http_timeout}
# Owners must be same, if not, there is still some election or change ownership in progress
BuiltIn.Should_Be_Equal_As_Integers ${owner_1} ${owner_2} Owners for device ${device_name} are not same
[Return] ${owner_1} ${candidate_list_1}
Get_Owner_And_Candidates_For_Device
- [Arguments] ${device_name} ${device_type} ${member_index}
+ [Arguments] ${device_name} ${device_type} ${member_index} ${http_timeout}=${EMPTY}
[Documentation] Returns the owner and a list of candidates for the SB device ${device_name} of type ${device_type}. Request is sent to member ${member_index}.
... If parsing as singleton failed, kw try to parse data in old way (without singleton).
... Candidates are all members that register to own a device, so the list of candiates includes the owner.
... It is recommended to use Get_Owner_And_Successors_For_Device instead of this keyword, see documentation there.
# Try singleton
${status} ${results} = BuiltIn.Run_Keyword_And_Ignore_Error Get_Owner_And_Candidates_For_Device_Singleton device_name=${device_name} device_type=${device_type} member_index=${member_index}
+ ... http_timeout=${http_timeout}
BuiltIn.Return_From_Keyword_If "${status}"=="PASS" ${results}
# If singleton failed, try parsing in old way
${status} ${results} = BuiltIn.Run_Keyword_And_Ignore_Error Get_Owner_And_Candidates_For_Device_Old device_name=${device_name} device_type=${device_type} member_index=${member_index}
+ ... http_timeout=${http_timeout}
# previous 3 lines (BuilIn.Return.., # If singleton..., ${status}....) could be deleted when old way will not be supported anymore
BuiltIn.Run_Keyword_If '${status}'=='FAIL' BuiltIn.Fail Could not parse owner and candidates for device ${device_name}
[Return] @{results}
Check_Old_Owner_Stays_Elected_For_Device
- [Arguments] ${device_name} ${device_type} ${old_owner} ${node_to_ask}
+ [Arguments] ${device_name} ${device_type} ${old_owner} ${node_to_ask} ${http_timeout}=${EMPTY}
[Documentation] Verify the owner remain the same as ${old_owner}
- ${owner} ${candidates} = Get_Owner_And_Candidates_For_Device ${device_name} ${device_type} ${node_to_ask}
+ ${owner} ${candidates} = Get_Owner_And_Candidates_For_Device ${device_name} ${device_type} ${node_to_ask} http_timeout=${http_timeout}
BuiltIn.Should_Be_Equal_As_numbers ${old_owner} ${owner}
BuiltIn.Return_From_Keyword ${owner} ${candidates}
Check_New_Owner_Got_Elected_For_Device
- [Arguments] ${device_name} ${device_type} ${old_owner} ${node_to_ask}
+ [Arguments] ${device_name} ${device_type} ${old_owner} ${node_to_ask} ${http_timeout}=${EMPTY}
[Documentation] Verify new owner was elected comparing to ${old_owner}
- ${owner} ${candidates} = Get_Owner_And_Candidates_For_Device ${device_name} ${device_type} ${node_to_ask}
+ ${owner} ${candidates} = Get_Owner_And_Candidates_For_Device ${device_name} ${device_type} ${node_to_ask} http_timeout=${http_timeout}
BuiltIn.Should_Not_Be_Equal_As_Numbers ${old_owner} ${owner}
BuiltIn.Return_From_Keyword ${owner} ${candidates}
Get_Owner_And_Candidates_For_Type_And_Id
- [Arguments] ${type} ${id} ${member_index} ${require_candidate_list}=${EMPTY}
+ [Arguments] ${type} ${id} ${member_index} ${require_candidate_list}=${EMPTY} ${http_timeout}=${EMPTY}
[Documentation] Returns the owner and a list of candidates for entity specified by ${type} and ${id}
... Request is sent to member ${member_index}.
... Candidates are all members that register to own a device, so the list of candiates includes the owner.
... namely you cannot use \${EMPTY} to stand for "full list" in this keyword.
BuiltIn.Comment TODO: Find a way to unify and deduplicate code blocks in Get_Owner_And_Candidates_* keywords.
${session} = Resolve_Http_Session_For_Member member_index=${member_index}
- ${data} = TemplatedRequests.Get_As_Json_From_Uri uri=${ENTITY_OWNER_URI} session=${session}
+ ${data} = TemplatedRequests.Get_As_Json_From_Uri uri=${ENTITY_OWNER_URI} session=${session} http_timeout=${http_timeout}
${candidate_list} = BuiltIn.Create_List
${json} = RequestsLibrary.To_Json ${data}
${entity_type_list} = Collections.Get_From_Dictionary &{json}[entity-owners] entity-type
Extract_OpenFlow_Device_Data
[Arguments] ${data}
[Documentation] Remove superfluous OpenFlow device data from Entity Owner printout.
- ${clear_data} = BuiltIn.Run Keyword If '${ODL_STREAM}' != 'beryllium' and '${ODL_OF_PLUGIN}' == 'lithium' String.Replace_String ${data} org.opendaylight.mdsal.ServiceEntityType openflow
- ... ELSE BuiltIn.Set_Variable ${data}
+ ${clear_data} = String.Replace_String ${data} org.opendaylight.mdsal.ServiceEntityType openflow
${clear_data} = String.Replace_String ${clear_data} /odl-general-entity:entity[odl-general-entity:name=' ${EMPTY}
${clear_data} = String.Replace_String ${clear_data} /general-entity:entity[general-entity:name=' ${EMPTY}
${clear_data} = String.Replace_String ${clear_data} '] ${EMPTY}
[Documentation] Convenience keyword that kills the specified member of the cluster.
... The KW will return a list of available members: \${updated index_list}=\${original_index_list}-\${member}
${index_list} = ClusterManagement__Build_List ${member}
+ ${member_ip} = Return_Member_IP ${member}
+ KarafKeywords.Log_Message_To_Controller_Karaf Killing ODL${member} ${member_ip}
${updated_index_list} = Kill_Members_From_List_Or_All ${index_list} ${original_index_list} ${confirm}
[Return] ${updated_index_list}
Start_Members_From_List_Or_All ${index_list} ${wait_for_sync} ${timeout}
Start_Members_From_List_Or_All
- [Arguments] ${member_index_list}=${EMPTY} ${wait_for_sync}=True ${timeout}=300s ${karaf_home}=${EMPTY} ${export_java_home}=${EMPTY}
+ [Arguments] ${member_index_list}=${EMPTY} ${wait_for_sync}=True ${timeout}=300s ${karaf_home}=${EMPTY} ${export_java_home}=${EMPTY} ${gc_log_dir}=${EMPTY}
[Documentation] If the list is empty, start all cluster members. Otherwise, start members based on present indices.
... If ${wait_for_sync}, wait for cluster sync on listed members.
... Optionally karaf_home can be overriden. Optionally specific JAVA_HOME is used for starting.
- ${base_command} = BuiltIn.Set_Variable_If "${karaf_home}" ${karaf_home}/bin/start ${NODE_START_COMMAND}
- ${command} = BuiltIn.Set_Variable_If "${export_java_home}" export JAVA_HOME="${export_java_home}"; ${base_command} ${base_command}
- Run_Bash_Command_On_List_Or_All command=${command} member_index_list=${member_index_list}
+ ... Garbage collection is unconditionally logged to files. TODO: Make that reasonable conditional?
+ ${base_command} = BuiltIn.Set_Variable_If """${karaf_home}""" != "" ${karaf_home}/bin/start ${NODE_START_COMMAND}
+ ${command} = BuiltIn.Set_Variable_If """${export_java_home}""" != "" export JAVA_HOME="${export_java_home}"; ${base_command} ${base_command}
+ ${epoch} = DateTime.Get_Current_Date time_zone=UTC result_format=epoch exclude_millis=False
+ ${gc_filepath} = BuiltIn.Set_Variable_If """${karaf_home}""" != "" ${karaf_home}/data/log/gc_${epoch}.log ${GC_LOG_PATH}/gc_${epoch}.log
+ ${gc_options} = BuiltIn.Set_Variable -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:${gc_filepath}
+ Run_Bash_Command_On_List_Or_All command=${command} ${gc_options} member_index_list=${member_index_list}
BuiltIn.Return_From_Keyword_If not ${wait_for_sync}
BuiltIn.Wait_Until_Keyword_Succeeds ${timeout} 10s Check_Cluster_Is_In_Sync member_index_list=${member_index_list}
# TODO: Do we also want to check Shard Leaders here?
+ [Teardown] Run_Bash_Command_On_List_Or_All command=netstat -pnatu member_index_list=${member_index_list}
Freeze_Single_Member
[Arguments] ${member}
# TODO: Migrate callers to Check_Bash_Command_*
${member_ip} = Collections.Get_From_Dictionary dictionary=${ClusterManagement__index_to_ip_mapping} key=${member_index}
${output} = SSHKeywords.Run_Keyword_Preserve_Connection Utils.Run_Command_On_Controller ${member_ip} ${command}
+ Log ${output}
[Return] ${output}
Run_Karaf_Command_On_List_Or_All
[Documentation] Verify that cluster node is in sync with others
${status} Get_Sync_Status_Of_Member ${controller_index}
BuiltIn.Should_Be_True ${status}
+
+Return_Member_IP
+ [Arguments] ${member_index}
+ [Documentation] Return the IP address of the member given the member_index.
+ ${member_int} = BuiltIn.Convert_To_Integer ${member_index}
+ ${member_ip} = Collections.Get_From_Dictionary dictionary=${ClusterManagement__index_to_ip_mapping} key=${member_int}
+ [Return] ${member_ip}
Code Review / integration / test.git / blobdiff