X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=csit%2Flibraries%2FClusterKeywords.robot;h=5f19dff7fd94ce99c9a5489618a7981a1df5dd10;hb=e9b687cb4c8ad74d10b70fab3fe5de5e1b1ae0f3;hp=1a951c2337cf8a962a22be0b3c22d9f317a3426e;hpb=4f0033ab5c4ac3355ed93bca80c61cf249adf852;p=integration%2Ftest.git diff --git a/csit/libraries/ClusterKeywords.robot b/csit/libraries/ClusterKeywords.robot index 1a951c2337..5f19dff7fd 100644 --- a/csit/libraries/ClusterKeywords.robot +++ b/csit/libraries/ClusterKeywords.robot @@ -1,11 +1,104 @@ *** Settings *** -Resource Utils.robot +Library RequestsLibrary Library Collections +Library UtilLibrary.py Library ClusterStateLibrary.py +Library ./HsfJson/hsf_json.py +Resource Utils.robot *** Variables *** +${jolokia_conf} /jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore +${jolokia_oper} /jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-operational,type=DistributedOperationalDatastore +${jolokia_read} /jolokia/read/org.opendaylight.controller *** Keywords *** +Create Controller Index List + [Documentation] Reads number of controllers and returns a list with all controllers indexes. + ${controller_index_list} Create List + ${NUM_ODL_SYSTEM}= Convert to Integer ${NUM_ODL_SYSTEM} + : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} + \ Append To List ${controller_index_list} ${i+1} + [Return] ${controller_index_list} + +Create Controller Sessions + [Documentation] Creates REST session to all controller instances. + ${NUM_ODL_SYSTEM}= Convert to Integer ${NUM_ODL_SYSTEM} + : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} + \ Log Create Session ${ODL_SYSTEM_${i+1}_IP} + \ Create Session controller${i+1} http://${ODL_SYSTEM_${i+1}_IP}:${RESTCONFPORT} auth=${AUTH} + +Get Cluster Shard Status + [Arguments] ${controller_index_list} ${shard_type} ${shard} + [Documentation] Checks ${shard} status and returns Leader index and a list of Followers from a ${controller_index_list}. + ... ${shard_type} is either config or operational. + ${lenght}= Get Length ${controller_index_list} + Run Keyword If '${shard_type}' == 'config' Set Test Variable ${type} DistributedConfigDatastore + Run Keyword If '${shard_type}' == 'operational' Set Test Variable ${type} DistributedOperationalDatastore + Should Not Be Empty ${type} Wrong type, valid values are config and operational. + ${leader}= Set Variable 0 + ${follower_list}= Create List + : FOR ${i} IN @{controller_index_list} + \ ${data}= Get Data From URI controller${i} ${jolokia_read}:Category=Shards,name=member-${i}-shard-${shard}-${shard_type},type=${type} + \ Log ${data} + \ ${json}= To Json ${data} + \ ${status}= Get From Dictionary &{json}[value] RaftState + \ Log Controller ${ODL_SYSTEM_${i}_IP} is ${status} for shard ${shard} + \ Run Keyword If '${status}' == 'Leader' Set Test Variable ${leader} ${i} + \ Run Keyword If '${status}' == 'Follower' Append To List ${follower_list} ${i} + Should Not Be Equal ${leader} 0 No Leader elected in shard ${shard_type} ${shard} + Length Should Be ${follower_list} ${lenght-1} Not enough or too many Followers in shard ${shard_type} ${shard} + [Return] ${leader} ${follower_list} + +Check Item Occurrence At URI In Cluster + [Arguments] ${controller_index_list} ${dictionary_item_occurrence} ${uri} + [Documentation] Send a GET with the supplied ${uri} to all cluster instances in ${controller_index_list} + ... and check for occurrences of items expressed in a dictionary ${dictionary_item_occurrence}. + : FOR ${i} IN @{controller_index_list} + \ ${data} Get Data From URI controller${i} ${uri} + \ Log ${data} + \ Check Item Occurrence ${data} ${dictionary_item_occurrence} + +Put And Check At URI In Cluster + [Arguments] ${controller_index_list} ${controller_index} ${uri} ${body} + [Documentation] Send a PUT with the supplied ${uri} and ${body} (json string) to a ${controller_index} + ... and check the data is replicated in all instances in ${controller_index_list}. + ${expected_body}= Hsf Json ${body} + Log ${body} + ${resp} RequestsLibrary.Put Request controller${controller_index} ${uri} ${body} ${HEADERS_YANG_JSON} + Log ${resp.content} + Log ${resp.status_code} + ${status_code}= Convert To String ${resp.status_code} + Should Match Regexp ${status_code} 20(0|1) + : FOR ${i} IN @{controller_index_list} + \ ${data} Wait Until Keyword Succeeds 5s 1s Get Data From URI controller${i} + \ ... ${uri} + \ Log ${data} + \ ${received_body} Hsf Json ${data} + \ Should Be Equal ${expected_body} ${received_body} + +Delete And Check At URI In Cluster + [Arguments] ${controller_index_list} ${controller_index} ${uri} + [Documentation] Send a DELETE with the supplied ${uri} to a ${controller_index} + ... and check the data is removed from all instances in ${controller_index_list}. + ${resp} RequestsLibrary.Delete Request controller${controller_index} ${uri} + Should Be Equal As Strings ${resp.status_code} 200 + : FOR ${i} IN @{controller_index_list} + \ Wait Until Keyword Succeeds 5s 1s No Content From URI controller${i} ${uri} + +Kill Multiple Controllers + [Arguments] @{controller_index_list} + [Documentation] Give this keyword a scalar or list of controllers to be stopped. + : FOR ${i} IN @{controller_index_list} + \ ${output}= Run Command On Controller ${ODL_SYSTEM_${i}_IP} ps axf | grep karaf | grep -v grep | awk '{print \"kill -9 \" $1}' | sh + \ Controller Down Check ${ODL_SYSTEM_${i}_IP} + +Start Multiple Controllers + [Arguments] ${timeout} @{controller_index_list} + [Documentation] Give this keyword a scalar or list of controllers to be started. + : FOR ${i} IN @{controller_index_list} + \ ${output}= Run Command On Controller ${ODL_SYSTEM_${i}_IP} ${WORKSPACE}/${BUNDLEFOLDER}/bin/start + \ Wait For Controller Sync ${timeout} ${ODL_SYSTEM_${i}_IP} + Get Controller List [Arguments] ${exclude_controller}=${EMPTY} [Documentation] Creates a list of all controllers minus any excluded controller. @@ -20,167 +113,35 @@ Get Leader And Verify [Documentation] Returns the IP addr or hostname of the leader of the specified shard. ... Controllers are specifed in the pybot command line. ${searchlist} Get Controller List ${old_leader} - ${leader} GetLeader ${shard_name} ${3} ${3} ${1} ${PORT} + ${leader} GetLeader ${shard_name} ${3} ${3} ${1} ${RESTCONFPORT} ... @{searchlist} Should Not Be Equal As Strings ${leader} None Run Keyword If '${old_leader}'!='${EMPTY}' Should Not Be Equal ${old_leader} ${leader} [Return] ${leader} -Wait For Leader To Be Found +Expect No Leader [Arguments] ${shard_name} - [Documentation] Waits until the leader of the specified shard is found. - ${leader} Wait Until Keyword Succeeds 12s 2s Get Leader And Verify ${shard_name} - Log ${leader} - [Return] ${leader} - -Switch Leader - [Arguments] ${shard_name} ${current_leader} - [Documentation] Forces a change of leadership by shutting down the current leader. - Stop One Or More Controllers ${current_leader} - ${new_leader} Wait Until Keyword Succeeds 60s 2s Get Leader And Verify ${shard_name} ${current_leader} - # TODO: Future enhanement: make sure the other controller is a follower and not a master or candidate. - Log ${new_leader} - [Return] ${new_leader} + [Documentation] No leader is elected in the car shard + ${leader} GetLeader ${shard_name} ${3} ${1} ${1} ${RESTCONFPORT} + ... ${CURRENT_CAR_LEADER} + Should Be Equal As Strings ${leader} None Get All Followers [Arguments] ${shard_name} ${exclude_controller}=${EMPTY} [Documentation] Returns the IP addresses or hostnames of all followers of the specified shard. ${searchlist} Get Controller List ${exclude_controller} - ${followers} GetFollowers ${shard_name} ${3} ${3} ${1} ${PORT} + ${followers} GetFollowers ${shard_name} ${3} ${3} ${1} ${RESTCONFPORT} ... @{searchlist} Log ${followers} Should Not Be Empty ${followers} [Return] ${followers} -Add Cars And Verify - [Arguments] ${controller_ip} ${num_cars} ${timeout}=12s - [Documentation] Initializes shard and then adds the specified number of cars and performs a GET as a check. - ${resp} InitCar ${controller_ip} ${PORT} - Should Be Equal As Strings ${resp.status_code} 204 - ${resp} AddCar ${controller_ip} ${RESTCONFPORT} ${num_cars} 204 - Should Be Equal As Strings ${resp.status_code} 204 - Wait Until Keyword Succeeds ${timeout} 2s Get Cars And Verify ${controller_ip} ${num_cars} - -Add Cars And Verify Without Init - [Arguments] ${controller_ip} ${num_cars} ${timeout}=12s - [Documentation] Adds cars to an initialized cars shard then performs a GET as a check. - Comment First car add may return 409, but subsequent should be 204 - ${resp} AddCar ${controller_ip} ${RESTCONFPORT} ${num_cars} 204 409 - Should Be Equal As Strings ${resp.status_code} 204 - Wait Until Keyword Succeeds ${timeout} 2s Get Cars And Verify ${controller_ip} ${num_cars} - -Get Cars And Verify - [Arguments] ${controller_ip} ${num_cars} - [Documentation] Gets cars and verifies that the manufacturer is correct. - # TODO: Future enhanement: verify all fields. - ${resp} Getcars ${controller_ip} ${RESTCONFPORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - : FOR ${i} IN RANGE 1 ${num_cars}+1 - \ Should Contain ${resp.content} manufacturer${i} - -Add People And Verify - [Arguments] ${controller_ip} ${num_people} - [Documentation] Note: The first AddPerson call passed with 0 posts directly to the data store to get - ... the people container created so the subsequent AddPerson RPC calls that put to the - ... person list will succeed. - ${resp} AddPerson ${controller_ip} ${RESTCONFPORT} ${0} 204 - Should Be Equal As Strings ${resp.status_code} 204 - Wait Until Keyword Succeeds 12s 2s Get One Person And Verify ${controller_ip} ${0} - ${resp} AddPerson ${controller_ip} ${RESTCONFPORT} ${num_people} 200 - Wait Until Keyword Succeeds 12s 2s Get People And Verify ${controller_ip} ${num_people} - -Get One Person And Verify - [Arguments] ${controller_ip} ${number} - [Documentation] Gets a person and verifies that the user ID is correct. - # TODO: Future enhanement: verify all fields. - ${resp} GetPersons ${controller_ip} ${RESTCONFPORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user${number} - -Get People And Verify - [Arguments] ${controller_ip} ${num_people} - [Documentation] Gets multiple people and verifies that the user IDs are correct. - # TODO: Future enhanement: verify all fields. - ${resp} GetPersons ${controller_ip} ${RESTCONFPORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - : FOR ${i} IN RANGE 1 ${num_people}+1 - \ Should Contain ${resp.content} user${i} - -Add Car Person And Verify - [Arguments] ${controller_ip} - [Documentation] Add a car-person via the data store and get the car-person from Leader. - ... Note: This is done to get the car-people container created so subsequent - ... BuyCar RPC puts to the car-person list will succeed. - AddCarPerson ${controller_ip} ${RESTCONFPORT} ${0} - Wait Until Keyword Succeeds 60s 2s Get One Car-Person Mapping And Verify ${controller_ip} ${0} - -Get One Car-Person Mapping And Verify - [Arguments] ${controller_ip} ${number} - [Documentation] Gets a car person mapping and verifies that the user ID is correct. - ${resp} GetCarPersonMappings ${controller_ip} ${RESTCONFPORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user${number} - -Get Car-Person Mappings And Verify - [Arguments] ${controller_ip} ${num_entries} - ${resp} GetCarPersonMappings ${controller_ip} ${RESTCONFPORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - : FOR ${i} IN RANGE 1 ${num_entries}+1 - \ Should Contain ${resp.content} user${i} - -Buy Cars And Verify - [Arguments] ${controller_ip} ${num_entries} ${start}=${0} - Wait Until Keyword Succeeds 60s 2s BuyCar ${controller_ip} ${RESTCONFPORT} ${num_entries} - ... ${start} - -Check Cars Deleted - [Arguments] ${controller_ip} - ${resp} Getcars ${controller_ip} ${RESTCONFPORT} ${0} - Should Be Equal As Strings ${resp.status_code} 404 - -Delete All Cars And Verify - [Arguments] ${controller_ip} - DeleteAllCars ${controller_ip} ${RESTCONFPORT} ${0} - Wait Until Keyword Succeeds 60s 2s Check Cars Deleted ${controller_ip} - -Check People Deleted - [Arguments] ${controller_ip} - ${resp} GetPersons ${controller_ip} ${RESTCONFPORT} ${0} - Should Be Equal As Strings ${resp.status_code} 404 - -Delete All People And Verify - [Arguments] ${controller_ip} - DeleteAllPersons ${controller_ip} ${RESTCONFPORT} ${0} - Wait Until Keyword Succeeds 60s 2s Check People Deleted ${controller_ip} - -Check Cars-Persons Deleted - [Arguments] ${controller_ip} - ${resp} GetCarPersonMappings ${controller_ip} ${RESTCONFPORT} ${0} - Should Be Equal As Strings ${resp.status_code} 404 - -Delete All Cars-Persons And Verify - [Arguments] ${controller_ip} - DeleteAllCarsPersons ${controller_ip} ${RESTCONFPORT} ${0} - Wait Until Keyword Succeeds 60s 2s Check Cars-Persons Deleted ${controller_ip} - Stop One Or More Controllers [Arguments] @{controllers} [Documentation] Give this keyword a scalar or list of controllers to be stopped. ${cmd} = Set Variable ${KARAF_HOME}/bin/stop : FOR ${ip} IN @{controllers} \ Run Command On Remote System ${ip} ${cmd} - : FOR ${ip} IN @{controllers} - \ Wait Until Keyword Succeeds ${CONTROLLER_STOP_TIMEOUT} s 3 s Controller Down Check ${ip} - -Start One Or More Controllers - [Arguments] @{controllers} - [Documentation] Give this keyword a scalar or list of controllers to be started. - ${cmd} = Set Variable ${KARAF_HOME}/bin/start - : FOR ${ip} IN @{controllers} - \ Run Command On Remote System ${ip} ${cmd} - # TODO: This should throw an error if controller never comes up. - : FOR ${ip} IN @{controllers} - \ UtilLibrary.Wait For Controller Up ${ip} ${RESTCONFPORT} Kill One Or More Controllers [Arguments] @{controllers} @@ -189,17 +150,78 @@ Kill One Or More Controllers log ${cmd} : FOR ${ip} IN @{controllers} \ Run Command On Remote System ${ip} ${cmd} + +Wait For Cluster Down + [Arguments] ${timeout} @{controllers} + [Documentation] Waits for one or more clustered controllers to be down. : FOR ${ip} IN @{controllers} - \ Wait Until Keyword Succeeds 12 s 3 s Controller Down Check ${ip} + \ ${status}= Run Keyword And Return Status Wait For Controller Down ${timeout} ${ip} + \ Exit For Loop If '${status}' == 'FAIL' + +Wait For Controller Down + [Arguments] ${timeout} ${ip} + [Documentation] Waits for one controllers to be down. + Wait Until Keyword Succeeds ${timeout} 2s Controller Down Check ${ip} Controller Down Check [Arguments] ${ip} [Documentation] Checks to see if a controller is down by verifying that the karaf process isn't present. ${cmd} = Set Variable ps axf | grep karaf | grep -v grep | wc -l - ${response} Run Command On Remote System ${ip} ${cmd} + ${response} Run Command On COntroller ${ip} ${cmd} Log Number of controller instances running: ${response} Should Start With ${response} 0 Controller process found or there may be extra instances of karaf running on the host machine. +Start One Or More Controllers + [Arguments] @{controllers} + [Documentation] Give this keyword a scalar or list of controllers to be started. + ${cmd} = Set Variable ${KARAF_HOME}/bin/start + : FOR ${ip} IN @{controllers} + \ Run Command On Remote System ${ip} ${cmd} + +Wait For Cluster Sync + [Arguments] ${timeout} @{controllers} + [Documentation] Waits for one or more clustered controllers to report Sync Status as true. + : FOR ${ip} IN @{controllers} + \ ${status}= Run Keyword And Return Status Wait For Controller Sync ${timeout} ${ip} + \ Exit For Loop If '${status}' == 'FAIL' + +Wait For Controller Sync + [Arguments] ${timeout} ${ip} + [Documentation] Waits for one controllers to report Sync Status as true. + Wait Until Keyword Succeeds ${timeout} 2s Controller Sync Status Should Be True ${ip} + +Controller Sync Status Should Be True + [Arguments] ${ip} + [Documentation] Checks if Sync Status is true. + ${SyncStatus}= Get Controller Sync Status ${ip} + Should Be Equal ${SyncStatus} ${TRUE} + +Controller Sync Status Should Be False + [Arguments] ${ip} + [Documentation] Checks if Sync Status is false. + ${SyncStatus}= Get Controller Sync Status ${ip} + Should Be Equal ${SyncStatus} ${FALSE} + +Get Controller Sync Status + [Arguments] ${controller_ip} + [Documentation] Return Sync Status. + Create_Session session http://${controller_ip}:${RESTCONFPORT} headers=${HEADERS} auth=${AUTH} + ${data}= Get Data From URI session ${jolokia_conf} + Log ${data} + ${json}= To Json ${data} + ${value}= Get From Dictionary ${json} value + ${ConfSyncStatus}= Get From Dictionary ${value} SyncStatus + Log Configuration Sync Status: ${ConfSyncStatus} + ${data}= Get Data From URI session ${jolokia_oper} + Log ${data} + ${json}= To Json ${data} + ${value}= Get From Dictionary ${json} value + ${OperSyncStatus}= Get From Dictionary ${value} SyncStatus + Log Operational Sync Status: ${OperSyncStatus} + Run Keyword If ${OperSyncStatus} and ${ConfSyncStatus} Set Test Variable ${SyncStatus} ${TRUE} + ... ELSE Set Test Variable ${SyncStatus} ${FALSE} + [Return] ${SyncStatus} + Clean One Or More Journals [Arguments] @{controllers} [Documentation] Give this keyword a scalar or list of controllers on which to clean journals. @@ -237,36 +259,6 @@ Show Cluster Configuation Files : FOR ${ip} IN @{controllers} \ Run Command On Remote System ${ip} ${cmd} -Check Cars - [Arguments] ${selected controller} ${PORT} ${nth car} - [Documentation] Verifies that the first through nth car is present. - ${resp} Getcars ${selected controller} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - : FOR ${INDEX} IN RANGE 1 ${nth car} - \ ${counter}= Convert to String ${INDEX} - \ Log manufacturer${counter} - \ Should Contain ${resp.content} manufacturer${counter} - -Check People - [Arguments] ${selected controller} ${PORT} ${nth person} - [Documentation] Verifies that the first through nth person is present. - ${resp} GetPersons ${selected controller} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - : FOR ${INDEX} IN RANGE 1 ${nth person} - \ ${counter}= Convert to String ${INDEX} - \ Log user${counter} - \ Should Contain ${resp.content} user${counter} - -Check CarPeople - [Arguments] ${selected controller} ${PORT} ${nth carperson} - [Documentation] Verifies that the first through nth car-person is present. - ${resp} GetCarPersonMappings ${selected controller} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - : FOR ${INDEX} IN RANGE 1 ${nth carperson} - \ ${counter}= Convert to String ${INDEX} - \ Log user${counter} - \ Should Contain ${resp.content} user${counter} - Isolate a Controller From Cluster [Arguments] ${isolated controller} @{controllers} [Documentation] Use IPTables to isolate one controller from the cluster.