From af48a74be55aec608b778b764cc05df1ec7d9ffa Mon Sep 17 00:00:00 2001 From: Vratko Polak Date: Thu, 3 Mar 2016 18:46:48 +0100 Subject: [PATCH] Rework of controller car/people cluster suite - Deleted the previous car/people cluster suites + New Resources: CarPeople, ClusterManagement + Updated/fixed: TemplatedRequests.robot, norm_json.py + New suite names, without number prefixes + Full-fields data files for car/people TemplatedRequests - Simplistic iterated values for the fields + controller-clustering testplan updated Change-Id: I56d0743129d6e8108b66e9b48c135e4c3bc13d5c Signed-off-by: Vratko Polak --- csit/libraries/CarPeople.robot | 60 +++++ csit/libraries/CarsAndPeople.robot | 241 ----------------- csit/libraries/ClusterManagement.robot | 242 ++++++++++++++++++ csit/libraries/TemplatedRequests.robot | 8 +- csit/libraries/norm_json.py | 35 +-- .../001_start_cluster.robot | 26 -- .../010_crud_on_leader.robot | 93 ------- .../020_crud_on_any_follower.robot | 72 ------ .../030_car_failover_crud_on_new_leader.robot | 78 ------ ...0_people_failover_crud_on_new_leader.robot | 68 ----- .../050_car_persistence_recovery.robot | 79 ------ .../140_recovery_restart_follower.robot | 76 ------ .../Clustering_Datastore/999_cleanup.robot | 20 -- .../Clustering_Datastore/__init__.robot | 5 - .../buycar_failover.robot | 122 +++++++++ .../car_failover_crud.robot | 125 +++++++++ .../car_outage_corners.robot | 102 ++++++++ .../car_persistence_recovery.robot | 62 +++++ .../Clustering_Datastore/carpeople_crud.robot | 103 ++++++++ .../Clustering_Datastore/cluster_ready.robot | 30 +++ .../001_start_cluster.robot | 21 -- .../023_routed_rpc_crud_test.robot | 129 ---------- csit/testplans/controller-clustering.txt | 13 +- .../carpeople/crud/add-person/location.uri | 1 + .../carpeople/crud/add-person/post_data.json | 9 + .../carpeople/crud/buy-car/location.uri | 1 + .../carpeople/crud/buy-car/post_data.json | 7 + .../crud/car-people/data.epilog.json | 3 + .../carpeople/crud/car-people/data.item.json | 4 + .../crud/car-people/data.prolog.json | 3 + .../carpeople/crud/car-people/location.uri | 1 + .../carpeople/crud/cars/data.epilog.json | 3 + .../carpeople/crud/cars/data.item.json | 7 + .../carpeople/crud/cars/data.prolog.json | 3 + .../carpeople/crud/cars/location.uri | 1 + .../carpeople/crud/cars/post_data.epilog.json | 2 + .../carpeople/crud/cars/post_data.prolog.json | 2 + .../carpeople/crud/people/data.epilog.json | 3 + .../carpeople/crud/people/data.item.json | 7 + .../carpeople/crud/people/data.prolog.json | 3 + .../carpeople/crud/people/location.uri | 1 + .../crud/people/post_data.epilog.json | 2 + .../crud/people/post_data.prolog.json | 2 + .../carpeople/libtest/purchase/post_data.json | 4 +- csit/variables/restconf/modules/location.uri | 1 + 45 files changed, 942 insertions(+), 938 deletions(-) create mode 100644 csit/libraries/CarPeople.robot delete mode 100644 csit/libraries/CarsAndPeople.robot create mode 100644 csit/libraries/ClusterManagement.robot delete mode 100644 csit/suites/controller/Clustering_Datastore/001_start_cluster.robot delete mode 100644 csit/suites/controller/Clustering_Datastore/010_crud_on_leader.robot delete mode 100644 csit/suites/controller/Clustering_Datastore/020_crud_on_any_follower.robot delete mode 100644 csit/suites/controller/Clustering_Datastore/030_car_failover_crud_on_new_leader.robot delete mode 100644 csit/suites/controller/Clustering_Datastore/040_people_failover_crud_on_new_leader.robot delete mode 100644 csit/suites/controller/Clustering_Datastore/050_car_persistence_recovery.robot delete mode 100644 csit/suites/controller/Clustering_Datastore/140_recovery_restart_follower.robot delete mode 100644 csit/suites/controller/Clustering_Datastore/999_cleanup.robot delete mode 100644 csit/suites/controller/Clustering_Datastore/__init__.robot create mode 100644 csit/suites/controller/Clustering_Datastore/buycar_failover.robot create mode 100644 csit/suites/controller/Clustering_Datastore/car_failover_crud.robot create mode 100644 csit/suites/controller/Clustering_Datastore/car_outage_corners.robot create mode 100644 csit/suites/controller/Clustering_Datastore/car_persistence_recovery.robot create mode 100644 csit/suites/controller/Clustering_Datastore/carpeople_crud.robot create mode 100644 csit/suites/controller/Clustering_Datastore/cluster_ready.robot delete mode 100644 csit/suites/controller/Clustering_Routedrpc/001_start_cluster.robot delete mode 100644 csit/suites/controller/Clustering_Routedrpc/023_routed_rpc_crud_test.robot create mode 100644 csit/variables/carpeople/crud/add-person/location.uri create mode 100644 csit/variables/carpeople/crud/add-person/post_data.json create mode 100644 csit/variables/carpeople/crud/buy-car/location.uri create mode 100644 csit/variables/carpeople/crud/buy-car/post_data.json create mode 100644 csit/variables/carpeople/crud/car-people/data.epilog.json create mode 100644 csit/variables/carpeople/crud/car-people/data.item.json create mode 100644 csit/variables/carpeople/crud/car-people/data.prolog.json create mode 100644 csit/variables/carpeople/crud/car-people/location.uri create mode 100644 csit/variables/carpeople/crud/cars/data.epilog.json create mode 100644 csit/variables/carpeople/crud/cars/data.item.json create mode 100644 csit/variables/carpeople/crud/cars/data.prolog.json create mode 100644 csit/variables/carpeople/crud/cars/location.uri create mode 100644 csit/variables/carpeople/crud/cars/post_data.epilog.json create mode 100644 csit/variables/carpeople/crud/cars/post_data.prolog.json create mode 100644 csit/variables/carpeople/crud/people/data.epilog.json create mode 100644 csit/variables/carpeople/crud/people/data.item.json create mode 100644 csit/variables/carpeople/crud/people/data.prolog.json create mode 100644 csit/variables/carpeople/crud/people/location.uri create mode 100644 csit/variables/carpeople/crud/people/post_data.epilog.json create mode 100644 csit/variables/carpeople/crud/people/post_data.prolog.json create mode 100644 csit/variables/restconf/modules/location.uri diff --git a/csit/libraries/CarPeople.robot b/csit/libraries/CarPeople.robot new file mode 100644 index 0000000000..bae50d90e8 --- /dev/null +++ b/csit/libraries/CarPeople.robot @@ -0,0 +1,60 @@ +*** Settings *** +Documentation Resource housing Keywords common to tests which interact with car/people models. +... +... Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... +... This resource is tightly coupled with "crud" cluster suite, +... as it is not straightforward to allow ${VAR_DIR} customization. +Resource ${CURDIR}/TemplatedRequests.robot + +*** Variables *** +${VAR_DIR} ${CURDIR}/../../../variables/carpeople/crud + +*** Keywords *** +Add_Several_People + [Arguments] ${session} ${iterations} ${iter_start}=1 + [Documentation] Simple loop for issuing add-person RPCs to session, one by one. + ... People need to be added via RPC, otherwise buy-car routed RPC will not find registered path. + ... See javadocs in RpcProviderRegistry.java + : FOR ${i} IN RANGE ${iter_start} ${iter_start}+${iterations} + \ TemplatedRequests.Post_As_Json_Templated folder=${VAR_DIR}/add-person mapping={"i": "${i}"} session=${session} + +Buy_Several_Cars + [Arguments] ${session} ${iterations} ${iter_start}=1 ${registration_delay}=20s + [Documentation] Simple loop for issuing buy-car RPCs to session, one by one. + ... This needs to be a separate Keyword mostly just because nested FOR loops are not allowed. + ... Actual fact of buying one car is done by inner Keyword. + : FOR ${iter} IN RANGE ${iter_start} ${iter_start}+${iterations} + \ Buy_Single_Car session=${session} iteration=${iter} registration_delay=${registration_delay} + +Buy_Single_Car + [Arguments] ${session} ${iteration}=1 ${registration_delay}=20s + [Documentation] Each buy-car RPC is routed, which means there is a delay between + ... the time add-car RPC is executed and the time member in question registers the route. + ... To distinguish functional bugs from performance ones, this Keyword waits up to 20 seconds + ... while retrying buy-car requests. + BuiltIn.Wait_Until_Keyword_Succeeds ${registration_delay} 1s TemplatedRequests.Post_As_Json_Templated folder=${VAR_DIR}/buy-car mapping={"i": "${iteration}"} session=${session} + +Set_Variables_For_Shard + [Arguments] ${shard_name} + [Documentation] Get leader and followers for given shard name and + ... set several suite variables related to member indices and sessions. + ... ClusterManagement Resource is assumed to be initialized. + ... TODO: car-people shard name causes dash in variable names. Should we convert to underscores? + ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=config + BuiltIn.Set_Suite_Variable \${${shard_name}_leader_index} ${leader} + BuiltIn.Set_Suite_Variable \${${shard_name}_follower_indices} ${follower_list} + ${leader_session} = ClusterManagement.Resolve_Http_Session_For_Member member_index=${leader} + BuiltIn.Set_Suite_Variable \${${shard_name}_leader_session} ${leader_session} + ${sessions} = BuiltIn.Create_List + : FOR ${follower_index} IN @{follower_list} + \ ${follower_session} = ClusterManagement.Resolve_Http_Session_For_Member member_index=${follower_index} + \ Collections.Append_To_List ${sessions} ${follower_session} + BuiltIn.Set_Suite_Variable \${${shard_name}_follower_sessions} ${sessions} + ${first_follower_session} = Collections.Get_From_List ${sessions} 0 + BuiltIn.Set_Suite_Variable \${${shard_name}_first_follower_session} ${first_follower_session} diff --git a/csit/libraries/CarsAndPeople.robot b/csit/libraries/CarsAndPeople.robot deleted file mode 100644 index a5bb4e1a98..0000000000 --- a/csit/libraries/CarsAndPeople.robot +++ /dev/null @@ -1,241 +0,0 @@ -*** Settings *** -Library Collections -Library CrudLibrary.py -Library SettingsLibrary.py -Resource DatastoreCRUD.robot - -*** Keywords *** -Add Cars And Verify - [Arguments] ${controller_ip} ${num_cars} ${timeout}=3s - [Documentation] Initializes shard and then adds the specified number of cars and performs a GET as a check. - ${resp} InitCar ${controller_ip} ${PORT} - Should Be Equal As Strings ${resp.status_code} 204 - ${resp} AddCar ${controller_ip} ${RESTCONFPORT} ${num_cars} 204 - Should Be Equal As Strings ${resp.status_code} 204 - Wait Until Keyword Succeeds ${timeout} 1s Get Cars And Verify ${controller_ip} ${num_cars} - -Add Cars And Verify Without Init - [Arguments] ${controller_ip} ${num_cars} ${timeout}=3s - [Documentation] Adds cars to an initialized cars shard then performs a GET as a check. - Comment First car add may return 409, but subsequent should be 204 - ${resp} AddCar ${controller_ip} ${RESTCONFPORT} ${num_cars} 204 409 - Should Be Equal As Strings ${resp.status_code} 204 - Wait Until Keyword Succeeds ${timeout} 1s Get Cars And Verify ${controller_ip} ${num_cars} - -Get Cars And Verify - [Arguments] ${controller_ip} ${num_cars} - [Documentation] Gets cars and verifies that the manufacturer is correct. - # TODO: Future enhanement: verify all fields. - ${resp} Getcars ${controller_ip} ${RESTCONFPORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - : FOR ${i} IN RANGE 1 ${num_cars}+1 - \ Should Contain ${resp.content} manufacturer${i} - -Add People And Verify - [Arguments] ${controller_ip} ${num_people} ${timeout}=3s - [Documentation] Note: The first AddPerson call passed with 0 posts directly to the data store to get - ... the people container created so the subsequent AddPerson RPC calls that put to the - ... person list will succeed. - ${resp} AddPerson ${controller_ip} ${RESTCONFPORT} ${0} 204 - Should Be Equal As Strings ${resp.status_code} 204 - Wait Until Keyword Succeeds ${timeout} 1s Get One Person And Verify ${controller_ip} ${0} - ${resp} AddPerson ${controller_ip} ${RESTCONFPORT} ${num_people} 200 - Wait Until Keyword Succeeds ${timeout} 1s Get People And Verify ${controller_ip} ${num_people} - -Add People And Verify Without Init - [Arguments] ${controller_ip} ${num_people} ${timeout}=3s - [Documentation] Adds people to an initialized people shard then performs a GET as a check. - ${resp} AddPerson ${controller_ip} ${RESTCONFPORT} ${num_people} 200 - Should Be Equal As Strings ${resp.status_code} 200 - Wait Until Keyword Succeeds ${timeout} 1s Get People And Verify ${controller_ip} ${num_people} - -Get One Person And Verify - [Arguments] ${controller_ip} ${number} - [Documentation] Gets a person and verifies that the user ID is correct. - # TODO: Future enhanement: verify all fields. - ${resp} GetPersons ${controller_ip} ${RESTCONFPORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user${number} - -Get People And Verify - [Arguments] ${controller_ip} ${num_people} - [Documentation] Gets multiple people and verifies that the user IDs are correct. - # TODO: Future enhanement: verify all fields. - ${resp} GetPersons ${controller_ip} ${RESTCONFPORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - : FOR ${i} IN RANGE 1 ${num_people}+1 - \ Should Contain ${resp.content} user${i} - -Add Car Person And Verify - [Arguments] ${controller_ip} ${timeout}=3s - [Documentation] Add a car-person via the data store and get the car-person from Leader. - ... Note: This is done to get the car-people container created so subsequent - ... BuyCar RPC puts to the car-person list will succeed. - AddCarPerson ${controller_ip} ${RESTCONFPORT} ${0} - Wait Until Keyword Succeeds ${timeout} 1s Get One Car-Person Mapping And Verify ${controller_ip} ${0} - -Get One Car-Person Mapping And Verify - [Arguments] ${controller_ip} ${number} - [Documentation] Gets a car person mapping and verifies that the user ID is correct. - ${resp} GetCarPersonMappings ${controller_ip} ${RESTCONFPORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user${number} - -Get Car-Person Mappings And Verify - [Arguments] ${controller_ip} ${num_entries} - ${resp} GetCarPersonMappings ${controller_ip} ${RESTCONFPORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - : FOR ${i} IN RANGE 1 ${num_entries}+1 - \ Should Contain ${resp.content} user${i} - -Buy Cars And Verify - [Arguments] ${controller_ip} ${num_entries} ${start}=${0} ${timeout}=3s - BuyCar ${controller_ip} ${RESTCONFPORT} ${num_entries} ${start} - ${total_entries} Evaluate ${start}+${num_entries} - Wait Until Keyword Succeeds ${timeout} 1s Get Car-Person Mappings And Verify ${controller_ip} ${total_entries} - -Check Cars Deleted - [Arguments] ${controller_ip} - ${resp} Getcars ${controller_ip} ${RESTCONFPORT} ${0} - Should Be Equal As Strings ${resp.status_code} 404 - -Delete All Cars And Verify - [Arguments] ${controller_ip} ${timeout}=3s - DeleteAllCars ${controller_ip} ${RESTCONFPORT} ${0} - Wait Until Keyword Succeeds ${timeout} 1s Check Cars Deleted ${controller_ip} - -Check People Deleted - [Arguments] ${controller_ip} - ${resp} GetPersons ${controller_ip} ${RESTCONFPORT} ${0} - Should Be Equal As Strings ${resp.status_code} 404 - -Delete All People And Verify - [Arguments] ${controller_ip} ${timeout}=3s - DeleteAllPersons ${controller_ip} ${RESTCONFPORT} ${0} - Wait Until Keyword Succeeds ${timeout} 1s Check People Deleted ${controller_ip} - -Check Cars-Persons Deleted - [Arguments] ${controller_ip} - ${resp} GetCarPersonMappings ${controller_ip} ${RESTCONFPORT} ${0} - Should Be Equal As Strings ${resp.status_code} 404 - -Delete All Cars-Persons And Verify - [Arguments] ${controller_ip} ${timeout}=3s - DeleteAllCarsPersons ${controller_ip} ${RESTCONFPORT} ${0} - Wait Until Keyword Succeeds ${timeout} 1s Check Cars-Persons Deleted ${controller_ip} - -Delete All Entries From Shards - [Arguments] @{controllers} - [Documentation] Delete All Shards. - : FOR ${ip} IN @{controllers} - \ Delete All Cars And Verify ${ip} - : FOR ${ip} IN @{controllers} - \ Delete All People And Verify ${ip} - : FOR ${ip} IN @{controllers} - \ Delete All Cars-Persons And Verify ${ip} - -Check Cars - [Arguments] ${controller_ip} ${nth_car} - [Documentation] Verifies that the first through nth car is present. - ${resp} Getcars ${controller_ip} ${RESTCONFPORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - : FOR ${INDEX} IN RANGE 1 ${nth_car} - \ ${counter}= Convert to String ${INDEX} - \ Log manufacturer${counter} - \ Should Contain ${resp.content} manufacturer${counter} - -Check People - [Arguments] ${controller_ip} ${nth_person} - [Documentation] Verifies that the first through nth person is present. - ${resp} GetPersons ${controller_ip} ${RESTCONFPORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - : FOR ${INDEX} IN RANGE 1 ${nth_person} - \ ${counter}= Convert to String ${INDEX} - \ Log user${counter} - \ Should Contain ${resp.content} user${counter} - -Check CarPeople - [Arguments] ${controller_ip} ${nth_carperson} - [Documentation] Verifies that the first through nth car-person is present. - ${resp} GetCarPersonMappings ${controller_ip} ${RESTCONFPORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - : FOR ${INDEX} IN RANGE 1 ${nth_carperson} - \ ${counter}= Convert to String ${INDEX} - \ Log user${counter} - \ Should Contain ${resp.content} user${counter} - -Check Elements In Shards - [Arguments] ${controller_ip} ${nth} - [Documentation] Check all shards for nth elements - wait until keyword succeeds 3 1 Check Cars ${controller_ip} ${nth} - wait until keyword succeeds 3 1 Check People ${controller_ip} ${nth} - wait until keyword succeeds 3 1 Check CarPeople ${controller_ip} ${nth} - -Initialize Cars - [Arguments] ${controller_ip} ${field bases} - [Documentation] Initializes the cars shard by creating a 0th car with POST then deleting it. - ... Field bases are a dictionary of datastore record field values onto which is appended - ... an incremental value to uniquely identify the record from which it came. - ... Typically, you will use the Create Dictionary keyword on arguments which look like this: - ... id=${EMPTY} category=coupe model=model manufacturer=mfg year=2 - ${node}= Set Variable ${EMPTY} - ${prefix}= Set Variable {"car:cars":{"car-entry":[{ - ${postfix}= Set Variable }]}} - Create Records ${controller_ip} ${node} ${0} ${0} ${prefix} ${field bases} - ... ${postfix} - ${node}= Set Variable car:cars/car-entry - Delete Records ${controller_ip} ${node} ${0} ${0} - -Create Cars - [Arguments] ${controller_ip} ${first} ${last} ${field bases} - [Documentation] Creates cars with record IDs of specified range using POST. - ... If first and last are equal, only one record is updated. - ... Field bases are a dictionary of datastore record field values onto which is appended - ... an incremental value to uniquely identify the record from which it came. - ... Typically, you will use the Create Dictionary keyword on an argument which looks like this: - ... id=${EMPTY} category=coupe model=model manufacturer=mfg year=2 - ${node}= Set Variable car:cars - ${prefix}= Set Variable {"car-entry":[{ - ${postfix}= Set Variable }]} - Create Records ${controller_ip} ${node} ${first} ${last} ${prefix} ${field bases} - ... ${postfix} - -Update Cars - [Arguments] ${controller_ip} ${first} ${last} ${field bases} - [Documentation] Updates cars with record IDs of the specified using PUT. - ... If first and last are equal, only one record is updated. - ... Field bases are a dictionary of datastore record field values onto which is appended - ... an incremental value to uniquely identify the record from which it came. - ... Typically, you will use the Create Dictionary keyword on arguments which look like this: - ... id=${EMPTY} category=coupe model=model manufacturer=mfg year=2 - ${node}= Set Variable car:cars/car-entry - ${prefix}= Set Variable {"car-entry":[{ - ${postfix}= Set Variable }]} - Update Records ${controller_ip} ${node} ${first} ${last} ${prefix} ${field bases} - ... ${postfix} - -Read All Cars - [Arguments] ${controller_ip} - [Documentation] Returns all records from the cars shard in JSON format. - ${node}= Set Variable car:cars - ${result}= Read Records ${controller_ip} ${node} - [Return] ${result} - -Read One Car - [Arguments] ${controller_ip} ${id} - [Documentation] Returns the specified record from the cars shard in JSON format. - ${node}= Set Variable car:cars/car-entry/${id} - ${result}= Read Records ${controller_ip} ${node} - [Return] ${result} - -Remove All Cars - [Arguments] ${controller_ip} - [Documentation] Deletes all records from the cars shard. - ${node}= Set Variable car:cars - Delete All Records ${controller_ip} ${node} - -Remove Cars - [Arguments] ${controller_ip} ${first} ${last} - [Documentation] Deletes the specified range of records from the cars shard. - ${node}= Set Variable car:cars/car-entry - Delete Records ${controller_ip} ${node} ${first} ${last} diff --git a/csit/libraries/ClusterManagement.robot b/csit/libraries/ClusterManagement.robot new file mode 100644 index 0000000000..fbd3fbe656 --- /dev/null +++ b/csit/libraries/ClusterManagement.robot @@ -0,0 +1,242 @@ +*** Settings *** +Documentation Resource housing Keywords common to several suites for cluster functional testing. +... +... Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... +... This resource holds private state (in suite variables), +... which is generated once at Setup. +... The state includes list with indexes (numbers enumerating cluster members), +... IP addresses and Http (RequestsLibrary) sessions. +... Most functionality deals with stopping/starting controllers +... and finding leaders/followers for a Shard. +... +... odl-jolokia is assumed to be installed. +... +... Keywords which run commands on ODL systems do not preserve active SSH session. +... TODO: Should they? +... +... Keywords are ordered from friendly ones to fiddly ones. +... TODO: Figure out more deterministic but still user-friendly ordering. +... +... TODO: Unify capitalization of Leaders and Followers. +... +... TODO: Move Keywords related to iptables manipulation from ClusterKeywords +... here, or to separate Resource. +Library RequestsLibrary # for Create_Session and To_Json +Library Collections +Resource ${CURDIR}/TemplatedRequests.robot # for Get_As_Json_From_Uri +Resource ${CURDIR}/Utils.robot # for Run_Command_On_Controller + +*** Variables *** +${JOLOKIA_CONF_SHARD_MANAGER_URI} jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore +${JOLOKIA_OPER_SHARD_MANAGER_URI} jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-operational,type=DistributedOperationalDatastore +${JOLOKIA_READ_URI} jolokia/read/org.opendaylight.controller +${KARAF_HOME} ${WORKSPACE}${/}${BUNDLEFOLDER} +${RESTCONF_MODULES_DIR} ${CURDIR}/../variables/restconf/modules + +*** Keywords *** +ClusterManagement_Setup + [Documentation] Detect repeated call, or detect number of members and initialize derived suite variables. + # Avoid multiple initialization by several downstream libraries. + ${already_done} = BuiltIn.Get_Variable_Value \${ClusterManagement__has_setup_run} False + BuiltIn.Return_From_Keyword_If ${already_done} + BuiltIn.Set_Suite_Variable \${ClusterManagement__has_setup_run} True + ${status} ${possibly_int_of_members} = BuiltIn.Run_Keyword_And_Ignore_Error BuiltIn.Convert_To_Integer ${NUM_ODL_SYSTEM} + ${int_of_members} = BuiltIn.Set_Variable_If '${status}' != 'PASS' ${1} ${possibly_int_of_members} + ClusterManagement__Compute_Derived_Variables int_of_members=${int_of_members} + +Kill_Members_From_List_Or_All + [Arguments] ${member_index_list}=${EMPTY} ${confirm}=True + [Documentation] If the list is empty, kill all ODL instances. Otherwise, kill members based on present indices. + ... If \${confirm} is True, sleep 1 second and verify killed instances are not there anymore. + ${command} = BuiltIn.Set_Variable ps axf | grep karaf | grep -v grep | awk '{print \"kill -9 \" $1}' | sh + Run_Command_On_List_Or_All command=${command} member_index_list=${member_index_list} + BuiltIn.Return_From_Keyword_If not ${confirm} + # TODO: Convert to WUKS with configurable timeout if it turns out 1 second is not enough. + BuiltIn.Sleep 1s Kill -9 closes open files, which may take longer than ssh overhead, but not long enough to warrant WUKS. + ${index_list} = ClusterManagement__Given_Or_Internal_Index_List given_list=${member_index_list} + : FOR ${index} IN @{index_list} + \ Verify_Karaf_Is_Not_Running_On_Member member_index=${index} + +Clean_Journals_And_Snapshots_On_List_Or_All + [Arguments] ${member_index_list}=${EMPTY} + [Documentation] Delete journal and snapshots directories on every node listed (or all). + ${index_list} = ClusterManagement__Given_Or_Internal_Index_List given_list=${member_index_list} + ${command} = Set Variable rm -rf "${KARAF_HOME}/journal" "${KARAF_HOME}/snapshots" + : FOR ${index} IN @{index_list} # usually: 1, 2, 3. + \ Run_Command_On_Member command=${command} member_index=${index} + +Start_Members_From_List_Or_All + [Arguments] ${member_index_list}=${EMPTY} ${wait_for_sync}=True ${timeout}=300s + [Documentation] If the list is empty, start all cluster members. Otherwise, start members based on present indices. + ... If ${wait_for_sync}, wait for cluster sync on listed members. + ${command} = BuiltIn.Set_Variable ${KARAF_HOME}/bin/start + Run_Command_On_List_Or_All command=${command} member_index_list=${member_index_list} + BuiltIn.Return_From_Keyword_If not ${wait_for_sync} + BuiltIn.Wait_Until_Keyword_Succeeds ${timeout} 1s Check_Cluster_Is_In_Sync member_index_list=${member_index_list} + # TODO: Do we also want to check Shard Leaders here? + +Verify_Leader_Exists_For_Each_Shard + [Arguments] ${shard_name_list} ${shard_type}=operational ${member_index_list}=${EMPTY} ${verify_restconf}=True + [Documentation] For each shard name, call Get_Leader_And_Followers_For_Shard. + ... Not much logic there, but single Keyword is useful when using BuiltIn.Wait_Until_Keyword_Succeeds. + : FOR ${shard_name} IN @{shard_name_list} + \ Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} validate=True member_index_list=${member_index_list} verify_restconf=${verify_restconf} + +Get_Leader_And_Followers_For_Shard + [Arguments] ${shard_name}=default ${shard_type}=operational ${validate}=True ${member_index_list}=${EMPTY} ${verify_restconf}=True + [Documentation] Get role lists, validate there is one leader, return the leader and list of followers. + ... Optionally, issue GET to a simple restconf URL to make sure subsequent operations will not encounter 503. + ${leader_list} ${follower_list} = Get_State_Info_For_Shard shard_name=${shard_name} shard_type=${shard_type} validate=True member_index_list=${member_index_list} verify_restconf=${verify_restconf} + ${leader_count} = BuiltIn.Get_Length ${leader_list} + BuiltIn.Run_Keyword_If ${leader_count} < 1 BuiltIn.Fail No leader found. + BuiltIn.Length_Should_Be ${leader_list} ${1} Too many Leaders. + ${leader} = Collections.Get_From_List ${leader_list} 0 + [Return] ${leader} ${follower_list} + +Resolve_Http_Session_For_Member + [Arguments] ${member_index} + [Documentation] Return RequestsLibrary session alias pointing to node of given index. + ${session} = BuiltIn.Set_Variable ClusterManagement__session_${member_index} + [Return] ${session} + +Get_State_Info_For_Shard + [Arguments] ${shard_name}=default ${shard_type}=operational ${validate}=False ${member_index_list}=${EMPTY} ${verify_restconf}=False + [Documentation] Return lists of Leader and Follower member indices from a given member index list + ... (or from the full list if empty). If \${shard_type} is not 'config', 'operational' is assumed. + ... If \${validate}, Fail if raft state is not Leader or Follower (for example on Candidate). + ... The biggest difference from Get_Leader_And_Followers_For_Shard + ... is that no check on number of Leaders is performed. + ${index_list} = ClusterManagement__Given_Or_Internal_Index_List given_list=${member_index_list} + # TODO: Support alternative capitalization of 'config'? + ${ds_type} = BuiltIn.Set_Variable_If '${shard_type}' != 'config' operational config + ${leader_list} = BuiltIn.Create_List + ${follower_list} = BuiltIn.Create_List + : FOR ${index} IN @{index_list} # usually: 1, 2, 3. + \ ${raft_state} = Get_Raft_State_Of_Shard_At_Member shard_name=${shard_name} shard_type=${ds_type} member_index=${index} verify_restconf=${verify_restconf} + \ BuiltIn.Run_Keyword_If 'Follower' == '${raft_state}' Collections.Append_To_List ${follower_list} ${index} + \ ... ELSE IF 'Leader' == '${raft_state}' Collections.Append_To_List ${leader_list} ${index} + \ ... ELSE IF ${validate} BuiltIn.Fail Unrecognized Raft state: ${raft_state} + [Return] ${leader_list} ${follower_list} + +Check_Cluster_Is_In_Sync + [Arguments] ${member_index_list}=${EMPTY} + [Documentation] Fail if no-sync is detected on a member from list (or any). + ${index_list} = ClusterManagement__Given_Or_Internal_Index_List given_list=${member_index_list} + : FOR ${index} IN @{index_list} # usually: 1, 2, 3. + \ ${status} = Get_Sync_Status_Of_Member member_index=${index} + \ # The previous line may have failed already. If not, check status. + \ BuiltIn.Continue_For_Loop_If 'True' == '${status}' + \ BuiltIn.Fail Index ${index} has incorrect status: ${status} + +Verify_Karaf_Is_Not_Running_On_Member + [Arguments] ${member_index} + [Documentation] Fail if non-zero karaf instances are counted on member of given index. + ${count} = Count_Running_Karafs_On_Member member_index=${member_index} + BuiltIn.Should_Be_Equal 0 ${count} Found running Karaf count: ${count} + +Verify_Single_Karaf_Is_Running_On_Member + [Arguments] ${member_index} + [Documentation] Fail if number of karaf instances on member of given index is not one. + ${count} = Count_Running_Karafs_On_Member member_index=${member_index} + BuiltIn.Should_Be_Equal 1 ${count} Wrong number of Karafs running: ${count} + +Run_Command_On_List_Or_All + [Arguments] ${command} ${member_index_list}=${EMPTY} + [Documentation] Cycle through indices (or all), run command on each. + ${index_list} = ClusterManagement__Given_Or_Internal_Index_List given_list=${member_index_list} + : FOR ${index} IN @{index_list} + \ Run_Command_On_Member command=${command} member_index=${index} + +Get_Sync_Status_Of_Member + [Arguments] ${member_index} + [Documentation] Obtain IP, two GETs from jolokia URIs, return combined sync status as string. + ${session} = Resolve_Http_Session_For_Member member_index=${member_index} + ${conf_text} = Get_As_Json_From_Uri uri=${JOLOKIA_CONF_SHARD_MANAGER_URI} session=${session} + ${conf_status} = ClusterManagement__Parse_Sync_Status shard_manager_text=${conf_text} + BuiltIn.Return_From_Keyword_If 'False' == ${conf_status} False + ${oper_text} = Get_As_Json_From_Uri uri=${JOLOKIA_OPER_SHARD_MANAGER_URI} session=${session} + ${oper_status} = ClusterManagement__Parse_Sync_Status shard_manager_text=${oper_text} + [Return] ${oper_status} + +Run_Command_On_Member + [Arguments] ${command} ${member_index} + [Documentation] Obtain IP, call Utils and return output. This does not preserve active ssh session. + ${member_ip} = Collections.Get_From_Dictionary dictionary=${ClusterManagement__index_to_ip_mapping} key=${member_index} + ${output} = Utils.Run_Command_On_Controller ${member_ip} ${command} + [Return] ${output} + +Count_Running_Karafs_On_Member + [Arguments] ${member_index} + [Documentation] Remotely execute grep for karaf process, return count as string. + ${command} = BuiltIn.Set_Variable ps axf | grep karaf | grep -v grep | wc -l + ${count} = Run_Command_On_Member command=${command} member_index=${member_index} + [Return] ${count} + +Get_Raft_State_Of_Shard_At_Member + [Arguments] ${shard_name} ${shard_type} ${member_index} ${verify_restconf}=False + [Documentation] Send request to Jolokia on indexed member, return extracted Raft status. + ... Optionally, check restconf works. + ${session} = Resolve_Http_Session_For_Member member_index=${member_index} + # TODO: Does the used URI tend to generate large data which floods log.html? + BuiltIn.Run_Keyword_If ${verify_restconf} TemplatedRequests.Get_As_Json_Templated session=${session} folder=${RESTCONF_MODULES_DIR} verify=False + ${type_class} = Resolve_Shard_Type_Class shard_type=${shard_type} + ${uri} = BuiltIn.Set_Variable ${JOLOKIA_READ_URI}:Category=Shards,name=member-${member_index}-shard-${shard_name}-${shard_type},type=${type_class} + ${data_text} = TemplatedRequests.Get_As_Json_From_Uri uri=${uri} session=${session} + ${data_object} = RequestsLibrary.To_Json ${data_text} + ${value} = Collections.Get_From_Dictionary ${data_object} value + ${raft_state} = Collections.Get_From_Dictionary ${value} RaftState + [Return] ${raft_state} + +Resolve_Shard_Type_Class + [Arguments] ${shard_type} + [Documentation] Simple lookup for class name corresponding to desired type. + BuiltIn.Run_Keyword_If '${shard_type}' == 'config' BuiltIn.Return_From_Keyword DistributedConfigDatastore + ... ELSE IF '${shard_type}' == 'operational' BuiltIn.Return_From_Keyword DistributedOperationalDatastore + BuiltIn.Fail Unrecognized shard type: ${shard_type} + +ClusterManagement__Parse_Sync_Status + [Arguments] ${shard_manager_text} + [Documentation] Return sync status parsed out of given text. Called twice by Get_Sync_Status_Of_Member. + BuiltIn.Log ${shard_manager_text} + ${manager_object} = RequestsLibrary.To_Json ${shard_manager_text} + ${value_object} = Collections.Get_From_Dictionary dictionary=${manager_object} key=value + ${sync_status} = Collections.Get_From_Dictionary dictionary=${value_object} key=SyncStatus + [Return] ${sync_status} + +ClusterManagement__Given_Or_Internal_Index_List + [Arguments] ${given_list}=${EMPTY} + [Documentation] Utility to allow \${EMPTY} as default argument value, as the internal list is computed at runtime. + ${given_length} = BuiltIn.Get_Length ${given_list} + ${return_list} = BuiltIn.Set_Variable_If ${given_length} > 0 ${given_list} ${ClusterManagement__member_index_list} + [Return] ${return_list} + +ClusterManagement__Compute_Derived_Variables + [Arguments] ${int_of_members} + [Documentation] Construct index list, session list and IP mapping, publish them as suite variables. + @{member_index_list} = BuiltIn.Create_List + @{session_list} = BuiltIn.Create_List + &{index_to_ip_mapping} = BuiltIn.Create_Dictionary + : FOR ${index} IN RANGE 1 ${int_of_members+1} + \ ClusterManagement__Include_Member_Index ${index} ${member_index_list} ${session_list} ${index_to_ip_mapping} + BuiltIn.Set_Suite_Variable \${ClusterManagement__member_index_list} ${member_index_list} + BuiltIn.Set_Suite_Variable \${ClusterManagement__index_to_ip_mapping} ${index_to_ip_mapping} + BuiltIn.Set_Suite_Variable \${ClusterManagement__session_list} ${session_list} + +ClusterManagement__Include_Member_Index + [Arguments] ${index} ${member_index_list} ${session_list} ${index_to_ip_mapping} + [Documentation] Add a corresponding item based on index into the last three arguments. + ... Create the Http session whose alias is added to list. + Collections.Append_To_List ${member_index_list} ${index} + ${member_ip} = BuiltIn.Set_Variable ${ODL_SYSTEM_${index}_IP} + # ${index} is int (not string) so "key=value" syntax does not work in the following line. + Collections.Set_To_Dictionary ${index_to_ip_mapping} ${index} ${member_ip} + # Http session, with ${AUTH}, without headers. + ${session_alias} = Resolve_Http_Session_For_Member member_index=${index} + RequestsLibrary.Create_Session ${session_alias} http://${member_ip}:${RESTCONFPORT} auth=${AUTH} + Collections.Append_To_List ${session_list} ${session_alias} diff --git a/csit/libraries/TemplatedRequests.robot b/csit/libraries/TemplatedRequests.robot index 9a2657443a..992bd9ee3a 100644 --- a/csit/libraries/TemplatedRequests.robot +++ b/csit/libraries/TemplatedRequests.robot @@ -286,7 +286,7 @@ Verify_Response_Templated # TODO: Support for XML-aware comparison could be added, but there are issues with namespaces and similar. ${expected_text} = Resolve_Text_From_Template_Folder folder=${folder} base_name=${base_name} extension=${extension} mapping=${mapping} endline=${endline} ... iterations=${iterations} iter_start=${iter_start} - BuiltIn.Run_Keyword_If ${normalize_json} Normalize_Jsons_And_Compare ${expected_text} ${response} + BuiltIn.Run_Keyword_If ${normalize_json} Normalize_Jsons_And_Compare expected_raw=${expected_text} actual_raw=${response} ... ELSE BuiltIn.Should_Be_Equal ${expected_text} ${response} Get_From_Uri @@ -369,7 +369,7 @@ Resolve_Text_From_Template_Folder ${separator} = BuiltIn.Set_Variable_If '${extension}' != 'json' ${endline} ,${endline} : FOR ${iteration} IN RANGE ${iter_start} ${iterations}+${iter_start} \ # Add separator only if we are beyond first item. - \ BuiltIn.Run_Keyword_If ${iteration} > 1 Collections.Append_To_List ${items} ${separator} + \ BuiltIn.Run_Keyword_If ${iteration} > ${iter_start} Collections.Append_To_List ${items} ${separator} \ ${item} = BuiltIn.Evaluate string.Template('''${item_template}''').substitute({"i":"${iteration}"}) modules=string \ Collections.Append_To_List ${items} ${item} # TODO: The following makes ugly result for iterations=0. Should we fix that? @@ -387,10 +387,10 @@ Resolve_Text_From_Template_File [Return] ${final_text} Normalize_Jsons_And_Compare - [Arguments] ${actual_raw} ${expected_raw} + [Arguments] ${expected_raw} ${actual_raw} [Documentation] Use norm_json to normalize both JSON arguments, call Should_Be_Equal. - ${actual_normalized} = norm_json.normalize_json_text ${actual_raw} ${expected_normalized} = norm_json.normalize_json_text ${expected_raw} + ${actual_normalized} = norm_json.normalize_json_text ${actual_raw} # Should_Be_Equal shall print nice diff-style line comparison. BuiltIn.Should_Be_Equal ${expected_normalized} ${actual_normalized} # TODO: Add garbage collection? Check whether the temporary data accumulates. diff --git a/csit/libraries/norm_json.py b/csit/libraries/norm_json.py index d46d6c78ff..7dde004504 100644 --- a/csit/libraries/norm_json.py +++ b/csit/libraries/norm_json.py @@ -18,7 +18,7 @@ __license__ = "Eclipse Public License v1.0" __email__ = "vrpolak@cisco.com" -# Internal details. +# Internal details; look down below for Robot Keywords. class _Hsfl(list): @@ -93,23 +93,12 @@ class _Decoder(_json.JSONDecoder): self.scan_once = _json.scanner.py_make_scanner(self) -# Robot Keywords. +# Robot Keywords; look above for internal details. def loads_sorted(text, strict=False): - """ - Return Python object with sorted arrays and dictionary keys. - - If strict is true, raise exception on parse error. - If strict is not true, return string with error message. - """ - try: - object_decoded = _json.loads(text, cls=_Decoder, object_hook=_Hsfod) - except ValueError as err: - if strict: - raise err - else: - return str(err) + '\n' + text + """Return Python object with sorted arrays and dictionary keys.""" + object_decoded = _json.loads(text, cls=_Decoder, object_hook=_Hsfod) return object_decoded @@ -126,7 +115,19 @@ def dumps_indented(obj, indent=1): def normalize_json_text(text, strict=False, indent=1): # pylint likes lowercase - """Return sorted indented JSON string, or an error message string.""" - object_decoded = loads_sorted(text, strict=strict) + """ + Attempt to return sorted indented JSON string. + + If parse error happens: + If strict is true, raise the exception. + If strict is not true, return original text with error message. + """ + try: + object_decoded = loads_sorted(text) + except ValueError as err: + if strict: + raise err + else: + return str(err) + '\n' + text pretty_json = dumps_indented(object_decoded, indent=indent) return pretty_json diff --git a/csit/suites/controller/Clustering_Datastore/001_start_cluster.robot b/csit/suites/controller/Clustering_Datastore/001_start_cluster.robot deleted file mode 100644 index 1a3eca67bd..0000000000 --- a/csit/suites/controller/Clustering_Datastore/001_start_cluster.robot +++ /dev/null @@ -1,26 +0,0 @@ -*** Settings *** -Documentation Start the controllers -Default Tags 3-node-cluster -Resource ../../../libraries/ClusterKeywords.robot - -*** Variables *** -@{controllers} ${ODL_SYSTEM_IP} ${ODL_SYSTEM_2_IP} ${ODL_SYSTEM_3_IP} -${KARAF_HOME} ${WORKSPACE}/${BUNDLEFOLDER} -${START_TIMEOUT} 300s -${STOP_TIMEOUT} 180s - -*** Test Cases *** -Stop All Controllers - [Documentation] Stop all the controllers in the cluster. - Stop One Or More Controllers @{controllers} - Wait For Cluster Down ${STOP_TIMEOUT} @{controllers} - -Clean All Journals - [Documentation] Clean the journals of all the controllers in the cluster - Clean One Or More Journals @{controllers} - Clean One Or More Snapshots @{controllers} - -Start All Controllers - [Documentation] Start all the controllers in the cluster - Start One Or More Controllers @{controllers} - Wait For Cluster Sync ${START_TIMEOUT} @{controllers} diff --git a/csit/suites/controller/Clustering_Datastore/010_crud_on_leader.robot b/csit/suites/controller/Clustering_Datastore/010_crud_on_leader.robot deleted file mode 100644 index 32fc8672be..0000000000 --- a/csit/suites/controller/Clustering_Datastore/010_crud_on_leader.robot +++ /dev/null @@ -1,93 +0,0 @@ -*** Settings *** -Documentation This test finds the leader for shards in a 3-Node cluster and executes CRUD operations on them -Default Tags 3-node-cluster -Resource ../../../libraries/ClusterKeywords.robot -Resource ../../../libraries/CarsAndPeople.robot -Variables ../../../variables/Variables.py - -*** Variables *** -${SHARD_CAR_NAME} shard-car-config -${SHARD_PEOPLE_NAME} shard-people-config -${SHARD_CAR_PERSON_NAME} shard-car-people-config -${NUM_ENTRIES} ${30} - -*** Test Cases *** -Get Car Leader And Followers - [Documentation] Find leader and followers in the car shard - ${CURRENT_CAR_LEADER} Get Leader And Verify ${SHARD_CAR_NAME} - Set Suite Variable ${CURRENT_CAR_LEADER} - ${CAR_FOLLOWERS} Get All Followers ${SHARD_CAR_NAME} - Set Suite Variable ${CAR_FOLLOWERS} - -Add Cars And Get Cars From Leader - [Documentation] Add some cars and get added cars from Leader - Add Cars And Verify ${CURRENT_CAR_LEADER} ${NUM_ENTRIES} - -Get Added Cars From Follower1 - [Documentation] Get added cars from Follower1 - Get Cars And Verify @{CAR_FOLLOWERS}[0] ${NUM_ENTRIES} - -Get Added Cars From Follower2 - [Documentation] Get added cars from Follower2 - Get Cars And Verify @{CAR_FOLLOWERS}[1] ${NUM_ENTRIES} - -Get People Leader And Followers - [Documentation] Find leader and followers in the people shard - ${CURRENT_PEOPLE_LEADER} Get Leader And Verify ${SHARD_PEOPLE_NAME} - Set Suite Variable ${CURRENT_PEOPLE_LEADER} - ${PEOPLE_FOLLOWERS} Get All Followers ${SHARD_PEOPLE_NAME} - Set Suite Variable ${PEOPLE_FOLLOWERS} - -Add People And Get People From Leader - [Documentation] Add some people and get people from Leader. - Add People And Verify ${CURRENT_PEOPLE_LEADER} ${NUM_ENTRIES} - -Get Added People From Follower1 - [Documentation] Get added people from Follower1 - Get People And Verify @{PEOPLE_FOLLOWERS}[0] ${NUM_ENTRIES} - -Get Added People From Follower2 - [Documentation] Get added people from Follower2 - Get People And Verify @{PEOPLE_FOLLOWERS}[1] ${NUM_ENTRIES} - -Get Car-Person Leader And Followers - [Documentation] Find leader and followers in the car-person shard - ${CURRENT_CAR_PERSON_LEADER} Get Leader And Verify ${SHARD_CAR_PERSON_NAME} - Set Suite Variable ${CURRENT_CAR_PERSON_LEADER} - ${CAR_PERSON_FOLLOWERS} Get All Followers ${SHARD_CAR_PERSON_NAME} - Set Suite Variable ${CAR_PERSON_FOLLOWERS} - -Add Car-Person Mapping And Get Car-Person Mapping From Leader - [Documentation] Initialize car-person shard - Add Car Person And Verify ${CURRENT_CAR_PERSON_LEADER} - -Purchase Cars On Leader - [Documentation] Purchase some cars on the Leader - ${NUM_BUY_CARS_ON_LEADER} Evaluate ${NUM_ENTRIES}/3 - ${NUM_BUY_CARS_ON_FOLLOWER1} Evaluate ${NUM_ENTRIES}/3 - ${NUM_BUY_CARS_ON_FOLLOWER2} Evaluate ${NUM_ENTRIES}-${NUM_BUY_CARS_ON_LEADER}-${NUM_BUY_CARS_ON_FOLLOWER1} - Set Suite Variable ${NUM_BUY_CARS_ON_LEADER} - Set Suite Variable ${NUM_BUY_CARS_ON_FOLLOWER1} - Set Suite Variable ${NUM_BUY_CARS_ON_FOLLOWER2} - Buy Cars And Verify ${CURRENT_CAR_PERSON_LEADER} ${NUM_BUY_CARS_ON_LEADER} - -Purchase Cars On Follower1 - [Documentation] Purchase some cars on Follower1 - Buy Cars And Verify @{CAR_PERSON_FOLLOWERS}[0] ${NUM_BUY_CARS_ON_FOLLOWER1} ${NUM_BUY_CARS_ON_LEADER} - -Purchase Cars On Follower2 - [Documentation] Purchase some cars on Follower2 - ${start} Evaluate ${NUM_BUY_CARS_ON_LEADER}+${NUM_BUY_CARS_ON_FOLLOWER1} - Buy Cars And Verify @{CAR_PERSON_FOLLOWERS}[1] ${NUM_BUY_CARS_ON_FOLLOWER2} ${start} - -Get Car-Person Mappings From Leader - [Documentation] Get car-person mappings from Leader to see all entries - Get Car-Person Mappings And Verify ${CURRENT_CAR_PERSON_LEADER} ${NUM_ENTRIES} - -Get Car-Person Mappings From Follower1 - [Documentation] Get car-person mappings from Follower1 to see all entries - Get Car-Person Mappings And Verify @{CAR_PERSON_FOLLOWERS}[0] ${NUM_ENTRIES} - -Get Car-Person Mappings From Follower2 - [Documentation] Get car-person mappings from Follower2 to see all entries - Get Car-Person Mappings And Verify @{CAR_PERSON_FOLLOWERS}[1] ${NUM_ENTRIES} diff --git a/csit/suites/controller/Clustering_Datastore/020_crud_on_any_follower.robot b/csit/suites/controller/Clustering_Datastore/020_crud_on_any_follower.robot deleted file mode 100644 index aa8ff496f2..0000000000 --- a/csit/suites/controller/Clustering_Datastore/020_crud_on_any_follower.robot +++ /dev/null @@ -1,72 +0,0 @@ -*** Settings *** -Documentation This test finds the followers of certain shards in a 3-Node cluster and executes CRUD operations on any one follower -Default Tags 3-node-cluster -Resource ../../../libraries/ClusterKeywords.robot -Resource ../../../libraries/CarsAndPeople.robot -Variables ../../../variables/Variables.py - -*** Variables *** -${SHARD_CAR_NAME} shard-car-config -${SHARD_PEOPLE_NAME} shard-people-config -${SHARD_CAR_PERSON_NAME} shard-car-people-config -${NUM_ENTRIES} ${40} - -*** Test Cases *** -Get Car Followers - [Documentation] Find followers in the car shard - ${CAR_FOLLOWERS} Get All Followers ${SHARD_CAR_NAME} - Set Suite Variable ${CAR_FOLLOWERS} - -Get People Followers - [Documentation] Find followers in the people shard - ${PEOPLE_FOLLOWERS} Get All Followers ${SHARD_PEOPLE_NAME} - Set Suite Variable ${PEOPLE_FOLLOWERS} - -Get Car-Person Followers - ${CAR_PERSON_FOLLOWERS} Get All Followers ${SHARD_CAR_PERSON_NAME} - Set Suite Variable ${CAR_PERSON_FOLLOWERS} - -Delete Cars From Follower1 - Delete All Cars And Verify @{CAR_FOLLOWERS}[0] - -Delete People From Follower1 - Delete All People And Verify @{PEOPLE_FOLLOWERS}[0] - -Delete Car-Persons from Follower1 - Delete All Cars-Persons And Verify @{CAR_PERSON_FOLLOWERS}[0] - -Add Cars And Get Cars From Follower1 - [Documentation] Add cars and get added cars from Follower1 - Add Cars And Verify @{CAR_FOLLOWERS}[0] ${NUM_ENTRIES} - -Get Added Cars From Follower2 - [Documentation] Get added cars from Follower2 - Get Cars And Verify @{CAR_FOLLOWERS}[1] ${NUM_ENTRIES} - -Add People And Get People From Follower1 - [Documentation] Add people and get people from Follower1 - Add People And Verify @{PEOPLE_FOLLOWERS}[0] ${NUM_ENTRIES} - -Get Added People From Follower2 - [Documentation] Get added people from Follower2 - Get People And Verify @{PEOPLE_FOLLOWERS}[1] ${NUM_ENTRIES} - -Add Car-Person Mapping And Get Car-Person Mapping From Follower1 - Add Car Person And Verify @{CAR_PERSON_FOLLOWERS}[0] - -Purchase Cars On Follower1 - [Documentation] Purchase cars using Follower1 - Buy Cars And Verify @{CAR_PERSON_FOLLOWERS}[0] ${NUM_ENTRIES} - -Get Car-Person Mappings From Follower1 - [Documentation] Get car-person mappings from Follower1 to see all entries - Get Car-Person Mappings And Verify @{CAR_PERSON_FOLLOWERS}[0] ${NUM_ENTRIES} - -Get Car-Person Mappings From Leader - [Documentation] Get car-person mappings from the Leader to see all entries - ${CURRENT_CAR_LEADER} Get Leader And Verify ${SHARD_CAR_PERSON_NAME} - Get Car-Person Mappings And Verify ${CURRENT_CAR_LEADER} ${NUM_ENTRIES} - -Get Car-Person Mappings From Follower2 - [Documentation] Get car-person mappings from Follower2 to see all entries - Get Car-Person Mappings And Verify @{CAR_PERSON_FOLLOWERS}[1] ${NUM_ENTRIES} diff --git a/csit/suites/controller/Clustering_Datastore/030_car_failover_crud_on_new_leader.robot b/csit/suites/controller/Clustering_Datastore/030_car_failover_crud_on_new_leader.robot deleted file mode 100644 index 8c8f432668..0000000000 --- a/csit/suites/controller/Clustering_Datastore/030_car_failover_crud_on_new_leader.robot +++ /dev/null @@ -1,78 +0,0 @@ -*** Settings *** -Documentation This test brings down the current leader of the "car" shard and then executes CRUD -... operations on the new leader -Default Tags 3-node-cluster -Resource ../../../libraries/ClusterKeywords.robot -Resource ../../../libraries/CarsAndPeople.robot -Variables ../../../variables/Variables.py - -*** Variables *** -${CAR_SHARD} shard-car-config -${NUM_CARS} ${50} -${NUM_ORIG_CARS} ${10} -${KARAF_HOME} ${WORKSPACE}/${BUNDLEFOLDER} -${START_TIMEOUT} 300s -${STOP_TIMEOUT} 180s - -*** Test Cases *** -Get Old Car Leader - [Documentation] Find leader in the car shard - ${OLD_CAR_LEADER} Get Leader And Verify ${CAR_SHARD} - Set Suite Variable ${OLD_CAR_LEADER} - -Delete Cars On Old Leader - [Documentation] Delete cars in Leader - Delete All Cars And Verify ${OLD_CAR_LEADER} - -Add Original Cars On Old Leader - [Documentation] Add new cars in Leader and verify - Add Cars And Verify ${OLD_CAR_LEADER} ${NUM_ORIG_CARS} - -Switch Car Leader - [Documentation] Stop the leader to cause a new leader to be elected - Stop One Or More Controllers ${OLD_CAR_LEADER} - Wait For Controller Down ${STOP_TIMEOUT} ${OLD_CAR_LEADER} - ${NEW_CAR_LEADER} Wait Until Keyword Succeeds 30s 2s Get Leader And Verify ${CAR_SHARD} ${OLD_CAR_LEADER} - Set Suite Variable ${NEW_CAR_LEADER} - -Get Original Cars On New Leader - [Documentation] Get cars in new Leader - Get Cars And Verify ${NEW_CAR_LEADER} ${NUM_ORIG_CARS} - -Delete Cars On New Leader - [Documentation] Delete cars in new Leader - Delete All Cars And Verify ${NEW_CAR_LEADER} - -Add New Cars And Get Cars From New Leader - [Documentation] Add cars and get added cars from the Leader - Add Cars And Verify ${NEW_CAR_LEADER} ${NUM_CARS} - -Get Car Followers - [Documentation] Find followers in the car shard - ${CAR_FOLLOWERS} Get All Followers ${CAR_SHARD} ${OLD_CAR_LEADER} - Set Suite Variable ${CAR_FOLLOWERS} - -Get Added Cars From Follower - [Documentation] Get the added cars from the Follower - Get Cars And Verify @{CAR_FOLLOWERS}[0] ${NUM_CARS} - -Delete Cars On Follower - [Documentation] Delete cars in follower - Delete All Cars And Verify @{CAR_FOLLOWERS}[0] - -Add Cars From Follower - [Documentation] Add more cars from the Follower - Add Cars And Verify @{CAR_FOLLOWERS}[0] ${NUM_CARS} - -Get Added Cars From New Leader - [Documentation] Get added cars from the new leader - Get Cars And Verify ${NEW_CAR_LEADER} ${NUM_CARS} - -Restart Old Car Leader - [Documentation] Start old car Leader - Start One Or More Controllers ${OLD_CAR_LEADER} - Wait For Controller Sync ${START_TIMEOUT} ${OLD_CAR_LEADER} - -Get Added Cars From Old Leader - [Documentation] Get the added cars from the old leader - Wait Until Keyword Succeeds ${START_TIMEOUT} 2s Get Cars And Verify ${OLD_CAR_LEADER} ${NUM_CARS} diff --git a/csit/suites/controller/Clustering_Datastore/040_people_failover_crud_on_new_leader.robot b/csit/suites/controller/Clustering_Datastore/040_people_failover_crud_on_new_leader.robot deleted file mode 100644 index 326d2ba0b6..0000000000 --- a/csit/suites/controller/Clustering_Datastore/040_people_failover_crud_on_new_leader.robot +++ /dev/null @@ -1,68 +0,0 @@ -*** Settings *** -Documentation This test brings down the current leader of the "car" shard and then executes CRUD operations on the new leader -Default Tags 3-node-cluster -Resource ../../../libraries/ClusterKeywords.robot -Resource ../../../libraries/CarsAndPeople.robot -Variables ../../../variables/Variables.py - -*** Variables *** -${PEOPLE_SHARD} shard-people-config -${NUM_ENTRIES} ${50} -${KARAF_HOME} ${WORKSPACE}/${BUNDLEFOLDER} -${START_TIMEOUT} 300s -${STOP_TIMEOUT} 180s - -*** Test Cases *** -Get Old People Leader - [Documentation] Find leader in the people shard - ${OLD_PEOPLE_LEADER} Get Leader And Verify ${PEOPLE_SHARD} - Set Suite Variable ${OLD_PEOPLE_LEADER} - -Switch People Leader - [Documentation] Stop the leader to cause a new leader to be elected - Stop One Or More Controllers ${OLD_PEOPLE_LEADER} - Wait For Controller Down ${STOP_TIMEOUT} ${OLD_PEOPLE_LEADER} - ${NEW_PEOPLE_LEADER} Wait Until Keyword Succeeds 30s 2s Get Leader And Verify ${PEOPLE_SHARD} ${OLD_PEOPLE_LEADER} - Set Suite Variable ${NEW_PEOPLE_LEADER} - -Delete People From New Leader - [Documentation] Delete people in new Leader - Delete All People And Verify ${NEW_PEOPLE_LEADER} - -Add People And Get From New Leader - [Documentation] Add people and get people from new leader - Add People And Verify ${NEW_PEOPLE_LEADER} ${NUM_ENTRIES} - -Get People Followers - [Documentation] Find followers in the people shard - ${PEOPLE_FOLLOWERS} Get All Followers ${PEOPLE_SHARD} ${OLD_PEOPLE_LEADER} - Set Suite Variable ${PEOPLE_FOLLOWERS} - -Get Added People From Follower - [Documentation] Get people in follower and verify - Get People And Verify @{PEOPLE_FOLLOWERS}[0] ${NUM_ENTRIES} - -Delete People From New Follower - [Documentation] Delete people in follower and verify - Delete All People And Verify @{PEOPLE_FOLLOWERS}[0] - -Add People From New Follower - [Documentation] Add people in follower and verify - Add People And Verify @{PEOPLE_FOLLOWERS}[0] ${NUM_ENTRIES} - -Get Added People From New Leader - [Documentation] Get people in Leader and verify - Get People And Verify ${NEW_PEOPLE_LEADER} ${NUM_ENTRIES} - -Restart Old People Leader - [Documentation] Start old people Leader - Start One Or More Controllers ${OLD_PEOPLE_LEADER} - Wait For Controller Sync ${START_TIMEOUT} ${OLD_PEOPLE_LEADER} - -Check Cars In Old People Leader - [Documentation] Check cars in new Leader. This is to avoid delay when RPC does not work. - Wait Until Keyword Succeeds ${START_TIMEOUT} 2s Check Cars ${OLD_PEOPLE_LEADER} ${NUM_ENTRIES} - -Get Added People From Old Leader - [Documentation] Get people in old Leader and verify - Get People And Verify ${OLD_PEOPLE_LEADER} ${NUM_ENTRIES} diff --git a/csit/suites/controller/Clustering_Datastore/050_car_persistence_recovery.robot b/csit/suites/controller/Clustering_Datastore/050_car_persistence_recovery.robot deleted file mode 100644 index 259425f708..0000000000 --- a/csit/suites/controller/Clustering_Datastore/050_car_persistence_recovery.robot +++ /dev/null @@ -1,79 +0,0 @@ -*** Settings *** -Documentation This test restarts all controllers to verify recovery of car data from persistene -Default Tags 3-node-cluster -Resource ../../../libraries/ClusterKeywords.robot -Resource ../../../libraries/CarsAndPeople.robot -Variables ../../../variables/Variables.py - -*** Variables *** -${CAR_SHARD} shard-car-config -${NUM_CARS} ${50} -${KARAF_HOME} ${WORKSPACE}/${BUNDLEFOLDER} -@{controllers} ${ODL_SYSTEM_IP} ${ODL_SYSTEM_2_IP} ${ODL_SYSTEM_3_IP} -${START_TIMEOUT} 300s -${STOP_TIMEOUT} 180s - -*** Test Cases *** -Get Car Leader - [Documentation] Find leader in the car shard - ${CAR_LEADER} Get Leader And Verify ${CAR_SHARD} - Set Suite Variable ${CAR_LEADER} - -Delete Cars From Leader - [Documentation] Delete cars in Leader - Delete All Cars And Verify ${CAR_LEADER} - -Stop All Controllers After Delete - [Documentation] Stop all controllers - Stop One Or More Controllers @{controllers} - Wait For Cluster Down ${STOP_TIMEOUT} @{controllers} - -Start All Controllers After Delete - [Documentation] Start all controller - Start One Or More Controllers @{controllers} - Wait For Cluster Sync ${START_TIMEOUT} @{controllers} - -Get Car Leader After First Restart - [Documentation] Find leader in the car shard - ${CAR_LEADER} Get Leader And Verify ${CAR_SHARD} - Set Suite Variable ${CAR_LEADER} - -Verify No Cars On Leader After Restart - [Documentation] Verify no cars after restart - Wait Until Keyword Succeeds ${START_TIMEOUT} 2s Check Cars Deleted ${CAR_LEADER} - -Add Cars On Leader - [Documentation] Add cars in Leader - Add Cars And Verify ${CAR_LEADER} ${NUM_CARS} - -Stop All Controllers After Add - [Documentation] Stop all controllers - Stop One Or More Controllers @{controllers} - Wait For Cluster Down ${STOP_TIMEOUT} @{controllers} - -Start All Controllers After Add - [Documentation] Start all controllers - Start One Or More Controllers @{controllers} - Wait For Cluster Sync ${START_TIMEOUT} @{controllers} - -Get Car Leader After Second Restart - [Documentation] Find leader in the car shard - ${CAR_LEADER} Get Leader And Verify ${CAR_SHARD} - Set Suite Variable ${CAR_LEADER} - -Get Cars From Leader After Restart - [Documentation] Get cars from Leader and verify - Wait Until Keyword Succeeds ${START_TIMEOUT} 2s Get Cars And Verify ${CAR_LEADER} ${NUM_CARS} - -Get Car Followers - [Documentation] Find followers in the car shard - ${CAR_FOLLOWERS} Get All Followers ${CAR_SHARD} - Set Suite Variable ${CAR_FOLLOWERS} - -Get Cars From Follower1 After Restart - [Documentation] Get cars in follower and verify - Wait Until Keyword Succeeds ${START_TIMEOUT} 2s Get Cars And Verify @{CAR_FOLLOWERS}[0] ${NUM_CARS} - -Get Cars From Follower2 After Restart - [Documentation] Get cars in follower and verify - Wait Until Keyword Succeeds ${START_TIMEOUT} 2s Get Cars And Verify @{CAR_FOLLOWERS}[1] ${NUM_CARS} diff --git a/csit/suites/controller/Clustering_Datastore/140_recovery_restart_follower.robot b/csit/suites/controller/Clustering_Datastore/140_recovery_restart_follower.robot deleted file mode 100644 index 8a9a313081..0000000000 --- a/csit/suites/controller/Clustering_Datastore/140_recovery_restart_follower.robot +++ /dev/null @@ -1,76 +0,0 @@ -*** Settings *** -Documentation This test kills any of the followers and verifies that when that follower is restarted it can join the cluster -Default Tags 3-node-cluster -Library Collections -Library RequestsLibrary -Library ../../../libraries/CrudLibrary.py -Resource ../../../libraries/ClusterKeywords.robot -Resource ../../../libraries/CarsAndPeople.robot -Variables ../../../variables/Variables.py - -*** Variables *** -${CAR_SHARD} shard-car-config -${NUM_CARS} ${60} -@{controllers} ${ODL_SYSTEM_IP} ${ODL_SYSTEM_2_IP} ${ODL_SYSTEM_3_IP} -${KARAF_HOME} ${WORKSPACE}/${BUNDLEFOLDER} -${START_TIMEOUT} 300s -${STOP_TIMEOUT} 180s - -*** Test Cases *** -Stop All Controllers - [Documentation] Stop all the controllers in the cluster - Stop One Or More Controllers @{controllers} - Wait For Cluster Down ${STOP_TIMEOUT} @{controllers} - -Clean All Journals - [Documentation] Clean the journals of all the controllers in the cluster - Clean One Or More Journals @{controllers} - Clean One Or More Snapshots @{controllers} - -Start All Controllers - [Documentation] Start all the controllers in the cluster - Start One Or More Controllers @{controllers} - Wait For Cluster Sync ${START_TIMEOUT} @{controllers} - -Get Car Leader And Followers - [Documentation] Find leader and followers in the car shard - ${CURRENT_CAR_LEADER} Get Leader And Verify ${CAR_SHARD} - Set Suite Variable ${CURRENT_CAR_LEADER} - ${CAR_FOLLOWERS} Get All Followers ${CAR_SHARD} - Set Suite Variable ${CAR_FOLLOWERS} - -Verify No Cars On Leader After Restart - [Documentation] Verify no cars after restart - Wait Until Keyword Succeeds ${START_TIMEOUT} 2s Check Cars Deleted ${CURRENT_CAR_LEADER} - -Stop Both Of The Followers - [Documentation] Stop car followers - @{followers} = Create List @{CAR_FOLLOWERS}[0] @{CAR_FOLLOWERS}[1] - Stop One Or More Controllers @{followers} - Wait For Cluster Down ${STOP_TIMEOUT} @{followers} - -Attempt To Add A Car To The Leader - [Documentation] Add car should fail as both followers are down - AddCar ${CURRENT_CAR_LEADER} ${RESTCONFPORT} ${1} 500 503 - Sleep 2 - ${resp} Getcars ${CURRENT_CAR_LEADER} ${RESTCONFPORT} ${1} - Should Not Be Equal As Strings ${resp.status_code} 200 - -Restart The First Follower - [Documentation] Start one follower - Start One Or More Controllers @{CAR_FOLLOWERS}[0] - Wait For Controller Sync ${START_TIMEOUT} @{CAR_FOLLOWERS}[0] - -Add Cars To The First Follower - [Documentation] Add cars to the follower and verify - Log Adding ${NUM_CARS} cars to @{CAR_FOLLOWERS}[0] - Wait Until Keyword Succeeds ${START_TIMEOUT} 2s Add Cars And Verify Without Init @{CAR_FOLLOWERS}[0] ${NUM_CARS} - -Restart The Second Follower - [Documentation] Start another follower - Start One Or More Controllers @{CAR_FOLLOWERS}[1] - Wait For Controller Sync ${START_TIMEOUT} @{CAR_FOLLOWERS}[1] - -Get All The Cars From The Second Follower - [Documentation] Add cars to the follower and verify - Wait Until Keyword Succeeds ${START_TIMEOUT} 2s Get Cars And Verify @{CAR_FOLLOWERS}[1] ${NUM_CARS} diff --git a/csit/suites/controller/Clustering_Datastore/999_cleanup.robot b/csit/suites/controller/Clustering_Datastore/999_cleanup.robot deleted file mode 100644 index 667b425e59..0000000000 --- a/csit/suites/controller/Clustering_Datastore/999_cleanup.robot +++ /dev/null @@ -1,20 +0,0 @@ -*** Settings *** -Documentation Test cleanup -Default Tags 3-node-cluster -Resource ../../../libraries/ClusterKeywords.robot -Library ../../../libraries/UtilLibrary.py -Variables ../../../variables/Variables.py - -*** Variables *** -@{controllers} ${ODL_SYSTEM_IP} ${ODL_SYSTEM_2_IP} ${ODL_SYSTEM_3_IP} -${KARAF_HOME} ${WORKSPACE}/${BUNDLEFOLDER} - -*** Test Cases *** -Kill All Controllers - [Documentation] Kill all the karaf processes in the cluster - Kill One Or More Controllers @{controllers} - -Clean All Journals - [Documentation] Clean the journals of all the controllers in the cluster - Clean One Or More Journals @{controllers} - Clean One Or More Snapshots @{controllers} diff --git a/csit/suites/controller/Clustering_Datastore/__init__.robot b/csit/suites/controller/Clustering_Datastore/__init__.robot deleted file mode 100644 index 604f9af3ef..0000000000 --- a/csit/suites/controller/Clustering_Datastore/__init__.robot +++ /dev/null @@ -1,5 +0,0 @@ -*** Settings *** -Documentation Test suite for Clustering Datastore -Library SSHLibrary - -*** Keywords *** diff --git a/csit/suites/controller/Clustering_Datastore/buycar_failover.robot b/csit/suites/controller/Clustering_Datastore/buycar_failover.robot new file mode 100644 index 0000000000..421ba304ef --- /dev/null +++ b/csit/suites/controller/Clustering_Datastore/buycar_failover.robot @@ -0,0 +1,122 @@ +*** Settings *** +Documentation This test focuses on testing buy-car RPC over 3 Leader reboots. +... +... Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... +... All purchases are against the same node, which is the first one to get rebooted. +... +... All data is deleted at the end of the suite. +... This suite expects car, people and car-people modules to have separate Shards. +Suite Setup Setup +Default Tags clustering carpeople critical +Library Collections +Resource ${CURDIR}/../../../libraries/CarPeople.robot +Resource ${CURDIR}/../../../libraries/ClusterManagement.robot +Resource ${CURDIR}/../../../libraries/TemplatedRequests.robot +Variables ${CURDIR}/../../../variables/Variables.py + +*** Variables *** +${CARPEOPLE_ITEMS} ${100} +${MEMBER_START_TIMEOUT} 300s +@{SHARD_NAME_LIST} car people car-people +${VAR_DIR} ${CURDIR}/../../../variables/carpeople/crud + +*** Test Cases *** +Add_Cars_To_Leader_And_Verify + [Documentation] Add all needed cars to car Leader, verify on each member. + ${car_items} = BuiltIn.Evaluate ${CARPEOPLE_ITEMS} * 4 + TemplatedRequests.Put_As_Json_Templated folder=${VAR_DIR}/cars session=${car_leader_session} iterations=${car_items} + : FOR ${session} IN @{ClusterManagement__session_list} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} verify=True iterations=${car_items} + +Add_People_To_First_Follower_And_Verify + [Documentation] Add all needed people to people first Follower, verify on each member. + ${people_items} = BuiltIn.Evaluate ${CARPEOPLE_ITEMS} * 4 + CarPeople.Add_Several_People session=${people_first_follower_session} iterations=${people_items} + : FOR ${session} IN @{ClusterManagement__session_list} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/people session=${session} verify=True iterations=${people_items} + +Buy_Cars_After_0_Reboots_And_Verify + [Documentation] Buy some cars on the test member. + ${iter_start} = BuiltIn.Evaluate 0 * ${CARPEOPLE_ITEMS} + 1 + CarPeople.Buy_Several_Cars session=${buying_session} iterations=${CARPEOPLE_ITEMS} iter_start=${iter_start} + ${total_iterations} = BuiltIn.Evaluate 1 * ${CARPEOPLE_ITEMS} + : FOR ${session} IN @{ClusterManagement__session_list} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} verify=True iterations=${total_iterations} + +Reboot_1 + [Documentation] Previous car-people Leader is rebooted (without persistence cleanup). + ${index_to_reboot} = Collections.Remove_From_List ${list_to_reboot} 0 + ${index_list} = BuiltIn.Create_List ${index_to_reboot} + ClusterManagement.Kill_Members_From_List_Or_All member_index_list=${index_list} confirm=True + ClusterManagement.Start_Members_From_List_Or_All member_index_list=${index_list} wait_for_sync=True timeout=${MEMBER_START_TIMEOUT} + BuiltIn.Wait_Until_Keyword_Succeeds 30s 2s ClusterManagement.Verify_Leader_Exists_For_Each_Shard shard_name_list=${SHARD_NAME_LIST} shard_type=config + +Buy_Cars_After_1_Reboots_And_Verify + [Documentation] Buy some cars on the test member. + ${iter_start} = BuiltIn.Evaluate 1 * ${CARPEOPLE_ITEMS} + 1 + CarPeople.Buy_Several_Cars session=${buying_session} iterations=${CARPEOPLE_ITEMS} iter_start=${iter_start} + ${total_iterations} = BuiltIn.Evaluate 2 * ${CARPEOPLE_ITEMS} + : FOR ${session} IN @{ClusterManagement__session_list} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} verify=True iterations=${total_iterations} + +Reboot_2 + [Documentation] Previous car-people Leader is rebooted (without persistence cleanup). + ${index_to_reboot} = Collections.Remove_From_List ${list_to_reboot} 0 + ${index_list} = BuiltIn.Create_List ${index_to_reboot} + ClusterManagement.Kill_Members_From_List_Or_All member_index_list=${index_list} confirm=True + ClusterManagement.Start_Members_From_List_Or_All member_index_list=${index_list} wait_for_sync=True timeout=${MEMBER_START_TIMEOUT} + BuiltIn.Wait_Until_Keyword_Succeeds 30s 2s ClusterManagement.Verify_Leader_Exists_For_Each_Shard shard_name_list=${SHARD_NAME_LIST} shard_type=config + +Buy_Cars_After_2_Reboots_And_Verify + [Documentation] Buy some cars on the test member. + ${iter_start} = BuiltIn.Evaluate 2 * ${CARPEOPLE_ITEMS} + 1 + CarPeople.Buy_Several_Cars session=${buying_session} iterations=${CARPEOPLE_ITEMS} iter_start=${iter_start} + ${total_iterations} = BuiltIn.Evaluate 3 * ${CARPEOPLE_ITEMS} + : FOR ${session} IN @{ClusterManagement__session_list} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} verify=True iterations=${total_iterations} + +Reboot_3 + [Documentation] Previous car-people Leader is rebooted (without persistence cleanup). + ${index_to_reboot} = Collections.Remove_From_List ${list_to_reboot} 0 + ${index_list} = BuiltIn.Create_List ${index_to_reboot} + ClusterManagement.Kill_Members_From_List_Or_All member_index_list=${index_list} confirm=True + ClusterManagement.Start_Members_From_List_Or_All member_index_list=${index_list} wait_for_sync=True timeout=${MEMBER_START_TIMEOUT} + BuiltIn.Wait_Until_Keyword_Succeeds 30s 2s ClusterManagement.Verify_Leader_Exists_For_Each_Shard shard_name_list=${SHARD_NAME_LIST} shard_type=config + +Buy_Cars_After_3_Reboots_And_Verify + [Documentation] Buy some cars on the test member. + ${iter_start} = BuiltIn.Evaluate 3 * ${CARPEOPLE_ITEMS} + 1 + CarPeople.Buy_Several_Cars session=${buying_session} iterations=${CARPEOPLE_ITEMS} iter_start=${iter_start} + ${total_iterations} = BuiltIn.Evaluate 4 * ${CARPEOPLE_ITEMS} + : FOR ${session} IN @{ClusterManagement__session_list} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} verify=True iterations=${total_iterations} + +Delete_All_CarPeople + [Documentation] DELETE car-people container. No verification beyond http status. + TemplatedRequests.Delete_Templated folder=${VAR_DIR}/car-people session=${buying_session} + +Delete_All_People + [Documentation] DELETE people container. No verification beyond http status. + TemplatedRequests.Delete_Templated folder=${VAR_DIR}/people session=${people_leader_session} + +Delete_All_Cars + [Documentation] DELETE cars container. No verification beyond http status. + TemplatedRequests.Delete_Templated folder=${VAR_DIR}/cars session=${car_leader_session} + +*** Keywords *** +Setup + [Documentation] Initialize resources, memorize shard leaders, compute item distribution. + ClusterManagement.ClusterManagement_Setup + Set_Variables_For_Shard shard_name=car + Set_Variables_For_Shard shard_name=people + Set_Variables_For_Shard shard_name=car-people + ${leader_list} = BuiltIn.Create_List ${car-people_leader_index} + ${reboot_list} = Collections.Combine_Lists ${leader_list} ${car-people_follower_indices} + BuiltIn.Set_Suite_Variable \${list_to_reboot} ${reboot_list} + BuiltIn.Set_Suite_Variable \${buying_session} ${car-people_leader_session} diff --git a/csit/suites/controller/Clustering_Datastore/car_failover_crud.robot b/csit/suites/controller/Clustering_Datastore/car_failover_crud.robot new file mode 100644 index 0000000000..e33a57e296 --- /dev/null +++ b/csit/suites/controller/Clustering_Datastore/car_failover_crud.robot @@ -0,0 +1,125 @@ +*** Settings *** +Documentation Suite mixing basic operations with restart of car Leader. +... +... Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... +... This test kills the current leader of the "car" shard and then executes CRD +... operations on the new leader and a new follower. The killed member is brought back. +... This suite uses 3 different car sets, same size but different starting ID. +... +... Other models and shards (people, car-people) are not accessed by this suite. +... +... All data is deleted at the end of the suite. +... This suite expects car module to have a separate Shard. +Suite Setup Setup +Default Tags clustering carpeople critical +Library Collections +Resource ${CURDIR}/../../../libraries/CarPeople.robot +Resource ${CURDIR}/../../../libraries/ClusterManagement.robot +Resource ${CURDIR}/../../../libraries/TemplatedRequests.robot +Variables ${CURDIR}/../../../variables/Variables.py + +*** Variables *** +${CAR_ITEMS} 30 +${FOLLOWER_2NODE_START_I} 300 +${LEADER_2NODE_START_I} 200 +${MEMBER_START_TIMEOUT} 300s +${ORIGINAL_START_I} 100 +@{SHARD_NAME_LIST} car +${VAR_DIR} ${CURDIR}/../../../variables/carpeople/crud + +*** Test Cases *** +Add_Original_Cars_On_Old_Leader + [Documentation] Add initial cars on car Leader. + TemplatedRequests.Put_As_Json_Templated folder=${VAR_DIR}/cars session=${car_leader_session} iterations=${CAR_ITEMS} iter_start=${ORIGINAL_START_I} + +Kill_Original_Car_Leader + [Documentation] Kill the car Leader to cause a new leader to get elected. + ${kill_list} = BuiltIn.Create_List ${car_leader_index} + ClusterManagement.Kill_Members_From_List_Or_All member_index_list=${kill_list} confirm=True + +Wait_For_New_Leader + [Documentation] Wait until new car Leader is elected. + BuiltIn.Wait_Until_Keyword_Succeeds 30s 2s Set_Variables_For_Less_Nodes member_index_list=${car_follower_indices} + +See_Original_Cars_On_New_Leader + [Documentation] GET cars from new Leader, should be the initial ones. + TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${new_leader_session} verify=True iterations=${CAR_ITEMS} iter_start=${ORIGINAL_START_I} + +See_Original_Cars_On_New_Followers + [Documentation] The same check on other existing member(s). + : FOR ${session} IN @{new_follower_sessions} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} verify=True iterations=${CAR_ITEMS} iter_start=${ORIGINAL_START_I} + +Delete_Original_Cars_On_New_Leader + [Documentation] Delete cars on the new Leader. + TemplatedRequests.Delete_Templated folder=${VAR_DIR}/cars session=${new_leader_session} + +Add_Leader_Cars_On_New_Leader + [Documentation] Add cars on the new Leader. + TemplatedRequests.Put_As_Json_Templated folder=${VAR_DIR}/cars session=${new_leader_session} iterations=${CAR_ITEMS} iter_start=${LEADER_2NODE_START_I} + +See_Leader_Cars_On_New_Leader + [Documentation] GET cars from new Leader, should be the new ones. + TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${new_leader_session} verify=True iterations=${CAR_ITEMS} iter_start=${LEADER_2NODE_START_I} + +See_Leader_Cars_On_New_Followers + [Documentation] The same check on other existing members. + : FOR ${session} IN @{new_follower_sessions} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} verify=True iterations=${CAR_ITEMS} iter_start=${LEADER_2NODE_START_I} + +Delete_Leader_Cars_On_New_First_Follower + [Documentation] Delete cars in new first Follower. + TemplatedRequests.Delete_Templated folder=${VAR_DIR}/cars session=${new_first_follower_session} + +Add_Follower_Cars_On_New_First_Follower + [Documentation] Add cars on the new first Follower. + TemplatedRequests.Put_As_Json_Templated folder=${VAR_DIR}/cars session=${new_first_follower_session} iterations=${CAR_ITEMS} iter_start=${FOLLOWER_2NODE_START_I} + +See_Folower_Cars_On_New_Leader + [Documentation] Get cars from the new Leader, should be the ones added on follower. + TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${new_leader_session} verify=True iterations=${CAR_ITEMS} iter_start=${FOLLOWER_2NODE_START_I} + +See_Follower_Cars_On_New_Followers + [Documentation] The same check on other existing members. + : FOR ${session} IN @{new_follower_sessions} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} verify=True iterations=${CAR_ITEMS} iter_start=${FOLLOWER_2NODE_START_I} + +Start_Old_Car_Leader + [Documentation] Start the killed member without deleting the persisted data. + ${revive_list} = BuiltIn.Create_List ${car_leader_index} + ClusterManagement.Start_Members_From_List_Or_All member_index_list=${revive_list} wait_for_sync=True timeout=${MEMBER_START_TIMEOUT} + BuiltIn.Wait_Until_Keyword_Succeeds 30s 2s ClusterManagement.Verify_Leader_Exists_For_Each_Shard shard_name_list=${SHARD_NAME_LIST} shard_type=config + +See_Folower_Cars_On_Old_Leader + [Documentation] GET cars from the restarted member, should be the ones added on follower. + TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${car_leader_session} verify=True iterations=${CAR_ITEMS} iter_start=${FOLLOWER_2NODE_START_I} + +Delete_Follower_Cars_On_New_Leader + [Documentation] Delete cars on the last Leader. + TemplatedRequests.Delete_Templated folder=${VAR_DIR}/cars session=${new_leader_session} + +*** Keywords *** +Setup + [Documentation] Initialize resources, memorize car shard leader and followers. + ClusterManagement.ClusterManagement_Setup + CarPeople.Set_Variables_For_Shard shard_name=car + +Set_Variables_For_Less_Nodes + [Arguments] ${member_index_list} + [Documentation] Get current leader and followers for car shard, set additional suite variables. + ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=car shard_type=config member_index_list=${member_index_list} + ${leader_session} = ClusterManagement.Resolve_Http_Session_For_Member member_index=${leader} + BuiltIn.Set_Suite_Variable \${new_leader_session} ${leader_session} + ${sessions} = BuiltIn.Create_List + : FOR ${follower_index} IN @{follower_list} + \ ${follower_session} = ClusterManagement.Resolve_Http_Session_For_Member member_index=${follower_index} + \ Collections.Append_To_List ${sessions} ${follower_session} + BuiltIn.Set_Suite_Variable \${new_follower_sessions} ${sessions} + ${first_follower_session} = Collections.Get_From_List ${sessions} 0 + BuiltIn.Set_Suite_Variable \${new_first_follower_session} ${first_follower_session} diff --git a/csit/suites/controller/Clustering_Datastore/car_outage_corners.robot b/csit/suites/controller/Clustering_Datastore/car_outage_corners.robot new file mode 100644 index 0000000000..3d39e170e9 --- /dev/null +++ b/csit/suites/controller/Clustering_Datastore/car_outage_corners.robot @@ -0,0 +1,102 @@ +*** Settings *** +Documentation Cluster suite for testing minimal and sum-minimal member population behavior. +... +... Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... +... This test kills majority of the followers and verifies car addition is not possible, +... then resumes single follower (first from original list) and checks that addition works. +... Then remaining members are brought up. +... Leader member is always up and assumed to remain Leading during the whole suite run. +... +... TODO: Use initial data to check more operations. +... TODO: Perhaps merge with car_failover_crud suite. +... +... Other modules and Shards (people, car-people) are not accessed by this suite. +... +... All data is deleted at the end of the suite. +... This suite expects car module to have a separate Shard. +Suite Setup Setup +Default Tags clustering carpeople critical +Library Collections +Resource ${CURDIR}/../../../libraries/CarPeople.robot +Resource ${CURDIR}/../../../libraries/ClusterManagement.robot +Resource ${CURDIR}/../../../libraries/TemplatedRequests.robot +Variables ${CURDIR}/../../../variables/Variables.py + +*** Variables *** +${CAR_ITEMS} 50 +${MINORITY_START_I} 300 +${MAJORITY_START_I} 200 +${MEMBER_START_TIMEOUT} 300s +@{SHARD_NAME_LIST} car +${VAR_DIR} ${CURDIR}/../../../variables/carpeople/crud + +*** Test Cases *** +Kill_Majority_Of_The_Followers + [Documentation] Kill half plus one car Follower members. + ClusterManagement.Kill_Members_From_List_Or_All member_index_list=${list_of_killing} confirm=True + +Attempt_To_Add_Cars_To_Leader + [Documentation] Adding cars should fail, as majority of Followers are down. + ${status} ${message} = BuiltIn.Run_Keyword_And_Ignore_Error TemplatedRequests.Put_As_Json_Templated folder=${VAR_DIR}/cars session=${car_leader_session} iterations=${CAR_ITEMS} + ... iter_start=${MINORITY_START_I} + # TODO: Is there a specific status and mesage to require in this scenario? + BuiltIn.Should_Contain ${message} '50 + +Clean_And_Start_Tipping_Follower + [Documentation] Start one Follower member without persisted data. + ClusterManagement.Clean_Journals_And_Snapshots_On_List_Or_All member_index_list=${list_of_tipping} + ClusterManagement.Start_Members_From_List_Or_All member_index_list=${list_of_tipping} wait_for_sync=True timeout=${MEMBER_START_TIMEOUT} + BuiltIn.Wait_Until_Keyword_Succeeds 30s 2s ClusterManagement.Verify_Leader_Exists_For_Each_Shard shard_name_list=${SHARD_NAME_LIST} shard_type=config member_index_list=${list_of_majority} + +Add_Cars_On_Tipping_Follower + [Documentation] Add cars on the tipping Follower. + TemplatedRequests.Put_As_Json_Templated folder=${VAR_DIR}/cars session=${car_first_follower_session} iterations=${CAR_ITEMS} iter_start=${MAJORITY_START_I} + +See_Cars_On_Existing_Members + [Documentation] On each up member: GET cars, should match the ones added on tipping Follower. + : FOR ${session} IN @{list_of_majority} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} verify=True iterations=${CAR_ITEMS} iter_start=${MAJORITY_START_I} + +Clean_And_Start_Other_Followers + [Documentation] Start other followers without persisted data. + ClusterManagement.Start_Members_From_List_Or_All member_index_list=${list_of_reviving} wait_for_sync=True timeout=${MEMBER_START_TIMEOUT} + BuiltIn.Wait_Until_Keyword_Succeeds 30s 2s ClusterManagement.Verify_Leader_Exists_For_Each_Shard shard_name_list=${SHARD_NAME_LIST} shard_type=config + +See_Cars_On_New_Follower_Leader + [Documentation] GET cars from a new follower to see that the current state was replicated. + TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${car_last_follower_session} verify=True iterations=${CAR_ITEMS} iter_start=${MAJORITY_START_I} + +Delete_Cars_On_Leader + [Documentation] Delete cars on Leader. + TemplatedRequests.Delete_Templated folder=${VAR_DIR}/cars session=${car_leader_session} + +*** Keywords *** +Setup + [Documentation] Initialize resources, memorize shard leaders, pre-compute member lists. + ClusterManagement.ClusterManagement_Setup + CarPeople.Set_Variables_For_Shard shard_name=car + Set_Additional_Variables + +Set_Additional_Variables + [Documentation] Compute various lists useful for test cases in this suite. + # TODO: Migrate this Keyword to CarPeople Resource if more suites want that. + ${last_follower_session} = Collections.Get_From_List ${car_follower_sessions} -1 + BuiltIn.Set_Suite_Variable \${car_last_follower_session} ${last_follower_session} + ${number_followers} = BuiltIn.Get_Length ${car_follower_indices} + ${half_followers} = BuiltIn.Evaluate ${number_followers} / 2 + ${majority_follower_list} = Collections.Get_Slice_From_List ${car_follower_indices} 0 ${half_followers} + ${leader_list} = BuiltIn.Create_List ${car_leader_index} + ${majority_list} = Collections.Combine_Lists ${leader_list} ${majority_follower_list} + BuiltIn.Set_Suite_Variable \${list_of_majority} ${majority_list} + ${tipping_list} = Collections.Get_Slice_From_List ${majority_follower_list} 0 1 + BuiltIn.Set_Suite_Variable \${list_of_tipping} ${tipping_list} + ${revive_list} = Collections.Get_Slice_From_List ${car_follower_indices} ${half_followers} ${number_followers} + BuiltIn.Set_Suite_Variable \${list_of_reviving} ${revive_list} + ${kill_list} = Collections.Combine_Lists ${tipping_list} ${revive_list} + BuiltIn.Set_Suite_Variable \${list_of_killing} ${kill_list} diff --git a/csit/suites/controller/Clustering_Datastore/car_persistence_recovery.robot b/csit/suites/controller/Clustering_Datastore/car_persistence_recovery.robot new file mode 100644 index 0000000000..9092935c1b --- /dev/null +++ b/csit/suites/controller/Clustering_Datastore/car_persistence_recovery.robot @@ -0,0 +1,62 @@ +*** Settings *** +Documentation This test restarts all controllers to verify recovery of car data from persistence. +... +... Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... +... Other models and shards (people, car-people) are not accessed by this suite. +... +... All data is deleted at the end of the suite. +... This suite expects car module to have a separate Shard. +Suite Setup Setup +Default Tags clustering carpeople critical +Library Collections +Resource ${CURDIR}/../../../libraries/CarPeople.robot +Resource ${CURDIR}/../../../libraries/ClusterManagement.robot +Resource ${CURDIR}/../../../libraries/TemplatedRequests.robot +Variables ${CURDIR}/../../../variables/Variables.py + +*** Variables *** +${CAR_ITEMS} 50 +${MEMBER_START_TIMEOUT} 300s +${VAR_DIR} ${CURDIR}/../../../variables/carpeople/crud + +*** Test Cases *** +Add_Cars_On_Leader + [Documentation] Single big PUT to datastore to add cars to car Leader. + TemplatedRequests.Put_As_Json_Templated folder=${VAR_DIR}/cars session=${car_leader_session} iterations=${CAR_ITEMS} + +Kill_All_Members + [Documentation] Kill all controllers. + ClusterManagement.Kill_Members_From_List_Or_All confirm=True + +Start_All_Members + [Documentation] Start all controllers (should restore the persisted data). + ClusterManagement.Start_Members_From_List_Or_All wait_for_sync=True timeout=${MEMBER_START_TIMEOUT} + +Memorize_Leader_And_Followers + [Documentation] Locate current Leader of car Shard. + BuiltIn.Wait_Until_Keyword_Succeeds 30s 2s CarPeople.Set_Variables_For_Shard shard_name=car + +See_Cars_On_Leader + [Documentation] GET cars from Leader, should match the PUT data. + TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${car_leader_session} verify=True iterations=${CAR_ITEMS} + +See_Cars_On_Followers + [Documentation] The same check on other members. + : FOR ${session} IN @{car_follower_sessions} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} verify=True iterations=${CAR_ITEMS} + +Delete_Cars_On_Leader + [Documentation] Delete cars on the new Leader. + TemplatedRequests.Delete_Templated folder=${VAR_DIR}/cars session=${car_leader_session} + +*** Keywords *** +Setup + [Documentation] Initialize resources, memorize car shard leader and followers. + ClusterManagement.ClusterManagement_Setup + CarPeople.Set_Variables_For_Shard shard_name=car diff --git a/csit/suites/controller/Clustering_Datastore/carpeople_crud.robot b/csit/suites/controller/Clustering_Datastore/carpeople_crud.robot new file mode 100644 index 0000000000..5b844baa79 --- /dev/null +++ b/csit/suites/controller/Clustering_Datastore/carpeople_crud.robot @@ -0,0 +1,103 @@ +*** Settings *** +Documentation Suite for performing basic car/people CRUD operations on leaders and followers. +... +... Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... +... More precisely, Update operation is not executed, but some operations +... are using specific RPCs which goes beyond "basic CRUD". +... +... Cars are added by one big PUT to datastore on car Leader. +... People are added in a loop with add-person RPC on a people Follower. +... Cars are bought by chunks on each member, by loop with buy-car RPC. +... +... All data is deleted at the end of the suite. +... This suite expects car, people and car-people modules to have separate Shards. +Suite Setup Setup +Default Tags clustering carpeople critical +Library Collections +Resource ${CURDIR}/../../../libraries/CarPeople.robot +Resource ${CURDIR}/../../../libraries/ClusterManagement.robot +Resource ${CURDIR}/../../../libraries/TemplatedRequests.robot +Variables ${CURDIR}/../../../variables/Variables.py + +*** Variables *** +${CARPEOPLE_ITEMS} ${30} +${VAR_DIR} ${CURDIR}/../../../variables/carpeople/crud + +*** Test Cases *** +Add_Cars_To_Leader + [Documentation] Add ${CARPEOPLE_ITEMS} cars to car Leader by one big PUT. + TemplatedRequests.Put_As_Json_Templated folder=${VAR_DIR}/cars session=${car_leader_session} iterations=${CARPEOPLE_ITEMS} + +See_Added_Cars_On_Leader + [Documentation] GET response from Leader should match the PUT data. + TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${car_leader_session} verify=True iterations=${CARPEOPLE_ITEMS} + +See_Added_Cars_On_Followers + [Documentation] The same check on other members. + : FOR ${session} IN @{car_follower_sessions} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} verify=True iterations=${CARPEOPLE_ITEMS} + +Add_People_To_First_Follower + [Documentation] Add ${CARPEOPLE_ITEMS} people to people first Follower, loop of add-person. + CarPeople.Add_Several_People session=${people_first_follower_session} iterations=${CARPEOPLE_ITEMS} + +See_Added_People_On_Leader + [Documentation] GET response from Leader should match the added people. + TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/people session=${people_leader_session} verify=True iterations=${CARPEOPLE_ITEMS} + +See_Added_People_On_Followers + [Documentation] The same check on other members. + : FOR ${session} IN @{people_follower_sessions} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/people session=${session} verify=True iterations=${CARPEOPLE_ITEMS} + +Buy_Cars_On_Leader + [Documentation] Buy some cars on car-people Leader, loop of buy-car, ending segment of IDs. + # Cars are numbered, leader gets chunk at the end, as that is few keypresses shorter. + ${start_id} = BuiltIn.Evaluate (${NUM_ODL_SYSTEM} - 1) * ${items_per_follower} + 1 + CarPeople.Buy_Several_Cars session=${car-people_leader_session} iterations=${items_per_leader} iter_start=${start_id} + +Buy_Cars_On_Followers + [Documentation] On each Follower buy corresponding ID segment of cars in buy-car loop. + ${start_id} = BuiltIn.Set_Variable 0 + : FOR ${session} IN @{car-people_follower_sessions} + \ CarPeople.Buy_Several_Cars session=${session} iterations=${items_per_follower} iter_start=${start_id} + \ ${start_id} = BuiltIn.Evaluate ${start_id} + ${items_per_follower} + +See_Added_CarPeople_On_Leader + [Documentation] GET car-person mappings from Leader to see all entries. + TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${car-people_leader_session} verify=True iterations=${CARPEOPLE_ITEMS} + +See_Added_CarPeople_On_Followers + [Documentation] The same check on other members. + : FOR ${session} IN @{car-people_follower_sessions} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} verify=True iterations=${CARPEOPLE_ITEMS} + +Delete_All_CarPeople_On_Leader + [Documentation] DELETE car-people container. No verification beyond http status. + TemplatedRequests.Delete_Templated folder=${VAR_DIR}/car-people session=${car-people_leader_session} + +Delete_All_People_On_Leader + [Documentation] DELETE people container. No verification beyond http status. + TemplatedRequests.Delete_Templated folder=${VAR_DIR}/people session=${people_leader_session} + +Delete_All_Cars_On_Leader + [Documentation] DELETE cars container. No verification beyond http status. + TemplatedRequests.Delete_Templated folder=${VAR_DIR}/cars session=${car_leader_session} + +*** Keywords *** +Setup + [Documentation] Initialize resources, memorize shard leaders, compute item distribution. + ClusterManagement.ClusterManagement_Setup + CarPeople.Set_Variables_For_Shard shard_name=car + CarPeople.Set_Variables_For_Shard shard_name=people + CarPeople.Set_Variables_For_Shard shard_name=car-people + ${follower_number} = BuiltIn.Evaluate ${CARPEOPLE_ITEMS} / ${NUM_ODL_SYSTEM} + BuiltIn.Set_Suite_Variable ${items_per_follower} ${follower_number} + ${leader_number} = BuiltIn.Evaluate ${CARPEOPLE_ITEMS} - (${NUM_ODL_SYSTEM} - 1) * ${follower_number} + BuiltIn.Set_Suite_Variable ${items_per_leader} ${leader_number} diff --git a/csit/suites/controller/Clustering_Datastore/cluster_ready.robot b/csit/suites/controller/Clustering_Datastore/cluster_ready.robot new file mode 100644 index 0000000000..33ad3cbc80 --- /dev/null +++ b/csit/suites/controller/Clustering_Datastore/cluster_ready.robot @@ -0,0 +1,30 @@ +*** Settings *** +Documentation This test waits until cluster appears to be ready. +... +... Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... +... Intended use is at a start of testplan, so that suites can assume cluster works. +... +... This suite expects car, people and car-people modules to have separate Shards. +Suite Setup ClusterManagement.ClusterManagement_Setup +Default Tags clustering carpeople critical +Resource ${CURDIR}/../../../libraries/ClusterManagement.robot + +*** Variables *** +${CLUSTER_BOOTUP_SYNC_TIMEOUT} 180s +@{SHARD_NAME_LIST} car people car-people + +*** Test Cases *** +Wait_For_Sync_And_Shards + [Documentation] Repeatedly check for cluster sync status and Shard Leaders, fail when timeout is exceeded. + BuiltIn.Wait_Until_Keyword_Succeeds ${CLUSTER_BOOTUP_SYNC_TIMEOUT} 10s Check_Sync_And_Shards + +*** Keywords *** +Check_Sync_And_Shards + ClusterManagement.Check_Cluster_Is_In_Sync + ClusterManagement.Verify_Leader_Exists_For_Each_Shard shard_name_list=${SHARD_NAME_LIST} shard_type=config diff --git a/csit/suites/controller/Clustering_Routedrpc/001_start_cluster.robot b/csit/suites/controller/Clustering_Routedrpc/001_start_cluster.robot deleted file mode 100644 index 69f2ece2c9..0000000000 --- a/csit/suites/controller/Clustering_Routedrpc/001_start_cluster.robot +++ /dev/null @@ -1,21 +0,0 @@ -*** Settings *** -Documentation Start the controllers -Default Tags 3-node-cluster -Resource ../../../libraries/ClusterKeywords.robot - -*** Variables *** -@{controllers} ${ODL_SYSTEM_IP} ${ODL_SYSTEM_2_IP} ${ODL_SYSTEM_3_IP} -${KARAF_HOME} ${WORKSPACE}/${BUNDLEFOLDER} - -*** Test Cases *** -Stop All Controllers - [Documentation] Stop all the controllers in the cluster - Stop One Or More Controllers @{controllers} - -Clean All Journals - [Documentation] Clean the journals of all the controllers in the cluster - Clean One Or More Journals @{controllers} - -Start All Controllers - [Documentation] Start all the controllers in the cluster - Start One Or More Controllers @{controllers} diff --git a/csit/suites/controller/Clustering_Routedrpc/023_routed_rpc_crud_test.robot b/csit/suites/controller/Clustering_Routedrpc/023_routed_rpc_crud_test.robot deleted file mode 100644 index 087d9fd130..0000000000 --- a/csit/suites/controller/Clustering_Routedrpc/023_routed_rpc_crud_test.robot +++ /dev/null @@ -1,129 +0,0 @@ -*** Settings *** -Documentation Test suite for Routed RPC. -Library Collections -Library RequestsLibrary -Library ../../../libraries/CrudLibrary.py -Resource ../../../libraries/ClusterKeywords.robot -Resource ../../../libraries/CarsAndPeople.robot -Variables ../../../variables/Variables.py - -*** Variables *** -@{controllers} ${ODL_SYSTEM_IP} ${ODL_SYSTEM_2_IP} ${ODL_SYSTEM_3_IP} -${SHARD_CAR_NAME} shard-car-config -${SHARD_PEOPLE_NAME} shard-people-config -${SHARD_CAR_PERSON_NAME} shard-car-people-config -${KARAF_HOME} ${WORKSPACE}/${BUNDLEFOLDER} -${NUM_ENTRIES} ${100} -${START_TIMEOUT} 300s -${STOP_TIMEOUT} 180s - -*** Test Cases *** -Delete All Entries - [Documentation] Make sure the shards are cleared for testing. - Delete All Entries From Shards @{controllers} - -Get Car Leader And Followers - [Documentation] Find leader and followers in the car shard - ${CURRENT_CAR_LEADER} Get Leader And Verify ${SHARD_CAR_NAME} - Set Suite Variable ${CURRENT_CAR_LEADER} - @{CAR_FOLLOWERS} Get All Followers ${SHARD_CAR_NAME} - Set Suite Variable ${CAR_FOLLOWERS} - -Add Cars And Get Cars From Leader - [Documentation] Add 100 cars and get added cars from Leader - Add Cars And Verify ${CURRENT_CAR_LEADER} ${NUM_ENTRIES} - -Add Persons And Get Persons From Car Leader - [Documentation] Add 100 persons and get persons from Leader - Add People And Verify ${CURRENT_CAR_LEADER} ${NUM_ENTRIES} - -Add Car-Person Mapping And Get Car-Person Mapping From Car Follower1 - [Documentation] Add car-person and get car-person from Leader - Add Car Person And Verify @{CAR_FOLLOWERS}[0] - -Purchase 100 Cars Using Car Follower1 - [Documentation] Purchase 100 cars using Follower1 - Buy Cars And Verify @{CAR_FOLLOWERS}[0] ${NUM_ENTRIES} - -Check Contents Of Car Leader Shards - [Documentation] Check all content using Leader - Check Elements In Shards ${CURRENT_CAR_LEADER} ${NUM_ENTRIES} - -Check Contents Of Car Follower1 Shards - [Documentation] Check all content using first follower - Check Elements In Shards @{CAR_FOLLOWERS}[0] ${NUM_ENTRIES} - -Check Contents Of Car Follower2 Shards - [Documentation] Check all content using second follower - Check Elements In Shards @{CAR_FOLLOWERS}[1] ${NUM_ENTRIES} - -Get Old Car Leader - [Documentation] Find leader in the car shard - ${OLD_CAR_LEADER} Get Leader And Verify ${SHARD_CAR_NAME} - Set Suite Variable ${OLD_CAR_LEADER} - -Switch Car Leader - [Documentation] Stop the leader to cause a new leader to be elected - Stop One Or More Controllers ${OLD_CAR_LEADER} - Wait For Controller Down ${STOP_TIMEOUT} ${OLD_CAR_LEADER} - ${NEW_CAR_LEADER} Wait Until Keyword Succeeds 30s 2s Get Leader And Verify ${SHARD_CAR_NAME} ${OLD_CAR_LEADER} - Set Suite Variable ${NEW_CAR_LEADER} - -Get New Car Followers - [Documentation] Find the new followers for the car shard. - @{CAR_FOLLOWERS} Get All Followers ${SHARD_CAR_NAME} - Set Suite Variable ${CAR_FOLLOWERS} - Log @{CAR_FOLLOWERS}[0] - -Check Cars In New Car Leader - [Documentation] Check cars in new Leader - Wait Until Keyword Succeeds ${START_TIMEOUT} 2s Check Cars ${NEW_CAR_LEADER} ${NUM_ENTRIES} - -Check Contents Of New Car Leader Shards - [Documentation] Check all content using new Leader - Check Elements In Shards ${NEW_CAR_LEADER} ${NUM_ENTRIES} - -Check Contents Of New Car Follower1 Shards - [Documentation] Check all content using first follower - Check Elements In Shards @{CAR_FOLLOWERS}[0] ${NUM_ENTRIES} - -Readd People From New Car Leader - [Documentation] Add 100 persons and get persons from Leader - Add People And Verify Without Init ${NEW_CAR_LEADER} ${NUM_ENTRIES} - -Repurchase 100 Cars Using New Car Follower1 - [Documentation] Repurchase 100 cars using Follower1 - Buy Cars And Verify @{CAR_FOLLOWERS}[0] ${NUM_ENTRIES} - -Check Contents Of New Car Leader Shards After Repurchase - [Documentation] Check all content using new Leader - Check Elements In Shards ${NEW_CAR_LEADER} ${NUM_ENTRIES} - -Check Contents Of New Car First Follower Shards After Repurchase - [Documentation] Check all content using first follower - Check Elements In Shards @{CAR_FOLLOWERS}[0] ${NUM_ENTRIES} - -Start Old Car Leader - [Documentation] Start Leader controller - Start One Or More Controllers ${OLD_CAR_LEADER} - Wait For Controller Sync ${START_TIMEOUT} ${OLD_CAR_LEADER} - -Check Cars In Old Car Leader - [Documentation] Check cars in old Leader - Wait Until Keyword Succeeds ${START_TIMEOUT} 2s Check Cars ${OLD_CAR_LEADER} ${NUM_ENTRIES} - -Check Contents Of Old Leader Shards - [Documentation] Check all content using old Leader - Check Elements In Shards ${OLD_CAR_LEADER} ${NUM_ENTRIES} - -Readd People From Old Leader - [Documentation] Add 100 persons and get persons from Leader - Wait Until Keyword Succeeds 30 2s Add People And Verify Without Init ${OLD_CAR_LEADER} ${NUM_ENTRIES} - -Repurchase 100 Cars Using Old Leader - [Documentation] Repurchase 100 cars using Follower1 - Buy Cars And Verify ${OLD_CAR_LEADER} ${NUM_ENTRIES} - -Check Contents Of Old Leader Shards After Repurchase - [Documentation] Check all content using Leader - Check Elements In Shards ${OLD_CAR_LEADER} ${NUM_ENTRIES} diff --git a/csit/testplans/controller-clustering.txt b/csit/testplans/controller-clustering.txt index 79630ac0d9..366a101fc7 100644 --- a/csit/testplans/controller-clustering.txt +++ b/csit/testplans/controller-clustering.txt @@ -1,8 +1,7 @@ # Place the suites in run order: -integration/test/csit/suites/controller/Clustering_Datastore/010_crud_on_leader.robot -integration/test/csit/suites/controller/Clustering_Datastore/020_crud_on_any_follower.robot -integration/test/csit/suites/controller/Clustering_Datastore/030_car_failover_crud_on_new_leader.robot -integration/test/csit/suites/controller/Clustering_Datastore/040_people_failover_crud_on_new_leader.robot -integration/test/csit/suites/controller/Clustering_Datastore/050_car_persistence_recovery.robot -integration/test/csit/suites/controller/Clustering_Datastore/140_recovery_restart_follower.robot -integration/test/csit/suites/controller/Clustering_Routedrpc/023_routed_rpc_crud_test.robot +integration/test/csit/suites/controller/Clustering_Datastore/cluster_ready.robot +integration/test/csit/suites/controller/Clustering_Datastore/carpeople_crud.robot +integration/test/csit/suites/controller/Clustering_Datastore/car_failover_crud.robot +integration/test/csit/suites/controller/Clustering_Datastore/car_persistence_recovery.robot +integration/test/csit/suites/controller/Clustering_Datastore/car_outage_corners.robot +integration/test/csit/suites/controller/Clustering_Datastore/buycar_failover.robot diff --git a/csit/variables/carpeople/crud/add-person/location.uri b/csit/variables/carpeople/crud/add-person/location.uri new file mode 100644 index 0000000000..e6b1ab0d52 --- /dev/null +++ b/csit/variables/carpeople/crud/add-person/location.uri @@ -0,0 +1 @@ +/restconf/operations/people:add-person diff --git a/csit/variables/carpeople/crud/add-person/post_data.json b/csit/variables/carpeople/crud/add-person/post_data.json new file mode 100644 index 0000000000..fd0d1a3561 --- /dev/null +++ b/csit/variables/carpeople/crud/add-person/post_data.json @@ -0,0 +1,9 @@ +{ + "input": { + "id": "localhost/people/person_id_$i", + "gender": "gender_$i", + "age": $i, + "address": "address_$i", + "contactNo": "contact_$i" + } +} diff --git a/csit/variables/carpeople/crud/buy-car/location.uri b/csit/variables/carpeople/crud/buy-car/location.uri new file mode 100644 index 0000000000..503b42c4b3 --- /dev/null +++ b/csit/variables/carpeople/crud/buy-car/location.uri @@ -0,0 +1 @@ +/restconf/operations/car-purchase:buy-car diff --git a/csit/variables/carpeople/crud/buy-car/post_data.json b/csit/variables/carpeople/crud/buy-car/post_data.json new file mode 100644 index 0000000000..b096bd686d --- /dev/null +++ b/csit/variables/carpeople/crud/buy-car/post_data.json @@ -0,0 +1,7 @@ +{ + "input": { + "person": "/people:people/people:person[people:id='localhost/people/person_id_$i']", + "car-id": "localhost/cars/car_id_$i", + "person-id": "localhost/people/person_id_$i" + } +} diff --git a/csit/variables/carpeople/crud/car-people/data.epilog.json b/csit/variables/carpeople/crud/car-people/data.epilog.json new file mode 100644 index 0000000000..e633845876 --- /dev/null +++ b/csit/variables/carpeople/crud/car-people/data.epilog.json @@ -0,0 +1,3 @@ + ] + } +} diff --git a/csit/variables/carpeople/crud/car-people/data.item.json b/csit/variables/carpeople/crud/car-people/data.item.json new file mode 100644 index 0000000000..27008bd149 --- /dev/null +++ b/csit/variables/carpeople/crud/car-people/data.item.json @@ -0,0 +1,4 @@ + { + "car-id": "localhost/cars/car_id_$i", + "person-id": "localhost/people/person_id_$i" + } diff --git a/csit/variables/carpeople/crud/car-people/data.prolog.json b/csit/variables/carpeople/crud/car-people/data.prolog.json new file mode 100644 index 0000000000..bc28565308 --- /dev/null +++ b/csit/variables/carpeople/crud/car-people/data.prolog.json @@ -0,0 +1,3 @@ +{ + "car-people": { + "car-person": [ diff --git a/csit/variables/carpeople/crud/car-people/location.uri b/csit/variables/carpeople/crud/car-people/location.uri new file mode 100644 index 0000000000..81ac4440fc --- /dev/null +++ b/csit/variables/carpeople/crud/car-people/location.uri @@ -0,0 +1 @@ +/restconf/config/car-people:car-people diff --git a/csit/variables/carpeople/crud/cars/data.epilog.json b/csit/variables/carpeople/crud/cars/data.epilog.json new file mode 100644 index 0000000000..e633845876 --- /dev/null +++ b/csit/variables/carpeople/crud/cars/data.epilog.json @@ -0,0 +1,3 @@ + ] + } +} diff --git a/csit/variables/carpeople/crud/cars/data.item.json b/csit/variables/carpeople/crud/cars/data.item.json new file mode 100644 index 0000000000..0f195370f4 --- /dev/null +++ b/csit/variables/carpeople/crud/cars/data.item.json @@ -0,0 +1,7 @@ + { + "id": "localhost/cars/car_id_$i", + "category": "car_category_$i", + "model": "car_model_$i", + "manufacturer": "car_manufacturer_$i", + "year": $i + } diff --git a/csit/variables/carpeople/crud/cars/data.prolog.json b/csit/variables/carpeople/crud/cars/data.prolog.json new file mode 100644 index 0000000000..632c77c526 --- /dev/null +++ b/csit/variables/carpeople/crud/cars/data.prolog.json @@ -0,0 +1,3 @@ +{ + "cars": { + "car-entry": [ diff --git a/csit/variables/carpeople/crud/cars/location.uri b/csit/variables/carpeople/crud/cars/location.uri new file mode 100644 index 0000000000..22a7f80ea1 --- /dev/null +++ b/csit/variables/carpeople/crud/cars/location.uri @@ -0,0 +1 @@ +/restconf/config/car:cars diff --git a/csit/variables/carpeople/crud/cars/post_data.epilog.json b/csit/variables/carpeople/crud/cars/post_data.epilog.json new file mode 100644 index 0000000000..fffc6c7c08 --- /dev/null +++ b/csit/variables/carpeople/crud/cars/post_data.epilog.json @@ -0,0 +1,2 @@ + ] +} diff --git a/csit/variables/carpeople/crud/cars/post_data.prolog.json b/csit/variables/carpeople/crud/cars/post_data.prolog.json new file mode 100644 index 0000000000..fe01eeacaa --- /dev/null +++ b/csit/variables/carpeople/crud/cars/post_data.prolog.json @@ -0,0 +1,2 @@ +{ + "car-entry": [ diff --git a/csit/variables/carpeople/crud/people/data.epilog.json b/csit/variables/carpeople/crud/people/data.epilog.json new file mode 100644 index 0000000000..e633845876 --- /dev/null +++ b/csit/variables/carpeople/crud/people/data.epilog.json @@ -0,0 +1,3 @@ + ] + } +} diff --git a/csit/variables/carpeople/crud/people/data.item.json b/csit/variables/carpeople/crud/people/data.item.json new file mode 100644 index 0000000000..d0419c4773 --- /dev/null +++ b/csit/variables/carpeople/crud/people/data.item.json @@ -0,0 +1,7 @@ + { + "id": "localhost/people/person_id_$i", + "gender": "gender_$i", + "age": $i, + "address": "address_$i", + "contactNo": "contact_$i" + } diff --git a/csit/variables/carpeople/crud/people/data.prolog.json b/csit/variables/carpeople/crud/people/data.prolog.json new file mode 100644 index 0000000000..31975cba3a --- /dev/null +++ b/csit/variables/carpeople/crud/people/data.prolog.json @@ -0,0 +1,3 @@ +{ + "people": { + "person": [ diff --git a/csit/variables/carpeople/crud/people/location.uri b/csit/variables/carpeople/crud/people/location.uri new file mode 100644 index 0000000000..91229c89d0 --- /dev/null +++ b/csit/variables/carpeople/crud/people/location.uri @@ -0,0 +1 @@ +/restconf/config/people:people diff --git a/csit/variables/carpeople/crud/people/post_data.epilog.json b/csit/variables/carpeople/crud/people/post_data.epilog.json new file mode 100644 index 0000000000..fffc6c7c08 --- /dev/null +++ b/csit/variables/carpeople/crud/people/post_data.epilog.json @@ -0,0 +1,2 @@ + ] +} diff --git a/csit/variables/carpeople/crud/people/post_data.prolog.json b/csit/variables/carpeople/crud/people/post_data.prolog.json new file mode 100644 index 0000000000..ee5a30359b --- /dev/null +++ b/csit/variables/carpeople/crud/people/post_data.prolog.json @@ -0,0 +1,2 @@ +{ + "person": [ diff --git a/csit/variables/carpeople/libtest/purchase/post_data.json b/csit/variables/carpeople/libtest/purchase/post_data.json index df97195154..fed080342c 100644 --- a/csit/variables/carpeople/libtest/purchase/post_data.json +++ b/csit/variables/carpeople/libtest/purchase/post_data.json @@ -1,7 +1,7 @@ { "input": { + "person": "/people:people/people:person[people:id='joe']", "car-id": "boogie", - "person-id": "joe", - "person": "/people:people/people:person[people:id='joe']" + "person-id": "joe" } } diff --git a/csit/variables/restconf/modules/location.uri b/csit/variables/restconf/modules/location.uri new file mode 100644 index 0000000000..6d618bcadd --- /dev/null +++ b/csit/variables/restconf/modules/location.uri @@ -0,0 +1 @@ +/restconf/modules -- 2.36.6