From: Shaleen Saxena Date: Thu, 18 Dec 2014 16:21:08 +0000 (-0500) Subject: Enhancements to 3-node cluster tests X-Git-Tag: release/helium-sr1.1^0 X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=commitdiff_plain;h=f069d23530893a6c048bc543f16a40f4240a2832;p=integration%2Ftest.git Enhancements to 3-node cluster tests Modified the stopController method to wait until the controller karaf process has stopped completed via the ps command. This eliminates the arbitrary wait which can cause sporadic failures if the controller takes too long to shutdown. Also added a stopAllControllers method that first executes /bin/stop on all controllers in parallel and then waits for completion. This speeds it up. Added additional checks in varous test cases, eg use a FOR loop to verify all the expected entries in the output from a REST GET instead of checking for just a couple entries. I also eliminated some test case files and consolidated the steps so that all the related steps for a particular test are in one test case file. I find this makes the steps easier to understand/follow without having to go to other files and figure out which test case files are stand-alone and which may be related to other files. This also eliminates redundant setup steps in each test case file. As part of consolidation, I separated Car and People leader failover tests into their own separate files. They basically run the same sequence of steps but on their respective data model/shard. However they're not exactly redundant because the People tests utilize an RPC. Change-Id: I8be1d219fa07b2384b4d622db67af1bc4e0bebec Signed-off-by: Shaleen Saxena --- diff --git a/test/csit/libraries/ClusterStateLibrary.py b/test/csit/libraries/ClusterStateLibrary.py index 1255623ac7..8f25e415d7 100644 --- a/test/csit/libraries/ClusterStateLibrary.py +++ b/test/csit/libraries/ClusterStateLibrary.py @@ -34,6 +34,7 @@ def getClusterRoles(shardName,numOfShards=3,numOfTries=3,sleepBetweenRetriesInSe resp = UtilLibrary.get(url) print resp if(resp.status_code != 200): + sleep(sleepBetweenRetriesInSecs) continue data = json.loads(resp.text) if('value' in data): @@ -76,10 +77,11 @@ def isLeader(shardName,numOfShards,numOfRetries,sleepFor,port,ipAddress): # Or None # def getLeader(shardName,numOfShards=3,numOfTries=3,sleepBetweenRetriesInSecs=1,port=8181,*ips): - dict = getClusterRoles(shardName,numOfShards,numOfTries,sleepBetweenRetriesInSecs,port,*ips) - for ip in dict.keys(): - if(dict[ip]=='Leader'): - return ip + for i in range(3): # Try 3 times to find a leader + dict = getClusterRoles(shardName,numOfShards,numOfTries,sleepBetweenRetriesInSecs,port,*ips) + for ip in dict.keys(): + if(dict[ip]=='Leader'): + return ip return None diff --git a/test/csit/libraries/CrudLibrary.py b/test/csit/libraries/CrudLibrary.py index f1a23d6fa0..62fd4d5a60 100644 --- a/test/csit/libraries/CrudLibrary.py +++ b/test/csit/libraries/CrudLibrary.py @@ -27,6 +27,7 @@ def addCar(hostname,port,numberOfCars): print("the response of the POST to add car=") print(resp) + time.sleep(5) # Let the add finish return resp #TBD: Detailed validation @@ -202,7 +203,7 @@ def deleteAllPersons(hostname,port,ignore): # # def deleteAllCarsPersons(hostname,port,ignore): - UtilLibrary.delete(SettingsLibrary.getCarPersonsUrl(hostname,port),"admin","admin") + UtilLibrary.delete(SettingsLibrary.getCarPersonUrl(hostname,port),"admin","admin") resp = getPersons(hostname,port,ignore) print("Persons in store after deletion:"+ str(resp)) diff --git a/test/csit/libraries/UtilLibrary.py b/test/csit/libraries/UtilLibrary.py index 2524c57331..0312744e55 100644 --- a/test/csit/libraries/UtilLibrary.py +++ b/test/csit/libraries/UtilLibrary.py @@ -8,6 +8,7 @@ import requests from SSHLibrary import SSHLibrary import robot +import time global _cache @@ -15,29 +16,35 @@ global _cache #Helps in making GET REST calls # -def get(url, userId=None, password=None): +def get(url, userId='admin', password='admin'): headers = {} headers['Accept']= 'application/xml' # Send the GET request session = _cache.switch("CLUSTERING_GET") - resp = session.get(url,headers=headers) - + resp = session.get(url,headers=headers,auth=(userId,password)) + #resp = session.get(url,headers=headers,auth={userId,password}) # Read the response return resp # #Helps in making POST REST calls without outputs # -def nonprintpost(url, userId, password,data): +def nonprintpost(url, userId, password, data): + + if userId == None: + userId = 'admin' + + if password == None: + password = 'admin' headers = {} headers['Content-Type'] = 'application/json' #headers['Accept']= 'application/xml' session = _cache.switch("CLUSTERING_POST") - resp = session.post(url,data.encode('utf-8'),headers=headers) + resp = session.post(url,data.encode('utf-8'),headers=headers,auth=(userId,password)) return resp @@ -45,27 +52,36 @@ def nonprintpost(url, userId, password,data): # #Helps in making POST REST calls # -def post(url, userId, password,data): +def post(url, userId, password, data): + + if userId == None: + userId = 'admin' + + if password == None: + password = 'admin' + print("post request with url "+url) print("post request with data "+data) headers = {} headers['Content-Type'] = 'application/json' #headers['Accept']= 'application/xml' session = _cache.switch("CLUSTERING_POST") - resp = session.post(url,data.encode('utf-8'),headers=headers) + resp = session.post(url,data.encode('utf-8'),headers=headers,auth=(userId,password)) #print (resp.raise_for_status()) print (resp.headers) + if (resp.status_code >= 500): + print (resp.text) return resp # #Helps in making DELET REST calls # -def delete(url,userId,password): +def delete(url, userId='admin', password='admin'): print("delete all resources belonging to url"+url) session = _cache.switch("CLUSTERING_DELETE") - resp=session.delete(url) + resp=session.delete(url,auth=(userId,password)) def Should_Not_Be_Type_None(var): '''Keyword to check if the given variable is of type NoneType. If the @@ -88,12 +104,83 @@ def execute_ssh_command(ip, username, password, command): print "command executed : " + command lib.close_connection() -def startcontroller(ip,username,password,karafhome): +def wait_for_controller_up(ip, port="8181"): + url = "http://" + ip + ":" + str(port) + \ + "/restconf/config/opendaylight-inventory:nodes/node/controller-config/yang-ext:mount/config:modules" + + print "Waiting for controller " + ip + " up." + # Try 30*10s=5 minutes for the controller to be up. + for i in xrange(30): + try: + print "attempt " + str(i) + " to url " + url + resp = get(url, "admin", "admin") + print "attempt " + str(i) + " response is " + str(resp) + print resp.text + if ('clustering-it-provider' in resp.text): + print "Wait for controller " + ip + " succeeded" + return True + except Exception as e: + print e + time.sleep(10) + + print "Wait for controller " + ip + " failed" + return False + +def startAllControllers(username, password, karafhome, port, *ips): + # Start all controllers + for ip in ips: + execute_ssh_command(ip, username, password, karafhome+"/bin/start") + + # Wait for all of them to be up + for ip in ips: + rc = wait_for_controller_up(ip, port) + if (rc == False): + return False + return True + +def startcontroller(ip,username,password,karafhome,port): execute_ssh_command(ip, username, password, karafhome+"/bin/start") + return wait_for_controller_up(ip, port) def stopcontroller(ip,username,password,karafhome): + executeStopController(ip,username,password,karafhome) + + wait_for_controller_stopped(ip,username,password,karafhome) + +def executeStopController(ip,username,password,karafhome): execute_ssh_command(ip, username, password, karafhome+"/bin/stop") +def stopAllControllers(username,password,karafhome,*ips): + for ip in ips: + executeStopController(ip,username,password,karafhome) + + for ip in ips: + wait_for_controller_stopped(ip, username, password, karafhome) + +def wait_for_controller_stopped(ip, username, password, karafHome): + lib = SSHLibrary() + lib.open_connection(ip) + lib.login(username=username,password=password) + + # Wait 1 minute for the controller to stop gracefully + tries = 20 + i=1 + while i <= tries: + stdout = lib.execute_command("ps -axf | grep karaf | grep -v grep | wc -l") + #print "stdout: "+stdout + processCnt = stdout[0].strip('\n') + print "processCnt: "+processCnt + if processCnt == '0': + break; + i = i+1 + time.sleep(3) + + lib.close_connection() + + if i > tries: + print "Killing controller" + kill_controller(ip, username, password, karafHome) + def clean_journal(ip, username, password, karafHome): execute_ssh_command(ip, username, password, "rm -rf " + karafHome + "/journal") diff --git a/test/csit/suites/clustering/datastore/001_start_cluster.txt b/test/csit/suites/clustering/datastore/001_start_cluster.txt index da9264a2f4..910230ec2a 100644 --- a/test/csit/suites/clustering/datastore/001_start_cluster.txt +++ b/test/csit/suites/clustering/datastore/001_start_cluster.txt @@ -14,13 +14,7 @@ ${REST_CONTEXT} /restconf/config/ *** Test Cases *** Stop All Controllers [Documentation] Stop all the controllers in the cluster - Stopcontroller ${MEMBER1} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Stopcontroller ${MEMBER2} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Stopcontroller ${MEMBER3} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Sleep 30 - KillController ${MEMBER1} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - KillController ${MEMBER2} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - KillController ${MEMBER3} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} + StopAllControllers ${USER_NAME} ${PASSWORD} ${KARAF_HOME} ${MEMBER1} ${MEMBER2} ${MEMBER3} Clean All Journals @@ -32,7 +26,6 @@ Clean All Journals Start All Controllers [Documentation] Start all the controllers in the cluster - Startcontroller ${MEMBER1} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Startcontroller ${MEMBER2} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Startcontroller ${MEMBER3} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Sleep 120 \ No newline at end of file + ${rc} StartAllControllers ${USER_NAME} ${PASSWORD} ${KARAF_HOME} ${RESTCONFPORT} + ... ${MEMBER1} ${MEMBER2} ${MEMBER3} + Should Be True ${rc} diff --git a/test/csit/suites/clustering/datastore/010_crud_on_leader.txt b/test/csit/suites/clustering/datastore/010_crud_on_leader.txt index f09db2f15f..e929c86a8f 100644 --- a/test/csit/suites/clustering/datastore/010_crud_on_leader.txt +++ b/test/csit/suites/clustering/datastore/010_crud_on_leader.txt @@ -16,29 +16,52 @@ ${REST_CONTEXT} /restconf/config/ ${SHARD_CAR_NAME} shard-car-config ${SHARD_PEOPLE_NAME} shard-people-config ${SHARD_CAR_PERSON_NAME} shard-car-people-config - +${NUM_ENTRIES} ${30} *** Test Cases *** Add cars and get cars from Leader - [Documentation] Add 100 cars and get added cars from Leader + [Documentation] Add some cars and get added cars from Leader ${CURRENT_CAR_LEADER} GetLeader ${SHARD_CAR_NAME} ${3} ${3} ${1} ${PORT} ${MEMBER1} ${MEMBER2} ${MEMBER3} Log CURRENT_CAR_SHARD_LEADER ${CURRENT_CAR_LEADER} + Should Not Be Equal As Strings ${CURRENT_CAR_LEADER} None Set Suite Variable ${CURRENT_CAR_LEADER} - ${resp} AddCar ${CURRENT_CAR_LEADER} ${PORT} ${100} - ${resp} Getcars ${CURRENT_CAR_LEADER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} manufacturer1 + ${resp} AddCar ${CURRENT_CAR_LEADER} ${PORT} ${NUM_ENTRIES} + ${resp} Getcars ${CURRENT_CAR_LEADER} ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_ENTRIES} + \ Should Contain ${resp.content} manufacturer${i} + +Get added cars using Follower1 + [Documentation] Get added cars using Follower1 + ${FOLLOWERS} GetFollowers ${SHARD_CAR_PERSON_NAME} ${3} ${3} ${1} ${PORT} ${MEMBER1} ${MEMBER2} ${MEMBER3} + Log ${FOLLOWERS} + SET SUITE VARIABLE ${FOLLOWERS} + ${resp} Getcars @{FOLLOWERS}[0] ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_ENTRIES} + \ Should Contain ${resp.content} manufacturer${i} + +Get added cars using Follower2 + [Documentation] Get added cars using Follower2 + ${resp} Getcars @{FOLLOWERS}[1] ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_ENTRIES} + \ Should Contain ${resp.content} manufacturer${i} Add persons and get persons from Leader - [Documentation] Add 100 persons and get persons + [Documentation] Add some persons and get persons [Documentation] Note: There should be one person added first to enable rpc ${CURRENT_PEOPLE_LEADER} GetLeader ${SHARD_PEOPLE_NAME} ${3} ${3} ${1} ${PORT} ${MEMBER1} ${MEMBER2} ${MEMBER3} + Log CURRENT_PEOPLE_SHARD_LEADER ${CURRENT_PEOPLE_LEADER} + Should Not Be Equal As Strings ${CURRENT_PEOPLE_LEADER} None Set Suite Variable ${CURRENT_PEOPLE_LEADER} - ${resp} AddPerson ${CURRENT_PEOPLE_LEADER} ${PORT} ${0} - ${resp} AddPerson ${CURRENT_PEOPLE_LEADER} ${PORT} ${100} - ${resp} GetPersons ${CURRENT_PEOPLE_LEADER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user5 + ${resp} AddPerson ${CURRENT_PEOPLE_LEADER} ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 204 + ${resp} AddPerson ${CURRENT_PEOPLE_LEADER} ${PORT} ${NUM_ENTRIES} + ${resp} GetPersons ${CURRENT_PEOPLE_LEADER} ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_ENTRIES} + \ Should Contain ${resp.content} user${i} Add car-person mapping and get car-person mapping from Leader [Documentation] Add car-person and get car-person from Leader @@ -46,38 +69,33 @@ Add car-person mapping and get car-person mapping from Leader ${CURRENT_CAR_PERSON_LEADER} GetLeader ${SHARD_CAR_PERSON_NAME} ${3} ${3} ${1} ${PORT} ${MEMBER1} ${MEMBER2} ${MEMBER3} Set Suite Variable ${CURRENT_CAR_PERSON_LEADER} ${resp} AddCarPerson ${CURRENT_CAR_PERSON_LEADER} ${PORT} ${0} - Sleep 2 + Sleep 5 ${resp} GetCarPersonMappings ${CURRENT_CAR_PERSON_LEADER} ${PORT} ${0} Should Be Equal As Strings ${resp.status_code} 200 Should Contain ${resp.content} user0 -Purchase 100 cars using Leader - [Documentation] Purchase 100 cars using Leader - ${resp} BuyCar ${CURRENT_CAR_PERSON_LEADER} ${PORT} ${100} - Sleep 2 - ${resp} GetCarPersonMappings ${CURRENT_CAR_PERSON_LEADER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 +Purchase cars using Leader + [Documentation] Purchase some cars using Leader + ${resp} BuyCar ${CURRENT_CAR_PERSON_LEADER} ${PORT} ${NUM_ENTRIES} + Sleep 5 Get car-person mappings using Leader [Documentation] Get car-person mappings using Leader to see 100 entry - ${resp} GetCarPersonMappings ${CURRENT_CAR_PERSON_LEADER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user100 + ${resp} GetCarPersonMappings ${CURRENT_CAR_PERSON_LEADER} ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_ENTRIES} + \ Should Contain ${resp.content} user${i} Get car-person mappings using Follower1 [Documentation] Get car-person mappings using Follower1 to see 100 entry - ${FOLLOWERS} GetFollowers ${SHARD_CAR_PERSON_NAME} ${3} ${3} ${1} ${PORT} ${MEMBER1} ${MEMBER2} ${MEMBER3} - Log ${FOLLOWERS} - SET SUITE VARIABLE ${FOLLOWERS} - ${resp} GetCarPersonMappings ${FOLLOWERS[0]} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user100 - Should Contain ${resp.content} user5 - + ${resp} GetCarPersonMappings @{FOLLOWERS}[0] ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_ENTRIES} + \ Should Contain ${resp.content} user${i} Get car-person mappings using Follower2 [Documentation] Get car-person mappings using Follower2 to see 100 entry - ${resp} GetCarPersonMappings ${FOLLOWERS[1]} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user0 - Should Contain ${resp.content} user100 + ${resp} GetCarPersonMappings @{FOLLOWERS}[1] ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_ENTRIES} + \ Should Contain ${resp.content} user${i} diff --git a/test/csit/suites/clustering/datastore/020_crud_on_any_follower.txt b/test/csit/suites/clustering/datastore/020_crud_on_any_follower.txt index baaa24e44e..5e177387f2 100644 --- a/test/csit/suites/clustering/datastore/020_crud_on_any_follower.txt +++ b/test/csit/suites/clustering/datastore/020_crud_on_any_follower.txt @@ -15,67 +15,100 @@ ${REST_CONTEXT} /restconf/config/ ${SHARD_CAR_NAME} shard-car-config ${SHARD_PEOPLE_NAME} shard-people-config ${SHARD_CAR_PERSON_NAME} shard-car-people-config - +${NUM_ENTRIES} ${40} *** Test Cases *** +Get Car Followers + ${CAR_FOLLOWERS} GetFollowers ${SHARD_CAR_NAME} ${3} ${3} ${1} ${PORT} ${MEMBER1} ${MEMBER2} ${MEMBER3} + Log ${CAR_FOLLOWERS} + SET SUITE VARIABLE ${CAR_FOLLOWERS} + +Get People Followers + ${PEOPLE_FOLLOWERS} GetFollowers ${SHARD_PEOPLE_NAME} ${3} ${3} ${1} ${PORT} ${MEMBER1} ${MEMBER2} ${MEMBER3} + Log ${PEOPLE_FOLLOWERS} + SET SUITE VARIABLE ${PEOPLE_FOLLOWERS} + +Get Car-Person Followers + ${CAR_PERSON_FOLLOWERS} GetFollowers ${SHARD_CAR_NAME} ${3} ${3} ${1} ${PORT} ${MEMBER1} ${MEMBER2} ${MEMBER3} + Log ${CAR_PERSON_FOLLOWERS} + SET SUITE VARIABLE ${CAR_PERSON_FOLLOWERS} + +Delete cars from Follower1 + DeleteAllCars @{CAR_FOLLOWERS}[0] ${PORT} ${0} + ${resp} Getcars @{CAR_FOLLOWERS}[0] ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 404 + +Delete people from Follower1 + DeleteAllPersons @{PEOPLE_FOLLOWERS}[0] ${PORT} ${0} + ${resp} GetPersons @{PEOPLE_FOLLOWERS}[0] ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 404 + +Delete car-persons from Follower1 + DeleteAllCarsPersons @{CAR_PERSON_FOLLOWERS}[0] ${PORT} ${0} + ${resp} GetCarPersonMappings @{CAR_PERSON_FOLLOWERS}[0] ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 404 + Add cars and get cars from Follower1 - [Documentation] Add 100 cars and get added cars from Follower1 - ${FOLLOWERS} GetFollowers ${SHARD_CAR_PERSON_NAME} ${3} ${3} ${1} ${PORT} ${MEMBER1} ${MEMBER2} ${MEMBER3} - Log ${FOLLOWERS} - SET SUITE VARIABLE ${FOLLOWERS} + [Documentation] Add cars and get added cars from Follower1 + ${resp} AddCar @{CAR_FOLLOWERS}[0] ${PORT} ${NUM_ENTRIES} + ${resp} Getcars @{CAR_FOLLOWERS}[0] ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_ENTRIES} + \ Should Contain ${resp.content} manufacturer${i} - ${resp} AddCar ${FOLLOWERS[0]} ${PORT} ${100} - Sleep 1 - ${resp} Getcars ${FOLLOWERS[0]} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} manufacturer1 +Get added cars using Follower2 + [Documentation] Get added cars using Follower2 + ${resp} Getcars @{CAR_FOLLOWERS}[1] ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_ENTRIES} + \ Should Contain ${resp.content} manufacturer${i} Add persons and get persons from Follower1 - [Documentation] Add 100 persons and get persons from Follower1 + [Documentation] Add persons and get persons from Follower1 [Documentation] Note: There should be one person added first to enable rpc - ${resp} AddPerson ${FOLLOWERS[0]} ${PORT} ${0} - ${resp} AddPerson ${FOLLOWERS[0]} ${PORT} ${100} - Sleep 1 - ${resp} GetPersons ${FOLLOWERS[0]} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user5 + ${resp} AddPerson @{PEOPLE_FOLLOWERS}[0] ${PORT} ${0} + ${resp} AddPerson @{PEOPLE_FOLLOWERS}[0] ${PORT} ${NUM_ENTRIES} + Sleep 5 + ${resp} GetPersons @{PEOPLE_FOLLOWERS}[0] ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_ENTRIES} + \ Should Contain ${resp.content} user${i} Add car-person mapping and get car-person mapping from Follower1 - [Documentation] Add car-person and get car-person from Follower1 - [Documentation] Note: This is done to enable working of rpc - - ${resp} AddCarPerson ${FOLLOWERS[0]} ${PORT} ${0} - ${resp} GetCarPersonMappings ${FOLLOWERS[0]} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user0 + [Documentation] Add car-person and get car-person from Follower1 + [Documentation] Note: This is done to enable working of rpc -Purchase 100 cars using Follower1 - [Documentation] Purchase 100 cars using Follower1 + ${resp} AddCarPerson @{CAR_PERSON_FOLLOWERS}[0] ${PORT} ${0} + Sleep 1 + ${resp} GetCarPersonMappings @{CAR_PERSON_FOLLOWERS}[0] ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + Should Contain ${resp.content} user0 - ${resp} BuyCar ${FOLLOWERS[0]} ${PORT} ${100} - Sleep 1 - ${resp} GetCarPersonMappings ${FOLLOWERS[0]} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 +Purchase cars using Follower1 + [Documentation] Purchase cars using Follower1 + ${resp} BuyCar @{CAR_PERSON_FOLLOWERS}[0] ${PORT} ${NUM_ENTRIES} + Sleep 5 Get car-person mappings using Follower1 - [Documentation] Get car-person mappings using follower1 to see 100 entry - ${resp} GetCarPersonMappings ${FOLLOWERS[0]} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user100 - Should Contain ${resp.content} user5 + [Documentation] Get car-person mappings using follower1 to see 100 entry + ${resp} GetCarPersonMappings @{CAR_PERSON_FOLLOWERS}[0] ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_ENTRIES} + \ Should Contain ${resp.content} user${i} Get car-person mappings using Leader - [Documentation] Get car-person mappings using Leader to see 100 entry + [Documentation] Get car-person mappings using Leader to see 100 entry ${CURRENT_CAR_LEADER} GetLeader ${SHARD_CAR_PERSON_NAME} ${3} ${3} ${1} ${PORT} ${MEMBER1} ${MEMBER2} ${MEMBER3} Log ${CURRENT_CAR_LEADER} Sleep 1 - ${resp} GetCarPersonMappings ${CURRENT_CAR_LEADER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user100 + ${resp} GetCarPersonMappings ${CURRENT_CAR_LEADER} ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_ENTRIES} + \ Should Contain ${resp.content} user${i} Get car-person mappings using Follower2 - [Documentation] Get car-person mappings using Follower2 to see 100 entry - ${resp} GetCarPersonMappings ${FOLLOWERS[1]} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user0 - Should Contain ${resp.content} user100 + [Documentation] Get car-person mappings using Follower2 to see 100 entry + ${resp} GetCarPersonMappings @{CAR_PERSON_FOLLOWERS}[1] ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_ENTRIES} + \ Should Contain ${resp.content} user${i} diff --git a/test/csit/suites/clustering/datastore/030_car_failover_crud_on_new_leader.txt b/test/csit/suites/clustering/datastore/030_car_failover_crud_on_new_leader.txt new file mode 100644 index 0000000000..77f865c92d --- /dev/null +++ b/test/csit/suites/clustering/datastore/030_car_failover_crud_on_new_leader.txt @@ -0,0 +1,103 @@ +*** Settings *** +Documentation This test brings down the current leader of the "car" shard and then executes CRUD operations on the new leader +Library ../../../libraries/CrudLibrary.py +Library ../../../libraries/UtilLibrary.py +Library ../../../libraries/ClusterStateLibrary.py + +*** Variables *** +${CAR_SHARD} shard-car-config +${NUM_CARS} ${50} +${NUM_ORIG_CARS} ${10} + +*** Test Cases *** +Get old car leader + ${OLD_CAR_LEADER} GetLeader ${CAR_SHARD} ${3} ${3} ${2} ${PORT} ${MEMBER1} ${MEMBER2} ${MEMBER3} + Should Not Be Equal As Strings ${OLD_CAR_LEADER} None + Set Suite Variable ${OLD_CAR_LEADER} + +Delete cars on old leader + DeleteAllCars ${OLD_CAR_LEADER} ${PORT} ${0} + ${resp} Getcars ${OLD_CAR_LEADER} ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 404 + +Add original cars on old leader + ${resp} AddCar ${OLD_CAR_LEADER} ${PORT} ${NUM_ORIG_CARS} + ${resp} Getcars ${OLD_CAR_LEADER} ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_ORIG_CARS} + \ Should Contain ${resp.content} manufacturer${i} + +Switch car leader + [Documentation] stop leader and elect new leader + Stopcontroller ${OLD_CAR_LEADER} ${USERNAME} ${PASSWORD} ${KARAFHOME} + Sleep 3 + ${NEW_CAR_LEADER} GetLeader ${CAR_SHARD} ${3} ${3} ${2} ${PORT} ${MEMBER1} ${MEMBER2} ${MEMBER3} + Log ${NEW_CAR_LEADER} + Should Not Be Equal As Strings ${NEW_CAR_LEADER} None + Should Not Be Equal ${OLD_CAR_LEADER} ${NEW_CAR_LEADER} + Set Suite Variable ${NEW_CAR_LEADER} + +Get original cars on new leader + ${resp} Getcars ${NEW_CAR_LEADER} ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_ORIG_CARS} + \ Should Contain ${resp.content} manufacturer${i} + +Delete cars at new leader + [Documentation] delete cars + DeleteAllCars ${NEW_CAR_LEADER} ${PORT} ${0} + ${resp} Getcars ${NEW_CAR_LEADER} ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 404 + +Add new cars and get cars from new leader + [Documentation] Add cars and get added cars from Leader + ${resp} AddCar ${NEW_CAR_LEADER} ${PORT} ${NUM_CARS} + ${resp} Getcars ${NEW_CAR_LEADER} ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_CARS} + \ Should Contain ${resp.content} manufacturer${i} + +Get Car Followers + ${CAR_FOLLOWERS} GetFollowers ${CAR_SHARD} ${3} ${3} ${1} ${PORT} ${MEMBER1} ${MEMBER2} ${MEMBER3} + Log ${CAR_FOLLOWERS} + SET SUITE VARIABLE ${CAR_FOLLOWERS} + +Get added cars from Follower + [Documentation] Get added cars using the Follower + ${resp} Getcars @{CAR_FOLLOWERS}[0] ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_CARS} + \ Should Contain ${resp.content} manufacturer${i} + +Delete cars from Follower + [Documentation] delete cars + DeleteAllCars @{CAR_FOLLOWERS}[0] ${PORT} ${0} + ${resp} Getcars @{CAR_FOLLOWERS}[0] ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 404 + +Add cars from follower + [Documentation] Add more cars using the follower + ${resp} AddCar @{CAR_FOLLOWERS}[0] ${PORT} ${NUM_CARS} + ${resp} Getcars @{CAR_FOLLOWERS}[0] ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_CARS} + \ Should Contain ${resp.content} manufacturer${i} + +Get added cars from new leader + [Documentation] Get added cars using the new leader + ${resp} Getcars ${NEW_CAR_LEADER} ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_CARS} + \ Should Contain ${resp.content} manufacturer${i} + +Restart old Car leader + Startcontroller ${OLD_CAR_LEADER} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} ${PORT} + Sleep 3 + +Get added cars from old leader + [Documentation] Get added cars using the old leader + ${resp} Getcars ${OLD_CAR_LEADER} ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_CARS} + \ Should Contain ${resp.content} manufacturer${i} + diff --git a/test/csit/suites/clustering/datastore/030_failover_crud_on_new_leader.txt b/test/csit/suites/clustering/datastore/030_failover_crud_on_new_leader.txt deleted file mode 100644 index e40888d365..0000000000 --- a/test/csit/suites/clustering/datastore/030_failover_crud_on_new_leader.txt +++ /dev/null @@ -1,70 +0,0 @@ -*** Settings *** -Documentation This test brings down the current leader of the "car" shard and then executes CRUD operations on the new leader -Library ../../../libraries/CrudLibrary.py -Library ../../../libraries/UtilLibrary.py -Library ../../../libraries/ClusterStateLibrary.py - -*** Variables *** -${SHARD} shard-car-config - -*** Test Cases *** -Switch Leader - [Documentation] stop leader and elect new leader - ${OLD_LEADER} GetLeader ${SHARD} ${3} ${3} ${2} ${8181} ${MEMBER1} ${MEMBER2} ${MEMBER3} - Stopcontroller ${OLD_LEADER} ${USERNAME} ${PASSWORD} ${KARAFHOME} - Sleep 30 - ${NEW_LEADER} GetLeader ${SHARD} ${3} ${3} ${2} ${8181} ${MEMBER1} ${MEMBER2} ${MEMBER3} - Log ${NEW_LEADER} - Set Suite Variable ${NEW_LEADER} - -Delete cars at new leader - [Documentation] delete cars - DeleteAllCars ${NEW_LEADER} ${PORT} ${0} - ${resp} Getcars ${NEW_LEADER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 404 - -Delete people at new leader - [Documentation] delete people - DeleteAllPersons ${NEW_LEADER} ${PORT} ${0} - ${resp} GetPersons ${NEW_LEADER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 404 - -Add cars and get cars from new leader - [Documentation] Add 50 cars and get added cars from Leader - ${resp} AddCar ${NEW_LEADER} ${PORT} ${50} - Sleep 10 - ${resp} Getcars ${NEW_LEADER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} manufacturer1 - Should Contain ${resp.content} manufacturer50 - Should Not Contain ${resp.content} manufacturer60 - -Add people and get people from leader - [Documentation] Add 50 persons and get people - [Documentation] Note: There should be one person added first to enable rpc - ${resp} AddPerson ${NEW_LEADER} ${PORT} ${0} - ${resp} AddPerson ${NEW_LEADER} ${PORT} ${50} - Sleep 10 - ${resp} GetPersons ${NEW_LEADER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user2 - Should Contain ${resp.content} user50 - Should Not Contain ${resp.content} user60 - -Purchase 50 cars at new leader - [Documentation] Add car-person and get car-person from Leader - [Documentation] Note: This is done to enable working of rpc - [Documentation] Purchase 50 cars using Leader - ${resp} AddCarPerson ${NEW_LEADER} ${PORT} ${0} - ${resp} GetCarPersonMappings ${NEW_LEADER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user0 - ${resp} BuyCar ${NEW_LEADER} ${PORT} ${50} - Sleep 10 - -Get car-person mappings at new Leader - [Documentation] Get car-person mappings using new leader to see 50 entry - ${resp} GetCarPersonMappings ${NEW_LEADER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user5 - Should Contain ${resp.content} user50 \ No newline at end of file diff --git a/test/csit/suites/clustering/datastore/040_failover_read_from_follower.txt b/test/csit/suites/clustering/datastore/040_failover_read_from_follower.txt deleted file mode 100644 index 9cfd3588d9..0000000000 --- a/test/csit/suites/clustering/datastore/040_failover_read_from_follower.txt +++ /dev/null @@ -1,38 +0,0 @@ -*** Settings *** -Documentation This test tries to read the data that was written by the previous test from any one follower -Library ../../../libraries/CrudLibrary.py -Library ../../../libraries/ClusterStateLibrary.py - -*** Variables *** -${SHARD} shard-car-config - -*** Test Cases *** -Find follower - [Documentation] find follower - ${FOLLOWERS} GetFollowers ${SHARD} ${3} ${3} ${2} ${8181} ${MEMBER1} ${MEMBER2} ${MEMBER3} - Log ${FOLLOWERS} - ${LAST_FOLLOWER} Set Variable ${FOLLOWERS[0]} - Set Suite Variable ${LAST_FOLLOWER} - -Get cars from last follower - [Documentation] get 50 cars from last follower - ${resp} Getcars ${LAST_FOLLOWER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} manufacturer1 - Should Contain ${resp.content} manufacturer50 - Should Not Contain ${resp.content} manufacturer60 - -Get people from last follower - [Documentation] get 50 people - ${resp} GetPersons ${LAST_FOLLOWER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user2 - Should Contain ${resp.content} user50 - Should Not Contain ${resp.content} user60 - -Get car-person mappings at last follower - [Documentation] Get car-person mappings using last to see 50 entry - ${resp} GetCarPersonMappings ${LAST_FOLLOWER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user5 - Should Contain ${resp.content} user50 \ No newline at end of file diff --git a/test/csit/suites/clustering/datastore/040_people_failover_crud_on_new_leader.txt b/test/csit/suites/clustering/datastore/040_people_failover_crud_on_new_leader.txt new file mode 100644 index 0000000000..0704abf88a --- /dev/null +++ b/test/csit/suites/clustering/datastore/040_people_failover_crud_on_new_leader.txt @@ -0,0 +1,83 @@ +*** Settings *** +Documentation This test brings down the current leader of the "car" shard and then executes CRUD operations on the new leader +Library ../../../libraries/CrudLibrary.py +Library ../../../libraries/UtilLibrary.py +Library ../../../libraries/ClusterStateLibrary.py + +*** Variables *** +${PEOPLE_SHARD} shard-people-config +${NUM_ENTRIES} ${50} + +*** Test Cases *** +Switch People Leader + [Documentation] stop leader and elect new leader + ${OLD_PEOPLE_LEADER} GetLeader ${PEOPLE_SHARD} ${3} ${3} ${2} ${PORT} ${MEMBER1} ${MEMBER2} ${MEMBER3} + Stopcontroller ${OLD_PEOPLE_LEADER} ${USERNAME} ${PASSWORD} ${KARAFHOME} + Sleep 3 + ${NEW_PEOPLE_LEADER} GetLeader ${PEOPLE_SHARD} ${3} ${3} ${2} ${PORT} ${MEMBER1} ${MEMBER2} ${MEMBER3} + Log ${NEW_PEOPLE_LEADER} + Should Not Be Equal As Strings ${NEW_PEOPLE_LEADER} None + Should Not Be Equal ${OLD_PEOPLE_LEADER} ${NEW_PEOPLE_LEADER} + Set Suite Variable ${NEW_PEOPLE_LEADER} + Set Suite Variable ${OLD_PEOPLE_LEADER} + +Delete people from new leader + [Documentation] delete people + DeleteAllPersons ${NEW_PEOPLE_LEADER} ${PORT} ${0} + ${resp} GetPersons ${NEW_PEOPLE_LEADER} ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 404 + +Add persons and get from new leader + [Documentation] Add persons and get persons from new leader + [Documentation] Note: There should be one person added first to enable rpc + ${resp} AddPerson ${NEW_PEOPLE_LEADER} ${PORT} ${0} + ${resp} AddPerson ${NEW_PEOPLE_LEADER} ${PORT} ${NUM_ENTRIES} + Sleep 10 + ${resp} GetPersons ${NEW_PEOPLE_LEADER} ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_ENTRIES} + \ Should Contain ${resp.content} user${i} + +Get People Followers + ${PEOPLE_FOLLOWERS} GetFollowers ${PEOPLE_SHARD} ${3} ${3} ${1} ${PORT} ${MEMBER1} ${MEMBER2} ${MEMBER3} + Log ${PEOPLE_FOLLOWERS} + SET SUITE VARIABLE ${PEOPLE_FOLLOWERS} + +Get added persons from follower + ${resp} GetPersons @{PEOPLE_FOLLOWERS}[0] ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_ENTRIES} + \ Should Contain ${resp.content} user${i} + +Delete people from new Follower + [Documentation] delete people + DeleteAllPersons @{PEOPLE_FOLLOWERS}[0] ${PORT} ${0} + ${resp} GetPersons @{PEOPLE_FOLLOWERS}[0] ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 404 + +Add persons from new Follower + [Documentation] Add persons and get persons from follower + [Documentation] Note: There should be one person added first to enable rpc + ${resp} AddPerson @{PEOPLE_FOLLOWERS}[0] ${PORT} ${0} + ${resp} AddPerson @{PEOPLE_FOLLOWERS}[0] ${PORT} ${NUM_ENTRIES} + Sleep 10 + ${resp} GetPersons @{PEOPLE_FOLLOWERS}[0] ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_ENTRIES} + \ Should Contain ${resp.content} user${i} + +Get added persons from new leader + ${resp} GetPersons ${NEW_PEOPLE_LEADER} ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_ENTRIES} + \ Should Contain ${resp.content} user${i} + +Restart old People leader + Startcontroller ${OLD_PEOPLE_LEADER} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} ${PORT} + Sleep 3 + +Get added persons from old leader + ${resp} GetPersons ${OLD_PEOPLE_LEADER} ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_ENTRIES} + \ Should Contain ${resp.content} user${i} diff --git a/test/csit/suites/clustering/datastore/050_car_persistence_recovery.txt b/test/csit/suites/clustering/datastore/050_car_persistence_recovery.txt new file mode 100644 index 0000000000..1a4d601c4f --- /dev/null +++ b/test/csit/suites/clustering/datastore/050_car_persistence_recovery.txt @@ -0,0 +1,55 @@ +*** Settings *** +Documentation This test restarts all controllers to verify recovery of car data from persistene +Library ../../../libraries/CrudLibrary.py +Library ../../../libraries/UtilLibrary.py +Library ../../../libraries/ClusterStateLibrary.py +Variables ../../../variables/Variables.py + +*** Variables *** +${CAR_SHARD} shard-car-config +${NUM_CARS} ${50} + +*** Test Cases *** +Get car leader + ${CAR_LEADER} GetLeader ${CAR_SHARD} ${3} ${3} ${2} ${PORT} ${MEMBER1} ${MEMBER2} ${MEMBER3} + Should Not Be Equal As Strings ${CAR_LEADER} None + Set Suite Variable ${CAR_LEADER} + +Delete cars from leader + DeleteAllCars ${CAR_LEADER} ${PORT} ${0} + ${resp} Getcars ${CAR_LEADER} ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 404 + +Stop all controllers after delete + StopAllControllers ${USER_NAME} ${PASSWORD} ${KARAF_HOME} ${MEMBER1} ${MEMBER2} ${MEMBER3} + +Start all controllers after delete + ${rc} StartAllControllers ${USER_NAME} ${PASSWORD} ${KARAF_HOME} ${RESTCONFPORT} + ... ${MEMBER1} ${MEMBER2} ${MEMBER3} + Should Be True ${rc} + +Verify no cars on leader after restart + ${resp} Getcars ${CAR_LEADER} ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 404 + +Add cars on leader + ${resp} AddCar ${CAR_LEADER} ${PORT} ${NUM_CARS} + ${resp} Getcars ${CAR_LEADER} ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_CARS} + \ Should Contain ${resp.content} manufacturer${i} + +Stop all controllers after add + StopAllControllers ${USER_NAME} ${PASSWORD} ${KARAF_HOME} ${MEMBER1} ${MEMBER2} ${MEMBER3} + +Start all controllers after add + ${rc} StartAllControllers ${USER_NAME} ${PASSWORD} ${KARAF_HOME} ${RESTCONFPORT} + ... ${MEMBER1} ${MEMBER2} ${MEMBER3} + Should Be True ${rc} + +Get cars from leader after restart + ${resp} Getcars ${CAR_LEADER} ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_CARS} + \ Should Contain ${resp.content} manufacturer${i} + diff --git a/test/csit/suites/clustering/datastore/050_failover_crud_on_any_follower.txt b/test/csit/suites/clustering/datastore/050_failover_crud_on_any_follower.txt deleted file mode 100644 index 89bb931b70..0000000000 --- a/test/csit/suites/clustering/datastore/050_failover_crud_on_any_follower.txt +++ /dev/null @@ -1,67 +0,0 @@ -*** Settings *** -Documentation This test executes CRUD operations on any one follower after the old leader has been brought down -Library ../../../libraries/CrudLibrary.py -Library ../../../libraries/ClusterStateLibrary.py - -*** Variables *** -${SHARD} shard-car-config - -*** Test Cases *** -Find follower - [Documentation] find follower - ${FOLLOWERS} GetFollowers ${SHARD} ${3} ${3} ${2} ${8181} ${MEMBER1} ${MEMBER2} ${MEMBER3} - Log ${FOLLOWERS} - ${LAST_FOLLOWER} Set Variable ${FOLLOWERS[0]} - Set Suite Variable ${LAST_FOLLOWER} - -Delete cars at last follower - [Documentation] delete cars - DeleteAllCars ${LAST_FOLLOWER} ${PORT} ${0} - ${resp} Getcars ${LAST_FOLLOWER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 404 - -Delete people at last follower - [Documentation] delete people - DeleteAllPersons ${LAST_FOLLOWER} ${PORT} ${0} - ${resp} GetPersons ${LAST_FOLLOWER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 404 - -Add cars and get cars from last follower - [Documentation] Add 40 cars and get added cars from last follower - ${resp} AddCar ${LAST_FOLLOWER} ${PORT} ${40} - Sleep 8 - ${resp} Getcars ${LAST_FOLLOWER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} manufacturer1 - Should Contain ${resp.content} manufacturer40 - Should Not Contain ${resp.content} manufacturer50 - -Add people and get people from last follower - [Documentation] Add 40 persons and get people - [Documentation] Note: There should be one person added first to enable rpc - ${resp} AddPerson ${LAST_FOLLOWER} ${PORT} ${0} - ${resp} AddPerson ${LAST_FOLLOWER} ${PORT} ${40} - Sleep 8 - ${resp} GetPersons ${LAST_FOLLOWER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user2 - Should Contain ${resp.content} user40 - Should Not Contain ${resp.content} user50 - -Purchase 40 cars at last follower - [Documentation] Add car-person and get car-person from last follower - [Documentation] Note: This is done to enable working of rpc - [Documentation] Purchase 40 cars using Leader - ${resp} AddCarPerson ${LAST_FOLLOWER} ${PORT} ${0} - ${resp} GetCarPersonMappings ${LAST_FOLLOWER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user0 - ${resp} BuyCar ${LAST_FOLLOWER} ${PORT} ${40} - Sleep 8 - -Get car-person mappings at last follower - [Documentation] Get car-person mappings using last follower to see 40 entry - ${resp} GetCarPersonMappings ${LAST_FOLLOWER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user5 - Should Contain ${resp.content} user40 \ No newline at end of file diff --git a/test/csit/suites/clustering/datastore/060_failover_read_from_new_leader.txt b/test/csit/suites/clustering/datastore/060_failover_read_from_new_leader.txt deleted file mode 100644 index d673aefb59..0000000000 --- a/test/csit/suites/clustering/datastore/060_failover_read_from_new_leader.txt +++ /dev/null @@ -1,37 +0,0 @@ -*** Settings *** -Documentation This test reads the data from the leader that was written to the follower by the previous test -Library ../../../libraries/CrudLibrary.py -Library ../../../libraries/ClusterStateLibrary.py - -*** Variables *** -${SHARD} shard-car-config - -*** Test Cases *** -Find Leader - [Documentation] find new leader - ${NEW_LEADER} GetLeader ${SHARD} ${3} ${3} ${2} ${8181} ${MEMBER1} ${MEMBER2} ${MEMBER3} - Log ${NEW_LEADER} - Set Suite Variable ${NEW_LEADER} - -Get cars from new leader - [Documentation] get 40 cars from new leader - ${resp} Getcars ${NEW_LEADER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} manufacturer1 - Should Contain ${resp.content} manufacturer40 - Should Not Contain ${resp.content} manufacturer50 - -Get people from new leader - [Documentation] get 40 people from new leader - ${resp} GetPersons ${NEW_LEADER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user2 - Should Contain ${resp.content} user40 - Should Not Contain ${resp.content} user50 - -Get car-person mappings at new leader - [Documentation] Get car-person mappings using last to see 40 entry - ${resp} GetCarPersonMappings ${NEW_LEADER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} user5 - Should Contain ${resp.content} user40 \ No newline at end of file diff --git a/test/csit/suites/clustering/datastore/130_recovery_restart_leader.txt b/test/csit/suites/clustering/datastore/130_recovery_restart_leader.txt deleted file mode 100644 index 567eb3f37b..0000000000 --- a/test/csit/suites/clustering/datastore/130_recovery_restart_leader.txt +++ /dev/null @@ -1,104 +0,0 @@ -*** Settings *** -Documentation This test kills the leader and verifies that on restart the old leader is able to rejoin the cluster -Library Collections -Library ../../../libraries/RequestsLibrary.py -Library ../../../libraries/Common.py -Library ../../../libraries/CrudLibrary.py -Library ../../../libraries/SettingsLibrary.py -Library ../../../libraries/UtilLibrary.py -Library ../../../libraries/ClusterStateLibrary.py -Variables ../../../variables/Variables.py - -*** Variables *** -${REST_CONTEXT} /restconf/config/ -${KARAF_HOME} /root/odl/dist -${USER_NAME} root -${PASSWORD} Ecp123 -${CAR_SHARD} shard-car-config - -*** Test Cases *** -Stop All Controllers - [Documentation] Stop all the controllers in the cluster - Stopcontroller ${MEMBER1} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Stopcontroller ${MEMBER2} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Stopcontroller ${MEMBER3} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Sleep 30 - KillController ${MEMBER1} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - KillController ${MEMBER2} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - KillController ${MEMBER3} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - - -Clean All Journals - [Documentation] Clean the journals of all the controllers in the cluster - CleanJournal ${MEMBER1} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - CleanJournal ${MEMBER2} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - CleanJournal ${MEMBER3} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Sleep 5 - -Start All Controllers - [Documentation] Start all the controllers in the cluster - Startcontroller ${MEMBER1} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Startcontroller ${MEMBER2} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Startcontroller ${MEMBER3} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Sleep 120 - -Delete all cars - [Documentation] Delete all the cars from the system - ${resp} DeleteAllCars ${MEMBER1} ${PORT} 0 - ${resp} GetCars ${MEMBER1} ${PORT} 0 - Should Be Equal As Strings ${resp.status_code} 404 - - -Delete all people - [Documentation] Delete all the people from the system - ${resp} DeleteAllPersons ${MEMBER1} ${PORT} 0 - ${resp} GetPersons ${MEMBER1} ${PORT} 0 - Should Be Equal As Strings ${resp.status_code} 404 - -Add 200 cars - [Documentation] Add 200 cars - ${resp} AddCar ${MEMBER1} ${PORT} ${200} - Should Be Equal As Strings ${resp.status_code} 204 - -Add 200 people - [Documentation] Add 200 people - ${resp} AddPerson ${MEMBER1} ${PORT} ${0} - ${resp} AddPerson ${MEMBER1} ${PORT} ${200} - Should Be Equal As Strings ${resp.status_code} 204 - -Add Car Person mapping - [Documentation] Add Car Persons - ${resp} AddCarPerson ${MEMBER1} ${PORT} ${0} - ${resp} BuyCar ${MEMBER1} ${PORT} ${200} - -Stop the Leader - ${CAR_LEADER} GetLeader ${CAR_SHARD} ${3} ${3} ${1} 8181 ${MEMBER1} ${MEMBER2} ${MEMBER3} - Set Suite Variable ${CAR_LEADER} - Stopcontroller ${CAR_LEADER} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Sleep 30 - KillController ${CAR_LEADER} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - -Get all the cars from Follower 1 - ${followers} GetFollowers ${CAR_SHARD} ${3} ${3} ${1} 8181 ${MEMBER1} ${MEMBER2} ${MEMBER3} - ${resp} Getcars ${followers[0]} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} manufacturer1 - -Restart the Leader - Startcontroller ${CAR_LEADER} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Sleep 120 - -Get all the cars from Leader - ${resp} Getcars ${CAR_LEADER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} manufacturer1 - -Cleanup All Controllers - [Documentation] Stop all the controllers in the cluster - Stopcontroller ${MEMBER1} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Stopcontroller ${MEMBER2} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Stopcontroller ${MEMBER3} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Sleep 30 - - - diff --git a/test/csit/suites/clustering/datastore/140_recovery_restart_follower.txt b/test/csit/suites/clustering/datastore/140_recovery_restart_follower.txt index 195ef3efd8..9943294cae 100644 --- a/test/csit/suites/clustering/datastore/140_recovery_restart_follower.txt +++ b/test/csit/suites/clustering/datastore/140_recovery_restart_follower.txt @@ -11,21 +11,13 @@ Variables ../../../variables/Variables.py *** Variables *** ${REST_CONTEXT} /restconf/config/ -${KARAF_HOME} /root/odl/dist -${USER_NAME} root -${PASSWORD} Ecp123 ${CAR_SHARD} shard-car-config +${NUM_CARS} ${60} *** Test Cases *** Stop All Controllers [Documentation] Stop all the controllers in the cluster - Stopcontroller ${MEMBER1} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Stopcontroller ${MEMBER2} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Stopcontroller ${MEMBER3} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Sleep 30 - KillController ${MEMBER1} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - KillController ${MEMBER2} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - KillController ${MEMBER3} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} + StopAllControllers ${USER_NAME} ${PASSWORD} ${KARAF_HOME} ${MEMBER1} ${MEMBER2} ${MEMBER3} Clean All Journals @@ -37,69 +29,48 @@ Clean All Journals Start All Controllers [Documentation] Start all the controllers in the cluster - Startcontroller ${MEMBER1} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Startcontroller ${MEMBER2} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Startcontroller ${MEMBER3} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Sleep 120 - -Delete all cars - [Documentation] Delete all the cars from the system - ${resp} DeleteAllCars ${MEMBER1} ${PORT} 0 - ${resp} GetCars ${MEMBER1} ${PORT} 0 - Should Be Equal As Strings ${resp.status_code} 404 - - -Delete all people - [Documentation] Delete all the people from the system - ${resp} DeleteAllPersons ${MEMBER1} ${PORT} 0 - ${resp} GetPersons ${MEMBER1} ${PORT} 0 - Should Be Equal As Strings ${resp.status_code} 404 - -Add 200 cars - [Documentation] Add 200 cars - ${resp} AddCar ${MEMBER1} ${PORT} ${200} - Should Be Equal As Strings ${resp.status_code} 204 - -Add 200 people - [Documentation] Add 200 people - ${resp} AddPerson ${MEMBER1} ${PORT} ${0} - ${resp} AddPerson ${MEMBER1} ${PORT} ${200} - Should Be Equal As Strings ${resp.status_code} 204 - -Add Car Person mapping - [Documentation] Add Car Persons - ${resp} AddCarPerson ${MEMBER1} ${PORT} ${0} - ${resp} BuyCar ${MEMBER1} ${PORT} ${200} - -Stop one of the followers - ${followers} GetFollowers ${CAR_SHARD} ${3} ${3} ${1} 8181 ${MEMBER1} ${MEMBER2} ${MEMBER3} - ${CAR_FOLLOWER} Set Variable ${followers[0]} - Set Suite Variable ${CAR_FOLLOWER} - Stopcontroller ${CAR_FOLLOWER} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Sleep 30 - KillController ${CAR_FOLLOWER} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - -Get all the cars from the other Follower - ${followers} GetFollowers ${CAR_SHARD} ${3} ${3} ${1} 8181 ${MEMBER1} ${MEMBER2} ${MEMBER3} - ${resp} Getcars ${followers[0]} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} manufacturer1 - -Restart the Stopped Follower - Startcontroller ${CAR_FOLLOWER} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Sleep 120 - -Get all the cars from Stopped Follower - ${resp} Getcars ${CAR_FOLLOWER} ${PORT} ${0} - Should Be Equal As Strings ${resp.status_code} 200 - Should Contain ${resp.content} manufacturer1 + ${rc} StartAllControllers ${USER_NAME} ${PASSWORD} ${KARAF_HOME} ${RESTCONFPORT} + ... ${MEMBER1} ${MEMBER2} ${MEMBER3} + Should Be True ${rc} + Sleep 3 + +Get car leader and followers + ${CURRENT_CAR_LEADER} GetLeader ${CAR_SHARD} ${3} ${3} ${1} ${PORT} ${MEMBER1} ${MEMBER2} ${MEMBER3} + Set Suite Variable ${CURRENT_CAR_LEADER} + ${CAR_FOLLOWERS} GetFollowers ${CAR_SHARD} ${3} ${3} ${1} ${PORT} ${MEMBER1} ${MEMBER2} ${MEMBER3} + Set Suite Variable ${CAR_FOLLOWERS} + +Stop both of the followers + StopAllControllers ${USER_NAME} ${PASSWORD} ${KARAF_HOME} @{CAR_FOLLOWERS}[0] @{CAR_FOLLOWERS}[1] + +Attempt to add a car from the leader + [Documentation] Should fail as both followers are down + ${resp} AddCar ${CURRENT_CAR_LEADER} ${PORT} ${1} + ${resp} Getcars ${CURRENT_CAR_LEADER} ${PORT} ${1} + Should Be Equal As Strings ${resp.status_code} 404 + +Restart the first follower + Startcontroller @{CAR_FOLLOWERS}[0] ${USER_NAME} ${PASSWORD} ${KARAF_HOME} ${PORT} + Sleep 3 + +Add cars from the first follower + ${resp} AddCar @{CAR_FOLLOWERS}[0] ${PORT} ${NUM_CARS} + Should Be Equal As Strings ${resp.status_code} 204 + ${resp} Getcars @{CAR_FOLLOWERS}[0] ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_CARS} + \ Should Contain ${resp.content} manufacturer${i} + +Restart the second follower + Startcontroller @{CAR_FOLLOWERS}[1] ${USER_NAME} ${PASSWORD} ${KARAF_HOME} ${PORT} + Sleep 3 + +Get all the cars from the second follower + ${resp} Getcars @{CAR_FOLLOWERS}[1] ${PORT} ${0} + Should Be Equal As Strings ${resp.status_code} 200 + :FOR ${i} IN RANGE 1 ${NUM_CARS} + \ Should Contain ${resp.content} manufacturer${i} Cleanup All Controllers [Documentation] Stop all the controllers in the cluster - Stopcontroller ${MEMBER1} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Stopcontroller ${MEMBER2} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Stopcontroller ${MEMBER3} ${USER_NAME} ${PASSWORD} ${KARAF_HOME} - Sleep 30 - - - + StopAllControllers ${USER_NAME} ${PASSWORD} ${KARAF_HOME} ${MEMBER1} ${MEMBER2} ${MEMBER3} diff --git a/test/csit/suites/clustering/datastore/__init__.txt b/test/csit/suites/clustering/datastore/__init__.txt index cf422c9cf5..604f9af3ef 100644 --- a/test/csit/suites/clustering/datastore/__init__.txt +++ b/test/csit/suites/clustering/datastore/__init__.txt @@ -1,6 +1,5 @@ *** Settings *** Documentation Test suite for Clustering Datastore -Library SSHLibrary +Library SSHLibrary - -** Keywords *** +*** Keywords ***