From d0cd46e5a33f120bf8866fac8c73006ffbc3a7bd Mon Sep 17 00:00:00 2001 From: Peter Gubka Date: Thu, 20 Apr 2017 15:14:58 +0200 Subject: [PATCH] Add prefix based shard dom-data-broker suites Change-Id: I54326b1aefa5e9beacaab5b8a418d9ab93a9c7e7 Signed-off-by: Peter Gubka --- csit/libraries/MdsalLowlevel.robot | 8 +- csit/libraries/MdsalLowlevelPy.py | 130 ++++++++--- csit/libraries/controller/DdbCommons.robot | 211 +++++++++++++++--- ...clean_leader_shutdown_prefbasedshard.robot | 33 +++ .../client_isolation_prefbasedshard.robot | 41 ++++ .../ddb-sanity-module-based.robot | 84 +++++++ .../ddb-sanity-prefix-based.robot | 86 +++++++ ...licit_leader_movement_prefbasedshard.robot | 37 +++ .../leader_isolation_prefbasedshard.robot | 32 +++ .../remote_listener_prefbasedshard.robot | 32 +++ csit/testplans/controller-clustering.txt | 7 + .../add_prefix_shard_replica/post_data.xml | 4 +- .../remove_prefix_shard_replica/post_data.xml | 4 +- .../become_prefix_leader/post_data.xml | 4 +- .../create_prefix_shard/location.uri | 2 +- .../create_prefix_shard/post_data.xml | 4 +- .../remove_prefix_shard/location.uri | 2 +- .../remove_prefix_shard/post_data.xml | 4 +- 18 files changed, 646 insertions(+), 79 deletions(-) create mode 100644 csit/suites/controller/dom_data_broker/clean_leader_shutdown_prefbasedshard.robot create mode 100644 csit/suites/controller/dom_data_broker/client_isolation_prefbasedshard.robot create mode 100644 csit/suites/controller/dom_data_broker/ddb-sanity-module-based.robot create mode 100644 csit/suites/controller/dom_data_broker/ddb-sanity-prefix-based.robot create mode 100644 csit/suites/controller/dom_data_broker/explicit_leader_movement_prefbasedshard.robot create mode 100644 csit/suites/controller/dom_data_broker/leader_isolation_prefbasedshard.robot create mode 100644 csit/suites/controller/dom_data_broker/remote_listener_prefbasedshard.robot diff --git a/csit/libraries/MdsalLowlevel.robot b/csit/libraries/MdsalLowlevel.robot index bbc188af90..0ee7049440 100644 --- a/csit/libraries/MdsalLowlevel.robot +++ b/csit/libraries/MdsalLowlevel.robot @@ -132,11 +132,11 @@ Produce_Transactions Create_Prefix_Shard [Arguments] ${member_index} ${prefix} ${replicas} - [Documentation] Create prefix based shard. + [Documentation] Create prefix based shard. ${replicas} is a list of cluster node indexes, taken e.g. from ClusterManagement.List_Indices_Or_All. ${session} = ClusterManagement.Resolve_Http_Session_For_Member member_index=${member_index} ${replicas_str} BuiltIn.Set_Variable ${EMPTY} : FOR ${replica} IN @{replicas} - \ ${replicas_str} BuiltIn.Set_Variable ${replicas_str}${replica} + \ ${replicas_str} BuiltIn.Set_Variable ${replicas_str}member-${replica} &{mapping} BuiltIn.Create_Dictionary PREFIX=${prefix} REPLICAS=${replicas_str} ${text} = TemplatedRequests.Post_As_Xml_Templated ${CREATE_PREFIX_SHARD_DIR} mapping=${mapping} session=${session} @@ -148,10 +148,10 @@ Remove_Prefix_Shard ${text} = TemplatedRequests.Post_As_Xml_Templated ${REMOVE_PREFIX_SHARD_DIR} mapping=${mapping} session=${session} Become_Prefix_Leader - [Arguments] ${member_index} ${prefix} + [Arguments] ${member_index} ${shard_name} ${id_prefix} [Documentation] Given node ask to become a shard leader. ${session} = ClusterManagement.Resolve_Http_Session_For_Member member_index=${member_index} - &{mapping} BuiltIn.Create_Dictionary PREFIX=${prefix} + &{mapping} BuiltIn.Create_Dictionary SHARD_NAME=${shard_name} ID=${id_prefix}${member_index} ${text} = TemplatedRequests.Post_As_Xml_Templated ${BECOME_PREFIX_LEADER_DIR} mapping=${mapping} session=${session} Subscribe_Dtcl diff --git a/csit/libraries/MdsalLowlevelPy.py b/csit/libraries/MdsalLowlevelPy.py index 012f87c2cf..d0506cd9a1 100644 --- a/csit/libraries/MdsalLowlevelPy.py +++ b/csit/libraries/MdsalLowlevelPy.py @@ -11,54 +11,124 @@ import threading _globals = {} -def start_write_transactions_on_nodes(host_list, id_prefix, duration, rate, chained_flag=True): - """Invoke publish notification rpcs and verify the response. +def _send_http_request_thread_impl(rqueue, url, data, http_timeout): + """Start either publish or write transactions rpc based on input. + + :param rqueue: result queue + :type rqueue: Queue.Queue + :param url: rpc url + :type url: string + :param data: http request content + :type data: string + :param http_timeout: http response timeout + :type http_timeout: int + """ + logger.info('rpc indoked with details: {}'.format(data)) + try: + resp = requests.post(url=url, headers={'Content-Type': 'application/xml'}, + data=data, auth=('admin', 'admin'), timeout=http_timeout) + except Exception as exc: + resp = exc + logger.debug(exc) + rqueue.put(resp) + + +def _initiate_rpcs(host_list, prefix_list, url_templ, data_templ, subst_dict): + """Initiate rpc on given hosts. + + :param host_list: list of ip address of odl nodes + :type host_list: list of strings + :param id_prefix: list of node indexes which coresponds to the ip addresses + :type id_prefix: string + :param url_templ: url template + :type url_templ: string.Template object + :param data_templ: http request data + :type data_templ: string.Template object + :param subst_dict: dictionary with key value pairs to be used with template + :type subst_dict: dict + """ + resqueue = _globals.pop('result_queue', Queue.Queue()) + lthreads = _globals.pop('threads', []) + for i, host in enumerate(host_list): + subst_dict.update({'ID': '{}{}'.format(subst_dict['ID_PREFIX'], prefix_list[i])}) + url = url_templ.substitute({'HOST': host}) + data = data_templ.substitute(subst_dict) + timeout = int(subst_dict['DURATION'])+60 + logger.info('url: {}, data: {}, timeout: {}'.format(url, data, timeout)) + t = threading.Thread(target=_send_http_request_thread_impl, + args=(resqueue, url, data, timeout)) + t.daemon = True + t.start() + lthreads.append(t) + + _globals.update({'threads': lthreads, 'result_queue': resqueue}) + + +def start_write_transactions_on_nodes(host_list, prefix_list, id_prefix, duration, rate, chained_flag=False): + """Invoke write-transactions rpc on given nodes. :param host_list: list of ip address of odl nodes :type host_list: list of strings + :param prefix_list: list of node indexes which coresponds to the ip addresses + :type prefix_list: list :param id_prefix: identifier prefix :type id_prefix: string :param duration: time in seconds :type duration: int - :param rate: writing transactions rate per second + :param rate: writing transactions rate in transactions per second :type rate: int :param chained_flag: specify chained vs. simple transactions :type chained_flag: bool """ - def _write_transactions(rqueue, url, grid, duration, rate, chained_flag): - dtmpl = string.Template(''' + logger.info( + "Input parameters: host_list:{}, prefix_list:{}, id_prefix:{}, duration:{}, rate:{}, chained_flag:{}".format( + host_list, prefix_list, id_prefix, duration, rate, chained_flag)) + datat = string.Template(''' $ID $DURATION $RATE $CHAINED_FLAG ''') - data = dtmpl.substitute({'ID': grid, 'DURATION': duration, 'RATE': rate, 'CHAINED_FLAG': chained_flag}) - logger.info('write-transactions rpc indoked with details: {}'.format(data)) - try: - resp = requests.post(url=url, headers={'Content-Type': 'application/xml'}, - data=data, auth=('admin', 'admin'), timeout=int(duration)+60) - except Exception as exc: - resp = exc - logger.debug(exc) - rqueue.put(resp) - - logger.info("Input parameters: host_list:{}, id_prefix:{}, duration:{}, rate:{}, chained_flag:{}".format( - host_list, id_prefix, duration, rate, chained_flag)) - resqueue = _globals.pop('result_queue', Queue.Queue()) - lthreads = _globals.pop('threads', []) - for i, host in enumerate(host_list): - url = 'http://{}:8181/restconf/operations/odl-mdsal-lowlevel-control:write-transactions'.format(host) - t = threading.Thread(target=_write_transactions, - args=(resqueue, url, '{}{}'.format(id_prefix, i), duration, rate, chained_flag)) - t.daemon = True - t.start() - lthreads.append(t) + subst_dict = {'ID_PREFIX': id_prefix, 'DURATION': duration, 'RATE': rate, 'CHAINED_FLAG': chained_flag} + urlt = string.Template('''http://$HOST:8181/restconf/operations/odl-mdsal-lowlevel-control:write-transactions''') + _initiate_rpcs(host_list, prefix_list, urlt, datat, subst_dict) - _globals.update({'threads': lthreads, 'result_queue': resqueue}) + +def start_produce_transactions_on_nodes(host_list, prefix_list, id_prefix, + duration, rate, isolated_transactions_flag=True): + """Invoke produce-transactions rpcs on given nodes. + + :param host_list: list of ip address of odl nodes + :type host_list: list of strings + :param prefix_list: list of node indexes which coresponds to the ip addresses + :type prefix_list: list + :param id_prefix: identifier prefix + :type id_prefix: string + :param duration: time in seconds + :type duration: int + :param rate: produce transactions rate in transactions per second + :type rate: int + :param isolated_transactions_flag: isolated transactions flag + :type isolated_transactions_flag: bool + """ + msg = "host_list:{}, prefix_list:{} ,id_prefix:{}, duration:{}, rate:{}, isolated_transactions:{}".format( + host_list, prefix_list, id_prefix, duration, rate, isolated_transactions_flag) + msg = "Input parameters: " + msg + logger.info(msg) + datat = string.Template(''' + $ID + $DURATION + $RATE + $ISOLATED_TRANSACTIONS +''') + subst_dict = {'ID_PREFIX': id_prefix, 'DURATION': duration, 'RATE': rate, + 'ISOLATED_TRANSACTIONS': isolated_transactions_flag} + urlt = string.Template('''http://$HOST:8181/restconf/operations/odl-mdsal-lowlevel-control:produce-transactions''') + _initiate_rpcs(host_list, prefix_list, urlt, datat, subst_dict) -def wait_for_write_transactions(): - """Blocking call, waiting for responses from all threads.""" +def wait_for_transactions(): + """Blocking call, waitig for responses from all threads.""" lthreads = _globals.pop('threads') resqueue = _globals.pop('result_queue') @@ -72,7 +142,7 @@ def wait_for_write_transactions(): return results -def get_next_write_transactions_response(): +def get_next_transactions_response(): """Get http response from write-transactions rpc if available.""" resqueue = _globals.get('result_queue') diff --git a/csit/libraries/controller/DdbCommons.robot b/csit/libraries/controller/DdbCommons.robot index 6d388066c1..af0c6d1c9f 100644 --- a/csit/libraries/controller/DdbCommons.robot +++ b/csit/libraries/controller/DdbCommons.robot @@ -6,6 +6,9 @@ Documentation DOMDataBroker testing: Common keywords ... This program and the accompanying materials are made available under the ... terms of the Eclipse Public License v1.0 which accompanies this distribution, ... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... This resource file implements various test cases templates. +... FIXME: add a link to a document (when published) where the scenarios are defined Library ${CURDIR}/../MdsalLowlevelPy.py Resource ${CURDIR}/../ClusterAdmin.robot Resource ${CURDIR}/../ClusterManagement.robot @@ -26,26 +29,46 @@ ${SIMPLE_TX} ${False} ${CHAINED_TX} ${True} ${HARD_TIMEOUT} ${60} @{TRANSACTION_FAILED} ${500} +${PREF_BASED_SHARD} id-ints +${TEST_LOG_LEVEL} info +@{TEST_LOG_COMPONENTS} org.opendaylight.controller.cluster.sharding org.opendaylight.controller.cluster.datastore *** Keywords *** Explicit_Leader_Movement_Test_Templ [Arguments] ${leader_from} ${leader_to} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE} [Documentation] Implements explicit leader movement test scenario. - ${idx_from} ${idx_to} ${idx_trans} = Get_Node_Indexes_For_The_ELM_Test ${leader_from} ${leader_to} + ${idx_from} ${idx_to} ${idx_trans} = Get_Node_Indexes_For_The_ELM_Test ${leader_from} ${leader_to} ${shard_name} + ... ${shard_type} ${ip_trans_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${idx_trans}_IP} - MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${ip_trans_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${False} + ${idx_trans_as_list} = BuiltIn.Create_List ${idx_trans} + MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${ip_trans_as_list} ${idx_trans_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${False} ClusterAdmin.Make_Leader_Local ${idx_to} ${shard_name} ${shard_type} ${new_leader} ${new_followers} = BuiltIn.Wait_Until_Keyword_Succeeds 10s 1s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name} ... ${shard_type} ${True} ${idx_from} BuiltIn.Should_Be_Equal ${idx_to} ${new_leader} - ${resp_list} = MdsalLowlevelPy.Wait_For_Write_Transactions + ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions + TemplatedRequests.Check_Status_Code @{resp_list}[0] + +Explicit_Leader_Movement_PrefBasedShard_Test_Templ + [Arguments] ${leader_from} ${leader_to} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE} + [Documentation] Implements explicit leader movement test scenario. + ${idx_from} ${idx_to} ${idx_trans} = Get_Node_Indexes_For_The_ELM_Test ${leader_from} ${leader_to} ${shard_name}!! + ... ${shard_type} + ${ip_trans_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${idx_trans}_IP} + ${idx_trans_as_list} = BuiltIn.Create_List ${idx_trans} + MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${ip_trans_as_list} ${idx_trans_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} + MdsalLowlevel.Become_Prefix_Leader ${idx_to} ${shard_name} ${ID_PREFIX} + ${new_leader} ${new_followers} = BuiltIn.Wait_Until_Keyword_Succeeds 10s 1s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!! + ... ${shard_type} ${True} ${idx_from} + BuiltIn.Should_Be_Equal ${idx_to} ${new_leader} + ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions TemplatedRequests.Check_Status_Code @{resp_list}[0] Get_Node_Indexes_For_The_ELM_Test - [Arguments] ${leader_from} ${leader_to} + [Arguments] ${leader_from} ${leader_to} ${shard_name} ${shard_type} [Documentation] Return indexes for explicit leader movement test case, indexes of present to next leader node and index where transaction ... producer should be deployed. - ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${SHARD_NAME} shard_type=${SHARD_TYPE} + ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} ${idx_from} = BuiltIn.Set_Variable ${leader} ${idx_to} = BuiltIn.Set_Variable @{follower_list}[0] ${idx_trans} = BuiltIn.Set_Variable_If "${leader_from}" == "remote" and "${leader_to}" == "remote" @{follower_list}[1] "${leader_from}" == "local" ${leader} "${leader_to}" == "local" @@ -56,60 +79,114 @@ Clean_Leader_Shutdown_Test_Templ [Arguments] ${leader_location} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE} [Documentation] Implements clean leader shutdown test scenario. ${removed} = BuiltIn.Set_Variable ${False} - ${producer_idx} ${actual_leader} = Get_Node_Indexes_For_Clean_Leader_Shutdown_Test ${leader_location} + ${producer_idx} ${actual_leader} ${follower_list} = Get_Node_Indexes_For_Clean_Leader_Shutdown_Test ${leader_location} ${shard_name} ${shard_type} ${producer_ip_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${producer_idx}_IP} - MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${producer_ip_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${False} + ${producer_idx_as_list} = BuiltIn.Create_List ${producer_idx} + MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${producer_ip_as_list} ${producer_idx_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${False} ClusterAdmin.Remove_Shard_Replica ${actual_leader} ${shard_name} member-${actual_leader} ${shard_type} ${removed} = BuiltIn.Set_Variable ${True} - ${resp_list} = MdsalLowlevelPy.Wait_For_Write_Transactions + ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions TemplatedRequests.Check_Status_Code @{resp_list}[0] [Teardown] ClusterAdmin.Add_Shard_Replica ${actual_leader} ${shard_name} ${shard_type} +Clean_Leader_Shutdown_PrefBasedShard_Test_Templ + [Arguments] ${leader_location} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE} + [Documentation] Implements clean leader shutdown test scenario. + ${producer_idx} ${actual_leader} ${follower_list} = Get_Node_Indexes_For_Clean_Leader_Shutdown_Test ${leader_location} ${shard_name}!! ${shard_type} + ${producer_ip_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${producer_idx}_IP} + ${producer_idx_as_list} = BuiltIn.Create_List ${producer_idx} + MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${producer_ip_as_list} ${producer_idx_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} + ClusterAdmin.Remove_Prefix_Shard_Replica ${actual_leader} ${shard_name} member-${actual_leader} ${shard_type} + BuiltIn.Wait_Until_Keyword_Succeeds 15s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${follower_list} + BuiltIn.Run_Keyword_And_Ignore_Error ClusterManagement.Get_Raft_State_Of_Shard_At_Member shard_name=${shard_name}!! shard_type=${shard_type} member_index=${actual_leader} + WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 10s 2s ClusterManagement.Get_Raft_State_Of_Shard_At_Member shard_name=${shard_name}!! shard_type=${shard_type} member_index=${actual_leader} + ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions + TemplatedRequests.Check_Status_Code @{resp_list}[0] + [Teardown] ClusterAdmin.Add_Prefix_Shard_Replica ${actual_leader} ${shard_name} ${shard_type} + Get_Node_Indexes_For_Clean_Leader_Shutdown_Test - [Arguments] ${leader_location} + [Arguments] ${leader_location} ${shard_name} ${shard_type} [Documentation] Return indexes for clean leader shudown test case, index where transaction producer shoudl be deployed and a shard leader index. - ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${SHARD_NAME} shard_type=${SHARD_TYPE} + ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} ${follower_list_leangth} = BuiltIn.Evaluate ${NUM_ODL_SYSTEM}-1 BuiltIn.Length_Should_Be ${follower_list} ${follower_list_leangth} ${producer_idx} = BuiltIn.Set_Variable_If "${leader_location}" == "local" ${leader} @{follower_list}[0] - BuiltIn.Return_From_Keyword ${producer_idx} ${leader} + BuiltIn.Return_From_Keyword ${producer_idx} ${leader} ${follower_list} Leader_Isolation_Test_Templ - [Arguments] ${heal_timeout} + [Arguments] ${heal_timeout} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE} [Documentation] Implements leader isolation test scenario. ${producing_transactions_time} = BuiltIn.Set_Variable ${${heal_timeout}+60} ${all_indices} = ClusterManagement.List_All_Indices - ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${SHARD_NAME} shard_type=${SHARD_TYPE} member_index_list=${all_indices} + ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices} ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices} - MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${all_ip_list} ${ID_PREFIX} ${producing_transactions_time} ${TRANSACTION_RATE_1K} chained_flag=${SIMPLE_TX} + MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${producing_transactions_time} ${TRANSACTION_RATE_1K} chained_flag=${SIMPLE_TX} ${date_start} = DateTime.Get_Current_Date ${date_end} = DateTime.Add_Time_To_Date ${date_start} ${producing_transactions_time} ClusterManagement.Isolate_Member_From_List_Or_All ${leader} - BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s ClusterManagement.Verify_Shard_Leader_Elected ${SHARD_NAME} ${SHARD_TYPE} ${True} + BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name} ${shard_type} ${True} ... ${leader} member_index_list=${follower_list} BuiltIn.Sleep ${heal_timeout} ClusterManagement.Rejoin_Member_From_List_Or_All ${leader} - BuiltIn.Wait_Until_Keyword_Succeeds 20s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard ${SHARD_NAME} ${SHARD_TYPE} + BuiltIn.Wait_Until_Keyword_Succeeds 20s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard ${shard_name} ${shard_type} ${time_to_finish} = Get_Seconds_To_Time ${date_end} BuiltIn.Run_Keyword_If ${heal_timeout} < ${TRANSACTION_TIMEOUT} Leader_Isolation_Heal_Within_Tt - ... ELSE Leader_Isolation_Heal_Default ${time_to_finish} + ... ELSE Module_Leader_Isolation_Heal_Default ${leader} ${time_to_finish} + +Leader_Isolation_PrefBasedShard_Test_Templ + [Arguments] ${heal_timeout} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE} + [Documentation] Implements leader isolation test scenario. + ${producing_transactions_time} = BuiltIn.Set_Variable ${${heal_timeout}+60} + ${all_indices} = ClusterManagement.List_All_Indices + ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices} + ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices} + MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${producing_transactions_time} ${TRANSACTION_RATE_1K} + ${date_start} = DateTime.Get_Current_Date + ${date_end} = DateTime.Add_Time_To_Date ${date_start} ${producing_transactions_time} + ClusterManagement.Isolate_Member_From_List_Or_All ${leader} + BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!! ${shard_type} ${True} + ... ${leader} member_index_list=${follower_list} + BuiltIn.Sleep ${heal_timeout} + ClusterManagement.Rejoin_Member_From_List_Or_All ${leader} + BuiltIn.Wait_Until_Keyword_Succeeds 20s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard ${shard_name}!! ${shard_type} + ${time_to_finish} = Get_Seconds_To_Time ${date_end} + BuiltIn.Run_Keyword_If ${heal_timeout} < ${TRANSACTION_TIMEOUT} Leader_Isolation_Heal_Within_Tt + ... ELSE Prefix_Leader_Isolation_Heal_Default ${leader} ${time_to_finish} Leader_Isolation_Heal_Within_Tt [Documentation] The leader isolation test case end if the heal happens within transaction timeout. All write transaction ... producers shoudl finish without error. - ${resp_list} = MdsalLowlevelPy.Wait_For_Write_Transactions + ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions : FOR ${resp} IN @{resp_list} \ TemplatedRequests.Check_Status_Code ${resp} -Leader_Isolation_Heal_Default +Module_Leader_Isolation_Heal_Default [Arguments] ${isolated_node} ${time_to_finish} [Documentation] The leader isolation test case end. The transaction producer on isolated node shoudl fail and should be restarted. Then all write transaction producers shoudl finish without error. - ${resp} = MdsalLowlevelPy.Get_Next_Write_Transactions_Response + ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response BuiltIn.Log ${resp} - ${restart_producer_node} BuiltIn.Create_List ${isolated_node} - MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${restart_producer_node} ${ID_PREFIX} ${time_to_finish} ${TRANSACTION_RATE_1K} chained_flag=${SIMPLE_TX} - ${resp_list} = MdsalLowlevelPy.Wait_For_Write_Transactions + # TODO: check on response status code + ${restart_producer_node_idx_as_list} BuiltIn.Create_List ${isolated_node} + ${restart_producer_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${isolated_node} + ${restart_producer_node_ip_as_list} BuiltIn.Create_List ${restart_producer_node_ip} + MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${restart_producer_node_ip_as_list} ${restart_producer_node_idx_as_list} ${ID_PREFIX} ${time_to_finish} ${TRANSACTION_RATE_1K} chained_flag=${SIMPLE_TX} + ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions + : FOR ${resp} IN @{resp_list} + \ TemplatedRequests.Check_Status_Code ${resp} + +Prefix_Leader_Isolation_Heal_Default + [Arguments] ${isolated_node} ${time_to_finish} + [Documentation] The leader isolation test case end. The transaction producer on isolated node shoudl fail and should be restarted. + Then all write transaction producers shoudl finish without error. + ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response + BuiltIn.Log ${resp} + # TODO: check on response status code + ${restart_producer_node_idx_as_list} BuiltIn.Create_List ${isolated_node} + ${restart_producer_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${isolated_node} + ${restart_producer_node_ip_as_list} BuiltIn.Create_List ${restart_producer_node_ip} + MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${restart_producer_node_ip_as_list} ${restart_producer_node_idx_as_list} ${ID_PREFIX} ${time_to_finish} ${TRANSACTION_RATE_1K} + ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions : FOR ${resp} IN @{resp_list} \ TemplatedRequests.Check_Status_Code ${resp} @@ -122,27 +199,51 @@ Client_Isolation_Test_Templ ${client_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1} ${client_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${client_node_dst} ${client_node_ip_as_list} BuiltIn.Create_List ${client_node_ip} - MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${client_node_ip_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${trans_chain_flag} + ${client_node_idx_as_list} BuiltIn.Create_List ${client_node_dst} + MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${client_node_ip_as_list} ${client_node_idx_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${trans_chain_flag} ${start_date} DateTime.Get_Current_Date ${timeout_date} = DateTime.Add_Time_To_Date ${start_date} ${TRANSACTION_TIMEOUT} ${abort_date} = DateTime.Add_Time_To_Date ${start_date} ${HARD_TIMEOUT} ClusterManagement.Isolate_Member_From_List_Or_All ${client_node_dst} - WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${TRANSACTION_TIMEOUT} 1s Write_Transactions_Not_Failed_Yet - WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 3s 1s Write_Transactions_Failed + WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${TRANSACTION_TIMEOUT} 1s Ongoing_Transactions_Not_Failed_Yet + WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 3s 1s Ongoing_Transactions_Failed ${abort_time} Get_Seconds_To_Time ${abort_date} WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${abort_time} 1s Verify_Client_Aborted ${False} WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 3s 1s Verify_Client_Aborted ${True} [Teardown] BuiltIn.Run Keywords ClusterManagement.Rejoin_Member_From_List_Or_All ${client_node_dst} - ... AND MdsalLowlevelPy.Wait_For_Write_Transactions + ... AND MdsalLowlevelPy.Wait_For_Transactions -Write_Transactions_Not_Failed_Yet +Client_Isolation_PrefBasedShard_Test_Templ + [Arguments] ${listener_node_role} ${trans_chain_flag} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE} + [Documentation] Implements client isolation test scenario. + ${all_indices} = ClusterManagement.List_All_Indices + ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices} + ${follower1} = Collections.Get_From_List ${follower_list} ${0} + ${client_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1} + ${client_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${client_node_dst} + ${client_node_ip_as_list} BuiltIn.Create_List ${client_node_ip} + ${client_node_idx_as_list} BuiltIn.Create_List ${client_node_dst} + MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${client_node_ip_as_list} ${client_node_idx_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} + ${start_date} DateTime.Get_Current_Date + ${timeout_date} = DateTime.Add_Time_To_Date ${start_date} ${TRANSACTION_TIMEOUT} + ${abort_date} = DateTime.Add_Time_To_Date ${start_date} ${HARD_TIMEOUT} + ClusterManagement.Isolate_Member_From_List_Or_All ${client_node_dst} + WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${TRANSACTION_TIMEOUT} 1s Ongoing_Transactions_Not_Failed_Yet + WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 3s 1s Ongoing_Transactions_Failed + ${abort_time} Get_Seconds_To_Time ${abort_date} + WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${abort_time} 1s Verify_Client_Aborted ${False} + WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 3s 1s Verify_Client_Aborted ${True} + [Teardown] BuiltIn.Run Keywords ClusterManagement.Rejoin_Member_From_List_Or_All ${client_node_dst} + ... AND MdsalLowlevelPy.Wait_For_Transactions + +Ongoing_Transactions_Not_Failed_Yet [Documentation] Verify that no write-transaction rpc finished, means they are still running. - ${resp} = MdsalLowlevelPy.Get_Next_Write_Transactions_Response + ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response BuiltIn.Should_Be_Equal ${None} ${resp} ${resp} not expected. -Write_Transactions_Failed +Ongoing_Transactions_Failed [Documentation] Verify if write-transaction failed. - ${resp} = MdsalLowlevelPy.Get_Next_Write_Transactions_Response + ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response Check_Status_Code ${resp} explicit_status_codes=${TRANSACTION_FAILED} Verify_Client_Aborted @@ -167,9 +268,53 @@ Remote_Listener_Test_Templ ${listener_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1} MdsalLowlevel.Subscribe_Dtcl ${listener_node_dst} ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices} - MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${all_ip_list} ${ID_PREFIX} ${DURATION_10S} ${TRANSACTION_RATE_1K} chained_flag=${SIMPLE_TX} + MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${DURATION_10S} ${TRANSACTION_RATE_1K} chained_flag=${SIMPLE_TX} + ClusterAdmin.Make_Leader_Local ${follower1} ${shard_name} ${shard_type} + ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions + : FOR ${resp} IN @{resp_list} + \ TemplatedRequests.Check_Status_Code ${resp} + [Teardown] MdsalLowlevel.Unsubscribe_Dtcl ${listener_node_dst} + +Remote_Listener_PrefBasedShard_Test_Templ + [Arguments] ${listener_node_role} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE} + [Documentation] Implements listener isolation test scenario. + ${all_indices} = ClusterManagement.List_All_Indices + ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices} + ${follower1} = Collections.Get_From_List ${follower_list} ${0} + ${follower2} = Collections.Get_From_List ${follower_list} ${1} + ${listener_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1} + MdsalLowlevel.Subscribe_Dtcl ${listener_node_dst} + ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices} + MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${DURATION_10S} ${TRANSACTION_RATE_1K} ClusterAdmin.Make_Leader_Local ${follower1} ${shard_name} ${shard_type} - ${resp_list} = MdsalLowlevelPy.Wait_For_Write_Transactions + ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions : FOR ${resp} IN @{resp_list} \ TemplatedRequests.Check_Status_Code ${resp} [Teardown] MdsalLowlevel.Unsubscribe_Dtcl ${listener_node_dst} + +Create_Prefix_Based_Shard_And_Verify + [Arguments] ${prefix}=${PREF_BASED_SHARD} + [Documentation] Create prefix based shard with replicas on all nodes + ${all_indices} = ClusterManagement.List_All_Indices + ${node_to_trigger} = Collections.Get_From_List ${all_indices} ${0} + MdsalLowlevel.Create_Prefix_Shard ${node_to_trigger} ${prefix} ${all_indices} + BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${prefix}!! shard_type=${SHARD_TYPE} member_index_list=${all_indices} + +Remove_Prefix_Based_Shard_And_Verify + [Arguments] ${prefix}=${PREF_BASED_SHARD} + [Documentation] Remove prefix based shard with all replicas + ${all_indices} = ClusterManagement.List_All_Indices + ${node_to_trigger} = Collections.Get_From_List ${all_indices} ${0} + MdsalLowlevel.Remove_Prefix_Shard ${node_to_trigger} ${prefix} + : FOR ${idx} IN @{all_indices} + \ BuiltIn.Wait_Until_Keyword_Succeeds 15s 2s Verify_Shard_Replica_Removed ${idx} ${prefix}!! + \ ... ${SHARD_TYPE} + +Verify_Shard_Replica_Removed + [Arguments] ${member_index} ${shard_name} ${shard_type} + [Documentation] Verify that shard is removed. Jolokia return 404 for shard memeber. + ${session} = Resolve_Http_Session_For_Member member_index=${member_index} + ${type_class} = Resolve_Shard_Type_Class shard_type=${shard_type} + ${uri} = BuiltIn.Set_Variable /jolokia/read/org.opendaylight.controller:Category=Shards,name=member-${member_index}-shard-${shard_name}-${shard_type},type=${type_class} + ${text} TemplatedRequests.Get_From_Uri uri=${uri} session=${session} + BuiltIn.Should_Contain ${text} "status":404 javax.management.InstanceNotFoundException diff --git a/csit/suites/controller/dom_data_broker/clean_leader_shutdown_prefbasedshard.robot b/csit/suites/controller/dom_data_broker/clean_leader_shutdown_prefbasedshard.robot new file mode 100644 index 0000000000..2f24faf732 --- /dev/null +++ b/csit/suites/controller/dom_data_broker/clean_leader_shutdown_prefbasedshard.robot @@ -0,0 +1,33 @@ +*** Settings *** +Documentation DOMDataBroker testing: Clean Leader Shutdown +... +... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... The goal is to ensure that applications do not observe disruption when a shard +... leader is shut down cleanly. This is performed by having a steady-stream +... producer execute operations against the shard and then initiate leader shard +... shutdown, then the producer is shut down cleanly. +Suite Setup BuiltIn.Run_Keywords SetupUtils.Setup_Utils_For_Setup_And_Teardown +... AND DdbCommons.Create_Prefix_Based_Shard +Suite Teardown BuiltIn.Run_Keywords DdbCommons.Remove_Prefix_Based_Shard +... AND SSHLibrary.Close_All_Connections +Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing +Test Teardown SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed +Default Tags critical +Test Template DdbCommons.Clean_Leader_Shutdown_PrefBasedShard_Test_Templ +Library SSHLibrary +Resource ${CURDIR}/../../../libraries/controller/DdbCommons.robot +Resource ${CURDIR}/../../../libraries/SetupUtils.robot + +*** Test Cases *** +Local_Leader_Shutdown + [Documentation] Shutdown the leader on the same node as transaction producer. + local + +Remote_Leader_Shutdown + [Documentation] Shutdown the leader on different node as transaction producer. + remote diff --git a/csit/suites/controller/dom_data_broker/client_isolation_prefbasedshard.robot b/csit/suites/controller/dom_data_broker/client_isolation_prefbasedshard.robot new file mode 100644 index 0000000000..1250107b40 --- /dev/null +++ b/csit/suites/controller/dom_data_broker/client_isolation_prefbasedshard.robot @@ -0,0 +1,41 @@ +*** Settings *** +Documentation DOMDataBroker testing: Client Isolation +... +... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... The purpose of this test is to ascertain that the failure modes of +... cds-access-client work as expected. This is performed by having a steady +... stream of transactions flowing from the frontend and isolating the node hosting +... the frontend from the rest of the cluster. +Suite Setup BuiltIn.Run_Keywords SetupUtils.Setup_Utils_For_Setup_And_Teardown +... AND DdbCommons.Create_Prefix_Based_Shard +Suite Teardown BuiltIn.Run_Keywords DdbCommons.Remove_Prefix_Based_Shard +... AND SSHLibrary.Close_All_Connections +Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing +Test Teardown SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed +Default Tags critical +Test Template DdbCommons.Client_Isolation_PrefBasedShard_Test_Templ +Library SSHLibrary +Resource ${CURDIR}/../../../libraries/controller/DdbCommons.robot +Resource ${CURDIR}/../../../libraries/SetupUtils.robot + +*** Test Cases *** +Producer_On_Shard_Leader_Node_ChainedTx + [Documentation] Client isolation with producer on shard leader with chained transactions. + leader ${CHAINED_TX} + +Producer_On_Shard_Leader_Node_SimpleTx + [Documentation] Client isolation with producer on shard leader with simple transactions. + leader ${SIMPLE_TX} + +Producer_On_Shard_Non_Leader_Node_ChainedTx + [Documentation] Client isolation with producer on shard non-leader with chained transactions. + non-leader ${CHAINED_TX} + +Producer_On_Shard_Non_Leader_Node_SimpleTx + [Documentation] Client isolation with producer on shard non-leader with simple transactions. + non-leader ${SIMPLE_TX} diff --git a/csit/suites/controller/dom_data_broker/ddb-sanity-module-based.robot b/csit/suites/controller/dom_data_broker/ddb-sanity-module-based.robot new file mode 100644 index 0000000000..fca0943d2e --- /dev/null +++ b/csit/suites/controller/dom_data_broker/ddb-sanity-module-based.robot @@ -0,0 +1,84 @@ +*** Settings *** +Documentation DOMDataBroker testing: Module based shards sanity suite +... +... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... The goal is to call several basic rpc form ClusterAdmin.robot and +... MdsalLowlevel.robot to ensute that those rpcs can be safely used in +... other suites. +... It also verify the ability of the odl-controller-test-app to perform +... several activities. +Suite Setup SetupUtils.Setup_Utils_For_Setup_And_Teardown +Suite Teardown SSHLibrary.Close_All_Connections +Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing +Test Teardown SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed +Default Tags critical +Library SSHLibrary +Library ${CURDIR}/../../../libraries/MdsalLowlevelPy.py +Resource ${CURDIR}/../../../libraries/ClusterAdmin.robot +Resource ${CURDIR}/../../../libraries/ClusterManagement.robot +Resource ${CURDIR}/../../../libraries/controller/DdbCommons.robot +Resource ${CURDIR}/../../../libraries/SetupUtils.robot + +*** Variables *** +${SHARD_NAME} default +${SHARD_TYPE} config +${TRANSACTION_RATE_1K} ${1000} +${DURATION_10S} ${10} +${SIMPLE_TX} ${False} +${CHAINED_TX} ${True} +${MODULE_SHARD_PREFIX} member- + +*** Test Cases *** +Make_Leader_Local + [Documentation] Make the loeader local and verify. + ${shard_name} = BuiltIn.Set_Variable ${SHARD_NAME} + ${shard_type} = BuiltIn.Set_Variable ${SHARD_TYPE} + ${all_indices} = ClusterManagement.List_All_Indices + ${old_leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${SHARD_NAME} shard_type=${SHARD_TYPE} member_index_list=${all_indices} + ${follower1} = Collections.Get_From_List ${follower_list} ${0} + ClusterAdmin.Make_Leader_Local ${follower1} ${shard_name} ${shard_type} + ${leader} ${follower_list} = BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name} + ... ${shard_type} ${True} ${old_leader} member_index_list=${EMPTY} + BuiltIn.Should_Be_Equal_As_Numbers ${follower1} ${leader} + +Remove_Leader_Shard_Replica_And_Add_It_Back + [Documentation] Remove and add shard replica adn verify it. + ${shard_name} = BuiltIn.Set_Variable ${SHARD_NAME} + ${shard_type} = BuiltIn.Set_Variable ${SHARD_TYPE} + ${all_indices} = ClusterManagement.List_All_Indices + ${old_leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices} + ClusterAdmin.Remove_Shard_Replica ${old_leader} ${shard_name} member-${old_leader} ${shard_type} + BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s DdbCommons.Verify_Shard_Replica_Removed ${old_leader} ${shard_name} ${shard_type} + ${actual_leader} ${actual_follower_list} = BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} + ... shard_type=${shard_type} member_index_list=${follower_list} + BuiltIn.Should_Not_Be_Equal_As_Numbers ${old_leader} ${actual_leader} + ClusterAdmin.Add_Shard_Replica ${old_leader} ${shard_name} ${shard_type} + BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices} + +Remove_Follower_Shard_Replica_And_Add_It_Back + [Documentation] Remove and add shard replica adn verify it. + ${shard_name} = BuiltIn.Set_Variable ${SHARD_NAME} + ${shard_type} = BuiltIn.Set_Variable ${SHARD_TYPE} + ${all_indices} = ClusterManagement.List_All_Indices + ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices} + ${follower1} = Collections.Get_From_List ${follower_list} ${0} + ClusterAdmin.Remove_Shard_Replica ${follower1} ${shard_name} member-${follower1} ${shard_type} + BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s DdbCommons.Verify_Shard_Replica_Removed ${follower1} ${shard_name} ${shard_type} + ${new_indices_list} = ClusterManagement.List_Indices_Minus_Member ${follower1} + ClusterManagement.Verify_Shard_Leader_Elected ${shard_name} ${shard_type} ${False} ${leader} member_index_list=${new_indices_list} + ClusterAdmin.Add_Shard_Replica ${follower1} ${shard_name} ${shard_type} + BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices} + +Write_Transactions + [Documentation] Write transactions. + ${all_indices} = ClusterManagement.List_All_Indices + ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices} + MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${MODULE_SHARD_PREFIX} ${DURATION_10S} ${TRANSACTION_RATE_1K} chained_flag=${SIMPLE_TX} + ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions + : FOR ${resp} IN @{resp_list} + \ TemplatedRequests.Check_Status_Code ${resp} diff --git a/csit/suites/controller/dom_data_broker/ddb-sanity-prefix-based.robot b/csit/suites/controller/dom_data_broker/ddb-sanity-prefix-based.robot new file mode 100644 index 0000000000..5472abf425 --- /dev/null +++ b/csit/suites/controller/dom_data_broker/ddb-sanity-prefix-based.robot @@ -0,0 +1,86 @@ +*** Settings *** +Documentation DOMDataBroker testing: Module based shards sanity suite +... +... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... The goal is to call several basic rpc form ClusterAdmin.robot and +... MdsalLowlevel.robot to ensute that those rpcs can be safely used in +... other suites. +... It also verify the ability of the odl-controller-test-app to perform +... several activities. +Suite Setup BuiltIn.Run_Keywords SetupUtils.Setup_Utils_For_Setup_And_Teardown +... AND DdbCommons.Create_Prefix_Based_Shard_And_Verify +Suite Teardown BuiltIn.Run_Keywords DdbCommons.Remove_Prefix_Based_Shard_And_Verify +... AND SSHLibrary.Close_All_Connections +Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing +Test Teardown SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed +Default Tags critical +Library SSHLibrary +Library ${CURDIR}/../../../libraries/MdsalLowlevelPy.py +Resource ${CURDIR}/../../../libraries/ClusterAdmin.robot +Resource ${CURDIR}/../../../libraries/ClusterManagement.robot +Resource ${CURDIR}/../../../libraries/controller/DdbCommons.robot +Resource ${CURDIR}/../../../libraries/SetupUtils.robot + +*** Variables *** +${PREF_BASED_SHARD} id-ints +${SHARD_TYPE} config +${TRANSACTION_RATE_1K} ${1000} +${DURATION_10S} ${10} +${SIMPLE_TX} ${False} +${CHAINED_TX} ${True} +${SHARD_PREFIX} member- +${ID_PREFIX} prefix- + +*** Test Cases *** +Produce_Transactions + [Documentation] Write transactions. + ${all_indices} = ClusterManagement.List_All_Indices + ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices} + MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${DURATION_10S} ${TRANSACTION_RATE_1K} + ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions + : FOR ${resp} IN @{resp_list} + \ TemplatedRequests.Check_Status_Code ${resp} + +Become_Prefix_Leader + [Documentation] Make the loeader local and verify. + ${shard_name} = BuiltIn.Set_Variable ${PREF_BASED_SHARD} + ${shard_type} = BuiltIn.Set_Variable ${SHARD_TYPE} + ${all_indices} = ClusterManagement.List_All_Indices + ${old_leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices} + ${follower1} = Collections.Get_From_List ${follower_list} ${0} + MdsalLowlevel.Become_Prefix_Leader ${follower1} ${shard_name} ${ID_PREFIX} + ${leader} ${follower_list} = BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!! + ... ${shard_type} ${True} ${old_leader} member_index_list=${EMPTY} + BuiltIn.Should_Be_Equal_As_Numbers ${follower1} ${leader} + +Remove_Leader_Prefix_Shard_Replica_And_Add_It_Back + [Documentation] Remove and add shard replica adn verify it. + ${shard_name} = BuiltIn.Set_Variable ${PREF_BASED_SHARD} + ${shard_type} = BuiltIn.Set_Variable ${SHARD_TYPE} + ${all_indices} = ClusterManagement.List_All_Indices + ${old_leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices} + ClusterAdmin.Remove_Prefix_Shard_Replica ${old_leader} ${shard_name} member-${old_leader} ${shard_type} + BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s Verify_Shard_Replica_Removed ${old_leader} ${shard_name}!! ${shard_type} + ${actual_leader} ${actual_follower_list} = BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! + ... shard_type=${shard_type} member_index_list=${follower_list} + ClusterAdmin.Add_Prefix_Shard_Replica ${old_leader} ${shard_name} ${shard_type} + BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices} + +Remove_Follower_Prefix_Shard_Replica_And_Add_It_Back + [Documentation] Remove and add shard replica adn verify it. + ${shard_name} = BuiltIn.Set_Variable ${PREF_BASED_SHARD} + ${shard_type} = BuiltIn.Set_Variable ${SHARD_TYPE} + ${all_indices} = ClusterManagement.List_All_Indices + ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices} + ${follower1} = Collections.Get_From_List ${follower_list} ${0} + ClusterAdmin.Remove_Prefix_Shard_Replica ${follower1} ${shard_name} member-${follower1} ${shard_type} + BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s DdbCommons.Verify_Shard_Replica_Removed ${follower1} ${shard_name}!! ${shard_type} + ${new_indices_list} = ClusterManagement.List_Indices_Minus_Member ${follower1} + ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!! ${shard_type} ${False} ${leader} member_index_list=${new_indices_list} + ClusterAdmin.Add_Prefix_Shard_Replica ${follower1} ${shard_name} ${shard_type} + BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices} diff --git a/csit/suites/controller/dom_data_broker/explicit_leader_movement_prefbasedshard.robot b/csit/suites/controller/dom_data_broker/explicit_leader_movement_prefbasedshard.robot new file mode 100644 index 0000000000..7d0b097db2 --- /dev/null +++ b/csit/suites/controller/dom_data_broker/explicit_leader_movement_prefbasedshard.robot @@ -0,0 +1,37 @@ +*** Settings *** +Documentation DOMDataBroker testing: Explicit Leader Movement +... +... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... The goal is to ensure that applications do not observe disruption when a shard +... leader is moved as the result of explicit application request. This is performed +... by having a steady-stream producer execute operations against the shard and then +... initiate shard leader shutdown, then the producer is shut down cleanly. +Suite Setup BuiltIn.Run_Keywords SetupUtils.Setup_Utils_For_Setup_And_Teardown +... AND DdbCommons.Create_Prefix_Based_Shard +Suite Teardown BuiltIn.Run_Keywords DdbCommons.Remove_Prefix_Based_Shard +... AND SSHLibrary.Close_All_Connections +Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing +Test Teardown SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed +Default Tags critical +Test Template DdbCommons.Explicit_Leader_Movement_PrefBasedShard_Test_Templ +Library SSHLibrary +Resource ${CURDIR}/../../../libraries/controller/DdbCommons.robot +Resource ${CURDIR}/../../../libraries/SetupUtils.robot + +*** Test Cases *** +Local_To_Remote_Movement + [Documentation] Leader moves from local to remote node during transaction producing. + local remote + +Remote_To_Remote_Movement + [Documentation] Leader moves from one remote to other remote node during transaction producing. + remote remote + +Remote_To_Local_Movement + [Documentation] Leader moves from remote to local node during transaction producing. + remote local diff --git a/csit/suites/controller/dom_data_broker/leader_isolation_prefbasedshard.robot b/csit/suites/controller/dom_data_broker/leader_isolation_prefbasedshard.robot new file mode 100644 index 0000000000..ed0ab26d89 --- /dev/null +++ b/csit/suites/controller/dom_data_broker/leader_isolation_prefbasedshard.robot @@ -0,0 +1,32 @@ +*** Settings *** +Documentation DOMDataBroker testing: Leader Isolation +... +... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... The goal is to ensure the datastore succeeds in basic isolation/rejoin scenario, +... simulating either a network partition, or a prolonged GC pause. +Suite Setup BuiltIn.Run_Keywords SetupUtils.Setup_Utils_For_Setup_And_Teardown +... AND DdbCommons.Create_Prefix_Based_Shard +Suite Teardown BuiltIn.Run_Keywords DdbCommons.Remove_Prefix_Based_Shard +... AND SSHLibrary.Close_All_Connections +Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing +Test Teardown SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed +Default Tags critical +Test Template DdbCommons.Leader_Isolation_PrefBasedShard_Test_Templ +Library SSHLibrary +Resource ${CURDIR}/../../../libraries/controller/DdbCommons.robot +Resource ${CURDIR}/../../../libraries/SetupUtils.robot + +*** Test Cases *** +Healing_Within_Transaction_Timeout + [Documentation] The isolated node (leader) is rejoined as soon as new leader is elected and + ... and within transaction timeout. + ${0} + +Healing_After_2x_Transaction_Timeout + [Documentation] The isolated node (leader) is rejoined after 2x transaction timeout. + ${${TRANSACTION_TIMEOUT_2X}+1} diff --git a/csit/suites/controller/dom_data_broker/remote_listener_prefbasedshard.robot b/csit/suites/controller/dom_data_broker/remote_listener_prefbasedshard.robot new file mode 100644 index 0000000000..cf7ccd2e49 --- /dev/null +++ b/csit/suites/controller/dom_data_broker/remote_listener_prefbasedshard.robot @@ -0,0 +1,32 @@ +*** Settings *** +Documentation DOMDataBroker testing: Remote Listener +... +... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... The goal is to ensure listeners do no observe disruption when the leader moves. +... This is performed by having a steady stream of transactions being observed by +... the listeners and having the leader move. +Suite Setup BuiltIn.Run_Keywords SetupUtils.Setup_Utils_For_Setup_And_Teardown +... AND DdbCommons.Create_Prefix_Based_Shard +Suite Teardown BuiltIn.Run_Keywords DdbCommons.Remove_Prefix_Based_Shard +... AND SSHLibrary.Close_All_Connections +Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing +Test Teardown SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed +Default Tags critical +Test Template DdbCommons.Remote_Listener_PrefBasedShard_Test_Templ +Library SSHLibrary +Resource ${CURDIR}/../../../libraries/controller/DdbCommons.robot +Resource ${CURDIR}/../../../libraries/SetupUtils.robot + +*** Test Cases *** +Listener_On_Shard_Leader_Node + [Documentation] Listener runs on leader node when leader changed. + leader + +Listener_On_Shard_Non_Leader_Node + [Documentation] Listener runs on non-leader node when leader changed. + non-leader diff --git a/csit/testplans/controller-clustering.txt b/csit/testplans/controller-clustering.txt index 768de885ec..cfa74a8929 100644 --- a/csit/testplans/controller-clustering.txt +++ b/csit/testplans/controller-clustering.txt @@ -13,6 +13,8 @@ integration/test/csit/suites/controller/dom_data_broker/leader_isolation.robot integration/test/csit/suites/controller/dom_data_broker/client_isolation.robot integration/test/csit/suites/controller/dom_data_broker/remote_listener.robot integration/test/csit/suites/controller/dom_data_broker/restart_odl_with_tell_based_true.robot +integration/test/csit/suites/controller/dom_data_broker/ddb-sanity-prefix-based.robot +integration/test/csit/suites/controller/dom_data_broker/ddb-sanity-module-based.robot integration/test/csit/suites/controller/Clustering_Datastore/cluster_ready.robot integration/test/csit/suites/controller/cluster_singleton/master_stability.robot integration/test/csit/suites/controller/cluster_singleton/partition_and_heal.robot @@ -22,6 +24,11 @@ integration/test/csit/suites/controller/dom_data_broker/explicit_leader_movement integration/test/csit/suites/controller/dom_data_broker/leader_isolation.robot integration/test/csit/suites/controller/dom_data_broker/client_isolation.robot integration/test/csit/suites/controller/dom_data_broker/remote_listener.robot +integration/test/csit/suites/controller/dom_data_broker/leader_isolation_prefbasedshard.robot +integration/test/csit/suites/controller/dom_data_broker/explicit_leader_movement_prefbasedshard.robot +integration/test/csit/suites/controller/dom_data_broker/remote_listener_prefbasedshard.robot +integration/test/csit/suites/controller/dom_data_broker/client_isolation_prefbasedshard.robot +integration/test/csit/suites/controller/dom_data_broker/clean_leader_shutdown_prefbasedshard.robot integration/test/csit/suites/controller/dom_data_broker/restart_odl_with_tell_based_false.robot integration/test/csit/suites/controller/Clustering_Datastore/cluster_ready.robot integration/test/csit/suites/controller/singleton_service/global_rpc_kill.robot diff --git a/csit/variables/mdsal/clusteradmin/add_prefix_shard_replica/post_data.xml b/csit/variables/mdsal/clusteradmin/add_prefix_shard_replica/post_data.xml index 921061f501..96852b8fbf 100644 --- a/csit/variables/mdsal/clusteradmin/add_prefix_shard_replica/post_data.xml +++ b/csit/variables/mdsal/clusteradmin/add_prefix_shard_replica/post_data.xml @@ -1,5 +1,5 @@ - - $SHARD_PREFIX + + /odl-mdsal-lowlevel-target:$SHARD_PREFIX $DATA_STORE_TYPE diff --git a/csit/variables/mdsal/clusteradmin/remove_prefix_shard_replica/post_data.xml b/csit/variables/mdsal/clusteradmin/remove_prefix_shard_replica/post_data.xml index 933fa5a624..c711c632b9 100644 --- a/csit/variables/mdsal/clusteradmin/remove_prefix_shard_replica/post_data.xml +++ b/csit/variables/mdsal/clusteradmin/remove_prefix_shard_replica/post_data.xml @@ -1,5 +1,5 @@ - - $SHARD_PREFIX + + /odl-mdsal-lowlevel-target:$SHARD_PREFIX $MEMBER_NAME $DATA_STORE_TYPE diff --git a/csit/variables/mdsal/lowlevelrpc/become_prefix_leader/post_data.xml b/csit/variables/mdsal/lowlevelrpc/become_prefix_leader/post_data.xml index f0f32ff0d1..11b7b3f231 100644 --- a/csit/variables/mdsal/lowlevelrpc/become_prefix_leader/post_data.xml +++ b/csit/variables/mdsal/lowlevelrpc/become_prefix_leader/post_data.xml @@ -1,3 +1,3 @@ - - $PREFIX + + /odl-mdsal-lowlevel-target:$SHARD_NAME/odl-mdsal-lowlevel-target:id-int[odl-mdsal-lowlevel-target:id="$ID"] diff --git a/csit/variables/mdsal/lowlevelrpc/create_prefix_shard/location.uri b/csit/variables/mdsal/lowlevelrpc/create_prefix_shard/location.uri index 43991e3ecc..4f360714ca 100644 --- a/csit/variables/mdsal/lowlevelrpc/create_prefix_shard/location.uri +++ b/csit/variables/mdsal/lowlevelrpc/create_prefix_shard/location.uri @@ -1 +1 @@ -/restconf/operations/odl-mdsal-lowlevel-control:add-prefix-shard +/restconf/operations/odl-mdsal-lowlevel-control:create-prefix-shard diff --git a/csit/variables/mdsal/lowlevelrpc/create_prefix_shard/post_data.xml b/csit/variables/mdsal/lowlevelrpc/create_prefix_shard/post_data.xml index 3a5ac96616..041f4c3eb8 100644 --- a/csit/variables/mdsal/lowlevelrpc/create_prefix_shard/post_data.xml +++ b/csit/variables/mdsal/lowlevelrpc/create_prefix_shard/post_data.xml @@ -1,4 +1,4 @@ - - $PREFIX + + /odl-mdsal-lowlevel-target:$PREFIX $REPLICAS diff --git a/csit/variables/mdsal/lowlevelrpc/remove_prefix_shard/location.uri b/csit/variables/mdsal/lowlevelrpc/remove_prefix_shard/location.uri index fb88de7c5d..65d8867222 100644 --- a/csit/variables/mdsal/lowlevelrpc/remove_prefix_shard/location.uri +++ b/csit/variables/mdsal/lowlevelrpc/remove_prefix_shard/location.uri @@ -1 +1 @@ -/restconf/operations/odl-mdsal-lowlevel-control:become-prefix-leader \ No newline at end of file +/restconf/operations/odl-mdsal-lowlevel-control:remove-prefix-shard diff --git a/csit/variables/mdsal/lowlevelrpc/remove_prefix_shard/post_data.xml b/csit/variables/mdsal/lowlevelrpc/remove_prefix_shard/post_data.xml index f0f32ff0d1..c64abb0c98 100644 --- a/csit/variables/mdsal/lowlevelrpc/remove_prefix_shard/post_data.xml +++ b/csit/variables/mdsal/lowlevelrpc/remove_prefix_shard/post_data.xml @@ -1,3 +1,3 @@ - - $PREFIX + + /odl-mdsal-lowlevel-target:$PREFIX -- 2.36.6