Create_Prefix_Shard
[Arguments] ${member_index} ${prefix} ${replicas}
- [Documentation] Create prefix based shard.
+ [Documentation] Create prefix based shard. ${replicas} is a list of cluster node indexes, taken e.g. from ClusterManagement.List_Indices_Or_All.
${session} = ClusterManagement.Resolve_Http_Session_For_Member member_index=${member_index}
${replicas_str} BuiltIn.Set_Variable ${EMPTY}
: FOR ${replica} IN @{replicas}
- \ ${replicas_str} BuiltIn.Set_Variable ${replicas_str}<replica>${replica}</replica>
+ \ ${replicas_str} BuiltIn.Set_Variable ${replicas_str}<replicas>member-${replica}</replicas>
&{mapping} BuiltIn.Create_Dictionary PREFIX=${prefix} REPLICAS=${replicas_str}
${text} = TemplatedRequests.Post_As_Xml_Templated ${CREATE_PREFIX_SHARD_DIR} mapping=${mapping} session=${session}
${text} = TemplatedRequests.Post_As_Xml_Templated ${REMOVE_PREFIX_SHARD_DIR} mapping=${mapping} session=${session}
Become_Prefix_Leader
- [Arguments] ${member_index} ${prefix}
+ [Arguments] ${member_index} ${shard_name} ${id_prefix}
[Documentation] Given node ask to become a shard leader.
${session} = ClusterManagement.Resolve_Http_Session_For_Member member_index=${member_index}
- &{mapping} BuiltIn.Create_Dictionary PREFIX=${prefix}
+ &{mapping} BuiltIn.Create_Dictionary SHARD_NAME=${shard_name} ID=${id_prefix}${member_index}
${text} = TemplatedRequests.Post_As_Xml_Templated ${BECOME_PREFIX_LEADER_DIR} mapping=${mapping} session=${session}
Subscribe_Dtcl
_globals = {}
-def start_write_transactions_on_nodes(host_list, id_prefix, duration, rate, chained_flag=True):
- """Invoke publish notification rpcs and verify the response.
+def _send_http_request_thread_impl(rqueue, url, data, http_timeout):
+ """Start either publish or write transactions rpc based on input.
+
+ :param rqueue: result queue
+ :type rqueue: Queue.Queue
+ :param url: rpc url
+ :type url: string
+ :param data: http request content
+ :type data: string
+ :param http_timeout: http response timeout
+ :type http_timeout: int
+ """
+ logger.info('rpc indoked with details: {}'.format(data))
+ try:
+ resp = requests.post(url=url, headers={'Content-Type': 'application/xml'},
+ data=data, auth=('admin', 'admin'), timeout=http_timeout)
+ except Exception as exc:
+ resp = exc
+ logger.debug(exc)
+ rqueue.put(resp)
+
+
+def _initiate_rpcs(host_list, prefix_list, url_templ, data_templ, subst_dict):
+ """Initiate rpc on given hosts.
+
+ :param host_list: list of ip address of odl nodes
+ :type host_list: list of strings
+ :param id_prefix: list of node indexes which coresponds to the ip addresses
+ :type id_prefix: string
+ :param url_templ: url template
+ :type url_templ: string.Template object
+ :param data_templ: http request data
+ :type data_templ: string.Template object
+ :param subst_dict: dictionary with key value pairs to be used with template
+ :type subst_dict: dict
+ """
+ resqueue = _globals.pop('result_queue', Queue.Queue())
+ lthreads = _globals.pop('threads', [])
+ for i, host in enumerate(host_list):
+ subst_dict.update({'ID': '{}{}'.format(subst_dict['ID_PREFIX'], prefix_list[i])})
+ url = url_templ.substitute({'HOST': host})
+ data = data_templ.substitute(subst_dict)
+ timeout = int(subst_dict['DURATION'])+60
+ logger.info('url: {}, data: {}, timeout: {}'.format(url, data, timeout))
+ t = threading.Thread(target=_send_http_request_thread_impl,
+ args=(resqueue, url, data, timeout))
+ t.daemon = True
+ t.start()
+ lthreads.append(t)
+
+ _globals.update({'threads': lthreads, 'result_queue': resqueue})
+
+
+def start_write_transactions_on_nodes(host_list, prefix_list, id_prefix, duration, rate, chained_flag=False):
+ """Invoke write-transactions rpc on given nodes.
:param host_list: list of ip address of odl nodes
:type host_list: list of strings
+ :param prefix_list: list of node indexes which coresponds to the ip addresses
+ :type prefix_list: list
:param id_prefix: identifier prefix
:type id_prefix: string
:param duration: time in seconds
:type duration: int
- :param rate: writing transactions rate per second
+ :param rate: writing transactions rate in transactions per second
:type rate: int
:param chained_flag: specify chained vs. simple transactions
:type chained_flag: bool
"""
- def _write_transactions(rqueue, url, grid, duration, rate, chained_flag):
- dtmpl = string.Template('''<input xmlns="tag:opendaylight.org,2017:controller:yang:lowlevel:control">
+ logger.info(
+ "Input parameters: host_list:{}, prefix_list:{}, id_prefix:{}, duration:{}, rate:{}, chained_flag:{}".format(
+ host_list, prefix_list, id_prefix, duration, rate, chained_flag))
+ datat = string.Template('''<input xmlns="tag:opendaylight.org,2017:controller:yang:lowlevel:control">
<id>$ID</id>
<seconds>$DURATION</seconds>
<transactions-per-second>$RATE</transactions-per-second>
<chained-transactions>$CHAINED_FLAG</chained-transactions>
</input>''')
- data = dtmpl.substitute({'ID': grid, 'DURATION': duration, 'RATE': rate, 'CHAINED_FLAG': chained_flag})
- logger.info('write-transactions rpc indoked with details: {}'.format(data))
- try:
- resp = requests.post(url=url, headers={'Content-Type': 'application/xml'},
- data=data, auth=('admin', 'admin'), timeout=int(duration)+60)
- except Exception as exc:
- resp = exc
- logger.debug(exc)
- rqueue.put(resp)
-
- logger.info("Input parameters: host_list:{}, id_prefix:{}, duration:{}, rate:{}, chained_flag:{}".format(
- host_list, id_prefix, duration, rate, chained_flag))
- resqueue = _globals.pop('result_queue', Queue.Queue())
- lthreads = _globals.pop('threads', [])
- for i, host in enumerate(host_list):
- url = 'http://{}:8181/restconf/operations/odl-mdsal-lowlevel-control:write-transactions'.format(host)
- t = threading.Thread(target=_write_transactions,
- args=(resqueue, url, '{}{}'.format(id_prefix, i), duration, rate, chained_flag))
- t.daemon = True
- t.start()
- lthreads.append(t)
+ subst_dict = {'ID_PREFIX': id_prefix, 'DURATION': duration, 'RATE': rate, 'CHAINED_FLAG': chained_flag}
+ urlt = string.Template('''http://$HOST:8181/restconf/operations/odl-mdsal-lowlevel-control:write-transactions''')
+ _initiate_rpcs(host_list, prefix_list, urlt, datat, subst_dict)
- _globals.update({'threads': lthreads, 'result_queue': resqueue})
+
+def start_produce_transactions_on_nodes(host_list, prefix_list, id_prefix,
+ duration, rate, isolated_transactions_flag=True):
+ """Invoke produce-transactions rpcs on given nodes.
+
+ :param host_list: list of ip address of odl nodes
+ :type host_list: list of strings
+ :param prefix_list: list of node indexes which coresponds to the ip addresses
+ :type prefix_list: list
+ :param id_prefix: identifier prefix
+ :type id_prefix: string
+ :param duration: time in seconds
+ :type duration: int
+ :param rate: produce transactions rate in transactions per second
+ :type rate: int
+ :param isolated_transactions_flag: isolated transactions flag
+ :type isolated_transactions_flag: bool
+ """
+ msg = "host_list:{}, prefix_list:{} ,id_prefix:{}, duration:{}, rate:{}, isolated_transactions:{}".format(
+ host_list, prefix_list, id_prefix, duration, rate, isolated_transactions_flag)
+ msg = "Input parameters: " + msg
+ logger.info(msg)
+ datat = string.Template('''<input xmlns="tag:opendaylight.org,2017:controller:yang:lowlevel:control">
+ <id>$ID</id>
+ <seconds>$DURATION</seconds>
+ <transactions-per-second>$RATE</transactions-per-second>
+ <isolated-transactions>$ISOLATED_TRANSACTIONS</isolated-transactions>
+</input>''')
+ subst_dict = {'ID_PREFIX': id_prefix, 'DURATION': duration, 'RATE': rate,
+ 'ISOLATED_TRANSACTIONS': isolated_transactions_flag}
+ urlt = string.Template('''http://$HOST:8181/restconf/operations/odl-mdsal-lowlevel-control:produce-transactions''')
+ _initiate_rpcs(host_list, prefix_list, urlt, datat, subst_dict)
-def wait_for_write_transactions():
- """Blocking call, waiting for responses from all threads."""
+def wait_for_transactions():
+ """Blocking call, waitig for responses from all threads."""
lthreads = _globals.pop('threads')
resqueue = _globals.pop('result_queue')
return results
-def get_next_write_transactions_response():
+def get_next_transactions_response():
"""Get http response from write-transactions rpc if available."""
resqueue = _globals.get('result_queue')
... This program and the accompanying materials are made available under the
... terms of the Eclipse Public License v1.0 which accompanies this distribution,
... and is available at http://www.eclipse.org/legal/epl-v10.html
+...
+... This resource file implements various test cases templates.
+... FIXME: add a link to a document (when published) where the scenarios are defined
Library ${CURDIR}/../MdsalLowlevelPy.py
Resource ${CURDIR}/../ClusterAdmin.robot
Resource ${CURDIR}/../ClusterManagement.robot
${CHAINED_TX} ${True}
${HARD_TIMEOUT} ${60}
@{TRANSACTION_FAILED} ${500}
+${PREF_BASED_SHARD} id-ints
+${TEST_LOG_LEVEL} info
+@{TEST_LOG_COMPONENTS} org.opendaylight.controller.cluster.sharding org.opendaylight.controller.cluster.datastore
*** Keywords ***
Explicit_Leader_Movement_Test_Templ
[Arguments] ${leader_from} ${leader_to} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
[Documentation] Implements explicit leader movement test scenario.
- ${idx_from} ${idx_to} ${idx_trans} = Get_Node_Indexes_For_The_ELM_Test ${leader_from} ${leader_to}
+ ${idx_from} ${idx_to} ${idx_trans} = Get_Node_Indexes_For_The_ELM_Test ${leader_from} ${leader_to} ${shard_name}
+ ... ${shard_type}
${ip_trans_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${idx_trans}_IP}
- MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${ip_trans_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${False}
+ ${idx_trans_as_list} = BuiltIn.Create_List ${idx_trans}
+ MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${ip_trans_as_list} ${idx_trans_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${False}
ClusterAdmin.Make_Leader_Local ${idx_to} ${shard_name} ${shard_type}
${new_leader} ${new_followers} = BuiltIn.Wait_Until_Keyword_Succeeds 10s 1s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}
... ${shard_type} ${True} ${idx_from}
BuiltIn.Should_Be_Equal ${idx_to} ${new_leader}
- ${resp_list} = MdsalLowlevelPy.Wait_For_Write_Transactions
+ ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
+ TemplatedRequests.Check_Status_Code @{resp_list}[0]
+
+Explicit_Leader_Movement_PrefBasedShard_Test_Templ
+ [Arguments] ${leader_from} ${leader_to} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
+ [Documentation] Implements explicit leader movement test scenario.
+ ${idx_from} ${idx_to} ${idx_trans} = Get_Node_Indexes_For_The_ELM_Test ${leader_from} ${leader_to} ${shard_name}!!
+ ... ${shard_type}
+ ${ip_trans_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${idx_trans}_IP}
+ ${idx_trans_as_list} = BuiltIn.Create_List ${idx_trans}
+ MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${ip_trans_as_list} ${idx_trans_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K}
+ MdsalLowlevel.Become_Prefix_Leader ${idx_to} ${shard_name} ${ID_PREFIX}
+ ${new_leader} ${new_followers} = BuiltIn.Wait_Until_Keyword_Succeeds 10s 1s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!!
+ ... ${shard_type} ${True} ${idx_from}
+ BuiltIn.Should_Be_Equal ${idx_to} ${new_leader}
+ ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
TemplatedRequests.Check_Status_Code @{resp_list}[0]
Get_Node_Indexes_For_The_ELM_Test
- [Arguments] ${leader_from} ${leader_to}
+ [Arguments] ${leader_from} ${leader_to} ${shard_name} ${shard_type}
[Documentation] Return indexes for explicit leader movement test case, indexes of present to next leader node and index where transaction
... producer should be deployed.
- ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${SHARD_NAME} shard_type=${SHARD_TYPE}
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type}
${idx_from} = BuiltIn.Set_Variable ${leader}
${idx_to} = BuiltIn.Set_Variable @{follower_list}[0]
${idx_trans} = BuiltIn.Set_Variable_If "${leader_from}" == "remote" and "${leader_to}" == "remote" @{follower_list}[1] "${leader_from}" == "local" ${leader} "${leader_to}" == "local"
[Arguments] ${leader_location} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
[Documentation] Implements clean leader shutdown test scenario.
${removed} = BuiltIn.Set_Variable ${False}
- ${producer_idx} ${actual_leader} = Get_Node_Indexes_For_Clean_Leader_Shutdown_Test ${leader_location}
+ ${producer_idx} ${actual_leader} ${follower_list} = Get_Node_Indexes_For_Clean_Leader_Shutdown_Test ${leader_location} ${shard_name} ${shard_type}
${producer_ip_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${producer_idx}_IP}
- MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${producer_ip_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${False}
+ ${producer_idx_as_list} = BuiltIn.Create_List ${producer_idx}
+ MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${producer_ip_as_list} ${producer_idx_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${False}
ClusterAdmin.Remove_Shard_Replica ${actual_leader} ${shard_name} member-${actual_leader} ${shard_type}
${removed} = BuiltIn.Set_Variable ${True}
- ${resp_list} = MdsalLowlevelPy.Wait_For_Write_Transactions
+ ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
TemplatedRequests.Check_Status_Code @{resp_list}[0]
[Teardown] ClusterAdmin.Add_Shard_Replica ${actual_leader} ${shard_name} ${shard_type}
+Clean_Leader_Shutdown_PrefBasedShard_Test_Templ
+ [Arguments] ${leader_location} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
+ [Documentation] Implements clean leader shutdown test scenario.
+ ${producer_idx} ${actual_leader} ${follower_list} = Get_Node_Indexes_For_Clean_Leader_Shutdown_Test ${leader_location} ${shard_name}!! ${shard_type}
+ ${producer_ip_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${producer_idx}_IP}
+ ${producer_idx_as_list} = BuiltIn.Create_List ${producer_idx}
+ MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${producer_ip_as_list} ${producer_idx_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K}
+ ClusterAdmin.Remove_Prefix_Shard_Replica ${actual_leader} ${shard_name} member-${actual_leader} ${shard_type}
+ BuiltIn.Wait_Until_Keyword_Succeeds 15s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${follower_list}
+ BuiltIn.Run_Keyword_And_Ignore_Error ClusterManagement.Get_Raft_State_Of_Shard_At_Member shard_name=${shard_name}!! shard_type=${shard_type} member_index=${actual_leader}
+ WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 10s 2s ClusterManagement.Get_Raft_State_Of_Shard_At_Member shard_name=${shard_name}!! shard_type=${shard_type} member_index=${actual_leader}
+ ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
+ TemplatedRequests.Check_Status_Code @{resp_list}[0]
+ [Teardown] ClusterAdmin.Add_Prefix_Shard_Replica ${actual_leader} ${shard_name} ${shard_type}
+
Get_Node_Indexes_For_Clean_Leader_Shutdown_Test
- [Arguments] ${leader_location}
+ [Arguments] ${leader_location} ${shard_name} ${shard_type}
[Documentation] Return indexes for clean leader shudown test case, index where transaction producer shoudl be deployed and a shard leader index.
- ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${SHARD_NAME} shard_type=${SHARD_TYPE}
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type}
${follower_list_leangth} = BuiltIn.Evaluate ${NUM_ODL_SYSTEM}-1
BuiltIn.Length_Should_Be ${follower_list} ${follower_list_leangth}
${producer_idx} = BuiltIn.Set_Variable_If "${leader_location}" == "local" ${leader} @{follower_list}[0]
- BuiltIn.Return_From_Keyword ${producer_idx} ${leader}
+ BuiltIn.Return_From_Keyword ${producer_idx} ${leader} ${follower_list}
Leader_Isolation_Test_Templ
- [Arguments] ${heal_timeout}
+ [Arguments] ${heal_timeout} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
[Documentation] Implements leader isolation test scenario.
${producing_transactions_time} = BuiltIn.Set_Variable ${${heal_timeout}+60}
${all_indices} = ClusterManagement.List_All_Indices
- ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${SHARD_NAME} shard_type=${SHARD_TYPE} member_index_list=${all_indices}
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices}
${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
- MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${all_ip_list} ${ID_PREFIX} ${producing_transactions_time} ${TRANSACTION_RATE_1K} chained_flag=${SIMPLE_TX}
+ MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${producing_transactions_time} ${TRANSACTION_RATE_1K} chained_flag=${SIMPLE_TX}
${date_start} = DateTime.Get_Current_Date
${date_end} = DateTime.Add_Time_To_Date ${date_start} ${producing_transactions_time}
ClusterManagement.Isolate_Member_From_List_Or_All ${leader}
- BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s ClusterManagement.Verify_Shard_Leader_Elected ${SHARD_NAME} ${SHARD_TYPE} ${True}
+ BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name} ${shard_type} ${True}
... ${leader} member_index_list=${follower_list}
BuiltIn.Sleep ${heal_timeout}
ClusterManagement.Rejoin_Member_From_List_Or_All ${leader}
- BuiltIn.Wait_Until_Keyword_Succeeds 20s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard ${SHARD_NAME} ${SHARD_TYPE}
+ BuiltIn.Wait_Until_Keyword_Succeeds 20s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard ${shard_name} ${shard_type}
${time_to_finish} = Get_Seconds_To_Time ${date_end}
BuiltIn.Run_Keyword_If ${heal_timeout} < ${TRANSACTION_TIMEOUT} Leader_Isolation_Heal_Within_Tt
- ... ELSE Leader_Isolation_Heal_Default ${time_to_finish}
+ ... ELSE Module_Leader_Isolation_Heal_Default ${leader} ${time_to_finish}
+
+Leader_Isolation_PrefBasedShard_Test_Templ
+ [Arguments] ${heal_timeout} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
+ [Documentation] Implements leader isolation test scenario.
+ ${producing_transactions_time} = BuiltIn.Set_Variable ${${heal_timeout}+60}
+ ${all_indices} = ClusterManagement.List_All_Indices
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices}
+ ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
+ MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${producing_transactions_time} ${TRANSACTION_RATE_1K}
+ ${date_start} = DateTime.Get_Current_Date
+ ${date_end} = DateTime.Add_Time_To_Date ${date_start} ${producing_transactions_time}
+ ClusterManagement.Isolate_Member_From_List_Or_All ${leader}
+ BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!! ${shard_type} ${True}
+ ... ${leader} member_index_list=${follower_list}
+ BuiltIn.Sleep ${heal_timeout}
+ ClusterManagement.Rejoin_Member_From_List_Or_All ${leader}
+ BuiltIn.Wait_Until_Keyword_Succeeds 20s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard ${shard_name}!! ${shard_type}
+ ${time_to_finish} = Get_Seconds_To_Time ${date_end}
+ BuiltIn.Run_Keyword_If ${heal_timeout} < ${TRANSACTION_TIMEOUT} Leader_Isolation_Heal_Within_Tt
+ ... ELSE Prefix_Leader_Isolation_Heal_Default ${leader} ${time_to_finish}
Leader_Isolation_Heal_Within_Tt
[Documentation] The leader isolation test case end if the heal happens within transaction timeout. All write transaction
... producers shoudl finish without error.
- ${resp_list} = MdsalLowlevelPy.Wait_For_Write_Transactions
+ ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
: FOR ${resp} IN @{resp_list}
\ TemplatedRequests.Check_Status_Code ${resp}
-Leader_Isolation_Heal_Default
+Module_Leader_Isolation_Heal_Default
[Arguments] ${isolated_node} ${time_to_finish}
[Documentation] The leader isolation test case end. The transaction producer on isolated node shoudl fail and should be restarted.
Then all write transaction producers shoudl finish without error.
- ${resp} = MdsalLowlevelPy.Get_Next_Write_Transactions_Response
+ ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
BuiltIn.Log ${resp}
- ${restart_producer_node} BuiltIn.Create_List ${isolated_node}
- MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${restart_producer_node} ${ID_PREFIX} ${time_to_finish} ${TRANSACTION_RATE_1K} chained_flag=${SIMPLE_TX}
- ${resp_list} = MdsalLowlevelPy.Wait_For_Write_Transactions
+ # TODO: check on response status code
+ ${restart_producer_node_idx_as_list} BuiltIn.Create_List ${isolated_node}
+ ${restart_producer_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${isolated_node}
+ ${restart_producer_node_ip_as_list} BuiltIn.Create_List ${restart_producer_node_ip}
+ MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${restart_producer_node_ip_as_list} ${restart_producer_node_idx_as_list} ${ID_PREFIX} ${time_to_finish} ${TRANSACTION_RATE_1K} chained_flag=${SIMPLE_TX}
+ ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
+ : FOR ${resp} IN @{resp_list}
+ \ TemplatedRequests.Check_Status_Code ${resp}
+
+Prefix_Leader_Isolation_Heal_Default
+ [Arguments] ${isolated_node} ${time_to_finish}
+ [Documentation] The leader isolation test case end. The transaction producer on isolated node shoudl fail and should be restarted.
+ Then all write transaction producers shoudl finish without error.
+ ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
+ BuiltIn.Log ${resp}
+ # TODO: check on response status code
+ ${restart_producer_node_idx_as_list} BuiltIn.Create_List ${isolated_node}
+ ${restart_producer_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${isolated_node}
+ ${restart_producer_node_ip_as_list} BuiltIn.Create_List ${restart_producer_node_ip}
+ MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${restart_producer_node_ip_as_list} ${restart_producer_node_idx_as_list} ${ID_PREFIX} ${time_to_finish} ${TRANSACTION_RATE_1K}
+ ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
: FOR ${resp} IN @{resp_list}
\ TemplatedRequests.Check_Status_Code ${resp}
${client_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
${client_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${client_node_dst}
${client_node_ip_as_list} BuiltIn.Create_List ${client_node_ip}
- MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${client_node_ip_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${trans_chain_flag}
+ ${client_node_idx_as_list} BuiltIn.Create_List ${client_node_dst}
+ MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${client_node_ip_as_list} ${client_node_idx_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${trans_chain_flag}
${start_date} DateTime.Get_Current_Date
${timeout_date} = DateTime.Add_Time_To_Date ${start_date} ${TRANSACTION_TIMEOUT}
${abort_date} = DateTime.Add_Time_To_Date ${start_date} ${HARD_TIMEOUT}
ClusterManagement.Isolate_Member_From_List_Or_All ${client_node_dst}
- WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${TRANSACTION_TIMEOUT} 1s Write_Transactions_Not_Failed_Yet
- WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 3s 1s Write_Transactions_Failed
+ WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${TRANSACTION_TIMEOUT} 1s Ongoing_Transactions_Not_Failed_Yet
+ WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 3s 1s Ongoing_Transactions_Failed
${abort_time} Get_Seconds_To_Time ${abort_date}
WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${abort_time} 1s Verify_Client_Aborted ${False}
WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 3s 1s Verify_Client_Aborted ${True}
[Teardown] BuiltIn.Run Keywords ClusterManagement.Rejoin_Member_From_List_Or_All ${client_node_dst}
- ... AND MdsalLowlevelPy.Wait_For_Write_Transactions
+ ... AND MdsalLowlevelPy.Wait_For_Transactions
-Write_Transactions_Not_Failed_Yet
+Client_Isolation_PrefBasedShard_Test_Templ
+ [Arguments] ${listener_node_role} ${trans_chain_flag} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
+ [Documentation] Implements client isolation test scenario.
+ ${all_indices} = ClusterManagement.List_All_Indices
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices}
+ ${follower1} = Collections.Get_From_List ${follower_list} ${0}
+ ${client_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
+ ${client_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${client_node_dst}
+ ${client_node_ip_as_list} BuiltIn.Create_List ${client_node_ip}
+ ${client_node_idx_as_list} BuiltIn.Create_List ${client_node_dst}
+ MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${client_node_ip_as_list} ${client_node_idx_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K}
+ ${start_date} DateTime.Get_Current_Date
+ ${timeout_date} = DateTime.Add_Time_To_Date ${start_date} ${TRANSACTION_TIMEOUT}
+ ${abort_date} = DateTime.Add_Time_To_Date ${start_date} ${HARD_TIMEOUT}
+ ClusterManagement.Isolate_Member_From_List_Or_All ${client_node_dst}
+ WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${TRANSACTION_TIMEOUT} 1s Ongoing_Transactions_Not_Failed_Yet
+ WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 3s 1s Ongoing_Transactions_Failed
+ ${abort_time} Get_Seconds_To_Time ${abort_date}
+ WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${abort_time} 1s Verify_Client_Aborted ${False}
+ WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 3s 1s Verify_Client_Aborted ${True}
+ [Teardown] BuiltIn.Run Keywords ClusterManagement.Rejoin_Member_From_List_Or_All ${client_node_dst}
+ ... AND MdsalLowlevelPy.Wait_For_Transactions
+
+Ongoing_Transactions_Not_Failed_Yet
[Documentation] Verify that no write-transaction rpc finished, means they are still running.
- ${resp} = MdsalLowlevelPy.Get_Next_Write_Transactions_Response
+ ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
BuiltIn.Should_Be_Equal ${None} ${resp} ${resp} not expected.
-Write_Transactions_Failed
+Ongoing_Transactions_Failed
[Documentation] Verify if write-transaction failed.
- ${resp} = MdsalLowlevelPy.Get_Next_Write_Transactions_Response
+ ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
Check_Status_Code ${resp} explicit_status_codes=${TRANSACTION_FAILED}
Verify_Client_Aborted
${listener_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
MdsalLowlevel.Subscribe_Dtcl ${listener_node_dst}
${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
- MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${all_ip_list} ${ID_PREFIX} ${DURATION_10S} ${TRANSACTION_RATE_1K} chained_flag=${SIMPLE_TX}
+ MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${DURATION_10S} ${TRANSACTION_RATE_1K} chained_flag=${SIMPLE_TX}
+ ClusterAdmin.Make_Leader_Local ${follower1} ${shard_name} ${shard_type}
+ ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
+ : FOR ${resp} IN @{resp_list}
+ \ TemplatedRequests.Check_Status_Code ${resp}
+ [Teardown] MdsalLowlevel.Unsubscribe_Dtcl ${listener_node_dst}
+
+Remote_Listener_PrefBasedShard_Test_Templ
+ [Arguments] ${listener_node_role} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
+ [Documentation] Implements listener isolation test scenario.
+ ${all_indices} = ClusterManagement.List_All_Indices
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices}
+ ${follower1} = Collections.Get_From_List ${follower_list} ${0}
+ ${follower2} = Collections.Get_From_List ${follower_list} ${1}
+ ${listener_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
+ MdsalLowlevel.Subscribe_Dtcl ${listener_node_dst}
+ ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
+ MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${DURATION_10S} ${TRANSACTION_RATE_1K}
ClusterAdmin.Make_Leader_Local ${follower1} ${shard_name} ${shard_type}
- ${resp_list} = MdsalLowlevelPy.Wait_For_Write_Transactions
+ ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
: FOR ${resp} IN @{resp_list}
\ TemplatedRequests.Check_Status_Code ${resp}
[Teardown] MdsalLowlevel.Unsubscribe_Dtcl ${listener_node_dst}
+
+Create_Prefix_Based_Shard_And_Verify
+ [Arguments] ${prefix}=${PREF_BASED_SHARD}
+ [Documentation] Create prefix based shard with replicas on all nodes
+ ${all_indices} = ClusterManagement.List_All_Indices
+ ${node_to_trigger} = Collections.Get_From_List ${all_indices} ${0}
+ MdsalLowlevel.Create_Prefix_Shard ${node_to_trigger} ${prefix} ${all_indices}
+ BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${prefix}!! shard_type=${SHARD_TYPE} member_index_list=${all_indices}
+
+Remove_Prefix_Based_Shard_And_Verify
+ [Arguments] ${prefix}=${PREF_BASED_SHARD}
+ [Documentation] Remove prefix based shard with all replicas
+ ${all_indices} = ClusterManagement.List_All_Indices
+ ${node_to_trigger} = Collections.Get_From_List ${all_indices} ${0}
+ MdsalLowlevel.Remove_Prefix_Shard ${node_to_trigger} ${prefix}
+ : FOR ${idx} IN @{all_indices}
+ \ BuiltIn.Wait_Until_Keyword_Succeeds 15s 2s Verify_Shard_Replica_Removed ${idx} ${prefix}!!
+ \ ... ${SHARD_TYPE}
+
+Verify_Shard_Replica_Removed
+ [Arguments] ${member_index} ${shard_name} ${shard_type}
+ [Documentation] Verify that shard is removed. Jolokia return 404 for shard memeber.
+ ${session} = Resolve_Http_Session_For_Member member_index=${member_index}
+ ${type_class} = Resolve_Shard_Type_Class shard_type=${shard_type}
+ ${uri} = BuiltIn.Set_Variable /jolokia/read/org.opendaylight.controller:Category=Shards,name=member-${member_index}-shard-${shard_name}-${shard_type},type=${type_class}
+ ${text} TemplatedRequests.Get_From_Uri uri=${uri} session=${session}
+ BuiltIn.Should_Contain ${text} "status":404 javax.management.InstanceNotFoundException
--- /dev/null
+*** Settings ***
+Documentation DOMDataBroker testing: Clean Leader Shutdown
+...
+... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
+...
+... This program and the accompanying materials are made available under the
+... terms of the Eclipse Public License v1.0 which accompanies this distribution,
+... and is available at http://www.eclipse.org/legal/epl-v10.html
+...
+... The goal is to ensure that applications do not observe disruption when a shard
+... leader is shut down cleanly. This is performed by having a steady-stream
+... producer execute operations against the shard and then initiate leader shard
+... shutdown, then the producer is shut down cleanly.
+Suite Setup BuiltIn.Run_Keywords SetupUtils.Setup_Utils_For_Setup_And_Teardown
+... AND DdbCommons.Create_Prefix_Based_Shard
+Suite Teardown BuiltIn.Run_Keywords DdbCommons.Remove_Prefix_Based_Shard
+... AND SSHLibrary.Close_All_Connections
+Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
+Test Teardown SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
+Default Tags critical
+Test Template DdbCommons.Clean_Leader_Shutdown_PrefBasedShard_Test_Templ
+Library SSHLibrary
+Resource ${CURDIR}/../../../libraries/controller/DdbCommons.robot
+Resource ${CURDIR}/../../../libraries/SetupUtils.robot
+
+*** Test Cases ***
+Local_Leader_Shutdown
+ [Documentation] Shutdown the leader on the same node as transaction producer.
+ local
+
+Remote_Leader_Shutdown
+ [Documentation] Shutdown the leader on different node as transaction producer.
+ remote
--- /dev/null
+*** Settings ***
+Documentation DOMDataBroker testing: Client Isolation
+...
+... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
+...
+... This program and the accompanying materials are made available under the
+... terms of the Eclipse Public License v1.0 which accompanies this distribution,
+... and is available at http://www.eclipse.org/legal/epl-v10.html
+...
+... The purpose of this test is to ascertain that the failure modes of
+... cds-access-client work as expected. This is performed by having a steady
+... stream of transactions flowing from the frontend and isolating the node hosting
+... the frontend from the rest of the cluster.
+Suite Setup BuiltIn.Run_Keywords SetupUtils.Setup_Utils_For_Setup_And_Teardown
+... AND DdbCommons.Create_Prefix_Based_Shard
+Suite Teardown BuiltIn.Run_Keywords DdbCommons.Remove_Prefix_Based_Shard
+... AND SSHLibrary.Close_All_Connections
+Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
+Test Teardown SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
+Default Tags critical
+Test Template DdbCommons.Client_Isolation_PrefBasedShard_Test_Templ
+Library SSHLibrary
+Resource ${CURDIR}/../../../libraries/controller/DdbCommons.robot
+Resource ${CURDIR}/../../../libraries/SetupUtils.robot
+
+*** Test Cases ***
+Producer_On_Shard_Leader_Node_ChainedTx
+ [Documentation] Client isolation with producer on shard leader with chained transactions.
+ leader ${CHAINED_TX}
+
+Producer_On_Shard_Leader_Node_SimpleTx
+ [Documentation] Client isolation with producer on shard leader with simple transactions.
+ leader ${SIMPLE_TX}
+
+Producer_On_Shard_Non_Leader_Node_ChainedTx
+ [Documentation] Client isolation with producer on shard non-leader with chained transactions.
+ non-leader ${CHAINED_TX}
+
+Producer_On_Shard_Non_Leader_Node_SimpleTx
+ [Documentation] Client isolation with producer on shard non-leader with simple transactions.
+ non-leader ${SIMPLE_TX}
--- /dev/null
+*** Settings ***
+Documentation DOMDataBroker testing: Module based shards sanity suite
+...
+... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
+...
+... This program and the accompanying materials are made available under the
+... terms of the Eclipse Public License v1.0 which accompanies this distribution,
+... and is available at http://www.eclipse.org/legal/epl-v10.html
+...
+... The goal is to call several basic rpc form ClusterAdmin.robot and
+... MdsalLowlevel.robot to ensute that those rpcs can be safely used in
+... other suites.
+... It also verify the ability of the odl-controller-test-app to perform
+... several activities.
+Suite Setup SetupUtils.Setup_Utils_For_Setup_And_Teardown
+Suite Teardown SSHLibrary.Close_All_Connections
+Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
+Test Teardown SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
+Default Tags critical
+Library SSHLibrary
+Library ${CURDIR}/../../../libraries/MdsalLowlevelPy.py
+Resource ${CURDIR}/../../../libraries/ClusterAdmin.robot
+Resource ${CURDIR}/../../../libraries/ClusterManagement.robot
+Resource ${CURDIR}/../../../libraries/controller/DdbCommons.robot
+Resource ${CURDIR}/../../../libraries/SetupUtils.robot
+
+*** Variables ***
+${SHARD_NAME} default
+${SHARD_TYPE} config
+${TRANSACTION_RATE_1K} ${1000}
+${DURATION_10S} ${10}
+${SIMPLE_TX} ${False}
+${CHAINED_TX} ${True}
+${MODULE_SHARD_PREFIX} member-
+
+*** Test Cases ***
+Make_Leader_Local
+ [Documentation] Make the loeader local and verify.
+ ${shard_name} = BuiltIn.Set_Variable ${SHARD_NAME}
+ ${shard_type} = BuiltIn.Set_Variable ${SHARD_TYPE}
+ ${all_indices} = ClusterManagement.List_All_Indices
+ ${old_leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${SHARD_NAME} shard_type=${SHARD_TYPE} member_index_list=${all_indices}
+ ${follower1} = Collections.Get_From_List ${follower_list} ${0}
+ ClusterAdmin.Make_Leader_Local ${follower1} ${shard_name} ${shard_type}
+ ${leader} ${follower_list} = BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}
+ ... ${shard_type} ${True} ${old_leader} member_index_list=${EMPTY}
+ BuiltIn.Should_Be_Equal_As_Numbers ${follower1} ${leader}
+
+Remove_Leader_Shard_Replica_And_Add_It_Back
+ [Documentation] Remove and add shard replica adn verify it.
+ ${shard_name} = BuiltIn.Set_Variable ${SHARD_NAME}
+ ${shard_type} = BuiltIn.Set_Variable ${SHARD_TYPE}
+ ${all_indices} = ClusterManagement.List_All_Indices
+ ${old_leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices}
+ ClusterAdmin.Remove_Shard_Replica ${old_leader} ${shard_name} member-${old_leader} ${shard_type}
+ BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s DdbCommons.Verify_Shard_Replica_Removed ${old_leader} ${shard_name} ${shard_type}
+ ${actual_leader} ${actual_follower_list} = BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}
+ ... shard_type=${shard_type} member_index_list=${follower_list}
+ BuiltIn.Should_Not_Be_Equal_As_Numbers ${old_leader} ${actual_leader}
+ ClusterAdmin.Add_Shard_Replica ${old_leader} ${shard_name} ${shard_type}
+ BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices}
+
+Remove_Follower_Shard_Replica_And_Add_It_Back
+ [Documentation] Remove and add shard replica adn verify it.
+ ${shard_name} = BuiltIn.Set_Variable ${SHARD_NAME}
+ ${shard_type} = BuiltIn.Set_Variable ${SHARD_TYPE}
+ ${all_indices} = ClusterManagement.List_All_Indices
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices}
+ ${follower1} = Collections.Get_From_List ${follower_list} ${0}
+ ClusterAdmin.Remove_Shard_Replica ${follower1} ${shard_name} member-${follower1} ${shard_type}
+ BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s DdbCommons.Verify_Shard_Replica_Removed ${follower1} ${shard_name} ${shard_type}
+ ${new_indices_list} = ClusterManagement.List_Indices_Minus_Member ${follower1}
+ ClusterManagement.Verify_Shard_Leader_Elected ${shard_name} ${shard_type} ${False} ${leader} member_index_list=${new_indices_list}
+ ClusterAdmin.Add_Shard_Replica ${follower1} ${shard_name} ${shard_type}
+ BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices}
+
+Write_Transactions
+ [Documentation] Write transactions.
+ ${all_indices} = ClusterManagement.List_All_Indices
+ ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
+ MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${MODULE_SHARD_PREFIX} ${DURATION_10S} ${TRANSACTION_RATE_1K} chained_flag=${SIMPLE_TX}
+ ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
+ : FOR ${resp} IN @{resp_list}
+ \ TemplatedRequests.Check_Status_Code ${resp}
--- /dev/null
+*** Settings ***
+Documentation DOMDataBroker testing: Module based shards sanity suite
+...
+... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
+...
+... This program and the accompanying materials are made available under the
+... terms of the Eclipse Public License v1.0 which accompanies this distribution,
+... and is available at http://www.eclipse.org/legal/epl-v10.html
+...
+... The goal is to call several basic rpc form ClusterAdmin.robot and
+... MdsalLowlevel.robot to ensute that those rpcs can be safely used in
+... other suites.
+... It also verify the ability of the odl-controller-test-app to perform
+... several activities.
+Suite Setup BuiltIn.Run_Keywords SetupUtils.Setup_Utils_For_Setup_And_Teardown
+... AND DdbCommons.Create_Prefix_Based_Shard_And_Verify
+Suite Teardown BuiltIn.Run_Keywords DdbCommons.Remove_Prefix_Based_Shard_And_Verify
+... AND SSHLibrary.Close_All_Connections
+Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
+Test Teardown SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
+Default Tags critical
+Library SSHLibrary
+Library ${CURDIR}/../../../libraries/MdsalLowlevelPy.py
+Resource ${CURDIR}/../../../libraries/ClusterAdmin.robot
+Resource ${CURDIR}/../../../libraries/ClusterManagement.robot
+Resource ${CURDIR}/../../../libraries/controller/DdbCommons.robot
+Resource ${CURDIR}/../../../libraries/SetupUtils.robot
+
+*** Variables ***
+${PREF_BASED_SHARD} id-ints
+${SHARD_TYPE} config
+${TRANSACTION_RATE_1K} ${1000}
+${DURATION_10S} ${10}
+${SIMPLE_TX} ${False}
+${CHAINED_TX} ${True}
+${SHARD_PREFIX} member-
+${ID_PREFIX} prefix-
+
+*** Test Cases ***
+Produce_Transactions
+ [Documentation] Write transactions.
+ ${all_indices} = ClusterManagement.List_All_Indices
+ ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
+ MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${DURATION_10S} ${TRANSACTION_RATE_1K}
+ ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
+ : FOR ${resp} IN @{resp_list}
+ \ TemplatedRequests.Check_Status_Code ${resp}
+
+Become_Prefix_Leader
+ [Documentation] Make the loeader local and verify.
+ ${shard_name} = BuiltIn.Set_Variable ${PREF_BASED_SHARD}
+ ${shard_type} = BuiltIn.Set_Variable ${SHARD_TYPE}
+ ${all_indices} = ClusterManagement.List_All_Indices
+ ${old_leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices}
+ ${follower1} = Collections.Get_From_List ${follower_list} ${0}
+ MdsalLowlevel.Become_Prefix_Leader ${follower1} ${shard_name} ${ID_PREFIX}
+ ${leader} ${follower_list} = BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!!
+ ... ${shard_type} ${True} ${old_leader} member_index_list=${EMPTY}
+ BuiltIn.Should_Be_Equal_As_Numbers ${follower1} ${leader}
+
+Remove_Leader_Prefix_Shard_Replica_And_Add_It_Back
+ [Documentation] Remove and add shard replica adn verify it.
+ ${shard_name} = BuiltIn.Set_Variable ${PREF_BASED_SHARD}
+ ${shard_type} = BuiltIn.Set_Variable ${SHARD_TYPE}
+ ${all_indices} = ClusterManagement.List_All_Indices
+ ${old_leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices}
+ ClusterAdmin.Remove_Prefix_Shard_Replica ${old_leader} ${shard_name} member-${old_leader} ${shard_type}
+ BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s Verify_Shard_Replica_Removed ${old_leader} ${shard_name}!! ${shard_type}
+ ${actual_leader} ${actual_follower_list} = BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!!
+ ... shard_type=${shard_type} member_index_list=${follower_list}
+ ClusterAdmin.Add_Prefix_Shard_Replica ${old_leader} ${shard_name} ${shard_type}
+ BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices}
+
+Remove_Follower_Prefix_Shard_Replica_And_Add_It_Back
+ [Documentation] Remove and add shard replica adn verify it.
+ ${shard_name} = BuiltIn.Set_Variable ${PREF_BASED_SHARD}
+ ${shard_type} = BuiltIn.Set_Variable ${SHARD_TYPE}
+ ${all_indices} = ClusterManagement.List_All_Indices
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices}
+ ${follower1} = Collections.Get_From_List ${follower_list} ${0}
+ ClusterAdmin.Remove_Prefix_Shard_Replica ${follower1} ${shard_name} member-${follower1} ${shard_type}
+ BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s DdbCommons.Verify_Shard_Replica_Removed ${follower1} ${shard_name}!! ${shard_type}
+ ${new_indices_list} = ClusterManagement.List_Indices_Minus_Member ${follower1}
+ ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!! ${shard_type} ${False} ${leader} member_index_list=${new_indices_list}
+ ClusterAdmin.Add_Prefix_Shard_Replica ${follower1} ${shard_name} ${shard_type}
+ BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices}
--- /dev/null
+*** Settings ***
+Documentation DOMDataBroker testing: Explicit Leader Movement
+...
+... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
+...
+... This program and the accompanying materials are made available under the
+... terms of the Eclipse Public License v1.0 which accompanies this distribution,
+... and is available at http://www.eclipse.org/legal/epl-v10.html
+...
+... The goal is to ensure that applications do not observe disruption when a shard
+... leader is moved as the result of explicit application request. This is performed
+... by having a steady-stream producer execute operations against the shard and then
+... initiate shard leader shutdown, then the producer is shut down cleanly.
+Suite Setup BuiltIn.Run_Keywords SetupUtils.Setup_Utils_For_Setup_And_Teardown
+... AND DdbCommons.Create_Prefix_Based_Shard
+Suite Teardown BuiltIn.Run_Keywords DdbCommons.Remove_Prefix_Based_Shard
+... AND SSHLibrary.Close_All_Connections
+Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
+Test Teardown SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
+Default Tags critical
+Test Template DdbCommons.Explicit_Leader_Movement_PrefBasedShard_Test_Templ
+Library SSHLibrary
+Resource ${CURDIR}/../../../libraries/controller/DdbCommons.robot
+Resource ${CURDIR}/../../../libraries/SetupUtils.robot
+
+*** Test Cases ***
+Local_To_Remote_Movement
+ [Documentation] Leader moves from local to remote node during transaction producing.
+ local remote
+
+Remote_To_Remote_Movement
+ [Documentation] Leader moves from one remote to other remote node during transaction producing.
+ remote remote
+
+Remote_To_Local_Movement
+ [Documentation] Leader moves from remote to local node during transaction producing.
+ remote local
--- /dev/null
+*** Settings ***
+Documentation DOMDataBroker testing: Leader Isolation
+...
+... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
+...
+... This program and the accompanying materials are made available under the
+... terms of the Eclipse Public License v1.0 which accompanies this distribution,
+... and is available at http://www.eclipse.org/legal/epl-v10.html
+...
+... The goal is to ensure the datastore succeeds in basic isolation/rejoin scenario,
+... simulating either a network partition, or a prolonged GC pause.
+Suite Setup BuiltIn.Run_Keywords SetupUtils.Setup_Utils_For_Setup_And_Teardown
+... AND DdbCommons.Create_Prefix_Based_Shard
+Suite Teardown BuiltIn.Run_Keywords DdbCommons.Remove_Prefix_Based_Shard
+... AND SSHLibrary.Close_All_Connections
+Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
+Test Teardown SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
+Default Tags critical
+Test Template DdbCommons.Leader_Isolation_PrefBasedShard_Test_Templ
+Library SSHLibrary
+Resource ${CURDIR}/../../../libraries/controller/DdbCommons.robot
+Resource ${CURDIR}/../../../libraries/SetupUtils.robot
+
+*** Test Cases ***
+Healing_Within_Transaction_Timeout
+ [Documentation] The isolated node (leader) is rejoined as soon as new leader is elected and
+ ... and within transaction timeout.
+ ${0}
+
+Healing_After_2x_Transaction_Timeout
+ [Documentation] The isolated node (leader) is rejoined after 2x transaction timeout.
+ ${${TRANSACTION_TIMEOUT_2X}+1}
--- /dev/null
+*** Settings ***
+Documentation DOMDataBroker testing: Remote Listener
+...
+... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
+...
+... This program and the accompanying materials are made available under the
+... terms of the Eclipse Public License v1.0 which accompanies this distribution,
+... and is available at http://www.eclipse.org/legal/epl-v10.html
+...
+... The goal is to ensure listeners do no observe disruption when the leader moves.
+... This is performed by having a steady stream of transactions being observed by
+... the listeners and having the leader move.
+Suite Setup BuiltIn.Run_Keywords SetupUtils.Setup_Utils_For_Setup_And_Teardown
+... AND DdbCommons.Create_Prefix_Based_Shard
+Suite Teardown BuiltIn.Run_Keywords DdbCommons.Remove_Prefix_Based_Shard
+... AND SSHLibrary.Close_All_Connections
+Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
+Test Teardown SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
+Default Tags critical
+Test Template DdbCommons.Remote_Listener_PrefBasedShard_Test_Templ
+Library SSHLibrary
+Resource ${CURDIR}/../../../libraries/controller/DdbCommons.robot
+Resource ${CURDIR}/../../../libraries/SetupUtils.robot
+
+*** Test Cases ***
+Listener_On_Shard_Leader_Node
+ [Documentation] Listener runs on leader node when leader changed.
+ leader
+
+Listener_On_Shard_Non_Leader_Node
+ [Documentation] Listener runs on non-leader node when leader changed.
+ non-leader
integration/test/csit/suites/controller/dom_data_broker/client_isolation.robot
integration/test/csit/suites/controller/dom_data_broker/remote_listener.robot
integration/test/csit/suites/controller/dom_data_broker/restart_odl_with_tell_based_true.robot
+integration/test/csit/suites/controller/dom_data_broker/ddb-sanity-prefix-based.robot
+integration/test/csit/suites/controller/dom_data_broker/ddb-sanity-module-based.robot
integration/test/csit/suites/controller/Clustering_Datastore/cluster_ready.robot
integration/test/csit/suites/controller/cluster_singleton/master_stability.robot
integration/test/csit/suites/controller/cluster_singleton/partition_and_heal.robot
integration/test/csit/suites/controller/dom_data_broker/leader_isolation.robot
integration/test/csit/suites/controller/dom_data_broker/client_isolation.robot
integration/test/csit/suites/controller/dom_data_broker/remote_listener.robot
+integration/test/csit/suites/controller/dom_data_broker/leader_isolation_prefbasedshard.robot
+integration/test/csit/suites/controller/dom_data_broker/explicit_leader_movement_prefbasedshard.robot
+integration/test/csit/suites/controller/dom_data_broker/remote_listener_prefbasedshard.robot
+integration/test/csit/suites/controller/dom_data_broker/client_isolation_prefbasedshard.robot
+integration/test/csit/suites/controller/dom_data_broker/clean_leader_shutdown_prefbasedshard.robot
integration/test/csit/suites/controller/dom_data_broker/restart_odl_with_tell_based_false.robot
integration/test/csit/suites/controller/Clustering_Datastore/cluster_ready.robot
integration/test/csit/suites/controller/singleton_service/global_rpc_kill.robot
-<input xmlns="urn:opendaylight:params:xml:ns:yang:controller:md:sal:cluster:admin">
- <shard-prefix>$SHARD_PREFIX</shard-prefix>
+<input xmlns="urn:opendaylight:params:xml:ns:yang:controller:md:sal:cluster:admin" xmlns:odl-mdsal-lowlevel-target="tag:opendaylight.org,2017:controller:yang:lowlevel:target">
+ <shard-prefix>/odl-mdsal-lowlevel-target:$SHARD_PREFIX</shard-prefix>
<data-store-type>$DATA_STORE_TYPE</data-store-type>
</input>
-<input xmlns="urn:opendaylight:params:xml:ns:yang:controller:md:sal:cluster:admin">
- <shard-prefix>$SHARD_PREFIX</shard-prefix>
+<input xmlns="urn:opendaylight:params:xml:ns:yang:controller:md:sal:cluster:admin" xmlns:odl-mdsal-lowlevel-target="tag:opendaylight.org,2017:controller:yang:lowlevel:target">
+ <shard-prefix>/odl-mdsal-lowlevel-target:$SHARD_PREFIX</shard-prefix>
<member-name>$MEMBER_NAME</member-name>
<data-store-type>$DATA_STORE_TYPE</data-store-type>
</input>
-<input xmlns="tag:opendaylight.org,2017:controller:yang:lowlevel:control">
- <prefix>$PREFIX</prefix>
+<input xmlns="tag:opendaylight.org,2017:controller:yang:lowlevel:control" xmlns:odl-mdsal-lowlevel-target="tag:opendaylight.org,2017:controller:yang:lowlevel:target">
+ <prefix>/odl-mdsal-lowlevel-target:$SHARD_NAME/odl-mdsal-lowlevel-target:id-int[odl-mdsal-lowlevel-target:id="$ID"]</prefix>
</input>
-/restconf/operations/odl-mdsal-lowlevel-control:add-prefix-shard
+/restconf/operations/odl-mdsal-lowlevel-control:create-prefix-shard
-<input xmlns="tag:opendaylight.org,2017:controller:yang:lowlevel:control">
- <prefix>$PREFIX</prefix>
+<input xmlns="tag:opendaylight.org,2017:controller:yang:lowlevel:control" xmlns:odl-mdsal-lowlevel-target="tag:opendaylight.org,2017:controller:yang:lowlevel:target">
+ <prefix>/odl-mdsal-lowlevel-target:$PREFIX</prefix>
$REPLICAS
</input>
-/restconf/operations/odl-mdsal-lowlevel-control:become-prefix-leader
\ No newline at end of file
+/restconf/operations/odl-mdsal-lowlevel-control:remove-prefix-shard
-<input xmlns="tag:opendaylight.org,2017:controller:yang:lowlevel:control">
- <prefix>$PREFIX</prefix>
+<input xmlns="tag:opendaylight.org,2017:controller:yang:lowlevel:control" xmlns:odl-mdsal-lowlevel-target="tag:opendaylight.org,2017:controller:yang:lowlevel:target">
+ <prefix>/odl-mdsal-lowlevel-target:$PREFIX</prefix>
</input>
Code Review / integration / test.git / commitdiff