--- /dev/null
+*** Settings ***
+Documentation Test when a car shard leader is isolated while configuring cars.
+...
+... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
+...
+... This program and the accompanying materials are made available under the
+... terms of the Eclipse Public License v1.0 which accompanies this distribution,
+... and is available at http://www.eclipse.org/legal/epl-v10.html
+...
+... This test suite requires odl-restconf and odl-clustering-test-app modules.
+... The script cluster_rest_script.py is used for generating requests for
+... PUTing car items while the car shard leader is isolated.
+Suite Setup Start_Suite
+Suite Teardown Stop_Suite
+Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
+Library RequestsLibrary
+Library SSHLibrary
+Resource ${CURDIR}/../../../variables/Variables.robot
+Resource ${CURDIR}/../../../libraries/Utils.robot
+Resource ${CURDIR}/../../../libraries/SetupUtils.robot
+Resource ${CURDIR}/../../../libraries/ClusterManagement.robot
+Resource ${CURDIR}/../../../libraries/CarPeople.robot
+
+*** Variables ***
+${ITEM_COUNT} ${10000}
+${THREADS} 10
+${ADDCMD} python ${TOOL_NAME} --port ${RESTCONFPORT} add-with-retries --itemtype car --itemcount ${ITEM_COUNT} --threads ${THREADS}
+${CARURL} /restconf/config/car:cars
+${SHARD_NAME} car
+${SHARD_TYPE} config
+${TEST_LOG_LEVEL} info
+@{TEST_LOG_COMPONENTS} org.opendaylight.controller
+${TOOL_OPTIONS} ${EMPTY}
+${TOOL_NAME} cluster_rest_script.py
+
+*** Test Cases ***
+Start_Adding_Cars_To_Follower
+ [Documentation] Start the script to configure ${ITEM_COUNT} cars in the background.
+ ${idx} = Collections.Get_From_List ${car_follower_indices} 0
+ ${follower_ip} = ClusterManagement.Resolve_IP_Address_For_Member member_index=${idx}
+ Start Tool ${ADDCMD} --host ${follower_ip} ${TOOL_OPTIONS}
+ ${session} = Resolve_Http_Session_For_Member member_index=${car_leader_index}
+ BuiltIn.Wait_Until_Keyword_Succeeds 5x 2s Ensure_Cars_Being_Configured ${session}
+
+Isolate_Current_Car_Leader
+ [Documentation] Isolating cluster node which is the car shard leader.
+ ClusterManagement.Isolate_Member_From_List_Or_All ${car_leader_index}
+ BuiltIn.Set Suite variable ${old_car_leader} ${car_leader_index}
+ BuiltIn.Set Suite variable ${old_car_followers} ${car_follower_indices}
+
+Verify_New_Car_Leader_Elected
+ [Documentation] Verify new owner of the car shard is elected.
+ [Tags] critical
+ BuiltIn.Wait_Until_Keyword_Succeeds 5x 2s ClusterManagement.Verify_Shard_Leader_Elected ${SHARD_NAME} ${SHARD_TYPE} ${True}
+ ... ${old_car_leader} member_index_list=${old_car_followers}
+ CarPeople.Set_Tmp_Variables_For_Shard_For_Nodes ${old_car_followers} shard_name=${SHARD_NAME} shard_type=${SHARD_TYPE}
+
+Verify_Cars_Configured
+ [Documentation] Verify that all cars are configured.
+ [Tags] critical
+ BuiltIn.Wait_Until_Keyword_Succeeds 120x 2s SSHLibrary.Read_Until_Prompt
+ ${session} = Resolve_Http_Session_For_Member member_index=${new_leader_index}
+ Verify_Cars_Count ${ITEM_COUNT} ${session}
+
+Rejoin_Isolated_Member
+ [Documentation] Rejoin isolated node
+ [Tags] @{NO_TAGS}
+ ClusterManagement.Rejoin_Member_From_List_Or_All ${old_car_leader}
+
+Delete Cars
+ [Documentation] Remove cars from the datastore
+ ${session} = Resolve_Http_Session_For_Member member_index=${new_leader_index}
+ ${rsp}= RequestsLibrary.Delete Request ${session} ${CARURL}
+ Should Be Equal As Numbers 200 ${rsp.status_code}
+ ${rsp}= RequestsLibrary.Get Request ${session} ${CARURL}
+ Should Be Equal As Numbers 404 ${rsp.status_code}
+
+*** Keywords ***
+Start Suite
+ [Documentation] Upload the script file and create a virtual env
+ SetupUtils.Setup_Utils_For_Setup_And_Teardown
+ SetupUtils.Setup_Logging_For_Debug_Purposes_On_List_Or_All ${TEST_LOG_LEVEL} ${TEST_LOG_COMPONENTS}
+ ${mininet_conn_id} = SSHKeywords.Open_Connection_To_Tools_System
+ Builtin.Set Suite Variable ${mininet_conn_id}
+ SSHLibrary.Put File ${CURDIR}/../../../../tools/odl-mdsal-clustering-tests/scripts/${TOOL_NAME} .
+ ${stdout} ${stderr} ${rc}= SSHLibrary.Execute Command ls return_stdout=True return_stderr=True
+ ... return_rc=True
+ ${out_file} = Utils.Get_Log_File_Name ${TOOL_NAME}
+ BuiltIn.Set_Suite_Variable ${out_file}
+ SSHKeywords.Virtual_Env_Create
+ SSHKeywords.Virtual_Env_Install_Package requests
+ CarPeople.Set_Variables_For_Shard ${SHARD_NAME} shard_type=${SHARD_TYPE}
+
+Stop Suite
+ [Documentation] Stop the tool, remove virtual env and close ssh connection towards tools vm.
+ Stop_Tool
+ SSHKeywords.Virtual_Env_Delete
+ Store_File_To_Workspace ${out_file} ${out_file}
+ SSHLibrary.Close All Connections
+
+Start_Tool
+ [Arguments] ${command} ${tool_opt}
+ [Documentation] Start the tool
+ # TODO: https://trello.com/c/rXsMu7iz/444-create-keywords-for-the-tool-start-and-stop-in-remotebash-robot
+ BuiltIn.Log ${command}
+ SSHKeywords.Virtual_Env_Activate_On_Current_Session log_output=${True}
+ ${output}= SSHLibrary.Write ${command} ${tool_opt} 2>&1 | tee ${out_file}
+ BuiltIn.Log ${output}
+
+Stop_Tool
+ [Documentation] Stop the tool if still running.
+ # TODO: https://trello.com/c/rXsMu7iz/444-create-keywords-for-the-tool-start-and-stop-in-remotebash-robot
+ ${output}= SSHLibrary.Read
+ BuiltIn.Log ${output}
+ Utils.Write_Bare_Ctrl_C
+ ${output}= SSHLibrary.Read_Until_Prompt
+ BuiltIn.Log ${output}
+ SSHKeywords.Virtual_Env_Deactivate_On_Current_Session log_output=${True}
+
+Verify_Cars_Count
+ [Arguments] ${exp_count} ${session}
+ [Documentation] Count car items in config ds and compare with expected number.
+ ${count} = Get_Cars_Count ${session}
+ BuiltIn.Should_Be_Equal_As_Numbers ${count} ${exp_count}
+
+Get_Cars_Count
+ [Arguments] ${session}
+ [Documentation] Count car items in config ds.
+ ${resp}= RequestsLibrary.Get_Request ${session} ${CARURL}
+ ${count} = BuiltIn.Evaluate len(${resp.json()}["cars"]["car-entry"])
+ BuiltIn.Return_From_Keyword ${count}
+
+Ensure_Cars_Being_Configured
+ [Arguments] ${session}
+ ${count1} = Get_Cars_Count ${session}
+ ${count2} = Get_Cars_Count ${session}
+ BuiltIn.Should_Not_Be_Equal_As_Integers ${count1} ${count2}
+
+Store_File_To_Workspace
+ [Arguments] ${source_file_name} ${target_file_name}
+ [Documentation] Store the ${source_file_name} to the workspace as ${target_file_name}.
+ SSHLibrary.Get_File ${source_file_name} ${target_file_name}
import copy
import argparse
import logging
+import time
_template_add_car = {
def _request_sender(thread_id, preparing_function, auth, in_queue=None,
- exit_event=None, odl_ip="127.0.0.1", port="8181", out_queue=None):
+ exit_event=None, odl_ip="127.0.0.1", port="8181", out_queue=None,
+ req_timeout=60, retry_timeout=15, retry_rcs=[]):
"""The funcion sends http requests.
Runs in the working thread. It reads out flow details from the queue and
:param out_queue: queue where the results should be put
+ :param req_timeout: http request timeout
+
+ :param retry_timeout: timout to give up retry attempts to send http requests
+
+ :param retry_rcs: list of return codes when retry should be performed
+
Returns:
None (results is put into the output queue)
"""
continue
req = preparing_function(odl_ip, port, item_list, auth)
prep = req.prepare()
- try:
- rsp = ses.send(prep, timeout=600)
- except requests.exceptions.Timeout:
- counter[99] += 1
- logger.error("No response from %s", odl_ip)
- continue
- logger.debug("%s %s", rsp.request, rsp.request.url)
- logger.debug("Headers %s:", rsp.request.headers)
- logger.debug("Body: %s", rsp.request.body)
- logger.debug("Response: %s", rsp.text)
- logger.debug("%s %s", rsp, rsp.reason)
- counter[rsp.status_code] += 1
+ start_time = time_now = time.time()
+ while start_time + retry_timeout > time_now:
+ try:
+ rsp = ses.send(prep, timeout=req_timeout)
+ except requests.exceptions.Timeout:
+ counter[99] += 1
+ logger.error("No response from %s", odl_ip)
+ rc = 99
+ else:
+ logger.debug("%s %s", rsp.request, rsp.request.url)
+ logger.debug("Headers %s:", rsp.request.headers)
+ logger.debug("Body: %s", rsp.request.body)
+ logger.debug("Response: %s", rsp.text)
+ logger.debug("%s %s", rsp, rsp.reason)
+ counter[rsp.status_code] += 1
+ rc = rsp.status_code
+ if rc not in retry_rcs:
+ break
+ time_now = time.time()
responses = {}
for response_code, count in enumerate(counter):
if count > 0:
def _task_executor(preparing_function, odl_ip="127.0.0.1", port="8181",
thread_count=1, item_count=1, items_per_request=1,
- auth=('admin', 'admin')):
+ auth=('admin', 'admin'), req_timeout=600, retry_timeout=15, retry_rcs=[]):
"""The main function which drives sending of http requests.
Creates 2 queues and requested number of "working threads".
:param auth: authentication credentials
+ :param req_timeout: http request timeout
+
+ :param retry_timeout: timout to give up retry attempts to send http requests
+
+ :param retry_rcs: list of return codes when retry should be performed
+
Returns:
:returns dict: dictionary of http response counts like
{"http_status_code1: "count1", etc.}
args=(i, preparing_function, auth),
kwargs={"in_queue": send_queue, "exit_event": exit_event,
"odl_ip": hosts[i % nrhosts], "port": port,
- "out_queue": result_queue})
+ "out_queue": result_queue, "req_timeout": req_timeout,
+ "retry_timeout": retry_timeout, "retry_rcs": retry_rcs})
threads.append(thr)
thr.start()
raise Exception("Not all cars were configured: " + repr(res))
+def add_car_with_retries(odl_ip, port, thread_count, item_count, auth, items_per_request):
+ """Configure car entries to the config datastore.
+
+ Args:
+ :param odl_ip: ip address of ODL
+
+ :param port: restconf port
+
+ :param thread_count: number of threads used to send http requests; default=1
+
+ :param item_count: number of items to be configured
+
+ :param auth: authentication credentials
+
+ :param items_per_request: items per request, not used here,
+ just to keep the same api
+
+ Returns:
+ None
+ """
+
+ logger.info("Add %s car(s) to %s:%s (%s per request)",
+ item_count, odl_ip, port, items_per_request)
+ retry_rcs = [401, 404, 503]
+ res = _task_executor(_prepare_add_car, odl_ip=odl_ip, port=port,
+ thread_count=thread_count, item_count=item_count,
+ items_per_request=items_per_request, auth=auth,
+ req_timeout=15, retry_timeout=30, retry_rcs=retry_rcs)
+ acceptable_rcs = [204] + retry_rcs
+ for key in res.keys():
+ if key not in acceptable_rcs:
+ logger.error("Problems during cars' configuration appeared: " + repr(res))
+ raise Exception("Problems during cars' configuration appeared: " + repr(res))
+
+
def add_people_rpc(odl_ip, port, thread_count, item_count, auth, items_per_request):
"""Configure people entries to the config datastore.
raise Exception("Not all rpc calls passed: " + repr(res))
-_actions = ["add", "get", "delete", "add-rpc"]
+_actions = ["add", "get", "delete", "add-rpc", "add-with-retries"]
_items = ["car", "people", "car-people"]
_handler_matrix = {
"get": {"car": get_car, "people": get_people, "car-people": get_car_people},
"delete": {"car": delete_car, "people": delete_people, "car-people": delete_car_people},
"add-rpc": {"car-people": add_car_people_rpc, "people": add_people_rpc},
+ "add-with-retries": {"car": add_car_with_retries},
}
Code Review / integration / test.git / commitdiff