2 Documentation Test when a car shard leader is isolated while configuring cars.
4 ... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
6 ... This program and the accompanying materials are made available under the
7 ... terms of the Eclipse Public License v1.0 which accompanies this distribution,
8 ... and is available at http://www.eclipse.org/legal/epl-v10.html
10 ... This test suite requires odl-restconf and odl-clustering-test-app modules.
11 ... The script cluster_rest_script.py is used for generating requests for
12 ... PUTing car items while the car shard leader is isolated.
13 Suite Setup Start_Suite
14 Suite Teardown Stop_Suite
15 Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
17 Library RequestsLibrary
19 Resource ${CURDIR}/../../../libraries/CarPeople.robot
20 Resource ${CURDIR}/../../../libraries/ClusterManagement.robot
21 Resource ${CURDIR}/../../../libraries/RemoteBash.robot
22 Resource ${CURDIR}/../../../libraries/SetupUtils.robot
23 Resource ${CURDIR}/../../../libraries/TemplatedRequests.robot
24 Resource ${CURDIR}/../../../libraries/Utils.robot
25 Resource ${CURDIR}/../../../variables/Variables.robot
28 ${ITEM_COUNT} ${10000}
30 ${ADDCMD} python ${TOOL_NAME} --port ${RESTCONFPORT} add-with-retries --itemtype car --itemcount ${ITEM_COUNT} --threads ${THREADS}
31 ${CARURL} /restconf/config/car:cars
34 ${TEST_LOG_LEVEL} info
35 @{TEST_LOG_COMPONENTS} org.opendaylight.controller
36 ${TOOL_OPTIONS} ${EMPTY}
37 ${TOOL_NAME} cluster_rest_script.py
40 Start_Adding_Cars_To_Follower
41 [Documentation] Start the script to configure ${ITEM_COUNT} cars in the background.
42 ${idx} = Collections.Get_From_List ${car_follower_indices} 0
43 ${follower_ip} = ClusterManagement.Resolve_IP_Address_For_Member member_index=${idx}
44 Start Tool ${ADDCMD} --host ${follower_ip} ${TOOL_OPTIONS}
45 ${session} = Resolve_Http_Session_For_Member member_index=${car_leader_index}
46 BuiltIn.Wait_Until_Keyword_Succeeds 10x 5s Ensure_Cars_Being_Configured ${session}
48 Isolate_Current_Car_Leader
49 [Documentation] Isolating cluster node which is the car shard leader.
50 ClusterManagement.Isolate_Member_From_List_Or_All ${car_leader_index}
51 BuiltIn.Set Suite variable ${old_car_leader} ${car_leader_index}
52 BuiltIn.Set Suite variable ${old_car_followers} ${car_follower_indices}
54 Verify_New_Car_Leader_Elected
55 [Documentation] Verify new owner of the car shard is elected.
56 BuiltIn.Wait_Until_Keyword_Succeeds 10x 5s ClusterManagement.Verify_Shard_Leader_Elected ${SHARD_NAME} ${SHARD_TYPE} ${True}
57 ... ${old_car_leader} member_index_list=${old_car_followers}
58 CarPeople.Set_Tmp_Variables_For_Shard_For_Nodes ${old_car_followers} shard_name=${SHARD_NAME} shard_type=${SHARD_TYPE}
60 Verify_Cars_Configured
61 [Documentation] Verify that all cars are configured.
62 BuiltIn.Wait_Until_Keyword_Succeeds 120x 2s SSHLibrary.Read_Until_Prompt
63 ${session} = Resolve_Http_Session_For_Member member_index=${new_leader_index}
64 BuiltIn.Wait_Until_Keyword_Succeeds 5x 2s Verify_Cars_Count ${ITEM_COUNT} ${session}
66 Rejoin_Isolated_Member
67 [Documentation] Rejoin isolated node
68 ClusterManagement.Rejoin_Member_From_List_Or_All ${old_car_leader}
71 [Documentation] Remove cars from the datastore
72 ${session} = Resolve_Http_Session_For_Member member_index=${new_leader_index}
73 ${rsp}= RequestsLibrary.Delete Request ${session} ${CARURL}
74 Should Be Equal As Numbers 200 ${rsp.status_code}
75 ${rsp}= RequestsLibrary.Get Request ${session} ${CARURL}
76 Should Contain ${DELETED_STATUS_CODES} ${rsp.status_code}
80 [Documentation] Upload the script file and create a virtual env
81 SetupUtils.Setup_Utils_For_Setup_And_Teardown
82 SetupUtils.Setup_Logging_For_Debug_Purposes_On_List_Or_All ${TEST_LOG_LEVEL} ${TEST_LOG_COMPONENTS}
83 ${mininet_conn_id} = SSHKeywords.Open_Connection_To_Tools_System prompt=~]>
84 Builtin.Set Suite Variable ${mininet_conn_id}
85 SSHLibrary.Put File ${CURDIR}/../../../../tools/odl-mdsal-clustering-tests/scripts/${TOOL_NAME} .
86 ${stdout} ${stderr} ${rc}= SSHLibrary.Execute Command ls return_stdout=True return_stderr=True
88 ${out_file} = Utils.Get_Log_File_Name ${TOOL_NAME}
89 BuiltIn.Set_Suite_Variable ${out_file}
90 SSHKeywords.Virtual_Env_Create
91 SSHKeywords.Virtual_Env_Install_Package requests
92 CarPeople.Set_Variables_For_Shard ${SHARD_NAME} shard_type=${SHARD_TYPE}
95 [Documentation] Stop the tool, remove virtual env and close ssh connection towards tools vm.
97 ${session} = Resolve_Http_Session_For_Member member_index=${new_leader_index}
98 # best effort to make sure cars are deleted in case more suites will run after this and the delete test case had trouble
99 ${rsp}= RequestsLibrary.Delete Request ${session} ${CARURL}
100 BuiltIn.Log ${rsp.status_code} : ${rsp.text}
101 SSHKeywords.Virtual_Env_Delete
102 Store_File_To_Workspace ${out_file} ${out_file}
103 SSHLibrary.Close All Connections
106 [Arguments] ${command} ${tool_opt}
107 [Documentation] Start the tool
108 # TODO: https://trello.com/c/rXsMu7iz/444-create-keywords-for-the-tool-start-and-stop-in-remotebash-robot
109 BuiltIn.Log ${command}
110 SSHKeywords.Virtual_Env_Activate_On_Current_Session log_output=${True}
111 ${output}= SSHLibrary.Write ${command} ${tool_opt} 2>&1 | tee ${out_file}
112 BuiltIn.Log ${output}
115 [Documentation] Stop the tool if still running.
116 # TODO: https://trello.com/c/rXsMu7iz/444-create-keywords-for-the-tool-start-and-stop-in-remotebash-robot
117 ${output}= SSHLibrary.Read
118 BuiltIn.Log ${output}
119 RemoteBash.Write_Bare_Ctrl_C
120 ${output}= SSHLibrary.Read_Until_Prompt
121 BuiltIn.Log ${output}
122 SSHKeywords.Virtual_Env_Deactivate_On_Current_Session log_output=${True}
125 [Arguments] ${exp_count} ${session}
126 [Documentation] Count car items in config ds and compare with expected number.
127 ${count} = Get_Cars_Count ${session}
128 BuiltIn.Should_Be_Equal_As_Numbers ${count} ${exp_count}
131 [Arguments] ${session}
132 [Documentation] Count car items in config ds.
133 ${resp}= RequestsLibrary.Get_Request ${session} ${CARURL}
134 ${count} = BuiltIn.Evaluate len(${resp.json()}[cars][car-entry])
135 BuiltIn.Return_From_Keyword ${count}
137 Ensure_Cars_Being_Configured
138 [Arguments] ${session}
139 [Documentation] FIXME: Add a documentation.
140 ${count1} = Get_Cars_Count ${session}
141 ${count2} = Get_Cars_Count ${session}
142 BuiltIn.Should_Not_Be_Equal_As_Integers ${count1} ${count2}
144 Store_File_To_Workspace
145 [Arguments] ${source_file_name} ${target_file_name}
146 [Documentation] Store the ${source_file_name} to the workspace as ${target_file_name}.
147 SSHLibrary.Get_File ${source_file_name} ${target_file_name}