5 Resource DataModels.robot
7 Resource SSHKeywords.robot
9 Resource ../variables/Variables.robot
10 Variables ../variables/coe/Modules.py
11 Resource ToolsSystem.robot
15 ${BUSY_BOX} ${CURDIR}/../variables/coe/busy-box.yaml
16 ${CNI_BINARY_FILE} /opt/cni/bin/odlovs-cni
17 ${CONFIG_FILE} /etc/cni/net.d/odlovs-cni.conf
18 ${CONFIG_FILE_TEMPLATE} ${CURDIR}/../variables/coe/odlovs-cni.conf.j2
19 ${HOST_INVENTORY} ${CURDIR}/../variables/coe/hosts.yaml
20 ${K8s_MASTER_IP} ${TOOLS_SYSTEM_1_IP}
21 ${HOSTS_FILE_TEMPLATE} ${CURDIR}/../variables/coe/minions_template.yaml
22 ${NODE_READY_STATUS} \\sReady # The check using this variable should not mess up with NotReady
23 ${PLAYBOOK_FILE} ${CURDIR}/../variables/coe/coe_play.yaml
24 ${POD_RUNNING_STATUS} \\sRunning
25 ${VARIABLES_PATH} ${CURDIR}/../variables/coe
26 ${WATCHER_COE} ${CURDIR}/../variables/coe/coe.yaml
27 @{COE_DIAG_SERVICES} OPENFLOW IFM ITM DATASTORE ELAN OVSDB
32 [Documentation] COE project requires start suite to be executed only for the first test suite.This keyword find the current suite,compares it with the stored first suite value and executes Coe.Start suite only if the cuurent suite is equal to the first suite.
33 ToolsSystem.Get Tools System Nodes Data
34 Coe.Set Connection ids and Bridge
35 Coe.Derive Coe Data Models
36 ${current suite} ${suite names updated} = Extract current suite name
37 ${first_suite} = Set Variable ${suite names updated[0]}
38 ${status} = BuiltIn.Evaluate '${first_suite}' == '${current suite}'
39 IF '${status}' == 'True' Coe.Start Suite
42 [Documentation] Suite setup keyword.
43 Coe.Configuration Playbook
44 Coe.Verify Config Files
45 Coe.Verify Watcher Is Running
46 BuiltIn.Wait Until Keyword Succeeds 40s 2s Coe.Check Node Status Is Ready
48 BuiltIn.Wait Until Keyword Succeeds
51 ... ClusterManagement.Check Status Of Services Is OPERATIONAL
52 ... @{COE_DIAG_SERVICES}
54 Set Connection ids and Bridge
55 [Documentation] Sets the connection ids for all the nodes and get the bridge from configuration file .
56 FOR ${conn_id} IN @{TOOLS_SYSTEM_ALL_CONN_IDS}
57 SSHLibrary.Switch Connection ${conn_id}
58 SSHKeywords.Flexible_SSH_Login ${DEFAULT_USER} ${DEFAULT_PASSWORD}
60 ${file} = OperatingSystem.Get File ${CONFIG_FILE_TEMPLATE}
61 ${ovs bridge output} ${bridge} = BuiltIn.Should Match Regexp ${file} "ovsBridge": "(\\w.*)"
62 BuiltIn.Set Suite Variable ${bridge}
64 Configuration Playbook
65 [Documentation] Ansible playbook which does all basic configuration for kubernetes nodes.
66 ${playbook minions} ${playbook hosts} ${host file} = Modifying templates in playbook
67 ${playbook} = OperatingSystem.Get File ${PLAYBOOK_FILE}
68 ${playbook} = String.Replace String ${playbook} coe-hosts ${playbook hosts}
69 ${playbook} = String.Replace String ${playbook} coe-minions ${playbook minions}
70 OperatingSystem.Create File ${PLAYBOOK_FILE} ${playbook}
71 OperatingSystem.Create File ${USER_HOME}/hosts.yaml ${host file}
72 ${watcher} = OperatingSystem.Get File ${WATCHER_COE}
73 ${watcher} = String.Replace String ${watcher} odlip ${ODL_SYSTEM_IP}
74 ${watcher} = String.Replace String ${watcher} port ${RESTCONFPORT}
75 OperatingSystem.Create File ${WATCHER_COE} ${watcher}
76 SSHKeywords.Copy_File_To_Remote_System ${K8s_MASTER_IP} ${WATCHER_COE} ${USER_HOME}
77 OperatingSystem.Copy File ${PLAYBOOK_FILE} ${USER_HOME}
78 ${branch_ref_spec} = BuiltIn.Catenate SEPARATOR= refs/heads/ ${GERRIT_BRANCH}
79 ${gerrit_ref_spec} = BuiltIn.Set Variable If
80 ... '${GERRIT_PROJECT}' != 'coe'
81 ... ${branch_ref_spec}
83 Run Coe Playbook ${gerrit_ref_spec}
86 [Arguments] ${gerrit_ref_spec}
87 ${play_output} = OperatingSystem.Run
88 ... ansible-playbook -v ${USER_HOME}/coe_play.yaml -i ${USER_HOME}/hosts.yaml --extra-vars '{"gerrit_branch":"FETCH_HEAD","gerrit_refspec":"${gerrit_ref_spec}"}'
89 BuiltIn.Log ${play_output}
91 Modifying templates in playbook
92 ${inventory} = OperatingSystem.Get File ${HOST_INVENTORY}
93 ${template} = OperatingSystem.Get File ${HOSTS_FILE_TEMPLATE}
94 ${template} = String.Replace String ${template} minion_ip ${TOOLS_SYSTEM_ALL_IPS[0]}
95 @{minions} = Create List coe-minion
96 ${hosts} = Set Variable coe-master:
97 FOR ${i} IN RANGE 1 ${NUM_TOOLS_SYSTEM}
98 Append To List ${minions} coe-minion${i}
99 ${hosts} = Catenate ${hosts} coe-minion${i}:
101 ${hosts} = Replace String Using Regexp ${hosts} :$ ${EMPTY}
102 ${hosts} = Remove Space on String ${hosts}
103 ${minion hosts} = Replace String Using Regexp ${hosts} ^[\\w-]+: ${EMPTY}
104 FOR ${i} IN RANGE 1 ${NUM_TOOLS_SYSTEM}
105 ${j} = Evaluate ${i}+1
106 ${template} = String.Replace String ${template} ${minions[${i}-1]} ${minions[${i}]}
107 ${template} = String.Replace String
109 ... ${TOOLS_SYSTEM_ALL_IPS[${i}-1]}
110 ... ${TOOLS_SYSTEM_ALL_IPS[${i}]}
111 ${template} = String.Replace String ${template} 192.168.50.1${i} 192.168.50.1${j}
112 ${template} = String.Replace String ${template} 10.11.${i}.0/24 10.11.${j}.0/24
113 ${template} = String.Replace String ${template} 10.11.${i}.1 10.11.${j}.1
114 Append To File ${HOST_INVENTORY} ${template}
116 ${host file} = OperatingSystem.Get File ${HOST_INVENTORY}
117 ${host file} = String.Replace String ${host file} master_ip ${TOOLS_SYSTEM_ALL_IPS[0]}
118 ${host file} = String.Replace String ${host file} odl_ip ${ODL_SYSTEM_IP}
119 ${host file} = String.Replace String ${host file} mport ${OVSDBPORT}
120 ${host file} = String.Replace String ${host file} cport ${ODL_OF_PORT_6653}
121 ${host file} = String.Replace String ${host file} filepath ${CONFIG_FILE_TEMPLATE}
122 ${host file} = String.Replace String ${host file} yamlpath ${USER_HOME}/coe.yaml
124 RETURN ${minion hosts} ${hosts} ${host file}
127 [Documentation] Checks if the configuration files are present in all nodes
128 FOR ${nodes} IN @{TOOLS_SYSTEM_ALL_IPS}
129 Utils.Verify File Exists On Remote System ${nodes} ${CONFIG_FILE}
131 FOR ${nodes} IN @{TOOLS_SYSTEM_ALL_IPS}
132 Utils.Verify File Exists On Remote System ${nodes} ${CNI_BINARY_FILE}
135 Verify Watcher Is Running
136 [Documentation] Checks if watcher is running in the background
137 ${watcher status} = Utils.Run Command On Remote System ${K8s_MASTER_IP} ps -ef | grep watcher
138 BuiltIn.Should Match Regexp ${watcher status} .* watcher odl
140 Check Node Status Is Ready
141 [Documentation] Checks the status of nodes.This keyword is repeated until the status of all nodes is Ready
142 ${nodes} = Utils.Run Command On Remote System And Log
144 ... kubectl get nodes
146 ... ${DEFAULT_PASSWORD}
147 ... ${DEFAULT_LINUX_PROMPT_STRICT}
148 ${node_status} = String.Get Lines Matching Regexp ${nodes} ${NODE_READY_STATUS} partial_match=True
149 ${lines_containing_ready} = String.Get Line Count ${node_status}
150 BuiltIn.Should Be Equal As Strings ${lines_containing_ready} ${NUM_TOOLS_SYSTEM}
153 [Documentation] Create labels for minions so that random allocation of pods to minions is avoided
154 ${i} = BuiltIn.Set Variable 1
155 ${get nodes} = Utils.Run Command On Remote System And Log ${K8s_MASTER_IP} kubectl get nodes
156 @{get nodes} = String.Split To Lines ${get nodes} 2
157 FOR ${status} IN @{get nodes}
158 ${minion} = BuiltIn.Should Match Regexp ${status} ^\\w+-.*-\\d+
159 Utils.Run Command On Remote System And Log ${K8s_MASTER_IP} kubectl label nodes ${minion} disktype=ss${i}
160 ${i} = BuiltIn.Evaluate ${i}+1
162 Utils.Run Command On Remote System And Log ${K8s_MASTER_IP} kubectl get nodes --show-labels
164 Check Pod Status Is Running
165 [Documentation] Checks the status of pods.This keyword is repeated until the status of all pods is Running
166 ${pods} = Utils.Run Command On Remote System And Log
168 ... kubectl get pods -o wide
170 ... ${DEFAULT_PASSWORD}
171 ... ${DEFAULT_LINUX_PROMPT_STRICT}
172 @{cluster} = String.Split To Lines ${pods} 1
173 FOR ${pod} IN @{cluster}
174 BuiltIn.Should Match Regexp ${pod} ${POD_RUNNING_STATUS}
178 [Documentation] Test teardown to get dumpflows,ovsconfig,model dump,node status,pod status and to dump config files \ and delete pods.
179 FOR ${conn_id} IN @{TOOLS_SYSTEM_ALL_CONN_IDS}
180 OVSDB.Get DumpFlows And Ovsconfig ${conn_id} ${bridge}
182 BuiltIn.Run Keyword And Ignore Error DataModels.Get Model Dump ${ODL_SYSTEM_IP} ${coe_data_models}
184 Utils.Run Command On Remote System And Log
186 ... kubectl get nodes
188 ... ${DEFAULT_PASSWORD}
189 ... ${DEFAULT_LINUX_PROMPT_STRICT}
190 Utils.Run Command On Remote System And Log
192 ... kubectl get pods -o wide
194 ... ${DEFAULT_PASSWORD}
195 ... ${DEFAULT_LINUX_PROMPT_STRICT}
199 [Documentation] Waits till the keyword delete status succeeds implying that all pods created have been deleted
200 ${get pods} = Utils.Run Command On Remote System ${K8s_MASTER_IP} kubectl get pods -o wide
201 @{get pods} = String.Split To Lines ${get pods} 1
202 FOR ${status} IN @{get pods}
203 ${pod_name} = BuiltIn.Should Match Regexp ${status} ^\\w+-\\w+
204 Utils.Run Command On Remote System ${K8s_MASTER_IP} kubectl delete pods ${pod_name}
206 BuiltIn.Wait Until Keyword Succeeds 60s 3s Coe.Check If Pods Are Terminated
207 Coe.Check For Stale veth Ports
209 Check If Pods Are Terminated
210 [Documentation] Checks if the pods created have been terminated.The keyword is repeated until the pods are deleted
211 ${status} = Utils.Run Command On Remote System
213 ... kubectl get pods -o wide
215 ... ${DEFAULT_PASSWORD}
216 ... ${DEFAULT_LINUX_PROMPT_STRICT}
217 ... ${DEFAULT_TIMEOUT}
218 ... return_stdout=False
219 ... return_stderr=True
220 BuiltIn.Should Contain ${status} No resources
223 [Documentation] Logs the configuration files present in all nodes
224 FOR ${nodes} IN @{TOOLS_SYSTEM_ALL_IPS}
225 Utils.Run Command On Remote System And Log ${nodes} cat ${CONFIG_FILE}
229 [Documentation] Suite teardown keyword
230 Coe.Collect Watcher Log
231 Coe.Collect Journalctl Log
234 SSHLibrary.Close All Connections
237 [Documentation] Watcher running in background logs into watcher.out which is copied to ${JENKINS_WORKSPACE}/archives/watcher.log
238 SSHLibrary.Switch Connection ${TOOLS_SYSTEM_ALL_CONN_IDS[0]}
239 SSHLibrary.Get File /tmp/watcher.out ${JENKINS_WORKSPACE}/archives/watcher.log
241 Collect Journalctl Log
242 [Documentation] Logs of the command journalctl -u kubelet is copied to ${JENKINS_WORKSPACE}/archives/journal.log
243 Utils.Run Command On Remote System And Log
245 ... sudo journalctl -u kubelet > ${USER_HOME}/journal.txt
246 SSHLibrary.Switch Connection ${TOOLS_SYSTEM_ALL_CONN_IDS[0]}
247 SSHLibrary.Get File ${USER_HOME}/journal.txt ${JENKINS_WORKSPACE}/archives/journalctl.log
250 [Documentation] Kill the watcher running at the background after completion of tests cases
251 ${watcher status} = Utils.Run Command On Remote System ${K8s_MASTER_IP} ps -ef | grep watcher
252 ${watcher} ${pid} = BuiltIn.Should Match Regexp ${watcher status} \\w+\\s+(\\d+).*watcher odl
253 Utils.Run Command On Remote System ${K8s_MASTER_IP} kill -9 ${pid}
256 [Documentation] Reset K8s to clear up all stale entries
257 FOR ${nodes} IN @{TOOLS_SYSTEM_ALL_IPS}
258 ${kube} = Utils.Run Command On Remote System And Log ${nodes} sudo kubeadm reset
259 BuiltIn.Should Contain ${kube} Stopping the kubelet service.
263 [Documentation] Creates pods using the labels of the nodes and busy box names passed as arguments.
264 [Arguments] ${label} ${yaml} ${name}
265 ${busybox} = OperatingSystem.Get File ${BUSY_BOX}
266 ${busybox} = String.Replace String ${busybox} string ${label}
267 ${busybox} = String.Replace String ${busybox} busyboxname ${name}
268 OperatingSystem.Create File ${VARIABLES_PATH}/${yaml} ${busybox}
269 SSHKeywords.Move_file_To_Remote_System ${K8s_MASTER_IP} ${VARIABLES_PATH}/${yaml} ${USER_HOME}
270 Utils.Run Command On Remote System And Log ${K8s_MASTER_IP} kubectl create -f ${yaml}
272 Collect Pod Names and Ping
273 [Documentation] This keyword collects the pod names and checks connectivity between each and every pod with respect to one another.
274 SSHLibrary.Switch Connection ${TOOLS_SYSTEM_ALL_CONN_IDS[0]}
275 ${get pods} = Write Commands Until Expected Prompt kubectl get pods -o wide ${DEFAULT_LINUX_PROMPT_STRICT}
276 @{pod ips} = String.Get Regexp Matches ${get pods} \\d+\\.\\d+\\.\\d+\\.\\d+
277 @{pod names} = String.Get Regexp Matches ${get pods} ss\\w+-\\w+
278 FOR ${pod_name} IN @{pod names}
279 ${logs} = Log Statements ${pod ips} ${pod names} ${pod_name}
280 Ping Pods ${pod_name} ${pod ips} ${logs}
284 [Arguments] ${pod ips} ${pod names} ${pod_name}
285 @{log statement} = Create List
286 ${i} = Set Variable 0
287 FOR ${pod_ip} IN @{pod ips}
288 ${ping statement} = Set Variable Ping from ${pod_name} to ${pod names[${i}]} (${pod ip})
289 Append To List ${log statement} ${ping statement}
290 ${i} = Evaluate ${i}+1
292 RETURN @{log statement}
295 [Arguments] ${pod_name} ${pod ips} ${logs}
296 ${i} = Set Variable 0
297 FOR ${ping info} IN @{logs}
298 ${ping} = Write Commands Until Expected Prompt
299 ... kubectl exec -it ${pod_name} -- ping -c 3 ${pod ips[${i}]}
300 ... ${DEFAULT_LINUX_PROMPT_STRICT}
302 BuiltIn.Should Contain ${ping} 64 bytes
303 ${i} = Evaluate ${i}+1
307 [Documentation] COE project requires stop suite to be executed only for the last test suite.This keyword find the current suite,compares it with the stored last suite value and executes Coe.Stop suite only if the cuurent suite is equal to the last suite.
308 ${current suite} ${suite names updated} = Extract current suite name
309 ${last_suite} = Set Variable ${suite names updated[-1]}
310 ${status} = BuiltIn.Evaluate '${last_suite}' == '${current suite}'
311 IF '${status}' == 'True' Coe.Stop Suite
313 Extract current suite name
314 [Documentation] This keyword returns the name of current test suite.Appropriate replacement in text is done to make test suite names in SUITES and SUITE_NAME similar.
315 BuiltIn.Log SUITE_NAME: ${SUITE_NAME}
316 BuiltIn.Log SUITES: ${SUITES}
317 @{suite_names} = Get Regexp Matches ${SUITES} coe\\/(\\w+).robot 1
318 @{suite_names_updated} = Create List
319 FOR ${suite} IN @{suite_names}
320 ${suite} = Replace String ${suite} _ ${SPACE}
321 Append To List ${suite_names_updated} ${suite}
323 ${num_suites} = BuiltIn.Get Length ${suite_names_updated}
324 IF ${num_suites} > ${1}
325 ${suite line} ${current_suite} = Should Match Regexp ${SUITE_NAME} .txt.(\\w.*)
327 ${suite line} ${current_suite} = BuiltIn.Set Variable
328 ... ${suite_names_updated}[0]
329 ... ${suite_names_updated}[0]
331 RETURN ${current_suite} ${suite_names_updated}
333 Check For Stale veth Ports
334 [Documentation] Check on switches(except master) where pods were created and deleted to ensure there are no stale veth ports left behind.
335 FOR ${minion_index} IN RANGE 2 ${NUM_TOOLS_SYSTEM}+1
336 ${switch output} = Utils.Run Command On Remote System And Log
337 ... ${TOOLS_SYSTEM_${minion_index}_IP}
338 ... sudo ovs-vsctl show
339 BuiltIn.Should Not Contain ${switch output} veth