2 Library RequestsLibrary
5 Library ClusterStateLibrary.py
9 ${smc_node} /org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore
13 [Arguments] ${exclude_controller}=${EMPTY}
14 [Documentation] Creates a list of all controllers minus any excluded controller.
15 Log ${exclude_controller}
16 @{searchlist} Create List ${CONTROLLER} ${CONTROLLER1} ${CONTROLLER2}
17 Remove Values From List ${searchlist} ${exclude_controller}
19 [Return] ${searchlist}
22 [Arguments] ${shard_name} ${old_leader}=${EMPTY}
23 [Documentation] Returns the IP addr or hostname of the leader of the specified shard.
24 ... Controllers are specifed in the pybot command line.
25 ${searchlist} Get Controller List ${old_leader}
26 ${leader} GetLeader ${shard_name} ${3} ${3} ${1} ${RESTCONFPORT}
28 Should Not Be Equal As Strings ${leader} None
29 Run Keyword If '${old_leader}'!='${EMPTY}' Should Not Be Equal ${old_leader} ${leader}
33 [Arguments] ${shard_name}
34 [Documentation] No leader is elected in the car shard
35 ${leader} GetLeader ${shard_name} ${3} ${1} ${1} ${RESTCONFPORT}
36 ... ${CURRENT_CAR_LEADER}
37 Should Be Equal As Strings ${leader} None
40 [Arguments] ${shard_name} ${exclude_controller}=${EMPTY}
41 [Documentation] Returns the IP addresses or hostnames of all followers of the specified shard.
42 ${searchlist} Get Controller List ${exclude_controller}
43 ${followers} GetFollowers ${shard_name} ${3} ${3} ${1} ${RESTCONFPORT}
46 Should Not Be Empty ${followers}
49 Stop One Or More Controllers
50 [Arguments] @{controllers}
51 [Documentation] Give this keyword a scalar or list of controllers to be stopped.
52 ${cmd} = Set Variable ${KARAF_HOME}/bin/stop
53 : FOR ${ip} IN @{controllers}
54 \ Run Command On Remote System ${ip} ${cmd}
56 Kill One Or More Controllers
57 [Arguments] @{controllers}
58 [Documentation] Give this keyword a scalar or list of controllers to be stopped.
59 ${cmd} = Set Variable ps axf | grep karaf | grep -v grep | awk '{print \"kill -9 \" $1}' | sh
61 : FOR ${ip} IN @{controllers}
62 \ Run Command On Remote System ${ip} ${cmd}
65 [Arguments] ${timeout} @{controllers}
66 [Documentation] Waits for one or more clustered controllers to be down.
67 : FOR ${ip} IN @{controllers}
68 \ ${status}= Run Keyword And Return Status Wait For Controller Down ${timeout} ${ip}
69 \ Exit For Loop If '${status}' == 'FAIL'
71 Wait For Controller Down
72 [Arguments] ${timeout} ${ip}
73 [Documentation] Waits for one controllers to be down.
74 Wait Until Keyword Succeeds ${timeout} 2s Controller Down Check ${ip}
78 [Documentation] Checks to see if a controller is down by verifying that the karaf process isn't present.
79 ${cmd} = Set Variable ps axf | grep karaf | grep -v grep | wc -l
80 ${response} Run Command On Remote System ${ip} ${cmd}
81 Log Number of controller instances running: ${response}
82 Should Start With ${response} 0 Controller process found or there may be extra instances of karaf running on the host machine.
84 Start One Or More Controllers
85 [Arguments] @{controllers}
86 [Documentation] Give this keyword a scalar or list of controllers to be started.
87 ${cmd} = Set Variable ${KARAF_HOME}/bin/start
88 : FOR ${ip} IN @{controllers}
89 \ Run Command On Remote System ${ip} ${cmd}
92 [Arguments] ${timeout} @{controllers}
93 [Documentation] Waits for one or more clustered controllers to report Sync Status as true.
94 : FOR ${ip} IN @{controllers}
95 \ ${status}= Run Keyword And Return Status Wait For Controller Sync ${timeout} ${ip}
96 \ Exit For Loop If '${status}' == 'FAIL'
98 Wait For Controller Sync
99 [Arguments] ${timeout} ${ip}
100 [Documentation] Waits for one controllers to report Sync Status as true.
101 Wait Until Keyword Succeeds ${timeout} 2s Controller Sync Status Should Be True ${ip}
103 Controller Sync Status Should Be True
105 [Documentation] Checks if Sync Status is true.
106 ${SyncStatus}= Get Controller Sync Status ${ip}
107 Should Be Equal ${SyncStatus} ${True}
109 Controller Sync Status Should Be False
111 [Documentation] Checks if Sync Status is false.
112 ${SyncStatus}= Get Controller Sync Status ${ip}
113 Should Be Equal ${SyncStatus} ${False}
115 Get Controller Sync Status
116 [Arguments] ${controller_ip}
117 [Documentation] Return Sync Status.
118 ${api} Set Variable /jolokia/read
119 Create_Session session http://${controller_ip}:${RESTCONFPORT}${api} headers=${HEADERS} auth=${AUTH}
120 ${resp}= RequestsLibrary.Get session ${smc_node}
123 ${json}= Set Variable ${resp.json()}
124 ${value}= Get From Dictionary ${json} value
126 ${SyncStatus}= Get From Dictionary ${value} SyncStatus
127 Log SyncSatus: ${SyncStatus}
128 [Return] ${SyncStatus}
130 Clean One Or More Journals
131 [Arguments] @{controllers}
132 [Documentation] Give this keyword a scalar or list of controllers on which to clean journals.
133 ${del_cmd} = Set Variable rm -rf ${KARAF_HOME}/journal
134 : FOR ${ip} IN @{controllers}
135 \ Run Command On Remote System ${ip} ${del_cmd}
137 Clean One Or More Snapshots
138 [Arguments] @{controllers}
139 [Documentation] Give this keyword a scalar or list of controllers on which to clean snapshots.
140 ${del_cmd} = Set Variable rm -rf ${KARAF_HOME}/snapshots
141 : FOR ${ip} IN @{controllers}
142 \ Run Command On Remote System ${ip} ${del_cmd}
144 Show Cluster Configuation Files
145 [Arguments] @{controllers}
146 [Documentation] Prints out the cluster configuration files for one or more controllers.
147 Log controllers: @{controllers}
148 ${cmd} = Set Variable cat ${KARAF_HOME}/configuration/initial/akka.conf
149 : FOR ${ip} IN @{controllers}
150 \ Run Command On Remote System ${ip} ${cmd}
151 ${cmd} = Set Variable cat ${KARAF_HOME}/configuration/initial/modules.conf
152 : FOR ${ip} IN @{controllers}
153 \ Run Command On Remote System ${ip} ${cmd}
154 ${cmd} = Set Variable cat ${KARAF_HOME}/configuration/initial/module-shards.conf
155 : FOR ${ip} IN @{controllers}
156 \ Run Command On Remote System ${ip} ${cmd}
157 ${cmd} = Set Variable cat ${KARAF_HOME}/configuration/initial/jolokia.xml
158 : FOR ${ip} IN @{controllers}
159 \ Run Command On Remote System ${ip} ${cmd}
160 ${cmd} = Set Variable cat ${KARAF_HOME}/etc/initial/org.apache.karaf.management.cfg
161 : FOR ${ip} IN @{controllers}
162 \ Run Command On Remote System ${ip} ${cmd}
163 ${cmd} = Set Variable cat ${KARAF_HOME}/etc/org.apache.karaf.features.cfg
164 : FOR ${ip} IN @{controllers}
165 \ Run Command On Remote System ${ip} ${cmd}
167 Isolate a Controller From Cluster
168 [Arguments] ${isolated controller} @{controllers}
169 [Documentation] Use IPTables to isolate one controller from the cluster.
170 ... On the isolated controller it blocks IP traffic to and from each of the other controllers.
171 : FOR ${controller} IN @{controllers}
172 \ ${other controller}= Evaluate "${isolated controller}" != "${controller}"
173 \ Run Keyword If ${other controller} Isolate One Controller From Another ${isolated controller} ${controller}
175 Rejoin a Controller To Cluster
176 [Arguments] ${isolated controller} @{controllers}
177 [Documentation] Use IPTables to rejoin one controller to the cluster.
178 ... On the isolated controller it unblocks IP traffic to and from each of the other controllers.
179 : FOR ${controller} IN @{controllers}
180 \ ${other controller}= Evaluate "${isolated controller}" != "${controller}"
181 \ Run Keyword If ${other controller} Rejoin One Controller To Another ${isolated controller} ${controller}
183 Isolate One Controller From Another
184 [Arguments] ${isolated controller} ${controller}
185 [Documentation] Inserts an IPTable rule to disconnect one controller from another controller in the cluster.
186 Modify IPTables ${isolated controller} ${controller} -I
188 Rejoin One Controller To Another
189 [Arguments] ${isolated controller} ${controller}
190 [Documentation] Deletes an IPTable rule, allowing one controller to reconnect to another controller in the cluster.
191 Modify IPTables ${isolated controller} ${controller} -D
194 [Arguments] ${isolated controller} ${controller} ${rule type}
195 [Documentation] Adds a rule, usually inserting or deleting an entry between two controllers.
196 ${base string} Set Variable sudo iptables ${rule type} OUTPUT -p all --source
197 ${cmd string} Catenate ${base string} ${isolated controller} --destination ${controller} -j DROP
198 Run Command On Remote System ${isolated controller} ${cmd string}
199 ${cmd string} Catenate ${base string} ${controller} --destination ${isolated controller} -j DROP
200 Run Command On Remote System ${isolated controller} ${cmd string}
201 ${cmd string} Set Variable sudo iptables -L -n
202 ${return string}= Run Command On Remote System ${isolated controller} ${cmd string}
204 Run Keyword If "${rule type}" == '-I' Should Match Regexp ${return string} [\s\S]*DROP *all *-- *${isolated controller} *${controller}[\s\S]*
205 Run Keyword If "${rule type}" == '-I' Should Match Regexp ${return string} [\s\S]*DROP *all *-- *${controller} *${isolated controller}[\s\S]*
207 Run Keyword If "${rule type}" == '-D' Should Match Regexp ${return string} (?![\s\S]*DROP *all *-- *${isolated controller} *${controller}[\s\S]*)
208 Run Keyword If "${rule type}" == '-D' Should Match Regexp ${return string} (?![\s\S]*DROP *all *-- *${controller} *${isolated controller}[\s\S]*)
210 Rejoin All Isolated Controllers
211 [Arguments] @{controllers}
212 [Documentation] Wipe all IPTables rules from all controllers, thus rejoining all controllers.
213 : FOR ${isolated controller} IN @{controllers}
214 \ Flush IPTables ${isolated controller}
217 [Arguments] ${isolated controller}
218 [Documentation] This keyword is generally not called from a test case but supports a complete wipe of all rules on
220 ${cmd string} Set Variable sudo iptables -v -F
221 ${return string}= Run Command On Remote System ${isolated controller} ${cmd string}
222 Log return: ${return string}
223 Should Contain ${return string} Flushing chain `INPUT'
224 Should Contain ${return string} Flushing chain `FORWARD'
225 Should Contain ${return string} Flushing chain `OUTPUT'