git.opendaylight Code Review - integration/test.git/blob

1 *** Settings ***

2 Documentation Karaf library. This library is useful to deal with controller Karaf console for ssh sessions in cluster.

3 ... Running Setup_Karaf_Keywords is necessary. If SetupUtils initialization is called, this gets initialized as well.

4 ... If this gets initialized, ClusterManagement gets initialized as well.

5 Library SSHLibrary

6 Library OperatingSystem

7 Resource ${CURDIR}/ClusterManagement.robot

8 Resource ${CURDIR}/SSHKeywords.robot

9 Resource ${CURDIR}/Utils.robot

10 Variables ${CURDIR}/../variables/Variables.py

11

12 *** Variables ***

13 ${WORKSPACE} /tmp

14 ${connection_index_dict} &{EMPTY}

15

16 *** Keywords ***

17 Setup_Karaf_Keywords

18 [Arguments] ${http_timeout}=1

19 [Documentation] Initialize ClusterManagement. Open ssh karaf connections to each ODL.

20 ClusterManagement.ClusterManagement_Setup http_timeout=${http_timeout}

21 BuiltIn.Comment First connections to Karaf console may fail, so WUKS is used. TODO: Track as a Bug.

22 : FOR ${index} IN @{ClusterManagement__member_index_list}

23 \ BuiltIn.Run_Keyword_And_Ignore_Error BuiltIn.Wait_Until_Keyword_Succeeds 3s 1s Open_Controller_Karaf_Console_On_Background member_index=${index}

24

25 Verify_Feature_Is_Installed

26 [Arguments] ${feature_name} ${controller}=${ODL_SYSTEM_IP} ${karaf_port}=${KARAF_SHELL_PORT}

27 [Documentation] Will Succeed if the given ${feature_name} is found in the output of "feature:list -i"

28 ${output} = Issue_Command_On_Karaf_Console feature:list -i | grep ${feature_name} ${controller} ${karaf_port}

29 BuiltIn.Should_Contain ${output} ${feature_name}

30 [Return] ${output}

31

32 Issue_Command_On_Karaf_Console

33 [Arguments] ${cmd} ${controller}=${ODL_SYSTEM_IP} ${karaf_port}=${KARAF_SHELL_PORT} ${timeout}=10 ${loglevel}=INFO

34 [Documentation] Will execute the given ${cmd} by ssh'ing to the karaf console running on ${controller}

35 ... Note that this keyword will open&close new SSH connection, without switching back to previously current session.

36 SSHLibrary.Open_Connection ${controller} port=${karaf_port} prompt=${KARAF_PROMPT} timeout=${timeout}

37 SSHLibrary.Login ${KARAF_USER} ${KARAF_PASSWORD} loglevel=${loglevel}

38 SSHLibrary.Write ${cmd}

39 ${output} SSHLibrary.Read_Until ${KARAF_PROMPT}

40 SSHLibrary.Close_Connection

41 BuiltIn.Log ${output}

42 [Return] ${output}

43

44 Safe_Issue_Command_On_Karaf_Console

45 [Arguments] ${cmd} ${controller}=${ODL_SYSTEM_IP} ${karaf_port}=${KARAF_SHELL_PORT} ${timeout}=10 ${loglevel}=INFO

46 [Documentation] Run Issue_Command_On_Karaf_Console but restore previous connection afterwards.

47 BuiltIn.Run_Keyword_And_Return SSHKeywords.Run_Keyword_Preserve_Connection Issue_Command_On_Karaf_Console ${cmd} ${controller} ${karaf_port} ${timeout}

48 ... ${loglevel}

49

50 Check For Elements On Karaf Command Output Message

51 [Arguments] ${cmd} ${elements} ${controller}=${ODL_SYSTEM_IP} ${karaf_port}=${KARAF_SHELL_PORT} ${timeout}=5

52 [Documentation] Will execute the command using Issue Command On Karaf Console then check for the given elements

53 ... in the command output message

54 ${output} Issue_Command_On_Karaf_Console ${cmd} ${controller} ${karaf_port} ${timeout}

55 : FOR ${i} IN @{elements}

56 \ BuiltIn.Should_Contain ${output} ${i}

57

58 Verify_Bundle_Is_Installed

59 [Arguments] ${bundle_name} ${controller}=${ODL_SYSTEM_IP} ${karaf_port}=${KARAF_SHELL_PORT}

60 [Documentation] Will succeed if the given ${bundle name} is present in the output of "bundle:list -s "

61 ${output} = Issue_Command_On_Karaf_Console bundle:list -s | grep ${bundle_name} ${controller} ${karaf_port}

62 BuiltIn.Should_Contain ${output} ${bundle_name}

63 [Return] ${output}

64

65 Verify_Bundle_Is_Not_Installed

66 [Arguments] ${bundle_name} ${controller}=${ODL_SYSTEM_IP} ${karaf_port}=${KARAF_SHELL_PORT}

67 [Documentation] Will succeed if the given ${bundle_name} is NOT found in the output of "bundle:list -s"

68 ${output} = Issue_Command_On_Karaf_Console bundle:list -i | grep ${bundle_name} ${controller} ${karaf_port}

69 BuiltIn.Should_Not_Contain ${output} ${bundle_name}

70 [Return] ${output}

71

72 Check_Karaf_Log_Has_Messages

73 [Arguments] ${filter_string} @{message_list}

74 [Documentation] Will succeed if the @{messages} are found in \ the output of "log:display"

75 ${output} = Issue_Command_On_Karaf_Console log:display | grep ${filter_string}

76 : FOR ${message} IN @{message_list}

77 \ BuiltIn.Should_Contain ${output} ${message}

78 [Return] ${output}

79

80 Install_A_Feature

81 [Arguments] ${feature_name} ${controller}=${ODL_SYSTEM_IP} ${karaf_port}=${KARAF_SHELL_PORT} ${timeout}=180

82 [Documentation] Will Install the given ${feature_name}

83 BuiltIn.Log ${timeout}

84 ${output} = Issue_Command_On_Karaf_Console feature:install ${feature_name} ${controller} ${karaf_port} ${timeout}

85 BuiltIn.Log ${output}

86 [Return] ${output}

87

88 Install_A_Feature_Using_Active_Connection

89 [Arguments] ${feature_name}

90 [Documentation] Will Install the given ${feature_name} using active connection

91 ${cmd} = BuiltIn.Set_Variable feature:install ${feature_name}

92 SSHLibrary.Write ${cmd}

93 ${output} SSHLibrary.Read_Until ${KARAF_PROMPT}

94 BuiltIn.Log ${output}

95 [Return] ${output}

96

97 Uninstall_A_Feature

98 [Arguments] ${feature_name} ${controller}=${ODL_SYSTEM_IP} ${karaf_port}=${KARAF_SHELL_PORT} ${timeout}=180

99 [Documentation] Will UnInstall the given ${feature_name}

100 ${output} = Issue_Command_On_Karaf_Console feature:uninstall ${feature_name} ${controller} ${karaf_port} ${timeout}

101 BuiltIn.Log ${output}

102 [Return] ${output}

103

104 Open_Controller_Karaf_Console_On_Background

105 [Arguments] ${member_index}=${1} ${timeout}=10 ${loglevel}=INFO

106 [Documentation] If there is a stored ssh connection index of connection to the controller's karaf console for ${member_index},

107 ... close the previous connection. In any case create a new connection

108 ... to karaf console for ${member_index}, set correct prompt set and login to karaf console.

109 ... Store connection index for ${member_index} and restore the previous active connection.

110 ${current_ssh_connection_object}= SSHLibrary.Get_Connection

111 BuiltIn.Log ${connection_index_dict}

112 BuiltIn.Log ${member_index}

113 ${status} ${old_connection_index} = BuiltIn.Run_Keyword_And_Ignore_Error Get From Dictionary ${connection_index_dict} ${member_index}

114 BuiltIn.Run_Keyword_If '${status}'=='PASS' BuiltIn.Run_Keywords SSHLibrary.Switch_Connection ${old_connection_index}

115 ... AND SSHLibrary.Close_Connection

116 ${odl_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${member_index}

117 SSHLibrary.Open_Connection ${odl_ip} port=${KARAF_SHELL_PORT} prompt=${KARAF_DETAILED_PROMPT} timeout=${timeout}

118 ${karaf_connection_object} = SSHLibrary.Get_Connection

119 Collections.Set_To_Dictionary ${connection_index_dict} ${member_index} ${karaf_connection_object.index}

120 SSHLibrary.Login ${KARAF_USER} ${KARAF_PASSWORD} loglevel=${loglevel}

121 [Teardown] SSHKeywords.Restore_Current_Ssh_Connection_From_Index ${current_ssh_connection_object.index}

122

123 Open_Controller_Karaf_Console_With_Timeout

124 [Arguments] ${member_index}=${1} ${timeout}=3s

125 [Documentation] Open new connection to karaf console for member index with specified timeout.

126 BuiltIn.Log ${member_index}

127 ${odl_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${member_index}

128 SSHLibrary.Open_Connection ${odl_ip} port=${KARAF_SHELL_PORT} prompt=${KARAF_DETAILED_PROMPT} timeout=${timeout}

129 SSHLibrary.Login ${KARAF_USER} ${KARAF_PASSWORD}

130

131 Configure_Timeout_For_Karaf_Console

132 [Arguments] ${timeout} ${member_index_list}=${EMPTY}

133 [Documentation] Configure a different timeout for each Karaf console.

134 ${index_list} = ClusterManagement.List_Indices_Or_All given_list=${member_index_list}

135 ${current_connection_object} = SSHLibrary.Get_Connection

136 : FOR ${member_index} IN @{index_list} # usually: 1, 2, 3

137 \ ${karaf_connection_index} = Collections.Get_From_Dictionary ${connection_index_dict} ${member_index}

138 \ SSHLibrary.Switch_Connection ${karaf_connection_index}

139 \ SSHLibrary.Set_Client_Configuration timeout=${timeout}

140 [Teardown] SSHKeywords.Restore_Current_Ssh_Connection_From_Index ${current_connection_object.index}

141

142 Execute_Controller_Karaf_Command_On_Background

143 [Arguments] ${command} ${member_index}=${1}

144 [Documentation] Send command to karaf without affecting current SSH connection. Read, log and return response.

145 ${karaf_connection_index} = Collections.Get_From_Dictionary ${connection_index_dict} ${member_index}

146 ${current_connection_index} = SSHLibrary.Switch_Connection ${karaf_connection_index}

147 ${status_write} ${message_write} = BuiltIn.Run_Keyword_And_Ignore_Error SSHLibrary.Write ${command}

148 ${status_wait} ${message_wait} = BuiltIn.Run_Keyword_And_Ignore_Error SSHLibrary.Read_Until_Prompt

149 BuiltIn.Run Keyword If '${status_write}' != 'PASS' BuiltIn.Fail Failed to send the command: ${command}

150 BuiltIn.Log ${message_wait}

151 BuiltIn.Run_Keyword_If '${status_wait}' != 'PASS' BuiltIn.Fail Failed to see prompt after sending the command: ${command}

152 [Teardown] SSHKeywords.Restore_Current_Ssh_Connection_From_Index ${current_connection_index}

153 [Return] ${message_wait}

154

155 Execute_Controller_Karaf_Command_With_Retry_On_Background

156 [Arguments] ${command} ${member_index}=${1}

157 [Documentation] Attemp to send command to karaf for ${member_index}, if fail then open connection and try again.

158 ${status} ${message} = BuiltIn.Run_Keyword_And_Ignore_Error Execute_Controller_Karaf_Command_On_Background ${command} ${member_index}

159 BuiltIn.Return_From_Keyword_If '${status}' == 'PASS' ${message}

160 # TODO: Verify this does not leak connections indices.

161 Open_Controller_Karaf_Console_On_Background ${member_index}

162 ${message} = Execute_Controller_Karaf_Command_On_Background ${command} ${member_index}

163 [Return] ${message}

164

165 Log_Message_To_Controller_Karaf

166 [Arguments] ${message} ${member_index_list}=${EMPTY} ${tolerate_failure}=True

167 [Documentation] Make sure this resource is initialized. Send a message into the controller's karaf log file on every node listed (or all).

168 ... By default, failure while processing a node is silently ignored, unless ${tolerate_failure} is False.

169 ${index_list} = ClusterManagement.List_Indices_Or_All given_list=${member_index_list}

170 : FOR ${index} IN @{index_list} # usually: 1, 2, 3.

171 \ ${status} ${output} = BuiltIn.Run_Keyword_And_Ignore_Error Execute_Controller_Karaf_Command_With_Retry_On_Background log:log "ROBOT MESSAGE: ${message}" member_index=${index}

172 \ BuiltIn.Run_Keyword_Unless ${tolerate_failure} or "${status}" == "PASS" BuiltIn.Fail ${output}

173

174 Log_Test_Suite_Start_To_Controller_Karaf

175 [Arguments] ${member_index_list}=${EMPTY}

176 [Documentation] Log suite name to karaf log, useful in suite setup.

177 Log_Message_To_Controller_Karaf Starting suite ${SUITE_SOURCE} ${member_index_list}

178

179 Log_Testcase_Start_To_Controller_Karaf

180 [Arguments] ${member_index_list}=${EMPTY}

181 [Documentation] Log test case name to karaf log, useful in test case setup.

182 Log_Message_To_Controller_Karaf Starting test ${TEST_NAME} ${member_index_list}

183

184 Set_Bgpcep_Log_Levels

185 [Arguments] ${bgpcep_level}=${DEFAULT_BGPCEP_LOG_LEVEL} ${protocol_level}=${DEFAULT_PROTOCOL_LOG_LEVEL} ${member_index_list}=${EMPTY}

186 [Documentation] Assuming OCKCOB was used, set logging level on bgpcep and protocol loggers without affecting current SSH session.

187 # FIXME: Move to appropriate Resource

188 ${index_list} = ClusterManagement.List_Indices_Or_All given_list=${member_index_list}

189 : FOR ${index} IN @{index_list} # usually: 1, 2, 3.

190 \ Execute_Controller_Karaf_Command_On_Background log:set ${bgpcep_level} org.opendaylight.bgpcep member_index=${index}

191 \ Execute_Controller_Karaf_Command_On_Background log:set ${protocol_level} org.opendaylight.protocol member_index=${index}

192

193 Wait_For_Karaf_Log

194 [Arguments] ${message} ${timeout}=60 ${member_index}=${1}

195 [Documentation] Read karaf logs until message appear

196 # TODO: refactor this keyword to use the new workflow to account for multiple controllers. Initial work was done

197 # in this patch https://git.opendaylight.org/gerrit/#/c/45596/

198 # however, the consumers of this keyword were breaking after that change. Initial theory is that a previous

199 # keyword used before this "Wait For Karaf Log" keyword was closing the karaf console connection, so the

200 # "Flexible SSH Login" keyword from the patch above (45596) was failing.

201 BuiltIn.Log Waiting for '${message}' in karaf log

202 SSHLibrary.Open_Connection ${ODL_SYSTEM_IP} port=${KARAF_SHELL_PORT} prompt=${KARAF_PROMPT} timeout=${timeout}

203 SSHLibrary.Login ${KARAF_USER} ${KARAF_PASSWORD} loglevel=${loglevel}

204 SSHLibrary.Write log:tail

205 SSHLibrary.Read_Until ${message}

206 SSHLibrary.Close_Connection

207

208 Restart_Bundle

209 [Arguments] ${bundle_id}

210 [Documentation] Restarts bundle passed as argument. Note this operation is only for testing and not production environments

211 # TODO: prepare this for cluster environment and multiple controllers

212 Safe_Issue_Command_On_Karaf_Console bundle:restart -f $(bundle:id '${bundle_id}')

213

214 Restart_Karaf

215 [Documentation] Restarts Karaf and polls log to detect when Karaf is up and running again

216 # TODO: prepare this for cluster environment and multiple controllers

217 Safe_Issue_Command_On_Karaf_Console log:clear

218 Issue_Command_On_Karaf_Console shutdown -r -f

219 BuiltIn.Run_Keyword_And_Return_Status BuiltIn.Wait_Until_Keyword_Succeeds 240s 60s Wait_For_Karaf_Log Karaf started in

220

221 Start Karaf

222 [Documentation] Starts Karaf and polls log to detect when Karaf is up and running

223 # TODO: prepare this for cluster environment and multiple controllers

224 Run Command On Remote System ${ODL_SYSTEM_IP} /usr/bin/env JAVA_HOME=${JAVA_HOME} ${WORKSPACE}/${BUNDLEFOLDER}/bin/start

225 Run Keyword And Return Status Wait Until Keyword Succeeds 240s 60s Wait For Karaf Log Karaf started in

226

227 Stop Karaf

228 [Documentation] Stops Karaf and checks Karaf is not running any more

229 # TODO: prepare this for cluster environment and multiple controllers

230 Run Keyword And Ignore Error Safe_Issue_Command_On_Karaf_Console log:clear

231 Run Command On Remote System ${ODL_SYSTEM_IP} /usr/bin/env JAVA_HOME=${JAVA_HOME} ${WORKSPACE}/${BUNDLEFOLDER}/bin/stop

232 Wait Until Keyword Succeeds 240s 60s Karaf Status Not Running

233

234 Karaf Status Not Running

235 [Documentation] Checks Karaf status is Not Running

236 ${status} = Run Command On Remote System ${ODL_SYSTEM_IP} /usr/bin/env JAVA_HOME=${JAVA_HOME} ${WORKSPACE}/${BUNDLEFOLDER}/bin/status

237 Should Contain ${status} Not Running

238 [Return] ${status}

239

240 Restart Jetty

241 [Documentation] Restarts jetty bundle (to reload certificates or key/truststore information)

242 Safe_Issue_Command_On_Karaf_Console log:clear

243 Restart_Bundle OPS4J Pax Web - Jetty

244 Wait_For_Karaf_Log Instantiated the Application class org.opendaylight.restconf.RestconfApplication

245 Wait_For_Karaf_Log Instantiated the Application class org.opendaylight.netconf.sal.rest.impl.RestconfApplication

246 Wait_For_Karaf_Log Instantiated the Application class org.opendaylight.aaa.idm.IdmLightApplication