csit/libraries/ClusterManagement.robot

   1 *** Settings ***
   2 Documentation       Resource housing Keywords common to several suites for cluster functional testing.
   3 ...
   4 ...                 Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
   5 ...                 Copyright (c) 2016 Brocade Communications Systems, Inc. and others. All rights reserved.
   6 ...
   7 ...                 This program and the accompanying materials are made available under the
   8 ...                 terms of the Eclipse Public License v1.0 which accompanies this distribution,
   9 ...                 and is available at http://www.eclipse.org/legal/epl-v10.html
  10 ...
  11 ...
  12 ...                 This resource holds private state (in suite variables),
  13 ...                 which is generated once at Setup with ClusterManagement_Setup KW.
  14 ...                 The state includes member indexes, IP addresses and Http (RequestsLibrary) sessions.
  15 ...                 Cluster Keywords normally use member index, member list or nothing (all members) as argument.
  16 ...
  17 ...                 All index lists returned should be sorted numerically, fix if not.
  18 ...
  19 ...                 Requirements:
  20 ...                 odl-jolokia is assumed to be installed.
  21 ...
  22 ...                 Keywords are ordered as follows:
  23 ...                 - Cluster Setup
  24 ...                 - Shard state, leader and followers
  25 ...                 - Entity Owner, candidates and successors
  26 ...                 - Kill, Stop and Start Member
  27 ...                 - Isolate and Rejoin Member
  28 ...                 - Run Commands On Member
  29 ...                 - REST requests and checks on Members
  30 ...
  31 ...                 TODO: Unify capitalization of Leaders and Followers.
  32
  33 Library             RequestsLibrary    # for Create_Session and To_Json
  34 Library             Collections
  35 Library             String
  36 Library             ClusterEntities.py
  37 Resource            ${CURDIR}/CompareStream.robot
  38 Resource            ${CURDIR}/KarafKeywords.robot
  39 Resource            ${CURDIR}/SSHKeywords.robot
  40 Resource            ${CURDIR}/TemplatedRequests.robot    # for Get_As_Json_From_Uri
  41 Resource            ${CURDIR}/Utils.robot    # for Run_Command_On_Controller
  42 Resource            ../variables/Variables.robot
  43
  44
  45 *** Variables ***
  46 ${RESTCONF_URI}                                 rests
  47 ${GC_LOG_PATH}                                  ${KARAF_HOME}/data/log
  48 ${JAVA_HOME}                                    ${EMPTY}    # releng/builder scripts should provide correct value
  49 ${JOLOKIA_CONF_SHARD_MANAGER_URI}
  50 ...                                             jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore
  51 ${JOLOKIA_OPER_SHARD_MANAGER_URI}
  52 ...                                             jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-operational,type=DistributedOperationalDatastore
  53 ${JOLOKIA_CONFIG_LOCAL_SHARDS_URI}
  54 ...                                             jolokia/read/org.opendaylight.controller:type=DistributedConfigDatastore,Category=ShardManager,name=shard-manager-config/LocalShards
  55 ${JOLOKIA_OPER_LOCAL_SHARDS_URI}
  56 ...                                             jolokia/read/org.opendaylight.controller:type=DistributedOperationalDatastore,Category=ShardManager,name=shard-manager-operational/LocalShards
  57 ${JOLOKIA_READ_URI}                             jolokia/read/org.opendaylight.controller
  58 # Bug 9044 workaround: delete etc/host.key before restart.
  59 @{ODL_DEFAULT_DATA_PATHS}
  60 ...                                             tmp/
  61 ...                                             data/
  62 ...                                             cache/
  63 ...                                             snapshots/
  64 ...                                             journal/
  65 ...                                             segmented-journal/
  66 ...                                             etc/opendaylight/current/
  67 ...                                             etc/host.key
  68 ${RESTCONF_MODULES_DIR}                         ${CURDIR}/../variables/restconf/modules
  69 ${SINGLETON_NETCONF_DEVICE_ID_PREFIX}
  70 ...                                             KeyedInstanceIdentifier{targetType=interface org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node, path=[org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NetworkTopology, org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.Topology[key=TopologyKey{_topologyId=Uri{_value=topology-netconf}}], org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node[key=NodeKey{_nodeId=Uri{_value=
  71 ${SINGLETON_NETCONF_DEVICE_ID_SUFFIX}           }}]]}
  72 ${SINGLETON_BGPCEP_DEVICE_ID_SUFFIX}            -service-group
  73 &{SINGLETON_DEVICE_ID_PREFIX}
  74 ...                                             bgpcep=${EMPTY}
  75 ...                                             netconf=${SINGLETON_NETCONF_DEVICE_ID_PREFIX}
  76 ...                                             openflow=${EMPTY}
  77 ...                                             sxp=${EMPTY}
  78 &{SINGLETON_DEVICE_ID_SUFFIX}
  79 ...                                             bgpcep=${SINGLETON_BGPCEP_DEVICE_ID_SUFFIX}
  80 ...                                             netconf=${SINGLETON_NETCONF_DEVICE_ID_SUFFIX}
  81 ...                                             openflow=${EMPTY}
  82 ...                                             sxp=${EMPTY}
  83 ${SINGLETON_ELECTION_ENTITY_TYPE}               org.opendaylight.mdsal.ServiceEntityType
  84 ${SINGLETON_CHANGE_OWNERSHIP_ENTITY_TYPE}       org.opendaylight.mdsal.AsyncServiceCloseEntityType
  85 ${NODE_ROLE_INDEX_START}                        1
  86 ${NODE_START_COMMAND}                           ${KARAF_HOME}/bin/start
  87 ${NODE_STOP_COMMAND}                            ${KARAF_HOME}/bin/stop
  88 ${NODE_KARAF_COUNT_COMMAND}                     ps axf | grep org.apache.karaf | grep -v grep | wc -l
  89 ${NODE_KILL_COMMAND}
  90 ...                                             ps axf | grep org.apache.karaf | grep -v grep | awk '{print \"kill -9 \" $1}' | sh
  91 ${NODE_FREEZE_COMMAND}
  92 ...                                             ps axf | grep org.apache.karaf | grep -v grep | awk '{print \"kill -STOP \" $1}' | sh
  93 ${NODE_UNFREEZE_COMMAND}
  94 ...                                             ps axf | grep org.apache.karaf | grep -v grep | awk '{print \"kill -CONT \" $1}' | sh
  95
  96
  97 *** Keywords ***
  98 ClusterManagement_Setup
  99     [Documentation]    Detect repeated call, or detect number of members and initialize derived suite variables.
 100     ...    Http sessions are created with parameters to not waste time when ODL is no accepting connections properly.
 101     [Arguments]    ${http_timeout}=${DEFAULT_TIMEOUT_HTTP}    ${http_retries}=0
 102     # Avoid multiple initialization by several downstream libraries.
 103     ${already_done} =    BuiltIn.Get_Variable_Value    \${ClusterManagement__has_setup_run}    False
 104     IF    ${already_done}    RETURN
 105     BuiltIn.Set_Suite_Variable    \${ClusterManagement__has_setup_run}    True
 106     ${cluster_size} =    BuiltIn.Get_Variable_Value    \${NUM_ODL_SYSTEM}    1
 107     ${status}    ${possibly_int_of_members} =    BuiltIn.Run_Keyword_And_Ignore_Error
 108     ...    BuiltIn.Convert_To_Integer
 109     ...    ${cluster_size}
 110     ${int_of_members} =    BuiltIn.Set_Variable_If    '${status}' != 'PASS'    ${1}    ${possibly_int_of_members}
 111     ClusterManagement__Compute_Derived_Variables
 112     ...    int_of_members=${int_of_members}
 113     ...    http_timeout=${http_timeout}
 114     ...    http_retries=${http_retries}
 115
 116 Check_Cluster_Is_In_Sync
 117     [Documentation]    Fail if no-sync is detected on a member from list (or any).
 118     [Arguments]    ${member_index_list}=${EMPTY}
 119     ${index_list} =    List_Indices_Or_All    given_list=${member_index_list}
 120     FOR    ${index}    IN    @{index_list}    # usually: 1, 2, 3.
 121         ${status} =    Get_Sync_Status_Of_Member    member_index=${index}
 122         IF    'True' == '${status}'            CONTINUE
 123         BuiltIn.Fail    Index ${index} has incorrect status: ${status}
 124     END
 125
 126 Get_Sync_Status_Of_Member
 127     [Documentation]    Obtain IP, two GETs from jolokia URIs, return combined sync status as string.
 128     [Arguments]    ${member_index}
 129     ${session} =    Resolve_Http_Session_For_Member    member_index=${member_index}
 130     ${conf_text} =    TemplatedRequests.Get_As_Json_From_Uri
 131     ...    uri=${JOLOKIA_CONF_SHARD_MANAGER_URI}
 132     ...    session=${session}
 133     ${conf_status} =    ClusterManagement__Parse_Sync_Status    shard_manager_text=${conf_text}
 134     IF    'False' == ${conf_status}    RETURN    False
 135     ${oper_text} =    TemplatedRequests.Get_As_Json_From_Uri
 136     ...    uri=${JOLOKIA_OPER_SHARD_MANAGER_URI}
 137     ...    session=${session}
 138     ${oper_status} =    ClusterManagement__Parse_Sync_Status    shard_manager_text=${oper_text}
 139     RETURN    ${oper_status}
 140
 141 Verify_Leader_Exists_For_Each_Shard
 142     [Documentation]    For each shard name, call Get_Leader_And_Followers_For_Shard.
 143     ...    Not much logic there, but single Keyword is useful when using BuiltIn.Wait_Until_Keyword_Succeeds.
 144     [Arguments]    ${shard_name_list}    ${shard_type}=operational    ${member_index_list}=${EMPTY}    ${verify_restconf}=True
 145     FOR    ${shard_name}    IN    @{shard_name_list}
 146         Get_Leader_And_Followers_For_Shard
 147         ...    shard_name=${shard_name}
 148         ...    shard_type=${shard_type}
 149         ...    validate=True
 150         ...    member_index_list=${member_index_list}
 151         ...    verify_restconf=${verify_restconf}
 152     END
 153
 154 Get_Leader_And_Followers_For_Shard
 155     [Documentation]    Get role lists, validate there is one leader, return the leader and list of followers.
 156     ...    Optionally, issue GET to a simple restconf URL to make sure subsequent operations will not encounter 503.
 157     [Arguments]    ${shard_name}=default    ${shard_type}=operational    ${validate}=True    ${member_index_list}=${EMPTY}    ${verify_restconf}=True    ${http_timeout}=${EMPTY}
 158     ${leader_list}    ${follower_list} =    Get_State_Info_For_Shard
 159     ...    shard_name=${shard_name}
 160     ...    shard_type=${shard_type}
 161     ...    validate=True
 162     ...    member_index_list=${member_index_list}
 163     ...    verify_restconf=${verify_restconf}
 164     ...    http_timeout=${http_timeout}
 165     ${leader_count} =    BuiltIn.Get_Length    ${leader_list}
 166     IF    ${leader_count} < 1    BuiltIn.Fail    No leader found.
 167     BuiltIn.Length_Should_Be    ${leader_list}    ${1}    Too many Leaders.
 168     ${leader} =    Collections.Get_From_List    ${leader_list}    0
 169     RETURN    ${leader}    ${follower_list}
 170
 171 Get_State_Info_For_Shard
 172     [Documentation]    Return lists of Leader and Follower member indices from a given member index list
 173     ...    (or from the full list if empty). If \${shard_type} is not 'config', 'operational' is assumed.
 174     ...    If \${validate}, Fail if raft state is not Leader or Follower (for example on Candidate).
 175     ...    The biggest difference from Get_Leader_And_Followers_For_Shard
 176     ...    is that no check on number of Leaders is performed.
 177     [Arguments]    ${shard_name}=default    ${shard_type}=operational    ${validate}=False    ${member_index_list}=${EMPTY}    ${verify_restconf}=False    ${http_timeout}=${EMPTY}
 178     ${index_list} =    List_Indices_Or_All    given_list=${member_index_list}
 179     Collections.Sort_List    ${index_list}    # to guarantee return values are also sorted lists
 180     # TODO: Support alternative capitalization of 'config'?
 181     ${ds_type} =    BuiltIn.Set_Variable_If    '${shard_type}' != 'config'    operational    config
 182     ${leader_list} =    BuiltIn.Create_List
 183     ${follower_list} =    BuiltIn.Create_List
 184     FOR    ${index}    IN    @{index_list}    # usually: 1, 2, 3.
 185         ${raft_state} =    Get_Raft_State_Of_Shard_At_Member
 186         ...    shard_name=${shard_name}
 187         ...    shard_type=${ds_type}
 188         ...    member_index=${index}
 189         ...    verify_restconf=${verify_restconf}
 190         ...    http_timeout=${http_timeout}
 191         IF    'Follower' == '${raft_state}'
 192             Collections.Append_To_List    ${follower_list}    ${index}
 193         ELSE IF    'Leader' == '${raft_state}'
 194             Collections.Append_To_List    ${leader_list}    ${index}
 195         ELSE IF    ${validate}
 196             BuiltIn.Fail    Unrecognized Raft state: ${raft_state}
 197         END
 198     END
 199     RETURN    ${leader_list}    ${follower_list}
 200
 201 Get_Raft_State_Of_Shard_At_Member
 202     [Documentation]    Send request to Jolokia on indexed member, return extracted Raft status.
 203     ...    Optionally, check restconf works.
 204     [Arguments]    ${shard_name}    ${shard_type}    ${member_index}    ${verify_restconf}=False    ${http_timeout}=${EMPTY}
 205     ${raft_state} =    Get_Raft_Property_From_Shard_Member
 206     ...    RaftState
 207     ...    ${shard_name}
 208     ...    ${shard_type}
 209     ...    ${member_index}
 210     ...    verify_restconf=${verify_restconf}
 211     ...    http_timeout=${http_timeout}
 212     RETURN    ${raft_state}
 213
 214 Get_Raft_State_Of_Shard_Of_All_Member_Nodes
 215     [Documentation]    Get raft state of shard of all member nodes
 216     [Arguments]    ${shard_name}=default    ${shard_type}=config    ${member_index_list}=${EMPTY}
 217     ${index_list} =    List_Indices_Or_All    given_list=${member_index_list}
 218     Collections.Sort_List    ${index_list}
 219     FOR    ${index}    IN    @{index_list}
 220         ClusterManagement.Get Raft State Of Shard At Member
 221         ...    shard_name=${shard_name}
 222         ...    shard_type=${shard_type}
 223         ...    member_index=${index}
 224     END
 225
 226 Get_Raft_Property_From_Shard_Member
 227     [Documentation]    Send request to Jolokia on indexed member, return extracted Raft property.
 228     ...    Optionally, check restconf works.
 229     [Arguments]    ${property}    ${shard_name}    ${shard_type}    ${member_index}    ${verify_restconf}=False    ${http_timeout}=${EMPTY}
 230     ${session} =    Resolve_Http_Session_For_Member    member_index=${member_index}
 231     # TODO: Does the used URI tend to generate large data which floods log.html?
 232     IF    ${verify_restconf}
 233         TemplatedRequests.Get_As_Json_Templated
 234         ...    session=${session}
 235         ...    folder=${RESTCONF_MODULES_DIR}
 236         ...    verify=False
 237         ...    http_timeout=${http_timeout}
 238     END
 239     ${type_class} =    Resolve_Shard_Type_Class    shard_type=${shard_type}
 240     ${cluster_index} =    Evaluate    ${member_index}+${NODE_ROLE_INDEX_START}-1
 241     ${uri} =    BuiltIn.Set_Variable
 242     ...    ${JOLOKIA_READ_URI}:Category=Shards,name=member-${cluster_index}-shard-${shard_name}-${shard_type},type=${type_class}
 243     ${data_text} =    TemplatedRequests.Get_As_Json_From_Uri
 244     ...    uri=${uri}
 245     ...    session=${session}
 246     ...    http_timeout=${http_timeout}
 247     ${data_object} =    RequestsLibrary.To_Json    ${data_text}
 248     ${value} =    Collections.Get_From_Dictionary    ${data_object}    value
 249     ${raft_property} =    Collections.Get_From_Dictionary    ${value}    ${property}
 250     RETURN    ${raft_property}
 251
 252 Verify_Shard_Leader_Elected
 253     [Documentation]    Verify new leader was elected or remained the same. Bool paramter ${new_elected} indicates if
 254     ...    new leader is elected or should remained the same as ${old_leader}
 255     [Arguments]    ${shard_name}    ${shard_type}    ${new_elected}    ${old_leader}    ${member_index_list}=${EMPTY}    ${verify_restconf}=True
 256     ${leader}    ${followers} =    Get_Leader_And_Followers_For_Shard
 257     ...    shard_name=${shard_name}
 258     ...    shard_type=${shard_type}
 259     ...    member_index_list=${member_index_list}
 260     ...    verify_restconf=${verify_restconf}
 261     IF    ${new_elected}
 262         BuiltIn.Should_Not_Be_Equal_As_Numbers    ${old_leader}    ${leader}
 263     END
 264     IF    not ${new_elected}
 265         BuiltIn.Should_Be_Equal_As_numbers    ${old_leader}    ${leader}
 266     END
 267     RETURN    ${leader}    ${followers}
 268
 269 Verify_Owner_And_Successors_For_Device
 270     [Documentation]    Returns the owner and successors for the SB device ${device_name} of type ${device_type}. Request is sent to member ${member_index}.
 271     ...    For Boron and beyond, candidates are not removed on node down or isolation,
 272     ...    so this keyword expects candidates to be all members from Boron on.
 273     ...    Extra check is done to verify owner and successors are within the ${candidate_list}. This KW is useful when combined with WUKS.
 274     ...    ${candidate_list} minus owner is returned as ${successor list}.
 275     ...    Users can still use Get_Owner_And_Successors_For_Device if they are interested in downed candidates,
 276     ...    or for testing heterogeneous clusters.
 277     [Arguments]    ${device_name}    ${device_type}    ${member_index}    ${candidate_list}=${EMPTY}    ${after_stop}=False
 278     ${index_list} =    List_Indices_Or_All    given_list=${candidate_list}
 279     ${owner}    ${successor_list} =    Get_Owner_And_Successors_For_Device
 280     ...    device_name=${device_name}
 281     ...    device_type=${device_type}
 282     ...    member_index=${member_index}
 283     Collections.List_Should_Contain_Value
 284     ...    ${index_list}
 285     ...    ${owner}
 286     ...    Owner ${owner} is not in candidate list ${index_list}
 287     # In Beryllium or after stopping an instance, the removed instance does not show in the candidate list.
 288     ${expected_candidate_list_origin} =    BuiltIn.Set_Variable_If
 289     ...    ${after_stop}
 290     ...    ${index_list}
 291     ...    ${ClusterManagement__member_index_list}
 292     # We do not want to manipulate either origin list.
 293     ${expected_successor_list} =    BuiltIn.Create_List    @{expected_candidate_list_origin}
 294     Collections.Remove_Values_From_List    ${expected_successor_list}    ${owner}
 295     Collections.Lists_Should_Be_Equal
 296     ...    ${expected_successor_list}
 297     ...    ${successor_list}
 298     ...    Successor list ${successor_list} is not the came as expected ${expected_successor_list}
 299     # User expects the returned successor list to be the provided candidate list minus the owner.
 300     Collections.Remove_Values_From_List    ${index_list}    ${owner}
 301     RETURN    ${owner}    ${index_list}
 302
 303 Get_Owner_And_Successors_For_Device
 304     [Documentation]    Returns the owner and a list of successors for the SB device ${device_name} of type ${device_type}. Request is sent to member ${member_index}.
 305     ...    Successors are those device candidates not elected as owner. The list of successors = (list of candidates) - (owner).
 306     ...    The returned successor list is sorted numerically.
 307     ...    Note that "candidate list" definition currently differs between Beryllium and Boron.
 308     ...    Use Verify_Owner_And_Successors_For_Device if you want the older semantics (inaccessible nodes not present in the list).
 309     [Arguments]    ${device_name}    ${device_type}    ${member_index}    ${http_timeout}=${EMPTY}
 310     # TODO: somewhere to introduce ${DEFAULT_RESTCONF_DATASTORE_TIMEOUT}. Value may depend on protocol (ask vs tell) and perhaps stream.
 311     ${owner}    ${candidate_list} =    Get_Owner_And_Candidates_For_Device
 312     ...    device_name=${device_name}
 313     ...    device_type=${device_type}
 314     ...    member_index=${member_index}
 315     ...    http_timeout=${http_timeout}
 316     # Copy operation is not required, but new variable name requires a line anyway.
 317     ${successor_list} =    BuiltIn.Create_List
 318     ...    @{candidate_list}
 319     Collections.Remove_Values_From_List    ${successor_list}    ${owner}
 320     RETURN    ${owner}    ${successor_list}
 321
 322 Get_Owner_And_Candidates_For_Device_Rpc
 323     [Documentation]    Returns the owner and a list of candidates for the SB device ${device_name} of type ${device_type}. Request is sent to member ${member_index}.
 324     ...    Candidates are all members that register to own a device, so the list of candiates includes the owner.
 325     ...    The returned candidate list is sorted numerically.
 326     ...    Note that "candidate list" definition currently differs between Beryllium and Boron.
 327     ...    It is recommended to use Get_Owner_And_Successors_For_Device instead of this keyword, see documentation there.
 328     [Arguments]    ${device_name}    ${device_type}    ${member_index}    ${http_timeout}=${EMPTY}
 329     BuiltIn.Comment    TODO: Can this implementation be changed to call Get_Owner_And_Candidates_For_Type_And_Id?
 330     ${index} =    BuiltIn.Convert_To_Integer    ${member_index}
 331     ${ip} =    Resolve_IP_Address_For_Member    member_index=${index}
 332     ${entity_type} =    BuiltIn.Set_Variable_If
 333     ...    '${device_type}' == 'netconf'
 334     ...    netconf-node/${device_name}
 335     ...    ${device_type}
 336     ${url} =    BuiltIn.Catenate    SEPARATOR=    http://    ${ip}    :8181/    ${RESTCONF_URI}
 337     ${entity_result} =    ClusterEntities.Get_Entity    ${url}    ${entity_type}    ${device_name}
 338     ${entity_candidates} =    Collections.Get_From_Dictionary    ${entity_result}    candidates
 339     ${entity_owner} =    Collections.Get_From_Dictionary    ${entity_result}    owner
 340     BuiltIn.Should_Not_Be_Empty    ${entity_owner}    No owner found for ${device_name}
 341     ${owner} =    String.Replace_String    ${entity_owner}    member-    ${EMPTY}
 342     ${owner} =    BuiltIn.Convert_To_Integer    ${owner}
 343     ${candidate_list} =    BuiltIn.Create_List
 344     FOR    ${entity_candidate}    IN    @{entity_candidates}
 345         ${candidate} =    String.Replace_String    ${entity_candidate}    member-    ${EMPTY}
 346         ${candidate} =    BuiltIn.Convert_To_Integer    ${candidate}
 347         Collections.Append_To_List    ${candidate_list}    ${candidate}
 348     END
 349     Collections.Sort_List    ${candidate_list}
 350     RETURN    ${owner}    ${candidate_list}
 351
 352 Get_Owner_And_Candidates_For_Device_Singleton
 353     [Documentation]    Returns the owner and a list of candidates for the SB device ${device_name} of type ${device_type}. Request is sent to member ${member_index}.
 354     [Arguments]    ${device_name}    ${device_type}    ${member_index}    ${http_timeout}=${EMPTY}
 355     # Normalize device type to the lowercase as in ${SINGLETON_DEVICE_ID_PREFIX} & ${SINGLETON_DEVICE_ID_SUFFIX}
 356     ${device_type} =    String.Convert To Lower Case    ${device_type}
 357     # Set device ID prefix
 358     Collections.Dictionary Should Contain Key    ${SINGLETON_DEVICE_ID_PREFIX}    ${device_type}
 359     ${device_id_prefix} =    Collections.Get From Dictionary    ${SINGLETON_DEVICE_ID_PREFIX}    ${device_type}
 360     Log    ${device_id_prefix}
 361     # Set device ID suffix
 362     Collections.Dictionary Should Contain Key    ${SINGLETON_DEVICE_ID_SUFFIX}    ${device_type}
 363     ${device_id_suffix} =    Collections.Get From Dictionary    ${SINGLETON_DEVICE_ID_SUFFIX}    ${device_type}
 364     Log    ${device_id_suffix}
 365     # Set device ID
 366     ${id} =    BuiltIn.Set_Variable    ${device_id_prefix}${device_name}${device_id_suffix}
 367     Log    ${id}
 368     # Get election entity type results
 369     ${type} =    BuiltIn.Set_Variable    ${SINGLETON_ELECTION_ENTITY_TYPE}
 370     ${owner_1}    ${candidate_list_1} =    Get_Owner_And_Candidates_For_Type_And_Id
 371     ...    ${type}
 372     ...    ${id}
 373     ...    ${member_index}
 374     ...    http_timeout=${http_timeout}
 375     # Get change ownership entity type results
 376     ${type} =    BuiltIn.Set_Variable    ${SINGLETON_CHANGE_OWNERSHIP_ENTITY_TYPE}
 377     ${owner_2}    ${candidate_list_2} =    Get_Owner_And_Candidates_For_Type_And_Id
 378     ...    ${type}
 379     ...    ${id}
 380     ...    ${member_index}
 381     ...    http_timeout=${http_timeout}
 382     # Owners must be same, if not, there is still some election or change ownership in progress
 383     BuiltIn.Should_Be_Equal_As_Integers    ${owner_1}    ${owner_2}    Owners for device ${device_name} are not same
 384     RETURN    ${owner_1}    ${candidate_list_1}
 385
 386 Get_Owner_And_Candidates_For_Device
 387     [Documentation]    Returns the owner and a list of candidates for the SB device ${device_name} of type ${device_type}. Request is sent to member ${member_index}.
 388     ...    If parsing as singleton failed, kw try to parse data in old way (without singleton).
 389     ...    Candidates are all members that register to own a device, so the list of candiates includes the owner.
 390     ...    The returned candidate list is sorted numerically.
 391     ...    Note that "candidate list" definition currently differs between Beryllium and Boron.
 392     ...    It is recommended to use Get_Owner_And_Successors_For_Device instead of this keyword, see documentation there.
 393     [Arguments]    ${device_name}    ${device_type}    ${member_index}    ${http_timeout}=${EMPTY}
 394     # Try singleton
 395     ${status}    ${results} =    BuiltIn.Run_Keyword_And_Ignore_Error
 396     ...    Get_Owner_And_Candidates_For_Device_Singleton
 397     ...    device_name=${device_name}
 398     ...    device_type=${device_type}
 399     ...    member_index=${member_index}
 400     ...    http_timeout=${http_timeout}
 401     IF    "${status}"=="PASS"    RETURN    ${results}
 402     # If singleton failed, try parsing in old way
 403     ${status}    ${results} =    BuiltIn.Run_Keyword_And_Ignore_Error
 404     ...    Get_Owner_And_Candidates_For_Device_Rpc
 405     ...    device_name=${device_name}
 406     ...    device_type=${device_type}
 407     ...    member_index=${member_index}
 408     ...    http_timeout=${http_timeout}
 409     # previous 3 lines (BuilIn.Return.., # If singleton..., ${status}....) could be deleted when old way will not be supported anymore
 410     IF    '${status}'=='FAIL'
 411         BuiltIn.Fail    Could not parse owner and candidates for device ${device_name}
 412     END
 413     RETURN    @{results}
 414
 415 Check_Old_Owner_Stays_Elected_For_Device
 416     [Documentation]    Verify the owner remain the same as ${old_owner}
 417     [Arguments]    ${device_name}    ${device_type}    ${old_owner}    ${node_to_ask}    ${http_timeout}=${EMPTY}
 418     ${owner}    ${candidates} =    Get_Owner_And_Candidates_For_Device
 419     ...    ${device_name}
 420     ...    ${device_type}
 421     ...    ${node_to_ask}
 422     ...    http_timeout=${http_timeout}
 423     BuiltIn.Should_Be_Equal_As_numbers    ${old_owner}    ${owner}
 424     RETURN    ${owner}    ${candidates}
 425
 426 Check_New_Owner_Got_Elected_For_Device
 427     [Documentation]    Verify new owner was elected comparing to ${old_owner}
 428     [Arguments]    ${device_name}    ${device_type}    ${old_owner}    ${node_to_ask}    ${http_timeout}=${EMPTY}
 429     ${owner}    ${candidates} =    Get_Owner_And_Candidates_For_Device
 430     ...    ${device_name}
 431     ...    ${device_type}
 432     ...    ${node_to_ask}
 433     ...    http_timeout=${http_timeout}
 434     BuiltIn.Should_Not_Be_Equal_As_Numbers    ${old_owner}    ${owner}
 435     RETURN    ${owner}    ${candidates}
 436
 437 Get_Owner_And_Candidates_For_Type_And_Id
 438     [Documentation]    Returns the owner and a list of candidates for entity specified by ${type} and ${id}
 439     ...    Request is sent to member ${member_index}.
 440     ...    Candidates are all members that register to own a device, so the list of candiates includes the owner.
 441     ...    Bear in mind that for Boron and beyond, candidates are not removed on node down or isolation.
 442     ...    If ${require_candidate_list} is not \${EMPTY}, check whether the actual list of candidates matches.
 443     ...    Note that differs from "given list" semantics used in other keywords,
 444     ...    namely you cannot use \${EMPTY} to stand for "full list" in this keyword.
 445     [Arguments]    ${type}    ${id}    ${member_index}    ${require_candidate_list}=${EMPTY}    ${http_timeout}=${EMPTY}
 446     BuiltIn.Comment    TODO: Find a way to unify and deduplicate code blocks in Get_Owner_And_Candidates_* keywords.
 447     ${owner}    ${candidates} =    Get_Owner_And_Candidates_For_Device_Rpc
 448     ...    ${id}
 449     ...    ${type}
 450     ...    ${member_index}
 451     ...    http_timeout=${http_timeout}
 452     RETURN    ${owner}    ${candidates}
 453
 454 Extract_Service_Entity_Type
 455     [Documentation]    Remove superfluous device data from Entity Owner printout.
 456     [Arguments]    ${data}
 457     ${clear_data} =    String.Replace_String
 458     ...    ${data}
 459     ...    /odl-general-entity:entity[odl-general-entity:name='
 460     ...    ${EMPTY}
 461     ${clear_data} =    String.Replace_String    ${clear_data}    -service-group']    ${EMPTY}
 462     Log    ${clear_data}
 463     RETURN    ${clear_data}
 464
 465 Extract_OpenFlow_Device_Data
 466     [Documentation]    Remove superfluous OpenFlow device data from Entity Owner printout.
 467     [Arguments]    ${data}
 468     ${clear_data} =    String.Replace_String    ${data}    org.opendaylight.mdsal.ServiceEntityType    openflow
 469     ${clear_data} =    String.Replace_String
 470     ...    ${clear_data}
 471     ...    /odl-general-entity:entity[odl-general-entity:name='
 472     ...    ${EMPTY}
 473     ${clear_data} =    String.Replace_String
 474     ...    ${clear_data}
 475     ...    /general-entity:entity[general-entity:name='
 476     ...    ${EMPTY}
 477     ${clear_data} =    String.Replace_String    ${clear_data}    ']    ${EMPTY}
 478     Log    ${clear_data}
 479     RETURN    ${clear_data}
 480
 481 Extract_Ovsdb_Device_Data
 482     [Documentation]    Remove superfluous OVSDB device data from Entity Owner printout.
 483     [Arguments]    ${data}
 484     ${clear_data} =    String.Replace_String
 485     ...    ${data}
 486     ...    /network-topology:network-topology/network-topology:topology[network-topology:topology-id='ovsdb:1']/network-topology:node[network-topology:node-id='
 487     ...    ${EMPTY}
 488     ${clear_data} =    String.Replace_String    ${clear_data}    ']    ${EMPTY}
 489     Log    ${clear_data}
 490     RETURN    ${clear_data}
 491
 492 Kill_Single_Member
 493     [Documentation]    Convenience keyword that kills the specified member of the cluster.
 494     ...    The KW will return a list of available members: \${updated index_list}=\${original_index_list}-\${member}
 495     [Arguments]    ${member}    ${original_index_list}=${EMPTY}    ${confirm}=True
 496     ${index_list} =    ClusterManagement__Build_List    ${member}
 497     ${member_ip} =    Return_Member_IP    ${member}
 498     KarafKeywords.Log_Message_To_Controller_Karaf    Killing ODL${member} ${member_ip}
 499     ${updated_index_list} =    Kill_Members_From_List_Or_All    ${index_list}    ${original_index_list}    ${confirm}
 500     RETURN    ${updated_index_list}
 501
 502 Kill_Members_From_List_Or_All
 503     [Documentation]    If the list is empty, kill all ODL instances. Otherwise, kill members based on \${kill_index_list}
 504     ...    If \${confirm} is True, sleep 1 second and verify killed instances are not there anymore.
 505     ...    The KW will return a list of available members: \${updated index_list}=\${original_index_list}-\${member_index_list}
 506     [Arguments]    ${member_index_list}=${EMPTY}    ${original_index_list}=${EMPTY}    ${confirm}=True
 507     ${kill_index_list} =    List_Indices_Or_All    given_list=${member_index_list}
 508     ${index_list} =    List_Indices_Or_All    given_list=${original_index_list}
 509     Run_Bash_Command_On_List_Or_All    command=${NODE_KILL_COMMAND}    member_index_list=${member_index_list}
 510     ${updated_index_list} =    BuiltIn.Create_List    @{index_list}
 511     Collections.Remove_Values_From_List    ${updated_index_list}    @{kill_index_list}
 512     IF    not ${confirm}    RETURN    ${updated_index_list}
 513     # TODO: Convert to WUKS with configurable timeout if it turns out 1 second is not enough.
 514     BuiltIn.Sleep
 515     ...    1s
 516     ...    Kill -9 closes open files, which may take longer than ssh overhead, but not long enough to warrant WUKS.
 517     FOR    ${index}    IN    @{kill_index_list}
 518         Verify_Karaf_Is_Not_Running_On_Member    member_index=${index}
 519     END
 520     Run_Bash_Command_On_List_Or_All    command=netstat -pnatu | grep 2550
 521     RETURN    ${updated_index_list}
 522
 523 Stop_Single_Member
 524     [Documentation]    Convenience keyword that stops the specified member of the cluster.
 525     ...    The KW will return a list of available members: \${updated index_list}=\${original_index_list}-\${member}
 526     [Arguments]    ${member}    ${original_index_list}=${EMPTY}    ${confirm}=True    ${msg}=${EMPTY}
 527     ${index_list} =    ClusterManagement__Build_List    ${member}
 528     ${member_ip} =    Return_Member_IP    ${member}
 529     ${msg} =    Builtin.Set Variable If
 530     ...    "${msg}" == "${EMPTY}"
 531     ...    Stopping ODL${member} ${member_ip}
 532     ...    Stopping ODL${member} ${member_ip}, ${msg}
 533     KarafKeywords.Log_Message_To_Controller_Karaf    ${msg}
 534     ${updated_index_list} =    Stop_Members_From_List_Or_All    ${index_list}    ${original_index_list}    ${confirm}
 535     RETURN    ${updated_index_list}
 536
 537 Stop_Members_From_List_Or_All
 538     [Documentation]    If the list is empty, stops all ODL instances. Otherwise stop members based on \${stop_index_list}
 539     ...    If \${confirm} is True, verify stopped instances are not there anymore.
 540     ...    The KW will return a list of available members: \${updated index_list}=\${original_index_list}-\${member_index_list}
 541     [Arguments]    ${member_index_list}=${EMPTY}    ${original_index_list}=${EMPTY}    ${confirm}=True    ${timeout}=360s
 542     ${stop_index_list} =    List_Indices_Or_All    given_list=${member_index_list}
 543     ${index_list} =    List_Indices_Or_All    given_list=${original_index_list}
 544     Run_Bash_Command_On_List_Or_All    command=${NODE_STOP_COMMAND}    member_index_list=${member_index_list}
 545     ${updated_index_list} =    BuiltIn.Create_List    @{index_list}
 546     Collections.Remove_Values_From_List    ${updated_index_list}    @{stop_index_list}
 547     IF    not ${confirm}    RETURN    ${updated_index_list}
 548     FOR    ${index}    IN    @{stop_index_list}
 549         BuiltIn.Wait Until Keyword Succeeds
 550         ...    ${timeout}
 551         ...    2s
 552         ...    Verify_Karaf_Is_Not_Running_On_Member
 553         ...    member_index=${index}
 554     END
 555     Run_Bash_Command_On_List_Or_All    command=netstat -pnatu | grep 2550
 556     RETURN    ${updated_index_list}
 557
 558 Start_Single_Member
 559     [Documentation]    Convenience keyword that starts the specified member of the cluster.
 560     [Arguments]    ${member}    ${wait_for_sync}=True    ${timeout}=300s    ${msg}=${EMPTY}    ${check_system_status}=False    ${verify_restconf}=True
 561     ...    ${service_list}=${EMPTY_LIST}
 562     ${index_list} =    ClusterManagement__Build_List    ${member}
 563     ${member_ip} =    Return_Member_IP    ${member}
 564     ${msg} =    Builtin.Set Variable If
 565     ...    "${msg}" == "${EMPTY}"
 566     ...    Starting ODL${member} ${member_ip}
 567     ...    Starting ODL${member} ${member_ip}, ${msg}
 568     KarafKeywords.Log_Message_To_Controller_Karaf    ${msg}
 569     Start_Members_From_List_Or_All
 570     ...    ${index_list}
 571     ...    ${wait_for_sync}
 572     ...    ${timeout}
 573     ...    check_system_status=${check_system_status}
 574     ...    verify_restconf=${verify_restconf}
 575     ...    service_list=${service_list}
 576
 577 Start_Members_From_List_Or_All
 578     [Documentation]    If the list is empty, start all cluster members. Otherwise, start members based on present indices.
 579     ...    If ${wait_for_sync}, wait for cluster sync on listed members.
 580     ...    Optionally karaf_home can be overriden. Optionally specific JAVA_HOME is used for starting.
 581     ...    Garbage collection is unconditionally logged to files. TODO: Make that reasonable conditional?
 582     [Arguments]    ${member_index_list}=${EMPTY}    ${wait_for_sync}=True    ${timeout}=360s    ${karaf_home}=${EMPTY}    ${export_java_home}=${EMPTY}    ${gc_log_dir}=${EMPTY}
 583     ...    ${check_system_status}=False    ${verify_restconf}=True    ${service_list}=${EMPTY_LIST}
 584     ${base_command} =    BuiltIn.Set_Variable_If
 585     ...    """${karaf_home}""" != ""
 586     ...    ${karaf_home}/bin/start
 587     ...    ${NODE_START_COMMAND}
 588     ${command} =    BuiltIn.Set_Variable_If
 589     ...    """${export_java_home}""" != ""
 590     ...    export JAVA_HOME="${export_java_home}"; ${base_command}
 591     ...    ${base_command}
 592     ${epoch} =    DateTime.Get_Current_Date    time_zone=UTC    result_format=epoch    exclude_millis=False
 593     ${gc_filepath} =    BuiltIn.Set_Variable_If
 594     ...    """${karaf_home}""" != ""
 595     ...    ${karaf_home}/data/log/gc_${epoch}.log
 596     ...    ${GC_LOG_PATH}/gc_${epoch}.log
 597     ${gc_options} =    BuiltIn.Set_Variable_If
 598     ...    "docker" not in """${node_start_command}"""
 599     ...    -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:${gc_filepath}
 600     ...    ${EMPTY}
 601     Run_Bash_Command_On_List_Or_All    command=${command} ${gc_options}    member_index_list=${member_index_list}
 602     BuiltIn.Wait_Until_Keyword_Succeeds
 603     ...    ${timeout}
 604     ...    10s
 605     ...    Verify_Members_Are_Ready
 606     ...    ${member_index_list}
 607     ...    ${wait_for_sync}
 608     ...    ${verify_restconf}
 609     ...    ${check_system_status}
 610     ...    ${service_list}
 611     [Teardown]    Run_Bash_Command_On_List_Or_All    command=netstat -pnatu | grep 2550
 612
 613 Verify_Members_Are_Ready
 614     [Documentation]    Verifies the specified readiness conditions for the given listed members after startup.
 615     ...    If ${verify_cluster_sync}, verifies the datastores have synced with the rest of the cluster.
 616     ...    If ${verify_restconf}, verifies RESTCONF is available.
 617     ...    If ${verify_system_status}, verifies the system services are OPERATIONAL.
 618     [Arguments]    ${member_index_list}    ${verify_cluster_sync}    ${verify_restconf}    ${verify_system_status}    ${service_list}
 619     IF    ${verify_cluster_sync}
 620         Check_Cluster_Is_In_Sync    ${member_index_list}
 621     END
 622     IF    ${verify_restconf}
 623         Verify_Restconf_Is_Available    ${member_index_list}
 624     END
 625     # for backward compatibility, some consumers might not be passing @{service_list}, but since we can't set a list to a default
 626     # value, we need to check here if it's empty in order to skip the check which would throw an error
 627     IF    ${verify_system_status} and ("${service_list}" != "[[]]")
 628         ClusterManagement.Check Status Of Services Is OPERATIONAL    @{service_list}
 629     END
 630
 631 Verify_Restconf_Is_Available
 632     [Arguments]    ${member_index_list}
 633     ${index_list} =    List_Indices_Or_All    given_list=${member_index_list}
 634     FOR    ${index}    IN    @{index_list}
 635         ${session} =    Resolve_Http_Session_For_Member    member_index=${index}
 636         TemplatedRequests.Get_As_Json_Templated    session=${session}    folder=${RESTCONF_MODULES_DIR}    verify=False
 637     END
 638
 639 Freeze_Single_Member
 640     [Documentation]    Convenience keyword that stops the specified member of the cluster by freezing the jvm.
 641     [Arguments]    ${member}
 642     ${index_list} =    ClusterManagement__Build_List    ${member}
 643     Freeze_Or_Unfreeze_Members_From_List_Or_All    ${NODE_FREEZE_COMMAND}    ${index_list}
 644
 645 Unfreeze_Single_Member
 646     [Documentation]    Convenience keyword that "continues" the specified member of the cluster by unfreezing the jvm.
 647     [Arguments]    ${member}    ${wait_for_sync}=True    ${timeout}=60s
 648     ${index_list} =    ClusterManagement__Build_List    ${member}
 649     Freeze_Or_Unfreeze_Members_From_List_Or_All    ${NODE_UNFREEZE_COMMAND}    ${index_list}
 650     BuiltIn.Wait_Until_Keyword_Succeeds    ${timeout}    10s    Check_Cluster_Is_In_Sync
 651
 652 Freeze_Or_Unfreeze_Members_From_List_Or_All
 653     [Documentation]    If the list is empty, stops/runs all ODL instances. Otherwise stop/run members based on \${stop_index_list}
 654     ...    For command parameter only ${NODE_FREEZE_COMMAND} and ${NODE_UNFREEZE_COMMAND} should be used
 655     [Arguments]    ${command}    ${member_index_list}=${EMPTY}
 656     ${freeze_index_list} =    List_Indices_Or_All    given_list=${member_index_list}
 657     Run_Bash_Command_On_List_Or_All    command=${command}    member_index_list=${member_index_list}
 658
 659 Clean_Journals_Data_And_Snapshots_On_List_Or_All
 660     [Documentation]    Delete journal and snapshots directories on every node listed (or all).
 661     ...    BEWARE: If only a subset of members is cleaned, this causes RetiredGenerationException in Carbon after the affected node re-start.
 662     ...    See https://bugs.opendaylight.org/show_bug.cgi?id=8138
 663     [Arguments]    ${member_index_list}=${EMPTY}    ${karaf_home}=${KARAF_HOME}
 664     ${index_list} =    List_Indices_Or_All    given_list=${member_index_list}
 665     ${command} =    Set Variable    rm -rf "${karaf_home}/"*journal "${karaf_home}/snapshots" "${karaf_home}/data"
 666     FOR    ${index}    IN    @{index_list}    # usually: 1, 2, 3.
 667         Run_Bash_Command_On_Member    command=${command}    member_index=${index}
 668     END
 669
 670 Verify_Karaf_Is_Not_Running_On_Member
 671     [Documentation]    Fail if non-zero karaf instances are counted on member of given index.
 672     [Arguments]    ${member_index}
 673     ${count} =    Count_Running_Karafs_On_Member    member_index=${member_index}
 674     BuiltIn.Should_Be_Equal    0    ${count}    Found running Karaf count: ${count}
 675
 676 Verify_Single_Karaf_Is_Running_On_Member
 677     [Documentation]    Fail if number of karaf instances on member of given index is not one.
 678     [Arguments]    ${member_index}
 679     ${count} =    Count_Running_Karafs_On_Member    member_index=${member_index}
 680     BuiltIn.Should_Be_Equal    1    ${count}    Wrong number of Karafs running: ${count}
 681
 682 Count_Running_Karafs_On_Member
 683     [Documentation]    Remotely execute grep for karaf process, return count as string.
 684     [Arguments]    ${member_index}
 685     ${command} =    BuiltIn.Set_Variable    ${NODE_KARAF_COUNT_COMMAND}
 686     ${count} =    Run_Bash_Command_On_Member    command=${command}    member_index=${member_index}
 687     RETURN    ${count}
 688
 689 Isolate_Member_From_List_Or_All
 690     [Documentation]    If the list is empty, isolate member from all ODL instances. Otherwise, isolate member based on present indices.
 691     ...    The KW will return a list of available members: \${updated index_list}=\${member_index_list}-\${isolate_member_index}
 692     [Arguments]    ${isolate_member_index}    ${member_index_list}=${EMPTY}    ${protocol}=all    ${port}=${EMPTY}
 693     ${index_list} =    List_Indices_Or_All    given_list=${member_index_list}
 694     ${source} =    Collections.Get_From_Dictionary
 695     ...    ${ClusterManagement__index_to_ip_mapping}
 696     ...    ${isolate_member_index}
 697     ${dport} =    BuiltIn.Set_Variable_If    '${port}' != '${EMPTY}'    --dport ${port}    ${EMPTY}
 698     FOR    ${index}    IN    @{index_list}
 699         ${destination} =    Collections.Get_From_Dictionary    ${ClusterManagement__index_to_ip_mapping}    ${index}
 700         ${command} =    BuiltIn.Set_Variable
 701         ...    sudo /sbin/iptables -I OUTPUT -p ${protocol} ${dport} --source ${source} --destination ${destination} -j DROP
 702         IF    "${index}" != "${isolate_member_index}"
 703             Run_Bash_Command_On_Member    command=${command}    member_index=${isolate_member_index}
 704         END
 705     END
 706     ${command} =    BuiltIn.Set_Variable    sudo /sbin/iptables -L -n
 707     ${output} =    Run_Bash_Command_On_Member    command=${command}    member_index=${isolate_member_index}
 708     BuiltIn.Log    ${output}
 709     ${updated_index_list} =    BuiltIn.Create_List    @{index_list}
 710     Collections.Remove_Values_From_List    ${updated_index_list}    ${isolate_member_index}
 711     RETURN    ${updated_index_list}
 712
 713 Rejoin_Member_From_List_Or_All
 714     [Documentation]    If the list is empty, rejoin member from all ODL instances. Otherwise, rejoin member based on present indices.
 715     [Arguments]    ${rejoin_member_index}    ${member_index_list}=${EMPTY}    ${protocol}=all    ${port}=${EMPTY}    ${timeout}=60s
 716     ${index_list} =    List_Indices_Or_All    given_list=${member_index_list}
 717     ${source} =    Collections.Get_From_Dictionary
 718     ...    ${ClusterManagement__index_to_ip_mapping}
 719     ...    ${rejoin_member_index}
 720     ${dport} =    BuiltIn.Set_Variable_If    '${port}' != '${EMPTY}'    --dport ${port}    ${EMPTY}
 721     FOR    ${index}    IN    @{index_list}
 722         ${destination} =    Collections.Get_From_Dictionary    ${ClusterManagement__index_to_ip_mapping}    ${index}
 723         ${command} =    BuiltIn.Set_Variable
 724         ...    sudo /sbin/iptables -D OUTPUT -p ${protocol} ${dport} --source ${source} --destination ${destination} -j DROP
 725         IF    "${index}" != "${rejoin_member_index}"
 726             Run_Bash_Command_On_Member    command=${command}    member_index=${rejoin_member_index}
 727         END
 728     END
 729     ${command} =    BuiltIn.Set_Variable    sudo /sbin/iptables -L -n
 730     ${output} =    Run_Bash_Command_On_Member    command=${command}    member_index=${rejoin_member_index}
 731     BuiltIn.Log    ${output}
 732     BuiltIn.Wait_Until_Keyword_Succeeds    ${timeout}    10s    Check_Cluster_Is_In_Sync
 733
 734 Flush_Iptables_From_List_Or_All
 735     [Documentation]    If the list is empty, flush IPTables in all ODL instances. Otherwise, flush member based on present indices.
 736     [Arguments]    ${member_index_list}=${EMPTY}
 737     ${command} =    BuiltIn.Set_Variable    sudo iptables -v -F
 738     ${output} =    Run_Bash_Command_On_List_Or_All    command=${command}    member_index_list=${member_index_list}
 739
 740 Check_Bash_Command_On_List_Or_All
 741     [Documentation]    Cycle through indices (or all), run bash command on each, using temporary SSH session and restoring the previously active one.
 742     [Arguments]    ${command}    ${member_index_list}=${EMPTY}    ${return_success_only}=False    ${log_on_success}=True    ${log_on_failure}=True    ${stderr_must_be_empty}=True
 743     ${index_list} =    List_Indices_Or_All    given_list=${member_index_list}
 744     FOR    ${index}    IN    @{index_list}
 745         Check_Bash_Command_On_Member
 746         ...    command=${command}
 747         ...    member_index=${index}
 748         ...    return_success_only=${return_success_only}
 749         ...    log_on_success=${log_on_success}
 750         ...    log_on_failure=${log_on_failure}
 751         ...    stderr_must_be_empty=${stderr_must_be_empty}
 752     END
 753
 754 Check_Bash_Command_On_Member
 755     [Documentation]    Open SSH session, call SSHKeywords.Execute_Command_Passes, close session, restore previously active session and return output.
 756     [Arguments]    ${command}    ${member_index}    ${return_success_only}=False    ${log_on_success}=True    ${log_on_failure}=True    ${stderr_must_be_empty}=True
 757     BuiltIn.Run_Keyword_And_Return
 758     ...    SSHKeywords.Run_Keyword_Preserve_Connection
 759     ...    Check_Unsafely_Bash_Command_On_Member
 760     ...    ${command}
 761     ...    ${member_index}
 762     ...    return_success_only=${return_success_only}
 763     ...    log_on_success=${log_on_success}
 764     ...    log_on_failure=${log_on_failure}
 765     ...    stderr_must_be_empty=${stderr_must_be_empty}
 766
 767 Check_Unsafely_Bash_Command_On_Member
 768     [Documentation]    Obtain Ip address, open session, call SSHKeywords.Execute_Command_Passes, close session and return output. This affects which SSH session is active.
 769     [Arguments]    ${command}    ${member_index}    ${return_success_only}=False    ${log_on_success}=True    ${log_on_failure}=True    ${stderr_must_be_empty}=True
 770     ${member_ip} =    Resolve_Ip_Address_For_Member    ${member_index}
 771     BuiltIn.Run_Keyword_And_Return
 772     ...    SSHKeywords.Run_Unsafely_Keyword_Over_Temporary_Odl_Session
 773     ...    ${member_ip}
 774     ...    Execute_Command_Passes
 775     ...    ${command}
 776     ...    return_success_only=${return_success_only}
 777     ...    log_on_success=${log_on_success}
 778     ...    log_on_failure=${log_on_failure}
 779     ...    stderr_must_be_empty=${stderr_must_be_empty}
 780
 781 Run_Bash_Command_On_List_Or_All
 782     [Documentation]    Cycle through indices (or all), run command on each.
 783     [Arguments]    ${command}    ${member_index_list}=${EMPTY}
 784     # TODO: Migrate callers to Check_Bash_Command_*
 785     ${index_list} =    List_Indices_Or_All    given_list=${member_index_list}
 786     FOR    ${index}    IN    @{index_list}
 787         Run_Bash_Command_On_Member    command=${command}    member_index=${index}
 788     END
 789
 790 Run_Bash_Command_On_Member
 791     [Documentation]    Obtain IP, call Utils and return output. This keeps previous ssh session active.
 792     [Arguments]    ${command}    ${member_index}
 793     # TODO: Migrate callers to Check_Bash_Command_*
 794     ${member_ip} =    Collections.Get_From_Dictionary
 795     ...    dictionary=${ClusterManagement__index_to_ip_mapping}
 796     ...    key=${member_index}
 797     ${output} =    SSHKeywords.Run_Keyword_Preserve_Connection
 798     ...    Utils.Run_Command_On_Controller
 799     ...    ${member_ip}
 800     ...    ${command}
 801     Log    ${output}
 802     RETURN    ${output}
 803
 804 Run_Karaf_Command_On_List_Or_All
 805     [Documentation]    Cycle through indices (or all), run karaf command on each.
 806     [Arguments]    ${command}    ${member_index_list}=${EMPTY}    ${timeout}=10s
 807     ${index_list} =    List_Indices_Or_All    given_list=${member_index_list}
 808     FOR    ${index}    IN    @{index_list}
 809         ${member_ip} =    Collections.Get_From_Dictionary
 810         ...    dictionary=${ClusterManagement__index_to_ip_mapping}
 811         ...    key=${index}
 812         KarafKeywords.Safe_Issue_Command_On_Karaf_Console    ${command}    ${member_ip}    timeout=${timeout}
 813     END
 814
 815 Run_Karaf_Command_On_Member
 816     [Documentation]    Obtain IP address, call KarafKeywords and return output. This does not preserve active ssh session.
 817     ...    This keyword is not used by Run_Karaf_Command_On_List_Or_All, but returned output may be useful.
 818     [Arguments]    ${command}    ${member_index}    ${timeout}=10s
 819     ${member_ip} =    Collections.Get_From_Dictionary
 820     ...    dictionary=${ClusterManagement__index_to_ip_mapping}
 821     ...    key=${member_index}
 822     ${output} =    KarafKeywords.Safe_Issue_Command_On_Karaf_Console
 823     ...    ${command}
 824     ...    controller=${member_ip}
 825     ...    timeout=${timeout}
 826     RETURN    ${output}
 827
 828 Install_Feature_On_List_Or_All
 829     [Documentation]    Attempt installation on each member from list (or all). Then look for failures.
 830     [Arguments]    ${feature_name}    ${member_index_list}=${EMPTY}    ${timeout}=60s
 831     ${index_list} =    List_Indices_Or_All    given_list=${member_index_list}
 832     ${status_list} =    BuiltIn.Create_List
 833     FOR    ${index}    IN    @{index_list}
 834         ${status}    ${text} =    BuiltIn.Run_Keyword_And_Ignore_Error
 835         ...    Install_Feature_On_Member
 836         ...    feature_name=${feature_name}
 837         ...    member_index=${index}
 838         ...    timeout=${timeout}
 839         BuiltIn.Log    ${text}
 840         Collections.Append_To_List    ${status_list}    ${status}
 841     END
 842     FOR    ${status}    IN    @{status_list}
 843         IF    "${status}" != "PASS"
 844             BuiltIn.Fail    ${feature_name} installation failed, see log.
 845         END
 846     END
 847
 848 Install_Feature_On_Member
 849     [Documentation]    Run feature:install karaf command, fail if installation was not successful. Return output.
 850     [Arguments]    ${feature_name}    ${member_index}    ${timeout}=60s
 851     ${status}    ${output} =    BuiltIn.Run_Keyword_And_Ignore_Error
 852     ...    Run_Karaf_Command_On_Member
 853     ...    command=feature:install ${feature_name}
 854     ...    member_index=${member_index}
 855     ...    timeout=${timeout}
 856     IF    "${status}" != "PASS"
 857         BuiltIn.Fail    Failed to install ${feature_name}: ${output}
 858     END
 859     BuiltIn.Should_Not_Contain    ${output}    Can't install    Failed to install ${feature_name}: ${output}
 860     RETURN    ${output}
 861
 862 With_Ssh_To_List_Or_All_Run_Keyword
 863     [Documentation]    For each index in given list (or all): activate SSH connection, run given Keyword, close active connection. Return None.
 864     ...    Beware that in order to avoid "got positional argument after named arguments", first two arguments in the call should not be named.
 865     [Arguments]    ${member_index_list}    ${keyword_name}    @{args}    &{kwargs}
 866     BuiltIn.Comment    This keyword is experimental and there is high risk of being replaced by another approach.
 867     # TODO: For_Index_From_List_Or_All_Run_Keyword applied to With_Ssh_To_Member_Run_Keyword?
 868     # TODO: Imagine another keyword, using ScalarClosures and adding member index as first argument for each call. Worth it?
 869     ${index_list} =    List_Indices_Or_All    given_list=${member_index_list}
 870     FOR    ${member_index}    IN    @{index_list}
 871         ${member_ip} =    Resolve_IP_Address_For_Member    ${member_index}
 872         SSHKeywords.Run_Unsafely_Keyword_Over_Temporary_Odl_Session
 873         ...    ${member_ip}
 874         ...    ${keyword_name}
 875         ...    @{args}
 876         ...    &{kwargs}
 877     END
 878
 879 Safe_With_Ssh_To_List_Or_All_Run_Keyword
 880     [Documentation]    Remember active ssh connection index, call With_Ssh_To_List_Or_All_Run_Keyword, return None. Restore the conection index on teardown.
 881     [Arguments]    ${member_index_list}    ${keyword_name}    @{args}    &{kwargs}
 882     SSHKeywords.Run_Keyword_Preserve_Connection
 883     ...    With_Ssh_To_List_Or_All_Run_Keyword
 884     ...    ${member_index_list}
 885     ...    ${keyword_name}
 886     ...    @{args}
 887     ...    &{kwargs}
 888
 889 Clean_Directories_On_List_Or_All
 890     [Documentation]    Clear @{directory_list} or @{ODL_DEFAULT_DATA_PATHS} for members in given list or all. Return None.
 891     ...    If \${tmp_dir} is nonempty, use that location to preserve data/log/.
 892     ...    This is intended to return Karaf (offline) to the state it was upon the first boot.
 893     [Arguments]    ${member_index_list}=${EMPTY}    ${directory_list}=${EMPTY}    ${karaf_home}=${KARAF_HOME}    ${tmp_dir}=${EMPTY}
 894     ${path_list} =    Builtin.Set Variable If
 895     ...    "${directory_list}" == "${EMPTY}"
 896     ...    ${ODL_DEFAULT_DATA_PATHS}
 897     ...    ${directory_list}
 898     IF    """${tmp_dir}""" != ""
 899         Check_Bash_Command_On_List_Or_All
 900         ...    mkdir -p '${tmp_dir}' && rm -vrf '${tmp_dir}/log' && mv -vf '${karaf_home}/data/log' '${tmp_dir}/'
 901         ...    ${member_index_list}
 902     END
 903     Safe_With_Ssh_To_List_Or_All_Run_Keyword
 904     ...    ${member_index_list}
 905     ...    ClusterManagement__Clean_Directories
 906     ...    ${path_list}
 907     ...    ${karaf_home}
 908     IF    """${tmp_dir}""" != ""
 909         Check_Bash_Command_On_List_Or_All
 910         ...    mkdir -p '${karaf_home}/data' && rm -vrf '${karaf_home}/log' && mv -vf '${tmp_dir}/log' '${karaf_home}/data/'
 911         ...    ${member_index_list}
 912     END
 913
 914 Store_Karaf_Log_On_List_Or_All
 915     [Documentation]    Saves karaf.log to the ${dst_dir} for members in given list or all. Return None.
 916     [Arguments]    ${member_index_list}=${EMPTY}    ${dst_dir}=/tmp    ${karaf_home}=${KARAF_HOME}
 917     Safe_With_Ssh_To_List_Or_All_Run_Keyword
 918     ...    ${member_index_list}
 919     ...    SSHKeywords.Execute_Command_Should_Pass
 920     ...    cp ${karaf_home}/data/log/karaf.log ${dst_dir}
 921
 922 Restore_Karaf_Log_On_List_Or_All
 923     [Documentation]    Places stored karaf.log to the ${karaf_home}/data/log for members in given list or all. Return None.
 924     [Arguments]    ${member_index_list}=${EMPTY}    ${src_dir}=/tmp    ${karaf_home}=${KARAF_HOME}
 925     Safe_With_Ssh_To_List_Or_All_Run_Keyword
 926     ...    ${member_index_list}
 927     ...    SSHKeywords.Execute_Command_Should_Pass
 928     ...    cp ${src_dir}/karaf.log ${karaf_home}/data/log/
 929
 930 ClusterManagement__Clean_Directories
 931     [Documentation]    For each relative path, remove files with respect to ${karaf_home}. Return None.
 932     [Arguments]    ${relative_path_list}    ${karaf_home}
 933     FOR    ${relative_path}    IN    @{relative_path_list}
 934         SSHLibrary.Execute_Command    rm -rf ${karaf_home}${/}${relative_path}
 935     END
 936
 937 Put_As_Json_And_Check_Member_List_Or_All
 938     [Documentation]    Send a PUT with the supplied uri ${uri} and body ${data} to member ${member_index}.
 939     ...    Then check data is replicated in all or some members defined in ${member_index_list}.
 940     [Arguments]    ${uri}    ${data}    ${member_index}    ${member_index_list}=${EMPTY}
 941     ${response_text} =    Put_As_Json_To_Member    uri=${uri}    data=${data}    member_index=${member_index}
 942     Wait Until Keyword Succeeds
 943     ...    5s
 944     ...    1s
 945     ...    Check_Json_Member_List_Or_All
 946     ...    uri=${uri}?content=config
 947     ...    expected_data=${data}
 948     ...    member_index_list=${member_index_list}
 949     RETURN    ${response_text}
 950
 951 Put_As_Json_To_Member
 952     [Documentation]    Send a PUT with the supplied uri and data to member ${member_index}.
 953     [Arguments]    ${uri}    ${data}    ${member_index}
 954     ${session} =    Resolve_Http_Session_For_Member    member_index=${member_index}
 955     ${response_text} =    TemplatedRequests.Put_As_Json_To_Uri    uri=${uri}    data=${data}    session=${session}
 956     RETURN    ${response_text}
 957
 958 Post_As_Json_To_Member
 959     [Documentation]    Send a POST with the supplied uri and data to member ${member_index}.
 960     [Arguments]    ${uri}    ${data}    ${member_index}
 961     ${session} =    Resolve_Http_Session_For_Member    member_index=${member_index}
 962     ${response_text} =    TemplatedRequests.Post_As_Json_To_Uri    uri=${uri}    data=${data}    session=${session}
 963     RETURN    ${response_text}
 964
 965 Delete_And_Check_Member_List_Or_All
 966     [Documentation]    Send a DELETE with the supplied uri to the member ${member_index}.
 967     ...    Then check the data is removed from all members in ${member_index_list}.
 968     [Arguments]    ${uri}    ${member_index}    ${member_index_list}=${EMPTY}
 969     ${response_text} =    Delete_From_Member    ${uri}    ${member_index}
 970     BuiltIn.Wait_Until_Keyword_Succeeds
 971     ...    5s
 972     ...    1s
 973     ...    Check_No_Content_Member_List_Or_All
 974     ...    uri=${uri}
 975     ...    member_index_list=${member_index_list}
 976     RETURN    ${response_text}
 977
 978 Delete_From_Member
 979     [Documentation]    Send a DELETE with the supplied uri to member ${member_index}.
 980     [Arguments]    ${uri}    ${member_index}
 981     ${session} =    Resolve_Http_Session_For_Member    member_index=${member_index}
 982     ${response_text} =    TemplatedRequests.Delete_From_Uri    uri=${uri}    session=${session}
 983     RETURN    ${response_text}
 984
 985 Check_Json_Member_List_Or_All
 986     [Documentation]    Send a GET with the supplied uri to all or some members defined in ${member_index_list}.
 987     ...    Then check received data is = ${expected data}.
 988     [Arguments]    ${uri}    ${expected_data}    ${member_index_list}=${EMPTY}
 989     ${index_list} =    List_Indices_Or_All    given_list=${member_index_list}
 990     FOR    ${index}    IN    @{index_list}
 991         ${data} =    Get_From_Member    uri=${uri}    member_index=${index}
 992         TemplatedRequests.Normalize_Jsons_And_Compare    ${expected_data}    ${data}
 993     END
 994
 995 Check_Item_Occurrence_Member_List_Or_All
 996     [Documentation]    Send a GET with the supplied uri to all or some members defined in ${member_index_list}.
 997     ...    Then check received for occurrences of items expressed in a dictionary ${dictionary}.
 998     [Arguments]    ${uri}    ${dictionary}    ${member_index_list}=${EMPTY}
 999     ${index_list} =    List_Indices_Or_All    given_list=${member_index_list}
1000     FOR    ${index}    IN    @{index_list}
1001         ${data} =    Get_From_Member    uri=${uri}    member_index=${index}
1002         Utils.Check Item Occurrence    ${data}    ${dictionary}
1003     END
1004
1005 Check_No_Content_Member_List_Or_All
1006     [Documentation]    Send a GET with the supplied uri to all or some members defined in ${member_index_list}.
1007     ...    Then check there is no content.
1008     [Arguments]    ${uri}    ${member_index_list}=${EMPTY}
1009     ${index_list} =    List_Indices_Or_All    given_list=${member_index_list}
1010     FOR    ${index}    IN    @{index_list}
1011         ${session} =    Resolve_Http_Session_For_Member    member_index=${index}
1012         Utils.No_Content_From_URI    ${session}    ${uri}
1013     END
1014
1015 Get_From_Member
1016     [Documentation]    Send a GET with the supplied uri to member ${member_index}.
1017     [Arguments]    ${uri}    ${member_index}    ${access}=${ACCEPT_EMPTY}
1018     ${session} =    Resolve_Http_Session_For_Member    member_index=${member_index}
1019     ${response_text} =    TemplatedRequests.Get_From_Uri    uri=${uri}    accept=${access}    session=${session}
1020     RETURN    ${response_text}
1021
1022 Resolve_IP_Address_For_Member
1023     [Documentation]    Return node IP address of given index.
1024     [Arguments]    ${member_index}
1025     ${ip_address} =    Collections.Get From Dictionary
1026     ...    dictionary=${ClusterManagement__index_to_ip_mapping}
1027     ...    key=${member_index}
1028     RETURN    ${ip_address}
1029
1030 Resolve_IP_Address_For_Members
1031     [Documentation]    Return a list of IP address of given indexes.
1032     [Arguments]    ${member_index_list}
1033     ${member_ip_list} =    BuiltIn.Create_List
1034     FOR    ${index}    IN    @{member_index_list}
1035         ${ip_address} =    Collections.Get From Dictionary
1036         ...    dictionary=${ClusterManagement__index_to_ip_mapping}
1037         ...    key=${index}
1038         Collections.Append_To_List    ${member_ip_list}    ${ip_address}
1039     END
1040     RETURN    ${member_ip_list}
1041
1042 Resolve_Http_Session_For_Member
1043     [Documentation]    Return RequestsLibrary session alias pointing to node of given index.
1044     [Arguments]    ${member_index}
1045     ${session} =    BuiltIn.Set_Variable    ClusterManagement__session_${member_index}
1046     RETURN    ${session}
1047
1048 Resolve_Shard_Type_Class
1049     [Documentation]    Simple lookup for class name corresponding to desired type.
1050     [Arguments]    ${shard_type}
1051     IF    '${shard_type}' == 'config'
1052         RETURN    DistributedConfigDatastore
1053     ELSE IF    '${shard_type}' == 'operational'
1054         RETURN    DistributedOperationalDatastore
1055     END
1056     BuiltIn.Fail    Unrecognized shard type: ${shard_type}
1057
1058 ClusterManagement__Build_List
1059     [Arguments]    ${member}
1060     ${member_int} =    BuiltIn.Convert_To_Integer    ${member}
1061     ${index_list} =    BuiltIn.Create_List    ${member_int}
1062     RETURN    ${index_list}
1063
1064 ClusterManagement__Parse_Sync_Status
1065     [Documentation]    Return sync status parsed out of given text. Called twice by Get_Sync_Status_Of_Member.
1066     [Arguments]    ${shard_manager_text}
1067     BuiltIn.Log    ${shard_manager_text}
1068     ${manager_object} =    RequestsLibrary.To_Json    ${shard_manager_text}
1069     ${value_object} =    Collections.Get_From_Dictionary    dictionary=${manager_object}    key=value
1070     ${sync_status} =    Collections.Get_From_Dictionary    dictionary=${value_object}    key=SyncStatus
1071     RETURN    ${sync_status}
1072
1073 List_All_Indices
1074     [Documentation]    Create a new list of all indices.
1075     BuiltIn.Run_Keyword_And_Return    List_Indices_Or_All
1076
1077 List_Indices_Or_All
1078     [Documentation]    Utility to allow \${EMPTY} as default argument value, as the internal list is computed at runtime.
1079     ...    This keyword always returns a (shallow) copy of given or default list,
1080     ...    so operations with the returned list should not affect other lists.
1081     ...    Also note that this keyword does not consider empty list to be \${EMPTY}.
1082     [Arguments]    ${given_list}=${EMPTY}
1083     ${return_list_reference} =    BuiltIn.Set_Variable_If
1084     ...    """${given_list}""" != ""
1085     ...    ${given_list}
1086     ...    ${ClusterManagement__member_index_list}
1087     ${return_list_copy} =    BuiltIn.Create_List    @{return_list_reference}
1088     RETURN    ${return_list_copy}
1089
1090 List_Indices_Minus_Member
1091     [Documentation]    Create a new list which contains indices from ${member_index_list} (or all) without ${member_index}.
1092     [Arguments]    ${member_index}    ${member_index_list}=${EMPTY}
1093     ${index_list} =    List_Indices_Or_All    ${member_index_list}
1094     Collections.Remove Values From List    ${index_list}    ${member_index}
1095     RETURN    ${index_list}
1096
1097 ClusterManagement__Compute_Derived_Variables
1098     [Documentation]    Construct index list, session list and IP mapping, publish them as suite variables.
1099     [Arguments]    ${int_of_members}    ${http_timeout}=${DEFAULT_TIMEOUT_HTTP}    ${http_retries}=0
1100     @{member_index_list} =    BuiltIn.Create_List
1101     @{session_list} =    BuiltIn.Create_List
1102     &{index_to_ip_mapping} =    BuiltIn.Create_Dictionary
1103     FOR    ${index}    IN RANGE    1    ${int_of_members+1}
1104         ClusterManagement__Include_Member_Index
1105         ...    ${index}
1106         ...    ${member_index_list}
1107         ...    ${session_list}
1108         ...    ${index_to_ip_mapping}
1109         ...    http_timeout=${http_timeout}
1110         ...    http_retries=${http_retries}
1111     END
1112     BuiltIn.Set_Suite_Variable    \${ClusterManagement__member_index_list}    ${member_index_list}
1113     BuiltIn.Set_Suite_Variable    \${ClusterManagement__index_to_ip_mapping}    ${index_to_ip_mapping}
1114     BuiltIn.Set_Suite_Variable    \${ClusterManagement__session_list}    ${session_list}
1115
1116 ClusterManagement__Include_Member_Index
1117     [Documentation]    Add a corresponding item based on index into the last three arguments.
1118     ...    Create the Http session whose alias is added to list.
1119     [Arguments]    ${index}    ${member_index_list}    ${session_list}    ${index_to_ip_mapping}    ${http_timeout}=${DEFAULT_TIMEOUT_HTTP}    ${http_retries}=0
1120     Collections.Append_To_List    ${member_index_list}    ${index}
1121     ${member_ip} =    BuiltIn.Set_Variable    ${ODL_SYSTEM_${index}_IP}
1122     # ${index} is int (not string) so "key=value" syntax does not work in the following line.
1123     Collections.Set_To_Dictionary    ${index_to_ip_mapping}    ${index}    ${member_ip}
1124     # Http session, with ${AUTH}, without headers.
1125     ${session_alias} =    Resolve_Http_Session_For_Member    member_index=${index}
1126     RequestsLibrary.Create_Session
1127     ...    ${session_alias}
1128     ...    http://${member_ip}:${RESTCONFPORT}
1129     ...    auth=${AUTH}
1130     ...    timeout=${http_timeout}
1131     ...    max_retries=${http_retries}
1132     Collections.Append_To_List    ${session_list}    ${session_alias}
1133
1134 Sync_Status_Should_Be_False
1135     [Documentation]    Verify that cluster node is not in sync with others
1136     [Arguments]    ${controller_index}
1137     ${status} =    Get_Sync_Status_Of_Member    ${controller_index}
1138     BuiltIn.Should_Not_Be_True    ${status}
1139
1140 Sync_Status_Should_Be_True
1141     [Documentation]    Verify that cluster node is in sync with others
1142     [Arguments]    ${controller_index}
1143     ${status} =    Get_Sync_Status_Of_Member    ${controller_index}
1144     BuiltIn.Should_Be_True    ${status}
1145
1146 Return_Member_IP
1147     [Documentation]    Return the IP address of the member given the member_index.
1148     [Arguments]    ${member_index}
1149     ${member_int} =    BuiltIn.Convert_To_Integer    ${member_index}
1150     ${member_ip} =    Collections.Get_From_Dictionary
1151     ...    dictionary=${ClusterManagement__index_to_ip_mapping}
1152     ...    key=${member_int}
1153     RETURN    ${member_ip}
1154
1155 Check Service Status
1156     [Documentation]    Issues the karaf shell command showSvcStatus to verify the ready and service states are the same as the arguments passed
1157     [Arguments]    ${odl_ip}    ${system_ready_state}    ${service_state}    @{service_list}
1158     IF    ${NUM_ODL_SYSTEM} > 1
1159         ${service_status_output} =    KarafKeywords.Issue_Command_On_Karaf_Console
1160         ...    showSvcStatus -n ${odl_ip}
1161         ...    ${odl_ip}
1162         ...    ${KARAF_SHELL_PORT}
1163     ELSE
1164         ${service_status_output} =    KarafKeywords.Issue_Command_On_Karaf_Console
1165         ...    showSvcStatus
1166         ...    ${odl_ip}
1167         ...    ${KARAF_SHELL_PORT}
1168     END
1169     BuiltIn.Should Contain    ${service_status_output}    ${system_ready_state}
1170     FOR    ${service}    IN    @{service_list}
1171         BuiltIn.Should Match Regexp    ${service_status_output}    ${service} +: ${service_state}
1172     END
1173
1174 Check Status Of Services Is OPERATIONAL
1175     [Documentation]    This keyword will verify whether all the services are operational in all the ODL nodes
1176     [Arguments]    @{service_list}
1177     FOR    ${i}    IN RANGE    ${NUM_ODL_SYSTEM}
1178         ClusterManagement.Check Service Status    ${ODL_SYSTEM_${i+1}_IP}    ACTIVE    OPERATIONAL    @{service_list}
1179     END