csit/suites/controller/ThreeNodes_Datastore/puts_during_isolation.robot

   1 *** Settings ***
   2 Documentation       Test when a car shard leader is isolated while configuring cars.
   3 ...
   4 ...                 Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...                 This program and the accompanying materials are made available under the
   7 ...                 terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...                 and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...                 This test suite requires odl-restconf and odl-clustering-test-app modules.
  11 ...                 The script cluster_rest_script.py is used for generating requests for
  12 ...                 PUTing car items while the car shard leader is isolated.
  13
  14 Library             RequestsLibrary
  15 Library             SSHLibrary
  16 Resource            ${CURDIR}/../../../libraries/CarPeople.robot
  17 Resource            ${CURDIR}/../../../libraries/ClusterManagement.robot
  18 Resource            ${CURDIR}/../../../libraries/RemoteBash.robot
  19 Resource            ${CURDIR}/../../../libraries/SetupUtils.robot
  20 Resource            ${CURDIR}/../../../libraries/TemplatedRequests.robot
  21 Resource            ${CURDIR}/../../../libraries/Utils.robot
  22 Resource            ${CURDIR}/../../../variables/Variables.robot
  23
  24 Suite Setup         Start_Suite
  25 Suite Teardown      Stop_Suite
  26 Test Setup          SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  27
  28 Default Tags        critical
  29
  30
  31 *** Variables ***
  32 ${ITEM_COUNT}               ${10000}
  33 ${THREADS}                  10
  34 ${ADDCMD}
  35 ...                         python ${TOOL_NAME} --port ${RESTCONFPORT} add-with-retries --itemtype car --itemcount ${ITEM_COUNT} --threads ${THREADS}
  36 ${CARURL}                   /restconf/config/car:cars
  37 ${SHARD_NAME}               car
  38 ${SHARD_TYPE}               config
  39 ${TEST_LOG_LEVEL}           info
  40 @{TEST_LOG_COMPONENTS}      org.opendaylight.controller
  41 ${TOOL_OPTIONS}             ${EMPTY}
  42 ${TOOL_NAME}                cluster_rest_script.py
  43
  44
  45 *** Test Cases ***
  46 Start_Adding_Cars_To_Follower
  47     [Documentation]    Start the script to configure ${ITEM_COUNT} cars in the background.
  48     ${idx} =    Collections.Get_From_List    ${car_follower_indices}    0
  49     ${follower_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    member_index=${idx}
  50     Start Tool    ${ADDCMD}    --host ${follower_ip} ${TOOL_OPTIONS}
  51     ${session} =    Resolve_Http_Session_For_Member    member_index=${car_leader_index}
  52     BuiltIn.Wait_Until_Keyword_Succeeds    10x    5s    Ensure_Cars_Being_Configured    ${session}
  53
  54 Isolate_Current_Car_Leader
  55     [Documentation]    Isolating cluster node which is the car shard leader.
  56     ClusterManagement.Isolate_Member_From_List_Or_All    ${car_leader_index}
  57     BuiltIn.Set Suite variable    ${old_car_leader}    ${car_leader_index}
  58     BuiltIn.Set Suite variable    ${old_car_followers}    ${car_follower_indices}
  59
  60 Verify_New_Car_Leader_Elected
  61     [Documentation]    Verify new owner of the car shard is elected.
  62     BuiltIn.Wait_Until_Keyword_Succeeds
  63     ...    10x
  64     ...    5s
  65     ...    ClusterManagement.Verify_Shard_Leader_Elected
  66     ...    ${SHARD_NAME}
  67     ...    ${SHARD_TYPE}
  68     ...    ${True}
  69     ...    ${old_car_leader}
  70     ...    member_index_list=${old_car_followers}
  71     CarPeople.Set_Tmp_Variables_For_Shard_For_Nodes
  72     ...    ${old_car_followers}
  73     ...    shard_name=${SHARD_NAME}
  74     ...    shard_type=${SHARD_TYPE}
  75
  76 Verify_Cars_Configured
  77     [Documentation]    Verify that all cars are configured.
  78     BuiltIn.Wait_Until_Keyword_Succeeds    120x    2s    SSHLibrary.Read_Until_Prompt
  79     ${session} =    Resolve_Http_Session_For_Member    member_index=${new_leader_index}
  80     BuiltIn.Wait_Until_Keyword_Succeeds    5x    2s    Verify_Cars_Count    ${ITEM_COUNT}    ${session}
  81
  82 Rejoin_Isolated_Member
  83     [Documentation]    Rejoin isolated node
  84     ClusterManagement.Rejoin_Member_From_List_Or_All    ${old_car_leader}
  85
  86 Delete Cars
  87     [Documentation]    Remove cars from the datastore
  88     ${session} =    Resolve_Http_Session_For_Member    member_index=${new_leader_index}
  89     ${rsp} =    RequestsLibrary.Delete Request    ${session}    ${CARURL}
  90     Should Be Equal As Numbers    200    ${rsp.status_code}
  91     ${rsp} =    RequestsLibrary.Get Request    ${session}    ${CARURL}
  92     Should Contain    ${DELETED_STATUS_CODES}    ${rsp.status_code}
  93
  94
  95 *** Keywords ***
  96 Start Suite
  97     [Documentation]    Upload the script file and create a virtual env
  98     SetupUtils.Setup_Utils_For_Setup_And_Teardown
  99     SetupUtils.Setup_Logging_For_Debug_Purposes_On_List_Or_All    ${TEST_LOG_LEVEL}    ${TEST_LOG_COMPONENTS}
 100     ${mininet_conn_id} =    SSHKeywords.Open_Connection_To_Tools_System    prompt=~]>
 101     Builtin.Set Suite Variable    ${mininet_conn_id}
 102     SSHLibrary.Put File    ${CURDIR}/../../../../tools/odl-mdsal-clustering-tests/scripts/${TOOL_NAME}    .
 103     ${stdout}    ${stderr}    ${rc} =    SSHLibrary.Execute Command    ls    return_stdout=True    return_stderr=True
 104     ...    return_rc=True
 105     ${out_file} =    Utils.Get_Log_File_Name    ${TOOL_NAME}
 106     BuiltIn.Set_Suite_Variable    ${out_file}
 107     SSHKeywords.Virtual_Env_Create
 108     SSHKeywords.Virtual_Env_Install_Package    requests
 109     CarPeople.Set_Variables_For_Shard    ${SHARD_NAME}    shard_type=${SHARD_TYPE}
 110
 111 Stop Suite
 112     [Documentation]    Stop the tool, remove virtual env and close ssh connection towards tools vm.
 113     Stop_Tool
 114     ${session} =    Resolve_Http_Session_For_Member    member_index=${new_leader_index}
 115     # best effort to make sure cars are deleted in case more suites will run after this and the delete test case had trouble
 116     ${rsp} =    RequestsLibrary.Delete Request    ${session}    ${CARURL}
 117     BuiltIn.Log    ${rsp.status_code} : ${rsp.text}
 118     SSHKeywords.Virtual_Env_Delete
 119     Store_File_To_Workspace    ${out_file}    ${out_file}
 120     SSHLibrary.Close All Connections
 121
 122 Start_Tool
 123     [Documentation]    Start the tool
 124     [Arguments]    ${command}    ${tool_opt}
 125     # TODO: https://trello.com/c/rXsMu7iz/444-create-keywords-for-the-tool-start-and-stop-in-remotebash-robot
 126     BuiltIn.Log    ${command}
 127     SSHKeywords.Virtual_Env_Activate_On_Current_Session    log_output=${True}
 128     ${output} =    SSHLibrary.Write    ${command} ${tool_opt} 2>&1 | tee ${out_file}
 129     BuiltIn.Log    ${output}
 130
 131 Stop_Tool
 132     [Documentation]    Stop the tool if still running.
 133     # TODO: https://trello.com/c/rXsMu7iz/444-create-keywords-for-the-tool-start-and-stop-in-remotebash-robot
 134     ${output} =    SSHLibrary.Read
 135     BuiltIn.Log    ${output}
 136     RemoteBash.Write_Bare_Ctrl_C
 137     ${output} =    SSHLibrary.Read_Until_Prompt
 138     BuiltIn.Log    ${output}
 139     SSHKeywords.Virtual_Env_Deactivate_On_Current_Session    log_output=${True}
 140
 141 Verify_Cars_Count
 142     [Documentation]    Count car items in config ds and compare with expected number.
 143     [Arguments]    ${exp_count}    ${session}
 144     ${count} =    Get_Cars_Count    ${session}
 145     BuiltIn.Should_Be_Equal_As_Numbers    ${count}    ${exp_count}
 146
 147 Get_Cars_Count
 148     [Documentation]    Count car items in config ds.
 149     [Arguments]    ${session}
 150     ${resp} =    RequestsLibrary.Get_Request    ${session}    ${CARURL}
 151     ${count} =    BuiltIn.Evaluate    len(${resp.json()}[cars][car-entry])
 152     RETURN    ${count}
 153
 154 Ensure_Cars_Being_Configured
 155     [Documentation]    FIXME: Add a documentation.
 156     [Arguments]    ${session}
 157     ${count1} =    Get_Cars_Count    ${session}
 158     ${count2} =    Get_Cars_Count    ${session}
 159     BuiltIn.Should_Not_Be_Equal_As_Integers    ${count1}    ${count2}
 160
 161 Store_File_To_Workspace
 162     [Documentation]    Store the ${source_file_name} to the workspace as ${target_file_name}.
 163     [Arguments]    ${source_file_name}    ${target_file_name}
 164     SSHLibrary.Get_File    ${source_file_name}    ${target_file_name}