csit/suites/controller/ThreeNodes_Datastore/puts_during_isolation.robot

   1 *** Settings ***
   2 Documentation     Test when a car shard leader is isolated while configuring cars.
   3 ...
   4 ...               Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...               This test suite requires odl-restconf and odl-clustering-test-app modules.
  11 ...               The script cluster_rest_script.py is used for generating requests for
  12 ...               PUTing car items while the car shard leader is isolated.
  13 Suite Setup       Start_Suite
  14 Suite Teardown    Stop_Suite
  15 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  16 Default Tags      critical
  17 Library           RequestsLibrary
  18 Library           SSHLibrary
  19 Resource          ${CURDIR}/../../../variables/Variables.robot
  20 Resource          ${CURDIR}/../../../libraries/Utils.robot
  21 Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
  22 Resource          ${CURDIR}/../../../libraries/RemoteBash.robot
  23 Resource          ${CURDIR}/../../../libraries/ClusterManagement.robot
  24 Resource          ${CURDIR}/../../../libraries/CarPeople.robot
  25
  26 *** Variables ***
  27 ${ITEM_COUNT}     ${10000}
  28 ${THREADS}        10
  29 ${ADDCMD}         python ${TOOL_NAME} --port ${RESTCONFPORT} add-with-retries --itemtype car --itemcount ${ITEM_COUNT} --threads ${THREADS}
  30 ${CARURL}         /restconf/config/car:cars
  31 ${SHARD_NAME}     car
  32 ${SHARD_TYPE}     config
  33 ${TEST_LOG_LEVEL}    info
  34 @{TEST_LOG_COMPONENTS}    org.opendaylight.controller
  35 ${TOOL_OPTIONS}    ${EMPTY}
  36 ${TOOL_NAME}      cluster_rest_script.py
  37
  38 *** Test Cases ***
  39 Start_Adding_Cars_To_Follower
  40     [Documentation]    Start the script to configure ${ITEM_COUNT} cars in the background.
  41     ${idx} =    Collections.Get_From_List    ${car_follower_indices}    0
  42     ${follower_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    member_index=${idx}
  43     Start Tool    ${ADDCMD}    --host ${follower_ip} ${TOOL_OPTIONS}
  44     ${session} =    Resolve_Http_Session_For_Member    member_index=${car_leader_index}
  45     BuiltIn.Wait_Until_Keyword_Succeeds    10x    5s    Ensure_Cars_Being_Configured    ${session}
  46
  47 Isolate_Current_Car_Leader
  48     [Documentation]    Isolating cluster node which is the car shard leader.
  49     ClusterManagement.Isolate_Member_From_List_Or_All    ${car_leader_index}
  50     BuiltIn.Set Suite variable    ${old_car_leader}    ${car_leader_index}
  51     BuiltIn.Set Suite variable    ${old_car_followers}    ${car_follower_indices}
  52
  53 Verify_New_Car_Leader_Elected
  54     [Documentation]    Verify new owner of the car shard is elected.
  55     BuiltIn.Wait_Until_Keyword_Succeeds    10x    5s    ClusterManagement.Verify_Shard_Leader_Elected    ${SHARD_NAME}    ${SHARD_TYPE}    ${True}
  56     ...    ${old_car_leader}    member_index_list=${old_car_followers}
  57     CarPeople.Set_Tmp_Variables_For_Shard_For_Nodes    ${old_car_followers}    shard_name=${SHARD_NAME}    shard_type=${SHARD_TYPE}
  58
  59 Verify_Cars_Configured
  60     [Documentation]    Verify that all cars are configured.
  61     BuiltIn.Wait_Until_Keyword_Succeeds    120x    2s    SSHLibrary.Read_Until_Prompt
  62     ${session} =    Resolve_Http_Session_For_Member    member_index=${new_leader_index}
  63     BuiltIn.Wait_Until_Keyword_Succeeds    5x    2s    Verify_Cars_Count    ${ITEM_COUNT}    ${session}
  64
  65 Rejoin_Isolated_Member
  66     [Documentation]    Rejoin isolated node
  67     ClusterManagement.Rejoin_Member_From_List_Or_All    ${old_car_leader}
  68
  69 Delete Cars
  70     [Documentation]    Remove cars from the datastore
  71     ${session} =    Resolve_Http_Session_For_Member    member_index=${new_leader_index}
  72     ${rsp}=    RequestsLibrary.Delete Request    ${session}    ${CARURL}
  73     Should Be Equal As Numbers    200    ${rsp.status_code}
  74     ${rsp}=    RequestsLibrary.Get Request    ${session}    ${CARURL}
  75     Should Be Equal As Numbers    404    ${rsp.status_code}
  76
  77 *** Keywords ***
  78 Start Suite
  79     [Documentation]    Upload the script file and create a virtual env
  80     SetupUtils.Setup_Utils_For_Setup_And_Teardown
  81     SetupUtils.Setup_Logging_For_Debug_Purposes_On_List_Or_All    ${TEST_LOG_LEVEL}    ${TEST_LOG_COMPONENTS}
  82     ${mininet_conn_id} =    SSHKeywords.Open_Connection_To_Tools_System    prompt=~]>
  83     Builtin.Set Suite Variable    ${mininet_conn_id}
  84     SSHLibrary.Put File    ${CURDIR}/../../../../tools/odl-mdsal-clustering-tests/scripts/${TOOL_NAME}    .
  85     ${stdout}    ${stderr}    ${rc}=    SSHLibrary.Execute Command    ls    return_stdout=True    return_stderr=True
  86     ...    return_rc=True
  87     ${out_file} =    Utils.Get_Log_File_Name    ${TOOL_NAME}
  88     BuiltIn.Set_Suite_Variable    ${out_file}
  89     SSHKeywords.Virtual_Env_Create
  90     SSHKeywords.Virtual_Env_Install_Package    requests
  91     CarPeople.Set_Variables_For_Shard    ${SHARD_NAME}    shard_type=${SHARD_TYPE}
  92
  93 Stop Suite
  94     [Documentation]    Stop the tool, remove virtual env and close ssh connection towards tools vm.
  95     Stop_Tool
  96     ${session} =    Resolve_Http_Session_For_Member    member_index=${new_leader_index}
  97     # best effort to make sure cars are deleted in case more suites will run after this and the delete test case had trouble
  98     ${rsp}=    RequestsLibrary.Delete Request    ${session}    ${CARURL}
  99     BuiltIn.Log    ${rsp.status_code} : ${rsp.text}
 100     SSHKeywords.Virtual_Env_Delete
 101     Store_File_To_Workspace    ${out_file}    ${out_file}
 102     SSHLibrary.Close All Connections
 103
 104 Start_Tool
 105     [Arguments]    ${command}    ${tool_opt}
 106     [Documentation]    Start the tool
 107     # TODO: https://trello.com/c/rXsMu7iz/444-create-keywords-for-the-tool-start-and-stop-in-remotebash-robot
 108     BuiltIn.Log    ${command}
 109     SSHKeywords.Virtual_Env_Activate_On_Current_Session    log_output=${True}
 110     ${output}=    SSHLibrary.Write    ${command} ${tool_opt} 2>&1 | tee ${out_file}
 111     BuiltIn.Log    ${output}
 112
 113 Stop_Tool
 114     [Documentation]    Stop the tool if still running.
 115     # TODO: https://trello.com/c/rXsMu7iz/444-create-keywords-for-the-tool-start-and-stop-in-remotebash-robot
 116     ${output}=    SSHLibrary.Read
 117     BuiltIn.Log    ${output}
 118     RemoteBash.Write_Bare_Ctrl_C
 119     ${output}=    SSHLibrary.Read_Until_Prompt
 120     BuiltIn.Log    ${output}
 121     SSHKeywords.Virtual_Env_Deactivate_On_Current_Session    log_output=${True}
 122
 123 Verify_Cars_Count
 124     [Arguments]    ${exp_count}    ${session}
 125     [Documentation]    Count car items in config ds and compare with expected number.
 126     ${count} =    Get_Cars_Count    ${session}
 127     BuiltIn.Should_Be_Equal_As_Numbers    ${count}    ${exp_count}
 128
 129 Get_Cars_Count
 130     [Arguments]    ${session}
 131     [Documentation]    Count car items in config ds.
 132     ${resp}=    RequestsLibrary.Get_Request    ${session}    ${CARURL}
 133     ${count} =    BuiltIn.Evaluate    len(${resp.json()}[cars][car-entry])
 134     BuiltIn.Return_From_Keyword    ${count}
 135
 136 Ensure_Cars_Being_Configured
 137     [Arguments]    ${session}
 138     [Documentation]    FIXME: Add a documentation.
 139     ${count1} =    Get_Cars_Count    ${session}
 140     ${count2} =    Get_Cars_Count    ${session}
 141     BuiltIn.Should_Not_Be_Equal_As_Integers    ${count1}    ${count2}
 142
 143 Store_File_To_Workspace
 144     [Arguments]    ${source_file_name}    ${target_file_name}
 145     [Documentation]    Store the ${source_file_name} to the workspace as ${target_file_name}.
 146     SSHLibrary.Get_File    ${source_file_name}    ${target_file_name}