csit/suites/controller/ThreeNodes_Datastore/puts_during_isolation.robot

   1 *** Settings ***
   2 Documentation     Test when a car shard leader is isolated while configuring cars.
   3 ...
   4 ...               Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...               This test suite requires odl-restconf and odl-clustering-test-app modules.
  11 ...               The script cluster_rest_script.py is used for generating requests for
  12 ...               PUTing car items while the car shard leader is isolated.
  13 Suite Setup       Start_Suite
  14 Suite Teardown    Stop_Suite
  15 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  16 Default Tags      critical
  17 Library           RequestsLibrary
  18 Library           SSHLibrary
  19 Resource          ${CURDIR}/../../../variables/Variables.robot
  20 Resource          ${CURDIR}/../../../libraries/Utils.robot
  21 Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
  22 Resource          ${CURDIR}/../../../libraries/ClusterManagement.robot
  23 Resource          ${CURDIR}/../../../libraries/CarPeople.robot
  24
  25 *** Variables ***
  26 ${ITEM_COUNT}     ${10000}
  27 ${THREADS}        10
  28 ${ADDCMD}         python ${TOOL_NAME} --port ${RESTCONFPORT} add-with-retries --itemtype car --itemcount ${ITEM_COUNT} --threads ${THREADS}
  29 ${CARURL}         /restconf/config/car:cars
  30 ${SHARD_NAME}     car
  31 ${SHARD_TYPE}     config
  32 ${TEST_LOG_LEVEL}    info
  33 @{TEST_LOG_COMPONENTS}    org.opendaylight.controller
  34 ${TOOL_OPTIONS}    ${EMPTY}
  35 ${TOOL_NAME}      cluster_rest_script.py
  36
  37 *** Test Cases ***
  38 Start_Adding_Cars_To_Follower
  39     [Documentation]    Start the script to configure ${ITEM_COUNT} cars in the background.
  40     ${idx} =    Collections.Get_From_List    ${car_follower_indices}    0
  41     ${follower_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    member_index=${idx}
  42     Start Tool    ${ADDCMD}    --host ${follower_ip} ${TOOL_OPTIONS}
  43     ${session} =    Resolve_Http_Session_For_Member    member_index=${car_leader_index}
  44     BuiltIn.Wait_Until_Keyword_Succeeds    5x    2s    Ensure_Cars_Being_Configured    ${session}
  45
  46 Isolate_Current_Car_Leader
  47     [Documentation]    Isolating cluster node which is the car shard leader.
  48     ClusterManagement.Isolate_Member_From_List_Or_All    ${car_leader_index}
  49     BuiltIn.Set Suite variable    ${old_car_leader}    ${car_leader_index}
  50     BuiltIn.Set Suite variable    ${old_car_followers}    ${car_follower_indices}
  51
  52 Verify_New_Car_Leader_Elected
  53     [Documentation]    Verify new owner of the car shard is elected.
  54     BuiltIn.Wait_Until_Keyword_Succeeds    5x    2s    ClusterManagement.Verify_Shard_Leader_Elected    ${SHARD_NAME}    ${SHARD_TYPE}    ${True}
  55     ...    ${old_car_leader}    member_index_list=${old_car_followers}
  56     CarPeople.Set_Tmp_Variables_For_Shard_For_Nodes    ${old_car_followers}    shard_name=${SHARD_NAME}    shard_type=${SHARD_TYPE}
  57
  58 Verify_Cars_Configured
  59     [Documentation]    Verify that all cars are configured.
  60     BuiltIn.Wait_Until_Keyword_Succeeds    120x    2s    SSHLibrary.Read_Until_Prompt
  61     ${session} =    Resolve_Http_Session_For_Member    member_index=${new_leader_index}
  62     Verify_Cars_Count    ${ITEM_COUNT}    ${session}
  63
  64 Rejoin_Isolated_Member
  65     [Documentation]    Rejoin isolated node
  66     ClusterManagement.Rejoin_Member_From_List_Or_All    ${old_car_leader}
  67
  68 Delete Cars
  69     [Documentation]    Remove cars from the datastore
  70     ${session} =    Resolve_Http_Session_For_Member    member_index=${new_leader_index}
  71     ${rsp}=    RequestsLibrary.Delete Request    ${session}    ${CARURL}
  72     Should Be Equal As Numbers    200    ${rsp.status_code}
  73     ${rsp}=    RequestsLibrary.Get Request    ${session}    ${CARURL}
  74     Should Be Equal As Numbers    404    ${rsp.status_code}
  75
  76 *** Keywords ***
  77 Start Suite
  78     [Documentation]    Upload the script file and create a virtual env
  79     SetupUtils.Setup_Utils_For_Setup_And_Teardown
  80     SetupUtils.Setup_Logging_For_Debug_Purposes_On_List_Or_All    ${TEST_LOG_LEVEL}    ${TEST_LOG_COMPONENTS}
  81     ${mininet_conn_id} =    SSHKeywords.Open_Connection_To_Tools_System
  82     Builtin.Set Suite Variable    ${mininet_conn_id}
  83     SSHLibrary.Put File    ${CURDIR}/../../../../tools/odl-mdsal-clustering-tests/scripts/${TOOL_NAME}    .
  84     ${stdout}    ${stderr}    ${rc}=    SSHLibrary.Execute Command    ls    return_stdout=True    return_stderr=True
  85     ...    return_rc=True
  86     ${out_file} =    Utils.Get_Log_File_Name    ${TOOL_NAME}
  87     BuiltIn.Set_Suite_Variable    ${out_file}
  88     SSHKeywords.Virtual_Env_Create
  89     SSHKeywords.Virtual_Env_Install_Package    requests
  90     CarPeople.Set_Variables_For_Shard    ${SHARD_NAME}    shard_type=${SHARD_TYPE}
  91
  92 Stop Suite
  93     [Documentation]    Stop the tool, remove virtual env and close ssh connection towards tools vm.
  94     Stop_Tool
  95     SSHKeywords.Virtual_Env_Delete
  96     Store_File_To_Workspace    ${out_file}    ${out_file}
  97     SSHLibrary.Close All Connections
  98
  99 Start_Tool
 100     [Arguments]    ${command}    ${tool_opt}
 101     [Documentation]    Start the tool
 102     # TODO: https://trello.com/c/rXsMu7iz/444-create-keywords-for-the-tool-start-and-stop-in-remotebash-robot
 103     BuiltIn.Log    ${command}
 104     SSHKeywords.Virtual_Env_Activate_On_Current_Session    log_output=${True}
 105     ${output}=    SSHLibrary.Write    ${command} ${tool_opt} 2>&1 | tee ${out_file}
 106     BuiltIn.Log    ${output}
 107
 108 Stop_Tool
 109     [Documentation]    Stop the tool if still running.
 110     # TODO: https://trello.com/c/rXsMu7iz/444-create-keywords-for-the-tool-start-and-stop-in-remotebash-robot
 111     ${output}=    SSHLibrary.Read
 112     BuiltIn.Log    ${output}
 113     Utils.Write_Bare_Ctrl_C
 114     ${output}=    SSHLibrary.Read_Until_Prompt
 115     BuiltIn.Log    ${output}
 116     SSHKeywords.Virtual_Env_Deactivate_On_Current_Session    log_output=${True}
 117
 118 Verify_Cars_Count
 119     [Arguments]    ${exp_count}    ${session}
 120     [Documentation]    Count car items in config ds and compare with expected number.
 121     ${count} =    Get_Cars_Count    ${session}
 122     BuiltIn.Should_Be_Equal_As_Numbers    ${count}    ${exp_count}
 123
 124 Get_Cars_Count
 125     [Arguments]    ${session}
 126     [Documentation]    Count car items in config ds.
 127     ${resp}=    RequestsLibrary.Get_Request    ${session}    ${CARURL}
 128     ${count} =    BuiltIn.Evaluate    len(${resp.json()}["cars"]["car-entry"])
 129     BuiltIn.Return_From_Keyword    ${count}
 130
 131 Ensure_Cars_Being_Configured
 132     [Arguments]    ${session}
 133     [Documentation]    FIXME: Add a documentation.
 134     ${count1} =    Get_Cars_Count    ${session}
 135     ${count2} =    Get_Cars_Count    ${session}
 136     BuiltIn.Should_Not_Be_Equal_As_Integers    ${count1}    ${count2}
 137
 138 Store_File_To_Workspace
 139     [Arguments]    ${source_file_name}    ${target_file_name}
 140     [Documentation]    Store the ${source_file_name} to the workspace as ${target_file_name}.
 141     SSHLibrary.Get_File    ${source_file_name}    ${target_file_name}