csit/suites/controller/ThreeNodes_Datastore/puts_during_isolation.robot

   1 *** Settings ***
   2 Documentation     Test when a car shard leader is isolated while configuring cars.
   3 ...
   4 ...               Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...               This test suite requires odl-restconf and odl-clustering-test-app modules.
  11 ...               The script cluster_rest_script.py is used for generating requests for
  12 ...               PUTing car items while the car shard leader is isolated.
  13 Suite Setup       Start_Suite
  14 Suite Teardown    Stop_Suite
  15 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  16 Library           RequestsLibrary
  17 Library           SSHLibrary
  18 Resource          ${CURDIR}/../../../variables/Variables.robot
  19 Resource          ${CURDIR}/../../../libraries/Utils.robot
  20 Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
  21 Resource          ${CURDIR}/../../../libraries/ClusterManagement.robot
  22 Resource          ${CURDIR}/../../../libraries/CarPeople.robot
  23
  24 *** Variables ***
  25 ${ITEM_COUNT}     ${10000}
  26 ${THREADS}        10
  27 ${ADDCMD}         python ${TOOL_NAME} --port ${RESTCONFPORT} add-with-retries --itemtype car --itemcount ${ITEM_COUNT} --threads ${THREADS}
  28 ${CARURL}         /restconf/config/car:cars
  29 ${SHARD_NAME}     car
  30 ${SHARD_TYPE}     config
  31 ${TEST_LOG_LEVEL}    info
  32 @{TEST_LOG_COMPONENTS}    org.opendaylight.controller
  33 ${TOOL_OPTIONS}    ${EMPTY}
  34 ${TOOL_NAME}      cluster_rest_script.py
  35
  36 *** Test Cases ***
  37 Start_Adding_Cars_To_Follower
  38     [Documentation]    Start the script to configure ${ITEM_COUNT} cars in the background.
  39     ${idx} =    Collections.Get_From_List    ${car_follower_indices}    0
  40     ${follower_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    member_index=${idx}
  41     Start Tool    ${ADDCMD}    --host ${follower_ip} ${TOOL_OPTIONS}
  42     ${session} =    Resolve_Http_Session_For_Member    member_index=${car_leader_index}
  43     BuiltIn.Wait_Until_Keyword_Succeeds    5x    2s    Ensure_Cars_Being_Configured    ${session}
  44
  45 Isolate_Current_Car_Leader
  46     [Documentation]    Isolating cluster node which is the car shard leader.
  47     ClusterManagement.Isolate_Member_From_List_Or_All    ${car_leader_index}
  48     BuiltIn.Set Suite variable    ${old_car_leader}    ${car_leader_index}
  49     BuiltIn.Set Suite variable    ${old_car_followers}    ${car_follower_indices}
  50
  51 Verify_New_Car_Leader_Elected
  52     [Documentation]    Verify new owner of the car shard is elected.
  53     [Tags]    critical
  54     BuiltIn.Wait_Until_Keyword_Succeeds    5x    2s    ClusterManagement.Verify_Shard_Leader_Elected    ${SHARD_NAME}    ${SHARD_TYPE}    ${True}
  55     ...    ${old_car_leader}    member_index_list=${old_car_followers}
  56     CarPeople.Set_Tmp_Variables_For_Shard_For_Nodes    ${old_car_followers}    shard_name=${SHARD_NAME}    shard_type=${SHARD_TYPE}
  57
  58 Verify_Cars_Configured
  59     [Documentation]    Verify that all cars are configured.
  60     [Tags]    critical
  61     BuiltIn.Wait_Until_Keyword_Succeeds    120x    2s    SSHLibrary.Read_Until_Prompt
  62     ${session} =    Resolve_Http_Session_For_Member    member_index=${new_leader_index}
  63     Verify_Cars_Count    ${ITEM_COUNT}    ${session}
  64
  65 Rejoin_Isolated_Member
  66     [Documentation]    Rejoin isolated node
  67     [Tags]    @{NO_TAGS}
  68     ClusterManagement.Rejoin_Member_From_List_Or_All    ${old_car_leader}
  69
  70 Delete Cars
  71     [Documentation]    Remove cars from the datastore
  72     ${session} =    Resolve_Http_Session_For_Member    member_index=${new_leader_index}
  73     ${rsp}=    RequestsLibrary.Delete Request    ${session}    ${CARURL}
  74     Should Be Equal As Numbers    200    ${rsp.status_code}
  75     ${rsp}=    RequestsLibrary.Get Request    ${session}    ${CARURL}
  76     Should Be Equal As Numbers    404    ${rsp.status_code}
  77
  78 *** Keywords ***
  79 Start Suite
  80     [Documentation]    Upload the script file and create a virtual env
  81     SetupUtils.Setup_Utils_For_Setup_And_Teardown
  82     SetupUtils.Setup_Logging_For_Debug_Purposes_On_List_Or_All    ${TEST_LOG_LEVEL}    ${TEST_LOG_COMPONENTS}
  83     ${mininet_conn_id} =    SSHKeywords.Open_Connection_To_Tools_System
  84     Builtin.Set Suite Variable    ${mininet_conn_id}
  85     SSHLibrary.Put File    ${CURDIR}/../../../../tools/odl-mdsal-clustering-tests/scripts/${TOOL_NAME}    .
  86     ${stdout}    ${stderr}    ${rc}=    SSHLibrary.Execute Command    ls    return_stdout=True    return_stderr=True
  87     ...    return_rc=True
  88     ${out_file} =    Utils.Get_Log_File_Name    ${TOOL_NAME}
  89     BuiltIn.Set_Suite_Variable    ${out_file}
  90     SSHKeywords.Virtual_Env_Create
  91     SSHKeywords.Virtual_Env_Install_Package    requests
  92     CarPeople.Set_Variables_For_Shard    ${SHARD_NAME}    shard_type=${SHARD_TYPE}
  93
  94 Stop Suite
  95     [Documentation]    Stop the tool, remove virtual env and close ssh connection towards tools vm.
  96     Stop_Tool
  97     SSHKeywords.Virtual_Env_Delete
  98     Store_File_To_Workspace    ${out_file}    ${out_file}
  99     SSHLibrary.Close All Connections
 100
 101 Start_Tool
 102     [Arguments]    ${command}    ${tool_opt}
 103     [Documentation]    Start the tool
 104     # TODO: https://trello.com/c/rXsMu7iz/444-create-keywords-for-the-tool-start-and-stop-in-remotebash-robot
 105     BuiltIn.Log    ${command}
 106     SSHKeywords.Virtual_Env_Activate_On_Current_Session    log_output=${True}
 107     ${output}=    SSHLibrary.Write    ${command} ${tool_opt} 2>&1 | tee ${out_file}
 108     BuiltIn.Log    ${output}
 109
 110 Stop_Tool
 111     [Documentation]    Stop the tool if still running.
 112     # TODO: https://trello.com/c/rXsMu7iz/444-create-keywords-for-the-tool-start-and-stop-in-remotebash-robot
 113     ${output}=    SSHLibrary.Read
 114     BuiltIn.Log    ${output}
 115     Utils.Write_Bare_Ctrl_C
 116     ${output}=    SSHLibrary.Read_Until_Prompt
 117     BuiltIn.Log    ${output}
 118     SSHKeywords.Virtual_Env_Deactivate_On_Current_Session    log_output=${True}
 119
 120 Verify_Cars_Count
 121     [Arguments]    ${exp_count}    ${session}
 122     [Documentation]    Count car items in config ds and compare with expected number.
 123     ${count} =    Get_Cars_Count    ${session}
 124     BuiltIn.Should_Be_Equal_As_Numbers    ${count}    ${exp_count}
 125
 126 Get_Cars_Count
 127     [Arguments]    ${session}
 128     [Documentation]    Count car items in config ds.
 129     ${resp}=    RequestsLibrary.Get_Request    ${session}    ${CARURL}
 130     ${count} =    BuiltIn.Evaluate    len(${resp.json()}["cars"]["car-entry"])
 131     BuiltIn.Return_From_Keyword    ${count}
 132
 133 Ensure_Cars_Being_Configured
 134     [Arguments]    ${session}
 135     ${count1} =    Get_Cars_Count    ${session}
 136     ${count2} =    Get_Cars_Count    ${session}
 137     BuiltIn.Should_Not_Be_Equal_As_Integers    ${count1}    ${count2}
 138
 139 Store_File_To_Workspace
 140     [Arguments]    ${source_file_name}    ${target_file_name}
 141     [Documentation]    Store the ${source_file_name} to the workspace as ${target_file_name}.
 142     SSHLibrary.Get_File    ${source_file_name}    ${target_file_name}