Initial works for OF cluster test 34/29234/27
authorLuis Gomez <ecelgp@gmail.com>
Wed, 4 Nov 2015 02:43:30 +0000 (18:43 -0800)
committerGerrit Code Review <gerrit@opendaylight.org>
Thu, 12 Nov 2015 22:48:00 +0000 (22:48 +0000)
Initial OpenFlow Cluster test suite
New Cluster Keywords to manage Shard and Entity Owner status.
New Cluster Keywords to Check, Get, Put and Delete URI
New Mininet library including multiple controller setup.
Fix Run Command On Remote System to do execute command
New Utils keyword to extract dictionary from list.
New Utils Keyword to check item occurrence in a string.
Add missing variables for MININET.

Change-Id: I3d3f2503140f3af4be80a64f96ebcdde45737930
Signed-off-by: Luis Gomez <ecelgp@gmail.com>
csit/libraries/ClusterKeywords.robot
csit/libraries/MininetKeywords.robot [new file with mode: 0644]
csit/libraries/Utils.robot
csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot [new file with mode: 0644]
csit/testplans/openflowplugin-clustering.txt
csit/variables/Variables.py
csit/variables/openflowplugin/add_flow_rpc.json [new file with mode: 0644]
csit/variables/openflowplugin/delete_flow_rpc.json [new file with mode: 0644]
csit/variables/openflowplugin/sample_flow_1.json [new file with mode: 0644]
csit/variables/openflowplugin/sample_flow_2.json [new file with mode: 0644]

index 8ba9dbadf4c8aa83d523ea05293f9f329221dfbc..ccd29bec4a20e864a1afbd980738bcd5ae0b4498 100644 (file)
@@ -7,8 +7,110 @@ Resource          Utils.robot
 
 *** Variables ***
 ${smc_node}       /org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore
+${jolokia_read}    /jolokia/read/org.opendaylight.controller
 
 *** Keywords ***
+Create Controller Index List
+    [Documentation]    Reads number of controllers and returns a list with all controllers indexes.
+    ${controller_index_list}    Create List
+    ${NUM_ODL_SYSTEM}=    Convert to Integer    ${NUM_ODL_SYSTEM}
+    : FOR    ${i}    IN RANGE    ${NUM_ODL_SYSTEM}
+    \    Append To List    ${controller_index_list}    ${i+1}
+    [Return]    ${controller_index_list}
+
+Create Controller Sessions
+    [Documentation]    Creates REST session to all controller instances.
+    ${NUM_ODL_SYSTEM}=    Convert to Integer    ${NUM_ODL_SYSTEM}
+    : FOR    ${i}    IN RANGE    ${NUM_ODL_SYSTEM}
+    \    Log    Create Session ${ODL_SYSTEM_${i+1}_IP}
+    \    Create Session    controller${i+1}    http://${ODL_SYSTEM_${i+1}_IP}:${RESTCONFPORT}    auth=${AUTH}
+
+Get Cluster Shard Status
+    [Arguments]    ${controller_index_list}    ${shard_type}    ${shard}
+    [Documentation]    Checks ${shard} status and returns Leader index and a list of Followers from a ${controller_index_list}.
+    ...    ${shard_type} is either config or operational.
+    ${lenght}=    Get Length    ${controller_index_list}
+    Run Keyword If    '${shard_type}' == 'config'    Set Test Variable    ${type}    DistributedConfigDatastore
+    Run Keyword If    '${shard_type}' == 'operational'    Set Test Variable    ${type}    DistributedOperationalDatastore
+    Should Not Be Empty    ${type}    Wrong type, valid values are config and operational.
+    ${leader}=    Set Variable    0
+    ${follower_list}=    Create List
+    : FOR    ${i}    IN    @{controller_index_list}
+    \    ${data}=    Get Data From URI    controller${i}    ${jolokia_read}:Category=Shards,name=member-${i}-shard-${shard}-${shard_type},type=${type}
+    \    Log    ${data}
+    \    ${json}=    To Json    ${data}
+    \    ${status}=    Get From Dictionary    &{json}[value]    RaftState
+    \    Log    Controller ${ODL_SYSTEM_${i}_IP} is ${status} for shard ${shard}
+    \    Run Keyword If    '${status}' == 'Leader'    Set Test Variable    ${leader}    ${i}
+    \    Run Keyword If    '${status}' == 'Follower'    Append To List    ${follower_list}    ${i}
+    Should Not Be Equal    ${leader}    0    No Leader elected in shard ${shard_type} ${shard}
+    Length Should Be    ${follower_list}    ${lenght-1}    Not enough or too many Followers in shard ${shard_type} ${shard}
+    [Return]    ${leader}    ${follower_list}
+
+Get Cluster Entity Owner Status
+    [Arguments]    ${controller_index_list}    ${device_type}    ${device}
+    [Documentation]    Checks Entity Owner status for a ${device} and returns owner index and list of candidates from a ${controller_index_list}.
+    ...    ${device_type} is openflow, ovsdb, etc...
+    ${length}=    Get Length    ${controller_index_list}
+    ${candidates_list}=    Create List
+    ${data}=    Get Data From URI    controller@{controller_index_list}[0]    /restconf/operational/entity-owners:entity-owners
+    Log    ${data}
+    ${data}=    Replace String    ${data}    /general-entity:entity[general-entity:name='    ${EMPTY}
+    ${clear_data}=    Replace String    ${data}    ']    ${EMPTY}
+    Log    ${clear_data}
+    ${json}=    To Json    ${clear_data}
+    ${entity_type_list}=    Get From Dictionary    &{json}[entity-owners]    entity-type
+    ${entity_type_index}=    Get Index From List Of Dictionaries    ${entity_type_list}    type    ${device_type}
+    Should Not Be Equal    ${entity_type_index}    -1    No Entity Owner found for ${device_type}
+    ${entity_list}=    Get From Dictionary    @{entity_type_list}[${entity_type_index}]    entity
+    ${entity_index}=    Get Index From List Of Dictionaries    ${entity_list}    id    ${device}
+    Should Not Be Equal    ${entity_index}    -1    Device ${device} not found in Entity Owner ${device_type}
+    ${entity_owner}=    Get From Dictionary    @{entity_list}[${entity_index}]    owner
+    Should Not Be Empty    ${entity_owner}    No owner found for ${device}
+    ${owner}=    Replace String    ${entity_owner}    member-    ${EMPTY}
+    ${owner}=    Convert To Integer    ${owner}
+    ${entity_candidates_list}=    Get From Dictionary    @{entity_list}[${entity_index}]    candidate
+    ${list_length}=    Get Length    ${entity_candidates_list}
+    Should Be Equal    ${list_length}    ${length}    Not enough or too many candidates for ${device}
+    : FOR    ${entity_candidate}    IN    @{entity_candidates_list}
+    \    ${candidate}=    Replace String    &{entity_candidate}[name]    member-    ${EMPTY}
+    \    ${candidate}=    Convert To Integer    ${candidate}
+    \    Run Keyword If    '${candidate}' != '${owner}'    Append To List    ${candidates_list}    ${candidate}
+    [Return]    ${owner}    ${candidates_list}
+
+Check Item Occurrence At URI In Cluster
+    [Arguments]    ${controller_index_list}    ${dictionary_item_occurrence}    ${uri}    ${headers}=${HEADERS}
+    [Documentation]    Send a GET with the supplied ${uri} to all cluster instances in ${controller_index_list}
+    ...    and check for occurrences of items expressed in a dictionary ${dictionary_item_occurrence}.
+    : FOR    ${i}    IN    @{controller_index_list}
+    \    ${data}    Get Data From URI    controller${i}    ${uri}    ${headers}
+    \    Log    ${data}
+    \    Check Item Occurrence    ${data}    ${dictionary_item_occurrence}
+
+Put And Check At URI In Cluster
+    [Arguments]    ${controller_index_list}    ${controller_index}    ${uri}    ${body}    ${headers}=${HEADERS}
+    [Documentation]    Send a PUT with the supplied ${uri} and ${body} to a ${controller_index}
+    ...    and check the data is replicated in all instances in ${controller_index_list}.
+    ${expected_body}=    To Json    ${body}
+    ${resp}    RequestsLibrary.Put Request    controller${controller_index}    ${uri}    ${body}    ${headers}
+    Should Be Equal As Strings    ${resp.status_code}    200
+    : FOR    ${i}    IN    @{controller_index_list}
+    \    ${data}    Wait Until Keyword Succeeds    5s    1s    Get Data From URI    controller${i}
+    \    ...    ${uri}    ${headers}
+    \    Log    ${data}
+    \    ${received_body}    To Json    ${data}
+    \    Should Be Equal    ${received_body}    ${expected_body}
+
+Delete And Check At URI In Cluster
+    [Arguments]    ${controller_index_list}    ${controller_index}    ${uri}    ${headers}=${HEADERS}
+    [Documentation]    Send a DELETE with the supplied ${uri} to a ${controller_index}
+    ...    and check the data is removed from all instances in ${controller_index_list}.
+    ${resp}    RequestsLibrary.Delete Request    controller${controller_index}    ${uri}    ${headers}
+    Should Be Equal As Strings    ${resp.status_code}    200
+    : FOR    ${i}    IN    @{controller_index_list}
+    \    Wait Until Keyword Succeeds    5s    1s    No Content From URI    controller${i}    ${uri}
+    \    ...    ${headers}
+
 Get Controller List
     [Arguments]    ${exclude_controller}=${EMPTY}
     [Documentation]    Creates a list of all controllers minus any excluded controller.
diff --git a/csit/libraries/MininetKeywords.robot b/csit/libraries/MininetKeywords.robot
new file mode 100644 (file)
index 0000000..f4c155a
--- /dev/null
@@ -0,0 +1,81 @@
+*** Settings ***
+Library           SSHLibrary
+Resource          Utils.robot
+Variables         ../variables/Variables.py
+
+*** Keywords ***
+Start Mininet Single Controller
+    [Arguments]    ${mininet}=${TOOLS_SYSTEM_IP}    ${controller}=${ODL_SYSTEM_IP}    ${options}=--topo tree,1 --switch ovsk,protocols=OpenFlow13    ${custom}=${EMPTY}    ${ofport}=6633
+    [Documentation]    Start Mininet with custom topology and connect to controller.
+    Log    Clear any existing mininet
+    Clean Mininet System    ${mininet}
+    ${mininet_conn_id}=    Open Connection    ${mininet}    prompt=${TOOLS_SYSTEM_PROMPT}    timeout=${DEFAULT_TIMEOUT}
+    Set Suite Variable    ${mininet_conn_id}
+    Flexible Mininet Login
+    Run Keyword If    '${custom}' != '${EMPTY}'    Put File    ${custom}
+    Log    Start mininet ${options} to ${controller}
+    Write    sudo mn --controller 'remote,ip=${controller},port=${ofport}' ${options}
+    Read Until    mininet>
+    ${output}=    Run Command On Mininet    ${mininet}    sudo ovs-vsctl show
+    Log    ${output}
+    [Return]    ${mininet_conn_id}
+
+Start Mininet Multiple Controllers
+    [Arguments]    ${mininet}    ${controller_index_list}    ${options}=--topo tree,1 --switch ovsk,protocols=OpenFlow13    ${custom}=${EMPTY}    ${ofport}=6633
+    [Documentation]    Start Mininet with custom topology and connect to all controllers in the ${controller_index_list}.
+    Log    Clear any existing mininet
+    Clean Mininet System    ${mininet}
+    ${mininet_conn_id}=    Open Connection    ${mininet}    prompt=${TOOLS_SYSTEM_PROMPT}    timeout=${DEFAULT_TIMEOUT}
+    Set Suite Variable    ${mininet_conn_id}
+    Flexible Mininet Login
+    Run Keyword If    '${custom}' != '${EMPTY}'    Put File    ${custom}
+    Log    Start mininet ${options}
+    Write    sudo mn ${options}
+    Read Until    mininet>
+    Log    Create controller configuration
+    ${ovs_opt}=    Set Variable
+    : FOR    ${index}    IN    @{controller_index_list}
+    \    ${ovs_opt}=    Catenate    ${ovs_opt}    ${SPACE}tcp:${ODL_SYSTEM_${index}_IP}:${ofport}
+    \    Log    ${ovs_opt}
+    Log    Find Number of OVS bridges
+    ${num_bridges}    Run Command On Mininet    ${mininet}    sudo ovs-vsctl show | grep Bridge | wc -l
+    ${num_bridges}=    Convert To Integer    ${num_bridges}
+    Log    Configure OVS controllers ${ovs_opt} in all bridges
+    : FOR    ${i}    IN RANGE    1    ${num_bridges+1}
+    \    ${bridge}=    Run Command On Mininet    ${mininet}    sudo ovs-vsctl show | grep Bridge | cut -c 12- | sort | head -${i} | tail -1
+    \    Run Command On Mininet    ${mininet}    sudo ovs-vsctl set-controller ${bridge} ${ovs_opt}
+    Log    Check OVS configuratiom
+    ${output}=    Run Command On Mininet    ${mininet}    sudo ovs-vsctl show
+    Log    ${output}
+    [Return]    ${mininet_conn_id}
+
+Send Mininet Command
+    [Arguments]    ${mininet_conn_id}    ${cmd}=help
+    [Documentation]    Sends Command ${cmd} to Mininet session ${mininet_conn_id} and returns read buffer response.
+    Switch Connection    ${mininet_conn_id}
+    SSHLibrary.Write    ${cmd}
+    ${output}=    Read Until    mininet>
+    [Return]    ${output}
+
+Send Mininet Command Multiple Sessions
+    [Arguments]    ${mininet_conn_list}    ${cmd}=help
+    [Documentation]    Sends Command ${cmd} to Mininet sessions in ${mininet_conn_list} and returns list of read buffer responses.
+    ${output_list}=    Create List
+    : FOR    ${mininet_conn_id}    IN    @{mininet_conn_list}
+    \    ${output}=    Send Mininet Command    ${mininet_conn_id}    ${cmd}
+    \    Append To List    ${output_list}    ${output}
+    [Return]    ${output_list}
+
+Stop Mininet And Exit
+    [Arguments]    ${mininet_conn_id}
+    [Documentation]    Stops Mininet and exits session ${mininet_conn_id}
+    Switch Connection    ${mininet_conn_id}
+    SSHLibrary.Write    exit
+    Read Until    ${TOOLS_SYSTEM_PROMPT}
+    Close Connection
+
+Stop Mininet And Exit Multiple Sessions
+    [Arguments]    ${mininet_conn_list}
+    [Documentation]    Stops Mininet and exits sessions in ${mininet_conn_list}.
+    : FOR    ${mininet_conn_id}    IN    @{mininet_conn_list}
+    \    Stop Mininet And Exit    ${mininet_conn_id}
index d62ff87dc43cff46ed76e3e4d9804a8573842095..df967dda00d78eed4e3bb0f2a9d9b2dcf695248f 100644 (file)
@@ -131,7 +131,7 @@ Check For Elements Not At URI
     \    Should Not Contain    ${resp.content}    ${i}
 
 Clean Mininet System
-    [Arguments]    ${system}=${MININET}
+    [Arguments]    ${system}=${TOOLS_SYSTEM_IP}
     Run Command On Mininet    ${system}    sudo mn -c
     Run Command On Mininet    ${system}    sudo ps -elf | egrep 'usr/local/bin/mn' | egrep python | awk '{print "sudo kill -9",$4}' | sh
 
@@ -192,28 +192,27 @@ Flexible SSH Login
     BuiltIn.Run Keyword And Return    SSHLibrary.Login With Public Key    ${user}    ${USER_HOME}/.ssh/${SSH_KEY}    ${KEYFILE_PASS}    delay=${delay}
 
 Flexible Mininet Login
-    [Arguments]    ${user}=${MININET_USER}    ${password}=${MININET_PASSWORD}    ${delay}=0.5s
+    [Arguments]    ${user}=${TOOLS_SYSTEM_USER}    ${password}=${TOOLS_SYSTEM_PASSWORD}    ${delay}=0.5s
     [Documentation]    Call Flexible SSH Login, but with default values suitable for Mininet machine.
     BuiltIn.Run Keyword And Return    Flexible SSH Login    user=${user}    password=${password}    delay=${delay}
 
 Flexible Controller Login
-    [Arguments]    ${user}=${CONTROLLER_USER}    ${password}=${CONTROLLER_PASSWORD}    ${delay}=0.5s
+    [Arguments]    ${user}=${ODL_SYSTEM_USER}    ${password}=${ODL_SYSTEM_PASSWORD}    ${delay}=0.5s
     [Documentation]    Call Flexible SSH Login, but with default values suitable for Controller machine.
     BuiltIn.Run Keyword And Return    Flexible SSH Login    user=${user}    password=${password}    delay=${delay}
 
 Run Command On Remote System
-    [Arguments]    ${system}    ${cmd}    ${user}=${MININET_USER}    ${password}=${EMPTY}    ${prompt}=${DEFAULT_LINUX_PROMPT}    ${prompt_timeout}=30s
+    [Arguments]    ${system}    ${cmd}    ${user}=${DEFAULT_USER}    ${password}=${EMPTY}    ${prompt}=${DEFAULT_LINUX_PROMPT}    ${prompt_timeout}=${DEFAULT_TIMEOUT}
     [Documentation]    Reduces the common work of running a command on a remote system to a single higher level
     ...    robot keyword, taking care to log in with a public key and. The command given is written
     ...    and the output returned. No test conditions are checked.
     Log    Attempting to execute ${cmd} on ${system} by ${user} with ${keyfile_pass} and ${prompt}
     ${conn_id}=    SSHLibrary.Open Connection    ${system}    prompt=${prompt}    timeout=${prompt_timeout}
     Flexible SSH Login    ${user}    ${password}
-    SSHLibrary.Write    ${cmd}
-    ${output}=    SSHLibrary.Read Until    ${prompt}
+    ${stdout}    ${stderr}    SSHLibrary.Execute Command    ${cmd}    return_stderr=True
     SSHLibrary.Close Connection
-    Log    ${output}
-    [Return]    ${output}
+    Log    ${stderr}
+    [Return]    ${stdout}
 
 Write_Bare_Ctrl_C
     [Documentation]    Construct ctrl+c character and SSH-write it (without endline) to the current SSH connection.
@@ -228,16 +227,14 @@ Write Bare Ctrl D
     SSHLibrary.Write Bare    ${ctrl_d}
 
 Run Command On Mininet
-    [Arguments]    ${system}=${MININET}    ${cmd}=echo    ${user}=${MININET_USER}    ${password}=${MININET_PASSWORD}    ${prompt}=${DEFAULT_LINUX_PROMPT}    ${prompt_timeout}=30s
+    [Arguments]    ${system}=${TOOLS_SYSTEM_IP}    ${cmd}=echo    ${user}=${TOOLS_SYSTEM_USER}    ${password}=${TOOLS_SYSTEM_PASSWORD}    ${prompt}=${TOOLS_SYSTEM_PROMPT}
     [Documentation]    Call Run Comand On Remote System, but with default values suitable for Mininet machine.
-    BuiltIn.Run Keyword And Return    Run Command On Remote System    ${system}    ${cmd}    user=${user}    password=${password}    prompt=${prompt}
-    ...    prompt_timeout=${prompt_timeout}
+    BuiltIn.Run Keyword And Return    Run Command On Remote System    ${system}    ${cmd}    ${user}    ${password}    prompt=${prompt}
 
 Run Command On Controller
-    [Arguments]    ${system}=${CONTROLLER}    ${cmd}=echo    ${user}=${CONTROLLER_USER}    ${password}=${CONTROLLER_PASSWORD}    ${prompt}=${DEFAULT_LINUX_PROMPT}    ${prompt_timeout}=30s
+    [Arguments]    ${system}=${ODL_SYSTEM_IP}    ${cmd}=echo    ${user}=${ODL_SYSTEM_USER}    ${password}=${ODL_SYSTEM_PASSWORD}    ${prompt}=${ODL_SYSTEM_PROMPT}
     [Documentation]    Call Run Comand On Remote System, but with default values suitable for Controller machine.
-    BuiltIn.Run Keyword And Return    Run Command On Remote System    ${system}    ${cmd}    user=${user}    password=${password}    prompt=${prompt}
-    ...    prompt_timeout=${prompt_timeout}
+    BuiltIn.Run Keyword And Return    Run Command On Remote System    ${system}    ${cmd}    ${user}    ${password}    prompt=${prompt}
 
 Verify File Exists On Remote System
     [Arguments]    ${system}    ${file}    ${user}=${MININET_USER}    ${password}=${MININET_PASSWORD}    ${prompt}=${DEFAULT_LINUX_PROMPT}    ${prompt_timeout}=5s
@@ -338,3 +335,30 @@ Get Data From URI
     Builtin.Return_From_Keyword_If    ${response.status_code} == 200    ${response.text}
     Builtin.Log    ${response.text}
     Builtin.Fail    The request failed with code ${response.status_code}
+
+No Content From URI
+    [Arguments]    ${session}    ${uri}    ${headers}=${NONE}
+    [Documentation]    Issue a GET request and return on error 404 (No content) or will fail and log the content.
+    ...    Issues a GET request for ${uri} in ${session} using headers from
+    ...    ${headers}. If the request returns a HTTP error, fails. Otherwise
+    ...    returns the data obtained by the request.
+    ${response}=    RequestsLibrary.Get Request    ${session}    ${uri}    ${headers}
+    Builtin.Return_From_Keyword_If    ${response.status_code} == 404
+    Builtin.Log    ${response.text}
+    Builtin.Fail    The request failed with code ${response.status_code}
+
+Get Index From List Of Dictionaries
+    [Arguments]    ${dictionary_list}    ${key}    ${value}
+    [Documentation]    Extract index for the dictionary in a list that contains a key-value pair. Returns -1 if key-value is not found.
+    ${length}=    Get Length    ${dictionary_list}
+    ${index}=    Set Variable    -1
+    : FOR    ${i}    IN RANGE    ${length}
+    \    ${dictionary}=    Get From List    ${dictionary_list}    ${i}
+    \    Run Keyword If    '&{dictionary}[${key}]' == '${value}'    Set Test Variable    ${index}    ${i}
+    [Return]    ${index}
+
+Check Item Occurrence
+    [Arguments]    ${string}    ${dictionary_item_occurrence}
+    [Documentation]    Check string for occurrences of items expressed in a list of dictionaries {item=occurrences}. 0 occurences means item is not present.
+    : FOR    ${item}    IN    @{dictionary_item_occurrence}
+    \    Should Contain X Times    ${string}    ${item}    &{dictionary_item_occurrence}[${item}]
diff --git a/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot b/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot
new file mode 100644 (file)
index 0000000..ea6198e
--- /dev/null
@@ -0,0 +1,145 @@
+*** Settings ***
+Suite Setup       Create Controller Sessions
+Suite Teardown    Delete All Sessions
+Library           RequestsLibrary
+Resource          ../../../libraries/ClusterKeywords.robot
+Resource          ../../../libraries/MininetKeywords.robot
+Variables         ../../../variables/Variables.py
+
+*** Variables ***
+${config_table_0}    ${CONFIG_NODES_API}/node/openflow:1/table/0
+${operational_table_0}    ${OPERATIONAL_NODES_API}/node/openflow:1/table/0
+${operational_port_1}    ${OPERATIONAL_NODES_API}/node/openflow:1/node-connector/openflow:1:1
+
+*** Test Cases ***
+Check OpenFlow Shards Status
+    [Documentation]    Create original cluster list and Check Status for all shards in OpenFlow application.
+    ${original_cluster_list}    Create Controller Index List
+    Set Suite Variable    ${original_cluster_list}
+    ${inv_conf_leader}    ${inv_conf_followers_list}    Get Cluster Shard Status    ${original_cluster_list}    config    inventory
+    ${inv_oper_leader}    ${inv_oper_followers_list}    Get Cluster Shard Status    ${original_cluster_list}    operational    inventory
+    ${topo_oper_leader}    ${topo_oper_followers_list}    Get Cluster Shard Status    ${original_cluster_list}    operational    topology
+    Log    config inventory Leader is ${inv_conf_leader} and followers are ${inv_conf_followers_list}
+    Log    operational inventory Leader is ${inv_oper_leader} and followers are ${inv_oper_followers_list}
+    Log    operational topology Leader is ${topo_oper_leader} and followers are ${topo_oper_followers_list}
+
+Start Mininet Multiple Connections
+    [Documentation]    Start mininet with connection to all cluster instances.
+    ${mininet_conn_id}=    Start Mininet Multiple Controllers    ${TOOLS_SYSTEM_IP}    ${original_cluster_list}
+    Set Suite Variable    ${mininet_conn_id}
+
+Check Entity Owner Status And Find Owner and Candidate
+    [Documentation]    Check Entity Owner Status and identify owner and candidate.
+    ${original_owner}    ${original_candidates_list}    Wait Until Keyword Succeeds    5s    1s    Get Cluster Entity Owner Status    ${original_cluster_list}
+    ...    openflow    openflow:1
+    ${original_candidate}=    Get From List    ${original_candidates_list}    0
+    Set Suite Variable    ${original_owner}
+    Set Suite Variable    ${original_candidate}
+
+Check Network Operational Information
+    [Documentation]    Check device is in operational inventory and topology in all cluster instances.
+    ...    Inventory should show 1x node_id per device 1x node_id per connector. Topology should show 2x node_id per device + 3x node_id per connector.
+    ${dictionary}    Create Dictionary    openflow:1=4
+    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${OPERATIONAL_NODES_API}
+    ${dictionary}    Create Dictionary    openflow:1=11
+    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${OPERATIONAL_TOPO_API}
+
+Add Flow In Owner and Verify
+    [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
+    ${body}=    OperatingSystem.Get File    ${CURDIR}/../../../variables/openflowplugin/sample_flow_1.json
+    ${dictionary}=    Create Dictionary    10.0.1.0/24=1
+    Put And Check At URI In Cluster    ${original_cluster_list}    ${original_owner}    ${config_table_0}/flow/1    ${body}    ${HEADERS}
+    Wait Until Keyword Succeeds    10s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_table_0}
+
+Modify Flow In Owner and Verify
+    [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.
+    ${body}=    OperatingSystem.Get File    ${CURDIR}/../../../variables/openflowplugin/sample_flow_2.json
+    ${dictionary}=    Create Dictionary    10.0.2.0/24=1
+    Put And Check At URI In Cluster    ${original_cluster_list}    ${original_owner}    ${config_table_0}/flow/1    ${body}    ${HEADERS}
+    Wait Until Keyword Succeeds    10s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_table_0}
+
+Delete Flow In Owner and Verify
+    [Documentation]    Delete Flow in Owner and verify it gets applied from all instances.
+    ${dictionary}=    Create Dictionary    10.0.2.0/24=0
+    Delete And Check At URI In Cluster    ${original_cluster_list}    ${original_owner}    ${config_table_0}/flow/1
+    Wait Until Keyword Succeeds    10s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_table_0}
+
+Add Flow In Candidate and Verify
+    [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
+    ${body}=    OperatingSystem.Get File    ${CURDIR}/../../../variables/openflowplugin/sample_flow_1.json
+    ${dictionary}=    Create Dictionary    10.0.1.0/24=1
+    Put And Check At URI In Cluster    ${original_cluster_list}    ${original_candidate}    ${config_table_0}/flow/1    ${body}    ${HEADERS}
+    Wait Until Keyword Succeeds    10s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_table_0}
+
+Modify Flow In Candidate and Verify
+    [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.
+    ${body}=    OperatingSystem.Get File    ${CURDIR}/../../../variables/openflowplugin/sample_flow_2.json
+    ${dictionary}=    Create Dictionary    10.0.2.0/24=1
+    Put And Check At URI In Cluster    ${original_cluster_list}    ${original_candidate}    ${config_table_0}/flow/1    ${body}    ${HEADERS}
+    Wait Until Keyword Succeeds    10s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_table_0}
+
+Delete Flow In Candidate and Verify
+    [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.
+    ${dictionary}=    Create Dictionary    10.0.2.0/24=0
+    Delete And Check At URI In Cluster    ${original_cluster_list}    ${original_candidate}    ${config_table_0}/flow/1
+    Wait Until Keyword Succeeds    10s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_table_0}
+
+Send RPC Add Flow to Owner and Verify
+    [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
+    ${body}=    OperatingSystem.Get File    ${CURDIR}/../../../variables/openflowplugin/add_flow_rpc.json
+    ${dictionary}=    Create Dictionary    10.0.1.0/24=1
+    ${resp}    RequestsLibrary.Post Request    controller${original_owner}    /restconf/operations/sal-flow:add-flow    ${body}    ${HEADERS}
+    Log    ${resp.content}
+    Should Be Equal As Strings    ${resp.status_code}    200
+    Wait Until Keyword Succeeds    10s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_table_0}
+
+Send RPC Delete Flow to Owner and Verify
+    [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.
+    ${body}=    OperatingSystem.Get File    ${CURDIR}/../../../variables/openflowplugin/delete_flow_rpc.json
+    ${dictionary}=    Create Dictionary    10.0.1.0/24=0
+    ${resp}    RequestsLibrary.Post Request    controller${original_owner}    /restconf/operations/sal-flow:remove-flow    ${body}    ${HEADERS}
+    Log    ${resp.content}
+    Should Be Equal As Strings    ${resp.status_code}    200
+    Wait Until Keyword Succeeds    10s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_table_0}
+
+Send RPC Add Flow to Candidate and Verify
+    [Documentation]    Add Flow in Candidate and verify it gets applied from all instances.
+    ${body}=    OperatingSystem.Get File    ${CURDIR}/../../../variables/openflowplugin/add_flow_rpc.json
+    ${dictionary}=    Create Dictionary    10.0.1.0/24=1
+    ${resp}    RequestsLibrary.Post Request    controller${original_candidate}    /restconf/operations/sal-flow:add-flow    ${body}    ${HEADERS}
+    Log    ${resp.content}
+    Should Be Equal As Strings    ${resp.status_code}    200
+    Wait Until Keyword Succeeds    10s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_table_0}
+
+Send RPC Delete Flow to Candidate and Verify
+    [Documentation]    Delete Flow in Candidate and verify it gets removed from all instances.
+    ${body}=    OperatingSystem.Get File    ${CURDIR}/../../../variables/openflowplugin/delete_flow_rpc.json
+    ${dictionary}=    Create Dictionary    10.0.1.0/24=0
+    ${resp}    RequestsLibrary.Post Request    controller${original_candidate}    /restconf/operations/sal-flow:remove-flow    ${body}    ${HEADERS}
+    Log    ${resp.content}
+    Should Be Equal As Strings    ${resp.status_code}    200
+    Wait Until Keyword Succeeds    10s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_table_0}
+
+Take a Link Down and Verify
+    [Documentation]    Take a link down and verify port status in all instances.
+    ${dictionary}=    Create Dictionary    "link-down":true=1
+    ${ouput}=    Send Mininet Command    ${mininet_conn_id}    link s1 h1 down
+    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_port_1}
+
+Take a Link Up and Verify
+    [Documentation]    Take the link up and verify port status in all instances.
+    ${dictionary}=    Create Dictionary    "link-down":true=0
+    ${ouput}=    Send Mininet Command    ${mininet_conn_id}    link s1 h1 up
+    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_port_1}
+
+Stop Mininet and Exit
+    [Documentation]    Stop mininet and exit connection.
+    Stop Mininet And Exit    ${mininet_conn_id}
+    Clean Mininet System
+
+Check No Network Operational Information
+    [Documentation]    Check device is not in operational inventory or topology in all cluster instances.
+    ${dictionary}    Create Dictionary    openflow:1=0
+    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${OPERATIONAL_NODES_API}
+    ${dictionary}    Create Dictionary    openflow:1=0
+    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${OPERATIONAL_TOPO_API}
index 2c7feda44ae9ae4bfc4c9749cb208b6ae1860507..b200f39fc90d88ee40ed174a75d6605ffc485557 100644 (file)
@@ -1,2 +1,3 @@
 # Place the suites in run order:
+integration/test/csit/suites/openflowplugin/Clustering
 integration/test/csit/suites/openflowplugin/Sanity3Node
index c62e175e416804e26b7c0752cc9d2692474428f8..8c4f71c86707ef64b5450be622c085f6d63188a2 100644 (file)
@@ -8,17 +8,19 @@ Edited: Many times by many people
 
 # VM Environment defaults
 DEFAULT_LINUX_PROMPT = '>'
+DEFAULT_USER = 'jenkins'
+DEFAULT_TIMEOUT = '30s'
 
 # ODL system variables
 ODL_SYSTEM_IP = '127.0.0.1'  # Override if ODL is not running locally to pybot
 ODL_SYSTEM_IP_LIST = ['ODL_SYSTEM_1_IP', 'ODL_SYSTEM_2_IP', 'ODL_SYSTEM_3_IP']
-ODL_SYSTEM_USER = 'jenkins'
+ODL_SYSTEM_USER = DEFAULT_USER
 ODL_SYSTEM_PASSWORD = ''  # empty means use public key authentication
 ODL_SYSTEM_PROMPT = DEFAULT_LINUX_PROMPT
 
 # "Tools" system variables (mininet etc).
 TOOLS_SYSTEM_IP = '127.0.0.1'  # Override if tools are not run locally to pybot
-TOOLS_SYSTEM_USER = 'jenkins'
+TOOLS_SYSTEM_USER = DEFAULT_USER
 TOOLS_SYSTEM_PASSWORD = ''  # empty means use public key authentication
 TOOLS_SYSTEM_PROMPT = DEFAULT_LINUX_PROMPT
 
@@ -177,6 +179,8 @@ CONTROLLER = ODL_SYSTEM_IP
 CONTROLLERS = ['CONTROLLER', 'CONTROLLER1', 'CONTROLLER2']
 CONTROLLER_PASSWORD = ODL_SYSTEM_PASSWORD
 CONTROLLER_PROMPT = ODL_SYSTEM_PROMPT
+MININET = TOOLS_SYSTEM_IP
+MININET_USER = TOOLS_SYSTEM_USER
 MININET_PASSWORD = TOOLS_SYSTEM_PASSWORD
 MININET_PROMPT = TOOLS_SYSTEM_PROMPT
 PROMPT = '>'  # TODO: remove this as it's vague.  need to fix any occurances of it first.
diff --git a/csit/variables/openflowplugin/add_flow_rpc.json b/csit/variables/openflowplugin/add_flow_rpc.json
new file mode 100644 (file)
index 0000000..c4125ac
--- /dev/null
@@ -0,0 +1,32 @@
+{
+    "input": {
+      "opendaylight-flow-service:node":"/opendaylight-inventory:nodes/opendaylight-inventory:node[opendaylight-inventory:id='openflow:1']",
+      "priority": 2,
+      "table_id": 0,
+      "instructions": {
+        "instruction": [
+          {
+            "order": 0,
+            "apply-actions": {
+              "action": [
+                {
+                  "order": 0,
+                  "output-action": {
+                    "output-node-connector": "1"
+                  }
+                }
+              ]
+            }
+          }
+        ]
+      },
+      "match": {
+        "ipv4-destination": "10.0.1.0/24",
+        "ethernet-match": {
+          "ethernet-type": {
+            "type": 2048
+          }
+        }
+      }
+    }
+}
diff --git a/csit/variables/openflowplugin/delete_flow_rpc.json b/csit/variables/openflowplugin/delete_flow_rpc.json
new file mode 100644 (file)
index 0000000..de8703d
--- /dev/null
@@ -0,0 +1,18 @@
+{
+    "input": {
+      "opendaylight-flow-service:node":"/opendaylight-inventory:nodes/opendaylight-inventory:node[opendaylight-inventory:id='openflow:1']",
+      "priority": 2,
+      "flow-name": "flow1",
+      "table_id": 0,
+      "hard-timeout": 0,
+      "idle-timeout": 0,
+      "match": {
+        "ipv4-destination": "10.0.1.0/24",
+        "ethernet-match": {
+          "ethernet-type": {
+            "type": 2048
+          }
+        }
+      }
+    }
+}
diff --git a/csit/variables/openflowplugin/sample_flow_1.json b/csit/variables/openflowplugin/sample_flow_1.json
new file mode 100644 (file)
index 0000000..0e36847
--- /dev/null
@@ -0,0 +1,35 @@
+{
+  "flow-node-inventory:flow": [
+    {
+      "id": "1",
+      "table_id": 0,
+      "priority": 2,
+      "flow-name": "flow1",
+      "instructions": {
+        "instruction": [
+          {
+            "order": 0,
+            "apply-actions": {
+              "action": [
+                {
+                  "order": 0,
+                  "output-action": {
+                    "output-node-connector": "1"
+                  }
+                }
+              ]
+            }
+          }
+        ]
+      },
+      "match": {
+        "ipv4-destination": "10.0.1.0/24",
+        "ethernet-match": {
+          "ethernet-type": {
+            "type": 2048
+          }
+        }
+      }
+    }
+  ]
+}
diff --git a/csit/variables/openflowplugin/sample_flow_2.json b/csit/variables/openflowplugin/sample_flow_2.json
new file mode 100644 (file)
index 0000000..d486455
--- /dev/null
@@ -0,0 +1,35 @@
+{
+  "flow-node-inventory:flow": [
+    {
+      "id": "1",
+      "table_id": 0,
+      "priority": 2,
+      "flow-name": "flow1",
+      "instructions": {
+        "instruction": [
+          {
+            "order": 0,
+            "apply-actions": {
+              "action": [
+                {
+                  "order": 0,
+                  "output-action": {
+                    "output-node-connector": "2"
+                  }
+                }
+              ]
+            }
+          }
+        ]
+      },
+      "match": {
+        "ipv4-destination": "10.0.2.0/24",
+        "ethernet-match": {
+          "ethernet-type": {
+            "type": 2048
+          }
+        }
+      }
+    }
+  ]
+}