CSIT test for ODL upgrade

Robot test for ODL upgrade. The upgrade steps are:
1. Create setup
2. Ping VMs created to ensure basic instance connectivity
3. Stop ODL
4. Disconnect OVS
5. Delete data, journal, snapshot
7. Start ODL
8. Re-connect OVS
9. Wait for full-sync to finish
10. Ping VMs initially created to ensure full sync worked
11. Clean up

Depends-On: https://git.opendaylight.org/gerrit/66723
Change-Id: Id27ec4b5a001dba6b60765e27265b017e356b5f6
Signed-off-By: Janki Chhatbar <jchhatba@redhat.com>
Signed-off-by: Jamo Luhrsen <jluhrsen@redhat.com>

diff --git a/csit/libraries/OVSDB.robot b/csit/libraries/OVSDB.robot
index e1db8fd375d3ffc541b09d12c067b38132be8857..d0a72cd71f4b2e081cff833c2736157381f86a96 100644 (file)
--- a/csit/libraries/OVSDB.robot
+++ b/csit/libraries/OVSDB.robot
@@ -382,3 +382,38 @@ Get Bridge Data
     ${uuid} =    String.Get Line    ${result}    0
     ${line}    ${bridge_name}    Builtin.Should Match Regexp    ${result}    Bridge "(\\w+)"
     [Return]    ${uuid}    ${bridge_name}
+
+Delete OVS Controller
+    [Arguments]    ${ovs_ip}    ${bridge}=${INTEGRATION_BRIDGE}
+    [Documentation]    Delete controller from OVS
+    ${del_ctr} =    Utils.Run Command On Remote System    ${ovs_ip}    sudo ovs-vsctl del-controller ${bridge}
+    BuiltIn.Log    ${del_ctr}
+
+Delete OVS Manager
+    [Arguments]    ${ovs_ip}
+    [Documentation]    Delete manager from OVS
+    ${del_mgr} =    Utils.Run Command On Remote System    ${ovs_ip}    sudo ovs-vsctl del-manager
+    BuiltIn.Log    ${del_mgr}
+
+Delete Groups On Bridge
+    [Arguments]    ${ovs_ip}    ${br}=${INTEGRATION_BRIDGE}
+    [Documentation]    Delete OVS groups from ${br}
+    ${del_grp} =    Utils.Run Command On Remote System    ${ovs_ip}    sudo ovs-ofctl -O Openflow13 del-groups ${br}
+    BuiltIn.Log    ${del_grp}
+
+Get Ports From Bridge By Type
+    [Arguments]    ${ovs_ip}    ${br}    ${type}
+    [Documentation]    Get ${type} ports for a bridge ${br} on node ${ovs_ip}.
+    ${ports} =    Utils.Run Command On Remote System    ${ovs_ip}    sudo ovs-vsctl list-ports ${br} | grep "${type}"
+    ${ports_list} =    String.Split to lines    ${ports}
+    [Return]    ${ports_list}
+
+Delete Ports On Bridge By Type
+    [Arguments]    ${ovs_ip}    ${br}    ${type}
+    [Documentation]    List all ports of ${br} and delete ${type} ports
+    ${ports_present} =    Get Ports From Bridge By Type    ${ovs_ip}    ${br}    ${type}
+    : FOR    ${port}    IN    @{ports_present}
+    \    ${del-ports} =    Utils.Run Command On Remote System    ${ovs_ip}    sudo ovs-vsctl del-port ${br} ${port}
+    \    BuiltIn.Log    ${del-ports}
+    ${ports_present_after_delete} =    Get Ports From Bridge By Type    ${ovs_ip}    ${br}    ${type}
+    BuiltIn.Log    ${ports_present_after_delete}

diff --git a/csit/libraries/OpenStackOperations.robot b/csit/libraries/OpenStackOperations.robot
index f216a158ad9983959f33b47683912f2a495256fd..48dad4dc2f828597cc51aaf59b8f49fa22e138bb 100644 (file)
--- a/csit/libraries/OpenStackOperations.robot
+++ b/csit/libraries/OpenStackOperations.robot
@@ -831,6 +831,14 @@ Update Port Rest
     BuiltIn.Should Be Equal As Strings    ${resp.status_code}    200
     [Return]    ${resp.content}
 
+Get Neutron Network Rest
+    [Arguments]    ${net_id}
+    [Documentation]    Keyword to get the specific network details in Neutron (Using REST).
+    ${resp} =    RequestsLibrary.Get Request    session    ${NETWORK_URL}/network/${net_id}
+    BuiltIn.Log    ${resp.content}
+    BuiltIn.Should Be Equal As Strings    ${resp.status_code}    200
+    [Return]    ${resp.content}
+
 Create And Configure Security Group
     [Arguments]    ${sg-name}
     [Documentation]    Create Security Group with given name, and default allow rules for TCP/UDP/ICMP protocols.

diff --git a/csit/suites/netvirt/upgrade/upgrade.robot b/csit/suites/netvirt/upgrade/upgrade.robot
new file mode 100644 (file)
index 0000000..aba7c9e
--- /dev/null
+++ b/csit/suites/netvirt/upgrade/upgrade.robot
@@ -0,0 +1,147 @@
+*** Settings ***
+Documentation     Test suite for ODL Upgrade. It is assumed that OLD + OpenStack
+...               integrated environment is deployed and ready.
+Suite Setup       OpenStackOperations.OpenStack Suite Setup
+Suite Teardown    Upgrade Suite Teardown
+Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
+Test Teardown     Get Test Teardown Debugs
+Library           OperatingSystem
+Library           RequestsLibrary
+Library           SSHLibrary
+Resource          ../../../libraries/ClusterManagement.robot
+Resource          ../../../libraries/DevstackUtils.robot
+Resource          ../../../libraries/KarafKeywords.robot
+Resource          ../../../libraries/OpenStackOperations.robot
+Resource          ../../../libraries/OVSDB.robot
+Resource          ../../../libraries/SetupUtils.robot
+Resource          ../../../libraries/Utils.robot
+
+*** Variables ***
+${SECURITY_GROUP}    upgrade_sg
+@{NETWORKS}       upgrade_net_1    upgrade_net_2
+@{SUBNETS}        upgrade_sub_1    upgrade_sub_2
+@{NET_1_VMS}      upgrade_net_1_vm_1    upgrade_net_1_vm_2
+@{NET_2_VMS}      upgrade_net_2_vm_1    upgrade_net_2_vm_2
+@{SUBNETS_RANGE}    91.0.0.0/24    92.0.0.0/24
+${ROUTER}         upgrade_router_1
+${TYPE}           tun
+${PASSIVE_MANAGER}    ptcp:6641:127.0.0.1
+@{DEBUG_LOG_COMPONENTS}    org.opendaylight.ovsdb    org.opendaylight.ovsdb.lib    org.opendaylight.netvirt    org.opendaylight.genius
+
+*** Test Cases ***
+Create Setup And Verify Instance Connectivity
+    [Documentation]    Create 2 VXLAN networks, subnets with 2 VMs each and a router. Ping all 4 VMs.
+    Create Resources
+    Check Resource Connectivity
+    DevstackUtils.Set Node Data For Control Only Node Setup
+    Dump Debug With Annotations    POST_SETUP
+
+Stop ODL
+    ClusterManagement.Stop_Members_From_List_Or_All
+
+Disconnect OVS
+    [Documentation]    Delete OVS manager, controller and groups and tun ports
+    : FOR    ${node}    IN    @{OS_ALL_IPS}
+    \    OVSDB.Delete OVS Manager    ${node}
+    \    OVSDB.Delete OVS Controller    ${node}
+    \    OVSDB.Delete Groups On Bridge    ${node}    ${INTEGRATION_BRIDGE}
+    \    OVSDB.Delete Ports On Bridge By Type    ${node}    ${INTEGRATION_BRIDGE}    ${TYPE}
+
+Wipe Local Data
+    [Documentation]    Delete data/, journal/, snapshots/
+    ClusterManagement.Clean_Journals_Data_And_Snapshots_On_List_Or_All
+
+Start ODL
+    [Documentation]    Start controller, wait for it to come "UP" and make sure netvirt is installed
+    ClusterManagement.Start_Members_From_List_Or_All    wait_for_sync=True
+    Wait Until Keyword Succeeds    100s    5s    Utils.Check Diagstatus
+    BuiltIn.Set_Suite_Variable    \${ClusterManagement__has_setup_run}    False
+    KarafKeywords.Verify_Feature_Is_Installed    odl-netvirt-openstack
+    Set Custom Component Logging To    DEBUG
+
+Wait For Full Sync
+    [Documentation]    Wait for networking_odl to sync neutron configuration
+    Wait Until Keyword Succeeds    90s    5s    Canary Network Should Exist
+
+Set OVS Manager And Controller
+    [Documentation]    Set controller and manager on each OpenStack node and check that egress flows are present
+    : FOR    ${node}    IN    @{OS_ALL_IPS}
+    \    Run Command On Remote System    ${node}    sudo ovs-vsctl set-manager tcp:${ODL_SYSTEM_IP}:${OVSDBPORT} ${PASSIVE_MANAGER}
+    Wait Until Keyword Succeeds    180s    15s    Check OVS Nodes Have Egress Flows
+
+Check Connectivity With Previously Created Resources And br-int Info
+    [Documentation]    Check that pre-existing instance connectivity still works after the new controller is brought
+    ...    up and config is sync'd
+    Dump Debug With Annotations    POST_UPGRADE
+    Wait Until Keyword Succeeds    90s    10s    Check Resource Connectivity
+
+*** Keywords ***
+Create Resources
+    [Documentation]    Create 2 VXLAN networks, subnets with 2 VMs each and a router. Ping all 4 VMs.
+    : FOR    ${net}    IN    @{NETWORKS}
+    \    OpenStackOperations.Create Network    ${net}
+    OpenStackOperations.Create SubNet    @{NETWORKS}[0]    @{SUBNETS}[0]    @{SUBNETS_RANGE}[0]
+    OpenStackOperations.Create SubNet    @{NETWORKS}[1]    @{SUBNETS}[1]    @{SUBNETS_RANGE}[1]
+    OpenStackOperations.Create Allow All SecurityGroup    ${SECURITY_GROUP}
+    OpenStackOperations.Create Nano Flavor
+    : FOR    ${vm}    IN    @{NET_1_VMS}
+    \    OpenStackOperations.Create Vm Instance On Compute Node    @{NETWORKS}[0]    ${vm}    ${OS_CMP1_HOSTNAME}    sg=${SECURITY_GROUP}
+    : FOR    ${vm}    IN    @{NET_2_VMS}
+    \    OpenStackOperations.Create Vm Instance On Compute Node    @{NETWORKS}[1]    ${vm}    ${OS_CMP2_HOSTNAME}    sg=${SECURITY_GROUP}
+    OpenStackOperations.Create Router    ${ROUTER}
+    : FOR    ${interface}    IN    @{SUBNETS}
+    \    OpenStackOperations.Add Router Interface    ${ROUTER}    ${interface}
+    @{NET1_VM_IPS}    ${NET1_DHCP_IP} =    OpenStackOperations.Get VM IPs    @{NET_1_VMS}
+    @{NET2_VM_IPS}    ${NET2_DHCP_IP} =    OpenStackOperations.Get VM IPs    @{NET_2_VMS}
+    BuiltIn.Set Suite Variable    @{NET1_VM_IPS}
+    BuiltIn.Set Suite Variable    @{NET2_VM_IPS}
+    BuiltIn.Should Not Contain    ${NET1_VM_IPS}    None
+    BuiltIn.Should Not Contain    ${NET2_VM_IPS}    None
+    BuiltIn.Should Not Contain    ${NET1_DHCP_IP}    None
+    BuiltIn.Should Not Contain    ${NET2_DHCP_IP}    None
+
+Check Resource Connectivity
+    [Documentation]    Ping 2 VMs in the same net and 1 from another net.
+    OpenStackOperations.Ping Vm From DHCP Namespace    upgrade_net_1    @{NET1_VM_IPS}[0]
+    OpenStackOperations.Ping Vm From DHCP Namespace    upgrade_net_1    @{NET1_VM_IPS}[1]
+    OpenStackOperations.Ping Vm From DHCP Namespace    upgrade_net_1    @{NET2_VM_IPS}[0]
+    OpenStackOperations.Ping Vm From DHCP Namespace    upgrade_net_2    @{NET2_VM_IPS}[0]
+    OpenStackOperations.Ping Vm From DHCP Namespace    upgrade_net_2    @{NET2_VM_IPS}[1]
+    OpenStackOperations.Ping Vm From DHCP Namespace    upgrade_net_2    @{NET1_VM_IPS}[0]
+
+Check OVS Nodes Have Egress Flows
+    [Documentation]    Loop over all openstack nodes to ensure they have the proper flows installed.
+    : FOR    ${node}    IN    @{OS_ALL_IPS}
+    \    Does OVS Have Multiple Egress Flows    ${node}
+
+Does OVS Have Multiple Egress Flows
+    [Arguments]    ${ip}
+    [Documentation]    Verifies that at least 1 flow exists on the node for the ${EGRESS_L2_FWD_TABLE}
+    ${flows} =    Utils.Run Command On Remote System    ${ip}    sudo ovs-ofctl -O OpenFlow13 dump-flows ${INTEGRATION_BRIDGE}
+    ${egress_flows} =    String.Get Lines Containing String    ${flows}    table=${EGRESS_LPORT_DISPATCHER_TABLE}
+    ${num_egress_flows} =    String.Get Line Count    ${egress_flows}
+    BuiltIn.Should Be True    ${num_egress_flows} > 1
+
+Set Custom Component Logging To
+    [Arguments]    ${level}
+    SetupUtils.Setup_Logging_For_Debug_Purposes_On_List_Or_All    ${level}    ${DEBUG_LOG_COMPONENTS}
+    KarafKeywords.Issue_Command_On_Karaf_Console    log:list
+
+Dump Debug With Annotations
+    [Arguments]    ${tag}
+    [Documentation]    Dump tons of debug logs for each OS node but also emit tags to make parsing easier
+    Builtin.Log    Start dumping at phase ${tag}
+    : FOR    ${node}    IN    @{OS_ALL_IPS}
+    \    ${conn_id} =    DevstackUtils.Open Connection    ${node}_CONNECTION_NAME    ${node}
+    \    Builtin.Log    Start dumping for ${node} at phase ${tag}
+    \    OpenStackOperations.Get DumpFlows And Ovsconfig    ${conn_id}
+    \    Builtin.Log    End dumping for ${node} at phase ${tag}
+    \    SSHLibrary.Close Connection
+    Builtin.Log    End dumping at phase ${tag}
+
+Canary Network Should Exist
+    OpenStackOperations.Get Neutron Network Rest    bd8db3a8-2b30-4083-a8b3-b3fd46401142
+
+Upgrade Suite Teardown
+    Set Custom Component Logging To    INFO
+    OpenStackOperations.OpenStack Suite Teardown

diff --git a/csit/testplans/netvirt-1node-openstack-upgrade.txt b/csit/testplans/netvirt-1node-openstack-upgrade.txt
new file mode 100644 (file)
index 0000000..24a8584
--- /dev/null
+++ b/csit/testplans/netvirt-1node-openstack-upgrade.txt
@@ -0,0 +1 @@
+integration/test/csit/suites/netvirt/upgrade/upgrade.robot

diff --git a/csit/variables/netvirt/Variables.robot b/csit/variables/netvirt/Variables.robot
index cff165a088287622b61ed06f33a17746f9172028..cbe67f60b91ec54dd2b1b1803b19bc970d9176be 100644 (file)
--- a/csit/variables/netvirt/Variables.robot
+++ b/csit/variables/netvirt/Variables.robot
@@ -63,7 +63,7 @@ ${INTERVAL_0}     {"tunnel-monitor-interval":{"interval":0}}
 ${INTERVAL_NEG}    {"tunnel-monitor-interval":{"interval":-100}}
 ${INTERVAL_31000}    {"tunnel-monitor-interval":{"interval":31000}}
 #ODL Rest URLs
-${NETWORK_URL}    ${CONFIG_API}/neutron:neutron/networks/
+${NETWORK_URL}    ${CONFIG_API}/neutron:neutron/networks
 ${SUBNETWORK_URL}    ${CONFIG_API}/neutron:neutron/subnets/
 ${PORT_URL}       ${CONFIG_API}/neutron:neutron/ports/
 ${ROUTER_URL}     ${CONFIG_API}/neutron:neutron/routers/
@@ -77,12 +77,13 @@ ${TUNNEL_INTERFACES}    ${CONFIG_API}/ietf-interfaces:interfaces/
 ${LEARNT_VIP}     ${OPERATIONAL_API}/odl-l3vpn:learnt-vpn-vip-to-port-data/
 ${DISPATCHER_TABLE}    17
 ${GWMAC_TABLE}    19
-${IPV6_TABLE}     45
 ${L3_TABLE}       21
+${IPV6_TABLE}     45
 ${ELAN_BASETABLE}    48
 ${ELAN_SMACTABLE}    50
 ${ELAN_DMACTABLE}    51
 ${ELAN_UNKNOWNMACTABLE}    52
 ${INGRESS_ACL_REMOTE_ACL_TABLE}    211
 ${EGRESS_ACL_TABLE}    240
+${EGRESS_LPORT_DISPATCHER_TABLE}    220
 ${EGRESS_LEARN_ACL_FILTER_TABLE}    244