csit/suites/netvirt/vpnservice/vpn_basic_ipv6.robot

   1 *** Settings ***
   2 Documentation     Test suite to validate IPv6 vpnservice functionality in an Openstack integrated environment.
   3 ...               The assumption of this suite is that the environment is already configured with the proper
   4 ...               integration bridges and vxlan tunnels.
   5 Suite Setup       BuiltIn.Run Keywords    OpenStackOperations.Get OvsDebugInfo
   6 ...               AND    VpnOperations.Basic Vpnservice Suite Setup
   7 ...               AND    OpenStackOperations.Get OvsDebugInfo
   8 Suite Teardown    SSHLibrary.Close All Connections
   9 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  10 Test Teardown     OpenStackOperations.Get Test Teardown Debugs
  11 Library           OperatingSystem
  12 Library           RequestsLibrary
  13 Resource          ../../../libraries/Utils.robot
  14 Resource          ../../../libraries/OpenStackOperations.robot
  15 Resource          ../../../libraries/DevstackUtils.robot
  16 Resource          ../../../libraries/VpnOperations.robot
  17 Resource          ../../../libraries/OVSDB.robot
  18 Resource          ../../../libraries/SetupUtils.robot
  19 Resource          ../../../libraries/Tcpdump.robot
  20 Resource          ../../../variables/Variables.robot
  21 Resource          ../../../variables/netvirt/Variables.robot
  22
  23 *** Variables ***
  24 ${SECURITY_GROUP}    vpn6_sg
  25 @{NETWORKS}       vpn6_net_1    vpn6_net_2
  26 @{SUBNETS}        vpn6_sub_1    vpn6_sub_2
  27 @{SUBNETS_CIDR}    2001:db8:0:2::/64    2001:db8:0:3::/64
  28 @{PORTS}          vpn6_net_1_port_1    vpn6_net_1_port_2    vpn6_net_2_port_1    vpn6_net_2_port_2
  29 @{NET_1_VM_INSTANCES}    vpn6_net_1_vm_1    vpn6_net_1_vm_2
  30 @{NET_2_VM_INSTANCES}    vpn6_net_2_vm_1    vpn6_net_2_vm_2
  31 ${ROUTER}         vpn6_router
  32 @{EXTRA_NW_IP}    2001:db9:cafe:d::10    2001:db9:abcd:d::20
  33 @{EXTRA_NW_SUBNET}    2001:db9:cafe:d::/64    2001:db9:abcd:d::/64
  34 ${UPDATE_NETWORK}    UpdateNetworkV6
  35 ${UPDATE_SUBNET}    UpdateSubnetV6
  36 ${UPDATE_PORT}    UpdatePortV6
  37 @{VPN_INSTANCE_IDS}    4ae8cd92-48ca-49b5-94e1-b2921a261661    4ae8cd92-48ca-49b5-94e1-b2921a261662    4ae8cd92-48ca-49b5-94e1-b2921a261663
  38 @{VPN_NAME}       vpn6_1    vpn6_2    vpn6_3
  39 @{RDS}            ["2206:2"]    ["2306:2"]    ["2406:2"]
  40
  41 *** Test Cases ***
  42 Create Neutron Networks
  43     OpenStackOperations.Create Network    @{NETWORKS}[0]
  44     OpenStackOperations.Create Network    @{NETWORKS}[1]
  45     BuiltIn.Wait Until Keyword Succeeds    3s    1s    Utils.Check For Elements At URI    ${NETWORK_URL}    ${NETWORKS}
  46     OpenStackOperations.Update Network    @{NETWORKS}[0]    additional_args=--description ${UPDATE_NETWORK}
  47     ${output} =    OpenStackOperations.Show Network    @{NETWORKS}[0]
  48     BuiltIn.Should Contain    ${output}    ${UPDATE_NETWORK}
  49
  50 Create Neutron Subnets
  51     ${net1_additional_args}=    BuiltIn.Catenate    --ip-version=6 --ipv6-address-mode=slaac --ipv6-ra-mode=slaac ${NET1_IPV6_ADDR_POOL}
  52     ${net2_additional_args}=    BuiltIn.Catenate    --ip-version=6 --ipv6-address-mode=slaac --ipv6-ra-mode=slaac ${NET2_IPV6_ADDR_POOL}
  53     OpenStackOperations.Create SubNet    @{NETWORKS}[0]    @{SUBNETS}[0]    @{SUBNETS_CIDR}[0]    ${net1_additional_args}
  54     OpenStackOperations.Create SubNet    @{NETWORKS}[1]    @{SUBNETS}[1]    @{SUBNETS_CIDR}[1]    ${net2_additional_args}
  55     BuiltIn.Wait Until Keyword Succeeds    3s    1s    Utils.Check For Elements At URI    ${SUBNETWORK_URL}    ${SUBNETS}
  56     OpenStackOperations.Update SubNet    @{SUBNETS}[0]    additional_args=--description ${UPDATE_SUBNET}
  57     ${output} =    OpenStackOperations.Show SubNet    @{SUBNETS}[0]
  58     BuiltIn.Should Contain    ${output}    ${UPDATE_SUBNET}
  59
  60 Create Router
  61     OpenStackOperations.Create Router    ${ROUTER}
  62     ${router_list} =    BuiltIn.Create List    ${ROUTER}
  63     BuiltIn.Wait Until Keyword Succeeds    3s    1s    Utils.Check For Elements At URI    ${ROUTER_URL}    ${router_list}
  64
  65 Add Interfaces To Router
  66     : FOR    ${interface}    IN    @{SUBNETS}
  67     \    OpenStackOperations.Add Router Interface    ${ROUTER}    ${interface}
  68     ${interface_output} =    OpenStackOperations.Show Router Interface    ${ROUTER}
  69     : FOR    ${interface}    IN    @{SUBNETS}
  70     \    ${subnet_id} =    OpenStackOperations.Get Subnet Id    ${interface}    ${devstack_conn_id}
  71     \    BuiltIn.Should Contain    ${interface_output}    ${subnet_id}
  72     ${GWMAC_ADDRS}    ${GWIP_ADDRS} =    VpnOperations.Get Gateway MAC And IP Address    ${ROUTER}    ${IP6_REGEX}
  73     BuiltIn.Set Suite Variable    ${GWMAC_ADDRS}
  74     BuiltIn.Set Suite Variable    ${GWIP_ADDRS}
  75
  76 Add Ssh V6 Allow All Rule
  77     OpenStackOperations.Create Allow All SecurityGroup    ${SECURITY_GROUP}    IPv6
  78
  79 Create Neutron Ports
  80     ${allowed_address_pairs_args}=    Set Variable If    '${OPENSTACK_BRANCH}'=='stable/newton'    --allowed-address ip_address=@{EXTRA_NW_SUBNET}[0] --allowed-address ip_address=@{EXTRA_NW_SUBNET}[1]    --allowed-address ip-address=@{EXTRA_NW_SUBNET}[0] --allowed-address ip-address=@{EXTRA_NW_SUBNET}[1]
  81     Create Port    @{NETWORKS}[0]    @{PORTS}[0]    sg=${SECURITY_GROUP}    additional_args=${allowed_address_pairs_args}
  82     Create Port    @{NETWORKS}[0]    @{PORTS}[1]    sg=${SECURITY_GROUP}    additional_args=${allowed_address_pairs_args}
  83     Create Port    @{NETWORKS}[1]    @{PORTS}[2]    sg=${SECURITY_GROUP}    additional_args=${allowed_address_pairs_args}
  84     Create Port    @{NETWORKS}[1]    @{PORTS}[3]    sg=${SECURITY_GROUP}    additional_args=${allowed_address_pairs_args}
  85     Wait Until Keyword Succeeds    3s    1s    Check For Elements At URI    ${PORT_URL}    ${PORTS}
  86     Update Port    @{PORTS}[0]    additional_args=--name ${UPDATE_PORT}
  87     ${output} =    Show Port    ${UPDATE_PORT}
  88     Should Contain    ${output}    ${UPDATE_PORT}
  89     Update Port    ${UPDATE_PORT}    additional_args=--name @{PORTS}[0]
  90
  91 Create Nova VMs
  92     OpenStackOperations.Create Vm Instance With Port On Compute Node    @{PORTS}[0]    @{NET_1_VM_INSTANCES}[0]    ${OS_COMPUTE_1_IP}    sg=${SECURITY_GROUP}
  93     OpenStackOperations.Create Vm Instance With Port On Compute Node    @{PORTS}[1]    @{NET_1_VM_INSTANCES}[1]    ${OS_COMPUTE_2_IP}    sg=${SECURITY_GROUP}
  94     OpenStackOperations.Create Vm Instance With Port On Compute Node    @{PORTS}[2]    @{NET_2_VM_INSTANCES}[0]    ${OS_COMPUTE_1_IP}    sg=${SECURITY_GROUP}
  95     OpenStackOperations.Create Vm Instance With Port On Compute Node    @{PORTS}[3]    @{NET_2_VM_INSTANCES}[1]    ${OS_COMPUTE_2_IP}    sg=${SECURITY_GROUP}
  96     ${vms}=    BuiltIn.Create List    @{NET_1_VM_INSTANCES}    @{NET_2_VM_INSTANCES}
  97     : FOR    ${vm}    IN    @{vms}
  98     \    OpenStackOperations.Poll VM Is ACTIVE    ${vm}
  99     BuiltIn.Wait Until Keyword Succeeds    30s    10s    Wait For Routes To Propogate    ${NETWORKS}    ${SUBNETS_CIDR}
 100     ${prefix_net10} =    Replace String    @{SUBNETS_CIDR}[0]    ::/64    (:[a-f0-9]{,4}){,4}
 101     ${status}    ${message}    Run Keyword And Ignore Error    BuiltIn.Wait Until Keyword Succeeds    3x    60s    OpenStackOperations.Collect VM IPv6 SLAAC Addresses
 102     ...    true    ${prefix_net10}    @{NET_1_VM_INSTANCES}
 103     ${prefix_net20} =    Replace String    @{SUBNETS_CIDR}[1]    ::/64    (:[a-f0-9]{,4}){,4}
 104     ${status}    ${message}    Run Keyword And Ignore Error    BuiltIn.Wait Until Keyword Succeeds    3x    60s    OpenStackOperations.Collect VM IPv6 SLAAC Addresses
 105     ...    true    ${prefix_net20}    @{NET_2_VM_INSTANCES}
 106     ${VM_IP_NET10} =    OpenStackOperations.Collect VM IPv6 SLAAC Addresses    false    ${prefix_net10}    @{NET_1_VM_INSTANCES}
 107     ${VM_IP_NET20} =    OpenStackOperations.Collect VM IPv6 SLAAC Addresses    false    ${prefix_net20}    @{NET_2_VM_INSTANCES}
 108     ${VM_INSTANCES} =    Collections.Combine Lists    ${NET_1_VM_INSTANCES}    ${NET_2_VM_INSTANCES}
 109     ${VM_IPS}=    Collections.Combine Lists    ${VM_IP_NET10}    ${VM_IP_NET20}
 110     ${LOOP_COUNT}    BuiltIn.Get Length    ${NET_1_VM_INSTANCES}
 111     : FOR    ${index}    IN RANGE    0    ${LOOP_COUNT}
 112     \    ${status}    ${message}    Run Keyword And Ignore Error    BuiltIn.Should Not Contain    @{VM_IPS}[${index}]    None
 113     \    Run Keyword If    '${status}' == 'FAIL'    DevstackUtils.Write Commands Until Prompt    openstack console log show @{VM_INSTANCES}[${index}]    30s
 114     BuiltIn.Set Suite Variable    ${VM_IP_NET10}
 115     BuiltIn.Set Suite Variable    ${VM_IP_NET20}
 116     BuiltIn.Should Not Contain    ${VM_IP_NET10}    None
 117     BuiltIn.Should Not Contain    ${VM_IP_NET20}    None
 118     [Teardown]    BuiltIn.Run Keywords    OpenStackOperations.Show Debugs    @{NET_1_VM_INSTANCES}    @{NET_2_VM_INSTANCES}
 119     ...    AND    OpenStackOperations.Get Test Teardown Debugs
 120
 121 Check ELAN Datapath Traffic Within The Networks
 122     ${output}=    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    @{VM_IP_NET10}[0]    ping6 -c 3 @{VM_IP_NET10}[1]
 123     BuiltIn.Should Contain    ${output}    64 bytes
 124     ${output}=    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[1]    @{VM_IP_NET20}[0]    ping6 -c 3 @{VM_IP_NET20}[1]
 125     BuiltIn.Should Contain    ${output}    64 bytes
 126
 127 Check L3_Datapath Traffic Across Networks With Router
 128     BuiltIn.Log    Verification of FIB Entries and Flow
 129     ${cn1_conn_id} =    Tcpdump.Start Packet Capture on Node    ${OS_COMPUTE_1_IP}    file_Name=tcpDumpCN1
 130     ${cn2_conn_id} =    Tcpdump.Start Packet Capture on Node    ${OS_COMPUTE_2_IP}    file_Name=tcpDumpCN2
 131     ${os_conn_id} =    Tcpdump.Start Packet Capture on Node    ${OS_CONTROL_NODE_IP}    file_Name=tcpDumpOS
 132     ${vm_ips} =    BuiltIn.Create List    @{VM_IP_NET10}    @{VM_IP_NET20}
 133     BuiltIn.Wait Until Keyword Succeeds    30s    10s    Utils.Check For Elements At URI    ${FIB_ENTRY_URL}    ${vm_ips}
 134     BuiltIn.Wait Until Keyword Succeeds    30s    10s    VpnOperations.Verify Flows Are Present For L3VPN    ${OS_COMPUTE_1_IP}    ${VM_IP_NET10}
 135     BuiltIn.Wait Until Keyword Succeeds    30s    10s    VpnOperations.Verify Flows Are Present For L3VPN    ${OS_COMPUTE_1_IP}    ${VM_IP_NET20}
 136     BuiltIn.Wait Until Keyword Succeeds    30s    10s    VpnOperations.Verify GWMAC Entry On ODL    ${GWMAC_ADDRS}
 137     BuiltIn.Wait Until Keyword Succeeds    30s    10s    VpnOperations.Verify GWMAC Flow Entry On Flow Table    ${OS_COMPUTE_1_IP}    ipv6
 138     BuiltIn.Wait Until Keyword Succeeds    30s    10s    VpnOperations.Verify GWMAC Flow Entry On Flow Table    ${OS_COMPUTE_2_IP}    ipv6
 139     ${dst_ip_list} =    BuiltIn.Create List    @{VM_IP_NET10}[1]    @{VM_IP_NET20}
 140     OpenStackOperations.Test Operations From Vm Instance    @{NETWORKS}[0]    @{VM_IP_NET10}[0]    ${dst_ip_list}
 141     ${dst_ip_list} =    BuiltIn.Create List    @{VM_IP_NET20}[1]    @{VM_IP_NET10}
 142     OpenStackOperations.Test Operations From Vm Instance    @{NETWORKS}[1]    @{VM_IP_NET20}[0]    ${dst_ip_list}
 143     [Teardown]    VpnOperations.Test Teardown With Tcpdump Stop    ${cn1_conn_id}    ${cn2_conn_id}    ${os_conn_id}
 144
 145 Add Multiple Extra Routes And Check Datapath Before L3VPN Creation
 146     ${CONFIG_EXTRA_ROUTE_IP1} =    BuiltIn.Catenate    sudo ip -6 addr add @{EXTRA_NW_IP}[0]/64 dev eth0
 147     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    @{VM_IP_NET10}[0]    ${CONFIG_EXTRA_ROUTE_IP1}
 148     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    @{VM_IP_NET10}[0]    ip -6 a
 149     ${CONFIG_EXTRA_ROUTE_IP2} =    BuiltIn.Catenate    sudo ip -6 addr add @{EXTRA_NW_IP}[1]/64 dev eth0
 150     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    @{VM_IP_NET10}[0]    ${CONFIG_EXTRA_ROUTE_IP2}
 151     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    @{VM_IP_NET10}[0]    ip -6 a
 152     ${EXT_RT1} =    Set Variable    destination=@{EXTRA_NW_SUBNET}[0],gateway=@{VM_IP_NET10}[0]
 153     ${EXT_RT2} =    Set Variable    destination=@{EXTRA_NW_SUBNET}[1],gateway=@{VM_IP_NET10}[0]
 154     ${cmd} =    BuiltIn.Catenate    ${RT_OPTIONS}    ${EXT_RT1}    ${RT_OPTIONS}    ${EXT_RT2}
 155     OpenStackOperations.Update Router    ${ROUTER}    ${cmd}
 156     OpenStackOperations.Show Router    ${ROUTER}    -D
 157     ${vm_ips} =    BuiltIn.Create List    @{EXTRA_NW_SUBNET}
 158     BuiltIn.Wait Until Keyword Succeeds    30s    10s    Utils.Check For Elements At URI    ${FIB_ENTRY_URL}    ${vm_ips}
 159     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    @{VM_IP_NET10}[1]    ping6 -c 3 @{EXTRA_NW_IP}[1]
 160     BuiltIn.Should Contain    ${output}    64 bytes
 161     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[1]    @{VM_IP_NET20}[1]    ping6 -c 3 @{EXTRA_NW_IP}[1]
 162     BuiltIn.Should Contain    ${output}    64 bytes
 163     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    @{VM_IP_NET10}[1]    ping6 -c 3 @{EXTRA_NW_IP}[0]
 164     BuiltIn.Should Contain    ${output}    64 bytes
 165
 166 Delete Extra Route
 167     OpenStackOperations.Update Router    ${ROUTER}    ${RT_CLEAR}
 168     OpenStackOperations.Show Router    ${ROUTER}    -D
 169
 170 Delete And Recreate Extra Route
 171     [Documentation]    Recreate multiple extra route and check data path before L3VPN creation
 172     ${CONFIG_EXTRA_ROUTE_IP1} =    BuiltIn.Catenate    sudo ip -6 addr add @{EXTRA_NW_IP}[1]/64 dev eth0
 173     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    @{VM_IP_NET10}[0]    ${CONFIG_EXTRA_ROUTE_IP1}
 174     ${EXT_RT1} =    Set Variable    destination=@{EXTRA_NW_SUBNET}[0],gateway=@{VM_IP_NET10}[0]
 175     ${cmd} =    BuiltIn.Catenate    ${RT_OPTIONS}    ${EXT_RT1}
 176     OpenStackOperations.Update Router    ${ROUTER}    ${cmd}
 177     OpenStackOperations.Show Router    ${ROUTER}    -D
 178     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    @{VM_IP_NET10}[1]    ping6 -c 3 @{EXTRA_NW_IP}[0]
 179     BuiltIn.Should Contain    ${output}    64 bytes
 180     # clear off extra-routes before the next set of tests
 181     [Teardown]    BuiltIn.Run Keywords    OpenStackOperations.Update Router    ${ROUTER}    ${RT_CLEAR}
 182     ...    AND    OpenStackOperations.Show Router    ${ROUTER}    -D
 183     ...    AND    OpenStackOperations.Get Test Teardown Debugs
 184
 185 Create L3VPN
 186     ${net_id} =    OpenStackOperations.Get Net Id    @{NETWORKS}[0]    ${devstack_conn_id}
 187     ${tenant_id} =    OpenStackOperations.Get Tenant ID From Network    ${net_id}
 188     VpnOperations.VPN Create L3VPN    vpnid=@{VPN_INSTANCE_IDS}[0]    name=@{VPN_NAME}[0]    rd=@{RDS}[0]    exportrt=@{RDS}[0]    importrt=@{RDS}[0]    tenantid=${tenant_id}
 189     ${resp}=    VpnOperations.VPN Get L3VPN    vpnid=@{VPN_INSTANCE_IDS}[0]
 190     BuiltIn.Should Contain    ${resp}    @{VPN_INSTANCE_IDS}[0]
 191
 192 Associate L3VPN To Routers
 193     ${router_id} =    OpenStackOperations.Get Router Id    ${ROUTER}    ${devstack_conn_id}
 194     VpnOperations.Associate VPN to Router    routerid=${router_id}    vpnid=@{VPN_INSTANCE_IDS}[0]
 195     ${resp}=    VpnOperations.VPN Get L3VPN    vpnid=@{VPN_INSTANCE_IDS}[0]
 196     BuiltIn.Should Contain    ${resp}    ${router_id}
 197
 198 Verify L3VPN Datapath With Router Association
 199     BuiltIn.Log    Verify VPN interfaces, FIB entries and Flow table
 200     ${vm_instances} =    BuiltIn.Create List    @{VM_IP_NET10}    @{VM_IP_NET20}
 201     BuiltIn.Wait Until Keyword Succeeds    30s    10s    Utils.Check For Elements At URI    ${VPN_IFACES_URL}    ${vm_instances}
 202     ${RD} =    Strip String    @{RDS}[0]    characters="[]
 203     BuiltIn.Wait Until Keyword Succeeds    60s    15s    Utils.Check For Elements At URI    ${CONFIG_API}/odl-fib:fibEntries/vrfTables/${RD}/    ${vm_instances}
 204     BuiltIn.Wait Until Keyword Succeeds    60s    15s    VpnOperations.Verify Flows Are Present For L3VPN    ${OS_COMPUTE_1_IP}    ${vm_instances}
 205     BuiltIn.Wait Until Keyword Succeeds    60s    15s    VpnOperations.Verify Flows Are Present For L3VPN    ${OS_COMPUTE_2_IP}    ${vm_instances}
 206     BuiltIn.Wait Until Keyword Succeeds    30s    10s    VpnOperations.Verify GWMAC Entry On ODL    ${GWMAC_ADDRS}
 207     BuiltIn.Wait Until Keyword Succeeds    30s    10s    VpnOperations.Verify GWMAC Flow Entry On Flow Table    ${OS_COMPUTE_1_IP}    ipv6
 208     BuiltIn.Wait Until Keyword Succeeds    30s    10s    VpnOperations.Verify GWMAC Flow Entry On Flow Table    ${OS_COMPUTE_2_IP}    ipv6
 209     BuiltIn.Log    Check datapath from network1 to network2
 210     ${dst_ip_list} =    BuiltIn.Create List    @{VM_IP_NET10}[1]    @{VM_IP_NET20}
 211     OpenStackOperations.Test Operations From Vm Instance    @{NETWORKS}[0]    @{VM_IP_NET10}[0]    ${dst_ip_list}
 212     BuiltIn.Log    Check datapath from network2 to network1
 213     ${dst_ip_list} =    BuiltIn.Create List    @{VM_IP_NET20}[1]    @{VM_IP_NET10}
 214     OpenStackOperations.Test Operations From Vm Instance    @{NETWORKS}[1]    @{VM_IP_NET20}[0]    ${dst_ip_list}
 215
 216 Dissociate L3VPN From Routers
 217     ${router_id}=    OpenStackOperations.Get Router Id    ${ROUTER}    ${devstack_conn_id}
 218     VpnOperations.Dissociate VPN to Router    routerid=${router_id}    vpnid=@{VPN_INSTANCE_IDS}[0]
 219     ${resp}=    VpnOperations.VPN Get L3VPN    vpnid=@{VPN_INSTANCE_IDS}[0]
 220     BuiltIn.Should Not Contain    ${resp}    ${router_id}
 221
 222 Delete Router And Router Interfaces With L3VPN
 223     ${router_id}=    OpenStackOperations.Get Router Id    ${ROUTER}    ${devstack_conn_id}
 224     VpnOperations.Associate VPN to Router    routerid=${router_id}    vpnid=@{VPN_INSTANCE_IDS}[0]
 225     ${resp} =    VpnOperations.VPN Get L3VPN    vpnid=@{VPN_INSTANCE_IDS}[0]
 226     BuiltIn.Should Contain    ${resp}    ${router_id}
 227     : FOR    ${INTERFACE}    IN    @{SUBNETS}
 228     \    OpenStackOperations.Remove Interface    ${ROUTER}    ${INTERFACE}
 229     ${interface_output} =    OpenStackOperations.Show Router Interface    ${ROUTER}
 230     : FOR    ${INTERFACE}    IN    @{SUBNETS}
 231     \    ${subnet_id} =    OpenStackOperations.Get Subnet Id    ${INTERFACE}    ${devstack_conn_id}
 232     \    BuiltIn.Should Not Contain    ${interface_output}    ${subnet_id}
 233     OpenStackOperations.Delete Router    ${ROUTER}
 234     ${router_output} =    OpenStackOperations.List Routers
 235     BuiltIn.Should Not Contain    ${router_output}    ${ROUTER}
 236     @{router_list} =    BuiltIn.Create List    ${ROUTER}
 237     BuiltIn.Wait Until Keyword Succeeds    3s    1s    Utils.Check For Elements Not At URI    ${ROUTER_URL}    ${router_list}
 238     ${resp}=    VpnOperations.VPN Get L3VPN    vpnid=@{VPN_INSTANCE_IDS}[0]
 239     BuiltIn.Should Not Contain    ${resp}    ${router_id}
 240     BuiltIn.Wait Until Keyword Succeeds    30s    10s    Verify GWMAC Flow Entry Removed From Flow Table    ${OS_COMPUTE_1_IP}
 241     BuiltIn.Wait Until Keyword Succeeds    30s    10s    Verify GWMAC Flow Entry Removed From Flow Table    ${OS_COMPUTE_2_IP}
 242
 243 Delete Router With NonExistentRouter Name
 244     ${rc}    ${output}=    Run And Return Rc And Output    neutron router-delete nonExistentRouter
 245     BuiltIn.Should Match Regexp    ${output}    Unable to find router with name or id 'nonExistentRouter'|Unable to find router\\(s\\) with id\\(s\\) 'nonExistentRouter'
 246
 247 Associate L3VPN To Networks
 248     ${network1_id} =    OpenStackOperations.Get Net Id    @{NETWORKS}[0]    ${devstack_conn_id}
 249     ${network2_id} =    OpenStackOperations.Get Net Id    @{NETWORKS}[1]    ${devstack_conn_id}
 250     VpnOperations.Associate L3VPN To Network    networkid=${network1_id}    vpnid=@{VPN_INSTANCE_IDS}[0]
 251     ${resp} =    VpnOperations.VPN Get L3VPN    vpnid=@{VPN_INSTANCE_IDS}[0]
 252     BuiltIn.Should Contain    ${resp}    ${network1_id}
 253     VpnOperations.Associate L3VPN To Network    networkid=${network2_id}    vpnid=@{VPN_INSTANCE_IDS}[0]
 254     ${resp} =    VpnOperations.VPN Get L3VPN    vpnid=@{VPN_INSTANCE_IDS}[0]
 255     BuiltIn.Should Contain    ${resp}    ${network2_id}
 256
 257 Dissociate L3VPN From Networks
 258     ${network1_id} =    OpenStackOperations.Get Net Id    @{NETWORKS}[0]    ${devstack_conn_id}
 259     ${network2_id} =    OpenStackOperations.Get Net Id    @{NETWORKS}[1]    ${devstack_conn_id}
 260     VpnOperations.Dissociate L3VPN From Networks    networkid=${network1_id}    vpnid=@{VPN_INSTANCE_IDS}[0]
 261     ${resp}=    VpnOperations.VPN Get L3VPN    vpnid=@{VPN_INSTANCE_IDS}[0]
 262     BuiltIn.Should Not Contain    ${resp}    ${network1_id}
 263     VpnOperations.Dissociate L3VPN From Networks    networkid=${network2_id}    vpnid=@{VPN_INSTANCE_IDS}[0]
 264     ${resp}=    VpnOperations.VPN Get L3VPN    vpnid=@{VPN_INSTANCE_IDS}[0]
 265     BuiltIn.Should Not Contain    ${resp}    ${network2_id}
 266
 267 Delete L3VPN
 268     VpnOperations.VPN Delete L3VPN    vpnid=@{VPN_INSTANCE_IDS}[0]
 269
 270 Create Multiple L3VPN
 271     ${net_id} =    Get Net Id    @{NETWORKS}[0]    ${devstack_conn_id}
 272     ${tenant_id} =    Get Tenant ID From Network    ${net_id}
 273     VpnOperations.VPN Create L3VPN    vpnid=@{VPN_INSTANCE_IDS}[0]    name=@{VPN_NAME}[0]    rd=@{RDS}[0]    exportrt=@{RDS}[0]    importrt=@{RDS}[0]    tenantid=${tenant_id}
 274     VpnOperations.VPN Create L3VPN    vpnid=@{VPN_INSTANCE_IDS}[1]    name=@{VPN_NAME}[1]    rd=@{RDS}[1]    exportrt=@{RDS}[1]    importrt=@{RDS}[1]    tenantid=${tenant_id}
 275     VpnOperations.VPN Create L3VPN    vpnid=@{VPN_INSTANCE_IDS}[2]    name=@{VPN_NAME}[2]    rd=@{RDS}[2]    exportrt=@{RDS}[2]    importrt=@{RDS}[2]    tenantid=${tenant_id}
 276     ${resp}=    VpnOperations.VPN Get L3VPN    vpnid=@{VPN_INSTANCE_IDS}[0]
 277     BuiltIn.Should Contain    ${resp}    @{VPN_INSTANCE_IDS}[0]
 278     ${resp}=    VpnOperations.VPN Get L3VPN    vpnid=@{VPN_INSTANCE_IDS}[1]
 279     BuiltIn.Should Contain    ${resp}    @{VPN_INSTANCE_IDS}[1]
 280     ${resp}=    VpnOperations.VPN Get L3VPN    vpnid=@{VPN_INSTANCE_IDS}[2]
 281     BuiltIn.Should Contain    ${resp}    @{VPN_INSTANCE_IDS}[2]
 282
 283 Delete Multiple L3VPN
 284     VpnOperations.VPN Delete L3VPN    vpnid=@{VPN_INSTANCE_IDS}[0]
 285     VpnOperations.VPN Delete L3VPN    vpnid=@{VPN_INSTANCE_IDS}[1]
 286     VpnOperations.VPN Delete L3VPN    vpnid=@{VPN_INSTANCE_IDS}[2]
 287
 288 Cleanup
 289     @{vms} =    BuiltIn.Create List    @{NET_1_VM_INSTANCES}    @{NET_2_VM_INSTANCES}
 290     @{sgs} =    BuiltIn.Create List    ${SECURITY_GROUP}
 291     Basic Vpnservice Suite Cleanup    ${VPN_INSTANCE_IDS}    ${vms}    ${NETWORKS}    ${SUBNETS}    ${PORTS}    ${sgs}