csit/suites/netvirt/vpnservice/vpn_basic_dualstack_one_router.robot

   1 *** Settings ***
   2 Documentation     Test suite to validate dualstack (IPv4 + IPv6) vpnservice functionality in an Openstack
   3 ...               integrated environment.
   4 ...               The assumption of this suite is that the environment is already configured with the proper
   5 ...               integration bridges and vxlan tunnels.
   6 Suite Setup       VpnOperations.Basic Suite Setup
   7 Suite Teardown    VpnOperations.Basic Vpnservice Suite Cleanup
   8 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
   9 Test Teardown     OpenStackOperations.Get Test Teardown Debugs
  10 Library           Collections
  11 Library           OperatingSystem
  12 Library           RequestsLibrary
  13 Resource          ../../../libraries/Utils.robot
  14 Resource          ../../../libraries/OpenStackOperations.robot
  15 Resource          ../../../libraries/DevstackUtils.robot
  16 Resource          ../../../libraries/VpnOperations.robot
  17 Resource          ../../../libraries/OVSDB.robot
  18 Resource          ../../../libraries/SetupUtils.robot
  19 Resource          ../../../libraries/Tcpdump.robot
  20 Resource          ../../../variables/Variables.robot
  21 Resource          ../../../variables/netvirt/Variables.robot
  22
  23 *** Variables ***
  24 ${SECURITY_GROUP}    vpn_sg_dualstack
  25 @{NETWORKS}       vpn_net_1_dualstack    vpn_net_2_dualstack
  26 @{SUBNETS4}       vpn_net_ipv4_1_dualstack    vpn_net_ipv4_2_dualstack
  27 @{SUBNETS6}       vpn_net_ipv6_1_dualstack    vpn_net_ipv6_2_dualstack
  28 @{SUBNETS4_CIDR}    30.1.1.0/24    40.1.1.0/24
  29 @{SUBNETS6_CIDR}    2001:db5:0:2::/64    2001:db5:0:3::/64
  30 ${SUBNET_ADDITIONAL_ARGS}    --ip-version=6 --ipv6-address-mode=slaac --ipv6-ra-mode=slaac
  31 @{PORTS}          vpn_port_1_dualstack    vpn_port_2_dualstack    vpn_port_3_dualstack    vpn_port_4_dualstack
  32 @{NET_1_VM_INSTANCES}    vpn_net_1_vm_1_dualstack    vpn_net_1_vm_2_dualstack
  33 @{NET_2_VM_INSTANCES}    vpn_net_2_vm_1_dualstack    vpn_net_2_vm_2_dualstack
  34 @{EXTRA_NW_IPV4}    76.1.1.2    77.1.1.2
  35 @{EXTRA_NW_IPV6}    3001:db9:cafe:d::10    3001:db9:abcd:d::20
  36 @{EXTRA_NW_SUBNET_IPv4}    76.1.1.0/24    77.1.1.0/24
  37 @{EXTRA_NW_SUBNET_IPv6}    3001:db9:cafe:d::/64    3001:db9:abcd:d::/64
  38 ${ROUTER}         vpn_router_dualstack
  39 ${UPDATE_NETWORK}    UpdateNetwork_dualstack
  40 ${UPDATE_SUBNET}    UpdateSubnet_dualstack
  41 ${UPDATE_PORT}    UpdatePort_dualstack
  42 @{VPN_INSTANCE_ID}    1bc8cd92-48ca-49b5-94e1-b2921a261661    1bc8cd92-48ca-49b5-94e1-b2921a261662    1bc8cd92-48ca-49b5-94e1-b2921a261663
  43 @{VPN_NAME}       vpn1_dualstack    vpn2_dualstack    vpn3_dualstack
  44 @{RDS}            ["2506:2"]    ["2606:2"]    ["2706:2"]
  45
  46 *** Test Cases ***
  47 Create Neutron Networks
  48     [Documentation]    Create two networks.
  49     OpenStackOperations.Create Network    @{NETWORKS}[0]
  50     OpenStackOperations.Create Network    @{NETWORKS}[1]
  51     ${NET_LIST} =    OpenStackOperations.List Networks
  52     BuiltIn.Wait Until Keyword Succeeds    3s    1s    Utils.Check For Elements At URI    ${NETWORK_URL}    ${NETWORKS}
  53     OpenStackOperations.Update Network    @{NETWORKS}[0]    additional_args=--description ${UPDATE_NETWORK}
  54     ${output} =    OpenStackOperations.Show Network    @{NETWORKS}[0]
  55     BuiltIn.Should Contain    ${output}    ${UPDATE_NETWORK}
  56
  57 Create Neutron Subnets
  58     [Documentation]    Create subnets for previously created networks.
  59     OpenStackOperations.Create SubNet    @{NETWORKS}[0]    @{SUBNETS4}[0]    @{SUBNETS4_CIDR}[0]
  60     OpenStackOperations.Create SubNet    @{NETWORKS}[0]    @{SUBNETS6}[0]    @{SUBNETS6_CIDR}[0]    ${SUBNET_ADDITIONAL_ARGS}
  61     OpenStackOperations.Create SubNet    @{NETWORKS}[1]    @{SUBNETS4}[1]    @{SUBNETS4_CIDR}[1]
  62     OpenStackOperations.Create SubNet    @{NETWORKS}[1]    @{SUBNETS6}[1]    @{SUBNETS6_CIDR}[1]    ${SUBNET_ADDITIONAL_ARGS}
  63     ${SUB_LIST} =    OpenStackOperations.List Subnets
  64     BuiltIn.Wait Until Keyword Succeeds    3s    1s    Utils.Check For Elements At URI    ${SUBNETWORK_URL}    ${SUBNETS4}
  65     BuiltIn.Wait Until Keyword Succeeds    3s    1s    Utils.Check For Elements At URI    ${SUBNETWORK_URL}    ${SUBNETS6}
  66     OpenStackOperations.Update SubNet    @{SUBNETS4}[0]    additional_args=--description ${UPDATE_SUBNET}
  67     ${output} =    OpenStackOperations.Show SubNet    @{SUBNETS4}[0]
  68     BuiltIn.Should Contain    ${output}    ${UPDATE_SUBNET}
  69
  70 Create Router
  71     [Documentation]    Create Router.
  72     OpenStackOperations.Create Router    ${ROUTER}
  73     @{ROUTER_LIST} =    BuiltIn.Create List    ${ROUTER}
  74     BuiltIn.Wait Until Keyword Succeeds    3s    1s    Utils.Check For Elements At URI    ${ROUTER_URL}    ${ROUTER_LIST}
  75
  76 Add Router Ports
  77     [Documentation]    Add created subnets to router.
  78     : FOR    ${PORT}    IN    @{SUBNETS4}
  79     \    OpenStackOperations.Add Router Interface    ${ROUTER}    ${PORT}
  80     ${interface_output} =    OpenStackOperations.Show Router Interface    ${ROUTER}
  81     ${GW1_MAC_ADDRS}    ${GW_IPV4_ADDRS} =    VpnOperations.Get Gateway MAC And IP Address    ${ROUTER}
  82     : FOR    ${PORT}    IN    @{SUBNETS6}
  83     \    OpenStackOperations.Add Router Interface    ${ROUTER}    ${PORT}
  84     ${interface_output} =    OpenStackOperations.Show Router Interface    ${ROUTER}
  85     ${GW2_MAC_ADDRS}    ${GW_IPV6_ADDRS} =    VpnOperations.Get Gateway MAC And IP Address    ${ROUTER}    ${IP6_REGEX}
  86     ${GW_MAC_ADDRS} =    BuiltIn.Create List    @{GW1_MAC_ADDRS}    @{GW2_MAC_ADDRS}
  87     BuiltIn.Set Suite Variable    ${GW1_MAC_ADDRS}
  88     BuiltIn.Set Suite Variable    ${GW2_MAC_ADDRS}
  89     BuiltIn.Set Suite Variable    ${GW_MAC_ADDRS}
  90     BuiltIn.Set Suite Variable    ${GW_IPV4_ADDRS}
  91     BuiltIn.Set Suite Variable    ${GW_IPV6_ADDRS}
  92
  93 Create Allow All Security Group IPv4+IPv6
  94     [Documentation]    Create neutron security group with Allow All rule set for IPv4 ethertype.
  95     ...    Then add in this group Allow All rule set for IPv6 ethertype.
  96     OpenStackOperations.Create Allow All SecurityGroup    ${SECURITY_GROUP}    IPv4
  97     OpenStackOperations.Neutron Security Group Rule Create    ${SECURITY_GROUP}    direction=ingress    ethertype=IPv6    port_range_max=65535    port_range_min=1    protocol=tcp
  98     OpenStackOperations.Neutron Security Group Rule Create    ${SECURITY_GROUP}    direction=egress    ethertype=IPv6    port_range_max=65535    port_range_min=1    protocol=tcp
  99     OpenStackOperations.Neutron Security Group Rule Create    ${SECURITY_GROUP}    direction=ingress    ethertype=IPv6    protocol=icmp
 100     OpenStackOperations.Neutron Security Group Rule Create    ${SECURITY_GROUP}    direction=egress    ethertype=IPv6    protocol=icmp
 101     OpenStackOperations.Neutron Security Group Rule Create    ${SECURITY_GROUP}    direction=ingress    ethertype=IPv6    port_range_max=65535    port_range_min=1    protocol=udp
 102     OpenStackOperations.Neutron Security Group Rule Create    ${SECURITY_GROUP}    direction=egress    ethertype=IPv6    port_range_max=65535    port_range_min=1    protocol=udp
 103
 104 Create Neutron Ports
 105     [Documentation]    Create 2 ports in previously created IPv4 subnets and 2 ports in previously created IPv6 subnets.
 106     ${allowed_address_pairs_args} =    BuiltIn.Set Variable    --allowed-address ip-address=@{EXTRA_NW_SUBNET_IPV4}[0] --allowed-address ip-address=@{EXTRA_NW_SUBNET_IPV4}[1] --allowed-address ip-address=@{EXTRA_NW_SUBNET_IPV6}[0] --allowed-address ip-address=@{EXTRA_NW_SUBNET_IPV6}[1]
 107     OpenStackOperations.Create Port    @{NETWORKS}[0]    @{PORTS}[0]    sg=${SECURITY_GROUP}    additional_args=${allowed_address_pairs_args}
 108     OpenStackOperations.Create Port    @{NETWORKS}[0]    @{PORTS}[1]    sg=${SECURITY_GROUP}    additional_args=${allowed_address_pairs_args}
 109     OpenStackOperations.Create Port    @{NETWORKS}[1]    @{PORTS}[2]    sg=${SECURITY_GROUP}    additional_args=${allowed_address_pairs_args}
 110     OpenStackOperations.Create Port    @{NETWORKS}[1]    @{PORTS}[3]    sg=${SECURITY_GROUP}    additional_args=${allowed_address_pairs_args}
 111     BuiltIn.Wait Until Keyword Succeeds    3s    1s    Utils.Check For Elements At URI    ${PORT_URL}    ${PORTS}
 112     ${PORTS_MACADDR} =    OpenStackOperations.Get Ports MacAddr    ${PORTS}
 113     BuiltIn.Set Suite Variable    ${PORTS_MACADDR}
 114     OpenStackOperations.Update Port    @{PORTS}[0]    additional_args=--name ${UPDATE_PORT}
 115     ${output} =    Show Port    ${UPDATE_PORT}
 116     BuiltIn.Should Contain    ${output}    ${UPDATE_PORT}
 117     OpenStackOperations.Update Port    ${UPDATE_PORT}    additional_args=--name @{PORTS}[0]
 118
 119 Create Nova VMs
 120     [Documentation]    Launch a VM for each previously created port.
 121     OpenStackOperations.Create Vm Instance With Port On Compute Node    @{PORTS}[0]    ${NET_1_VM_INSTANCES[0]}    ${OS_CMP1_HOSTNAME}    sg=${SECURITY_GROUP}
 122     OpenStackOperations.Create Vm Instance With Port On Compute Node    @{PORTS}[1]    ${NET_1_VM_INSTANCES[1]}    ${OS_CMP2_HOSTNAME}    sg=${SECURITY_GROUP}
 123     OpenStackOperations.Create Vm Instance With Port On Compute Node    @{PORTS}[2]    ${NET_2_VM_INSTANCES[0]}    ${OS_CMP1_HOSTNAME}    sg=${SECURITY_GROUP}
 124     OpenStackOperations.Create Vm Instance With Port On Compute Node    @{PORTS}[3]    ${NET_2_VM_INSTANCES[1]}    ${OS_CMP2_HOSTNAME}    sg=${SECURITY_GROUP}
 125     ${VM_INSTANCES} =    BuiltIn.Create List    @{NET_1_VM_INSTANCES}    @{NET_2_VM_INSTANCES}
 126     BuiltIn.Set Suite Variable    ${VM_INSTANCES}
 127     BuiltIn.Wait Until Keyword Succeeds    30s    10s    OpenStackOperations.Wait For Routes To Propogate    ${NETWORKS}    ${SUBNETS4_CIDR}
 128     BuiltIn.Wait Until Keyword Succeeds    30s    10s    OpenStackOperations.Wait For Routes To Propogate    ${NETWORKS}    ${SUBNETS6_CIDR}
 129     @{NET_1_VM_IPV4}    ${NET_1_DHCP_IPV4} =    OpenStackOperations.Get VM IPs    @{NET_1_VM_INSTANCES}
 130     @{NET_2_VM_IPV4}    ${NET_2_DHCP_IPV4} =    OpenStackOperations.Get VM IPs    @{NET_2_VM_INSTANCES}
 131     BuiltIn.Should Not Contain    ${NET_1_VM_IPV4}    None
 132     BuiltIn.Should Not Contain    ${NET_2_VM_IPV4}    None
 133     BuiltIn.Should Not Contain    ${NET_1_DHCP_IPV4}    None
 134     BuiltIn.Should Not Contain    ${NET_2_DHCP_IPV4}    None
 135     BuiltIn.Log    Collect VMs IPv6 addresses
 136     ${prefix_net10} =    String.Replace String    @{SUBNETS6_CIDR}[0]    ${IP6_SUBNET_CIDR_SUFFIX}    ${IP6_ADDR_SUFFIX}
 137     ${prefix_net20} =    String.Replace String    @{SUBNETS6_CIDR}[1]    ${IP6_SUBNET_CIDR_SUFFIX}    ${IP6_ADDR_SUFFIX}
 138     ${status}    ${message}    Run Keyword And Ignore Error    BuiltIn.Wait Until Keyword Succeeds    3x    60s    OpenStackOperations.Collect VM IPv6 SLAAC Addresses
 139     ...    true    ${NET_1_VM_INSTANCES}    @{NETWORKS}[0]    ${prefix_net10}
 140     ${status}    ${message}    Run Keyword And Ignore Error    BuiltIn.Wait Until Keyword Succeeds    3x    60s    OpenStackOperations.Collect VM IPv6 SLAAC Addresses
 141     ...    true    ${NET_2_VM_INSTANCES}    @{NETWORKS}[1]    ${prefix_net20}
 142     ${NET_1_VM_IPV6} =    OpenStackOperations.Collect VM IPv6 SLAAC Addresses    false    ${NET_1_VM_INSTANCES}    @{NETWORKS}[0]    ${prefix_net10}
 143     ${NET_2_VM_IPV6} =    OpenStackOperations.Collect VM IPv6 SLAAC Addresses    false    ${NET_2_VM_INSTANCES}    @{NETWORKS}[1]    ${prefix_net20}
 144     ${LOOP_COUNT}    Get Length    ${NET_1_VM_INSTANCES}
 145     : FOR    ${index}    IN RANGE    0    ${LOOP_COUNT}
 146     \    ${status}    ${message}    Run Keyword And Ignore Error    BuiltIn.Should Not Contain    @{NET_1_VM_IPV6}[${index}]    None
 147     \    Run Keyword If    '${status}' == 'FAIL'    Write Commands Until Prompt    nova console-log @{NET_1_VM_INSTANCES}[${index}]    30s
 148     \    ${status}    ${message}    Run Keyword And Ignore Error    BuiltIn.Should Not Contain    @{NET_2_VM_IPV6}[${index}]    None
 149     \    Run Keyword If    '${status}' == 'FAIL'    Write Commands Until Prompt    nova console-log @{NET_2_VM_INSTANCES}[${index}]    30s
 150     BuiltIn.Set Suite Variable    ${NET_1_VM_IPV4}
 151     BuiltIn.Set Suite Variable    ${NET_2_VM_IPV4}
 152     BuiltIn.Set Suite Variable    ${NET_1_VM_IPV6}
 153     BuiltIn.Set Suite Variable    ${NET_2_VM_IPV6}
 154     ${VM_IPS} =    BuiltIn.Create List    @{NET_1_VM_IPV4}    @{NET_2_VM_IPV4}    @{NET_1_VM_IPV6}    @{NET_2_VM_IPV6}
 155     BuiltIn.Set Suite Variable    ${VM_IPS}
 156     [Teardown]    BuiltIn.Run Keywords    OpenStackOperations.Show Debugs    @{NET_1_VM_INSTANCES}    @{NET_2_VM_INSTANCES}
 157     ...    AND    OpenStackOperations.Get Test Teardown Debugs
 158
 159 Check ELAN Datapath Traffic Within The Networks
 160     [Documentation]    Checks datapath within the same network with different vlans.
 161     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV4[0]}    ping -I ${NET_1_VM_IPV4[0]} -c 3 ${NET_1_VM_IPV4[1]}
 162     BuiltIn.Should Contain    ${output}    64 bytes
 163     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV6[0]}    ping6 -I ${NET_1_VM_IPV6[0]} -c 3 ${NET_1_VM_IPV6[1]}
 164     BuiltIn.Should Contain    ${output}    64 bytes
 165     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[1]    ${NET_2_VM_IPV4[0]}    ping -I ${NET_2_VM_IPV4[0]} -c 3 ${NET_2_VM_IPV4[1]}
 166     BuiltIn.Should Contain    ${output}    64 bytes
 167     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[1]    ${NET_2_VM_IPV6[0]}    ping6 -I ${NET_2_VM_IPV6[0]} -c 3 ${NET_2_VM_IPV6[1]}
 168     BuiltIn.Should Contain    ${output}    64 bytes
 169
 170 Check L3_Datapath Traffic Across Networks With Router
 171     [Documentation]    L3 Datapath test across networks using previously created router.
 172     BuiltIn.Log    Verification of FIB Entries and Flow
 173     @{tcpdump_conn_ids} =    OpenStackOperations.Start Packet Capture On Nodes    tcpdump_vpn_ds    ${EMPTY}    @{OS_ALL_IPS}
 174     ${vm_instances} =    BuiltIn.Create List    @{NET_1_VM_IPV4}    @{NET_2_VM_IPV4}    @{NET_1_VM_IPV6}    @{NET_2_VM_IPV6}
 175     BuiltIn.Wait Until Keyword Succeeds    30s    5s    Utils.Check For Elements At URI    ${FIB_ENTRY_URL}    ${vm_instances}
 176     : FOR    ${VM}    IN    ${vm_instances}
 177     \    BuiltIn.Wait Until Keyword Succeeds    30s    5s    VpnOperations.Verify Flows Are Present For L3VPN    ${OS_CMP1_IP}    ${VM}
 178     BuiltIn.Wait Until Keyword Succeeds    30s    5s    VpnOperations.Verify GWMAC Entry On ODL    ${GW_MAC_ADDRS}
 179     Verify GWMAC Flow Entry On Flow Table On All Compute Nodes
 180     BuiltIn.Log    L3 Datapath test across the networks using router
 181     ${dst_ipv4_list1} =    BuiltIn.Create List    ${NET_1_VM_IPV4[1]}    @{NET_2_VM_IPV4}
 182     Test Operations From Vm Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV4[0]}    ${dst_ipv4_list1}
 183     ${dst_ipv4_list2} =    BuiltIn.Create List    ${NET_2_VM_IPV4[1]}    @{NET_1_VM_IPV4}
 184     Test Operations From Vm Instance    @{NETWORKS}[1]    ${NET_2_VM_IPV4[0]}    ${dst_ipv4_list2}
 185     ${dst_ipv6_list1} =    BuiltIn.Create List    ${NET_1_VM_IPV6[1]}    @{NET_2_VM_IPV6}
 186     Test Operations From Vm Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV6[0]}    ${dst_ipv6_list1}
 187     ${dst_ipv6_list2} =    BuiltIn.Create List    ${NET_2_VM_IPV6[1]}    @{NET_1_VM_IPV6}
 188     Test Operations From Vm Instance    @{NETWORKS}[1]    ${NET_2_VM_IPV6[0]}    ${dst_ipv6_list2}
 189     [Teardown]    VpnOperations.Test Teardown With Tcpdump Stop    ${tcpdump_conn_ids}
 190
 191 Add Multiple Extra Routes And Check Datapath Before L3VPN Creation
 192     [Documentation]    Add multiple extra routes and check data path before L3VPN creation.
 193     BuiltIn.Log    Add extraroutes to VM
 194     : FOR    ${extra_ip}    IN    @{EXTRA_NW_IPV4}
 195     \    ${cmd} =    BuiltIn.Catenate    sudo ip addr add ${extra_ip}/24 dev eth0
 196     \    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    @{NET_1_VM_IPV4}[0]    ${cmd}
 197     \    ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    @{NET_1_VM_IPV4}[0]    ip a
 198     \    BuiltIn.Should Contain    ${output}    ${extra_ip}/24
 199     ${ext_rt1} =    BuiltIn.Set Variable    destination=@{EXTRA_NW_SUBNET_IPV4}[0],gateway=@{NET_1_VM_IPV4}[0]
 200     ${ext_rt2} =    BuiltIn.Set Variable    destination=@{EXTRA_NW_SUBNET_IPV4}[1],gateway=@{NET_1_VM_IPV4}[0]
 201     ${cmd} =    BuiltIn.Catenate    ${RT_OPTIONS}    ${ext_rt1}    ${RT_OPTIONS}    ${ext_rt2}
 202     OpenStackOperations.Update Router    ${ROUTER}    ${cmd}
 203     OpenStackOperations.Show Router    ${ROUTER}    -D
 204     : FOR    ${extra_ip}    IN    @{EXTRA_NW_IPV6}
 205     \    ${cmd} =    BuiltIn.Catenate    sudo ip -6 addr add ${extra_ip}/64 dev eth0
 206     \    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    @{NET_1_VM_IPV6}[0]    ${cmd}
 207     \    ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    @{NET_1_VM_IPV6}[0]    ip -6 a
 208     \    BuiltIn.Should Contain    ${output}    ${extra_ip}/64
 209     ${ext_rt3} =    BuiltIn.Set Variable    destination=@{EXTRA_NW_SUBNET_IPV6}[0],gateway=@{NET_1_VM_IPV6}[0]
 210     ${ext_rt4} =    BuiltIn.Set Variable    destination=@{EXTRA_NW_SUBNET_IPV6}[1],gateway=@{NET_1_VM_IPV6}[0]
 211     ${cmd} =    BuiltIn.Catenate    ${RT_OPTIONS}    ${ext_rt3}    ${RT_OPTIONS}    ${ext_rt4}
 212     OpenStackOperations.Update Router    ${ROUTER}    ${cmd}
 213     OpenStackOperations.Show Router    ${ROUTER}    -D
 214     BuiltIn.Log    Verify FIB table
 215     ${vm_ips} =    BuiltIn.Create List    @{EXTRA_NW_SUBNET_IPV4}    @{EXTRA_NW_SUBNET_IPV6}
 216     BuiltIn.Wait Until Keyword Succeeds    30s    5s    Utils.Check For Elements At URI    ${FIB_ENTRY_URL}    ${vm_ips}
 217     : FOR    ${extra_ip}    IN    @{EXTRA_NW_IPV4}
 218     \    ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV4[1]}    ping -c 3 ${extra_ip}
 219     \    BuiltIn.Should Contain    ${output}    64 bytes
 220     \    ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[1]    ${NET_2_VM_IPV4[0]}    ping -c 3 ${extra_ip}
 221     \    BuiltIn.Should Contain    ${output}    64 bytes
 222     \    ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[1]    ${NET_2_VM_IPV4[1]}    ping -c 3 ${extra_ip}
 223     \    BuiltIn.Should Contain    ${output}    64 bytes
 224     : FOR    ${extra_ip}    IN    @{EXTRA_NW_IPV6}
 225     \    ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV6[1]}    ping6 -c 3 ${extra_ip}
 226     \    BuiltIn.Should Contain    ${output}    64 bytes
 227     \    ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[1]    ${NET_2_VM_IPV6[0]}    ping6 -c 3 ${extra_ip}
 228     \    BuiltIn.Should Contain    ${output}    64 bytes
 229     \    ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[1]    ${NET_2_VM_IPV6[1]}    ping6 -c 3 ${extra_ip}
 230     \    BuiltIn.Should Contain    ${output}    64 bytes
 231
 232 Delete And Recreate Extra Route
 233     [Documentation]    Delete IPv4 and IPv6 extra routes and recreate it.
 234     ...    Then check data path before L3VPN creation.
 235     BuiltIn.Log    Delete all extra routes
 236     OpenStackOperations.Update Router    ${ROUTER}    ${RT_CLEAR}
 237     OpenStackOperations.Show Router    ${ROUTER}    -D
 238     ${cmd}=    BuiltIn.Catenate    sudo ip addr add @{EXTRA_NW_IPV4}[0]/24 dev eth0
 239     OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    @{NET_1_VM_IPV4}[0]    ${cmd}
 240     ${ext_rt_ipv4} =    BuiltIn.Set Variable    destination=@{EXTRA_NW_SUBNET_IPV4}[0],gateway=@{NET_1_VM_IPV4}[0]
 241     ${cmd} =    BuiltIn.Catenate    ${RT_OPTIONS}    ${ext_rt_ipv4}
 242     OpenStackOperations.Update Router    ${ROUTER}    ${cmd}
 243     OpenStackOperations.Show Router    ${ROUTER}    -D
 244     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    @{NET_1_VM_IPV4}[1]    ping -c 3 @{EXTRA_NW_IPV4}[0]
 245     BuiltIn.Should Contain    ${output}    64 bytes
 246     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[1]    @{NET_2_VM_IPV4}[0]    ping -c 3 @{EXTRA_NW_IPV4}[0]
 247     BuiltIn.Should Contain    ${output}    64 bytes
 248     ${cmd} =    BuiltIn.Catenate    sudo ip -6 addr add @{EXTRA_NW_IPV6}[0]/64 dev eth0
 249     OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    @{NET_1_VM_IPV6}[0]    ${cmd}
 250     ${ext_rt2} =    BuiltIn.Set Variable    destination=@{EXTRA_NW_SUBNET_IPV6}[0],gateway=@{NET_1_VM_IPV6}[0]
 251     ${cmd} =    BuiltIn.Catenate    ${RT_OPTIONS}    ${ext_rt2}
 252     OpenStackOperations.Update Router    ${ROUTER}    ${cmd}
 253     OpenStackOperations.Show Router    ${ROUTER}    -D
 254     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    @{NET_1_VM_IPV6}[1]    ping6 -c 3 @{EXTRA_NW_IPV6}[0]
 255     BuiltIn.Should Contain    ${output}    64 bytes
 256     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[1]    @{NET_2_VM_IPV6}[0]    ping6 -c 3 @{EXTRA_NW_IPV6}[0]
 257     BuiltIn.Should Contain    ${output}    64 bytes
 258     # clear off extra-routes before the next set of tests
 259     OpenStackOperations.Update Router    ${ROUTER}    ${RT_CLEAR}
 260     [Teardown]    BuiltIn.Run Keywords    OpenStackOperations.Update Router    ${ROUTER}    ${RT_CLEAR}
 261     ...    AND    OpenStackOperations.Show Router    ${ROUTER}    -D
 262     ...    AND    OpenStackOperations.Get Test Teardown Debugs
 263
 264 Create L3VPN
 265     ${net_id} =    OpenStackOperations.Get Net Id    @{NETWORKS}[0]
 266     ${tenant_id} =    OpenStackOperations.Get Tenant ID From Network    ${net_id}
 267     VpnOperations.VPN Create L3VPN    vpnid=${VPN_INSTANCE_ID[0]}    name=${VPN_NAME[0]}    rd=@{RDS}[0]    exportrt=@{RDS}[0]    importrt=@{RDS}[0]    tenantid=${tenant_id}
 268     ${resp} =    VpnOperations.VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 269     BuiltIn.Should Contain    ${resp}    ${VPN_INSTANCE_ID[0]}
 270
 271 Associate L3VPN To Routers
 272     [Documentation]    Associating router to L3VPN.
 273     ${router_id} =    OpenStackOperations.Get Router Id    ${ROUTER}
 274     VpnOperations.Associate VPN to Router    routerid=${router_id}    vpnid=${VPN_INSTANCE_ID[0]}
 275     ${resp} =    VpnOperations.VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 276     BuiltIn.Should Contain    ${resp}    ${router_id}
 277
 278 Verify L3VPN Datapath With Router Association
 279     [Documentation]    Datapath test across networks using L3VPN associated with router.
 280     BuiltIn.Log    Verify VPN interfaces, FIB entries and Flow table
 281     BuiltIn.Wait Until Keyword Succeeds    30s    5s    Utils.Check For Elements At URI    ${VPN_IFACES_URL}    ${VM_IPS}
 282     ${RD} =    Strip String    ${RDS[0]}    characters="[]
 283     BuiltIn.Wait Until Keyword Succeeds    60s    15s    Utils.Check For Elements At URI    ${CONFIG_API}/odl-fib:fibEntries/vrfTables/${RD}/    ${VM_IPS}
 284     Verify Flows Are Present For L3VPN On All Compute Nodes    ${VM_IPS}
 285     BuiltIn.Wait Until Keyword Succeeds    30s    5s    VpnOperations.Verify GWMAC Entry On ODL    ${GW_MAC_ADDRS}
 286     Verify GWMAC Flow Entry On Flow Table On All Compute Nodes
 287     BuiltIn.Log    L3 Datapath test across the networks using L3VPN
 288     ${dst_ipv4_list1} =    BuiltIn.Create List    ${NET_1_VM_IPV4[1]}    @{NET_2_VM_IPV4}
 289     Test Operations From Vm Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV4[0]}    ${dst_ipv4_list1}
 290     ${dst_ipv4_list2} =    BuiltIn.Create List    ${NET_2_VM_IPV4[1]}    @{NET_1_VM_IPV4}
 291     Test Operations From Vm Instance    @{NETWORKS}[1]    ${NET_2_VM_IPV4[0]}    ${dst_ipv4_list2}
 292     ${dst_ipv6_list1} =    BuiltIn.Create List    ${NET_1_VM_IPV6[1]}    @{NET_2_VM_IPV6}
 293     Test Operations From Vm Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV6[0]}    ${dst_ipv6_list1}
 294     ${dst_ipv6_list2} =    BuiltIn.Create List    ${NET_2_VM_IPV6[1]}    @{NET_1_VM_IPV6}
 295     Test Operations From Vm Instance    @{NETWORKS}[1]    ${NET_2_VM_IPV6[0]}    ${dst_ipv6_list2}
 296
 297 Delete IPv6 Subnet And Check IPv4 datapath
 298     [Documentation]    Delete IPv6 subnet from router and check IPv4 datapath before L3VPN creation.
 299     ...    Then recreate IPv6 subnet.
 300     BuiltIn.Log    Delete extra routes
 301     OpenStackOperations.Update Router    ${ROUTER}    ${RT_CLEAR}
 302     OpenStackOperations.Show Router    ${ROUTER}    -D
 303     BuiltIn.Log    Delete IPv6 subnet
 304     : FOR    ${PORT}    IN    @{SUBNETS6}
 305     \    Remove Interface    ${ROUTER}    ${PORT}
 306     BuiltIn.Log    Test L2 datapath
 307     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV4[0]}    ping -I ${NET_1_VM_IPV4[0]} -c 3 ${NET_1_VM_IPV4[1]}
 308     BuiltIn.Should Contain    ${output}    64 bytes
 309     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[1]    ${NET_2_VM_IPV4[0]}    ping -I ${NET_2_VM_IPV4[0]} -c 3 ${NET_2_VM_IPV4[1]}
 310     BuiltIn.Should Contain    ${output}    64 bytes
 311     BuiltIn.Log    Test L3 datapath
 312     ${dst_ipv4_list1} =    BuiltIn.Create List    ${NET_1_VM_IPV4[1]}    @{NET_2_VM_IPV4}
 313     Test Operations From Vm Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV4[0]}    ${dst_ipv4_list1}
 314     ${dst_ipv4_list2} =    BuiltIn.Create List    ${NET_2_VM_IPV4[1]}    @{NET_1_VM_IPV4}
 315     Test Operations From Vm Instance    @{NETWORKS}[1]    ${NET_2_VM_IPV4[0]}    ${dst_ipv4_list2}
 316
 317 Dissociate L3VPN From Routers
 318     [Documentation]    Dissociating router from L3VPN.
 319     ${router_id} =    OpenStackOperations.Get Router Id    ${ROUTER}
 320     VpnOperations.Dissociate VPN to Router    routerid=${router_id}    vpnid=${VPN_INSTANCE_ID[0]}
 321     ${resp} =    VpnOperations.VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 322     BuiltIn.Should Not Contain    ${resp}    ${router_id}
 323
 324 Delete Router And Router Interfaces With L3VPN
 325     [Documentation]    Delete associate with L3VPN Router and its Ports.
 326     # Asscoiate router with L3VPN
 327     ${router_id} =    OpenStackOperations.Get Router Id    ${ROUTER}
 328     VpnOperations.Associate VPN to Router    routerid=${router_id}    vpnid=${VPN_INSTANCE_ID[0]}
 329     ${resp} =    VpnOperations.VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 330     BuiltIn.Should Contain    ${resp}    ${router_id}
 331     # Delete Interface
 332     : FOR    ${iface}    IN    @{SUBNETS4}
 333     \    Remove Interface    ${ROUTER}    ${iface}
 334     \    ${subnet_id} =    Get Subnet Id    ${iface}
 335     \    ${rt_port_list} =    OpenStackOperations.Show Router Interface    ${ROUTER}
 336     \    BuiltIn.Should Not Contain    ${rt_port_list}    ${subnet_id}
 337     # Delete Router and Interface to the subnets.
 338     Delete Router    ${ROUTER}
 339     ${router_output} =    OpenStackOperations.List Routers
 340     BuiltIn.Should Not Contain    ${router_output}    ${ROUTER}
 341     ${router_list} =    BuiltIn.Create List    ${ROUTER}
 342     BuiltIn.Wait Until Keyword Succeeds    3s    1s    Utils.Check For Elements Not At URI    ${ROUTER_URL}    ${router_list}
 343     # Verify Router Entry removed from L3VPN
 344     ${resp} =    VpnOperations.VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 345     BuiltIn.Should Not Contain    ${resp}    ${router_id}
 346     Verify GWMAC Flow Entry Removed From Flow Table On All Compute Nodes
 347
 348 Delete Router With NonExistentRouter Name
 349     [Documentation]    Delete router with nonExistentRouter name.
 350     ${rc}    ${output} =    Run And Return Rc And Output    openstack router delete nonExistentRouter
 351     BuiltIn.Should Match Regexp    ${output}    Failed to delete router with name or ID 'nonExistentRouter'|Failed to delete router\\(s\\) with name or ID\\(s\\) 'nonExistentRouter'
 352
 353 Associate L3VPN To Networks
 354     [Documentation]    Associates L3VPN to networks.
 355     ${network1_id} =    OpenStackOperations.Get Net Id    @{NETWORKS}[0]
 356     ${network2_id} =    OpenStackOperations.Get Net Id    @{NETWORKS}[1]
 357     VpnOperations.Associate L3VPN To Network    networkid=${network1_id}    vpnid=${VPN_INSTANCE_ID[0]}
 358     ${resp} =    VpnOperations.VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 359     BuiltIn.Should Contain    ${resp}    ${network1_id}
 360     VpnOperations.Associate L3VPN To Network    networkid=${network2_id}    vpnid=${VPN_INSTANCE_ID[0]}
 361     ${resp} =    VpnOperations.VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 362     BuiltIn.Should Contain    ${resp}    ${network2_id}
 363
 364 Dissociate L3VPN From Networks
 365     [Documentation]    Dissociate L3VPN from networks.
 366     ${network1_id} =    OpenStackOperations.Get Net Id    @{NETWORKS}[0]
 367     ${network2_id} =    OpenStackOperations.Get Net Id    @{NETWORKS}[1]
 368     VpnOperations.Dissociate L3VPN From Networks    networkid=${network1_id}    vpnid=${VPN_INSTANCE_ID[0]}
 369     ${resp} =    VpnOperations.VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 370     BuiltIn.Should Not Contain    ${resp}    ${network1_id}
 371     VpnOperations.Dissociate L3VPN From Networks    networkid=${network2_id}    vpnid=${VPN_INSTANCE_ID[0]}
 372     ${resp} =    VpnOperations.VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 373     BuiltIn.Should Not Contain    ${resp}    ${network2_id}
 374
 375 Delete L3VPN
 376     [Documentation]    Delete L3VPN.
 377     VpnOperations.VPN Delete L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 378
 379 Create Multiple L3VPN
 380     [Documentation]    Creates three L3VPNs and then check the same.
 381     ${net_id} =    OpenStackOperations.Get Net Id    @{NETWORKS}[0]
 382     ${tenant_id} =    OpenStackOperations.Get Tenant ID From Network    ${net_id}
 383     VpnOperations.VPN Create L3VPN    vpnid=${VPN_INSTANCE_ID[0]}    name=${VPN_NAME[0]}    rd=${RDS[0]}    exportrt=${RDS[0]}    importrt=${RDS[0]}    tenantid=${tenant_id}
 384     VpnOperations.VPN Create L3VPN    vpnid=${VPN_INSTANCE_ID[1]}    name=${VPN_NAME[1]}    rd=${RDS[1]}    exportrt=${RDS[1]}    importrt=${RDS[1]}    tenantid=${tenant_id}
 385     VpnOperations.VPN Create L3VPN    vpnid=${VPN_INSTANCE_ID[2]}    name=${VPN_NAME[2]}    rd=${RDS[2]}    exportrt=${RDS[2]}    importrt=${RDS[2]}    tenantid=${tenant_id}
 386     ${resp} =    VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 387     BuiltIn.Should Contain    ${resp}    ${VPN_INSTANCE_ID[0]}
 388     ${resp} =    VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[1]}
 389     BuiltIn.Should Contain    ${resp}    ${VPN_INSTANCE_ID[1]}
 390     ${resp} =    VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[2]}
 391     BuiltIn.Should Contain    ${resp}    ${VPN_INSTANCE_ID[2]}
 392
 393 *** Keywords ***
 394 Verify GWMAC Flow Entry On Flow Table
 395     [Arguments]    ${cnIp}
 396     [Documentation]    Verify GWMAC Table, ARP Response table and Dispatcher table.
 397     ${flow_output} =    Run Command On Remote System    ${cnIp}    sudo ovs-ofctl -O OpenFlow13 dump-flows ${INTEGRATION_BRIDGE}
 398     ${group_output} =    Run Command On Remote System    ${cnIp}    sudo ovs-ofctl -O OpenFlow13 dump-groups ${INTEGRATION_BRIDGE}
 399     BuiltIn.Should Contain    ${flow_output}    table=${DISPATCHER_TABLE}
 400     ${dispatcher_table} =    Get Lines Containing String    ${flow_output}    table=${DISPATCHER_TABLE}
 401     BuiltIn.Should Contain    ${dispatcher_table}    goto_table:${GWMAC_TABLE}
 402     BuiltIn.Should Not Contain    ${dispatcher_table}    goto_table:${ARP_RESPONSE_TABLE}
 403     BuiltIn.Should Contain    ${flow_output}    table=${GWMAC_TABLE}
 404     ${gwmac_table} =    Get Lines Containing String    ${flow_output}    table=${GWMAC_TABLE}
 405     # Verify GWMAC address present in table ${L3_TABLE}
 406     : FOR    ${macAdd}    IN    @{GW_MAC_ADDRS}
 407     \    BuiltIn.Should Contain    ${gwmac_table}    dl_dst=${macAdd} actions=goto_table:${L3_TABLE}
 408     # Verify Miss entry
 409     BuiltIn.Should Contain    ${gwmac_table}    actions=resubmit(,17)
 410     # Verify ARP_CHECK_TABLE - ${ARP_CHECK_TABLE}
 411     ${arpchk_table} =    Get Lines Containing String    ${flow_output}    table=${ARP_CHECK_TABLE}
 412     BuiltIn.Should Match Regexp    ${arpchk_table}    ${ARP_RESPONSE_REGEX}
 413     ${match} =    BuiltIn.Should Match Regexp    ${arpchk_table}    ${ARP_REQUEST_REGEX}
 414     ${groupID} =    Split String    ${match}    separator=:
 415     BuiltIn.Should Contain    ${flow_output}    table=${IPV6_TABLE}
 416     ${icmp_ipv6_flows} =    Get Lines Containing String    ${flow_output}    icmp_type=${ICMP_TYPE}
 417     # Verify IPv6 icmp_type=135
 418     : FOR    ${ip_addr}    IN    @{GW_IPV6_ADDRS}
 419     \    ${rule} =    BuiltIn.Set Variable    icmp_type=${ICMP_TYPE},icmp_code=0,nd_target=${ip_addr} actions=CONTROLLER:65535
 420     \    BuiltIn.Should Match Regexp    ${icmp_ipv6_flows}    ${rule}
 421     VpnOperations.Verify ARP REQUEST in groupTable    ${group_output}    ${groupID[1]}
 422     # Verify ARP_RESPONSE_TABLE - ${ARP_RESPONSE_TABLE}
 423     BuiltIn.Should Contain    ${flow_output}    table=${ARP_RESPONSE_TABLE}
 424     ${arpResponder_table} =    Get Lines Containing String    ${flow_output}    table=${ARP_RESPONSE_TABLE}
 425     BuiltIn.Should Contain    ${arpResponder_table}    priority=0 actions=drop
 426     : FOR    ${macAdd}    ${ipAdd}    IN ZIP    ${GW1_MAC_ADDRS}    ${GW_IPV4_ADDRS}
 427     \    ${ARP_RESPONSE_IP_MAC_REGEX} =    BuiltIn.Set Variable    arp_tpa=${ipAdd},arp_op=1 actions=.*,set_field:${macAdd}->eth_src
 428     \    BuiltIn.Should Match Regexp    ${arpResponder_table}    ${ARP_RESPONSE_IP_MAC_REGEX}