csit/suites/netvirt/vpnservice/vpn_basic_dualstack_one_router.robot

   1 *** Settings ***
   2 Documentation     Test suite to validate dualstack (IPv4 + IPv6) vpnservice functionality in an Openstack
   3 ...               integrated environment.
   4 ...               The assumption of this suite is that the environment is already configured with the proper
   5 ...               integration bridges and vxlan tunnels.
   6 Suite Setup       VpnOperations.Basic Suite Setup
   7 Suite Teardown    VpnOperations.Basic Vpnservice Suite Cleanup
   8 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
   9 Test Teardown     OpenStackOperations.Get Test Teardown Debugs
  10 Library           Collections
  11 Library           OperatingSystem
  12 Library           RequestsLibrary
  13 Resource          ../../../libraries/Utils.robot
  14 Resource          ../../../libraries/OpenStackOperations.robot
  15 Resource          ../../../libraries/DevstackUtils.robot
  16 Resource          ../../../libraries/VpnOperations.robot
  17 Resource          ../../../libraries/OVSDB.robot
  18 Resource          ../../../libraries/SetupUtils.robot
  19 Resource          ../../../libraries/Tcpdump.robot
  20 Resource          ../../../variables/Variables.robot
  21 Resource          ../../../variables/netvirt/Variables.robot
  22
  23 *** Variables ***
  24 ${SECURITY_GROUP}    vpn_sg_dualstack
  25 @{NETWORKS}       vpn_net_1_dualstack    vpn_net_2_dualstack
  26 @{SUBNETS4}       vpn_net_ipv4_1_dualstack    vpn_net_ipv4_2_dualstack
  27 @{SUBNETS6}       vpn_net_ipv6_1_dualstack    vpn_net_ipv6_2_dualstack
  28 @{SUBNETS4_CIDR}    30.1.1.0/24    40.1.1.0/24
  29 @{SUBNETS6_CIDR}    2001:db5:0:2::/64    2001:db5:0:3::/64
  30 ${SUBNET_ADDITIONAL_ARGS}    --ip-version=6 --ipv6-address-mode=slaac --ipv6-ra-mode=slaac
  31 @{PORTS}          vpn_port_1_dualstack    vpn_port_2_dualstack    vpn_port_3_dualstack    vpn_port_4_dualstack
  32 @{NET_1_VM_INSTANCES}    vpn_net_1_vm_1_dualstack    vpn_net_1_vm_2_dualstack
  33 @{NET_2_VM_INSTANCES}    vpn_net_2_vm_1_dualstack    vpn_net_2_vm_2_dualstack
  34 @{EXTRA_NW_IPV4}    76.1.1.2    77.1.1.2
  35 @{EXTRA_NW_IPV6}    3001:db9:cafe:d::10    3001:db9:abcd:d::20
  36 @{EXTRA_NW_SUBNET}    76.1.1.0/24    77.1.1.0/24    3001:db9:cafe:d::/64    3001:db9:abcd:d::/64
  37 ${ROUTER}         vpn_router_dualstack
  38 ${UPDATE_NETWORK}    UpdateNetwork_dualstack
  39 ${UPDATE_SUBNET}    UpdateSubnet_dualstack
  40 ${UPDATE_PORT}    UpdatePort_dualstack
  41 @{VPN_INSTANCE_ID}    1bc8cd92-48ca-49b5-94e1-b2921a261661    1bc8cd92-48ca-49b5-94e1-b2921a261662    1bc8cd92-48ca-49b5-94e1-b2921a261663
  42 @{VPN_NAME}       vpn1_dualstack    vpn2_dualstack    vpn3_dualstack
  43 @{RDS}            ["2506:2"]    ["2606:2"]    ["2706:2"]
  44
  45 *** Test Cases ***
  46 Create Neutron Networks
  47     [Documentation]    Create two networks.
  48     OpenStackOperations.Create Network    @{NETWORKS}[0]
  49     OpenStackOperations.Create Network    @{NETWORKS}[1]
  50     ${NET_LIST} =    OpenStackOperations.List Networks
  51     BuiltIn.Wait Until Keyword Succeeds    3s    1s    Utils.Check For Elements At URI    ${NETWORK_URL}    ${NETWORKS}
  52     OpenStackOperations.Update Network    @{NETWORKS}[0]    additional_args=--description ${UPDATE_NETWORK}
  53     ${output} =    OpenStackOperations.Show Network    @{NETWORKS}[0]
  54     BuiltIn.Should Contain    ${output}    ${UPDATE_NETWORK}
  55
  56 Create Neutron Subnets
  57     [Documentation]    Create subnets for previously created networks.
  58     OpenStackOperations.Create SubNet    @{NETWORKS}[0]    @{SUBNETS4}[0]    @{SUBNETS4_CIDR}[0]
  59     OpenStackOperations.Create SubNet    @{NETWORKS}[0]    @{SUBNETS6}[0]    @{SUBNETS6_CIDR}[0]    ${SUBNET_ADDITIONAL_ARGS}
  60     OpenStackOperations.Create SubNet    @{NETWORKS}[1]    @{SUBNETS4}[1]    @{SUBNETS4_CIDR}[1]
  61     OpenStackOperations.Create SubNet    @{NETWORKS}[1]    @{SUBNETS6}[1]    @{SUBNETS6_CIDR}[1]    ${SUBNET_ADDITIONAL_ARGS}
  62     ${SUB_LIST} =    OpenStackOperations.List Subnets
  63     BuiltIn.Wait Until Keyword Succeeds    3s    1s    Utils.Check For Elements At URI    ${SUBNETWORK_URL}    ${SUBNETS4}
  64     BuiltIn.Wait Until Keyword Succeeds    3s    1s    Utils.Check For Elements At URI    ${SUBNETWORK_URL}    ${SUBNETS6}
  65     OpenStackOperations.Update SubNet    @{SUBNETS4}[0]    additional_args=--description ${UPDATE_SUBNET}
  66     ${output} =    OpenStackOperations.Show SubNet    @{SUBNETS4}[0]
  67     BuiltIn.Should Contain    ${output}    ${UPDATE_SUBNET}
  68
  69 Create Router
  70     [Documentation]    Create Router.
  71     OpenStackOperations.Create Router    ${ROUTER}
  72     @{ROUTER_LIST} =    BuiltIn.Create List    ${ROUTER}
  73     BuiltIn.Wait Until Keyword Succeeds    3s    1s    Utils.Check For Elements At URI    ${ROUTER_URL}    ${ROUTER_LIST}
  74
  75 Add Router Ports
  76     [Documentation]    Add created subnets to router.
  77     : FOR    ${PORT}    IN    @{SUBNETS4}
  78     \    OpenStackOperations.Add Router Interface    ${ROUTER}    ${PORT}
  79     ${interface_output} =    OpenStackOperations.Show Router Interface    ${ROUTER}
  80     ${GW1_MAC_ADDRS}    ${GW_IPV4_ADDRS} =    VpnOperations.Get Gateway MAC And IP Address    ${ROUTER}
  81     : FOR    ${PORT}    IN    @{SUBNETS6}
  82     \    OpenStackOperations.Add Router Interface    ${ROUTER}    ${PORT}
  83     ${interface_output} =    OpenStackOperations.Show Router Interface    ${ROUTER}
  84     ${GW2_MAC_ADDRS}    ${GW_IPV6_ADDRS} =    VpnOperations.Get Gateway MAC And IP Address    ${ROUTER}    ${IP6_REGEX}
  85     ${GW_MAC_ADDRS} =    BuiltIn.Create List    @{GW1_MAC_ADDRS}    @{GW2_MAC_ADDRS}
  86     BuiltIn.Set Suite Variable    ${GW1_MAC_ADDRS}
  87     BuiltIn.Set Suite Variable    ${GW2_MAC_ADDRS}
  88     BuiltIn.Set Suite Variable    ${GW_MAC_ADDRS}
  89     BuiltIn.Set Suite Variable    ${GW_IPV4_ADDRS}
  90     BuiltIn.Set Suite Variable    ${GW_IPV6_ADDRS}
  91
  92 Create Allow All Security Group IPv4+IPv6
  93     [Documentation]    Create neutron security group with Allow All rule set for IPv4 ethertype.
  94     ...    Then add in this group Allow All rule set for IPv6 ethertype.
  95     OpenStackOperations.Create Allow All SecurityGroup    ${SECURITY_GROUP}    IPv4
  96     OpenStackOperations.Neutron Security Group Rule Create    ${SECURITY_GROUP}    direction=ingress    ethertype=IPv6    port_range_max=65535    port_range_min=1    protocol=tcp
  97     OpenStackOperations.Neutron Security Group Rule Create    ${SECURITY_GROUP}    direction=egress    ethertype=IPv6    port_range_max=65535    port_range_min=1    protocol=tcp
  98     OpenStackOperations.Neutron Security Group Rule Create    ${SECURITY_GROUP}    direction=ingress    ethertype=IPv6    protocol=icmp
  99     OpenStackOperations.Neutron Security Group Rule Create    ${SECURITY_GROUP}    direction=egress    ethertype=IPv6    protocol=icmp
 100     OpenStackOperations.Neutron Security Group Rule Create    ${SECURITY_GROUP}    direction=ingress    ethertype=IPv6    port_range_max=65535    port_range_min=1    protocol=udp
 101     OpenStackOperations.Neutron Security Group Rule Create    ${SECURITY_GROUP}    direction=egress    ethertype=IPv6    port_range_max=65535    port_range_min=1    protocol=udp
 102
 103 Create Neutron Ports
 104     [Documentation]    Create 2 ports in previously created IPv4 subnets and 2 ports in previously created IPv6 subnets.
 105     ${allowed_address_pairs_args} =    BuiltIn.Set Variable    --allowed-address ip-address=${EXTRA_NW_SUBNET[0]} --allowed-address ip-address=${EXTRA_NW_SUBNET[1]} --allowed-address ip-address=${EXTRA_NW_SUBNET[2]} --allowed-address ip-address=${EXTRA_NW_SUBNET[3]}
 106     OpenStackOperations.Create Port    @{NETWORKS}[0]    @{PORTS}[0]    sg=${SECURITY_GROUP}    additional_args=${allowed_address_pairs_args}
 107     OpenStackOperations.Create Port    @{NETWORKS}[0]    @{PORTS}[1]    sg=${SECURITY_GROUP}    additional_args=${allowed_address_pairs_args}
 108     OpenStackOperations.Create Port    @{NETWORKS}[1]    @{PORTS}[2]    sg=${SECURITY_GROUP}    additional_args=${allowed_address_pairs_args}
 109     OpenStackOperations.Create Port    @{NETWORKS}[1]    @{PORTS}[3]    sg=${SECURITY_GROUP}    additional_args=${allowed_address_pairs_args}
 110     BuiltIn.Wait Until Keyword Succeeds    3s    1s    Utils.Check For Elements At URI    ${PORT_URL}    ${PORTS}
 111     ${PORTS_MACADDR} =    OpenStackOperations.Get Ports MacAddr    ${PORTS}
 112     BuiltIn.Set Suite Variable    ${PORTS_MACADDR}
 113     OpenStackOperations.Update Port    @{PORTS}[0]    additional_args=--name ${UPDATE_PORT}
 114     ${output} =    Show Port    ${UPDATE_PORT}
 115     BuiltIn.Should Contain    ${output}    ${UPDATE_PORT}
 116     OpenStackOperations.Update Port    ${UPDATE_PORT}    additional_args=--name @{PORTS}[0]
 117
 118 Create Nova VMs
 119     [Documentation]    Launch a VM for each previously created port.
 120     OpenStackOperations.Create Vm Instance With Port On Compute Node    @{PORTS}[0]    ${NET_1_VM_INSTANCES[0]}    ${OS_CMP1_HOSTNAME}    sg=${SECURITY_GROUP}
 121     OpenStackOperations.Create Vm Instance With Port On Compute Node    @{PORTS}[1]    ${NET_1_VM_INSTANCES[1]}    ${OS_CMP2_HOSTNAME}    sg=${SECURITY_GROUP}
 122     OpenStackOperations.Create Vm Instance With Port On Compute Node    @{PORTS}[2]    ${NET_2_VM_INSTANCES[0]}    ${OS_CMP1_HOSTNAME}    sg=${SECURITY_GROUP}
 123     OpenStackOperations.Create Vm Instance With Port On Compute Node    @{PORTS}[3]    ${NET_2_VM_INSTANCES[1]}    ${OS_CMP2_HOSTNAME}    sg=${SECURITY_GROUP}
 124     ${VM_INSTANCES} =    BuiltIn.Create List    @{NET_1_VM_INSTANCES}    @{NET_2_VM_INSTANCES}
 125     BuiltIn.Set Suite Variable    ${VM_INSTANCES}
 126     BuiltIn.Wait Until Keyword Succeeds    30s    10s    OpenStackOperations.Wait For Routes To Propogate    ${NETWORKS}    ${SUBNETS4_CIDR}
 127     BuiltIn.Wait Until Keyword Succeeds    30s    10s    OpenStackOperations.Wait For Routes To Propogate    ${NETWORKS}    ${SUBNETS6_CIDR}
 128     @{NET_1_VM_IPV4}    ${NET_1_DHCP_IPV4} =    OpenStackOperations.Get VM IPs    @{NET_1_VM_INSTANCES}
 129     @{NET_2_VM_IPV4}    ${NET_2_DHCP_IPV4} =    OpenStackOperations.Get VM IPs    @{NET_2_VM_INSTANCES}
 130     BuiltIn.Should Not Contain    ${NET_1_VM_IPV4}    None
 131     BuiltIn.Should Not Contain    ${NET_2_VM_IPV4}    None
 132     BuiltIn.Should Not Contain    ${NET_1_DHCP_IPV4}    None
 133     BuiltIn.Should Not Contain    ${NET_2_DHCP_IPV4}    None
 134     BuiltIn.Log    Collect VMs IPv6 addresses
 135     ${prefix_net10} =    String.Replace String    @{SUBNETS6_CIDR}[0]    ${IP6_SUBNET_CIDR_SUFFIX}    ${IP6_ADDR_SUFFIX}
 136     ${prefix_net20} =    String.Replace String    @{SUBNETS6_CIDR}[1]    ${IP6_SUBNET_CIDR_SUFFIX}    ${IP6_ADDR_SUFFIX}
 137     ${status}    ${message}    Run Keyword And Ignore Error    BuiltIn.Wait Until Keyword Succeeds    3x    60s    OpenStackOperations.Collect VM IPv6 SLAAC Addresses
 138     ...    true    ${NET_1_VM_INSTANCES}    @{NETWORKS}[0]    ${prefix_net10}
 139     ${status}    ${message}    Run Keyword And Ignore Error    BuiltIn.Wait Until Keyword Succeeds    3x    60s    OpenStackOperations.Collect VM IPv6 SLAAC Addresses
 140     ...    true    ${NET_2_VM_INSTANCES}    @{NETWORKS}[1]    ${prefix_net20}
 141     ${NET_1_VM_IPV6} =    OpenStackOperations.Collect VM IPv6 SLAAC Addresses    false    ${NET_1_VM_INSTANCES}    @{NETWORKS}[0]    ${prefix_net10}
 142     ${NET_2_VM_IPV6} =    OpenStackOperations.Collect VM IPv6 SLAAC Addresses    false    ${NET_2_VM_INSTANCES}    @{NETWORKS}[1]    ${prefix_net20}
 143     ${LOOP_COUNT}    Get Length    ${NET_1_VM_INSTANCES}
 144     : FOR    ${index}    IN RANGE    0    ${LOOP_COUNT}
 145     \    ${status}    ${message}    Run Keyword And Ignore Error    BuiltIn.Should Not Contain    @{NET_1_VM_IPV6}[${index}]    None
 146     \    Run Keyword If    '${status}' == 'FAIL'    Write Commands Until Prompt    nova console-log @{NET_1_VM_INSTANCES}[${index}]    30s
 147     \    ${status}    ${message}    Run Keyword And Ignore Error    BuiltIn.Should Not Contain    @{NET_2_VM_IPV6}[${index}]    None
 148     \    Run Keyword If    '${status}' == 'FAIL'    Write Commands Until Prompt    nova console-log @{NET_2_VM_INSTANCES}[${index}]    30s
 149     BuiltIn.Set Suite Variable    ${NET_1_VM_IPV4}
 150     BuiltIn.Set Suite Variable    ${NET_2_VM_IPV4}
 151     BuiltIn.Set Suite Variable    ${NET_1_VM_IPV6}
 152     BuiltIn.Set Suite Variable    ${NET_2_VM_IPV6}
 153     ${VM_IPS} =    BuiltIn.Create List    @{NET_1_VM_IPV4}    @{NET_2_VM_IPV4}    @{NET_1_VM_IPV6}    @{NET_2_VM_IPV6}
 154     BuiltIn.Set Suite Variable    ${VM_IPS}
 155     [Teardown]    BuiltIn.Run Keywords    OpenStackOperations.Show Debugs    @{NET_1_VM_INSTANCES}    @{NET_2_VM_INSTANCES}
 156     ...    AND    OpenStackOperations.Get Test Teardown Debugs
 157
 158 Check ELAN Datapath Traffic Within The Networks
 159     [Documentation]    Checks datapath within the same network with different vlans.
 160     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV4[0]}    ping -I ${NET_1_VM_IPV4[0]} -c 3 ${NET_1_VM_IPV4[1]}
 161     BuiltIn.Should Contain    ${output}    64 bytes
 162     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV6[0]}    ping6 -I ${NET_1_VM_IPV6[0]} -c 3 ${NET_1_VM_IPV6[1]}
 163     BuiltIn.Should Contain    ${output}    64 bytes
 164     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[1]    ${NET_2_VM_IPV4[0]}    ping -I ${NET_2_VM_IPV4[0]} -c 3 ${NET_2_VM_IPV4[1]}
 165     BuiltIn.Should Contain    ${output}    64 bytes
 166     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[1]    ${NET_2_VM_IPV6[0]}    ping6 -I ${NET_2_VM_IPV6[0]} -c 3 ${NET_2_VM_IPV6[1]}
 167     BuiltIn.Should Contain    ${output}    64 bytes
 168
 169 Check L3_Datapath Traffic Across Networks With Router
 170     [Documentation]    L3 Datapath test across networks using previously created router.
 171     BuiltIn.Log    Verification of FIB Entries and Flow
 172     ${cn1_conn_id} =    Tcpdump.Start Packet Capture on Node    ${OS_COMPUTE_1_IP}    file_Name=tcpDumpCN1
 173     ${cn2_conn_id} =    Tcpdump.Start Packet Capture on Node    ${OS_COMPUTE_2_IP}    file_Name=tcpDumpCN2
 174     ${os_conn_id} =    Tcpdump.Start Packet Capture on Node    ${OS_CONTROL_NODE_IP}    file_Name=tcpDumpOS
 175     ${vm_instances} =    BuiltIn.Create List    @{NET_1_VM_IPV4}    @{NET_2_VM_IPV4}    @{NET_1_VM_IPV6}    @{NET_2_VM_IPV6}
 176     BuiltIn.Wait Until Keyword Succeeds    30s    5s    Utils.Check For Elements At URI    ${FIB_ENTRY_URL}    ${vm_instances}
 177     : FOR    ${VM}    IN    ${vm_instances}
 178     \    BuiltIn.Wait Until Keyword Succeeds    30s    5s    VpnOperations.Verify Flows Are Present For L3VPN    ${OS_COMPUTE_1_IP}    ${VM}
 179     BuiltIn.Wait Until Keyword Succeeds    30s    5s    VpnOperations.Verify GWMAC Entry On ODL    ${GW_MAC_ADDRS}
 180     BuiltIn.Wait Until Keyword Succeeds    30s    5s    Verify GWMAC Flow Entry On Flow Table    ${OS_COMPUTE_1_IP}
 181     BuiltIn.Wait Until Keyword Succeeds    30s    5s    Verify GWMAC Flow Entry On Flow Table    ${OS_COMPUTE_2_IP}
 182     BuiltIn.Log    L3 Datapath test across the networks using router
 183     ${dst_ipv4_list1} =    BuiltIn.Create List    ${NET_1_VM_IPV4[1]}    @{NET_2_VM_IPV4}
 184     Test Operations From Vm Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV4[0]}    ${dst_ipv4_list1}
 185     ${dst_ipv4_list2} =    BuiltIn.Create List    ${NET_2_VM_IPV4[1]}    @{NET_1_VM_IPV4}
 186     Test Operations From Vm Instance    @{NETWORKS}[1]    ${NET_2_VM_IPV4[0]}    ${dst_ipv4_list2}
 187     ${dst_ipv6_list1} =    BuiltIn.Create List    ${NET_1_VM_IPV6[1]}    @{NET_2_VM_IPV6}
 188     Test Operations From Vm Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV6[0]}    ${dst_ipv6_list1}
 189     ${dst_ipv6_list2} =    BuiltIn.Create List    ${NET_2_VM_IPV6[1]}    @{NET_1_VM_IPV6}
 190     Test Operations From Vm Instance    @{NETWORKS}[1]    ${NET_2_VM_IPV6[0]}    ${dst_ipv6_list2}
 191     [Teardown]    VpnOperations.Test Teardown With Tcpdump Stop
 192
 193 Add Multiple Extra Routes And Check Datapath Before L3VPN Creation
 194     [Documentation]    Add multiple extra routes and check data path before L3VPN creation.
 195     BuiltIn.Log    Add extraroutes to VM
 196     : FOR    ${VM}    IN    @{NET_1_VM_IPV4}
 197     \    ${CONFIG_EXTRA_ROUTE_IP1} =    BuiltIn.Catenate    sudo ifconfig eth0:1 @{EXTRA_NW_IPV4}[0] netmask 255.255.255.0 up
 198     \    ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${VM}    ${CONFIG_EXTRA_ROUTE_IP1}
 199     \    ${CONFIG_EXTRA_ROUTE_IP2} =    BuiltIn.Catenate    sudo ifconfig eth0:2 @{EXTRA_NW_IPV4}[1] netmask 255.255.255.0 up
 200     \    ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${VM}    ${CONFIG_EXTRA_ROUTE_IP2}
 201     \    ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${VM}    ip a
 202     ${ext_rt1} =    BuiltIn.Set Variable    destination=${EXTRA_NW_SUBNET[0]},gateway=${NET_1_VM_IPV4[0]}
 203     ${ext_rt2} =    BuiltIn.Set Variable    destination=${EXTRA_NW_SUBNET[1]},gateway=${NET_1_VM_IPV4[0]}
 204     ${cmd} =    BuiltIn.Catenate    ${RT_OPTIONS}    ${ext_rt1}    ${RT_OPTIONS}    ${ext_rt2}
 205     OpenStackOperations.Update Router    ${ROUTER}    ${cmd}
 206     OpenStackOperations.Show Router    ${ROUTER}    -D
 207     : FOR    ${VM}    IN    @{NET_1_VM_IPV6}
 208     \    ${CONFIG_EXTRA_ROUTE_IP3} =    BuiltIn.Catenate    sudo ip -6 addr add ${EXTRA_NW_IPV6[0]}/64 dev eth0
 209     \    ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${VM}    ${CONFIG_EXTRA_ROUTE_IP3}
 210     \    ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${VM}    ip -6 a
 211     \    ${CONFIG_EXTRA_ROUTE_IP4} =    BuiltIn.Catenate    sudo ip -6 addr add ${EXTRA_NW_IPV6[0]}/64 dev eth0
 212     \    ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV6[0]}    ${CONFIG_EXTRA_ROUTE_IP4}
 213     \    ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV6[0]}    ip -6 a
 214     ${ext_rt3} =    BuiltIn.Set Variable    destination=${EXTRA_NW_SUBNET[2]},gateway=${NET_1_VM_IPV6[0]}
 215     ${ext_rt4} =    BuiltIn.Set Variable    destination=${EXTRA_NW_SUBNET[3]},gateway=${NET_1_VM_IPV6[0]}
 216     ${cmd} =    BuiltIn.Catenate    ${RT_OPTIONS}    ${ext_rt3}    ${RT_OPTIONS}    ${ext_rt4}
 217     OpenStackOperations.Update Router    ${ROUTER}    ${cmd}
 218     OpenStackOperations.Show Router    ${ROUTER}    -D
 219     BuiltIn.Log    Verify FIB table
 220     ${vm_instances} =    BuiltIn.Create List    @{EXTRA_NW_SUBNET}
 221     BuiltIn.Wait Until Keyword Succeeds    30s    5s    Utils.Check For Elements At URI    ${FIB_ENTRY_URL}    ${vm_instances}
 222     : FOR    ${EXTRA_NW_IP}    IN    @{EXTRA_NW_IPV4}
 223     \    ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV4[0]}    ping -c 3 ${EXTRA_NW_IP}
 224     \    BuiltIn.Should Contain    ${output}    64 bytes
 225     \    ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV4[1]}    ping -c 3 ${EXTRA_NW_IP}
 226     \    BuiltIn.Should Contain    ${output}    64 bytes
 227     : FOR    ${EXTRA_NW_IP}    IN    @{EXTRA_NW_IPV6}
 228     \    ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV6[0]}    ping6 -c 3 ${EXTRA_NW_IP}
 229     \    BuiltIn.Should Contain    ${output}    64 bytes
 230     \    ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV6[1]}    ping6 -c 3 ${EXTRA_NW_IP}
 231     \    BuiltIn.Should Contain    ${output}    64 bytes
 232
 233 Delete And Recreate Extra Route
 234     [Documentation]    Delete IPv4 and IPv6 extra routes and recreate it.
 235     ...    Then check data path before L3VPN creation.
 236     BuiltIn.Log    Delete all extra routes
 237     OpenStackOperations.Update Router    ${ROUTER}    ${RT_CLEAR}
 238     OpenStackOperations.Show Router    ${ROUTER}    -D
 239     BuiltIn.Log    Adding extra route to VM
 240     ${CONFIG_EXTRA_ROUTE_IPV4}=    BuiltIn.Catenate    sudo ifconfig eth0:1 @{EXTRA_NW_IPV4}[0] netmask 255.255.255.0 up
 241     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV4[0]}    ${CONFIG_EXTRA_ROUTE_IPV4}
 242     ${ext_rt_ipv4} =    BuiltIn.Set Variable    destination=${EXTRA_NW_SUBNET[0]},gateway=${NET_1_VM_IPV4[0]}
 243     ${cmd} =    BuiltIn.Catenate    ${RT_OPTIONS}    ${ext_rt_ipv4}
 244     OpenStackOperations.Update Router    ${ROUTER}    ${cmd}
 245     OpenStackOperations.Show Router    ${ROUTER}    -D
 246     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV4[1]}    ping -c 3 @{EXTRA_NW_IPV4}[0]
 247     BuiltIn.Should Contain    ${output}    64 bytes
 248     ${CONFIG_EXTRA_ROUTE_IPV6} =    BuiltIn.Catenate    sudo ip -6 addr add ${EXTRA_NW_IPV6[0]}/64 dev eth0
 249     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV6[1]}    ${CONFIG_EXTRA_ROUTE_IPV6}
 250     ${ext_rt2} =    BuiltIn.Set Variable    destination=${EXTRA_NW_SUBNET[2]},gateway=${NET_1_VM_IPV6[1]}
 251     ${cmd} =    BuiltIn.Catenate    ${RT_OPTIONS}    ${ext_rt2}
 252     OpenStackOperations.Update Router    ${ROUTER}    ${cmd}
 253     OpenStackOperations.Show Router    ${ROUTER}    -D
 254     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV6[1]}    ping6 -c 3 @{EXTRA_NW_IPV6}[0]
 255     BuiltIn.Should Contain    ${output}    64 bytes
 256     # clear off extra-routes before the next set of tests
 257     OpenStackOperations.Update Router    ${ROUTER}    ${RT_CLEAR}
 258     [Teardown]    BuiltIn.Run Keywords    OpenStackOperations.Update Router    ${ROUTER}    ${RT_CLEAR}
 259     ...    AND    OpenStackOperations.Get Test Teardown Debugs
 260
 261 Create L3VPN
 262     ${net_id} =    OpenStackOperations.Get Net Id    @{NETWORKS}[0]
 263     ${tenant_id} =    OpenStackOperations.Get Tenant ID From Network    ${net_id}
 264     VpnOperations.VPN Create L3VPN    vpnid=${VPN_INSTANCE_ID[0]}    name=${VPN_NAME[0]}    rd=@{RDS}[0]    exportrt=@{RDS}[0]    importrt=@{RDS}[0]    tenantid=${tenant_id}
 265     ${resp} =    VpnOperations.VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 266     BuiltIn.Should Contain    ${resp}    ${VPN_INSTANCE_ID[0]}
 267
 268 Associate L3VPN To Routers
 269     [Documentation]    Associating router to L3VPN.
 270     ${router_id} =    OpenStackOperations.Get Router Id    ${ROUTER}
 271     VpnOperations.Associate VPN to Router    routerid=${router_id}    vpnid=${VPN_INSTANCE_ID[0]}
 272     ${resp} =    VpnOperations.VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 273     BuiltIn.Should Contain    ${resp}    ${router_id}
 274
 275 Verify L3VPN Datapath With Router Association
 276     [Documentation]    Datapath test across networks using L3VPN associated with router.
 277     BuiltIn.Log    Verify VPN interfaces, FIB entries and Flow table
 278     BuiltIn.Wait Until Keyword Succeeds    30s    5s    Utils.Check For Elements At URI    ${VPN_IFACES_URL}    ${VM_IPS}
 279     ${RD} =    Strip String    ${RDS[0]}    characters="[]
 280     BuiltIn.Wait Until Keyword Succeeds    60s    15s    Utils.Check For Elements At URI    ${CONFIG_API}/odl-fib:fibEntries/vrfTables/${RD}/    ${VM_IPS}
 281     BuiltIn.Wait Until Keyword Succeeds    60s    5s    VpnOperations.Verify Flows Are Present For L3VPN    ${OS_COMPUTE_1_IP}    ${VM_IPS}
 282     BuiltIn.Wait Until Keyword Succeeds    60s    5s    VpnOperations.Verify Flows Are Present For L3VPN    ${OS_COMPUTE_2_IP}    ${VM_IPS}
 283     BuiltIn.Wait Until Keyword Succeeds    30s    5s    VpnOperations.Verify GWMAC Entry On ODL    ${GW_MAC_ADDRS}
 284     BuiltIn.Wait Until Keyword Succeeds    30s    5s    Verify GWMAC Flow Entry On Flow Table    ${OS_COMPUTE_1_IP}
 285     BuiltIn.Wait Until Keyword Succeeds    30s    5s    Verify GWMAC Flow Entry On Flow Table    ${OS_COMPUTE_2_IP}
 286     BuiltIn.Log    L3 Datapath test across the networks using L3VPN
 287     ${dst_ipv4_list1} =    BuiltIn.Create List    ${NET_1_VM_IPV4[1]}    @{NET_2_VM_IPV4}
 288     Test Operations From Vm Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV4[0]}    ${dst_ipv4_list1}
 289     ${dst_ipv4_list2} =    BuiltIn.Create List    ${NET_2_VM_IPV4[1]}    @{NET_1_VM_IPV4}
 290     Test Operations From Vm Instance    @{NETWORKS}[1]    ${NET_2_VM_IPV4[0]}    ${dst_ipv4_list2}
 291     ${dst_ipv6_list1} =    BuiltIn.Create List    ${NET_1_VM_IPV6[1]}    @{NET_2_VM_IPV6}
 292     Test Operations From Vm Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV6[0]}    ${dst_ipv6_list1}
 293     ${dst_ipv6_list2} =    BuiltIn.Create List    ${NET_2_VM_IPV6[1]}    @{NET_1_VM_IPV6}
 294     Test Operations From Vm Instance    @{NETWORKS}[1]    ${NET_2_VM_IPV6[0]}    ${dst_ipv6_list2}
 295
 296 Delete IPv6 Subnet And Check IPv4 datapath
 297     [Documentation]    Delete IPv6 subnet from router and check IPv4 datapath before L3VPN creation.
 298     ...    Then recreate IPv6 subnet.
 299     BuiltIn.Log    Delete extra routes
 300     OpenStackOperations.Update Router    ${ROUTER}    ${RT_CLEAR}
 301     OpenStackOperations.Show Router    ${ROUTER}    -D
 302     BuiltIn.Log    Delete IPv6 subnet
 303     : FOR    ${PORT}    IN    @{SUBNETS6}
 304     \    Remove Interface    ${ROUTER}    ${PORT}
 305     BuiltIn.Log    Test L2 datapath
 306     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV4[0]}    ping -I ${NET_1_VM_IPV4[0]} -c 3 ${NET_1_VM_IPV4[1]}
 307     BuiltIn.Should Contain    ${output}    64 bytes
 308     ${output} =    OpenStackOperations.Execute Command on VM Instance    @{NETWORKS}[1]    ${NET_2_VM_IPV4[0]}    ping -I ${NET_2_VM_IPV4[0]} -c 3 ${NET_2_VM_IPV4[1]}
 309     BuiltIn.Should Contain    ${output}    64 bytes
 310     BuiltIn.Log    Test L3 datapath
 311     ${dst_ipv4_list1} =    BuiltIn.Create List    ${NET_1_VM_IPV4[1]}    @{NET_2_VM_IPV4}
 312     Test Operations From Vm Instance    @{NETWORKS}[0]    ${NET_1_VM_IPV4[0]}    ${dst_ipv4_list1}
 313     ${dst_ipv4_list2} =    BuiltIn.Create List    ${NET_2_VM_IPV4[1]}    @{NET_1_VM_IPV4}
 314     Test Operations From Vm Instance    @{NETWORKS}[1]    ${NET_2_VM_IPV4[0]}    ${dst_ipv4_list2}
 315
 316 Dissociate L3VPN From Routers
 317     [Documentation]    Dissociating router from L3VPN.
 318     ${router_id} =    OpenStackOperations.Get Router Id    ${ROUTER}
 319     VpnOperations.Dissociate VPN to Router    routerid=${router_id}    vpnid=${VPN_INSTANCE_ID[0]}
 320     ${resp} =    VpnOperations.VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 321     BuiltIn.Should Not Contain    ${resp}    ${router_id}
 322
 323 Delete Router And Router Interfaces With L3VPN
 324     [Documentation]    Delete associate with L3VPN Router and its Ports.
 325     # Asscoiate router with L3VPN
 326     ${router_id} =    OpenStackOperations.Get Router Id    ${ROUTER}
 327     VpnOperations.Associate VPN to Router    routerid=${router_id}    vpnid=${VPN_INSTANCE_ID[0]}
 328     ${resp} =    VpnOperations.VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 329     BuiltIn.Should Contain    ${resp}    ${router_id}
 330     # Delete Interface
 331     : FOR    ${iface}    IN    @{SUBNETS4}
 332     \    Remove Interface    ${ROUTER}    ${iface}
 333     \    ${subnet_id} =    Get Subnet Id    ${iface}
 334     \    ${rt_port_list} =    OpenStackOperations.Show Router Interface    ${ROUTER}
 335     \    BuiltIn.Should Not Contain    ${rt_port_list}    ${subnet_id}
 336     # Delete Router and Interface to the subnets.
 337     Delete Router    ${ROUTER}
 338     ${router_output} =    OpenStackOperations.List Routers
 339     BuiltIn.Should Not Contain    ${router_output}    ${ROUTER}
 340     ${router_list} =    BuiltIn.Create List    ${ROUTER}
 341     BuiltIn.Wait Until Keyword Succeeds    3s    1s    Utils.Check For Elements Not At URI    ${ROUTER_URL}    ${router_list}
 342     # Verify Router Entry removed from L3VPN
 343     ${resp} =    VpnOperations.VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 344     BuiltIn.Should Not Contain    ${resp}    ${router_id}
 345     BuiltIn.Wait Until Keyword Succeeds    30s    5s    VpnOperations.Verify GWMAC Flow Entry Removed From Flow Table    ${OS_COMPUTE_1_IP}
 346     BuiltIn.Wait Until Keyword Succeeds    30s    5s    VpnOperations.Verify GWMAC Flow Entry Removed From Flow Table    ${OS_COMPUTE_2_IP}
 347
 348 Delete Router With NonExistentRouter Name
 349     [Documentation]    Delete router with nonExistentRouter name.
 350     ${rc}    ${output} =    Run And Return Rc And Output    openstack router delete nonExistentRouter
 351     BuiltIn.Should Match Regexp    ${output}    Failed to delete router with name or ID 'nonExistentRouter'|Failed to delete router\\(s\\) with name or ID\\(s\\) 'nonExistentRouter'
 352
 353 Associate L3VPN To Networks
 354     [Documentation]    Associates L3VPN to networks.
 355     ${network1_id} =    OpenStackOperations.Get Net Id    @{NETWORKS}[0]
 356     ${network2_id} =    OpenStackOperations.Get Net Id    @{NETWORKS}[1]
 357     VpnOperations.Associate L3VPN To Network    networkid=${network1_id}    vpnid=${VPN_INSTANCE_ID[0]}
 358     ${resp} =    VpnOperations.VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 359     BuiltIn.Should Contain    ${resp}    ${network1_id}
 360     VpnOperations.Associate L3VPN To Network    networkid=${network2_id}    vpnid=${VPN_INSTANCE_ID[0]}
 361     ${resp} =    VpnOperations.VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 362     BuiltIn.Should Contain    ${resp}    ${network2_id}
 363
 364 Dissociate L3VPN From Networks
 365     [Documentation]    Dissociate L3VPN from networks.
 366     ${network1_id} =    OpenStackOperations.Get Net Id    @{NETWORKS}[0]
 367     ${network2_id} =    OpenStackOperations.Get Net Id    @{NETWORKS}[1]
 368     VpnOperations.Dissociate L3VPN From Networks    networkid=${network1_id}    vpnid=${VPN_INSTANCE_ID[0]}
 369     ${resp} =    VpnOperations.VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 370     BuiltIn.Should Not Contain    ${resp}    ${network1_id}
 371     VpnOperations.Dissociate L3VPN From Networks    networkid=${network2_id}    vpnid=${VPN_INSTANCE_ID[0]}
 372     ${resp} =    VpnOperations.VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 373     BuiltIn.Should Not Contain    ${resp}    ${network2_id}
 374
 375 Delete L3VPN
 376     [Documentation]    Delete L3VPN.
 377     VpnOperations.VPN Delete L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 378
 379 Create Multiple L3VPN
 380     [Documentation]    Creates three L3VPNs and then check the same.
 381     ${net_id} =    OpenStackOperations.Get Net Id    @{NETWORKS}[0]
 382     ${tenant_id} =    OpenStackOperations.Get Tenant ID From Network    ${net_id}
 383     VpnOperations.VPN Create L3VPN    vpnid=${VPN_INSTANCE_ID[0]}    name=${VPN_NAME[0]}    rd=${RDS[0]}    exportrt=${RDS[0]}    importrt=${RDS[0]}    tenantid=${tenant_id}
 384     VpnOperations.VPN Create L3VPN    vpnid=${VPN_INSTANCE_ID[1]}    name=${VPN_NAME[1]}    rd=${RDS[1]}    exportrt=${RDS[1]}    importrt=${RDS[1]}    tenantid=${tenant_id}
 385     VpnOperations.VPN Create L3VPN    vpnid=${VPN_INSTANCE_ID[2]}    name=${VPN_NAME[2]}    rd=${RDS[2]}    exportrt=${RDS[2]}    importrt=${RDS[2]}    tenantid=${tenant_id}
 386     ${resp} =    VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[0]}
 387     BuiltIn.Should Contain    ${resp}    ${VPN_INSTANCE_ID[0]}
 388     ${resp} =    VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[1]}
 389     BuiltIn.Should Contain    ${resp}    ${VPN_INSTANCE_ID[1]}
 390     ${resp} =    VPN Get L3VPN    vpnid=${VPN_INSTANCE_ID[2]}
 391     BuiltIn.Should Contain    ${resp}    ${VPN_INSTANCE_ID[2]}
 392
 393 *** Keywords ***
 394 Verify GWMAC Flow Entry On Flow Table
 395     [Arguments]    ${cnIp}
 396     [Documentation]    Verify GWMAC Table, ARP Response table and Dispatcher table.
 397     ${flow_output} =    Run Command On Remote System    ${cnIp}    sudo ovs-ofctl -O OpenFlow13 dump-flows br-int
 398     ${group_output} =    Run Command On Remote System    ${cnIp}    sudo ovs-ofctl -O OpenFlow13 dump-groups br-int
 399     BuiltIn.Should Contain    ${flow_output}    table=${DISPATCHER_TABLE}
 400     ${dispatcher_table} =    Get Lines Containing String    ${flow_output}    table=${DISPATCHER_TABLE}
 401     BuiltIn.Should Contain    ${dispatcher_table}    goto_table:${GWMAC_TABLE}
 402     BuiltIn.Should Not Contain    ${dispatcher_table}    goto_table:${ARP_RESPONSE_TABLE}
 403     BuiltIn.Should Contain    ${flow_output}    table=${GWMAC_TABLE}
 404     ${gwmac_table} =    Get Lines Containing String    ${flow_output}    table=${GWMAC_TABLE}
 405     # Verify GWMAC address present in table ${L3_TABLE}
 406     : FOR    ${macAdd}    IN    @{GW_MAC_ADDRS}
 407     \    BuiltIn.Should Contain    ${gwmac_table}    dl_dst=${macAdd} actions=goto_table:${L3_TABLE}
 408     # Verify Miss entry
 409     BuiltIn.Should Contain    ${gwmac_table}    actions=resubmit(,17)
 410     # Verify ARP_CHECK_TABLE - ${ARP_CHECK_TABLE}
 411     ${arpchk_table} =    Get Lines Containing String    ${flow_output}    table=${ARP_CHECK_TABLE}
 412     BuiltIn.Should Match Regexp    ${arpchk_table}    ${ARP_RESPONSE_REGEX}
 413     ${match} =    BuiltIn.Should Match Regexp    ${arpchk_table}    ${ARP_REQUEST_REGEX}
 414     ${groupID} =    Split String    ${match}    separator=:
 415     BuiltIn.Should Contain    ${flow_output}    table=${IPV6_TABLE}
 416     ${icmp_ipv6_flows} =    Get Lines Containing String    ${flow_output}    icmp_type=${ICMP_TYPE}
 417     # Verify IPv6 icmp_type=135
 418     : FOR    ${ip_addr}    IN    @{GW_IPV6_ADDRS}
 419     \    ${rule} =    BuiltIn.Set Variable    icmp_type=${ICMP_TYPE},icmp_code=0,nd_target=${ip_addr} actions=CONTROLLER:65535
 420     \    BuiltIn.Should Match Regexp    ${icmp_ipv6_flows}    ${rule}
 421     VpnOperations.Verify ARP REQUEST in groupTable    ${group_output}    ${groupID[1]}
 422     # Verify ARP_RESPONSE_TABLE - ${ARP_RESPONSE_TABLE}
 423     BuiltIn.Should Contain    ${flow_output}    table=${ARP_RESPONSE_TABLE}
 424     ${arpResponder_table} =    Get Lines Containing String    ${flow_output}    table=${ARP_RESPONSE_TABLE}
 425     BuiltIn.Should Contain    ${arpResponder_table}    priority=0 actions=drop
 426     : FOR    ${macAdd}    ${ipAdd}    IN ZIP    ${GW1_MAC_ADDRS}    ${GW_IPV4_ADDRS}
 427     \    ${ARP_RESPONSE_IP_MAC_REGEX} =    BuiltIn.Set Variable    arp_tpa=${ipAdd},arp_op=1 actions=.*,set_field:${macAdd}->eth_src
 428     \    BuiltIn.Should Match Regexp    ${arpResponder_table}    ${ARP_RESPONSE_IP_MAC_REGEX}