X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=csit%2Flibraries%2FSxpClusterLib.robot;h=10bc9ef6b2dc2a2dc77cadb3c2a65f89479d2b51;hb=94651ca9711d8eb316721775561e815830bc6c7e;hp=ca593bdff229057a88c8117d9436db39d5ee468a;hpb=d0b1ef8d87d486787c777763ce9e73a38e49db54;p=integration%2Ftest.git

diff --git a/csit/libraries/SxpClusterLib.robot b/csit/libraries/SxpClusterLib.robot
index ca593bdff2..10bc9ef6b2 100644
--- a/csit/libraries/SxpClusterLib.robot
+++ b/csit/libraries/SxpClusterLib.robot
@@ -11,7 +11,6 @@ Resource          ./SxpLib.robot
 @{SXP_PACKAGE}    org.opendaylight.sxp
 ${DEVICE_SESSION}    device_1
 ${CONTROLLER_SESSION}    ClusterManagement__session_1
-${SXP_LOG_LEVEL}    INFO
 ${VIRTUAL_IP}     ${TOOLS_SYSTEM_2_IP}
 ${VIRTUAL_IP_MASK}    255.255.255.0
 ${VIRTUAL_INTERFACE}    dummy0
@@ -25,7 +24,7 @@ Setup SXP Cluster Session
     [Documentation]    Create sessions asociated with SXP cluster setup
     ClusterManagement.ClusterManagement_Setup
     SetupUtils.Setup_Utils_For_Setup_And_Teardown
-    SetupUtils.Setup_Logging_For_Debug_Purposes_On_List_Or_All    ${SXP_LOG_LEVEL}    ${SXP_PACKAGE}
+    SetupUtils.Setup_Logging_For_Debug_Purposes_On_List_Or_All    DEBUG    ${SXP_PACKAGE}
 
 Setup Device Session
     [Documentation]    Create session on the SXP device
@@ -39,6 +38,8 @@ Setup SXP Cluster Session With Device
 Clean SXP Cluster Session
     [Documentation]    Clean sessions asociated with SXP cluster setup
     ClusterManagement.Flush_Iptables_From_List_Or_All
+    BuiltIn.Wait Until Keyword Succeeds    60x    1s    ClusterManagement.Verify_Members_Are_Ready    member_index_list=${EMPTY}    verify_cluster_sync=True    verify_restconf=True
+    ...    verify_system_status=False    service_list=@{EMPTY}
     RequestsLibrary.Delete All Sessions
     SetupUtils.Setup_Logging_For_Debug_Purposes_On_List_Or_All    INFO    ${SXP_PACKAGE}
 
@@ -52,39 +53,48 @@ Setup SXP Cluster
     [Arguments]    ${peer_mode}=listener
     [Documentation]    Setup and connect SXP cluster topology
     SxpLib.Add Node    ${DEVICE_NODE_ID}    session=${DEVICE_SESSION}
-    : FOR    ${i}    IN RANGE    ${NUM_ODL_SYSTEM}
-    \    SxpLib.Add Connection    version4    ${peer_mode}    ${ODL_SYSTEM_${i+1}_IP}    64999    node=${DEVICE_NODE_ID}
-    \    ...    session=${DEVICE_SESSION}
+    BuiltIn.Wait Until Keyword Succeeds    240x    1s    SxpLib.Check Node Started    ${DEVICE_NODE_ID}    session=${DEVICE_SESSION}
+    FOR    ${i}    IN RANGE    ${NUM_ODL_SYSTEM}
+        SxpLib.Add Connection    version4    ${peer_mode}    ${ODL_SYSTEM_${i+1}_IP}    64999    node=${DEVICE_NODE_ID}
+        ...    session=${DEVICE_SESSION}
+    END
     ${cluster_mode} =    Sxp.Get Opposing Mode    ${peer_mode}
     SxpLib.Add Node    ${INADDR_ANY}    session=${CONTROLLER_SESSION}
+    BuiltIn.Wait Until Keyword Succeeds    240x    1s    Check Cluster Node Started    ${INADDR_ANY}    ip=${EMPTY}
     SxpLib.Add Connection    version4    ${cluster_mode}    ${DEVICE_NODE_ID}    64999    ${INADDR_ANY}    session=${CONTROLLER_SESSION}
-    BuiltIn.Wait Until Keyword Succeeds    12x    10s    SxpLib.Check Node Started    ${DEVICE_NODE_ID}    session=${DEVICE_SESSION}
-    BuiltIn.Wait Until Keyword Succeeds    12x    10s    Check Cluster Node started    ${INADDR_ANY}    ip=${EMPTY}
-    BuiltIn.Wait Until Keyword Succeeds    48x    10s    Check Device is Connected    ${DEVICE_NODE_ID}    session=${DEVICE_SESSION}
+    BuiltIn.Wait Until Keyword Succeeds    480x    1s    Check Device is Connected    ${DEVICE_NODE_ID}    session=${DEVICE_SESSION}
 
 Clean SXP Cluster
     [Documentation]    Disconnect SXP cluster topology
     SxpLib.Delete Node    ${DEVICE_NODE_ID}    session=${DEVICE_SESSION}
-    BuiltIn.Wait Until Keyword Succeeds    3x    10s    SxpLib.Delete Node    ${INADDR_ANY}    session=${CONTROLLER_SESSION}
+    BuiltIn.Wait Until Keyword Succeeds    240x    1s    SxpLib.Check Node Stopped    ${DEVICE_NODE_ID}    session=${DEVICE_SESSION}
+    BuiltIn.Wait Until Keyword Succeeds    60x    1s    SxpLib.Delete Node    ${INADDR_ANY}    session=${CONTROLLER_SESSION}
+    BuiltIn.Wait Until Keyword Succeeds    240x    1s    SxpClusterLib.Check Cluster Node Stopped    ${INADDR_ANY}    ip=${EMPTY}
 
-Check Cluster Node started
+Check Cluster Node Started
     [Arguments]    ${node}    ${port}=64999    ${ip}=${node}
     [Documentation]    Verify that SxpNode has data written to Operational datastore and Node is running on one of cluster nodes
+    ${resp} =    RequestsLibrary.Get Request    ${CONTROLLER_SESSION}    /restconf/operational/network-topology:network-topology/topology/sxp/node/${node}/
+    BuiltIn.Should Be Equal As Strings    ${resp.status_code}    200
     ${started} =    BuiltIn.Set Variable    ${False}
-    : FOR    ${i}    IN RANGE    ${NUM_ODL_SYSTEM}
-    \    ${rc} =    Utils.Run Command On Remote System    ${ODL_SYSTEM_${i+1}_IP}    netstat -tln | grep -q ${ip}:${port} && echo 0 || echo 1    ${ODL_SYSTEM_USER}    ${ODL_SYSTEM_PASSWORD}
-    \    ...    prompt=${ODL_SYSTEM_PROMPT}
-    \    ${started} =    BuiltIn.Set Variable If    '${rc}' == '0'    ${True}    ${started}
+    FOR    ${i}    IN RANGE    ${NUM_ODL_SYSTEM}
+        ${rc} =    Utils.Run Command On Remote System    ${ODL_SYSTEM_${i+1}_IP}    netstat -tln | grep -q ${ip}:${port} && echo 0 || echo 1    ${ODL_SYSTEM_USER}    ${ODL_SYSTEM_PASSWORD}
+        ...    prompt=${ODL_SYSTEM_PROMPT}
+        ${started} =    BuiltIn.Set Variable If    '${rc}' == '0'    ${True}    ${started}
+    END
     BuiltIn.Should Be True    ${started}
 
-Check Cluster Node stopped
+Check Cluster Node Stopped
     [Arguments]    ${node}    ${port}=64999    ${ip}=${node}
     [Documentation]    Verify that SxpNode has data removed from Operational datastore and Node is stopped
+    ${resp} =    RequestsLibrary.Get Request    ${CONTROLLER_SESSION}    /restconf/operational/network-topology:network-topology/topology/sxp/node/${node}/
+    BuiltIn.Should Be Equal As Strings    ${resp.status_code}    404
     ${stopped} =    BuiltIn.Set Variable    ${False}
-    : FOR    ${i}    IN RANGE    ${NUM_ODL_SYSTEM}
-    \    ${rc} =    Utils.Run Command On Remote System    ${ODL_SYSTEM_${i+1}_IP}    netstat -tln | grep -q ${ip}:${port} && echo 0 || echo 1    ${ODL_SYSTEM_USER}    ${ODL_SYSTEM_PASSWORD}
-    \    ...    prompt=${ODL_SYSTEM_PROMPT}
-    \    ${stopped} =    BuiltIn.Set Variable If    '${rc}' == '1'    ${True}    ${stopped}
+    FOR    ${i}    IN RANGE    ${NUM_ODL_SYSTEM}
+        ${rc} =    Utils.Run Command On Remote System    ${ODL_SYSTEM_${i+1}_IP}    netstat -tln | grep -q ${ip}:${port} && echo 0 || echo 1    ${ODL_SYSTEM_USER}    ${ODL_SYSTEM_PASSWORD}
+        ...    prompt=${ODL_SYSTEM_PROMPT}
+        ${stopped} =    BuiltIn.Set Variable If    '${rc}' == '1'    ${True}    ${stopped}
+    END
     BuiltIn.Should Be True    ${stopped}
 
 Check Device is Connected
@@ -92,10 +102,11 @@ Check Device is Connected
     [Documentation]    Checks if SXP device is connected to the cluster. It means it has connection in state "on" with one of the cluster members.
     ${resp} =    SxpLib.Get Connections    node=${node}    session=${session}
     ${is_connected} =    BuiltIn.Set Variable    ${False}
-    : FOR    ${i}    IN RANGE    ${NUM_ODL_SYSTEM}
-    \    ${is_connected} =    Sxp.Find Connection    ${resp}    ${version}    any    ${ODL_SYSTEM_${i+1}_IP}
-    \    ...    ${port}    on
-    \    BuiltIn.Exit For Loop If    ${is_connected}
+    FOR    ${i}    IN RANGE    ${NUM_ODL_SYSTEM}
+        ${is_connected} =    Sxp.Find Connection    ${resp}    ${version}    any    ${ODL_SYSTEM_${i+1}_IP}
+        ...    ${port}    on
+        BuiltIn.Exit For Loop If    ${is_connected}
+    END
     BuiltIn.Should Be True    ${is_connected}
 
 Check Cluster is Connected
@@ -107,7 +118,7 @@ Check Cluster is Connected
 Get Owner Controller
     [Arguments]    ${running_member}=1
     [Documentation]    Find cluster controller that is marked as cluster owner by requesting ownership data from ${running_member} node of the cluster
-    ${owner}    ${candidates} =    BuiltIn.Wait Until Keyword Succeeds    5x    2s    ClusterManagement.Get_Owner_And_Successors_For_Device    org.opendaylight.sxp.controller.boot.SxpControllerInstance
+    ${owner}    ${candidates} =    BuiltIn.Wait Until Keyword Succeeds    60x    1s    ClusterManagement.Get_Owner_And_Successors_For_Device    org.opendaylight.sxp.controller.boot.SxpControllerInstance
     ...    Sxp    ${running_member}
     [Return]    ${owner}
 
@@ -125,9 +136,10 @@ Get Any Controller
 Map Followers To Mac Addresses
     [Documentation]    Creates Map containing ODL_SYSTEM_IP to corresponding MAC-ADDRESS
     ${mac_addresses} =    BuiltIn.Create dictionary
-    : FOR    ${i}    IN RANGE    ${NUM_ODL_SYSTEM}
-    \    ${mac_address}    Find Mac Address Of Ip Address    ${ODL_SYSTEM_${i+1}_IP}
-    \    Collections.Set To Dictionary    ${mac_addresses}    ${ODL_SYSTEM_${i+1}_IP}    ${mac_address}
+    FOR    ${i}    IN RANGE    ${NUM_ODL_SYSTEM}
+        ${mac_address}    Find Mac Address Of Ip Address    ${ODL_SYSTEM_${i+1}_IP}
+        Collections.Set To Dictionary    ${mac_addresses}    ${ODL_SYSTEM_${i+1}_IP}    ${mac_address}
+    END
     BuiltIn.Log    ${mac_addresses}
     [Return]    ${mac_addresses}
 
@@ -162,13 +174,15 @@ Shutdown Tools Node
 
 Create Virtual Interface
     [Documentation]    Create virtual interface on all of the cluster nodes
-    : FOR    ${i}    IN RANGE    ${NUM_ODL_SYSTEM}
-    \    Utils.Run Command On Remote System    ${ODL_SYSTEM_${i+1}_IP}    sudo modprobe dummy    ${ODL_SYSTEM_USER}    ${ODL_SYSTEM_PASSWORD}
-    \    Utils.Run Command On Remote System And Log    ${ODL_SYSTEM_${i+1}_IP}    sudo ip link show    ${ODL_SYSTEM_USER}    ${ODL_SYSTEM_PASSWORD}
+    FOR    ${i}    IN RANGE    ${NUM_ODL_SYSTEM}
+        Utils.Run Command On Remote System    ${ODL_SYSTEM_${i+1}_IP}    sudo modprobe dummy    ${ODL_SYSTEM_USER}    ${ODL_SYSTEM_PASSWORD}
+        Utils.Run Command On Remote System And Log    ${ODL_SYSTEM_${i+1}_IP}    sudo ip link show    ${ODL_SYSTEM_USER}    ${ODL_SYSTEM_PASSWORD}
+    END
 
 Delete Virtual Interface
     [Documentation]    Create virtual interface on all of the cluster nodes
-    : FOR    ${i}    IN RANGE    ${NUM_ODL_SYSTEM}
-    \    Utils.Run Command On Remote System    ${ODL_SYSTEM_${i+1}_IP}    sudo ip link delete ${VIRTUAL_INTERFACE} type dummy    ${ODL_SYSTEM_USER}    ${ODL_SYSTEM_PASSWORD}
-    \    Utils.Run Command On Remote System    ${ODL_SYSTEM_${i+1}_IP}    sudo rmmod dummy    ${ODL_SYSTEM_USER}    ${ODL_SYSTEM_PASSWORD}
-    \    Utils.Run Command On Remote System And Log    ${ODL_SYSTEM_${i+1}_IP}    sudo ip link show    ${ODL_SYSTEM_USER}    ${ODL_SYSTEM_PASSWORD}
+    FOR    ${i}    IN RANGE    ${NUM_ODL_SYSTEM}
+        Utils.Run Command On Remote System    ${ODL_SYSTEM_${i+1}_IP}    sudo ip link delete ${VIRTUAL_INTERFACE} type dummy    ${ODL_SYSTEM_USER}    ${ODL_SYSTEM_PASSWORD}
+        Utils.Run Command On Remote System    ${ODL_SYSTEM_${i+1}_IP}    sudo rmmod dummy    ${ODL_SYSTEM_USER}    ${ODL_SYSTEM_PASSWORD}
+        Utils.Run Command On Remote System And Log    ${ODL_SYSTEM_${i+1}_IP}    sudo ip link show    ${ODL_SYSTEM_USER}    ${ODL_SYSTEM_PASSWORD}
+    END