SXP: Improve reliability of basic suite

- check that node is started after add
- check that node is stopped after delete

- increase timeouts in all connection
related tests

JIRA: SXP-150
Change-Id: I440a90c7b471594642d843ed87a9b638743a6485
Signed-off-by: Ivan Hrasko <ihrasko@cisco.com>

diff --git a/csit/libraries/SxpLib.robot b/csit/libraries/SxpLib.robot
index eba1eba36be083b39a1d23b902f4fc7a709c0dff..c38372d169ad7774963a40fcaa04d1726f987667 100644 (file)
--- a/csit/libraries/SxpLib.robot
+++ b/csit/libraries/SxpLib.robot
@@ -350,6 +350,7 @@ Clean SXP Environment
     : FOR    ${num}    IN RANGE    1    ${node_range}+1
     \    ${ip} =    Sxp.Get Ip From Number    ${num}
     \    Delete Node    ${ip}
+    \    BuiltIn.Wait Until Keyword Succeeds    12x    10s    Check Node Stopped    ${ip}
     RequestsLibrary.Delete All Sessions
 
 Get Routing Configuration From Controller

diff --git a/csit/suites/sxp/basic/020_Restconf_CRUD.robot b/csit/suites/sxp/basic/020_Restconf_CRUD.robot
index c377c1c42c2879a81a0f45bbf5fca7084d45b193..abd35a742922fd22cfa4d57159871b3d17f23e23 100644 (file)
--- a/csit/suites/sxp/basic/020_Restconf_CRUD.robot
+++ b/csit/suites/sxp/basic/020_Restconf_CRUD.robot
@@ -15,51 +15,51 @@ Test Add Bindings
     [Tags]    Restconf CRUD    SXP
     ${resp}    Get Bindings
     Add Bindings    5230    1.1.1.1/32
-    Wait Until Keyword Succeeds    3x    250ms    Bindings Should Contain    5230    1.1.1.1/32
+    Wait Until Keyword Succeeds    30x    1s    Bindings Should Contain    5230    1.1.1.1/32
     Add Bindings    30    2001:0:0:0:0:0:0:0/128
-    Wait Until Keyword Succeeds    3x    250ms    Bindings Should Contain    30    2001:0:0:0:0:0:0:0/128
+    Wait Until Keyword Succeeds    30x    1s    Bindings Should Contain    30    2001:0:0:0:0:0:0:0/128
 
 Test Update Bindings
     [Documentation]    Test if bindings can be updated to different SGT values by new incoming bindings for the same IP prefix
     [Tags]    Restconf CRUD    SXP
     Add Bindings    30    1.1.1.10/32
-    Wait Until Keyword Succeeds    3x    250ms    Bindings Should Contain    30    1.1.1.10/32
+    Wait Until Keyword Succeeds    30x    1s    Bindings Should Contain    30    1.1.1.10/32
     Sleep    1s    New binding must be at least 1s newer
     Add Bindings    40    1.1.1.10/32
-    Wait Until Keyword Succeeds    3x    250ms    Bindings Should Not Contain    30    1.1.1.10/32
-    Wait Until Keyword Succeeds    3x    250ms    Bindings Should Contain    40    1.1.1.10/32
+    Wait Until Keyword Succeeds    30x    1s    Bindings Should Not Contain    30    1.1.1.10/32
+    Wait Until Keyword Succeeds    30x    1s    Bindings Should Contain    40    1.1.1.10/32
 
 Test Delete Bindings
     [Documentation]    Test if bindings are deleted from Master DB
     [Tags]    Restconf CRUD    SXP
     Add Bindings    52301    12.1.1.1/32
-    Wait Until Keyword Succeeds    3x    250ms    Bindings Should Contain    52301    12.1.1.1/32
+    Wait Until Keyword Succeeds    30x    1s    Bindings Should Contain    52301    12.1.1.1/32
     Run Keyword And Expect Error    *    Delete Bindings    2631    12.1.1.1/32
-    Wait Until Keyword Succeeds    3x    250ms    Bindings Should Contain    52301    12.1.1.1/32
+    Wait Until Keyword Succeeds    30x    1s    Bindings Should Contain    52301    12.1.1.1/32
     Delete Bindings    52301    12.1.1.1/32
-    Wait Until Keyword Succeeds    3x    250ms    Bindings Should Not Contain    52301    12.1.1.1/32
+    Wait Until Keyword Succeeds    30x    1s    Bindings Should Not Contain    52301    12.1.1.1/32
 
 Test Add Connection
     [Documentation]    Test if connections are added to Node
     [Tags]    Restconf CRUD    SXP
     Add Connection    version4    speaker    10.1.0.0    60000
-    Wait Until Keyword Succeeds    3x    250ms    Connections Should Contain    10.1.0.0    60000    speaker
+    Wait Until Keyword Succeeds    30x    1s    Connections Should Contain    10.1.0.0    60000    speaker
     ...    version4
     Add Connection    version1    listener    105.12.0.50    64000
-    Wait Until Keyword Succeeds    3x    250ms    Connections Should Contain    105.12.0.50    64000    listener
+    Wait Until Keyword Succeeds    30x    1s    Connections Should Contain    105.12.0.50    64000    listener
     ...    version1
 
 Test Delete Connection
     [Documentation]    Test if conncetions are removed from Node
     [Tags]    Restconf CRUD    SXP
     Add Connection    version4    speaker    127.1.0.30    60000
-    Wait Until Keyword Succeeds    3x    250ms    Connections Should Contain    127.1.0.30    60000    speaker
+    Wait Until Keyword Succeeds    30x    1s    Connections Should Contain    127.1.0.30    60000    speaker
     ...    version4
     Run Keyword And Expect Error    *    Delete Connections    127.1.0.30    65000
-    Wait Until Keyword Succeeds    3x    250ms    Connections Should Contain    127.1.0.30    60000    speaker
+    Wait Until Keyword Succeeds    30x    1s    Connections Should Contain    127.1.0.30    60000    speaker
     ...    version4
     Delete Connections    127.1.0.30    60000
-    Wait Until Keyword Succeeds    3x    250ms    Connections Should Not Contain    127.1.0.30    60000    speaker
+    Wait Until Keyword Succeeds    30x    1s    Connections Should Not Contain    127.1.0.30    60000    speaker
     ...    version4
 
 *** Keywords ***

diff --git a/csit/suites/sxp/basic/030_Connectivity.robot b/csit/suites/sxp/basic/030_Connectivity.robot
index 1907c035c78edee6aef5e07a46a06a015c21d810..be4cd94bea3f1c0965c85ae54c7007d830a2e3ee 100644 (file)
--- a/csit/suites/sxp/basic/030_Connectivity.robot
+++ b/csit/suites/sxp/basic/030_Connectivity.robot
@@ -70,18 +70,18 @@ Test Nodes
     \    ...    ${PASSWORD}
     \    SxpLib.Add Connection    ${version}    speaker    127.0.0.1    64999    127.0.0.2
     \    ...    ${PASSWORD}
-    \    BuiltIn.Wait Until Keyword Succeeds    15    1    SxpLib.Verify Connection    ${cmp_version}    listener
+    \    BuiltIn.Wait Until Keyword Succeeds    120x    1s    SxpLib.Verify Connection    ${cmp_version}    listener
     \    ...    127.0.0.2    64999    127.0.0.1
-    \    BuiltIn.Wait Until Keyword Succeeds    15    1    SxpLib.Verify Connection    ${cmp_version}    speaker
+    \    BuiltIn.Wait Until Keyword Succeeds    120x    1s    SxpLib.Verify Connection    ${cmp_version}    speaker
     \    ...    127.0.0.1    64999    127.0.0.2
     \    BuiltIn.Log    OK ${r_version}:listener ${version}:speaker
     \    SxpLib.Add Connection    ${version}    listener    127.0.0.2    64999    127.0.0.3
     \    ...    ${PASSWORD}
     \    SxpLib.Add Connection    ${r_version}    speaker    127.0.0.3    64999    127.0.0.2
     \    ...    ${PASSWORD}
-    \    BuiltIn.Wait Until Keyword Succeeds    15    1    SxpLib.Verify Connection    ${cmp_version}    listener
+    \    BuiltIn.Wait Until Keyword Succeeds    120x    1s    SxpLib.Verify Connection    ${cmp_version}    listener
     \    ...    127.0.0.2    64999    127.0.0.3
-    \    BuiltIn.Wait Until Keyword Succeeds    15    1    SxpLib.Verify Connection    ${cmp_version}    speaker
+    \    BuiltIn.Wait Until Keyword Succeeds    120x    1s    SxpLib.Verify Connection    ${cmp_version}    speaker
     \    ...    127.0.0.3    64999    127.0.0.2
     \    BuiltIn.Log    OK ${version}:listener ${r_version}:speaker
     \    BuiltIn.Run Keyword If    '${version}' == 'version4' and '${r_version}' == 'version4'    Test Both    ${version}    ${r_version}    ${PASSWORD}
@@ -93,9 +93,9 @@ Test Both
     ${cmp_version}    Sxp.Lower Version    ${r_version}    ${version}
     SxpLib.Add Connection    ${r_version}    both    127.0.0.3    64999    127.0.0.1    ${PASSWORD}
     SxpLib.Add Connection    ${version}    both    127.0.0.1    64999    127.0.0.3    ${PASSWORD}
-    BuiltIn.Wait Until Keyword Succeeds    15    1    SxpLib.Verify Connection    ${cmp_version}    both    127.0.0.3
+    BuiltIn.Wait Until Keyword Succeeds    120x    1s    SxpLib.Verify Connection    ${cmp_version}    both    127.0.0.3
     ...    64999    127.0.0.1
-    BuiltIn.Wait Until Keyword Succeeds    15    1    SxpLib.Verify Connection    ${cmp_version}    both    127.0.0.1
+    BuiltIn.Wait Until Keyword Succeeds    120x    1s    SxpLib.Verify Connection    ${cmp_version}    both    127.0.0.1
     ...    64999    127.0.0.3
     BuiltIn.Log    OK ${r_version}:both ${version}:both
 

diff --git a/csit/suites/sxp/basic/040_Custom_Passwords.robot b/csit/suites/sxp/basic/040_Custom_Passwords.robot
index a91710f451f4d31276ede40336f62ccc2ecbc98b..a58e0b6b4b9600dfdcb9571f8252ea65bea0ddeb 100644 (file)
--- a/csit/suites/sxp/basic/040_Custom_Passwords.robot
+++ b/csit/suites/sxp/basic/040_Custom_Passwords.robot
@@ -54,21 +54,21 @@ Test Mode
     [Documentation]    Setup connection Speaker => Listener / Listener => Speaker / Both <=> Both for specific versions
     Add Connection    ${version}    ${mode_local}    127.0.0.3    64999    127.0.0.1    CUSTOM
     Add Connection    ${version}    ${mode_remote}    127.0.0.1    64999    127.0.0.3    ${EMPTY}
-    Wait Until Keyword Succeeds    15    1    Verify Connection    ${version}    ${mode_local}    127.0.0.3
+    Wait Until Keyword Succeeds    120x    1s    Verify Connection    ${version}    ${mode_local}    127.0.0.3
     ...    64999    127.0.0.1
-    Wait Until Keyword Succeeds    15    1    Verify Connection    ${version}    ${mode_remote}    127.0.0.1
+    Wait Until Keyword Succeeds    120x    1s    Verify Connection    ${version}    ${mode_remote}    127.0.0.1
     ...    64999    127.0.0.3
     Add Connection    ${version}    ${mode_local}    127.0.0.2    64999    127.0.0.1    ${EMPTY}
     Add Connection    ${version}    ${mode_remote}    127.0.0.1    64999    127.0.0.2    ${EMPTY}
-    Wait Until Keyword Succeeds    15    1    Verify Connection    ${version}    ${mode_local}    127.0.0.2
+    Wait Until Keyword Succeeds    120x    1s    Verify Connection    ${version}    ${mode_local}    127.0.0.2
     ...    64999    127.0.0.1
-    Wait Until Keyword Succeeds    15    1    Verify Connection    ${version}    ${mode_remote}    127.0.0.1
+    Wait Until Keyword Succeeds    120x    1s    Verify Connection    ${version}    ${mode_remote}    127.0.0.1
     ...    64999    127.0.0.2
     Add Connection    ${version}    ${mode_local}    127.0.0.3    64999    127.0.0.2    CUSTOM_2
     Add Connection    ${version}    ${mode_remote}    127.0.0.2    64999    127.0.0.3    CUSTOM_2
-    Wait Until Keyword Succeeds    15    1    Verify Connection    ${version}    ${mode_local}    127.0.0.3
+    Wait Until Keyword Succeeds    120x    1s    Verify Connection    ${version}    ${mode_local}    127.0.0.3
     ...    64999    127.0.0.2
-    Wait Until Keyword Succeeds    15    1    Verify Connection    ${version}    ${mode_remote}    127.0.0.2
+    Wait Until Keyword Succeeds    120x    1s    Verify Connection    ${version}    ${mode_remote}    127.0.0.2
     ...    64999    127.0.0.3
 
 Clean Nodes

diff --git a/csit/suites/sxp/basic/040_SSL_Passwords.robot b/csit/suites/sxp/basic/040_SSL_Passwords.robot
index 15b7f480066e3cf0b454a40c680066cccc63fc63..08a5b42cc284624303e5edba80c9514e18d8474b 100644 (file)
--- a/csit/suites/sxp/basic/040_SSL_Passwords.robot
+++ b/csit/suites/sxp/basic/040_SSL_Passwords.robot
@@ -21,25 +21,25 @@ SSL ConectivityCase 1
     [Tags]    SXP    SSL
     SxpLib.Add Connection    ${version}    speaker    127.0.0.2    64999    127.0.0.1    security_mode=TLS
     SxpLib.Add Connection    ${version}    listener    127.0.0.1    64999    127.0.0.2    security_mode=TLS
-    BuiltIn.Wait Until Keyword Succeeds    60    1    SxpLib.Verify Connection    ${version}    speaker    127.0.0.2
+    BuiltIn.Wait Until Keyword Succeeds    120x    1s    SxpLib.Verify Connection    ${version}    speaker    127.0.0.2
     ...    64999    127.0.0.1
-    BuiltIn.Wait Until Keyword Succeeds    60    1    SxpLib.Verify Connection    ${version}    listener    127.0.0.1
+    BuiltIn.Wait Until Keyword Succeeds    120x    1s    SxpLib.Verify Connection    ${version}    listener    127.0.0.1
     ...    64999    127.0.0.2
     SxpLib.Clean Connections    127.0.0.1
     SxpLib.Clean Connections    127.0.0.2
     SxpLib.Add Connection    ${version}    listener    127.0.0.2    64999    127.0.0.1    security_mode=TLS
     SxpLib.Add Connection    ${version}    speaker    127.0.0.1    64999    127.0.0.2    security_mode=TLS
-    BuiltIn.Wait Until Keyword Succeeds    60    1    SxpLib.Verify Connection    ${version}    listener    127.0.0.2
+    BuiltIn.Wait Until Keyword Succeeds    120x    1s    SxpLib.Verify Connection    ${version}    listener    127.0.0.2
     ...    64999    127.0.0.1
-    BuiltIn.Wait Until Keyword Succeeds    60    1    SxpLib.Verify Connection    ${version}    speaker    127.0.0.1
+    BuiltIn.Wait Until Keyword Succeeds    120x    1s    SxpLib.Verify Connection    ${version}    speaker    127.0.0.1
     ...    64999    127.0.0.2
     SxpLib.Clean Connections    127.0.0.1
     SxpLib.Clean Connections    127.0.0.2
     SxpLib.Add Connection    ${version}    both    127.0.0.2    64999    127.0.0.1    security_mode=TLS
     SxpLib.Add Connection    ${version}    both    127.0.0.1    64999    127.0.0.2    security_mode=TLS
-    BuiltIn.Wait Until Keyword Succeeds    60    1    SxpLib.Verify Connection    ${version}    both    127.0.0.2
+    BuiltIn.Wait Until Keyword Succeeds    120x    1s    SxpLib.Verify Connection    ${version}    both    127.0.0.2
     ...    64999    127.0.0.1
-    BuiltIn.Wait Until Keyword Succeeds    60    1    SxpLib.Verify Connection    ${version}    both    127.0.0.1
+    BuiltIn.Wait Until Keyword Succeeds    120x    1s    SxpLib.Verify Connection    ${version}    both    127.0.0.1
     ...    64999    127.0.0.2
 
 SSL ConectivityCase 2
@@ -101,29 +101,29 @@ SSL ConectivityCase 4
     [Tags]    SXP    SSL
     SxpLib.Add Connection    ${version}    listener    127.0.0.2    64999    127.0.0.1    security_mode=TLS
     SxpLib.Add Connection    ${version}    speaker    127.0.0.1    64999    127.0.0.2    security_mode=TLS
-    BuiltIn.Wait Until Keyword Succeeds    60    1    SxpLib.Verify Connection    ${version}    listener    127.0.0.2
+    BuiltIn.Wait Until Keyword Succeeds    120x    1s    SxpLib.Verify Connection    ${version}    listener    127.0.0.2
     ...    64999    127.0.0.1
-    BuiltIn.Wait Until Keyword Succeeds    60    1    SxpLib.Verify Connection    ${version}    speaker    127.0.0.1
+    BuiltIn.Wait Until Keyword Succeeds    120x    1s    SxpLib.Verify Connection    ${version}    speaker    127.0.0.1
     ...    64999    127.0.0.2
     SxpLib.Add Connection    ${version}    listener    127.0.0.3    64999    127.0.0.1    paswd
     SxpLib.Add Connection    ${version}    speaker    127.0.0.1    64999    127.0.0.3    paswd
-    BuiltIn.Wait Until Keyword Succeeds    60    1    SxpLib.Verify Connection    ${version}    listener    127.0.0.3
+    BuiltIn.Wait Until Keyword Succeeds    120x    1s    SxpLib.Verify Connection    ${version}    listener    127.0.0.3
     ...    64999    127.0.0.1
-    BuiltIn.Wait Until Keyword Succeeds    60    1    SxpLib.Verify Connection    ${version}    speaker    127.0.0.1
+    BuiltIn.Wait Until Keyword Succeeds    120x    1s    SxpLib.Verify Connection    ${version}    speaker    127.0.0.1
     ...    64999    127.0.0.3
     SxpLib.Add Connection    ${version}    listener    127.0.0.4    64999    127.0.0.1
     SxpLib.Add Connection    ${version}    speaker    127.0.0.1    64999    127.0.0.4
-    BuiltIn.Wait Until Keyword Succeeds    60    1    SxpLib.Verify Connection    ${version}    listener    127.0.0.4
+    BuiltIn.Wait Until Keyword Succeeds    120x    1s    SxpLib.Verify Connection    ${version}    listener    127.0.0.4
     ...    64999    127.0.0.1
-    BuiltIn.Wait Until Keyword Succeeds    60    1    SxpLib.Verify Connection    ${version}    speaker    127.0.0.1
+    BuiltIn.Wait Until Keyword Succeeds    120x    1s    SxpLib.Verify Connection    ${version}    speaker    127.0.0.1
     ...    64999    127.0.0.4
     SxpLib.Add Connection    ${version}    speaker    127.0.0.5    64999    127.0.0.1    security_mode=TLS
     SxpLib.Add Connection    ${version}    listener    127.0.0.1    64999    127.0.0.5    security_mode=TLS
-    BuiltIn.Wait Until Keyword Succeeds    60    1    SxpLib.Verify Connection    ${version}    speaker    127.0.0.5
+    BuiltIn.Wait Until Keyword Succeeds    120x    1s    SxpLib.Verify Connection    ${version}    speaker    127.0.0.5
     ...    64999    127.0.0.1
-    BuiltIn.Wait Until Keyword Succeeds    60    1    SxpLib.Verify Connection    ${version}    listener    127.0.0.1
+    BuiltIn.Wait Until Keyword Succeeds    120x    1s    SxpLib.Verify Connection    ${version}    listener    127.0.0.1
     ...    64999    127.0.0.5
-    BuiltIn.Wait Until Keyword Succeeds    60    1    Verify Topology Bindings    5
+    BuiltIn.Wait Until Keyword Succeeds    120x    1s    Verify Topology Bindings    5
 
 *** Keywords ***
 Setup SXP Environment Local
@@ -153,6 +153,7 @@ Setup SXP Environment Local
     \    ${SSL}    BuiltIn.Create Dictionary    truststore=${ssl_stores}/csit-truststore-${node}    keystore=${ssl_stores}/csit-keystore-${node}    password=${password}
     \    ${rnd_retry_time} =    BuiltIn.Evaluate    random.randint(1, 5)    modules=random
     \    SxpLib.Add Node    127.0.0.${node}    ${EMPTY}    ssl_stores=${SSL}    retry_open_timer=${rnd_retry_time}
+    \    BuiltIn.Wait Until Keyword Succeeds    20x    10s    SxpLib.Check Node started    127.0.0.${node}
     \    SxpLib.Add Bindings    ${node}00    1.1.1.${node}/32    127.0.0.${node}
     \    SxpLib.Add Bindings    ${node}00    2.2.2.${node}/32    127.0.0.${node}