Add bug references to bgp-ingest suites 98/33898/21
authorRadovan Sajben <rsajben@cisco.com>
Tue, 2 Feb 2016 13:16:59 +0000 (14:16 +0100)
committerRadovan Sajben <rsajben@cisco.com>
Thu, 11 Feb 2016 15:30:43 +0000 (16:30 +0100)
- implement Report_Failure_And_Point_To_Linked_Bugs to Utils
- update SetupUtils library to use the keyword
- update BGP ingest suites to use such feature
- CHECK tests not to be skipped after WAIT test failures

Change-Id: I3a8598b08fb4c4d850f506cfd21860fe3c93fedf
Signed-off-by: Radovan Sajben <rsajben@cisco.com>
csit/libraries/SetupUtils.robot
csit/libraries/Utils.robot
csit/suites/bgpcep/bgpingest/bgp_app_peer_prefixcount.robot
csit/suites/bgpcep/bgpingest/manypeers_changecount.robot
csit/suites/bgpcep/bgpingest/manypeers_prefixcount.robot
csit/suites/bgpcep/bgpingest/singlepeer_changecount.robot
csit/suites/bgpcep/bgpingest/singlepeer_prefixcount.robot

index b042c48515e371c3a0b00691e01be6cc1c4ec29c..573677787e45ec0fd177ba960e30d33d11a5c113 100644 (file)
@@ -23,3 +23,12 @@ Setup_Test_With_Logging_And_Without_Fast_Failing
     ...    Needed if the recommended default is to be overriden.
     FailFast.Run_Even_When_Failing_Fast
     BuiltIn.Run Keyword And Ignore Error    KarafKeywords.Log_Testcase_Start_To_Controller_Karaf
+
+Teardown_Test_Show_Bugs_And_Start_Fast_Failing_If_Test_Failed
+    [Documentation]    Test case teardown. Show linked bugs and start fast failing in case of failure.
+    BuiltIn.Run_Keyword_If_Test_Failed    BuiltIn.Set_Suite_Variable    ${SuiteFastFail}    True
+    Utils.Report_Failure_And_Point_To_Linked_Bugs
+
+Teardown_Test_Show_Bugs_If_Test_Failed
+    [Documentation]    Test case teardown. Show linked bugs in case of failure.
+    Utils.Report_Failure_And_Point_To_Linked_Bugs
index ee16ad0f7d42b395aa94ff95eeb1cef82a7bc593..0d6f09ec2b4166473750357969350f86cd4f1c1d 100644 (file)
@@ -4,6 +4,8 @@ Library           SSHLibrary
 Library           String
 Library           DateTime
 Library           Process
+Library           Collections
+Library           RequestsLibrary
 Library           ./UtilLibrary.py
 Resource          KarafKeywords.robot
 Variables         ../variables/Variables.py
@@ -63,14 +65,37 @@ Report_Failure_Due_To_Bug
     [Arguments]    ${number}
     [Documentation]    Report that a test failed due to a known Bugzilla bug whose
     ...    number is provided as an argument.
-    ...    This must be used in the [Teardown] setting of the affected test
+    ...    Not FAILED (incl. SKIPPED) test are not reported.
+    ...    This keyword must be used in the [Teardown] setting of the affected test
     ...    or as the first line of the test if FastFail module is not being
     ...    used. It reports the URL of the bug on console and also puts it
     ...    into the Robot log file.
+    ${test_skipped}=    BuiltIn.Evaluate    len(re.findall('SKIPPED', '''${TEST_MESSAGE}''')) > 0    modules=re
+    BuiltIn.Return From Keyword If    ('${TEST_STATUS}' != 'FAIL') or ${test_skipped}
+    ${newline}=    BuiltIn.Evaluate    chr(10)
     ${msg}=    BuiltIn.Set_Variable    This test fails due to https://bugs.opendaylight.org/show_bug.cgi?id=${number}
+    BuiltIn.Set Test Message    ${msg}${newline}${newline}${TEST_MESSAGE}
+    BuiltIn.Log    ${msg}
+
+Report_Failure_And_Point_To_Linked_Bugs
+    [Documentation]    Report that a test failed and point to linked Bugzilla bug(s).
+    ...    Linked bugs must contain the ${reference} inside comments (workaround
+    ...    becasue of currently missing suitable field for external references and
+    ...    not correctly working the CONTENT MATCHES filter).
+    ...    Not FAILED (incl. SKIPPED) test are not reported.
+    ...    This keyword must be used in the [Teardown] setting of the affected test
+    ...    or as the first line of the test if FastFail module is not being
+    ...    used. It reports the URL of the bug on console and also puts it
+    ...    into the Robot log file.
+    ${test_skipped}=    BuiltIn.Evaluate    len(re.findall('SKIPPED', '''${TEST_MESSAGE}''')) > 0    modules=re
+    BuiltIn.Return From Keyword If    ('${TEST_STATUS}' != 'FAIL') or ${test_skipped}
     ${newline}=    BuiltIn.Evaluate    chr(10)
-    Run Keyword If    "${TEST STATUS}"=="FAIL"    BuiltIn.Set Test Message    ${msg}${newline}${newline}${TEST_MESSAGE}
-    Run Keyword If    "${TEST STATUS}"=="FAIL"    BuiltIn.Log    ${msg}
+    ${reference}=    String.Replace_String_Using_Regexp    ${SUITE_NAME}_${TEST_NAME}    [ /\.-]    _
+    ${reference}=    String.Convert_To_Lowercase    ${reference}
+    ${msg}=    BuiltIn.Set_Variable    ... click for list of related bugs or create a new one if needed (with the${newline}"${reference}"${newline}reference somewhere inside)
+    ${bugs}=    BuiltIn.Set_Variable    "https://bugs.opendaylight.org/buglist.cgi?f1=longdesc&o1=substring&v1=${reference}&order=bug_status"
+    BuiltIn.Set Test Message    ${msg}${newline}${bugs}${newline}${newline}${TEST_MESSAGE}
+    BuiltIn.Log    ${msg}${newline}${bugs}
 
 Ensure All Nodes Are In Response
     [Arguments]    ${URI}    ${node_list}
index 375b499a2177862bd256244037ef1fc3524ee5d7..1fb1ced9bdd05fb7d9be2ac6a5704bb4912ba8d9 100644 (file)
@@ -26,8 +26,8 @@ Documentation     BGP performance of ingesting from 1 BGP application peer
 ...               Bug 4791 - BGPSessionImpl: Failed to send message Update logged even all UPDATE mesages received by iBGP peer
 Suite Setup       Setup_Everything
 Suite Teardown    Teardown_Everything
-Test Setup        Setup_Test_With_Logging_And_Without_Fast_Failing
-Test Teardown     FailFast.Start_Failing_Fast_If_This_Failed
+Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
+Test Teardown     SetupUtils.Teardown_Test_Show_Bugs_And_Start_Fast_Failing_If_Test_Failed
 Force Tags        critical
 Library           SSHLibrary    timeout=10s
 Library           RequestsLibrary
@@ -96,13 +96,11 @@ BGP_Application_Peer_Prefill_Routes
 Wait_For_Ipv4_Topology_Is_Prefilled
     [Documentation]    Wait until example-ipv4-topology reaches the target prfix count.
     BuiltIn.Wait Until Keyword Succeeds    ${bgp_filling_timeout}    10s    PrefixCounting.Check_Ipv4_Topology_Count    ${PREFILL}
-    [Teardown]    Report_Failure_Due_To_Bug    4689
 
 Check_Bgp_Peer_Updates_For_Prefilled_Routes
     [Documentation]    Count the routes introduced by updates.
     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Fast_Failing
     BuiltIn.Wait Until Keyword Succeeds    ${bgp_filling_timeout}    1s    Check_File_For_Word_Count    bgp_peer.log    total_received_nlri_prefix_counter: ${PREFILL}    2
-    [Teardown]    Report_Failure_Due_To_Bug    4689
 
 BGP_Application_Peer_Introduce_Single_Routes
     [Documentation]    Start BGP application peer tool and introduce routes.
@@ -114,13 +112,11 @@ BGP_Application_Peer_Introduce_Single_Routes
 Wait_For_Ipv4_Topology_Is_Filled
     [Documentation]    Wait until example-ipv4-topology reaches the target prfix count.
     BuiltIn.Wait Until Keyword Succeeds    ${bgp_filling_timeout}    10s    PrefixCounting.Check_Ipv4_Topology_Count    ${COUNT_APP_PEER_PREFIX_COUNT}
-    [Teardown]    Report_Failure_Due_To_Bug    4689
 
 Check_Bgp_Peer_Updates_For_All_Routes
     [Documentation]    Count the routes introduced by updates.
     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Fast_Failing
     BuiltIn.Wait Until Keyword Succeeds    ${bgp_filling_timeout}    1s    Check_File_For_Word_Count    bgp_peer.log    total_received_nlri_prefix_counter: ${COUNT_APP_PEER_PREFIX_COUNT}    2
-    [Teardown]    Report_Failure_Due_To_Bug    4689
 
 Disconnect_BGP_Peer
     [Documentation]    Stop BGP peer tool
@@ -136,6 +132,7 @@ Reconnect_BGP_Peer
 
 Check_Bgp_Peer_Updates_For_Reintroduced_Routes
     [Documentation]    Count the routes introduced by updates.
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Fast_Failing
     BuiltIn.Wait Until Keyword Succeeds    ${bgp_filling_timeout}    1s    Check_File_For_Word_Count    bgp_peer.log    total_received_nlri_prefix_counter: ${COUNT_APP_PEER_PREFIX_COUNT}    2
 
 BGP_Application_Peer_Delete_All_Routes
@@ -157,7 +154,6 @@ Check_Bgp_Peer_Updates_For_Prefix_Withdrawals
     [Documentation]    Count the routes withdrawn by updates.
     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Fast_Failing
     BuiltIn.Wait Until Keyword Succeeds    ${bgp_emptying_timeout}    1s    Check_File_For_Word_Count    bgp_peer.log    total_received_withdrawn_prefix_counter: ${COUNT_APP_PEER_PREFIX_COUNT}    2
-    [Teardown]    Report_Failure_Due_To_Bug    4689
 
 Stop_BGP_Peer
     [Documentation]    Stop BGP peer tool
@@ -175,10 +171,9 @@ Delete_Bgp_Application_Peer_Configuration
     ${template_as_string}=    BuiltIn.Set_Variable    {'NAME': 'example-bgp-peer-app'}
     ConfigViaRestconf.Delete_Xml_Template_Folder_Config_Via_Restconf    ${BGP_VARIABLES_FOLDER}${/}bgp_application_peer    ${template_as_string}
 
-Check_Controller_Logs
+Check_Bug_4791
     [Documentation]    Check controller's log for errors
     Check Karaf Log File Does Not Have Messages    ${ODL_SYSTEM_IP}    Failed to send message Update
-    [Teardown]    Report_Failure_Due_To_Bug    4791
 
 *** Keywords ***
 Setup_Everything
index 71126b0e5a3901d2fcee0beaf6a4b7b87844dc33..545eca8946e2cea03cd848e299c0e45bc5e73add 100644 (file)
@@ -44,7 +44,7 @@ Documentation     BGP performance of ingesting from many iBGP peers, data change
 Suite Setup       Setup_Everything
 Suite Teardown    Teardown_Everything
 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Fast_Failing
-Test Teardown     FailFast.Start_Failing_Fast_If_This_Failed
+Test Teardown     SetupUtils.Teardown_Test_Show_Bugs_And_Start_Fast_Failing_If_Test_Failed
 Library           DateTime
 Library           RequestsLibrary
 Library           SSHLibrary    timeout=10s
@@ -127,6 +127,7 @@ Wait_For_Stable_Talking_Ipv4_Topology
 Check_Talking_Ipv4_Topology_Count
     [Documentation]    Count the routes in example-ipv4-topology and fail if the count is not correct.
     [Tags]    critical
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
     PrefixCounting.Check_Ipv4_Topology_Count    ${COUNT_CHANGE_COUNT_MANY}
 
 Kill_Talking_BGP_Speakers
@@ -136,7 +137,7 @@ Kill_Talking_BGP_Speakers
     BGPSpeaker.Kill_BGP_Speaker
     FailFast.Do_Not_Fail_Fast_From_Now_On
     # NOTE: It is still possible to remain failing fast, if both previous and this test have failed.
-    [Teardown]    FailFast.Do_Not_Start_Failing_If_This_Failed
+    [Teardown]    SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
 
 Wait_For_Stable_Ipv4_Topology_After_Talking
     [Documentation]    Wait until example-ipv4-topology becomes stable again.
@@ -146,6 +147,7 @@ Wait_For_Stable_Ipv4_Topology_After_Talking
 Check_For_Empty_Ipv4_Topology_After_Talking
     [Documentation]    Example-ipv4-topology should be empty now.
     [Tags]    critical
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
     PrefixCounting.Check_Ipv4_Topology_Is_Empty
 
 Restore_Karaf_Logging_Levels
index ae795679e8e19b7e6e31195045f720ca6c0741d4..072ca3969a236ed2377ab0a3681732c36de9ed50 100644 (file)
@@ -41,7 +41,7 @@ Documentation     BGP performance of ingesting from many iBGP peers, data change
 Suite Setup       Setup_Everything
 Suite Teardown    Teardown_Everything
 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Fast_Failing
-Test Teardown     FailFast.Start_Failing_Fast_If_This_Failed
+Test Teardown     SetupUtils.Teardown_Test_Show_Bugs_And_Start_Fast_Failing_If_Test_Failed
 Library           DateTime
 Library           RequestsLibrary
 Library           SSHLibrary    timeout=10s
@@ -113,6 +113,7 @@ Wait_For_Stable_Talking_Ipv4_Topology
 Check_Talking_Ipv4_Topology_Count
     [Documentation]    Count the routes in example-ipv4-topology and fail if the count is not correct.
     [Tags]    critical
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
     PrefixCounting.Check_Ipv4_Topology_Count    ${COUNT_PREFIX_COUNT_MANY}
 
 Kill_Talking_BGP_Speakers
@@ -121,7 +122,7 @@ Kill_Talking_BGP_Speakers
     BGPSpeaker.Kill_BGP_Speaker
     FailFast.Do_Not_Fail_Fast_From_Now_On
     # NOTE: It is still possible to remain failing fast, if both previous and this test have failed.
-    [Teardown]    FailFast.Do_Not_Start_Failing_If_This_Failed
+    [Teardown]    SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
 
 Wait_For_Stable_Ipv4_Topology_After_Talking
     [Documentation]    Wait until example-ipv4-topology becomes stable again.
@@ -134,6 +135,7 @@ Wait_For_Stable_Ipv4_Topology_After_Talking
 Check_For_Empty_Ipv4_Topology_After_Talking
     [Documentation]    Example-ipv4-topology should be empty now.
     [Tags]    critical
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
     PrefixCounting.Check_Ipv4_Topology_Is_Empty
 
 Restore_Karaf_Logging_Levels
index 43303921af477190829722a48340e727816db467..ff757a4c7dd5339296413445e91fe1f56f04f84e 100644 (file)
@@ -32,7 +32,7 @@ Documentation     BGP performance of ingesting from 1 iBGP peer, data change cou
 Suite Setup       Setup_Everything
 Suite Teardown    Teardown_Everything
 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Fast_Failing
-Test Teardown     FailFast.Start_Failing_Fast_If_This_Failed
+Test Teardown     SetupUtils.Teardown_Test_Show_Bugs_And_Start_Fast_Failing_If_Test_Failed
 Library           SSHLibrary    timeout=10s
 Library           RequestsLibrary
 Variables         ${CURDIR}/../../../variables/Variables.py
@@ -110,6 +110,7 @@ Wait_For_Stable_Talking_Ipv4_Topology
 Check_Talking_Ipv4_Topology_Count
     [Documentation]    Count the routes in example-ipv4-topology and fail if the count is not correct.
     [Tags]    critical
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
     PrefixCounting.Check_Ipv4_Topology_Count    ${COUNT_CHANGE_COUNT_SINGLE}
 
 Kill_Talking_BGP_Speaker
@@ -120,7 +121,7 @@ Kill_Talking_BGP_Speaker
     BGPSpeaker.Kill_BGP_Speaker
     FailFast.Do_Not_Fail_Fast_From_Now_On
     # NOTE: It is still possible to remain failing fast, if both previous and this test have failed.
-    [Teardown]    FailFast.Do_Not_Start_Failing_If_This_Failed
+    [Teardown]    SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
 
 Store_Results_For_Talking_BGP_Speaker
     [Documentation]    Store results for plotting
@@ -138,6 +139,7 @@ Wait_For_Stable_Ipv4_Topology_After_Talking
 Check_For_Empty_Ipv4_Topology_After_Talking
     [Documentation]    Example-ipv4-topology should be empty now.
     [Tags]    critical
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
     PrefixCounting.Check_Ipv4_Topology_Is_Empty
 
 Start_Listening_BGP_Speaker
@@ -157,6 +159,7 @@ Wait_For_Stable_Listening_Ipv4_Topology
 Check_Listening_Ipv4_Topology_Count
     [Documentation]    Count the routes in example-ipv4-topology and fail if the count is not correct.
     [Tags]    critical
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
     PrefixCounting.Check_Ipv4_Topology_Count    ${COUNT_CHANGE_COUNT_SINGLE}
 
 Kill_Listening_BGP_Speaker
@@ -167,7 +170,7 @@ Kill_Listening_BGP_Speaker
     BGPSpeaker.Kill_BGP_Speaker
     FailFast.Do_Not_Fail_Fast_From_Now_On
     # NOTE: It is still possible to remain failing fast, if both previous and this test have failed.
-    [Teardown]    FailFast.Do_Not_Start_Failing_If_This_Failed
+    [Teardown]    SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
 
 Store_Results_For_Listening_BGP_Speaker
     [Documentation]    Store results for plotting
@@ -185,6 +188,7 @@ Wait_For_Stable_Ipv4_Topology_After_Listening
 Check_For_Empty_Ipv4_Topology_After_Listening
     [Documentation]    Example-ipv4-topology should be empty now.
     [Tags]    critical
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
     PrefixCounting.Check_Ipv4_Topology_Is_Empty
 
 Restore_Karaf_Logging_Levels
index 5f2b161dd21e068944ff27c89166192a1693d4b5..adc3434441fd31972b27b3d066c8d4b034290e39 100644 (file)
@@ -25,7 +25,7 @@ Documentation     BGP performance of ingesting from 1 iBGP peer, data change cou
 Suite Setup       Setup_Everything
 Suite Teardown    Teardown_Everything
 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Fast_Failing
-Test Teardown     FailFast.Start_Failing_Fast_If_This_Failed
+Test Teardown     SetupUtils.Teardown_Test_Show_Bugs_And_Start_Fast_Failing_If_Test_Failed
 Library           SSHLibrary    timeout=10s
 Library           RequestsLibrary
 Variables         ${CURDIR}/../../../variables/Variables.py
@@ -92,6 +92,7 @@ Wait_For_Stable_Talking_Ipv4_Topology
 Check_Talking_Ipv4_Topology_Count
     [Documentation]    Count the routes in example-ipv4-topology and fail if the count is not correct.
     [Tags]    critical
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
     PrefixCounting.Check_Ipv4_Topology_Count    ${COUNT_PREFIX_COUNT_SINGLE}
 
 Kill_Talking_BGP_Speaker
@@ -101,7 +102,7 @@ Kill_Talking_BGP_Speaker
     BGPSpeaker.Kill_BGP_Speaker
     FailFast.Do_Not_Fail_Fast_From_Now_On
     # NOTE: It is still possible to remain failing fast, if both previous and this test have failed.
-    [Teardown]    FailFast.Do_Not_Start_Failing_If_This_Failed
+    [Teardown]    SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
 
 Store_Results_For_Talking_BGP_Speaker
     [Documentation]    Store results for plotting
@@ -122,6 +123,7 @@ Wait_For_Stable_Ipv4_Topology_After_Talking
 Check_For_Empty_Ipv4_Topology_After_Talking
     [Documentation]    Example-ipv4-topology should be empty now.
     [Tags]    critical
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
     PrefixCounting.Check_Ipv4_Topology_Is_Empty
 
 Start_Listening_BGP_Speaker
@@ -140,6 +142,7 @@ Wait_For_Stable_Listening_Ipv4_Topology
 Check_Listening_Ipv4_Topology_Count
     [Documentation]    Count the routes in example-ipv4-topology and fail if the count is not correct.
     [Tags]    critical
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
     PrefixCounting.Check_Ipv4_Topology_Count    ${COUNT_PREFIX_COUNT_SINGLE}
 
 Kill_Listening_BGP_Speaker
@@ -149,7 +152,7 @@ Kill_Listening_BGP_Speaker
     BGPSpeaker.Kill_BGP_Speaker
     FailFast.Do_Not_Fail_Fast_From_Now_On
     # NOTE: It is still possible to remain failing fast, if both previous and this test have failed.
-    [Teardown]    FailFast.Do_Not_Start_Failing_If_This_Failed
+    [Teardown]    SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
 
 Store_Results_For_Listening_BGP_Speaker
     [Documentation]    Store results for plotting
@@ -167,6 +170,7 @@ Wait_For_Stable_Ipv4_Topology_After_Listening
 Check_For_Empty_Ipv4_Topology_After_Listening
     [Documentation]    Example-ipv4-topology should be empty now.
     [Tags]    critical
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
     PrefixCounting.Check_Ipv4_Topology_Is_Empty
 
 Restore_Karaf_Logging_Levels