X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=csit%2Flibraries%2FCoe.robot;h=c0ab41652dccd002c26ab2a4ae329b3b7d87c44b;hb=69c34cf13db1d6db66ba7a05ad1ce8d08c4be8b4;hp=90034b4d2fcee7f7a5abde196129fb4822a53c59;hpb=029fd685ab7e9ad117a667523974f17c0150329f;p=integration%2Ftest.git

diff --git a/csit/libraries/Coe.robot b/csit/libraries/Coe.robot
index 90034b4d2f..c0ab41652d 100644
--- a/csit/libraries/Coe.robot
+++ b/csit/libraries/Coe.robot
@@ -21,17 +21,12 @@ ${CONFIG_FILE}    /etc/cni/net.d/odlovs-cni.conf
 ${CONFIG_FILE_TEMPLATE}    ${CURDIR}/../variables/coe/odlovs-cni.conf.j2
 ${HOST_INVENTORY}    ${CURDIR}/../variables/coe/hosts.yaml
 ${K8s_MASTER_IP}    ${TOOLS_SYSTEM_1_IP}
-${K8s_MINION1_IP}    ${TOOLS_SYSTEM_2_IP}
-${K8s_MINION2_IP}    ${TOOLS_SYSTEM_3_IP}
-${K8s_MINION3_IP}    ${TOOLS_SYSTEM_4_IP}
-${K8s_MINION4_IP}    ${TOOLS_SYSTEM_5_IP}
 ${HOSTS_FILE_TEMPLATE}    ${CURDIR}/../variables/coe/minions_template.yaml
-${NODE_READY_STATUS}    \\sReady
+${NODE_READY_STATUS}    \\sReady    # The check using this variable should not mess up with NotReady
 ${PLAYBOOK_FILE}    ${CURDIR}/../variables/coe/coe_play.yaml
 ${POD_RUNNING_STATUS}    \\sRunning
 ${VARIABLES_PATH}    ${CURDIR}/../variables/coe
 ${WATCHER_COE}    ${CURDIR}/../variables/coe/coe.yaml
-@{NODE_IPs}       ${K8s_MASTER_IP}    ${K8s_MINION1_IP}    ${K8s_MINION2_IP}    ${K8s_MINION3_IP}    ${K8s_MINION4_IP}
 @{COE_DIAG_SERVICES}    OPENFLOW    IFM    ITM    DATASTORE    ELAN    OVSDB
 
 *** Keywords ***
@@ -39,6 +34,7 @@ Coe Suite Setup
     [Documentation]    COE project requires start suite to be executed only for the first test suite.This keyword find the current suite,compares it with the stored first suite value and executes Coe.Start suite only if the cuurent suite is equal to the first suite.
     ToolsSystem.Get Tools System Nodes Data
     Coe.Set Connection ids and Bridge
+    Coe.Derive Coe Data Models
     ${current suite}    ${suite names updated}    Extract current suite name
     ${first_suite} =    Set Variable    ${suite names updated[0]}
     ${status} =    BuiltIn.Evaluate    '${first_suite}' == '${current suite}'
@@ -53,7 +49,6 @@ Start Suite
     Coe.Label Nodes
     BuiltIn.Wait Until Keyword Succeeds    60    2    ClusterManagement.Check Status Of Services Is OPERATIONAL    @{COE_DIAG_SERVICES}
     BuiltIn.Wait Until Keyword Succeeds    85    2    Genius.Verify Tunnel Status as UP    default-transport-zone
-    Coe.Derive Coe Data Models
 
 Set Connection ids and Bridge
     [Documentation]    Sets the connection ids for all the nodes and get the bridge from configuration file .
@@ -61,8 +56,8 @@ Set Connection ids and Bridge
     \    SSHLibrary.Switch Connection    ${conn_id}
     \    SSHKeywords.Flexible_SSH_Login    ${DEFAULT_USER}    ${DEFAULT_PASSWORD}
     ${file} =    OperatingSystem.Get File    ${CONFIG_FILE_TEMPLATE}
-    ${ovs bridge output}    ${bridge} =    Should Match Regexp    ${file}    "ovsBridge": "(\\w.*)"
-    Set Suite Variable    ${bridge}
+    ${ovs bridge output}    ${bridge} =    BuiltIn.Should Match Regexp    ${file}    "ovsBridge": "(\\w.*)"
+    BuiltIn.Set Suite Variable    ${bridge}
 
 Configuration Playbook
     [Documentation]    Ansible playbook which does all basic configuration for kubernetes nodes.
@@ -78,7 +73,13 @@ Configuration Playbook
     OperatingSystem.Create File    ${WATCHER_COE}    ${watcher}
     SSHKeywords.Copy_File_To_Remote_System    ${K8s_MASTER_IP}    ${WATCHER_COE}    ${USER_HOME}
     OperatingSystem.Copy File    ${PLAYBOOK_FILE}    ${USER_HOME}
-    ${play_output} =    OperatingSystem.Run    ansible-playbook -v ${USER_HOME}/coe_play.yaml -i ${USER_HOME}/hosts.yaml --extra-vars '{"gerrit_branch":"FETCH_HEAD","gerrit_refspec":"${GERRIT_REFSPEC}"}'
+    ${branch_ref_spec} =    BuiltIn.Catenate    SEPARATOR=    refs/heads/    ${GERRIT_BRANCH}
+    ${gerrit_ref_spec} =    BuiltIn.Set Variable If    '${GERRIT_PROJECT}' != 'coe'    ${branch_ref_spec}    ${GERRIT_REFSPEC}
+    Run Coe Playbook    ${gerrit_ref_spec}
+
+Run Coe Playbook
+    [Arguments]    ${gerrit_ref_spec}
+    ${play_output} =    OperatingSystem.Run    ansible-playbook -v ${USER_HOME}/coe_play.yaml -i ${USER_HOME}/hosts.yaml --extra-vars '{"gerrit_branch":"FETCH_HEAD","gerrit_refspec":"${gerrit_ref_spec}"}'
     BuiltIn.Log    ${play_output}
 
 Modifying templates in playbook
@@ -87,13 +88,13 @@ Modifying templates in playbook
     ${template} =    String.Replace String    ${template}    minion_ip    ${TOOLS_SYSTEM_ALL_IPS[0]}
     @{minions}    Create List    coe-minion
     ${hosts}    Set Variable    coe-master:
-    : FOR    ${i}    INRANGE    1    ${NUM_TOOLS_SYSTEM}
+    : FOR    ${i}    IN RANGE    1    ${NUM_TOOLS_SYSTEM}
     \    Append To List    ${minions}    coe-minion${i}
     \    ${hosts} =    Catenate    ${hosts}    coe-minion${i}:
     ${hosts} =    Replace String Using Regexp    ${hosts}    :$    ${EMPTY}
     ${hosts} =    Remove Space on String    ${hosts}
     ${minion hosts} =    Replace String Using Regexp    ${hosts}    ^[\\w-]+:    ${EMPTY}
-    : FOR    ${i}    INRANGE    1    ${NUM_TOOLS_SYSTEM}
+    : FOR    ${i}    IN RANGE    1    ${NUM_TOOLS_SYSTEM}
     \    ${j} =    Evaluate    ${i}+1
     \    ${template} =    String.Replace String    ${template}    ${minions[${i}-1]}    ${minions[${i}]}
     \    ${template} =    String.Replace String    ${template}    ${TOOLS_SYSTEM_ALL_IPS[${i}-1]}    ${TOOLS_SYSTEM_ALL_IPS[${i}]}
@@ -125,10 +126,10 @@ Verify Watcher Is Running
 
 Check Node Status Is Ready
     [Documentation]    Checks the status of nodes.This keyword is repeated until the status of all nodes is Ready
-    ${nodes} =    Utils.Run Command On Remote System    ${K8s_MASTER_IP}    kubectl get nodes    ${DEFAULT_USER}    ${DEFAULT_PASSWORD}    ${DEFAULT_LINUX_PROMPT_STRICT}
-    @{cluster} =    String.Split To Lines    ${nodes}    1
-    : FOR    ${node}    IN    @{cluster}
-    \    BuiltIn.Should Match Regexp    ${node}    ${NODE_READY_STATUS}
+    ${nodes} =    Utils.Run Command On Remote System And Log    ${K8s_MASTER_IP}    kubectl get nodes    ${DEFAULT_USER}    ${DEFAULT_PASSWORD}    ${DEFAULT_LINUX_PROMPT_STRICT}
+    ${node_status} =    String.Get Lines Matching Regexp    ${nodes}    ${NODE_READY_STATUS}    partial_match=True
+    ${lines_containing_ready} =    String.Get Line Count    ${node_status}
+    BuiltIn.Should Be Equal As Strings    ${lines_containing_ready}    ${NUM_TOOLS_SYSTEM}
 
 Label Nodes
     [Documentation]    Create labels for minions so that random allocation of pods to minions is avoided
@@ -148,7 +149,7 @@ Derive Coe Data Models
 
 Check Pod Status Is Running
     [Documentation]    Checks the status of pods.This keyword is repeated until the status of all pods is Running
-    ${pods} =    Utils.Run Command On Remote System    ${K8s_MASTER_IP}    kubectl get pods -o wide    ${DEFAULT_USER}    ${DEFAULT_PASSWORD}    ${DEFAULT_LINUX_PROMPT_STRICT}
+    ${pods} =    Utils.Run Command On Remote System And Log    ${K8s_MASTER_IP}    kubectl get pods -o wide    ${DEFAULT_USER}    ${DEFAULT_PASSWORD}    ${DEFAULT_LINUX_PROMPT_STRICT}
     @{cluster} =    String.Split To Lines    ${pods}    1
     : FOR    ${pod}    IN    @{cluster}
     \    BuiltIn.Should Match Regexp    ${pod}    ${POD_RUNNING_STATUS}
@@ -171,6 +172,7 @@ Delete Pods
     \    ${pod_name} =    BuiltIn.Should Match Regexp    ${status}    ^\\w+-\\w+
     \    Utils.Run Command On Remote System    ${K8s_MASTER_IP}    kubectl delete pods ${pod_name}
     BuiltIn.Wait Until Keyword Succeeds    60s    3s    Coe.Check If Pods Are Terminated
+    Coe.Check For Stale veth Ports
 
 Check If Pods Are Terminated
     [Documentation]    Checks if the pods created have been terminated.The keyword is repeated until the pods are deleted
@@ -210,7 +212,7 @@ Stop Watcher
 
 Kube reset
     [Documentation]    Reset K8s to clear up all stale entries
-    : FOR    ${nodes}    IN    @{NODE_IPs}
+    : FOR    ${nodes}    IN    @{TOOLS_SYSTEM_ALL_IPS}
     \    ${kube} =    Utils.Run Command On Remote System And Log    ${nodes}    sudo kubeadm reset
     \    BuiltIn.Should Contain    ${kube}    Stopping the kubelet service.
 
@@ -239,7 +241,7 @@ Log Statements
     @{log statement} =    Create List
     ${i} =    Set Variable    0
     : FOR    ${pod_ip}    IN    @{pod ips}
-    \    ${ping statement}    Set Variable    Ping ${pod_name} and ${pod names[${i}]} : ${pod ip}
+    \    ${ping statement}    Set Variable    Ping from ${pod_name} to ${pod names[${i}]} (${pod ip})
     \    Append To List    ${log statement}    ${ping statement}
     \    ${i} =    Evaluate    ${i}+1
     [Return]    @{log statement}
@@ -273,3 +275,9 @@ Extract current suite name
     ${suite line}    ${current_suite} =    BuiltIn.Run Keyword If    ${num_suites} > ${1}    Should Match Regexp    ${SUITE_NAME}    .txt.(\\w.*)
     ...    ELSE    BuiltIn.Set Variable    @{suite_names_updated}[0]    @{suite_names_updated}[0]
     [Return]    ${current_suite}    ${suite_names_updated}
+
+Check For Stale veth Ports
+    [Documentation]    Check on switches(except master) where pods were created and deleted to ensure there are no stale veth ports left behind.
+    : FOR    ${minion_index}    IN RANGE    2    ${NUM_TOOLS_SYSTEM}+1
+    \    ${switch output} =    Utils.Run Command On Remote System And Log    ${TOOLS_SYSTEM_${minion_index}_IP}    sudo ovs-vsctl show
+    \    BuiltIn.Should Not Contain    ${switch output}    veth