2 Documentation Library containing Keywords used for SXP cluster testing
3 Library RequestsLibrary
5 Resource ./SxpLib.robot
6 Resource ./ClusterManagement.robot
7 Variables ../variables/Variables.py
10 @{SHARD_OPER_LIST} inventory topology default entity-ownership
11 @{SHARD_CONF_LIST} inventory topology default
12 ${DEVICE_SESSION} device_1
13 ${DEVICE_NODE_ID} 1.1.1.1
14 ${CLUSTER_NODE_ID} 2.2.2.2
17 Setup SXP Cluster Session
18 [Documentation] Create sessions asociated with SXP cluster setup
19 Wait Until Keyword Succeeds 120 10 Prepare SSH Keys On Karaf ${TOOLS_SYSTEM_IP} ${TOOLS_SYSTEM_USER} ${TOOLS_SYSTEM_PASSWORD}
20 ... ${TOOLS_SYSTEM_PROMPT} /tmp
21 Setup SXP Session ${DEVICE_SESSION} ${TOOLS_SYSTEM_IP}
22 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
23 \ Wait Until Keyword Succeeds 120 10 Prepare SSH Keys On Karaf ${ODL_SYSTEM_${i+1}_IP}
24 \ Setup SXP Session controller${i+1} ${ODL_SYSTEM_${i+1}_IP}
25 ClusterManagement_Setup
27 Clean SXP Cluster Session
28 [Documentation] Clean sessions asociated with SXP cluster setup
29 Flush_Iptables_From_List_Or_All
30 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
31 \ Wait Until Keyword Succeeds 240 1 Sync_Status_Should_Be_True ${i+1}
35 [Documentation] Check Status for all shards in SXP application.
36 ClusterManagement.Check_Cluster_Is_In_Sync
37 ClusterManagement.Verify_Leader_Exists_For_Each_Shard shard_name_list=${SHARD_OPER_LIST} shard_type=operational
38 ClusterManagement.Verify_Leader_Exists_For_Each_Shard shard_name_list=${SHARD_CONF_LIST} shard_type=config
41 [Arguments] ${peer_mode}=listener
42 [Documentation] Setup and connect SXP cluster topology
43 Add Node ${DEVICE_NODE_ID} ip=0.0.0.0 session=${DEVICE_SESSION}
44 Wait Until Keyword Succeeds 20 1 Check Node Started ${DEVICE_NODE_ID} session=${DEVICE_SESSION} system=${TOOLS_SYSTEM_IP}
46 ${cluster_mode} Get Opposing Mode ${peer_mode}
47 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
48 \ Add Connection version4 ${peer_mode} ${ODL_SYSTEM_${i+1}_IP} 64999 ${DEVICE_NODE_ID}
49 \ ... session=${DEVICE_SESSION}
50 ${controller_id} Get Any Controller
51 Add Node ${CLUSTER_NODE_ID} ip=0.0.0.0 session=controller${controller_id}
52 Wait Until Keyword Succeeds 20 1 Check Cluster Node started ${CLUSTER_NODE_ID}
53 Add Connection version4 ${cluster_mode} ${TOOLS_SYSTEM_IP} 64999 ${CLUSTER_NODE_ID} session=controller${controller_id}
54 Wait Until Keyword Succeeds 120 1 Check Device is Connected ${DEVICE_NODE_ID} session=${DEVICE_SESSION}
57 [Documentation] Disconnect SXP cluster topology
58 Flush_Iptables_From_List_Or_All
59 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
60 \ Wait Until Keyword Succeeds 240 1 Sync_Status_Should_Be_True ${i+1}
61 ${controller_index} Get Active Controller
62 Delete Node ${DEVICE_NODE_ID} session=${DEVICE_SESSION}
63 Delete Node ${CLUSTER_NODE_ID} session=controller${controller_index}
65 Check Cluster Node started
66 [Arguments] ${node} ${port}=64999 ${ip}=${EMPTY}
67 [Documentation] Verify that SxpNode has data written to Operational datastore and Node is running on one of cluster nodes
68 ${started} Set Variable ${False}
69 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
70 \ ${rc} Run Command On Remote System ${ODL_SYSTEM_${i+1}_IP} netstat -tln | grep -q ${ip}:${port} && echo 0 || echo 1 ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD}
71 \ ... prompt=${ODL_SYSTEM_PROMPT}
72 \ ${started} Set Variable If '${rc}' == '0' ${True} ${started}
73 Should Be True ${started}
75 Check Device is Connected
76 [Arguments] ${node} ${version}=version4 ${port}=64999 ${session}=session
77 [Documentation] Checks if SXP device is connected to at least one cluster node
78 ${is_connected} Set Variable ${False}
79 ${resp} Get Connections node=${node} session=${session}
80 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
81 \ ${follower} Find Connection ${resp} ${version} any ${ODL_SYSTEM_${i+1}_IP}
83 \ ${is_connected} Run Keyword If ${follower} Set Variable ${True}
84 \ ... ELSE Set Variable ${is_connected}
85 Should Be True ${is_connected}
87 Check Cluster is Connected
88 [Arguments] ${node} ${version}=version4 ${port}=64999 ${mode}=speaker ${session}=session
89 [Documentation] Checks if SXP device is connected to at least one cluster node
90 ${resp} Get Connections node=${node} session=${session}
91 Should Contain Connection ${resp} ${TOOLS_SYSTEM_IP} ${port} ${mode} ${version}
94 [Documentation] Find cluster controller that is marked as leader for SXP service in cluster
96 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
97 \ ${resp} RequestsLibrary.Get Request controller${i+1} /restconf/operational/entity-owners:entity-owners
98 \ Continue For Loop If ${resp.status_code} != 200
99 \ ${controller} Get Active Controller From Json ${resp.content} SxpControllerInstance
100 \ Append To List ${votes} ${controller}
101 ${length} Get Length ${votes}
102 Should Not Be Equal As Integers ${length} 0
103 ${active_controller} Evaluate collections.Counter(${votes}).most_common(1)[0][0] collections
104 [Return] ${active_controller}
106 Get Inactive Controller
107 [Documentation] Find cluster controller that is not marked as leader for SXP service in cluster
108 ${active_controller} Get Active Controller
109 ${controller} Evaluate random.choice( filter( lambda i: i!=${active_controller}, range(1, ${NUM_ODL_SYSTEM} + 1))) random
110 [Return] ${controller}
113 [Documentation] Get any controller from cluster range
114 ${follower} Evaluate random.choice( range(1, ${NUM_ODL_SYSTEM} + 1)) random
117 Map Followers To Mac Addresses
118 [Documentation] Creates Map containing ODL_SYSTEM_IP to corresponding MAC-ADDRESS
119 ${mac_addresses} create dictionary
120 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
121 \ ${mac_address} Find Mac Address Of Ip Address ${ODL_SYSTEM_${i+1}_IP}
122 \ Set To Dictionary ${mac_addresses} ${ODL_SYSTEM_${i+1}_IP} ${mac_address}
124 [Return] ${mac_addresses}
126 Find Mac Address Of Ip Address
128 [Documentation] Finds out MAC-ADDRESS of specified IP by pinging it from TOOLS_SYSTEM machine
129 ${mac_address} Run Command On Remote System ${TOOLS_SYSTEM_IP} ping -c 1 -W 1 ${ip} >/dev/null && arp -n | grep ${ip} | awk '{print $3}' ${TOOLS_SYSTEM_USER} ${TOOLS_SYSTEM_PASSWORD}
130 [Return] ${mac_address}
132 Ip Addres Should Not Be Routed To Follower
133 [Arguments] ${mac_addresses} ${ip_address} ${follower_index}
134 [Documentation] Verify that IP-ADDRESS is not routed to follower specified by ID
135 ${mac_address_assigned} Get From Dictionary ${mac_addresses} ${ODL_SYSTEM_${follower_index}_IP}
136 ${mac_address_resolved} Find Mac Address Of Ip Address ${ip_address}
137 Should Not Be Equal As Strings ${mac_address_assigned} ${mac_address_resolved}
139 Ip Addres Should Be Routed To Follower
140 [Arguments] ${mac_addresses} ${ip_address} ${follower_index}
141 [Documentation] Verify that IP-ADDRESS is routed to follower specified by ID
142 ${mac_address_assigned} Get From Dictionary ${mac_addresses} ${ODL_SYSTEM_${follower_index}_IP}
143 ${mac_address_resolved} Find Mac Address Of Ip Address ${ip_address}
144 Should Not Be Empty ${mac_address_resolved}
145 Should Be Equal As Strings ${mac_address_assigned} ${mac_address_resolved}
148 [Arguments] ${ip_address}=${TOOLS_SYSTEM_2_IP} ${user}=${TOOLS_SYSTEM_USER} ${passwd}=${TOOLS_SYSTEM_PASSWORD}
149 [Documentation] Shutdown Tools node to avoid conflict in resolving virtual ip that is overlaping that node.
150 ${rc} Run And Return Rc ping -q -c 3 ${ip_address}
151 ${stdout} Run Keyword And Return If ${rc} == 0 Run Command On Remote System ${ip_address} sudo shutdown -P 0 ${user}