2 Documentation Library containing Keywords used for SXP cluster testing
3 Library RequestsLibrary
5 Resource ./SxpLib.robot
6 Resource ./ClusterManagement.robot
7 Resource ./SetupUtils.robot
8 Resource ../variables/Variables.robot
11 @{SHARD_OPER_LIST} inventory topology default entity-ownership
12 @{SHARD_CONF_LIST} inventory topology default
13 ${DEVICE_SESSION} device_1
14 ${DEVICE_NODE_ID} 1.1.1.1
15 ${CLUSTER_NODE_ID} 2.2.2.2
17 @{SXP_PACKAGE} org.opendaylight.sxp
20 Setup SXP Cluster Session
21 [Documentation] Create sessions asociated with SXP cluster setup
22 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
23 \ BuiltIn.Wait Until Keyword Succeeds 120 10 SxpLib.Prepare SSH Keys On Karaf ${ODL_SYSTEM_${i+1}_IP}
24 \ SxpLib.Setup SXP Session controller${i+1} ${ODL_SYSTEM_${i+1}_IP}
25 ClusterManagement.ClusterManagement_Setup
26 SetupUtils.Setup_Utils_For_Setup_And_Teardown
27 SetupUtils.Setup_Logging_For_Debug_Purposes_On_List_Or_All ${SXP_LOG_LEVEL} ${SXP_PACKAGE}
29 Clean SXP Cluster Session
30 [Documentation] Clean sessions asociated with SXP cluster setup
31 ClusterManagement.Flush_Iptables_From_List_Or_All
32 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
33 \ BuiltIn.Wait Until Keyword Succeeds 240 1 ClusterManagement.Sync_Status_Should_Be_True ${i+1}
34 SxpLib.Clean SXP Session
35 SetupUtils.Setup_Logging_For_Debug_Purposes_On_List_Or_All INFO ${SXP_PACKAGE}
38 [Documentation] Check Status for all shards in SXP application.
39 ClusterManagement.Check_Cluster_Is_In_Sync
40 ClusterManagement.Verify_Leader_Exists_For_Each_Shard shard_name_list=${SHARD_OPER_LIST} shard_type=operational
41 ClusterManagement.Verify_Leader_Exists_For_Each_Shard shard_name_list=${SHARD_CONF_LIST} shard_type=config
44 [Arguments] ${peer_mode}=listener
45 [Documentation] Setup and connect SXP cluster topology
46 SxpLib.Add Node ${DEVICE_NODE_ID} ip=0.0.0.0 session=${DEVICE_SESSION}
47 BuiltIn.Wait Until Keyword Succeeds 20 1 SxpLib.Check Node Started ${DEVICE_NODE_ID} session=${DEVICE_SESSION} system=${TOOLS_SYSTEM_IP}
49 ${cluster_mode} = Sxp.Get Opposing Mode ${peer_mode}
50 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
51 \ SxpLib.Add Connection version4 ${peer_mode} ${ODL_SYSTEM_${i+1}_IP} 64999 ${DEVICE_NODE_ID}
52 \ ... session=${DEVICE_SESSION}
53 ${controller_id} = Get Any Controller
54 SxpLib.Add Node ${CLUSTER_NODE_ID} ip=0.0.0.0 session=controller${controller_id}
55 BuiltIn.Wait Until Keyword Succeeds 20 1 Check Cluster Node started ${CLUSTER_NODE_ID}
56 SxpLib.Add Connection version4 ${cluster_mode} ${TOOLS_SYSTEM_IP} 64999 ${CLUSTER_NODE_ID} session=controller${controller_id}
57 BuiltIn.Wait Until Keyword Succeeds 120 1 Check Device is Connected ${DEVICE_NODE_ID} session=${DEVICE_SESSION}
60 [Documentation] Disconnect SXP cluster topology
61 ClusterManagement.Flush_Iptables_From_List_Or_All
62 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
63 \ BuiltIn.Wait Until Keyword Succeeds 240 1 ClusterManagement.Sync_Status_Should_Be_True ${i+1}
64 ${controller_index} = Get Active Controller
65 SxpLib.Delete Node ${DEVICE_NODE_ID} session=${DEVICE_SESSION}
66 SxpLib.Delete Node ${CLUSTER_NODE_ID} session=controller${controller_index}
68 Check Cluster Node started
69 [Arguments] ${node} ${port}=64999 ${ip}=${EMPTY}
70 [Documentation] Verify that SxpNode has data written to Operational datastore and Node is running on one of cluster nodes
71 ${started} = BuiltIn.Set Variable ${False}
72 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
73 \ ${rc} = Utils.Run Command On Remote System ${ODL_SYSTEM_${i+1}_IP} netstat -tln | grep -q ${ip}:${port} && echo 0 || echo 1 ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD}
74 \ ... prompt=${ODL_SYSTEM_PROMPT}
75 \ ${started} = BuiltIn.Set Variable If '${rc}' == '0' ${True} ${started}
76 BuiltIn.Should Be True ${started}
78 Check Device is Connected
79 [Arguments] ${node} ${version}=version4 ${port}=64999 ${session}=session
80 [Documentation] Checks if SXP device is connected to at least one cluster node
81 ${is_connected} = BuiltIn.Set Variable ${False}
82 ${resp} = SxpLib.Get Connections node=${node} session=${session}
83 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
84 \ ${follower} = Sxp.Find Connection ${resp} ${version} any ${ODL_SYSTEM_${i+1}_IP}
86 \ ${is_connected} = BuiltIn.Run Keyword If ${follower} BuiltIn.Set Variable ${True}
87 \ ... ELSE BuiltIn.Set Variable ${is_connected}
88 BuiltIn.Should Be True ${is_connected}
90 Check Cluster is Connected
91 [Arguments] ${node} ${version}=version4 ${port}=64999 ${mode}=speaker ${session}=session
92 [Documentation] Checks if SXP device is connected to at least one cluster node
93 ${resp} = SxpLib.Get Connections node=${node} session=${session}
94 SxpLib.Should Contain Connection ${resp} ${TOOLS_SYSTEM_IP} ${port} ${mode} ${version}
97 [Documentation] Find cluster controller that is marked as leader for SXP service in cluster
98 @{votes} = BuiltIn.Create List
99 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
100 \ ${resp} = RequestsLibrary.Get Request controller${i+1} /restconf/operational/entity-owners:entity-owners
101 \ BuiltIn.Continue For Loop If ${resp.status_code} != 200
102 \ ${controller} = Sxp.Get Active Controller From Json ${resp.content} SxpControllerInstance
103 \ Collections.Append To List ${votes} ${controller}
104 ${length} = BuiltIn.Get Length ${votes}
105 BuiltIn.Should Not Be Equal As Integers ${length} 0
106 ${active_controller} = BuiltIn.Evaluate collections.Counter(${votes}).most_common(1)[0][0] collections
107 [Return] ${active_controller}
109 Get Inactive Controller
110 [Documentation] Find cluster controller that is not marked as leader for SXP service in cluster
111 ${active_controller} = Get Active Controller
112 ${controller} = BuiltIn.Evaluate random.choice( filter( lambda i: i!=${active_controller}, range(1, ${NUM_ODL_SYSTEM} + 1))) random
113 [Return] ${controller}
116 [Documentation] Get any controller from cluster range
117 ${follower} = BuiltIn.Evaluate random.choice( range(1, ${NUM_ODL_SYSTEM} + 1)) random
120 Map Followers To Mac Addresses
121 [Documentation] Creates Map containing ODL_SYSTEM_IP to corresponding MAC-ADDRESS
122 ${mac_addresses} = BuiltIn.Create dictionary
123 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
124 \ ${mac_address} Find Mac Address Of Ip Address ${ODL_SYSTEM_${i+1}_IP}
125 \ Collections.Set To Dictionary ${mac_addresses} ${ODL_SYSTEM_${i+1}_IP} ${mac_address}
126 BuiltIn.Log ${mac_addresses}
127 [Return] ${mac_addresses}
129 Find Mac Address Of Ip Address
131 [Documentation] Finds out MAC-ADDRESS of specified IP by pinging it from TOOLS_SYSTEM machine
132 ${mac_address} = Utils.Run Command On Remote System ${TOOLS_SYSTEM_IP} ping -c 1 -W 1 ${ip} >/dev/null && arp -n | grep ${ip} | awk '{print $3}' ${TOOLS_SYSTEM_USER} ${TOOLS_SYSTEM_PASSWORD}
133 [Return] ${mac_address}
135 Ip Addres Should Not Be Routed To Follower
136 [Arguments] ${mac_addresses} ${ip_address} ${follower_index}
137 [Documentation] Verify that IP-ADDRESS is not routed to follower specified by ID
138 ${mac_address_assigned} = Collections.Get From Dictionary ${mac_addresses} ${ODL_SYSTEM_${follower_index}_IP}
139 ${mac_address_resolved} = Find Mac Address Of Ip Address ${ip_address}
140 BuiltIn.Should Not Be Equal As Strings ${mac_address_assigned} ${mac_address_resolved}
142 Ip Addres Should Be Routed To Follower
143 [Arguments] ${mac_addresses} ${ip_address} ${follower_index}
144 [Documentation] Verify that IP-ADDRESS is routed to follower specified by ID
145 ${mac_address_assigned} = Collections.Get From Dictionary ${mac_addresses} ${ODL_SYSTEM_${follower_index}_IP}
146 ${mac_address_resolved} = Find Mac Address Of Ip Address ${ip_address}
147 BuiltIn.Should Not Be Empty ${mac_address_resolved}
148 BuiltIn.Should Be Equal As Strings ${mac_address_assigned} ${mac_address_resolved}
151 [Arguments] ${ip_address}=${TOOLS_SYSTEM_2_IP} ${user}=${TOOLS_SYSTEM_USER} ${passwd}=${TOOLS_SYSTEM_PASSWORD}
152 [Documentation] Shutdown Tools node to avoid conflict in resolving virtual ip that is overlaping that node.
153 ${rc} = OperatingSystem.Run And Return Rc ping -q -c 3 ${ip_address}
154 ${stdout} = BuiltIn.Run Keyword And Return If ${rc} == 0 Utils.Run Command On Remote System ${ip_address} sudo shutdown -P 0 ${user}
156 BuiltIn.Log ${stdout}
158 Create Virtual Interface Eth0
159 [Documentation] Create virtual interface on all of the cluster nodes
160 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
161 \ Utils.Run Command On Remote System ${ODL_SYSTEM_${i+1}_IP} sudo modprobe dummy ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD}
162 \ Utils.Run Command On Remote System ${ODL_SYSTEM_${i+1}_IP} sudo ip link set name eth0 dev dummy0 ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD}
163 \ Utils.Run Command On Remote System And Log ${ODL_SYSTEM_${i+1}_IP} sudo ip link show ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD}
165 Delete Virtual Interface Eth0
166 [Documentation] Create virtual interface on all of the cluster nodes
167 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
168 \ Utils.Run Command On Remote System ${ODL_SYSTEM_${i+1}_IP} sudo ip link delete eth0 type dummy ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD}
169 \ Utils.Run Command On Remote System ${ODL_SYSTEM_${i+1}_IP} sudo rmmod dummy ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD}
170 \ Utils.Run Command On Remote System And Log ${ODL_SYSTEM_${i+1}_IP} sudo ip link show ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD}