2 Documentation Test suite for Cluster HA with Bulk Flows - Data consistency after cluster restart, leader restart and follower restart with one switch connected
3 Suite Setup Create Controller Sessions
4 Suite Teardown Delete All Sessions
5 Resource ../../../libraries/BulkomaticKeywords.robot
6 Resource ../../../libraries/MininetKeywords.robot
7 Resource ../../../libraries/ClusterKeywords.robot
8 Resource ../../../libraries/ClusterOpenFlow.robot
9 Resource ../../../libraries/Utils.robot
10 Variables ../../../variables/Variables.py
13 ${operation_timeout} 100s
14 ${restart_timeout} 350s
15 ${flow_count_per_switch} 1000
17 ${flow_count_after_add} 1000
18 ${flow_count_after_del} 0
19 ${orig_json_config_add} sal_add_bulk_flow_config.json
20 ${orig_json_config_get} sal_get_bulk_flow_config.json
21 ${orig_json_config_del} sal_del_bulk_flow_config.json
24 Create Original Cluster List
25 [Documentation] Create original cluster list.
26 ${original_cluster_list} ClusterKeywords.Create Controller Index List
27 Set Suite Variable ${original_cluster_list}
29 Check Shards Status Before Leader Restart
30 [Documentation] Check Status for all shards in OpenFlow application.
31 ClusterOpenFlow.Check OpenFlow Shards Status ${original_cluster_list}
33 Get Inventory Follower Before Cluster Restart
34 [Documentation] Find a follower in the inventory config shard
35 ${inventory_leader} ${inventory_followers} ClusterOpenFlow.Get InventoryConfig Shard Status ${original_cluster_list}
36 ${Follower_Node_1}= Get From List ${Inventory_Followers} 0
37 Set Suite Variable ${Follower_Node_1}
39 Start Mininet Connect To Follower Node1
40 [Documentation] Start mininet with connection to Follower Node1.
41 ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${Follower_Node_1}_IP}
42 Set Suite Variable ${mininet_conn_id}
44 Add Bulk Flow From Follower
45 [Documentation] 1000 Flows added via Follower Node1 and verify it gets applied in all instances.
46 ${temp_json_config_add} BulkomaticKeywords.Set DPN And Flow Count In Json Add ${orig_json_config_add} ${switch_count} ${flow_count_per_switch}
47 Set Suite Variable ${temp_json_config_add}
48 BulkomaticKeywords.Add Bulk Flow In Node ${Follower_Node_1} ${temp_json_config_add} ${operation_timeout}
50 Get Bulk Flows and Verify In Cluster
51 [Documentation] Initiate get operation and check flow count across cluster nodes
52 ${temp_json_config_get} BulkomaticKeywords.Set DPN And Flow Count In Json Get ${orig_json_config_get} ${switch_count} ${flow_count_after_add}
53 Set Suite Variable ${temp_json_config_get}
54 BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout} ${flow_count_after_add}
56 Verify Flows In Switch Before Cluster Restart
57 [Documentation] Verify flows are installed in switch before cluster restart.
58 MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count} ${flow_count_per_switch} ${operation_timeout}
60 Kill All Cluster Nodes
61 [Documentation] Kill All Nodes.
62 ClusterKeywords.Kill Multiple Controllers @{original_cluster_list}
64 Stop Mininet Connected To Follower Node1 and Exit
65 [Documentation] Stop mininet and exit connection.
66 MininetKeywords.Stop Mininet And Exit ${mininet_conn_id}
67 Utils.Clean Mininet System
69 Restart All Cluster Nodes
70 [Documentation] Restart all cluster nodes.
71 ClusterKeywords.Start Multiple Controllers ${restart_timeout} @{original_cluster_list}
73 Verify Data Recovery After Cluster Restart
74 [Documentation] 1000 Flows preserved in all controller instances.
75 Wait Until Keyword Succeeds ${restart_timeout} 2s BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout}
76 ... ${flow_count_after_add}
78 Start Mininet Again Connect To Follower Node1
79 [Documentation] Start mininet with connection to follower node1.
80 ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${Follower_Node_1}_IP}
81 Set Suite Variable ${mininet_conn_id}
83 Verify Flows In Switch After Cluster Restart
84 [Documentation] Verify flows are installed in switch after cluster restart.
85 MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count} ${flow_count_per_switch} ${operation_timeout}
87 Stop Mininet Connected To Follower Node1
88 [Documentation] Stop mininet and exit connection.
89 MininetKeywords.Stop Mininet And Exit ${mininet_conn_id}
90 Utils.Clean Mininet System
92 Delete All Flows From Follower Node1
93 [Documentation] 1000 Flows deleted via Follower Node1 and verify it gets applied in all instances.
94 ${temp_json_config_del} BulkomaticKeywords.Set DPN And Flow Count In Json Del ${orig_json_config_del} ${switch_count} ${flow_count_per_switch}
95 Set Suite Variable ${temp_json_config_del}
96 BulkomaticKeywords.Delete Bulk Flow In Node ${Follower_Node_1} ${temp_json_config_del} ${operation_timeout}
98 Verify No Flows In Cluster
99 [Documentation] Verify flow count is 0 across cluster nodes.
100 BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout} ${flow_count_after_del}
102 Get Inventory Leader Before Leader Restart
103 [Documentation] Find leader in the inventory config shard
104 ${inventory_leader} ${inventory_followers} ClusterOpenFlow.Get InventoryConfig Shard Status ${original_cluster_list}
105 Set Suite Variable ${Inventory_Leader}
107 Start Mininet Connect To Leader
108 [Documentation] Start mininet with connection to Leader Node.
109 ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${Inventory_Leader}_IP}
110 Set Suite Variable ${mininet_conn_id}
112 Add Bulk Flow From Leader
113 [Documentation] 1000 Flows added via Follower Node1 and verify it gets applied in all instances.
114 BulkomaticKeywords.Add Bulk Flow In Node ${Inventory_Leader} ${temp_json_config_add} ${operation_timeout}
116 Get Bulk Flows and Verify In Cluster Before Leader Restart
117 [Documentation] Initiate get operation and check flow count across cluster nodes
118 BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout} ${flow_count_after_add}
120 Verify Flows In Switch Before Leader Restart
121 [Documentation] Verify flows are installed in switch before leader restart.
122 MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count} ${flow_count_per_switch} ${operation_timeout}
124 Kill Leader From Cluster Node
125 [Documentation] Kill Leader Node.
126 ClusterKeywords.Kill Multiple Controllers ${Inventory_Leader}
128 Stop Mininet Connected To Leader Node
129 [Documentation] Stop mininet and exit connection.
130 MininetKeywords.Stop Mininet And Exit ${mininet_conn_id}
131 Utils.Clean Mininet System
133 Restart Leader from Cluster Node
134 [Documentation] Start Leader Node Up.
135 ClusterKeywords.Start Multiple Controllers ${restart_timeout} ${Inventory_Leader}
137 Verify Data Recovery After Leader Restart
138 [Documentation] 1000 Flows preserved in all controller instances.
139 Wait Until Keyword Succeeds ${restart_timeout} 2s BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout}
140 ... ${flow_count_after_add}
142 Start Mininet Again Connect To Leader
143 [Documentation] Start mininet with connection to Leader Node.
144 ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${Inventory_Leader}_IP}
145 Set Suite Variable ${mininet_conn_id}
147 Verify Flows In Switch After Leader Restart
148 [Documentation] Verify flows are installed in switch after leader restart.
149 MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count} ${flow_count_per_switch} ${operation_timeout}
151 Stop Mininet Connected To Leader Node After Leader Restart
152 [Documentation] Stop mininet and exit connection.
153 MininetKeywords.Stop Mininet And Exit ${mininet_conn_id}
154 Utils.Clean Mininet System
156 Delete All Flows From Leader Node
157 [Documentation] 1000 Flows deleted via Leader Node and verify it gets applied in all instances.
158 BulkomaticKeywords.Delete Bulk Flow In Node ${Inventory_Leader} ${temp_json_config_del} ${operation_timeout}
160 Verify No Flows In Cluster After Leader Restart
161 [Documentation] Verify flow count is 0 across cluster nodes.
162 BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout} ${flow_count_after_del}
164 Get Inventory Follower Before follower Restart
165 [Documentation] Find follower in the inventory config shard
166 ${inventory_leader} ${inventory_followers} ClusterOpenFlow.Get InventoryConfig Shard Status ${original_cluster_list}
167 ${Follower_Node_2}= Get From List ${Inventory_Followers} 1
168 Set Suite Variable ${Follower_Node_2}
170 Start Mininet Connect To Follower Node2
171 [Documentation] Start mininet with connection to Follower Node2.
172 ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${Follower_Node_2}_IP}
173 Set Suite Variable ${mininet_conn_id}
175 Add Bulk Flow From Follower Node2
176 [Documentation] 1000 Flows added via Follower Node2 and verify it gets applied in all instances.
177 BulkomaticKeywords.Add Bulk Flow In Node ${Follower_Node_2} ${temp_json_config_add} ${operation_timeout}
179 Get Bulk Flows and Verify In Cluster Before Follower Restart
180 [Documentation] Initiate get operation and check flow count across cluster nodes
181 BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout} ${flow_count_after_add}
183 Verify Flows In Switch Before Follower Restart
184 [Documentation] Verify flows are installed in switch before follower restart.
185 MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count} ${flow_count_per_switch} ${operation_timeout}
188 [Documentation] Kill Follower Node2.
189 ClusterKeywords.Kill Multiple Controllers ${Follower_Node_2}
191 Stop Mininet Connected To Follower Node2 and Exit
192 [Documentation] Stop mininet and exit connection.
193 MininetKeywords.Stop Mininet And Exit ${mininet_conn_id}
194 Utils.Clean Mininet System
196 Restart Follower Node2
197 [Documentation] Start Follower Node2 Up.
198 ClusterKeywords.Start Multiple Controllers ${restart_timeout} ${Follower_Node_2}
200 Verify Data Recovery After Follower Node2 Restart
201 [Documentation] 1000 Flows preserved in all controller instances.
202 Wait Until Keyword Succeeds ${restart_timeout} 2s BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout}
203 ... ${flow_count_after_add}
205 Start Mininet Again Connect To Follower Node2
206 [Documentation] Start mininet with connection to follower node1.
207 ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${Follower_Node_2}_IP}
208 Set Suite Variable ${mininet_conn_id}
210 Verify Flows In Switch After Follower Node2 Restart
211 [Documentation] Verify flows are installed in switch after follower restart.
212 MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count} ${flow_count_per_switch} ${operation_timeout}
214 Stop Mininet Connected To Follower Node2
215 [Documentation] Stop mininet and exit connection.
216 MininetKeywords.Stop Mininet And Exit ${mininet_conn_id}
217 Utils.Clean Mininet System
219 Delete All Flows From Follower Node 2
220 [Documentation] 1000 Flows deleted via Leader Node and verify it gets applied in all instances.
221 BulkomaticKeywords.Delete Bulk Flow In Node ${Follower_Node_2} ${temp_json_config_del} ${operation_timeout}
223 Verify No Flows In Cluster After Follower Node2 Restart
224 BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout} ${flow_count_after_del}