2 Documentation Test suite for 2Node Cluster HA with Bulk Flows - Cluster node convergance and Data consistency after leader and follower restart with one switch connected
3 Suite Setup Create Controller Sessions
4 Suite Teardown Delete All Sessions
5 Resource ../../../libraries/BulkomaticKeywords.robot
6 Resource ../../../libraries/MininetKeywords.robot
7 Resource ../../../libraries/ClusterKeywords.robot
8 Resource ../../../libraries/ClusterOpenFlow.robot
9 Resource ../../../libraries/Utils.robot
10 Variables ../../../variables/Variables.py
13 ${operation_timeout} 100s
14 ${restart_timeout} 350s
15 ${flow_count_per_switch} 1000
16 ${switch_count_per_node} 1
17 ${flow_count_after_add} 1000
18 ${flow_count_after_del} 0
19 ${add_small_config} sal_add_bulk_flow_small_config.json
20 ${get_small_config} sal_get_bulk_flow_small_config.json
21 ${del_small_config} sal_del_bulk_flow_small_config.json
24 Create Original Cluster List
25 [Documentation] Create original cluster list.
26 ${original_cluster_list} ClusterKeywords.Create Controller Index List
27 Set Suite Variable ${original_cluster_list}
29 Check Shards Status Before Leader Restart
30 [Documentation] Check Status for all shards in OpenFlow application.
31 ClusterOpenFlow.Check OpenFlow Shards Status ${original_cluster_list}
33 Get Inventory Follower Before Cluster Restart
34 [Documentation] Find a follower in the inventory config shard
35 ${inventory_leader} ${inventory_followers} ClusterOpenFlow.Get InventoryConfig Shard Status ${original_cluster_list}
36 ${Inventory_Leader_List}= Create List ${Inventory_Leader}
37 ${Follower_Node_1}= Get From List ${Inventory_Followers} 0
38 Set Suite Variable ${Inventory_Followers}
39 Set Suite Variable ${Follower_Node_1}
40 Set Suite Variable ${Inventory_Leader}
41 Set Suite Variable ${Inventory_Leader_List}
43 Shutdown Leader From Cluster Node
44 [Documentation] Shutdown Leader Node and Start it Up.
45 ClusterKeywords.Kill Multiple Controllers ${Inventory_Leader}
47 Check Shards Status After Leader Shutdown
48 [Documentation] Wait for node convergence and check status for all shards in OpenFlow application.
49 Wait Until Keyword Succeeds ${operation_timeout} 2s ClusterOpenFlow.Check OpenFlow Shards Status ${Inventory_Followers}
51 Start Mininet Connect To Follower Node1
52 [Documentation] Start mininet with connection to Follower Node1.
53 ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${Follower_Node_1}_IP}
54 Set Suite Variable ${mininet_conn_id}
56 Add Bulk Flow From Follower
57 [Documentation] 1000 Flows added via Follower Node1 and verify it gets applied in all instances.
58 BulkomaticKeywords.Add Bulk Flow In Node ${Follower_Node_1} ${add_small_config} ${operation_timeout}
60 Get Bulk Flows and Verify In Cluster
61 BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${Inventory_Followers} ${get_small_config} ${operation_timeout} ${flow_count_after_add}
63 Verify Flows In Switch Before Cluster Restart
64 [Documentation] Verify flows are installed in switch before cluster restart.
65 MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout}
67 Restart Leader From Cluster Node
68 [Documentation] Restart Leader Node.
69 ClusterKeywords.Start Multiple Controllers ${operation_timeout} ${Inventory_Leader}
71 Verify Data Recovery After Leader Restart
72 [Documentation] 1000 Flows preserved in all controller instances.
73 Wait Until Keyword Succeeds ${restart_timeout} 2s BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout}
74 ... ${flow_count_after_add}
76 Verify Flows In Switch After Leader Restart
77 [Documentation] Verify flows are installed in switch after cluster restart.
78 MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout}
80 Stop Mininet Connected To Follower Node1
81 [Documentation] Stop mininet and exit connection.
82 MininetKeywords.Stop Mininet And Exit ${mininet_conn_id}
83 Utils.Clean Mininet System
85 Delete All Flows From Follower Node1
86 [Documentation] Rest 1000 Flows deleted via Follower Node1 and verify it gets applied in all instances.
87 BulkomaticKeywords.Delete Bulk Flow In Node ${Follower_Node_1} ${del_small_config} ${operation_timeout}
89 Verify No Flows In Cluster
90 BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout} ${flow_count_after_del}
92 Get Inventory Follower Before Follower Restart
93 [Documentation] Find Leader and followers in the inventory config shard
94 ${inventory_leader} ${inventory_followers} ClusterOpenFlow.Get InventoryConfig Shard Status ${original_cluster_list}
95 ${Active_Nodes}= Create List
96 ${Follower_Node_1}= Get From List ${Inventory_Followers} 0
97 ${Follower_Node_2}= Get From List ${Inventory_Followers} 1
98 Append To List ${Active_Nodes} ${inventory_leader} ${Follower_Node_1}
99 Set Suite Variable ${Active_Nodes}
100 Set Suite Variable ${Follower_Node_1}
101 Set Suite Variable ${Follower_Node_2}
103 Shutdown Follower From Cluster Node
104 [Documentation] Shutdown Follower Node2 and Start it Up.
105 ClusterKeywords.Kill Multiple Controllers ${Follower_Node_2}
107 Check Shards Status After Follower Shutdown
108 [Documentation] Wait for node convergence and check status for all shards in OpenFlow application.
109 Wait Until Keyword Succeeds ${operation_timeout} 2s ClusterOpenFlow.Check OpenFlow Shards Status ${Active_Nodes}
111 Start Mininet Connect To Follower Node
112 [Documentation] Start mininet with connection to Follower Node1.
113 ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${Follower_Node_1}_IP}
114 Set Suite Variable ${mininet_conn_id}
116 Add Bulk Flow From Follower Node1
117 [Documentation] 1000 Flows added via Follower Node1 and verify it gets applied in all instances.
118 BulkomaticKeywords.Add Bulk Flow In Node ${Follower_Node_1} ${add_small_config} ${operation_timeout}
120 Get Bulk Flows and Verify In Cluster Before Follower Restart
121 BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${Active_Nodes} ${get_small_config} ${operation_timeout} ${flow_count_after_add}
123 Verify Flows In Switch Before Follower Restart
124 [Documentation] Verify flows are installed in switch before follower restart.
125 MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout}
127 Restart Follower From Cluster Node
128 [Documentation] Restart Follower Node2.
129 ClusterKeywords.Start Multiple Controllers ${operation_timeout} ${Follower_Node_2}
131 Verify Data Recovery After Follower Restart
132 [Documentation] 1000 Flows preserved in all controller instances.
133 Wait Until Keyword Succeeds ${restart_timeout} 2s BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout}
134 ... ${flow_count_after_add}
136 Verify Flows In Switch After Follower Restart
137 [Documentation] Verify flows are installed in switch after cluster restart.
138 MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout}
140 Stop Mininet Connected To Follower Node
141 [Documentation] Stop mininet and exit connection.
142 MininetKeywords.Stop Mininet And Exit ${mininet_conn_id}
143 Utils.Clean Mininet System
145 Delete All Flows From Follower Node
146 [Documentation] Rest 1000 Flows deleted via Follower Node1 and verify it gets applied in all instances.
147 BulkomaticKeywords.Delete Bulk Flow In Node ${Follower_Node_1} ${del_small_config} ${operation_timeout}
149 Verify No Flows In Cluster After Follower Restart
150 BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout} ${flow_count_after_del}