2 Documentation Test suite for Cluster HA - Data Recovery at Leader Follower failover and cluster restart
3 Suite Setup Create Controller Sessions
4 Suite Teardown Delete All Sessions
5 Library RequestsLibrary
6 Resource ../../../libraries/ClusterOpenFlow.robot
7 Resource ../../../libraries/ClusterKeywords.robot
8 Resource ../../../libraries/MininetKeywords.robot
9 Variables ../../../variables/Variables.py
12 ${INVENTORY_SHARD} shard-inventory-config
14 ${flow_count_per_switch} 1
15 ${switch_count_per_node} 1
16 ${operation_timeout} 15s
19 Create Original Cluster List
20 [Documentation] Create original cluster list.
21 ${original_cluster_list} ClusterKeywords.Create Controller Index List
22 Set Suite Variable ${original_cluster_list}
24 Check Shards Status Before Leader Restart
25 [Documentation] Check Status for all shards in OpenFlow application.
26 ClusterOpenFlow.Check OpenFlow Shards Status ${original_cluster_list}
28 Get inventory Leader Before Leader Restart
29 [Documentation] Find leader in the inventory config shard
30 ${inventory_leader} ${inventory_followers} ClusterOpenFlow.Get InventoryConfig Shard Status ${original_cluster_list}
31 ${follower_node_1}= Get From List ${inventory_followers} 0
32 ${follower_node_2}= Get From List ${inventory_followers} 1
33 Set Suite Variable ${follower_node_1}
34 Set Suite Variable ${follower_node_2}
35 Set Suite Variable ${inventory_leader}
37 Start Mininet Connect To Follower Node1
38 [Documentation] Start mininet with connection to Follower Node1.
39 ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${follower_node_1}_IP}
40 Set Suite Variable ${mininet_conn_id}
42 Add Flows In Follower Node2 and Verify Before Leader Restart
43 [Documentation] Add Flow via Follower Node2 and verify it gets applied from all instances.
44 ClusterOpenFlow.Add Sample Flow And Verify ${original_cluster_list} ${follower_node_2}
46 Stop Mininet Connected To Follower Node1 and Exit
47 [Documentation] Stop mininet and exit connection.
48 MininetKeywords.Stop Mininet And Exit ${mininet_conn_id}
49 Utils.Clean Mininet System
51 Restart Leader From Cluster Node
52 [Documentation] Kill Leader Node and Start it Up, Verify it is sync with other controller node.
53 ClusterKeywords.Kill Multiple Controllers ${inventory_leader}
54 ClusterKeywords.Start Multiple Controllers ${START_TIMEOUT} ${inventory_leader}
56 Get inventory Follower After Leader Restart
57 [Documentation] Find new Followers and Leader in the inventory config shard After Leader Restart.
58 ${inventory_leader} ${inventory_followers} ClusterOpenFlow.Get InventoryConfig Shard Status ${original_cluster_list}
59 ${follower_node_1}= Get From List ${inventory_followers} 0
60 ${follower_node_2}= Get From List ${inventory_followers} 1
61 Set Suite Variable ${follower_node_1}
62 Set Suite Variable ${follower_node_2}
63 Set Suite Variable ${inventory_leader}
65 Start Mininet Connect To Follower Node2
66 [Documentation] Start mininet with connection to cluster Follower Node2.
67 ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${follower_node_2}_IP}
68 Set Suite Variable ${mininet_conn_id}
70 Verify Flows In Switch After Leader Restart
71 [Documentation] Verify flows are installed in switch after leader restart.
72 MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout}
74 Stop Mininet Connected To Follower Node2 and Exit
75 [Documentation] Stop mininet and exit connection.
76 MininetKeywords.Stop Mininet And Exit ${mininet_conn_id}
77 Utils.Clean Mininet System
79 Restart Follower Node2
80 [Documentation] Kill Follower Node2 and Start it Up, Verify it is sync with other controller node.
81 ClusterKeywords.Kill Multiple Controllers ${follower_node_2}
82 ClusterKeywords.Start Multiple Controllers ${START_TIMEOUT} ${follower_node_2}
84 Get inventory Follower After Follower Restart
85 [Documentation] Find Followers and Leader in the inventory config shard After Follower Restart.
86 ${inventory_leader} ${inventory_followers} ClusterOpenFlow.Get InventoryConfig Shard Status ${original_cluster_list}
87 ${follower_node_1}= Get From List ${inventory_followers} 0
88 ${follower_node_2}= Get From List ${inventory_followers} 1
89 Set Suite Variable ${follower_node_1}
90 Set Suite Variable ${follower_node_2}
91 Set Suite Variable ${inventory_leader}
93 Start Mininet Connect To Leader
94 [Documentation] Start mininet with connection to Leader.
95 ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${inventory_leader}_IP}
96 Set Suite Variable ${mininet_conn_id}
98 Verify Flows In Switch After Follower Restart
99 [Documentation] Verify flows are installed in switch after follower restart.
100 MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout}
102 Stop Mininet Connected To Leader and Exit
103 [Documentation] Stop mininet Connected To Other Follower and exit connection.
104 MininetKeywords.Stop Mininet And Exit ${mininet_conn_id}
105 Utils.Clean Mininet System
108 [Documentation] Kill all Cluster Nodes and Start it Up All.
109 ClusterKeywords.Kill Multiple Controllers @{original_cluster_list}
110 ClusterKeywords.Start Multiple Controllers ${START_TIMEOUT} @{original_cluster_list}
112 Get inventory Status After Cluster Restart
113 [Documentation] Find New Followers and Leader in the inventory config shard After Cluster Restart.
114 ${inventory_leader} ${inventory_followers} ClusterOpenFlow.Get InventoryConfig Shard Status ${original_cluster_list}
115 ${follower_node_1}= Get From List ${inventory_followers} 0
116 ${follower_node_2}= Get From List ${inventory_followers} 1
117 Set Suite Variable ${follower_node_1}
118 Set Suite Variable ${follower_node_2}
119 Set Suite Variable ${inventory_leader}
121 Start Mininet Connect To Follower Node2 After Cluster Restart
122 [Documentation] Start mininet with connection to cluster Follower Node2.
123 ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${follower_node_2}_IP}
124 Set Suite Variable ${mininet_conn_id}
126 Verify Flows In Switch After Cluster Restart
127 [Documentation] Verify flows are installed in switch after cluster restart.
128 MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout}
130 Delete Flows In Follower Node1 and Verify After Leader Restart
131 [Documentation] Delete Flow in Follower Node1.
132 ClusterOpenFlow.Delete Sample Flow and Verify ${original_cluster_list} ${follower_node_1}
134 Stop Mininet Connected To Follower Node2 and Exit After Cluster Restart
135 [Documentation] Stop mininet Connected To Other Follower and exit connection.
136 MininetKeywords.Stop Mininet And Exit ${mininet_conn_id}
137 Utils.Clean Mininet System