--- /dev/null
+*** Settings ***
+Documentation Bulkomatic Keyword library contains keywords for performing bulkomatic operations
+... with a single bulkomatic API we can trigger bulk flows in config datastore which eventually populates switches and operational datastore
+... So far this library is only to be used by MD-SAL clustering and OpenFlowplugin clustering test as it is very specific for these tests
+Resource Utils.robot
+Variables ../variables/Variables.py
+
+*** Variables ***
+${ADD_BULK_CONFIG_NODES_API} /restconf/operations/sal-bulk-flow:flow-test
+${GET_BULK_CONFIG_NODES_API} /restconf/operations/sal-bulk-flow:read-flow-test
+${jolokia_write_op_status} /jolokia/read/org.opendaylight.openflowplugin.applications.bulk.o.matic:type=FlowCounter/WriteOpStatus
+${jolokia_read_op_status} /jolokia/read/org.opendaylight.openflowplugin.applications.bulk.o.matic:type=FlowCounter/ReadOpStatus
+${jolokia_flow_count_status} /jolokia/read/org.opendaylight.openflowplugin.applications.bulk.o.matic:type=FlowCounter/FlowCount
+
+*** Keywords ***
+Operation Status Check
+ [Arguments] ${controller_index} ${op_status_uri}
+ [Documentation] Checks to see if read or write operation is successfull in controller node.
+ ${data}= Utils.Get Data From URI controller${controller_index} ${op_status_uri}
+ Log ${data}
+ ${json}= To Json ${data}
+ ${value}= Get From Dictionary ${json} value
+ ${value}= Convert to String ${value}
+ ${two}= Convert to String 2
+ Should Start With ${value} ${two}
+
+Wait Until Write Finishes
+ [Arguments] ${controller_index} ${timeout}
+ [Documentation] Wait Until Write operation status is OK in ${controller_index}.
+ Wait Until Keyword Succeeds ${timeout} 1s BulkomaticKeywords.Operation Status Check ${controller_index} ${jolokia_write_op_status}
+
+Wait Until Read Finishes
+ [Arguments] ${controller_index} ${timeout}
+ [Documentation] Wait Until Read operation status is OK in ${controller_index}.
+ Wait Until Keyword Succeeds ${timeout} 1s BulkomaticKeywords.Operation Status Check ${controller_index} ${jolokia_read_op_status}
+
+Add Bulk Flow
+ [Arguments] ${controller_index} ${add_bulk_json_file}
+ [Documentation] Add Bulk Flow in ${controller_index} according to ${add_bulk_json_file}.
+ ${add_body}= OperatingSystem.Get File ${CURDIR}/../variables/openflowplugin/${add_bulk_json_file}
+ ${resp} Utils.Post Elements To URI ${ADD_BULK_CONFIG_NODES_API} ${add_body} headers=${HEADERS_YANG_JSON} session=controller${controller_index}
+
+Delete Bulk Flow
+ [Arguments] ${controller_index} ${del_bulk_json_file}
+ [Documentation] Delete Bulk Flow in ${controller_index} according to ${del_bulk_json_file}.
+ ${del_body}= OperatingSystem.Get File ${CURDIR}/../variables/openflowplugin/${del_bulk_json_file}
+ ${resp} Utils.Post Elements To URI ${ADD_BULK_CONFIG_NODES_API} ${del_body} headers=${HEADERS_YANG_JSON} session=controller${controller_index}
+
+Get Bulk Flow
+ [Arguments] ${controller_index} ${get_bulk_json_file}
+ [Documentation] Get Bulk Flow in ${controller_index} according to ${get_bulk_json_file}.
+ ${get_body}= OperatingSystem.Get File ${CURDIR}/../variables/openflowplugin/${get_bulk_json_file}
+ ${resp} Utils.Post Elements To URI ${GET_BULK_CONFIG_NODES_API} ${get_body} headers=${HEADERS_YANG_JSON} session=controller${controller_index}
+
+Get Bulk Flow Count
+ [Arguments] ${controller_index}
+ [Documentation] Get Flow count in ${controller_index}. New Flow Count is available after Get Bulk Flow operation.
+ ${data}= Utils.Get Data From URI controller${controller_index} ${jolokia_flow_count_status}
+ Log ${data}
+ [Return] ${data}
+
+Verify Flow Count
+ [Arguments] ${controller_index} ${flow_count}
+ [Documentation] Verify Flow Count in ${controller_index} matches ${flow_count}.
+ ${data}= Get Bulk Flow Count ${controller_index}
+ Log ${data}
+ ${json}= To Json ${data}
+ ${value}= Get From Dictionary ${json} value
+ Should Be Equal As Strings ${value} ${flow_count}
+
+Add Bulk Flow In Node
+ [Arguments] ${controller_index} ${add_bulk_json_file} ${timeout}
+ [Documentation] Add Bulk Flow in ${controller_index} and wait until operation is completed.
+ Add Bulk Flow ${controller_index} ${add_bulk_json_file}
+ Wait Until Write Finishes ${controller_index} ${timeout}
+
+Delete Bulk Flow In Node
+ [Arguments] ${controller_index} ${delete_bulk_json_file} ${timeout}
+ [Documentation] Delete Bulk Flow in ${controller_index} and wait until operation is completed.
+ Delete Bulk Flow ${controller_index} ${delete_bulk_json_file}
+ Wait Until Write Finishes ${controller_index} ${timeout}
+
+Get Bulk Flow And Verify Count In Cluster
+ [Arguments] ${controller_index_list} ${get_bulk_json_file} ${timeout} ${flow_count}
+ [Documentation] Get Bulk Flow and Verify Flow Count in ${controller_index_list} matches ${flow_count}.
+ : FOR ${index} IN @{controller_index_list}
+ \ Get Bulk Flow ${index} ${get_bulk_json_file}
+ : FOR ${index} IN @{controller_index_list}
+ \ Wait Until Read Finishes ${index} ${timeout}
+ : FOR ${index} IN @{controller_index_list}
+ \ Verify Flow Count ${index} ${flow_count}
--- /dev/null
+*** Settings ***
+Documentation Test suite for Cluster HA with Bulk Flows - Data consistency after cluster restart, leader restart and follower restart with one switch connected
+Suite Setup Create Controller Sessions
+Suite Teardown Delete All Sessions
+Resource ../../../libraries/BulkomaticKeywords.robot
+Resource ../../../libraries/MininetKeywords.robot
+Resource ../../../libraries/ClusterKeywords.robot
+Resource ../../../libraries/ClusterOpenFlow.robot
+Resource ../../../libraries/Utils.robot
+Variables ../../../variables/Variables.py
+
+*** Variables ***
+${operation_timeout} 100s
+${restart_timeout} 350s
+${flow_count_per_switch} 1000
+${switch_count_per_node} 1
+${flow_count_after_add} 1000
+${flow_count_after_del} 0
+${add_small_config} sal_add_bulk_flow_small_config.json
+${get_small_config} sal_get_bulk_flow_small_config.json
+${del_small_config} sal_del_bulk_flow_small_config.json
+
+*** Test Cases ***
+Create Original Cluster List
+ [Documentation] Create original cluster list.
+ ${original_cluster_list} ClusterKeywords.Create Controller Index List
+ Set Suite Variable ${original_cluster_list}
+
+Check Shards Status Before Leader Restart
+ [Documentation] Check Status for all shards in OpenFlow application.
+ ClusterOpenFlow.Check OpenFlow Shards Status ${original_cluster_list}
+
+Get Inventory Follower Before Cluster Restart
+ [Documentation] Find a follower in the inventory config shard
+ ${inventory_leader} ${inventory_followers} ClusterOpenFlow.Get InventoryConfig Shard Status ${original_cluster_list}
+ ${Follower_Node_1}= Get From List ${Inventory_Followers} 0
+ Set Suite Variable ${Follower_Node_1}
+
+Start Mininet Connect To Follower Node1
+ [Documentation] Start mininet with connection to Follower Node1.
+ ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${Follower_Node_1}_IP}
+ Set Suite Variable ${mininet_conn_id}
+
+Add Bulk Flow From Follower
+ [Documentation] 1000 Flows added via Follower Node1 and verify it gets applied in all instances.
+ BulkomaticKeywords.Add Bulk Flow In Node ${Follower_Node_1} ${add_small_config} ${operation_timeout}
+
+Get Bulk Flows and Verify In Cluster
+ BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout} ${flow_count_after_add}
+
+Verify Flows In Switch Before Cluster Restart
+ [Documentation] Verify flows are installed in switch before cluster restart.
+ MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout}
+
+Kill All Cluster Nodes
+ [Documentation] Kill All Nodes.
+ ClusterKeywords.Kill Multiple Controllers @{original_cluster_list}
+
+Stop Mininet Connected To Follower Node1 and Exit
+ [Documentation] Stop mininet and exit connection.
+ MininetKeywords.Stop Mininet And Exit ${mininet_conn_id}
+ Utils.Clean Mininet System
+
+Restart All Cluster Nodes
+ [Documentation] Restart all cluster nodes.
+ ClusterKeywords.Start Multiple Controllers ${restart_timeout} @{original_cluster_list}
+
+Verify Data Recovery After Cluster Restart
+ [Documentation] 1000 Flows preserved in all controller instances.
+ Wait Until Keyword Succeeds ${restart_timeout} 2s BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout}
+ ... ${flow_count_after_add}
+
+Start Mininet Again Connect To Follower Node1
+ [Documentation] Start mininet with connection to follower node1.
+ ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${Follower_Node_1}_IP}
+ Set Suite Variable ${mininet_conn_id}
+
+Verify Flows In Switch After Cluster Restart
+ [Documentation] Verify flows are installed in switch after cluster restart.
+ MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout}
+
+Stop Mininet Connected To Follower Node1
+ [Documentation] Stop mininet and exit connection.
+ MininetKeywords.Stop Mininet And Exit ${mininet_conn_id}
+ Utils.Clean Mininet System
+
+Delete All Flows From Follower Node1
+ [Documentation] Rest 1000 Flows deleted via Follower Node1 and verify it gets applied in all instances.
+ BulkomaticKeywords.Delete Bulk Flow In Node ${Follower_Node_1} ${del_small_config} ${operation_timeout}
+
+Verify No Flows In Cluster
+ BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout} ${flow_count_after_del}
+
+Get Inventory Leader Before Leader Restart
+ [Documentation] Find leader in the inventory config shard
+ ${inventory_leader} ${inventory_followers} ClusterOpenFlow.Get InventoryConfig Shard Status ${original_cluster_list}
+ Set Suite Variable ${Inventory_Leader}
+
+Start Mininet Connect To Leader
+ [Documentation] Start mininet with connection to Leader Node.
+ ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${Inventory_Leader}_IP}
+ Set Suite Variable ${mininet_conn_id}
+
+Add Bulk Flow From Leader
+ [Documentation] 1000 Flows added via Follower Node1 and verify it gets applied in all instances.
+ BulkomaticKeywords.Add Bulk Flow In Node ${Inventory_Leader} ${add_small_config} ${operation_timeout}
+
+Get Bulk Flows and Verify In Cluster Before Leader Restart
+ BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout} ${flow_count_after_add}
+
+Verify Flows In Switch Before Leader Restart
+ [Documentation] Verify flows are installed in switch before leader restart.
+ MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout}
+
+Kill Leader From Cluster Node
+ [Documentation] Kill Leader Node and Start it Up.
+ ClusterKeywords.Kill Multiple Controllers ${Inventory_Leader}
+
+Stop Mininet Connected To Leader Node
+ [Documentation] Stop mininet and exit connection.
+ MininetKeywords.Stop Mininet And Exit ${mininet_conn_id}
+ Utils.Clean Mininet System
+
+Restart Leader from Cluster Node
+ [Documentation] Start Leader Node Up.
+ ClusterKeywords.Start Multiple Controllers ${restart_timeout} ${Inventory_Leader}
+
+Verify Data Recovery After Leader Restart
+ [Documentation] 1000 Flows preserved in all controller instances.
+ Wait Until Keyword Succeeds ${restart_timeout} 2s BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout}
+ ... ${flow_count_after_add}
+
+Start Mininet Again Connect To Leader
+ [Documentation] Start mininet with connection to Leader Node.
+ ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${Inventory_Leader}_IP}
+ Set Suite Variable ${mininet_conn_id}
+
+Verify Flows In Switch After Leader Restart
+ [Documentation] Verify flows are installed in switch after leader restart.
+ MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout}
+
+Stop Mininet Connected To Leader Node After Leader Restart
+ [Documentation] Stop mininet and exit connection.
+ MininetKeywords.Stop Mininet And Exit ${mininet_conn_id}
+ Utils.Clean Mininet System
+
+Delete All Flows From Leader Node
+ [Documentation] Rest 1000 Flows deleted via Leader Node and verify it gets applied in all instances.
+ BulkomaticKeywords.Delete Bulk Flow In Node ${Inventory_Leader} ${del_small_config} ${operation_timeout}
+
+Verify No Flows In Cluster After Leader Restart
+ BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout} ${flow_count_after_del}
+
+Get Inventory Follower Before follower Restart
+ [Documentation] Find follower in the inventory config shard
+ ${inventory_leader} ${inventory_followers} ClusterOpenFlow.Get InventoryConfig Shard Status ${original_cluster_list}
+ ${Follower_Node_2}= Get From List ${Inventory_Followers} 1
+ Set Suite Variable ${Follower_Node_2}
+
+Start Mininet Connect To Follower Node2
+ [Documentation] Start mininet with connection to Follower Node2.
+ ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${Follower_Node_2}_IP}
+ Set Suite Variable ${mininet_conn_id}
+
+Add Bulk Flow From Follower Node2
+ [Documentation] 1000 Flows added via Follower Node2 and verify it gets applied in all instances.
+ BulkomaticKeywords.Add Bulk Flow In Node ${Follower_Node_2} ${add_small_config} ${operation_timeout}
+
+Get Bulk Flows and Verify In Cluster Before Follower Restart
+ BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout} ${flow_count_after_add}
+
+Verify Flows In Switch Before Follower Restart
+ [Documentation] Verify flows are installed in switch before follower restart.
+ MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout}
+
+Kill Follower Node2
+ [Documentation] Kill Follower Node2.
+ ClusterKeywords.Kill Multiple Controllers ${Follower_Node_2}
+
+Stop Mininet Connected To Follower Node2 and Exit
+ [Documentation] Stop mininet and exit connection.
+ MininetKeywords.Stop Mininet And Exit ${mininet_conn_id}
+ Utils.Clean Mininet System
+
+Restart Follower Node2
+ [Documentation] Start Follower Node2 Up.
+ ClusterKeywords.Start Multiple Controllers ${restart_timeout} ${Follower_Node_2}
+
+Verify Data Recovery After Follower Node2 Restart
+ [Documentation] 1000 Flows preserved in all controller instances.
+ Wait Until Keyword Succeeds ${restart_timeout} 2s BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout}
+ ... ${flow_count_after_add}
+
+Start Mininet Again Connect To Follower Node2
+ [Documentation] Start mininet with connection to follower node1.
+ ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${Follower_Node_2}_IP}
+ Set Suite Variable ${mininet_conn_id}
+
+Verify Flows In Switch After Follower Node2 Restart
+ [Documentation] Verify flows are installed in switch after follower restart.
+ MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout}
+
+Stop Mininet Connected To Follower Node2
+ [Documentation] Stop mininet and exit connection.
+ MininetKeywords.Stop Mininet And Exit ${mininet_conn_id}
+ Utils.Clean Mininet System
+
+Delete All Flows From Follower Node 2
+ [Documentation] Rest 1000 Flows deleted via Leader Node and verify it gets applied in all instances.
+ BulkomaticKeywords.Delete Bulk Flow In Node ${Follower_Node_2} ${del_small_config} ${operation_timeout}
+
+Verify No Flows In Cluster After Follower Node2 Restart
+ BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout} ${flow_count_after_del}