Increase topology recover timeout
[integration/test.git] / csit / libraries / ClusterManagement.robot
index e01f5e5b61cd0f481d45575437d599bd7d1b2f38..36d73e66d47efa6a54e6120008bc5286db1ff01b 100644 (file)
@@ -45,7 +45,8 @@ ${JOLOKIA_CONF_SHARD_MANAGER_URI}    jolokia/read/org.opendaylight.controller:Ca
 ${JOLOKIA_OPER_SHARD_MANAGER_URI}    jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-operational,type=DistributedOperationalDatastore
 ${JOLOKIA_READ_URI}    jolokia/read/org.opendaylight.controller
 ${KARAF_HOME}     ${WORKSPACE}${/}${BUNDLEFOLDER}    # TODO: Migrate to Variables.robot
-@{ODL_DEFAULT_DATA_PATHS}    tmp/    data/    cache/    snapshots/    journal/    etc/opendaylight/current/
+# Bug 9044 workaround: delete etc/host.key before restart.
+@{ODL_DEFAULT_DATA_PATHS}    tmp/    data/    cache/    snapshots/    journal/    etc/opendaylight/current/    etc/host.key
 ${RESTCONF_MODULES_DIR}    ${CURDIR}/../variables/restconf/modules
 ${SINGLETON_NETCONF_DEVICE_ID_PREFIX}    /odl-general-entity:entity[odl-general-entity:name='KeyedInstanceIdentifier{targetType=interface org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node, path=[org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NetworkTopology, org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.Topology[key=TopologyKey [_topologyId=Uri [_value=topology-netconf]]], org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node[key=NodeKey [_nodeId=Uri [_value=
 ${SINGLETON_NETCONF_DEVICE_ID_SUFFIX}    ]]]]}']
@@ -53,6 +54,7 @@ ${SINGLETON_ELECTION_ENTITY_TYPE}    org.opendaylight.mdsal.ServiceEntityType
 ${SINGLETON_CHANGE_OWNERSHIP_ENTITY_TYPE}    org.opendaylight.mdsal.AsyncServiceCloseEntityType
 ${NODE_START_COMMAND}    ${KARAF_HOME}/bin/start
 ${NODE_STOP_COMMAND}    ${KARAF_HOME}/bin/stop
+${NODE_KARAF_COUNT_COMMAND}    ps axf | grep org.apache.karaf | grep -v grep | wc -l
 ${NODE_KILL_COMMAND}    ps axf | grep org.apache.karaf | grep -v grep | awk '{print \"kill -9 \" $1}' | sh
 ${NODE_FREEZE_COMMAND}    ps axf | grep org.apache.karaf | grep -v grep | awk '{print \"kill -STOP \" $1}' | sh
 ${NODE_UNFREEZE_COMMAND}    ps axf | grep org.apache.karaf | grep -v grep | awk '{print \"kill -CONT \" $1}' | sh
@@ -355,6 +357,8 @@ Kill_Single_Member
     [Documentation]    Convenience keyword that kills the specified member of the cluster.
     ...    The KW will return a list of available members: \${updated index_list}=\${original_index_list}-\${member}
     ${index_list} =    ClusterManagement__Build_List    ${member}
+    ${member_ip} =    Return_Member_IP    ${member}
+    KarafKeywords.Log_Message_To_Controller_Karaf    Killing ODL${member} ${member_ip}
     ${updated_index_list} =    Kill_Members_From_List_Or_All    ${index_list}    ${original_index_list}    ${confirm}
     [Return]    ${updated_index_list}
 
@@ -373,6 +377,7 @@ Kill_Members_From_List_Or_All
     BuiltIn.Sleep    1s    Kill -9 closes open files, which may take longer than ssh overhead, but not long enough to warrant WUKS.
     : FOR    ${index}    IN    @{kill_index_list}
     \    Verify_Karaf_Is_Not_Running_On_Member    member_index=${index}
+    Run_Bash_Command_On_List_Or_All    command=netstat -pnatu | grep 2550
     [Return]    ${updated_index_list}
 
 Stop_Single_Member
@@ -396,6 +401,7 @@ Stop_Members_From_List_Or_All
     BuiltIn.Return_From_Keyword_If    not ${confirm}    ${updated_index_list}
     : FOR    ${index}    IN    @{stop_index_list}
     \    BuiltIn.Wait Until Keyword Succeeds    ${timeout}    2s    Verify_Karaf_Is_Not_Running_On_Member    member_index=${index}
+    Run_Bash_Command_On_List_Or_All    command=netstat -pnatu | grep 2550
     [Return]    ${updated_index_list}
 
 Start_Single_Member
@@ -414,11 +420,12 @@ Start_Members_From_List_Or_All
     ${command} =    BuiltIn.Set_Variable_If    """${export_java_home}""" != ""    export JAVA_HOME="${export_java_home}"; ${base_command}    ${base_command}
     ${epoch} =    DateTime.Get_Current_Date    time_zone=UTC    result_format=epoch    exclude_millis=False
     ${gc_filepath} =    BuiltIn.Set_Variable_If    """${karaf_home}""" != ""    ${karaf_home}/data/log/gc_${epoch}.log    ${GC_LOG_PATH}/gc_${epoch}.log
-    ${gc_options} =    BuiltIn.Set_Variable    -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:${gc_filepath}
+    ${gc_options} =    BuiltIn.Set_Variable_If    "docker" not in """${node_start_command}"""    -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:${gc_filepath}    ${EMPTY}
     Run_Bash_Command_On_List_Or_All    command=${command} ${gc_options}    member_index_list=${member_index_list}
     BuiltIn.Return_From_Keyword_If    not ${wait_for_sync}
     BuiltIn.Wait_Until_Keyword_Succeeds    ${timeout}    10s    Check_Cluster_Is_In_Sync    member_index_list=${member_index_list}
     # TODO: Do we also want to check Shard Leaders here?
+    [Teardown]    Run_Bash_Command_On_List_Or_All    command=netstat -pnatu | grep 2550
 
 Freeze_Single_Member
     [Arguments]    ${member}
@@ -465,7 +472,7 @@ Verify_Single_Karaf_Is_Running_On_Member
 Count_Running_Karafs_On_Member
     [Arguments]    ${member_index}
     [Documentation]    Remotely execute grep for karaf process, return count as string.
-    ${command} =    BuiltIn.Set_Variable    ps axf | grep org.apache.karaf | grep -v grep | wc -l
+    ${command} =    BuiltIn.Set_Variable    ${NODE_KARAF_COUNT_COMMAND}
     ${count} =    Run_Bash_Command_On_Member    command=${command}    member_index=${member_index}
     [Return]    ${count}
 
@@ -540,6 +547,7 @@ Run_Bash_Command_On_Member
     # TODO: Migrate callers to Check_Bash_Command_*
     ${member_ip} =    Collections.Get_From_Dictionary    dictionary=${ClusterManagement__index_to_ip_mapping}    key=${member_index}
     ${output} =    SSHKeywords.Run_Keyword_Preserve_Connection    Utils.Run_Command_On_Controller    ${member_ip}    ${command}
+    Log    ${output}
     [Return]    ${output}
 
 Run_Karaf_Command_On_List_Or_All
@@ -794,3 +802,10 @@ Sync_Status_Should_Be_True
     [Documentation]    Verify that cluster node is in sync with others
     ${status}    Get_Sync_Status_Of_Member    ${controller_index}
     BuiltIn.Should_Be_True    ${status}
+
+Return_Member_IP
+    [Arguments]    ${member_index}
+    [Documentation]    Return the IP address of the member given the member_index.
+    ${member_int} =    BuiltIn.Convert_To_Integer    ${member_index}
+    ${member_ip} =    Collections.Get_From_Dictionary    dictionary=${ClusterManagement__index_to_ip_mapping}    key=${member_int}
+    [Return]    ${member_ip}