From: Chandra Shekar S Date: Wed, 24 Jul 2019 09:49:01 +0000 (+0530) Subject: Check for the SHARD Status before opening the OVSDB port/HwvtepSouthboundProvider... X-Git-Tag: release/magnesium~10 X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=commitdiff_plain;h=be1d1857fcd2337fee1cbae751d91306791c7442;p=ovsdb.git Check for the SHARD Status before opening the OVSDB port/HwvtepSouthboundProvider intialization. JIRA: OVSDB-484 Currently the HwvtepSouthboundProvider is intialized and the Ovsdb port is opened. As soon as the Ovsdb port is opened the tors will connected to the HwvtepSouthboundPlugin. The HwvtepSouthboundPlugin will process the connected tors and tries to build the topology operational and further the topology config and tries to update these datastores. If the topology SHARDs are not up or the SHARD leaders are not elected by this time, it ends up in the exceptions while updating the datastores and the operational and config datastores will not be populated with the expected data. The fix is check for the topology config and operational SHARDs status up and Leaders for these are elected and then initialize HwvtepSouthboundProvider and start Ovsdb port when they are up. Signed-off-by: Chandra Shekar S Change-Id: Ia8599b9ed9cc3d5afbdb29a951cebee01c15ddbe --- diff --git a/hwvtepsouthbound/hwvtepsouthbound-impl/pom.xml b/hwvtepsouthbound/hwvtepsouthbound-impl/pom.xml index a7ba31b9e..738e7ed0a 100644 --- a/hwvtepsouthbound/hwvtepsouthbound-impl/pom.xml +++ b/hwvtepsouthbound/hwvtepsouthbound-impl/pom.xml @@ -140,6 +140,28 @@ and is available at http://www.eclipse.org/legal/epl-v10.html false + + org.codehaus.mojo + build-helper-maven-plugin + + + attach-artifacts + + attach-artifact + + package + + + + ${project.build.directory}/classes/initial/hwvtepsouthbound.cfg + cfg + config + + + + + + diff --git a/hwvtepsouthbound/hwvtepsouthbound-impl/src/main/java/org/opendaylight/ovsdb/hwvtepsouthbound/HwvtepSouthboundProvider.java b/hwvtepsouthbound/hwvtepsouthbound-impl/src/main/java/org/opendaylight/ovsdb/hwvtepsouthbound/HwvtepSouthboundProvider.java index 322fa4eb8..34b44c1f0 100644 --- a/hwvtepsouthbound/hwvtepsouthbound-impl/src/main/java/org/opendaylight/ovsdb/hwvtepsouthbound/HwvtepSouthboundProvider.java +++ b/hwvtepsouthbound/hwvtepsouthbound-impl/src/main/java/org/opendaylight/ovsdb/hwvtepsouthbound/HwvtepSouthboundProvider.java @@ -38,6 +38,7 @@ import org.opendaylight.ovsdb.hwvtepsouthbound.reconciliation.configuration.Hwvt import org.opendaylight.ovsdb.hwvtepsouthbound.transactions.md.TransactionInvoker; import org.opendaylight.ovsdb.hwvtepsouthbound.transactions.md.TransactionInvokerImpl; import org.opendaylight.ovsdb.lib.OvsdbConnection; +import org.opendaylight.ovsdb.utils.mdsal.utils.ShardStatusMonitor; import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NetworkTopology; import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.Topology; import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.TopologyBuilder; @@ -66,6 +67,7 @@ public class HwvtepSouthboundProvider implements ClusteredDataTreeChangeListener private HwvtepReconciliationManager hwvtepReconciliationManager; private final AtomicBoolean registered = new AtomicBoolean(false); private ListenerRegistration operTopologyRegistration; + private int shardStatusCheckRetryCount = 1000; @Inject public HwvtepSouthboundProvider(@Reference final DataBroker dataBroker, @@ -87,6 +89,27 @@ public class HwvtepSouthboundProvider implements ClusteredDataTreeChangeListener */ @PostConstruct public void init() { + boolean isDatastoreAvailable = false; + int retryCount = 0; + try { + while (retryCount < shardStatusCheckRetryCount) { + isDatastoreAvailable = ShardStatusMonitor.getShardStatus(ShardStatusMonitor.TOPOLOGY_SHARDS); + if (isDatastoreAvailable) { + break; + } + LOG.warn("Hwvtep: retrying shard status check for the {} time", ++retryCount); + Thread.sleep(2000); + } + if (isDatastoreAvailable) { + LOG.info("Hwvtep is UP"); + init2(); + } + } catch (InterruptedException e) { + LOG.error("Error in intializing the Hwvtep Southbound ", e); + } + } + + private void init2() { LOG.info("HwvtepSouthboundProvider Session Initiated"); txInvoker = new TransactionInvokerImpl(dataBroker); cm = new HwvtepConnectionManager(dataBroker, txInvoker, entityOwnershipService, ovsdbConnection); @@ -196,6 +219,10 @@ public class HwvtepSouthboundProvider implements ClusteredDataTreeChangeListener } } + public void setShardStatusCheckRetryCount(int retryCount) { + this.shardStatusCheckRetryCount = retryCount; + } + private static class HwvtepsbPluginInstanceEntityOwnershipListener implements EntityOwnershipListener { private final HwvtepSouthboundProvider hsp; private final EntityOwnershipListenerRegistration listenerRegistration; diff --git a/hwvtepsouthbound/hwvtepsouthbound-impl/src/main/java/org/opendaylight/ovsdb/hwvtepsouthbound/HwvtepSouthboundProviderConfigurator.java b/hwvtepsouthbound/hwvtepsouthbound-impl/src/main/java/org/opendaylight/ovsdb/hwvtepsouthbound/HwvtepSouthboundProviderConfigurator.java new file mode 100644 index 000000000..6ad47a328 --- /dev/null +++ b/hwvtepsouthbound/hwvtepsouthbound-impl/src/main/java/org/opendaylight/ovsdb/hwvtepsouthbound/HwvtepSouthboundProviderConfigurator.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2020 Ericsson India Global Services Pvt Ltd. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.ovsdb.hwvtepsouthbound; + +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Helper to let Blueprint XML configure {@link HwvtepSouthboundProvider}. + * + * @author Chandra Shekar S + */ +public class HwvtepSouthboundProviderConfigurator { + + private static final Logger LOG = LoggerFactory.getLogger(HwvtepSouthboundProviderConfigurator.class); + + private static final String SHARD_STATUS_CHECK_RETRY_COUNT = "shard-status-check-retry-count"; + + private final HwvtepSouthboundProvider hwvtepSouthboundProvider; + + public HwvtepSouthboundProviderConfigurator(HwvtepSouthboundProvider hwvtepSouthboundProvider) { + this.hwvtepSouthboundProvider = hwvtepSouthboundProvider; + } + + public void setShardStatusCheckRetryCount(int retryCount) { + hwvtepSouthboundProvider.setShardStatusCheckRetryCount(retryCount); + } + + + + public void updateConfigParameter(Map configParameters) { + if (configParameters != null && !configParameters.isEmpty()) { + LOG.debug("Config parameters received : {}", configParameters.entrySet()); + for (Map.Entry paramEntry : configParameters.entrySet()) { + if (paramEntry.getKey().equalsIgnoreCase(SHARD_STATUS_CHECK_RETRY_COUNT)) { + hwvtepSouthboundProvider + .setShardStatusCheckRetryCount(Integer.parseInt((String) paramEntry.getValue())); + } + } + } + } +} diff --git a/hwvtepsouthbound/hwvtepsouthbound-impl/src/main/resources/OSGI-INF/blueprint/hwvtepsouthbound.xml b/hwvtepsouthbound/hwvtepsouthbound-impl/src/main/resources/OSGI-INF/blueprint/hwvtepsouthbound.xml index b8937d063..7c6c2b53d 100644 --- a/hwvtepsouthbound/hwvtepsouthbound-impl/src/main/resources/OSGI-INF/blueprint/hwvtepsouthbound.xml +++ b/hwvtepsouthbound/hwvtepsouthbound-impl/src/main/resources/OSGI-INF/blueprint/hwvtepsouthbound.xml @@ -1,8 +1,28 @@ + + + + + + + + + + + + + + diff --git a/hwvtepsouthbound/hwvtepsouthbound-impl/src/main/resources/initial/hwvtepsouthbound.cfg b/hwvtepsouthbound/hwvtepsouthbound-impl/src/main/resources/initial/hwvtepsouthbound.cfg new file mode 100644 index 000000000..e4bbc75de --- /dev/null +++ b/hwvtepsouthbound/hwvtepsouthbound-impl/src/main/resources/initial/hwvtepsouthbound.cfg @@ -0,0 +1,7 @@ +#******************************************************************************************** +# Boot Time Configuration * +# Config knob changes will require controller restart * +#******************************************************************************************** +#HwvtepSouthboundPlugin will check the topology SHARDS for their status to be up. +#This the retry count to check the topoloy SHARDS status by the HwvtepSouthboundPlugin. +shard-status-check-retry-count = 1000 diff --git a/utils/mdsal-utils/src/main/java/org/opendaylight/ovsdb/utils/mdsal/utils/ShardStatusMonitor.java b/utils/mdsal-utils/src/main/java/org/opendaylight/ovsdb/utils/mdsal/utils/ShardStatusMonitor.java new file mode 100644 index 000000000..4a2842a97 --- /dev/null +++ b/utils/mdsal-utils/src/main/java/org/opendaylight/ovsdb/utils/mdsal/utils/ShardStatusMonitor.java @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2020 Ericsson India Global Services Pvt Ltd. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ + +package org.opendaylight.ovsdb.utils.mdsal.utils; + +import java.lang.management.ManagementFactory; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; + +import javax.management.InstanceNotFoundException; +import javax.management.MBeanException; +import javax.management.MBeanServer; +import javax.management.MalformedObjectNameException; +import javax.management.ObjectName; +import javax.management.ReflectionException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class ShardStatusMonitor { + + private static final Logger LOG = LoggerFactory.getLogger(ShardStatusMonitor.class); + private static final String TOPOLOGY_CONFIG_SHARD = "topology:config"; + private static final String TOPOLOGY_OPER_SHARD = "topology:oper"; + private static final String STATUS_OPERATIONAL = "OPERATIONAL"; + + private static final String JMX_OBJECT_NAME_LIST_OF_CONFIG_SHARDS = + "org.opendaylight.controller:type=DistributedConfigDatastore,Category=ShardManager,name=shard-manager-config"; + private static final String JMX_OBJECT_NAME_LIST_OF_OPER_SHARDS = + "org.opendaylight.controller:type=DistributedOperationalDatastore," + + "Category=ShardManager,name=shard-manager-operational"; + + public static final Collection TOPOLOGY_SHARDS = + Collections.unmodifiableList(Arrays.asList(TOPOLOGY_CONFIG_SHARD, TOPOLOGY_OPER_SHARD)); + + //To avoid the checkstyle errors + private ShardStatusMonitor() { + + } + + @SuppressWarnings("checkstyle:IllegalCatch") + public static String getLeaderJMX(String objectName, String atrName) { + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + String leader = ""; + if (mbs != null) { + try { + leader = (String)mbs.getAttribute(new ObjectName(objectName), atrName); + } catch (Exception e) { + LOG.error("Failed to get leader jmx {}", e.getMessage()); + } + } + return leader; + } + + public static boolean getShardStatus(Collection shards) { + boolean status = true; + for (String shard : shards) { + String[] params = shard.split(":"); + if (!getDataStoreStatus(params[0], params[1]).equalsIgnoreCase(STATUS_OPERATIONAL)) { + status = false; + break; + } + } + return status; + } + + @SuppressWarnings("checkstyle:IllegalCatch") + public static String getDataStoreStatus(String name, String type) { + boolean statusResult = true; + try { + ArrayList listOfShards; + if (type.equalsIgnoreCase("config")) { + listOfShards = getAttributeJMXCommand(JMX_OBJECT_NAME_LIST_OF_CONFIG_SHARDS, "LocalShards"); + } else { + listOfShards = getAttributeJMXCommand(JMX_OBJECT_NAME_LIST_OF_OPER_SHARDS, "LocalShards"); + } + if (listOfShards != null) { + for (int i = 0; i < listOfShards.size(); i++) { + if (listOfShards.get(i).toString().contains(name)) { + String jmxObjectShardStatus; + if (type.equalsIgnoreCase("config")) { + jmxObjectShardStatus = "org.opendaylight.controller:Category=Shards,name=" + + listOfShards.get(i) + ",type=DistributedConfigDatastore"; + } else { + jmxObjectShardStatus = "org.opendaylight.controller:Category=Shards,name=" + + listOfShards.get(i) + ",type=DistributedOperationalDatastore"; + } + String leader = getLeaderJMX(jmxObjectShardStatus,"Leader"); + if (leader != null && leader.length() > 1) { + if (type.equalsIgnoreCase("config")) { + LOG.info("{} ::Config DS has the Leader as:: {}", listOfShards.get(i), leader); + } else { + LOG.info("{} ::Oper DS has the Leader as:: {}", listOfShards.get(i), leader); + } + } else { + statusResult = false; + } + } + } + } + } catch (Exception e) { + LOG.error("ERROR::", e); + statusResult = false; + } + if (statusResult) { + return STATUS_OPERATIONAL; + } else { + return "ERROR"; + } + } + + @SuppressWarnings("checkstyle:IllegalCatch") + public static ArrayList getAttributeJMXCommand(String objectName, String attributeName) { + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + ArrayList listOfShards = new ArrayList(); + if (mbs != null) { + try { + listOfShards = (ArrayList) mbs.getAttribute(new ObjectName(objectName), attributeName); + } catch (MalformedObjectNameException monEx) { + LOG.error("CRITICAL EXCEPTION : Malformed Object Name Exception"); + } catch (MBeanException mbEx) { + LOG.error("CRITICAL EXCEPTION : MBean Exception"); + } catch (InstanceNotFoundException infEx) { + LOG.error("CRITICAL EXCEPTION : Instance Not Found Exception"); + } catch (ReflectionException rEx) { + LOG.error("CRITICAL EXCEPTION : Reflection Exception"); + } catch (Exception e) { + LOG.error("Attribute not found"); + } + } + return listOfShards; + } + +} \ No newline at end of file