Check for the SHARD Status before opening the OVSDB port/HwvtepSouthboundProvider... 29/83229/5
authorChandra Shekar S <chandra.shekar.s@ericsson.com>
Wed, 24 Jul 2019 09:49:01 +0000 (15:19 +0530)
committerChandra Shekar S <chandra.shekar.s@ericsson.com>
Wed, 8 Jan 2020 10:57:51 +0000 (16:27 +0530)
JIRA: OVSDB-484

Currently the HwvtepSouthboundProvider is intialized and the Ovsdb port is opened.
As soon as the Ovsdb port is opened the tors will connected to the HwvtepSouthboundPlugin.
The HwvtepSouthboundPlugin will process the connected tors and tries to build the topology operational
and further the topology config and tries to update these datastores.
If the topology SHARDs are not up or the SHARD leaders are not elected by this time, it ends up in the
exceptions while updating the datastores and the operational and config datastores will not be populated
with the expected data.

The fix is check for the topology config and operational SHARDs status up and Leaders for these are
elected and then initialize HwvtepSouthboundProvider and start Ovsdb port when they are up.

Signed-off-by: Chandra Shekar S <chandra.shekar.s@ericsson.com>
Change-Id: Ia8599b9ed9cc3d5afbdb29a951cebee01c15ddbe

hwvtepsouthbound/hwvtepsouthbound-impl/pom.xml
hwvtepsouthbound/hwvtepsouthbound-impl/src/main/java/org/opendaylight/ovsdb/hwvtepsouthbound/HwvtepSouthboundProvider.java
hwvtepsouthbound/hwvtepsouthbound-impl/src/main/java/org/opendaylight/ovsdb/hwvtepsouthbound/HwvtepSouthboundProviderConfigurator.java [new file with mode: 0644]
hwvtepsouthbound/hwvtepsouthbound-impl/src/main/resources/OSGI-INF/blueprint/hwvtepsouthbound.xml
hwvtepsouthbound/hwvtepsouthbound-impl/src/main/resources/initial/hwvtepsouthbound.cfg [new file with mode: 0644]
utils/mdsal-utils/src/main/java/org/opendaylight/ovsdb/utils/mdsal/utils/ShardStatusMonitor.java [new file with mode: 0644]

index a7ba31b9e86d1aedb110965ccf91ff15d1826d34..738e7ed0a542ced6fc3ca97bd12487f17d8899b9 100644 (file)
@@ -140,6 +140,28 @@ and is available at http://www.eclipse.org/legal/epl-v10.html
           <reuseForks>false</reuseForks>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>build-helper-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>attach-artifacts</id>
+            <goals>
+              <goal>attach-artifact</goal>
+            </goals>
+            <phase>package</phase>
+            <configuration>
+              <artifacts>
+                <artifact>
+                  <file>${project.build.directory}/classes/initial/hwvtepsouthbound.cfg</file>
+                  <type>cfg</type>
+                  <classifier>config</classifier>
+                </artifact>
+              </artifacts>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 
index 322fa4eb86ed988ba2a81ab82a2d790ed0a1e9a3..34b44c1f08b4c7e7d93ca425d381bde4a6d6f7de 100644 (file)
@@ -38,6 +38,7 @@ import org.opendaylight.ovsdb.hwvtepsouthbound.reconciliation.configuration.Hwvt
 import org.opendaylight.ovsdb.hwvtepsouthbound.transactions.md.TransactionInvoker;
 import org.opendaylight.ovsdb.hwvtepsouthbound.transactions.md.TransactionInvokerImpl;
 import org.opendaylight.ovsdb.lib.OvsdbConnection;
+import org.opendaylight.ovsdb.utils.mdsal.utils.ShardStatusMonitor;
 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NetworkTopology;
 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.Topology;
 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.TopologyBuilder;
@@ -66,6 +67,7 @@ public class HwvtepSouthboundProvider implements ClusteredDataTreeChangeListener
     private HwvtepReconciliationManager hwvtepReconciliationManager;
     private final AtomicBoolean registered = new AtomicBoolean(false);
     private ListenerRegistration<HwvtepSouthboundProvider> operTopologyRegistration;
+    private int shardStatusCheckRetryCount = 1000;
 
     @Inject
     public HwvtepSouthboundProvider(@Reference final DataBroker dataBroker,
@@ -87,6 +89,27 @@ public class HwvtepSouthboundProvider implements ClusteredDataTreeChangeListener
      */
     @PostConstruct
     public void init() {
+        boolean isDatastoreAvailable = false;
+        int retryCount = 0;
+        try {
+            while (retryCount < shardStatusCheckRetryCount) {
+                isDatastoreAvailable = ShardStatusMonitor.getShardStatus(ShardStatusMonitor.TOPOLOGY_SHARDS);
+                if (isDatastoreAvailable) {
+                    break;
+                }
+                LOG.warn("Hwvtep: retrying shard status check for the {} time", ++retryCount);
+                Thread.sleep(2000);
+            }
+            if (isDatastoreAvailable) {
+                LOG.info("Hwvtep is UP");
+                init2();
+            }
+        } catch (InterruptedException e) {
+            LOG.error("Error in intializing the Hwvtep Southbound ", e);
+        }
+    }
+
+    private void init2() {
         LOG.info("HwvtepSouthboundProvider Session Initiated");
         txInvoker = new TransactionInvokerImpl(dataBroker);
         cm = new HwvtepConnectionManager(dataBroker, txInvoker, entityOwnershipService, ovsdbConnection);
@@ -196,6 +219,10 @@ public class HwvtepSouthboundProvider implements ClusteredDataTreeChangeListener
         }
     }
 
+    public void setShardStatusCheckRetryCount(int retryCount) {
+        this.shardStatusCheckRetryCount = retryCount;
+    }
+
     private static class HwvtepsbPluginInstanceEntityOwnershipListener implements EntityOwnershipListener {
         private final HwvtepSouthboundProvider hsp;
         private final EntityOwnershipListenerRegistration listenerRegistration;
diff --git a/hwvtepsouthbound/hwvtepsouthbound-impl/src/main/java/org/opendaylight/ovsdb/hwvtepsouthbound/HwvtepSouthboundProviderConfigurator.java b/hwvtepsouthbound/hwvtepsouthbound-impl/src/main/java/org/opendaylight/ovsdb/hwvtepsouthbound/HwvtepSouthboundProviderConfigurator.java
new file mode 100644 (file)
index 0000000..6ad47a3
--- /dev/null
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2020 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.ovsdb.hwvtepsouthbound;
+
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Helper to let Blueprint XML configure {@link HwvtepSouthboundProvider}.
+ *
+ * @author Chandra Shekar S
+ */
+public class HwvtepSouthboundProviderConfigurator {
+
+    private static final Logger LOG = LoggerFactory.getLogger(HwvtepSouthboundProviderConfigurator.class);
+
+    private static final String SHARD_STATUS_CHECK_RETRY_COUNT = "shard-status-check-retry-count";
+
+    private final HwvtepSouthboundProvider hwvtepSouthboundProvider;
+
+    public HwvtepSouthboundProviderConfigurator(HwvtepSouthboundProvider hwvtepSouthboundProvider) {
+        this.hwvtepSouthboundProvider = hwvtepSouthboundProvider;
+    }
+
+    public void setShardStatusCheckRetryCount(int retryCount) {
+        hwvtepSouthboundProvider.setShardStatusCheckRetryCount(retryCount);
+    }
+
+
+
+    public void updateConfigParameter(Map<String, Object> configParameters) {
+        if (configParameters != null && !configParameters.isEmpty()) {
+            LOG.debug("Config parameters received : {}", configParameters.entrySet());
+            for (Map.Entry<String, Object> paramEntry : configParameters.entrySet()) {
+                if (paramEntry.getKey().equalsIgnoreCase(SHARD_STATUS_CHECK_RETRY_COUNT)) {
+                    hwvtepSouthboundProvider
+                            .setShardStatusCheckRetryCount(Integer.parseInt((String) paramEntry.getValue()));
+                }
+            }
+        }
+    }
+}
index b8937d0633a83e4df9fda7ba7240795643d6057a..7c6c2b53de46bfe8df1c7e1a6fba01097ef18f10 100644 (file)
@@ -1,8 +1,28 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <blueprint xmlns="http://www.osgi.org/xmlns/blueprint/v1.0.0"
   xmlns:odl="http://opendaylight.org/xmlns/blueprint/v1.0.0"
+  xmlns:cm="http://aries.apache.org/blueprint/xmlns/blueprint-cm/v1.1.0"
   odl:use-default-for-reference-types="true">
 
+    <!-- Read default values at startup and inject to HwvtepSouthboundProvider-->
+    <cm:property-placeholder persistent-id="org.opendaylight.ovsdb.hwvtepsouthbound" update-strategy="none">
+        <!-- Setting default values, in case hwvtepsouthbound.cfg file is not present
+        or config property is commented out. This will be overridden if user
+        specify the property in hwvtepsouthbound.cfg file-->
+        <cm:default-properties>
+            <cm:property name="shard-status-check-retry-count" value="1000"/>
+        </cm:default-properties>
+    </cm:property-placeholder>
+
+    <bean id="hwvtepSouthboundProviderConfigurator"
+          class="org.opendaylight.ovsdb.hwvtepsouthbound.HwvtepSouthboundProviderConfigurator">
+        <cm:managed-properties persistent-id="org.opendaylight.ovsdb.hwvtepsouthbound"
+                               update-strategy="component-managed"
+                               update-method="updateConfigParameter"/>
+        <argument ref="hwvtepSouthboundProvider" />
+        <property name="shardStatusCheckRetryCount" value="${shard-status-check-retry-count}"/>
+    </bean>
+
     <command-bundle xmlns="http://karaf.apache.org/xmlns/shell/v1.1.0">
     <command>
         <action class="org.opendaylight.ovsdb.hwvtepsouthbound.TransactionHistoryCmd">
diff --git a/hwvtepsouthbound/hwvtepsouthbound-impl/src/main/resources/initial/hwvtepsouthbound.cfg b/hwvtepsouthbound/hwvtepsouthbound-impl/src/main/resources/initial/hwvtepsouthbound.cfg
new file mode 100644 (file)
index 0000000..e4bbc75
--- /dev/null
@@ -0,0 +1,7 @@
+#********************************************************************************************
+#                               Boot Time Configuration                                     *
+#                   Config knob changes will require controller restart                     *
+#********************************************************************************************
+#HwvtepSouthboundPlugin will check the topology SHARDS for their status to be up.
+#This the retry count to check the topoloy SHARDS status by the HwvtepSouthboundPlugin.
+shard-status-check-retry-count = 1000
diff --git a/utils/mdsal-utils/src/main/java/org/opendaylight/ovsdb/utils/mdsal/utils/ShardStatusMonitor.java b/utils/mdsal-utils/src/main/java/org/opendaylight/ovsdb/utils/mdsal/utils/ShardStatusMonitor.java
new file mode 100644 (file)
index 0000000..4a2842a
--- /dev/null
@@ -0,0 +1,141 @@
+/*
+ * Copyright (c) 2020 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+
+package org.opendaylight.ovsdb.utils.mdsal.utils;
+
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+
+import javax.management.InstanceNotFoundException;
+import javax.management.MBeanException;
+import javax.management.MBeanServer;
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+import javax.management.ReflectionException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public final class ShardStatusMonitor {
+
+    private static final Logger LOG = LoggerFactory.getLogger(ShardStatusMonitor.class);
+    private static final String TOPOLOGY_CONFIG_SHARD = "topology:config";
+    private static final String TOPOLOGY_OPER_SHARD = "topology:oper";
+    private static final String STATUS_OPERATIONAL = "OPERATIONAL";
+
+    private static final String JMX_OBJECT_NAME_LIST_OF_CONFIG_SHARDS =
+        "org.opendaylight.controller:type=DistributedConfigDatastore,Category=ShardManager,name=shard-manager-config";
+    private static final String JMX_OBJECT_NAME_LIST_OF_OPER_SHARDS =
+        "org.opendaylight.controller:type=DistributedOperationalDatastore,"
+                + "Category=ShardManager,name=shard-manager-operational";
+
+    public static final Collection<String> TOPOLOGY_SHARDS =
+            Collections.unmodifiableList(Arrays.asList(TOPOLOGY_CONFIG_SHARD, TOPOLOGY_OPER_SHARD));
+
+    //To avoid the checkstyle errors
+    private ShardStatusMonitor() {
+
+    }
+
+    @SuppressWarnings("checkstyle:IllegalCatch")
+    public static String getLeaderJMX(String objectName, String atrName) {
+        MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+        String leader = "";
+        if (mbs != null) {
+            try {
+                leader  = (String)mbs.getAttribute(new ObjectName(objectName), atrName);
+            } catch (Exception e) {
+                LOG.error("Failed to get leader jmx {}", e.getMessage());
+            }
+        }
+        return leader;
+    }
+
+    public static boolean getShardStatus(Collection<String> shards) {
+        boolean status = true;
+        for (String shard : shards) {
+            String[] params = shard.split(":");
+            if (!getDataStoreStatus(params[0], params[1]).equalsIgnoreCase(STATUS_OPERATIONAL)) {
+                status = false;
+                break;
+            }
+        }
+        return status;
+    }
+
+    @SuppressWarnings("checkstyle:IllegalCatch")
+    public static String getDataStoreStatus(String name, String type) {
+        boolean statusResult = true;
+        try {
+            ArrayList listOfShards;
+            if (type.equalsIgnoreCase("config")) {
+                listOfShards = getAttributeJMXCommand(JMX_OBJECT_NAME_LIST_OF_CONFIG_SHARDS, "LocalShards");
+            } else  {
+                listOfShards = getAttributeJMXCommand(JMX_OBJECT_NAME_LIST_OF_OPER_SHARDS, "LocalShards");
+            }
+            if (listOfShards != null) {
+                for (int i = 0; i < listOfShards.size(); i++) {
+                    if (listOfShards.get(i).toString().contains(name)) {
+                        String jmxObjectShardStatus;
+                        if (type.equalsIgnoreCase("config")) {
+                            jmxObjectShardStatus = "org.opendaylight.controller:Category=Shards,name="
+                                    + listOfShards.get(i) + ",type=DistributedConfigDatastore";
+                        } else {
+                            jmxObjectShardStatus = "org.opendaylight.controller:Category=Shards,name="
+                                    + listOfShards.get(i) + ",type=DistributedOperationalDatastore";
+                        }
+                        String leader = getLeaderJMX(jmxObjectShardStatus,"Leader");
+                        if (leader != null && leader.length() > 1) {
+                            if (type.equalsIgnoreCase("config")) {
+                                LOG.info("{} ::Config DS has the Leader as:: {}", listOfShards.get(i), leader);
+                            } else {
+                                LOG.info("{} ::Oper DS has the Leader as:: {}", listOfShards.get(i), leader);
+                            }
+                        } else {
+                            statusResult = false;
+                        }
+                    }
+                }
+            }
+        } catch (Exception e) {
+            LOG.error("ERROR::", e);
+            statusResult = false;
+        }
+        if (statusResult) {
+            return STATUS_OPERATIONAL;
+        } else {
+            return "ERROR";
+        }
+    }
+
+    @SuppressWarnings("checkstyle:IllegalCatch")
+    public static ArrayList getAttributeJMXCommand(String objectName, String attributeName) {
+        MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+        ArrayList listOfShards = new ArrayList();
+        if (mbs != null) {
+            try {
+                listOfShards = (ArrayList) mbs.getAttribute(new ObjectName(objectName), attributeName);
+            } catch (MalformedObjectNameException monEx) {
+                LOG.error("CRITICAL EXCEPTION : Malformed Object Name Exception");
+            } catch (MBeanException mbEx) {
+                LOG.error("CRITICAL EXCEPTION : MBean Exception");
+            } catch (InstanceNotFoundException infEx) {
+                LOG.error("CRITICAL EXCEPTION : Instance Not Found Exception");
+            } catch (ReflectionException rEx) {
+                LOG.error("CRITICAL EXCEPTION : Reflection Exception");
+            } catch (Exception e) {
+                LOG.error("Attribute not found");
+            }
+        }
+        return listOfShards;
+    }
+
+}
\ No newline at end of file