Synchronize the HwvtepNode Delete and Add for connection flaps 55/86955/8
authorChandra Shekar S <chandra.shekar.s@ericsson.com>
Thu, 16 Jan 2020 09:21:53 +0000 (14:51 +0530)
committerChandra Shekar S <chandra.shekar.s@ericsson.com>
Tue, 28 Jan 2020 09:45:51 +0000 (09:45 +0000)
Signed-off-by: Chandra Shekar S <chandra.shekar.s@ericsson.com>
Change-Id: I55cbd9324995d61cf9f8f83445f294bbe6f34b0b

hwvtepsouthbound/hwvtepsouthbound-impl/src/main/java/org/opendaylight/ovsdb/hwvtepsouthbound/HwvtepConnectionManager.java
hwvtepsouthbound/hwvtepsouthbound-impl/src/main/java/org/opendaylight/ovsdb/hwvtepsouthbound/HwvtepOperGlobalListener.java
hwvtepsouthbound/hwvtepsouthbound-impl/src/main/java/org/opendaylight/ovsdb/hwvtepsouthbound/HwvtepSouthboundConstants.java
hwvtepsouthbound/hwvtepsouthbound-impl/src/main/java/org/opendaylight/ovsdb/hwvtepsouthbound/HwvtepSouthboundUtil.java

index af799f7a9d067e1f3cc7ce220508ea92086e722e..440899b842d815bca96a2042b3584173b33a51b1 100644 (file)
@@ -116,16 +116,17 @@ public class HwvtepConnectionManager implements OvsdbConnectionListener, AutoClo
 
     @Override
     public void connected(final OvsdbClient externalClient) {
-        LOG.info("Library connected {} from {}:{} to {}:{}",
-                externalClient.getConnectionInfo().getType(),
-                externalClient.getConnectionInfo().getRemoteAddress(),
-                externalClient.getConnectionInfo().getRemotePort(),
-                externalClient.getConnectionInfo().getLocalAddress(),
-                externalClient.getConnectionInfo().getLocalPort());
+        HwvtepConnectionInstance hwClient = null;
         try {
             List<String> databases = externalClient.getDatabases().get(DB_FETCH_TIMEOUT, TimeUnit.MILLISECONDS);
-            if (databases.contains(HwvtepSchemaConstants.HARDWARE_VTEP)) {
-                HwvtepConnectionInstance hwClient = connectedButCallBacksNotRegistered(externalClient);
+            if (databases != null && !databases.isEmpty() && databases.contains(HwvtepSchemaConstants.HARDWARE_VTEP)) {
+                LOG.info("Hwvtep Library connected {} from {}:{} to {}:{}",
+                        externalClient.getConnectionInfo().getType(),
+                        externalClient.getConnectionInfo().getRemoteAddress(),
+                        externalClient.getConnectionInfo().getRemotePort(),
+                        externalClient.getConnectionInfo().getLocalAddress(),
+                        externalClient.getConnectionInfo().getLocalPort());
+                hwClient = connectedButCallBacksNotRegistered(externalClient);
                 registerEntityForOwnership(hwClient);
             }
         } catch (InterruptedException | ExecutionException | TimeoutException e) {
@@ -136,47 +137,53 @@ public class HwvtepConnectionManager implements OvsdbConnectionListener, AutoClo
     }
 
     @Override
+    @SuppressWarnings("checkstyle:IllegalCatch")
     public void disconnected(final OvsdbClient client) {
-        LOG.info("Library disconnected {} from {}:{} to {}:{}. Cleaning up the operational data store",
-                client.getConnectionInfo().getType(),
-                client.getConnectionInfo().getRemoteAddress(),
-                client.getConnectionInfo().getRemotePort(),
-                client.getConnectionInfo().getLocalAddress(),
-                client.getConnectionInfo().getLocalPort());
-        ConnectionInfo key = HwvtepSouthboundMapper.createConnectionInfo(client);
-        HwvtepConnectionInstance hwvtepConnectionInstance = getConnectionInstance(key);
-        if (hwvtepConnectionInstance != null) {
-            if (hwvtepConnectionInstance.getInstanceIdentifier() != null) {
-                deviceUpdateHistory.get(hwvtepConnectionInstance.getInstanceIdentifier()).addToHistory(
-                        TransactionType.DELETE, new ClientConnected(client.getConnectionInfo().getRemotePort()));
-            }
+        HwvtepConnectionInstance hwvtepConnectionInstance = null;
+        try {
+            LOG.info("Library disconnected {} from {}:{} to {}:{}. Cleaning up the operational data store",
+                    client.getConnectionInfo().getType(),
+                    client.getConnectionInfo().getRemoteAddress(),
+                    client.getConnectionInfo().getRemotePort(),
+                    client.getConnectionInfo().getLocalAddress(),
+                    client.getConnectionInfo().getLocalPort());
+            ConnectionInfo key = HwvtepSouthboundMapper.createConnectionInfo(client);
+            hwvtepConnectionInstance = getConnectionInstance(key);
+            if (hwvtepConnectionInstance != null) {
+                if (hwvtepConnectionInstance.getInstanceIdentifier() != null) {
+                    deviceUpdateHistory.get(hwvtepConnectionInstance.getInstanceIdentifier()).addToHistory(
+                            TransactionType.DELETE, new ClientConnected(client.getConnectionInfo().getRemotePort()));
+                }
 
 
-            // Unregister Entity ownership as soon as possible ,so this instance should
-            // not be used as a candidate in Entity election (given that this instance is
-            // about to disconnect as well), if current owner get disconnected from
-            // HWVTEP device.
-            if (hwvtepConnectionInstance.getHasDeviceOwnership()) {
-                unregisterEntityForOwnership(hwvtepConnectionInstance);
-                txInvoker.invoke(new HwvtepGlobalRemoveCommand(hwvtepConnectionInstance, null, null));
-            } else {
-                unregisterEntityForOwnership(hwvtepConnectionInstance);
-                //Do not delete if client disconnected from follower HwvtepGlobalRemoveCommand
-            }
+                // Unregister Entity ownership as soon as possible ,so this instance should
+                // not be used as a candidate in Entity election (given that this instance is
+                // about to disconnect as well), if current owner get disconnected from
+                // HWVTEP device.
+                if (hwvtepConnectionInstance.getHasDeviceOwnership()) {
+                    unregisterEntityForOwnership(hwvtepConnectionInstance);
+                    txInvoker.invoke(new HwvtepGlobalRemoveCommand(hwvtepConnectionInstance, null, null));
+                } else {
+                    unregisterEntityForOwnership(hwvtepConnectionInstance);
+                    //Do not delete if client disconnected from follower HwvtepGlobalRemoveCommand
+                }
 
-            removeConnectionInstance(key);
+                removeConnectionInstance(key);
 
-            //Controller initiated connection can be terminated from switch side.
-            //So cleanup the instance identifier cache.
-            removeInstanceIdentifier(key);
-            removeConnectionInstance(hwvtepConnectionInstance.getInstanceIdentifier());
-            retryConnection(hwvtepConnectionInstance.getInstanceIdentifier(),
-                    hwvtepConnectionInstance.getHwvtepGlobalAugmentation(),
-                    ConnectionReconciliationTriggers.ON_DISCONNECT);
-        } else {
-            LOG.warn("HWVTEP disconnected event did not find connection instance for {}", key);
+                //Controller initiated connection can be terminated from switch side.
+                //So cleanup the instance identifier cache.
+                removeInstanceIdentifier(key);
+                removeConnectionInstance(hwvtepConnectionInstance.getInstanceIdentifier());
+                retryConnection(hwvtepConnectionInstance.getInstanceIdentifier(),
+                        hwvtepConnectionInstance.getHwvtepGlobalAugmentation(),
+                        ConnectionReconciliationTriggers.ON_DISCONNECT);
+            } else {
+                LOG.warn("HWVTEP disconnected event did not find connection instance for {}", key);
+            }
+            LOG.trace("HwvtepConnectionManager exit disconnected client: {}", client);
+        } catch (Exception e) {
+            LOG.error("Failed to execute disconnected ",e);
         }
-        LOG.trace("HwvtepConnectionManager exit disconnected client: {}", client);
     }
 
     public OvsdbClient connect(final InstanceIdentifier<Node> iid, final HwvtepGlobalAugmentation hwvtepGlobal)
@@ -233,8 +240,8 @@ public class HwvtepConnectionManager implements OvsdbConnectionListener, AutoClo
             removeConnectionInstance(key);
         }
 
-        hwvtepConnectionInstance = new HwvtepConnectionInstance(this, key, externalClient, getInstanceIdentifier(key),
-                txInvoker, db);
+        hwvtepConnectionInstance = new HwvtepConnectionInstance(this, key,
+                externalClient, getInstanceIdentifier(key), txInvoker, db);
         hwvtepConnectionInstance.createTransactInvokers();
         return hwvtepConnectionInstance;
     }
@@ -246,22 +253,10 @@ public class HwvtepConnectionManager implements OvsdbConnectionListener, AutoClo
     }
 
     void putConnectionInstance(final InstanceIdentifier<Node> nodeIid,
-            final HwvtepConnectionInstance connectionInstance) {
+                                       final HwvtepConnectionInstance connectionInstance) {
         nodeIidVsConnectionInstance.put(nodeIid, connectionInstance);
     }
 
-    public HwvtepConnectionInstance getConnectionInstanceFromNodeIid(final InstanceIdentifier<Node> nodeIid) {
-        HwvtepConnectionInstance hwvtepConnectionInstance = nodeIidVsConnectionInstance.get(nodeIid);
-        if (hwvtepConnectionInstance != null) {
-            return hwvtepConnectionInstance;
-        }
-        InstanceIdentifier<Node> globalNodeIid = HwvtepSouthboundUtil.getGlobalNodeIid(nodeIid);
-        if (globalNodeIid != null) {
-            return nodeIidVsConnectionInstance.get(globalNodeIid);
-        }
-        return null;
-    }
-
     public HwvtepConnectionInstance getConnectionInstance(final ConnectionInfo key) {
         if (key == null) {
             return null;
@@ -300,6 +295,18 @@ public class HwvtepConnectionManager implements OvsdbConnectionListener, AutoClo
         }
     }
 
+    public HwvtepConnectionInstance getConnectionInstanceFromNodeIid(final InstanceIdentifier<Node> nodeIid) {
+        HwvtepConnectionInstance hwvtepConnectionInstance = nodeIidVsConnectionInstance.get(nodeIid);
+        if (hwvtepConnectionInstance != null) {
+            return hwvtepConnectionInstance;
+        }
+        InstanceIdentifier<Node> globalNodeIid = HwvtepSouthboundUtil.getGlobalNodeIid(nodeIid);
+        if (globalNodeIid != null) {
+            return nodeIidVsConnectionInstance.get(globalNodeIid);
+        }
+        return null;
+    }
+
     public void stopConfigurationReconciliation(final InstanceIdentifier<Node> nodeIid) {
         final ReconciliationTask task = new HwvtepReconciliationTask(
                 reconciliationManager, HwvtepConnectionManager.this, nodeIid, null, null, db);
@@ -346,11 +353,31 @@ public class HwvtepConnectionManager implements OvsdbConnectionListener, AutoClo
         return getConnectionInstance(connectionInfo).getOvsdbClient();
     }
 
-    private void registerEntityForOwnership(final HwvtepConnectionInstance hwvtepConnectionInstance) {
+    @SuppressWarnings("checkstyle:IllegalCatch")
+    private void registerCallbacks(HwvtepConnectionInstance hwvtepConnectionInstance) {
+        LOG.info("HWVTEP entity {} is owned by this controller registering callbacks",
+                hwvtepConnectionInstance.getConnectionInfo());
+        try {
+            hwvtepOperGlobalListener.runAfterNodeDeleted(
+                hwvtepConnectionInstance.getInstanceIdentifier(), () -> {
+                    cleanupOperationalNode(hwvtepConnectionInstance.getInstanceIdentifier());
+                    hwvtepConnectionInstance.registerCallbacks();
+                    return null;
+                });
+        } catch (Exception e) {
+            LOG.error("Failed to register callbacks for HWVTEP entity {} ",
+                    hwvtepConnectionInstance.getConnectionInfo(), e);
+        }
+    }
+
+
+    private void registerEntityForOwnership(HwvtepConnectionInstance hwvtepConnectionInstance) {
 
         Entity candidateEntity = getEntityFromConnectionInstance(hwvtepConnectionInstance);
         if (entityConnectionMap.get(candidateEntity) != null) {
+            InstanceIdentifier<Node> iid = hwvtepConnectionInstance.getInstanceIdentifier();
             disconnected(entityConnectionMap.get(candidateEntity).getOvsdbClient());
+            hwvtepConnectionInstance.setInstanceIdentifier(iid);
             putConnectionInstance(hwvtepConnectionInstance.getInstanceIdentifier(), hwvtepConnectionInstance);
         }
         entityConnectionMap.put(candidateEntity, hwvtepConnectionInstance);
@@ -361,14 +388,10 @@ public class HwvtepConnectionManager implements OvsdbConnectionListener, AutoClo
                     entityOwnershipService.registerCandidate(candidateEntity);
             hwvtepConnectionInstance.setDeviceOwnershipCandidateRegistration(registration);
             LOG.info("HWVTEP entity {} is registered for ownership.", candidateEntity);
-
-            //If entity already has owner, it won't get notification from EntityOwnershipService
-            //so cache the connection instances.
-            handleOwnershipState(candidateEntity, hwvtepConnectionInstance);
         } catch (CandidateAlreadyRegisteredException e) {
             LOG.warn("OVSDB entity {} was already registered for ownership", candidateEntity, e);
         }
-
+        handleOwnershipState(candidateEntity, hwvtepConnectionInstance);
     }
 
     private void handleOwnershipState(final Entity candidateEntity,
@@ -387,19 +410,12 @@ public class HwvtepConnectionManager implements OvsdbConnectionListener, AutoClo
                                     + "instance, so *this* instance is NOT an OWNER of the device",
                             hwvtepConnectionInstance.getConnectionInfo());
                 } else {
-                    afterTakingOwnership(hwvtepConnectionInstance);
+                    registerCallbacks(hwvtepConnectionInstance);
                 }
             }
         }
     }
 
-    private void afterTakingOwnership(final HwvtepConnectionInstance hwvtepConnectionInstance) {
-        txInvoker.invoke(new HwvtepGlobalRemoveCommand(hwvtepConnectionInstance, null, null));
-        putConnectionInstance(hwvtepConnectionInstance.getMDConnectionInfo(), hwvtepConnectionInstance);
-        hwvtepConnectionInstance.setHasDeviceOwnership(true);
-        hwvtepConnectionInstance.registerCallbacks();
-    }
-
     private static Global getHwvtepGlobalTableEntry(final HwvtepConnectionInstance connectionInstance) {
         final TypedDatabaseSchema dbSchema;
         try {
@@ -486,7 +502,11 @@ public class HwvtepConnectionManager implements OvsdbConnectionListener, AutoClo
     }
 
     private void retryConnection(final InstanceIdentifier<Node> iid, final HwvtepGlobalAugmentation hwvtepNode,
-                                 final ConnectionReconciliationTriggers trigger) {
+                                 ConnectionReconciliationTriggers trigger) {
+        if (hwvtepNode == null) {
+            //switch initiated connection
+            return;
+        }
         final ReconciliationTask task = new ConnectionReconciliationTask(
                 reconciliationManager,
                 this,
@@ -561,8 +581,28 @@ public class HwvtepConnectionManager implements OvsdbConnectionListener, AutoClo
             // might went down abruptly and didn't get a chance to clean up the operational data store.
             if (!ownershipChange.getState().hasOwner()) {
                 LOG.debug("{} has no owner, cleaning up the operational data store", ownershipChange.getEntity());
-                // If first cleanEntityOperationalData() was called, this call will be no-op.
-                cleanEntityOperationalData(ownershipChange.getEntity());
+                // Below code might look weird but it's required. We want to give first opportunity to the
+                // previous owner of the device to clean up the operational data store if there is no owner now.
+                // That way we will avoid lot of nasty md-sal exceptions because of concurrent delete.
+                InstanceIdentifier<Node> nodeIid =
+                        (InstanceIdentifier<Node>) ownershipChange.getEntity().getIdentifier();
+                hwvtepOperGlobalListener.scheduleOldConnectionNodeDelete(nodeIid);
+                /*
+                Assuming node1 was the owner earlier.
+                If the owner relinquished he would have cleaned it already in which case the above would be a no op
+                If the owner crashed then the above would clean the node after the scheduled delay
+                The live nodes (two and three) will try to cleanup but that is ok one of them ends up cleaning.
+                But if the southbound connects again that connection can itself trigger the pending cleanup and
+                the above op would become noop again.
+
+                In summary
+                In The following cases it would be a noop
+                1) The southbound connects again within the scheduled cleanup delay.
+                2) The owner node1 which is not crashed cleaned the node properly.
+
+                In the following case both node2 and node3 will try to clean it (one of them will succeed ).
+                 1) node1 which was the owner crashed
+                 */
             }
             return;
         }
@@ -583,7 +623,7 @@ public class HwvtepConnectionManager implements OvsdbConnectionListener, AutoClo
 
             //*this* instance of southbound plugin is owner of the device,
             //so register for monitor callbacks
-            afterTakingOwnership(hwvtepConnectionInstance);
+            registerCallbacks(hwvtepConnectionInstance);
 
         } else {
             //You were owner of the device, but now you are not. With the current ownership
@@ -598,12 +638,6 @@ public class HwvtepConnectionManager implements OvsdbConnectionListener, AutoClo
         }
     }
 
-    private void cleanEntityOperationalData(final Entity entity) {
-        @SuppressWarnings("unchecked")
-        final InstanceIdentifier<Node> nodeIid = (InstanceIdentifier<Node>) entity.getIdentifier();
-        txInvoker.invoke(new HwvtepGlobalRemoveCommand(nodeIid));
-    }
-
     private HwvtepConnectionInstance getConnectionInstanceFromEntity(final Entity entity) {
         return entityConnectionMap.get(entity);
     }
@@ -653,4 +687,8 @@ public class HwvtepConnectionManager implements OvsdbConnectionListener, AutoClo
     public Map<InstanceIdentifier<Node>, HwvtepConnectionInstance> getAllConnectedInstances() {
         return Collections.unmodifiableMap(nodeIidVsConnectionInstance);
     }
+
+    public void cleanupOperationalNode(InstanceIdentifier<Node> nodeIid) {
+        txInvoker.invoke(new HwvtepGlobalRemoveCommand(nodeIid));
+    }
 }
index 27f236570cf8d9dd5f269a5c582fe3038cc867e0..b3b44e9fe7aefa241f097a578f64864ae11f02a6 100644 (file)
@@ -7,13 +7,15 @@
  */
 package org.opendaylight.ovsdb.hwvtepsouthbound;
 
+import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Timer;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
 import org.opendaylight.controller.md.sal.binding.api.ClusteredDataTreeChangeListener;
 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
 import org.opendaylight.controller.md.sal.binding.api.DataObjectModification;
@@ -21,6 +23,8 @@ import org.opendaylight.controller.md.sal.binding.api.DataObjectModification.Mod
 import org.opendaylight.controller.md.sal.binding.api.DataTreeIdentifier;
 import org.opendaylight.controller.md.sal.binding.api.DataTreeModification;
 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepGlobalAugmentation;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.ConnectionInfo;
 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NetworkTopology;
 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.Topology;
 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.TopologyKey;
@@ -31,15 +35,19 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class HwvtepOperGlobalListener implements ClusteredDataTreeChangeListener<Node>, AutoCloseable {
+
     private static final Logger LOG = LoggerFactory.getLogger(HwvtepOperGlobalListener.class);
+    private static final Map<InstanceIdentifier<Node>, ConnectionInfo> NODE_CONNECTION_INFO = new ConcurrentHashMap<>();
 
-    private final Timer timer = new Timer();
     private ListenerRegistration<HwvtepOperGlobalListener> registration;
     private final HwvtepConnectionManager hcm;
     private final DataBroker db;
+    private static final Map<InstanceIdentifier<Node>, List<Callable<Void>>> NODE_DELET_WAITING_JOBS
+            = new ConcurrentHashMap<>();
     private static final Map<InstanceIdentifier<Node>, Node> CONNECTED_NODES = new ConcurrentHashMap<>();
 
-    HwvtepOperGlobalListener(final DataBroker db, final HwvtepConnectionManager hcm) {
+
+    HwvtepOperGlobalListener(final DataBroker db, HwvtepConnectionManager hcm) {
         LOG.info("Registering HwvtepOperGlobalListener");
         this.db = db;
         this.hcm = hcm;
@@ -48,7 +56,7 @@ public class HwvtepOperGlobalListener implements ClusteredDataTreeChangeListener
 
     private void registerListener() {
         final DataTreeIdentifier<Node> treeId =
-                        new DataTreeIdentifier<>(LogicalDatastoreType.OPERATIONAL, getWildcardPath());
+                        new DataTreeIdentifier<Node>(LogicalDatastoreType.OPERATIONAL, getWildcardPath());
 
         registration = db.registerDataTreeChangeListener(treeId, HwvtepOperGlobalListener.this);
     }
@@ -61,33 +69,133 @@ public class HwvtepOperGlobalListener implements ClusteredDataTreeChangeListener
     }
 
     @Override
+    @SuppressWarnings("checkstyle:IllegalCatch")
     public void onDataTreeChanged(final Collection<DataTreeModification<Node>> changes) {
-        changes.forEach(change -> {
+        LOG.trace("onDataTreeChanged: ");
+        try {
+            connect(changes);
+            updated(changes);
+            disconnect(changes);
+        } catch (Exception e) {
+            LOG.error("Failed to handle dcn event ", e);
+        }
+    }
+
+    public void runAfterNodeDeleted(InstanceIdentifier<Node> iid, Callable<Void> job) throws Exception {
+        synchronized (HwvtepOperGlobalListener.class) {
+            if (NODE_DELET_WAITING_JOBS.containsKey(iid)) {
+                LOG.error("Node present in the cache {} adding to delete queue", iid);
+                NODE_DELET_WAITING_JOBS.get(iid).add(job);
+                //Also delete the node so that reconciliation kicks in
+                deleteTheNodeOfOldConnection(iid, getNodeConnectionInfo(iid));
+                HwvtepSouthboundUtil.getScheduledExecutorService().schedule(() -> {
+                    runPendingJobs(iid);
+                }, HwvtepSouthboundConstants.HWVTEP_REGISTER_CALLBACKS_WAIT_TIMEOUT, TimeUnit.SECONDS);
+            } else {
+                LOG.info("Node not present in the cache {} running the job now", iid);
+                job.call();
+            }
+        }
+    }
+
+    @SuppressWarnings("checkstyle:IllegalCatch")
+    private synchronized void runPendingJobs(InstanceIdentifier<Node> iid) {
+        List<Callable<Void>> jobs = NODE_DELET_WAITING_JOBS.remove(iid);
+        if (jobs != null && !jobs.isEmpty()) {
+            jobs.forEach((job) -> {
+                try {
+                    LOG.error("Node disconnected job found {} running it now ", iid);
+                    job.call();
+                } catch (Exception e) {
+                    LOG.error("Failed to run callable ", e);
+                }
+            });
+            jobs.clear();
+        }
+    }
+
+    private void connect(Collection<DataTreeModification<Node>> changes) {
+        changes.forEach((change) -> {
             InstanceIdentifier<Node> key = change.getRootPath().getRootIdentifier();
             DataObjectModification<Node> mod = change.getRootNode();
-            InstanceIdentifier<Node> nodeIid = change.getRootPath().getRootIdentifier();
             Node node = getCreated(mod);
+            if (node == null) {
+                return;
+            }
+            CONNECTED_NODES.put(key, node);
+            HwvtepGlobalAugmentation globalAugmentation = node.augmentation(HwvtepGlobalAugmentation.class);
+            if (globalAugmentation != null) {
+                ConnectionInfo connectionInfo = globalAugmentation.getConnectionInfo();
+                if (connectionInfo != null) {
+                    NODE_CONNECTION_INFO.put(key, connectionInfo);
+                }
+            }
+            if (node != null) {
+                synchronized (HwvtepOperGlobalListener.class) {
+                    NODE_DELET_WAITING_JOBS.putIfAbsent(key, new ArrayList<>());
+                }
+            }
+        });
+    }
+
+    private void updated(Collection<DataTreeModification<Node>> changes) {
+        changes.forEach((change) -> {
+            InstanceIdentifier<Node> key = change.getRootPath().getRootIdentifier();
+            DataObjectModification<Node> mod = change.getRootNode();
+            Node node = getUpdated(mod);
             if (node != null) {
                 CONNECTED_NODES.put(key, node);
             }
-            node = getRemoved(mod);
+        });
+    }
+
+    public static Node getNode(final InstanceIdentifier<Node> key) {
+        return CONNECTED_NODES.get(key);
+    }
+
+    private void disconnect(Collection<DataTreeModification<Node>> changes) {
+        changes.forEach((change) -> {
+            InstanceIdentifier<Node> key = change.getRootPath().getRootIdentifier();
+            DataObjectModification<Node> mod = change.getRootNode();
+            Node node = getRemoved(mod);
             if (node != null) {
                 CONNECTED_NODES.remove(key);
-                HwvtepConnectionInstance connectionInstance = hcm.getConnectionInstanceFromNodeIid(nodeIid);
-                if (connectionInstance != null && Objects.equals(connectionInstance.getConnectionInfo().getRemotePort(),
-                            HwvtepSouthboundUtil.getRemotePort(node))) {
-                    //Oops some one deleted the node held by me This should never happen
-                    try {
-                        connectionInstance.refreshOperNode();
-                    } catch (ExecutionException | InterruptedException e) {
-                        LOG.error("Failed to refresh operational nodes ", e);
-                    }
+                NODE_CONNECTION_INFO.remove(key);
+                synchronized (HwvtepOperGlobalListener.class) {
+                    runPendingJobs(key);
                 }
-
             }
         });
     }
 
+    private static String getNodeId(InstanceIdentifier<Node> iid) {
+        return iid.firstKeyOf(Node.class).getNodeId().getValue();
+    }
+
+    public void scheduleOldConnectionNodeDelete(InstanceIdentifier<Node> iid) {
+        ConnectionInfo oldConnectionInfo = getNodeConnectionInfo(iid);
+        HwvtepSouthboundUtil.getScheduledExecutorService().schedule(() -> {
+            deleteTheNodeOfOldConnection(iid, oldConnectionInfo);
+        }, HwvtepSouthboundConstants.STALE_HWVTEP_CLEANUP_DELAY_SECS, TimeUnit.SECONDS);
+    }
+
+    private void deleteTheNodeOfOldConnection(InstanceIdentifier<Node> iid,
+                                                    ConnectionInfo oldConnectionInfo) {
+        if (oldConnectionInfo == null) {
+            return;
+        }
+        ConnectionInfo latestConnectionInfo = getNodeConnectionInfo(iid);
+        if (Objects.equals(latestConnectionInfo, oldConnectionInfo)) {
+            //Still old connection node is not deleted
+            LOG.debug("Delete Node {} from oper ", getNodeId(iid));
+            hcm.cleanupOperationalNode(iid);
+        }
+    }
+
+    private static ConnectionInfo getNodeConnectionInfo(InstanceIdentifier<Node> iid) {
+        return NODE_CONNECTION_INFO.get(iid);
+    }
+
     private static Node getCreated(final DataObjectModification<Node> mod) {
         if (mod.getModificationType() == ModificationType.WRITE && mod.getDataBefore() == null) {
             return mod.getDataAfter();
@@ -102,17 +210,28 @@ public class HwvtepOperGlobalListener implements ClusteredDataTreeChangeListener
         return null;
     }
 
-    public Map<InstanceIdentifier<Node>, Node> getConnectedNodes() {
-        return Collections.unmodifiableMap(CONNECTED_NODES);
-    }
-
     private static InstanceIdentifier<Node> getWildcardPath() {
         return InstanceIdentifier.create(NetworkTopology.class)
                 .child(Topology.class, new TopologyKey(HwvtepSouthboundConstants.HWVTEP_TOPOLOGY_ID))
                 .child(Node.class);
     }
 
-    public static Node getNode(final InstanceIdentifier<Node> key) {
-        return CONNECTED_NODES.get(key);
+    private Node getUpdated(DataObjectModification<Node> mod) {
+        Node node = null;
+        switch (mod.getModificationType()) {
+            case SUBTREE_MODIFIED:
+                node = mod.getDataAfter();
+                break;
+            case WRITE:
+                if (mod.getDataBefore() !=  null) {
+                    node = mod.getDataAfter();
+                }
+                break;
+            default:
+                break;
+        }
+        return node;
     }
+
+
 }
index bbef5b98956f3cf9526073c9a75b6a9ea2877463..c5ea525e00a5ee3683bf4725470669c902050e6a 100644 (file)
@@ -42,10 +42,15 @@ public interface HwvtepSouthboundConstants {
     int WAITING_QUEUE_CAPACITY = Integer.getInteger("hwvtep.wait.queue.capacity", 1000);
     long WAITING_JOB_EXPIRY_TIME_MILLIS = Integer.getInteger(
             "hwvtep.wait.job.expiry.time.millis", 90000);
+    Integer STALE_HWVTEP_CLEANUP_DELAY_SECS
+            = Integer.getInteger("stale.hwvtep.node.cleanup.delay.secs", 240);
+    Integer HWVTEP_REGISTER_CALLBACKS_WAIT_TIMEOUT
+            = Integer.getInteger("hwvtep.max.oper.wait.time.secs", 10);
     long IN_TRANSIT_STATE_EXPIRY_TIME_MILLIS = Integer.getInteger(
             "hwvtep.intransit.job.expiry.time.millis", 10000);
     long IN_TRANSIT_STATE_CHECK_PERIOD_MILLIS = Integer.getInteger(
             "hwvtep.intransit.job.check.period.millis", 30000);
     long CONFIG_NODE_UPDATE_MAX_DELAY_MS = Integer.getInteger(
             "config.node.update.max.delay.ms", 10000);
+
 }
index e7023b43d02be7d3a5bea0de14add17788ff3b7d..2f931527ad5d70d0d848941046f50938d9f8a7ce 100644 (file)
@@ -10,9 +10,14 @@ package org.opendaylight.ovsdb.hwvtepsouthbound;
 
 import com.google.common.base.Optional;
 import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
 import java.util.Collection;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+
 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
 import org.opendaylight.controller.md.sal.binding.api.ReadTransaction;
 import org.opendaylight.controller.md.sal.binding.api.ReadWriteTransaction;
@@ -43,6 +48,10 @@ public final class HwvtepSouthboundUtil {
 
     private static InstanceIdentifierCodec instanceIdentifierCodec;
 
+    private static ScheduledExecutorService scheduledExecutorService = Executors
+            .newSingleThreadScheduledExecutor(new ThreadFactoryBuilder()
+                    .setNameFormat("hwvteputil-executor-service-%d").build());
+
     private HwvtepSouthboundUtil() {
         // Prevent instantiating a utility class
     }
@@ -221,4 +230,8 @@ public final class HwvtepSouthboundUtil {
         }
         return 0;
     }
+
+    public static ScheduledExecutorService getScheduledExecutorService() {
+        return scheduledExecutorService;
+    }
 }