Migrate to MD-SAL APIs
[bgpcep.git] / bgp / rib-impl / src / main / java / org / opendaylight / protocol / bgp / rib / impl / AdjRibInWriter.java
index b630e7b05722c369551c56fca6bf9654378f9c43..3b020082c7655a2f6302e69852ef4bc2a1b6d711 100644 (file)
  */
 package org.opendaylight.protocol.bgp.rib.impl;
 
-import com.google.common.base.Preconditions;
+import static java.util.Objects.requireNonNull;
+import static org.opendaylight.protocol.bgp.rib.spi.RIBNodeIdentifiers.ADJRIBIN_NID;
+import static org.opendaylight.protocol.bgp.rib.spi.RIBNodeIdentifiers.ADJRIBOUT_NID;
+import static org.opendaylight.protocol.bgp.rib.spi.RIBNodeIdentifiers.ATTRIBUTES_NID;
+import static org.opendaylight.protocol.bgp.rib.spi.RIBNodeIdentifiers.EFFRIBIN_NID;
+import static org.opendaylight.protocol.bgp.rib.spi.RIBNodeIdentifiers.TABLES_NID;
+import static org.opendaylight.protocol.bgp.rib.spi.RIBNodeIdentifiers.UPTODATE_NID;
+
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableMap.Builder;
+import com.google.common.util.concurrent.FluentFuture;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.MoreExecutors;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Optional;
 import java.util.Set;
-import javax.annotation.concurrent.NotThreadSafe;
-import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
-import org.opendaylight.controller.md.sal.dom.api.DOMDataWriteTransaction;
-import org.opendaylight.controller.md.sal.dom.api.DOMTransactionChain;
-import org.opendaylight.protocol.bgp.rib.spi.RIBExtensionConsumerContext;
-import org.opendaylight.protocol.bgp.rib.spi.RIBSupport;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.message.rev130919.PathAttributes;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.multiprotocol.rev130919.update.path.attributes.MpReachNlri;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.multiprotocol.rev130919.update.path.attributes.MpUnreachNlri;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.rib.rev130925.bgp.rib.rib.peer.AdjRibIn;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.rib.rev130925.rib.Tables;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.rib.rev130925.rib.TablesKey;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.rib.rev130925.rib.tables.Attributes;
-import org.opendaylight.yangtools.yang.binding.util.BindingReflections;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.stream.Collectors;
+import org.checkerframework.checker.lock.qual.GuardedBy;
+import org.eclipse.jdt.annotation.NonNull;
+import org.eclipse.jdt.annotation.Nullable;
+import org.opendaylight.mdsal.common.api.CommitInfo;
+import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
+import org.opendaylight.mdsal.dom.api.DOMDataTreeReadTransaction;
+import org.opendaylight.mdsal.dom.api.DOMDataTreeWriteTransaction;
+import org.opendaylight.protocol.bgp.rib.impl.ApplicationPeer.RegisterAppPeerListener;
+import org.opendaylight.protocol.bgp.rib.impl.spi.PeerTransactionChain;
+import org.opendaylight.protocol.bgp.rib.impl.spi.RIBSupportContext;
+import org.opendaylight.protocol.bgp.rib.impl.spi.RIBSupportContextRegistry;
+import org.opendaylight.protocol.bgp.rib.spi.IdentifierUtils;
+import org.opendaylight.protocol.bgp.rib.spi.PeerRoleUtil;
+import org.opendaylight.protocol.bgp.rib.spi.RIBNormalizedNodes;
+import org.opendaylight.protocol.bgp.rib.spi.RIBQNames;
+import org.opendaylight.protocol.bgp.rib.spi.RibSupportUtils;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.multiprotocol.rev180329.SendReceive;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.multiprotocol.rev180329.update.attributes.MpReachNlri;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.multiprotocol.rev180329.update.attributes.MpUnreachNlri;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.rib.rev180329.PeerId;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.rib.rev180329.PeerRole;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.rib.rev180329.bgp.rib.rib.Peer;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.rib.rev180329.bgp.rib.rib.peer.SupportedTables;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.rib.rev180329.rib.TablesKey;
 import org.opendaylight.yangtools.yang.common.QName;
 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.InstanceIdentifierBuilder;
 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifier;
 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifierWithPredicates;
 import org.opendaylight.yangtools.yang.data.api.schema.ContainerNode;
-import org.opendaylight.yangtools.yang.data.api.schema.LeafNode;
+import org.opendaylight.yangtools.yang.data.api.schema.MapEntryNode;
+import org.opendaylight.yangtools.yang.data.api.schema.MapNode;
+import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
 import org.opendaylight.yangtools.yang.data.impl.schema.Builders;
 import org.opendaylight.yangtools.yang.data.impl.schema.ImmutableNodes;
+import org.opendaylight.yangtools.yang.data.impl.schema.builder.api.DataContainerNodeBuilder;
+import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableMapNodeBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * Writer of Adjacency-RIB-In for a single peer. An instance of this object
- * is attached to each {@link BGPPeer} and {@link ApplicationPeer}.
+ * Writer of Adjacency-RIB-In for a single peer. An instance of this object is attached to each {@link BGPPeer} and
+ * {@link ApplicationPeer}. This class is NOT thread-safe.
  */
-@NotThreadSafe
 final class AdjRibInWriter {
+
     private static final Logger LOG = LoggerFactory.getLogger(AdjRibInWriter.class);
 
+    private static final QName PEER_ROLE_QNAME = QName.create(Peer.QNAME, "peer-role").intern();
+    private static final NodeIdentifier PEER_ID = NodeIdentifier.create(RIBQNames.PEER_ID_QNAME);
+    private static final NodeIdentifier PEER_ROLE = NodeIdentifier.create(PEER_ROLE_QNAME);
+    private static final NodeIdentifier PEER_TABLES = NodeIdentifier.create(SupportedTables.QNAME);
+    private static final QName SEND_RECEIVE = QName.create(SupportedTables.QNAME, "send-receive").intern();
+
     // FIXME: is there a utility method to construct this?
-    private static final ContainerNode EMPTY_ADJRIBIN = Builders.containerBuilder().withNodeIdentifier(new NodeIdentifier(AdjRibIn.QNAME)).addChild(ImmutableNodes.mapNodeBuilder(Tables.QNAME).build()).build();
-    private static final LeafNode<Boolean> ATTRIBUTES_UPTODATE_FALSE = ImmutableNodes.leafNode(QName.create(Attributes.QNAME, "uptodate"), Boolean.FALSE);
-    private static final LeafNode<Boolean> ATTRIBUTES_UPTODATE_TRUE = ImmutableNodes.leafNode(ATTRIBUTES_UPTODATE_FALSE.getNodeType(), Boolean.TRUE);
-    private static final QName AFI_QNAME = QName.create(Tables.QNAME, "afi");
-    private static final QName SAFI_QNAME = QName.create(Tables.QNAME, "safi");
+    private static final MapNode EMPTY_TABLES = ImmutableNodes.mapNodeBuilder(TABLES_NID).build();
+    private static final ContainerNode EMPTY_ADJRIBIN = Builders.containerBuilder()
+            .withNodeIdentifier(ADJRIBIN_NID).addChild(EMPTY_TABLES).build();
+    private static final ContainerNode EMPTY_EFFRIBIN = Builders.containerBuilder()
+            .withNodeIdentifier(EFFRIBIN_NID).addChild(EMPTY_TABLES).build();
+    private static final ContainerNode EMPTY_ADJRIBOUT = Builders.containerBuilder()
+            .withNodeIdentifier(ADJRIBOUT_NID).addChild(EMPTY_TABLES).build();
 
     private final Map<TablesKey, TableContext> tables;
-    private final YangInstanceIdentifier adjRibInRoot;
-    private final DOMTransactionChain chain;
+    private final YangInstanceIdentifier ribPath;
+    private final PeerTransactionChain chain;
+    private final PeerRole role;
+    @GuardedBy("this")
+    private final Map<TablesKey, Collection<NodeIdentifierWithPredicates>> staleRoutesRegistry = new HashMap<>();
+    @GuardedBy("this")
+    private FluentFuture<? extends CommitInfo> submitted;
 
-    /*
-     * FIXME: transaction chain has to be instantiated in caller, so it can terminate us when it fails.
-     */
-    private AdjRibInWriter(final DOMTransactionChain chain, final YangInstanceIdentifier adjRibInRoot, final Map<TablesKey, TableContext> tables) {
-        this.chain = Preconditions.checkNotNull(chain);
-        this.adjRibInRoot = Preconditions.checkNotNull(adjRibInRoot);
-        this.tables = Preconditions.checkNotNull(tables);
+    private AdjRibInWriter(final YangInstanceIdentifier ribPath, final PeerTransactionChain chain, final PeerRole role,
+            final Map<TablesKey, TableContext> tables) {
+        this.ribPath = requireNonNull(ribPath);
+        this.chain = requireNonNull(chain);
+        this.tables = requireNonNull(tables);
+        this.role = requireNonNull(role);
     }
 
-    static AdjRibInWriter create(final DOMTransactionChain chain, final YangInstanceIdentifier peer) {
-        // Not used often, no need to optimize it via builder
-        final YangInstanceIdentifier adjRibInRoot = peer.node(EMPTY_ADJRIBIN.getIdentifier());
-
-        // Create top-level AdjRibIn with an empty table list
-        final DOMDataWriteTransaction tx = chain.newWriteOnlyTransaction();
-        tx.put(LogicalDatastoreType.OPERATIONAL, adjRibInRoot, EMPTY_ADJRIBIN);
-        tx.submit();
-
-        return new AdjRibInWriter(chain, adjRibInRoot, Collections.<TablesKey, TableContext>emptyMap());
+    /**
+     * Create a new writer using a transaction chain.
+     *
+     * @param role                peer's role
+     * @param chain               transaction chain  @return A fresh writer instance
+     */
+    static AdjRibInWriter create(final @NonNull YangInstanceIdentifier ribId, final @NonNull PeerRole role,
+            final @NonNull PeerTransactionChain chain) {
+        return new AdjRibInWriter(ribId, chain, role, Collections.emptyMap());
     }
 
     /**
@@ -86,105 +129,308 @@ final class AdjRibInWriter {
      * Empty tables are created for new entries and old tables are deleted. Once this
      * method returns, the old instance must not be reasonably used.
      *
-     * @param tableTypes New tables, must not be null
+     * @param newPeerId         new peer BGP identifier
+     * @param peerPath          path of the peer in the datastore
+     * @param registry          RIB extension registry
+     * @param tableTypes        New tables, must not be null
+     * @param addPathTablesType supported add path tables
      * @return New writer
      */
-    AdjRibInWriter changeTableTypes(final RIBExtensionConsumerContext registry, final Set<TablesKey> tableTypes) {
-        if (tableTypes.equals(tables.keySet())) {
-            return this;
-        }
+    AdjRibInWriter transform(final PeerId newPeerId, final YangInstanceIdentifier peerPath,
+            final RIBSupportContextRegistry registry,
+            final Set<TablesKey> tableTypes, final Map<TablesKey, SendReceive> addPathTablesType) {
+        return transform(newPeerId, peerPath, registry, tableTypes, addPathTablesType, null);
+    }
 
-        final DOMDataWriteTransaction tx = chain.newWriteOnlyTransaction();
+    AdjRibInWriter transform(final PeerId newPeerId, final YangInstanceIdentifier peerPath,
+            final RIBSupportContextRegistry registry, final Set<TablesKey> tableTypes,
+            final Map<TablesKey, SendReceive> addPathTablesType,
+            final @Nullable RegisterAppPeerListener registerAppPeerListener) {
+        final DOMDataTreeWriteTransaction tx = this.chain.getDomChain().newWriteOnlyTransaction();
 
-        // Wipe tables which are not present in the new types
-        for (Entry<TablesKey, TableContext> e : tables.entrySet()) {
-            if (!tableTypes.contains(e.getKey())) {
-                e.getValue().removeTable(tx);
-            }
-        }
+        createEmptyPeerStructure(newPeerId, peerPath, tx);
+        final ImmutableMap<TablesKey, TableContext> tb = createNewTableInstances(peerPath, registry, tableTypes,
+                addPathTablesType, tx);
 
-        final Builder<TablesKey, TableContext> tb = ImmutableMap.builder();
-        for (TablesKey k : tableTypes) {
-            TableContext ctx = tables.get(k);
-            if (ctx == null) {
-                final RIBSupport rs = registry.getRIBSupport(k);
-                if (rs == null) {
-                    LOG.warn("No support for table type {}, skipping it", k);
-                    continue;
+        tx.commit().addCallback(new FutureCallback<CommitInfo>() {
+            @Override
+            public void onSuccess(final CommitInfo result) {
+                if (registerAppPeerListener != null) {
+                    LOG.trace("Application Peer Listener registered");
+                    registerAppPeerListener.register();
                 }
+            }
 
-                // We will use table keys very often, make sure they are optimized
-                final InstanceIdentifierBuilder idb = YangInstanceIdentifier.builder(adjRibInRoot);
+            @Override
+            public void onFailure(final Throwable throwable) {
+                if (registerAppPeerListener != null) {
+                    LOG.error("Failed to create Empty Structure, Application Peer Listener won't be registered",
+                            throwable);
+                } else {
+                    LOG.error("Failed to create Empty Structure", throwable);
+                }
+            }
+        }, MoreExecutors.directExecutor());
+        return new AdjRibInWriter(this.ribPath, this.chain, this.role, tb);
+    }
 
-                // FIXME: use codec to translate the key
-                final Map<QName, Object> keyValues = ImmutableMap.<QName, Object>of(AFI_QNAME, BindingReflections.getQName(k.getAfi()), SAFI_QNAME, BindingReflections.getQName(k.getSafi()));
-                final NodeIdentifierWithPredicates key = new NodeIdentifierWithPredicates(Tables.QNAME, keyValues);
-                idb.nodeWithKey(key.getNodeType(), keyValues);
+    /**
+     * Create new table instances, potentially creating their empty entries.
+     */
+    private static ImmutableMap<TablesKey, TableContext> createNewTableInstances(
+            final YangInstanceIdentifier newPeerPath, final RIBSupportContextRegistry registry,
+            final Set<TablesKey> tableTypes, final Map<TablesKey, SendReceive> addPathTablesType,
+            final DOMDataTreeWriteTransaction tx) {
 
-                ctx = new TableContext(rs, idb.build());
-                ctx.clearTable(tx);
-            } else {
-                tx.merge(LogicalDatastoreType.OPERATIONAL, ctx.getTableId().node(Attributes.QNAME).node(ATTRIBUTES_UPTODATE_FALSE.getNodeType()), ATTRIBUTES_UPTODATE_FALSE);
+        final Builder<TablesKey, TableContext> tb = ImmutableMap.builder();
+        for (final TablesKey tableKey : tableTypes) {
+            final RIBSupportContext rs = registry.getRIBSupportContext(tableKey);
+            // TODO: Use returned value once Instance Identifier builder allows for it.
+            final NodeIdentifierWithPredicates instanceIdentifierKey = RibSupportUtils.toYangTablesKey(tableKey);
+            if (rs == null) {
+                LOG.warn("No support for table type {}, skipping it", tableKey);
+                continue;
             }
-
-            tb.put(k, ctx);
+            installAdjRibsOutTables(newPeerPath, rs, instanceIdentifierKey, tableKey,
+                    addPathTablesType.get(tableKey), tx);
+            installAdjRibInTables(newPeerPath, tableKey, rs, instanceIdentifierKey, tx, tb);
         }
+        return tb.build();
+    }
 
-        tx.submit();
+    private static void installAdjRibInTables(final YangInstanceIdentifier newPeerPath, final TablesKey tableKey,
+            final RIBSupportContext rs, final NodeIdentifierWithPredicates instanceIdentifierKey,
+            final DOMDataTreeWriteTransaction tx, final Builder<TablesKey, TableContext> tb) {
+        // We will use table keys very often, make sure they are optimized
+        final InstanceIdentifierBuilder idb = YangInstanceIdentifier.builder(newPeerPath
+                .node(EMPTY_ADJRIBIN.getIdentifier()).node(TABLES_NID));
+        idb.nodeWithKey(instanceIdentifierKey.getNodeType(), instanceIdentifierKey.getKeyValues());
 
-        return new AdjRibInWriter(chain, adjRibInRoot, tb.build());
-    }
+        final TableContext ctx = new TableContext(rs, idb.build());
+        ctx.createEmptyTableStructure(tx);
 
-    /**
-     * Clean all routes in specified tables
-     *
-     * @param tableTypes Tables to clean.
-     */
-    void cleanTables(final Collection<TablesKey> tableTypes) {
-        final DOMDataWriteTransaction tx = chain.newWriteOnlyTransaction();
+        tx.merge(LogicalDatastoreType.OPERATIONAL, ctx.getTableId().node(ATTRIBUTES_NID).node(UPTODATE_NID),
+            RIBNormalizedNodes.ATTRIBUTES_UPTODATE_FALSE);
+        LOG.debug("Created table instance {}", ctx.getTableId());
+        tb.put(tableKey, ctx);
+    }
 
-        for (TablesKey k : tableTypes) {
-            LOG.debug("Clearing table {}", k);
-            tables.get(k).clearTable(tx);
+    private static void installAdjRibsOutTables(final YangInstanceIdentifier newPeerPath, final RIBSupportContext rs,
+            final NodeIdentifierWithPredicates instanceIdentifierKey, final TablesKey tableKey,
+            final SendReceive sendReceive, final DOMDataTreeWriteTransaction tx) {
+        final NodeIdentifierWithPredicates supTablesKey = RibSupportUtils.toYangKey(SupportedTables.QNAME, tableKey);
+        final DataContainerNodeBuilder<NodeIdentifierWithPredicates, MapEntryNode> tt =
+                Builders.mapEntryBuilder().withNodeIdentifier(supTablesKey);
+        for (final Entry<QName, Object> e : supTablesKey.getKeyValues().entrySet()) {
+            tt.withChild(ImmutableNodes.leafNode(e.getKey(), e.getValue()));
+        }
+        if (sendReceive != null) {
+            tt.withChild(ImmutableNodes.leafNode(SEND_RECEIVE, sendReceive.toString().toLowerCase(Locale.ENGLISH)));
         }
+        tx.put(LogicalDatastoreType.OPERATIONAL, newPeerPath.node(PEER_TABLES).node(supTablesKey), tt.build());
+        rs.createEmptyTableStructure(tx, newPeerPath.node(EMPTY_ADJRIBOUT.getIdentifier())
+                .node(TABLES_NID).node(instanceIdentifierKey));
+    }
+
+    private void createEmptyPeerStructure(final PeerId newPeerId,
+            final YangInstanceIdentifier peerPath, final DOMDataTreeWriteTransaction tx) {
+        final NodeIdentifierWithPredicates peerKey = IdentifierUtils.domPeerId(newPeerId);
 
-        tx.submit();
+        tx.put(LogicalDatastoreType.OPERATIONAL, peerPath, peerSkeleton(peerKey, newPeerId.getValue()));
+        LOG.debug("New peer {} structure installed.", peerPath);
     }
 
-    void markTablesUptodate(final Collection<TablesKey> tableTypes) {
-        final DOMDataWriteTransaction tx = chain.newWriteOnlyTransaction();
+    @VisibleForTesting
+    MapEntryNode peerSkeleton(final NodeIdentifierWithPredicates peerKey, final String peerId) {
+        final DataContainerNodeBuilder<NodeIdentifierWithPredicates, MapEntryNode> pb = Builders.mapEntryBuilder();
+        pb.withNodeIdentifier(peerKey);
+        pb.withChild(ImmutableNodes.leafNode(PEER_ID, peerId));
+        pb.withChild(ImmutableNodes.leafNode(PEER_ROLE, PeerRoleUtil.roleForString(this.role)));
+        pb.withChild(ImmutableMapNodeBuilder.create().withNodeIdentifier(PEER_TABLES).build());
+        pb.withChild(EMPTY_ADJRIBIN);
+        pb.withChild(EMPTY_EFFRIBIN);
+        pb.withChild(EMPTY_ADJRIBOUT);
+        return pb.build();
+    }
 
-        for (TablesKey k : tableTypes) {
-            final TableContext ctx = tables.get(k);
-            tx.merge(LogicalDatastoreType.OPERATIONAL, ctx.getTableId().node(Attributes.QNAME).node(ATTRIBUTES_UPTODATE_TRUE.getNodeType()), ATTRIBUTES_UPTODATE_TRUE);
-        }
+    void markTableUptodate(final TablesKey tableTypes) {
+        final DOMDataTreeWriteTransaction tx = this.chain.getDomChain().newWriteOnlyTransaction();
+        final TableContext ctx = this.tables.get(tableTypes);
+        tx.merge(LogicalDatastoreType.OPERATIONAL, ctx.getTableId().node(ATTRIBUTES_NID).node(UPTODATE_NID),
+            RIBNormalizedNodes.ATTRIBUTES_UPTODATE_TRUE);
+        tx.commit().addCallback(new FutureCallback<CommitInfo>() {
+            @Override
+            public void onSuccess(final CommitInfo result) {
+                LOG.trace("Write Attributes uptodate, succeed");
+            }
 
-        tx.submit();
+            @Override
+            public void onFailure(final Throwable throwable) {
+                LOG.error("Write Attributes uptodate failed", throwable);
+            }
+        }, MoreExecutors.directExecutor());
     }
 
-    void updateRoutes(final MpReachNlri nlri, final PathAttributes attributes) {
+    void updateRoutes(final MpReachNlri nlri, final org.opendaylight.yang.gen.v1.urn.opendaylight.params
+            .xml.ns.yang.bgp.message.rev180329.path.attributes.Attributes attributes) {
         final TablesKey key = new TablesKey(nlri.getAfi(), nlri.getSafi());
-        final TableContext ctx = tables.get(key);
+        final TableContext ctx = this.tables.get(key);
         if (ctx == null) {
             LOG.debug("No table for {}, not accepting NLRI {}", key, nlri);
             return;
         }
 
-        final DOMDataWriteTransaction tx = chain.newWriteOnlyTransaction();
-        ctx.writeRoutes(null, tx, nlri, attributes);
-        tx.submit();
+        final DOMDataTreeWriteTransaction tx = this.chain.getDomChain().newWriteOnlyTransaction();
+        final Collection<NodeIdentifierWithPredicates> routeKeys = ctx.writeRoutes(tx, nlri, attributes);
+        final Collection<NodeIdentifierWithPredicates> staleRoutes = this.staleRoutesRegistry.get(key);
+        if (staleRoutes != null) {
+            staleRoutes.removeAll(routeKeys);
+        }
+        LOG.trace("Write routes {}", nlri);
+        final FluentFuture<? extends CommitInfo> future = tx.commit();
+        this.submitted = future;
+        future.addCallback(new FutureCallback<CommitInfo>() {
+            @Override
+            public void onSuccess(final CommitInfo result) {
+                LOG.trace("Write routes {}, succeed", nlri);
+            }
+
+            @Override
+            public void onFailure(final Throwable throwable) {
+                LOG.error("Write routes failed", throwable);
+            }
+        }, MoreExecutors.directExecutor());
     }
 
     void removeRoutes(final MpUnreachNlri nlri) {
         final TablesKey key = new TablesKey(nlri.getAfi(), nlri.getSafi());
-        final TableContext ctx = tables.get(key);
+        final TableContext ctx = this.tables.get(key);
         if (ctx == null) {
             LOG.debug("No table for {}, not accepting NLRI {}", key, nlri);
             return;
         }
+        LOG.trace("Removing routes {}", nlri);
+        final DOMDataTreeWriteTransaction tx = this.chain.getDomChain().newWriteOnlyTransaction();
+        ctx.removeRoutes(tx, nlri);
+        final FluentFuture<? extends CommitInfo> future = tx.commit();
+        this.submitted = future;
+        future.addCallback(new FutureCallback<CommitInfo>() {
+            @Override
+            public void onSuccess(final CommitInfo result) {
+                LOG.trace("Removing routes {}, succeed", nlri);
+            }
+
+            @Override
+            public void onFailure(final Throwable throwable) {
+                LOG.error("Removing routes failed", throwable);
+            }
+        }, MoreExecutors.directExecutor());
+    }
+
+    void releaseChain() {
+        if (this.submitted != null) {
+            try {
+                this.submitted.get();
+            } catch (final InterruptedException | ExecutionException throwable) {
+                LOG.error("Write routes failed", throwable);
+            }
+        }
+    }
+
+    void storeStaleRoutes(final Set<TablesKey> gracefulTables) {
+        final CountDownLatch latch = new CountDownLatch(gracefulTables.size());
+
+        try (DOMDataTreeReadTransaction tx = this.chain.getDomChain().newReadOnlyTransaction()) {
+            for (TablesKey tablesKey : gracefulTables) {
+                final TableContext ctx = this.tables.get(tablesKey);
+                if (ctx == null) {
+                    LOG.warn("Missing table for address family {}", tablesKey);
+                    latch.countDown();
+                    continue;
+                }
+
+                tx.read(LogicalDatastoreType.OPERATIONAL, ctx.routesPath()).addCallback(
+                    new FutureCallback<Optional<NormalizedNode<?, ?>>>() {
+                        @Override
+                        public void onSuccess(final Optional<NormalizedNode<?, ?>> routesOptional) {
+                            try {
+                                if (routesOptional.isPresent()) {
+                                    synchronized (AdjRibInWriter.this.staleRoutesRegistry) {
+                                        final MapNode routesNode = (MapNode) routesOptional.get();
+                                        final List<NodeIdentifierWithPredicates> routes = routesNode.getValue().stream()
+                                                .map(MapEntryNode::getIdentifier)
+                                                .collect(Collectors.toList());
+                                        if (!routes.isEmpty()) {
+                                            AdjRibInWriter.this.staleRoutesRegistry.put(tablesKey, routes);
+                                        }
+                                    }
+                                }
+                            } finally {
+                                latch.countDown();
+                            }
+                        }
+
+                        @Override
+                        public void onFailure(final Throwable throwable) {
+                            LOG.warn("Failed to store stale routes for table {}", tablesKey, throwable);
+                            latch.countDown();
+                        }
+                    }, MoreExecutors.directExecutor());
+            }
+        }
+
+        try {
+            latch.await();
+        } catch (InterruptedException e) {
+            LOG.warn("Interrupted while waiting to store stale routes with {} tasks of {} to finish", latch.getCount(),
+                gracefulTables, e);
+        }
+    }
+
+    void removeStaleRoutes(final TablesKey tableKey) {
+        final TableContext ctx = this.tables.get(tableKey);
+        if (ctx == null) {
+            LOG.debug("No table for {}, not removing any stale routes", tableKey);
+            return;
+        }
+        final Collection<NodeIdentifierWithPredicates> routeKeys = this.staleRoutesRegistry.get(tableKey);
+        if (routeKeys == null || routeKeys.isEmpty()) {
+            LOG.debug("No stale routes present in table {}", tableKey);
+            return;
+        }
+        LOG.trace("Removing routes {}", routeKeys);
+        final DOMDataTreeWriteTransaction tx = this.chain.getDomChain().newWriteOnlyTransaction();
+        routeKeys.forEach(routeKey -> {
+            tx.delete(LogicalDatastoreType.OPERATIONAL, ctx.routePath(routeKey));
+        });
+        final FluentFuture<? extends CommitInfo> future = tx.commit();
+        this.submitted = future;
+        future.addCallback(new FutureCallback<CommitInfo>() {
+            @Override
+            public void onSuccess(final CommitInfo result) {
+                LOG.trace("Removing routes {}, succeed", routeKeys);
+                synchronized (AdjRibInWriter.this.staleRoutesRegistry) {
+                    staleRoutesRegistry.remove(tableKey);
+                }
+            }
+
+            @Override
+            public void onFailure(final Throwable throwable) {
+                LOG.warn("Removing routes {}, failed", routeKeys, throwable);
+            }
+        }, MoreExecutors.directExecutor());
+    }
+
+    FluentFuture<? extends CommitInfo> clearTables(final Set<TablesKey> tablesToClear) {
+        if (tablesToClear == null || tablesToClear.isEmpty()) {
+            return CommitInfo.emptyFluentFuture();
+        }
 
-        final DOMDataWriteTransaction tx = chain.newWriteOnlyTransaction();
-        ctx.removeRoutes(null, tx, nlri);
-        tx.submit();
+        final DOMDataTreeWriteTransaction wtx = this.chain.getDomChain().newWriteOnlyTransaction();
+        tablesToClear.forEach(tableKey -> {
+            final TableContext ctx = this.tables.get(tableKey);
+            wtx.delete(LogicalDatastoreType.OPERATIONAL, ctx.routesPath().getParent());
+        });
+        return wtx.commit();
     }
 }