Write failed node data on recovery to a file 87/36987/6
authorTom Pantelis <tpanteli@brocade.com>
Thu, 31 Mar 2016 21:52:17 +0000 (17:52 -0400)
committerTom Pantelis <tpanteli@brocade.com>
Fri, 10 Jun 2016 09:55:14 +0000 (09:55 +0000)
If a snapshot or journal batch fails schema validation on recovery, it's
helpful for troubleshooting to see the data so I added code to write the
failed node or modification to a file under the data directory.

Change-Id: I054798ae589837a6d4f6f511a22f0b478b6995bf
Signed-off-by: Tom Pantelis <tpanteli@brocade.com>
opendaylight/md-sal/sal-distributed-datastore/pom.xml
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardRecoveryCoordinator.java
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/AbstractBatchedModificationsCursor.java
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/AbstractDataTreeModificationCursor.java [new file with mode: 0644]
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/DataTreeModificationOutput.java [new file with mode: 0644]
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/NormalizedNodeXMLOutput.java [new file with mode: 0644]
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/PruningDataTreeModification.java

index b138574..7eef556 100644 (file)
     </dependency>
 
     <!-- OpenDaylight -->
+    <dependency>
+      <groupId>net.java.dev.stax-utils</groupId>
+      <artifactId>stax-utils</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.opendaylight.controller</groupId>
       <artifactId>config-api</artifactId>
index dae3383..f3cbc86 100644 (file)
@@ -8,7 +8,10 @@
 package org.opendaylight.controller.cluster.datastore;
 
 import com.google.common.base.Preconditions;
+import java.io.File;
 import java.io.IOException;
+import org.opendaylight.controller.cluster.datastore.utils.DataTreeModificationOutput;
+import org.opendaylight.controller.cluster.datastore.utils.NormalizedNodeXMLOutput;
 import org.opendaylight.controller.cluster.datastore.utils.PruningDataTreeModification;
 import org.opendaylight.controller.cluster.datastore.utils.SerializationUtils;
 import org.opendaylight.controller.cluster.raft.RaftActorRecoveryCohort;
@@ -84,8 +87,13 @@ class ShardRecoveryCoordinator implements RaftActorRecoveryCohort {
         log.debug("{}: Applying current log recovery batch with size {}", shardName, size);
         try {
             commitTransaction(transaction);
-        } catch (DataValidationFailedException e) {
-            log.error("{}: Failed to apply recovery batch", shardName, e);
+        } catch (Exception e) {
+            File file = new File(System.getProperty("karaf.data", "."),
+                    "failed-recovery-batch-" + shardName + ".out");
+            DataTreeModificationOutput.toFile(file, transaction.getResultingModification());
+            throw new RuntimeException(String.format(
+                    "%s: Failed to apply recovery batch. Modification data was written to file %s",
+                    shardName, file), e);
         }
         transaction = null;
     }
@@ -105,8 +113,13 @@ class ShardRecoveryCoordinator implements RaftActorRecoveryCohort {
         tx.write(YangInstanceIdentifier.EMPTY, node);
         try {
             commitTransaction(tx);
-        } catch (DataValidationFailedException e) {
-            log.error("{}: Failed to apply recovery snapshot", shardName, e);
+        } catch (Exception e) {
+            File file = new File(System.getProperty("karaf.data", "."),
+                    "failed-recovery-snapshot-" + shardName + ".xml");
+            NormalizedNodeXMLOutput.toFile(file, node);
+            throw new RuntimeException(String.format(
+                    "%s: Failed to apply recovery snapshot. Node data was written to file %s",
+                    shardName, file), e);
         }
     }
 
index 02f8abb..ecddebc 100644 (file)
@@ -7,88 +7,33 @@
  */
 package org.opendaylight.controller.cluster.datastore.utils;
 
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import java.util.ArrayDeque;
-import java.util.Deque;
-import javax.annotation.Nonnull;
 import org.opendaylight.controller.cluster.datastore.messages.BatchedModifications;
 import org.opendaylight.controller.cluster.datastore.modification.DeleteModification;
 import org.opendaylight.controller.cluster.datastore.modification.MergeModification;
 import org.opendaylight.controller.cluster.datastore.modification.WriteModification;
-import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.PathArgument;
 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
-import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeModificationCursor;
 
 /**
  * Base class for a DataTreeModificationCursor that publishes to BatchedModifications instance(s).
  *
  * @author Thomas Pantelis
  */
-public abstract class AbstractBatchedModificationsCursor implements DataTreeModificationCursor {
-    private final Deque<YangInstanceIdentifier> stack = new ArrayDeque<>();
-
-    protected AbstractBatchedModificationsCursor() {
-        stack.push(YangInstanceIdentifier.EMPTY);
-    }
-
+public abstract class AbstractBatchedModificationsCursor extends AbstractDataTreeModificationCursor {
     protected abstract BatchedModifications getModifications();
 
     @Override
     public void delete(final PathArgument child) {
-        getModifications().addModification(new DeleteModification(stack.peek().node(child)));
+        getModifications().addModification(new DeleteModification(next(child)));
     }
 
     @Override
     public void merge(final PathArgument child, final NormalizedNode<?, ?> data) {
-        getModifications().addModification(new MergeModification(stack.peek().node(child), data));
+        getModifications().addModification(new MergeModification(next(child), data));
     }
 
     @Override
     public void write(final PathArgument child, final NormalizedNode<?, ?> data) {
-        getModifications().addModification(new WriteModification(stack.peek().node(child), data));
-    }
-
-    @Override
-    public void enter(@Nonnull final PathArgument child) {
-        stack.push(stack.peek().node(child));
-    }
-
-    @Override
-    public void enter(@Nonnull final PathArgument... path) {
-        for (PathArgument arg : path) {
-            enter(arg);
-        }
-    }
-
-    @Override
-    public void enter(@Nonnull final Iterable<PathArgument> path) {
-        for (PathArgument arg : path) {
-            enter(arg);
-        }
-    }
-
-    @Override
-    public void exit() {
-        stack.pop();
-    }
-
-    @Override
-    public void exit(final int depth) {
-        Preconditions.checkArgument(depth < stack.size(), "Stack holds only %s elements, cannot exit %s levels", stack.size(), depth);
-        for (int i = 0; i < depth; ++i) {
-            stack.pop();
-        }
-    }
-
-    @Override
-    public Optional<NormalizedNode<?, ?>> readNode(@Nonnull final PathArgument child) {
-        throw new UnsupportedOperationException("Not implemented");
-    }
-
-    @Override
-    public void close() {
-        // No-op
+        getModifications().addModification(new WriteModification(next(child), data));
     }
 }
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/AbstractDataTreeModificationCursor.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/AbstractDataTreeModificationCursor.java
new file mode 100644 (file)
index 0000000..c434509
--- /dev/null
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2016 Brocade Communications Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.controller.cluster.datastore.utils;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import javax.annotation.Nonnull;
+import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
+import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.PathArgument;
+import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
+import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeModificationCursor;
+
+/**
+ * Base class for a DataTreeModificationCursor.
+ *
+ * @author Thomas Pantelis
+ */
+public abstract class AbstractDataTreeModificationCursor implements DataTreeModificationCursor {
+    private final Deque<YangInstanceIdentifier> stack = new ArrayDeque<>();
+
+    protected AbstractDataTreeModificationCursor() {
+        stack.push(YangInstanceIdentifier.EMPTY);
+    }
+
+    protected YangInstanceIdentifier next(@Nonnull final PathArgument child) {
+        return stack.peek().node(child);
+    }
+
+    @Override
+    public void enter(@Nonnull final PathArgument child) {
+        stack.push(stack.peek().node(child));
+    }
+
+    @Override
+    public void enter(@Nonnull final PathArgument... path) {
+        for (PathArgument arg : path) {
+            enter(arg);
+        }
+    }
+
+    @Override
+    public void enter(@Nonnull final Iterable<PathArgument> path) {
+        for (PathArgument arg : path) {
+            enter(arg);
+        }
+    }
+
+    @Override
+    public void exit() {
+        stack.pop();
+    }
+
+    @Override
+    public void exit(final int depth) {
+        Preconditions.checkArgument(depth < stack.size(), "Stack holds only %s elements, cannot exit %s levels", stack.size(), depth);
+        for (int i = 0; i < depth; ++i) {
+            stack.pop();
+        }
+    }
+
+    @Override
+    public Optional<NormalizedNode<?, ?>> readNode(@Nonnull final PathArgument child) {
+        throw new UnsupportedOperationException("Not implemented");
+    }
+
+    @Override
+    public void close() {
+        // No-op
+    }
+}
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/DataTreeModificationOutput.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/DataTreeModificationOutput.java
new file mode 100644 (file)
index 0000000..17d4c4a
--- /dev/null
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2016 Brocade Communications Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.controller.cluster.datastore.utils;
+
+import com.google.common.base.Throwables;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import javax.xml.stream.XMLStreamException;
+import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.PathArgument;
+import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
+import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeModification;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility class to output DataTreeModifications in readable format.
+ *
+ * @author Thomas Pantelis
+ */
+public final class DataTreeModificationOutput {
+    private static final Logger LOG = LoggerFactory.getLogger(DataTreeModificationOutput.class);
+
+    private DataTreeModificationOutput() {
+    }
+
+    public static void toFile(File file, DataTreeModification modification) {
+        try(FileOutputStream outStream = new FileOutputStream(file)) {
+            modification.applyToCursor(new DataTreeModificationOutputCursor(new DataOutputStream(outStream)));
+        } catch(Exception e) {
+            LOG.error("Error writing DataTreeModification to file {}", file, e);
+        }
+    }
+
+    private static class DataTreeModificationOutputCursor extends AbstractDataTreeModificationCursor {
+        private final DataOutputStream output;
+
+        DataTreeModificationOutputCursor(DataOutputStream output) {
+            this.output = output;
+        }
+
+        @Override
+        public void delete(PathArgument child) {
+            try {
+                output.write("\nDELETE -> ".getBytes());
+                output.write(next(child).toString().getBytes());
+                output.writeByte('\n');
+            } catch(IOException e) {
+                Throwables.propagate(e);
+            }
+        }
+
+        @Override
+        public void merge(PathArgument child, NormalizedNode<?, ?> data) {
+            outputPathAndNode("MERGE", child, data);
+        }
+
+        @Override
+        public void write(PathArgument child, NormalizedNode<?, ?> data) {
+            outputPathAndNode("WRITE", child, data);
+        }
+
+        private void outputPathAndNode(String name, PathArgument child, NormalizedNode<?, ?> data) {
+            try {
+                output.writeByte('\n');
+                output.write(name.getBytes());
+                output.write(" -> ".getBytes());
+                output.write(next(child).toString().getBytes());
+                output.write(": \n".getBytes());
+                NormalizedNodeXMLOutput.toStream(output, data);
+                output.writeByte('\n');
+            } catch(IOException | XMLStreamException e) {
+                Throwables.propagate(e);
+            }
+        }
+    }
+}
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/NormalizedNodeXMLOutput.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/NormalizedNodeXMLOutput.java
new file mode 100644 (file)
index 0000000..ffcc00e
--- /dev/null
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2016 Brocade Communications Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.controller.cluster.datastore.utils;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import javax.xml.stream.XMLOutputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
+import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
+import org.opendaylight.yangtools.yang.data.api.schema.stream.NormalizedNodeStreamWriter;
+import org.opendaylight.yangtools.yang.data.api.schema.stream.NormalizedNodeWriter;
+import org.opendaylight.yangtools.yang.data.impl.codec.xml.XMLStreamNormalizedNodeStreamWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import javanet.staxutils.IndentingXMLStreamWriter;
+
+/**
+ * Utility class to output NormalizedNodes as XML.
+ *
+ * @author Thomas Pantelis
+ */
+public final class NormalizedNodeXMLOutput {
+    private static final Logger LOG = LoggerFactory.getLogger(NormalizedNodeXMLOutput.class);
+
+    private NormalizedNodeXMLOutput() {
+    }
+
+    public static void toStream(OutputStream outStream, NormalizedNode<?, ?> node)
+            throws XMLStreamException, IOException {
+        XMLOutputFactory xmlFactory = XMLOutputFactory.newFactory();
+        xmlFactory.setProperty(XMLOutputFactory.IS_REPAIRING_NAMESPACES, true);
+        XMLStreamWriter xmlWriter = xmlFactory.createXMLStreamWriter(outStream);
+
+        IndentingXMLStreamWriter indenting = new IndentingXMLStreamWriter(xmlWriter);
+        try(NormalizedNodeStreamWriter streamWriter = XMLStreamNormalizedNodeStreamWriter.createSchemaless(
+                indenting)) {
+            NormalizedNodeWriter nodeWriter = NormalizedNodeWriter.forStreamWriter(streamWriter);
+            nodeWriter.write(node);
+            nodeWriter.flush();
+        }
+    }
+
+    public static void toFile(File file, NormalizedNode<?, ?> node) {
+        try(FileOutputStream outStream = new FileOutputStream(file)) {
+            toStream(outStream, node);
+        } catch(IOException | XMLStreamException e) {
+            LOG.error("Error writing NormalizedNode to file {}", file, e);
+        }
+    }
+}
index 0e97263..a44b686 100644 (file)
@@ -10,11 +10,7 @@ package org.opendaylight.controller.cluster.datastore.utils;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
 import java.io.IOException;
-import java.util.ArrayDeque;
-import java.util.Deque;
-import javax.annotation.Nonnull;
 import org.opendaylight.controller.cluster.datastore.node.utils.transformer.NormalizedNodePruner;
 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.PathArgument;
@@ -147,8 +143,7 @@ public class PruningDataTreeModification implements DataTreeModification {
         return delegate;
     }
 
-    private static class PruningDataTreeModificationCursor implements DataTreeModificationCursor {
-        private final Deque<YangInstanceIdentifier> stack = new ArrayDeque<>();
+    private static class PruningDataTreeModificationCursor extends AbstractDataTreeModificationCursor {
         private final DataTreeModification toModification;
         private final PruningDataTreeModification pruningModification;
 
@@ -156,12 +151,11 @@ public class PruningDataTreeModification implements DataTreeModification {
                 PruningDataTreeModification pruningModification) {
             this.toModification = toModification;
             this.pruningModification = pruningModification;
-            stack.push(YangInstanceIdentifier.EMPTY);
         }
 
         @Override
         public void write(PathArgument child, NormalizedNode<?, ?> data) {
-            YangInstanceIdentifier path = stack.peek().node(child);
+            YangInstanceIdentifier path = next(child);
             NormalizedNode<?, ?> prunedNode = pruningModification.pruneNormalizedNode(path, data);
             if(prunedNode != null) {
                 toModification.write(path, prunedNode);
@@ -170,7 +164,7 @@ public class PruningDataTreeModification implements DataTreeModification {
 
         @Override
         public void merge(PathArgument child, NormalizedNode<?, ?> data) {
-            YangInstanceIdentifier path = stack.peek().node(child);
+            YangInstanceIdentifier path = next(child);
             NormalizedNode<?, ?> prunedNode = pruningModification.pruneNormalizedNode(path, data);
             if(prunedNode != null) {
                 toModification.merge(path, prunedNode);
@@ -180,51 +174,10 @@ public class PruningDataTreeModification implements DataTreeModification {
         @Override
         public void delete(PathArgument child) {
             try {
-                toModification.delete(stack.peek().node(child));
+                toModification.delete(next(child));
             } catch(SchemaValidationFailedException e) {
                 // Ignoring since we would've already logged this in the call to the original modification.
             }
         }
-
-        @Override
-        public void enter(@Nonnull final PathArgument child) {
-            stack.push(stack.peek().node(child));
-        }
-
-        @Override
-        public void enter(@Nonnull final PathArgument... path) {
-            for (PathArgument arg : path) {
-                enter(arg);
-            }
-        }
-
-        @Override
-        public void enter(@Nonnull final Iterable<PathArgument> path) {
-            for (PathArgument arg : path) {
-                enter(arg);
-            }
-        }
-
-        @Override
-        public void exit() {
-            stack.pop();
-        }
-
-        @Override
-        public void exit(final int depth) {
-            Preconditions.checkArgument(depth < stack.size(), "Stack holds only %s elements, cannot exit %s levels", stack.size(), depth);
-            for (int i = 0; i < depth; ++i) {
-                stack.pop();
-            }
-        }
-
-        @Override
-        public Optional<NormalizedNode<?, ?>> readNode(@Nonnull final PathArgument child) {
-            throw new UnsupportedOperationException("Not implemented");
-        }
-
-        @Override
-        public void close() {
-        }
     }
 }