From 34c6032dd81c2d76720cce53478c38f5e5cdddc4 Mon Sep 17 00:00:00 2001 From: Tom Pantelis Date: Thu, 31 Mar 2016 17:52:17 -0400 Subject: [PATCH] Write failed node data on recovery to a file If a snapshot or journal batch fails schema validation on recovery, it's helpful for troubleshooting to see the data so I added code to write the failed node or modification to a file under the data directory. Change-Id: I054798ae589837a6d4f6f511a22f0b478b6995bf Signed-off-by: Tom Pantelis --- .../md-sal/sal-distributed-datastore/pom.xml | 4 + .../datastore/ShardRecoveryCoordinator.java | 21 ++++- .../AbstractBatchedModificationsCursor.java | 63 +------------- .../AbstractDataTreeModificationCursor.java | 77 +++++++++++++++++ .../utils/DataTreeModificationOutput.java | 83 +++++++++++++++++++ .../utils/NormalizedNodeXMLOutput.java | 58 +++++++++++++ .../utils/PruningDataTreeModification.java | 55 +----------- 7 files changed, 247 insertions(+), 114 deletions(-) create mode 100644 opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/AbstractDataTreeModificationCursor.java create mode 100644 opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/DataTreeModificationOutput.java create mode 100644 opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/NormalizedNodeXMLOutput.java diff --git a/opendaylight/md-sal/sal-distributed-datastore/pom.xml b/opendaylight/md-sal/sal-distributed-datastore/pom.xml index b138574add..7eef556619 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/pom.xml +++ b/opendaylight/md-sal/sal-distributed-datastore/pom.xml @@ -85,6 +85,10 @@ + + net.java.dev.stax-utils + stax-utils + org.opendaylight.controller config-api diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardRecoveryCoordinator.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardRecoveryCoordinator.java index dae3383a2e..f3cbc8649d 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardRecoveryCoordinator.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardRecoveryCoordinator.java @@ -8,7 +8,10 @@ package org.opendaylight.controller.cluster.datastore; import com.google.common.base.Preconditions; +import java.io.File; import java.io.IOException; +import org.opendaylight.controller.cluster.datastore.utils.DataTreeModificationOutput; +import org.opendaylight.controller.cluster.datastore.utils.NormalizedNodeXMLOutput; import org.opendaylight.controller.cluster.datastore.utils.PruningDataTreeModification; import org.opendaylight.controller.cluster.datastore.utils.SerializationUtils; import org.opendaylight.controller.cluster.raft.RaftActorRecoveryCohort; @@ -84,8 +87,13 @@ class ShardRecoveryCoordinator implements RaftActorRecoveryCohort { log.debug("{}: Applying current log recovery batch with size {}", shardName, size); try { commitTransaction(transaction); - } catch (DataValidationFailedException e) { - log.error("{}: Failed to apply recovery batch", shardName, e); + } catch (Exception e) { + File file = new File(System.getProperty("karaf.data", "."), + "failed-recovery-batch-" + shardName + ".out"); + DataTreeModificationOutput.toFile(file, transaction.getResultingModification()); + throw new RuntimeException(String.format( + "%s: Failed to apply recovery batch. Modification data was written to file %s", + shardName, file), e); } transaction = null; } @@ -105,8 +113,13 @@ class ShardRecoveryCoordinator implements RaftActorRecoveryCohort { tx.write(YangInstanceIdentifier.EMPTY, node); try { commitTransaction(tx); - } catch (DataValidationFailedException e) { - log.error("{}: Failed to apply recovery snapshot", shardName, e); + } catch (Exception e) { + File file = new File(System.getProperty("karaf.data", "."), + "failed-recovery-snapshot-" + shardName + ".xml"); + NormalizedNodeXMLOutput.toFile(file, node); + throw new RuntimeException(String.format( + "%s: Failed to apply recovery snapshot. Node data was written to file %s", + shardName, file), e); } } diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/AbstractBatchedModificationsCursor.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/AbstractBatchedModificationsCursor.java index 02f8abb572..ecddebcc55 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/AbstractBatchedModificationsCursor.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/AbstractBatchedModificationsCursor.java @@ -7,88 +7,33 @@ */ package org.opendaylight.controller.cluster.datastore.utils; -import com.google.common.base.Optional; -import com.google.common.base.Preconditions; -import java.util.ArrayDeque; -import java.util.Deque; -import javax.annotation.Nonnull; import org.opendaylight.controller.cluster.datastore.messages.BatchedModifications; import org.opendaylight.controller.cluster.datastore.modification.DeleteModification; import org.opendaylight.controller.cluster.datastore.modification.MergeModification; import org.opendaylight.controller.cluster.datastore.modification.WriteModification; -import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier; import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.PathArgument; import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode; -import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeModificationCursor; /** * Base class for a DataTreeModificationCursor that publishes to BatchedModifications instance(s). * * @author Thomas Pantelis */ -public abstract class AbstractBatchedModificationsCursor implements DataTreeModificationCursor { - private final Deque stack = new ArrayDeque<>(); - - protected AbstractBatchedModificationsCursor() { - stack.push(YangInstanceIdentifier.EMPTY); - } - +public abstract class AbstractBatchedModificationsCursor extends AbstractDataTreeModificationCursor { protected abstract BatchedModifications getModifications(); @Override public void delete(final PathArgument child) { - getModifications().addModification(new DeleteModification(stack.peek().node(child))); + getModifications().addModification(new DeleteModification(next(child))); } @Override public void merge(final PathArgument child, final NormalizedNode data) { - getModifications().addModification(new MergeModification(stack.peek().node(child), data)); + getModifications().addModification(new MergeModification(next(child), data)); } @Override public void write(final PathArgument child, final NormalizedNode data) { - getModifications().addModification(new WriteModification(stack.peek().node(child), data)); - } - - @Override - public void enter(@Nonnull final PathArgument child) { - stack.push(stack.peek().node(child)); - } - - @Override - public void enter(@Nonnull final PathArgument... path) { - for (PathArgument arg : path) { - enter(arg); - } - } - - @Override - public void enter(@Nonnull final Iterable path) { - for (PathArgument arg : path) { - enter(arg); - } - } - - @Override - public void exit() { - stack.pop(); - } - - @Override - public void exit(final int depth) { - Preconditions.checkArgument(depth < stack.size(), "Stack holds only %s elements, cannot exit %s levels", stack.size(), depth); - for (int i = 0; i < depth; ++i) { - stack.pop(); - } - } - - @Override - public Optional> readNode(@Nonnull final PathArgument child) { - throw new UnsupportedOperationException("Not implemented"); - } - - @Override - public void close() { - // No-op + getModifications().addModification(new WriteModification(next(child), data)); } } diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/AbstractDataTreeModificationCursor.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/AbstractDataTreeModificationCursor.java new file mode 100644 index 0000000000..c4345091f3 --- /dev/null +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/AbstractDataTreeModificationCursor.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2016 Brocade Communications Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.controller.cluster.datastore.utils; + +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import java.util.ArrayDeque; +import java.util.Deque; +import javax.annotation.Nonnull; +import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier; +import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.PathArgument; +import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode; +import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeModificationCursor; + +/** + * Base class for a DataTreeModificationCursor. + * + * @author Thomas Pantelis + */ +public abstract class AbstractDataTreeModificationCursor implements DataTreeModificationCursor { + private final Deque stack = new ArrayDeque<>(); + + protected AbstractDataTreeModificationCursor() { + stack.push(YangInstanceIdentifier.EMPTY); + } + + protected YangInstanceIdentifier next(@Nonnull final PathArgument child) { + return stack.peek().node(child); + } + + @Override + public void enter(@Nonnull final PathArgument child) { + stack.push(stack.peek().node(child)); + } + + @Override + public void enter(@Nonnull final PathArgument... path) { + for (PathArgument arg : path) { + enter(arg); + } + } + + @Override + public void enter(@Nonnull final Iterable path) { + for (PathArgument arg : path) { + enter(arg); + } + } + + @Override + public void exit() { + stack.pop(); + } + + @Override + public void exit(final int depth) { + Preconditions.checkArgument(depth < stack.size(), "Stack holds only %s elements, cannot exit %s levels", stack.size(), depth); + for (int i = 0; i < depth; ++i) { + stack.pop(); + } + } + + @Override + public Optional> readNode(@Nonnull final PathArgument child) { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public void close() { + // No-op + } +} diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/DataTreeModificationOutput.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/DataTreeModificationOutput.java new file mode 100644 index 0000000000..17d4c4a71d --- /dev/null +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/DataTreeModificationOutput.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2016 Brocade Communications Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.controller.cluster.datastore.utils; + +import com.google.common.base.Throwables; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import javax.xml.stream.XMLStreamException; +import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.PathArgument; +import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode; +import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeModification; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class to output DataTreeModifications in readable format. + * + * @author Thomas Pantelis + */ +public final class DataTreeModificationOutput { + private static final Logger LOG = LoggerFactory.getLogger(DataTreeModificationOutput.class); + + private DataTreeModificationOutput() { + } + + public static void toFile(File file, DataTreeModification modification) { + try(FileOutputStream outStream = new FileOutputStream(file)) { + modification.applyToCursor(new DataTreeModificationOutputCursor(new DataOutputStream(outStream))); + } catch(Exception e) { + LOG.error("Error writing DataTreeModification to file {}", file, e); + } + } + + private static class DataTreeModificationOutputCursor extends AbstractDataTreeModificationCursor { + private final DataOutputStream output; + + DataTreeModificationOutputCursor(DataOutputStream output) { + this.output = output; + } + + @Override + public void delete(PathArgument child) { + try { + output.write("\nDELETE -> ".getBytes()); + output.write(next(child).toString().getBytes()); + output.writeByte('\n'); + } catch(IOException e) { + Throwables.propagate(e); + } + } + + @Override + public void merge(PathArgument child, NormalizedNode data) { + outputPathAndNode("MERGE", child, data); + } + + @Override + public void write(PathArgument child, NormalizedNode data) { + outputPathAndNode("WRITE", child, data); + } + + private void outputPathAndNode(String name, PathArgument child, NormalizedNode data) { + try { + output.writeByte('\n'); + output.write(name.getBytes()); + output.write(" -> ".getBytes()); + output.write(next(child).toString().getBytes()); + output.write(": \n".getBytes()); + NormalizedNodeXMLOutput.toStream(output, data); + output.writeByte('\n'); + } catch(IOException | XMLStreamException e) { + Throwables.propagate(e); + } + } + } +} diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/NormalizedNodeXMLOutput.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/NormalizedNodeXMLOutput.java new file mode 100644 index 0000000000..ffcc00e557 --- /dev/null +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/NormalizedNodeXMLOutput.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2016 Brocade Communications Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.controller.cluster.datastore.utils; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import javax.xml.stream.XMLOutputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamWriter; +import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode; +import org.opendaylight.yangtools.yang.data.api.schema.stream.NormalizedNodeStreamWriter; +import org.opendaylight.yangtools.yang.data.api.schema.stream.NormalizedNodeWriter; +import org.opendaylight.yangtools.yang.data.impl.codec.xml.XMLStreamNormalizedNodeStreamWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import javanet.staxutils.IndentingXMLStreamWriter; + +/** + * Utility class to output NormalizedNodes as XML. + * + * @author Thomas Pantelis + */ +public final class NormalizedNodeXMLOutput { + private static final Logger LOG = LoggerFactory.getLogger(NormalizedNodeXMLOutput.class); + + private NormalizedNodeXMLOutput() { + } + + public static void toStream(OutputStream outStream, NormalizedNode node) + throws XMLStreamException, IOException { + XMLOutputFactory xmlFactory = XMLOutputFactory.newFactory(); + xmlFactory.setProperty(XMLOutputFactory.IS_REPAIRING_NAMESPACES, true); + XMLStreamWriter xmlWriter = xmlFactory.createXMLStreamWriter(outStream); + + IndentingXMLStreamWriter indenting = new IndentingXMLStreamWriter(xmlWriter); + try(NormalizedNodeStreamWriter streamWriter = XMLStreamNormalizedNodeStreamWriter.createSchemaless( + indenting)) { + NormalizedNodeWriter nodeWriter = NormalizedNodeWriter.forStreamWriter(streamWriter); + nodeWriter.write(node); + nodeWriter.flush(); + } + } + + public static void toFile(File file, NormalizedNode node) { + try(FileOutputStream outStream = new FileOutputStream(file)) { + toStream(outStream, node); + } catch(IOException | XMLStreamException e) { + LOG.error("Error writing NormalizedNode to file {}", file, e); + } + } +} diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/PruningDataTreeModification.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/PruningDataTreeModification.java index 0e9726373a..a44b6861ba 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/PruningDataTreeModification.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/PruningDataTreeModification.java @@ -10,11 +10,7 @@ package org.opendaylight.controller.cluster.datastore.utils; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Optional; -import com.google.common.base.Preconditions; import java.io.IOException; -import java.util.ArrayDeque; -import java.util.Deque; -import javax.annotation.Nonnull; import org.opendaylight.controller.cluster.datastore.node.utils.transformer.NormalizedNodePruner; import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier; import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.PathArgument; @@ -147,8 +143,7 @@ public class PruningDataTreeModification implements DataTreeModification { return delegate; } - private static class PruningDataTreeModificationCursor implements DataTreeModificationCursor { - private final Deque stack = new ArrayDeque<>(); + private static class PruningDataTreeModificationCursor extends AbstractDataTreeModificationCursor { private final DataTreeModification toModification; private final PruningDataTreeModification pruningModification; @@ -156,12 +151,11 @@ public class PruningDataTreeModification implements DataTreeModification { PruningDataTreeModification pruningModification) { this.toModification = toModification; this.pruningModification = pruningModification; - stack.push(YangInstanceIdentifier.EMPTY); } @Override public void write(PathArgument child, NormalizedNode data) { - YangInstanceIdentifier path = stack.peek().node(child); + YangInstanceIdentifier path = next(child); NormalizedNode prunedNode = pruningModification.pruneNormalizedNode(path, data); if(prunedNode != null) { toModification.write(path, prunedNode); @@ -170,7 +164,7 @@ public class PruningDataTreeModification implements DataTreeModification { @Override public void merge(PathArgument child, NormalizedNode data) { - YangInstanceIdentifier path = stack.peek().node(child); + YangInstanceIdentifier path = next(child); NormalizedNode prunedNode = pruningModification.pruneNormalizedNode(path, data); if(prunedNode != null) { toModification.merge(path, prunedNode); @@ -180,51 +174,10 @@ public class PruningDataTreeModification implements DataTreeModification { @Override public void delete(PathArgument child) { try { - toModification.delete(stack.peek().node(child)); + toModification.delete(next(child)); } catch(SchemaValidationFailedException e) { // Ignoring since we would've already logged this in the call to the original modification. } } - - @Override - public void enter(@Nonnull final PathArgument child) { - stack.push(stack.peek().node(child)); - } - - @Override - public void enter(@Nonnull final PathArgument... path) { - for (PathArgument arg : path) { - enter(arg); - } - } - - @Override - public void enter(@Nonnull final Iterable path) { - for (PathArgument arg : path) { - enter(arg); - } - } - - @Override - public void exit() { - stack.pop(); - } - - @Override - public void exit(final int depth) { - Preconditions.checkArgument(depth < stack.size(), "Stack holds only %s elements, cannot exit %s levels", stack.size(), depth); - for (int i = 0; i < depth; ++i) { - stack.pop(); - } - } - - @Override - public Optional> readNode(@Nonnull final PathArgument child) { - throw new UnsupportedOperationException("Not implemented"); - } - - @Override - public void close() { - } } } -- 2.36.6