opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/JsonExportTest.java

   1 /*
   2  * Copyright (c) 2021 PANTHEON.tech, s.r.o. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.cluster.datastore;
   9
  10 import static org.awaitility.Awaitility.await;
  11 import static org.junit.Assert.assertEquals;
  12
  13 import java.io.File;
  14 import java.io.IOException;
  15 import java.nio.file.Files;
  16 import java.nio.file.Path;
  17 import java.util.HashSet;
  18 import java.util.Set;
  19 import java.util.concurrent.TimeUnit;
  20 import org.junit.Before;
  21 import org.junit.Rule;
  22 import org.junit.Test;
  23 import org.junit.rules.TemporaryFolder;
  24 import org.opendaylight.controller.cluster.raft.persisted.ApplyJournalEntries;
  25 import org.opendaylight.controller.cluster.raft.persisted.SimpleReplicatedLogEntry;
  26 import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
  27 import org.opendaylight.controller.md.cluster.datastore.model.TestModel;
  28 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.config.distributed.datastore.provider.rev231229.DataStoreProperties.ExportOnRecovery;
  29 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
  30 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifier;
  31 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifierWithPredicates;
  32 import org.opendaylight.yangtools.yang.data.spi.node.ImmutableNodes;
  33
  34 public class JsonExportTest extends AbstractShardTest {
  35     private static final String DUMMY_DATA = "Dummy data as snapshot sequence number is set to 0 in "
  36             + "InMemorySnapshotStore and journal recovery seq number will start from 1";
  37     private static final String EXPECTED_JOURNAL_FILE = "expectedJournalExport.json";
  38     private static final String EXPECTED_SNAPSHOT_FILE = "expectedSnapshotExport.json";
  39     private static String actualJournalFilePath;
  40     private static String actualSnapshotFilePath;
  41     private DatastoreContext datastoreContext;
  42
  43     @Rule
  44     public TemporaryFolder temporaryFolder = new TemporaryFolder();
  45
  46     @Override
  47     @Before
  48     public void setUp() throws Exception {
  49         super.setUp();
  50         final var exportTmpFolder = temporaryFolder.newFolder("persistence-export");
  51         actualJournalFilePath = exportTmpFolder.getAbsolutePath() + "/journals/"
  52             + "member-1-shard-inventory-config" + nextShardNum + "-journal.json";
  53         actualSnapshotFilePath = exportTmpFolder.getAbsolutePath() + "/snapshots/"
  54             + "member-1-shard-inventory-config" + nextShardNum + "-snapshot.json";
  55         datastoreContext = DatastoreContext.newBuilder().shardJournalRecoveryLogBatchSize(1)
  56             .shardSnapshotBatchCount(5000).shardHeartbeatIntervalInMillis(HEARTBEAT_MILLIS).persistent(true)
  57             .exportOnRecovery(ExportOnRecovery.Json)
  58             .recoveryExportBaseDir(exportTmpFolder.getAbsolutePath()).build();
  59     }
  60
  61     @Override
  62     protected DatastoreContext newDatastoreContext() {
  63         return datastoreContext;
  64     }
  65
  66     @Test
  67     public void testJsonExport() throws Exception {
  68         // Set up the InMemorySnapshotStore.
  69         final var source = setupInMemorySnapshotStore();
  70
  71         final var writeMod = source.takeSnapshot().newModification();
  72         writeMod.write(TestModel.OUTER_LIST_PATH, ImmutableNodes.newSystemMapBuilder()
  73             .withNodeIdentifier(new NodeIdentifier(TestModel.OUTER_LIST_QNAME))
  74             .build());
  75         writeMod.ready();
  76         InMemoryJournal.addEntry(shardID.toString(), 0, DUMMY_DATA);
  77
  78         // Set up the InMemoryJournal.
  79         InMemoryJournal.addEntry(shardID.toString(), 1, new SimpleReplicatedLogEntry(0, 1,
  80                 payloadForModification(source, writeMod, nextTransactionId())));
  81
  82         final int nListEntries = 16;
  83         final Set<Integer> listEntryKeys = new HashSet<>();
  84
  85         // Add some ModificationPayload entries
  86         for (int i = 1; i <= nListEntries; i++) {
  87             final Integer value = i;
  88             listEntryKeys.add(value);
  89
  90             final var path = YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH)
  91                     .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, value).build();
  92
  93             final var mod = source.takeSnapshot().newModification();
  94             mod.merge(path, ImmutableNodes.newMapEntryBuilder()
  95                 .withNodeIdentifier(
  96                     NodeIdentifierWithPredicates.of(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, value))
  97                 .withChild(ImmutableNodes.leafNode(TestModel.ID_QNAME, value))
  98                 .build());
  99             mod.ready();
 100
 101             InMemoryJournal.addEntry(shardID.toString(), i + 1, new SimpleReplicatedLogEntry(i, 1,
 102                     payloadForModification(source, mod, nextTransactionId())));
 103         }
 104
 105         InMemoryJournal.addEntry(shardID.toString(), nListEntries + 2,
 106                 new ApplyJournalEntries(nListEntries));
 107
 108         testRecovery(listEntryKeys, false);
 109
 110         verifyJournalExport();
 111         verifySnapshotExport();
 112     }
 113
 114     private static void verifyJournalExport() throws IOException {
 115         final String expectedJournalData = readExpectedFile(EXPECTED_JOURNAL_FILE);
 116         final String actualJournalData = readActualFile(actualJournalFilePath);
 117         assertEquals("Exported journal is not expected ", expectedJournalData, actualJournalData);
 118     }
 119
 120     private static void verifySnapshotExport() throws IOException {
 121         final String expectedSnapshotData = readExpectedFile(EXPECTED_SNAPSHOT_FILE);
 122         final String actualSnapshotData = readActualFile(actualSnapshotFilePath);
 123         assertEquals("Exported snapshot is not expected ", expectedSnapshotData, actualSnapshotData);
 124     }
 125
 126     private static String readExpectedFile(final String filePath) throws IOException {
 127         final File exportFile = new File(JsonExportTest.class.getClassLoader().getResource(filePath).getFile());
 128         return new String(Files.readAllBytes(Path.of(exportFile.getPath())));
 129     }
 130
 131     private static String readActualFile(final String filePath) throws IOException {
 132         final File exportFile = new File(filePath);
 133         await().atMost(10, TimeUnit.SECONDS).until(exportFile::exists);
 134         return new String(Files.readAllBytes(Path.of(filePath)));
 135     }
 136 }