From 67b69d39102b1bb2b057ef08d5406c9200012d8e Mon Sep 17 00:00:00 2001 From: Robert Varga Date: Wed, 27 Mar 2024 00:20:26 +0100 Subject: [PATCH 01/16] Do not use BufferOverflowException for EOF signalling A neat benefit of well-structured code is that it makes it clear what the contracts are. In this patch we eliminate a source of BufferOverflowExceptions and opt to report the fact next segment is needed via a @Nullable return. JIRA: CONTROLLER-2100 Change-Id: I1b9535053561709dc2a4b60446cd7cabd19da659 Signed-off-by: Robert Varga --- .../storage/journal/JournalSegmentWriter.java | 19 ++-- .../atomix/storage/journal/JournalWriter.java | 96 ++++++++++--------- .../journal/SegmentedJournalWriter.java | 14 ++- .../storage/journal/StorageException.java | 4 + 4 files changed, 71 insertions(+), 62 deletions(-) diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegmentWriter.java b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegmentWriter.java index df8ca35068..c7c035be65 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegmentWriter.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegmentWriter.java @@ -20,15 +20,18 @@ import static java.util.Objects.requireNonNull; import com.esotericsoftware.kryo.KryoException; import io.atomix.storage.journal.index.JournalIndex; -import java.nio.BufferOverflowException; import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.util.zip.CRC32; import org.eclipse.jdt.annotation.NonNull; import org.eclipse.jdt.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; abstract sealed class JournalSegmentWriter permits DiskJournalSegmentWriter, MappedJournalSegmentWriter { + private static final Logger LOG = LoggerFactory.getLogger(JournalSegmentWriter.class); + final @NonNull FileChannel channel; final @NonNull JournalSegment segment; private final @NonNull JournalIndex index; @@ -88,12 +91,12 @@ abstract sealed class JournalSegmentWriter permits DiskJournalSegmentWriter, } /** - * Appends an entry to the journal. + * Tries to append an entry to the journal. * * @param entry The entry to append. - * @return The appended indexed entry. + * @return The appended indexed entry, or {@code null} if there is not enough space available */ - final Indexed append(final T entry) { + final @Nullable Indexed append(final T entry) { // Store the entry index. final long index = getNextIndex(); final int position = currentPosition; @@ -102,7 +105,8 @@ abstract sealed class JournalSegmentWriter permits DiskJournalSegmentWriter, final int bodyPosition = position + HEADER_BYTES; final int avail = maxSegmentSize - bodyPosition; if (avail < 0) { - throw new BufferOverflowException(); + LOG.trace("Not enough space for {} at {}", index, position); + return null; } final var writeLimit = Math.min(avail, maxEntrySize); @@ -112,11 +116,12 @@ abstract sealed class JournalSegmentWriter permits DiskJournalSegmentWriter, } catch (KryoException e) { if (writeLimit != maxEntrySize) { // We have not provided enough capacity, signal to roll to next segment - throw new BufferOverflowException(); + LOG.trace("Tail serialization with {} bytes available failed", writeLimit, e); + return null; } // Just reset the buffer. There's no need to zero the bytes since we haven't written the length or checksum. - throw new StorageException.TooLarge("Entry size exceeds maximum allowed bytes (" + maxEntrySize + ")"); + throw new StorageException.TooLarge("Entry size exceeds maximum allowed bytes (" + maxEntrySize + ")", e); } final int length = diskEntry.position() - HEADER_BYTES; diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalWriter.java b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalWriter.java index b8acf1e593..1462463e9d 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalWriter.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalWriter.java @@ -15,64 +15,66 @@ */ package io.atomix.storage.journal; +import org.eclipse.jdt.annotation.NonNull; + /** * Log writer. * * @author Jordan Halterman */ public interface JournalWriter { - /** - * Returns the last written index. - * - * @return The last written index. - */ - long getLastIndex(); + /** + * Returns the last written index. + * + * @return The last written index. + */ + long getLastIndex(); - /** - * Returns the last entry written. - * - * @return The last entry written. - */ - Indexed getLastEntry(); + /** + * Returns the last entry written. + * + * @return The last entry written. + */ + Indexed getLastEntry(); - /** - * Returns the next index to be written. - * - * @return The next index to be written. - */ - long getNextIndex(); + /** + * Returns the next index to be written. + * + * @return The next index to be written. + */ + long getNextIndex(); - /** - * Appends an entry to the journal. - * - * @param entry The entry to append. - * @return The appended indexed entry. - */ - Indexed append(T entry); + /** + * Appends an entry to the journal. + * + * @param entry The entry to append. + * @return The appended indexed entry. + */ + @NonNull Indexed append(T entry); - /** - * Commits entries up to the given index. - * - * @param index The index up to which to commit entries. - */ - void commit(long index); + /** + * Commits entries up to the given index. + * + * @param index The index up to which to commit entries. + */ + void commit(long index); - /** - * Resets the head of the journal to the given index. - * - * @param index the index to which to reset the head of the journal - */ - void reset(long index); + /** + * Resets the head of the journal to the given index. + * + * @param index the index to which to reset the head of the journal + */ + void reset(long index); - /** - * Truncates the log to the given index. - * - * @param index The index to which to truncate the log. - */ - void truncate(long index); + /** + * Truncates the log to the given index. + * + * @param index The index to which to truncate the log. + */ + void truncate(long index); - /** - * Flushes written entries to disk. - */ - void flush(); + /** + * Flushes written entries to disk. + */ + void flush(); } diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalWriter.java b/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalWriter.java index 389c06aa4e..a95622e553 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalWriter.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalWriter.java @@ -15,7 +15,7 @@ */ package io.atomix.storage.journal; -import java.nio.BufferOverflowException; +import static com.google.common.base.Verify.verifyNotNull; /** * Raft log writer. @@ -70,19 +70,17 @@ final class SegmentedJournalWriter implements JournalWriter { @Override public Indexed append(T entry) { - try { - return currentWriter.append(entry); - } catch (BufferOverflowException e) { - if (currentSegment.firstIndex() == currentWriter.getNextIndex()) { - throw e; - } + var indexed = currentWriter.append(entry); + if (indexed != null) { + return indexed; } + // Slow path: we do not have enough capacity currentWriter.flush(); currentSegment.releaseWriter(); currentSegment = journal.getNextSegment(); currentWriter = currentSegment.acquireWriter(); - return currentWriter.append(entry); + return verifyNotNull(currentWriter.append(entry)); } @Override diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/StorageException.java b/atomix-storage/src/main/java/io/atomix/storage/journal/StorageException.java index e5dd9a2073..0a220ec652 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/StorageException.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/StorageException.java @@ -50,6 +50,10 @@ public class StorageException extends RuntimeException { public TooLarge(final String message) { super(message); } + + public TooLarge(final String message, final Throwable cause) { + super(message, cause); + } } /** -- 2.36.6 From 5efa65cd92e63c9eba8fdfd6b935463c314b4ad7 Mon Sep 17 00:00:00 2001 From: Robert Varga Date: Wed, 27 Mar 2024 05:09:55 +0100 Subject: [PATCH 02/16] Defer up to maxEntrySize flushes by default Batching flushes has performance benefits in terms of throughput. Keeping the flush size at maxEntrySize makes the flush times more consistent, as the journal's syncs will be amortized to fit writing maxEntrySize'd entries. JIRA: CONTROLLER-2108 Change-Id: Ie385738f65d9503fdeeed6a9e0b5ced37fde7fd3 Signed-off-by: Robert Varga --- .../akka/segjournal/SegmentedFileJournal.java | 3 +-- .../src/test/resources/SegmentedFileJournalTest.conf | 1 - .../src/main/resources/initial/factory-akka.conf | 10 ++++++---- .../src/test/resources/segmented.conf | 1 - 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/opendaylight/md-sal/sal-akka-segmented-journal/src/main/java/org/opendaylight/controller/akka/segjournal/SegmentedFileJournal.java b/opendaylight/md-sal/sal-akka-segmented-journal/src/main/java/org/opendaylight/controller/akka/segjournal/SegmentedFileJournal.java index 41a6add0ed..b9320998c9 100644 --- a/opendaylight/md-sal/sal-akka-segmented-journal/src/main/java/org/opendaylight/controller/akka/segjournal/SegmentedFileJournal.java +++ b/opendaylight/md-sal/sal-akka-segmented-journal/src/main/java/org/opendaylight/controller/akka/segjournal/SegmentedFileJournal.java @@ -45,7 +45,6 @@ public class SegmentedFileJournal extends AsyncWriteJournal { public static final String STORAGE_MAX_SEGMENT_SIZE = "max-segment-size"; public static final int STORAGE_MAX_SEGMENT_SIZE_DEFAULT = STORAGE_MAX_ENTRY_SIZE_DEFAULT * 8; public static final String STORAGE_MAX_UNFLUSHED_BYTES = "max-unflushed-bytes"; - public static final int STORAGE_MAX_UNFLUSHED_BYTES_DEFAULT = 0; public static final String STORAGE_MEMORY_MAPPED = "memory-mapped"; private static final Logger LOG = LoggerFactory.getLogger(SegmentedFileJournal.class); @@ -67,7 +66,7 @@ public class SegmentedFileJournal extends AsyncWriteJournal { maxEntrySize = getBytes(config, STORAGE_MAX_ENTRY_SIZE, STORAGE_MAX_ENTRY_SIZE_DEFAULT); maxSegmentSize = getBytes(config, STORAGE_MAX_SEGMENT_SIZE, STORAGE_MAX_SEGMENT_SIZE_DEFAULT); - maxUnflushedBytes = getBytes(config, STORAGE_MAX_UNFLUSHED_BYTES, STORAGE_MAX_UNFLUSHED_BYTES_DEFAULT); + maxUnflushedBytes = getBytes(config, STORAGE_MAX_UNFLUSHED_BYTES, maxEntrySize); if (config.hasPath(STORAGE_MEMORY_MAPPED)) { storage = config.getBoolean(STORAGE_MEMORY_MAPPED) ? StorageLevel.MAPPED : StorageLevel.DISK; diff --git a/opendaylight/md-sal/sal-akka-segmented-journal/src/test/resources/SegmentedFileJournalTest.conf b/opendaylight/md-sal/sal-akka-segmented-journal/src/test/resources/SegmentedFileJournalTest.conf index e391a0dd5d..8fce93b097 100644 --- a/opendaylight/md-sal/sal-akka-segmented-journal/src/test/resources/SegmentedFileJournalTest.conf +++ b/opendaylight/md-sal/sal-akka-segmented-journal/src/test/resources/SegmentedFileJournalTest.conf @@ -8,7 +8,6 @@ akka { root-directory = "target/segmented-journal" max-entry-size = 8M max-segment-size = 32M - max-unflushed-bytes = 256K memory-mapped = false } } diff --git a/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/factory-akka.conf b/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/factory-akka.conf index 00be0551f6..9834e08ea8 100644 --- a/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/factory-akka.conf +++ b/opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/factory-akka.conf @@ -163,8 +163,9 @@ odl-cluster-data { max-entry-size = 16M # Maximum size of a segment max-segment-size = 128M - # Maximum number of bytes that are written without synchronizing storage - max-unflushed-bytes = 1M + # Maximum number of bytes that are written without synchronizing storage. Defaults to max-entry-size. + # Set to <= 0 to flush immediately. + #max-unflushed-bytes = 1M # Map each segment into memory. Defaults to true, use false to keep a heap-based # buffer instead. memory-mapped = true @@ -183,8 +184,9 @@ odl-cluster-data { max-entry-size = 512K # Maximum size of a segment max-segment-size = 1M - # Maximum number of bytes that are written without synchronizing storage - max-unflushed-bytes = 128K + # Maximum number of bytes that are written without synchronizing storage. Defaults to max-entry-size. + # Set to <= 0 to flush immediately. + #max-unflushed-bytes = 128K # Map each segment into memory. Note that while this can improve performance, # it will also place additional burden on system resources. memory-mapped = false diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/resources/segmented.conf b/opendaylight/md-sal/sal-distributed-datastore/src/test/resources/segmented.conf index a21cc3e178..7a0b83212f 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/test/resources/segmented.conf +++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/resources/segmented.conf @@ -28,7 +28,6 @@ Member1 { root-directory = "target/segmented-journal" max-entry-size = 8M max-segment-size = 32M - max-unflushed-bytes = 256K memory-mapped = false } } -- 2.36.6 From 775c2e4fe9d30ad7bc23cf61bfde8d4fab98dcfc Mon Sep 17 00:00:00 2001 From: jenkins-releng Date: Wed, 27 Mar 2024 10:37:57 +0000 Subject: [PATCH 03/16] Release controller --- akka/pom.xml | 2 +- akka/repackaged-akka-jar/pom.xml | 2 +- akka/repackaged-akka/pom.xml | 2 +- artifacts/pom.xml | 2 +- atomix-storage/pom.xml | 2 +- benchmark/api/pom.xml | 2 +- benchmark/dsbenchmark/pom.xml | 2 +- benchmark/ntfbenchmark/pom.xml | 2 +- benchmark/pom.xml | 2 +- benchmark/rpcbenchmark/pom.xml | 2 +- bundle-parent/pom.xml | 4 ++-- docs/pom.xml | 2 +- features/features-controller-experimental/pom.xml | 2 +- features/features-controller-testing/pom.xml | 2 +- features/features-controller/pom.xml | 2 +- features/odl-clustering-test-app/pom.xml | 2 +- features/odl-controller-akka/pom.xml | 2 +- features/odl-controller-blueprint/pom.xml | 2 +- features/odl-controller-broker-local/pom.xml | 2 +- features/odl-controller-mdsal-common/pom.xml | 2 +- features/odl-controller-scala/pom.xml | 2 +- features/odl-jolokia/pom.xml | 2 +- features/odl-mdsal-benchmark/pom.xml | 2 +- features/odl-mdsal-broker/pom.xml | 2 +- features/odl-mdsal-clustering-commons/pom.xml | 2 +- features/odl-mdsal-distributed-datastore/pom.xml | 2 +- features/odl-mdsal-remoterpc-connector/pom.xml | 2 +- features/odl-toaster/pom.xml | 2 +- features/pom.xml | 2 +- features/single-feature-parent/pom.xml | 4 ++-- jolokia/pom.xml | 2 +- karaf/pom.xml | 2 +- opendaylight/blueprint/pom.xml | 2 +- opendaylight/md-sal/cds-access-api/pom.xml | 2 +- opendaylight/md-sal/cds-access-client/pom.xml | 2 +- opendaylight/md-sal/cds-dom-api/pom.xml | 2 +- opendaylight/md-sal/cds-mgmt-api/pom.xml | 2 +- opendaylight/md-sal/eos-dom-akka/pom.xml | 2 +- opendaylight/md-sal/mdsal-it-base/pom.xml | 2 +- opendaylight/md-sal/mdsal-it-parent/pom.xml | 4 ++-- opendaylight/md-sal/parent/pom.xml | 4 ++-- opendaylight/md-sal/pom.xml | 2 +- opendaylight/md-sal/sal-akka-raft-example/pom.xml | 2 +- opendaylight/md-sal/sal-akka-raft/pom.xml | 2 +- opendaylight/md-sal/sal-akka-segmented-journal/pom.xml | 2 +- opendaylight/md-sal/sal-binding-it/pom.xml | 2 +- opendaylight/md-sal/sal-cluster-admin-api/pom.xml | 2 +- opendaylight/md-sal/sal-cluster-admin-impl/pom.xml | 2 +- opendaylight/md-sal/sal-cluster-admin-karaf-cli/pom.xml | 2 +- opendaylight/md-sal/sal-clustering-commons/pom.xml | 2 +- opendaylight/md-sal/sal-clustering-config/pom.xml | 2 +- opendaylight/md-sal/sal-common-util/pom.xml | 2 +- opendaylight/md-sal/sal-distributed-datastore/pom.xml | 2 +- opendaylight/md-sal/sal-dummy-distributed-datastore/pom.xml | 2 +- opendaylight/md-sal/sal-remoterpc-connector/pom.xml | 2 +- opendaylight/md-sal/sal-test-model/pom.xml | 2 +- .../md-sal/samples/clustering-test-app/configuration/pom.xml | 2 +- .../md-sal/samples/clustering-test-app/karaf-cli/pom.xml | 2 +- opendaylight/md-sal/samples/clustering-test-app/model/pom.xml | 2 +- opendaylight/md-sal/samples/clustering-test-app/pom.xml | 2 +- .../md-sal/samples/clustering-test-app/provider/pom.xml | 2 +- opendaylight/md-sal/samples/pom.xml | 2 +- opendaylight/md-sal/samples/toaster-consumer/pom.xml | 2 +- opendaylight/md-sal/samples/toaster-it/pom.xml | 2 +- opendaylight/md-sal/samples/toaster-provider/pom.xml | 2 +- opendaylight/md-sal/samples/toaster/pom.xml | 2 +- pom.xml | 2 +- 67 files changed, 71 insertions(+), 71 deletions(-) diff --git a/akka/pom.xml b/akka/pom.xml index d32e4f035a..ba744c2dab 100644 --- a/akka/pom.xml +++ b/akka/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller akka-aggregator - 9.0.2-SNAPSHOT + 9.0.2 pom diff --git a/akka/repackaged-akka-jar/pom.xml b/akka/repackaged-akka-jar/pom.xml index 9204b4ed84..8f8a218f80 100644 --- a/akka/repackaged-akka-jar/pom.xml +++ b/akka/repackaged-akka-jar/pom.xml @@ -20,7 +20,7 @@ org.opendaylight.controller repackaged-akka-jar jar - 9.0.2-SNAPSHOT + 9.0.2 ${project.artifactId} diff --git a/akka/repackaged-akka/pom.xml b/akka/repackaged-akka/pom.xml index 2ac52cfe53..a0eafa2d3d 100644 --- a/akka/repackaged-akka/pom.xml +++ b/akka/repackaged-akka/pom.xml @@ -13,7 +13,7 @@ org.opendaylight.controller bundle-parent - 9.0.2-SNAPSHOT + 9.0.2 ../../bundle-parent diff --git a/artifacts/pom.xml b/artifacts/pom.xml index cfc8e78ad3..dad0864304 100644 --- a/artifacts/pom.xml +++ b/artifacts/pom.xml @@ -20,7 +20,7 @@ org.opendaylight.controller controller-artifacts - 9.0.2-SNAPSHOT + 9.0.2 pom diff --git a/atomix-storage/pom.xml b/atomix-storage/pom.xml index ef8c475223..9f9fc98896 100644 --- a/atomix-storage/pom.xml +++ b/atomix-storage/pom.xml @@ -20,7 +20,7 @@ org.opendaylight.controller bundle-parent - 9.0.2-SNAPSHOT + 9.0.2 ../bundle-parent diff --git a/benchmark/api/pom.xml b/benchmark/api/pom.xml index ff092f1388..70c6eed9e1 100644 --- a/benchmark/api/pom.xml +++ b/benchmark/api/pom.xml @@ -11,7 +11,7 @@ and is available at http://www.eclipse.org/legal/epl-v10.html org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../../opendaylight/md-sal/parent diff --git a/benchmark/dsbenchmark/pom.xml b/benchmark/dsbenchmark/pom.xml index 353e9a65b2..cf9bb974b0 100644 --- a/benchmark/dsbenchmark/pom.xml +++ b/benchmark/dsbenchmark/pom.xml @@ -12,7 +12,7 @@ and is available at http://www.eclipse.org/legal/epl-v10.html org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../../opendaylight/md-sal/parent diff --git a/benchmark/ntfbenchmark/pom.xml b/benchmark/ntfbenchmark/pom.xml index ca7c4ba3f6..442966852a 100644 --- a/benchmark/ntfbenchmark/pom.xml +++ b/benchmark/ntfbenchmark/pom.xml @@ -12,7 +12,7 @@ and is available at http://www.eclipse.org/legal/epl-v10.html org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../../opendaylight/md-sal/parent diff --git a/benchmark/pom.xml b/benchmark/pom.xml index 51f562b856..9a5d703c99 100644 --- a/benchmark/pom.xml +++ b/benchmark/pom.xml @@ -16,7 +16,7 @@ and is available at http://www.eclipse.org/legal/epl-v10.html org.opendaylight.controller benchmark-aggregator - 9.0.2-SNAPSHOT + 9.0.2 pom diff --git a/benchmark/rpcbenchmark/pom.xml b/benchmark/rpcbenchmark/pom.xml index fa2e5c10cb..2385b96627 100644 --- a/benchmark/rpcbenchmark/pom.xml +++ b/benchmark/rpcbenchmark/pom.xml @@ -12,7 +12,7 @@ and is available at http://www.eclipse.org/legal/epl-v10.html org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../../opendaylight/md-sal/parent diff --git a/bundle-parent/pom.xml b/bundle-parent/pom.xml index d76fe9a2df..b20f4ff2d0 100644 --- a/bundle-parent/pom.xml +++ b/bundle-parent/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller bundle-parent - 9.0.2-SNAPSHOT + 9.0.2 pom @@ -25,7 +25,7 @@ org.opendaylight.controller controller-artifacts - 9.0.2-SNAPSHOT + 9.0.2 pom import diff --git a/docs/pom.xml b/docs/pom.xml index 6b7828da71..3b9826234f 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -19,7 +19,7 @@ org.opendaylight.controller controller-docs jar - 9.0.2-SNAPSHOT + 9.0.2 ${project.artifactId} Controller documentation diff --git a/features/features-controller-experimental/pom.xml b/features/features-controller-experimental/pom.xml index ee7549f168..fc43757dc8 100644 --- a/features/features-controller-experimental/pom.xml +++ b/features/features-controller-experimental/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller features-controller-experimental - 9.0.2-SNAPSHOT + 9.0.2 feature OpenDaylight :: Controller :: Experimental Features Controller Experimental Features diff --git a/features/features-controller-testing/pom.xml b/features/features-controller-testing/pom.xml index 6f0673a094..787568d5ca 100644 --- a/features/features-controller-testing/pom.xml +++ b/features/features-controller-testing/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller features-controller-testing - 9.0.2-SNAPSHOT + 9.0.2 feature OpenDaylight :: Controller :: Features to support CSIT testing Controller CSIT Features diff --git a/features/features-controller/pom.xml b/features/features-controller/pom.xml index 05ffc20fef..20dde35670 100644 --- a/features/features-controller/pom.xml +++ b/features/features-controller/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller features-controller - 9.0.2-SNAPSHOT + 9.0.2 feature OpenDaylight :: Controller :: Features Controller Production Features diff --git a/features/odl-clustering-test-app/pom.xml b/features/odl-clustering-test-app/pom.xml index ac9f486d09..ffc0ebb176 100644 --- a/features/odl-clustering-test-app/pom.xml +++ b/features/odl-clustering-test-app/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2-SNAPSHOT + 9.0.2 ../single-feature-parent diff --git a/features/odl-controller-akka/pom.xml b/features/odl-controller-akka/pom.xml index 7c7c1b64db..15591e120d 100644 --- a/features/odl-controller-akka/pom.xml +++ b/features/odl-controller-akka/pom.xml @@ -14,7 +14,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2-SNAPSHOT + 9.0.2 ../single-feature-parent diff --git a/features/odl-controller-blueprint/pom.xml b/features/odl-controller-blueprint/pom.xml index 1b3114bbeb..7b964de5e4 100644 --- a/features/odl-controller-blueprint/pom.xml +++ b/features/odl-controller-blueprint/pom.xml @@ -12,7 +12,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2-SNAPSHOT + 9.0.2 ../single-feature-parent diff --git a/features/odl-controller-broker-local/pom.xml b/features/odl-controller-broker-local/pom.xml index 8dd49ae86b..667c436692 100644 --- a/features/odl-controller-broker-local/pom.xml +++ b/features/odl-controller-broker-local/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2-SNAPSHOT + 9.0.2 ../single-feature-parent diff --git a/features/odl-controller-mdsal-common/pom.xml b/features/odl-controller-mdsal-common/pom.xml index 659cf80a01..b06739a2ba 100644 --- a/features/odl-controller-mdsal-common/pom.xml +++ b/features/odl-controller-mdsal-common/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2-SNAPSHOT + 9.0.2 ../single-feature-parent diff --git a/features/odl-controller-scala/pom.xml b/features/odl-controller-scala/pom.xml index 7191939adc..4093670604 100644 --- a/features/odl-controller-scala/pom.xml +++ b/features/odl-controller-scala/pom.xml @@ -14,7 +14,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2-SNAPSHOT + 9.0.2 ../single-feature-parent diff --git a/features/odl-jolokia/pom.xml b/features/odl-jolokia/pom.xml index 296e5ee9d0..a3f4789a6a 100644 --- a/features/odl-jolokia/pom.xml +++ b/features/odl-jolokia/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2-SNAPSHOT + 9.0.2 ../single-feature-parent diff --git a/features/odl-mdsal-benchmark/pom.xml b/features/odl-mdsal-benchmark/pom.xml index 9362051945..fb58a3b5fe 100644 --- a/features/odl-mdsal-benchmark/pom.xml +++ b/features/odl-mdsal-benchmark/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2-SNAPSHOT + 9.0.2 ../single-feature-parent diff --git a/features/odl-mdsal-broker/pom.xml b/features/odl-mdsal-broker/pom.xml index 53e3ef3c36..b279a98b8f 100644 --- a/features/odl-mdsal-broker/pom.xml +++ b/features/odl-mdsal-broker/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2-SNAPSHOT + 9.0.2 ../single-feature-parent diff --git a/features/odl-mdsal-clustering-commons/pom.xml b/features/odl-mdsal-clustering-commons/pom.xml index 16120918d5..7152fbdddb 100644 --- a/features/odl-mdsal-clustering-commons/pom.xml +++ b/features/odl-mdsal-clustering-commons/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2-SNAPSHOT + 9.0.2 ../single-feature-parent diff --git a/features/odl-mdsal-distributed-datastore/pom.xml b/features/odl-mdsal-distributed-datastore/pom.xml index 14b9622c78..59c1b5b838 100644 --- a/features/odl-mdsal-distributed-datastore/pom.xml +++ b/features/odl-mdsal-distributed-datastore/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2-SNAPSHOT + 9.0.2 ../single-feature-parent diff --git a/features/odl-mdsal-remoterpc-connector/pom.xml b/features/odl-mdsal-remoterpc-connector/pom.xml index 3c1033d4fa..312830da7a 100644 --- a/features/odl-mdsal-remoterpc-connector/pom.xml +++ b/features/odl-mdsal-remoterpc-connector/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2-SNAPSHOT + 9.0.2 ../single-feature-parent diff --git a/features/odl-toaster/pom.xml b/features/odl-toaster/pom.xml index e19baa3ced..18e7f79a1a 100644 --- a/features/odl-toaster/pom.xml +++ b/features/odl-toaster/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2-SNAPSHOT + 9.0.2 ../single-feature-parent diff --git a/features/pom.xml b/features/pom.xml index a4d1590b3e..8078cee14f 100644 --- a/features/pom.xml +++ b/features/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller features-aggregator - 9.0.2-SNAPSHOT + 9.0.2 pom diff --git a/features/single-feature-parent/pom.xml b/features/single-feature-parent/pom.xml index e4f636aff1..5e441c0f45 100644 --- a/features/single-feature-parent/pom.xml +++ b/features/single-feature-parent/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2-SNAPSHOT + 9.0.2 pom @@ -25,7 +25,7 @@ org.opendaylight.controller bundle-parent - 9.0.2-SNAPSHOT + 9.0.2 pom import diff --git a/jolokia/pom.xml b/jolokia/pom.xml index 23351a6593..5f214047f2 100644 --- a/jolokia/pom.xml +++ b/jolokia/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller odl-jolokia-osgi - 9.0.2-SNAPSHOT + 9.0.2 jar diff --git a/karaf/pom.xml b/karaf/pom.xml index a3b5f66b69..dbf97e0595 100644 --- a/karaf/pom.xml +++ b/karaf/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller controller-test-karaf - 9.0.2-SNAPSHOT + 9.0.2 pom diff --git a/opendaylight/blueprint/pom.xml b/opendaylight/blueprint/pom.xml index 0921465d3b..a1a3077947 100644 --- a/opendaylight/blueprint/pom.xml +++ b/opendaylight/blueprint/pom.xml @@ -20,7 +20,7 @@ blueprint bundle ${project.artifactId} - 9.0.2-SNAPSHOT + 9.0.2 diff --git a/opendaylight/md-sal/cds-access-api/pom.xml b/opendaylight/md-sal/cds-access-api/pom.xml index 15b61598b6..4e92431326 100644 --- a/opendaylight/md-sal/cds-access-api/pom.xml +++ b/opendaylight/md-sal/cds-access-api/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../parent diff --git a/opendaylight/md-sal/cds-access-client/pom.xml b/opendaylight/md-sal/cds-access-client/pom.xml index 6ca2c69cdb..8186d5ac03 100644 --- a/opendaylight/md-sal/cds-access-client/pom.xml +++ b/opendaylight/md-sal/cds-access-client/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../parent diff --git a/opendaylight/md-sal/cds-dom-api/pom.xml b/opendaylight/md-sal/cds-dom-api/pom.xml index 05d5f8a175..ab7eb9eec2 100644 --- a/opendaylight/md-sal/cds-dom-api/pom.xml +++ b/opendaylight/md-sal/cds-dom-api/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../parent diff --git a/opendaylight/md-sal/cds-mgmt-api/pom.xml b/opendaylight/md-sal/cds-mgmt-api/pom.xml index 14958b9368..76ec086465 100644 --- a/opendaylight/md-sal/cds-mgmt-api/pom.xml +++ b/opendaylight/md-sal/cds-mgmt-api/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../parent diff --git a/opendaylight/md-sal/eos-dom-akka/pom.xml b/opendaylight/md-sal/eos-dom-akka/pom.xml index 213b179271..2124643f7e 100644 --- a/opendaylight/md-sal/eos-dom-akka/pom.xml +++ b/opendaylight/md-sal/eos-dom-akka/pom.xml @@ -14,7 +14,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../parent diff --git a/opendaylight/md-sal/mdsal-it-base/pom.xml b/opendaylight/md-sal/mdsal-it-base/pom.xml index fa3976239e..60d5307623 100644 --- a/opendaylight/md-sal/mdsal-it-base/pom.xml +++ b/opendaylight/md-sal/mdsal-it-base/pom.xml @@ -18,7 +18,7 @@ and is available at http://www.eclipse.org/legal/epl-v10.html org.opendaylight.controller mdsal-it-base - 9.0.2-SNAPSHOT + 9.0.2 bundle diff --git a/opendaylight/md-sal/mdsal-it-parent/pom.xml b/opendaylight/md-sal/mdsal-it-parent/pom.xml index 548fea9bdb..5c6498d804 100644 --- a/opendaylight/md-sal/mdsal-it-parent/pom.xml +++ b/opendaylight/md-sal/mdsal-it-parent/pom.xml @@ -19,7 +19,7 @@ and is available at http://www.eclipse.org/legal/epl-v10.html org.opendaylight.controller mdsal-it-parent - 9.0.2-SNAPSHOT + 9.0.2 pom @@ -37,7 +37,7 @@ and is available at http://www.eclipse.org/legal/epl-v10.html org.opendaylight.controller controller-artifacts - 9.0.2-SNAPSHOT + 9.0.2 pom import diff --git a/opendaylight/md-sal/parent/pom.xml b/opendaylight/md-sal/parent/pom.xml index daf0fc982f..483b0d2b16 100644 --- a/opendaylight/md-sal/parent/pom.xml +++ b/opendaylight/md-sal/parent/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 pom @@ -25,7 +25,7 @@ org.opendaylight.controller bundle-parent - 9.0.2-SNAPSHOT + 9.0.2 pom import diff --git a/opendaylight/md-sal/pom.xml b/opendaylight/md-sal/pom.xml index 37b0c08ba5..295065c91b 100644 --- a/opendaylight/md-sal/pom.xml +++ b/opendaylight/md-sal/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller mdsal-aggregator - 9.0.2-SNAPSHOT + 9.0.2 pom diff --git a/opendaylight/md-sal/sal-akka-raft-example/pom.xml b/opendaylight/md-sal/sal-akka-raft-example/pom.xml index 3ca9784f02..1d4dbf09b7 100644 --- a/opendaylight/md-sal/sal-akka-raft-example/pom.xml +++ b/opendaylight/md-sal/sal-akka-raft-example/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../parent diff --git a/opendaylight/md-sal/sal-akka-raft/pom.xml b/opendaylight/md-sal/sal-akka-raft/pom.xml index f6a88435b9..0d9eeffbe9 100644 --- a/opendaylight/md-sal/sal-akka-raft/pom.xml +++ b/opendaylight/md-sal/sal-akka-raft/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../parent diff --git a/opendaylight/md-sal/sal-akka-segmented-journal/pom.xml b/opendaylight/md-sal/sal-akka-segmented-journal/pom.xml index bc3ae13c1c..84e826d7ba 100644 --- a/opendaylight/md-sal/sal-akka-segmented-journal/pom.xml +++ b/opendaylight/md-sal/sal-akka-segmented-journal/pom.xml @@ -12,7 +12,7 @@ and is available at http://www.eclipse.org/legal/epl-v10.html org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../parent diff --git a/opendaylight/md-sal/sal-binding-it/pom.xml b/opendaylight/md-sal/sal-binding-it/pom.xml index de90ea3b95..75e8bf1dbc 100644 --- a/opendaylight/md-sal/sal-binding-it/pom.xml +++ b/opendaylight/md-sal/sal-binding-it/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-it-parent - 9.0.2-SNAPSHOT + 9.0.2 ../mdsal-it-parent sal-binding-it diff --git a/opendaylight/md-sal/sal-cluster-admin-api/pom.xml b/opendaylight/md-sal/sal-cluster-admin-api/pom.xml index 7a01f80feb..c785004ce3 100644 --- a/opendaylight/md-sal/sal-cluster-admin-api/pom.xml +++ b/opendaylight/md-sal/sal-cluster-admin-api/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../parent diff --git a/opendaylight/md-sal/sal-cluster-admin-impl/pom.xml b/opendaylight/md-sal/sal-cluster-admin-impl/pom.xml index 0dcf88e7f8..684fc0079d 100644 --- a/opendaylight/md-sal/sal-cluster-admin-impl/pom.xml +++ b/opendaylight/md-sal/sal-cluster-admin-impl/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../parent diff --git a/opendaylight/md-sal/sal-cluster-admin-karaf-cli/pom.xml b/opendaylight/md-sal/sal-cluster-admin-karaf-cli/pom.xml index f3482edbd3..1550067264 100644 --- a/opendaylight/md-sal/sal-cluster-admin-karaf-cli/pom.xml +++ b/opendaylight/md-sal/sal-cluster-admin-karaf-cli/pom.xml @@ -5,7 +5,7 @@ mdsal-parent org.opendaylight.controller - 9.0.2-SNAPSHOT + 9.0.2 ../parent/pom.xml 4.0.0 diff --git a/opendaylight/md-sal/sal-clustering-commons/pom.xml b/opendaylight/md-sal/sal-clustering-commons/pom.xml index cb1a89af81..50448fc0f4 100644 --- a/opendaylight/md-sal/sal-clustering-commons/pom.xml +++ b/opendaylight/md-sal/sal-clustering-commons/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../parent diff --git a/opendaylight/md-sal/sal-clustering-config/pom.xml b/opendaylight/md-sal/sal-clustering-config/pom.xml index 53cf8ddd43..23f735fefb 100644 --- a/opendaylight/md-sal/sal-clustering-config/pom.xml +++ b/opendaylight/md-sal/sal-clustering-config/pom.xml @@ -18,7 +18,7 @@ org.opendaylight.controller sal-clustering-config - 9.0.2-SNAPSHOT + 9.0.2 jar Configuration files for md-sal clustering diff --git a/opendaylight/md-sal/sal-common-util/pom.xml b/opendaylight/md-sal/sal-common-util/pom.xml index 398c5b3e17..3e49aa652a 100644 --- a/opendaylight/md-sal/sal-common-util/pom.xml +++ b/opendaylight/md-sal/sal-common-util/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../parent diff --git a/opendaylight/md-sal/sal-distributed-datastore/pom.xml b/opendaylight/md-sal/sal-distributed-datastore/pom.xml index 082f5041f7..948ef79df8 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/pom.xml +++ b/opendaylight/md-sal/sal-distributed-datastore/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../parent diff --git a/opendaylight/md-sal/sal-dummy-distributed-datastore/pom.xml b/opendaylight/md-sal/sal-dummy-distributed-datastore/pom.xml index 0ddae4a0f2..0aae26c23a 100644 --- a/opendaylight/md-sal/sal-dummy-distributed-datastore/pom.xml +++ b/opendaylight/md-sal/sal-dummy-distributed-datastore/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../parent diff --git a/opendaylight/md-sal/sal-remoterpc-connector/pom.xml b/opendaylight/md-sal/sal-remoterpc-connector/pom.xml index f7be176b0a..6e1823bf3b 100644 --- a/opendaylight/md-sal/sal-remoterpc-connector/pom.xml +++ b/opendaylight/md-sal/sal-remoterpc-connector/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../parent diff --git a/opendaylight/md-sal/sal-test-model/pom.xml b/opendaylight/md-sal/sal-test-model/pom.xml index 61448bb025..aa037c80b5 100644 --- a/opendaylight/md-sal/sal-test-model/pom.xml +++ b/opendaylight/md-sal/sal-test-model/pom.xml @@ -6,7 +6,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../parent diff --git a/opendaylight/md-sal/samples/clustering-test-app/configuration/pom.xml b/opendaylight/md-sal/samples/clustering-test-app/configuration/pom.xml index 1bcbccf2e9..450dbf5bfb 100644 --- a/opendaylight/md-sal/samples/clustering-test-app/configuration/pom.xml +++ b/opendaylight/md-sal/samples/clustering-test-app/configuration/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller.samples clustering-it-config - 9.0.2-SNAPSHOT + 9.0.2 jar diff --git a/opendaylight/md-sal/samples/clustering-test-app/karaf-cli/pom.xml b/opendaylight/md-sal/samples/clustering-test-app/karaf-cli/pom.xml index d69fe53059..94ac773c6b 100644 --- a/opendaylight/md-sal/samples/clustering-test-app/karaf-cli/pom.xml +++ b/opendaylight/md-sal/samples/clustering-test-app/karaf-cli/pom.xml @@ -12,7 +12,7 @@ mdsal-parent org.opendaylight.controller - 9.0.2-SNAPSHOT + 9.0.2 ../../../parent/pom.xml 4.0.0 diff --git a/opendaylight/md-sal/samples/clustering-test-app/model/pom.xml b/opendaylight/md-sal/samples/clustering-test-app/model/pom.xml index 8181a38774..88c689aa68 100644 --- a/opendaylight/md-sal/samples/clustering-test-app/model/pom.xml +++ b/opendaylight/md-sal/samples/clustering-test-app/model/pom.xml @@ -5,7 +5,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../../../parent diff --git a/opendaylight/md-sal/samples/clustering-test-app/pom.xml b/opendaylight/md-sal/samples/clustering-test-app/pom.xml index 30c33b0910..3416a9e7d6 100644 --- a/opendaylight/md-sal/samples/clustering-test-app/pom.xml +++ b/opendaylight/md-sal/samples/clustering-test-app/pom.xml @@ -10,7 +10,7 @@ org.opendaylight.controller.samples clustering-test-app - 9.0.2-SNAPSHOT + 9.0.2 pom diff --git a/opendaylight/md-sal/samples/clustering-test-app/provider/pom.xml b/opendaylight/md-sal/samples/clustering-test-app/provider/pom.xml index 031d7cd0b6..6d95e8b8e1 100644 --- a/opendaylight/md-sal/samples/clustering-test-app/provider/pom.xml +++ b/opendaylight/md-sal/samples/clustering-test-app/provider/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../../../parent diff --git a/opendaylight/md-sal/samples/pom.xml b/opendaylight/md-sal/samples/pom.xml index 1411bb79d7..447a4f6cbb 100644 --- a/opendaylight/md-sal/samples/pom.xml +++ b/opendaylight/md-sal/samples/pom.xml @@ -10,7 +10,7 @@ org.opendaylight.controller.samples samples-aggregator - 9.0.2-SNAPSHOT + 9.0.2 pom diff --git a/opendaylight/md-sal/samples/toaster-consumer/pom.xml b/opendaylight/md-sal/samples/toaster-consumer/pom.xml index 5064523bf7..04074b7dfa 100644 --- a/opendaylight/md-sal/samples/toaster-consumer/pom.xml +++ b/opendaylight/md-sal/samples/toaster-consumer/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../../parent diff --git a/opendaylight/md-sal/samples/toaster-it/pom.xml b/opendaylight/md-sal/samples/toaster-it/pom.xml index a23aa81dea..6e324907ef 100644 --- a/opendaylight/md-sal/samples/toaster-it/pom.xml +++ b/opendaylight/md-sal/samples/toaster-it/pom.xml @@ -5,7 +5,7 @@ org.opendaylight.controller mdsal-it-parent - 9.0.2-SNAPSHOT + 9.0.2 ../../mdsal-it-parent sample-toaster-it diff --git a/opendaylight/md-sal/samples/toaster-provider/pom.xml b/opendaylight/md-sal/samples/toaster-provider/pom.xml index f00049a629..653ca4bd9a 100644 --- a/opendaylight/md-sal/samples/toaster-provider/pom.xml +++ b/opendaylight/md-sal/samples/toaster-provider/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../../parent diff --git a/opendaylight/md-sal/samples/toaster/pom.xml b/opendaylight/md-sal/samples/toaster/pom.xml index 29fa5f0e8e..08e292bff9 100644 --- a/opendaylight/md-sal/samples/toaster/pom.xml +++ b/opendaylight/md-sal/samples/toaster/pom.xml @@ -5,7 +5,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2-SNAPSHOT + 9.0.2 ../../parent diff --git a/pom.xml b/pom.xml index 60c9f19ad6..63a9be0355 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ org.opendaylight.controller releasepom - 9.0.2-SNAPSHOT + 9.0.2 pom controller -- 2.36.6 From 9aabc6eeae3b35b8ef5cc294378b7cceb29f3c08 Mon Sep 17 00:00:00 2001 From: Robert Varga Date: Wed, 27 Mar 2024 13:19:56 +0100 Subject: [PATCH 04/16] Bump versions to 9.0.3-SNAPSHOT This starts the next development iteration. Change-Id: I4263aa380442f375497222c0c2050e2a585c6adf Signed-off-by: Robert Varga --- akka/pom.xml | 2 +- akka/repackaged-akka-jar/pom.xml | 2 +- akka/repackaged-akka/pom.xml | 2 +- artifacts/pom.xml | 2 +- atomix-storage/pom.xml | 2 +- benchmark/api/pom.xml | 2 +- benchmark/dsbenchmark/pom.xml | 2 +- benchmark/ntfbenchmark/pom.xml | 2 +- benchmark/pom.xml | 2 +- benchmark/rpcbenchmark/pom.xml | 2 +- bundle-parent/pom.xml | 4 ++-- docs/pom.xml | 2 +- features/features-controller-experimental/pom.xml | 2 +- features/features-controller-testing/pom.xml | 2 +- features/features-controller/pom.xml | 2 +- features/odl-clustering-test-app/pom.xml | 2 +- features/odl-controller-akka/pom.xml | 2 +- features/odl-controller-blueprint/pom.xml | 2 +- features/odl-controller-broker-local/pom.xml | 2 +- features/odl-controller-mdsal-common/pom.xml | 2 +- features/odl-controller-scala/pom.xml | 2 +- features/odl-jolokia/pom.xml | 2 +- features/odl-mdsal-benchmark/pom.xml | 2 +- features/odl-mdsal-broker/pom.xml | 2 +- features/odl-mdsal-clustering-commons/pom.xml | 2 +- features/odl-mdsal-distributed-datastore/pom.xml | 2 +- features/odl-mdsal-remoterpc-connector/pom.xml | 2 +- features/odl-toaster/pom.xml | 2 +- features/pom.xml | 2 +- features/single-feature-parent/pom.xml | 4 ++-- jolokia/pom.xml | 2 +- karaf/pom.xml | 2 +- opendaylight/blueprint/pom.xml | 2 +- opendaylight/md-sal/cds-access-api/pom.xml | 2 +- opendaylight/md-sal/cds-access-client/pom.xml | 2 +- opendaylight/md-sal/cds-dom-api/pom.xml | 2 +- opendaylight/md-sal/cds-mgmt-api/pom.xml | 2 +- opendaylight/md-sal/eos-dom-akka/pom.xml | 2 +- opendaylight/md-sal/mdsal-it-base/pom.xml | 2 +- opendaylight/md-sal/mdsal-it-parent/pom.xml | 4 ++-- opendaylight/md-sal/parent/pom.xml | 4 ++-- opendaylight/md-sal/pom.xml | 2 +- opendaylight/md-sal/sal-akka-raft-example/pom.xml | 2 +- opendaylight/md-sal/sal-akka-raft/pom.xml | 2 +- opendaylight/md-sal/sal-akka-segmented-journal/pom.xml | 2 +- opendaylight/md-sal/sal-binding-it/pom.xml | 2 +- opendaylight/md-sal/sal-cluster-admin-api/pom.xml | 2 +- opendaylight/md-sal/sal-cluster-admin-impl/pom.xml | 2 +- opendaylight/md-sal/sal-cluster-admin-karaf-cli/pom.xml | 2 +- opendaylight/md-sal/sal-clustering-commons/pom.xml | 2 +- opendaylight/md-sal/sal-clustering-config/pom.xml | 2 +- opendaylight/md-sal/sal-common-util/pom.xml | 2 +- opendaylight/md-sal/sal-distributed-datastore/pom.xml | 2 +- opendaylight/md-sal/sal-dummy-distributed-datastore/pom.xml | 2 +- opendaylight/md-sal/sal-remoterpc-connector/pom.xml | 2 +- opendaylight/md-sal/sal-test-model/pom.xml | 2 +- .../md-sal/samples/clustering-test-app/configuration/pom.xml | 2 +- .../md-sal/samples/clustering-test-app/karaf-cli/pom.xml | 2 +- opendaylight/md-sal/samples/clustering-test-app/model/pom.xml | 2 +- opendaylight/md-sal/samples/clustering-test-app/pom.xml | 2 +- .../md-sal/samples/clustering-test-app/provider/pom.xml | 2 +- opendaylight/md-sal/samples/pom.xml | 2 +- opendaylight/md-sal/samples/toaster-consumer/pom.xml | 2 +- opendaylight/md-sal/samples/toaster-it/pom.xml | 2 +- opendaylight/md-sal/samples/toaster-provider/pom.xml | 2 +- opendaylight/md-sal/samples/toaster/pom.xml | 2 +- pom.xml | 2 +- 67 files changed, 71 insertions(+), 71 deletions(-) diff --git a/akka/pom.xml b/akka/pom.xml index ba744c2dab..afd11d7410 100644 --- a/akka/pom.xml +++ b/akka/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller akka-aggregator - 9.0.2 + 9.0.3-SNAPSHOT pom diff --git a/akka/repackaged-akka-jar/pom.xml b/akka/repackaged-akka-jar/pom.xml index 8f8a218f80..6c62c5d247 100644 --- a/akka/repackaged-akka-jar/pom.xml +++ b/akka/repackaged-akka-jar/pom.xml @@ -20,7 +20,7 @@ org.opendaylight.controller repackaged-akka-jar jar - 9.0.2 + 9.0.3-SNAPSHOT ${project.artifactId} diff --git a/akka/repackaged-akka/pom.xml b/akka/repackaged-akka/pom.xml index a0eafa2d3d..cc222188b5 100644 --- a/akka/repackaged-akka/pom.xml +++ b/akka/repackaged-akka/pom.xml @@ -13,7 +13,7 @@ org.opendaylight.controller bundle-parent - 9.0.2 + 9.0.3-SNAPSHOT ../../bundle-parent diff --git a/artifacts/pom.xml b/artifacts/pom.xml index dad0864304..79e8d88f08 100644 --- a/artifacts/pom.xml +++ b/artifacts/pom.xml @@ -20,7 +20,7 @@ org.opendaylight.controller controller-artifacts - 9.0.2 + 9.0.3-SNAPSHOT pom diff --git a/atomix-storage/pom.xml b/atomix-storage/pom.xml index 9f9fc98896..886ad6ac50 100644 --- a/atomix-storage/pom.xml +++ b/atomix-storage/pom.xml @@ -20,7 +20,7 @@ org.opendaylight.controller bundle-parent - 9.0.2 + 9.0.3-SNAPSHOT ../bundle-parent diff --git a/benchmark/api/pom.xml b/benchmark/api/pom.xml index 70c6eed9e1..52fde32640 100644 --- a/benchmark/api/pom.xml +++ b/benchmark/api/pom.xml @@ -11,7 +11,7 @@ and is available at http://www.eclipse.org/legal/epl-v10.html org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../../opendaylight/md-sal/parent diff --git a/benchmark/dsbenchmark/pom.xml b/benchmark/dsbenchmark/pom.xml index cf9bb974b0..1595fb8111 100644 --- a/benchmark/dsbenchmark/pom.xml +++ b/benchmark/dsbenchmark/pom.xml @@ -12,7 +12,7 @@ and is available at http://www.eclipse.org/legal/epl-v10.html org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../../opendaylight/md-sal/parent diff --git a/benchmark/ntfbenchmark/pom.xml b/benchmark/ntfbenchmark/pom.xml index 442966852a..82b3c3aabf 100644 --- a/benchmark/ntfbenchmark/pom.xml +++ b/benchmark/ntfbenchmark/pom.xml @@ -12,7 +12,7 @@ and is available at http://www.eclipse.org/legal/epl-v10.html org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../../opendaylight/md-sal/parent diff --git a/benchmark/pom.xml b/benchmark/pom.xml index 9a5d703c99..cf915b7bc2 100644 --- a/benchmark/pom.xml +++ b/benchmark/pom.xml @@ -16,7 +16,7 @@ and is available at http://www.eclipse.org/legal/epl-v10.html org.opendaylight.controller benchmark-aggregator - 9.0.2 + 9.0.3-SNAPSHOT pom diff --git a/benchmark/rpcbenchmark/pom.xml b/benchmark/rpcbenchmark/pom.xml index 2385b96627..8eefdf6664 100644 --- a/benchmark/rpcbenchmark/pom.xml +++ b/benchmark/rpcbenchmark/pom.xml @@ -12,7 +12,7 @@ and is available at http://www.eclipse.org/legal/epl-v10.html org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../../opendaylight/md-sal/parent diff --git a/bundle-parent/pom.xml b/bundle-parent/pom.xml index b20f4ff2d0..5d373d3f37 100644 --- a/bundle-parent/pom.xml +++ b/bundle-parent/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller bundle-parent - 9.0.2 + 9.0.3-SNAPSHOT pom @@ -25,7 +25,7 @@ org.opendaylight.controller controller-artifacts - 9.0.2 + 9.0.3-SNAPSHOT pom import diff --git a/docs/pom.xml b/docs/pom.xml index 3b9826234f..22c11bcb3c 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -19,7 +19,7 @@ org.opendaylight.controller controller-docs jar - 9.0.2 + 9.0.3-SNAPSHOT ${project.artifactId} Controller documentation diff --git a/features/features-controller-experimental/pom.xml b/features/features-controller-experimental/pom.xml index fc43757dc8..6401adb5fe 100644 --- a/features/features-controller-experimental/pom.xml +++ b/features/features-controller-experimental/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller features-controller-experimental - 9.0.2 + 9.0.3-SNAPSHOT feature OpenDaylight :: Controller :: Experimental Features Controller Experimental Features diff --git a/features/features-controller-testing/pom.xml b/features/features-controller-testing/pom.xml index 787568d5ca..fb095fe8a1 100644 --- a/features/features-controller-testing/pom.xml +++ b/features/features-controller-testing/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller features-controller-testing - 9.0.2 + 9.0.3-SNAPSHOT feature OpenDaylight :: Controller :: Features to support CSIT testing Controller CSIT Features diff --git a/features/features-controller/pom.xml b/features/features-controller/pom.xml index 20dde35670..64d72cf7c7 100644 --- a/features/features-controller/pom.xml +++ b/features/features-controller/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller features-controller - 9.0.2 + 9.0.3-SNAPSHOT feature OpenDaylight :: Controller :: Features Controller Production Features diff --git a/features/odl-clustering-test-app/pom.xml b/features/odl-clustering-test-app/pom.xml index ffc0ebb176..672ac82c36 100644 --- a/features/odl-clustering-test-app/pom.xml +++ b/features/odl-clustering-test-app/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2 + 9.0.3-SNAPSHOT ../single-feature-parent diff --git a/features/odl-controller-akka/pom.xml b/features/odl-controller-akka/pom.xml index 15591e120d..b1f43ef15b 100644 --- a/features/odl-controller-akka/pom.xml +++ b/features/odl-controller-akka/pom.xml @@ -14,7 +14,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2 + 9.0.3-SNAPSHOT ../single-feature-parent diff --git a/features/odl-controller-blueprint/pom.xml b/features/odl-controller-blueprint/pom.xml index 7b964de5e4..b6bba0b97c 100644 --- a/features/odl-controller-blueprint/pom.xml +++ b/features/odl-controller-blueprint/pom.xml @@ -12,7 +12,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2 + 9.0.3-SNAPSHOT ../single-feature-parent diff --git a/features/odl-controller-broker-local/pom.xml b/features/odl-controller-broker-local/pom.xml index 667c436692..9bb7bc3a64 100644 --- a/features/odl-controller-broker-local/pom.xml +++ b/features/odl-controller-broker-local/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2 + 9.0.3-SNAPSHOT ../single-feature-parent diff --git a/features/odl-controller-mdsal-common/pom.xml b/features/odl-controller-mdsal-common/pom.xml index b06739a2ba..be8cc89a13 100644 --- a/features/odl-controller-mdsal-common/pom.xml +++ b/features/odl-controller-mdsal-common/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2 + 9.0.3-SNAPSHOT ../single-feature-parent diff --git a/features/odl-controller-scala/pom.xml b/features/odl-controller-scala/pom.xml index 4093670604..5653979564 100644 --- a/features/odl-controller-scala/pom.xml +++ b/features/odl-controller-scala/pom.xml @@ -14,7 +14,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2 + 9.0.3-SNAPSHOT ../single-feature-parent diff --git a/features/odl-jolokia/pom.xml b/features/odl-jolokia/pom.xml index a3f4789a6a..9ea689a4a0 100644 --- a/features/odl-jolokia/pom.xml +++ b/features/odl-jolokia/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2 + 9.0.3-SNAPSHOT ../single-feature-parent diff --git a/features/odl-mdsal-benchmark/pom.xml b/features/odl-mdsal-benchmark/pom.xml index fb58a3b5fe..42b2d9e401 100644 --- a/features/odl-mdsal-benchmark/pom.xml +++ b/features/odl-mdsal-benchmark/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2 + 9.0.3-SNAPSHOT ../single-feature-parent diff --git a/features/odl-mdsal-broker/pom.xml b/features/odl-mdsal-broker/pom.xml index b279a98b8f..686971b49f 100644 --- a/features/odl-mdsal-broker/pom.xml +++ b/features/odl-mdsal-broker/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2 + 9.0.3-SNAPSHOT ../single-feature-parent diff --git a/features/odl-mdsal-clustering-commons/pom.xml b/features/odl-mdsal-clustering-commons/pom.xml index 7152fbdddb..5d1bbbeb51 100644 --- a/features/odl-mdsal-clustering-commons/pom.xml +++ b/features/odl-mdsal-clustering-commons/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2 + 9.0.3-SNAPSHOT ../single-feature-parent diff --git a/features/odl-mdsal-distributed-datastore/pom.xml b/features/odl-mdsal-distributed-datastore/pom.xml index 59c1b5b838..35b1d52094 100644 --- a/features/odl-mdsal-distributed-datastore/pom.xml +++ b/features/odl-mdsal-distributed-datastore/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2 + 9.0.3-SNAPSHOT ../single-feature-parent diff --git a/features/odl-mdsal-remoterpc-connector/pom.xml b/features/odl-mdsal-remoterpc-connector/pom.xml index 312830da7a..68033905be 100644 --- a/features/odl-mdsal-remoterpc-connector/pom.xml +++ b/features/odl-mdsal-remoterpc-connector/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2 + 9.0.3-SNAPSHOT ../single-feature-parent diff --git a/features/odl-toaster/pom.xml b/features/odl-toaster/pom.xml index 18e7f79a1a..01bfd40440 100644 --- a/features/odl-toaster/pom.xml +++ b/features/odl-toaster/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2 + 9.0.3-SNAPSHOT ../single-feature-parent diff --git a/features/pom.xml b/features/pom.xml index 8078cee14f..97ade422e2 100644 --- a/features/pom.xml +++ b/features/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller features-aggregator - 9.0.2 + 9.0.3-SNAPSHOT pom diff --git a/features/single-feature-parent/pom.xml b/features/single-feature-parent/pom.xml index 5e441c0f45..556d3acd76 100644 --- a/features/single-feature-parent/pom.xml +++ b/features/single-feature-parent/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller single-feature-parent - 9.0.2 + 9.0.3-SNAPSHOT pom @@ -25,7 +25,7 @@ org.opendaylight.controller bundle-parent - 9.0.2 + 9.0.3-SNAPSHOT pom import diff --git a/jolokia/pom.xml b/jolokia/pom.xml index 5f214047f2..1d98663b1f 100644 --- a/jolokia/pom.xml +++ b/jolokia/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller odl-jolokia-osgi - 9.0.2 + 9.0.3-SNAPSHOT jar diff --git a/karaf/pom.xml b/karaf/pom.xml index dbf97e0595..56869cc048 100644 --- a/karaf/pom.xml +++ b/karaf/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller controller-test-karaf - 9.0.2 + 9.0.3-SNAPSHOT pom diff --git a/opendaylight/blueprint/pom.xml b/opendaylight/blueprint/pom.xml index a1a3077947..2b7bfeb106 100644 --- a/opendaylight/blueprint/pom.xml +++ b/opendaylight/blueprint/pom.xml @@ -20,7 +20,7 @@ blueprint bundle ${project.artifactId} - 9.0.2 + 9.0.3-SNAPSHOT diff --git a/opendaylight/md-sal/cds-access-api/pom.xml b/opendaylight/md-sal/cds-access-api/pom.xml index 4e92431326..fb0bc37268 100644 --- a/opendaylight/md-sal/cds-access-api/pom.xml +++ b/opendaylight/md-sal/cds-access-api/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../parent diff --git a/opendaylight/md-sal/cds-access-client/pom.xml b/opendaylight/md-sal/cds-access-client/pom.xml index 8186d5ac03..78a4e73946 100644 --- a/opendaylight/md-sal/cds-access-client/pom.xml +++ b/opendaylight/md-sal/cds-access-client/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../parent diff --git a/opendaylight/md-sal/cds-dom-api/pom.xml b/opendaylight/md-sal/cds-dom-api/pom.xml index ab7eb9eec2..a28781c07d 100644 --- a/opendaylight/md-sal/cds-dom-api/pom.xml +++ b/opendaylight/md-sal/cds-dom-api/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../parent diff --git a/opendaylight/md-sal/cds-mgmt-api/pom.xml b/opendaylight/md-sal/cds-mgmt-api/pom.xml index 76ec086465..2ea3c28f04 100644 --- a/opendaylight/md-sal/cds-mgmt-api/pom.xml +++ b/opendaylight/md-sal/cds-mgmt-api/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../parent diff --git a/opendaylight/md-sal/eos-dom-akka/pom.xml b/opendaylight/md-sal/eos-dom-akka/pom.xml index 2124643f7e..dce797ca4f 100644 --- a/opendaylight/md-sal/eos-dom-akka/pom.xml +++ b/opendaylight/md-sal/eos-dom-akka/pom.xml @@ -14,7 +14,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../parent diff --git a/opendaylight/md-sal/mdsal-it-base/pom.xml b/opendaylight/md-sal/mdsal-it-base/pom.xml index 60d5307623..77e751dd0e 100644 --- a/opendaylight/md-sal/mdsal-it-base/pom.xml +++ b/opendaylight/md-sal/mdsal-it-base/pom.xml @@ -18,7 +18,7 @@ and is available at http://www.eclipse.org/legal/epl-v10.html org.opendaylight.controller mdsal-it-base - 9.0.2 + 9.0.3-SNAPSHOT bundle diff --git a/opendaylight/md-sal/mdsal-it-parent/pom.xml b/opendaylight/md-sal/mdsal-it-parent/pom.xml index 5c6498d804..3ea3d8e4a0 100644 --- a/opendaylight/md-sal/mdsal-it-parent/pom.xml +++ b/opendaylight/md-sal/mdsal-it-parent/pom.xml @@ -19,7 +19,7 @@ and is available at http://www.eclipse.org/legal/epl-v10.html org.opendaylight.controller mdsal-it-parent - 9.0.2 + 9.0.3-SNAPSHOT pom @@ -37,7 +37,7 @@ and is available at http://www.eclipse.org/legal/epl-v10.html org.opendaylight.controller controller-artifacts - 9.0.2 + 9.0.3-SNAPSHOT pom import diff --git a/opendaylight/md-sal/parent/pom.xml b/opendaylight/md-sal/parent/pom.xml index 483b0d2b16..2cd8d0ac1b 100644 --- a/opendaylight/md-sal/parent/pom.xml +++ b/opendaylight/md-sal/parent/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT pom @@ -25,7 +25,7 @@ org.opendaylight.controller bundle-parent - 9.0.2 + 9.0.3-SNAPSHOT pom import diff --git a/opendaylight/md-sal/pom.xml b/opendaylight/md-sal/pom.xml index 295065c91b..86f5e203f2 100644 --- a/opendaylight/md-sal/pom.xml +++ b/opendaylight/md-sal/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller mdsal-aggregator - 9.0.2 + 9.0.3-SNAPSHOT pom diff --git a/opendaylight/md-sal/sal-akka-raft-example/pom.xml b/opendaylight/md-sal/sal-akka-raft-example/pom.xml index 1d4dbf09b7..295d0d0552 100644 --- a/opendaylight/md-sal/sal-akka-raft-example/pom.xml +++ b/opendaylight/md-sal/sal-akka-raft-example/pom.xml @@ -11,7 +11,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../parent diff --git a/opendaylight/md-sal/sal-akka-raft/pom.xml b/opendaylight/md-sal/sal-akka-raft/pom.xml index 0d9eeffbe9..306e7561a5 100644 --- a/opendaylight/md-sal/sal-akka-raft/pom.xml +++ b/opendaylight/md-sal/sal-akka-raft/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../parent diff --git a/opendaylight/md-sal/sal-akka-segmented-journal/pom.xml b/opendaylight/md-sal/sal-akka-segmented-journal/pom.xml index 84e826d7ba..243f3b0bfc 100644 --- a/opendaylight/md-sal/sal-akka-segmented-journal/pom.xml +++ b/opendaylight/md-sal/sal-akka-segmented-journal/pom.xml @@ -12,7 +12,7 @@ and is available at http://www.eclipse.org/legal/epl-v10.html org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../parent diff --git a/opendaylight/md-sal/sal-binding-it/pom.xml b/opendaylight/md-sal/sal-binding-it/pom.xml index 75e8bf1dbc..dd32609d30 100644 --- a/opendaylight/md-sal/sal-binding-it/pom.xml +++ b/opendaylight/md-sal/sal-binding-it/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-it-parent - 9.0.2 + 9.0.3-SNAPSHOT ../mdsal-it-parent sal-binding-it diff --git a/opendaylight/md-sal/sal-cluster-admin-api/pom.xml b/opendaylight/md-sal/sal-cluster-admin-api/pom.xml index c785004ce3..ad8d996edc 100644 --- a/opendaylight/md-sal/sal-cluster-admin-api/pom.xml +++ b/opendaylight/md-sal/sal-cluster-admin-api/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../parent diff --git a/opendaylight/md-sal/sal-cluster-admin-impl/pom.xml b/opendaylight/md-sal/sal-cluster-admin-impl/pom.xml index 684fc0079d..cb905343c8 100644 --- a/opendaylight/md-sal/sal-cluster-admin-impl/pom.xml +++ b/opendaylight/md-sal/sal-cluster-admin-impl/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../parent diff --git a/opendaylight/md-sal/sal-cluster-admin-karaf-cli/pom.xml b/opendaylight/md-sal/sal-cluster-admin-karaf-cli/pom.xml index 1550067264..481adfd32f 100644 --- a/opendaylight/md-sal/sal-cluster-admin-karaf-cli/pom.xml +++ b/opendaylight/md-sal/sal-cluster-admin-karaf-cli/pom.xml @@ -5,7 +5,7 @@ mdsal-parent org.opendaylight.controller - 9.0.2 + 9.0.3-SNAPSHOT ../parent/pom.xml 4.0.0 diff --git a/opendaylight/md-sal/sal-clustering-commons/pom.xml b/opendaylight/md-sal/sal-clustering-commons/pom.xml index 50448fc0f4..14056f32ca 100644 --- a/opendaylight/md-sal/sal-clustering-commons/pom.xml +++ b/opendaylight/md-sal/sal-clustering-commons/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../parent diff --git a/opendaylight/md-sal/sal-clustering-config/pom.xml b/opendaylight/md-sal/sal-clustering-config/pom.xml index 23f735fefb..4c62912307 100644 --- a/opendaylight/md-sal/sal-clustering-config/pom.xml +++ b/opendaylight/md-sal/sal-clustering-config/pom.xml @@ -18,7 +18,7 @@ org.opendaylight.controller sal-clustering-config - 9.0.2 + 9.0.3-SNAPSHOT jar Configuration files for md-sal clustering diff --git a/opendaylight/md-sal/sal-common-util/pom.xml b/opendaylight/md-sal/sal-common-util/pom.xml index 3e49aa652a..a0bf479f16 100644 --- a/opendaylight/md-sal/sal-common-util/pom.xml +++ b/opendaylight/md-sal/sal-common-util/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../parent diff --git a/opendaylight/md-sal/sal-distributed-datastore/pom.xml b/opendaylight/md-sal/sal-distributed-datastore/pom.xml index 948ef79df8..2d397cdcf5 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/pom.xml +++ b/opendaylight/md-sal/sal-distributed-datastore/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../parent diff --git a/opendaylight/md-sal/sal-dummy-distributed-datastore/pom.xml b/opendaylight/md-sal/sal-dummy-distributed-datastore/pom.xml index 0aae26c23a..7e1cffa2ed 100644 --- a/opendaylight/md-sal/sal-dummy-distributed-datastore/pom.xml +++ b/opendaylight/md-sal/sal-dummy-distributed-datastore/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../parent diff --git a/opendaylight/md-sal/sal-remoterpc-connector/pom.xml b/opendaylight/md-sal/sal-remoterpc-connector/pom.xml index 6e1823bf3b..c98a44d8eb 100644 --- a/opendaylight/md-sal/sal-remoterpc-connector/pom.xml +++ b/opendaylight/md-sal/sal-remoterpc-connector/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../parent diff --git a/opendaylight/md-sal/sal-test-model/pom.xml b/opendaylight/md-sal/sal-test-model/pom.xml index aa037c80b5..ce09fbb5fb 100644 --- a/opendaylight/md-sal/sal-test-model/pom.xml +++ b/opendaylight/md-sal/sal-test-model/pom.xml @@ -6,7 +6,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../parent diff --git a/opendaylight/md-sal/samples/clustering-test-app/configuration/pom.xml b/opendaylight/md-sal/samples/clustering-test-app/configuration/pom.xml index 450dbf5bfb..0fc796cac8 100644 --- a/opendaylight/md-sal/samples/clustering-test-app/configuration/pom.xml +++ b/opendaylight/md-sal/samples/clustering-test-app/configuration/pom.xml @@ -17,7 +17,7 @@ org.opendaylight.controller.samples clustering-it-config - 9.0.2 + 9.0.3-SNAPSHOT jar diff --git a/opendaylight/md-sal/samples/clustering-test-app/karaf-cli/pom.xml b/opendaylight/md-sal/samples/clustering-test-app/karaf-cli/pom.xml index 94ac773c6b..450c703289 100644 --- a/opendaylight/md-sal/samples/clustering-test-app/karaf-cli/pom.xml +++ b/opendaylight/md-sal/samples/clustering-test-app/karaf-cli/pom.xml @@ -12,7 +12,7 @@ mdsal-parent org.opendaylight.controller - 9.0.2 + 9.0.3-SNAPSHOT ../../../parent/pom.xml 4.0.0 diff --git a/opendaylight/md-sal/samples/clustering-test-app/model/pom.xml b/opendaylight/md-sal/samples/clustering-test-app/model/pom.xml index 88c689aa68..962989a790 100644 --- a/opendaylight/md-sal/samples/clustering-test-app/model/pom.xml +++ b/opendaylight/md-sal/samples/clustering-test-app/model/pom.xml @@ -5,7 +5,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../../../parent diff --git a/opendaylight/md-sal/samples/clustering-test-app/pom.xml b/opendaylight/md-sal/samples/clustering-test-app/pom.xml index 3416a9e7d6..b83c335a94 100644 --- a/opendaylight/md-sal/samples/clustering-test-app/pom.xml +++ b/opendaylight/md-sal/samples/clustering-test-app/pom.xml @@ -10,7 +10,7 @@ org.opendaylight.controller.samples clustering-test-app - 9.0.2 + 9.0.3-SNAPSHOT pom diff --git a/opendaylight/md-sal/samples/clustering-test-app/provider/pom.xml b/opendaylight/md-sal/samples/clustering-test-app/provider/pom.xml index 6d95e8b8e1..5a4c62eaf6 100644 --- a/opendaylight/md-sal/samples/clustering-test-app/provider/pom.xml +++ b/opendaylight/md-sal/samples/clustering-test-app/provider/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../../../parent diff --git a/opendaylight/md-sal/samples/pom.xml b/opendaylight/md-sal/samples/pom.xml index 447a4f6cbb..80752933e0 100644 --- a/opendaylight/md-sal/samples/pom.xml +++ b/opendaylight/md-sal/samples/pom.xml @@ -10,7 +10,7 @@ org.opendaylight.controller.samples samples-aggregator - 9.0.2 + 9.0.3-SNAPSHOT pom diff --git a/opendaylight/md-sal/samples/toaster-consumer/pom.xml b/opendaylight/md-sal/samples/toaster-consumer/pom.xml index 04074b7dfa..99d9c76c55 100644 --- a/opendaylight/md-sal/samples/toaster-consumer/pom.xml +++ b/opendaylight/md-sal/samples/toaster-consumer/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../../parent diff --git a/opendaylight/md-sal/samples/toaster-it/pom.xml b/opendaylight/md-sal/samples/toaster-it/pom.xml index 6e324907ef..6b8c35e645 100644 --- a/opendaylight/md-sal/samples/toaster-it/pom.xml +++ b/opendaylight/md-sal/samples/toaster-it/pom.xml @@ -5,7 +5,7 @@ org.opendaylight.controller mdsal-it-parent - 9.0.2 + 9.0.3-SNAPSHOT ../../mdsal-it-parent sample-toaster-it diff --git a/opendaylight/md-sal/samples/toaster-provider/pom.xml b/opendaylight/md-sal/samples/toaster-provider/pom.xml index 653ca4bd9a..ea6ee62081 100644 --- a/opendaylight/md-sal/samples/toaster-provider/pom.xml +++ b/opendaylight/md-sal/samples/toaster-provider/pom.xml @@ -4,7 +4,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../../parent diff --git a/opendaylight/md-sal/samples/toaster/pom.xml b/opendaylight/md-sal/samples/toaster/pom.xml index 08e292bff9..57230fa79d 100644 --- a/opendaylight/md-sal/samples/toaster/pom.xml +++ b/opendaylight/md-sal/samples/toaster/pom.xml @@ -5,7 +5,7 @@ org.opendaylight.controller mdsal-parent - 9.0.2 + 9.0.3-SNAPSHOT ../../parent diff --git a/pom.xml b/pom.xml index 63a9be0355..7b3c5253d3 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ org.opendaylight.controller releasepom - 9.0.2 + 9.0.3-SNAPSHOT pom controller -- 2.36.6 From 0324432d000896c3f27c7be4e1863231bd50cadc Mon Sep 17 00:00:00 2001 From: lubos-cicut Date: Wed, 10 Apr 2024 10:48:50 +0200 Subject: [PATCH 05/16] Fix followerDistributedDataStore tear down Fix closing of followerDistributedDataStore in DistributedDataStoreRemotingIntegrationTest#tearDown. Change-Id: Iba35cae665f29a9da0430baff1f792191a4d1287 Signed-off-by: lubos-cicut --- .../datastore/DistributedDataStoreRemotingIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DistributedDataStoreRemotingIntegrationTest.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DistributedDataStoreRemotingIntegrationTest.java index 738ff58382..91c00f7eb1 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DistributedDataStoreRemotingIntegrationTest.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DistributedDataStoreRemotingIntegrationTest.java @@ -199,7 +199,7 @@ public class DistributedDataStoreRemotingIntegrationTest extends AbstractTest { @After public void tearDown() { if (followerDistributedDataStore != null) { - leaderDistributedDataStore.close(); + followerDistributedDataStore.close(); } if (leaderDistributedDataStore != null) { leaderDistributedDataStore.close(); -- 2.36.6 From 08d6b66f08fd997ab92e97e86d18059480eb693b Mon Sep 17 00:00:00 2001 From: Ruslan Kashapov Date: Wed, 3 Apr 2024 12:53:20 +0300 Subject: [PATCH 06/16] atomic-storage: remove type dependency at segment level I/O Due to object de- and serialization was served by segment I/O artifacts it required these artifacts holding serializer instance and being dependent on its type (generics). Moving serialization to upper level (journal) allows segment reader/writer operate on byte level only with no extra data conversions. Serializer interface simplified for further isolation and removal of Kryo serializer. JIRA: CONTROLLER-2115 Change-Id: I8f6bac3192a0f38b627150be4c8ea128f1e233e5 Signed-off-by: Ruslan Kashapov Signed-off-by: Robert Varga --- atomix-storage/pom.xml | 4 + .../journal/CommitsSegmentJournalReader.java | 2 +- .../journal/DiskJournalSegmentWriter.java | 28 ++--- .../storage/journal/JournalSegment.java | 24 ++-- .../storage/journal/JournalSegmentReader.java | 40 +++---- .../storage/journal/JournalSegmentWriter.java | 112 ++++++++---------- .../atomix/storage/journal/JournalSerdes.java | 3 + .../storage/journal/JournalSerializer.java | 48 ++++++++ .../journal/MappedJournalSegmentWriter.java | 28 ++--- .../storage/journal/SegmentedJournal.java | 93 ++++++++------- .../journal/SegmentedJournalReader.java | 22 ++-- .../journal/SegmentedJournalWriter.java | 21 ++-- features/odl-mdsal-clustering-commons/pom.xml | 6 + .../src/main/feature/feature.xml | 1 + 14 files changed, 243 insertions(+), 189 deletions(-) create mode 100644 atomix-storage/src/main/java/io/atomix/storage/journal/JournalSerializer.java diff --git a/atomix-storage/pom.xml b/atomix-storage/pom.xml index 886ad6ac50..8fc7ca3709 100644 --- a/atomix-storage/pom.xml +++ b/atomix-storage/pom.xml @@ -38,6 +38,10 @@ com.google.guava guava + + io.netty + netty-buffer + org.eclipse.jdt org.eclipse.jdt.annotation diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/CommitsSegmentJournalReader.java b/atomix-storage/src/main/java/io/atomix/storage/journal/CommitsSegmentJournalReader.java index ac80fed96b..65f4de6496 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/CommitsSegmentJournalReader.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/CommitsSegmentJournalReader.java @@ -19,7 +19,7 @@ package io.atomix.storage.journal; * A {@link JournalReader} traversing only committed entries. */ final class CommitsSegmentJournalReader extends SegmentedJournalReader { - CommitsSegmentJournalReader(SegmentedJournal journal, JournalSegment segment) { + CommitsSegmentJournalReader(SegmentedJournal journal, JournalSegment segment) { super(journal, segment); } diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/DiskJournalSegmentWriter.java b/atomix-storage/src/main/java/io/atomix/storage/journal/DiskJournalSegmentWriter.java index 266320127b..54feee1ada 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/DiskJournalSegmentWriter.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/DiskJournalSegmentWriter.java @@ -39,28 +39,28 @@ import java.nio.channels.FileChannel; * * @author Jordan Halterman */ -final class DiskJournalSegmentWriter extends JournalSegmentWriter { +final class DiskJournalSegmentWriter extends JournalSegmentWriter { private static final ByteBuffer ZERO_ENTRY_HEADER = ByteBuffer.wrap(new byte[HEADER_BYTES]); - private final JournalSegmentReader reader; + private final JournalSegmentReader reader; private final ByteBuffer buffer; - DiskJournalSegmentWriter(final FileChannel channel, final JournalSegment segment, final int maxEntrySize, - final JournalIndex index, final JournalSerdes namespace) { - super(channel, segment, maxEntrySize, index, namespace); + DiskJournalSegmentWriter(final FileChannel channel, final JournalSegment segment, final int maxEntrySize, + final JournalIndex index) { + super(channel, segment, maxEntrySize, index); buffer = DiskFileReader.allocateBuffer(maxSegmentSize, maxEntrySize); - reader = new JournalSegmentReader<>(segment, - new DiskFileReader(segment.file().file().toPath(), channel, buffer), maxEntrySize, namespace); + reader = new JournalSegmentReader(segment, + new DiskFileReader(segment.file().file().toPath(), channel, buffer), maxEntrySize); reset(0); } - DiskJournalSegmentWriter(final JournalSegmentWriter previous) { + DiskJournalSegmentWriter(final JournalSegmentWriter previous) { super(previous); buffer = DiskFileReader.allocateBuffer(maxSegmentSize, maxEntrySize); - reader = new JournalSegmentReader<>(segment, - new DiskFileReader(segment.file().file().toPath(), channel, buffer), maxEntrySize, namespace); + reader = new JournalSegmentReader(segment, + new DiskFileReader(segment.file().file().toPath(), channel, buffer), maxEntrySize); } @Override @@ -69,17 +69,17 @@ final class DiskJournalSegmentWriter extends JournalSegmentWriter { } @Override - MappedJournalSegmentWriter toMapped() { - return new MappedJournalSegmentWriter<>(this); + MappedJournalSegmentWriter toMapped() { + return new MappedJournalSegmentWriter(this); } @Override - DiskJournalSegmentWriter toFileChannel() { + DiskJournalSegmentWriter toFileChannel() { return this; } @Override - JournalSegmentReader reader() { + JournalSegmentReader reader() { return reader; } diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegment.java b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegment.java index 9239f86d25..45405aae68 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegment.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegment.java @@ -34,18 +34,17 @@ import org.eclipse.jdt.annotation.Nullable; * * @author Jordan Halterman */ -final class JournalSegment implements AutoCloseable { +final class JournalSegment implements AutoCloseable { private final JournalSegmentFile file; private final JournalSegmentDescriptor descriptor; private final StorageLevel storageLevel; private final int maxEntrySize; private final JournalIndex journalIndex; - private final JournalSerdes namespace; - private final Set> readers = ConcurrentHashMap.newKeySet(); + private final Set readers = ConcurrentHashMap.newKeySet(); private final AtomicInteger references = new AtomicInteger(); private final FileChannel channel; - private JournalSegmentWriter writer; + private JournalSegmentWriter writer; private boolean open = true; JournalSegment( @@ -53,13 +52,11 @@ final class JournalSegment implements AutoCloseable { JournalSegmentDescriptor descriptor, StorageLevel storageLevel, int maxEntrySize, - double indexDensity, - JournalSerdes namespace) { + double indexDensity) { this.file = file; this.descriptor = descriptor; this.storageLevel = storageLevel; this.maxEntrySize = maxEntrySize; - this.namespace = namespace; journalIndex = new SparseJournalIndex(indexDensity); try { channel = FileChannel.open(file.file().toPath(), @@ -68,9 +65,8 @@ final class JournalSegment implements AutoCloseable { throw new StorageException(e); } writer = switch (storageLevel) { - case DISK -> new DiskJournalSegmentWriter<>(channel, this, maxEntrySize, journalIndex, namespace); - case MAPPED -> new MappedJournalSegmentWriter<>(channel, this, maxEntrySize, journalIndex, namespace) - .toFileChannel(); + case DISK -> new DiskJournalSegmentWriter(channel, this, maxEntrySize, journalIndex); + case MAPPED -> new MappedJournalSegmentWriter(channel, this, maxEntrySize, journalIndex).toFileChannel(); }; } @@ -161,7 +157,7 @@ final class JournalSegment implements AutoCloseable { * * @return The segment writer. */ - JournalSegmentWriter acquireWriter() { + JournalSegmentWriter acquireWriter() { checkOpen(); acquire(); @@ -180,7 +176,7 @@ final class JournalSegment implements AutoCloseable { * * @return A new segment reader. */ - JournalSegmentReader createReader() { + JournalSegmentReader createReader() { checkOpen(); acquire(); @@ -188,7 +184,7 @@ final class JournalSegment implements AutoCloseable { final var path = file.file().toPath(); final var fileReader = buffer != null ? new MappedFileReader(path, buffer) : new DiskFileReader(path, channel, descriptor.maxSegmentSize(), maxEntrySize); - final var reader = new JournalSegmentReader<>(this, fileReader, maxEntrySize, namespace); + final var reader = new JournalSegmentReader(this, fileReader, maxEntrySize); reader.setPosition(JournalSegmentDescriptor.BYTES); readers.add(reader); return reader; @@ -199,7 +195,7 @@ final class JournalSegment implements AutoCloseable { * * @param reader the closed segment reader */ - void closeReader(JournalSegmentReader reader) { + void closeReader(JournalSegmentReader reader) { if (readers.remove(reader)) { release(); } diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegmentReader.java b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegmentReader.java index 93ccd1748e..d89c720c67 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegmentReader.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegmentReader.java @@ -18,30 +18,28 @@ package io.atomix.storage.journal; import static com.google.common.base.Verify.verify; import static java.util.Objects.requireNonNull; -import com.esotericsoftware.kryo.KryoException; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; import java.util.zip.CRC32; import org.eclipse.jdt.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -final class JournalSegmentReader { +final class JournalSegmentReader { private static final Logger LOG = LoggerFactory.getLogger(JournalSegmentReader.class); - private final JournalSegment segment; - private final JournalSerdes namespace; + private final JournalSegment segment; private final FileReader fileReader; private final int maxSegmentSize; private final int maxEntrySize; private int position; - JournalSegmentReader(final JournalSegment segment, final FileReader fileReader, - final int maxEntrySize, final JournalSerdes namespace) { + JournalSegmentReader(final JournalSegment segment, final FileReader fileReader, final int maxEntrySize) { this.segment = requireNonNull(segment); this.fileReader = requireNonNull(fileReader); maxSegmentSize = segment.descriptor().maxSegmentSize(); this.maxEntrySize = maxEntrySize; - this.namespace = requireNonNull(namespace); } /** @@ -73,12 +71,12 @@ final class JournalSegmentReader { } /** - * Reads the next entry, assigning it specified index. + * Reads the next binary data block * * @param index entry index - * @return The entry, or {@code null} + * @return The binary data, or {@code null} */ - @Nullable Indexed readEntry(final long index) { + @Nullable ByteBuf readBytes(final long index) { // Check if there is enough in the buffer remaining final int remaining = maxSegmentSize - position - SegmentEntry.HEADER_BYTES; if (remaining < 0) { @@ -102,10 +100,10 @@ final class JournalSegmentReader { final int checksum = buffer.getInt(Integer.BYTES); // Slice off the entry's bytes - final var entryBytes = buffer.slice(SegmentEntry.HEADER_BYTES, length); + final var entryBuffer = buffer.slice(SegmentEntry.HEADER_BYTES, length); // Compute the checksum for the entry bytes. final var crc32 = new CRC32(); - crc32.update(entryBytes); + crc32.update(entryBuffer); // If the stored checksum does not equal the computed checksum, do not proceed further final var computed = (int) crc32.getValue(); @@ -115,20 +113,12 @@ final class JournalSegmentReader { return null; } - // Attempt to deserialize - final E entry; - try { - entry = namespace.deserialize(entryBytes.rewind()); - } catch (KryoException e) { - // TODO: promote this to a hard error, as it should never happen - LOG.debug("Failed to deserialize entry", e); - invalidateCache(); - return null; - } + // update position + position += SegmentEntry.HEADER_BYTES + length; - // We are all set. Update the position. - position = position + SegmentEntry.HEADER_BYTES + length; - return new Indexed<>(index, entry, length); + // return bytes + entryBuffer.rewind(); + return Unpooled.buffer(length).writeBytes(entryBuffer); } /** diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegmentWriter.java b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegmentWriter.java index c7c035be65..c2e0a258c9 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegmentWriter.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegmentWriter.java @@ -18,8 +18,8 @@ package io.atomix.storage.journal; import static io.atomix.storage.journal.SegmentEntry.HEADER_BYTES; import static java.util.Objects.requireNonNull; -import com.esotericsoftware.kryo.KryoException; import io.atomix.storage.journal.index.JournalIndex; +import io.netty.buffer.ByteBuf; import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; @@ -29,37 +29,36 @@ import org.eclipse.jdt.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -abstract sealed class JournalSegmentWriter permits DiskJournalSegmentWriter, MappedJournalSegmentWriter { +abstract sealed class JournalSegmentWriter permits DiskJournalSegmentWriter, MappedJournalSegmentWriter { private static final Logger LOG = LoggerFactory.getLogger(JournalSegmentWriter.class); final @NonNull FileChannel channel; - final @NonNull JournalSegment segment; + final @NonNull JournalSegment segment; private final @NonNull JournalIndex index; - final @NonNull JournalSerdes namespace; final int maxSegmentSize; final int maxEntrySize; - private Indexed lastEntry; private int currentPosition; + private Long lastIndex; + private ByteBuf lastWritten; - JournalSegmentWriter(final FileChannel channel, final JournalSegment segment, final int maxEntrySize, - final JournalIndex index, final JournalSerdes namespace) { + JournalSegmentWriter(final FileChannel channel, final JournalSegment segment, final int maxEntrySize, + final JournalIndex index) { this.channel = requireNonNull(channel); this.segment = requireNonNull(segment); this.index = requireNonNull(index); - this.namespace = requireNonNull(namespace); maxSegmentSize = segment.descriptor().maxSegmentSize(); this.maxEntrySize = maxEntrySize; } - JournalSegmentWriter(final JournalSegmentWriter previous) { + JournalSegmentWriter(final JournalSegmentWriter previous) { channel = previous.channel; segment = previous.segment; index = previous.index; - namespace = previous.namespace; maxSegmentSize = previous.maxSegmentSize; maxEntrySize = previous.maxEntrySize; - lastEntry = previous.lastEntry; + lastWritten = previous.lastWritten; + lastIndex = previous.lastIndex; currentPosition = previous.currentPosition; } @@ -69,16 +68,16 @@ abstract sealed class JournalSegmentWriter permits DiskJournalSegmentWriter, * @return The last written index. */ final long getLastIndex() { - return lastEntry != null ? lastEntry.index() : segment.firstIndex() - 1; + return lastIndex != null ? lastIndex : segment.firstIndex() - 1; } /** - * Returns the last entry written. + * Returns the last data written. * - * @return The last entry written. + * @return The last data written. */ - final Indexed getLastEntry() { - return lastEntry; + final ByteBuf getLastWritten() { + return lastWritten == null ? null : lastWritten.slice(); } /** @@ -87,63 +86,52 @@ abstract sealed class JournalSegmentWriter permits DiskJournalSegmentWriter, * @return The next index to be written. */ final long getNextIndex() { - return lastEntry != null ? lastEntry.index() + 1 : segment.firstIndex(); + return lastIndex != null ? lastIndex + 1 : segment.firstIndex(); } /** - * Tries to append an entry to the journal. + * Tries to append a binary data to the journal. * - * @param entry The entry to append. - * @return The appended indexed entry, or {@code null} if there is not enough space available + * @param buf binary data to append + * @return The index of appended data, or {@code null} if segment has no space */ - final @Nullable Indexed append(final T entry) { + final Long append(final ByteBuf buf) { + final var length = buf.readableBytes(); + if (length > maxEntrySize) { + throw new StorageException.TooLarge("Serialized entry size exceeds maximum allowed bytes (" + + maxEntrySize + ")"); + } + // Store the entry index. final long index = getNextIndex(); final int position = currentPosition; - // Serialize the entry. - final int bodyPosition = position + HEADER_BYTES; - final int avail = maxSegmentSize - bodyPosition; - if (avail < 0) { + // check space available + final int nextPosition = position + HEADER_BYTES + length; + if (nextPosition >= maxSegmentSize) { LOG.trace("Not enough space for {} at {}", index, position); return null; } - final var writeLimit = Math.min(avail, maxEntrySize); - final var diskEntry = startWrite(position, writeLimit + HEADER_BYTES).position(HEADER_BYTES); - try { - namespace.serialize(entry, diskEntry); - } catch (KryoException e) { - if (writeLimit != maxEntrySize) { - // We have not provided enough capacity, signal to roll to next segment - LOG.trace("Tail serialization with {} bytes available failed", writeLimit, e); - return null; - } - - // Just reset the buffer. There's no need to zero the bytes since we haven't written the length or checksum. - throw new StorageException.TooLarge("Entry size exceeds maximum allowed bytes (" + maxEntrySize + ")", e); - } - - final int length = diskEntry.position() - HEADER_BYTES; + // allocate buffer and write data + final var writeBuffer = startWrite(position, length + HEADER_BYTES).position(HEADER_BYTES); + writeBuffer.put(buf.nioBuffer()); // Compute the checksum for the entry. final var crc32 = new CRC32(); - crc32.update(diskEntry.flip().position(HEADER_BYTES)); + crc32.update(writeBuffer.flip().position(HEADER_BYTES)); // Create a single byte[] in memory for the entire entry and write it as a batch to the underlying buffer. - diskEntry.putInt(0, length).putInt(Integer.BYTES, (int) crc32.getValue()); - commitWrite(position, diskEntry.rewind()); + writeBuffer.putInt(0, length).putInt(Integer.BYTES, (int) crc32.getValue()); + commitWrite(position, writeBuffer.rewind()); // Update the last entry with the correct index/term/length. - final var indexedEntry = new Indexed(index, entry, length); - lastEntry = indexedEntry; + currentPosition = nextPosition; + lastWritten = buf; + lastIndex = index; this.index.index(index, position); - currentPosition = bodyPosition + length; - - @SuppressWarnings("unchecked") - final var ugly = (Indexed) indexedEntry; - return ugly; + return index; } abstract ByteBuffer startWrite(int position, int size); @@ -167,9 +155,9 @@ abstract sealed class JournalSegmentWriter permits DiskJournalSegmentWriter, } } - abstract JournalSegmentReader reader(); + abstract JournalSegmentReader reader(); - private void resetWithBuffer(final JournalSegmentReader reader, final long index) { + private void resetWithBuffer(final JournalSegmentReader reader, final long index) { long nextIndex = segment.firstIndex(); // Clear the buffer indexes and acquire ownership of the buffer @@ -177,17 +165,18 @@ abstract sealed class JournalSegmentWriter permits DiskJournalSegmentWriter, reader.setPosition(JournalSegmentDescriptor.BYTES); while (index == 0 || nextIndex <= index) { - final var entry = reader.readEntry(nextIndex); - if (entry == null) { + final var buf = reader.readBytes(nextIndex); + if (buf == null) { break; } - lastEntry = entry; + lastWritten = buf; + lastIndex = nextIndex; this.index.index(nextIndex, currentPosition); nextIndex++; // Update the current position for indexing. - currentPosition = currentPosition + HEADER_BYTES + entry.size(); + currentPosition += HEADER_BYTES + buf.readableBytes(); } } @@ -202,8 +191,9 @@ abstract sealed class JournalSegmentWriter permits DiskJournalSegmentWriter, return; } - // Reset the last entry. - lastEntry = null; + // Reset the last written + lastIndex = null; + lastWritten = null; // Truncate the index. this.index.truncate(index); @@ -245,7 +235,7 @@ abstract sealed class JournalSegmentWriter permits DiskJournalSegmentWriter, */ abstract @Nullable MappedByteBuffer buffer(); - abstract @NonNull MappedJournalSegmentWriter toMapped(); + abstract @NonNull MappedJournalSegmentWriter toMapped(); - abstract @NonNull DiskJournalSegmentWriter toFileChannel(); + abstract @NonNull DiskJournalSegmentWriter toFileChannel(); } diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSerdes.java b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSerdes.java index 32fc8d30c6..a970882edf 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSerdes.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSerdes.java @@ -26,7 +26,10 @@ import java.nio.ByteBuffer; /** * Support for serialization of {@link Journal} entries. + * + * @deprecated due to dependency on outdated Kryo library, {@link JournalSerializer} to be used instead. */ +@Deprecated(forRemoval = true, since="9.0.3") public interface JournalSerdes { /** * Serializes given object to byte array. diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSerializer.java b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSerializer.java new file mode 100644 index 0000000000..eff9af8559 --- /dev/null +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSerializer.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2024 PANTHEON.tech s.r.o. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package io.atomix.storage.journal; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufUtil; +import io.netty.buffer.Unpooled; + +/** + * Support for serialization of {@link Journal} entries. + */ +public interface JournalSerializer { + + /** + * Serializes given object to byte array. + * + * @param obj Object to serialize + * @return serialized bytes as {@link ByteBuf} + */ + ByteBuf serialize(T obj) ; + + /** + * Deserializes given byte array to Object. + * + * @param buf serialized bytes as {@link ByteBuf} + * @return deserialized Object + */ + T deserialize(final ByteBuf buf); + + static JournalSerializer wrap(final JournalSerdes serdes) { + return new JournalSerializer<>() { + @Override + public ByteBuf serialize(final E obj) { + return Unpooled.wrappedBuffer(serdes.serialize(obj)); + } + + @Override + public E deserialize(final ByteBuf buf) { + return serdes.deserialize(ByteBufUtil.getBytes(buf)); + } + }; + } +} diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/MappedJournalSegmentWriter.java b/atomix-storage/src/main/java/io/atomix/storage/journal/MappedJournalSegmentWriter.java index 00dd4c6cec..48d9e764f6 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/MappedJournalSegmentWriter.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/MappedJournalSegmentWriter.java @@ -38,29 +38,29 @@ import org.eclipse.jdt.annotation.NonNull; * * @author Jordan Halterman */ -final class MappedJournalSegmentWriter extends JournalSegmentWriter { +final class MappedJournalSegmentWriter extends JournalSegmentWriter { private final @NonNull MappedByteBuffer mappedBuffer; - private final JournalSegmentReader reader; + private final JournalSegmentReader reader; private final ByteBuffer buffer; - MappedJournalSegmentWriter(final FileChannel channel, final JournalSegment segment, final int maxEntrySize, - final JournalIndex index, final JournalSerdes namespace) { - super(channel, segment, maxEntrySize, index, namespace); + MappedJournalSegmentWriter(final FileChannel channel, final JournalSegment segment, final int maxEntrySize, + final JournalIndex index) { + super(channel, segment, maxEntrySize, index); mappedBuffer = mapBuffer(channel, maxSegmentSize); buffer = mappedBuffer.slice(); - reader = new JournalSegmentReader<>(segment, new MappedFileReader(segment.file().file().toPath(), mappedBuffer), - maxEntrySize, namespace); + reader = new JournalSegmentReader(segment, new MappedFileReader(segment.file().file().toPath(), mappedBuffer), + maxEntrySize); reset(0); } - MappedJournalSegmentWriter(final JournalSegmentWriter previous) { + MappedJournalSegmentWriter(final JournalSegmentWriter previous) { super(previous); mappedBuffer = mapBuffer(channel, maxSegmentSize); buffer = mappedBuffer.slice(); - reader = new JournalSegmentReader<>(segment, new MappedFileReader(segment.file().file().toPath(), mappedBuffer), - maxEntrySize, namespace); + reader = new JournalSegmentReader(segment, new MappedFileReader(segment.file().file().toPath(), mappedBuffer), + maxEntrySize); } private static @NonNull MappedByteBuffer mapBuffer(final FileChannel channel, final int maxSegmentSize) { @@ -77,18 +77,18 @@ final class MappedJournalSegmentWriter extends JournalSegmentWriter { } @Override - MappedJournalSegmentWriter toMapped() { + MappedJournalSegmentWriter toMapped() { return this; } @Override - DiskJournalSegmentWriter toFileChannel() { + DiskJournalSegmentWriter toFileChannel() { close(); - return new DiskJournalSegmentWriter<>(this); + return new DiskJournalSegmentWriter(this); } @Override - JournalSegmentReader reader() { + JournalSegmentReader reader() { return reader; } diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournal.java b/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournal.java index ef1a4cf028..7289d3deef 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournal.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournal.java @@ -15,6 +15,10 @@ */ package io.atomix.storage.journal; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; @@ -31,10 +35,6 @@ import java.util.concurrent.ConcurrentSkipListMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static java.util.Objects.requireNonNull; - /** * Segmented journal. */ @@ -54,7 +54,7 @@ public final class SegmentedJournal implements Journal { private final String name; private final StorageLevel storageLevel; private final File directory; - private final JournalSerdes namespace; + private final JournalSerializer serializer; private final int maxSegmentSize; private final int maxEntrySize; private final int maxEntriesPerSegment; @@ -63,9 +63,9 @@ public final class SegmentedJournal implements Journal { private final SegmentedJournalWriter writer; private volatile long commitIndex; - private final ConcurrentNavigableMap> segments = new ConcurrentSkipListMap<>(); - private final Collection> readers = ConcurrentHashMap.newKeySet(); - private JournalSegment currentSegment; + private final ConcurrentNavigableMap segments = new ConcurrentSkipListMap<>(); + private final Collection readers = ConcurrentHashMap.newKeySet(); + private JournalSegment currentSegment; private volatile boolean open = true; @@ -82,7 +82,7 @@ public final class SegmentedJournal implements Journal { this.name = requireNonNull(name, "name cannot be null"); this.storageLevel = requireNonNull(storageLevel, "storageLevel cannot be null"); this.directory = requireNonNull(directory, "directory cannot be null"); - this.namespace = requireNonNull(namespace, "namespace cannot be null"); + this.serializer = JournalSerializer.wrap(requireNonNull(namespace, "namespace cannot be null")); this.maxSegmentSize = maxSegmentSize; this.maxEntrySize = maxEntrySize; this.maxEntriesPerSegment = maxEntriesPerSegment; @@ -166,7 +166,7 @@ public final class SegmentedJournal implements Journal { * * @return the collection of journal segments */ - public Collection> segments() { + public Collection segments() { return segments.values(); } @@ -176,10 +176,19 @@ public final class SegmentedJournal implements Journal { * @param index the starting index * @return the journal segments starting with indexes greater than or equal to the given index */ - public Collection> segments(long index) { + public Collection segments(long index) { return segments.tailMap(index).values(); } + /** + * Returns serializer instance. + * + * @return serializer instance + */ + JournalSerializer serializer() { + return serializer; + } + /** * Returns the total size of the journal. * @@ -230,7 +239,7 @@ public final class SegmentedJournal implements Journal { */ private synchronized void open() { // Load existing log segments from disk. - for (JournalSegment segment : loadSegments()) { + for (JournalSegment segment : loadSegments()) { segments.put(segment.descriptor().index(), segment); } @@ -274,7 +283,7 @@ public final class SegmentedJournal implements Journal { * Resets the current segment, creating a new segment if necessary. */ private synchronized void resetCurrentSegment() { - JournalSegment lastSegment = getLastSegment(); + JournalSegment lastSegment = getLastSegment(); if (lastSegment != null) { currentSegment = lastSegment; } else { @@ -297,16 +306,16 @@ public final class SegmentedJournal implements Journal { * @param index the starting index of the journal * @return the first segment */ - JournalSegment resetSegments(long index) { + JournalSegment resetSegments(long index) { assertOpen(); // If the index already equals the first segment index, skip the reset. - JournalSegment firstSegment = getFirstSegment(); + JournalSegment firstSegment = getFirstSegment(); if (index == firstSegment.firstIndex()) { return firstSegment; } - for (JournalSegment segment : segments.values()) { + for (JournalSegment segment : segments.values()) { segment.close(); segment.delete(); } @@ -328,9 +337,9 @@ public final class SegmentedJournal implements Journal { * * @throws IllegalStateException if the segment manager is not open */ - JournalSegment getFirstSegment() { + JournalSegment getFirstSegment() { assertOpen(); - Map.Entry> segment = segments.firstEntry(); + Map.Entry segment = segments.firstEntry(); return segment != null ? segment.getValue() : null; } @@ -339,9 +348,9 @@ public final class SegmentedJournal implements Journal { * * @throws IllegalStateException if the segment manager is not open */ - JournalSegment getLastSegment() { + JournalSegment getLastSegment() { assertOpen(); - Map.Entry> segment = segments.lastEntry(); + Map.Entry segment = segments.lastEntry(); return segment != null ? segment.getValue() : null; } @@ -351,11 +360,11 @@ public final class SegmentedJournal implements Journal { * @return The next segment. * @throws IllegalStateException if the segment manager is not open */ - synchronized JournalSegment getNextSegment() { + synchronized JournalSegment getNextSegment() { assertOpen(); assertDiskSpace(); - JournalSegment lastSegment = getLastSegment(); + JournalSegment lastSegment = getLastSegment(); JournalSegmentDescriptor descriptor = JournalSegmentDescriptor.builder() .withId(lastSegment != null ? lastSegment.descriptor().id() + 1 : 1) .withIndex(currentSegment.lastIndex() + 1) @@ -375,8 +384,8 @@ public final class SegmentedJournal implements Journal { * @param index The segment index with which to look up the next segment. * @return The next segment for the given index. */ - JournalSegment getNextSegment(long index) { - Map.Entry> nextSegment = segments.higherEntry(index); + JournalSegment getNextSegment(long index) { + Map.Entry nextSegment = segments.higherEntry(index); return nextSegment != null ? nextSegment.getValue() : null; } @@ -386,7 +395,7 @@ public final class SegmentedJournal implements Journal { * @param index The index for which to return the segment. * @throws IllegalStateException if the segment manager is not open */ - synchronized JournalSegment getSegment(long index) { + synchronized JournalSegment getSegment(long index) { assertOpen(); // Check if the current segment contains the given index first in order to prevent an unnecessary map lookup. if (currentSegment != null && index > currentSegment.firstIndex()) { @@ -394,7 +403,7 @@ public final class SegmentedJournal implements Journal { } // If the index is in another segment, get the entry with the next lowest first index. - Map.Entry> segment = segments.floorEntry(index); + Map.Entry segment = segments.floorEntry(index); if (segment != null) { return segment.getValue(); } @@ -406,7 +415,7 @@ public final class SegmentedJournal implements Journal { * * @param segment The segment to remove. */ - synchronized void removeSegment(JournalSegment segment) { + synchronized void removeSegment(JournalSegment segment) { segments.remove(segment.firstIndex()); segment.close(); segment.delete(); @@ -416,7 +425,7 @@ public final class SegmentedJournal implements Journal { /** * Creates a new segment. */ - JournalSegment createSegment(JournalSegmentDescriptor descriptor) { + JournalSegment createSegment(JournalSegmentDescriptor descriptor) { File segmentFile = JournalSegmentFile.createSegmentFile(name, directory, descriptor.id()); RandomAccessFile raf; @@ -443,7 +452,7 @@ public final class SegmentedJournal implements Journal { } catch (IOException e) { } } - JournalSegment segment = newSegment(new JournalSegmentFile(segmentFile), descriptor); + JournalSegment segment = newSegment(new JournalSegmentFile(segmentFile), descriptor); LOG.debug("Created segment: {}", segment); return segment; } @@ -455,21 +464,21 @@ public final class SegmentedJournal implements Journal { * @param descriptor The segment descriptor. * @return The segment instance. */ - protected JournalSegment newSegment(JournalSegmentFile segmentFile, JournalSegmentDescriptor descriptor) { - return new JournalSegment<>(segmentFile, descriptor, storageLevel, maxEntrySize, indexDensity, namespace); + protected JournalSegment newSegment(JournalSegmentFile segmentFile, JournalSegmentDescriptor descriptor) { + return new JournalSegment(segmentFile, descriptor, storageLevel, maxEntrySize, indexDensity); } /** * Loads a segment. */ - private JournalSegment loadSegment(long segmentId) { + private JournalSegment loadSegment(long segmentId) { File segmentFile = JournalSegmentFile.createSegmentFile(name, directory, segmentId); ByteBuffer buffer = ByteBuffer.allocate(JournalSegmentDescriptor.BYTES); try (FileChannel channel = openChannel(segmentFile)) { channel.read(buffer); buffer.flip(); JournalSegmentDescriptor descriptor = new JournalSegmentDescriptor(buffer); - JournalSegment segment = newSegment(new JournalSegmentFile(segmentFile), descriptor); + JournalSegment segment = newSegment(new JournalSegmentFile(segmentFile), descriptor); LOG.debug("Loaded disk segment: {} ({})", descriptor.id(), segmentFile.getName()); return segment; } catch (IOException e) { @@ -490,11 +499,11 @@ public final class SegmentedJournal implements Journal { * * @return A collection of segments for the log. */ - protected Collection> loadSegments() { + protected Collection loadSegments() { // Ensure log directories are created. directory.mkdirs(); - TreeMap> segments = new TreeMap<>(); + TreeMap segments = new TreeMap<>(); // Iterate through all files in the log directory. for (File file : directory.listFiles(File::isFile)) { @@ -513,7 +522,7 @@ public final class SegmentedJournal implements Journal { JournalSegmentDescriptor descriptor = new JournalSegmentDescriptor(buffer); // Load the segment. - JournalSegment segment = loadSegment(descriptor.id()); + JournalSegment segment = loadSegment(descriptor.id()); // Add the segment to the segments list. LOG.debug("Found segment: {} ({})", segment.descriptor().id(), segmentFile.file().getName()); @@ -522,11 +531,11 @@ public final class SegmentedJournal implements Journal { } // Verify that all the segments in the log align with one another. - JournalSegment previousSegment = null; + JournalSegment previousSegment = null; boolean corrupted = false; - Iterator>> iterator = segments.entrySet().iterator(); + Iterator> iterator = segments.entrySet().iterator(); while (iterator.hasNext()) { - JournalSegment segment = iterator.next().getValue(); + JournalSegment segment = iterator.next().getValue(); if (previousSegment != null && previousSegment.lastIndex() != segment.firstIndex() - 1) { LOG.warn("Journal is inconsistent. {} is not aligned with prior segment {}", segment.file().file(), previousSegment.file().file()); corrupted = true; @@ -584,7 +593,7 @@ public final class SegmentedJournal implements Journal { * @return indicates whether a segment can be removed from the journal */ public boolean isCompactable(long index) { - Map.Entry> segmentEntry = segments.floorEntry(index); + Map.Entry segmentEntry = segments.floorEntry(index); return segmentEntry != null && segments.headMap(segmentEntry.getValue().firstIndex()).size() > 0; } @@ -595,7 +604,7 @@ public final class SegmentedJournal implements Journal { * @return the starting index of the last segment in the log */ public long getCompactableIndex(long index) { - Map.Entry> segmentEntry = segments.floorEntry(index); + Map.Entry segmentEntry = segments.floorEntry(index); return segmentEntry != null ? segmentEntry.getValue().firstIndex() : 0; } @@ -612,7 +621,7 @@ public final class SegmentedJournal implements Journal { final var compactSegments = segments.headMap(segmentEntry.getValue().firstIndex()); if (!compactSegments.isEmpty()) { LOG.debug("{} - Compacting {} segment(s)", name, compactSegments.size()); - for (JournalSegment segment : compactSegments.values()) { + for (JournalSegment segment : compactSegments.values()) { LOG.trace("Deleting segment: {}", segment); segment.close(); segment.delete(); diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalReader.java b/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalReader.java index cc0fe0d31c..42f40e0c72 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalReader.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalReader.java @@ -24,12 +24,12 @@ import static java.util.Objects.requireNonNull; sealed class SegmentedJournalReader implements JournalReader permits CommitsSegmentJournalReader { final SegmentedJournal journal; - private JournalSegment currentSegment; - private JournalSegmentReader currentReader; + private JournalSegment currentSegment; + private JournalSegmentReader currentReader; private Indexed currentEntry; private long nextIndex; - SegmentedJournalReader(final SegmentedJournal journal, final JournalSegment segment) { + SegmentedJournalReader(final SegmentedJournal journal, final JournalSegment segment) { this.journal = requireNonNull(journal); currentSegment = requireNonNull(segment); currentReader = segment.createReader(); @@ -99,7 +99,7 @@ sealed class SegmentedJournalReader implements JournalReader permits Commi */ private void rewind(final long index) { if (currentSegment.firstIndex() >= index) { - JournalSegment segment = journal.getSegment(index - 1); + JournalSegment segment = journal.getSegment(index - 1); if (segment != null) { currentReader.close(); @@ -113,8 +113,8 @@ sealed class SegmentedJournalReader implements JournalReader permits Commi @Override public Indexed tryNext() { - var next = currentReader.readEntry(nextIndex); - if (next == null) { + var buf = currentReader.readBytes(nextIndex); + if (buf == null) { final var nextSegment = journal.getNextSegment(currentSegment.firstIndex()); if (nextSegment == null || nextSegment.firstIndex() != nextIndex) { return null; @@ -124,15 +124,15 @@ sealed class SegmentedJournalReader implements JournalReader permits Commi currentSegment = nextSegment; currentReader = currentSegment.createReader(); - next = currentReader.readEntry(nextIndex); - if (next == null) { + buf = currentReader.readBytes(nextIndex); + if (buf == null) { return null; } } - nextIndex = nextIndex + 1; - currentEntry = next; - return next; + final var entry = journal.serializer().deserialize(buf); + currentEntry = new Indexed<>(nextIndex++, entry, buf.readableBytes()); + return currentEntry; } @Override diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalWriter.java b/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalWriter.java index a95622e553..9ff535284d 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalWriter.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalWriter.java @@ -22,8 +22,8 @@ import static com.google.common.base.Verify.verifyNotNull; */ final class SegmentedJournalWriter implements JournalWriter { private final SegmentedJournal journal; - private JournalSegment currentSegment; - private JournalSegmentWriter currentWriter; + private JournalSegment currentSegment; + private JournalSegmentWriter currentWriter; SegmentedJournalWriter(SegmentedJournal journal) { this.journal = journal; @@ -38,7 +38,12 @@ final class SegmentedJournalWriter implements JournalWriter { @Override public Indexed getLastEntry() { - return currentWriter.getLastEntry(); + final var lastWritten = currentWriter.getLastWritten(); + if (lastWritten == null) { + return null; + } + final E deserialized = journal.serializer().deserialize(lastWritten); + return new Indexed<>(currentWriter.getLastIndex(), deserialized, lastWritten.readableBytes()) ; } @Override @@ -70,9 +75,10 @@ final class SegmentedJournalWriter implements JournalWriter { @Override public Indexed append(T entry) { - var indexed = currentWriter.append(entry); - if (indexed != null) { - return indexed; + final var bytes = journal.serializer().serialize(entry); + var index = currentWriter.append(bytes); + if (index != null) { + return new Indexed<>(index, entry, bytes.readableBytes()); } // Slow path: we do not have enough capacity @@ -80,7 +86,8 @@ final class SegmentedJournalWriter implements JournalWriter { currentSegment.releaseWriter(); currentSegment = journal.getNextSegment(); currentWriter = currentSegment.acquireWriter(); - return verifyNotNull(currentWriter.append(entry)); + final var newIndex = verifyNotNull(currentWriter.append(bytes)); + return new Indexed<>(newIndex, entry, bytes.readableBytes()); } @Override diff --git a/features/odl-mdsal-clustering-commons/pom.xml b/features/odl-mdsal-clustering-commons/pom.xml index 5d1bbbeb51..0d98ae7098 100644 --- a/features/odl-mdsal-clustering-commons/pom.xml +++ b/features/odl-mdsal-clustering-commons/pom.xml @@ -39,6 +39,12 @@ xml features + + org.opendaylight.odlparent + odl-netty-4 + xml + features + org.opendaylight.odlparent odl-servlet-api diff --git a/features/odl-mdsal-clustering-commons/src/main/feature/feature.xml b/features/odl-mdsal-clustering-commons/src/main/feature/feature.xml index f4034164f1..7a41fc13bb 100644 --- a/features/odl-mdsal-clustering-commons/src/main/feature/feature.xml +++ b/features/odl-mdsal-clustering-commons/src/main/feature/feature.xml @@ -10,6 +10,7 @@ odl-apache-commons-lang3 odl-dropwizard-metrics + odl-netty-4 odl-servlet-api odl-yangtools-data odl-yangtools-codec -- 2.36.6 From 0107a12894fefcc986c66ab154c442b08019e628 Mon Sep 17 00:00:00 2001 From: Robert Varga Date: Tue, 16 Apr 2024 14:23:15 +0200 Subject: [PATCH 07/16] Fixup checkstyle Updated checkstyle is picking up a few new warnings. Fix them up. Change-Id: Ib5fa92b84c7e4570098a1bd1163f0947a86ed7a9 Signed-off-by: Robert Varga --- .../org/opendaylight/controller/cluster/example/Main.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/opendaylight/md-sal/sal-akka-raft-example/src/main/java/org/opendaylight/controller/cluster/example/Main.java b/opendaylight/md-sal/sal-akka-raft-example/src/main/java/org/opendaylight/controller/cluster/example/Main.java index ae8ea82295..871d3dfc2c 100644 --- a/opendaylight/md-sal/sal-akka-raft-example/src/main/java/org/opendaylight/controller/cluster/example/Main.java +++ b/opendaylight/md-sal/sal-akka-raft-example/src/main/java/org/opendaylight/controller/cluster/example/Main.java @@ -15,7 +15,6 @@ import java.io.InputStreamReader; import java.nio.charset.Charset; import java.util.Arrays; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Optional; import org.opendaylight.controller.cluster.example.messages.KeyValue; @@ -49,11 +48,10 @@ public final class Main { withoutPeer("example-3"), Optional.empty()), "example-3"); - List examples = Arrays.asList(example1Actor, example2Actor, example3Actor); + final var examples = Arrays.asList(example1Actor, example2Actor, example3Actor); - ActorRef clientActor = ACTOR_SYSTEM.actorOf(ClientActor.props(example1Actor)); - BufferedReader br = - new BufferedReader(new InputStreamReader(System.in, Charset.defaultCharset())); + final var clientActor = ACTOR_SYSTEM.actorOf(ClientActor.props(example1Actor)); + final var br = new BufferedReader(new InputStreamReader(System.in, Charset.defaultCharset())); System.out.println("Usage :"); System.out.println("s <1-3> to start a peer"); -- 2.36.6 From 285074c11981fdc96ba8922563ea5e74036e2afd Mon Sep 17 00:00:00 2001 From: Ruslan Kashapov Date: Fri, 29 Mar 2024 17:47:43 +0200 Subject: [PATCH 08/16] Segmented journal benchmark Performance test for sal-akka-segmented-journal moved and refactored to standalone benchmark module, making self-contained executable jar; 'benchmark' profile executes benchmark via maven build using current configuration (taken from sal-clustering-config) JIRA: CONTROLLER-2114 Change-Id: I108f7aa99b089bdc1ae12b99fcc6ff4bf53bbd38 Signed-off-by: Ruslan Kashapov --- benchmark/pom.xml | 1 + benchmark/segjournal-benchmark/pom.xml | 161 ++++++++++++ .../akka/segjournal/BenchmarkMain.java | 238 ++++++++++++++++++ .../akka/segjournal/BenchmarkUtils.java | 216 ++++++++++++++++ .../src/main/resources/logback.xml | 20 ++ .../akka/segjournal/PerformanceTest.java | 196 --------------- 6 files changed, 636 insertions(+), 196 deletions(-) create mode 100644 benchmark/segjournal-benchmark/pom.xml create mode 100644 benchmark/segjournal-benchmark/src/main/java/org/opendaylight/controller/akka/segjournal/BenchmarkMain.java create mode 100644 benchmark/segjournal-benchmark/src/main/java/org/opendaylight/controller/akka/segjournal/BenchmarkUtils.java create mode 100644 benchmark/segjournal-benchmark/src/main/resources/logback.xml delete mode 100644 opendaylight/md-sal/sal-akka-segmented-journal/src/test/java/org/opendaylight/controller/akka/segjournal/PerformanceTest.java diff --git a/benchmark/pom.xml b/benchmark/pom.xml index cf915b7bc2..b79dbd5b8c 100644 --- a/benchmark/pom.xml +++ b/benchmark/pom.xml @@ -29,5 +29,6 @@ and is available at http://www.eclipse.org/legal/epl-v10.html dsbenchmark ntfbenchmark rpcbenchmark + segjournal-benchmark diff --git a/benchmark/segjournal-benchmark/pom.xml b/benchmark/segjournal-benchmark/pom.xml new file mode 100644 index 0000000000..72197da341 --- /dev/null +++ b/benchmark/segjournal-benchmark/pom.xml @@ -0,0 +1,161 @@ + + + + 4.0.0 + + org.opendaylight.controller + mdsal-parent + 9.0.3-SNAPSHOT + ../../opendaylight/md-sal/parent + + + segjournal-benchmark + jar + + + true + true + true + + + + + com.github.spotbugs + spotbugs-annotations + true + + + org.eclipse.jdt + org.eclipse.jdt.annotation + + + net.sourceforge.argparse4j + argparse4j + + + org.slf4j + slf4j-api + + + ch.qos.logback + logback-classic + compile + + + commons-io + commons-io + + + io.dropwizard.metrics + metrics-core + + + org.opendaylight.controller + repackaged-akka + + + org.opendaylight.controller + sal-clustering-commons + + + org.opendaylight.controller + atomix-storage + + + org.opendaylight.controller + sal-akka-segmented-journal + + + org.opendaylight.controller + sal-clustering-config + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + false + + + + package + + shade + + + true + executable + + + *:* + + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + org.opendaylight.controller.akka.segjournal.BenchmarkMain + + + + + + + + + + + + benchmarks + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.1 + + + execute-segmented-journal-benchmark + integration-test + + exec + + + + + java + true + + -classpath + + + org.opendaylight.controller.akka.segjournal.BenchmarkMain + + --current + + -n100000 + + -p100K + + + + + + + + \ No newline at end of file diff --git a/benchmark/segjournal-benchmark/src/main/java/org/opendaylight/controller/akka/segjournal/BenchmarkMain.java b/benchmark/segjournal-benchmark/src/main/java/org/opendaylight/controller/akka/segjournal/BenchmarkMain.java new file mode 100644 index 0000000000..5c711088e3 --- /dev/null +++ b/benchmark/segjournal-benchmark/src/main/java/org/opendaylight/controller/akka/segjournal/BenchmarkMain.java @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2024 PANTHEON.tech s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.controller.akka.segjournal; + +import static org.opendaylight.controller.akka.segjournal.BenchmarkUtils.buildConfig; +import static org.opendaylight.controller.akka.segjournal.BenchmarkUtils.formatBytes; +import static org.opendaylight.controller.akka.segjournal.BenchmarkUtils.formatNanos; +import static org.opendaylight.controller.akka.segjournal.BenchmarkUtils.toMetricId; + +import akka.actor.ActorRef; +import akka.actor.ActorSystem; +import akka.persistence.AtomicWrite; +import akka.persistence.PersistentRepr; +import com.google.common.base.Stopwatch; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.io.Serializable; +import java.util.Optional; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import org.apache.commons.io.FileUtils; +import org.opendaylight.controller.akka.segjournal.BenchmarkUtils.BenchmarkConfig; +import org.opendaylight.controller.akka.segjournal.SegmentedJournalActor.WriteMessages; +import org.opendaylight.controller.cluster.common.actor.MeteringBehavior; +import org.opendaylight.controller.cluster.reporting.MetricsReporter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.concurrent.Future; + +public final class BenchmarkMain { + private static final String BENCHMARK = "benchmark"; + private static final Logger LOG = LoggerFactory.getLogger("benchmark"); + + public static void main(String[] args) { + final var config = buildConfig(args); + final var benchmark = new BenchmarkMain(config); + Runtime.getRuntime().addShutdownHook(new Thread(benchmark::shutdown)); + benchmark.execute(); + System.exit(0); + } + + private final BenchmarkConfig config; + private final ActorSystem system; + private final ScheduledExecutorService executor; + private ActorRef actor; + + private BenchmarkMain(BenchmarkConfig config) { + this.config = config; + system = ActorSystem.create(BENCHMARK); + executor = Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat("progress-check-%d").build()); + } + + void execute() { + LOG.info("Starting with settings"); + LOG.info("\tstorage : {}", config.storage()); + LOG.info("\tworking dir : {}", config.workingDir().getAbsolutePath()); + LOG.info("\tmaxEntrySize : {}", formatBytes(config.maxEntrySize())); + LOG.info("\tmaxSegmentSize : {}", formatBytes(config.maxSegmentSize())); + LOG.info("\tmaxUnflushedBytes : {}", formatBytes(config.maxUnflushedBytes())); + + final var minLoadSize = Math.round(config.payloadSize() * 0.8f); + final var maxLoadSize = Math.min(Math.round(config.payloadSize() * 1.2f), config.maxEntrySize()); + LOG.info("Preparing load"); + LOG.info("\tnumber of messages : {}", config.messagesNum()); + LOG.info("\tpayload size : {} .. {}", formatBytes(minLoadSize), formatBytes(maxLoadSize)); + + // reset metrics + final var metricsRegistry = MetricsReporter.getInstance(MeteringBehavior.DOMAIN).getMetricsRegistry(); + final var keys = metricsRegistry.getMetrics().keySet(); + keys.forEach(metricsRegistry::remove); + + // get actor + actor = system.actorOf( + SegmentedJournalActor.props("perf", config.workingDir(), config.storage(), + config.maxEntrySize(), config.maxSegmentSize(), config.maxUnflushedBytes())); + + // randomize payloads + final var random = ThreadLocalRandom.current(); + final var payloads = new Payload[1_000]; + for (int i = 0; i < payloads.length; ++i) { + final var bytes = new byte[random.nextInt(minLoadSize, maxLoadSize)]; + random.nextBytes(bytes); + payloads[i] = new Payload(bytes); + } + + // enable periodic check for completed writes + final var results = new ConcurrentLinkedQueue>>(); + final var progressReporter = + new ProgressReporter(executor, results, config.messagesNum(), 10, TimeUnit.SECONDS); + + // start async message writing + final var sw = Stopwatch.createStarted(); + for (int i = 0; i < config.messagesNum(); ++i) { + results.add(writeMessage(i, payloads[random.nextInt(payloads.length)])); + } + LOG.info("{} Messages sent to akka in {}", config.messagesNum(), sw); + + // await completion + try { + progressReporter.awaitCompletion(); + } catch (InterruptedException e) { + LOG.error("Interrupted", e); + } + LOG.info("Messages written in {}", sw.stop()); + + // report + LOG.info("Following metrics collected"); + // meters + metricsRegistry.getMeters().forEach((key, meter) -> { + LOG.info("Meter '{}'", toMetricId(key)); + LOG.info("\tCount = {}", meter.getCount()); + LOG.info("\tMean Rate = {}", meter.getMeanRate()); + LOG.info("\t1 Min Rate = {}", meter.getOneMinuteRate()); + LOG.info("\t5 Min Rate = {}", meter.getFiveMinuteRate()); + LOG.info("\t15 Min Rate = {}", meter.getFifteenMinuteRate()); + }); + // timers + metricsRegistry.getTimers().forEach((key, timer) -> { + LOG.info("Timer '{}'", toMetricId(key)); + final var snap = timer.getSnapshot(); + LOG.info("\tMin = {}", formatNanos(snap.getMin())); + LOG.info("\tMax = {}", formatNanos(snap.getMax())); + LOG.info("\tMean = {}", formatNanos(snap.getMean())); + LOG.info("\tStdDev = {}", formatNanos(snap.getStdDev())); + LOG.info("\tMedian = {}", formatNanos(snap.getMedian())); + LOG.info("\t75th = {}", formatNanos(snap.get75thPercentile())); + LOG.info("\t95th = {}", formatNanos(snap.get95thPercentile())); + LOG.info("\t98th = {}", formatNanos(snap.get98thPercentile())); + LOG.info("\t99th = {}", formatNanos(snap.get99thPercentile())); + LOG.info("\t99.9th = {}", formatNanos(snap.get999thPercentile())); + }); + // histograms + metricsRegistry.getHistograms().forEach((key, histogram) -> { + LOG.info("Histogram '{}'", toMetricId(key)); + final var snap = histogram.getSnapshot(); + LOG.info("\tMin = {}", snap.getMin()); + LOG.info("\tMax = {}", snap.getMax()); + LOG.info("\tMean = {}", snap.getMean()); + LOG.info("\tStdDev = {}", snap.getStdDev()); + LOG.info("\tMedian = {}", snap.getMedian()); + LOG.info("\t75th = {}", snap.get75thPercentile()); + LOG.info("\t95th = {}", snap.get95thPercentile()); + LOG.info("\t98th = {}", snap.get98thPercentile()); + LOG.info("\t99th = {}", snap.get99thPercentile()); + LOG.info("\t99.9th = {}", snap.get999thPercentile()); + }); + } + + Future> writeMessage(final long seqNum, final Payload payload) { + final var writeMessage = new WriteMessages(); + final var result = writeMessage.add(AtomicWrite.apply( + PersistentRepr.apply(payload, seqNum, BENCHMARK, null, false, ActorRef.noSender(), "uuid"))); + actor.tell(writeMessage, ActorRef.noSender()); + return result; + } + + void shutdown() { + LOG.info("shutting down ..."); + executor.shutdown(); + if (actor != null) { + system.stop(actor); + } + if (config.workingDir().exists()) { + FileUtils.deleteQuietly(config.workingDir()); + } + system.terminate(); + LOG.info("Done."); + } + + private static final class Payload implements Serializable { + @java.io.Serial + private static final long serialVersionUID = 1L; + final byte[] bytes; + + Payload(final byte[] bytes) { + this.bytes = bytes; + } + } + + private static final class ProgressReporter implements Runnable { + final ScheduledExecutorService executor; + final CountDownLatch latch = new CountDownLatch(1); + final Queue>> queue; + final long total; + final int checkInterval; + final TimeUnit timeUnit; + long completed; + long errors; + + ProgressReporter(final ScheduledExecutorService executor, final Queue>> queue, + final long total, final int checkInterval, final TimeUnit timeUnit) { + this.executor = executor; + this.queue = queue; + this.total = total; + this.checkInterval = checkInterval; + this.timeUnit = timeUnit; + scheduleNextCheck(); + } + + @Override + public void run() { + // release completed from the beginning of the queue + while (!queue.isEmpty() && queue.peek().isCompleted()) { + final var future = queue.poll(); + completed++; + if (!future.value().get().get().isEmpty()) { + errors++; + } + } + LOG.info("{} of {} = {}% messages written, {} in queue", + completed, total, completed * 100 / total, queue.size()); + if (total == completed) { + LOG.info("Check completed, errors found : {}", errors); + latch.countDown(); + return; + } + scheduleNextCheck(); + } + + void scheduleNextCheck() { + executor.schedule(this, checkInterval, timeUnit); + } + + void awaitCompletion() throws InterruptedException { + latch.await(); + } + } +} diff --git a/benchmark/segjournal-benchmark/src/main/java/org/opendaylight/controller/akka/segjournal/BenchmarkUtils.java b/benchmark/segjournal-benchmark/src/main/java/org/opendaylight/controller/akka/segjournal/BenchmarkUtils.java new file mode 100644 index 0000000000..30ff871a51 --- /dev/null +++ b/benchmark/segjournal-benchmark/src/main/java/org/opendaylight/controller/akka/segjournal/BenchmarkUtils.java @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2024 PANTHEON.tech s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.controller.akka.segjournal; + +import static org.opendaylight.controller.akka.segjournal.SegmentedFileJournal.STORAGE_MAX_ENTRY_SIZE; +import static org.opendaylight.controller.akka.segjournal.SegmentedFileJournal.STORAGE_MAX_ENTRY_SIZE_DEFAULT; +import static org.opendaylight.controller.akka.segjournal.SegmentedFileJournal.STORAGE_MAX_SEGMENT_SIZE; +import static org.opendaylight.controller.akka.segjournal.SegmentedFileJournal.STORAGE_MAX_SEGMENT_SIZE_DEFAULT; +import static org.opendaylight.controller.akka.segjournal.SegmentedFileJournal.STORAGE_MAX_UNFLUSHED_BYTES; +import static org.opendaylight.controller.akka.segjournal.SegmentedFileJournal.STORAGE_MEMORY_MAPPED; + +import com.google.common.base.Stopwatch; +import com.google.common.base.Ticker; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import io.atomix.storage.journal.StorageLevel; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; +import net.sourceforge.argparse4j.ArgumentParsers; +import net.sourceforge.argparse4j.impl.Arguments; +import net.sourceforge.argparse4j.inf.ArgumentParser; +import net.sourceforge.argparse4j.inf.ArgumentParserException; + +@SuppressWarnings("RegexpSinglelineJava") +final class BenchmarkUtils { + + static final String PROG_NAME = "segjourlan-benchmark"; + + static final String BENCHMARK_USE_CURRENT = "current"; + static final String BENCHMARK_NUMBER_OF_MESSAGES = "messages-num"; + static final String BENCHMARK_PAYLOAD_SIZE = "payload-size"; + static final String BENCHMARK_PAYLOAD_SIZE_DEFAULT = "10K"; + + static final String CURRENT_CONFIG_RESOURCE = "/initial/factory-akka.conf"; + static final String CURRENT_CONFIG_PATH = "odl-cluster-data.akka.persistence.journal.segmented-file"; + + private static final String[] BYTE_SFX = {"G", "M", "K"}; + private static final int[] BYTE_THRESH = {1024 * 1024 * 1024, 1024 * 1024, 1024}; + + record BenchmarkConfig(StorageLevel storage, File workingDir, int maxEntrySize, int maxSegmentSize, + int maxUnflushedBytes, int payloadSize, int messagesNum) { + } + + private BenchmarkUtils() { + // utility class + } + + static BenchmarkConfig buildConfig(final String[] args) { + final var parser = getArgumentParser(); + final var paramsMap = new HashMap(); + try { + parser.parseArgs(args, paramsMap); + } catch (ArgumentParserException e) { + parser.handleError(e); + System.exit(1); + return null; + } + return toConfig(paramsMap); + } + + private static ArgumentParser getArgumentParser() { + final var parser = ArgumentParsers.newArgumentParser(PROG_NAME).defaultHelp(true); + + parser.description("Performs asynchronous write to segmented journal, collects and prints variety of metrics"); + + parser.addArgument("--current") + .type(Boolean.class).setDefault(Boolean.FALSE) + .action(Arguments.storeConst()).setConst(Boolean.TRUE) + .dest(BENCHMARK_USE_CURRENT) + .help("indicates base configuration to be taken from current cluster configuration, " + + "all other arguments excepting 'requests' and 'payload size' will be ignored"); + + parser.addArgument("--memory-mapped") + .type(Boolean.class).setDefault(Boolean.FALSE) + .action(Arguments.storeConst()).setConst(Boolean.TRUE) + .dest(STORAGE_MEMORY_MAPPED) + .help("indicates mapping journal segments to memory, otherwise file system is used"); + + parser.addArgument("-e", "--max-entry-size") + .type(String.class).setDefault(formatBytes(STORAGE_MAX_ENTRY_SIZE_DEFAULT)) + .dest(STORAGE_MAX_ENTRY_SIZE) + .help("max entry size, bytes format"); + + parser.addArgument("-s", "--max-segment-size") + .type(String.class).setDefault(formatBytes(STORAGE_MAX_SEGMENT_SIZE_DEFAULT)) + .dest(STORAGE_MAX_SEGMENT_SIZE) + .help("max segment size, bytes "); + + parser.addArgument("-u", "--max-unflushed-bytes") + .type(String.class) + .dest(STORAGE_MAX_UNFLUSHED_BYTES) + .help("max unflushed bytes, bytes format, " + + "if not defined the value is taken from 'max-entry-size'"); + + parser.addArgument("-n", "--messages-num") + .type(Integer.class).required(true) + .dest(BENCHMARK_NUMBER_OF_MESSAGES) + .setDefault(10_000) + .help("number of messages to write"); + + parser.addArgument("-p", "--payload-size") + .type(String.class).setDefault(BENCHMARK_PAYLOAD_SIZE_DEFAULT) + .dest(BENCHMARK_PAYLOAD_SIZE) + .help("median for request payload size, bytes format supported, " + + "actual size is variable 80% to 120% from defined median value"); + + return parser; + } + + static BenchmarkConfig toConfig(final Map paramsMap) { + final var inputConfig = ConfigFactory.parseMap(paramsMap); + final var finalConfig = (Boolean) paramsMap.get(BENCHMARK_USE_CURRENT) + ? currentConfig().withFallback(inputConfig) : inputConfig; + + final var benchmarkConfig = new BenchmarkConfig( + finalConfig.getBoolean(STORAGE_MEMORY_MAPPED) ? StorageLevel.MAPPED : StorageLevel.DISK, + createTempDirectory(), + bytes(finalConfig, STORAGE_MAX_ENTRY_SIZE), + bytes(finalConfig, STORAGE_MAX_SEGMENT_SIZE), + finalConfig.hasPath(STORAGE_MAX_UNFLUSHED_BYTES) + ? bytes(finalConfig, STORAGE_MAX_UNFLUSHED_BYTES) : bytes(finalConfig, STORAGE_MAX_ENTRY_SIZE), + bytes(finalConfig, BENCHMARK_PAYLOAD_SIZE), + finalConfig.getInt(BENCHMARK_NUMBER_OF_MESSAGES) + ); + // validate + if (benchmarkConfig.payloadSize > benchmarkConfig.maxEntrySize) { + printAndExit("payloadSize should be less than maxEntrySize"); + } + return benchmarkConfig; + } + + private static int bytes(final Config config, final String key) { + final var bytesLong = config.getBytes(key); + if (bytesLong <= 0 || bytesLong > Integer.MAX_VALUE) { + printAndExit( + key + " value (" + bytesLong + ") is invalid, expected in range 1 .. " + Integer.MAX_VALUE); + } + return bytesLong.intValue(); + } + + static Config currentConfig() { + try (var in = BenchmarkUtils.class.getResourceAsStream(CURRENT_CONFIG_RESOURCE)) { + final var content = new String(in.readAllBytes(), StandardCharsets.UTF_8); + final var globalConfig = ConfigFactory.parseString(content); + final var currentConfig = globalConfig.getConfig(CURRENT_CONFIG_PATH); + System.out.println("Current configuration loaded from " + CURRENT_CONFIG_RESOURCE); + return currentConfig; + + } catch (IOException e) { + printAndExit("Error loading current configuration from resource " + CURRENT_CONFIG_RESOURCE, e); + return null; + } + } + + private static File createTempDirectory() { + try { + return Files.createTempDirectory(PROG_NAME).toFile(); + } catch (IOException e) { + printAndExit("Cannot create temp directory", e); + } + return null; + } + + private static void printAndExit(final String message) { + printAndExit(message, null); + } + + private static void printAndExit(final String message, final Exception exception) { + System.err.println(message); + if (exception != null) { + exception.printStackTrace(System.err); + } + System.exit(1); + } + + static String formatBytes(int bytes) { + for (int i = 0; i < 3; i++) { + if (bytes > BYTE_THRESH[i]) { + return bytes / BYTE_THRESH[i] + BYTE_SFX[i]; + } + } + return String.valueOf(bytes); + } + + static String formatNanos(final double nanos) { + return formatNanos(Math.round(nanos)); + } + + static String formatNanos(final long nanos) { + return Stopwatch.createStarted(new Ticker() { + boolean started; + + @Override + public long read() { + if (started) { + return nanos; + } + started = true; + return 0; + } + }).toString(); + } + + static String toMetricId(final String metricKey) { + return metricKey.substring(metricKey.lastIndexOf('.') + 1); + } +} diff --git a/benchmark/segjournal-benchmark/src/main/resources/logback.xml b/benchmark/segjournal-benchmark/src/main/resources/logback.xml new file mode 100644 index 0000000000..1ed4b9f412 --- /dev/null +++ b/benchmark/segjournal-benchmark/src/main/resources/logback.xml @@ -0,0 +1,20 @@ + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + \ No newline at end of file diff --git a/opendaylight/md-sal/sal-akka-segmented-journal/src/test/java/org/opendaylight/controller/akka/segjournal/PerformanceTest.java b/opendaylight/md-sal/sal-akka-segmented-journal/src/test/java/org/opendaylight/controller/akka/segjournal/PerformanceTest.java deleted file mode 100644 index 636a5e1d3f..0000000000 --- a/opendaylight/md-sal/sal-akka-segmented-journal/src/test/java/org/opendaylight/controller/akka/segjournal/PerformanceTest.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright (c) 2024 PANTHEON.tech, s.r.o. and others. All rights reserved. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License v1.0 which accompanies this distribution, - * and is available at http://www.eclipse.org/legal/epl-v10.html - */ -package org.opendaylight.controller.akka.segjournal; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import akka.actor.PoisonPill; -import akka.persistence.AtomicWrite; -import akka.persistence.PersistentRepr; -import akka.testkit.CallingThreadDispatcher; -import akka.testkit.javadsl.TestKit; -import com.codahale.metrics.Histogram; -import com.codahale.metrics.UniformReservoir; -import com.google.common.base.Stopwatch; -import com.google.common.base.Ticker; -import io.atomix.storage.journal.StorageLevel; -import java.io.File; -import java.io.Serializable; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ThreadLocalRandom; -import org.apache.commons.io.FileUtils; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.opendaylight.controller.akka.segjournal.SegmentedJournalActor.WriteMessages; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import scala.concurrent.Future; - -class PerformanceTest { - private static final class Payload implements Serializable { - @java.io.Serial - private static final long serialVersionUID = 1L; - - final byte[] bytes; - - Payload(final int size, final ThreadLocalRandom random) { - bytes = new byte[size]; - random.nextBytes(bytes); - } - } - - private static final class Request { - final WriteMessages write = new WriteMessages(); - final Future> future; - - Request(final AtomicWrite atomicWrite) { - future = write.add(atomicWrite); - } - } - - private static final Logger LOG = LoggerFactory.getLogger(PerformanceTest.class); - private static final File DIRECTORY = new File("target/sfj-perf"); - - private static ActorSystem SYSTEM; - - private TestKit kit; - private ActorRef actor; - - @BeforeAll - static void beforeClass() { - SYSTEM = ActorSystem.create("test"); - } - - @AfterAll - static void afterClass() { - TestKit.shutdownActorSystem(SYSTEM); - SYSTEM = null; - } - - @BeforeEach - void before() { - kit = new TestKit(SYSTEM); - FileUtils.deleteQuietly(DIRECTORY); - } - - @AfterEach - void after() { - if (actor != null) { - actor.tell(PoisonPill.getInstance(), ActorRef.noSender()); - } - FileUtils.deleteQuietly(DIRECTORY); - } - - @Disabled("Disable due to being an extensive time hog") - @ParameterizedTest - @MethodSource - void writeRequests(final StorageLevel storage, final int maxEntrySize, final int maxSegmentSize, - final int payloadSize, final int requestCount) { - LOG.info("Test {} entrySize={} segmentSize={} payload={} count={}", storage, maxEntrySize, maxSegmentSize, - payloadSize, requestCount); - - actor = kit.childActorOf( - SegmentedJournalActor.props("perf", DIRECTORY, storage, maxEntrySize, maxSegmentSize, maxEntrySize) - .withDispatcher(CallingThreadDispatcher.Id())); - - final var random = ThreadLocalRandom.current(); - final var sw = Stopwatch.createStarted(); - final var payloads = new Payload[1_000]; - for (int i = 0; i < payloads.length; ++i) { - payloads[i] = new Payload(payloadSize, random); - } - LOG.info("{} payloads created in {}", payloads.length, sw.stop()); - - sw.reset().start(); - final var requests = new Request[requestCount]; - for (int i = 0; i < requests.length; ++i) { - requests[i] = new Request(AtomicWrite.apply(PersistentRepr.apply(payloads[random.nextInt(payloads.length)], - i, "foo", null, false, kit.getRef(), "uuid"))); - } - LOG.info("{} requests created in {}", requests.length, sw.stop()); - - final var histogram = new Histogram(new UniformReservoir(requests.length)); - sw.reset().start(); - long started = System.nanoTime(); - for (var req : requests) { - actor.tell(req.write, ActorRef.noSender()); - assertTrue(req.future.isCompleted()); - assertTrue(req.future.value().get().get().isEmpty()); - - final long now = System.nanoTime(); - histogram.update(now - started); - started = now; - } - sw.stop(); - final var snap = histogram.getSnapshot(); - - LOG.info("{} requests completed in {}", requests.length, sw); - LOG.info("Minimum: {}", formatNanos(snap.getMin())); - LOG.info("Maximum: {}", formatNanos(snap.getMax())); - LOG.info("Mean: {}", formatNanos(snap.getMean())); - LOG.info("StdDev: {}", formatNanos(snap.getStdDev())); - LOG.info("Median: {}", formatNanos(snap.getMedian())); - LOG.info("75th: {}", formatNanos(snap.get75thPercentile())); - LOG.info("95th: {}", formatNanos(snap.get95thPercentile())); - LOG.info("98th: {}", formatNanos(snap.get98thPercentile())); - LOG.info("99th: {}", formatNanos(snap.get99thPercentile())); - LOG.info("99.9th: {}", formatNanos(snap.get999thPercentile())); - } - - private static List writeRequests() { - return List.of( - // DISK: - // 100K requests, 10K each, 16M max, 128M segment - Arguments.of(StorageLevel.DISK, 16 * 1024 * 1024, 128 * 1024 * 1024, 10_000, 100_000), - // 100K requests, 10K each, 1M max, 16M segment - Arguments.of(StorageLevel.DISK, 1024 * 1024, 16 * 1024 * 1024, 10_000, 100_000), - // 10K requests, 100K each, 1M max, 16M segment - Arguments.of(StorageLevel.DISK, 1024 * 1024, 16 * 1024 * 1024, 100_000, 10_000), - // 1K requests, 1M each, 1M max, 16M segment - Arguments.of(StorageLevel.DISK, 1024 * 1024, 16 * 1024 * 1024, 1_000_000, 1_000), - - // MAPPED: - // 100K requests, 10K each, 16M max, 128M segment - Arguments.of(StorageLevel.MAPPED, 16 * 1024 * 1024, 128 * 1024 * 1024, 10_000, 100_000), - // 100K requests, 10K each, 1M max, 16M segment - Arguments.of(StorageLevel.MAPPED, 1024 * 1024, 16 * 1024 * 1024, 10_000, 100_000), - // 10K requests, 100K each, 1M max, 16M segment - Arguments.of(StorageLevel.MAPPED, 1024 * 1024, 16 * 1024 * 1024, 100_000, 10_000), - // 1K requests, 1M each, 1M max, 16M segment - Arguments.of(StorageLevel.MAPPED, 1024 * 1024, 16 * 1024 * 1024, 1_000_000, 1_000)); - - } - - private static String formatNanos(final double nanos) { - return formatNanos(Math.round(nanos)); - } - - private static String formatNanos(final long nanos) { - return Stopwatch.createStarted(new Ticker() { - boolean started; - - @Override - public long read() { - if (started) { - return nanos; - } - started = true; - return 0; - } - }).toString(); - } -} -- 2.36.6 From a35bb2b0d7c688eaa83c38996373b1784e513694 Mon Sep 17 00:00:00 2001 From: Robert Varga Date: Wed, 27 Mar 2024 01:15:59 +0100 Subject: [PATCH 09/16] Factor out FileWriter interface We have two specializations of JournalSegmentWriter just to provide specialized write access. Factor out the require interface into FileWriter and specialize that instead of JournalSegmentWriter. JIRA: CONTROLLER-2100 Change-Id: Ie6a2497f49d7e1e00f32836cccbf0680b17b15e3 Signed-off-by: Robert Varga Signed-off-by: Ruslan Kashapov --- ...SegmentWriter.java => DiskFileWriter.java} | 76 ++++++----------- .../io/atomix/storage/journal/FileWriter.java | 81 +++++++++++++++++++ .../storage/journal/JournalSegment.java | 22 ++--- .../storage/journal/JournalSegmentWriter.java | 67 ++++++++------- ...gmentWriter.java => MappedFileWriter.java} | 67 +++++---------- 5 files changed, 174 insertions(+), 139 deletions(-) rename atomix-storage/src/main/java/io/atomix/storage/journal/{DiskJournalSegmentWriter.java => DiskFileWriter.java} (52%) create mode 100644 atomix-storage/src/main/java/io/atomix/storage/journal/FileWriter.java rename atomix-storage/src/main/java/io/atomix/storage/journal/{MappedJournalSegmentWriter.java => MappedFileWriter.java} (55%) diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/DiskJournalSegmentWriter.java b/atomix-storage/src/main/java/io/atomix/storage/journal/DiskFileWriter.java similarity index 52% rename from atomix-storage/src/main/java/io/atomix/storage/journal/DiskJournalSegmentWriter.java rename to atomix-storage/src/main/java/io/atomix/storage/journal/DiskFileWriter.java index 54feee1ada..5f468d46a1 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/DiskJournalSegmentWriter.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/DiskFileWriter.java @@ -1,6 +1,5 @@ /* - * Copyright 2017-2022 Open Networking Foundation and others. All rights reserved. - * Copyright (c) 2024 PANTHEON.tech, s.r.o. + * Copyright (c) 2024 PANTHEON.tech, s.r.o. and others. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,49 +17,30 @@ package io.atomix.storage.journal; import static io.atomix.storage.journal.SegmentEntry.HEADER_BYTES; -import io.atomix.storage.journal.index.JournalIndex; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; +import java.nio.file.Path; /** - * Segment writer. - *

- * The format of an entry in the log is as follows: - *

    - *
  • 64-bit index
    • - *
  • 8-bit boolean indicating whether a term change is contained in the entry
    • - *
  • 64-bit optional term
    • - *
  • 32-bit signed entry length, including the entry type ID
    • - *
  • 8-bit signed entry type ID
    • - *
  • n-bit entry bytes
    • - *
- * - * @author Jordan Halterman + * A {@link StorageLevel#DISK} {@link FileWriter}. */ -final class DiskJournalSegmentWriter extends JournalSegmentWriter { +final class DiskFileWriter extends FileWriter { private static final ByteBuffer ZERO_ENTRY_HEADER = ByteBuffer.wrap(new byte[HEADER_BYTES]); - private final JournalSegmentReader reader; + private final DiskFileReader reader; private final ByteBuffer buffer; - DiskJournalSegmentWriter(final FileChannel channel, final JournalSegment segment, final int maxEntrySize, - final JournalIndex index) { - super(channel, segment, maxEntrySize, index); - + DiskFileWriter(final Path path, final FileChannel channel, final int maxSegmentSize, final int maxEntrySize) { + super(path, channel, maxSegmentSize, maxEntrySize); buffer = DiskFileReader.allocateBuffer(maxSegmentSize, maxEntrySize); - reader = new JournalSegmentReader(segment, - new DiskFileReader(segment.file().file().toPath(), channel, buffer), maxEntrySize); - reset(0); + reader = new DiskFileReader(path, channel, buffer); } - DiskJournalSegmentWriter(final JournalSegmentWriter previous) { - super(previous); - - buffer = DiskFileReader.allocateBuffer(maxSegmentSize, maxEntrySize); - reader = new JournalSegmentReader(segment, - new DiskFileReader(segment.file().file().toPath(), channel, buffer), maxEntrySize); + @Override + DiskFileReader reader() { + return reader; } @Override @@ -69,18 +49,23 @@ final class DiskJournalSegmentWriter extends JournalSegmentWriter { } @Override - MappedJournalSegmentWriter toMapped() { - return new MappedJournalSegmentWriter(this); + MappedFileWriter toMapped() { + flush(); + return new MappedFileWriter(path, channel, maxSegmentSize, maxEntrySize); } @Override - DiskJournalSegmentWriter toFileChannel() { - return this; + DiskFileWriter toDisk() { + return null; } @Override - JournalSegmentReader reader() { - return reader; + void writeEmptyHeader(final int position) { + try { + channel.write(ZERO_ENTRY_HEADER.asReadOnlyBuffer(), position); + } catch (IOException e) { + throw new StorageException(e); + } } @Override @@ -97,23 +82,14 @@ final class DiskJournalSegmentWriter extends JournalSegmentWriter { } } - @Override - void writeEmptyHeader(final int position) { - try { - channel.write(ZERO_ENTRY_HEADER.asReadOnlyBuffer(), position); - } catch (IOException e) { - throw new StorageException(e); - } - } - @Override void flush() { - try { - if (channel.isOpen()) { + if (channel.isOpen()) { + try { channel.force(true); + } catch (IOException e) { + throw new StorageException(e); } - } catch (IOException e) { - throw new StorageException(e); } } diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/FileWriter.java b/atomix-storage/src/main/java/io/atomix/storage/journal/FileWriter.java new file mode 100644 index 0000000000..4ead89bfb3 --- /dev/null +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/FileWriter.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2024 PANTHEON.tech, s.r.o. and others. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.storage.journal; + +import static java.util.Objects.requireNonNull; + +import com.google.common.base.MoreObjects; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import org.eclipse.jdt.annotation.Nullable; + +/** + * An abstraction over how to write a {@link JournalSegmentFile}. + */ +abstract sealed class FileWriter permits DiskFileWriter, MappedFileWriter { + final Path path; + final FileChannel channel; + final int maxSegmentSize; + final int maxEntrySize; + + FileWriter(final Path path, final FileChannel channel, final int maxSegmentSize, final int maxEntrySize) { + this.path = requireNonNull(path); + this.channel = requireNonNull(channel); + this.maxSegmentSize = maxSegmentSize; + this.maxEntrySize = maxEntrySize; + } + + /** + * Return the internal {@link FileReader}. + * + * @return the internal FileReader + */ + abstract FileReader reader(); + + /** + * Write {@link SegmentEntry#HEADER_BYTES} worth of zeroes at specified position. + * + * @param position position to write to + */ + abstract void writeEmptyHeader(int position); + + abstract ByteBuffer startWrite(int position, int size); + + abstract void commitWrite(int position, ByteBuffer entry); + + /** + * Flushes written entries to disk. + */ + abstract void flush(); + + /** + * Closes this writer. + */ + abstract void close(); + + @Override + public final String toString() { + return MoreObjects.toStringHelper(this).add("path", path).toString(); + } + + abstract @Nullable MappedByteBuffer buffer(); + + abstract @Nullable MappedFileWriter toMapped(); + + abstract @Nullable DiskFileWriter toDisk(); +} diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegment.java b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegment.java index 45405aae68..02921bed2b 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegment.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegment.java @@ -48,11 +48,11 @@ final class JournalSegment implements AutoCloseable { private boolean open = true; JournalSegment( - JournalSegmentFile file, - JournalSegmentDescriptor descriptor, - StorageLevel storageLevel, - int maxEntrySize, - double indexDensity) { + final JournalSegmentFile file, + final JournalSegmentDescriptor descriptor, + final StorageLevel storageLevel, + final int maxEntrySize, + final double indexDensity) { this.file = file; this.descriptor = descriptor; this.storageLevel = storageLevel; @@ -64,10 +64,14 @@ final class JournalSegment implements AutoCloseable { } catch (IOException e) { throw new StorageException(e); } - writer = switch (storageLevel) { - case DISK -> new DiskJournalSegmentWriter(channel, this, maxEntrySize, journalIndex); - case MAPPED -> new MappedJournalSegmentWriter(channel, this, maxEntrySize, journalIndex).toFileChannel(); + + final var fileWriter = switch (storageLevel) { + case DISK -> new DiskFileWriter(file.file().toPath(), channel, descriptor.maxSegmentSize(), maxEntrySize); + case MAPPED -> new MappedFileWriter(file.file().toPath(), channel, descriptor.maxSegmentSize(), maxEntrySize); }; + writer = new JournalSegmentWriter(fileWriter, this, maxEntrySize, journalIndex) + // relinquish mapped memory + .toFileChannel(); } /** @@ -125,7 +129,7 @@ final class JournalSegment implements AutoCloseable { * @param index the index to lookup * @return the position of the given index or a lesser index, or {@code null} */ - @Nullable Position lookup(long index) { + @Nullable Position lookup(final long index) { return journalIndex.lookup(index); } diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegmentWriter.java b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegmentWriter.java index c2e0a258c9..f00e0daddd 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegmentWriter.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegmentWriter.java @@ -20,19 +20,17 @@ import static java.util.Objects.requireNonNull; import io.atomix.storage.journal.index.JournalIndex; import io.netty.buffer.ByteBuf; -import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; -import java.nio.channels.FileChannel; import java.util.zip.CRC32; import org.eclipse.jdt.annotation.NonNull; import org.eclipse.jdt.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -abstract sealed class JournalSegmentWriter permits DiskJournalSegmentWriter, MappedJournalSegmentWriter { +final class JournalSegmentWriter { private static final Logger LOG = LoggerFactory.getLogger(JournalSegmentWriter.class); - final @NonNull FileChannel channel; + private final FileWriter fileWriter; final @NonNull JournalSegment segment; private final @NonNull JournalIndex index; final int maxSegmentSize; @@ -42,17 +40,18 @@ abstract sealed class JournalSegmentWriter permits DiskJournalSegmentWriter, Map private Long lastIndex; private ByteBuf lastWritten; - JournalSegmentWriter(final FileChannel channel, final JournalSegment segment, final int maxEntrySize, + JournalSegmentWriter(final FileWriter fileWriter, final JournalSegment segment, final int maxEntrySize, final JournalIndex index) { - this.channel = requireNonNull(channel); + this.fileWriter = requireNonNull(fileWriter); this.segment = requireNonNull(segment); this.index = requireNonNull(index); maxSegmentSize = segment.descriptor().maxSegmentSize(); this.maxEntrySize = maxEntrySize; + // adjust lastEntry value + reset(0); } - JournalSegmentWriter(final JournalSegmentWriter previous) { - channel = previous.channel; + JournalSegmentWriter(final JournalSegmentWriter previous, final FileWriter fileWriter) { segment = previous.segment; index = previous.index; maxSegmentSize = previous.maxSegmentSize; @@ -60,6 +59,7 @@ abstract sealed class JournalSegmentWriter permits DiskJournalSegmentWriter, Map lastWritten = previous.lastWritten; lastIndex = previous.lastIndex; currentPosition = previous.currentPosition; + this.fileWriter = requireNonNull(fileWriter); } /** @@ -114,7 +114,7 @@ abstract sealed class JournalSegmentWriter permits DiskJournalSegmentWriter, Map } // allocate buffer and write data - final var writeBuffer = startWrite(position, length + HEADER_BYTES).position(HEADER_BYTES); + final var writeBuffer = fileWriter.startWrite(position, length + HEADER_BYTES).position(HEADER_BYTES); writeBuffer.put(buf.nioBuffer()); // Compute the checksum for the entry. @@ -123,7 +123,7 @@ abstract sealed class JournalSegmentWriter permits DiskJournalSegmentWriter, Map // Create a single byte[] in memory for the entire entry and write it as a batch to the underlying buffer. writeBuffer.putInt(0, length).putInt(Integer.BYTES, (int) crc32.getValue()); - commitWrite(position, writeBuffer.rewind()); + fileWriter.commitWrite(position, writeBuffer.rewind()); // Update the last entry with the correct index/term/length. currentPosition = nextPosition; @@ -134,10 +134,6 @@ abstract sealed class JournalSegmentWriter permits DiskJournalSegmentWriter, Map return index; } - abstract ByteBuffer startWrite(int position, int size); - - abstract void commitWrite(int position, ByteBuffer entry); - /** * Resets the head of the segment to the given index. * @@ -145,23 +141,21 @@ abstract sealed class JournalSegmentWriter permits DiskJournalSegmentWriter, Map */ final void reset(final long index) { // acquire ownership of cache and make sure reader does not see anything we've done once we're done - final var reader = reader(); - reader.invalidateCache(); + final var fileReader = fileWriter.reader(); try { - resetWithBuffer(reader, index); + resetWithBuffer(fileReader, index); } finally { // Make sure reader does not see anything we've done - reader.invalidateCache(); + fileReader.invalidateCache(); } } - abstract JournalSegmentReader reader(); - - private void resetWithBuffer(final JournalSegmentReader reader, final long index) { + private void resetWithBuffer(final FileReader fileReader, final long index) { long nextIndex = segment.firstIndex(); // Clear the buffer indexes and acquire ownership of the buffer currentPosition = JournalSegmentDescriptor.BYTES; + final var reader = new JournalSegmentReader(segment, fileReader, maxEntrySize); reader.setPosition(JournalSegmentDescriptor.BYTES); while (index == 0 || nextIndex <= index) { @@ -207,25 +201,22 @@ abstract sealed class JournalSegmentWriter permits DiskJournalSegmentWriter, Map } // Zero the entry header at current channel position. - writeEmptyHeader(currentPosition); + fileWriter.writeEmptyHeader(currentPosition); } - /** - * Write {@link SegmentEntry#HEADER_BYTES} worth of zeroes at specified position. - * - * @param position position to write to - */ - abstract void writeEmptyHeader(int position); - /** * Flushes written entries to disk. */ - abstract void flush(); + void flush() { + fileWriter.flush(); + } /** * Closes this writer. */ - abstract void close(); + void close() { + fileWriter.close(); + } /** * Returns the mapped buffer underlying the segment writer, or {@code null} if the writer does not have such a @@ -233,9 +224,17 @@ abstract sealed class JournalSegmentWriter permits DiskJournalSegmentWriter, Map * * @return the mapped buffer underlying the segment writer, or {@code null}. */ - abstract @Nullable MappedByteBuffer buffer(); + @Nullable MappedByteBuffer buffer() { + return fileWriter.buffer(); + } - abstract @NonNull MappedJournalSegmentWriter toMapped(); + @NonNull JournalSegmentWriter toMapped() { + final var newWriter = fileWriter.toMapped(); + return newWriter == null ? this : new JournalSegmentWriter(this, newWriter); + } - abstract @NonNull DiskJournalSegmentWriter toFileChannel(); + @NonNull JournalSegmentWriter toFileChannel() { + final var newWriter = fileWriter.toDisk(); + return newWriter == null ? this : new JournalSegmentWriter(this, newWriter); + } } diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/MappedJournalSegmentWriter.java b/atomix-storage/src/main/java/io/atomix/storage/journal/MappedFileWriter.java similarity index 55% rename from atomix-storage/src/main/java/io/atomix/storage/journal/MappedJournalSegmentWriter.java rename to atomix-storage/src/main/java/io/atomix/storage/journal/MappedFileWriter.java index 48d9e764f6..d1bbd7d37e 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/MappedJournalSegmentWriter.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/MappedFileWriter.java @@ -1,6 +1,5 @@ /* - * Copyright 2017-2022 Open Networking Foundation and others. All rights reserved. - * Copyright (c) 2024 PANTHEON.tech, s.r.o. + * Copyright (c) 2024 PANTHEON.tech, s.r.o. and others. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,51 +15,27 @@ */ package io.atomix.storage.journal; -import io.atomix.storage.journal.index.JournalIndex; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; +import java.nio.file.Path; import org.eclipse.jdt.annotation.NonNull; /** - * Segment writer. - *

- * The format of an entry in the log is as follows: - *

    - *
  • 64-bit index
    • - *
  • 8-bit boolean indicating whether a term change is contained in the entry
    • - *
  • 64-bit optional term
    • - *
  • 32-bit signed entry length, including the entry type ID
    • - *
  • 8-bit signed entry type ID
    • - *
  • n-bit entry bytes
    • - *
- * - * @author Jordan Halterman + * A {@link StorageLevel#MAPPED} {@link FileWriter}. */ -final class MappedJournalSegmentWriter extends JournalSegmentWriter { +final class MappedFileWriter extends FileWriter { private final @NonNull MappedByteBuffer mappedBuffer; - private final JournalSegmentReader reader; + private final MappedFileReader reader; private final ByteBuffer buffer; - MappedJournalSegmentWriter(final FileChannel channel, final JournalSegment segment, final int maxEntrySize, - final JournalIndex index) { - super(channel, segment, maxEntrySize, index); - - mappedBuffer = mapBuffer(channel, maxSegmentSize); - buffer = mappedBuffer.slice(); - reader = new JournalSegmentReader(segment, new MappedFileReader(segment.file().file().toPath(), mappedBuffer), - maxEntrySize); - reset(0); - } - - MappedJournalSegmentWriter(final JournalSegmentWriter previous) { - super(previous); + MappedFileWriter(final Path path, final FileChannel channel, final int maxSegmentSize, final int maxEntrySize) { + super(path, channel, maxSegmentSize, maxEntrySize); mappedBuffer = mapBuffer(channel, maxSegmentSize); buffer = mappedBuffer.slice(); - reader = new JournalSegmentReader(segment, new MappedFileReader(segment.file().file().toPath(), mappedBuffer), - maxEntrySize); + reader = new MappedFileReader(path, mappedBuffer); } private static @NonNull MappedByteBuffer mapBuffer(final FileChannel channel, final int maxSegmentSize) { @@ -72,24 +47,30 @@ final class MappedJournalSegmentWriter extends JournalSegmentWriter { } @Override - @NonNull MappedByteBuffer buffer() { + MappedFileReader reader() { + return reader; + } + + @Override + MappedByteBuffer buffer() { return mappedBuffer; } @Override - MappedJournalSegmentWriter toMapped() { - return this; + MappedFileWriter toMapped() { + return null; } @Override - DiskJournalSegmentWriter toFileChannel() { + DiskFileWriter toDisk() { close(); - return new DiskJournalSegmentWriter(this); + return new DiskFileWriter(path, channel, maxSegmentSize, maxEntrySize); } @Override - JournalSegmentReader reader() { - return reader; + void writeEmptyHeader(final int position) { + // Note: we issue a single putLong() instead of two putInt()s. + buffer.putLong(position, 0L); } @Override @@ -102,12 +83,6 @@ final class MappedJournalSegmentWriter extends JournalSegmentWriter { // No-op, buffer is write-through } - @Override - void writeEmptyHeader(final int position) { - // Note: we issue a single putLong() instead of two putInt()s. - buffer.putLong(position, 0L); - } - @Override void flush() { mappedBuffer.force(); -- 2.36.6 From f2a918522421164b7ca3cc06e1863cd30f759a5f Mon Sep 17 00:00:00 2001 From: Robert Varga Date: Sun, 21 Apr 2024 14:04:40 +0200 Subject: [PATCH 10/16] Cap index to sequenceNr before tryNext() Rather than relying on Indexed.index(), which requires us to attempt to read the next entry, use a simple getNextIndex() check. JIRA: CONTROLLER-2115 Change-Id: Id1d7638dda0d909b8383af8f1e201fb9e7f192e0 Signed-off-by: Robert Varga --- .../controller/akka/segjournal/DataJournalV0.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opendaylight/md-sal/sal-akka-segmented-journal/src/main/java/org/opendaylight/controller/akka/segjournal/DataJournalV0.java b/opendaylight/md-sal/sal-akka-segmented-journal/src/main/java/org/opendaylight/controller/akka/segjournal/DataJournalV0.java index 243a064b80..fdb8c97d6a 100644 --- a/opendaylight/md-sal/sal-akka-segmented-journal/src/main/java/org/opendaylight/controller/akka/segjournal/DataJournalV0.java +++ b/opendaylight/md-sal/sal-akka-segmented-journal/src/main/java/org/opendaylight/controller/akka/segjournal/DataJournalV0.java @@ -90,9 +90,9 @@ final class DataJournalV0 extends DataJournal { private void handleReplayMessages(final JournalReader reader, final ReplayMessages message) { int count = 0; - while (count < message.max) { + while (count < message.max && reader.getNextIndex() <= message.toSequenceNr) { final var next = reader.tryNext(); - if (next == null || next.index() > message.toSequenceNr) { + if (next == null) { break; } -- 2.36.6 From 5eaf0a77b90c04d4a26f403c1fbf824bf8d6ed0f Mon Sep 17 00:00:00 2001 From: Robert Varga Date: Sun, 21 Apr 2024 14:51:20 +0200 Subject: [PATCH 11/16] Remove JournalReader.getCurrentEntry() JournalReader.currentEntry is just a single-entry cache which is not used anywhere but tests. Remove the public method and private field. While we are at it, make SegmentedJournalReader.tryNext() state index management more obvious. JIRA: CONTROLLER-2115 Change-Id: If92293e8393e71a4c1f402a0370973872a9593ac Signed-off-by: Robert Varga --- .../atomix/storage/journal/JournalReader.java | 7 ------ .../journal/SegmentedJournalReader.java | 19 +++++---------- .../storage/journal/AbstractJournalTest.java | 23 ++++--------------- 3 files changed, 11 insertions(+), 38 deletions(-) diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalReader.java b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalReader.java index 700f40d1b6..40b700d070 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalReader.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalReader.java @@ -44,13 +44,6 @@ public interface JournalReader extends AutoCloseable { */ long getFirstIndex(); - /** - * Returns the last read entry. - * - * @return The last read entry. - */ - Indexed getCurrentEntry(); - /** * Returns the next reader index. * diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalReader.java b/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalReader.java index 42f40e0c72..0a137090d8 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalReader.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalReader.java @@ -26,7 +26,6 @@ sealed class SegmentedJournalReader implements JournalReader permits Commi private JournalSegment currentSegment; private JournalSegmentReader currentReader; - private Indexed currentEntry; private long nextIndex; SegmentedJournalReader(final SegmentedJournal journal, final JournalSegment segment) { @@ -34,7 +33,6 @@ sealed class SegmentedJournalReader implements JournalReader permits Commi currentSegment = requireNonNull(segment); currentReader = segment.createReader(); nextIndex = currentSegment.firstIndex(); - currentEntry = null; } @Override @@ -42,11 +40,6 @@ sealed class SegmentedJournalReader implements JournalReader permits Commi return journal.getFirstSegment().firstIndex(); } - @Override - public final Indexed getCurrentEntry() { - return currentEntry; - } - @Override public final long getNextIndex() { return nextIndex; @@ -59,7 +52,6 @@ sealed class SegmentedJournalReader implements JournalReader permits Commi currentSegment = journal.getFirstSegment(); currentReader = currentSegment.createReader(); nextIndex = currentSegment.firstIndex(); - currentEntry = null; } @Override @@ -113,10 +105,11 @@ sealed class SegmentedJournalReader implements JournalReader permits Commi @Override public Indexed tryNext() { - var buf = currentReader.readBytes(nextIndex); + final var index = nextIndex; + var buf = currentReader.readBytes(index); if (buf == null) { final var nextSegment = journal.getNextSegment(currentSegment.firstIndex()); - if (nextSegment == null || nextSegment.firstIndex() != nextIndex) { + if (nextSegment == null || nextSegment.firstIndex() != index) { return null; } @@ -124,15 +117,15 @@ sealed class SegmentedJournalReader implements JournalReader permits Commi currentSegment = nextSegment; currentReader = currentSegment.createReader(); - buf = currentReader.readBytes(nextIndex); + buf = currentReader.readBytes(index); if (buf == null) { return null; } } final var entry = journal.serializer().deserialize(buf); - currentEntry = new Indexed<>(nextIndex++, entry, buf.readableBytes()); - return currentEntry; + nextIndex = index + 1; + return new Indexed<>(index, entry, buf.readableBytes()); } @Override diff --git a/atomix-storage/src/test/java/io/atomix/storage/journal/AbstractJournalTest.java b/atomix-storage/src/test/java/io/atomix/storage/journal/AbstractJournalTest.java index 14e59e5d7a..58ae35d711 100644 --- a/atomix-storage/src/test/java/io/atomix/storage/journal/AbstractJournalTest.java +++ b/atomix-storage/src/test/java/io/atomix/storage/journal/AbstractJournalTest.java @@ -16,6 +16,7 @@ */ package io.atomix.storage.journal; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -119,14 +120,12 @@ public abstract class AbstractJournalTest { var entry1 = reader.tryNext(); assertNotNull(entry1); assertEquals(1, entry1.index()); - assertEquals(entry1, reader.getCurrentEntry()); // Test reading a second entry assertEquals(2, reader.getNextIndex()); var entry2 = reader.tryNext(); assertNotNull(entry2); assertEquals(2, entry2.index()); - assertEquals(entry2, reader.getCurrentEntry()); assertEquals(3, reader.getNextIndex()); assertNull(reader.tryNext()); @@ -135,13 +134,11 @@ public abstract class AbstractJournalTest { entry1 = reader.tryNext(); assertNotNull(entry1); assertEquals(1, entry1.index()); - assertEquals(entry1, reader.getCurrentEntry()); assertEquals(2, reader.getNextIndex()); entry2 = reader.tryNext(); assertNotNull(entry2); assertEquals(2, entry2.index()); - assertEquals(entry2, reader.getCurrentEntry()); assertNull(reader.tryNext()); // Reset the reader. @@ -152,13 +149,11 @@ public abstract class AbstractJournalTest { entry1 = reader.tryNext(); assertNotNull(entry1); assertEquals(1, entry1.index()); - assertEquals(entry1, reader.getCurrentEntry()); assertEquals(2, reader.getNextIndex()); entry2 = reader.tryNext(); assertNotNull(entry2); assertEquals(2, entry2.index()); - assertEquals(entry2, reader.getCurrentEntry()); assertNull(reader.tryNext()); // Truncate the journal and write a different entry. @@ -173,14 +168,10 @@ public abstract class AbstractJournalTest { // Reset the reader to a specific index and read the last entry again. reader.reset(2); - final var current = reader.getCurrentEntry(); - assertNotNull(current); - assertEquals(1, current.index()); assertEquals(2, reader.getNextIndex()); entry2 = reader.tryNext(); assertNotNull(entry2); assertEquals(2, entry2.index()); - assertEquals(entry2, reader.getCurrentEntry()); assertNull(reader.tryNext()); } } @@ -267,19 +258,17 @@ public abstract class AbstractJournalTest { var entry = reader.tryNext(); assertNotNull(entry); assertEquals(i, entry.index()); - assertEquals(32, entry.entry().bytes().length); + assertArrayEquals(ENTRY.bytes(), entry.entry().bytes()); reader.reset(i); entry = reader.tryNext(); assertNotNull(entry); assertEquals(i, entry.index()); - assertEquals(32, entry.entry().bytes().length); + assertArrayEquals(ENTRY.bytes(), entry.entry().bytes()); if (i > 6) { reader.reset(i - 5); - final var current = reader.getCurrentEntry(); - assertNotNull(current); - assertEquals(i - 6, current.index()); assertEquals(i - 5, reader.getNextIndex()); + assertNotNull(reader.tryNext()); reader.reset(i + 1); } @@ -291,7 +280,7 @@ public abstract class AbstractJournalTest { entry = reader.tryNext(); assertNotNull(entry); assertEquals(i, entry.index()); - assertEquals(32, entry.entry().bytes().length); + assertArrayEquals(ENTRY.bytes(), entry.entry().bytes()); } } } @@ -358,13 +347,11 @@ public abstract class AbstractJournalTest { journal.compact(entriesPerSegment * 5 + 1); - assertNull(uncommittedReader.getCurrentEntry()); assertEquals(entriesPerSegment * 5 + 1, uncommittedReader.getNextIndex()); var entry = uncommittedReader.tryNext(); assertNotNull(entry); assertEquals(entriesPerSegment * 5 + 1, entry.index()); - assertNull(committedReader.getCurrentEntry()); assertEquals(entriesPerSegment * 5 + 1, committedReader.getNextIndex()); entry = committedReader.tryNext(); assertNotNull(entry); -- 2.36.6 From 903e61b1eb9496d74b6c330cca05caaf30cd8158 Mon Sep 17 00:00:00 2001 From: Robert Varga Date: Sun, 21 Apr 2024 15:28:18 +0200 Subject: [PATCH 12/16] Refactor JournalReader.tryNext() We are usually translating an entry to an internal representation, encode that in the API signature. Since we have an API method to carry Indexed.* fields, we reduce our reliance on this DTO. This leads to more efficient advance during reset, as we do not need to instantiate Indexed instance. JIRA: CONTROLLER-2115 Change-Id: I1d399e14997fb93b100397afab785778e47f985b Signed-off-by: Robert Varga --- .../journal/CommitsSegmentJournalReader.java | 9 +- .../atomix/storage/journal/JournalReader.java | 26 +++- .../storage/journal/SegmentedJournal.java | 2 +- .../journal/SegmentedJournalReader.java | 23 +++- .../storage/journal/AbstractJournalTest.java | 120 ++++++++---------- .../akka/segjournal/DataJournalV0.java | 26 ++-- 6 files changed, 118 insertions(+), 88 deletions(-) diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/CommitsSegmentJournalReader.java b/atomix-storage/src/main/java/io/atomix/storage/journal/CommitsSegmentJournalReader.java index 65f4de6496..767e67fa46 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/CommitsSegmentJournalReader.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/CommitsSegmentJournalReader.java @@ -15,16 +15,19 @@ */ package io.atomix.storage.journal; +import org.eclipse.jdt.annotation.NonNullByDefault; + /** * A {@link JournalReader} traversing only committed entries. */ +@NonNullByDefault final class CommitsSegmentJournalReader extends SegmentedJournalReader { - CommitsSegmentJournalReader(SegmentedJournal journal, JournalSegment segment) { + CommitsSegmentJournalReader(final SegmentedJournal journal, final JournalSegment segment) { super(journal, segment); } @Override - public Indexed tryNext() { - return getNextIndex() <= journal.getCommitIndex() ? super.tryNext() : null; + public T tryNext(final EntryMapper mapper) { + return getNextIndex() <= journal.getCommitIndex() ? super.tryNext(mapper) : null; } } diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalReader.java b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalReader.java index 40b700d070..a3c6ea5366 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalReader.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalReader.java @@ -15,6 +15,7 @@ */ package io.atomix.storage.journal; +import org.eclipse.jdt.annotation.NonNullByDefault; import org.eclipse.jdt.annotation.Nullable; /** @@ -22,6 +23,7 @@ import org.eclipse.jdt.annotation.Nullable; * * @author Jordan Halterman */ +@NonNullByDefault public interface JournalReader extends AutoCloseable { /** * Raft log reader mode. @@ -37,6 +39,25 @@ public interface JournalReader extends AutoCloseable { COMMITS, } + /** + * A journal entry processor. Responsible for transforming entries into their internal representation. + * + * @param Entry type + * @param Internal representation type + */ + @FunctionalInterface + interface EntryMapper { + /** + * Process an entry. + * + * @param index entry index + * @param entry entry itself + * @param size entry size + * @return resulting internal representation + */ + T mapEntry(long index, E entry, int size); + } + /** * Returns the first index in the journal. * @@ -54,9 +75,10 @@ public interface JournalReader extends AutoCloseable { /** * Try to move to the next entry. * - * @return The next entry in the reader, or {@code null} if there is no next entry. + * @param mapper callback to be invoked for the entry + * @return processed entry, or {@code null} */ - @Nullable Indexed tryNext(); + @Nullable T tryNext(EntryMapper mapper); /** * Resets the reader to the start. diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournal.java b/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournal.java index 7289d3deef..4b2f0334aa 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournal.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournal.java @@ -226,7 +226,7 @@ public final class SegmentedJournal implements Journal { // Forward reader to specified index long next = reader.getNextIndex(); - while (index > next && reader.tryNext() != null) { + while (index > next && reader.tryAdvance()) { next = reader.getNextIndex(); } diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalReader.java b/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalReader.java index 0a137090d8..a5deb6382e 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalReader.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalReader.java @@ -18,10 +18,15 @@ package io.atomix.storage.journal; import static java.util.Objects.requireNonNull; +import org.eclipse.jdt.annotation.NonNull; + /** * A {@link JournalReader} traversing all entries. */ sealed class SegmentedJournalReader implements JournalReader permits CommitsSegmentJournalReader { + // Marker non-null object for tryAdvance() + private static final @NonNull Object ADVANCED = new Object(); + final SegmentedJournal journal; private JournalSegment currentSegment; @@ -64,7 +69,7 @@ sealed class SegmentedJournalReader implements JournalReader permits Commi if (index < nextIndex) { rewind(index); } else if (index > nextIndex) { - while (index > nextIndex && tryNext() != null) { + while (index > nextIndex && tryAdvance()) { // Nothing else } } else { @@ -81,7 +86,7 @@ sealed class SegmentedJournalReader implements JournalReader permits Commi nextIndex = currentSegment.firstIndex(); currentReader.setPosition(JournalSegmentDescriptor.BYTES); } - while (nextIndex < index && tryNext() != null) { + while (nextIndex < index && tryAdvance()) { // Nothing else } } @@ -104,7 +109,7 @@ sealed class SegmentedJournalReader implements JournalReader permits Commi } @Override - public Indexed tryNext() { + public T tryNext(final EntryMapper mapper) { final var index = nextIndex; var buf = currentReader.readBytes(index); if (buf == null) { @@ -124,8 +129,18 @@ sealed class SegmentedJournalReader implements JournalReader permits Commi } final var entry = journal.serializer().deserialize(buf); + final var ret = requireNonNull(mapper.mapEntry(index, entry, buf.readableBytes())); nextIndex = index + 1; - return new Indexed<>(index, entry, buf.readableBytes()); + return ret; + } + + /** + * Try to move to the next entry. + * + * @return {@code true} if there was a next entry and this reader has moved to it + */ + final boolean tryAdvance() { + return tryNext((index, entry, size) -> ADVANCED) != null; } @Override diff --git a/atomix-storage/src/test/java/io/atomix/storage/journal/AbstractJournalTest.java b/atomix-storage/src/test/java/io/atomix/storage/journal/AbstractJournalTest.java index 58ae35d711..97d7d54c20 100644 --- a/atomix-storage/src/test/java/io/atomix/storage/journal/AbstractJournalTest.java +++ b/atomix-storage/src/test/java/io/atomix/storage/journal/AbstractJournalTest.java @@ -30,6 +30,8 @@ import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.List; +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jdt.annotation.Nullable; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -110,69 +112,60 @@ public abstract class AbstractJournalTest { assertEquals(2, writer.getNextIndex()); writer.append(ENTRY); reader.reset(2); - indexed = reader.tryNext(); - assertNotNull(indexed); + indexed = assertNext(reader); assertEquals(2, indexed.index()); - assertNull(reader.tryNext()); + assertNoNext(reader); // Test reading an entry reader.reset(); - var entry1 = reader.tryNext(); - assertNotNull(entry1); + var entry1 = assertNext(reader); assertEquals(1, entry1.index()); // Test reading a second entry assertEquals(2, reader.getNextIndex()); - var entry2 = reader.tryNext(); - assertNotNull(entry2); + var entry2 = assertNext(reader); assertEquals(2, entry2.index()); assertEquals(3, reader.getNextIndex()); - assertNull(reader.tryNext()); + assertNoNext(reader); // Test opening a new reader and reading from the journal. reader = journal.openReader(1); - entry1 = reader.tryNext(); - assertNotNull(entry1); + entry1 = assertNext(reader); assertEquals(1, entry1.index()); assertEquals(2, reader.getNextIndex()); - entry2 = reader.tryNext(); - assertNotNull(entry2); + entry2 = assertNext(reader); assertEquals(2, entry2.index()); - assertNull(reader.tryNext()); + assertNoNext(reader); // Reset the reader. reader.reset(); // Test opening a new reader and reading from the journal. reader = journal.openReader(1); - entry1 = reader.tryNext(); - assertNotNull(entry1); + entry1 = assertNext(reader); assertEquals(1, entry1.index()); assertEquals(2, reader.getNextIndex()); - entry2 = reader.tryNext(); - assertNotNull(entry2); + entry2 = assertNext(reader); assertEquals(2, entry2.index()); - assertNull(reader.tryNext()); + assertNoNext(reader); // Truncate the journal and write a different entry. writer.truncate(1); assertEquals(2, writer.getNextIndex()); writer.append(ENTRY); reader.reset(2); - indexed = reader.tryNext(); - assertNotNull(indexed); + indexed = assertNext(reader); assertEquals(2, indexed.index()); // Reset the reader to a specific index and read the last entry again. reader.reset(2); assertEquals(2, reader.getNextIndex()); - entry2 = reader.tryNext(); - assertNotNull(entry2); + entry2 = assertNext(reader); assertEquals(2, entry2.index()); - assertNull(reader.tryNext()); + assertNoNext(reader); } } @@ -189,8 +182,7 @@ public abstract class AbstractJournalTest { assertEquals(0, writer.getLastIndex()); writer.append(ENTRY); - var indexed = reader.tryNext(); - assertNotNull(indexed); + var indexed = assertNext(reader); assertEquals(1, indexed.index()); writer.reset(1); assertEquals(0, writer.getLastIndex()); @@ -198,8 +190,7 @@ public abstract class AbstractJournalTest { assertEquals(1, writer.getLastIndex()); assertEquals(1, writer.getLastEntry().index()); - indexed = reader.tryNext(); - assertNotNull(indexed); + indexed = assertNext(reader); assertEquals(1, indexed.index()); writer.truncate(0); @@ -209,8 +200,7 @@ public abstract class AbstractJournalTest { assertEquals(1, writer.getLastIndex()); assertEquals(1, writer.getLastEntry().index()); - indexed = reader.tryNext(); - assertNotNull(indexed); + indexed = assertNext(reader); assertEquals(1, indexed.index()); } } @@ -227,21 +217,18 @@ public abstract class AbstractJournalTest { } for (int j = 1; j <= i - 2; j++) { - final var indexed = reader.tryNext(); - assertNotNull(indexed); - assertEquals(j, indexed.index()); + assertEquals(j, assertNext(reader).index()); } writer.truncate(i - 2); - assertNull(reader.tryNext()); + assertNoNext(reader); assertEquals(i - 1, writer.append(new TestEntry(32)).index()); assertEquals(i, writer.append(new TestEntry(32)).index()); - Indexed entry = reader.tryNext(); - assertNotNull(entry); + var entry = assertNext(reader); assertEquals(i - 1, entry.index()); - entry = reader.tryNext(); + entry = assertNext(reader); assertNotNull(entry); assertEquals(i, entry.index()); } @@ -255,30 +242,27 @@ public abstract class AbstractJournalTest { for (int i = 1; i <= entriesPerSegment * 5; i++) { writer.append(ENTRY); - var entry = reader.tryNext(); - assertNotNull(entry); + var entry = assertNext(reader); assertEquals(i, entry.index()); assertArrayEquals(ENTRY.bytes(), entry.entry().bytes()); reader.reset(i); - entry = reader.tryNext(); - assertNotNull(entry); + entry = assertNext(reader); assertEquals(i, entry.index()); assertArrayEquals(ENTRY.bytes(), entry.entry().bytes()); if (i > 6) { reader.reset(i - 5); assertEquals(i - 5, reader.getNextIndex()); - assertNotNull(reader.tryNext()); + assertNext(reader); reader.reset(i + 1); } writer.truncate(i - 1); writer.append(ENTRY); - assertNotNull(reader.tryNext()); + assertNext(reader); reader.reset(i); - entry = reader.tryNext(); - assertNotNull(entry); + entry = assertNext(reader); assertEquals(i, entry.index()); assertArrayEquals(ENTRY.bytes(), entry.entry().bytes()); } @@ -293,17 +277,15 @@ public abstract class AbstractJournalTest { for (int i = 1; i <= entriesPerSegment * 5; i++) { writer.append(ENTRY); - assertNull(reader.tryNext()); + assertNoNext(reader); writer.commit(i); - var entry = reader.tryNext(); - assertNotNull(entry); + var entry = assertNext(reader); assertEquals(i, entry.index()); - assertEquals(32, entry.entry().bytes().length); + assertArrayEquals(ENTRY.bytes(), entry.entry().bytes()); reader.reset(i); - entry = reader.tryNext(); - assertNotNull(entry); + entry = assertNext(reader); assertEquals(i, entry.index()); - assertEquals(32, entry.entry().bytes().length); + assertArrayEquals(ENTRY.bytes(), entry.entry().bytes()); } } } @@ -323,38 +305,34 @@ public abstract class AbstractJournalTest { assertEquals(1, committedReader.getNextIndex()); // This creates asymmetry, as uncommitted reader will move one step ahead... - assertNotNull(uncommittedReader.tryNext()); + assertNext(uncommittedReader); assertEquals(2, uncommittedReader.getNextIndex()); - assertNull(committedReader.tryNext()); + assertNoNext(committedReader); assertEquals(1, committedReader.getNextIndex()); writer.commit(entriesPerSegment * 9); // ... so here we catch up ... - assertNotNull(committedReader.tryNext()); + assertNext(committedReader); assertEquals(2, committedReader.getNextIndex()); // ... and continue from the second entry for (int i = 2; i <= entriesPerSegment * 2.5; i++) { - var entry = uncommittedReader.tryNext(); - assertNotNull(entry); + var entry = assertNext(uncommittedReader); assertEquals(i, entry.index()); - entry = committedReader.tryNext(); - assertNotNull(entry); + entry = assertNext(committedReader); assertEquals(i, entry.index()); } journal.compact(entriesPerSegment * 5 + 1); assertEquals(entriesPerSegment * 5 + 1, uncommittedReader.getNextIndex()); - var entry = uncommittedReader.tryNext(); - assertNotNull(entry); + var entry = assertNext(uncommittedReader); assertEquals(entriesPerSegment * 5 + 1, entry.index()); assertEquals(entriesPerSegment * 5 + 1, committedReader.getNextIndex()); - entry = committedReader.tryNext(); - assertNotNull(entry); + entry = assertNext(committedReader); assertEquals(entriesPerSegment * 5 + 1, entry.index()); } } @@ -387,9 +365,7 @@ public abstract class AbstractJournalTest { // Ensure the reader starts at the first physical index in the journal. assertEquals(entriesPerSegment + 1, reader.getNextIndex()); assertEquals(reader.getFirstIndex(), reader.getNextIndex()); - final var indexed = reader.tryNext(); - assertNotNull(indexed); - assertEquals(entriesPerSegment + 1, indexed.index()); + assertEquals(entriesPerSegment + 1, assertNext(reader).index()); assertEquals(entriesPerSegment + 2, reader.getNextIndex()); } } @@ -413,4 +389,18 @@ public abstract class AbstractJournalTest { }); } } + + private static @NonNull Indexed assertNext(final JournalReader reader) { + final var ret = tryNext(reader); + assertNotNull(ret); + return ret; + } + + private static void assertNoNext(final JournalReader reader) { + assertNull(tryNext(reader)); + } + + private static @Nullable Indexed tryNext(final JournalReader reader) { + return reader.tryNext(Indexed::new); + } } diff --git a/opendaylight/md-sal/sal-akka-segmented-journal/src/main/java/org/opendaylight/controller/akka/segjournal/DataJournalV0.java b/opendaylight/md-sal/sal-akka-segmented-journal/src/main/java/org/opendaylight/controller/akka/segjournal/DataJournalV0.java index fdb8c97d6a..ad4c110bc8 100644 --- a/opendaylight/md-sal/sal-akka-segmented-journal/src/main/java/org/opendaylight/controller/akka/segjournal/DataJournalV0.java +++ b/opendaylight/md-sal/sal-akka-segmented-journal/src/main/java/org/opendaylight/controller/akka/segjournal/DataJournalV0.java @@ -91,22 +91,22 @@ final class DataJournalV0 extends DataJournal { private void handleReplayMessages(final JournalReader reader, final ReplayMessages message) { int count = 0; while (count < message.max && reader.getNextIndex() <= message.toSequenceNr) { - final var next = reader.tryNext(); - if (next == null) { + final var repr = reader.tryNext((index, entry, size) -> { + LOG.trace("{}: replay index={} entry={}", persistenceId, index, entry); + updateLargestSize(size); + if (entry instanceof FromPersistence fromPersistence) { + return fromPersistence.toRepr(persistenceId, index); + } + throw new VerifyException("Unexpected entry " + entry); + }); + + if (repr == null) { break; } - LOG.trace("{}: replay {}", persistenceId, next); - updateLargestSize(next.size()); - final var entry = next.entry(); - if (entry instanceof FromPersistence fromPersistence) { - final var repr = fromPersistence.toRepr(persistenceId, next.index()); - LOG.debug("{}: replaying {}", persistenceId, repr); - message.replayCallback.accept(repr); - count++; - } else { - throw new VerifyException("Unexpected entry " + entry); - } + LOG.debug("{}: replaying {}", persistenceId, repr); + message.replayCallback.accept(repr); + count++; } LOG.debug("{}: successfully replayed {} entries", persistenceId, count); } -- 2.36.6 From e1577ac90a997cc9e4839fa33f7fe60051c1fc7e Mon Sep 17 00:00:00 2001 From: Robert Varga Date: Mon, 22 Apr 2024 10:20:31 +0200 Subject: [PATCH 13/16] Use Netty to clean mapped buffers Netty offers PlatformDependent.freeDirectBuffer(), use that to clean our buffers. Change-Id: Id737c9e951bc1c77376133d7053287e2840bafc6 Signed-off-by: Robert Varga --- atomix-storage/pom.xml | 4 + .../atomix/storage/journal/BufferCleaner.java | 145 ------------------ .../io/atomix/storage/journal/Cleaner.java | 28 ---- .../storage/journal/MappedFileWriter.java | 7 +- 4 files changed, 6 insertions(+), 178 deletions(-) delete mode 100644 atomix-storage/src/main/java/io/atomix/storage/journal/BufferCleaner.java delete mode 100644 atomix-storage/src/main/java/io/atomix/storage/journal/Cleaner.java diff --git a/atomix-storage/pom.xml b/atomix-storage/pom.xml index 8fc7ca3709..bb07137137 100644 --- a/atomix-storage/pom.xml +++ b/atomix-storage/pom.xml @@ -42,6 +42,10 @@ io.netty netty-buffer + + io.netty + netty-common + org.eclipse.jdt org.eclipse.jdt.annotation diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/BufferCleaner.java b/atomix-storage/src/main/java/io/atomix/storage/journal/BufferCleaner.java deleted file mode 100644 index 8244e5743c..0000000000 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/BufferCleaner.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright 2019-2022 Open Networking Foundation and others. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.atomix.storage.journal; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.nio.ByteBuffer; -import java.security.AccessController; -import java.security.PrivilegedAction; -import java.util.Objects; - -import static java.lang.invoke.MethodHandles.constant; -import static java.lang.invoke.MethodHandles.dropArguments; -import static java.lang.invoke.MethodHandles.filterReturnValue; -import static java.lang.invoke.MethodHandles.guardWithTest; -import static java.lang.invoke.MethodHandles.lookup; -import static java.lang.invoke.MethodType.methodType; - -/** - * Utility class which allows explicit calls to the DirectByteBuffer cleaner method instead of relying on GC. - */ -public class BufferCleaner { - - private static final Logger LOGGER = LoggerFactory.getLogger(BufferCleaner.class); - - /** - * Reference to a Cleaner that does unmapping; no-op if not supported. - */ - private static final Cleaner CLEANER; - - static { - final Object hack = AccessController.doPrivileged((PrivilegedAction) BufferCleaner::unmapHackImpl); - if (hack instanceof Cleaner) { - CLEANER = (Cleaner) hack; - LOGGER.debug("java.nio.DirectByteBuffer.cleaner(): available"); - } else { - CLEANER = (ByteBuffer buffer) -> { - // noop - }; - LOGGER.debug("java.nio.DirectByteBuffer.cleaner(): unavailable: {}", hack); - } - } - - private static Object unmapHackImpl() { - final MethodHandles.Lookup lookup = lookup(); - try { - try { - // *** sun.misc.Unsafe unmapping (Java 9+) *** - final Class unsafeClass = Class.forName("sun.misc.Unsafe"); - // first check if Unsafe has the right method, otherwise we can give up - // without doing any security critical stuff: - final MethodHandle unmapper = lookup.findVirtual(unsafeClass, "invokeCleaner", - methodType(void.class, ByteBuffer.class)); - // fetch the unsafe instance and bind it to the virtual MH: - final Field f = unsafeClass.getDeclaredField("theUnsafe"); - f.setAccessible(true); - final Object theUnsafe = f.get(null); - return newBufferCleaner(ByteBuffer.class, unmapper.bindTo(theUnsafe)); - } catch (SecurityException se) { - // rethrow to report errors correctly (we need to catch it here, as we also catch RuntimeException below!): - throw se; - } catch (ReflectiveOperationException | RuntimeException e) { - // *** sun.misc.Cleaner unmapping (Java 8) *** - final Class directBufferClass = Class.forName("java.nio.DirectByteBuffer"); - - final Method m = directBufferClass.getMethod("cleaner"); - m.setAccessible(true); - final MethodHandle directBufferCleanerMethod = lookup.unreflect(m); - final Class cleanerClass = directBufferCleanerMethod.type().returnType(); - - /* "Compile" a MH that basically is equivalent to the following code: - * void unmapper(ByteBuffer byteBuffer) { - * sun.misc.Cleaner cleaner = ((java.nio.DirectByteBuffer) byteBuffer).cleaner(); - * if (Objects.nonNull(cleaner)) { - * cleaner.clean(); - * } else { - * noop(cleaner); // the noop is needed because MethodHandles#guardWithTest always needs ELSE - * } - * } - */ - final MethodHandle cleanMethod = lookup.findVirtual(cleanerClass, "clean", methodType(void.class)); - final MethodHandle nonNullTest = lookup.findStatic(Objects.class, "nonNull", methodType(boolean.class, Object.class)) - .asType(methodType(boolean.class, cleanerClass)); - final MethodHandle noop = dropArguments(constant(Void.class, null).asType(methodType(void.class)), 0, cleanerClass); - final MethodHandle unmapper = filterReturnValue(directBufferCleanerMethod, guardWithTest(nonNullTest, cleanMethod, noop)) - .asType(methodType(void.class, ByteBuffer.class)); - return newBufferCleaner(directBufferClass, unmapper); - } - } catch (SecurityException se) { - return "Unmapping is not supported, because not all required permissions are given to the Lucene JAR file: " - + se + " [Please grant at least the following permissions: RuntimePermission(\"accessClassInPackage.sun.misc\") " - + " and ReflectPermission(\"suppressAccessChecks\")]"; - } catch (ReflectiveOperationException | RuntimeException e) { - return "Unmapping is not supported on this platform, because internal Java APIs are not compatible with this Atomix version: " + e; - } - } - - private static Cleaner newBufferCleaner(final Class unmappableBufferClass, final MethodHandle unmapper) { - return (ByteBuffer buffer) -> { - if (!buffer.isDirect()) { - return; - } - if (!unmappableBufferClass.isInstance(buffer)) { - throw new IllegalArgumentException("buffer is not an instance of " + unmappableBufferClass.getName()); - } - final Throwable error = AccessController.doPrivileged((PrivilegedAction) () -> { - try { - unmapper.invokeExact(buffer); - return null; - } catch (Throwable t) { - return t; - } - }); - if (error != null) { - throw new IOException("Unable to unmap the mapped buffer", error); - } - }; - } - - /** - * Free {@link ByteBuffer} if possible. - */ - public static void freeBuffer(ByteBuffer buffer) throws IOException { - CLEANER.freeBuffer(buffer); - } -} diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/Cleaner.java b/atomix-storage/src/main/java/io/atomix/storage/journal/Cleaner.java deleted file mode 100644 index d81268038f..0000000000 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/Cleaner.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2017-2022 Open Networking Foundation and others. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.atomix.storage.journal; - -import java.io.IOException; -import java.nio.ByteBuffer; - -@FunctionalInterface -interface Cleaner { - - /** - * Free {@link ByteBuffer} if possible. - */ - void freeBuffer(ByteBuffer buffer) throws IOException; -} diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/MappedFileWriter.java b/atomix-storage/src/main/java/io/atomix/storage/journal/MappedFileWriter.java index d1bbd7d37e..47f26ba151 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/MappedFileWriter.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/MappedFileWriter.java @@ -15,6 +15,7 @@ */ package io.atomix.storage.journal; +import io.netty.util.internal.PlatformDependent; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; @@ -91,10 +92,6 @@ final class MappedFileWriter extends FileWriter { @Override void close() { flush(); - try { - BufferCleaner.freeBuffer(mappedBuffer); - } catch (IOException e) { - throw new StorageException(e); - } + PlatformDependent.freeDirectBuffer(mappedBuffer); } } -- 2.36.6 From 4ba8766b812c7df0e8fc63cf672eb032ba875672 Mon Sep 17 00:00:00 2001 From: Oleksandr Zharov Date: Mon, 11 Dec 2023 17:08:51 +0100 Subject: [PATCH 14/16] Remove bierman02 terminology from dev guide Updated dev guide terminology after bierman02 removal. JIRA: CONTROLLER-2088 Change-Id: I6e77b54c4c2fe1629b4a1014944c26110735a075 Signed-off-by: Oleksandr Zharov --- docs/dev-guide.rst | 72 ++++++++++++++++++++++---------------------- docs/images/Get.png | Bin 35496 -> 42623 bytes 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/docs/dev-guide.rst b/docs/dev-guide.rst index 686cfefcde..7e8d867cfe 100644 --- a/docs/dev-guide.rst +++ b/docs/dev-guide.rst @@ -874,7 +874,7 @@ RESTCONF operations overview .. note:: - | Each request must start with the URI /restconf. + | Each request must start with the URI /rests. | RESTCONF listens on port 8080 for HTTP requests. RESTCONF supports **OPTIONS**, **GET**, **PUT**, **POST**, and @@ -901,16 +901,16 @@ Identifier can represent a data node which is a list or container yang built-in type. If the data node is a list, there must be defined keys of the list behind the data node name for example, - //. + =,. - | The format : has to be used in this case as well: | Module A has node A1. Module B augments node A1 by adding node X. Module C augments node A1 by adding node X. For clarity, it has to be known which node is X (for example: C:X). For more details about - encoding, see: `RESTCONF 02 - Encoding YANG Instance Identifiers in + encoding, see: `RESTCONF RFC 8040 - Encoding YANG Instance Identifiers in the Request - URI. `__ + URI. `__ Mount point ~~~~~~~~~~~ @@ -927,29 +927,29 @@ Mount point HTTP methods ~~~~~~~~~~~~ -OPTIONS /restconf -^^^^^^^^^^^^^^^^^ +OPTIONS /rests +^^^^^^^^^^^^^^ - Returns the XML description of the resources with the required request and response media types in Web Application Description Language (WADL) -GET /restconf/config/ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +GET /rests/data/?content=config +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Returns a data node from the Config datastore. - points to a data node which must be retrieved. -GET /restconf/operational/ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +GET /rests/data/?content=nonconfig +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- Returns the value of the data node from the Operational datastore. +- Returns the value of the data node from the non-configuration datastore. - points to a data node which must be retrieved. -PUT /restconf/config/ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +PUT /rests/data/ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Updates or creates data in the Config datastore and returns the state about success. @@ -960,7 +960,7 @@ PUT /restconf/config/ :: - PUT http://:8080/restconf/config/module1:foo/bar + PUT http://:8080/rests/data/module1:foo/bar Content-Type: applicaton/xml … @@ -970,14 +970,14 @@ PUT /restconf/config/ :: - PUT http://:8080/restconf/config/module1:foo1/foo2/yang-ext:mount/module2:foo/bar + PUT http://:8080/rests/data/module1:foo1/foo2/yang-ext:mount/module2:foo/bar Content-Type: applicaton/xml … -POST /restconf/config -^^^^^^^^^^^^^^^^^^^^^ +POST /rests/data +^^^^^^^^^^^^^^^^ - Creates the data if it does not exist @@ -985,7 +985,7 @@ POST /restconf/config :: - POST URL: http://localhost:8080/restconf/config/ + POST URL: http://localhost:8080/rests/data/ content-type: application/yang.data+json JSON payload: @@ -998,8 +998,8 @@ POST /restconf/config } } -POST /restconf/config/ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +POST /rests/data/ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Creates the data if it does not exist in the Config datastore, and returns the state about success. @@ -1013,7 +1013,7 @@ POST /restconf/config/ :: - POST http://:8080/restconf/config/module1:foo + POST http://:8080/rests/data/module1:foo Content-Type: applicaton/xml/ … @@ -1023,14 +1023,14 @@ POST /restconf/config/ :: - http://:8080/restconf/config/module1:foo1/foo2/yang-ext:mount/module2:foo + http://:8080/rests/data/module1:foo1/foo2/yang-ext:mount/module2:foo Content-Type: applicaton/xml … -POST /restconf/operations/: -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +POST /rests/operations/: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Invokes RPC. @@ -1046,7 +1046,7 @@ POST /restconf/operations/: :: - POST http://:8080/restconf/operations/module1:fooRpc + POST http://:8080/rests/operations/module1:fooRpc Content-Type: applicaton/xml Accept: applicaton/xml @@ -1062,7 +1062,7 @@ POST /restconf/operations/: :: - POST http://localhost:8080/restconf/operations/toaster:make-toast + POST http://localhost:8080/rests/operations/toaster:make-toast Content-Type: application/yang.data+json { "input" : @@ -1077,8 +1077,8 @@ POST /restconf/operations/: Even though this is a default for the toasterToastType value in the yang, you still need to define it. -DELETE /restconf/config/ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +DELETE /rests/data/ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Removes the data node in the Config datastore and returns the state about success. @@ -1086,7 +1086,7 @@ DELETE /restconf/config/ - points to a data node which must be removed. More information is available in the `RESTCONF -RFC `__. +RFC `__. How RESTCONF works ~~~~~~~~~~~~~~~~~~ @@ -1126,8 +1126,8 @@ CompositeNode GET in action ~~~~~~~~~~~~~ -Figure 1 shows the GET operation with URI restconf/config/M:N where M is -the module name, and N is the node name. +Figure 1 shows the GET operation with URI rests/data/M:N?content=config +where M is the module name, and N is the node name. .. figure:: ./images/Get.png :alt: Get @@ -1154,7 +1154,7 @@ the module name, and N is the node name. PUT in action ~~~~~~~~~~~~~ -Figure 2 shows the PUT operation with the URI restconf/config/M:N where +Figure 2 shows the PUT operation with the URI rests/data/M:N where M is the module name, and N is the node name. Data is sent in the request either in the XML or JSON format. @@ -1192,7 +1192,7 @@ Something practical :: Operation: POST - URI: http://192.168.11.1:8080/restconf/config/opendaylight-inventory:nodes/node/openflow:1/table/2 + URI: http://192.168.11.1:8080/rests/data/opendaylight-inventory:nodes/node=openflow:1/table=2 Content-Type: application/xml :: @@ -1247,7 +1247,7 @@ Something practical :: Operation: PUT - URI: http://192.168.11.1:8080/restconf/config/opendaylight-inventory:nodes/node/openflow:1/table/2/flow/111 + URI: http://192.168.11.1:8080/rests/data/opendaylight-inventory:nodes/node=openflow:1/table=2/flow=111 Content-Type: application/xml :: @@ -1302,7 +1302,7 @@ Something practical :: Operation: GET - URI: http://192.168.11.1:8080/restconf/config/opendaylight-inventory:nodes/node/openflow:1/table/2/flow/111 + URI: http://192.168.11.1:8080/rests/data/opendaylight-inventory:nodes/node=openflow:1/table=2/flow=111?content=config Accept: application/xml | **HTTP response** @@ -1357,7 +1357,7 @@ Something practical :: Operation: DELETE - URI: http://192.168.11.1:8080/restconf/config/opendaylight-inventory:nodes/node/openflow:1/table/2/flow/111 + URI: http://192.168.11.1:8080/rests/data/opendaylight-inventory:nodes/node=openflow:1/table=2/flow=111 | **HTTP response** diff --git a/docs/images/Get.png b/docs/images/Get.png index 5c1f48445c29a15e6d75a000d830082ce37cd7c1..74f1a92e88dfd7b919e06ad73b5ce759d29c1051 100644 GIT binary patch literal 42623 zcma%j1#p|qvSr9Ilg!M_%yukWW@ct)W=PBoF~-cycFYhn#W6EOVrG^tdzJs)d*5!o zt*xDsN+Z>L`g(eLTGQR;mw;j$kI~cDsSGr*MIZoEehO6=qD~O{w&a! zx2`IZz&AC&2oIreK3Iy%i@teNACLHK3yrT+_^L9&dq-h>S*t4xC^2U zLRLeopTb0kQPzP&lL5^5zMi$Y?ss{!TpO(Tj#Bc82wMD6U^)KW``Y&Q zHJe9U{_pKN2j#C9x4NE&$a#5N2Sacc&yo6@KAau#oXrlhe!K~Hk(Q36|2xp{?R=x! zX7gN!Va+C9m^IzcBOYvhq}_KAsj;!KJywX?(%;r;`{AEqlzKgl_4SV{aE;(t5KcdC z9uI(vi>s`vN{tFSofwuppVXt()EhTv$S+noStE^}`U4YOoEz9i?!F;t`{Rz$RJ1j- z4bF~@o!F;l{YU<1Uk z{NL*BD~L80G{UYN|9h_m!NzwMPlJDZdakhlW0>(BLU1g!ReM{&{Je|)+iUyt5h}sI z*G@PDIzg`=ComrgpiTGsCqnuu&flKv1kwLff`17&6#iFe)VYO*dT8@rt@^wi@VCdc zxYi4m^Zyw2|2MV&;}X<#^I~IDXtT}He%lB!`P-{$3x>-4Z`(sPo*w*HA8A+YnfoP6 z=V`C9()q@#o8BZnwOAI&>5wjry`L>Cp3+1-KR-`PM<;4wLG2LeFJkbpYsM7{74j0h z<;52{r>6u5fPsNdOdR+5^Bb-~Pp5y@cdI7gI`F2aQp<_08eu(<^7&t;ULv%Un{8P2 z5wGO7tgK2!Y@x3znDT#WF`%$EU`Cj){Cw0V!dNFPrc0!P%xL%czAS&CpX+&Vv&H2B z9%@uAkGqJ~BnBh50{yJrKlC9(1<}^(&~1X$?eDgoW`DS%ZoQ5nDfo%6NL77lrVJ_?TNbI;g<3BIM9Y9OO1cM`IA)3;Cr^g*1C- zH2quuPiH5t_s$aT5dDY9&6~`YwTdBrZ0MFzz?)>XkKo!A>J6$47F!HlFR`4*g{m1k z?bO1Cb{%9k{*yx!9jr><_80Sdp8VGU_aJ-|e*VBthirV4^yXoE>HtTS#urE@ z{yTu8APAXFZanGn8r23)>uPv)UTL`RKNI6ZDLlXzB-ch}!{24~zM?Iv@y0H%7&sRH z*+*5#B90Viv|7?pUo4fb|w!Oldg2ErJlM%gA2_qWOc6rj0T19%!`Li zQOl6tp#_b<7O$zx6;@Afac?8Z%Ja`AbQU{e(KNbbIi63KFqnSJWhR{ck1Y>j{j+-; zad4}xH^PBeX9FX`in~Ljj|ycP=B5^88a4gAxzoCP z){vkg0r@nF_|&eoB%0CI=Q`yIrG7u6v%BGw)+{00z4?rExEn$n;cV8;l0TxEf^wOE zm=v@BuTuVX73nCGQe@nzrk5EEbzV={Wb{&c@Lw&Aye<)S+BlCG+~4SOZ^YXV&Xr0h z+v^4CRSn}m3HCLU^j_p=&`vuw1^_;ZC4IjgF;^!kkI+s}ZiN}21261oSms*{dq0{F zjvkbkco~E*zoZkTaqP|OWWpEO^W>g+XEr&maBptzUnE0qlND)}&1hO3^~>8A+-V+S z48@b6h^bHND-Bso0x0`AYgo$Lq7@;=ouk-Q^J07K+!XQzGhrBxp?|ioVv@ow52Vy9 zb}b;%g;R}Qh<9z2g%R@jk+`d&>G=zi8^!5x80QJ&S8Y$JtBa@JL=JbQ-@GJbzKBA*@>G2mWJy4P(-T>v)HbmKcfCbB5-&gvEmC5CvS+Zx#+#DMV4&S7u zPLKOggSMIxVYh2FQHD2b@Z&hb-BVZ3X-|Be*)6)qHsF?ov@)l45#V`tq6G!J;*6@V zE*Lj%(68o-sFSR3>bRx__F~;RQx}#jDjN_`J#vC;=hGFXbU%N~hD^0w^L8FFg|2Nc zMQcvpU%K0lDmPrDUBDnni3aJA*e;=iIQR2GoFL2o z#rn<`j+DdU<6*Dfr6pT0<6&dodpC{Xou3#2re-}}dP3##dS|`k;oIG^;Un9&)c3vR z?*V8h@6p9ff z^pHhzR#J%p4j#-YawjkxU(Wek2gDd=lr5_l-m)tmgR%M8vZa@Q?1?$ys(h!|^p z);$9KNqivc+W8pfJ)6CpXIu3^y3&bYdN|L*S-)lvVZS&T_#h)e%R&3jq_@>st4=ev z`JF|5JIZRb<}Z?P&4Kq?EV`cYUbLHyW32I>)$XL1pnOk?=~c#`lc_Q3I}~?L0%I3A zg<(SEsiTdt;v{LhTs)?TR{+WRV|`bB;pRyq0Fz&sj@ezyXpxA?!Xw1XG5J#z(DR!T1p@~nBq zVoRCu!)0+!s|5_Z7Di5e>V_ITdwFB7ulj$~i`*KTp09VO=a!mQ-DRBi5$vlPac3wryj!0WFfpjdt#b_T_DvwQ-=u_eBA9#| z5)(D4TX4*GFNVOluWC$pf?S~3@Ag^1k2E68PoCAId33OKEEsVi7tkI~pa!psZpLO30R_s>b|5gG+|NsuP$x-L#?;u0cu0W~zs3 z&6CUX1}FBAxl3{HJNfzYnssn-X^Ez#Bx1Orwc4GeaZ_%;&*0A2S)aT>8O~qC)9OC+ zDu6!ohpnp1t|HjTF{wG+!*xCtgudsm2z&}TEdH6A@<+kAnPLwKmggtvqb?ns*@6Mn zCg80zwsh7i4i)~;-3)1e=VIDY`=qTl8xu!9mgR(;dW8)JKv>tn-RO8K%`XyVZT;w7 znBP}ghYz=piU;Vy4)EM|>zPcy%)c{ISiQcUp9HuVWQ%E2(PA!c-H_=l6)|8AX=!LP z&HKc*hMAFx{vnI)4(`;0usZgC{CT_&+vAKV9{J}ngTpdQaB)Rf5rF}H_SE7g#+r-< z5!>oIX`)`_*h+us_gYBFTuq-C)R9d(>|W6Od$*+axr4XsnFubHtdtdyshs;z7zx9_ zAr7KUjXt@=8AC&e8!D#KT2~YUtm}nrjd<@bYyO@DeH$au=FjPMrdfYD!p87a3s+0a zLjdgV?f%`C^+P#UeGoAGzCEun_euF0Zi8jkImG|C-rUsx*kS9X4C1*?OjJrO$R&6h zZ)BRm|YMXPo9AiKvF7J<5zh{<0 z3Qu`!T`eeDjxU$)e@PmWnG_Thpy7J8cWqEZ&jSV?YD@0gT*Iun?WkCPjObB$&b`b9#vXT|P6SF?+<&7z=fkDxs_%1h^clq+ib`LvB~64vvv zAHe_yef|J6(@MZ(+cVF166lIl*ue0|hqxn0SEA}T+~g^!P@YJQ_s5%LQu5cJ7nJ6; z0&D~LgZwVj24EsZ5q1j$G$^6R=$z0R;1*hwUVt-9fa2tEyJ4Jkq9mC+GkZMY+&MoF2WbH$(8jpOfBs|rNk!hWi>C~UDJ);t z^AJfH@0?`ITd7P))r5xCKeoNEQl+(5z z7*k|!qsWKTDWMj(=7-Y=-+bzMO*MG+OgWnW@N4e ziicSO=gh$7dv6SELztgFvdJAULvZyQI03n}aiVb8SA;^2?1n^ozd)*i`yrZv$EvK_ zgy%I~YQbCdFwqIt?Djm&*7(+IW^I;h zW|iKRzSy2z3cEcCkn5duh}OQ~<{`5o&GuR~9FW(-dN;^`-gir=*>!b@@9a3QJ1^sP zIV}#-Y|XvRM#~#e2db5w$cmHi?l=o25Gn$ToeeNnHgI&~rPz`JI<<~7v9QRy_~$gs zTdQQkMwD?$;05R@C`2Cy<0b1Dx;}*o%{+heE@pR&8<6~r#>Huvx0I`{i!y$D49_5O z7oV0R0FYPn&LtlzuU(1vZLGgG5>EB&BC*B;sZ^oshVnj1<96v&#AAwH&w9i~-zw#@ zX5_|h50Iez zgCI*kulKG#V{;%t_;CY;jx#pF>FHES_@jF?_70y~3O|RhOdy>BPqgdR^$THmx~+N$ zT*t6BF<;kwmQCOdy@ly-AlGZMwTYZ)e{@Ndxu9ovNuvUU1nhn}dm>{GQ9ElTk8|#- zO2yC96pF9~iAkp3XGH7Pu6$0ZybZvFI#yi5b(xUDZJyGoTWjEGD`9@T z?}}v0G4}m*e)1*nENJ}%;zq}9mmVHaAzXUoH|#LXz$s9vVKlw;jaTt7z1x)pQ+MW~ zGcNPuC8+UxZ?`Fd8`ICQSo55Kzb?)~pw2Mjs{OY-&%dKGPfl$&wFRPa{Z)8>1L|V933&QMr9;V@c5bV5O1!I9Gkxi!_o9`+o;!uI z_(mmU7@@e8{9JjGlAtDI3APZHTFUX;`mM#*u_3#YR7C5w!YOr9fD$jst@BY5Vdz;l zc^wm%Bm!lDq)Esc%I$dt_LEqU0%HGcoOvD^Yf_kJ;=8hFG%?XH0W;BN*r}2RC2Ya` zqjh2#BK(3{4rziaZo;Fck_9$TFM4#e`ef7M`(`&k6lMe@_f>sn^}v18FGS-^DZm-H z*uX|eIuc7!@&l!yK3V)`T$5fEwn>W;oRBQgk!v5hSH@ZYU}f&?8w;^3vgmz+d_TgY zlLtlj9m|idK)j$qN!$8NyM8}t49W##BgYnF-fC0qVaVXC>HQuIQCp zw>mb9MqkV+1nAh!nz+*z<=Jn`n=TOC2uLRXKcGq;1Qk+$oyE5YS`( zfCOK5p+u&XoG>%9Yuj&$B>i<5{2h&>NfDF3OpILed{-i&weaH8m4`oAtX;&g>0zYA&Xg$w=;2oNHIF} z4xPgnVt{*yIp=eai@EsvL&IGLoY0N?&TqIKR-wZO@1=WsaXXCZgw;{1x)6!KhvRU2 z5gGKIo{tbcd78H1bBf_aLDQQvH_lF`g=lT&Z0RbvXcjN{SeyG58l2~JPDts z^ShPiG$Tur5*J5*KF#7ngFu4t!~Xcuzn>U>b4g!N5SMmY`B#lESxTi&Zfdt5<5h zH-g+6aveXG;zg%ynKz1GkU3*hi3Cs~n>yz=*c+El#TbWP4CAa!L9LRx&#NeFWoM@^ z`a{-zy|Y3rY0CGV=^iY^n6b`W^3TLI#ocT@it5K0+#WOz{xZ*X_!qubfP%}oo*P#? z#q=)272U?mvlN}>u5{~%^^B>xO}^ z*s+c0ny<_dv)39)YHo`TVzhm`CB7epNqPY5o9yVtKj%NxeAiF@B$2rY-`b&F>oc77 zKl((7i7jr63oR~ToCQdpk>>lPq;_v&LGThZ9|X9cg$Xh})}spfHuFo`SD1g`hFNz! z#CC(FpK5vkCN#^8l_E&|k0XIcWtWQfk_d_8KN*rnDa%;qzR`H}ZV|K5q8Er=-^}e? zTmlaB3trUca;?Yc=-AcbQ`w%du_NhMhW>0e);&3WVJt+GIJy=;E@2f={%&&IZTFm! zN`v8-phfB|85G~?l3bG0>13Im93!kBG6Hy#*vva(_@LRdN2|>@d#Th@&r3GKOgN8x z4*hlSAsO?mIG|NT`?x%Ns%nX*#tlIYWhVrD1LF_Bx!Z=$#wcgo?unqr!` z-wt)THm+IJK%Ac&L09iyHtgg{V%TFL!p!ew*BH;kdj($lsx!fedXhq)M@GfP=JkV} z;&@696Tp}f=}^n+n`5(OIs!k96;5VeA;;ZIS=i^;+Ref+op_r|Cx-vdQ0H`*H)QPS zALJB-A84+kY(gO6yVsO#|E9-{3xWb29XM^vNIAuzxbW5_(Z_eYGAhAnpmfl?c4iE6mUdC_u*HUE8_W) zM{^e>vNw?k^;z8x{()BM15v^(rzF^KpJ{0F2Ut4#r_evuA0G52#3lY>BhuHB z%EL4VKT;>sVYo)z&Nf?T{mMpi#2lZl|4K`VUK|a*evIzsD>!7@#|+mzVhNjwK%arn z@_inj^)PE21p%MJ6IOx3==l3_=5|kWNlY1-HVF!s?_SxXunUm4*)Yq^C<~U!^}! z`zav%is>c|ytnVCZG{66-%POhcY)n9a+~wh2o|f)QJEyKr}=Lnh3o*UIgz8JiNlh7 z|GmiB7U@+|j+)#)xLfz>I?Oa1dgSAAX*LU99yN*U*)|Fm(%Vw<$rN;cOHPv0MnW9w z7DwJqH7nHch_ zb~XyqQ5?s7GB%C*o3kjH;nS;ho}G2t^Rscy%mJU(^eq#?U99Q(98zTv-8#pWj4p^H z;Z4CQ$tYbtA`L?#=ARSq!{638)*8n|z)CSxX|SX*rGCMCr)?#I_F?{jsJ+|gqck_c z+l~7N^G!CA!E2J@s~JK~a)*LJG26p{L%*{*oT~Mw)e_6 zT5YjK(GAAk_B|Jy0DO;?xY>H%U(t`vxC}cf7r>HTG{OU#kornmKDx3%ll z$Wh_OwxXyaLeFL!nbuxDca&U%(Gk0qfS}5%rVo$aZGA3EdTJ57AoYh^FH#r10t_EY z&~~S)FIllF5_V7Vog~~!NjT2emLqPXr}ZV3qNpOveGO#~m<2Pd{`F?3wt)K|cN~MV5-i-l%Yydp;>MkN z=kGc#v#rDiUWBC+2dH)%5t1|Wf`wmbe&02j@-k(Ye5*3d3&-IP#pbpkBK`#vS?*ge zTb*{-LV$8KT(Ik*yRx_N1J~d|0v4I2b5`EVxL+J+Vmlk0U`!YRyTnG;PQLRM9lbWF z0IwzHBsU~x&KDh5i7_xPSn;mp0D$#e*TPnyg+uo-Yar>d!9y@23;|QA&tftb%m$HrQECyFrSoj z1oO(>L88>`{i5~SqZVhPdr^*ms_&FlSy#*TaDPM`s-z4{EtnDtNZ9rJ-U#bqNhQ6V z;We6kj>XaGBGHsK&UD zgJhK~SKrAi_1@reS*=ojq`Pq@=$5zs$&X}$@ax@}v>RU}2khDXBIux*6k0lst_cgp ziNemw9ZT zOw1uTkqM5*?oLmAXicvH2T{cWCwG{ZEU8yUREvFs|Ecc*?PjC~mY4^Sz7qjl*wB4Q z>Y0`*KIl7KqATyREDk77OB7+zP$=Ar*p9Fjmjrycae)tRqN0#6q)~4Apv#do%^_{K z`CH57p2=}bmLH0!Uj8u`qCrDk>drJM3cyibcRJ5g z5iA4?D;qMl#HA1YjJEUh#67_O&B%YW>-2?L%H`tcAHo=Xt{-hI*i>()TUe?B-imD} z6I}k>W{r_newx{GpJKBbzg39Di8}OJPYb!7K7iPZnwb{A&mh3BDzf2sR}n+n{j~AP zf+syZQvp~lMb>O%ejqM%cz`D@=k4trsif4%v5(F_yJSlw)_!C&RJ`tt!#o1QC%BD zS$*qIp(9nW5km(`{&$*!<_v(bcBuoet6N|U{m-Em6*s&mgGlRORil;fQC2@YRhIBT z$%rsiQs*lZ(=>8$?c;7_Wl7zLEw9*N9cnZ2b2oPNa=>7KNkZD_v?L?a`k;rEA$}vu zPGvSwqr@!@vdLZI%bPmlm321LQ(eDmqLcRV{;(nx`ODklyrx~GS-X&f3Urg>gBtm= z8d{LLH~fqLCepXIa;_&g<<3BDZM>_itBbxP)7Hmlbeky8>G;H#ugcE>+TK(u*YBO` zc#*iiqi7oE1u}sL7j}valpmfmSJ46Z5qU;|NPNYWQQMA;?qVOzwTP*@@|1WR(M0_- zjvI*ofO1QnU8dLRvlNa|v6FmY7BD+Pj+gV5giPDr`&*F?IS{ej3?#ztms2adRm1$h zd-GH8qlq&4T>uA0UW~!b>mfK}XcEIs{M(^teFOyX>&$bLyJ_dM${Ik(TL$Mdyvc$F zm2sz0-l;&0o5hDGs};xM_ggMGI#I8F&lC;<9f&3-CTRE-{^b3OxX1T_{?3>!dQ)`) zCwIP4FW-N(ef6IV`CI{{w&~;PWm)2*Zfj@6R&S#~$4(9!DZ>>X9@?e{aVOK3n@i3U}MiSKBBH>HkbahIR z<~5sTl*1gWlaj-Ha`DCb^-N_daZPs+o&or}I2E9eC@c`A%VM`A)>b|hoYMR)M^>8i zSy|`z+JzJf188%}W&k8fDkG?;S@RT(ssE+-EiOBITO{%u`8UfLCTPsk#6`4;k#rxD z3Hv-gh|u>Iz(<>Z~E9yY(}f+ zZ-zd{p?|x~A6&|JEy~0ctYEg*$)54b@pi;G>&Ro++1WwE#$GTfsBCJ&3yy9!u*&2pD|I252dvIv#%5F~shOdIKB*gqgs!5YqeHdR^eo^x zN^pS@G(OqG1%?9W-ec=;0Q4=UX1dD?t+Qk8WpVF9g?;2Sw5z=V8HoO7>B0tbBxnvV zD;XdCmmZI#{RalG`@^!ojf^n#+v6Y&C<2}Qn+_A?JkM1#T;$itYT)(LOgnlc<=tT_ zK)aXgG&M=f?{zU&gd=NCPV5BlwvE?3mvOlUJErPY1Dl_}%J?;Qyuk}qw~3NaQ^Ep+ zQb^h8rJju_(9rRy81YH`Jh*~FwhDZWE=KnbRS;QE@v9Lk2jfUk3ZsMpEJyj{%wCe* z(WQp-8Rqa2z9-Bvt~e{9tZ7E}=T-cNJbsG?3vlzsL)-$+AJnaEY5YNw`H z*KWpu^@0j1Uc z)snT9zW6oX3IaD7wF0!RtN{F^_Au^JOu%%anSwV#IVKxeW)|UX{jJ8tf3J<6 zzIp()BfhY#3PC7|yr_)6x(ui%4j1$ea(Vo=Ue2H#sj%plYA|G{(lBoHY7^ym;aHNT zG~sSJ1Y?g)`Tb)Wq>Yy}F1mS@g%8lhFF9sPkG)Ebp7KqT`A_BGg|M>=aV9S@e_ZDd zf*sDdMx?~Tl4!Qm_JITMSAF`la&+@1)DBb}qt|-!wn>W6rI<*#)KzGv^3z&QYGovQ z+Vxp#*spz<0|9-X{>#<0bGuuKqRV{0;Y{lXOA6w#*e~5ZDh38&=EwIY@vR;}$$U#W zE+0uv3PxU~?k>uF5UM6V3OsxY>Y`?9R+%HMs#MgZ=WJASiP%_jA3Y2Q~F(DC; zzAU>S)uRlVrUM=*!KkSiFbRpm45p~GLh&eDO=QyP?ChXB|AO0Ck_Ei|>=k*R1h8jp z9~AT8bSFKkGLit-0YxqhAiR~>sZxUz1_(DX17u2X;)koDz6gq;IQ4h}1rLBb$o1e# zW?^ZJ!Ary@n_m+0-pCPd;6(zPrd=qvtOBj|Y2XvN^kY0@abj(pnW=M@VA7%?QfR}H ztJ#!8(o)lhOqjbV>fcpxzoyfp?;jyK$9?pS|I&lzloalnxL>12=v(^>omcY&F?-gg z9^)!`_7!Qr8wJdD!x_Igy;1iS=Hth)cr%~Pg{|fUG%`J)(<*-Bp?mDzpy9LByFsyu zRS8Eb>-B*U(faf?5u_yWgR#~ArvY&gIdeD0jsO)Ia}7p8f$8tyRB;?88WM1~?-_?i zICBqs#lUivgW*;p@fTd177?T1L%(WMOy)%$3HNW>-m==N&9x$xeil2qJ-I!d+|57o z>tOCv{k=_h0DG=v)ZE00iPP*yU&L{QByS)lxI^HF?tL4fzj(E6QG>kiA|!(?0_+!~ z_lIz($Ui8_$WhF&P5T|S8&GG#h8%2lx}0HmF`zm$_NJ5REP!8^(0dXG{*WMm{D7L8 z#`vwe+JY47>_V3gHy(iV5qNOIn-ET9-3Hwn`tpR;T=X0E9(!`)!C~A zUA4HT7!y0d?1-%7h6q4!0L|IJC7S*MQp4!X9G@>9(T_kz%0Q1q%%|w?8Xd(%(WJ_Z zO2V?-y|0L|zc0u*!%eAMX>tASTkYMb6&Y}=IKp<5jvK(n<{Y6YCC(_MH(1BSj*xF` ztWtJ}su{l+Z;X4K(ELWTL~c-ptD+vxm09O=uuY6{78>;2_2W}WNSL%Uh&*~gqY6#` z=3QI?aZr$02rZ&n{UED-f(DMhnx&EzDLg(vdM~tq^~coKr@zpIAGE+WH0*)iDibwz zTS)r5lqph2bnk!>2JxIcIJ&$eVhuerRuhV#V%P}a4YN>JDbi-4yM~(eG0miwg~2x@ zTS{lKrzA{_Fw!W9%a;N4Q%^0a&q*+_kK?BaJaIrYX{$uZs1L-Ez0lM)-E&!%pjQ*o z3_sZy`X*qi`p9lRB>zx(yVr#e_Rmp7T=2kZWY;dM&8K%R13oi{S&^+)MmtziA(n8znaJfCP|}evSpLhAkBvHc?%26)($E(@TgJU?5g-x z2a0AIW2z0S>eq$?#?&s@oE4@~u3)l!h%Y$47_SglGq=M5aB6O0{ZV5+;D-J=oCSvD zkqLpe11;Et{w(enGX@V~9$^|t?fS-J?<7rxU-W|Cd%52lbhdfqqKT&1FvHFnMx|1- z0Ms4knYGdA1X=l*k=6lr9u)B)J+p``sAFRmPoD|%u)-P|Q#tnqU1ut*id|dnGzkZV zIx*n%#r5GHrfb+ifgWbI_enNFV3RYjPF3!zW~}b}#~(%EwoY|1lcnJmeyvgAjIpql zN*_M!Ir;SO-fgQ4f4WMR~ zTP=pBMdhU2~>PAz~QH@S&i4w>5 zZl<00v2ehP+e^;#+?Q%Y22croB{yp|^s>s+857!LDuCp%ikNIB0;^sQ%cQ-Gd_^KFtYNBrzVmX9-6tQn&O5YclBD>Km8H8zDRlyc&@$Q z;!Lw9n*Iyo|K6xu+j8UL1k5g#NBqmiyBY%Dr$=%uT)em z6oTeuKVd+p;viT`@(&;sKJn(yidqpd6R=GyuJ1x-j8;xj7bg^F#xs7z_U0+y@7rj? zS7$xGdLDqO*zX3+KRmX6j%ySPlG0qXoe3&6ss%_Z*r6-{$`z|DAda$C>i>*UbAgC=LYFO~_0IAxljp($VrD!N_HYt)us#VE~5WZG3G|>n= zS=wP{f&}29%)zc%qe^m2Fd|Q_6`>tOfky{0$o5G0_lUg62k`Pzw8-6I6*~33`giO8 zy1&}@t&@$w1js;3uyK|mOLY5WC>GeekEntDo&xMe>5r#I#MpkVs%PU+z)W%8)rZrk z0`-nPHi+lEgI}YN7~D^rHiH!we-i9y{|72AN$LU8^9_MxmCO0b;iCI!aU!D4ueJB% zx6r^>#{yD|$Nv0PFoaVR7Y4Q`<%(z4(?_ETX8%j9~0TsM)RU{IZngvlDDHF!Q zw>@&VuM+u|m*{)t%3;Pw`&+L=#$$#&QoXmw7^-S?f6?aCR_@cFgorU_+O+LJOcf~; z(erTe4MzGxYwbsKl?XISC9AAXU2tczT5zY7k0CR_u!(U((mxw+7kogV2L_LHh(qfJ zA>9GsFr4b#A?b(Kvs{`RQ}Wp^%_hgD+#%BfD0xR=MZ{ZQ zI3#bdOiiyE8#Z8d6PS1}+FQV~8o?iySc3B!DZccYQvDN`9PROsv#MpX&0q}&f5zHlzgjhkx zr|wUehp2~FrXzdbeD{wbOE9i-FlE>S{H{9-E;~=x2{$z76ZM}X;g`9fLs5|plygf7>-Sh0Dp)_1up;hipy$C z-o6q*3bxLC7qX<`C$76cL^Y!BDqz}NMQZ&q45ueAMb;O)yyRFfp6kv##9kz5TP*h7 z{0> zmcODWwdr9Hi%$dWCgf@eQmw|Yt03lG-0Agrv9_-5+Nn?u0!R#;#phiT7N=Is@36qq z&k%Ys7q5w)$q8U~kkk!Oy+N>ZO0tE%b=-w5 zn4cU5aJ)-=hM1MvMffVo<<4Xl5|mNwzHkK;f^1L-e3M?s zS12?D6TgolHX&T$q~q*61&cup{v8}qtvw~UHAL}`g&q37v0B##*FWqme&)ZVFK1aB zdhajLJ=s6ts@nA7Ts4qZMoBcMcp8V?vEyi8yif8je5ORcFZE*>8u5>AIWMWAZQ{~| z-qHB+7k%3z&tK-*{{;dktjm9*j=LX+*a*33&j-q<(wG+bub#3B3H`(0n&@-9nO9ut zM@<6fRE6P|n2#(~`eYz-cL&7d+Xz)uRw}Eg1pWLexAp=K5cGy}wFdE@&$Hoe;BfyT z4iSZB>&BOnZH0|5mXn29`>u6({Z9^;R#$){Jtq5WV012y*h;D7Us}?OdHMqB^*+fQ z+QpcBOJ#msXvbfo-)u$Qfej&JJ3lux2)=t4UGt~#UYWUeWPo8)IF#hhQ?os~XJS+f zntp*Q3DMYn`c^SoDA|xZq6Q_>RwoZ-t@%L&TVIagk1(`QTRDG@42r7Gz)wb~cc--E zYl!Yn`km`|9^G`p`|zkbIMP?ST?QNdpqi5woUp{p*0(=N(77fGN`s2>Qi|vIl!{NgLQ3t|vgF}|k;gFFr_BoU)alY}7T$lekjJh`%T&;aU>w!g^Y19tqjyd;h}+W4p~4A!T4-P}1{5@@*i-#;=l5kd4avQUkwF!O9Bv z@^aE@ncn;^nt4(wo91$af}dtdMq89z0O@^n2MV#_FhB1|v`xUq<&N%wXhx2m)66i~ej;?&w*>GTz5!3;m5aFd&LSaQx#qoKr z*Y2n7tkV(XeP0>YS1WsrXv7EAr?68bZS9GOSqHUmEq9=oj7Oi({P*CU+04NH_+<59nIh%{u z&wOQ4WsBj9$^0R{&h3yWc}Z5h0@gl}_p(PIqO;u>MC9=?g}pGWKGrPy3R zDC7Zs5fz0)LJAE$ zN@6VXvT~6t1^gdn!`A>LqxM!?I657}imgm0nmf zD*tCGSOoZeJTmCEyP-x-t}@74$N`^6>7_w2BtgXmGq0fJC=3_S>%SQ7*;3c7*Yz~%BV&j=pQ@)ijk$nxr^fg#U_ zxPTKEmykvO{;(Ne$@iJK3%r|k%_3&}g`!?UuMkhM_?BmrK?DRvxy^Y03DAqF-4gicc3hkXOqAwq zMcqi0oj?feTP&eD!XJs9ElTRzbOZv7olqp5G<23bSNb+6$Q?`cyvV&@E>-LXhp6h7 zuHIeokukE8DoIEvNr6`;Q%$cu4J-IQ%hsj`?>Pu5D=XK?a84J~m=pve?b`pP(!<_7 z=y4yXdyz-M7T&QUW1vd2_sp=F6r`#Zc7orqUoYlIJ$cmEYcKqb?6~I3YVsX{iCbgW zq|JE)8sE`$MJb!f3(d#wmqNrZ7pE;5Lcn(L^}i*_KMS%HY;deHXSR)AQ^mXx3nzy? z#&1d%e`PDpjlHzgr|of;2$G{|vsFCRtGu4r&#Er}gYuVM-b8oe$Tqr4MVx-)!7fON z_uX7sX+}twtsWj~iVc?I zvLil*TGr0!#NAIm9g+;)ssQGm7jEww`#vx#Ke}a(5OQ`>hv^td(@L9^F7adk!<+6# z+#!r1O&q&aHU!AUJ>n-LBi243U8vuOAco<*W}R^$Rg%qKjgoPg$=kHw%S%f-%l~L( zs?4Cgyhc=Bo$D$(WQl)I!Y+lbJ|&Pp=sgA+9@#)Dns#!AUR1ReS+gC`4kjF)O=2?? zU?R&2HuuUL^;pKYaPzOIsJOU_ejRXJ4a5w^Uz}(|g@>Q>WX9j#PTYA!k6UgL2`M$u zi&7m{nEOzRsqgo)oMG%>1m&*FWs1iyqoTlv%5&;872%hbm&X<1b8*d|oI=xm3M4i`(G+-e)L<~YRq?;Mb~ zMdlF1VytqLil70FL$o=wq$T`lQnLR=*I!0O`Muu*umURG-7Vdn(nt>tLrB+93eq6m zC5;G@B0aREFqCw6NSAcSz;om0_kGrXJ!?I0xZkkWoVn+mEB4vf-u34a;PcpW+Y>>} z$Z~9&`o@veD^-)%uCec{YmRpi8zOLHP5fT1-jmGqf8fMR4w>6P6TKy)kxNT$47o*3 zY8ZnoK_D3a$s!aDNjuXqhl=?{hiX34O3w|Lv&);F_-vo7b0qVzev4Sn^h}$AvBCuS3=xkKdGTtdcaAn8>j-CaY=Bt#r|4)wdek2v)N$s4h(AGvY21wAPbI1+kui4-Of zVU3Lb!XJA^TCnwuCwf^yP7dlzTytzrTtvgC3z=$)V9&S}ufU^ahQxO3M!%UUZ7L#V zO>K)Akuc0t3S@n7=Fs#$Pt0b!9Yk&|pmQb^7v1sJI7P<=pQpVbT37xTm5Op-6~JZE z*8-KGw}C0^S)c96^9x9IQl6^x6Ge+I-HQ`ta`OQOGMY&CU2!w$6z!;cK9R$#%cG9= zockJ<7iwZ&Y>Zj%iTG{j5b6;^Ifb1WX&-;&6DwjHV$-M4DE7wDOL1 z>^l}?xO{HCWkE}rtpA({J|R^y^nss$Z52|D=`hbCHva*$jsNSkZkaea;+KXYRN|}f zz0f`T@)2F#Wsk#d-QU>SK=LJz{X>0Q>{^R&06mF}V1yD7L~~8quu^}L2J{yY`Rnep z>U+1u^Boj2=zlyO!aR(l^YRe9eir9UKf+f>s;TrE*-<;(R1t!^3XcVivBy)7m4*=zDQ3Z$vfVhO~-&t;*ypt|G zb!s*?m4{*HpWTV%9b~iOV1D}gw$94%!(X|7(~P=wx$L7AY|Tq;`J@#gpdfUbYGA96 z*D!SJ7NhGV{aBf+0e%Sn0{><-qWd9c@q5(oH=e0F=@u>V^#?n)fY;o{t9XxSCpUaW4f32?V;|HbCip%4kgh$#|Zmde=d#B zPI4FJ*_{^8k^!!sJ~~XSrA;lCCng>R1Ebr0#yC9S;Vh>FNgrzdqV3_= zf4Ur2TG>`oB=9PK){wPZ*L(O;!yu)+2rYTGL_$LAfdMM#SFaB-c%{mxXR(labUe(8 zrp@d!{Gx{IykJbz1!PNevHsbw{mo>kS&(GVBC8D0 z!Avz{+ZxU1lm=%@qA@pbpkEH>XgT#H#ai7Fv$MbXge?w%(QO993w4eQ0$Cr1g-?$zW=!M9@)gz>Ka8YIWCj7B9p4}WwCZujZ{o9J%c{d}~FwN8Zl&BJ}UB@>PTX1ktq13{$@0-BiYS3ImVresAaPx>O1 z->ikf3qX`i@nhNnvnCU!Yxcd;j&)NiNlbi10YX3N@lpjdae#1@&7+C%{WiSr8q}Z{ z_Ged==+8H7n>5aa;AZ)lZsUE~o9ds6uA-It+aaNz{=dYG=#gGUQuWsBu8!?{*zP9` zl=p6mpW8(rp93?4P2{x2m%k9i&VQ?vX)JhAYMm5gOS~tcA#C4iP+)EM#c?{iVDcVs zni+caP~kPRNR?#xYEBl~Zhq4p=Eq>WFhg>sSLQKtv3{J2?O4%UowtI?#>=+-F=tLB z`KNSl=+#PqBuD%G`>w98*;Zr3SD0NZ50`>$`c=Sr3SOuQrNJBt5?6*C57g)gV2${`m>isJ;=WC-Ku4Vkt zW%{_2O!-i>KsnylEJr1ZiKg&IBf<9Il;7Mz_&>dxd4h8RUTQ84G~NRI(+-q3zmfR~ z`Z$e)*T(5iF@D%Mr^&CF_-&e=j%KlSeXiXn`px6vndz}8sp#^z!yxAzZz9FRZVAiIeeO-xpKzsHdvVdK4& zk<|X7E9EdL6!BHELMcrzpHm#;9#lSMuc1U)00F`=f%Z~FPlT^4`)Oe`R_tP=Pi*nG=|<5mq+38j-Ibby z4fdCYLycNK8R1xy zjf*QDl4iWs6+HX?1TyAP&-5J8r@rjDu_>;RPD+aBb%lclL=h|CBGyA0 zDrdrRt1oeJl#-i+Zh>>{WJ!$G*w*sSfgK#$>wULs+5E#_)k&o`#u7c?9dH-6o8~Nz zsHLGbol~R~m&TU%mfzmqAq4dMrPzle3$g|ahWZwDzuGR5*KA!u&(FAqBpmC}GbZY1 zh`xV)W79A$ialf z!yo>Zt7+xSle=w`xW*8Am8LV}!5m2{k44v4{tq`c%+-B0DXFQ9TS4elSzKn)!Mp^S zk>%_^j=q)gm<~Nhq9#vFp~FsgefjY|J0xLLk*0#&T={Yi*PK;53@7Zj^CL*S)33hF zVluw8x(T)Ne%tl_e*GmV&vhuvCn0?iW0k(K)A2NTa&jfEk__ySE_yF3w`nH=b7NvU zY%eH4%E-uQgkQj~@7$B7@QW_=Lj%sU_l~P~V$$O`rSaXh@~t)K%1WxG{n>aqw}np8 zFmna=C{DO45+*EQ9B#FbEKqFP^e3SF_TbPJf?QE$P z`;Lf~WQA#iUp%-X({e;t=wvb4bt={aZHzp=I9-Tj>6g36M5fsN9;rb)10!S99XI*& z=XoYL#}vQ0RLI^55Ee|EqsLy)I^F!WbmnAvX z3e|PGs(=(7))*KWY`>f8(xB`b|wzXs}HKw2pd++{DdH795OES`eVh6WI7HG%npGAA8>_P$i~zfm-IeOcU|K$ zkbf$#jBJRE-WoTV^`ta>l88_@3*E@wI6scjdCKnDT({A4hQEvcjNB2LXOT+Q280u1 z8%YeAYz+eiH_(=ljegrp(g5!&mbkJ#s%qZo&`$W8D0~{5ddRN zvvOm<5vi^#`5E}1d|N};G-|yQru^kek^npr!_am$Uu$sNo%;kS)wwEIcAZ6=uUc9r zUjr=s_hyh^o3XBT2K;p+NiSYW^y;47FmDQ1te)U7VdzWs4X+hF# z6gGw{F8Hg6`)5KbD$W;aL)O6o@2c*H39sg=+kPLp>gbsb*$feCRB4f79JxLn;I!J3_GpYMzY<)EG{y9f=-<=i4`-~$b!ox|8xrS=LJ4rBPsC#2|IwHV2cs)vr67f-qmuWNL5a^ImGoq$PRZqM}%4S(MJ zSCvr_&WPgTBo6Pc8n|)mhCMa6)7DGmS?1^V7d!F(_Ep@Yuyo47b9wDH5{$QR+@ah* zdU~_8gUuG-nD2Shln9`Xop)+o4XA`twbWn}nMx=`RD2i%$`@u}oRDIf!?<9qHf?2v zNe1(E7Qbrh&-|a}(bb{5@cO|nLc#t8OgByq{ndp^v2b>>nG(%Er~BLeNTl;ujgV?8 zqE_v7hXH+xCp1Yv$D?R1S;D{n{QRVb+aCdL9NKGjBh-=~P&R{9-XUieMssTM`#hDe z*qh$IG;i~%E;l$1^et6#)-&+rrpP#ubnN_@-(1YiP9YZgz1caNqty_JVQBVzXnPzc z2XE}lcRwvg(ex#0H5RFD5hbOo|Lnj4jb`tcEskcl$YCc0JCYsv7x%OoFF>zv+DkRx zo$uPLPr4@6RdphJdl8mcecjj}sD1tqiz}WRn(h}cqiK80QNpX_HJ6Z>ZNRqZ`X-SR zH&G1`*9MQVkTu@&7n>1=lFDetuJs>*G)_fqSF7by@D7iK00< z!|ROtY@Nk@LrcPOJt}G1L4hR^-l+D~DleG*%gpK;t5XlZv?s}o9Up};gdmmNxoVac zE0v@EoU3zZ$CkOictlsGI~7o6tZM@;DPPHg%$`%G`~1}MzAklYQeYgWMk~V7fAzB8 zdkG>Afs|VvK7mr%TzC8c)Duh8>EK)rOC6Sr^1H=f(Di`vFfwMoTcavWz>>CA zY#j6?_&YZzv|6wJiesml3aW=;Dq*Fj3s_#; z&e`lAU75tIF9#4{MZG-zK42pA7zo25CujNITruLb7I4meyH-sOnTNkjb!=;nn?*f| zzKWc!wg8ke%bmQvuOG$EJ^^3A!Fq-D?gPl%bMFf)3>A)GS?*L%DNwJ=`%lELNJc7_ z8&x3}o9WhxNW2B)n_0{h`<7o~y?|6DJmj;n#v^-B=W{wbi>kkl2e6(l{+gpCcoUNr zjq5hz$m)G`W%n4&F5Ulze3v4>O2$L+30k5Fl$%Xz#VxIuj4@W2`PNjVIg}e&u2z>c zT6mfFGP?ZJvvPIy;4VT7gPjLJ+_dv*f2L0&SVXJHI95tV!QL!1Z@Gxe^{}_Qmj55V zlRHXg{X8)6vVb(V<83jA1ZYU=Ex&GFnV9k=^SjkBc`>EJVtUhxK%-Z;lE+RPi_ zai*r{>cU^Y9XWym%6|%~bm$Oe(Z3)L#a(v0Gqnof@YpdgZ8;Ufv}up~AkPjPTRQ@2 zfeW`ryL4$8m7xR_lixw+(RO!zKC*@30UHmGT7`z*o~Om1|Kznbo_EGfz<3>To(`Rx z(2kj^^IK-IP#}4yo4wimQCuFEFyR0SD+|K<`q{pb6ju9M!r+pHHqYELRjOaUnD?Q~ z?vxBH?@WYJW-v3H0&lc!dcD(Njt_3eS_#csn?gRW1bq!o=G!>Y>Po+jLE=q}KZ+9g-1WVL9|ihE z>bK)xBND!N9l5#VJ}F*o6i2N$LFnxFu4f_{hhIP2Ve`SezaRA$>OF$IBTzD@844fb zNz?SK*7Xm4q+Qr$?HDdbTqD>YOPFMJvx1NwxvO4ye9mKGQM#+*wxTI8W0~N^8CQs< zSedgIGinDDHR+W&l$<<#;Y9V3O3@VBZ(TDX3kC`(nx?#NIB8KRLDbzRv$Fmf$?xX< zk)(q^{ZF*VfVPbMD4ZqkQvm zb(@Lh?uEgNQpsT86>t#Jq|6zIg4)NI4=$S-*MAPu3VQeAUS(^2cvdRuM^xE* z9ztGZXY*AOZIh*I6i*YPqnBTMPmF2R8X_KkB<*x!?_Hj|g?*;oy&hfHik&V@i1Q$h zLqf8 zt2F31K6Rq+E*P5`rkkn$78*l!p`YsG(5JhrNlv!zLi<>YcC`M2Hm3mOJ;T8yQtD?bfd6E6}>b-yka{?t-EqG6aBTV0K- zg8=m8JbkpR>a#{a*H4A|VyJf0r;atu&$OO>5zyx?1Kj z_CNQWc0aSscx}`@-DUwaWmX|wx>lFk$u?in9y8nWa3~`G95XflF^5jM=H7h=#)u#9 zu2xR&;;3|(U0_sHxK~J_`0~H~8+gRUY*;Plg#h5U2X8Xl+F?e1#q)&dF$f_z^tk$M zd|JQ_@uY5eG>L7h^ag%F(5YtcRj?vNM@RcxQAyJa*zyiBJcpolFYeX9MyC{Bts09QE~9PRFI2Gqvaz;S$pZ{|~vt_W}^&wX4)1P+Vn~D{S6axSc=6-`FP1 zEjWFR)Lo(T0Yz!#S!Y{f@Z=|z+NH3*9sLlb@J>@ce3BlPQNbFg7B}=;4AQvQrba+! zSC{ZbtB$9FNp@fR1*CHf9Ksg@j`!6)-SLEKBMr88v4um!LPN%T`+md|VuY{$xN3(5Io#4RGrtWZ`TXaU->BWEs34C#{6lS7 zploKNX-Svx#BZiPf|-TY6K6J8Y1wWC+5T01Fp_yEq>pI9=ep;|+L5cla?J?!{q9(3 zL|kMVA^m2D+rR`YV;9A`QDv$l=vEB}Sx=geb?ZMrHA(#HI?Z=Z zWF8%&BWsCHdArT}UgTmf)8Cqt?a{s+-{_308F8h|fJPiFxO_=;z9X(NY!e0ocpcyA znvauSMzYVxol~Mg<$l}SW_5NII}ckp2QZK_v=;902q^i!Rj6E<=n~g3Fb-`c3GZI9 zpJU=Az_@Uek!dab)kZAQFZS+-apON0}ATrz7hlWvv&@ZWZz^5o6 z&FOu1Tue$uj;Q9E2#tzSZ?gU^Sxh#{R~#n)1-OVPuC_apNXX9o`GCLJ&7M8Ecy^`I zJf|^ko-6DeDs*&CQxbCW5@72PYQsSRL2I93iX(8en6D2YywLtm6Xclr=aQ1!Ate4f z%+hX4fSfWE>73i*)H0Ts|9Ou8F^kFL&TA45#bpj$gQiyww;?8)A!EK69ZdC616S*w z9q;i|;PeAAeDaW8_&Mmw1O+3UTs$6XSi}gv8=G?dRuUgmjh%{5MBDI= zOsR^z&L6|mj!qUJ(*oQQL$p&PkUsUrL|$1?{9wQ<hEK*d+q0VbW6 z!`)Na;BH+nb!|dSDOEZ1;Ooj@=)@lBF=@+dyQ zvtpH^EZaP42@!99j2Zi$L)ZIwRoT8efuT^?HwHr{YHt7y2MmS22Cy}+W>P$P{hZkn zCgQmHe$P3gfmxeU31}fKuhdHecApk}cRY}*P4aOX`&`~j8p0M7JY~wQw#gn?_Q9c! zUiQp6D^#ZaAYn7{1f}sb_citc0|@|2>fF4_Yb-<9OiAfuW=f{29q#`vSJ%k7Pshwc z2y43PsB&r@tI3v#Dt`-Z@yE#3zi&fTRysgnf8GAg*rfe-$0d*TpN0-R5jvBonr2L~74Y|EA@#w65|U?-=jm>A)G z7*M()&tG0;SrfPr&8oP$m6#h0DAg6Iotxw4FUa zr*LI~a6^%!W2CFFPqe=`EsIV@NOF)K83=m`Wh)1W3vMvGIo^L;6)Jz!m>cOst=-(8 zF`9>6I4}X);)KSIfb4x+{;1L`H~J1<$q~2$ zRQT)nc2+z}n&~UB8(U|-MM1>(JH^XQ`)9&lH+j!Z}R^4N{qivTTowCmD0n01H`4ZeUx4xmUdrE!-p)I0X9#r0N*zsnQRn=65} z<$txVSt|f0isuPp!n+YAsHrss$b*A+Ra*dhj+4N_N5<_9#Hkor+ymYrGueyOQl z$H?X8bEu{)sF4~Dzv;c`8xiM_GXS_cHY(YSGg1@g=6D;v+G+ql#%!*#ug?e1;Kk9m zynf-q$Sz&4OHXYgESxK^T5JQHDmmE!3CqMpZl14#UO#sPW!foljn2U(>Ml4d_n)Jf zU-$GoyS;cZR^9f9QP+J7@2y?(l#2L!5UvTTxRu1|Sj)Ol*^Q!s-**$h+^WyQ?s1 z85}CIU=!xw!~)-_*7EYP$;m^iqm;bLQIu4qG9KFIE_m;|o9^iAjqa+%jU-LYFZO4& z7pI*6?%D@eeIM}9hpX;DV~$SN>uo?r<5k2iMS*>nHthCYE1CAFfj^|IUKsGR^#3ja zm^{G+6(HjGN<96_rByz6+#w-}8h98;k74}S#7E|6M=>3Ul`@r-+a;gt!w(B-Ij7GD zoOKIHYlbQ4TXpCQB0NmL#+R=Iu)|&&w;FFOl@v22<%S$4#F~03$wPBhe+MBnhQh?07K6Q`!nRgYq|GM1vmon*sOC`;GB0In`m9U*_7wcrNt5>P*E}8 z*m_cTLZp!6k%QQ2^)a_8IeWsU?h>*y=|WQ>DUyBW#Z(liB}hKCKT|y%OIC9_NxAfHYUIBnnyEkfXv}%l0Ov@#u<8`LDkba zPN6vDi60&(2~u#GO2&M%3(a5}$qsqe&H->+=%OSv$!1WvJ8ARRyN+m-(4OPv4a7^M z>|${6S9G7oazBk&=t3I=%3STMv;>-5>D0OkwT3(YNVKvuH!2$WT}%&L9~leM-5t~H zF5!k2msfP8>QK$SAmUy<-A|mX5tOE*-Gt*SUL`fgw=gtDI2ntq-=loD7*7YO(fA}c zuH8&g^*I}7eD!hG2RHlc7VuY^{^y@jgaOh`%?~54lo|16XuC(e$D!`hbBVwJmXI?re2hmBpoaxd%v_nRX2v=mJ6Y@BS7zMpkWFD`lqf>&Y1<1ruKUz>s)I{3R}Z4VS{U@ zhD(sO^TgBcvD@9>Ru*ch!suKLZF{o0=wK};@IPHf?Nv~H6>r-PCoKP@%d2S-bQn#l zt%Fiph7%iM$lTaRMo7HD`GT}T9Gv|hw|m@fe^O&%eN~h*jV7>R@uHE%^$iJs<=3J@ zUn(&OzHUo1w%&>x6=SJ{>EUP#=Gsd_UF}QOYSh#joTodF(?vWv8R`->Vj3&Pb9n&I zrSvPkfT5M^w(qk&+-l9aye_UB*rikGGb@!2ik9GMx7tK)f%oND&HaI{yDqJxZNN|0 z9A_*ZN5EEIp4xmBzSpkwZu=-;w-{`b#LOLg8k}~E5fcdf!b4bPCFFMCe1D@#(9U6| zl6?$!W`qpzu9*d_XwWSX#2>BblB6+18&QiAU{!BI7MOg%-*An}>eIJedkUhfPJ{Id zfAM_w1G@2ohd4Q9!MqRev37BM)w`gcZVGM-nwG8rs@1;(RgZwDL9{@B8T%I>tS^LL%F%iKz=mmscqsY zszwnC2PStdg7wo;o>VXWprkH#GXwq$Z`e7q0P@V`OqB9Y%wzLG<@nxyjShR4uF05n zfwkBlpsfJzbLqQM|JE>c5njyEjsaAYPDtXZ=-Z)#8l9XcMp<gv{4dg!5KUroR{y zHwI+m2V8uI8M)|*sU#lNwRPaWV+Y|&;5RErAy6QD$cKAQRO9TG)%(Y zDADBt=w4jyi;V`C7C$sr*sDpUKibih|BRDWm-KBym{X|b5>>(q7{+LzW#d7;tA|I- zP&T2hv|SZtO>6#L4tPcta8V9vY@l~sy{&s;xc~qCbc^R|BJM%wPCxIab@RKXp~|2P zNyCV8AwuxMGbQTd4k8Du_14-&oHlEez6FLuWq5wC?AyTg1*PvAaD_#$3fEe7N};>k2jwi@Y<_UID4^48c*2$`zu5$) z#F(WwrNedSlcpvo`_VI#{WvQ}gJ)UB#lQqW1^z;V2jIaaUy6ew%GDxbqGa?v>4RiC zv9T3b0npXDA3qFS5m!az*bkVy@X}Cn(hv|I{c4F-P@E&S)r^ft#rxLV*6c$=a zgR$;)l}ciVHZx*hdwFpd_CLcRee8T2*2*2x0xOA67B`e#sppXH1#vJ`6T$#67l0Ee zprlgy5V1G*Dv&0(L>A3p!Q&XaZK;~T*{>GY)Nz>>R%r+e2ADUK{q*rV4<{-*dog4C zawvRi659wySMl4}^l_1gj(NgBQ_4&KPO*>e=KMq1y>;t(0~2IBk=SDUyQh56xFV36Qj^haYyan(1E6txlO@(OU!VN|N^eQ9Jq} zRpM4+rZJkcmp@)2`XtR5@k^uN7gP!&{{A0{55Jq7tz^E&WrV~)bJY@NBlo89P+p@e zczz#d&uo>Bg zydr)l0$Em0o|;1h6=&6~D)Nkiy$2>OJxQ(_Zi23zoJ#B1JL0*DHB+ql%GJ2ZypJS_ z%X86l*Q(HmS05rN#-ntno7QN$>9=?E7x7z`d zjLF+y6X0lyF=rh~nq?skit6k*4#w6oApnb%9#8%v5Q1NY607E~40i0+5Jd*gc`9+9 z;l_jCJh?B;{H)2AZ%)iCQJV9tWMWGQAX7Gqv7H694Z_6ucieH!ExP%~LaP@qR^Ud}ybVzm!t4X0Dw|kY7@(eaG747tYvZAM&pM){7U@4@wSdy`{7Mr4bVG zlJ5d`@*Ru%)eAamCfR$8AgUA;oHtaJfUXk8 zyg5l!-dsueMx237C&Ks~3XMFiR7sGULk^bdmNDkDh9iWajLF67Cmjg%^)>U4s%PRG z;TvA-(KHTjImZ2$`1ld+e}#3wrsXMowRVe{8tZ?n43C6-ZSJ?jua=`RTBdK&B06vP z#cay@DA0+Q)a@@z%|weURnKIWXGY;9^Tw7Kq3a`D(e1^KAKK6hqSiOx-~|sFwJVdA zuZu>W(PNTDHa@)WbKP5?n0@I(F*qFXp`4_4`r=H$@%GjWw)pF5o-oYJ@Y$(7C@5xc zkH?b8HwvDCfer4EhRF4nI78e&NAdu-JOEuSfwCg(b)PM*3wBgXF!*pW?;o+F0$yYkqR@5{$p+4CiNVP4^0Wi?(&%OQ%;Gg3-P8v(#Glz^#Q zw-Fz_af14dOW*Yns zGMTQy7UMKZFRy4iQT|JUdGsxW03CNopFM_)k8P1TCZWGJ`G~%%g-CuUH5lzU{KJ-6 zIC0B65c{F9G3(_^^_Gd-)ll8i7578$=G|30_Nb;xhtg6FV^CmbKPpRz>6>qahDwMu&bkP7<|j0>NXjP88N9KyvVh7g#W zm&n{>MVyQ9Ah~A{p;CKGeeSEUlVpgfrogE?rd8*80~KLi1!&t8|fs<2)BVhFX0HcZ@Jwlgf_Q7E{@^ zjPtqndYxh?94o$a>Mb7xHWiQ_L14caAwZ={7Ibpy>X>f#rIE3~&e4Lnhy$oAQ)@|S z>PtdrOYA?*vWUD#DG8ZHB{XN1jwrXq1-nK@;F=FGcIg%aCyXivWAe5WIsawZ+($oi zX-_{mxE!{ejkYIVqM8%bY<1&pG`>`&g7;+amPQa`C;YVUW1k-F1W)y$%IxD5CL7-8 z{cPSk?NmJL>`%$g9)Dudg;|&YP3pk0U}H^V%-s-BZ?vm^EDunv9S|+m=osjHTK76a zfnM}_ck3$JCMVgjvM{z?h=GBBP>Jry8$gnzDKRlkeVMh4ww~l|vGq%Z4c>o)=)Nmj zQ@d-QCH8J6%M4cSEGs2D!~UA1*fEB&Wa>>y*^cvz^5(K9{grT z1P%|KWy#}`rl|)yIb3O0MtR^(&^!&wphmxO(O^Qj51l>5Gmq}9VVVFYQ5I7EizwXe z;d3|675^R$K>$f8jqILB6MB9nyU6|h^|<70V5QE_uVv^Ek?aVl%%mrvwMvJ!QoRM( zPuiskdDWkW&i3=*ffJbqB#f=;lFbK#ZK5UY$%%=(vf0DT(lMV!SG-IZ|NS?5uG8dBmq|p8vnJFuj{D%nKZ87U*kt$LhBeiRJBSW4G-Xo?l7b4=@zjqf9}h(4 zIiHIHep$^3_SO;I1Z?m*$YQUC0S~j*GKW1noq%hBla+YlefrnOEz+k)0BgFazT3~- zkA2vSi3wp|XgcY+EFR_xgaTN9BTJJ$bA5crC@@L{Xla_Ie+(?O+5d(OzH#c(mAYjp zW+tPTv?eFO3!lA~WtR#g8%e!y!||yHVx9;(un|UYZ5YhY1b-+FD0XYD%)9ieq-lpD zNLs#w9SIDk%R~xX&R|Gc0{=Zv-ZWcMEEZSR(m+OEuc$32-}{?O`V<14eX{M00ft0# zXEJ2C)~_o)jl3!@15(0<5Y>Ody&u^T_uJH9X80BpYTL48n7FRtDs;}-*1$BkDHmqa z>y^l5`?6NN|9KeJAUK}gD?@dR*}HYE4;_S(-33vV^aS)FiM!^}VkR9@G;B|6e>$9a z<8AXXcFG4gtAz^tC+(Ffv@twnZ6|BiKDVAqjjsDE#=Ui2WVi1p{%)nmpAh&!-z^HU z&6di4i8Ls4a>(S|5&!GBC}hB^0Be0EALj+ASWFhr98Y;d>imfOT(*TtOeqos-Kt=oAmXk zCPe1@9Qdi_gcB^?+8DTC25HD98dOL1Y0B`o1h~^Or@Rkfs04+~?(O*1@BNd)(B&%H z^%gC>1u%OmCa%Rwxo(lVWFF5n>0q%Ip!ggM|L(uT~>ziE@<{=q=n_(dX3ALXk zmmH%#g+m4m16`dEn7YNf^U;H{~&*I($S`wJdU*)$8kZ@8j2 zFj!pgG^7ZPiAf}di#r}?HHIbpj6svyKCVx=>VdRoPn*@NZc>cTIGooKHwZ$KS@_nU z)kA!r+ts7FR^!vws@7%kElq4V1q^MKzxw{+q+aS-q~F{F&Zu(4QiK6?;HM$|MsVLKWbPJ~_9LE~iAnuE;vn*ooaqL;|`naT->tp<$ z0v?ZN{XHTO1TO%$RDd%I_<-7(g}lpbLOi?LiOso4U(i$Se96`sT6}S@HiIm zJ1swBGd9^cu{-G?ba?p;MA-?e6NmPBmx1<^>fy*?SPX$y_3ejS7tcZ|Nk_Vi;<0%r zsi&h--#Kh4s*wB9oYp7p@KcAFjhGKP)TeG`h*5aeQbIAACnq74iROpM<2>gMx>s3E z3+=AFt^W!RC-O?j0Xb!c2DTmt^9OU`7vhAO(H5eHihh}wRfeic<30@*hiE9|^ zL}nVEe*A$%@-Jwl69Rs}6V+vgfpQi$Yhul#JhWZo;+VTNR}+P^Rc4g8%+8aOa=+r1 z7%90j|GaG2Pv^m?AyOr<$`&`WHd&vpEqZs(lkSCQnLe9&m-wUt+?%v5AAvRTvH|67 zeu(~Tklzqy(uZa1ClO#|8fn_ZIGa4|=bA!4A)Fp7?+fgLDSzWhC%Bw;p;IsqERLmn zG1OajS#o%UperRZbjfQjIpjvZp~D=}tH2Hzozberv6G(d@;a`*ATPNDMG0G^=MT7_ z*9MS$Nwg-5l&Jk0!7WYw^Jme&m%wP(CBFIm)={IqtR&MOKkZNkbhAB=6lmIQS6qTD zV`aslG!REicf`eR!TLCc2P^{01~?0 zGtkoUcntToWsi#ySz$kMPv|m0r77$%voG@l?no)}qS0^-z2th2=hPp&1{^9oXWH8= ziA>Ao|Ib)vid>g+*JoKJ#<(`Gf_VP_$05hqwHZ^CzKx0A6-T#LZ))@T>AaA1t2gzq zH8vuKIlXu3rp29#M@TXOnpD1z;$HWVh)UnkY*0%V$UQW2;BpU)#ZXq|W8F7Fmpaoe z`=s$Ak1>uB7=a!J8dy^$$rUI|8{l_c{d3`nw@i|Je<9$Vg@EM zI2syh^xdqLW8q84=Iqw_u2>4X47(r>r_gOO$!(q8M9kfv>UPR)_u$!vs~<;cZGJM1 zrjVQk^lhWiO$NX@xb7L{rTUjP5c#^|Rc>g=_p_tJOfMXzD-*+y&$?4sbt12SPF};N zcRyT*-m_pk*$RkZxiMIJ}KTdseE-=p~y zQNI{zXgq%3^Lb`4V(tbgbUW1f+Qe0yz%f6!nE8?DFT@Qn>LhvoXRku{IBqrhVrE#T z^yDlM3uK~0{e2f*DJu-GDTkeuWHpZ)2ML!vhIi8Q|1n@W%oa9Kg}$T;5Rt-o9;`rL zN)xv0`FF9zt2UyB(LXBYzE?wM-vmJ~#a+r^IdaXJ2SE-_L{1)+P~`$30`z?39W=Sf zs6{|`Sq<>g=3m8vjIxByyG4nY`F%HJN{2E0_<2Val)U5L9y*<&5+UknoGvpOp2(r(G7K zjT^nbGj?Td(Mf2PGKO}@ljaHfp;Do9`m@@1a* zpm{=IuVdzaHyu+wW8H+}Dsfx)aRe)mBr8Wgu(vLucOG&93;c@6!k~|g&KKA)fsm8# zP=Y4}c!SVyL)S$onVyCd&bC1%OvazGlqG4el3kwe8KNMM_FNu?Cuo&ZHZ4c;b>DH_ zf6Yyu5NK7AnJv)nm3_^4%Vq(Dh%^O}9 zc6DPdtyFde1jnj0ldns&Di|lKZA3;DPu+!KzN^oJvP%^g`Vz*rcMSIH%-0Y>Of1rG z#;;sCj`%%)XG;Z$CFikAbvoeJ?&n8lLd5d6CMpb=IVe<@Yc*Rgy9HYG6oV9#n}1rX z{v=v-uDSVNx56JZPa{*cwi@}dIUs^197FHX4F51kSShvL1sEIr^MP+f*X*~eZ{517 ze%%3RBc*Q!^)q$5BK@R_tmeT_%3@Go}kCgp89bT1J z`33!c;e&sU(P!5Jq*P|6BL3c!bmB=UcxrP1SdCe9#Ag!@pB#2nI?E$Mm=Lv;%XPvM zYhaL!=ama_Pr*)ZxUi=@mZY*IM;G3@==q$qb_%9scBILO6EjnPm_xmbVp`0 zOz>M|oVp2NvC$t#=^YL_oI{O7nJq&fUjja23G)TZo~~-IA%=}4&Xz*)?b&89erdNM zp$FC?c1P*VD*4Y)k7$V7-JRb0oa9w!FM$q)5gb zof85nE~n}o72YBNbQX}Qb2mqhsKA*vY}h8AX>u^MN6W%vY1G%s!iSTU;hhNT%HK!1 z*$e)P-n}1QkGt8BP2=L(PT;jD&Z^`N`!2p3d|WGUD%r4|Go1LrGk4ucbodAif}MzY zrSF}$j(4I#f8+G)*?{>ZNIf;Z>-UC_+=eo}IXxGP_WeMTZEgO`Ptq>xnPKD1EZp(;@N2t(Im3Tq90nbRnL+Eb`o!VQF$Q~!pT9lg7qM}Ow4y=5HqDarw4u(`8h@2gu~7xRCIRIEf&T{}Plnb?c5FCXNB zET~15RdardPreK}&f+5&`_xOljlUb`ulOHS!B3fVCTStmQ-B8A+i30YkVhXaBC8B; zlvfAhTr>KB$gRQ8gf``YsN$_w<+2H!m-9?HD!^itwdr@HD3RnI^fPZaFWVI z2uZ(4^^AG#J;lNqqJ0S~NCy%Wk|L6;S?6{K}*kuo!c1}t0eUf+eY6)gt6zi z6YsYDQ`D*EO?d?%(=j5@%mts~m-YzuCH-!r=T{d=22Uh#nVYl6IuX zp?D9g!RCYM`3z&%E^6up0iy`C1Bl7v>0R0?8Ub!B?btMhh;lM8*SD-P6A2D?_=W$U z&aMNfscu_~iULYgI?@CYP^u6*2nvYwqV(Pcr1z#&Aq2rf?_KFtdXON!2nmEHRVfJ| zy(9$R3HslA|9jt^H}f)?$(hL{=j^l3UVD9Oed}9t%|8O)ac15Hn9!**L$=R@PhS#I zkP6+Qvw4ySJw4qs5iBFB*jHR>CBCJh z=j{z*#%>9eOZvp7j$$+~Y&ZWbPna#so08o>^C;Vm>Hrv(t&{@po7tS|? zMZx&6y+`m9LD%age?qe|B=xh^*Yc=Ty9Az3$6}F;Z=-G7UVMR;>al13iDTsPKa6GY za=kbW60x4Nc6yIRIir}Hmly)b{m4h63e|=`Y#aKr4jo7=e*i`w$Z|Hnxx#V5c8ph-DmS-KOqDHT5;5_#Ndc`O(2-orwFDIwR7+y__ zf<*J1o?M6lgA3+)xVYj&99r_11!ClbAJ&EB7?Kc?z3Ti<5Rzbe1F88yF^svdprD|q zudf65lev?ZHck(C^qNddpe^_Po6iHfG!`5-PH-tR2)}=cuHhT}2nX5N73N$pBc%oLvNl<2SF|&%L5g&b>&T?j_Jq7@X70WAtGIptV0-JXPZE#wIx|p_I0D1%ggq|eJlFCX=22ve^gkLUPSC0rqitj&k zwNqvAgticH&BQY@dX2F3aWrlE-Xrzk&nzy~Kj+pegLM+ZCBWWsHuoldRq%A+(z4X( zQC_rW#}I8(>RjDZRed`9hAOG6R%I!5T-q}~J=ZVrl<=yT$xpWQb#JEBdAHeR0`6zF zd)gm9LzwR@9RN*`cq!G!Rv5ue$2I}M(z9J8Te0qh~Z&%$X zIqS{K){ceCA4uByVc!ZGXV7*FI|q@C6!;Y=Qo$R;*N(kDF?G97x%HyWjrfK`GOWA< zcoFVi_b;vOdyXim1L8SzJiQ#EvvP+wBw>ptlH@P1Il{gihNg#F(oD%uTaVtSQ@vvv;>sp0OK`mYhHtel~7 ze$wN``^wY77hzFD@}+BjuA&u}AL-Knj#cbA#KvmEAsmD4Tb67O*QYf@OJ==dspQop zHNc--F}pV;@O{)?US7#-yQ}^>0(F2nD095LE~FS=jRH3z~>;d;0Lsn9Ak~ z9wv!%RK?oo-beN-j_1A`G-$oz5sIpGkH=gQdGS-paB<0!S=93W&CVzX z_UZ6^tyO~-LToCN{A$bKM1I1nDhD7y5rxTBUie}p+O|q;Tsog%6yOE zUPWhOZ1TVlG8cWklql7C_79tN_q%M}n4VLky(!i9%dM`AqnKq4qd?Y0E1Gpsz`A6< z?H>M->OVzRh6G&;T0MP5$A0lm8XCP|*w7mMMsddMjQODm07SCvF0p9t9dke`*WE@@ zK|%Sm!Xo%CAD=pjQ0fq_QQ}e~c5}RihFO~E`&i?7$8XUDcjVX{P`;^~)bvra(Ms`3 zSxEMuD*sF5Ks+yC6fZK$nOSq1Qjy0;b+b5&KS1A#{cHV$S-Fi<*6r{C2il~& zP&m7rjaa(CaKL8L6}q+W*bh9tN~?=t`6YRI`TV1pxVXenaElp7Q*nByff0>K0B!7B zEWh&*&yzmkO8}PdXa=zS_*P^~*Q33s{oMyCn?&q)XhIbaV-VmpujKf&Dnh^LJQN9K z(#F`Q%vsoTFJG&kW&q3+0$|?OfQTBBq8D|*>(&GdE0#aWpZB-|bkud$T`5#X+n?We zx-GKL&gynaU7bpPh&gxld>L%8&NToBP#(A}8V;PuStgw+X7w+_g^w@>K;ha3p!%lO z!xnv)L=q5;gE`B-?BbCUaS3BMkVde~JznPJ+PmVvI{CfZCgkSeb@y{vp_h0SUOJzk z#BhOlTvY9k+ZJH<(lUGEwwa2=B=k6p#srDBV6yGm=d{ZG2ZWZdv2Rn`Syt=UXZy`V zVLt=Ju_k@z0)=?g8D-ML22Ec6Ox)RCv6^heM72dowW&R(9M?5aH#MXu&s?FSmhfR{pd(H3Y1r+`6T8Z&M)`|YbK}D zM)@oS@(13ZOvY`FsI+9^;wsFh|M=*ceB=l+`cTPh;r1A;q3ZSeS{?pXGv}Lvpb*W( zAcjeg?f}bE7JtT$v=}%2SQD9tkos!}nk^u{hVaEVq2TcG4hKWZGdIDgtD0XScy=SQ;ZHEzq9 zSH~4U_wB9ZU#U=6LTYtwiA;TZ;Lad2!L|$)y~X&cWM3xi)R#+PaLQsHjm!kxaiTWU zkAB5Hj35tl&P*NVyBS_8Azzlm)y2Ne>Rp=HGvZVHlOM@< zC=Gk*B0qp9T_OgjYzYSr`bn0bXkQcfD8M-P`1Pgurdg# zd?!=kbC!z%B7VaF$8L@UukOdZHC^>!*Hl`Hl0V&nh<%+ZuDn}C^OK}sLW7UbrV_Vuvi1qo1{IzK+PHC>#OJ-2eWpKIwk^h>Z@!sL!gs?r;Qz@;+StmQ8w&`H1cj#OOuqRPv$VzvNk+#zALfU;!6x z&Cord3t72*yFdBnGLZ)RutLrf;Z91J&O8WHBDbr(lp#O5=$bkiV{g{v?Gc_w%;su< z)V_g`zRj0kXa_#N9+<}b4WGejAvTj8K0_aj`ko@^WT}=Qw8+IqJW`yvWc_~LRbC7g zJjQjNoA}G4VeLE;XsqsC8yBFr=lUnaU-{pETvvs*E0MDa| zOhwlLjkh8)x%1`I+Ox(Q;6aS?%jg)ly_$pztNl*Gho-=~-`l^Xjos-r;1+8!&{#GP zojDYkvC;qZ37(Vq#+0PmEtX4i$#(sFh|;e2WDRY3 z#|`KQ{qU+}sGaok!I-V%-SWw@=gok$Vp-c!dJV16Rf_>r-vjaV*2=Q&3UWi)fSfUu zPexRdpE#CjDO?cyZO+bD7RzKNJGY2NuaKTm@wMY0HT_zMKGGy&4{SDz$;Fu!5YTFL zlJJD+U)SYm@7dkQfYYk(E?g8>%aXx-8uX@Ij#=K&X2JkCQv*9l`<^S^^ zcCH!yyE*Bqy@99Ho_+jY^?#jJKnBPl;fGGG=*R#Q@Dlv*8yB8fBxQbJ^i1tHyMr3! znD$>1`+pzQp8@_p)$a)vh$TOxBnyDYod5mr^s_@=96=S;o)j=x2EN3HiMRJ1|3UNs z>Sb-Ah2z41pU)Rw5cJJU!aO892#O_Ao){7Y1qB0@r9W`r195>$D#o%E$&4q@2%j|! z-dj#BLQupb4y{EHr#%4XVK{%LP;S{G=jA2x`0?ZTl$7Xcv$@yewCYcFNFQLXI=|m+K<#BC@!M*9m0Vi-t4^&Kk11}BW?ziv%P;g9xhEyKO!UPln z{>WeUqmr^Ul1)PD@l`E3-kpP=iX$VQ2%?c5VaVvZ>i6~2v03vXiuMtlI!GGGumk&N z(Jg?QbViphBpt~^dOdq(L3gte9RYk!T}EWL_|JTai)_y3*yVB$#u_-bc*y2llDk>N zfu@W5)kGSxc6mKFYUt;!qRiK?;$6izosTenQR7X)Z2%=}n)Qb1s-!VY!`4XnBpeW~ z=dI~E53r~z=D>X_TfG`3Rs2gYc(jg=*@Qs^pA)o*AaU=PV#5HE-@6|x*>`W3#QAV@ zH(f7;NYN5gF18%gwFRrec#sw1e>nBb*?QLi@fs(mS&=i7E@b2?#>ePIS(p*Xw+Ml# z3MMe*6J9=G+fCx_(U`sitC0A3If7x?#!s9}>|NxNWc^LMpXzIDf}ja!wuNM=@1Dj! z7}d^cOLV%U8J!t4(qarKfcY&^hhMZ{p$5m0ffQd6KJgz?dygJz6_EAqde|IMM_Bwg zX)^&WbEm7>`tKsnJLP?K)5=R{SVIg)F`zsLY%?jZNv>oG(|ZgzQWhP^5EGoX9TqnM zJ8DY3H?Bpw8Hn_70b18SZltn}Yv;-M0jkM1=ohX5s8g7JhRaOIiul&8T`=MrbTKmg zCadt(mllgsmkB@1id}qZIU_6ZXwX8sc-!QOVQlJ09^Is*oTl9KPk!7I+AEdGh2;ul ztFY{$XZm%&*Mb6ZQdU5*69r&lHy(RJpbO$a6Q~bWaRgw#pNkmHwgkNXODxjBiC|=m z8Fgqm68|m8lSR9RtPcu_VAwe&cv;QHLb2Wx$4F&+;iRrjQ8;6H_Yz{Y{KHraJ%X-C zQCv&M-~$BgKb`@7JhrK-e2@4h1iHOHZk$F3$wggM;&C z&;z+B3V=syjFNgcYzMH)0VQOb5KG|77DwXrrD!1=( zTriJvC+mk^e6O?0C?#auj$xQJv0`G5>ny$)ele1jiA1Q_;XCh0+h>KwQ2|IWqygB zF5jspUux!HJE4YnG<4{lLk!kNs%A)P#==i(5iEqW+pCOBe#}Z zr)wC$SFZT&W8R{bOBR7}5Z)?_MOjYjO?yq?P6l~=K1#uy5cDZRt zGNc|f_(;0{1RYm3RlBk4I(ohcFGRsJiqLfZ1J3+pB`a71(E-9+@H!2Z6tI$LMK^-# zK?-;+Sq<()O)~_aJw1S0kn-h5=8!Qi<>+q`ve#TNaqBcwB$!P-h3`^05iU6IxW!t+7CvD( zccv3h*Tswb`oyof<@3u6GQQ1CZV|XalyeQk*W01?t3vXnI0Mi*kU7l97AteH*J^q3 z+SsGwLp2e=T-}k6?~$;^N(FL+M{;{>d<}b!c@2sksxgImRjd2x75PPVjqL|)FQt-J z%e1*(bsc>&1uh-?W%R;o>Z+gJL)Mp+`v+QnQ(ZP4$#Utyvgb~W9}mZYAm z8L5&XGTLaB5Ld=FzhP5VMQghF*(aNgIQw0B*zH@;#wS{l8&2&dF6HiqPqW584449G z0KT9O8z!fXA$+u8Ycqz)mwa_pSz^4&Fh3(RIyc6u~rq&-Qi%(vEM9Wp^Dr8A4| zrqmkh2s&+1mNGY4AL*ajdz(9iaL;_7dY|&^g~sEXHWx;@sjK+KYIP#_5yMmi!RlA0 z4R+lhU~eX8$-2BU1_+)s8Ql)VBHpsV!|$LF`sC=5tw9DvWoE3TjWvjX+0&o6Xfo_4 zsIube12hJ5CqFy=B*bu$^s4rp0RFcM* zfi#+ATg0dvuS>GpXEjU7E4rLzz)j#@8(ba}OjD$%X)gBe`5YFYR+wh6lql}+8g&HC zb_wB#mhEx~cWdz;ke>3(quR-J%N>3oHmF2Mf^@Hsv%mv}3Bw`JbsPEO0ZnDU^%U$o zuCTArZmR zE8MMg7_&p}r@`8C4YLrpYF@oy)&#W3nmoKW{qjqCMjk1V!R+^lF6pLk1TgQ;XCGA5H%-_&Q)z*XPem2~UxekBFA07jAs%ODpXAO0Np9W|z>=E3^VZO3}tM5*p+t+b|oA z(vOnEk>U-i>#&eshl0cuLA4X=uR>mjq!S0>`bZ`eWUD0rwV4&?*jTx6_{_B9+wdSf z?zL-%Mt5C$nbjHNxjyk8Q_)TvonSPohSYaMyzDe35JQ#ZSFAho)!kqYV^O}&!a07t zE`R9tT&GOStCd+nfX1qeXnRE%`W(|pu`gUODyj|_Osq2KJ}c|Mqc2NfU^+fiRK~1? z5c_j28#nYSs)$~{rd;go>R%K51Vyj?oaHr7o0|V1B$HjmXaPo zM>FwRc0jP+%CY>f;zT@7{tc45bw6{QaE9 z|NHj<#;y>+4{ajV?w?}=2nRs9rvR20KeWG(Z9(sjChRJF;X`@3qOvk^b#=Atp%GTg z^ouMxa9u9fudPNERjfw<&8^v_q=4dhf@d*5=P?-;$%%OX8SkZkeb881dbRsCi;|+^ zrRtg*Eki?|emwnuSq}2LvQnV4yPKVxn;3Wjf~n4ef4uM9HhiZq7+vv)4*nr z`vq}XSr#CBd7^0xxc%o5M?e=JMNiLKr{sqU3Tzx47f(;yer~j%Iq&&@ex9B603Z+s zVhxOq@BSJUlP>-1#O>|(k3kc*HN8;b&6Sm+X~JA;QsqXHe|~t_wI>ipWFjx#6iY?DU5;B-|8X1BXL8p z9bK0m0$)F9Nhi9c5e7UVKDGH1?h$C8NQ=N|gR@)o8(`3N*`51E-W#!JUG0x~bq;tb ME2uv#mp2dnAEu~sSO5S3 literal 35496 zcma&N1yEdF&;^JyxH|+G+}+(h5Fog_ySrt+JCodYqx5q z=FQa9dwu)1obEn1Qt7J{5{3ud7jzGgwQ3D|$@~Ou zNz?*w$OiCvIJ-w2BY|KNyAb#6JVx<#xQf~pRD67`N6s+EHtGTcf#9%1PUlv|Z#cmJ z(Et?aSyB!@1R`Q$r(uWC9^T?&V%8Nr!L6tws8FC6X%3_Ke-|QvVg$5*zlfkhfR;*j z<;9!>l8oyXqFo>_H@9IiLqs>?KLrP&D&YXL0v~0A!on6e#u2U_B0v{f&JeDo|FZ+4 z!nzS2P@4#FAi><(n7Y@=jhg37h)lMS!_K?Ns~YY%yZQy|n7@IQ74 z=-qV^z!CSs37+W>ZHKNhr~7|MC9=L@KdoeO2f!EhiF~`;vLG?NRmQ&}j6LW3^nzM* z2R-w4%3WYH_QyS9P(8*G$Xs*d{_R{maxUL#?;Z}L4eynTp(%9Jk0-_+{PQ2yRYZIi z%uVYH!n(`nf00|SL@%mfl{YJkTgMs5^rXrz94Eft!aTO6YD>mnR3Kmot*ff(PB4&k z#)^}_oaIV2hgOE&`bOQ(ljvsc>+6YqTdom)Cgy(O1TAUf3ufpBeE!OKs1L*)&I>ou z0h7;A^yT&Eiw8Z9DDN--Nxtq-)DXij_w8NN>G$$wDeqPH?5Iu)9P9y4|4;B2{G#eTm9zGMG7|-lzWFvXAhPQ6Sr!)Z5wV1vCv!8cVweyMw!s{%5pLBzXRl8+S*B zc7cYYrHs>HOke)_0U^lX|Cw>5RE00Oe8$uAaHsubb~Ch%5p~QN8*3JQ?SlrUrVds< ze01a*tB_x#&`N*j;zgcwDoS67;Go>Gz1-fay}6FGGz=QX#uvm4vGV7kNqIUJd>Y8w~D)P@U?!HBw8 z7}b2v`0ny2u93YtGtR8?Z=$6mA|YOPmC;4zLkHF-c+Qb%JlQ+!Xca+{?Xa+f3TdN8 z`)sd=kobDtv&Pp{sgnaTDc={}mudk1kr&41Kams%vz;Gtm^G=Mu47CP_$XE`_@|z6 zK%pSs4*EnCN@h(lUF}w-ZrKza;2Cm{+4^U5j?;x4N5Wy0ik(w5+ilaZINblQbmmFp z@mb@Z7;a{3uJ$CU@A2~nd!i(#OMsNsUg*?v&E@Wtm9cREb6FsP!ELO;F!A?xu8n})?9G=**cOM^VJ)t* zaFSN3-XU9+MctowaYfA~cl>h7c_b{62u*Q~83zRiDvP;&gvX6JjAvN|RdUiA$&(Bo z5PB#u>4pC66g&C!cGum?B6qXo;SILzao{HoD~|8O)OyxcwDLt$`L?y+zBL^$2Uy>E znwBQ`5=pWo9~LEr_H!PFxyWB`x#QbP*_kCa?nc^p9u0^cHj_EQB@}g%c#ZOUT&(SB z8^Lbs+V~!?w~U2f#yG4ME>ExeH;5rO5(5^;50TuCRv)VLcq~rtZMdzCd&gsrkrp|v z7=~Ap8baI_#T$J&wB~Jj#}X-#wcfJd+lLqN{+BVh;2NuGPgP{~Gq_Nmq(@R!M$L4k zXP|(x6AtrwaqL^uoM z1l~qQK>>^j*o1r!T%NP#b#_EDkjwZU24eW&+#D|&iE4MKcYE|Zdx#M-pB}PZ>a(%! z=u{+eTy)NcV!h=^$NQlNL*m-9JyFER+K4t;PuF1~VF@uBr!)eeAr%LA4o>guGdj47 zs{|V7b*&BwXMd4c*ASW>&Q0w2!=rt`!poa^3UvpU(;d&P-Lr%;e$Q%9DqKVhd>P{3 zvCy=Xb7*1GZUnhevzQfG`|WP;HommRH55<9-FzR1u%crZ)z4+9$z&w!2fF=8f%RC{ z#g5nGAO(B*UGyJo)$f6SVQA9Zec+XKteHZq+dYQNtU<-bTyS8wQC;5e0S`KTc{x%pllTj%HJ zH)vI8@TV=3-+vDqQ#113NF2^G#V%6#`2}6*Zqpls^r(Yrqx+(4CeQASo=?}cy>5gV zEx78#CGNbSX(avv<`Uws=poceubuQ=UfJPYtF{2wD#o*dS+hyZwC`Pv9G5O z&@&Dv$$o5fsU6asEQ#u8;SC_A#@F|{g_1w=wIua-n>wF7CIq*TuDNOXFvMuuZrq`m zJscr1Nsu^iv=uGYhS$4uvYo8lBQ0@u?}XRCS;4OQ^(dx2p&ZKdt|v@fOq#Mlhylr3 z;vv7jZgjmQ1$wtkFcEET=4E7}2p;$U9zPKJqpqr&A}^F!O5wZFv1Tq(0eNO>L9r{X zyt|)Rpj@jpg(TRLFxaxWFQl05$tob=d$X>+{LdRJr)s0ikRABb5iwT$!3d&BHc&x0 zhS}zSb{_hS70xDOeK=ak6K-TRWxA>;U|E-(c!&!F|4+1< zNB5`kb4P-W{C=L)hoi}*;Oe<$&!2@vX@j?UKFd)_#AEWS$!Iw;qOq58MPMe3z6<()BjR*J^qaLT%0pU;1>O~#7yQ+<|=m+7sgjgY{@9Y$R@?n1OK~nG(hO>HofDdPjz9R7Qj0POvt;HHA739}Mf@T@P z@^YVce71!--^WJk8r8qb&Su5rn{D)#*8Y_kv!s|CrbIc@aVL8g@>0cs9za;e(bfBB zb*m%e@A44EnRqCfXuce@cb69kX&xw6IV;b8L#i^K??L!fF0Jzy3ul>TFBM_8Lh5`rZ4h;Oj0`&Zd?) zU!_~l0BmagW!<+K2kC$N4)J8kKF!}x%_r@2}! zQ+HL#yVQJF0gSi=fhWu178NN;q>CY#L^USCUYInfdD6c_G2R}%@wSKglR;va3?G?D z%`jiCvMb&~I15cHy^{r>a$hRLL&#;^f_eO1P2r)dz_7ez> zm>P85Qco{J9A2vm-@PY0Jtk#XY}SVwITtT8rN*knG`l(Q+IkJ=Wx58$nk5gt2JM&G z<6m^4Zd0M_TVNmf9@Gw#z6E_Wt+rA0+#1g`PKC!yOY>VfeompRWwO^FhT<<{_iHUT zMd;iOMpGTt;mCYu!HLthwA)CtKGYu5c>L7-pyx{Rxa-);@P4>6ano*g`YKTCrHrwp zEEh>Q&c78mR#y~odh_%gWa~cDF;%NOf@Jx;fLwn;66Zi8GwD4w>*oa8J=Y)^uD3mD zF9esJ&=@$ExOeXdZ`19~6|KV1!U4^ei zWBGxwmpsCdy2)a4(3kgoO-Wwxk5tIgXB=(X#0ZM@uY!)7S1T>&WP`Q9()G=u7M_ zikS_?SwY(dP$!}eamYp`CrZ-Z(L)+jx-QbYO z7=MkbV24Lio~&P$mP-bpp8$JsOZbS&vQm=?{6C%UcS-WmBetH@tG|? z3Dl+WsI|_?-?Tj)7}HIW8aD$Va4Zvlo*4y0b;c|hvEU<;0qa>=+>^3~Y5km!s)PzM zt18&)+n)bS4SVnwvRU_`H(M+T?{cMQJ@Z)MCNKQxtiMg-0{1-P&UB^1%+U=Wme9T} zH|JjKww5`2k@p&b_5Zczs4J_qlOj(!SbpEc7@tzPYlQcw_~xo+yod?=f(75YeYsqx z&RBo+m#)E(yTz-&xAEn}#!ch;!vS+}C78?^O)o5e<%Fg*GM{OB<2m2UvBl`G{C?8_ zkDxFhI?tHQv@j^q~rfyP^%V6rp?U@5wKQur(B zyBicZ)MCcW_>9Uui|t_6dC&Q4k)8bsEZ2I8;5|DAR|>UFENM@T#EXty4_d^J6t)oU zzkE2op7EBsH?#w4oh`K&JKb+e59`}iaWX)am+;r}x<%BSH ze@+h#mUV6pOrrV9b99igYwF=SuL0cK#Yto1Ppb*Q9uugpbS8hP9oX9ulOvQtGr~3E zM9UY%Vg249We%G=?YAizVcfj;K2(vw8nWmRFZE&ib~}P%uEMe7o?r~8W9aeXQBbe;y>KYUh7UZvXSU63-Mr zPC{KE(+6;HfYJ1;;0=bplP2DR0165mlaUc@sW7)FgmjA=JKc}3pU_cJ3aESn5egHS z<(@RR@UKSS7Iz<6S;G;ZPhwHs@b0Krsu~2tWZs=VOB3&kgPi@;6wICfB;;5pBbs2; zJe&0NbcY2al>BfWF3??JMHXuJ!`iM_(ZU?h?QQ034mubhBx!5=`Rw#rMztdPp(*6c zUKbXPBDm|qm_{ii5c6hkitK*_^}LmMS*zqd>n9W~H0I^p1I5^A+g;YOqN=T;tEZ-= zri5SG1u?hXC(O`Pv;%8#1kT=_NgO?a~(LrrpRi(r1ms7%^AcBw1j>)eBTp{F@W+fe~aeAv-SO5rS2| zSL3$rq=FEV6R9_6qF$I;BHTxlRyl+W92vV4jYfSGX+o~Mz}tx30~G@QTo>xEIk3aO z*NU{wrl^YUYJ~n*zr@}FR+bL;y~MR_8R*XJ(qO9z(UZ^VhkcT286fn#x_P;Uno=7# zQ6MP)ii54o)ghFacH@NHMjPuHyf1@%+)5Z>lY&C<>B01>r&S$UMhr1}DRYRJ4oN6> z8GYAH&B(gH=mFvV*{eu=Qu>ZS?47;URgI@BJ+JNXyN>EdlC1imlUzm~MB034&0KT@ z6nQiWR}ov8W)Bpt)qKf&73;lLOm9h{Zn+v+n>C4;W)Hs4FIpaL4`)GdRaFagPpN&( zV(L}|RADi(|)@Cjrc^226~DN4P6O>_U4oHPR@qSKDZ|i=hRi9?e-+Oa%8fCFWDOqi+^tq_o)>e|R~GITh93SnACCLy78qFJtrl zF4sF{^ZqIVj$BAhzHqTde-ZO-{^-_VNLd;-bl|hZ_BhOXVOU~P!uzt~=1&*&ge#PJ z;@rrLSztK)eaY$DMS&jaw=)O;)ZB{*r<9gyY#pNwBls{q?G63~M8Bp-LVlyMX)!u_ zFrsu}UVJhS6${>GZmsZPj<~hk8Q+JqM_u~zFJcv@rxRDh4y%0!S43o|m5rwYMw{bi zjA`mdwy6SdIR#mymzE2dwr#3YgExkWkhUTbaBVE7KE?FCg)`BLiN zVu&yjoB=K-`Gnjl13zN-2iF^6Zi3{rw54yutA4t13N775Y~l`dPBd z6p5u5#e|PRo#7-Vn}(LB30*79VzB(zpUEmBORayO$4btYeQkURkDV{EOP@T1sZ}oe zD%h2B66u`^mzxD zseUX1mu4MY%C5dss@^I{cepKBsy0dONm}Rxd^w#(1{a!*OJp^J(3%reLIv$fWf3Um z?(^f64$|bNsvabXD)ZYL%`93jh!Q%z;+KX8oV7XMKXuj^d0lW;5Bv1_sWQ&! zE?eNxzmYrnGXbScNzAN<%ufrPzkbEA!$9W;Q=Uo``=G7zDGw`d^*OnC=zda9iP}__ zu#SaIS6{wl}x~ZtXsUnuF=^4;n&|+>+!xf=sw60 zvQ-1aUhRyEA_J!!$Gmd(i#FmHnNmB;)&8WW2wJMPfe@@jZo0M=EvMiThxU?Q%fpcq zk4%}TBB-1t!K$i2gslcd{484w)kBHSx zcV#@;5uUm$`+>c)?&KdCyBT^S3O^_#@d=Gj(~Fx$<=p1aD7zJloKmjnF5u0Topgv& zC>`p>q(*+5M6k!CymqmCHM?jPbQ|gWhO^A6@1EdU%FKnfZziZn8_4T(Us&q>C6_tJV>d z8*wHeA9i$MV`F1^`PqhVGI0iLMzk&6ozhCSFig(;A|BhQ1kOEtbkb=tIkwj8vvyUi z>gVG$UbM6g){J|0ql)ji}ei#8Q_n`_J6nb>6AHV*$8j zJ;$qLm@^i^(DS91_i&=uzBJTT6^6x;0jJ!YLR*yy(LsL=t5BGgLLW$DtI|`zyCKAd z|09?>-`OTDy}`T`wDN0WFw4?Qx~vo}`3|r`nFPH@umvkC2$3RHCb-4C=&if)>M+BT zj^FRyMa}cz_i_d|htLu8DJhF3hg&LG@l*_ZJ1-wPT`jTvcRy>n#I&^CWyTE0lPB%f zCn{JR0T0^Om@eZ*#W?>A@kF!Tu!*5~*7VdbpA6?gFyRVrAWsg7%EP2IqfESQiT+;g zN%HJ!5@@#>geVXIn_aF~Byp=1=ZG)e@8Zb@?Jv2&IvU zb2vtP6~>MG4^?YL0ywZ9Hp{rV(>}Z6Hd}Y^207n$1p?^>hf;Fc`&IaqtWA0IKgq0A@Tm=QHDh!a~iQalGFLm&$*4kP1_tGlndkP9gb zUe9M$*mXTNtajeESu1BNFs8%AE{2Yd*L3;^g0ANLIBROdGureA3xi_jbT@PgI`n-O z6o33yPf#nwOH|7^v>FC4$1X2_OYeALBYv3~tC->_z$=z}-2Ze^kc=KNX}Miw+D=6} zimALuifGK4ou;Z{c&)k(IPu+?6QeWNL&L;`YK<4PUx%1y%Rz{aiu!%o{M5Sph`xNKZ=knF6|w##T$%zu4{kv#Nel8siBX;ZY6fPR?QFKF z%RLA*JlN%ar56wp^C!)87~12rMJyh^nR-XDVk@5dc-<5-OwiuJv+&i`&}g<<8%REC z$c?iQu%WQwNd3_kr#DW$P2M1Ha$Zyc_h_86`n{AxhoM}_@8RHF6sglm{xfsdph?G8e^VUHs@zdR(cl!-2G(tif|IKFTV)N3nvfM1s zx7+fiL#{WqOpH7CmV>1pzZgj|yP3_rH2dM&+~h)^5Dip716qV=Qk1kX@hYZ%% zW43>f9Vym{=oXAC>FoWmqGEpBNw$#B_}r9@w{OROUHMXaSylRg4eP?f!Ub=J--T0> zbhr55@o#=+p{j^gdnB;A*kJ+kl;K)W`4wF8UR|!Ypuu{dp#RRhbGv{)>Fig2QrL6) z_o=wki&zIAkhoe?n3$OKFYMO$kP}mhC%?Aij!+k$Q!7xx%S3Oq1oha-*un*2OB3B~Z4z^*yc31&gX+m#dXwbJAl!Cnb3!d$9AfRGy^%HF(ULwD=4NfeSQXxC z7at#MBssql-dR;3FE=xFKe3geZh&ll5*;Q9u_eQgJk;Y=lZrMvpBpzj@Z}Vb*Vbqq z++%hC)Kw>4BAqaF5H^&=Ut&}!regR)SzuCvx~6JKgd%9hZ3%b0Wh=yLb>7l}&}_;C zR&{vwf#k|K-wqDmBaSxv=u{j!!i-TKibibIzk2ChUeqmUdRep8NbIuLZy?Rv6J&KO>r{ePK=-dL3O5sttc}7Hb@-bfrNk%~`?qWoEysLU0ruhv z3B4&n>^C%^phSP(RF{?uG?El@XWAPVdG7<{LUj&wmLRD{=BJDcvp`J7~CVWRJtwEUj@ zk1CK~6tfT}8YnG}2w~&eFDHg{Y$W27UU4qV^po#6@wKS$fj`eo7UOpy^iu&ku^e=B2 zh8L@^&+jws-*so(!R+I*xU`tQ?NwMA{I9X_Y)RmwGknvT<4rAm3Jftfa!kzhQOlDO zYfy7iG%>ln*A5_}w@WX*AH*36@*n~Y1!e|&UBGla8&FEEz*bSVWBHylSeQo;lRW@|wD&=P%Vh1#u8AOEUa=g102MJd8e zlK_RWSeWQRERy}RkQh%wB8gWpNQTe;GbGpR;YY6FP~7CkFMjvAnJQPYbh`y2y!=%K zlBQ-h9Db&pRe|ST;X#*yb78GjG6a zEQAhC9{Pq(Rxm_o-D=h@%)gW*$4N_F&z&s!k$n>T=WZo+K+j*ifZ3H6;7D`ts1Tt))N0a0UJ%0&=Q2Kl1#ZRxWA`V8gM132`?-fLmMJ#Wjq(0f8H`b z3kfwq$GH~uqXIk&BK>0S1-o*|TBw6m1Ph4x&-n$+us^30co%HvR&0mDnR#8yUbs1- zFWYk()^&{H+~rq7|M63nJrVzx8EHYJ7KIJL%x-d6Lc)1D!w^&nv#-=AoP-p|*Z-gt zve8v@4Mb=mK0%X@0YXg55S5xMMvSjby}WS}y|*rG5F?yr4Y%-3d0$YGFc`0gC9bGt z`MQCq6p2|>JALkZ85|&UKmeJ;?)c7+pz4%nDVNkX)@EF6Up>5;Xd4XyZx3lbyrur} z2ENzF%)<*R?&3bvq4V=qH&sJk-!HBvk`fz#j3Fxnjn!h#PNz3@)3VjD(brN4&_Sex z<9~jDieqBpqdT~7dcbv8_}zBH6Bzg7Ev4DiZ{60EYWc#8WJB6q5S=Gb#=3UosrMOT zaG@CkXPyf~5yVyS=Spp6AmW_z_wL$U#9qJqTGsfHGGZrzUPlC!kze*;yqy%dMgsA8 zc6Ff~-Md7@a%adaz^*B7=2no;5ff`YX^=dS>sEAhb#;}LsvVpU32v#JYO;=knFeRO zw|5NL13VH{hHfmjKX_m*z(fM5xG1W+s94F#L1{ip9LU(X;i4)q=TxqC`f00$5~mIb z*KtLP5%>A|UqhQ%p!$Cr+90+SF)WCjWeK7hF&>At#=A5%YMmlN6Ti2=_@}B@kD>}C zEHQx0$pw#B7N_%K;N?Xuo$;)c8yW6&6eZzPOgDo2>erELF4t-5A5yPrqY@U({iHL- z$`@#(4{soE8FzqB!N_AFsdlJlYHKZes=x5#uU=v|P2Imf*+!qD1!H9BGA&B#{jyoq zv@!Kt(@LRQMz%JGU{E5w6ym696EA6U$H`klj@OQ~@PTM(Jk=13vl7Ii6xIgJ?!Rl2y0^IhcsJep(ohKl%DTzf>aA@BQJGxxVIjvD#qBi1{>?g- zsFD5qu@4Ly9MXOtSwzJ4&rg|k{-1COeN`=XedeD@(gmUk!;gNoRUnp99^;*3YcS(k z-n7?(1^eIow4?%!U(y!usrN zM&Z+=Dy7nGw+D&@ak|Q=+Ml-B9iIb3pR|mn*}JZ1NKG9#4?|9&=+@PY;wr7E*S1PH zZk*BwRyFer3uo=PUlKTo$Nwm((N)E&IazS8n9)-Tn*KGo&~XF zDU4c?Mb-hT*%k~e(F<0M8vTEAArLHO8#SzNW!>>&BoVp=L#L-(i~kKQ1k4@S(Cait z?~kVTJA7GR;iSG%GfQg76={vyVQYex$gU;iACujsQ-Us601+6E2JmQc<;h=|*QfIH zez^8*(%G^f8~Sv@4h>K{MJdf}V$h=EG7ouFBp853+cI5!!etC`!H+-?AqjY7s}Toc zkA-sVM~@3joL3#Z%$aRn*2I$Ng6)z+(*K5kf3EYi&z|(HY=3dL%Tz$q8l#ei5}3ED zm=u~8V%ESxEx4W=!IF*C0RDZkW4k$>INVqhOkq1nv+sKQ5p9Ya86vdJL|s6IBihOe z!F`~EDzX}G{5omsN+GeZyP-z&+#98e$(b{Rz?w?fW>2P@ zTqJbm!B;DFiud9GC8432?pb2u?y1m02qlcBMQaRoC|sifOr*nvFBB~a0$Xn+Nf9@y zQIfXXtn~X7B^8FlRTzYJruxVlY>|dV?4&gupZI)t^R{!NjLD_7)?Na_46$eg-_Es{&(_ohkSg`X~J3J@# zNe)uVdz^&2B4qpA8>#VPg_1<~Sjc_`7-T;$Y=HQO5mI10!Hei*3M2DdIg7(4BJva8 z>%kE2WjI_z&7$$V@$y{1{Uba!Zk@dY2vkB2ch?w9G5X7acli~A+Co`|hJIFTj(+X(q~y^~&h&_}`0rt$+=1FWHmt!EU08n7&tooO z@zC9Rh| zRHr9amaNiXg@GIJzF_N{jp-)}7zLBo(%z3KXUXIo;Js>GJ3M#k{^>L7Ca40^%u=A@lr< z#B``jhR)Kf2F5IifDs{;LxeM}Wuf#6zmle&s~ikxwR;Tqi}yA(=mwJV%rs&Vdg5Q= zcM2vLMf^E!MB`itz*9p@qpM5|0Fq;{)K4v3t?^fDK8JeA8vX<<22jnLa(73 zo|x2XJE74Mz0S8Mbndg2O@iqM#=xCoiNJ?2Bwi@d3mX4R!tt@;zEn7<$+g{O zch#qF(Cg|)dw)?-kXP}7pKPHdsXC)Zv(dfxS``sTR)kp|LYw-ti|2nNN*C;iF(vZf zqx-Z;VMGB=L@`B>O7u_Am{67&^HLTp>Ctq`pOe0%?>E+7!JtSdA%9M?C^bwg+7`zg zaE}IZESlEbxM^f7#=1Fh1WCMsY?4R%8D%oOo9sq>QP!`uK!2a%mjEWU@!QC zOE8po#3BoqaAY%8atG-Pi^OBkc1xw)o`v?hTg9w2x&rCGhkSQLcMOm$yF~%OlaR?p z#3>ED07TH-w25NR0+~J^g6k>D5{qAPXB7cpY~v*B+~6+pnQw% z_#sLR1|n3u0CQw3vg;%Ze@FLKa0LG?4Jyrq3I1Yb^(zc83&rp)L=9F#+AJFmu=wNF~PPi@+uKe=zr_8CGtK1xyiuXb@j4zPU{ajTejpPQsJ)cD-xU zBWzHI!)Sh)zz+5pG3lo~B@SFmkkKg}1kzl;bTX(Lv5O|$n$xf%O!)H{(bTC$nIYM_ zVu8%EN6aisA{hW zI7~y+3{~`EwbwxGI~2rgNQvTm_3C$k1Q2fBDaT=ZvF>QaHb#EWxDgjI?C48ZQ)wwg zl+r2Al|PD5zN+Of+6v4lIr>1_s3t5e91in;OD+oby~dU$S4IuHa$__%Dj!6RXkz{I z7G*0L+*88A<2&Zkb0wWFkk~KL27^vz)6&h*yq|2|Tz=^LYf{29u0Y*jX>MsH>!dp6 z2?0m4mARpz;p5{YnMwcfH{M~xlAa1RrbC6$9V%COE|P7yk<0f} zp7`dg@zpA6wZ-QnHTMuo0*)V=?ayFM0Z%YaEk2v=o>g5Ryt?ak=2Jbvu+~;qvT|}L zgxpuxL&S4Wiw5c;uOeuNzkCeX6D}WWts{W!I1@o#IRFkVBHCW;-V>YK>w5jpCkvvO zmgB`0v8q_eZ^g=%1etdt4hX7;hbyov*vQyLO$=W@&#Yl!!^+>S%gVbg6q!(3B&IzlDJB=MNSG zv>~uUUq+*jY_mDxF6Z1=&z?!(7xp0o;8}NZ5hj-YBafrfTb#4IK}AUeDQ<zk7vXnkf}=wQ^GE?GNEx5l=bEFuq|E_ki1i_fydLG^tO6F7;{%Ytoy=c z>j!+lhixp{8qsb11=7sK>7lf(?Rl-~*x`6;{@ii#n8S8=AXDelZma9*V!7t+j1;km zkESM`!a9`dzu3BdQ=e|^euJ4CT9401EgByzLdyF|(m;GH1#tfvAzv`-dqiflm?Ik) zbl`2Z}Q~K{woTqlKtEDq2Dg?Rc3FAfe8_lMbD~}b5$lp8Q z>+25|Lk`}^cq06Kdea8hD=u8e9 zcV}Eq!va(eA|C0xMXhpSf<|Ry8pQG}$3e0^$0A<28N%bnK8$IW%Xaj)1T@6NhKrWu zu=1(u515C!Rl^7N2VrBl2#Ki;OI?(k(;=7IWdkhhSeA#Yj1F?TMZq+j4TB`(6qq!@ zb~@oV+x5F0O}m*d_6Sl^>{(s6hB=;Fgxr2_PaTh&YxzV?869~3h-iKWn~o< z{IWQiC6F$+fm#Yc8eC1p`Di0El{tBiD?_4abOj*~1FT%tqJ@`Jh#;B18b;~-nZUFJ znFFH)aq7RZTO4*t33fEta(;Rb+DuHj65b0D-?-$nN|Y^S_V_UKmE=2p&+19FlDVk< zi_9>3(*r=3_VPJ!YWW2Ze0^JfO|564WPM?tw=j=18wbiXGuqnrVFPA<)*T$vFNpINP|L8YA~2 z?bUV|fGugxSvP|f;)((qZC~67zU7-8EiqjNB(d#^Bso8y*U6o}@XPfU3?m>Ah)lR$ zS=!&<|G4GCzi~TMe!Tc)1os>dr|{#6-{TKj z-1ZY^&|))12Ra3A58j%@)QCo0Zl6@*z|Qa0kNWuNQg;GGui5vk0=}C?ZpO0E@y+_o z=w8Fy8iGY64gREP9S7KC^E=ljX7cMFt~k0BfG&HUTYvU4KYOI**Kv)(Rr8r>x@84v za!XB0zI`WWK%xm1PHhWE(hnrC7{V{GaF3cUpBXP#;l#*0v4-ore^FNKVv%7V2Uf4(FZiJ3aZ4E_Pg2#HpF{i+p%C7OVHfqy2o zELx{gAz62v?_c$@*F;tye<`DbkG`l4m#0($B#y+6swVec$Ux)fBS0;%O_?}6IXMXm z3JOhU(@*09o#?`+I9@0o$zc1z>T`aP8SUkafD~X}Fk(?$T1rhvLt8c6Xw`)ZKtjE& zYUZGKz<>S<**QVBIiRHeIx!hHw8_oE+zd?4dTsw|?H3lY@4r7N z^Px-A?negVHhAXus6!eR?{Sq+FW zF?LKTX}1~d+-BQ&5J6O*=-bniCA0Tsa69Hnq3kL_85tQ3#uCvbj~(Y_nqe6-og$V3 z-rwJYHH7hH_pNwq#mO-fR|FkRWKNN2krHQ`9~P);nc@v!(IZ_x^66Y!g_C6IlIaZ5 z;IZWvh9=j-jf8vE z5F2X8jH_92QuueBrt9kHPpz465DOTj{LD;|q)#$z>b)@duXvs8sTr92&2{gHJZmmX z{Y$#gDO9;+Q99I@q2Zg$cf*BP%C3MQ5<9Zfx8o*tM)2V7yB3rzZofO_uZt&-?!l5i zSCx)|A_9Ud!hb40EX+p(t{xbi4F2AhF0|ykh$H<3%go zKP0<{+qUC>RMG!XMFbzMFRy3%*`7|9lZFj1VPa7$TBuqQ%8`#pcT8zhzmN5uJY9kD zuv{eG&FNv@k+DB4TbG~e<0Ty;Nc7WQ(#KQ+Z@A~P+#gL%Ayx45*4__9h@@j|_t~F* zR8Hk*-Yrc@!EMbY1651B*Ca`@2jzeRrxtNwjVn-mVODY*vzvz*FQ$sWE~ma3)u`SN z9EINJYvF7$$2|rJ%bsQZ*J;@m-sNzEHO&5aZ&qRxR+htw`n&1YV>x`C3Qyx7-gahcE(da2P6<*$gf=YZ5i9A$Z{Rc zXi4Crql>no-(ZrSB)LIQ^_pt?%=7ULJG!)YIy7qo&#ym@LT=@k-**7XF8fc{r&dQ)|(*EjlNi{Z7d$* z7+{`bRQxiNAWw&W3{ihy8!a`_kV5I8M1PPe`);_A6MrLd~^o3}8JH7!|RBUngVYRt%GaO$h!c`X}ArMp3W zx>ChGFwAAsi)&uv?(JC~9N5Q{J zeK#Gbi#}YY`tv*lDVHRUX|?04LX9nk)VBsd5&o}}7h%4w>t(@cJ07M#SCL_>k2yJ@ zs3mEHJk{AtS{S@?Se`I!DjTZIfOXe2f1g{Zr_nNC!Hk+rI;k1FDA?vJUy$3#3AEbI z@Qf~+@{RX^su+(?X{MPw>bTqY-MV=5c+TV5I)5qn?8YroMZS(*a~jl`Fstiq-2c0x z=w2weQw%T->UN_QMc(`}lg7r&GSRi=wJ69kA%KuI{{mgCss(K;f*s*tly+hI^`B8z z!t&Ed0Bck!sfZQSigDjvks*~jhrp?&*QgoWbu_=h1V(g1a^P=NGMN0d+RlPjEF^pp zJbkkS2tAyo(YUXB({Nd@X7u~}eHtYjz`%W6SC5rbd?6SyzL8wWzt+PZOI!Ct}}Bm7l&_{0z@4i zI-)^F2zkvO2CHM>CgH*Pc#Zxpm@qA>zTz9AiWYbqY5KgR%pdg#g7FO)}`;! zp*M&c>>9NPy>tWU{>2A71^Y=pTB_F_;+XW<35rKE3aI3uKz`ZvB2SwP3-d_boKd$| z=MKs}esTg15A)Zg>O&$B5&oAGrfioVMbh+P5)o&3n|~1(T>tEj74kVnbIs$<;f1M$ zh$gBTH&aUER{ewt{+P2GAz$FlM24uAhQ*H_Fk15oVSqSIbVp0lT{A@6vGHim0F zEV$sK8FzsMDF%G)Y|xA{gmPB$56HNs{Hhi?t%vo}Wj;8X*iqx@bL4EZ=refePK@yV zc=UZf&xK}&rgpx)pc2VU?XRk*Pu44x=UtE>A?^CZCtn&rEs@*OsgLfZwvH?Xo@b4C zwJUW^4BJ-xQc>JMdLr$I$eZnfju82Fj{*q}o*gwgQqA76`<#OI=6#HVi^lc%>kng` zSd$xdWl<#~1>3>rtv_>a-peEAA03 zCbPCsEAxN?QvS#1SO1JpNPIQsr@`VDI_e5qt~g2?U~sx0!rNXc{C#Kqh%nQz)fKco zfI!w`!N7H{;!~q}U2)ZfId5G> zMJ-KDX{<40G;R!9j$mx+co8jcSv1ejp#136@qf(F`XGQ~Kk;E1Y>b9N4_r`$3Po}kHbF24b-rJ7OyUk+ zDe<7IqHNRn2beH)&!hnpmv#8O^vMX_d6gQA$-2wRS?BGMB&T{dL{tI6=h^w`1{8U) zP@Q@vyM=Yn3BMxV#hBcwg9B>OI*oGM7o~+}g@B;XUoXkNZj$-7G)WX(&OjbK`xi*a z`Fq#X##Q;?q@&E1na$?m{zZ<|xp`4B9pdvY3c9Insv^ zo%UL?0b~y>K;5*O{;;}+)}q$e`;*SHL9Nt$Txmh4c0MpENktjYC+Uwv7LSp6r3rfQ zhL5XWa52H$pNElr`<#$^R?;YF9@16N>FIdz( zuljx7oSqLxWY?P>%GSaBJ#1H40(eO`aLZVCuvAVeZKM4^$OmBbZd9pB@vhLt(O^g< zNI`=1A7ylvw^4p!VNt43@u%8hGiC48m{7tyvX>kb(I7TnMitX9olci_)7S|7(k+VxE#gilX~fG7xpcaVlReLm4eEHFGdJ8o59L#>N^M&=A?-6~lEPVFX_MGRZRZSH?EHdg% zRIDl_R3(;?<>R=BVVnARe+^>u0{Vy;GxYIxL*&(oxHXoZ z`_W_%kMO)rNokzU#oZn9VU5pfmM$;Yc4Q4{Ht;vpnQq)ran<_r`Q}BxsMtUP}HxG%w`}f*I4CHdhg(FGHAn%W6#5E+jI=PPrIjr-hloZC5tCPaR%(B(L=(}A3v;O>= zIj2=D83Tx1u#UX5>2H{XFf?c(zh-on=C+qNm*$R=$)H(y7a=a|o2q2jp6oy^KC^zcu6>~>w zDTSiYnnn*kbhY1q?PF|Wh!IQwF+$*0BD$+fY4{Z*oH$5xA5pA9kKBKBempDVMe-o49bnlqoIkCsxg7@xct3xkse7-1F++@+c8EJhxOC z`P^yzy4u?6+Ft{|iMt6?GEDqBh;9AM;tr1BXnZx(^67&d*7T!00Q#un=l!O~q@t?o zOI1$AP*GCJltLX?%4g_Z1mPDnht^2?#6tAMe_ioF1k)qS!)^7a;oF(veyW`#&fN({ z%_5sy18HPf^`bb6#u3R6B@_uVL;g^~Pf!*LC<;1v?er&Pw^BzBisA!4R^`%lqH=oQ zv$DFfe(9Fo`WgCOo3f>JdiY3RQR116;z#k|@jPoJjO*r1#mc6!e8fOZEiBHjug)(o zMt^NhWYq|?AI(laene6hGt!EuJXG3U(g0SFEdoH-v`+G5(}UyAF1+z;A0UL7{4`+0 zg>PVW@93=6(1n?h`E&6><0$slV zPS}*HEsPP0Gm^D14A~%6E+9UM?;4o-qrnkYQ$?8OSpu zLFU)l0cQX1%)Kx02wy>H8VyQ5?|l9fjaE(5!j)-|BQsw-W8 zsPhTYUMW(|Q8#3Xl+BnrN+gI>%JmpQV&Q2yO8YGQnap}VuWSvJQJ8p6bh=yH;i0T$ zK3YWd{H{b23VqI+uX|Hm7JPf`AZ+`+OF^Ku!41J)gf6jkdJ7?(uy{^YK+4o`J#CM# zbowaX+gpKYRS2{}7_A6iZgAK4wRP~KS|98Y;*VK@f_MeUaN4s78rR1CWC~vZsQx-K zJ6{r)zFTWxB=tn%HlU#D0`^xI@SxdegUM!)lf^t6@)zUoOM|9@0i{a1@Uq{E8 zg2_O6sL&bbU0!2lJ)U;03z8N8Y@tyJ6h@Led``fv5TdZTwhUZKqJOj7(-OyFKLQPb zKD(U!v$8e^zl<|HljhpWtr-ZMG;6RvQvQft)~DM=i!}{&P|(L5Ll9G5SLcxggFuN~ zB}paoPv8>aH==hL6GK!vNRD4PA+}JPqrIQDt$sty*EG6qAIF_1vpbFZN5=4L0}*xE zW}~~Qtghgnrt=K0cjw#4h|p(Kd8**jdhGA2E@Wy88RMO*QF`|=d2|hP$XTsSWmRqg zvIB8m^<20cIOur@aEoWqcyl^4zj!eSL6nU3fTTLfcR7#y054yxB^D!f0h9h8WoF8? z&&@XRNy5a%KeKq>F8HGO)Y8+e8q~z%mtqrdSTO>BE^^)niU-xo|-?qSH_l{3c|C0&}jHU1@}Il7r!NH)v4KvH|E;KK?F z#;8#xgzmtG3eqW?wops9x) zp#oI)d)|6<2{5vpK%<~w!kHPCbEXVY_h>)~n^qpmqU#?zp~?f<{@ZILP)_JVqpn;U z6J_DLieGC1XiNKw6Ir>NovJsPU+id~3&M5$=K>NX4^3RE+;022cRKLqQ>7`wm$afY zZg*FzCRyZFBbv=m5bP5)DwV_ynXcv+&kCP)a=PQHWU;-;*^_yi@-LF=41b;6P2vpK z#Ipfke+5lnfV)a`Qj-R6o_1aQ-LZ>xyFFXw-l+VAeC<>D&A+8&W1Mg%<mQlYocd&QOtO)QsM7#^E>a}=+zG1AYP-ZEmIR? zmP>HAD6Wh}2&+|jCW|}C`=n|H8CuRPD~l~kq%mS#E)zczZ=?IA;I zhI{QVy>Ib}-nAP!NzMsc);ZdrWL`}^MD}5CQ(F(;0?j&Qjg)&U0*fFQA9nMc1~A%y z!v-S2^Ljpo^%M74)U+{qq*Q_VD$A(^`WP5rpFlogj!(2M|K|h22;J|31u?@I6{n($ z$?tN?4n0=Lt3$Z#`rKZE&FuRzioDKr4`sPyP^QQ?XVaA;$IH@7c47DqT%jB(Kv7xH zz684_s4xB^D(bpB11NfTmo|Yxul$WlBkKk!2Io7VJfGh|fo~yXYTr&3td_eeR=w`8 zZSTq8&f3V0Dh1aJMJ{>;>?)U}JWS|Ndkp9eH&&o=z6*)u_6hTE-xW=Uh@?P5H92oU zc4rUE9`1L*BivYGt*5m;OO4dy+~b0@$TR~R)pOVs^I2?}5)*lfj2Ibe9QDtNgR(Lt z!Y6-CTS9!5O#F!8;X1&a0QiW9vp1o*&32l@(^zjjc2^nao{uyB2v4{QKpc_cCHhE` zSq`NYrLO<#o5WE~mki!w%Rzf+!Y#H+g8laC>-}2h7WXtRO{|!8>l;d-h{x+6o2+uz z=Y9OFDRAo@%e_Dw??;?`_@)!yFV);!$P)=B9)PqNprt`HKQ7lO?sFe-lj~oHU0g|h zb4wTc_&sd$G@U!?br$pUv+1kyk%oR2(zMdo{nzHw~Iugx>=9y$SzjuqIu6lW17T89bTj-ef{ zd?vNufHjQ{z@wB;Cy=vO8ALTp*KDe zZS0<4{=ILzPsAElliW62WM)Twe7HSNqDIrXe>aVDgB)-pdpy=Y%9zFTA)T$fEkW%D zJ~+x_Eh~~FsE-+%f5n}F43m(Ukx2)wfb==lkUqIsV;kr(^Itmrd38gPNk74s%nZa1 zzy(^doeA~T=NQx>CUods3ZO%yK1V-7zl4&&Pn*6>i5}29#=JCmJ%$=-u})~m>DkN2}INTYojD8 zolFes#_GO$hok2Z7D2b*(Bdfvh}*!UYYnJx$%mJ$Cj8yWb$P`rS+C_^`<(*RA3dnG&U2LBlP({*flGC0P@sj8-AeJG#c6=-b>R zPU+v!jNb>^=uny=LvkTFwnHLtd5#AQ`ceDa+{ zpc;SjL%))uwyL~}stzm&{V)t)gGQM=)E~1si9m^xo0(#aS`{B8EFBnlR#tV&H`n`3 zk7laG@0lGzT73GpJ=iruu$0=cft6FSpWt_4?eFk;ok5TC^V@R^8!K~5OG{e|+e=Fa z^)t~41P4bZbW3ly%^Lm|5m=mQh=Ez<^^VlNIF?>ng8gN;-UEt3+w!U;x0=pZRA8$k zp0r*>$`rzt5CFpGJp~jM1V$x9%+L17;8w~3@%eLL_dj?$h!OpQQAHUR2S3B9%=EL> z3}3ebXiNCr8WG1I_~Ap)=D+0BL3~<@DPSfa#QP1RS&6Jor2r33OvvzKU#|wPxPBF= zq_*_!P}aST&MS7r$(+wdwH^B%fo*{hJro9XVHGp15mTs?&bQC=DWkSW7Vtq~!_;y8 z(*cHdLgflWFQEAv86Emb!P*1uYrOa~HM>iK*^}7^L?4XA(&{#C_Gu^BHr{ppj!4CD zAhUsDNrh-fg1KPO5!5}c!vcaLAb!!*f-PE06vBg)E)G2>s!^X^&R!n+#kG`<4>MaQ zQQ~xL%IDI|%?EWmGQ+K~EZIT*b1tH0PY9hYNS}@&;S(MpqKA$P6aZa*>|g*Y%)>OG zJVg$}zI*QV&s=G>>w_&&%R%+8RWj%Av#p8PX4O6jff}AsbX0O{1Ob4hPXH+vM2EfI z78#)4`_tuShvq~y=#`NXIQQ-BJ_#){C zt-N`@LW>v~3apUeWMDQ-I+M*TB6%bM)d$kti=zfj-Q+&mN$P*@8x1gMP+`3gigsE4 z8@)-Miv zXIg!J7oz}tUy=}yXy~5ht*~<@I9$~Lg^XVlSs6c~J^N)U<~}8umq`6y9gM}ie%e}^ zptRHP!~ZT6{qRsG*vr54-5na2npy@+~FSnM1*g4M5uaPd^i z*$@IKVWIEZb2369)nY@945?Gfxg#Z>9<$8sEIlkG83+U%M18BQPFu38?|O^!CV2Og zf~!iIV;}6}tm)?-m#%MyuIl|y2hTuOVn7D&GK%(HtGrC^7XXlIGl&P*RsG7BLi`jf zCW@Bt&ShR6V7{L16-*jdN<1a?6?#`Y{;j}p|c!$_(@ArMD z(2lk1ik4IJ`;(2U6PD&O?v?L=NAdc8g(3ze__Gn}zkVZE)?F0NnJX`+2`}IkEQyPf z%b6ZDghdJQzk7oAk!k;ZM%13`Lk6H{(UktgB}%5Km7vnh8MMS&uwvFK#`i)@_Gc)I z1;68Rk@I^^bx?Ft%}Q~P1W~Bxlu%&#ks7<^l54%LZvDZ_rC)<+8JaUW;Nn!`k^}Iu z#L=g_k+rGNLxlUJ#e$NfIZZi)Lx|4aMYcp@P?M;_rt?T~LoC1L0)8GK1hdg^^!%K& z==$UFmA2`-BZXQPae{v_G)mL9MeAn+t&_~)yo2D~3aNuyfXsy$`2FZCc;H;W*Lkr4 znbQy+ivpxinuWvdQ;Lml;;3@uTl)TEoIxY$Jk-x46SQAl z`fJ=cO zC@jAeIr{9wRtRlmU$G`yJO|-EDf(`jAnVQqf{JvxsbVsJv6p&BI6$oUlx96L?JE(7 zn)IrOI8EG^Y6rtEk@taJ=!kQh|Gfc{xqgg`nzEn@yXI`v)E-rp&*NI#D530)e^y3_ z8>+{x;2RXkpjSJ_DxN<;hiyJLA1|=)+JhLPxL}a>#+WPDJ&TE)KOcVMaF!Ap&-o#R8{zul9RiFX zunKn$YEd`k9RR5)oI@k5rpVpOVe_K^@DZ zkTHM_odE?bWXyJu6vLwh#xdOJ^Cudojf4@?U_m@6MB`ni$GoDWfxJa%b-}`lO<5Cv zB7EtdzR)OPyUj{ppLKSVo6}x8*Yt4(ULimNnhgyn!8b(jnVV~DGZTjV19gO-iE7`$rue34I$V-&XMqwV zuA3BHO`oA@iqTF0Ptf3u%4#|C9NK580C3sEW6AAkzC@*goH!{sIP1lpO*y;@vid{^GYE^p3A*!U%#l1=E_hsIW96LD!SEYNO zKutqzX$99(Ryz%I6|DnK+9sNr@~VWW#6*`vdW#ik*?w*iPwt54fKhGX&=-NIhqc;4 zWwe#e9}|BZ&Iu+lKDw$5}i+hCjP)efMo@inY z@v%_;K}}0x4OiA0vS77U&2H66KR_WFY52$)3dGWM#|+qa?EP;WTVY%pCSOHRlVW4W;-zvqxROEe^Ihiw;m}&<%MbWzS?9b__cp@ zko|yC`YcT@bMzsoM!Vl+X180!9!t=Ay=1L%^rEjTPq$6dRSO8&O#r>S&blSOqJWHo z)^DlfliQh$3@T^bXFJp6v50#ijRXfodb>dt9uGk=+*@^^##%p6-Yc{1CVHCOImL-1 zWi4?rv^_b82;6O8BmF1F(Gv3s-_F58YYrQPI)H5I9=x<`O33)IX8F}gGa!=#_rA9@ zbY@tuEU(;iN2Se=YNWl~Q{xAMH}JbLU(GlYU!K_A9pd$ToY-iA(N2g8HvmuN@rwAW z5iCufpo=rMbW)qlc+Z&jqfcKs^~eYQ`O9jE1bF z-|Dxf#mqS6r;n0@02?-!?aZ;RU&0%eqp4hAc-RZ7#YsvURNF=|JpEhaRIZf%J}oP? z0Cx$?UU5M5vrPYSsizh?X-><`K5TO_txDOD!TdscRp}E$1UhylGY+~)UQOYhg7?zX zcvy9*o}lpWQ%t!4>WE(qa#+|~SjYzFqEv~40S2M`WCosPX!>aX}KRXFN>Mdlx6J=0Me~1z{V&{!KuICEJM_(yXb8%K4OhaTQ7ysJ9h;$ zFV29NM|~1ATRmNlZbZ+68+5X&N&XQueT0Og@{od%^Q*KBq0q1PD2gG8ub_(Sq#U~9 zS0@m4hYO3ww4+eH@l0f`;sp^3a)UnHuX4yOA8DE&pZTZ-e z>)kI=*fv7Kr8)TV{h}Q^DzCTK@1-^Uzh!+$kcMIHAEaXI0H{b>*9E*)KinH}`_1Bl z;{PGAM{6Ovu{A2sR*Q0ErTm4p4}YRQV#CKX<*$RGs-B59OCN~0x+c$!J7i`uH&ppG zKm86u4~n?&v(;O8GJ2<)2^K~{>sv}sAU0DIV)ZW?m#f4Lsj1vXETs<6hnkOk3n(Q# z60%uHRWw8W&}ZNP6kei+0_+PqeXWnsT=Ui^Ys(|9s~fg^=i7?T_;2}sW0dsKnwww0 zw{8s=Nej<{qTl~kV<`7(z}5r;aY%v~`c78DM@w{VK3>#SH{e3XJ4M*=kP{NU2f|t9 zM$IC+*2v;t#r)#p4&77rrh_`ZT8{ON`av{0HjNrw`;y6|as2pl*yK8>;|=xoDCaJ+ z!GHR?4QF|n(@y!vtM+`3)21iLH@uqeZQH9*mSyTMzT(@dE}?n`GjUSC4R0g9Ay%?& z3Xo&1d>G!{h~5V`zQ252Vq3}~t1pRnOFHf{Jti3vS$5g9g|9^8kZ*9>59Ox!(M+rI zSr@hH?`d2NT6{WfK~jA(+E%Phj-khm0ZH23*7{-9Z^^bWt22prKP_A)TnzhKC3lCV z4L<6mubf1wytmeaotGe?g7O%5j%!h0P7+@SRFD>ai+D(~xMzHad|-_-e}{ET=j1X{ zg{@H>rqn#w8Lrr%7fpaCY6@ zhHwzvP#;j{3h6RoMFBRdN+Sjk%S8*W31-V)O+2vuk zD!H5<_`MoUtd^7ump?O#@s(&5%2A_Jr)Ygj7$R4|xkQX@DH*z%U4>jrW^bbWhj##8 z$`vEl*WZ{HS!zNhYo#d*My#s+70jA{<>>`4e#JTCAyl&wg?D3R@dK`uw6q&1)GLw} zBhSIsdRU5oh;4q1km8*In(`j)GjSTu$7?c&y>3Mziuo$g#fICj1Iwb_gl~Di3+Gzz zp$_5Q!(7kokM6pwF3)jckPxxg_jZrHd5{C;Cf|}*>&n{|S3S1B(@-Zb>cFx^*!SPO zHip8RJ4btl03_9*2wUo3n3>RQu}O3=-QBY$bI@nh|HKu#CdcH2p;;FSVrM+Oz4jKa zvzCQ(6c)?6jxvdsltuEUwxx&gBVLsI;DU(G3hIw6Oyo_A5$Q%+a=6b$!fRY!;OAZq zgUkzl$%=?1iUpF?@GnM)B(him)v2Nk$)c0gy)TD4 z&FjPs&BZNGzT1_X99L+ONEcohW;NbtEPtFIaAw0W9F}hi75p1Pwm_jDE_yU5*n-vc z?PE4k+kl>`kRH3UFR9pZ30unn z9gXr{8R*;W?Q+rKWe3wimxNV`#&MN`g^@k^LX=#{3pC0-hyfLQ+L!d?uoJ76us^oT z&01HOR_mjmO3#m%D?Tm#YS{|&86pSdU)-Oy&^j|uL;tuk6ecAmYK}!J-k$c{c3@aV z{uD7A3pEvP%)lnJ5p4_-=}5x%bOd{-&){k2)ODKZ1)!m?wLXkhFFQp=zVs-tV>Wzf z531uj;R$$UA+Atf1;ocvn;R${7cD>PaXh%}CBKkCV+$j#X`+TofZNN3ATZFoj$EG( zMtOALKbTcDkOm|DVQU}!-EXC(_3Xs?iD5r_Z$!5_{#JiQ1NC^)Tz$RV%tp@%>&|&% zVR_+fDfh~<5`RgBNQ=|6GD5v-%zBY4x}KC-v?XW_5=W2&prrnR#>+2Z_?4Iiy|C0+ ziQkg3y95c)19Yp#{SrE4OqRNW+F$Catv@>I@bhhuRycG?JKWPX_HXa9- zyU*ehdH=I)F8A)Y)pRU0Z@wz2#s8dHg zp$Ih^8lm;m(?d!J&JyS?b=qt=>HL|(kYiH}Azq6RC_ADIe+yCfd3zeAv8BDCbBc8f;{g38l zGhIyuS2h@@$zX^vTlcn9Xaz>|-u}lRbPk(*hY{I;V@pk_D$jPkamN+s9pX`X)I);u z`H}rs6GlAgpj7FJ@n*Gj6-yp-jCp$t91#V)9LD}ZGHO8JtGa{ByOt?Z>r$_ZydPOE^4d&iTFEo2aiB}E(4qgz_PN%$~PB$mu^ z$rv?0wM*07l3m}*|CUYQ+pRB<)8QxR4uJ{>2Ys|jh%LeATt0oUD7zvaO44_aoUeE_ zQR+;Y(M13ny7))U4(S?aDY74-EcBg3?N7VbNhrc;dXeWO6H zWtdUNyjtaw1o$H1yADwHge3XmR}EIn;V9SY*N-MIgI1U8GwUHjP|4Uo+hI2T@jG@d zUpjQt=IA|cwS_vu7VL;jS>n|W*&X_~x%~7#58#$S^lI+Y`wW0@0Fs8G?+02Hn72yw z7c+f$bF(NI|HN(>f=H0FfGVoLKC*6gU{tJ?R?MZ8e4%QdL zk^IyBH|Hmc9g{?(p#JKvK}_++hC{)v3%tr;!tsmYZccVh3HyNjc%`-+Pu!1<;eG~+ zY*ez|zH!PCdJBPZl=#^o*`c_o82UgJ%#FSrR%?<>0bEU6JsxqEAVdRY(ugv3f=m${ zA|fIJLSPp%Vyuo=psjO&=knx+2l1!)&W@^_D3C5mDpZ($R0ff(v}(h{hI#2+-2)+0}}nM$DCC zYv^Kz@-^e}xx@qsX=r4)Xm=_5W4go0pzKUwUmetHkqxv1cW=q*VoNq4Lz_Ht+X0i9 zwB9K-@B0{+!};FJ>wky-is9DaVkiZ3kauDPb0$q$031LzoIofPB;--OfN!a3ijc5o zv=_Dpq7K*&N6oKGfT8{zs`=9rHJWr=78!Y@R#0Uu>mWY>+D%3ypE#!m<+1*{!kVda zVxG@2Uq-n8)5n!R$SOyiJL>M(Y32T4- zk;O_BNE9fP&G}J*J3)hkgMU{Ga9uyvT1)|2HG$g+ zwcKHAi%F)mt*gVhMS!*uR_-EJ0KBz!CZ1!0@X-}ncD_d^oThnSV>A{}j0{A%u5ae( zFYa+CqBh@Sie5e2RF|#hXsN5<+G`Yc}}ul1smZSb{cjrj8X%#Z%~^yE(;fGro1A%Rs`CHZ}un5^{=Q@QNcyVJ=9IGxJO zwLwOP=xM*Z&RHtw6tYxB@HHZ~>cziGT+a*(8sH7m26N-BP@6B3rk6=!05o9vZgh-( zJfqQROHkMo7DL>9Mx!+M&;v)1caGcbNtF2^1zldu>i=>!sjzlKiL{aWJJ?%OBU(NI zeQEANM(Uj2{b-_k(>f+ofj~hJP!!M%n?CXBqkhrm?6hi)4il@qn`i4FHfx!8c3aZz zH*F5xpM8B+DTL>%Nawapho42KW%Z_;G51;Ze}Ey<#sl_2pQfjx+tZA$VHIy?ztgwd}BD0 zp@2=v7wX=L0Q_J$HtT@5I}LWJEtPuI0Xx6&LJynt4WL}Oz_+`1$+sAKn&MeV2kLzDG&@R^sqc@x9&!!p>vS`iN<#7G3X_RII_ct|X^4{5#MZiyj-6}~f z(I=&Lz;y${DfMNZ^$PE6U@~(@GyO*Y}cy~v+ zlfON5I3d#Zesq}*OB)3kS(J_kYGc!2k0y2Mi;)MoM50JnSk}LdYrF-Lq^yM`>V0>Z~j!=EW=ut zNu<5~bhome`PaetMcl6z-sAd;zZbcNmHi$QCl48!X~t`Ni9WKG40?ZV2s%4h!Rv8Q zJe&**iuRZ1wjSx5*RQ1eJ2YotXoll|_FQ-1whi{t3oF0Cbg93)mz%+NUGA~qg;+1F zPFyKb0l#L1oj=$MjR(!jUYjAJbR_$+5@1KIE^P*p&s+{V>d!7a1Y-$19H;f$rElc9 zxB|x;>8<~AFa)nUO0@Hm;$zea+?d_Nj^$EGmmB96D^2LFPPL_`UJar*m$n|Tr z`@eh=u)CkrVchAji)i;EFFlC%I7wdliGW1Y-o1hggf>hJ3-8&vQYi|)zIKon_TPrL z`I?)gt#UtistioBauhL^u|k*n2(481NUT0Bxr#}p2gdua=vy792lDphBdC1GOP0dY z*reUrJSe>TSK8Lp>My(&3m!=FWzFOE_^a~jd^YiH0m#TP7xX;;dxyLyWX&qxQ<*>@ z>p%MD{#!R(-~Hmd;(xNj|NZ0tst?}Z05tgpUf=(K&bnJD$2%K``+@H{&Rg}%6~P8{$I=C|K7{D3d;Xqnb)rtY5(N34RS)xmJOJu`PqZthes>> zPj(x~^k2fiC*l6%hRSTx0pU5!M8Gftl<pkLEvhTC*a%F;w3hjvUlCQzX){3XkIGd2p2j)ZkapnF9FX%_%r4`u7 zOvgWNDA1>xp!?FQZ-xS0qNRf*glj`v@Bgm4rR&aTzJkQHrUu$;%ejJ&;Y*_gl1_K@Wu9eLh%prV^OL{9 zN8iXUA}3KK73hC3EP4CjR=A8?wggzE5RKi9+HlddP8ddWfe#5wICK3FzmBNe-o(fi zBXQ8#A4)Hgvl*>H{w~D(z$`^YfIS|q!rZL-)_3VL(Y%+)E3m1B4p*EqoDL`KgU`U} zmAlpRaO5Xx_|we9DmCvXiVBua<={9q=acHx&^Oyf?kvoZ#!srUT8=Z%7I~GJZX#Lw zZqBH!{6N?NbS1{@X&kJ{fQ!k`(c<+wVqhhPOO_{2SI70v9ugb8nToQHsYR?3nI@hj zR0z%`3|ziIHtuW_ljE&^#*0EAQE~GqdQ)vx%$9 zW{v&+``p!~2^Uc2y#Ne@?+XYlRVi?6o%R)aV6(ZmL*u&~s+aMPEOJd({p+2=FzZHj z7Mgx^Wss+es$(wnlfIZa!DOWgV;fCQJ3rx2I;=?K7_E zo%%#pSO(Mz^?!PDL&9ZBH=OO9ta^PFpY7}q?W#NeXe;$AwD$5FbwiUQ9p~KfpcWphYm$62-qxUszM|6#UARVPkhWzJ(xKn#OkqNqYR?B2 z;$)NG<-s=?yK56#m{_6S(Y=e67If8+KW{o#ly|Hnpyaps$Fb8mW`I;EcjJ&{M6N5!C(0j&v|x# z-n5pW4(U7_*8YCj20xMzKBUXX%I8W=!)uf{voiO>;3V8shYd)gk>XBJ@l>S=Tc_-t z({qx4Z+l%q+F<*L;e$`MSzT4Ih^3xZ|MPCo*?6p@eWfm8J`8q$nqg&vf0z-kG6ho8 z@|iU->%)(pa3PDog)2n7;$1rBBh?66&6B+B{KEuCV6KkN+b=2K3*-82T!XkjM1Fh- zbouyNQU>b4Y3U^Vhb~V{Hukh@v*Voc{Xln{3t#8j1zJamzghFH z*qJQPCO?X$T5gxs_@yA;oyaE=!+p2=7Dkhc z_v^Y`cnJE?gn^SD6#1;-AS<8zcyJ(#dd{jkqYp9-Y`Pm)3DrvljI$diTaLd6QYVxmGi+sxehKPPlf83HA| z4ebe9Q9jM8)9=KJ!_7x0mhT{?otgE$by!UfLvMlIh!N$rj9~Z>jgO= zOv>2L(^dJ1k~ey4#_>R>RoW`teRs|H<6~$U79qaCHm{9p2L;%dQF_1{)zGRr&$dP^ zws-(4?B#N(`O6?eg^%A->&1~iHdKCtBA{R(zFxaPm2-_zp=hOiif?gcTr*3^W6e65 z?vEoAc6^fy7W+xi_=|zO>!xqZ$BHyX?r6at1;Jpl-R#}MWyzFp?!OS8Spv~q)zeY2 zxz(+`q1FV2O=^xwcF(J_La>o-c;CIEJwGZyd&i9mJeSnJR1y#G3F#=x6nH7?l7Im4GrsBO*vCpW3ZMQL2ex<)cq zcXOwFWzKf~81m$|KYJ(TyY1dNkgs`Y;lgZHPFIAnooeQwTvr+|+30cQqeUU9{sMy| z+p&SGs@qpOum`%-P%*@20xs}jyub5hk?rm^wb9ewqOhpO!EL4E<}Us0j?EOYvZuw< zm1}kJB`MuODW$xof@Jj9o4#T#6!2}XkeOw;V|gNWA%~3NUZ+7&KZS!`!@Ocluofyp z(8?E&{aI#J0%L&-iP&k6S!F&t!p?Vqi#9((Z@Ab>X|&KjkV?5*13o z3vxC_npIAGztA@o_fcgkA2FQ%n^*>sTAz2J>DCIKmJ%FY(;m&oWz)~pyu8cA2&|7^ zlXv_PS>gLTL*@nFA1zxbu2=-Aq7l=d?WZe*((xY*CB3jdCPtKPLAbEhzb*0-d`Q<` znb23}JHl{%ED6e@t09MUjhPiZ=YD1>dr0Pq!?#CUeZY3Z1z}lImYquS{w)Jhb{9SP z8a~nCXj0E3`#J0zyv0q@{5M_U8iriArT0jQDR&;nuK;_5hLanXO9YiAv;Q-R!>bK5 z<~rn^|H*5f-^}P6YH{=a(gTk!L>{jGY|iqy?!>&Q)th%ph%Nt>%b01rF~9X$4*2ee z`zj?Ow*z*krwW;y?AWQl;7i<>OJ#Y7IIIJH=lk9iIIO$1X~yz3(ZM?y+xT=Utd4T^ z3rk;|Y!aEDUh}S`D6u5RCea5l(v+yc?L6*!uPvx8C+x`jXai&g}K3>{;@EYgZ z9k$X-cXFx!=eXT*=SATq`68949-HTTJ4Ur1cvTeT%#vw~d}(sEnx!Wba6gXl?8%d^ z-xrpIo?u;9dKtAP^2o8$*TJ9jMyCoquZ;1Gse4(jBhS>6abcX|&uKQHgv-JlMw zAds>3Sr0H=eo4W@Wy6LKo#%~T`&7g#dTe)76Wdc6o>EY-0e1iML0}W|P6xDo8W4Z| z+BGpz(WlR!i;Ih2zjDQaXFH$cggj=OSP?GQgoK2QjEsPQ2{C%*>mhAV|K#t$%Q24R z0>`*_?b_AV#kFG1>C>lq*_f|g=eWe%xV$?=>+9#w&F+0+tEZk!aawr$mrkN>vqIj& zjlV^Jel~j$Wi-?0!>@JMUq94uS;+SMnj*)g+=k`d5jtTSB3e_q{y9Y`2o$`&#e2<| zGW6$mdQ+XP`@-%IctV*`JzZ@9#- Date: Mon, 22 Apr 2024 12:49:02 +0200 Subject: [PATCH 15/16] Remove JournalWriter.getLastEntry() We really do not want to retain a written entry -- and there is only a single real user. Remove this method and update its sole caller. JIRA: CONTROLLER-2115 Change-Id: Ibea3d820fa7d1daa023be1cdcc1a513720744b6e Signed-off-by: Robert Varga Signed-off-by: Ruslan Kashapov --- .../storage/journal/JournalSegmentWriter.java | 24 ++++--------------- .../atomix/storage/journal/JournalWriter.java | 7 ------ .../journal/SegmentedJournalWriter.java | 10 -------- .../storage/journal/AbstractJournalTest.java | 9 ++++--- .../segjournal/SegmentedJournalActor.java | 5 ++-- 5 files changed, 12 insertions(+), 43 deletions(-) diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegmentWriter.java b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegmentWriter.java index f00e0daddd..e381bc25a7 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegmentWriter.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalSegmentWriter.java @@ -38,7 +38,6 @@ final class JournalSegmentWriter { private int currentPosition; private Long lastIndex; - private ByteBuf lastWritten; JournalSegmentWriter(final FileWriter fileWriter, final JournalSegment segment, final int maxEntrySize, final JournalIndex index) { @@ -56,7 +55,6 @@ final class JournalSegmentWriter { index = previous.index; maxSegmentSize = previous.maxSegmentSize; maxEntrySize = previous.maxEntrySize; - lastWritten = previous.lastWritten; lastIndex = previous.lastIndex; currentPosition = previous.currentPosition; this.fileWriter = requireNonNull(fileWriter); @@ -67,25 +65,16 @@ final class JournalSegmentWriter { * * @return The last written index. */ - final long getLastIndex() { + long getLastIndex() { return lastIndex != null ? lastIndex : segment.firstIndex() - 1; } - /** - * Returns the last data written. - * - * @return The last data written. - */ - final ByteBuf getLastWritten() { - return lastWritten == null ? null : lastWritten.slice(); - } - /** * Returns the next index to be written. * * @return The next index to be written. */ - final long getNextIndex() { + long getNextIndex() { return lastIndex != null ? lastIndex + 1 : segment.firstIndex(); } @@ -95,7 +84,7 @@ final class JournalSegmentWriter { * @param buf binary data to append * @return The index of appended data, or {@code null} if segment has no space */ - final Long append(final ByteBuf buf) { + Long append(final ByteBuf buf) { final var length = buf.readableBytes(); if (length > maxEntrySize) { throw new StorageException.TooLarge("Serialized entry size exceeds maximum allowed bytes (" @@ -127,7 +116,6 @@ final class JournalSegmentWriter { // Update the last entry with the correct index/term/length. currentPosition = nextPosition; - lastWritten = buf; lastIndex = index; this.index.index(index, position); @@ -139,7 +127,7 @@ final class JournalSegmentWriter { * * @param index the index to which to reset the head of the segment */ - final void reset(final long index) { + void reset(final long index) { // acquire ownership of cache and make sure reader does not see anything we've done once we're done final var fileReader = fileWriter.reader(); try { @@ -164,7 +152,6 @@ final class JournalSegmentWriter { break; } - lastWritten = buf; lastIndex = nextIndex; this.index.index(nextIndex, currentPosition); nextIndex++; @@ -179,7 +166,7 @@ final class JournalSegmentWriter { * * @param index The index to which to truncate the log. */ - final void truncate(final long index) { + void truncate(final long index) { // If the index is greater than or equal to the last index, skip the truncate. if (index >= getLastIndex()) { return; @@ -187,7 +174,6 @@ final class JournalSegmentWriter { // Reset the last written lastIndex = null; - lastWritten = null; // Truncate the index. this.index.truncate(index); diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalWriter.java b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalWriter.java index 1462463e9d..064fd019ec 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/JournalWriter.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/JournalWriter.java @@ -30,13 +30,6 @@ public interface JournalWriter { */ long getLastIndex(); - /** - * Returns the last entry written. - * - * @return The last entry written. - */ - Indexed getLastEntry(); - /** * Returns the next index to be written. * diff --git a/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalWriter.java b/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalWriter.java index 9ff535284d..71120891a1 100644 --- a/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalWriter.java +++ b/atomix-storage/src/main/java/io/atomix/storage/journal/SegmentedJournalWriter.java @@ -36,16 +36,6 @@ final class SegmentedJournalWriter implements JournalWriter { return currentWriter.getLastIndex(); } - @Override - public Indexed getLastEntry() { - final var lastWritten = currentWriter.getLastWritten(); - if (lastWritten == null) { - return null; - } - final E deserialized = journal.serializer().deserialize(lastWritten); - return new Indexed<>(currentWriter.getLastIndex(), deserialized, lastWritten.readableBytes()) ; - } - @Override public long getNextIndex() { return currentWriter.getNextIndex(); diff --git a/atomix-storage/src/test/java/io/atomix/storage/journal/AbstractJournalTest.java b/atomix-storage/src/test/java/io/atomix/storage/journal/AbstractJournalTest.java index 97d7d54c20..487c314141 100644 --- a/atomix-storage/src/test/java/io/atomix/storage/journal/AbstractJournalTest.java +++ b/atomix-storage/src/test/java/io/atomix/storage/journal/AbstractJournalTest.java @@ -186,19 +186,18 @@ public abstract class AbstractJournalTest { assertEquals(1, indexed.index()); writer.reset(1); assertEquals(0, writer.getLastIndex()); - writer.append(ENTRY); + indexed = writer.append(ENTRY); assertEquals(1, writer.getLastIndex()); - assertEquals(1, writer.getLastEntry().index()); + assertEquals(1, indexed.index()); indexed = assertNext(reader); assertEquals(1, indexed.index()); writer.truncate(0); assertEquals(0, writer.getLastIndex()); - assertNull(writer.getLastEntry()); - writer.append(ENTRY); + indexed = writer.append(ENTRY); assertEquals(1, writer.getLastIndex()); - assertEquals(1, writer.getLastEntry().index()); + assertEquals(1, indexed.index()); indexed = assertNext(reader); assertEquals(1, indexed.index()); diff --git a/opendaylight/md-sal/sal-akka-segmented-journal/src/main/java/org/opendaylight/controller/akka/segjournal/SegmentedJournalActor.java b/opendaylight/md-sal/sal-akka-segmented-journal/src/main/java/org/opendaylight/controller/akka/segjournal/SegmentedJournalActor.java index 9f63892d26..a3c28cac8a 100644 --- a/opendaylight/md-sal/sal-akka-segmented-journal/src/main/java/org/opendaylight/controller/akka/segjournal/SegmentedJournalActor.java +++ b/opendaylight/md-sal/sal-akka-segmented-journal/src/main/java/org/opendaylight/controller/akka/segjournal/SegmentedJournalActor.java @@ -486,8 +486,9 @@ abstract sealed class SegmentedJournalActor extends AbstractActor { final var sw = Stopwatch.createStarted(); deleteJournal = SegmentedJournal.builder().withDirectory(directory).withName("delete") .withNamespace(DELETE_NAMESPACE).withMaxSegmentSize(DELETE_SEGMENT_SIZE).build(); - final var lastEntry = deleteJournal.writer().getLastEntry(); - lastDelete = lastEntry == null ? 0 : lastEntry.entry(); + final var lastDeleteRecovered = deleteJournal.openReader(deleteJournal.writer().getLastIndex()) + .tryNext((index, value, length) -> value); + lastDelete = lastDeleteRecovered == null ? 0 : lastDeleteRecovered.longValue(); dataJournal = new DataJournalV0(persistenceId, messageSize, context().system(), storage, directory, maxEntrySize, maxSegmentSize); -- 2.36.6 From 373223e17636e3f4b0ffece56585098ffd719253 Mon Sep 17 00:00:00 2001 From: Ruslan Kashapov Date: Wed, 27 Mar 2024 10:52:41 +0200 Subject: [PATCH 16/16] Improve segmented journal actor metrics Update write time marked on actual flush not on flush request. JIRA: CONTROLLER-2108 Change-Id: I92a66ae775cbae6aeea69bddf654df741f473dbd Signed-off-by: Ruslan Kashapov Signed-off-by: Robert Varga --- .../segjournal/SegmentedJournalActor.java | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/opendaylight/md-sal/sal-akka-segmented-journal/src/main/java/org/opendaylight/controller/akka/segjournal/SegmentedJournalActor.java b/opendaylight/md-sal/sal-akka-segmented-journal/src/main/java/org/opendaylight/controller/akka/segjournal/SegmentedJournalActor.java index a3c28cac8a..73ffab6a05 100644 --- a/opendaylight/md-sal/sal-akka-segmented-journal/src/main/java/org/opendaylight/controller/akka/segjournal/SegmentedJournalActor.java +++ b/opendaylight/md-sal/sal-akka-segmented-journal/src/main/java/org/opendaylight/controller/akka/segjournal/SegmentedJournalActor.java @@ -121,7 +121,6 @@ abstract sealed class SegmentedJournalActor extends AbstractActor { void setFailure(final int index, final Exception cause) { results.get(index).success(Optional.of(cause)); - } void setSuccess(final int index) { @@ -193,7 +192,14 @@ abstract sealed class SegmentedJournalActor extends AbstractActor { } } - private final ArrayDeque unflushedWrites = new ArrayDeque<>(); + private record UnflushedWrite(WrittenMessages message, Stopwatch start, long count) { + UnflushedWrite { + requireNonNull(message); + requireNonNull(start); + } + } + + private final ArrayDeque unflushedWrites = new ArrayDeque<>(); private final Stopwatch unflushedDuration = Stopwatch.createUnstarted(); private final long maxUnflushedBytes; @@ -220,12 +226,12 @@ abstract sealed class SegmentedJournalActor extends AbstractActor { } @Override - void onWrittenMessages(final WrittenMessages message) { + void onWrittenMessages(final WrittenMessages message, final Stopwatch started, final long count) { boolean first = unflushedWrites.isEmpty(); if (first) { unflushedDuration.start(); } - unflushedWrites.addLast(message); + unflushedWrites.addLast(new UnflushedWrite(message, started, count)); unflushedBytes = unflushedBytes + message.writtenBytes; if (unflushedBytes >= maxUnflushedBytes) { LOG.debug("{}: reached {} unflushed journal bytes", persistenceId(), unflushedBytes); @@ -249,7 +255,7 @@ abstract sealed class SegmentedJournalActor extends AbstractActor { flushJournal(unflushedBytes, unsyncedSize); final var sw = Stopwatch.createStarted(); - unflushedWrites.forEach(WrittenMessages::complete); + unflushedWrites.forEach(write -> completeWriteMessages(write.message, write.start, write.count)); unflushedWrites.clear(); unflushedBytes = 0; unflushedDuration.reset(); @@ -264,9 +270,9 @@ abstract sealed class SegmentedJournalActor extends AbstractActor { } @Override - void onWrittenMessages(final WrittenMessages message) { + void onWrittenMessages(final WrittenMessages message, final Stopwatch started, final long count) { flushJournal(message.writtenBytes, 1); - message.complete(); + completeWriteMessages(message, started, count); } @Override @@ -453,25 +459,29 @@ abstract sealed class SegmentedJournalActor extends AbstractActor { private void handleWriteMessages(final WriteMessages message) { ensureOpen(); - final var sw = Stopwatch.createStarted(); + final var started = Stopwatch.createStarted(); final long start = dataJournal.lastWrittenSequenceNr(); final var writtenMessages = dataJournal.handleWriteMessages(message); - sw.stop(); - batchWriteTime.update(sw.elapsed(TimeUnit.NANOSECONDS), TimeUnit.NANOSECONDS); - messageWriteCount.mark(dataJournal.lastWrittenSequenceNr() - start); + onWrittenMessages(writtenMessages, started, dataJournal.lastWrittenSequenceNr() - start); + } + final void completeWriteMessages(final WrittenMessages message, final Stopwatch started, final long count) { + batchWriteTime.update(started.stop().elapsed(TimeUnit.NANOSECONDS), TimeUnit.NANOSECONDS); + messageWriteCount.mark(count); // log message after statistics are updated - LOG.debug("{}: write of {} bytes completed in {}", persistenceId, writtenMessages.writtenBytes, sw); - onWrittenMessages(writtenMessages); + LOG.debug("{}: write of {} bytes completed in {}", persistenceId, message.writtenBytes, started); + message.complete(); } /** * Handle a check of written messages. * * @param message Messages which were written + * @param started Stopwatch started when the write started + * @param count number of writes */ - abstract void onWrittenMessages(WrittenMessages message); + abstract void onWrittenMessages(WrittenMessages message, Stopwatch started, long count); private void handleUnknown(final Object message) { LOG.error("{}: Received unknown message {}", persistenceId, message); -- 2.36.6