From 003031286ffc17e48b1d3717eac29f9ab3aa2b01 Mon Sep 17 00:00:00 2001 From: Ruslan Kashapov Date: Tue, 25 Jul 2023 18:14:19 +0300 Subject: [PATCH] Standalone yang library data writer Due to yang library data is being written from multiple places it may cause data inconsistency and/or corruption. New standalone dedicated module is designed to replace existing writers with single one. JIRA: NETCONF-668 Change-Id: I34021ce5b26ee35b9662b8b156423fff5dbd9c80 Signed-off-by: Ruslan Kashapov --- apps/pom.xml | 1 + apps/yanglib-mdsal-writer/pom.xml | 67 +++++ .../writer/YangLibraryContentBuilderUtil.java | 237 +++++++++++++++++ .../YangLibrarySchemaSourceUrlProvider.java | 36 +++ .../yanglib/writer/YangLibraryWriter.java | 152 +++++++++++ .../yanglib/writer/YangLibraryWriterTest.java | 250 ++++++++++++++++++ .../src/test/resources/ietf-yang-library.yang | 10 + .../src/test/resources/test-module.yang | 16 ++ .../src/test/resources/test-more.yang | 22 ++ .../src/test/resources/test-submodule.yang | 10 + artifacts/pom.xml | 5 + 11 files changed, 806 insertions(+) create mode 100644 apps/yanglib-mdsal-writer/pom.xml create mode 100644 apps/yanglib-mdsal-writer/src/main/java/org/opendaylight/netconf/yanglib/writer/YangLibraryContentBuilderUtil.java create mode 100644 apps/yanglib-mdsal-writer/src/main/java/org/opendaylight/netconf/yanglib/writer/YangLibrarySchemaSourceUrlProvider.java create mode 100644 apps/yanglib-mdsal-writer/src/main/java/org/opendaylight/netconf/yanglib/writer/YangLibraryWriter.java create mode 100644 apps/yanglib-mdsal-writer/src/test/java/org/opendaylight/netconf/yanglib/writer/YangLibraryWriterTest.java create mode 100644 apps/yanglib-mdsal-writer/src/test/resources/ietf-yang-library.yang create mode 100644 apps/yanglib-mdsal-writer/src/test/resources/test-module.yang create mode 100644 apps/yanglib-mdsal-writer/src/test/resources/test-more.yang create mode 100644 apps/yanglib-mdsal-writer/src/test/resources/test-submodule.yang diff --git a/apps/pom.xml b/apps/pom.xml index d8b84fbdb0..a720fd5151 100644 --- a/apps/pom.xml +++ b/apps/pom.xml @@ -35,5 +35,6 @@ callhome-provider netconf-console netconf-nb + yanglib-mdsal-writer diff --git a/apps/yanglib-mdsal-writer/pom.xml b/apps/yanglib-mdsal-writer/pom.xml new file mode 100644 index 0000000000..3c6a835cbc --- /dev/null +++ b/apps/yanglib-mdsal-writer/pom.xml @@ -0,0 +1,67 @@ + + + + 4.0.0 + + + org.opendaylight.netconf + netconf-parent + 7.0.0-SNAPSHOT + ../../parent + + + yanglib-mdsal-writer + ${project.artifactId} + bundle + Yang-Library MDSAL Writer + + + + org.opendaylight.mdsal + mdsal-dom-api + + + org.opendaylight.mdsal + mdsal-binding-api + + + org.opendaylight.mdsal.binding.model.ietf + rfc6991-ietf-inet-types + + + org.opendaylight.mdsal.binding.model.ietf + rfc8525 + + + org.osgi + org.osgi.service.component.annotations + + + org.osgi + org.osgi.service.metatype.annotations + + + com.guicedee.services + javax.inject + true + + + jakarta.annotation + jakarta.annotation-api + true + + + + org.opendaylight.yangtools + yang-test-util + test + + + diff --git a/apps/yanglib-mdsal-writer/src/main/java/org/opendaylight/netconf/yanglib/writer/YangLibraryContentBuilderUtil.java b/apps/yanglib-mdsal-writer/src/main/java/org/opendaylight/netconf/yanglib/writer/YangLibraryContentBuilderUtil.java new file mode 100644 index 0000000000..00a0d9d334 --- /dev/null +++ b/apps/yanglib-mdsal-writer/src/main/java/org/opendaylight/netconf/yanglib/writer/YangLibraryContentBuilderUtil.java @@ -0,0 +1,237 @@ +/* + * Copyright (c) 2023 PANTHEON.tech s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.netconf.yanglib.writer; + +import static org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.module.list.Module.ConformanceType.Implement; +import static org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.module.list.Module.ConformanceType.Import; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jdt.annotation.Nullable; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.datastores.rev180214.Operational; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Uri; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.ModulesState; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.ModulesStateBuilder; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.RevisionIdentifier; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.YangLibrary; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.YangLibraryBuilder; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.module.list.CommonLeafs; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.module.list.module.Deviation; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.module.list.module.DeviationBuilder; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.module.list.module.DeviationKey; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.module.set.parameters.module.SubmoduleBuilder; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.yang.library.parameters.DatastoreBuilder; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.yang.library.parameters.ModuleSetBuilder; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.yang.library.parameters.SchemaBuilder; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.YangIdentifier; +import org.opendaylight.yangtools.yang.binding.util.BindingMap; +import org.opendaylight.yangtools.yang.common.QNameModule; +import org.opendaylight.yangtools.yang.model.api.EffectiveModelContext; +import org.opendaylight.yangtools.yang.model.api.FeatureDefinition; +import org.opendaylight.yangtools.yang.model.api.Module; +import org.opendaylight.yangtools.yang.model.api.ModuleLike; + +/** + * Utility class responsible for building ietf-yang-library content. + */ +// TODO: current artifact is part of integration with YangLibrarySupport from MDSAL project, +// it expected to be removed as extra once YangLibrarySupport is fully supporting required functionality. +// https://jira.opendaylight.org/browse/MDSAL-833 +// https://jira.opendaylight.org/browse/MDSAL-835 +final class YangLibraryContentBuilderUtil { + private static final CommonLeafs.Revision EMPTY_REVISION = new CommonLeafs.Revision(""); + + static final String DEFAULT_MODULE_SET_NAME = "ODL_modules"; + static final String DEFAULT_SCHEMA_NAME = "ODL_schema"; + + private YangLibraryContentBuilderUtil() { + // utility class + } + + /** + * Builds ietf-yang-library content based on model context. + * + * @param context effective model context + * @param urlProvider optional schema source URL provider + * @return content as YangLibrary object + */ + static YangLibrary buildYangLibrary(final @NonNull EffectiveModelContext context, + final @NonNull String contentId, final @Nullable YangLibrarySchemaSourceUrlProvider urlProvider) { + final var deviationsMap = getDeviationsMap(context); + return new YangLibraryBuilder() + .setModuleSet(BindingMap.of(new ModuleSetBuilder() + .setName(DEFAULT_MODULE_SET_NAME) + .setModule(context.getModules().stream() + .map(module -> buildModule(module, deviationsMap, urlProvider)) + .collect(BindingMap.toMap()) + ) + .build())) + .setSchema(BindingMap.of(new SchemaBuilder() + .setName(DEFAULT_SCHEMA_NAME) + .setModuleSet(Set.of(DEFAULT_MODULE_SET_NAME)) + .build())) + .setDatastore(BindingMap.of(new DatastoreBuilder() + .setName(Operational.VALUE) + .setSchema(DEFAULT_SCHEMA_NAME) + .build())) + .setContentId(contentId) + .build(); + } + + private static org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104 + .module.set.parameters.@NonNull Module buildModule(final @NonNull Module module, + final @NonNull Map> deviationsMap, + final @Nullable YangLibrarySchemaSourceUrlProvider urlProvider) { + return new org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library + .rev190104.module.set.parameters.ModuleBuilder() + .setName(buildModuleKeyName(module)) + .setRevision(buildRevision(module)) + .setNamespace(new Uri(module.getNamespace().toString())) + .setFeature(buildFeatures(module).orElse(null)) + .setDeviation(buildDeviations(module, deviationsMap).orElse(null)) + .setLocation(buildSchemaSourceUrl(module, urlProvider).map(Set::of).orElse(null)) + .setSubmodule(module.getSubmodules().stream() + .map(subModule -> new SubmoduleBuilder() + .setName(buildModuleKeyName(subModule)) + .setRevision(buildRevision(subModule)) + .setLocation(buildSchemaSourceUrl(subModule, urlProvider).map(Set::of).orElse(null)) + .build()) + .collect(BindingMap.toMap())) + .build(); + } + + /** + * Builds ietf-yang-library legacy content based on model context. + * + * @param context effective model context + * @param urlProvider optional schema source URL provider + * @return content as ModulesState object + * @deprecated due to model update via RFC 8525, the functionality serves backward compatibility. + */ + @Deprecated + static ModulesState buildModuleState(final @NonNull EffectiveModelContext context, + final @NonNull String moduleSetId, final @Nullable YangLibrarySchemaSourceUrlProvider urlProvider) { + final var deviationsMap = getDeviationsMap(context); + return new ModulesStateBuilder() + .setModule(context.getModules().stream() + .map(module -> buildLegacyModule(module, deviationsMap, urlProvider)) + .collect(BindingMap.toMap())) + .setModuleSetId(moduleSetId) + .build(); + } + + private static org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library + .rev190104.module.list.@NonNull Module buildLegacyModule(final @NonNull Module module, + final @NonNull Map> deviationsMap, + final @Nullable YangLibrarySchemaSourceUrlProvider urlProvider) { + + return new org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library + .rev190104.module.list.ModuleBuilder() + .setName(buildModuleKeyName(module)) + .setRevision(buildLegacyRevision(module)) + .setNamespace(new Uri(module.getNamespace().toString())) + .setFeature(buildFeatures(module).orElse(null)) + .setSchema(buildSchemaSourceUrl(module, urlProvider).orElse(null)) + .setConformanceType(hasDeviations(module) ? Implement : Import) + .setDeviation(buildLegacyDeviations(module, deviationsMap).orElse(null)) + .setSubmodule(module.getSubmodules().stream() + .map(subModule -> new org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library + .rev190104.module.list.module.SubmoduleBuilder() + .setName(buildModuleKeyName(subModule)) + .setRevision(buildLegacyRevision(subModule)) + .setSchema(buildSchemaSourceUrl(subModule, urlProvider).orElse(null)) + .build()) + .collect(BindingMap.toMap())) + .build(); + } + + private static RevisionIdentifier buildRevision(final ModuleLike module) { + return module.getQNameModule().getRevision().map(rev -> new RevisionIdentifier(rev.toString())).orElse(null); + } + + private static CommonLeafs.Revision buildLegacyRevision(final ModuleLike module) { + return module.getQNameModule().getRevision() + .map(rev -> new CommonLeafs.Revision(new RevisionIdentifier(rev.toString()))).orElse(EMPTY_REVISION); + } + + private static YangIdentifier buildModuleKeyName(final ModuleLike module) { + return new YangIdentifier(module.getName() + + module.getQNameModule().getRevision().map(revision -> "_" + revision).orElse("")); + } + + private static @NonNull Optional buildSchemaSourceUrl(final @NonNull ModuleLike module, + final @Nullable YangLibrarySchemaSourceUrlProvider urlProvider) { + return urlProvider == null ? Optional.empty() : + urlProvider.getSchemaSourceUrl(DEFAULT_MODULE_SET_NAME, module.getName(), + module.getRevision().orElse(null)); + } + + private static Optional> buildFeatures(final ModuleLike module) { + if (module.getFeatures() == null || module.getFeatures().isEmpty()) { + return Optional.empty(); + } + final var namespace = module.getQNameModule(); + final var features = module.getFeatures().stream() + .map(FeatureDefinition::getQName) + // ensure the features belong to same module + .filter(featureName -> namespace.equals(featureName.getModule())) + .map(featureName -> new YangIdentifier(featureName.getLocalName())) + .collect(Collectors.toUnmodifiableSet()); + return features.isEmpty() ? Optional.empty() : Optional.of(features); + } + + private static boolean hasDeviations(final Module module) { + return module.getDeviations() != null && !module.getDeviations().isEmpty(); + } + + private static Optional> buildDeviations(final Module module, + final Map> deviationsMap) { + final var deviationModules = deviationsMap.get(module.getQNameModule()); + if (deviationModules == null) { + return Optional.empty(); + } + return Optional.of(deviationModules.stream() + .map(devModule -> new YangIdentifier(buildModuleKeyName(devModule))) + .collect(ImmutableSet.toImmutableSet())); + } + + private static Optional> buildLegacyDeviations(final Module module, + final Map> deviationsMap) { + final var deviationModules = deviationsMap.get(module.getQNameModule()); + if (deviationModules == null) { + return Optional.empty(); + } + return Optional.of(deviationModules.stream() + .map(devModule -> new DeviationBuilder() + .setName(buildModuleKeyName(devModule)) + .setRevision(buildLegacyRevision(devModule)) + .build()) + .collect(BindingMap.toMap())); + } + + private static @NonNull Map> getDeviationsMap(final EffectiveModelContext context) { + final var result = new HashMap>(); + for (final var module : context.getModules()) { + if (module.getDeviations() == null || module.getDeviations().isEmpty()) { + continue; + } + for (final var deviation : module.getDeviations()) { + final var targetQname = deviation.getTargetPath().lastNodeIdentifier().getModule(); + result.computeIfAbsent(targetQname, key -> new HashSet<>()).add(module); + } + } + return ImmutableMap.copyOf(result); + } +} diff --git a/apps/yanglib-mdsal-writer/src/main/java/org/opendaylight/netconf/yanglib/writer/YangLibrarySchemaSourceUrlProvider.java b/apps/yanglib-mdsal-writer/src/main/java/org/opendaylight/netconf/yanglib/writer/YangLibrarySchemaSourceUrlProvider.java new file mode 100644 index 0000000000..f230a8a3b5 --- /dev/null +++ b/apps/yanglib-mdsal-writer/src/main/java/org/opendaylight/netconf/yanglib/writer/YangLibrarySchemaSourceUrlProvider.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2023 PANTHEON.tech s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.netconf.yanglib.writer; + +import java.util.Optional; +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jdt.annotation.Nullable; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Uri; +import org.opendaylight.yangtools.yang.common.Revision; + +/** + * The service providing URLs to yang schema sources. + */ +// TODO: current interface is a part of integration with YangLibrarySupport and expected +// to be removed once the similar interface is implemented there. +// Addresses https://jira.opendaylight.org/browse/MDSAL-833 +public interface YangLibrarySchemaSourceUrlProvider { + + /** + * Provides yang schema source URL where it can be downloaded from. + * + * @param moduleSetName the module set name the requested resource belongs to + * @param moduleName referenced module or submodule name + * @param revision optional revision + * + * @return optional of URL to requested resource + */ + Optional getSchemaSourceUrl(@NonNull String moduleSetName, @NonNull String moduleName, + @Nullable Revision revision); +} + diff --git a/apps/yanglib-mdsal-writer/src/main/java/org/opendaylight/netconf/yanglib/writer/YangLibraryWriter.java b/apps/yanglib-mdsal-writer/src/main/java/org/opendaylight/netconf/yanglib/writer/YangLibraryWriter.java new file mode 100644 index 0000000000..773e0371f6 --- /dev/null +++ b/apps/yanglib-mdsal-writer/src/main/java/org/opendaylight/netconf/yanglib/writer/YangLibraryWriter.java @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2023 PANTHEON.tech s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.netconf.yanglib.writer; + +import static java.util.Objects.requireNonNull; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.MoreExecutors; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.PreDestroy; +import javax.inject.Inject; +import javax.inject.Singleton; +import org.checkerframework.checker.lock.qual.GuardedBy; +import org.opendaylight.mdsal.binding.api.DataBroker; +import org.opendaylight.mdsal.common.api.CommitInfo; +import org.opendaylight.mdsal.common.api.LogicalDatastoreType; +import org.opendaylight.mdsal.dom.api.DOMSchemaService; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.ModulesState; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.YangLibrary; +import org.opendaylight.yangtools.concepts.Registration; +import org.opendaylight.yangtools.yang.binding.InstanceIdentifier; +import org.opendaylight.yangtools.yang.model.api.EffectiveModelContext; +import org.opendaylight.yangtools.yang.model.api.EffectiveModelContextListener; +import org.osgi.service.component.annotations.Activate; +import org.osgi.service.component.annotations.Component; +import org.osgi.service.component.annotations.Deactivate; +import org.osgi.service.component.annotations.Reference; +import org.osgi.service.component.annotations.ReferenceCardinality; +import org.osgi.service.metatype.annotations.AttributeDefinition; +import org.osgi.service.metatype.annotations.Designate; +import org.osgi.service.metatype.annotations.ObjectClassDefinition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Listens for updates on global schema context, transforms context to ietf-yang-library/yang-library and writes this + * state to operational data store. + */ +@Singleton +@Component(immediate = true, configurationPid = "org.opendaylight.netconf.yanglib") +@Designate(ocd = YangLibraryWriter.Configuration.class) +public final class YangLibraryWriter implements EffectiveModelContextListener, AutoCloseable { + + @ObjectClassDefinition + public @interface Configuration { + @AttributeDefinition(description = "Enables legacy content to be written") + boolean write$_$legacy() default false; + } + + private static final Logger LOG = LoggerFactory.getLogger(YangLibraryWriter.class); + private static final InstanceIdentifier YANG_LIBRARY_INSTANCE_IDENTIFIER = + InstanceIdentifier.create(YangLibrary.class); + private static final InstanceIdentifier MODULES_STATE_INSTANCE_IDENTIFIER = + InstanceIdentifier.create(ModulesState.class); + + private final AtomicLong idCounter = new AtomicLong(0L); + private final DataBroker dataBroker; + private final boolean writeLegacy; + + @Reference(cardinality = ReferenceCardinality.OPTIONAL) + volatile YangLibrarySchemaSourceUrlProvider schemaSourceUrlProvider; + + @GuardedBy("this") + private Registration reg; + + @Inject + @Activate + public YangLibraryWriter(final @Reference DOMSchemaService schemaService, + final @Reference DataBroker dataBroker, final Configuration configuration) { + this.dataBroker = requireNonNull(dataBroker); + this.writeLegacy = configuration.write$_$legacy(); + reg = schemaService.registerSchemaContextListener(this); + } + + @Deactivate + @PreDestroy + @Override + public synchronized void close() throws InterruptedException, ExecutionException { + if (reg == null) { + // Already shut down + return; + } + reg.close(); + reg = null; + + // FIXME: we should be using a transaction chain for this, but, really, this should be a dynamically-populated + // shard (i.e. no storage whatsoever)! + final var tx = dataBroker.newWriteOnlyTransaction(); + tx.delete(LogicalDatastoreType.OPERATIONAL, YANG_LIBRARY_INSTANCE_IDENTIFIER); + if (writeLegacy) { + tx.delete(LogicalDatastoreType.OPERATIONAL, MODULES_STATE_INSTANCE_IDENTIFIER); + } + + final var future = tx.commit(); + future.addCallback(new FutureCallback() { + @Override + public void onSuccess(final CommitInfo info) { + LOG.debug("YANG library cleared successfully"); + } + + @Override + public void onFailure(final Throwable throwable) { + LOG.warn("Unable to clear YANG library", throwable); + } + }, MoreExecutors.directExecutor()); + + // We need to synchronize here, otherwise we'd end up trampling over ourselves + future.get(); + } + + @Override + public void onModelContextUpdated(final EffectiveModelContext context) { + if (context.findModule(YangLibrary.QNAME.getModule()).isPresent()) { + updateYangLibrary(context); + } else { + LOG.warn("ietf-yang-library not present in context, skipping update"); + } + } + + private synchronized void updateYangLibrary(final EffectiveModelContext context) { + if (reg == null) { + // Already shut down, do not do anything + return; + } + final var nextId = String.valueOf(idCounter.incrementAndGet()); + final var tx = dataBroker.newWriteOnlyTransaction(); + tx.put(LogicalDatastoreType.OPERATIONAL, YANG_LIBRARY_INSTANCE_IDENTIFIER, + YangLibraryContentBuilderUtil.buildYangLibrary(context, nextId, schemaSourceUrlProvider)); + if (writeLegacy) { + tx.put(LogicalDatastoreType.OPERATIONAL, MODULES_STATE_INSTANCE_IDENTIFIER, + YangLibraryContentBuilderUtil.buildModuleState(context, nextId, schemaSourceUrlProvider)); + } + + tx.commit().addCallback(new FutureCallback() { + @Override + public void onSuccess(final CommitInfo result) { + LOG.debug("Yang library updated successfully"); + } + + @Override + public void onFailure(final Throwable throwable) { + LOG.warn("Failed to update yang library", throwable); + } + }, MoreExecutors.directExecutor()); + } +} diff --git a/apps/yanglib-mdsal-writer/src/test/java/org/opendaylight/netconf/yanglib/writer/YangLibraryWriterTest.java b/apps/yanglib-mdsal-writer/src/test/java/org/opendaylight/netconf/yanglib/writer/YangLibraryWriterTest.java new file mode 100644 index 0000000000..e7988d14d7 --- /dev/null +++ b/apps/yanglib-mdsal-writer/src/test/java/org/opendaylight/netconf/yanglib/writer/YangLibraryWriterTest.java @@ -0,0 +1,250 @@ +/* + * Copyright (c) 2023 PANTHEON.tech s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.netconf.yanglib.writer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.opendaylight.mdsal.common.api.CommitInfo.emptyFluentFuture; +import static org.opendaylight.mdsal.common.api.LogicalDatastoreType.OPERATIONAL; +import static org.opendaylight.netconf.yanglib.writer.YangLibraryContentBuilderUtil.DEFAULT_MODULE_SET_NAME; +import static org.opendaylight.netconf.yanglib.writer.YangLibraryContentBuilderUtil.DEFAULT_SCHEMA_NAME; +import static org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.module.list.Module.ConformanceType.Implement; +import static org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.module.list.Module.ConformanceType.Import; +import static org.opendaylight.yangtools.yang.test.util.YangParserTestUtils.parseYangResources; + +import java.util.Optional; +import java.util.Set; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opendaylight.mdsal.binding.api.DataBroker; +import org.opendaylight.mdsal.binding.api.WriteTransaction; +import org.opendaylight.mdsal.dom.api.DOMSchemaService; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.datastores.rev180214.Operational; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Uri; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.LegacyRevisionUtils; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.ModulesState; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.ModulesStateBuilder; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.RevisionIdentifier; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.RevisionUtils; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.YangLibrary; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.YangLibraryBuilder; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.module.list.CommonLeafs; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.module.list.module.DeviationBuilder; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.module.set.parameters.ModuleBuilder; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.module.set.parameters.module.SubmoduleBuilder; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.yang.library.parameters.DatastoreBuilder; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.yang.library.parameters.ModuleSetBuilder; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104.yang.library.parameters.SchemaBuilder; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.YangIdentifier; +import org.opendaylight.yangtools.concepts.ListenerRegistration; +import org.opendaylight.yangtools.yang.binding.InstanceIdentifier; +import org.opendaylight.yangtools.yang.binding.util.BindingMap; +import org.opendaylight.yangtools.yang.model.api.EffectiveModelContextListener; + +@ExtendWith(MockitoExtension.class) +class YangLibraryWriterTest { + private static final YangLibrarySchemaSourceUrlProvider URL_PROVIDER = (moduleSetName, moduleName, revision) -> + Optional.of(new Uri("/url/to/" + moduleName + (revision == null ? "" : "/" + revision))); + private static final InstanceIdentifier YANG_LIBRARY_PATH = + InstanceIdentifier.create(YangLibrary.class); + private static final InstanceIdentifier MODULES_STATE_PATH = + InstanceIdentifier.create(ModulesState.class); + private static final boolean WITH_LEGACY = true; + private static final boolean NO_LEGACY = false; + private static final boolean WITH_URLS = true; + private static final boolean NO_URLS = false; + + @Mock + private DOMSchemaService schemaService; + @Mock + private DataBroker dataBroker; + @Mock + private WriteTransaction writeTransaction; + @Mock + private ListenerRegistration registration; + @Mock + private YangLibraryWriter.Configuration config; + @Captor + private ArgumentCaptor yangLibraryCaptor; + @Captor + private ArgumentCaptor modulesStateCaptor; + private YangLibraryWriter writer; + + @BeforeEach + void beforeEach() { + doReturn(registration).when(schemaService).registerSchemaContextListener(any()); + } + + private YangLibraryWriter.Configuration setupConfig(final boolean writeLegacy) { + doReturn(writeLegacy).when(config).write$_$legacy(); + return config; + } + + @Test + @DisplayName("No update bc context has no ietf-yang-library") + void noUpdate() { + writer = new YangLibraryWriter(schemaService, dataBroker, setupConfig(NO_LEGACY)); + writer.onModelContextUpdated(parseYangResources(YangLibraryWriterTest.class, + "/test-module.yang", "/test-submodule.yang")); + verifyNoInteractions(dataBroker); + } + + @ParameterizedTest(name = "Write data -- with URLs: {0}, include legacy: {1}") + @MethodSource("writeContentArgs") + void writeContent(final boolean withUrls, final boolean writeLegacy, final YangLibrary expectedData, + final ModulesState expectedLegacyData) { + doReturn(writeTransaction).when(dataBroker).newWriteOnlyTransaction(); + doReturn(emptyFluentFuture()).when(writeTransaction).commit(); + + writer = new YangLibraryWriter(schemaService, dataBroker, setupConfig(writeLegacy)); + if (withUrls) { + writer.schemaSourceUrlProvider = URL_PROVIDER; + } + writer.onModelContextUpdated(parseYangResources(YangLibraryWriterTest.class, + "/test-module.yang", "/test-submodule.yang", "/test-more.yang", "/ietf-yang-library.yang")); + + verify(writeTransaction).put(eq(OPERATIONAL), eq(YANG_LIBRARY_PATH), yangLibraryCaptor.capture()); + assertEquals(expectedData, yangLibraryCaptor.getValue()); + if (writeLegacy) { + verify(writeTransaction).put(eq(OPERATIONAL), eq(MODULES_STATE_PATH), modulesStateCaptor.capture()); + assertEquals(expectedLegacyData, modulesStateCaptor.getValue()); + } else { + verify(writeTransaction, never()).put(eq(OPERATIONAL), eq(MODULES_STATE_PATH), any()); + } + verify(writeTransaction).commit(); + } + + private static Stream writeContentArgs() { + return Stream.of( + Arguments.of(NO_URLS, NO_LEGACY, buildYangLibrary(NO_URLS), null), + Arguments.of(NO_URLS, WITH_LEGACY, buildYangLibrary(NO_URLS), buildModulesState(NO_URLS)), + Arguments.of(WITH_URLS, NO_LEGACY, buildYangLibrary(WITH_URLS), null), + Arguments.of(WITH_URLS, WITH_LEGACY, buildYangLibrary(WITH_URLS), buildModulesState(WITH_URLS))); + } + + @ParameterizedTest(name = "Clear data on close -- include legacy: {0}") + @ValueSource(booleans = {false, true}) + void clearOnClose(final boolean writeLegacy) throws Exception { + doReturn(writeTransaction).when(dataBroker).newWriteOnlyTransaction(); + doReturn(emptyFluentFuture()).when(writeTransaction).commit(); + + new YangLibraryWriter(schemaService, dataBroker, setupConfig(writeLegacy)).close(); + verify(writeTransaction).delete(OPERATIONAL, YANG_LIBRARY_PATH); + if (writeLegacy) { + verify(writeTransaction).delete(OPERATIONAL, MODULES_STATE_PATH); + } else { + verify(writeTransaction, never()).delete(OPERATIONAL, MODULES_STATE_PATH); + } + verify(writeTransaction).commit(); + } + + private static YangLibrary buildYangLibrary(final boolean withUrls) { + return new YangLibraryBuilder() + .setModuleSet(BindingMap.of( + new ModuleSetBuilder() + .setName(DEFAULT_MODULE_SET_NAME) + .setModule(BindingMap.of( + new ModuleBuilder().setName(new YangIdentifier("test-module_2013-07-22")) + .setNamespace(new Uri("test:namespace")) + .setRevision(new RevisionIdentifier("2013-07-22")) + .setLocation(withUrls ? Set.of(new Uri("/url/to/test-module/2013-07-22")) : null) + .setSubmodule(BindingMap.of( + new SubmoduleBuilder() + .setName(new YangIdentifier("test-submodule")) + .setRevision(RevisionUtils.emptyRevision().getRevisionIdentifier()) + .setLocation(withUrls ? Set.of(new Uri("/url/to/test-submodule")) : null) + .build())) + .setDeviation(Set.of(new YangIdentifier("test-more_2023-07-25"))) + .build(), + new ModuleBuilder().setName(new YangIdentifier("test-more_2023-07-25")) + .setNamespace(new Uri("test:more")) + .setRevision(new RevisionIdentifier("2023-07-25")) + .setLocation(withUrls ? Set.of(new Uri("/url/to/test-more/2023-07-25")) : null) + .setFeature(Set.of( + new YangIdentifier("first-feature"), new YangIdentifier("second-feature"))) + .build(), + new ModuleBuilder().setName(new YangIdentifier("ietf-yang-library_2019-01-04")) + .setNamespace(new Uri("urn:ietf:params:xml:ns:yang:ietf-yang-library")) + .setRevision(new RevisionIdentifier("2019-01-04")) + .setLocation(withUrls ? Set.of(new Uri("/url/to/ietf-yang-library/2019-01-04")) : null) + .build())) + .build())) + .setSchema(BindingMap.of(new SchemaBuilder() + .setName(DEFAULT_SCHEMA_NAME) + .setModuleSet(Set.of(DEFAULT_MODULE_SET_NAME)) + .build())) + .setDatastore(BindingMap.of( + new DatastoreBuilder().setName(Operational.VALUE) + .setSchema(DEFAULT_SCHEMA_NAME) + .build())) + .setContentId("1") + .build(); + } + + private static ModulesState buildModulesState(final boolean withUrls) { + return new ModulesStateBuilder() + .setModule(BindingMap.of( + new org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104 + .module.list.ModuleBuilder() + .setName(new YangIdentifier("test-module_2013-07-22")) + .setNamespace(new Uri("test:namespace")) + .setRevision(new CommonLeafs.Revision(new RevisionIdentifier("2013-07-22"))) + .setSchema(withUrls ? new Uri("/url/to/test-module/2013-07-22") : null) + .setSubmodule(BindingMap.of( + new org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104 + .module.list.module.SubmoduleBuilder() + .setName(new YangIdentifier("test-submodule")) + .setRevision(LegacyRevisionUtils.emptyRevision()) + .setSchema(withUrls ? new Uri("/url/to/test-submodule") : null) + .build())) + .setDeviation(BindingMap.of( + new DeviationBuilder() + .setName(new YangIdentifier("test-more_2023-07-25")) + .setRevision(new CommonLeafs.Revision(new RevisionIdentifier("2023-07-25"))) + .build() + )) + .setConformanceType(Import) + .build(), + new org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104 + .module.list.ModuleBuilder() + .setName(new YangIdentifier("test-more_2023-07-25")) + .setNamespace(new Uri("test:more")) + .setRevision(new CommonLeafs.Revision(new RevisionIdentifier("2023-07-25"))) + .setSchema(withUrls ? new Uri("/url/to/test-more/2023-07-25") : null) + .setFeature(Set.of( + new YangIdentifier("first-feature"), new YangIdentifier("second-feature"))) + .setConformanceType(Implement) + .build(), + new org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.library.rev190104 + .module.list.ModuleBuilder() + .setName(new YangIdentifier("ietf-yang-library_2019-01-04")) + .setNamespace(new Uri("urn:ietf:params:xml:ns:yang:ietf-yang-library")) + .setRevision(new CommonLeafs.Revision(new RevisionIdentifier("2019-01-04"))) + .setSchema(withUrls ? new Uri("/url/to/ietf-yang-library/2019-01-04") : null) + .setConformanceType(Import) + .build())) + .setModuleSetId("1") + .build(); + } +} diff --git a/apps/yanglib-mdsal-writer/src/test/resources/ietf-yang-library.yang b/apps/yanglib-mdsal-writer/src/test/resources/ietf-yang-library.yang new file mode 100644 index 0000000000..ac7851ca44 --- /dev/null +++ b/apps/yanglib-mdsal-writer/src/test/resources/ietf-yang-library.yang @@ -0,0 +1,10 @@ +module ietf-yang-library { + yang-version 1.1; + namespace "urn:ietf:params:xml:ns:yang:ietf-yang-library"; + prefix yanglib; + + revision 2019-01-04; + + container yang-library { + } +} diff --git a/apps/yanglib-mdsal-writer/src/test/resources/test-module.yang b/apps/yanglib-mdsal-writer/src/test/resources/test-module.yang new file mode 100644 index 0000000000..395733827f --- /dev/null +++ b/apps/yanglib-mdsal-writer/src/test/resources/test-module.yang @@ -0,0 +1,16 @@ +module test-module { + yang-version 1.1; + namespace "test:namespace"; + prefix tm; + + include test-submodule; + + revision 2013-07-22; + + container cont { + leaf lf { + mandatory true; + type string; + } + } +} diff --git a/apps/yanglib-mdsal-writer/src/test/resources/test-more.yang b/apps/yanglib-mdsal-writer/src/test/resources/test-more.yang new file mode 100644 index 0000000000..5275e87af0 --- /dev/null +++ b/apps/yanglib-mdsal-writer/src/test/resources/test-more.yang @@ -0,0 +1,22 @@ +module test-more { + yang-version 1.1; + namespace "test:more"; + prefix mr; + + import test-module { + prefix tm; + } + + revision 2023-07-25; + + feature first-feature; + + feature second-feature; + + deviation "/tm:cont/tm:lf" { + deviate replace { + mandatory false; + type uint32; + } + } +} diff --git a/apps/yanglib-mdsal-writer/src/test/resources/test-submodule.yang b/apps/yanglib-mdsal-writer/src/test/resources/test-submodule.yang new file mode 100644 index 0000000000..53d097d860 --- /dev/null +++ b/apps/yanglib-mdsal-writer/src/test/resources/test-submodule.yang @@ -0,0 +1,10 @@ +submodule test-submodule { + yang-version 1.1; + + belongs-to test-module { + prefix tm; + } + + container sub { + } +} \ No newline at end of file diff --git a/artifacts/pom.xml b/artifacts/pom.xml index a7b9f0d6af..502893a9f0 100644 --- a/artifacts/pom.xml +++ b/artifacts/pom.xml @@ -334,6 +334,11 @@ netconf-topology-singleton ${project.version} + + ${project.groupId} + yanglib-mdsal-writer + ${project.version} + -- 2.36.6