2 * Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.netconf.client.mdsal.impl;
10 import static org.opendaylight.netconf.client.mdsal.impl.NetconfMessageTransformUtil.NETCONF_DATA_NODEID;
11 import static org.opendaylight.yang.svc.v1.urn.ietf.params.xml.ns.yang.ietf.netconf.monitoring.rev101004.YangModuleInfoImpl.qnameOf;
13 import com.google.common.annotations.VisibleForTesting;
14 import com.google.common.collect.ImmutableSet;
15 import com.google.common.collect.Sets;
16 import com.google.common.util.concurrent.AsyncFunction;
17 import com.google.common.util.concurrent.Futures;
18 import com.google.common.util.concurrent.ListenableFuture;
19 import com.google.common.util.concurrent.MoreExecutors;
20 import java.io.IOException;
21 import java.net.URISyntaxException;
22 import java.time.format.DateTimeParseException;
23 import java.util.HashSet;
24 import java.util.List;
26 import java.util.stream.Collectors;
27 import javax.xml.stream.XMLStreamException;
28 import javax.xml.transform.dom.DOMSource;
29 import org.eclipse.jdt.annotation.NonNull;
30 import org.eclipse.jdt.annotation.Nullable;
31 import org.opendaylight.mdsal.dom.api.DOMRpcResult;
32 import org.opendaylight.netconf.api.NamespaceURN;
33 import org.opendaylight.netconf.api.xml.XmlUtil;
34 import org.opendaylight.netconf.client.mdsal.api.NetconfDeviceSchemas;
35 import org.opendaylight.netconf.client.mdsal.api.NetconfDeviceSchemasResolver;
36 import org.opendaylight.netconf.client.mdsal.api.NetconfRpcService;
37 import org.opendaylight.netconf.client.mdsal.api.NetconfSessionPreferences;
38 import org.opendaylight.netconf.client.mdsal.api.ProvidedSources;
39 import org.opendaylight.netconf.client.mdsal.api.RemoteDeviceId;
40 import org.opendaylight.netconf.common.mdsal.NormalizedDataUtil;
41 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.netconf.base._1._0.rev110601.Get;
42 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.netconf.base._1._0.rev110601.GetInput;
43 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.netconf.base._1._0.rev110601.get.input.Filter;
44 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.netconf.monitoring.rev101004.NetconfState;
45 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.netconf.monitoring.rev101004.Yang;
46 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.netconf.monitoring.rev101004.netconf.state.Schemas;
47 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.netconf.monitoring.rev101004.netconf.state.schemas.Schema;
48 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.netconf.monitoring.rev101004.netconf.state.schemas.Schema.Location;
49 import org.opendaylight.yangtools.yang.common.ErrorSeverity;
50 import org.opendaylight.yangtools.yang.common.QName;
51 import org.opendaylight.yangtools.yang.common.Revision;
52 import org.opendaylight.yangtools.yang.common.XMLNamespace;
53 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifier;
54 import org.opendaylight.yangtools.yang.data.api.schema.AnyxmlNode;
55 import org.opendaylight.yangtools.yang.data.api.schema.ContainerNode;
56 import org.opendaylight.yangtools.yang.data.api.schema.DataContainerNode;
57 import org.opendaylight.yangtools.yang.data.api.schema.LeafNode;
58 import org.opendaylight.yangtools.yang.data.api.schema.MapEntryNode;
59 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
60 import org.opendaylight.yangtools.yang.data.api.schema.SystemLeafSetNode;
61 import org.opendaylight.yangtools.yang.data.api.schema.SystemMapNode;
62 import org.opendaylight.yangtools.yang.data.spi.node.ImmutableNodes;
63 import org.opendaylight.yangtools.yang.model.api.EffectiveModelContext;
64 import org.opendaylight.yangtools.yang.model.api.source.YangTextSource;
65 import org.opendaylight.yangtools.yang.model.api.stmt.FeatureSet;
66 import org.slf4j.Logger;
67 import org.slf4j.LoggerFactory;
68 import org.w3c.dom.Document;
69 import org.w3c.dom.Node;
70 import org.w3c.dom.traversal.DocumentTraversal;
71 import org.w3c.dom.traversal.NodeFilter;
72 import org.w3c.dom.traversal.TreeWalker;
73 import org.xml.sax.SAXException;
76 * Default implementation resolving schemas QNames from netconf-state or from modules-state.
78 public final class NetconfStateSchemasResolverImpl implements NetconfDeviceSchemasResolver {
79 private static final Logger LOG = LoggerFactory.getLogger(NetconfStateSchemasResolverImpl.class);
80 private static final String MONITORING_NAMESPACE = NetconfState.QNAME.getNamespace().toString();
81 private static final @NonNull NodeIdentifier SCHEMA_FORMAT_NODEID = NodeIdentifier.create(qnameOf("format"));
82 private static final @NonNull NodeIdentifier SCHEMA_LOCATION_NODEID = NodeIdentifier.create(qnameOf("location"));
83 private static final @NonNull NodeIdentifier SCHEMA_NAMESPACE_NODEID = NodeIdentifier.create(qnameOf("namespace"));
84 private static final @NonNull NodeIdentifier SCHEMA_IDENTIFIER_NODEID =
85 NodeIdentifier.create(qnameOf("identifier"));
86 private static final @NonNull NodeIdentifier SCHEMA_VERSION_NODEID = NodeIdentifier.create(qnameOf("version"));
87 private static final @NonNull String NETCONF_LOCATION = Location.Enumeration.NETCONF.getName();
88 private static final @NonNull ContainerNode GET_SCHEMAS_RPC;
91 final var document = XmlUtil.newDocument();
92 final var filterElem = document.createElementNS(NamespaceURN.BASE, "filter");
93 filterElem.setAttribute("type", "subtree");
95 final var stateElem = document.createElementNS(NetconfState.QNAME.getNamespace().toString(),
96 NetconfState.QNAME.getLocalName());
97 stateElem.appendChild(document.createElementNS(Schemas.QNAME.getNamespace().toString(),
98 Schemas.QNAME.getLocalName()));
99 filterElem.appendChild(stateElem);
101 GET_SCHEMAS_RPC = ImmutableNodes.newContainerBuilder()
102 .withNodeIdentifier(new NodeIdentifier(GetInput.QNAME))
103 .withChild(ImmutableNodes.newAnyxmlBuilder(DOMSource.class)
104 .withNodeIdentifier(new NodeIdentifier(Filter.QNAME))
105 .withValue(new DOMSource(filterElem))
111 public ListenableFuture<NetconfDeviceSchemas> resolve(final RemoteDeviceId deviceId,
112 final NetconfSessionPreferences sessionPreferences, final NetconfRpcService deviceRpc,
113 final EffectiveModelContext baseModelContext) {
114 // Find all schema sources provided via ietf-netconf-monitoring, if supported
115 final var monitoringFuture = sessionPreferences.isMonitoringSupported()
116 ? resolveMonitoringSources(deviceId, deviceRpc, baseModelContext)
117 : Futures.immediateFuture(List.<ProvidedSources<?>>of());
119 final AsyncFunction<List<ProvidedSources<?>>, NetconfDeviceSchemas> function;
120 LOG.debug("{}: resolving YANG 1.0 conformance", deviceId);
121 function = sources -> resolveYang10(deviceId, sessionPreferences, deviceRpc, baseModelContext, sources);
124 // urn:ietf:params:netconf:capability:yang-library:1.0?revision=<date>&module-set-id=<id>
126 // and then dispatch to resolveYang11(), which should resolve schemas based on RFC7950's conformance
127 // announcement via I <hello/> message, as defined in
128 // https://www.rfc-editor.org/rfc/rfc6020#section-5.6.4
130 // if (sessionPreferences.containsNonModuleCapability(CapabilityURN.YANG_LIBRARY)) {
131 // LOG.debug("{}: resolving YANG 1.1 conformance", deviceId);
132 // function = sources -> resolveYang11(deviceId, sessionPreferences, deviceRpc, baseModelContext,
137 // urn:ietf:params:netconf:capability:yang-library:1.1?revision=<date>&content-id=<content-id-value>
138 // where date is at least 2019-01-04
140 // and then dispatch to resolveNmda():
142 // if (sessionPreferences.containsNonModuleCapability(CapabilityURN.YANG_LIBRARY)) {
143 // LOG.debug("{}: resolving YANG 1.1 NMDA conformance", deviceId);
144 // function = sources -> resolveNmda(deviceId, sessionPreferences, deviceRpc, baseModelContext,
148 return Futures.transformAsync(monitoringFuture, function, MoreExecutors.directExecutor());
151 private static ListenableFuture<List<ProvidedSources<?>>> resolveMonitoringSources(
152 final RemoteDeviceId deviceId, final NetconfRpcService deviceRpc,
153 final EffectiveModelContext baseModelContext) {
154 return Futures.transform(deviceRpc.invokeNetconf(Get.QNAME, GET_SCHEMAS_RPC),
155 result -> resolveMonitoringSources(deviceId, deviceRpc, result, baseModelContext),
156 MoreExecutors.directExecutor());
159 private static List<ProvidedSources<?>> resolveMonitoringSources(final RemoteDeviceId deviceId,
160 final NetconfRpcService deviceRpc, final DOMRpcResult rpcResult,
161 final EffectiveModelContext baseModelContext) {
162 // Two-pass error reporting: first check if there is a hard error, then log any remaining warnings
163 final var errors = rpcResult.errors();
164 if (errors.stream().anyMatch(error -> error.getSeverity() == ErrorSeverity.ERROR)) {
165 LOG.warn("{}: failed to get netconf-state", errors);
168 for (var error : errors) {
169 LOG.info("{}: schema retrieval warning: {}", deviceId, error);
172 final var rpcOutput = rpcResult.value();
173 if (rpcOutput == null) {
174 LOG.warn("{}: missing RPC output", deviceId);
177 final var data = rpcOutput.childByArg(NETCONF_DATA_NODEID);
179 LOG.warn("{}: missing RPC data", deviceId);
182 if (!(data instanceof AnyxmlNode<?> anyxmlData)) {
183 LOG.warn("{}: unexpected data {}", deviceId, data.prettyTree());
186 final var dataBody = anyxmlData.body();
187 if (!(dataBody instanceof DOMSource domDataBody)) {
188 LOG.warn("{}: unexpected body {}", deviceId, dataBody);
192 // Server may include additional data which we do not understand. Make sure we trim the input before we try
194 // FIXME: we should not be going to NormalizedNode at all. We are interpreting a very limited set of data
195 // in the context of setting up the normalization schema. Everything we are dealing with are plain
196 // strings for which yang-common provides everything we need -- with the notable exception of identityref
197 // values. Those boil down into plain QNames -- so we can talk to XmlCodecs.identityRefCodec(). That
198 // operation needs to also handle IAE and ignore unknown values.
199 final var filteredBody = ietfMonitoringCopy(domDataBody);
201 // Now normalize the anyxml content to the selected model context
202 final NormalizedNode normalizedData;
204 normalizedData = NormalizedDataUtil.transformDOMSourceToNormalizedNode(baseModelContext, filteredBody)
206 } catch (XMLStreamException | URISyntaxException | IOException | SAXException e) {
207 LOG.warn("{}: failed to transform {}", deviceId, filteredBody, e);
211 // The result should be the root of datastore, hence a DataContainerNode
212 if (!(normalizedData instanceof DataContainerNode root)) {
213 LOG.warn("{}: unexpected normalized data {}", deviceId, normalizedData.prettyTree());
217 // container netconf-state
218 final var netconfState = root.childByArg(new NodeIdentifier(NetconfState.QNAME));
219 if (netconfState == null) {
220 LOG.warn("{}: missing netconf-state", deviceId);
223 if (!(netconfState instanceof ContainerNode netconfStateCont)) {
224 LOG.warn("{}: unexpected netconf-state {}", deviceId, netconfState.prettyTree());
229 final var schemas = netconfStateCont.childByArg(new NodeIdentifier(Schemas.QNAME));
230 if (schemas == null) {
231 LOG.warn("{}: missing schemas", deviceId);
234 if (!(schemas instanceof ContainerNode schemasNode)) {
235 LOG.warn("{}: unexpected schemas {}", deviceId, schemas.prettyTree());
239 return resolveMonitoringSources(deviceId, deviceRpc, schemasNode);
243 * Parse response of get(netconf-state/schemas) to find all schemas under netconf-state/schemas.
246 static List<ProvidedSources<?>> resolveMonitoringSources(final RemoteDeviceId deviceId,
247 final NetconfRpcService deviceRpc, final ContainerNode schemasNode) {
248 final var child = schemasNode.childByArg(new NodeIdentifier(Schema.QNAME));
250 LOG.warn("{}: missing schema", deviceId);
253 if (!(child instanceof SystemMapNode schemaMap)) {
254 LOG.warn("{}: unexpected schema {}", deviceId, child.prettyTree());
258 // FIXME: we are producing the wrong thing here and simply not handling all the use cases
259 // - instead of QName we want to say 'SourceIdentifier and XMLNamespace', because these are source files
260 // and there is some namespace guidance -- which we do not really need (because localName+revision is
261 // guaranteed to be unique and hence there cannot be a conflict on submodule names
262 // - we handle on "NETCONF" and completely ignore the URI case -- which is something useful for
263 // offloading model discovery
265 // At the end of the day, all this information is going into yang-parser-impl, i.e. it will need to go
266 // through SchemaSource and all the yang-repo-{api,spi} stuff. That implies policy and further control
267 // point which needs to be customizable as we want to plug in various providers and differing policies.
270 // - all URIs need to be resolved, which needs pluggable resolvers (https:// is obvious, but xri:// needs
271 // to hand this off to a dedicated resolver
272 // - we do not want to use URI.toURL().openConnection(), but leave it up to policy -- for example one
273 // would want to use java.net.http.HttpClient, which means authentication and content negotiation.
274 // Content negotiation is needed to establish byte stream encoding, plus
275 // - all sources of schema are subject to caching, perhaps even in IRSource form
277 // At the end of the day, we should just produce an instance of Schema.class and let others deal with
278 // translating it to the real world -- for example turning a String into a XMLNamespace or a local name.
279 final var builder = ImmutableSet.<QName>builderWithExpectedSize(schemaMap.size());
280 for (var schemaNode : schemaMap.body()) {
281 final var qname = createFromNormalizedNode(deviceId, schemaNode);
287 final var sources = builder.build();
288 return sources.isEmpty() ? List.of() : List.of(new ProvidedSources<>(YangTextSource.class,
289 new MonitoringSchemaSourceProvider(deviceId, deviceRpc), sources));
292 private static @Nullable QName createFromNormalizedNode(final RemoteDeviceId id,
293 final MapEntryNode schemaEntry) {
294 // These three are mandatory due to 'key "identifier version format"'
295 final var format = schemaEntry.getChildByArg(SCHEMA_FORMAT_NODEID).body();
296 // FIXME: we should support Yin as well
297 if (!Yang.QNAME.equals(format)) {
298 LOG.debug("{}: Ignoring schema due to unsupported format: {}", id, format);
301 // Note: module name or submodule name
302 final var identifier = (String) schemaEntry.getChildByArg(SCHEMA_IDENTIFIER_NODEID).body();
304 final var version = (String) schemaEntry.getChildByArg(SCHEMA_VERSION_NODEID).body();
306 // FIXME: we should be able to promote to 'getChildByArg()', IFF the normalizer is enforcing mandatory nodes
307 @SuppressWarnings("unchecked")
308 final var namespaceLeaf = (LeafNode<String>) schemaEntry.childByArg(SCHEMA_NAMESPACE_NODEID);
309 if (namespaceLeaf == null) {
310 LOG.warn("{}: Ignoring schema due to missing namespace", id);
314 @SuppressWarnings("unchecked")
315 final var location = (SystemLeafSetNode<String>) schemaEntry.childByArg(SCHEMA_LOCATION_NODEID);
316 if (location == null) {
317 LOG.debug("{}: Ignoring schema due to missing location", id);
321 boolean foundNetconf = false;
322 for (var locEntry : location.body()) {
323 final var loc = locEntry.body();
324 if (NETCONF_LOCATION.equals(loc)) {
329 // FIXME: the other string is an Uri, we should be exposing that as well
330 LOG.debug("{}: Ignoring schema due to unsupported location: {}", id, loc);
334 LOG.debug("{}: Ignoring schema due to no NETCONF location", id);
339 final var namespace = XMLNamespace.of(namespaceLeaf.body());
340 final var revision = version.isEmpty() ? null : Revision.of(version);
341 return QName.create(namespace, revision, identifier);
342 } catch (DateTimeParseException | IllegalArgumentException e) {
343 LOG.warn("{}: Ignoring malformed schema {}", id, schemaEntry.prettyTree(), e);
349 static DOMSource ietfMonitoringCopy(final DOMSource domSource) {
350 final var sourceDoc = XmlUtil.newDocument();
351 sourceDoc.appendChild(sourceDoc.importNode(domSource.getNode(), true));
353 final var treeWalker = ((DocumentTraversal) sourceDoc).createTreeWalker(sourceDoc.getDocumentElement(),
354 NodeFilter.SHOW_ALL, node -> {
355 final var namespace = node.getNamespaceURI();
356 return namespace == null || MONITORING_NAMESPACE.equals(namespace)
357 ? NodeFilter.FILTER_ACCEPT : NodeFilter.FILTER_REJECT;
360 final var filteredDoc = XmlUtil.newDocument();
361 filteredDoc.appendChild(filteredDoc.importNode(treeWalker.getRoot(), false));
362 final var filteredElement = filteredDoc.getDocumentElement();
363 copyChildren(treeWalker, filteredDoc, filteredElement);
365 return new DOMSource(filteredElement);
368 private static void copyChildren(final TreeWalker walker, final Document targetDoc, final Node targetNode) {
369 if (walker.firstChild() != null) {
370 for (var node = walker.getCurrentNode(); node != null; node = walker.nextSibling()) {
371 final var importedNode = targetDoc.importNode(node, false);
372 targetNode.appendChild(importedNode);
373 copyChildren(walker, targetDoc, importedNode);
374 walker.setCurrentNode(node);
379 // Resolve schemas based on RFC6020's conformance announcement in <hello/> message, as defined in
380 // https://www.rfc-editor.org/rfc/rfc6020#section-5.6.4
381 private static ListenableFuture<NetconfDeviceSchemas> resolveYang10(final RemoteDeviceId deviceId,
382 final NetconfSessionPreferences sessionPreferences, final NetconfRpcService deviceRpc,
383 final EffectiveModelContext baseModelContext, final List<ProvidedSources<?>> monitoringSources) {
384 final var providedSources = monitoringSources.stream()
385 .flatMap(sources -> sources.sources().stream())
386 .collect(Collectors.toSet());
387 LOG.debug("{}: Schemas exposed by ietf-netconf-monitoring: {}", deviceId, providedSources);
389 final var requiredSources = new HashSet<>(sessionPreferences.moduleBasedCaps().keySet());
390 final var requiredSourcesNotProvided = Sets.difference(requiredSources, providedSources);
391 if (!requiredSourcesNotProvided.isEmpty()) {
392 LOG.warn("{}: Netconf device does not provide all yang models reported in hello message capabilities,"
393 + " required but not provided: {}", deviceId, requiredSourcesNotProvided);
394 LOG.warn("{}: Attempting to build schema context from required sources", deviceId);
397 // Here all the sources reported in netconf monitoring are merged with those reported in hello.
398 // It is necessary to perform this since submodules are not mentioned in hello but still required.
399 // This clashes with the option of a user to specify supported yang models manually in configuration
400 // for netconf-connector and as a result one is not able to fully override yang models of a device.
401 // It is only possible to add additional models.
402 final var providedSourcesNotRequired = Sets.difference(providedSources, requiredSources);
403 if (!providedSourcesNotRequired.isEmpty()) {
404 LOG.warn("{}: Netconf device provides additional yang models not reported in "
405 + "hello message capabilities: {}", deviceId, providedSourcesNotRequired);
406 LOG.warn("{}: Adding provided but not required sources as required to prevent failures", deviceId);
407 LOG.debug("{}: Netconf device reported in hello: {}", deviceId, requiredSources);
408 requiredSources.addAll(providedSourcesNotRequired);
412 return Futures.immediateFuture(new NetconfDeviceSchemas(requiredSources,
413 // FIXME: determine features
414 FeatureSet.builder().build(),
415 // FIXME: use this instead of adjusted required sources