2 * Copyright (c) 2013 Cisco Systems, Inc. and others. All rights reserved.
3 * This program and the accompanying materials are made available under the
4 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
5 * and is available at http://www.eclipse.org/legal/epl-v10.html
7 package org.opendaylight.yangtools.yang.parser.impl;
9 import static com.google.common.base.Preconditions.checkNotNull;
10 import static org.opendaylight.yangtools.yang.parser.builder.impl.BuilderUtils.fillAugmentTarget;
11 import static org.opendaylight.yangtools.yang.parser.builder.impl.BuilderUtils.findBaseIdentity;
12 import static org.opendaylight.yangtools.yang.parser.builder.impl.BuilderUtils.findModuleFromContext;
13 import static org.opendaylight.yangtools.yang.parser.builder.impl.BuilderUtils.findSchemaNode;
14 import static org.opendaylight.yangtools.yang.parser.builder.impl.BuilderUtils.findSchemaNodeInModule;
15 import static org.opendaylight.yangtools.yang.parser.builder.impl.BuilderUtils.processAugmentation;
16 import static org.opendaylight.yangtools.yang.parser.builder.impl.TypeUtils.resolveType;
17 import static org.opendaylight.yangtools.yang.parser.builder.impl.TypeUtils.resolveTypeUnion;
19 import com.google.common.base.Optional;
20 import com.google.common.base.Preconditions;
21 import com.google.common.base.Splitter;
22 import com.google.common.collect.HashBiMap;
23 import com.google.common.io.ByteSource;
25 import java.io.IOException;
26 import java.io.InputStream;
28 import java.util.ArrayList;
29 import java.util.Collection;
30 import java.util.Collections;
31 import java.util.Date;
32 import java.util.HashMap;
33 import java.util.HashSet;
34 import java.util.LinkedHashMap;
35 import java.util.LinkedHashSet;
36 import java.util.List;
39 import java.util.TreeMap;
40 import javax.annotation.concurrent.Immutable;
41 import org.antlr.v4.runtime.ANTLRInputStream;
42 import org.antlr.v4.runtime.CommonTokenStream;
43 import org.antlr.v4.runtime.tree.ParseTree;
44 import org.antlr.v4.runtime.tree.ParseTreeWalker;
45 import org.opendaylight.yangtools.antlrv4.code.gen.YangLexer;
46 import org.opendaylight.yangtools.antlrv4.code.gen.YangParser;
47 import org.opendaylight.yangtools.antlrv4.code.gen.YangParser.YangContext;
48 import org.opendaylight.yangtools.yang.common.QName;
49 import org.opendaylight.yangtools.yang.common.QNameModule;
50 import org.opendaylight.yangtools.yang.model.api.ExtensionDefinition;
51 import org.opendaylight.yangtools.yang.model.api.Module;
52 import org.opendaylight.yangtools.yang.model.api.ModuleIdentifier;
53 import org.opendaylight.yangtools.yang.model.api.ModuleImport;
54 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
55 import org.opendaylight.yangtools.yang.model.api.SchemaPath;
56 import org.opendaylight.yangtools.yang.model.parser.api.YangContextParser;
57 import org.opendaylight.yangtools.yang.model.parser.api.YangSyntaxErrorException;
58 import org.opendaylight.yangtools.yang.parser.builder.api.AugmentationSchemaBuilder;
59 import org.opendaylight.yangtools.yang.parser.builder.api.AugmentationTargetBuilder;
60 import org.opendaylight.yangtools.yang.parser.builder.api.Builder;
61 import org.opendaylight.yangtools.yang.parser.builder.api.DataNodeContainerBuilder;
62 import org.opendaylight.yangtools.yang.parser.builder.api.DataSchemaNodeBuilder;
63 import org.opendaylight.yangtools.yang.parser.builder.api.ExtensionBuilder;
64 import org.opendaylight.yangtools.yang.parser.builder.api.GroupingBuilder;
65 import org.opendaylight.yangtools.yang.parser.builder.api.SchemaNodeBuilder;
66 import org.opendaylight.yangtools.yang.parser.builder.api.TypeAwareBuilder;
67 import org.opendaylight.yangtools.yang.parser.builder.api.UnknownSchemaNodeBuilder;
68 import org.opendaylight.yangtools.yang.parser.builder.api.UsesNodeBuilder;
69 import org.opendaylight.yangtools.yang.parser.builder.impl.BuilderUtils;
70 import org.opendaylight.yangtools.yang.parser.builder.impl.ChoiceBuilder;
71 import org.opendaylight.yangtools.yang.parser.builder.impl.ChoiceCaseBuilder;
72 import org.opendaylight.yangtools.yang.parser.builder.impl.GroupingUtils;
73 import org.opendaylight.yangtools.yang.parser.builder.impl.IdentitySchemaNodeBuilder;
74 import org.opendaylight.yangtools.yang.parser.builder.impl.IdentityrefTypeBuilder;
75 import org.opendaylight.yangtools.yang.parser.builder.impl.ModuleBuilder;
76 import org.opendaylight.yangtools.yang.parser.builder.impl.ModuleImpl;
77 import org.opendaylight.yangtools.yang.parser.builder.impl.UnionTypeBuilder;
78 import org.opendaylight.yangtools.yang.parser.builder.util.Comparators;
79 import org.opendaylight.yangtools.yang.parser.util.ModuleDependencySort;
80 import org.opendaylight.yangtools.yang.parser.util.NamedByteArrayInputStream;
81 import org.opendaylight.yangtools.yang.parser.util.NamedFileInputStream;
82 import org.opendaylight.yangtools.yang.parser.util.NamedInputStream;
83 import org.opendaylight.yangtools.yang.parser.util.YangParseException;
84 import org.slf4j.Logger;
85 import org.slf4j.LoggerFactory;
88 public final class YangParserImpl implements YangContextParser {
89 private static final Logger LOG = LoggerFactory.getLogger(YangParserImpl.class);
90 private static final Splitter COLON_SPLITTER = Splitter.on(':');
91 private static final YangParserImpl INSTANCE = new YangParserImpl();
93 public static YangParserImpl getInstance() {
99 public Set<Module> parseYangModels(final File yangFile, final File directory) {
101 return parseFile(yangFile, directory).getModules();
102 } catch (IOException | YangSyntaxErrorException e) {
103 throw new YangParseException("Failed to parse yang data", e);
108 public SchemaContext parseFile(final File yangFile, final File directory) throws IOException, YangSyntaxErrorException {
109 Preconditions.checkState(yangFile.exists(), yangFile + " does not exists");
110 Preconditions.checkState(directory.exists(), directory + " does not exists");
111 Preconditions.checkState(directory.isDirectory(), directory + " is not a directory");
113 final String yangFileName = yangFile.getName();
114 final String[] fileList = checkNotNull(directory.list(), directory + " not found or is not a directory");
116 Map<ByteSource, File> sourceToFile = new LinkedHashMap<>();
117 ByteSource mainFileSource = BuilderUtils.fileToByteSource(yangFile);
118 sourceToFile.put(mainFileSource, yangFile);
120 for (String fileName : fileList) {
121 if (fileName.equals(yangFileName)) {
124 File dependency = new File(directory, fileName);
125 if (dependency.isFile()) {
126 sourceToFile.put(BuilderUtils.fileToByteSource(dependency), dependency);
130 Map<ByteSource, ModuleBuilder> sourceToBuilder = parseSourcesToBuilders(sourceToFile.keySet(), null);
131 ModuleBuilder main = sourceToBuilder.get(mainFileSource);
133 List<ModuleBuilder> moduleBuilders = new ArrayList<>();
134 moduleBuilders.add(main);
135 filterImports(main, new ArrayList<>(sourceToBuilder.values()), moduleBuilders);
136 Collection<ModuleBuilder> resolved = resolveSubmodules(moduleBuilders);
138 // module builders sorted by dependencies
139 List<ModuleBuilder> sortedBuilders = ModuleDependencySort.sort(resolved);
140 LinkedHashMap<URI, TreeMap<Date, ModuleBuilder>> modules = resolveModulesWithImports(sortedBuilders, null);
141 Collection<Module> unsorted = build(modules).values();
142 Set<Module> result = new LinkedHashSet<>(
143 ModuleDependencySort.sort(unsorted.toArray(new Module[unsorted.size()])));
144 return resolveSchemaContext(result);
149 public Set<Module> parseYangModels(final List<File> yangFiles) {
150 return parseFiles(yangFiles).getModules();
154 public SchemaContext parseFiles(final Collection<File> yangFiles) {
155 Collection<Module> unsorted = parseYangModelsMapped(yangFiles).values();
156 Set<Module> sorted = new LinkedHashSet<>(
157 ModuleDependencySort.sort(unsorted.toArray(new Module[unsorted.size()])));
158 return resolveSchemaContext(sorted);
163 public Set<Module> parseYangModels(final List<File> yangFiles, final SchemaContext context) {
165 return parseFiles(yangFiles, context).getModules();
166 } catch (IOException | YangSyntaxErrorException e) {
167 throw new YangParseException("Failed to parse yang data", e);
172 public SchemaContext parseFiles(final Collection<File> yangFiles, final SchemaContext context) throws IOException,
173 YangSyntaxErrorException {
174 if (yangFiles == null) {
175 return resolveSchemaContext(Collections.<Module> emptySet());
178 Collection<ByteSource> sources = BuilderUtils.filesToByteSources(yangFiles);
179 return parseSources(sources, context);
184 public Set<Module> parseYangModelsFromStreams(final List<InputStream> yangModelStreams) {
186 Collection<ByteSource> sources = BuilderUtils.streamsToByteSources(yangModelStreams);
187 return parseSources(sources).getModules();
188 } catch (IOException | YangSyntaxErrorException e) {
189 throw new YangParseException("Failed to parse yang data", e);
194 public SchemaContext parseSources(final Collection<ByteSource> sources) throws IOException,YangSyntaxErrorException {
195 return assembleContext(parseYangModelSources(sources, null).values());
200 public Set<Module> parseYangModelsFromStreams(final List<InputStream> yangModelStreams, final SchemaContext context) {
202 Collection<ByteSource> sources = BuilderUtils.streamsToByteSources(yangModelStreams);
203 return parseSources(sources, context).getModules();
204 } catch (IOException | YangSyntaxErrorException e) {
205 throw new YangParseException("Failed to parse yang data", e);
210 public SchemaContext parseSources(final Collection<ByteSource> sources, final SchemaContext context)
211 throws IOException, YangSyntaxErrorException {
212 if (sources == null) {
213 return resolveSchemaContext(Collections.<Module> emptySet());
216 final List<ModuleBuilder> sorted = resolveModuleBuilders(sources, context);
217 final Map<URI, TreeMap<Date, ModuleBuilder>> modules = resolveModulesWithImports(sorted, context);
219 final Set<Module> unsorted = new LinkedHashSet<>(build(modules).values());
220 if (context != null) {
221 for (Module m : context.getModules()) {
222 if (!unsorted.contains(m)) {
227 Set<Module> result = new LinkedHashSet<>(
228 ModuleDependencySort.sort(unsorted.toArray(new Module[unsorted.size()])));
229 return resolveSchemaContext(result);
232 private static LinkedHashMap<URI, TreeMap<Date, ModuleBuilder>> resolveModulesWithImports(final List<ModuleBuilder> sorted,
233 final SchemaContext context) {
234 final LinkedHashMap<URI, TreeMap<Date, ModuleBuilder>> modules = orderModules(sorted);
235 for (ModuleBuilder module : sorted) {
236 if (module != null) {
237 for (ModuleImport imp : module.getImports().values()) {
238 String prefix = imp.getPrefix();
239 ModuleBuilder targetModule = BuilderUtils.findModuleFromBuilders(imp, sorted);
240 if (targetModule == null) {
241 Module result = findModuleFromContext(context, module, prefix, 0);
242 targetModule = new ModuleBuilder(result);
243 TreeMap<Date, ModuleBuilder> map = modules.get(prefix);
245 map = new TreeMap<>();
246 map.put(targetModule.getRevision(), targetModule);
247 modules.put(targetModule.getNamespace(), map);
249 map.put(targetModule.getRevision(), targetModule);
252 module.addImportedModule(prefix, targetModule);
260 public Map<File, Module> parseYangModelsMapped(final Collection<File> yangFiles) {
261 if (yangFiles == null || yangFiles.isEmpty()) {
262 return Collections.emptyMap();
265 Map<ByteSource, File> byteSourceToFile = new HashMap<>();
266 for (final File file : yangFiles) {
267 ByteSource source = new ByteSource() {
269 public InputStream openStream() throws IOException {
270 return new NamedFileInputStream(file, file.getPath());
273 byteSourceToFile.put(source, file);
276 Map<ByteSource, Module> byteSourceToModule;
278 byteSourceToModule = parseYangModelSources(byteSourceToFile.keySet(), null);
279 } catch (IOException | YangSyntaxErrorException e) {
280 throw new YangParseException("Failed to parse yang data", e);
282 Map<File, Module> result = new LinkedHashMap<>();
283 for (Map.Entry<ByteSource, Module> entry : byteSourceToModule.entrySet()) {
284 result.put(byteSourceToFile.get(entry.getKey()), entry.getValue());
290 public Map<InputStream, Module> parseYangModelsFromStreamsMapped(final Collection<InputStream> yangModelStreams) {
291 if (yangModelStreams == null || yangModelStreams.isEmpty()) {
292 return Collections.emptyMap();
295 Map<ByteSource, InputStream> sourceToStream = new HashMap<>();
296 for (final InputStream stream : yangModelStreams) {
297 ByteSource source = new ByteSource() {
299 public InputStream openStream() throws IOException {
300 return NamedByteArrayInputStream.create(stream);
303 sourceToStream.put(source, stream);
306 Map<ByteSource, Module> sourceToModule;
308 sourceToModule = parseYangModelSources(sourceToStream.keySet(), null);
309 } catch (IOException | YangSyntaxErrorException e) {
310 throw new YangParseException("Failed to parse yang data", e);
312 Map<InputStream, Module> result = new LinkedHashMap<>();
313 for (Map.Entry<ByteSource, Module> entry : sourceToModule.entrySet()) {
314 result.put(sourceToStream.get(entry.getKey()), entry.getValue());
320 public SchemaContext resolveSchemaContext(final Set<Module> modules) {
321 // after merging parse method with this one, add support for getting
322 // submodule sources.
323 Map<ModuleIdentifier, String> identifiersToSources = new HashMap<>();
324 for (Module module : modules) {
325 ModuleImpl moduleImpl = (ModuleImpl) module;
326 identifiersToSources.put(module, moduleImpl.getSource());
328 return new SchemaContextImpl(modules, identifiersToSources);
331 public Collection<Module> buildModules(final Collection<ModuleBuilder> builders) {
332 Collection<ModuleBuilder> unsorted = resolveSubmodules(builders);
333 List<ModuleBuilder> sorted = ModuleDependencySort.sort(unsorted);
334 Map<URI, TreeMap<Date, ModuleBuilder>> modules = resolveModulesWithImports(sorted, null);
335 Map<ModuleBuilder, Module> builderToModule = build(modules);
336 return builderToModule.values();
339 public SchemaContext assembleContext(final Collection<Module> modules) {
340 final Set<Module> sorted = new LinkedHashSet<>(
341 ModuleDependencySort.sort(modules.toArray(new Module[modules.size()])));
342 return resolveSchemaContext(sorted);
345 private Map<ByteSource, Module> parseYangModelSources(final Collection<ByteSource> sources, final SchemaContext context) throws IOException, YangSyntaxErrorException {
346 if (sources == null || sources.isEmpty()) {
347 return Collections.emptyMap();
350 Map<ByteSource, ModuleBuilder> sourceToBuilder = resolveSources(sources, context);
351 // sort and check for duplicates
352 List<ModuleBuilder> sorted = ModuleDependencySort.sort(sourceToBuilder.values());
353 Map<URI, TreeMap<Date, ModuleBuilder>> modules = resolveModulesWithImports(sorted, null);
354 Map<ModuleBuilder, Module> builderToModule = build(modules);
355 Map<ModuleBuilder, ByteSource> builderToSource = HashBiMap.create(sourceToBuilder).inverse();
356 sorted = ModuleDependencySort.sort(builderToModule.keySet());
358 Map<ByteSource, Module> result = new LinkedHashMap<>();
359 for (ModuleBuilder moduleBuilder : sorted) {
360 Module value = checkNotNull(builderToModule.get(moduleBuilder), "Cannot get module for %s", moduleBuilder);
361 result.put(builderToSource.get(moduleBuilder), value);
368 * Parse streams and resolve submodules.
371 * collection of streams to parse
372 * @return map, where key is source stream and value is module builder
374 * @throws YangSyntaxErrorException
376 // TODO: remove ByteSource result after removing YangModelParser
377 private Map<ByteSource, ModuleBuilder> resolveSources(final Collection<ByteSource> streams, final SchemaContext context) throws IOException, YangSyntaxErrorException {
378 Map<ByteSource, ModuleBuilder> builders = parseSourcesToBuilders(streams, context);
379 return resolveSubmodules(builders);
382 private Map<ByteSource, ModuleBuilder> parseSourcesToBuilders(final Collection<ByteSource> sources,
383 final SchemaContext context) throws IOException, YangSyntaxErrorException {
384 final ParseTreeWalker walker = new ParseTreeWalker();
385 final Map<ByteSource, ParseTree> sourceToTree = parseYangSources(sources);
386 final Map<ByteSource, ModuleBuilder> sourceToBuilder = new LinkedHashMap<>();
389 new YangModelBasicValidator(walker).validate(sourceToTree.values());
391 Map<String, TreeMap<Date, URI>> namespaceContext = BuilderUtils.createYangNamespaceContext(
392 sourceToTree.values(), Optional.fromNullable(context));
393 YangParserListenerImpl yangModelParser;
394 for (Map.Entry<ByteSource, ParseTree> entry : sourceToTree.entrySet()) {
395 ByteSource source = entry.getKey();
396 String path = null; // TODO refactor to Optional
397 // TODO refactor so that path can be retrieved without opening
398 // stream: NamedInputStream -> NamedByteSource ?
399 try (InputStream stream = source.openStream()) {
400 if (stream instanceof NamedInputStream) {
401 path = stream.toString();
404 yangModelParser = new YangParserListenerImpl(namespaceContext, path);
405 walker.walk(yangModelParser, entry.getValue());
406 ModuleBuilder moduleBuilder = yangModelParser.getModuleBuilder();
407 moduleBuilder.setSource(source);
408 sourceToBuilder.put(source, moduleBuilder);
410 return sourceToBuilder;
413 private Map<ByteSource, ModuleBuilder> resolveSubmodules(final Map<ByteSource, ModuleBuilder> builders) {
414 Map<ByteSource, ModuleBuilder> modules = new HashMap<>();
415 Map<String, TreeMap<Date, ModuleBuilder>> submodules = new HashMap<>();
416 for (Map.Entry<ByteSource, ModuleBuilder> entry : builders.entrySet()) {
417 ModuleBuilder builder = entry.getValue();
418 if (builder.isSubmodule()) {
419 String submoduleName = builder.getName();
420 TreeMap<Date, ModuleBuilder> map = submodules.get(submoduleName);
422 map = new TreeMap<>();
423 map.put(builder.getRevision(), builder);
424 submodules.put(submoduleName, map);
426 map.put(builder.getRevision(), builder);
429 modules.put(entry.getKey(), builder);
433 for (ModuleBuilder module : modules.values()) {
434 resolveSubmodules(module, submodules);
440 private Collection<ModuleBuilder> resolveSubmodules(final Collection<ModuleBuilder> builders) {
441 Collection<ModuleBuilder> modules = new HashSet<>();
442 Map<String, TreeMap<Date, ModuleBuilder>> submodules = new HashMap<>();
443 for (ModuleBuilder builder : builders) {
444 if (builder.isSubmodule()) {
445 String submoduleName = builder.getName();
446 TreeMap<Date, ModuleBuilder> map = submodules.get(submoduleName);
448 map = new TreeMap<>();
449 map.put(builder.getRevision(), builder);
450 submodules.put(submoduleName, map);
452 map.put(builder.getRevision(), builder);
455 modules.add(builder);
459 for (ModuleBuilder module : modules) {
460 resolveSubmodules(module, submodules);
467 * Traverse collection of builders, find builders representing submodule and
468 * add this submodule to its parent module.
473 * collection all loaded submodules
474 * @return collection of module builders with resolved submodules
476 private void resolveSubmodules(final ModuleBuilder module,
477 final Map<String, TreeMap<Date, ModuleBuilder>> submodules) {
478 Map<String, Date> includes = module.getIncludedModules();
479 for (Map.Entry<String, Date> entry : includes.entrySet()) {
480 TreeMap<Date, ModuleBuilder> subs = submodules.get(entry.getKey());
482 throw new YangParseException("Failed to find references submodule " + entry.getKey() + " in module "
485 Date rev = entry.getValue();
486 ModuleBuilder submodule;
488 submodule = subs.lastEntry().getValue();
490 submodule = subs.get(rev);
491 // FIXME an exception should be thrown after issue with
492 // submodule's revisions and namespaces will be resolved
493 if (submodule == null) {
494 submodule = subs.lastEntry().getValue();
498 if (submodule.getIncludedModules().size() > 0) {
499 resolveSubmodules(submodule, submodules);
501 addSubmoduleToModule(submodule, module);
505 private void addSubmoduleToModule(final ModuleBuilder submodule, final ModuleBuilder module) {
506 submodule.setParent(module);
507 module.getDirtyNodes().addAll(submodule.getDirtyNodes());
508 module.getImports().putAll(submodule.getImports());
509 module.getAugments().addAll(submodule.getAugments());
510 module.getAugmentBuilders().addAll(submodule.getAugmentBuilders());
511 module.getAllAugments().addAll(submodule.getAllAugments());
512 module.getChildNodeBuilders().addAll(submodule.getChildNodeBuilders());
513 module.getChildNodes().putAll(submodule.getChildNodes());
514 module.getGroupings().addAll(submodule.getGroupings());
515 module.getGroupingBuilders().addAll(submodule.getGroupingBuilders());
516 module.getTypeDefinitions().addAll(submodule.getTypeDefinitions());
517 module.getTypeDefinitionBuilders().addAll(submodule.getTypeDefinitionBuilders());
518 module.getUsesNodes().addAll(submodule.getUsesNodes());
519 module.getUsesNodeBuilders().addAll(submodule.getUsesNodeBuilders());
520 module.getAllGroupings().addAll(submodule.getAllGroupings());
521 module.getAllUsesNodes().addAll(submodule.getAllUsesNodes());
522 module.getRpcs().addAll(submodule.getRpcs());
523 module.getAddedRpcs().addAll(submodule.getAddedRpcs());
524 module.getNotifications().addAll(submodule.getNotifications());
525 module.getAddedNotifications().addAll(submodule.getAddedNotifications());
526 module.getIdentities().addAll(submodule.getIdentities());
527 module.getAddedIdentities().addAll(submodule.getAddedIdentities());
528 module.getFeatures().addAll(submodule.getFeatures());
529 module.getAddedFeatures().addAll(submodule.getAddedFeatures());
530 module.getDeviations().addAll(submodule.getDeviations());
531 module.getDeviationBuilders().addAll(submodule.getDeviationBuilders());
532 module.getExtensions().addAll(submodule.getExtensions());
533 module.getAddedExtensions().addAll(submodule.getAddedExtensions());
534 module.getUnknownNodes().addAll(submodule.getUnknownNodes());
535 module.getAllUnknownNodes().addAll(submodule.getAllUnknownNodes());
538 private List<ModuleBuilder> resolveModuleBuilders(final Collection<ByteSource> yangFileStreams,
539 final SchemaContext context) throws IOException, YangSyntaxErrorException {
540 Map<ByteSource, ModuleBuilder> parsedBuilders = resolveSources(yangFileStreams, context);
541 ModuleBuilder[] builders = new ModuleBuilder[parsedBuilders.size()];
542 parsedBuilders.values().toArray(builders);
544 // module dependency graph sorted
545 List<ModuleBuilder> sorted;
546 if (context == null) {
547 sorted = ModuleDependencySort.sort(builders);
549 sorted = ModuleDependencySort.sortWithContext(context, builders);
555 * Order modules by namespace and revision.
558 * topologically sorted modules
559 * @return modules ordered by namespace and revision
561 private static LinkedHashMap<URI, TreeMap<Date, ModuleBuilder>> orderModules(final List<ModuleBuilder> modules) {
562 final LinkedHashMap<URI, TreeMap<Date, ModuleBuilder>> result = new LinkedHashMap<>();
563 for (final ModuleBuilder builder : modules) {
564 if (builder == null) {
568 URI ns = builder.getNamespace();
569 Date rev = builder.getRevision();
574 TreeMap<Date, ModuleBuilder> builderByRevision = result.get(ns);
575 if (builderByRevision == null) {
576 builderByRevision = new TreeMap<>();
577 builderByRevision.put(rev, builder);
578 result.put(ns, builderByRevision);
580 builderByRevision.put(rev, builder);
588 * Find {@code main} dependencies from {@code other} and add them to
596 * collection to fill up
598 private void filterImports(final ModuleBuilder main, final Collection<ModuleBuilder> other,
599 final Collection<ModuleBuilder> filtered) {
600 Map<String, ModuleImport> imports = main.getImports();
602 // if this is submodule, add parent to filtered and pick its imports
603 if (main.isSubmodule()) {
604 TreeMap<Date, ModuleBuilder> dependencies = new TreeMap<>();
605 for (ModuleBuilder mb : other) {
606 if (mb.getName().equals(main.getBelongsTo())) {
607 dependencies.put(mb.getRevision(), mb);
610 ModuleBuilder parent = dependencies.get(dependencies.firstKey());
611 filtered.add(parent);
612 imports.putAll(parent.getImports());
615 for (ModuleImport mi : imports.values()) {
616 for (ModuleBuilder builder : other) {
617 if (mi.getModuleName().equals(builder.getModuleName())) {
618 if (mi.getRevision() == null) {
619 if (!filtered.contains(builder)) {
620 filtered.add(builder);
621 filterImports(builder, other, filtered);
624 if (mi.getRevision().equals(builder.getRevision())) {
625 if (!filtered.contains(builder)) {
626 filtered.add(builder);
627 filterImports(builder, other, filtered);
636 private Map<ByteSource, ParseTree> parseYangSources(final Collection<ByteSource> sources) throws IOException, YangSyntaxErrorException {
637 final Map<ByteSource, ParseTree> trees = new HashMap<>();
638 for (ByteSource source : sources) {
639 try (InputStream stream = source.openStream()) {
640 trees.put(source, parseYangSource(stream));
646 public static YangContext parseYangSource(final InputStream stream) throws IOException, YangSyntaxErrorException {
647 final YangLexer lexer = new YangLexer(new ANTLRInputStream(stream));
648 final CommonTokenStream tokens = new CommonTokenStream(lexer);
649 final YangParser parser = new YangParser(tokens);
650 parser.removeErrorListeners();
652 final YangErrorListener errorListener = new YangErrorListener();
653 parser.addErrorListener(errorListener);
655 final YangContext result = parser.yang();
656 errorListener.validate();
662 * Mini parser: This parsing context does not validate full YANG module,
663 * only parses header up to the revisions and imports.
665 * @see org.opendaylight.yangtools.yang.parser.impl.util.YangModelDependencyInfo
667 public static YangContext parseStreamWithoutErrorListeners(final InputStream yangStream) {
668 YangContext result = null;
670 final ANTLRInputStream input = new ANTLRInputStream(yangStream);
671 final YangLexer lexer = new YangLexer(input);
672 final CommonTokenStream tokens = new CommonTokenStream(lexer);
673 final YangParser parser = new YangParser(tokens);
674 parser.removeErrorListeners();
675 result = parser.yang();
676 } catch (IOException e) {
677 LOG.warn("Exception while reading yang file: " + yangStream, e);
683 * Creates builder-to-module map based on given modules. Method first
684 * resolve unresolved type references, instantiate groupings through uses
685 * statements and perform augmentation.
687 * Node resolving must be performed in following order:
690 * unresolved type references</li>
692 * uses in groupings</li>
694 * uses in other nodes</li>
701 * @return modules mapped on their builders
703 private Map<ModuleBuilder, Module> build(final Map<URI, TreeMap<Date, ModuleBuilder>> modules) {
704 resolveDirtyNodes(modules);
705 resolveAugmentsTargetPath(modules);
706 resolveUsesTargetGrouping(modules);
707 resolveUsesForGroupings(modules);
708 resolveUsesForNodes(modules);
709 resolveAugments(modules);
710 resolveIdentities(modules);
713 final Map<ModuleBuilder, Module> result = new LinkedHashMap<>();
714 for (Map.Entry<URI, TreeMap<Date, ModuleBuilder>> entry : modules.entrySet()) {
715 for (Map.Entry<Date, ModuleBuilder> childEntry : entry.getValue().entrySet()) {
716 final ModuleBuilder moduleBuilder = childEntry.getValue();
717 final Module module = moduleBuilder.build();
718 result.put(moduleBuilder, module);
725 * Resolve all unresolved type references.
730 private void resolveDirtyNodes(final Map<URI, TreeMap<Date, ModuleBuilder>> modules) {
731 for (Map.Entry<URI, TreeMap<Date, ModuleBuilder>> entry : modules.entrySet()) {
732 for (Map.Entry<Date, ModuleBuilder> childEntry : entry.getValue().entrySet()) {
733 final ModuleBuilder module = childEntry.getValue();
734 resolveUnknownNodes(modules, module);
735 resolveDirtyNodes(modules, module);
741 * Search for dirty nodes (node which contains UnknownType) and resolve
745 * all available modules
749 private void resolveDirtyNodes(final Map<URI, TreeMap<Date, ModuleBuilder>> modules, final ModuleBuilder module) {
750 final Set<TypeAwareBuilder> dirtyNodes = module.getDirtyNodes();
751 if (!dirtyNodes.isEmpty()) {
752 for (TypeAwareBuilder nodeToResolve : dirtyNodes) {
753 if (nodeToResolve instanceof UnionTypeBuilder) {
754 // special handling for union types
755 resolveTypeUnion((UnionTypeBuilder) nodeToResolve, modules, module);
756 } else if (nodeToResolve.getTypedef() instanceof IdentityrefTypeBuilder) {
757 // special handling for identityref types
758 IdentityrefTypeBuilder idref = (IdentityrefTypeBuilder) nodeToResolve.getTypedef();
759 IdentitySchemaNodeBuilder identity = findBaseIdentity(module, idref.getBaseString(),
761 if (identity == null) {
762 throw new YangParseException(module.getName(), idref.getLine(), "Failed to find base identity");
764 idref.setBaseIdentity(identity);
765 nodeToResolve.setType(idref.build());
767 resolveType(nodeToResolve, modules, module);
774 * Traverse through augmentations of modules and fix their child nodes
780 private void resolveAugmentsTargetPath(final Map<URI, TreeMap<Date, ModuleBuilder>> modules) {
781 // collect augments from all loaded modules
782 final List<AugmentationSchemaBuilder> allAugments = new ArrayList<>();
783 for (Map.Entry<URI, TreeMap<Date, ModuleBuilder>> entry : modules.entrySet()) {
784 for (Map.Entry<Date, ModuleBuilder> inner : entry.getValue().entrySet()) {
785 allAugments.addAll(inner.getValue().getAllAugments());
789 for (AugmentationSchemaBuilder augment : allAugments) {
790 setCorrectAugmentTargetPath(augment);
795 * Find augment target and set correct schema path for all its child nodes.
800 private void setCorrectAugmentTargetPath(final AugmentationSchemaBuilder augment) {
801 Builder parent = augment.getParent();
802 final SchemaPath targetNodeSchemaPath;
804 if (parent instanceof UsesNodeBuilder) {
805 targetNodeSchemaPath = findUsesAugmentTargetNodePath(((UsesNodeBuilder) parent).getParent(), augment);
807 targetNodeSchemaPath = augment.getTargetPath();
810 for (DataSchemaNodeBuilder childNode : augment.getChildNodeBuilders()) {
811 correctPathForAugmentNodes(childNode, targetNodeSchemaPath);
815 private SchemaPath findUsesAugmentTargetNodePath(DataNodeContainerBuilder usesParent,
816 AugmentationSchemaBuilder augment) {
817 QName parentQName = usesParent.getQName();
818 final QNameModule qnm;
819 if (parentQName == null) {
820 ModuleBuilder m = BuilderUtils.getParentModule(usesParent);
821 qnm = m.getQNameModule();
823 qnm = parentQName.getModule();
826 SchemaPath path = usesParent.getPath();
827 for (QName qname : augment.getTargetPath().getPathFromRoot()) {
828 path = path.createChild(QName.create(qnm, qname.getLocalName()));
835 * Set new schema path to node and all its child nodes based on given parent
836 * path. This method do not change the namespace.
839 * node which schema path should be updated
841 * schema path of parent node
843 private void correctPathForAugmentNodes(final DataSchemaNodeBuilder node, final SchemaPath parentPath) {
844 SchemaPath newPath = parentPath.createChild(node.getQName());
845 node.setPath(newPath);
846 if (node instanceof DataNodeContainerBuilder) {
847 for (DataSchemaNodeBuilder child : ((DataNodeContainerBuilder) node).getChildNodeBuilders()) {
848 correctPathForAugmentNodes(child, node.getPath());
851 if (node instanceof ChoiceBuilder) {
852 for (ChoiceCaseBuilder child : ((ChoiceBuilder) node).getCases()) {
853 correctPathForAugmentNodes(child, node.getPath());
859 * Check augments for mandatory nodes. If the target node is in another
860 * module, then nodes added by the augmentation MUST NOT be mandatory nodes.
861 * If mandatory node is found, throw an exception.
866 private void checkAugmentMandatoryNodes(final Collection<AugmentationSchemaBuilder> augments) {
867 for (AugmentationSchemaBuilder augment : augments) {
868 URI augmentTargetNs = augment.getTargetPath().getPathFromRoot().iterator().next().getNamespace();
869 Date augmentTargetRev = augment.getTargetPath().getPathFromRoot().iterator().next().getRevision();
870 ModuleBuilder module = BuilderUtils.getParentModule(augment);
872 if (augmentTargetNs.equals(module.getNamespace()) && augmentTargetRev.equals(module.getRevision())) {
876 for (DataSchemaNodeBuilder childNode : augment.getChildNodeBuilders()) {
877 if (childNode.getConstraints().isMandatory()) {
878 throw new YangParseException(augment.getModuleName(), augment.getLine(),
879 "Error in augment parsing: cannot augment mandatory node "
880 + childNode.getQName().getLocalName());
887 * Go through all augment definitions and resolve them.
890 * all loaded modules topologically sorted (based on dependencies
891 * between each other)
893 private void resolveAugments(final Map<URI, TreeMap<Date, ModuleBuilder>> modules) {
894 List<ModuleBuilder> all = new ArrayList<>();
895 for (Map.Entry<URI, TreeMap<Date, ModuleBuilder>> entry : modules.entrySet()) {
896 for (Map.Entry<Date, ModuleBuilder> inner : entry.getValue().entrySet()) {
897 all.add(inner.getValue());
901 for (ModuleBuilder mb : all) {
903 List<AugmentationSchemaBuilder> augments = mb.getAllAugments();
904 checkAugmentMandatoryNodes(augments);
905 Collections.sort(augments, Comparators.AUGMENT_BUILDER_COMP);
906 for (AugmentationSchemaBuilder augment : augments) {
907 if (!(augment.isResolved())) {
908 boolean resolved = resolveAugment(augment, mb, modules);
910 throw new YangParseException(augment.getModuleName(), augment.getLine(),
911 "Error in augment parsing: failed to find augment target: " + augment);
920 * Perform augmentation defined under uses statement.
928 * @return true if augment process succeed
930 private boolean resolveUsesAugment(final AugmentationSchemaBuilder augment, final ModuleBuilder module,
931 final Map<URI, TreeMap<Date, ModuleBuilder>> modules) {
932 if (augment.isResolved()) {
936 UsesNodeBuilder usesNode = (UsesNodeBuilder) augment.getParent();
937 DataNodeContainerBuilder parentNode = usesNode.getParent();
938 Optional<SchemaNodeBuilder> potentialTargetNode;
939 SchemaPath resolvedTargetPath = findUsesAugmentTargetNodePath(parentNode, augment);
940 if (parentNode instanceof ModuleBuilder && resolvedTargetPath.isAbsolute()) {
941 // Uses is directly used in module body, we lookup
942 // We lookup in data namespace to find correct augmentation target
943 potentialTargetNode = findSchemaNodeInModule(resolvedTargetPath, (ModuleBuilder) parentNode);
945 // Uses is used in local context (be it data namespace or grouping
947 // since all nodes via uses are imported to localName, it is safe to
948 // to proceed only with local names.
950 // Conflicting elements in other namespaces are still not present
951 // since resolveUsesAugment occurs before augmenting from external
953 potentialTargetNode = Optional.<SchemaNodeBuilder> fromNullable(findSchemaNode(augment.getTargetPath()
954 .getPathFromRoot(), (SchemaNodeBuilder) parentNode));
957 if (potentialTargetNode.isPresent()) {
958 SchemaNodeBuilder targetNode = potentialTargetNode.get();
959 if (targetNode instanceof AugmentationTargetBuilder) {
960 fillAugmentTarget(augment, targetNode);
961 ((AugmentationTargetBuilder) targetNode).addAugmentation(augment);
962 augment.setResolved(true);
966 "Error in module {} at line {}: Unsupported augment target: {}. Augmentation process skipped.",
967 module.getName(), augment.getLine(), potentialTargetNode);
968 augment.setResolved(true);
972 throw new YangParseException(module.getName(), augment.getLine(), String.format(
973 "Failed to resolve augment in uses. Invalid augment target path: %s", augment.getTargetPath()));
979 * Find augment target module and perform augmentation.
987 * @return true if augment process succeed
989 private boolean resolveAugment(final AugmentationSchemaBuilder augment, final ModuleBuilder module,
990 final Map<URI, TreeMap<Date, ModuleBuilder>> modules) {
991 if (augment.isResolved()) {
995 QName targetModuleName = augment.getTargetPath().getPathFromRoot().iterator().next();
996 ModuleBuilder targetModule = BuilderUtils.findModule(targetModuleName, modules);
997 if (targetModule == null) {
998 throw new YangParseException(module.getModuleName(), augment.getLine(), "Failed to resolve augment "
1002 return processAugmentation(augment, targetModule);
1006 * Go through identity statements defined in current module and resolve
1007 * their 'base' statement.
1010 * all loaded modules
1012 private void resolveIdentities(final Map<URI, TreeMap<Date, ModuleBuilder>> modules) {
1013 for (Map.Entry<URI, TreeMap<Date, ModuleBuilder>> entry : modules.entrySet()) {
1014 for (Map.Entry<Date, ModuleBuilder> inner : entry.getValue().entrySet()) {
1015 ModuleBuilder module = inner.getValue();
1016 final Set<IdentitySchemaNodeBuilder> identities = module.getAddedIdentities();
1017 for (IdentitySchemaNodeBuilder identity : identities) {
1018 resolveIdentity(module, identity);
1024 private void resolveIdentity(final ModuleBuilder module, final IdentitySchemaNodeBuilder identity) {
1025 final String baseIdentityName = identity.getBaseIdentityName();
1026 if (baseIdentityName != null) {
1027 IdentitySchemaNodeBuilder result = null;
1028 if (baseIdentityName.contains(":")) {
1029 final int line = identity.getLine();
1030 String[] splittedBase = baseIdentityName.split(":");
1031 if (splittedBase.length > 2) {
1032 throw new YangParseException(module.getName(), line, "Failed to parse identityref base: "
1033 + baseIdentityName);
1035 String prefix = splittedBase[0];
1036 String name = splittedBase[1];
1037 ModuleBuilder dependentModule = BuilderUtils.getModuleByPrefix(module, prefix);
1038 result = BuilderUtils.findIdentity(dependentModule.getAddedIdentities(), name);
1040 result = BuilderUtils.findIdentity(module.getAddedIdentities(), baseIdentityName);
1042 identity.setBaseIdentity(result);
1047 * Find and add reference of uses target grouping.
1050 * all loaded modules
1052 private void resolveUsesTargetGrouping(final Map<URI, TreeMap<Date, ModuleBuilder>> modules) {
1053 final List<UsesNodeBuilder> allUses = new ArrayList<>();
1054 for (Map.Entry<URI, TreeMap<Date, ModuleBuilder>> entry : modules.entrySet()) {
1055 for (Map.Entry<Date, ModuleBuilder> inner : entry.getValue().entrySet()) {
1056 allUses.addAll(inner.getValue().getAllUsesNodes());
1059 for (UsesNodeBuilder usesNode : allUses) {
1060 ModuleBuilder module = BuilderUtils.getParentModule(usesNode);
1061 final GroupingBuilder targetGroupingBuilder = GroupingUtils.getTargetGroupingFromModules(usesNode, modules,
1063 if (targetGroupingBuilder == null) {
1064 throw new YangParseException(module.getName(), usesNode.getLine(), "Referenced grouping '"
1065 + usesNode.getGroupingPath() + "' not found.");
1067 usesNode.setGrouping(targetGroupingBuilder);
1072 * Resolve uses statements defined in groupings.
1075 * all loaded modules
1077 private void resolveUsesForGroupings(final Map<URI, TreeMap<Date, ModuleBuilder>> modules) {
1078 final Set<GroupingBuilder> allGroupings = new HashSet<>();
1079 for (Map.Entry<URI, TreeMap<Date, ModuleBuilder>> entry : modules.entrySet()) {
1080 for (Map.Entry<Date, ModuleBuilder> inner : entry.getValue().entrySet()) {
1081 ModuleBuilder module = inner.getValue();
1082 allGroupings.addAll(module.getAllGroupings());
1085 final List<GroupingBuilder> sorted = GroupingSort.sort(allGroupings);
1086 for (GroupingBuilder gb : sorted) {
1087 List<UsesNodeBuilder> usesNodes = new ArrayList<>(GroupingSort.getAllUsesNodes(gb));
1088 Collections.sort(usesNodes, new GroupingUtils.UsesComparator());
1089 for (UsesNodeBuilder usesNode : usesNodes) {
1090 resolveUses(usesNode, modules);
1096 * Resolve uses statements.
1099 * all loaded modules
1101 private void resolveUsesForNodes(final Map<URI, TreeMap<Date, ModuleBuilder>> modules) {
1102 for (Map.Entry<URI, TreeMap<Date, ModuleBuilder>> entry : modules.entrySet()) {
1103 for (Map.Entry<Date, ModuleBuilder> inner : entry.getValue().entrySet()) {
1104 ModuleBuilder module = inner.getValue();
1105 List<UsesNodeBuilder> usesNodes = module.getAllUsesNodes();
1106 Collections.sort(usesNodes, new GroupingUtils.UsesComparator());
1107 for (UsesNodeBuilder usesNode : usesNodes) {
1108 resolveUses(usesNode, modules);
1115 * Find target grouping and copy its child nodes to current location with
1119 * uses node to resolve
1121 * all loaded modules
1123 private void resolveUses(final UsesNodeBuilder usesNode, final Map<URI, TreeMap<Date, ModuleBuilder>> modules) {
1124 if (!usesNode.isResolved()) {
1125 DataNodeContainerBuilder parent = usesNode.getParent();
1126 ModuleBuilder module = BuilderUtils.getParentModule(parent);
1127 GroupingBuilder target = GroupingUtils.getTargetGroupingFromModules(usesNode, modules, module);
1129 int index = nodeAfterUsesIndex(usesNode);
1130 List<DataSchemaNodeBuilder> targetNodes = target.instantiateChildNodes(parent);
1131 for (DataSchemaNodeBuilder targetNode : targetNodes) {
1132 parent.addChildNode(index++, targetNode);
1134 parent.getTypeDefinitionBuilders().addAll(target.instantiateTypedefs(parent));
1135 parent.getGroupingBuilders().addAll(target.instantiateGroupings(parent));
1136 parent.getUnknownNodes().addAll(target.instantiateUnknownNodes(parent));
1137 usesNode.setResolved(true);
1138 for (AugmentationSchemaBuilder augment : usesNode.getAugmentations()) {
1139 resolveUsesAugment(augment, module, modules);
1142 GroupingUtils.performRefine(usesNode);
1146 private int nodeAfterUsesIndex(final UsesNodeBuilder usesNode) {
1147 DataNodeContainerBuilder parent = usesNode.getParent();
1148 int usesLine = usesNode.getLine();
1150 List<DataSchemaNodeBuilder> childNodes = parent.getChildNodeBuilders();
1151 if (childNodes.isEmpty()) {
1155 DataSchemaNodeBuilder nextNodeAfterUses = null;
1156 for (DataSchemaNodeBuilder childNode : childNodes) {
1157 if (!(childNode.isAddedByUses()) && !(childNode.isAugmenting())) {
1158 if (childNode.getLine() > usesLine) {
1159 nextNodeAfterUses = childNode;
1165 // uses is declared after child nodes
1166 if (nextNodeAfterUses == null) {
1167 return childNodes.size();
1170 return parent.getChildNodeBuilders().indexOf(nextNodeAfterUses);
1174 * Try to find extension describing this unknown node and assign it to
1175 * unknown node builder.
1178 * all loaded modules
1182 private void resolveUnknownNodes(final Map<URI, TreeMap<Date, ModuleBuilder>> modules, final ModuleBuilder module) {
1183 for (UnknownSchemaNodeBuilder usnb : module.getAllUnknownNodes()) {
1184 QName nodeType = usnb.getNodeType();
1185 String localName = usnb.getNodeType().getLocalName();
1186 ModuleBuilder dependentModule = BuilderUtils.findModule(nodeType, modules);
1188 if (dependentModule == null) {
1190 "Error in module {} at line {}: Failed to resolve node {}: no such extension definition found.",
1191 module.getName(), usnb.getLine(), usnb);
1195 ExtensionBuilder extBuilder = findExtBuilder(localName, dependentModule.getAddedExtensions());
1196 if (extBuilder == null) {
1197 ExtensionDefinition extDef = findExtDef(localName, dependentModule.getExtensions());
1198 if (extDef == null) {
1200 "Error in module {} at line {}: Failed to resolve node {}: no such extension definition found.",
1201 module.getName(), usnb.getLine(), usnb);
1203 usnb.setExtensionDefinition(extDef);
1206 usnb.setExtensionBuilder(extBuilder);
1211 private ExtensionBuilder findExtBuilder(final String name, final Collection<ExtensionBuilder> extensions) {
1212 for (ExtensionBuilder extension : extensions) {
1213 if (extension.getQName().getLocalName().equals(name)) {
1220 private ExtensionDefinition findExtDef(final String name, final Collection<ExtensionDefinition> extensions) {
1221 for (ExtensionDefinition extension : extensions) {
1222 if (extension.getQName().getLocalName().equals(name)) {