35c04a8713bcae29908491b443b726a5a83aabe6
[controller.git] / opendaylight / sal / yang-prototype / code-generator / yang-model-parser-impl / src / main / java / org / opendaylight / controller / yang / model / parser / impl / YangModelParserImpl.java
1 /*
2  * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
3  *
4  * This program and the accompanying materials are made available under the
5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6  * and is available at http://www.eclipse.org/legal/epl-v10.html
7  */
8 package org.opendaylight.controller.yang.model.parser.impl;
9
10 import java.io.File;
11 import java.io.FileInputStream;
12 import java.io.FileNotFoundException;
13 import java.io.IOException;
14 import java.io.InputStream;
15 import java.net.URI;
16 import java.util.ArrayList;
17 import java.util.Collections;
18 import java.util.Date;
19 import java.util.HashMap;
20 import java.util.HashSet;
21 import java.util.Iterator;
22 import java.util.List;
23 import java.util.Map;
24 import java.util.NoSuchElementException;
25 import java.util.Set;
26 import java.util.TreeMap;
27
28 import org.antlr.v4.runtime.ANTLRInputStream;
29 import org.antlr.v4.runtime.CommonTokenStream;
30 import org.antlr.v4.runtime.tree.ParseTree;
31 import org.antlr.v4.runtime.tree.ParseTreeWalker;
32 import org.opendaylight.controller.antlrv4.code.gen.YangLexer;
33 import org.opendaylight.controller.antlrv4.code.gen.YangParser;
34 import org.opendaylight.controller.yang.common.QName;
35 import org.opendaylight.controller.yang.model.api.DataSchemaNode;
36 import org.opendaylight.controller.yang.model.api.ExtensionDefinition;
37 import org.opendaylight.controller.yang.model.api.Module;
38 import org.opendaylight.controller.yang.model.api.ModuleImport;
39 import org.opendaylight.controller.yang.model.api.MustDefinition;
40 import org.opendaylight.controller.yang.model.api.NotificationDefinition;
41 import org.opendaylight.controller.yang.model.api.RpcDefinition;
42 import org.opendaylight.controller.yang.model.api.SchemaContext;
43 import org.opendaylight.controller.yang.model.api.SchemaPath;
44 import org.opendaylight.controller.yang.model.api.TypeDefinition;
45 import org.opendaylight.controller.yang.model.api.type.BinaryTypeDefinition;
46 import org.opendaylight.controller.yang.model.api.type.DecimalTypeDefinition;
47 import org.opendaylight.controller.yang.model.api.type.IntegerTypeDefinition;
48 import org.opendaylight.controller.yang.model.api.type.LengthConstraint;
49 import org.opendaylight.controller.yang.model.api.type.PatternConstraint;
50 import org.opendaylight.controller.yang.model.api.type.RangeConstraint;
51 import org.opendaylight.controller.yang.model.api.type.StringTypeDefinition;
52 import org.opendaylight.controller.yang.model.parser.api.YangModelParser;
53 import org.opendaylight.controller.yang.model.parser.builder.api.AugmentationSchemaBuilder;
54 import org.opendaylight.controller.yang.model.parser.builder.api.AugmentationTargetBuilder;
55 import org.opendaylight.controller.yang.model.parser.builder.api.Builder;
56 import org.opendaylight.controller.yang.model.parser.builder.api.ChildNodeBuilder;
57 import org.opendaylight.controller.yang.model.parser.builder.api.DataSchemaNodeBuilder;
58 import org.opendaylight.controller.yang.model.parser.builder.api.GroupingBuilder;
59 import org.opendaylight.controller.yang.model.parser.builder.api.TypeAwareBuilder;
60 import org.opendaylight.controller.yang.model.parser.builder.api.TypeDefinitionBuilder;
61 import org.opendaylight.controller.yang.model.parser.builder.api.UsesNodeBuilder;
62 import org.opendaylight.controller.yang.model.parser.builder.impl.AnyXmlBuilder;
63 import org.opendaylight.controller.yang.model.parser.builder.impl.ChoiceBuilder;
64 import org.opendaylight.controller.yang.model.parser.builder.impl.ContainerSchemaNodeBuilder;
65 import org.opendaylight.controller.yang.model.parser.builder.impl.IdentitySchemaNodeBuilder;
66 import org.opendaylight.controller.yang.model.parser.builder.impl.IdentityrefTypeBuilder;
67 import org.opendaylight.controller.yang.model.parser.builder.impl.LeafListSchemaNodeBuilder;
68 import org.opendaylight.controller.yang.model.parser.builder.impl.LeafSchemaNodeBuilder;
69 import org.opendaylight.controller.yang.model.parser.builder.impl.ListSchemaNodeBuilder;
70 import org.opendaylight.controller.yang.model.parser.builder.impl.ModuleBuilder;
71 import org.opendaylight.controller.yang.model.parser.builder.impl.TypedefBuilder;
72 import org.opendaylight.controller.yang.model.parser.builder.impl.UnionTypeBuilder;
73 import org.opendaylight.controller.yang.model.parser.builder.impl.UnknownSchemaNodeBuilder;
74 import org.opendaylight.controller.yang.model.parser.util.ModuleDependencySort;
75 import org.opendaylight.controller.yang.model.parser.util.ModuleDependencySort.ModuleSimple;
76 import org.opendaylight.controller.yang.model.parser.util.ParserUtils;
77 import org.opendaylight.controller.yang.model.parser.util.RefineHolder;
78 import org.opendaylight.controller.yang.model.parser.util.TypeConstraints;
79 import org.opendaylight.controller.yang.model.parser.util.YangParseException;
80 import org.opendaylight.controller.yang.model.util.ExtendedType;
81 import org.opendaylight.controller.yang.model.util.IdentityrefType;
82 import org.opendaylight.controller.yang.model.util.UnknownType;
83 import org.opendaylight.controller.yang.model.validator.YangModelBasicValidator;
84 import org.slf4j.Logger;
85 import org.slf4j.LoggerFactory;
86
87 public class YangModelParserImpl implements YangModelParser {
88
89     private static final Logger logger = LoggerFactory
90             .getLogger(YangModelParserImpl.class);
91
92     @Override
93     public Set<Module> parseYangModels(final List<File> yangFiles) {
94         if (yangFiles != null) {
95             final List<InputStream> inputStreams = new ArrayList<InputStream>();
96
97             for (final File yangFile : yangFiles) {
98                 try {
99                     inputStreams.add(new FileInputStream(yangFile));
100                 } catch (FileNotFoundException e) {
101                     logger.warn("Exception while reading yang file: "
102                             + yangFile.getName(), e);
103                 }
104             }
105             final Map<String, TreeMap<Date, ModuleBuilder>> modules = resolveModuleBuilders(inputStreams);
106             return build(modules);
107         }
108         return Collections.emptySet();
109     }
110
111     @Override
112     public Set<Module> parseYangModelsFromStreams(
113             final List<InputStream> yangModelStreams) {
114         final Map<String, TreeMap<Date, ModuleBuilder>> modules = resolveModuleBuilders(yangModelStreams);
115         return build(modules);
116     }
117
118     @Override
119     public SchemaContext resolveSchemaContext(final Set<Module> modules) {
120         return new SchemaContextImpl(modules);
121     }
122
123     private Map<String, TreeMap<Date, ModuleBuilder>> resolveModuleBuilders(
124             final List<InputStream> yangFileStreams) {
125         final Map<String, TreeMap<Date, ModuleBuilder>> modules = new HashMap<String, TreeMap<Date, ModuleBuilder>>();
126         final ParseTreeWalker walker = new ParseTreeWalker();
127         final List<ParseTree> trees = parseStreams(yangFileStreams);
128         final ModuleBuilder[] builders = new ModuleBuilder[trees.size()];
129
130         // validate yang
131         new YangModelBasicValidator(walker).validate(trees);
132
133         YangModelParserListenerImpl yangModelParser = null;
134         for (int i = 0; i < trees.size(); i++) {
135             yangModelParser = new YangModelParserListenerImpl();
136             walker.walk(yangModelParser, trees.get(i));
137             builders[i] = yangModelParser.getModuleBuilder();
138         }
139
140         // module dependency graph sorted
141         List<ModuleSimple> sorted = new ModuleDependencySort(builders).sort();
142
143         for (ModuleBuilder builder : builders) {
144             final String builderName = builder.getName();
145             Date builderRevision = builder.getRevision();
146             if (builderRevision == null) {
147                 builderRevision = new Date(0L);
148             }
149             TreeMap<Date, ModuleBuilder> builderByRevision = modules
150                     .get(builderName);
151             if (builderByRevision == null) {
152                 builderByRevision = new TreeMap<Date, ModuleBuilder>();
153             }
154             builderByRevision.put(builderRevision, builder);
155             modules.put(builderName, builderByRevision);
156         }
157         return modules;
158     }
159
160     private List<ParseTree> parseStreams(final List<InputStream> yangStreams) {
161         final List<ParseTree> trees = new ArrayList<ParseTree>();
162         for (InputStream yangStream : yangStreams) {
163             trees.add(parseStream(yangStream));
164         }
165         return trees;
166     }
167
168     private ParseTree parseStream(final InputStream yangStream) {
169         ParseTree result = null;
170         try {
171             final ANTLRInputStream input = new ANTLRInputStream(yangStream);
172             final YangLexer lexer = new YangLexer(input);
173             final CommonTokenStream tokens = new CommonTokenStream(lexer);
174             final YangParser parser = new YangParser(tokens);
175             result = parser.yang();
176         } catch (IOException e) {
177             logger.warn("Exception while reading yang file: " + yangStream, e);
178         }
179         return result;
180     }
181
182     private Set<Module> build(
183             final Map<String, TreeMap<Date, ModuleBuilder>> modules) {
184         // fix unresolved nodes
185         for (Map.Entry<String, TreeMap<Date, ModuleBuilder>> entry : modules
186                 .entrySet()) {
187             for (Map.Entry<Date, ModuleBuilder> childEntry : entry.getValue()
188                     .entrySet()) {
189                 final ModuleBuilder moduleBuilder = childEntry.getValue();
190                 fixUnresolvedNodes(modules, moduleBuilder);
191             }
192         }
193         resolveAugments(modules);
194
195         // build
196         final Set<Module> result = new HashSet<Module>();
197         for (Map.Entry<String, TreeMap<Date, ModuleBuilder>> entry : modules
198                 .entrySet()) {
199             final Map<Date, Module> modulesByRevision = new HashMap<Date, Module>();
200             for (Map.Entry<Date, ModuleBuilder> childEntry : entry.getValue()
201                     .entrySet()) {
202                 final ModuleBuilder moduleBuilder = childEntry.getValue();
203                 final Module module = moduleBuilder.build();
204                 modulesByRevision.put(childEntry.getKey(), module);
205                 result.add(module);
206             }
207         }
208         return result;
209     }
210
211     private void fixUnresolvedNodes(
212             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
213             final ModuleBuilder builder) {
214         resolveDirtyNodes(modules, builder);
215         resolveIdentities(modules, builder);
216         resolveUses(modules, builder);
217         resolveUnknownNodes(modules, builder);
218     }
219
220     /**
221      * Search for dirty nodes (node which contains UnknownType) and resolve
222      * unknown types.
223      *
224      * @param modules
225      *            all available modules
226      * @param module
227      *            current module
228      */
229     private void resolveDirtyNodes(
230             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
231             final ModuleBuilder module) {
232         final Map<List<String>, TypeAwareBuilder> dirtyNodes = module
233                 .getDirtyNodes();
234         if (!dirtyNodes.isEmpty()) {
235             for (Map.Entry<List<String>, TypeAwareBuilder> entry : dirtyNodes
236                     .entrySet()) {
237
238                 final TypeAwareBuilder nodeToResolve = entry.getValue();
239                 // different handling for union types
240                 if (nodeToResolve instanceof UnionTypeBuilder) {
241                     final UnionTypeBuilder union = (UnionTypeBuilder) nodeToResolve;
242                     final List<TypeDefinition<?>> unionTypes = union.getTypes();
243                     final List<UnknownType> toRemove = new ArrayList<UnknownType>();
244                     for (TypeDefinition<?> td : unionTypes) {
245                         if (td instanceof UnknownType) {
246                             final UnknownType unknownType = (UnknownType) td;
247                             final TypeDefinitionBuilder resolvedType = resolveTypeUnion(
248                                     nodeToResolve, unknownType, modules, module);
249                             union.setType(resolvedType);
250                             toRemove.add(unknownType);
251                         }
252                     }
253                     unionTypes.removeAll(toRemove);
254                 } else if (nodeToResolve.getTypedef() instanceof IdentityrefTypeBuilder) {
255                     IdentityrefTypeBuilder idref = (IdentityrefTypeBuilder) nodeToResolve
256                             .getTypedef();
257                     nodeToResolve.setType(new IdentityrefType(findFullQName(
258                             modules, module, idref), idref.getPath()));
259                 } else {
260                     final TypeDefinitionBuilder resolvedType = resolveType(
261                             nodeToResolve, modules, module);
262                     nodeToResolve.setType(resolvedType);
263                 }
264             }
265         }
266     }
267
268     private TypeDefinitionBuilder resolveType(
269             final TypeAwareBuilder typeToResolve,
270             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
271             final ModuleBuilder builder) {
272         final TypeConstraints constraints = new TypeConstraints();
273
274         final TypeDefinitionBuilder targetType = getTypedefBuilder(
275                 typeToResolve, modules, builder);
276         final TypeConstraints tConstraints = findConstraints(typeToResolve,
277                 constraints, modules, builder);
278         targetType.setRanges(tConstraints.getRange());
279         targetType.setLengths(tConstraints.getLength());
280         targetType.setPatterns(tConstraints.getPatterns());
281         targetType.setFractionDigits(tConstraints.getFractionDigits());
282
283         return targetType;
284     }
285
286     private TypeDefinitionBuilder resolveTypeUnion(
287             final TypeAwareBuilder typeToResolve,
288             final UnknownType unknownType,
289             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
290             final ModuleBuilder builder) {
291         final TypeConstraints constraints = new TypeConstraints();
292
293         final TypeDefinitionBuilder targetType = getUnionBuilder(typeToResolve,
294                 unknownType, modules, builder);
295         final TypeConstraints tConstraints = findConstraints(typeToResolve,
296                 constraints, modules, builder);
297         targetType.setRanges(tConstraints.getRange());
298         targetType.setLengths(tConstraints.getLength());
299         targetType.setPatterns(tConstraints.getPatterns());
300         targetType.setFractionDigits(tConstraints.getFractionDigits());
301
302         return targetType;
303     }
304
305     private TypeDefinitionBuilder getTypedefBuilder(
306             final TypeAwareBuilder nodeToResolve,
307             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
308             final ModuleBuilder builder) {
309
310         final TypeDefinition<?> nodeToResolveBase = nodeToResolve.getType();
311         if (nodeToResolveBase != null
312                 && !(nodeToResolveBase instanceof UnknownType)) {
313             return (TypeDefinitionBuilder) nodeToResolve;
314         }
315
316         final UnknownType unknownType = (UnknownType) nodeToResolve.getType();
317         final QName unknownTypeQName = unknownType.getQName();
318
319         // search for module which contains referenced typedef
320         final ModuleBuilder dependentModule = findDependentModule(modules,
321                 builder, unknownTypeQName.getPrefix());
322         final TypeDefinitionBuilder lookedUpBuilder = findTypedefBuilderByName(
323                 dependentModule, unknownTypeQName.getLocalName());
324
325         final TypeDefinitionBuilder lookedUpBuilderCopy = copyTypedefBuilder(
326                 lookedUpBuilder, nodeToResolve instanceof TypeDefinitionBuilder);
327         final TypeDefinitionBuilder resolvedCopy = resolveCopiedBuilder(
328                 lookedUpBuilderCopy, modules, dependentModule);
329         return resolvedCopy;
330     }
331
332     private TypeDefinitionBuilder getUnionBuilder(
333             final TypeAwareBuilder nodeToResolve,
334             final UnknownType unknownType,
335             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
336             final ModuleBuilder module) {
337
338         final TypeDefinition<?> baseTypeToResolve = nodeToResolve.getType();
339         if (baseTypeToResolve != null
340                 && !(baseTypeToResolve instanceof UnknownType)) {
341             return (TypeDefinitionBuilder) nodeToResolve;
342         }
343
344         final QName unknownTypeQName = unknownType.getQName();
345         // search for module which contains referenced typedef
346         final ModuleBuilder dependentModule = findDependentModule(modules,
347                 module, unknownTypeQName.getPrefix());
348         final TypeDefinitionBuilder lookedUpBuilder = findTypedefBuilderByName(
349                 dependentModule, unknownTypeQName.getLocalName());
350
351         final TypeDefinitionBuilder lookedUpBuilderCopy = copyTypedefBuilder(
352                 lookedUpBuilder, nodeToResolve instanceof TypeDefinitionBuilder);
353         final TypeDefinitionBuilder resolvedCopy = resolveCopiedBuilder(
354                 lookedUpBuilderCopy, modules, dependentModule);
355         return resolvedCopy;
356     }
357
358     private TypeDefinitionBuilder copyTypedefBuilder(
359             final TypeDefinitionBuilder old, final boolean seekByTypedefBuilder) {
360         if (old instanceof UnionTypeBuilder) {
361             final UnionTypeBuilder oldUnion = (UnionTypeBuilder) old;
362             final UnionTypeBuilder newUnion = new UnionTypeBuilder(
363                     oldUnion.getActualPath(), oldUnion.getNamespace(),
364                     oldUnion.getRevision());
365             for (TypeDefinition<?> td : oldUnion.getTypes()) {
366                 newUnion.setType(td);
367             }
368             for (TypeDefinitionBuilder tdb : oldUnion.getTypedefs()) {
369                 newUnion.setType(copyTypedefBuilder(tdb, true));
370             }
371             return newUnion;
372         }
373
374         final QName oldName = old.getQName();
375         final QName newName = new QName(oldName.getNamespace(),
376                 oldName.getRevision(), oldName.getPrefix(),
377                 oldName.getLocalName());
378         final TypeDefinitionBuilder tdb = new TypedefBuilder(newName);
379
380         tdb.setRanges(old.getRanges());
381         tdb.setLengths(old.getLengths());
382         tdb.setPatterns(old.getPatterns());
383         tdb.setFractionDigits(old.getFractionDigits());
384         tdb.setPath(old.getPath());
385
386         final TypeDefinition<?> oldType = old.getType();
387         if (oldType == null) {
388             tdb.setType(old.getTypedef());
389         } else {
390             tdb.setType(oldType);
391         }
392
393         if (!seekByTypedefBuilder) {
394             tdb.setDescription(old.getDescription());
395             tdb.setReference(old.getReference());
396             tdb.setStatus(old.getStatus());
397             tdb.setDefaultValue(old.getDefaultValue());
398             tdb.setUnits(old.getUnits());
399         }
400         return tdb;
401     }
402
403     private TypeDefinitionBuilder resolveCopiedBuilder(
404             final TypeDefinitionBuilder copy,
405             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
406             final ModuleBuilder builder) {
407
408         if (copy instanceof UnionTypeBuilder) {
409             final UnionTypeBuilder union = (UnionTypeBuilder) copy;
410             final List<TypeDefinition<?>> unionTypes = union.getTypes();
411             final List<UnknownType> toRemove = new ArrayList<UnknownType>();
412             for (TypeDefinition<?> td : unionTypes) {
413                 if (td instanceof UnknownType) {
414                     final UnknownType unknownType = (UnknownType) td;
415                     final TypeDefinitionBuilder resolvedType = resolveTypeUnion(
416                             union, unknownType, modules, builder);
417                     union.setType(resolvedType);
418                     toRemove.add(unknownType);
419                 }
420             }
421             unionTypes.removeAll(toRemove);
422
423             return union;
424         }
425
426         final TypeDefinition<?> base = copy.getType();
427         final TypeDefinitionBuilder baseTdb = copy.getTypedef();
428         if (base != null && !(base instanceof UnknownType)) {
429             return copy;
430         } else if (base instanceof UnknownType) {
431             final UnknownType unknownType = (UnknownType) base;
432             final QName unknownTypeQName = unknownType.getQName();
433             final String unknownTypePrefix = unknownTypeQName.getPrefix();
434             final ModuleBuilder dependentModule = findDependentModule(modules,
435                     builder, unknownTypePrefix);
436             final TypeDefinitionBuilder utBuilder = getTypedefBuilder(copy,
437                     modules, dependentModule);
438             copy.setType(utBuilder);
439             return copy;
440         } else if (base == null && baseTdb != null) {
441             // make a copy of baseTypeDef and call again
442             final TypeDefinitionBuilder baseTdbCopy = copyTypedefBuilder(
443                     baseTdb, true);
444             final TypeDefinitionBuilder baseTdbCopyResolved = resolveCopiedBuilder(
445                     baseTdbCopy, modules, builder);
446             copy.setType(baseTdbCopyResolved);
447             return copy;
448         } else {
449             throw new IllegalStateException("Failed to resolve type "
450                     + copy.getQName().getLocalName());
451         }
452     }
453
454     private TypeDefinitionBuilder findTypedefBuilder(
455             final QName unknownTypeQName,
456             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
457             final ModuleBuilder builder) {
458         // search for module which contains referenced typedef
459         final ModuleBuilder dependentModule = findDependentModule(modules,
460                 builder, unknownTypeQName.getPrefix());
461         final TypeDefinitionBuilder lookedUpBuilder = findTypedefBuilderByName(
462                 dependentModule, unknownTypeQName.getLocalName());
463         return copyTypedefBuilder(lookedUpBuilder, true);
464     }
465
466     private TypeConstraints findConstraints(
467             final TypeAwareBuilder nodeToResolve,
468             final TypeConstraints constraints,
469             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
470             final ModuleBuilder builder) {
471         // union type cannot be restricted
472         if (nodeToResolve instanceof UnionTypeBuilder) {
473             return constraints;
474         }
475
476         // if referenced type is UnknownType again, search recursively with
477         // current constraints
478         final TypeDefinition<?> referencedType = nodeToResolve.getType();
479         List<RangeConstraint> ranges = Collections.emptyList();
480         List<LengthConstraint> lengths = Collections.emptyList();
481         List<PatternConstraint> patterns = Collections.emptyList();
482         Integer fractionDigits = null;
483         if (referencedType == null) {
484             final TypeDefinitionBuilder tdb = (TypeDefinitionBuilder) nodeToResolve;
485             ranges = tdb.getRanges();
486             constraints.addRanges(ranges);
487             lengths = tdb.getLengths();
488             constraints.addLengths(lengths);
489             patterns = tdb.getPatterns();
490             constraints.addPatterns(patterns);
491             fractionDigits = tdb.getFractionDigits();
492             constraints.setFractionDigits(fractionDigits);
493             return constraints;
494         } else if (referencedType instanceof ExtendedType) {
495             final ExtendedType ext = (ExtendedType) referencedType;
496             ranges = ext.getRanges();
497             constraints.addRanges(ranges);
498             lengths = ext.getLengths();
499             constraints.addLengths(lengths);
500             patterns = ext.getPatterns();
501             constraints.addPatterns(patterns);
502             fractionDigits = ext.getFractionDigits();
503             constraints.setFractionDigits(fractionDigits);
504             return findConstraints(
505                     findTypedefBuilder(ext.getQName(), modules, builder),
506                     constraints, modules, builder);
507         } else if (referencedType instanceof UnknownType) {
508             final UnknownType unknown = (UnknownType) referencedType;
509             ranges = unknown.getRangeStatements();
510             constraints.addRanges(ranges);
511             lengths = unknown.getLengthStatements();
512             constraints.addLengths(lengths);
513             patterns = unknown.getPatterns();
514             constraints.addPatterns(patterns);
515             fractionDigits = unknown.getFractionDigits();
516             constraints.setFractionDigits(fractionDigits);
517
518             String unknownTypePrefix = unknown.getQName().getPrefix();
519             if (unknownTypePrefix == null || "".equals(unknownTypePrefix)) {
520                 unknownTypePrefix = builder.getPrefix();
521             }
522             final ModuleBuilder dependentModule = findDependentModule(modules,
523                     builder, unknown.getQName().getPrefix());
524             final TypeDefinitionBuilder utBuilder = findTypedefBuilder(
525                     unknown.getQName(), modules, builder);
526             return findConstraints(utBuilder, constraints, modules,
527                     dependentModule);
528         } else {
529             // HANDLE BASE YANG TYPE
530             mergeConstraints(referencedType, constraints);
531             return constraints;
532         }
533     }
534
535     /**
536      * Go through all typedef statements from given module and search for one
537      * with given name
538      *
539      * @param typedefs
540      *            typedef statements to search
541      * @param name
542      *            name of searched typedef
543      * @return typedef with name equals to given name
544      */
545     private TypeDefinitionBuilder findTypedefBuilderByName(
546             final ModuleBuilder dependentModule, final String name) {
547         TypeDefinitionBuilder result = null;
548         final Set<TypeDefinitionBuilder> typedefs = dependentModule
549                 .getModuleTypedefs();
550         for (TypeDefinitionBuilder td : typedefs) {
551             if (td.getQName().getLocalName().equals(name)) {
552                 result = td;
553                 break;
554             }
555         }
556         if (result == null) {
557             throw new YangParseException("Target module '"
558                     + dependentModule.getName()
559                     + "' does not contain typedef '" + name + "'.");
560         }
561         return result;
562     }
563
564     /**
565      * Pull restriction from referenced type and add them to given constraints
566      *
567      * @param referencedType
568      * @param constraints
569      */
570     private void mergeConstraints(final TypeDefinition<?> referencedType,
571             final TypeConstraints constraints) {
572
573         if (referencedType instanceof DecimalTypeDefinition) {
574             constraints.addRanges(((DecimalTypeDefinition) referencedType)
575                     .getRangeStatements());
576             constraints
577                     .setFractionDigits(((DecimalTypeDefinition) referencedType)
578                             .getFractionDigits());
579         } else if (referencedType instanceof IntegerTypeDefinition) {
580             constraints.addRanges(((IntegerTypeDefinition) referencedType)
581                     .getRangeStatements());
582         } else if (referencedType instanceof StringTypeDefinition) {
583             constraints.addPatterns(((StringTypeDefinition) referencedType)
584                     .getPatterns());
585             constraints.addLengths(((StringTypeDefinition) referencedType)
586                     .getLengthStatements());
587         } else if (referencedType instanceof BinaryTypeDefinition) {
588             constraints.addLengths(((BinaryTypeDefinition) referencedType)
589                     .getLengthConstraints());
590         }
591     }
592
593     /**
594      * Go through all augmentation definitions and resolve them. This method
595      * also finds referenced node and add child nodes to it.
596      *
597      * @param modules
598      *            all available modules
599      */
600     private void resolveAugments(
601             final Map<String, TreeMap<Date, ModuleBuilder>> modules) {
602         final List<ModuleBuilder> allModulesList = new ArrayList<ModuleBuilder>();
603         final Set<ModuleBuilder> allModulesSet = new HashSet<ModuleBuilder>();
604         for (Map.Entry<String, TreeMap<Date, ModuleBuilder>> entry : modules
605                 .entrySet()) {
606             for (Map.Entry<Date, ModuleBuilder> inner : entry.getValue()
607                     .entrySet()) {
608                 allModulesList.add(inner.getValue());
609                 allModulesSet.add(inner.getValue());
610             }
611         }
612
613         for (int i = 0; i < allModulesList.size(); i++) {
614             final ModuleBuilder module = allModulesList.get(i);
615             // try to resolve augments in module
616             resolveAugment(modules, module);
617             // while all augments are not resolved
618             final Iterator<ModuleBuilder> allModulesIterator = allModulesSet
619                     .iterator();
620             while (!(module.getAugmentsResolved() == module.getAddedAugments()
621                     .size())) {
622                 ModuleBuilder nextModule = null;
623                 // try resolve other module augments
624                 try {
625                     nextModule = allModulesIterator.next();
626                     resolveAugment(modules, nextModule);
627                 } catch (NoSuchElementException e) {
628                     throw new YangParseException(
629                             "Failed to resolve augments in module '"
630                                     + module.getName() + "'.", e);
631                 }
632                 // then try to resolve first module again
633                 resolveAugment(modules, module);
634             }
635         }
636     }
637
638     /**
639      *
640      * @param modules
641      *            all available modules
642      * @param module
643      *            current module
644      */
645     private void resolveAugment(
646             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
647             final ModuleBuilder module) {
648         if (module.getAugmentsResolved() < module.getAddedAugments().size()) {
649             for (AugmentationSchemaBuilder augmentBuilder : module
650                     .getAddedAugments()) {
651                 final SchemaPath augmentTargetSchemaPath = augmentBuilder
652                         .getTargetPath();
653                 final List<QName> path = augmentTargetSchemaPath.getPath();
654
655                 int i = 0;
656                 final QName qname = path.get(i);
657                 String prefix = qname.getPrefix();
658                 if (prefix == null) {
659                     prefix = module.getPrefix();
660                 }
661
662                 DataSchemaNodeBuilder currentParent = null;
663                 final ModuleBuilder dependentModule = findDependentModule(
664                         modules, module, prefix);
665                 for (DataSchemaNodeBuilder child : dependentModule
666                         .getChildNodes()) {
667                     final QName childQName = child.getQName();
668                     if (childQName.getLocalName().equals(qname.getLocalName())) {
669                         currentParent = child;
670                         i++;
671                         break;
672                     }
673                 }
674
675                 for (; i < path.size(); i++) {
676                     final QName currentQName = path.get(i);
677                     DataSchemaNodeBuilder newParent = null;
678                     for (DataSchemaNodeBuilder child : ((ChildNodeBuilder) currentParent)
679                             .getChildNodes()) {
680                         final QName childQName = child.getQName();
681                         if (childQName.getLocalName().equals(
682                                 currentQName.getLocalName())) {
683                             newParent = child;
684                             break;
685                         }
686                     }
687                     if (newParent == null) {
688                         break; // node not found, quit search
689                     } else {
690                         currentParent = newParent;
691                     }
692                 }
693
694                 final QName currentQName = currentParent.getQName();
695                 final QName lastAugmentPathElement = path.get(path.size() - 1);
696                 if (currentQName.getLocalName().equals(
697                         lastAugmentPathElement.getLocalName())) {
698                     ParserUtils.fillAugmentTarget(augmentBuilder,
699                             (ChildNodeBuilder) currentParent);
700                     ((AugmentationTargetBuilder) currentParent)
701                             .addAugmentation(augmentBuilder);
702                     module.augmentResolved();
703                 }
704             }
705         }
706     }
707
708     /**
709      * Go through identity statements defined in current module and resolve
710      * their 'base' statement if present.
711      *
712      * @param modules
713      *            all modules
714      * @param module
715      *            module being resolved
716      */
717     private void resolveIdentities(
718             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
719             final ModuleBuilder module) {
720         final Set<IdentitySchemaNodeBuilder> identities = module
721                 .getAddedIdentities();
722         for (IdentitySchemaNodeBuilder identity : identities) {
723             final String baseIdentityName = identity.getBaseIdentityName();
724             if (baseIdentityName != null) {
725                 String baseIdentityPrefix = null;
726                 String baseIdentityLocalName = null;
727                 if (baseIdentityName.contains(":")) {
728                     final String[] splitted = baseIdentityName.split(":");
729                     baseIdentityPrefix = splitted[0];
730                     baseIdentityLocalName = splitted[1];
731                 } else {
732                     baseIdentityPrefix = module.getPrefix();
733                     baseIdentityLocalName = baseIdentityName;
734                 }
735                 final ModuleBuilder dependentModule = findDependentModule(
736                         modules, module, baseIdentityPrefix);
737
738                 final Set<IdentitySchemaNodeBuilder> dependentModuleIdentities = dependentModule
739                         .getAddedIdentities();
740                 for (IdentitySchemaNodeBuilder idBuilder : dependentModuleIdentities) {
741                     if (idBuilder.getQName().getLocalName()
742                             .equals(baseIdentityLocalName)) {
743                         identity.setBaseIdentity(idBuilder);
744                     }
745                 }
746             }
747         }
748     }
749
750     /**
751      * Go through uses statements defined in current module and resolve their
752      * refine statements.
753      *
754      * @param modules
755      *            all modules
756      * @param module
757      *            module being resolved
758      */
759     private void resolveUses(
760             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
761             final ModuleBuilder module) {
762         final Map<List<String>, UsesNodeBuilder> moduleUses = module
763                 .getAddedUsesNodes();
764         for (Map.Entry<List<String>, UsesNodeBuilder> entry : moduleUses
765                 .entrySet()) {
766             final List<String> key = entry.getKey();
767             final UsesNodeBuilder usesNode = entry.getValue();
768
769             final String groupingName = key.get(key.size() - 1);
770
771             for (RefineHolder refine : usesNode.getRefines()) {
772                 // refine statements
773                 final String defaultStr = refine.getDefaultStr();
774                 final Boolean mandatory = refine.isMandatory();
775                 final MustDefinition must = refine.getMust();
776                 final Boolean presence = refine.isPresence();
777                 final Integer min = refine.getMinElements();
778                 final Integer max = refine.getMaxElements();
779                 final List<UnknownSchemaNodeBuilder> unknownNodes = refine
780                         .getUnknownNodes();
781
782                 Builder refineTarget = getRefineTargetBuilder(groupingName,
783                         refine, modules, module);
784                 if (refineTarget instanceof LeafSchemaNodeBuilder) {
785                     final LeafSchemaNodeBuilder leaf = (LeafSchemaNodeBuilder) refineTarget;
786                     if (defaultStr != null && !("".equals(defaultStr))) {
787                         leaf.setDefaultStr(defaultStr);
788                     }
789                     if (mandatory != null) {
790                         leaf.getConstraints().setMandatory(mandatory);
791                     }
792                     if (must != null) {
793                         leaf.getConstraints().addMustDefinition(must);
794                     }
795                     if (unknownNodes != null) {
796                         for (UnknownSchemaNodeBuilder unknown : unknownNodes) {
797                             leaf.addUnknownSchemaNode(unknown);
798                         }
799                     }
800                     usesNode.addRefineNode(leaf);
801                 } else if (refineTarget instanceof ContainerSchemaNodeBuilder) {
802                     final ContainerSchemaNodeBuilder container = (ContainerSchemaNodeBuilder) refineTarget;
803                     if (presence != null) {
804                         container.setPresence(presence);
805                     }
806                     if (must != null) {
807                         container.getConstraints().addMustDefinition(must);
808                     }
809                     if (unknownNodes != null) {
810                         for (UnknownSchemaNodeBuilder unknown : unknownNodes) {
811                             container.addUnknownSchemaNode(unknown);
812                         }
813                     }
814                     usesNode.addRefineNode(container);
815                 } else if (refineTarget instanceof ListSchemaNodeBuilder) {
816                     final ListSchemaNodeBuilder list = (ListSchemaNodeBuilder) refineTarget;
817                     if (must != null) {
818                         list.getConstraints().addMustDefinition(must);
819                     }
820                     if (min != null) {
821                         list.getConstraints().setMinElements(min);
822                     }
823                     if (max != null) {
824                         list.getConstraints().setMaxElements(max);
825                     }
826                     if (unknownNodes != null) {
827                         for (UnknownSchemaNodeBuilder unknown : unknownNodes) {
828                             list.addUnknownSchemaNode(unknown);
829                         }
830                     }
831                 } else if (refineTarget instanceof LeafListSchemaNodeBuilder) {
832                     final LeafListSchemaNodeBuilder leafList = (LeafListSchemaNodeBuilder) getRefineTargetBuilder(
833                             groupingName, refine, modules, module);
834                     if (must != null) {
835                         leafList.getConstraints().addMustDefinition(must);
836                     }
837                     if (min != null) {
838                         leafList.getConstraints().setMinElements(min);
839                     }
840                     if (max != null) {
841                         leafList.getConstraints().setMaxElements(max);
842                     }
843                     if (unknownNodes != null) {
844                         for (UnknownSchemaNodeBuilder unknown : unknownNodes) {
845                             leafList.addUnknownSchemaNode(unknown);
846                         }
847                     }
848                 } else if (refineTarget instanceof ChoiceBuilder) {
849                     final ChoiceBuilder choice = (ChoiceBuilder) refineTarget;
850                     if (defaultStr != null) {
851                         choice.setDefaultCase(defaultStr);
852                     }
853                     if (mandatory != null) {
854                         choice.getConstraints().setMandatory(mandatory);
855                     }
856                     if (unknownNodes != null) {
857                         for (UnknownSchemaNodeBuilder unknown : unknownNodes) {
858                             choice.addUnknownSchemaNode(unknown);
859                         }
860                     }
861                 } else if (refineTarget instanceof AnyXmlBuilder) {
862                     final AnyXmlBuilder anyXml = (AnyXmlBuilder) refineTarget;
863                     if (mandatory != null) {
864                         anyXml.getConstraints().setMandatory(mandatory);
865                     }
866                     if (must != null) {
867                         anyXml.getConstraints().addMustDefinition(must);
868                     }
869                     if (unknownNodes != null) {
870                         for (UnknownSchemaNodeBuilder unknown : unknownNodes) {
871                             anyXml.addUnknownSchemaNode(unknown);
872                         }
873                     }
874                 }
875             }
876         }
877     }
878
879     /**
880      * Find original builder of refine node and return copy of this builder.
881      *
882      * @param groupingPath
883      *            path to grouping which contains node to refine
884      * @param refine
885      *            refine object containing informations about refine
886      * @param modules
887      *            all loaded modules
888      * @param module
889      *            current module
890      * @return copy of Builder object of node to be refined if it is present in
891      *         grouping, null otherwise
892      */
893     private Builder getRefineTargetBuilder(final String groupingPath,
894             final RefineHolder refine,
895             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
896             final ModuleBuilder module) {
897         Builder result = null;
898         final Builder lookedUpBuilder = findRefineTargetBuilder(groupingPath,
899                 refine.getName(), modules, module);
900         if (lookedUpBuilder instanceof LeafSchemaNodeBuilder) {
901             result = ParserUtils
902                     .copyLeafBuilder((LeafSchemaNodeBuilder) lookedUpBuilder);
903         } else if (lookedUpBuilder instanceof ContainerSchemaNodeBuilder) {
904             result = ParserUtils
905                     .copyContainerBuilder((ContainerSchemaNodeBuilder) lookedUpBuilder);
906         } else if (lookedUpBuilder instanceof ListSchemaNodeBuilder) {
907             result = ParserUtils
908                     .copyListBuilder((ListSchemaNodeBuilder) lookedUpBuilder);
909         } else if (lookedUpBuilder instanceof LeafListSchemaNodeBuilder) {
910             result = ParserUtils
911                     .copyLeafListBuilder((LeafListSchemaNodeBuilder) lookedUpBuilder);
912         } else if (lookedUpBuilder instanceof ChoiceBuilder) {
913             result = ParserUtils
914                     .copyChoiceBuilder((ChoiceBuilder) lookedUpBuilder);
915         } else if (lookedUpBuilder instanceof AnyXmlBuilder) {
916             result = ParserUtils
917                     .copyAnyXmlBuilder((AnyXmlBuilder) lookedUpBuilder);
918         } else {
919             throw new YangParseException("Target '" + refine.getName()
920                     + "' can not be refined");
921         }
922         return result;
923     }
924
925     /**
926      * Find builder of refine node.
927      *
928      * @param groupingPath
929      *            path to grouping which contains node to refine
930      * @param refineNodeName
931      *            name of node to be refined
932      * @param modules
933      *            all loaded modules
934      * @param module
935      *            current module
936      * @return Builder object of refine node if it is present in grouping, null
937      *         otherwise
938      */
939     private Builder findRefineTargetBuilder(final String groupingPath,
940             final String refineNodeName,
941             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
942             final ModuleBuilder module) {
943         final SchemaPath path = ParserUtils.parseUsesPath(groupingPath);
944         final List<String> builderPath = new ArrayList<String>();
945         String prefix = null;
946         for (QName qname : path.getPath()) {
947             builderPath.add(qname.getLocalName());
948             prefix = qname.getPrefix();
949         }
950         if (prefix == null) {
951             prefix = module.getPrefix();
952         }
953
954         final ModuleBuilder dependentModule = findDependentModule(modules,
955                 module, prefix);
956         builderPath.add(0, "grouping");
957         builderPath.add(0, dependentModule.getName());
958         final GroupingBuilder builder = (GroupingBuilder) dependentModule
959                 .getNode(builderPath);
960
961         return builder.getChildNode(refineNodeName);
962     }
963
964     private QName findFullQName(
965             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
966             final ModuleBuilder module, final IdentityrefTypeBuilder idref) {
967         QName result = null;
968         String baseString = idref.getBaseString();
969         if (baseString.contains(":")) {
970             String[] splittedBase = baseString.split(":");
971             if (splittedBase.length > 2) {
972                 throw new YangParseException(
973                         "Failed to parse identityref base: " + baseString);
974             }
975             String prefix = splittedBase[0];
976             String name = splittedBase[1];
977             ModuleBuilder dependentModule = findDependentModule(modules,
978                     module, prefix);
979             result = new QName(dependentModule.getNamespace(),
980                     dependentModule.getRevision(), prefix, name);
981         } else {
982             result = new QName(module.getNamespace(), module.getRevision(),
983                     module.getPrefix(), baseString);
984         }
985         return result;
986     }
987
988     private void resolveUnknownNodes(
989             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
990             final ModuleBuilder module) {
991         for (UnknownSchemaNodeBuilder usnb : module.getAddedUnknownNodes()) {
992             QName nodeType = usnb.getNodeType();
993             if (nodeType.getNamespace() == null
994                     || nodeType.getRevision() == null) {
995                 try {
996                     ModuleBuilder dependentModule = findDependentModule(
997                             modules, module, nodeType.getPrefix());
998                     QName newNodeType = new QName(
999                             dependentModule.getNamespace(),
1000                             dependentModule.getRevision(),
1001                             nodeType.getPrefix(), nodeType.getLocalName());
1002                     usnb.setNodeType(newNodeType);
1003                 } catch (YangParseException e) {
1004                     logger.debug("Failed to find unknown node type: "
1005                             + nodeType);
1006                 }
1007             }
1008         }
1009     }
1010
1011     /**
1012      * Find dependent module based on given prefix
1013      *
1014      * @param modules
1015      *            all available modules
1016      * @param module
1017      *            current module
1018      * @param prefix
1019      *            target module prefix
1020      * @return
1021      */
1022     private ModuleBuilder findDependentModule(
1023             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
1024             final ModuleBuilder module, final String prefix) {
1025         ModuleBuilder dependentModule = null;
1026         Date dependentModuleRevision = null;
1027
1028         if (prefix.equals(module.getPrefix())) {
1029             dependentModule = module;
1030         } else {
1031             final ModuleImport dependentModuleImport = ParserUtils
1032                     .getModuleImport(module, prefix);
1033             if (dependentModuleImport == null) {
1034                 throw new YangParseException("No import found with prefix '"
1035                         + prefix + "' in module " + module.getName() + "'.");
1036             }
1037             final String dependentModuleName = dependentModuleImport
1038                     .getModuleName();
1039             dependentModuleRevision = dependentModuleImport.getRevision();
1040
1041             final TreeMap<Date, ModuleBuilder> moduleBuildersByRevision = modules
1042                     .get(dependentModuleName);
1043             if (moduleBuildersByRevision == null) {
1044                 throw new YangParseException(
1045                         "Failed to find dependent module '"
1046                                 + dependentModuleName + "' needed by module '"
1047                                 + module.getName() + "'.");
1048             }
1049             if (dependentModuleRevision == null) {
1050                 dependentModule = moduleBuildersByRevision.lastEntry()
1051                         .getValue();
1052             } else {
1053                 dependentModule = moduleBuildersByRevision
1054                         .get(dependentModuleRevision);
1055             }
1056         }
1057
1058         if (dependentModule == null) {
1059             throw new YangParseException(
1060                     "Failed to find dependent module with prefix '" + prefix
1061                             + "' and revision '" + dependentModuleRevision
1062                             + "'.");
1063         }
1064         return dependentModule;
1065     }
1066
1067     private static class SchemaContextImpl implements SchemaContext {
1068         private final Set<Module> modules;
1069
1070         private SchemaContextImpl(final Set<Module> modules) {
1071             this.modules = modules;
1072         }
1073
1074         @Override
1075         public Set<DataSchemaNode> getDataDefinitions() {
1076             final Set<DataSchemaNode> dataDefs = new HashSet<DataSchemaNode>();
1077             for (Module m : modules) {
1078                 dataDefs.addAll(m.getChildNodes());
1079             }
1080             return dataDefs;
1081         }
1082
1083         @Override
1084         public Set<Module> getModules() {
1085             return modules;
1086         }
1087
1088         @Override
1089         public Set<NotificationDefinition> getNotifications() {
1090             final Set<NotificationDefinition> notifications = new HashSet<NotificationDefinition>();
1091             for (Module m : modules) {
1092                 notifications.addAll(m.getNotifications());
1093             }
1094             return notifications;
1095         }
1096
1097         @Override
1098         public Set<RpcDefinition> getOperations() {
1099             final Set<RpcDefinition> rpcs = new HashSet<RpcDefinition>();
1100             for (Module m : modules) {
1101                 rpcs.addAll(m.getRpcs());
1102             }
1103             return rpcs;
1104         }
1105
1106         @Override
1107         public Set<ExtensionDefinition> getExtensions() {
1108             final Set<ExtensionDefinition> extensions = new HashSet<ExtensionDefinition>();
1109             for (Module m : modules) {
1110                 extensions.addAll(m.getExtensionSchemaNodes());
1111             }
1112             return extensions;
1113         }
1114
1115         @Override
1116         public Module findModuleByName(final String name, final Date revision) {
1117             if ((name != null) && (revision != null)) {
1118                 for (final Module module : modules) {
1119                     if (module.getName().equals(name)
1120                             && module.getRevision().equals(revision)) {
1121                         return module;
1122                     }
1123                 }
1124             }
1125             return null;
1126         }
1127
1128         @Override
1129         public Module findModuleByNamespace(final URI namespace) {
1130             if (namespace != null) {
1131                 for (final Module module : modules) {
1132                     if (module.getNamespace().equals(namespace)) {
1133                         return module;
1134                     }
1135                 }
1136             }
1137             return null;
1138         }
1139     }
1140
1141 }