Fixed YANG union type parsing. Changed module builders storing from HashMap to Linked...
[controller.git] / opendaylight / sal / yang-prototype / code-generator / yang-model-parser-impl / src / main / java / org / opendaylight / controller / yang / parser / impl / YangParserImpl.java
1 /*
2  * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
3  *
4  * This program and the accompanying materials are made available under the
5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6  * and is available at http://www.eclipse.org/legal/epl-v10.html
7  */
8 package org.opendaylight.controller.yang.parser.impl;
9
10 import java.io.File;
11 import java.io.FileInputStream;
12 import java.io.FileNotFoundException;
13 import java.io.IOException;
14 import java.io.InputStream;
15 import java.net.URI;
16 import java.util.ArrayList;
17 import java.util.Collections;
18 import java.util.Date;
19 import java.util.HashMap;
20 import java.util.HashSet;
21 import java.util.Iterator;
22 import java.util.LinkedHashMap;
23 import java.util.LinkedHashSet;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.NoSuchElementException;
27 import java.util.Set;
28 import java.util.TreeMap;
29
30 import org.antlr.v4.runtime.ANTLRInputStream;
31 import org.antlr.v4.runtime.CommonTokenStream;
32 import org.antlr.v4.runtime.tree.ParseTree;
33 import org.antlr.v4.runtime.tree.ParseTreeWalker;
34 import org.opendaylight.controller.antlrv4.code.gen.YangLexer;
35 import org.opendaylight.controller.antlrv4.code.gen.YangParser;
36 import org.opendaylight.controller.yang.common.QName;
37 import org.opendaylight.controller.yang.model.api.DataSchemaNode;
38 import org.opendaylight.controller.yang.model.api.ExtensionDefinition;
39 import org.opendaylight.controller.yang.model.api.Module;
40 import org.opendaylight.controller.yang.model.api.ModuleImport;
41 import org.opendaylight.controller.yang.model.api.MustDefinition;
42 import org.opendaylight.controller.yang.model.api.NotificationDefinition;
43 import org.opendaylight.controller.yang.model.api.RpcDefinition;
44 import org.opendaylight.controller.yang.model.api.SchemaContext;
45 import org.opendaylight.controller.yang.model.api.SchemaPath;
46 import org.opendaylight.controller.yang.model.api.TypeDefinition;
47 import org.opendaylight.controller.yang.model.api.type.BinaryTypeDefinition;
48 import org.opendaylight.controller.yang.model.api.type.DecimalTypeDefinition;
49 import org.opendaylight.controller.yang.model.api.type.IntegerTypeDefinition;
50 import org.opendaylight.controller.yang.model.api.type.LengthConstraint;
51 import org.opendaylight.controller.yang.model.api.type.PatternConstraint;
52 import org.opendaylight.controller.yang.model.api.type.RangeConstraint;
53 import org.opendaylight.controller.yang.model.api.type.StringTypeDefinition;
54 import org.opendaylight.controller.yang.model.parser.api.YangModelParser;
55 import org.opendaylight.controller.yang.model.util.ExtendedType;
56 import org.opendaylight.controller.yang.model.util.IdentityrefType;
57 import org.opendaylight.controller.yang.model.util.UnknownType;
58 import org.opendaylight.controller.yang.parser.builder.api.AugmentationSchemaBuilder;
59 import org.opendaylight.controller.yang.parser.builder.api.AugmentationTargetBuilder;
60 import org.opendaylight.controller.yang.parser.builder.api.Builder;
61 import org.opendaylight.controller.yang.parser.builder.api.ChildNodeBuilder;
62 import org.opendaylight.controller.yang.parser.builder.api.DataSchemaNodeBuilder;
63 import org.opendaylight.controller.yang.parser.builder.api.GroupingBuilder;
64 import org.opendaylight.controller.yang.parser.builder.api.TypeAwareBuilder;
65 import org.opendaylight.controller.yang.parser.builder.api.TypeDefinitionBuilder;
66 import org.opendaylight.controller.yang.parser.builder.api.UsesNodeBuilder;
67 import org.opendaylight.controller.yang.parser.builder.impl.AnyXmlBuilder;
68 import org.opendaylight.controller.yang.parser.builder.impl.ChoiceBuilder;
69 import org.opendaylight.controller.yang.parser.builder.impl.ContainerSchemaNodeBuilder;
70 import org.opendaylight.controller.yang.parser.builder.impl.IdentitySchemaNodeBuilder;
71 import org.opendaylight.controller.yang.parser.builder.impl.IdentityrefTypeBuilder;
72 import org.opendaylight.controller.yang.parser.builder.impl.LeafListSchemaNodeBuilder;
73 import org.opendaylight.controller.yang.parser.builder.impl.LeafSchemaNodeBuilder;
74 import org.opendaylight.controller.yang.parser.builder.impl.ListSchemaNodeBuilder;
75 import org.opendaylight.controller.yang.parser.builder.impl.ModuleBuilder;
76 import org.opendaylight.controller.yang.parser.builder.impl.TypedefBuilder;
77 import org.opendaylight.controller.yang.parser.builder.impl.UnionTypeBuilder;
78 import org.opendaylight.controller.yang.parser.builder.impl.UnknownSchemaNodeBuilder;
79 import org.opendaylight.controller.yang.parser.util.ModuleDependencySort;
80 import org.opendaylight.controller.yang.parser.util.ModuleDependencySort.ModuleSimple;
81 import org.opendaylight.controller.yang.parser.util.ParserUtils;
82 import org.opendaylight.controller.yang.parser.util.RefineHolder;
83 import org.opendaylight.controller.yang.parser.util.TypeConstraints;
84 import org.opendaylight.controller.yang.parser.util.YangParseException;
85 import org.opendaylight.controller.yang.validator.YangModelBasicValidator;
86 import org.slf4j.Logger;
87 import org.slf4j.LoggerFactory;
88
89 public class YangParserImpl implements YangModelParser {
90
91     private static final Logger logger = LoggerFactory
92             .getLogger(YangParserImpl.class);
93
94     @Override
95     public Set<Module> parseYangModels(final List<File> yangFiles) {
96         if (yangFiles != null) {
97             final List<InputStream> inputStreams = new ArrayList<InputStream>();
98
99             for (final File yangFile : yangFiles) {
100                 try {
101                     inputStreams.add(new FileInputStream(yangFile));
102                 } catch (FileNotFoundException e) {
103                     logger.warn("Exception while reading yang file: "
104                             + yangFile.getName(), e);
105                 }
106             }
107             final Map<String, TreeMap<Date, ModuleBuilder>> modules = resolveModuleBuilders(inputStreams);
108             return build(modules);
109         }
110         return Collections.emptySet();
111     }
112
113     @Override
114     public Set<Module> parseYangModelsFromStreams(
115             final List<InputStream> yangModelStreams) {
116         final Map<String, TreeMap<Date, ModuleBuilder>> modules = resolveModuleBuilders(yangModelStreams);
117         return build(modules);
118     }
119
120     @Override
121     public SchemaContext resolveSchemaContext(final Set<Module> modules) {
122         return new SchemaContextImpl(modules);
123     }
124
125     private Map<String, TreeMap<Date, ModuleBuilder>> resolveModuleBuilders(
126             final List<InputStream> yangFileStreams) {
127         final Map<String, TreeMap<Date, ModuleBuilder>> modules = new LinkedHashMap<String, TreeMap<Date, ModuleBuilder>>();
128         final ParseTreeWalker walker = new ParseTreeWalker();
129         final List<ParseTree> trees = parseStreams(yangFileStreams);
130         final ModuleBuilder[] builders = new ModuleBuilder[trees.size()];
131
132         // validate yang
133         new YangModelBasicValidator(walker).validate(trees);
134
135         YangParserListenerImpl yangModelParser = null;
136         for (int i = 0; i < trees.size(); i++) {
137             yangModelParser = new YangParserListenerImpl();
138             walker.walk(yangModelParser, trees.get(i));
139             builders[i] = yangModelParser.getModuleBuilder();
140         }
141
142         // module dependency graph sorted
143         List<ModuleSimple> sorted = new ModuleDependencySort(builders).sort();
144
145         // TODO FIX THIS ASAP!
146         // FIXME this is just temp workaround the ModuleDependencySort MUST
147         // RETURN ordered List
148         // of SORTED and DEPENDECNY RESOLVED MODULE BUILDERS!!!!!!
149         final List<ModuleBuilder> orderedBuilders = new ArrayList<ModuleBuilder>();
150         for (final ModuleSimple ms : sorted) {
151             for (int i = 0; i < builders.length; ++i) {
152                 if (ms.getName().equals(builders[i].getName())
153                         && ms.getRevision().equals(builders[i].getRevision())) {
154                     orderedBuilders.add(builders[i]);
155                 }
156             }
157         }
158         // FIXME END OF WORKAROUND
159
160         for (ModuleBuilder builder : orderedBuilders) {
161             final String builderName = builder.getName();
162             Date builderRevision = builder.getRevision();
163             if (builderRevision == null) {
164                 builderRevision = new Date(0L);
165             }
166             TreeMap<Date, ModuleBuilder> builderByRevision = modules
167                     .get(builderName);
168             if (builderByRevision == null) {
169                 builderByRevision = new TreeMap<Date, ModuleBuilder>();
170             }
171             builderByRevision.put(builderRevision, builder);
172             modules.put(builderName, builderByRevision);
173         }
174         return modules;
175     }
176
177     private List<ParseTree> parseStreams(final List<InputStream> yangStreams) {
178         final List<ParseTree> trees = new ArrayList<ParseTree>();
179         for (InputStream yangStream : yangStreams) {
180             trees.add(parseStream(yangStream));
181         }
182         return trees;
183     }
184
185     private ParseTree parseStream(final InputStream yangStream) {
186         ParseTree result = null;
187         try {
188             final ANTLRInputStream input = new ANTLRInputStream(yangStream);
189             final YangLexer lexer = new YangLexer(input);
190             final CommonTokenStream tokens = new CommonTokenStream(lexer);
191             final YangParser parser = new YangParser(tokens);
192             result = parser.yang();
193         } catch (IOException e) {
194             logger.warn("Exception while reading yang file: " + yangStream, e);
195         }
196         return result;
197     }
198
199     private Set<Module> build(
200             final Map<String, TreeMap<Date, ModuleBuilder>> modules) {
201         // fix unresolved nodes
202         for (Map.Entry<String, TreeMap<Date, ModuleBuilder>> entry : modules
203                 .entrySet()) {
204             for (Map.Entry<Date, ModuleBuilder> childEntry : entry.getValue()
205                     .entrySet()) {
206                 final ModuleBuilder moduleBuilder = childEntry.getValue();
207                 fixUnresolvedNodes(modules, moduleBuilder);
208             }
209         }
210         resolveAugments(modules);
211
212         // build
213         final Set<Module> result = new LinkedHashSet<Module>();
214         for (Map.Entry<String, TreeMap<Date, ModuleBuilder>> entry : modules
215                 .entrySet()) {
216             final Map<Date, Module> modulesByRevision = new HashMap<Date, Module>();
217             for (Map.Entry<Date, ModuleBuilder> childEntry : entry.getValue()
218                     .entrySet()) {
219                 final ModuleBuilder moduleBuilder = childEntry.getValue();
220                 final Module module = moduleBuilder.build();
221                 modulesByRevision.put(childEntry.getKey(), module);
222                 result.add(module);
223             }
224         }
225         return result;
226     }
227
228     private void fixUnresolvedNodes(
229             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
230             final ModuleBuilder builder) {
231         resolveDirtyNodes(modules, builder);
232         resolveIdentities(modules, builder);
233         resolveUses(modules, builder);
234         resolveUnknownNodes(modules, builder);
235     }
236
237     /**
238      * Search for dirty nodes (node which contains UnknownType) and resolve
239      * unknown types.
240      *
241      * @param modules
242      *            all available modules
243      * @param module
244      *            current module
245      */
246     private void resolveDirtyNodes(
247             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
248             final ModuleBuilder module) {
249         final Map<List<String>, TypeAwareBuilder> dirtyNodes = module
250                 .getDirtyNodes();
251         if (!dirtyNodes.isEmpty()) {
252             for (Map.Entry<List<String>, TypeAwareBuilder> entry : dirtyNodes
253                     .entrySet()) {
254
255                 final TypeAwareBuilder nodeToResolve = entry.getValue();
256                 // different handling for union types
257                 if (nodeToResolve instanceof UnionTypeBuilder) {
258                     final UnionTypeBuilder union = (UnionTypeBuilder) nodeToResolve;
259                     final List<TypeDefinition<?>> unionTypes = union.getTypes();
260                     final List<UnknownType> toRemove = new ArrayList<UnknownType>();
261                     for (TypeDefinition<?> td : unionTypes) {
262                         if (td instanceof UnknownType) {
263                             final UnknownType unknownType = (UnknownType) td;
264                             final TypeDefinitionBuilder resolvedType = resolveTypeUnion(
265                                     nodeToResolve, unknownType, modules, module);
266                             union.setType(resolvedType);
267                             toRemove.add(unknownType);
268                         }
269                     }
270                     unionTypes.removeAll(toRemove);
271                 } else if (nodeToResolve.getTypedef() instanceof IdentityrefTypeBuilder) {
272                     IdentityrefTypeBuilder idref = (IdentityrefTypeBuilder) nodeToResolve
273                             .getTypedef();
274                     nodeToResolve.setType(new IdentityrefType(findFullQName(
275                             modules, module, idref), idref.getPath()));
276                 } else {
277                     final TypeDefinitionBuilder resolvedType = resolveType(
278                             nodeToResolve, modules, module);
279                     nodeToResolve.setType(resolvedType);
280                 }
281             }
282         }
283     }
284
285     private TypeDefinitionBuilder resolveType(
286             final TypeAwareBuilder typeToResolve,
287             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
288             final ModuleBuilder builder) {
289         final TypeConstraints constraints = new TypeConstraints();
290
291         final TypeDefinitionBuilder targetType = getTypedefBuilder(
292                 typeToResolve, modules, builder);
293         final TypeConstraints tConstraints = findConstraints(typeToResolve,
294                 constraints, modules, builder);
295         targetType.setRanges(tConstraints.getRange());
296         targetType.setLengths(tConstraints.getLength());
297         targetType.setPatterns(tConstraints.getPatterns());
298         targetType.setFractionDigits(tConstraints.getFractionDigits());
299
300         return targetType;
301     }
302
303     private TypeDefinitionBuilder resolveTypeUnion(
304             final TypeAwareBuilder typeToResolve,
305             final UnknownType unknownType,
306             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
307             final ModuleBuilder builder) {
308         final TypeConstraints constraints = new TypeConstraints();
309
310         final TypeDefinitionBuilder targetType = getUnionBuilder(typeToResolve,
311                 unknownType, modules, builder);
312         final TypeConstraints tConstraints = findConstraints(typeToResolve,
313                 constraints, modules, builder);
314         targetType.setRanges(tConstraints.getRange());
315         targetType.setLengths(tConstraints.getLength());
316         targetType.setPatterns(tConstraints.getPatterns());
317         targetType.setFractionDigits(tConstraints.getFractionDigits());
318
319         return targetType;
320     }
321
322     private TypeDefinitionBuilder getTypedefBuilder(
323             final TypeAwareBuilder nodeToResolve,
324             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
325             final ModuleBuilder builder) {
326
327         final TypeDefinition<?> nodeToResolveBase = nodeToResolve.getType();
328         if (nodeToResolveBase != null
329                 && !(nodeToResolveBase instanceof UnknownType)) {
330             return (TypeDefinitionBuilder) nodeToResolve;
331         }
332
333         final UnknownType unknownType = (UnknownType) nodeToResolve.getType();
334         final QName unknownTypeQName = unknownType.getQName();
335
336         // search for module which contains referenced typedef
337         final ModuleBuilder dependentModule = findDependentModule(modules,
338                 builder, unknownTypeQName.getPrefix(), nodeToResolve.getLine());
339         final TypeDefinitionBuilder lookedUpBuilder = findTypedefBuilderByName(
340                 dependentModule, unknownTypeQName.getLocalName(),
341                 builder.getName(), nodeToResolve.getLine());
342
343         final TypeDefinitionBuilder lookedUpBuilderCopy = copyTypedefBuilder(
344                 lookedUpBuilder, nodeToResolve instanceof TypeDefinitionBuilder);
345         final TypeDefinitionBuilder resolvedCopy = resolveCopiedBuilder(
346                 lookedUpBuilderCopy, modules, dependentModule);
347         return resolvedCopy;
348     }
349
350     private TypeDefinitionBuilder getUnionBuilder(
351             final TypeAwareBuilder nodeToResolve,
352             final UnknownType unknownType,
353             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
354             final ModuleBuilder module) {
355
356         final TypeDefinition<?> baseTypeToResolve = nodeToResolve.getType();
357         if (baseTypeToResolve != null
358                 && !(baseTypeToResolve instanceof UnknownType)) {
359             return (TypeDefinitionBuilder) nodeToResolve;
360         }
361
362         final QName unknownTypeQName = unknownType.getQName();
363         // search for module which contains referenced typedef
364         final ModuleBuilder dependentModule = findDependentModule(modules,
365                 module, unknownTypeQName.getPrefix(), nodeToResolve.getLine());
366         final TypeDefinitionBuilder lookedUpBuilder = findTypedefBuilderByName(
367                 dependentModule, unknownTypeQName.getLocalName(),
368                 module.getName(), nodeToResolve.getLine());
369
370         final TypeDefinitionBuilder lookedUpBuilderCopy = copyTypedefBuilder(
371                 lookedUpBuilder, nodeToResolve instanceof TypeDefinitionBuilder);
372         final TypeDefinitionBuilder resolvedCopy = resolveCopiedBuilder(
373                 lookedUpBuilderCopy, modules, dependentModule);
374         return resolvedCopy;
375     }
376
377     private TypeDefinitionBuilder copyTypedefBuilder(
378             final TypeDefinitionBuilder old, final boolean seekByTypedefBuilder) {
379         if (old instanceof UnionTypeBuilder) {
380             final UnionTypeBuilder oldUnion = (UnionTypeBuilder) old;
381             final UnionTypeBuilder newUnion = new UnionTypeBuilder(
382                     oldUnion.getActualPath(), oldUnion.getNamespace(),
383                     oldUnion.getRevision(), old.getLine());
384             for (TypeDefinition<?> td : oldUnion.getTypes()) {
385                 newUnion.setType(td);
386             }
387             for (TypeDefinitionBuilder tdb : oldUnion.getTypedefs()) {
388                 newUnion.setType(copyTypedefBuilder(tdb, true));
389             }
390             return newUnion;
391         }
392
393         final QName oldName = old.getQName();
394         final QName newName = new QName(oldName.getNamespace(),
395                 oldName.getRevision(), oldName.getPrefix(),
396                 oldName.getLocalName());
397         final TypeDefinitionBuilder tdb = new TypedefBuilder(newName,
398                 old.getLine());
399
400         tdb.setRanges(old.getRanges());
401         tdb.setLengths(old.getLengths());
402         tdb.setPatterns(old.getPatterns());
403         tdb.setFractionDigits(old.getFractionDigits());
404         tdb.setPath(old.getPath());
405
406         final TypeDefinition<?> oldType = old.getType();
407         if (oldType == null) {
408             tdb.setType(old.getTypedef());
409         } else {
410             tdb.setType(oldType);
411         }
412
413         if (!seekByTypedefBuilder) {
414             tdb.setDescription(old.getDescription());
415             tdb.setReference(old.getReference());
416             tdb.setStatus(old.getStatus());
417             tdb.setDefaultValue(old.getDefaultValue());
418             tdb.setUnits(old.getUnits());
419         }
420         return tdb;
421     }
422
423     private TypeDefinitionBuilder resolveCopiedBuilder(
424             final TypeDefinitionBuilder copy,
425             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
426             final ModuleBuilder builder) {
427
428         if (copy instanceof UnionTypeBuilder) {
429             final UnionTypeBuilder union = (UnionTypeBuilder) copy;
430             final List<TypeDefinition<?>> unionTypes = union.getTypes();
431             final List<UnknownType> toRemove = new ArrayList<UnknownType>();
432             for (TypeDefinition<?> td : unionTypes) {
433                 if (td instanceof UnknownType) {
434                     final UnknownType unknownType = (UnknownType) td;
435                     final TypeDefinitionBuilder resolvedType = resolveTypeUnion(
436                             union, unknownType, modules, builder);
437                     union.setType(resolvedType);
438                     toRemove.add(unknownType);
439                 }
440             }
441             unionTypes.removeAll(toRemove);
442
443             return union;
444         }
445
446         final TypeDefinition<?> base = copy.getType();
447         final TypeDefinitionBuilder baseTdb = copy.getTypedef();
448         if (base != null && !(base instanceof UnknownType)) {
449             return copy;
450         } else if (base instanceof UnknownType) {
451             final UnknownType unknownType = (UnknownType) base;
452             final QName unknownTypeQName = unknownType.getQName();
453             final String unknownTypePrefix = unknownTypeQName.getPrefix();
454             final ModuleBuilder dependentModule = findDependentModule(modules,
455                     builder, unknownTypePrefix, copy.getLine());
456             final TypeDefinitionBuilder utBuilder = getTypedefBuilder(copy,
457                     modules, dependentModule);
458             copy.setType(utBuilder);
459             return copy;
460         } else if (base == null && baseTdb != null) {
461             // make a copy of baseTypeDef and call again
462             final TypeDefinitionBuilder baseTdbCopy = copyTypedefBuilder(
463                     baseTdb, true);
464             final TypeDefinitionBuilder baseTdbCopyResolved = resolveCopiedBuilder(
465                     baseTdbCopy, modules, builder);
466             copy.setType(baseTdbCopyResolved);
467             return copy;
468         } else {
469             throw new IllegalStateException("Failed to resolve type "
470                     + copy.getQName().getLocalName());
471         }
472     }
473
474     private TypeDefinitionBuilder findTypedefBuilder(
475             final QName unknownTypeQName,
476             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
477             final ModuleBuilder builder, int line) {
478         // search for module which contains referenced typedef
479         final ModuleBuilder dependentModule = findDependentModule(modules,
480                 builder, unknownTypeQName.getPrefix(), line);
481         final TypeDefinitionBuilder lookedUpBuilder = findTypedefBuilderByName(
482                 dependentModule, unknownTypeQName.getLocalName(),
483                 builder.getName(), line);
484         return copyTypedefBuilder(lookedUpBuilder, true);
485     }
486
487     private TypeConstraints findConstraints(
488             final TypeAwareBuilder nodeToResolve,
489             final TypeConstraints constraints,
490             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
491             final ModuleBuilder builder) {
492         // union type cannot be restricted
493         if (nodeToResolve instanceof UnionTypeBuilder) {
494             return constraints;
495         }
496
497         // if referenced type is UnknownType again, search recursively with
498         // current constraints
499         final TypeDefinition<?> referencedType = nodeToResolve.getType();
500         List<RangeConstraint> ranges = Collections.emptyList();
501         List<LengthConstraint> lengths = Collections.emptyList();
502         List<PatternConstraint> patterns = Collections.emptyList();
503         Integer fractionDigits = null;
504         if (referencedType == null) {
505             final TypeDefinitionBuilder tdb = nodeToResolve.getTypedef();
506             ranges = tdb.getRanges();
507             constraints.addRanges(ranges);
508             lengths = tdb.getLengths();
509             constraints.addLengths(lengths);
510             patterns = tdb.getPatterns();
511             constraints.addPatterns(patterns);
512             fractionDigits = tdb.getFractionDigits();
513             constraints.setFractionDigits(fractionDigits);
514             return constraints;
515         } else if (referencedType instanceof ExtendedType) {
516             final ExtendedType ext = (ExtendedType) referencedType;
517             ranges = ext.getRanges();
518             constraints.addRanges(ranges);
519             lengths = ext.getLengths();
520             constraints.addLengths(lengths);
521             patterns = ext.getPatterns();
522             constraints.addPatterns(patterns);
523             fractionDigits = ext.getFractionDigits();
524             constraints.setFractionDigits(fractionDigits);
525             return findConstraints(
526                     findTypedefBuilder(ext.getQName(), modules, builder,
527                             nodeToResolve.getLine()), constraints, modules,
528                     builder);
529         } else if (referencedType instanceof UnknownType) {
530             final UnknownType unknown = (UnknownType) referencedType;
531             ranges = unknown.getRangeStatements();
532             constraints.addRanges(ranges);
533             lengths = unknown.getLengthStatements();
534             constraints.addLengths(lengths);
535             patterns = unknown.getPatterns();
536             constraints.addPatterns(patterns);
537             fractionDigits = unknown.getFractionDigits();
538             constraints.setFractionDigits(fractionDigits);
539
540             String unknownTypePrefix = unknown.getQName().getPrefix();
541             if (unknownTypePrefix == null || "".equals(unknownTypePrefix)) {
542                 unknownTypePrefix = builder.getPrefix();
543             }
544             final ModuleBuilder dependentModule = findDependentModule(modules,
545                     builder, unknown.getQName().getPrefix(),
546                     nodeToResolve.getLine());
547             final TypeDefinitionBuilder utBuilder = findTypedefBuilder(
548                     unknown.getQName(), modules, builder,
549                     nodeToResolve.getLine());
550             return findConstraints(utBuilder, constraints, modules,
551                     dependentModule);
552         } else {
553             // HANDLE BASE YANG TYPE
554             mergeConstraints(referencedType, constraints);
555             return constraints;
556         }
557     }
558
559     /**
560      * Go through all typedef statements from given module and search for one
561      * with given name
562      *
563      * @param typedefs
564      *            typedef statements to search
565      * @param name
566      *            name of searched typedef
567      * @return typedef with name equals to given name
568      */
569     private TypeDefinitionBuilder findTypedefBuilderByName(
570             final ModuleBuilder dependentModule, final String name,
571             final String currentModuleName, final int line) {
572         TypeDefinitionBuilder result = null;
573         final Set<TypeDefinitionBuilder> typedefs = dependentModule
574                 .getModuleTypedefs();
575         for (TypeDefinitionBuilder td : typedefs) {
576             if (td.getQName().getLocalName().equals(name)) {
577                 result = td;
578                 break;
579             }
580         }
581         if (result == null) {
582             throw new YangParseException(currentModuleName, line,
583                     "Target module '" + dependentModule.getName()
584                             + "' does not contain typedef '" + name + "'.");
585         }
586         return result;
587     }
588
589     /**
590      * Pull restriction from referenced type and add them to given constraints
591      *
592      * @param referencedType
593      * @param constraints
594      */
595     private void mergeConstraints(final TypeDefinition<?> referencedType,
596             final TypeConstraints constraints) {
597
598         if (referencedType instanceof DecimalTypeDefinition) {
599             constraints.addRanges(((DecimalTypeDefinition) referencedType)
600                     .getRangeStatements());
601             constraints
602                     .setFractionDigits(((DecimalTypeDefinition) referencedType)
603                             .getFractionDigits());
604         } else if (referencedType instanceof IntegerTypeDefinition) {
605             constraints.addRanges(((IntegerTypeDefinition) referencedType)
606                     .getRangeStatements());
607         } else if (referencedType instanceof StringTypeDefinition) {
608             constraints.addPatterns(((StringTypeDefinition) referencedType)
609                     .getPatterns());
610             constraints.addLengths(((StringTypeDefinition) referencedType)
611                     .getLengthStatements());
612         } else if (referencedType instanceof BinaryTypeDefinition) {
613             constraints.addLengths(((BinaryTypeDefinition) referencedType)
614                     .getLengthConstraints());
615         }
616     }
617
618     /**
619      * Go through all augmentation definitions and resolve them. This method
620      * also finds referenced node and add child nodes to it.
621      *
622      * @param modules
623      *            all available modules
624      */
625     private void resolveAugments(
626             final Map<String, TreeMap<Date, ModuleBuilder>> modules) {
627         final List<ModuleBuilder> allModulesList = new ArrayList<ModuleBuilder>();
628         final Set<ModuleBuilder> allModulesSet = new HashSet<ModuleBuilder>();
629         for (Map.Entry<String, TreeMap<Date, ModuleBuilder>> entry : modules
630                 .entrySet()) {
631             for (Map.Entry<Date, ModuleBuilder> inner : entry.getValue()
632                     .entrySet()) {
633                 allModulesList.add(inner.getValue());
634                 allModulesSet.add(inner.getValue());
635             }
636         }
637
638         for (int i = 0; i < allModulesList.size(); i++) {
639             final ModuleBuilder module = allModulesList.get(i);
640             // try to resolve augments in module
641             resolveAugment(modules, module);
642             // while all augments are not resolved
643             final Iterator<ModuleBuilder> allModulesIterator = allModulesSet
644                     .iterator();
645             while (!(module.getAugmentsResolved() == module.getAddedAugments()
646                     .size())) {
647                 ModuleBuilder nextModule = null;
648                 // try resolve other module augments
649                 try {
650                     nextModule = allModulesIterator.next();
651                     resolveAugment(modules, nextModule);
652                 } catch (NoSuchElementException e) {
653                     throw new YangParseException(
654                             "Failed to resolve augments in module '"
655                                     + module.getName() + "'.", e);
656                 }
657                 // then try to resolve first module again
658                 resolveAugment(modules, module);
659             }
660         }
661     }
662
663     /**
664      *
665      * @param modules
666      *            all available modules
667      * @param module
668      *            current module
669      */
670     private void resolveAugment(
671             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
672             final ModuleBuilder module) {
673         if (module.getAugmentsResolved() < module.getAddedAugments().size()) {
674             for (AugmentationSchemaBuilder augmentBuilder : module
675                     .getAddedAugments()) {
676
677                 if (!augmentBuilder.isResolved()) {
678                     final SchemaPath augmentTargetSchemaPath = augmentBuilder
679                             .getTargetPath();
680                     final List<QName> path = augmentTargetSchemaPath.getPath();
681
682                     int i = 0;
683                     final QName qname = path.get(i);
684                     String prefix = qname.getPrefix();
685                     if (prefix == null) {
686                         prefix = module.getPrefix();
687                     }
688
689                     DataSchemaNodeBuilder currentParent = null;
690                     final ModuleBuilder dependentModule = findDependentModule(
691                             modules, module, prefix, augmentBuilder.getLine());
692                     for (DataSchemaNodeBuilder child : dependentModule
693                             .getChildNodes()) {
694                         final QName childQName = child.getQName();
695                         if (childQName.getLocalName().equals(
696                                 qname.getLocalName())) {
697                             currentParent = child;
698                             i++;
699                             break;
700                         }
701                     }
702
703                     for (; i < path.size(); i++) {
704                         final QName currentQName = path.get(i);
705                         DataSchemaNodeBuilder newParent = null;
706                         for (DataSchemaNodeBuilder child : ((ChildNodeBuilder) currentParent)
707                                 .getChildNodes()) {
708                             final QName childQName = child.getQName();
709                             if (childQName.getLocalName().equals(
710                                     currentQName.getLocalName())) {
711                                 newParent = child;
712                                 break;
713                             }
714                         }
715                         if (newParent == null) {
716                             break; // node not found, quit search
717                         } else {
718                             currentParent = newParent;
719                         }
720                     }
721
722                     final QName currentQName = currentParent.getQName();
723                     final QName lastAugmentPathElement = path
724                             .get(path.size() - 1);
725                     if (currentQName.getLocalName().equals(
726                             lastAugmentPathElement.getLocalName())) {
727                         ParserUtils.fillAugmentTarget(augmentBuilder,
728                                 (ChildNodeBuilder) currentParent);
729                         ((AugmentationTargetBuilder) currentParent)
730                                 .addAugmentation(augmentBuilder);
731                         SchemaPath oldPath = currentParent.getPath();
732                         augmentBuilder.setTargetPath(new SchemaPath(oldPath
733                                 .getPath(), oldPath.isAbsolute()));
734                         augmentBuilder.setResolved(true);
735                         module.augmentResolved();
736                     }
737                 }
738
739             }
740         }
741     }
742
743     /**
744      * Go through identity statements defined in current module and resolve
745      * their 'base' statement if present.
746      *
747      * @param modules
748      *            all modules
749      * @param module
750      *            module being resolved
751      */
752     private void resolveIdentities(
753             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
754             final ModuleBuilder module) {
755         final Set<IdentitySchemaNodeBuilder> identities = module
756                 .getAddedIdentities();
757         for (IdentitySchemaNodeBuilder identity : identities) {
758             final String baseIdentityName = identity.getBaseIdentityName();
759             if (baseIdentityName != null) {
760                 String baseIdentityPrefix = null;
761                 String baseIdentityLocalName = null;
762                 if (baseIdentityName.contains(":")) {
763                     final String[] splitted = baseIdentityName.split(":");
764                     baseIdentityPrefix = splitted[0];
765                     baseIdentityLocalName = splitted[1];
766                 } else {
767                     baseIdentityPrefix = module.getPrefix();
768                     baseIdentityLocalName = baseIdentityName;
769                 }
770                 final ModuleBuilder dependentModule = findDependentModule(
771                         modules, module, baseIdentityPrefix, identity.getLine());
772
773                 final Set<IdentitySchemaNodeBuilder> dependentModuleIdentities = dependentModule
774                         .getAddedIdentities();
775                 for (IdentitySchemaNodeBuilder idBuilder : dependentModuleIdentities) {
776                     if (idBuilder.getQName().getLocalName()
777                             .equals(baseIdentityLocalName)) {
778                         identity.setBaseIdentity(idBuilder);
779                     }
780                 }
781             }
782         }
783     }
784
785     /**
786      * Go through uses statements defined in current module and resolve their
787      * refine statements.
788      *
789      * @param modules
790      *            all modules
791      * @param module
792      *            module being resolved
793      */
794     private void resolveUses(
795             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
796             final ModuleBuilder module) {
797         final Map<List<String>, UsesNodeBuilder> moduleUses = module
798                 .getAddedUsesNodes();
799         for (Map.Entry<List<String>, UsesNodeBuilder> entry : moduleUses
800                 .entrySet()) {
801             final List<String> key = entry.getKey();
802             final UsesNodeBuilder usesNode = entry.getValue();
803
804             final String groupingName = key.get(key.size() - 1);
805
806             for (RefineHolder refine : usesNode.getRefines()) {
807                 // refine statements
808                 final String defaultStr = refine.getDefaultStr();
809                 final Boolean mandatory = refine.isMandatory();
810                 final MustDefinition must = refine.getMust();
811                 final Boolean presence = refine.isPresence();
812                 final Integer min = refine.getMinElements();
813                 final Integer max = refine.getMaxElements();
814                 final List<UnknownSchemaNodeBuilder> unknownNodes = refine
815                         .getUnknownNodes();
816
817                 Builder refineTarget = getRefineTargetBuilder(groupingName,
818                         refine, modules, module);
819                 if (refineTarget instanceof LeafSchemaNodeBuilder) {
820                     final LeafSchemaNodeBuilder leaf = (LeafSchemaNodeBuilder) refineTarget;
821                     if (defaultStr != null && !("".equals(defaultStr))) {
822                         leaf.setDefaultStr(defaultStr);
823                     }
824                     if (mandatory != null) {
825                         leaf.getConstraints().setMandatory(mandatory);
826                     }
827                     if (must != null) {
828                         leaf.getConstraints().addMustDefinition(must);
829                     }
830                     if (unknownNodes != null) {
831                         for (UnknownSchemaNodeBuilder unknown : unknownNodes) {
832                             leaf.addUnknownSchemaNode(unknown);
833                         }
834                     }
835                     usesNode.addRefineNode(leaf);
836                 } else if (refineTarget instanceof ContainerSchemaNodeBuilder) {
837                     final ContainerSchemaNodeBuilder container = (ContainerSchemaNodeBuilder) refineTarget;
838                     if (presence != null) {
839                         container.setPresence(presence);
840                     }
841                     if (must != null) {
842                         container.getConstraints().addMustDefinition(must);
843                     }
844                     if (unknownNodes != null) {
845                         for (UnknownSchemaNodeBuilder unknown : unknownNodes) {
846                             container.addUnknownSchemaNode(unknown);
847                         }
848                     }
849                     usesNode.addRefineNode(container);
850                 } else if (refineTarget instanceof ListSchemaNodeBuilder) {
851                     final ListSchemaNodeBuilder list = (ListSchemaNodeBuilder) refineTarget;
852                     if (must != null) {
853                         list.getConstraints().addMustDefinition(must);
854                     }
855                     if (min != null) {
856                         list.getConstraints().setMinElements(min);
857                     }
858                     if (max != null) {
859                         list.getConstraints().setMaxElements(max);
860                     }
861                     if (unknownNodes != null) {
862                         for (UnknownSchemaNodeBuilder unknown : unknownNodes) {
863                             list.addUnknownSchemaNode(unknown);
864                         }
865                     }
866                 } else if (refineTarget instanceof LeafListSchemaNodeBuilder) {
867                     final LeafListSchemaNodeBuilder leafList = (LeafListSchemaNodeBuilder) getRefineTargetBuilder(
868                             groupingName, refine, modules, module);
869                     if (must != null) {
870                         leafList.getConstraints().addMustDefinition(must);
871                     }
872                     if (min != null) {
873                         leafList.getConstraints().setMinElements(min);
874                     }
875                     if (max != null) {
876                         leafList.getConstraints().setMaxElements(max);
877                     }
878                     if (unknownNodes != null) {
879                         for (UnknownSchemaNodeBuilder unknown : unknownNodes) {
880                             leafList.addUnknownSchemaNode(unknown);
881                         }
882                     }
883                 } else if (refineTarget instanceof ChoiceBuilder) {
884                     final ChoiceBuilder choice = (ChoiceBuilder) refineTarget;
885                     if (defaultStr != null) {
886                         choice.setDefaultCase(defaultStr);
887                     }
888                     if (mandatory != null) {
889                         choice.getConstraints().setMandatory(mandatory);
890                     }
891                     if (unknownNodes != null) {
892                         for (UnknownSchemaNodeBuilder unknown : unknownNodes) {
893                             choice.addUnknownSchemaNode(unknown);
894                         }
895                     }
896                 } else if (refineTarget instanceof AnyXmlBuilder) {
897                     final AnyXmlBuilder anyXml = (AnyXmlBuilder) refineTarget;
898                     if (mandatory != null) {
899                         anyXml.getConstraints().setMandatory(mandatory);
900                     }
901                     if (must != null) {
902                         anyXml.getConstraints().addMustDefinition(must);
903                     }
904                     if (unknownNodes != null) {
905                         for (UnknownSchemaNodeBuilder unknown : unknownNodes) {
906                             anyXml.addUnknownSchemaNode(unknown);
907                         }
908                     }
909                 }
910             }
911         }
912     }
913
914     /**
915      * Find original builder of refine node and return copy of this builder.
916      *
917      * @param groupingPath
918      *            path to grouping which contains node to refine
919      * @param refine
920      *            refine object containing informations about refine
921      * @param modules
922      *            all loaded modules
923      * @param module
924      *            current module
925      * @return copy of Builder object of node to be refined if it is present in
926      *         grouping, null otherwise
927      */
928     private Builder getRefineTargetBuilder(final String groupingPath,
929             final RefineHolder refine,
930             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
931             final ModuleBuilder module) {
932         Builder result = null;
933         final Builder lookedUpBuilder = findRefineTargetBuilder(groupingPath,
934                 refine, modules, module);
935         if (lookedUpBuilder instanceof LeafSchemaNodeBuilder) {
936             result = ParserUtils
937                     .copyLeafBuilder((LeafSchemaNodeBuilder) lookedUpBuilder);
938         } else if (lookedUpBuilder instanceof ContainerSchemaNodeBuilder) {
939             result = ParserUtils
940                     .copyContainerBuilder((ContainerSchemaNodeBuilder) lookedUpBuilder);
941         } else if (lookedUpBuilder instanceof ListSchemaNodeBuilder) {
942             result = ParserUtils
943                     .copyListBuilder((ListSchemaNodeBuilder) lookedUpBuilder);
944         } else if (lookedUpBuilder instanceof LeafListSchemaNodeBuilder) {
945             result = ParserUtils
946                     .copyLeafListBuilder((LeafListSchemaNodeBuilder) lookedUpBuilder);
947         } else if (lookedUpBuilder instanceof ChoiceBuilder) {
948             result = ParserUtils
949                     .copyChoiceBuilder((ChoiceBuilder) lookedUpBuilder);
950         } else if (lookedUpBuilder instanceof AnyXmlBuilder) {
951             result = ParserUtils
952                     .copyAnyXmlBuilder((AnyXmlBuilder) lookedUpBuilder);
953         } else {
954             throw new YangParseException(module.getName(), refine.getLine(),
955                     "Target '" + refine.getName() + "' can not be refined");
956         }
957         return result;
958     }
959
960     /**
961      * Find builder of refine node.
962      *
963      * @param groupingPath
964      *            path to grouping which contains node to refine
965      * @param refineNodeName
966      *            name of node to be refined
967      * @param modules
968      *            all loaded modules
969      * @param module
970      *            current module
971      * @return Builder object of refine node if it is present in grouping, null
972      *         otherwise
973      */
974     private Builder findRefineTargetBuilder(final String groupingPath,
975             final RefineHolder refine,
976             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
977             final ModuleBuilder module) {
978         final SchemaPath path = ParserUtils.parseUsesPath(groupingPath);
979         final List<String> builderPath = new ArrayList<String>();
980         String prefix = null;
981         for (QName qname : path.getPath()) {
982             builderPath.add(qname.getLocalName());
983             prefix = qname.getPrefix();
984         }
985         if (prefix == null) {
986             prefix = module.getPrefix();
987         }
988
989         final ModuleBuilder dependentModule = findDependentModule(modules,
990                 module, prefix, refine.getLine());
991         builderPath.add(0, "grouping");
992         builderPath.add(0, dependentModule.getName());
993         final GroupingBuilder builder = (GroupingBuilder) dependentModule
994                 .getNode(builderPath);
995
996         return builder.getChildNode(refine.getName());
997     }
998
999     private QName findFullQName(
1000             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
1001             final ModuleBuilder module, final IdentityrefTypeBuilder idref) {
1002         QName result = null;
1003         String baseString = idref.getBaseString();
1004         if (baseString.contains(":")) {
1005             String[] splittedBase = baseString.split(":");
1006             if (splittedBase.length > 2) {
1007                 throw new YangParseException(module.getName(), idref.getLine(),
1008                         "Failed to parse identityref base: " + baseString);
1009             }
1010             String prefix = splittedBase[0];
1011             String name = splittedBase[1];
1012             ModuleBuilder dependentModule = findDependentModule(modules,
1013                     module, prefix, idref.getLine());
1014             result = new QName(dependentModule.getNamespace(),
1015                     dependentModule.getRevision(), prefix, name);
1016         } else {
1017             result = new QName(module.getNamespace(), module.getRevision(),
1018                     module.getPrefix(), baseString);
1019         }
1020         return result;
1021     }
1022
1023     private void resolveUnknownNodes(
1024             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
1025             final ModuleBuilder module) {
1026         for (UnknownSchemaNodeBuilder usnb : module.getAddedUnknownNodes()) {
1027             QName nodeType = usnb.getNodeType();
1028             if (nodeType.getNamespace() == null
1029                     || nodeType.getRevision() == null) {
1030                 try {
1031                     ModuleBuilder dependentModule = findDependentModule(
1032                             modules, module, nodeType.getPrefix(),
1033                             usnb.getLine());
1034                     QName newNodeType = new QName(
1035                             dependentModule.getNamespace(),
1036                             dependentModule.getRevision(),
1037                             nodeType.getPrefix(), nodeType.getLocalName());
1038                     usnb.setNodeType(newNodeType);
1039                 } catch (YangParseException e) {
1040                     logger.debug(module.getName(), usnb.getLine(),
1041                             "Failed to find unknown node type: " + nodeType);
1042                 }
1043             }
1044         }
1045     }
1046
1047     /**
1048      * Find dependent module based on given prefix
1049      *
1050      * @param modules
1051      *            all available modules
1052      * @param module
1053      *            current module
1054      * @param prefix
1055      *            target module prefix
1056      * @return
1057      */
1058     private ModuleBuilder findDependentModule(
1059             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
1060             final ModuleBuilder module, final String prefix, final int line) {
1061         ModuleBuilder dependentModule = null;
1062         Date dependentModuleRevision = null;
1063
1064         if (prefix.equals(module.getPrefix())) {
1065             dependentModule = module;
1066         } else {
1067             final ModuleImport dependentModuleImport = ParserUtils
1068                     .getModuleImport(module, prefix);
1069             if (dependentModuleImport == null) {
1070                 throw new YangParseException(module.getName(), line,
1071                         "No import found with prefix '" + prefix + "'.");
1072             }
1073             final String dependentModuleName = dependentModuleImport
1074                     .getModuleName();
1075             dependentModuleRevision = dependentModuleImport.getRevision();
1076
1077             final TreeMap<Date, ModuleBuilder> moduleBuildersByRevision = modules
1078                     .get(dependentModuleName);
1079             if (moduleBuildersByRevision == null) {
1080                 throw new YangParseException(module.getName(), line,
1081                         "Failed to find dependent module '"
1082                                 + dependentModuleName + "'.");
1083             }
1084             if (dependentModuleRevision == null) {
1085                 dependentModule = moduleBuildersByRevision.lastEntry()
1086                         .getValue();
1087             } else {
1088                 dependentModule = moduleBuildersByRevision
1089                         .get(dependentModuleRevision);
1090             }
1091         }
1092
1093         if (dependentModule == null) {
1094             throw new YangParseException(module.getName(), line,
1095                     "Failed to find dependent module with prefix '" + prefix
1096                             + "' and revision '" + dependentModuleRevision
1097                             + "'.");
1098         }
1099         return dependentModule;
1100     }
1101
1102     private static class SchemaContextImpl implements SchemaContext {
1103         private final Set<Module> modules;
1104
1105         private SchemaContextImpl(final Set<Module> modules) {
1106             this.modules = modules;
1107         }
1108
1109         @Override
1110         public Set<DataSchemaNode> getDataDefinitions() {
1111             final Set<DataSchemaNode> dataDefs = new HashSet<DataSchemaNode>();
1112             for (Module m : modules) {
1113                 dataDefs.addAll(m.getChildNodes());
1114             }
1115             return dataDefs;
1116         }
1117
1118         @Override
1119         public Set<Module> getModules() {
1120             return modules;
1121         }
1122
1123         @Override
1124         public Set<NotificationDefinition> getNotifications() {
1125             final Set<NotificationDefinition> notifications = new HashSet<NotificationDefinition>();
1126             for (Module m : modules) {
1127                 notifications.addAll(m.getNotifications());
1128             }
1129             return notifications;
1130         }
1131
1132         @Override
1133         public Set<RpcDefinition> getOperations() {
1134             final Set<RpcDefinition> rpcs = new HashSet<RpcDefinition>();
1135             for (Module m : modules) {
1136                 rpcs.addAll(m.getRpcs());
1137             }
1138             return rpcs;
1139         }
1140
1141         @Override
1142         public Set<ExtensionDefinition> getExtensions() {
1143             final Set<ExtensionDefinition> extensions = new HashSet<ExtensionDefinition>();
1144             for (Module m : modules) {
1145                 extensions.addAll(m.getExtensionSchemaNodes());
1146             }
1147             return extensions;
1148         }
1149
1150         @Override
1151         public Module findModuleByName(final String name, final Date revision) {
1152             if (name != null) {
1153                 for (final Module module : modules) {
1154                     if (revision == null) {
1155                         if (module.getName().equals(name)) {
1156                             return module;
1157                         }
1158                     } else if (module.getName().equals(name)
1159                             && module.getRevision().equals(revision)) {
1160                         return module;
1161                     }
1162                 }
1163             }
1164             return null;
1165         }
1166
1167         @Override
1168         public Module findModuleByNamespace(final URI namespace) {
1169             if (namespace != null) {
1170                 for (final Module module : modules) {
1171                     if (module.getNamespace().equals(namespace)) {
1172                         return module;
1173                     }
1174                 }
1175             }
1176             return null;
1177         }
1178     }
1179
1180 }