Modified construction of built-in yang types.
[controller.git] / opendaylight / sal / yang-prototype / code-generator / yang-model-parser-impl / src / main / java / org / opendaylight / controller / yang / parser / impl / YangParserImpl.java
1 /*
2  * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
3  *
4  * This program and the accompanying materials are made available under the
5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6  * and is available at http://www.eclipse.org/legal/epl-v10.html
7  */
8 package org.opendaylight.controller.yang.parser.impl;
9
10 import java.io.File;
11 import java.io.FileInputStream;
12 import java.io.FileNotFoundException;
13 import java.io.IOException;
14 import java.io.InputStream;
15 import java.util.ArrayList;
16 import java.util.Collections;
17 import java.util.Date;
18 import java.util.HashMap;
19 import java.util.HashSet;
20 import java.util.Iterator;
21 import java.util.LinkedHashMap;
22 import java.util.List;
23 import java.util.Map;
24 import java.util.Map.Entry;
25 import java.util.NoSuchElementException;
26 import java.util.Set;
27 import java.util.TreeMap;
28
29 import org.antlr.v4.runtime.ANTLRInputStream;
30 import org.antlr.v4.runtime.CommonTokenStream;
31 import org.antlr.v4.runtime.tree.ParseTree;
32 import org.antlr.v4.runtime.tree.ParseTreeWalker;
33 import org.opendaylight.controller.antlrv4.code.gen.YangLexer;
34 import org.opendaylight.controller.antlrv4.code.gen.YangParser;
35 import org.opendaylight.controller.yang.common.QName;
36 import org.opendaylight.controller.yang.model.api.Module;
37 import org.opendaylight.controller.yang.model.api.ModuleImport;
38 import org.opendaylight.controller.yang.model.api.SchemaContext;
39 import org.opendaylight.controller.yang.model.api.SchemaPath;
40 import org.opendaylight.controller.yang.model.api.TypeDefinition;
41 import org.opendaylight.controller.yang.model.api.type.BinaryTypeDefinition;
42 import org.opendaylight.controller.yang.model.api.type.DecimalTypeDefinition;
43 import org.opendaylight.controller.yang.model.api.type.IntegerTypeDefinition;
44 import org.opendaylight.controller.yang.model.api.type.LengthConstraint;
45 import org.opendaylight.controller.yang.model.api.type.PatternConstraint;
46 import org.opendaylight.controller.yang.model.api.type.RangeConstraint;
47 import org.opendaylight.controller.yang.model.api.type.StringTypeDefinition;
48 import org.opendaylight.controller.yang.model.parser.api.YangModelParser;
49 import org.opendaylight.controller.yang.model.util.ExtendedType;
50 import org.opendaylight.controller.yang.model.util.IdentityrefType;
51 import org.opendaylight.controller.yang.model.util.UnknownType;
52 import org.opendaylight.controller.yang.model.util.YangTypesConverter;
53 import org.opendaylight.controller.yang.parser.builder.api.AugmentationSchemaBuilder;
54 import org.opendaylight.controller.yang.parser.builder.api.AugmentationTargetBuilder;
55 import org.opendaylight.controller.yang.parser.builder.api.Builder;
56 import org.opendaylight.controller.yang.parser.builder.api.DataNodeContainerBuilder;
57 import org.opendaylight.controller.yang.parser.builder.api.DataSchemaNodeBuilder;
58 import org.opendaylight.controller.yang.parser.builder.api.GroupingBuilder;
59 import org.opendaylight.controller.yang.parser.builder.api.SchemaNodeBuilder;
60 import org.opendaylight.controller.yang.parser.builder.api.TypeAwareBuilder;
61 import org.opendaylight.controller.yang.parser.builder.api.TypeDefinitionBuilder;
62 import org.opendaylight.controller.yang.parser.builder.api.UsesNodeBuilder;
63 import org.opendaylight.controller.yang.parser.builder.impl.AnyXmlBuilder;
64 import org.opendaylight.controller.yang.parser.builder.impl.ChoiceBuilder;
65 import org.opendaylight.controller.yang.parser.builder.impl.ContainerSchemaNodeBuilder;
66 import org.opendaylight.controller.yang.parser.builder.impl.IdentitySchemaNodeBuilder;
67 import org.opendaylight.controller.yang.parser.builder.impl.IdentityrefTypeBuilder;
68 import org.opendaylight.controller.yang.parser.builder.impl.LeafListSchemaNodeBuilder;
69 import org.opendaylight.controller.yang.parser.builder.impl.LeafSchemaNodeBuilder;
70 import org.opendaylight.controller.yang.parser.builder.impl.ListSchemaNodeBuilder;
71 import org.opendaylight.controller.yang.parser.builder.impl.ModuleBuilder;
72 import org.opendaylight.controller.yang.parser.builder.impl.RpcDefinitionBuilder;
73 import org.opendaylight.controller.yang.parser.builder.impl.TypeDefinitionBuilderImpl;
74 import org.opendaylight.controller.yang.parser.builder.impl.UnionTypeBuilder;
75 import org.opendaylight.controller.yang.parser.builder.impl.UnknownSchemaNodeBuilder;
76 import org.opendaylight.controller.yang.parser.util.ModuleDependencySort;
77 import org.opendaylight.controller.yang.parser.util.ParserUtils;
78 import org.opendaylight.controller.yang.parser.util.RefineHolder;
79 import org.opendaylight.controller.yang.parser.util.TypeConstraints;
80 import org.opendaylight.controller.yang.parser.util.YangParseException;
81 import org.opendaylight.controller.yang.validator.YangModelBasicValidator;
82 import org.slf4j.Logger;
83 import org.slf4j.LoggerFactory;
84
85 import com.google.common.collect.Lists;
86 import com.google.common.collect.Maps;
87 import com.google.common.collect.Sets;
88
89 public final class YangParserImpl implements YangModelParser {
90
91     private static final Logger logger = LoggerFactory
92             .getLogger(YangParserImpl.class);
93
94     @Override
95     public Map<File, Module> parseYangModelsMapped(List<File> yangFiles) {
96         if (yangFiles != null) {
97             final Map<InputStream, File> inputStreams = Maps.newHashMap();
98
99             for (final File yangFile : yangFiles) {
100                 try {
101                     inputStreams.put(new FileInputStream(yangFile), yangFile);
102                 } catch (FileNotFoundException e) {
103                     logger.warn("Exception while reading yang file: "
104                             + yangFile.getName(), e);
105                 }
106             }
107
108             Map<ModuleBuilder, InputStream> builderToStreamMap = Maps
109                     .newHashMap();
110
111             final Map<String, TreeMap<Date, ModuleBuilder>> modules = resolveModuleBuilders(
112                     Lists.newArrayList(inputStreams.keySet()),
113                     builderToStreamMap);
114             // return new LinkedHashSet<Module>(build(modules).values());
115
116             Map<File, Module> retVal = Maps.newLinkedHashMap();
117             Map<ModuleBuilder, Module> builderToModuleMap = build(modules);
118
119             for (Entry<ModuleBuilder, Module> builderToModule : builderToModuleMap
120                     .entrySet()) {
121                 retVal.put(inputStreams.get(builderToStreamMap
122                         .get(builderToModule.getKey())), builderToModule
123                         .getValue());
124             }
125
126             return retVal;
127         }
128         return Collections.emptyMap();
129     }
130
131     @Override
132     public Set<Module> parseYangModels(final List<File> yangFiles) {
133         return Sets.newLinkedHashSet(parseYangModelsMapped(yangFiles).values());
134     }
135
136     @Override
137     public Set<Module> parseYangModelsFromStreams(
138             final List<InputStream> yangModelStreams) {
139         return Sets.newHashSet(parseYangModelsFromStreamsMapped(
140                 yangModelStreams).values());
141     }
142
143     @Override
144     public Map<InputStream, Module> parseYangModelsFromStreamsMapped(
145             final List<InputStream> yangModelStreams) {
146         Map<ModuleBuilder, InputStream> builderToStreamMap = Maps.newHashMap();
147
148         final Map<String, TreeMap<Date, ModuleBuilder>> modules = resolveModuleBuilders(
149                 yangModelStreams, builderToStreamMap);
150         Map<InputStream, Module> retVal = Maps.newLinkedHashMap();
151         Map<ModuleBuilder, Module> builderToModuleMap = build(modules);
152
153         for (Entry<ModuleBuilder, Module> builderToModule : builderToModuleMap
154                 .entrySet()) {
155             retVal.put(builderToStreamMap.get(builderToModule.getKey()),
156                     builderToModule.getValue());
157         }
158         return retVal;
159     }
160
161     @Override
162     public SchemaContext resolveSchemaContext(final Set<Module> modules) {
163         return new SchemaContextImpl(modules);
164     }
165
166     private ModuleBuilder[] parseModuleBuilders(List<InputStream> inputStreams,
167             Map<ModuleBuilder, InputStream> streamToBuilderMap) {
168
169         final ParseTreeWalker walker = new ParseTreeWalker();
170         final List<ParseTree> trees = parseStreams(inputStreams);
171         final ModuleBuilder[] builders = new ModuleBuilder[trees.size()];
172
173         // validate yang
174         new YangModelBasicValidator(walker).validate(trees);
175
176         YangParserListenerImpl yangModelParser = null;
177         for (int i = 0; i < trees.size(); i++) {
178             yangModelParser = new YangParserListenerImpl();
179             walker.walk(yangModelParser, trees.get(i));
180             ModuleBuilder moduleBuilder = yangModelParser.getModuleBuilder();
181
182             // We expect the order of trees and streams has to be the same
183             streamToBuilderMap.put(moduleBuilder, inputStreams.get(i));
184             builders[i] = moduleBuilder;
185         }
186         return builders;
187     }
188
189     private Map<String, TreeMap<Date, ModuleBuilder>> resolveModuleBuilders(
190             final List<InputStream> yangFileStreams,
191             Map<ModuleBuilder, InputStream> streamToBuilderMap) {
192
193         final ModuleBuilder[] builders = parseModuleBuilders(yangFileStreams,
194                 streamToBuilderMap);
195
196         // Linked Hash Map MUST be used because Linked Hash Map preserves ORDER
197         // of items stored in map.
198         final LinkedHashMap<String, TreeMap<Date, ModuleBuilder>> modules = new LinkedHashMap<String, TreeMap<Date, ModuleBuilder>>();
199
200         // module dependency graph sorted
201         List<ModuleBuilder> sorted = ModuleDependencySort.sort(builders);
202
203         for (ModuleBuilder builder : sorted) {
204             final String builderName = builder.getName();
205             Date builderRevision = builder.getRevision();
206             if (builderRevision == null) {
207                 builderRevision = new Date(0L);
208             }
209             TreeMap<Date, ModuleBuilder> builderByRevision = modules
210                     .get(builderName);
211             if (builderByRevision == null) {
212                 builderByRevision = new TreeMap<Date, ModuleBuilder>();
213             }
214             builderByRevision.put(builderRevision, builder);
215             modules.put(builderName, builderByRevision);
216         }
217         return modules;
218     }
219
220     private List<ParseTree> parseStreams(final List<InputStream> yangStreams) {
221         final List<ParseTree> trees = new ArrayList<ParseTree>();
222         for (InputStream yangStream : yangStreams) {
223             trees.add(parseStream(yangStream));
224         }
225         return trees;
226     }
227
228     private ParseTree parseStream(final InputStream yangStream) {
229         ParseTree result = null;
230         try {
231             final ANTLRInputStream input = new ANTLRInputStream(yangStream);
232             final YangLexer lexer = new YangLexer(input);
233             final CommonTokenStream tokens = new CommonTokenStream(lexer);
234             final YangParser parser = new YangParser(tokens);
235             result = parser.yang();
236         } catch (IOException e) {
237             logger.warn("Exception while reading yang file: " + yangStream, e);
238         }
239         return result;
240     }
241
242     private Map<ModuleBuilder, Module> build(
243             final Map<String, TreeMap<Date, ModuleBuilder>> modules) {
244         // fix unresolved nodes
245         for (Map.Entry<String, TreeMap<Date, ModuleBuilder>> entry : modules
246                 .entrySet()) {
247             for (Map.Entry<Date, ModuleBuilder> childEntry : entry.getValue()
248                     .entrySet()) {
249                 final ModuleBuilder moduleBuilder = childEntry.getValue();
250                 fixUnresolvedNodes(modules, moduleBuilder);
251             }
252         }
253         resolveAugments(modules);
254
255         // build
256         // LinkedHashMap MUST be used otherwise the values will not maintain
257         // order!
258         // http://docs.oracle.com/javase/6/docs/api/java/util/LinkedHashMap.html
259         final Map<ModuleBuilder, Module> result = new LinkedHashMap<ModuleBuilder, Module>();
260         for (Map.Entry<String, TreeMap<Date, ModuleBuilder>> entry : modules
261                 .entrySet()) {
262             final Map<Date, Module> modulesByRevision = new HashMap<Date, Module>();
263             for (Map.Entry<Date, ModuleBuilder> childEntry : entry.getValue()
264                     .entrySet()) {
265                 final ModuleBuilder moduleBuilder = childEntry.getValue();
266                 final Module module = moduleBuilder.build();
267                 modulesByRevision.put(childEntry.getKey(), module);
268                 result.put(moduleBuilder, module);
269             }
270         }
271         return result;
272     }
273
274     private void fixUnresolvedNodes(
275             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
276             final ModuleBuilder builder) {
277         resolveDirtyNodes(modules, builder);
278         resolveIdentities(modules, builder);
279         resolveUsesRefines(modules, builder);
280         resolveUnknownNodes(modules, builder);
281     }
282
283     /**
284      * Search for dirty nodes (node which contains UnknownType) and resolve
285      * unknown types.
286      *
287      * @param modules
288      *            all available modules
289      * @param module
290      *            current module
291      */
292     private void resolveDirtyNodes(
293             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
294             final ModuleBuilder module) {
295         final Map<List<String>, TypeAwareBuilder> dirtyNodes = module
296                 .getDirtyNodes();
297         if (!dirtyNodes.isEmpty()) {
298             for (Map.Entry<List<String>, TypeAwareBuilder> entry : dirtyNodes
299                     .entrySet()) {
300
301                 final TypeAwareBuilder nodeToResolve = entry.getValue();
302                 // different handling for union types
303                 if (nodeToResolve instanceof UnionTypeBuilder) {
304                     final UnionTypeBuilder union = (UnionTypeBuilder) nodeToResolve;
305                     final List<TypeDefinition<?>> unionTypes = union.getTypes();
306                     final List<UnknownType> toRemove = new ArrayList<UnknownType>();
307                     for (TypeDefinition<?> td : unionTypes) {
308                         if (td instanceof UnknownType) {
309                             final UnknownType unknownType = (UnknownType) td;
310                             final TypeDefinitionBuilder resolvedType = resolveTypeUnion(
311                                     nodeToResolve, unknownType, modules, module);
312                             union.setTypedef(resolvedType);
313                             toRemove.add(unknownType);
314                         }
315                     }
316                     unionTypes.removeAll(toRemove);
317                 } else if (nodeToResolve.getTypedef() instanceof IdentityrefTypeBuilder) {
318                     // different handling for identityref types
319                     IdentityrefTypeBuilder idref = (IdentityrefTypeBuilder) nodeToResolve
320                             .getTypedef();
321                     nodeToResolve.setType(new IdentityrefType(findFullQName(
322                             modules, module, idref), idref.getPath()));
323                 } else {
324                     final TypeDefinitionBuilder resolvedType = resolveType(
325                             nodeToResolve, modules, module);
326                     nodeToResolve.setTypedef(resolvedType);
327                 }
328             }
329         }
330     }
331
332     private TypeDefinitionBuilder resolveType(
333             final TypeAwareBuilder nodeToResolve,
334             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
335             final ModuleBuilder builder) {
336         final TypeConstraints constraints = new TypeConstraints();
337
338         final TypeDefinitionBuilder targetTypeBuilder = getTypeDefinitionBuilderFromDirtyNode(
339                 nodeToResolve, modules, builder);
340         final TypeConstraints tConstraints = findConstraints(nodeToResolve,
341                 constraints, modules, builder);
342         targetTypeBuilder.setRanges(tConstraints.getRange());
343         targetTypeBuilder.setLengths(tConstraints.getLength());
344         targetTypeBuilder.setPatterns(tConstraints.getPatterns());
345         targetTypeBuilder.setFractionDigits(tConstraints.getFractionDigits());
346
347         return targetTypeBuilder;
348     }
349
350     private TypeDefinitionBuilder resolveTypeUnion(
351             final TypeAwareBuilder typeToResolve,
352             final UnknownType unknownType,
353             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
354             final ModuleBuilder builder) {
355         final TypeConstraints constraints = new TypeConstraints();
356
357         final TypeDefinitionBuilder targetTypeBuilder = getUnionBuilder(
358                 typeToResolve, unknownType, modules, builder);
359         final TypeConstraints tConstraints = findConstraints(typeToResolve,
360                 constraints, modules, builder);
361         targetTypeBuilder.setRanges(tConstraints.getRange());
362         targetTypeBuilder.setLengths(tConstraints.getLength());
363         targetTypeBuilder.setPatterns(tConstraints.getPatterns());
364         targetTypeBuilder.setFractionDigits(tConstraints.getFractionDigits());
365
366         return targetTypeBuilder;
367     }
368
369     private TypeDefinitionBuilder getTypeDefinitionBuilderFromDirtyNode(
370             final TypeAwareBuilder nodeToResolve,
371             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
372             final ModuleBuilder module) {
373
374         final UnknownType unknownType = (UnknownType) nodeToResolve.getType();
375         final QName unknownTypeQName = unknownType.getQName();
376
377         // search for module which contains referenced typedef
378         final ModuleBuilder dependentModule = findDependentModule(modules,
379                 module, unknownTypeQName.getPrefix(), nodeToResolve.getLine());
380
381         final TypeDefinitionBuilder lookedUpBuilder = findTypeDefinitionBuilder(
382                 nodeToResolve.getPath(), dependentModule,
383                 unknownTypeQName.getLocalName(), module.getName(),
384                 nodeToResolve.getLine());
385
386         final TypeDefinitionBuilder lookedUpBuilderCopy = copyTypedefBuilder(
387                 lookedUpBuilder, nodeToResolve instanceof TypeDefinitionBuilder);
388         final TypeDefinitionBuilder resolvedCopy = resolveCopiedBuilder(
389                 lookedUpBuilderCopy, modules, dependentModule);
390         return resolvedCopy;
391     }
392
393     private TypeDefinitionBuilder getUnionBuilder(
394             final TypeAwareBuilder nodeToResolve,
395             final UnknownType unknownType,
396             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
397             final ModuleBuilder module) {
398
399         final TypeDefinition<?> baseTypeToResolve = nodeToResolve.getType();
400         if (baseTypeToResolve != null
401                 && !(baseTypeToResolve instanceof UnknownType)) {
402             return (TypeDefinitionBuilder) nodeToResolve;
403         }
404
405         final QName unknownTypeQName = unknownType.getQName();
406         // search for module which contains referenced typedef
407         final ModuleBuilder dependentModule = findDependentModule(modules,
408                 module, unknownTypeQName.getPrefix(), nodeToResolve.getLine());
409         final TypeDefinitionBuilder lookedUpBuilder = findTypeDefinitionBuilder(
410                 nodeToResolve.getPath(), dependentModule,
411                 unknownTypeQName.getLocalName(), module.getName(),
412                 nodeToResolve.getLine());
413
414         final TypeDefinitionBuilder lookedUpBuilderCopy = copyTypedefBuilder(
415                 lookedUpBuilder, nodeToResolve instanceof TypeDefinitionBuilder);
416         final TypeDefinitionBuilder resolvedCopy = resolveCopiedBuilder(
417                 lookedUpBuilderCopy, modules, dependentModule);
418         return resolvedCopy;
419     }
420
421     private TypeDefinitionBuilder copyTypedefBuilder(
422             final TypeDefinitionBuilder old, final boolean seekByTypedefBuilder) {
423         if (old instanceof UnionTypeBuilder) {
424             final UnionTypeBuilder oldUnion = (UnionTypeBuilder) old;
425             final UnionTypeBuilder newUnion = new UnionTypeBuilder(
426                     old.getLine());
427             for (TypeDefinition<?> td : oldUnion.getTypes()) {
428                 newUnion.setType(td);
429             }
430             for (TypeDefinitionBuilder tdb : oldUnion.getTypedefs()) {
431                 newUnion.setTypedef(copyTypedefBuilder(tdb, true));
432             }
433             newUnion.setPath(old.getPath());
434             return newUnion;
435         }
436
437         final QName oldName = old.getQName();
438         final QName newName = new QName(oldName.getNamespace(),
439                 oldName.getRevision(), oldName.getPrefix(),
440                 oldName.getLocalName());
441         final TypeDefinitionBuilder tdb = new TypeDefinitionBuilderImpl(
442                 newName, old.getLine());
443
444         tdb.setRanges(old.getRanges());
445         tdb.setLengths(old.getLengths());
446         tdb.setPatterns(old.getPatterns());
447         tdb.setFractionDigits(old.getFractionDigits());
448         tdb.setPath(old.getPath());
449
450         final TypeDefinition<?> oldType = old.getType();
451         if (oldType == null) {
452             tdb.setTypedef(old.getTypedef());
453         } else {
454             tdb.setType(oldType);
455         }
456
457         if (!seekByTypedefBuilder) {
458             tdb.setDescription(old.getDescription());
459             tdb.setReference(old.getReference());
460             tdb.setStatus(old.getStatus());
461             tdb.setDefaultValue(old.getDefaultValue());
462             tdb.setUnits(old.getUnits());
463         }
464         return tdb;
465     }
466
467     private TypeDefinitionBuilder resolveCopiedBuilder(
468             final TypeDefinitionBuilder copy,
469             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
470             final ModuleBuilder builder) {
471
472         if (copy instanceof UnionTypeBuilder) {
473             final UnionTypeBuilder union = (UnionTypeBuilder) copy;
474             final List<TypeDefinition<?>> unionTypes = union.getTypes();
475             final List<UnknownType> toRemove = new ArrayList<UnknownType>();
476             for (TypeDefinition<?> td : unionTypes) {
477                 if (td instanceof UnknownType) {
478                     final UnknownType unknownType = (UnknownType) td;
479                     final TypeDefinitionBuilder resolvedType = resolveTypeUnion(
480                             union, unknownType, modules, builder);
481                     union.setTypedef(resolvedType);
482                     toRemove.add(unknownType);
483                 }
484             }
485             unionTypes.removeAll(toRemove);
486
487             return union;
488         }
489
490         final TypeDefinition<?> base = copy.getType();
491         final TypeDefinitionBuilder baseTdb = copy.getTypedef();
492         if (base != null && !(base instanceof UnknownType)) {
493             return copy;
494         } else if (base instanceof UnknownType) {
495             final UnknownType unknownType = (UnknownType) base;
496             final QName unknownTypeQName = unknownType.getQName();
497             final String unknownTypePrefix = unknownTypeQName.getPrefix();
498             final ModuleBuilder dependentModule = findDependentModule(modules,
499                     builder, unknownTypePrefix, copy.getLine());
500             final TypeDefinitionBuilder utBuilder = getTypeDefinitionBuilderFromDirtyNode(
501                     copy, modules, dependentModule);
502             copy.setTypedef(utBuilder);
503             return copy;
504         } else if (base == null && baseTdb != null) {
505             // make a copy of baseTypeDef and call again
506             final TypeDefinitionBuilder baseTdbCopy = copyTypedefBuilder(
507                     baseTdb, true);
508             final TypeDefinitionBuilder baseTdbCopyResolved = resolveCopiedBuilder(
509                     baseTdbCopy, modules, builder);
510             copy.setTypedef(baseTdbCopyResolved);
511             return copy;
512         } else {
513             throw new YangParseException(copy.getLine(),
514                     "Failed to resolve type " + copy.getQName().getLocalName());
515         }
516     }
517
518     private TypeConstraints findConstraints(
519             final TypeAwareBuilder nodeToResolve,
520             final TypeConstraints constraints,
521             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
522             final ModuleBuilder builder) {
523         // union type cannot be restricted
524         if (nodeToResolve instanceof UnionTypeBuilder) {
525             return constraints;
526         }
527
528         // if referenced type is UnknownType again, search recursively with
529         // current constraints
530         final TypeDefinition<?> referencedType = nodeToResolve.getType();
531         List<RangeConstraint> ranges = Collections.emptyList();
532         List<LengthConstraint> lengths = Collections.emptyList();
533         List<PatternConstraint> patterns = Collections.emptyList();
534         Integer fractionDigits = null;
535         if (referencedType == null) {
536             final TypeDefinitionBuilder tdb = nodeToResolve.getTypedef();
537             ranges = tdb.getRanges();
538             constraints.addRanges(ranges);
539             lengths = tdb.getLengths();
540             constraints.addLengths(lengths);
541             patterns = tdb.getPatterns();
542             constraints.addPatterns(patterns);
543             fractionDigits = tdb.getFractionDigits();
544             constraints.setFractionDigits(fractionDigits);
545             return constraints;
546         } else if (referencedType instanceof ExtendedType) {
547             final ExtendedType ext = (ExtendedType) referencedType;
548             ranges = ext.getRanges();
549             constraints.addRanges(ranges);
550             lengths = ext.getLengths();
551             constraints.addLengths(lengths);
552             patterns = ext.getPatterns();
553             constraints.addPatterns(patterns);
554             fractionDigits = ext.getFractionDigits();
555             constraints.setFractionDigits(fractionDigits);
556             if(YangTypesConverter.isBaseYangType(ext.getBaseType().getQName().getLocalName())) {
557                 mergeConstraints(ext.getBaseType(), constraints);
558                 return constraints;
559             } else {
560                 return findConstraints(
561                         findTypeDefinitionBuilder(nodeToResolve.getPath(), builder,
562                                 ext.getQName().getLocalName(), builder.getName(),
563                                 nodeToResolve.getLine()), constraints, modules,
564                         builder);
565             }
566         } else if (referencedType instanceof UnknownType) {
567             final UnknownType unknown = (UnknownType) referencedType;
568             ranges = unknown.getRangeStatements();
569             constraints.addRanges(ranges);
570             lengths = unknown.getLengthStatements();
571             constraints.addLengths(lengths);
572             patterns = unknown.getPatterns();
573             constraints.addPatterns(patterns);
574             fractionDigits = unknown.getFractionDigits();
575             constraints.setFractionDigits(fractionDigits);
576
577             String unknownTypePrefix = unknown.getQName().getPrefix();
578             if (unknownTypePrefix == null || "".equals(unknownTypePrefix)) {
579                 unknownTypePrefix = builder.getPrefix();
580             }
581             final ModuleBuilder dependentModule = findDependentModule(modules,
582                     builder, unknown.getQName().getPrefix(),
583                     nodeToResolve.getLine());
584             final TypeDefinitionBuilder utBuilder = findTypeDefinitionBuilder(
585                     nodeToResolve.getPath(), dependentModule, unknown
586                             .getQName().getLocalName(), builder.getName(),
587                     nodeToResolve.getLine());
588             return findConstraints(utBuilder, constraints, modules,
589                     dependentModule);
590         } else {
591             // HANDLE BASE YANG TYPE
592             mergeConstraints(referencedType, constraints);
593             return constraints;
594         }
595     }
596
597     /**
598      * Search for type definition builder by name.
599      *
600      * @param dirtyNodeSchemaPath
601      *            schema path of node which contains unresolved type
602      * @param dependentModule
603      *            module which should contains referenced type
604      * @param typeName
605      *            name of type definition
606      * @param currentModuleName
607      *            name of current module
608      * @param line
609      *            current line in yang model
610      * @return
611      */
612     private TypeDefinitionBuilder findTypeDefinitionBuilder(
613             SchemaPath dirtyNodeSchemaPath,
614             final ModuleBuilder dependentModule, final String typeName,
615             final String currentModuleName, final int line) {
616         final List<QName> path = dirtyNodeSchemaPath.getPath();
617         TypeDefinitionBuilder result = null;
618
619         Set<TypeDefinitionBuilder> typedefs = dependentModule
620                 .getModuleTypedefs();
621         result = findTdb(typedefs, typeName);
622
623         if (result == null) {
624             Builder currentNode = null;
625             final List<String> currentPath = new ArrayList<String>();
626             currentPath.add(dependentModule.getName());
627
628             for (int i = 0; i < path.size(); i++) {
629                 QName qname = path.get(i);
630                 currentPath.add(qname.getLocalName());
631                 currentNode = dependentModule.getModuleNode(currentPath);
632
633                 if (currentNode instanceof RpcDefinitionBuilder) {
634                     typedefs = ((RpcDefinitionBuilder) currentNode)
635                             .getTypeDefinitions();
636                 } else if (currentNode instanceof DataNodeContainerBuilder) {
637                     typedefs = ((DataNodeContainerBuilder) currentNode)
638                             .getTypeDefinitions();
639                 } else {
640                     typedefs = Collections.emptySet();
641                 }
642
643                 result = findTdb(typedefs, typeName);
644                 if (result != null) {
645                     break;
646                 }
647             }
648         }
649
650         if (result != null) {
651             return result;
652         }
653         throw new YangParseException(currentModuleName, line,
654                 "Referenced type '" + typeName + "' not found.");
655     }
656
657     private TypeDefinitionBuilder findTdb(Set<TypeDefinitionBuilder> types,
658             String name) {
659         for (TypeDefinitionBuilder td : types) {
660             if (td.getQName().getLocalName().equals(name)) {
661                 return td;
662             }
663         }
664         return null;
665     }
666
667     /**
668      * Pull restriction from referenced type and add them to given constraints
669      *
670      * @param referencedType
671      * @param constraints
672      */
673     private void mergeConstraints(final TypeDefinition<?> referencedType,
674             final TypeConstraints constraints) {
675
676         if (referencedType instanceof DecimalTypeDefinition) {
677             constraints.addRanges(((DecimalTypeDefinition) referencedType)
678                     .getRangeStatements());
679             constraints
680                     .setFractionDigits(((DecimalTypeDefinition) referencedType)
681                             .getFractionDigits());
682         } else if (referencedType instanceof IntegerTypeDefinition) {
683             constraints.addRanges(((IntegerTypeDefinition) referencedType)
684                     .getRangeStatements());
685         } else if (referencedType instanceof StringTypeDefinition) {
686             constraints.addPatterns(((StringTypeDefinition) referencedType)
687                     .getPatterns());
688             constraints.addLengths(((StringTypeDefinition) referencedType)
689                     .getLengthStatements());
690         } else if (referencedType instanceof BinaryTypeDefinition) {
691             constraints.addLengths(((BinaryTypeDefinition) referencedType)
692                     .getLengthConstraints());
693         }
694     }
695
696     /**
697      * Go through all augment definitions and resolve them. This method also
698      * finds augment target node and add child nodes to it.
699      *
700      * @param modules
701      *            all available modules
702      */
703     private void resolveAugments(
704             final Map<String, TreeMap<Date, ModuleBuilder>> modules) {
705         final List<ModuleBuilder> allModulesList = new ArrayList<ModuleBuilder>();
706         final Set<ModuleBuilder> allModulesSet = new HashSet<ModuleBuilder>();
707         for (Map.Entry<String, TreeMap<Date, ModuleBuilder>> entry : modules
708                 .entrySet()) {
709             for (Map.Entry<Date, ModuleBuilder> inner : entry.getValue()
710                     .entrySet()) {
711                 allModulesList.add(inner.getValue());
712                 allModulesSet.add(inner.getValue());
713             }
714         }
715
716         for (int i = 0; i < allModulesList.size(); i++) {
717             final ModuleBuilder module = allModulesList.get(i);
718             // try to resolve augments in module
719             resolveAugment(modules, module);
720             // while all augments are not resolved
721             final Iterator<ModuleBuilder> allModulesIterator = allModulesSet
722                     .iterator();
723             while (!(module.getAugmentsResolved() == module.getAugments()
724                     .size())) {
725                 ModuleBuilder nextModule = null;
726                 // try resolve other module augments
727                 try {
728                     nextModule = allModulesIterator.next();
729                     resolveAugment(modules, nextModule);
730                 } catch (NoSuchElementException e) {
731                     throw new YangParseException(
732                             "Failed to resolve augments in module '"
733                                     + module.getName() + "'.", e);
734                 }
735                 // then try to resolve first module again
736                 resolveAugment(modules, module);
737             }
738         }
739     }
740
741     /**
742      *
743      * @param modules
744      *            all available modules
745      * @param module
746      *            current module
747      */
748     private void resolveAugment(
749             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
750             final ModuleBuilder module) {
751         if (module.getAugmentsResolved() < module.getAugments().size()) {
752             for (AugmentationSchemaBuilder augmentBuilder : module
753                     .getAugments()) {
754
755                 if (!augmentBuilder.isResolved()) {
756                     final SchemaPath augmentTargetSchemaPath = augmentBuilder
757                             .getTargetPath();
758                     final List<QName> path = augmentTargetSchemaPath.getPath();
759
760                     final QName qname = path.get(0);
761                     String prefix = qname.getPrefix();
762                     if (prefix == null) {
763                         prefix = module.getPrefix();
764                     }
765
766                     DataSchemaNodeBuilder currentParent = null;
767                     final ModuleBuilder dependentModule = findDependentModule(
768                             modules, module, prefix, augmentBuilder.getLine());
769                     for (DataSchemaNodeBuilder child : dependentModule
770                             .getChildNodes()) {
771                         final QName childQName = child.getQName();
772                         if (childQName.getLocalName().equals(
773                                 qname.getLocalName())) {
774                             currentParent = child;
775                             break;
776                         }
777                     }
778
779                     if (currentParent == null) {
780                         continue;
781                     }
782
783                     for (int i = 1; i < path.size(); i++) {
784                         final QName currentQName = path.get(i);
785                         DataSchemaNodeBuilder newParent = null;
786                         for (DataSchemaNodeBuilder child : ((DataNodeContainerBuilder) currentParent)
787                                 .getChildNodes()) {
788                             final QName childQName = child.getQName();
789                             if (childQName.getLocalName().equals(
790                                     currentQName.getLocalName())) {
791                                 newParent = child;
792                                 break;
793                             }
794                         }
795                         if (newParent == null) {
796                             break; // node not found, quit search
797                         } else {
798                             currentParent = newParent;
799                         }
800                     }
801
802                     final QName currentQName = currentParent.getQName();
803                     final QName lastAugmentPathElement = path
804                             .get(path.size() - 1);
805                     if (currentQName.getLocalName().equals(
806                             lastAugmentPathElement.getLocalName())) {
807
808                         if (currentParent instanceof ChoiceBuilder) {
809                             ParserUtils.fillAugmentTarget(augmentBuilder,
810                                     (ChoiceBuilder) currentParent);
811                         } else {
812                             ParserUtils.fillAugmentTarget(augmentBuilder,
813                                     (DataNodeContainerBuilder) currentParent);
814                         }
815                         ((AugmentationTargetBuilder) currentParent)
816                                 .addAugmentation(augmentBuilder);
817                         SchemaPath oldPath = currentParent.getPath();
818                         augmentBuilder.setTargetPath(new SchemaPath(oldPath
819                                 .getPath(), oldPath.isAbsolute()));
820                         augmentBuilder.setResolved(true);
821                         module.augmentResolved();
822                     }
823                 }
824
825             }
826         }
827     }
828
829     /**
830      * Go through identity statements defined in current module and resolve
831      * their 'base' statement if present.
832      *
833      * @param modules
834      *            all modules
835      * @param module
836      *            module being resolved
837      */
838     private void resolveIdentities(
839             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
840             final ModuleBuilder module) {
841         final Set<IdentitySchemaNodeBuilder> identities = module
842                 .getIdentities();
843         for (IdentitySchemaNodeBuilder identity : identities) {
844             final String baseIdentityName = identity.getBaseIdentityName();
845             if (baseIdentityName != null) {
846                 String baseIdentityPrefix = null;
847                 String baseIdentityLocalName = null;
848                 if (baseIdentityName.contains(":")) {
849                     final String[] splitted = baseIdentityName.split(":");
850                     baseIdentityPrefix = splitted[0];
851                     baseIdentityLocalName = splitted[1];
852                 } else {
853                     baseIdentityPrefix = module.getPrefix();
854                     baseIdentityLocalName = baseIdentityName;
855                 }
856                 final ModuleBuilder dependentModule = findDependentModule(
857                         modules, module, baseIdentityPrefix, identity.getLine());
858
859                 final Set<IdentitySchemaNodeBuilder> dependentModuleIdentities = dependentModule
860                         .getIdentities();
861                 for (IdentitySchemaNodeBuilder idBuilder : dependentModuleIdentities) {
862                     if (idBuilder.getQName().getLocalName()
863                             .equals(baseIdentityLocalName)) {
864                         identity.setBaseIdentity(idBuilder);
865                     }
866                 }
867             }
868         }
869     }
870
871     /**
872      * Go through uses statements defined in current module and resolve their
873      * refine statements.
874      *
875      * @param modules
876      *            all modules
877      * @param module
878      *            module being resolved
879      */
880     private void resolveUsesRefines(
881             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
882             final ModuleBuilder module) {
883         final Map<List<String>, UsesNodeBuilder> moduleUses = module
884                 .getUsesNodes();
885         for (Map.Entry<List<String>, UsesNodeBuilder> entry : moduleUses
886                 .entrySet()) {
887             final List<String> key = entry.getKey();
888             final UsesNodeBuilder usesNode = entry.getValue();
889             final int line = usesNode.getLine();
890
891             final String groupingName = key.get(key.size() - 1);
892
893             for (RefineHolder refine : usesNode.getRefines()) {
894                 SchemaNodeBuilder refineTarget = getRefineNodeBuilderCopy(
895                         groupingName, refine, modules, module);
896                 ParserUtils.checkRefine(refineTarget, refine);
897                 ParserUtils.refineDefault(refineTarget, refine, line);
898                 if (refineTarget instanceof LeafSchemaNodeBuilder) {
899                     final LeafSchemaNodeBuilder leaf = (LeafSchemaNodeBuilder) refineTarget;
900                     ParserUtils.refineLeaf(leaf, refine, line);
901                     usesNode.addRefineNode(leaf);
902                 } else if (refineTarget instanceof ContainerSchemaNodeBuilder) {
903                     final ContainerSchemaNodeBuilder container = (ContainerSchemaNodeBuilder) refineTarget;
904                     ParserUtils.refineContainer(container, refine, line);
905                     usesNode.addRefineNode(container);
906                 } else if (refineTarget instanceof ListSchemaNodeBuilder) {
907                     final ListSchemaNodeBuilder list = (ListSchemaNodeBuilder) refineTarget;
908                     ParserUtils.refineList(list, refine, line);
909                     usesNode.addRefineNode(list);
910                 } else if (refineTarget instanceof LeafListSchemaNodeBuilder) {
911                     final LeafListSchemaNodeBuilder leafList = (LeafListSchemaNodeBuilder) refineTarget;
912                     ParserUtils.refineLeafList(leafList, refine, line);
913                     usesNode.addRefineNode(leafList);
914                 } else if (refineTarget instanceof ChoiceBuilder) {
915                     final ChoiceBuilder choice = (ChoiceBuilder) refineTarget;
916                     ParserUtils.refineChoice(choice, refine, line);
917                     usesNode.addRefineNode(choice);
918                 } else if (refineTarget instanceof AnyXmlBuilder) {
919                     final AnyXmlBuilder anyXml = (AnyXmlBuilder) refineTarget;
920                     ParserUtils.refineAnyxml(anyXml, refine, line);
921                     usesNode.addRefineNode(anyXml);
922                 } else if (refineTarget instanceof GroupingBuilder) {
923                     usesNode.addRefineNode(refineTarget);
924                 } else if (refineTarget instanceof TypeDefinitionBuilder) {
925                     usesNode.addRefineNode(refineTarget);
926                 }
927             }
928         }
929     }
930
931     /**
932      * Find original builder of node to refine and return copy of this builder.
933      * <p>
934      * We must create and use a copy of builder to preserve original builder
935      * state, because this object will be refined (modified) and later added to
936      * {@link UsesNodeBuilder}.
937      * </p>
938      *
939      * @param groupingPath
940      *            path to grouping which contains node to refine
941      * @param refine
942      *            refine object containing informations about refine
943      * @param modules
944      *            all loaded modules
945      * @param module
946      *            current module
947      * @return copy of node to be refined if it is present in grouping, null
948      *         otherwise
949      */
950     private SchemaNodeBuilder getRefineNodeBuilderCopy(
951             final String groupingPath, final RefineHolder refine,
952             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
953             final ModuleBuilder module) {
954         Builder result = null;
955         final Builder lookedUpBuilder = findRefineTargetBuilder(groupingPath,
956                 refine, modules, module);
957         if (lookedUpBuilder instanceof LeafSchemaNodeBuilder) {
958             result = ParserUtils
959                     .copyLeafBuilder((LeafSchemaNodeBuilder) lookedUpBuilder);
960         } else if (lookedUpBuilder instanceof ContainerSchemaNodeBuilder) {
961             result = ParserUtils
962                     .copyContainerBuilder((ContainerSchemaNodeBuilder) lookedUpBuilder);
963         } else if (lookedUpBuilder instanceof ListSchemaNodeBuilder) {
964             result = ParserUtils
965                     .copyListBuilder((ListSchemaNodeBuilder) lookedUpBuilder);
966         } else if (lookedUpBuilder instanceof LeafListSchemaNodeBuilder) {
967             result = ParserUtils
968                     .copyLeafListBuilder((LeafListSchemaNodeBuilder) lookedUpBuilder);
969         } else if (lookedUpBuilder instanceof ChoiceBuilder) {
970             result = ParserUtils
971                     .copyChoiceBuilder((ChoiceBuilder) lookedUpBuilder);
972         } else if (lookedUpBuilder instanceof AnyXmlBuilder) {
973             result = ParserUtils
974                     .copyAnyXmlBuilder((AnyXmlBuilder) lookedUpBuilder);
975         } else if (lookedUpBuilder instanceof GroupingBuilder) {
976             result = ParserUtils
977                     .copyGroupingBuilder((GroupingBuilder) lookedUpBuilder);
978         } else if (lookedUpBuilder instanceof TypeDefinitionBuilder) {
979             result = ParserUtils
980                     .copyTypedefBuilder((TypeDefinitionBuilderImpl) lookedUpBuilder);
981         } else {
982             throw new YangParseException(module.getName(), refine.getLine(),
983                     "Target '" + refine.getName() + "' can not be refined");
984         }
985         return (SchemaNodeBuilder) result;
986     }
987
988     /**
989      * Find builder of refine node.
990      *
991      * @param groupingPath
992      *            path to grouping which contains node to refine
993      * @param refine
994      *            object containing refine information
995      * @param modules
996      *            all loaded modules
997      * @param module
998      *            current module
999      * @return Builder object of refine node if it is present in grouping, null
1000      *         otherwise
1001      */
1002     private Builder findRefineTargetBuilder(final String groupingPath,
1003             final RefineHolder refine,
1004             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
1005             final ModuleBuilder module) {
1006         final String refineNodeName = refine.getName();
1007         final SchemaPath path = ParserUtils.parseUsesPath(groupingPath);
1008         final List<String> builderPath = new ArrayList<String>();
1009         String prefix = null;
1010         for (QName qname : path.getPath()) {
1011             builderPath.add(qname.getLocalName());
1012             prefix = qname.getPrefix();
1013         }
1014         if (prefix == null) {
1015             prefix = module.getPrefix();
1016         }
1017
1018         final ModuleBuilder dependentModule = findDependentModule(modules,
1019                 module, prefix, refine.getLine());
1020         builderPath.add(0, dependentModule.getName());
1021         final GroupingBuilder builder = dependentModule
1022                 .getGrouping(builderPath);
1023
1024         Builder result = builder.getChildNode(refineNodeName);
1025         if (result == null) {
1026             Set<GroupingBuilder> grps = builder.getGroupings();
1027             for (GroupingBuilder gr : grps) {
1028                 if (gr.getQName().getLocalName().equals(refineNodeName)) {
1029                     result = gr;
1030                     break;
1031                 }
1032             }
1033         }
1034         if (result == null) {
1035             Set<TypeDefinitionBuilder> typedefs = builder.getTypeDefinitions();
1036             for (TypeDefinitionBuilder typedef : typedefs) {
1037                 if (typedef.getQName().getLocalName().equals(refineNodeName)) {
1038                     result = typedef;
1039                     break;
1040                 }
1041             }
1042         }
1043         return result;
1044     }
1045
1046     private QName findFullQName(
1047             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
1048             final ModuleBuilder module, final IdentityrefTypeBuilder idref) {
1049         QName result = null;
1050         String baseString = idref.getBaseString();
1051         if (baseString.contains(":")) {
1052             String[] splittedBase = baseString.split(":");
1053             if (splittedBase.length > 2) {
1054                 throw new YangParseException(module.getName(), idref.getLine(),
1055                         "Failed to parse identityref base: " + baseString);
1056             }
1057             String prefix = splittedBase[0];
1058             String name = splittedBase[1];
1059             ModuleBuilder dependentModule = findDependentModule(modules,
1060                     module, prefix, idref.getLine());
1061             result = new QName(dependentModule.getNamespace(),
1062                     dependentModule.getRevision(), prefix, name);
1063         } else {
1064             result = new QName(module.getNamespace(), module.getRevision(),
1065                     module.getPrefix(), baseString);
1066         }
1067         return result;
1068     }
1069
1070     private void resolveUnknownNodes(
1071             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
1072             final ModuleBuilder module) {
1073         for (UnknownSchemaNodeBuilder usnb : module.getUnknownNodes()) {
1074             QName nodeType = usnb.getNodeType();
1075             if (nodeType.getNamespace() == null
1076                     || nodeType.getRevision() == null) {
1077                 try {
1078                     ModuleBuilder dependentModule = findDependentModule(
1079                             modules, module, nodeType.getPrefix(),
1080                             usnb.getLine());
1081                     QName newNodeType = new QName(
1082                             dependentModule.getNamespace(),
1083                             dependentModule.getRevision(),
1084                             nodeType.getPrefix(), nodeType.getLocalName());
1085                     usnb.setNodeType(newNodeType);
1086                 } catch (YangParseException e) {
1087                     logger.debug(module.getName(), usnb.getLine(),
1088                             "Failed to find unknown node type: " + nodeType);
1089                 }
1090             }
1091         }
1092     }
1093
1094     /**
1095      * Find dependent module based on given prefix
1096      *
1097      * @param modules
1098      *            all available modules
1099      * @param module
1100      *            current module
1101      * @param prefix
1102      *            target module prefix
1103      * @param line
1104      *            current line in yang model
1105      * @return
1106      */
1107     private ModuleBuilder findDependentModule(
1108             final Map<String, TreeMap<Date, ModuleBuilder>> modules,
1109             final ModuleBuilder module, final String prefix, final int line) {
1110         ModuleBuilder dependentModule = null;
1111         Date dependentModuleRevision = null;
1112
1113         if (prefix.equals(module.getPrefix())) {
1114             dependentModule = module;
1115         } else {
1116             final ModuleImport dependentModuleImport = ParserUtils
1117                     .getModuleImport(module, prefix);
1118             if (dependentModuleImport == null) {
1119                 throw new YangParseException(module.getName(), line,
1120                         "No import found with prefix '" + prefix + "'.");
1121             }
1122             final String dependentModuleName = dependentModuleImport
1123                     .getModuleName();
1124             dependentModuleRevision = dependentModuleImport.getRevision();
1125
1126             final TreeMap<Date, ModuleBuilder> moduleBuildersByRevision = modules
1127                     .get(dependentModuleName);
1128             if (moduleBuildersByRevision == null) {
1129                 throw new YangParseException(module.getName(), line,
1130                         "Failed to find dependent module '"
1131                                 + dependentModuleName + "'.");
1132             }
1133             if (dependentModuleRevision == null) {
1134                 dependentModule = moduleBuildersByRevision.lastEntry()
1135                         .getValue();
1136             } else {
1137                 dependentModule = moduleBuildersByRevision
1138                         .get(dependentModuleRevision);
1139             }
1140         }
1141
1142         if (dependentModule == null) {
1143             throw new YangParseException(module.getName(), line,
1144                     "Failed to find dependent module with prefix '" + prefix
1145                             + "' and revision '" + dependentModuleRevision
1146                             + "'.");
1147         }
1148         return dependentModule;
1149     }
1150
1151 }