2 * Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.yangtools.yang.parser.stmt.reactor;
10 import static com.google.common.base.Preconditions.checkState;
11 import static com.google.common.base.Verify.verify;
12 import static com.google.common.base.Verify.verifyNotNull;
13 import static java.util.Objects.requireNonNull;
15 import com.google.common.collect.HashBasedTable;
16 import com.google.common.collect.ImmutableMap;
17 import com.google.common.collect.ImmutableSet;
18 import com.google.common.collect.ImmutableSetMultimap;
19 import com.google.common.collect.SetMultimap;
20 import com.google.common.collect.Table;
21 import com.google.common.collect.TreeBasedTable;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.HashMap;
25 import java.util.HashSet;
26 import java.util.Iterator;
27 import java.util.List;
29 import java.util.Objects;
30 import java.util.Optional;
32 import java.util.SortedMap;
33 import org.eclipse.jdt.annotation.NonNull;
34 import org.opendaylight.yangtools.yang.common.Empty;
35 import org.opendaylight.yangtools.yang.common.QName;
36 import org.opendaylight.yangtools.yang.common.QNameModule;
37 import org.opendaylight.yangtools.yang.common.Revision;
38 import org.opendaylight.yangtools.yang.common.UnresolvedQName.Unqualified;
39 import org.opendaylight.yangtools.yang.common.YangVersion;
40 import org.opendaylight.yangtools.yang.model.api.meta.DeclaredStatement;
41 import org.opendaylight.yangtools.yang.model.api.meta.EffectiveStatement;
42 import org.opendaylight.yangtools.yang.model.repo.api.FeatureSet;
43 import org.opendaylight.yangtools.yang.model.repo.api.SourceIdentifier;
44 import org.opendaylight.yangtools.yang.parser.spi.ParserNamespaces;
45 import org.opendaylight.yangtools.yang.parser.spi.meta.DerivedNamespaceBehaviour;
46 import org.opendaylight.yangtools.yang.parser.spi.meta.ModelProcessingPhase;
47 import org.opendaylight.yangtools.yang.parser.spi.meta.MutableStatement;
48 import org.opendaylight.yangtools.yang.parser.spi.meta.NamespaceBehaviour;
49 import org.opendaylight.yangtools.yang.parser.spi.meta.NamespaceBehaviour.NamespaceStorageNode;
50 import org.opendaylight.yangtools.yang.parser.spi.meta.NamespaceBehaviour.StorageNodeType;
51 import org.opendaylight.yangtools.yang.parser.spi.meta.NamespaceNotAvailableException;
52 import org.opendaylight.yangtools.yang.parser.spi.meta.ParserNamespace;
53 import org.opendaylight.yangtools.yang.parser.spi.meta.ReactorException;
54 import org.opendaylight.yangtools.yang.parser.spi.meta.SomeModifiersUnresolvedException;
55 import org.opendaylight.yangtools.yang.parser.spi.meta.StatementSupportBundle;
56 import org.opendaylight.yangtools.yang.parser.spi.source.SourceException;
57 import org.opendaylight.yangtools.yang.parser.spi.source.StatementStreamSource;
58 import org.opendaylight.yangtools.yang.parser.spi.validation.ValidationBundles;
59 import org.opendaylight.yangtools.yang.parser.spi.validation.ValidationBundles.ValidationBundleType;
60 import org.opendaylight.yangtools.yang.parser.stmt.reactor.SourceSpecificContext.PhaseCompletionProgress;
61 import org.slf4j.Logger;
62 import org.slf4j.LoggerFactory;
64 final class BuildGlobalContext extends NamespaceStorageSupport {
65 private static final Logger LOG = LoggerFactory.getLogger(BuildGlobalContext.class);
67 private static final ModelProcessingPhase[] PHASE_EXECUTION_ORDER = {
68 ModelProcessingPhase.SOURCE_PRE_LINKAGE,
69 ModelProcessingPhase.SOURCE_LINKAGE,
70 ModelProcessingPhase.STATEMENT_DEFINITION,
71 ModelProcessingPhase.FULL_DECLARATION,
72 ModelProcessingPhase.EFFECTIVE_MODEL
75 private final Table<YangVersion, QName, StatementDefinitionContext<?, ?, ?>> definitions = HashBasedTable.create();
76 private final Map<QName, StatementDefinitionContext<?, ?, ?>> modelDefinedStmtDefs = new HashMap<>();
77 private final Map<ParserNamespace<?, ?>, NamespaceBehaviourWithListeners<?, ?>> supportedNamespaces =
79 private final List<MutableStatement> mutableStatementsToSeal = new ArrayList<>();
80 private final ImmutableMap<ModelProcessingPhase, StatementSupportBundle> supports;
81 private final Set<SourceSpecificContext> sources = new HashSet<>();
82 private final ImmutableSet<YangVersion> supportedVersions;
84 private Set<SourceSpecificContext> libSources = new HashSet<>();
85 private ModelProcessingPhase currentPhase = ModelProcessingPhase.INIT;
86 private ModelProcessingPhase finishedPhase = ModelProcessingPhase.INIT;
88 BuildGlobalContext(final ImmutableMap<ModelProcessingPhase, StatementSupportBundle> supports,
89 final ImmutableMap<ValidationBundleType, Collection<?>> supportedValidation) {
90 this.supports = requireNonNull(supports, "BuildGlobalContext#supports cannot be null");
92 final var behavior = getNamespaceBehaviour(ValidationBundles.NAMESPACE);
93 for (var validationBundle : supportedValidation.entrySet()) {
94 behavior.addTo(this, validationBundle.getKey(), validationBundle.getValue());
97 supportedVersions = ImmutableSet.copyOf(
98 verifyNotNull(supports.get(ModelProcessingPhase.INIT)).getSupportedVersions());
101 StatementSupportBundle getSupportsForPhase(final ModelProcessingPhase phase) {
102 return supports.get(phase);
105 void addSource(final @NonNull StatementStreamSource source) {
106 sources.add(new SourceSpecificContext(this, source));
109 void addLibSource(final @NonNull StatementStreamSource libSource) {
110 checkState(currentPhase == ModelProcessingPhase.INIT,
111 "Add library source is allowed in ModelProcessingPhase.INIT only");
112 libSources.add(new SourceSpecificContext(this, libSource));
115 void setSupportedFeatures(final Set<QName> supportedFeatures) {
116 if (supportedFeatures instanceof FeatureSet) {
117 addToNamespace(ParserNamespaces.SUPPORTED_FEATURES, Empty.value(), supportedFeatures);
119 addToNamespace(ParserNamespaces.SUPPORTED_FEATURES, Empty.value(), ImmutableSet.copyOf(supportedFeatures));
123 void setModulesDeviatedByModules(final SetMultimap<QNameModule, QNameModule> modulesDeviatedByModules) {
124 addToNamespace(ParserNamespaces.MODULES_DEVIATED_BY, Empty.value(),
125 ImmutableSetMultimap.copyOf(modulesDeviatedByModules));
129 public StorageNodeType getStorageNodeType() {
130 return StorageNodeType.GLOBAL;
134 public NamespaceStorageNode getParentNamespaceStorage() {
139 <K, V> NamespaceBehaviourWithListeners<K, V> getNamespaceBehaviour(final ParserNamespace<K, V> type) {
140 NamespaceBehaviourWithListeners<?, ?> potential = supportedNamespaces.get(type);
141 if (potential == null) {
142 final var potentialRaw = verifyNotNull(supports.get(currentPhase)).namespaceBehaviourOf(type);
143 if (potentialRaw != null) {
144 potential = createNamespaceContext(potentialRaw);
145 supportedNamespaces.put(type, potential);
147 throw new NamespaceNotAvailableException("Namespace " + type + " is not available in phase "
152 verify(type.equals(potential.getIdentifier()));
154 * Safe cast, previous checkState checks equivalence of key from which
155 * type argument are derived
157 return (NamespaceBehaviourWithListeners<K, V>) potential;
160 @SuppressWarnings({ "unchecked", "rawtypes" })
161 private <K, V> NamespaceBehaviourWithListeners<K, V> createNamespaceContext(
162 final NamespaceBehaviour<K, V> potentialRaw) {
163 if (potentialRaw instanceof DerivedNamespaceBehaviour derived) {
164 final VirtualNamespaceContext derivedContext = new VirtualNamespaceContext(derived);
165 getNamespaceBehaviour(derived.getDerivedFrom()).addDerivedNamespace(derivedContext);
166 return derivedContext;
168 return new SimpleNamespaceContext<>(potentialRaw);
171 StatementDefinitionContext<?, ?, ?> getStatementDefinition(final YangVersion version, final QName name) {
172 StatementDefinitionContext<?, ?, ?> potential = definitions.get(version, name);
173 if (potential == null) {
174 final var potentialRaw = verifyNotNull(supports.get(currentPhase)).getStatementDefinition(version, name);
175 if (potentialRaw != null) {
176 potential = new StatementDefinitionContext<>(potentialRaw);
177 definitions.put(version, name, potential);
183 StatementDefinitionContext<?, ?, ?> getModelDefinedStatementDefinition(final QName name) {
184 return modelDefinedStmtDefs.get(name);
187 void putModelDefinedStatementDefinition(final QName name, final StatementDefinitionContext<?, ?, ?> def) {
188 modelDefinedStmtDefs.put(name, def);
191 private void executePhases() throws ReactorException {
192 for (final ModelProcessingPhase phase : PHASE_EXECUTION_ORDER) {
194 loadPhaseStatements();
195 completePhaseActions();
200 ReactorDeclaredModel build() throws ReactorException {
205 EffectiveSchemaContext buildEffective() throws ReactorException {
207 return transformEffective();
210 private ReactorDeclaredModel transform() {
211 checkState(finishedPhase == ModelProcessingPhase.EFFECTIVE_MODEL);
212 final var rootStatements = new ArrayList<DeclaredStatement<?>>(sources.size());
213 for (var source : sources) {
214 rootStatements.add(source.declaredRoot());
216 return new ReactorDeclaredModel(rootStatements);
219 private SomeModifiersUnresolvedException propagateException(final SourceSpecificContext source,
220 final RuntimeException cause) throws SomeModifiersUnresolvedException {
221 final SourceIdentifier sourceId = source.identifySource();
222 if (!(cause instanceof SourceException)) {
224 * This should not be happening as all our processing should provide SourceExceptions.
225 * We will wrap the exception to provide enough information to identify the problematic model,
226 * but also emit a warning so the offending codepath will get fixed.
228 LOG.warn("Unexpected error processing source {}. Please file an issue with this model attached.",
232 throw new SomeModifiersUnresolvedException(currentPhase, sourceId, cause);
235 @SuppressWarnings("checkstyle:illegalCatch")
236 private EffectiveSchemaContext transformEffective() throws ReactorException {
237 checkState(finishedPhase == ModelProcessingPhase.EFFECTIVE_MODEL);
238 final var rootStatements = new ArrayList<DeclaredStatement<?>>(sources.size());
239 final var rootEffectiveStatements = new ArrayList<EffectiveStatement<?, ?>>(sources.size());
241 for (var source : sources) {
243 rootStatements.add(source.declaredRoot());
244 rootEffectiveStatements.add(source.effectiveRoot());
245 } catch (final RuntimeException ex) {
246 throw propagateException(source, ex);
250 sealMutableStatements();
251 return EffectiveSchemaContext.create(rootStatements, rootEffectiveStatements);
254 private void startPhase(final ModelProcessingPhase phase) {
255 checkState(Objects.equals(finishedPhase, phase.getPreviousPhase()));
256 startPhaseFor(phase, sources);
257 startPhaseFor(phase, libSources);
259 currentPhase = phase;
260 LOG.debug("Global phase {} started", phase);
263 private static void startPhaseFor(final ModelProcessingPhase phase, final Set<SourceSpecificContext> sources) {
264 for (final SourceSpecificContext source : sources) {
265 source.startPhase(phase);
269 private void loadPhaseStatements() throws ReactorException {
270 checkState(currentPhase != null);
271 loadPhaseStatementsFor(sources);
272 loadPhaseStatementsFor(libSources);
275 @SuppressWarnings("checkstyle:illegalCatch")
276 private void loadPhaseStatementsFor(final Set<SourceSpecificContext> srcs) throws ReactorException {
277 for (final SourceSpecificContext source : srcs) {
279 source.loadStatements();
280 } catch (final RuntimeException ex) {
281 throw propagateException(source, ex);
286 private SomeModifiersUnresolvedException addSourceExceptions(final List<SourceSpecificContext> sourcesToProgress) {
287 boolean addedCause = false;
288 SomeModifiersUnresolvedException buildFailure = null;
289 for (final SourceSpecificContext failedSource : sourcesToProgress) {
290 final Optional<SourceException> optSourceEx = failedSource.failModifiers(currentPhase);
291 if (optSourceEx.isEmpty()) {
295 final SourceException sourceEx = optSourceEx.get();
296 // Workaround for broken logging implementations which ignore
297 // suppressed exceptions
298 final Throwable cause = sourceEx.getCause() != null ? sourceEx.getCause() : sourceEx;
299 if (LOG.isDebugEnabled()) {
300 LOG.error("Failed to parse YANG from source {}", failedSource, sourceEx);
302 LOG.error("Failed to parse YANG from source {}: {}", failedSource, cause.getMessage());
305 final Throwable[] suppressed = sourceEx.getSuppressed();
306 if (suppressed.length > 0) {
307 LOG.error("{} additional errors reported:", suppressed.length);
310 for (final Throwable t : suppressed) {
311 LOG.error("Error {}: {}", count, t.getMessage());
318 final SourceIdentifier sourceId = failedSource.identifySource();
319 buildFailure = new SomeModifiersUnresolvedException(currentPhase, sourceId, sourceEx);
321 buildFailure.addSuppressed(sourceEx);
327 @SuppressWarnings("checkstyle:illegalCatch")
328 private void completePhaseActions() throws ReactorException {
329 checkState(currentPhase != null);
330 final List<SourceSpecificContext> sourcesToProgress = new ArrayList<>(sources);
331 if (!libSources.isEmpty()) {
332 checkState(currentPhase == ModelProcessingPhase.SOURCE_PRE_LINKAGE,
333 "Yang library sources should be empty after ModelProcessingPhase.SOURCE_PRE_LINKAGE, "
334 + "but current phase was %s", currentPhase);
335 sourcesToProgress.addAll(libSources);
338 boolean progressing = true;
339 while (progressing) {
340 // We reset progressing to false.
342 final Iterator<SourceSpecificContext> currentSource = sourcesToProgress.iterator();
343 while (currentSource.hasNext()) {
344 final SourceSpecificContext nextSourceCtx = currentSource.next();
346 final PhaseCompletionProgress sourceProgress =
347 nextSourceCtx.tryToCompletePhase(currentPhase.executionOrder());
348 switch (sourceProgress) {
350 currentSource.remove();
351 // we were able to make progress in computation
361 throw new IllegalStateException("Unsupported phase progress " + sourceProgress);
363 } catch (final RuntimeException ex) {
364 throw propagateException(nextSourceCtx, ex);
369 if (!libSources.isEmpty()) {
370 final Set<SourceSpecificContext> requiredLibs = getRequiredSourcesFromLib();
371 sources.addAll(requiredLibs);
372 libSources = ImmutableSet.of();
374 * We want to report errors of relevant sources only, so any others can
377 sourcesToProgress.retainAll(sources);
380 if (!sourcesToProgress.isEmpty()) {
381 final SomeModifiersUnresolvedException buildFailure = addSourceExceptions(sourcesToProgress);
382 if (buildFailure != null) {
388 private Set<SourceSpecificContext> getRequiredSourcesFromLib() {
389 checkState(currentPhase == ModelProcessingPhase.SOURCE_PRE_LINKAGE,
390 "Required library sources can be collected only in ModelProcessingPhase.SOURCE_PRE_LINKAGE phase,"
391 + " but current phase was %s", currentPhase);
392 final TreeBasedTable<Unqualified, Optional<Revision>, SourceSpecificContext> libSourcesTable =
393 TreeBasedTable.create(Unqualified::compareTo, Revision::compare);
394 for (final SourceSpecificContext libSource : libSources) {
395 final SourceIdentifier libSourceIdentifier = requireNonNull(libSource.getRootIdentifier());
396 libSourcesTable.put(libSourceIdentifier.name(),
397 Optional.ofNullable(libSourceIdentifier.revision()), libSource);
400 final Set<SourceSpecificContext> requiredLibs = new HashSet<>();
401 for (final SourceSpecificContext source : sources) {
402 collectRequiredSourcesFromLib(libSourcesTable, requiredLibs, source);
403 removeConflictingLibSources(source, requiredLibs);
408 private void collectRequiredSourcesFromLib(
409 final TreeBasedTable<Unqualified, Optional<Revision>, SourceSpecificContext> libSourcesTable,
410 final Set<SourceSpecificContext> requiredLibs, final SourceSpecificContext source) {
411 for (final SourceIdentifier requiredSource : source.getRequiredSources()) {
412 final SourceSpecificContext libSource = getRequiredLibSource(requiredSource, libSourcesTable);
413 if (libSource != null && requiredLibs.add(libSource)) {
414 collectRequiredSourcesFromLib(libSourcesTable, requiredLibs, libSource);
419 private static SourceSpecificContext getRequiredLibSource(final SourceIdentifier requiredSource,
420 final TreeBasedTable<Unqualified, Optional<Revision>, SourceSpecificContext> libSourcesTable) {
421 final var revision = requiredSource.revision();
422 return revision != null ? libSourcesTable.get(requiredSource.name(), Optional.of(revision))
423 : getLatestRevision(libSourcesTable.row(requiredSource.name()));
426 private static SourceSpecificContext getLatestRevision(
427 final SortedMap<Optional<Revision>, SourceSpecificContext> sourceMap) {
428 return sourceMap != null && !sourceMap.isEmpty() ? sourceMap.get(sourceMap.lastKey()) : null;
431 // removes required library sources which would cause namespace/name conflict with one of the main sources
432 // later in the parsing process. this can happen if we add a parent module or a submodule as a main source
433 // and the same parent module or submodule is added as one of the library sources.
434 // such situation may occur when using the yang-system-test artifact - if a parent module/submodule is specified
435 // as its argument and the same dir is specified as one of the library dirs through -p option).
436 private static void removeConflictingLibSources(final SourceSpecificContext source,
437 final Set<SourceSpecificContext> requiredLibs) {
438 final Iterator<SourceSpecificContext> requiredLibsIter = requiredLibs.iterator();
439 while (requiredLibsIter.hasNext()) {
440 final SourceSpecificContext currentReqSource = requiredLibsIter.next();
441 if (source.getRootIdentifier().equals(currentReqSource.getRootIdentifier())) {
442 requiredLibsIter.remove();
447 private void endPhase(final ModelProcessingPhase phase) {
448 checkState(currentPhase == phase);
449 finishedPhase = currentPhase;
450 LOG.debug("Global phase {} finished", phase);
453 Set<SourceSpecificContext> getSources() {
457 public Set<YangVersion> getSupportedVersions() {
458 return supportedVersions;
461 void addMutableStmtToSeal(final MutableStatement mutableStatement) {
462 mutableStatementsToSeal.add(mutableStatement);
465 void sealMutableStatements() {
466 for (final MutableStatement mutableStatement : mutableStatementsToSeal) {
467 mutableStatement.seal();
469 mutableStatementsToSeal.clear();