yang/yang-parser-impl/src/main/java/org/opendaylight/yangtools/yang/parser/stmt/reactor/BuildGlobalContext.java

   1 /*
   2  * Copyright (c) 2015 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.yangtools.yang.parser.stmt.reactor;
   9
  10 import com.google.common.base.Preconditions;
  11 import com.google.common.base.Verify;
  12 import com.google.common.collect.HashBasedTable;
  13 import com.google.common.collect.ImmutableList;
  14 import com.google.common.collect.ImmutableMap;
  15 import com.google.common.collect.ImmutableSet;
  16 import com.google.common.collect.Lists;
  17 import com.google.common.collect.Table;
  18 import com.google.common.collect.TreeBasedTable;
  19 import java.util.ArrayList;
  20 import java.util.Collection;
  21 import java.util.Date;
  22 import java.util.HashMap;
  23 import java.util.HashSet;
  24 import java.util.Iterator;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.Map.Entry;
  28 import java.util.Objects;
  29 import java.util.Optional;
  30 import java.util.Set;
  31 import java.util.SortedMap;
  32 import javax.annotation.Nonnull;
  33 import org.opendaylight.yangtools.yang.common.QName;
  34 import org.opendaylight.yangtools.yang.common.QNameModule;
  35 import org.opendaylight.yangtools.yang.common.SimpleDateFormatUtil;
  36 import org.opendaylight.yangtools.yang.common.YangVersion;
  37 import org.opendaylight.yangtools.yang.model.api.ModuleIdentifier;
  38 import org.opendaylight.yangtools.yang.model.api.meta.DeclaredStatement;
  39 import org.opendaylight.yangtools.yang.model.api.meta.EffectiveStatement;
  40 import org.opendaylight.yangtools.yang.model.api.meta.IdentifierNamespace;
  41 import org.opendaylight.yangtools.yang.model.repo.api.SourceIdentifier;
  42 import org.opendaylight.yangtools.yang.model.repo.api.StatementParserMode;
  43 import org.opendaylight.yangtools.yang.parser.spi.meta.DerivedNamespaceBehaviour;
  44 import org.opendaylight.yangtools.yang.parser.spi.meta.ModelProcessingPhase;
  45 import org.opendaylight.yangtools.yang.parser.spi.meta.MutableStatement;
  46 import org.opendaylight.yangtools.yang.parser.spi.meta.NamespaceBehaviour;
  47 import org.opendaylight.yangtools.yang.parser.spi.meta.NamespaceBehaviour.NamespaceStorageNode;
  48 import org.opendaylight.yangtools.yang.parser.spi.meta.NamespaceBehaviour.StorageNodeType;
  49 import org.opendaylight.yangtools.yang.parser.spi.meta.NamespaceNotAvailableException;
  50 import org.opendaylight.yangtools.yang.parser.spi.meta.ReactorException;
  51 import org.opendaylight.yangtools.yang.parser.spi.meta.SomeModifiersUnresolvedException;
  52 import org.opendaylight.yangtools.yang.parser.spi.meta.StatementSupport;
  53 import org.opendaylight.yangtools.yang.parser.spi.meta.StatementSupportBundle;
  54 import org.opendaylight.yangtools.yang.parser.spi.source.ModulesDeviatedByModules;
  55 import org.opendaylight.yangtools.yang.parser.spi.source.ModulesDeviatedByModules.SupportedModules;
  56 import org.opendaylight.yangtools.yang.parser.spi.source.SourceException;
  57 import org.opendaylight.yangtools.yang.parser.spi.source.StatementStreamSource;
  58 import org.opendaylight.yangtools.yang.parser.spi.source.SupportedFeaturesNamespace;
  59 import org.opendaylight.yangtools.yang.parser.spi.source.SupportedFeaturesNamespace.SupportedFeatures;
  60 import org.opendaylight.yangtools.yang.parser.spi.validation.ValidationBundlesNamespace;
  61 import org.opendaylight.yangtools.yang.parser.spi.validation.ValidationBundlesNamespace.ValidationBundleType;
  62 import org.opendaylight.yangtools.yang.parser.stmt.reactor.SourceSpecificContext.PhaseCompletionProgress;
  63 import org.opendaylight.yangtools.yang.parser.stmt.rfc6020.RecursiveObjectLeaker;
  64 import org.opendaylight.yangtools.yang.parser.stmt.rfc6020.Utils;
  65 import org.opendaylight.yangtools.yang.parser.stmt.rfc6020.effective.EffectiveSchemaContext;
  66 import org.slf4j.Logger;
  67 import org.slf4j.LoggerFactory;
  68
  69 class BuildGlobalContext extends NamespaceStorageSupport implements NamespaceBehaviour.Registry {
  70     private static final Logger LOG = LoggerFactory.getLogger(BuildGlobalContext.class);
  71
  72     private static final List<ModelProcessingPhase> PHASE_EXECUTION_ORDER = ImmutableList
  73             .<ModelProcessingPhase> builder().add(ModelProcessingPhase.SOURCE_PRE_LINKAGE)
  74             .add(ModelProcessingPhase.SOURCE_LINKAGE).add(ModelProcessingPhase.STATEMENT_DEFINITION)
  75             .add(ModelProcessingPhase.FULL_DECLARATION).add(ModelProcessingPhase.EFFECTIVE_MODEL).build();
  76
  77     private final Table<YangVersion, QName, StatementDefinitionContext<?, ?, ?>> definitions = HashBasedTable.create();
  78     private final Map<Class<?>, NamespaceBehaviourWithListeners<?, ?, ?>> supportedNamespaces = new HashMap<>();
  79     private final List<MutableStatement> mutableStatementsToSeal = new ArrayList<>();
  80     private final Map<ModelProcessingPhase, StatementSupportBundle> supports;
  81     private final Set<SourceSpecificContext> sources = new HashSet<>();
  82     private final Set<YangVersion> supportedVersions;
  83     private final boolean enabledSemanticVersions;
  84
  85     private Set<SourceSpecificContext> libSources = new HashSet<>();
  86     private ModelProcessingPhase currentPhase = ModelProcessingPhase.INIT;
  87     private ModelProcessingPhase finishedPhase = ModelProcessingPhase.INIT;
  88
  89     BuildGlobalContext(final Map<ModelProcessingPhase, StatementSupportBundle> supports,
  90             final Map<ValidationBundleType, Collection<?>> supportedValidation,
  91             final StatementParserMode statementParserMode) {
  92         this.supports = Preconditions.checkNotNull(supports, "BuildGlobalContext#supports cannot be null");
  93
  94         switch (statementParserMode) {
  95             case DEFAULT_MODE:
  96                 enabledSemanticVersions = false;
  97                 break;
  98             case SEMVER_MODE:
  99                 enabledSemanticVersions = true;
 100                 break;
 101             default:
 102                 throw new IllegalArgumentException("Unhandled parser mode " + statementParserMode);
 103         }
 104
 105         for (final Entry<ValidationBundleType, Collection<?>> validationBundle : supportedValidation.entrySet()) {
 106             addToNs(ValidationBundlesNamespace.class, validationBundle.getKey(), validationBundle.getValue());
 107         }
 108
 109         this.supportedVersions = ImmutableSet.copyOf(supports.get(ModelProcessingPhase.INIT).getSupportedVersions());
 110     }
 111
 112     boolean isEnabledSemanticVersioning() {
 113         return enabledSemanticVersions;
 114     }
 115
 116     StatementSupportBundle getSupportsForPhase(final ModelProcessingPhase currentPhase) {
 117         return supports.get(currentPhase);
 118     }
 119
 120     void addSource(@Nonnull final StatementStreamSource source) {
 121         sources.add(new SourceSpecificContext(this, source));
 122     }
 123
 124     void addLibSource(@Nonnull final StatementStreamSource libSource) {
 125         Preconditions.checkState(!isEnabledSemanticVersioning(),
 126                 "Library sources are not supported in semantic version mode currently.");
 127         Preconditions.checkState(currentPhase == ModelProcessingPhase.INIT,
 128                 "Add library source is allowed in ModelProcessingPhase.INIT only");
 129         libSources.add(new SourceSpecificContext(this, libSource));
 130     }
 131
 132     void setSupportedFeatures(final Set<QName> supportedFeatures) {
 133         addToNs(SupportedFeaturesNamespace.class, SupportedFeatures.SUPPORTED_FEATURES,
 134                     ImmutableSet.copyOf(supportedFeatures));
 135     }
 136
 137     void setModulesDeviatedByModules(final Map<QNameModule, Set<QNameModule>> modulesDeviatedByModules) {
 138         addToNs(ModulesDeviatedByModules.class, SupportedModules.SUPPORTED_MODULES,
 139                     ImmutableMap.copyOf(modulesDeviatedByModules));
 140     }
 141
 142     @Override
 143     public StorageNodeType getStorageNodeType() {
 144         return StorageNodeType.GLOBAL;
 145     }
 146
 147     @Override
 148     public NamespaceStorageNode getParentNamespaceStorage() {
 149         return null;
 150     }
 151
 152     @Override
 153     public NamespaceBehaviour.Registry getBehaviourRegistry() {
 154         return this;
 155     }
 156
 157     @Override
 158     public <K, V, N extends IdentifierNamespace<K, V>> NamespaceBehaviourWithListeners<K, V, N> getNamespaceBehaviour(
 159             final Class<N> type) {
 160         NamespaceBehaviourWithListeners<?, ?, ?> potential = supportedNamespaces.get(type);
 161         if (potential == null) {
 162             final NamespaceBehaviour<K, V, N> potentialRaw = supports.get(currentPhase).getNamespaceBehaviour(type);
 163             if (potentialRaw != null) {
 164                 potential = createNamespaceContext(potentialRaw);
 165                 supportedNamespaces.put(type, potential);
 166             } else {
 167                 throw new NamespaceNotAvailableException("Namespace " + type + " is not available in phase "
 168                         + currentPhase);
 169             }
 170         }
 171
 172         Verify.verify(type.equals(potential.getIdentifier()));
 173         /*
 174          * Safe cast, previous checkState checks equivalence of key from which
 175          * type argument are derived
 176          */
 177         return (NamespaceBehaviourWithListeners<K, V, N>) potential;
 178     }
 179
 180     @SuppressWarnings({ "unchecked", "rawtypes" })
 181     private <K, V, N extends IdentifierNamespace<K, V>> NamespaceBehaviourWithListeners<K, V, N> createNamespaceContext(
 182             final NamespaceBehaviour<K, V, N> potentialRaw) {
 183         if (potentialRaw instanceof DerivedNamespaceBehaviour) {
 184             final VirtualNamespaceContext derivedContext = new VirtualNamespaceContext(
 185                     (DerivedNamespaceBehaviour) potentialRaw);
 186             getNamespaceBehaviour(((DerivedNamespaceBehaviour) potentialRaw).getDerivedFrom()).addDerivedNamespace(
 187                     derivedContext);
 188             return derivedContext;
 189         }
 190         return new SimpleNamespaceContext<>(potentialRaw);
 191     }
 192
 193     StatementDefinitionContext<?, ?, ?> getStatementDefinition(final YangVersion version, final QName name) {
 194         StatementDefinitionContext<?, ?, ?> potential = definitions.get(version, name);
 195         if (potential == null) {
 196             final StatementSupport<?, ?, ?> potentialRaw = supports.get(currentPhase).getStatementDefinition(version,
 197                     name);
 198             if (potentialRaw != null) {
 199                 potential = new StatementDefinitionContext<>(potentialRaw);
 200                 definitions.put(version, name, potential);
 201             }
 202         }
 203         return potential;
 204     }
 205
 206     private void executePhases() throws ReactorException {
 207         for (final ModelProcessingPhase phase : PHASE_EXECUTION_ORDER) {
 208             startPhase(phase);
 209             loadPhaseStatements();
 210             completePhaseActions();
 211             endPhase(phase);
 212         }
 213     }
 214
 215     EffectiveModelContext build() throws ReactorException {
 216         executePhases();
 217         return transform();
 218     }
 219
 220     EffectiveSchemaContext buildEffective() throws ReactorException {
 221         executePhases();
 222         return transformEffective();
 223     }
 224
 225     private EffectiveModelContext transform() {
 226         Preconditions.checkState(finishedPhase == ModelProcessingPhase.EFFECTIVE_MODEL);
 227         final List<DeclaredStatement<?>> rootStatements = new ArrayList<>(sources.size());
 228         for (final SourceSpecificContext source : sources) {
 229             rootStatements.add(source.getRoot().buildDeclared());
 230         }
 231         return new EffectiveModelContext(rootStatements);
 232     }
 233
 234     private SomeModifiersUnresolvedException propagateException(final SourceSpecificContext source,
 235             final RuntimeException cause) throws SomeModifiersUnresolvedException {
 236         final SourceIdentifier sourceId = Utils.createSourceIdentifier(source.getRoot());
 237         if (!(cause instanceof SourceException)) {
 238             /*
 239              * This should not be happening as all our processing should provide SourceExceptions.
 240              * We will wrap the exception to provide enough information to identify the problematic model,
 241              * but also emit a warning so the offending codepath will get fixed.
 242              */
 243             LOG.warn("Unexpected error processing source {}. Please file an issue with this model attached.",
 244                 sourceId, cause);
 245         }
 246
 247         throw new SomeModifiersUnresolvedException(currentPhase, sourceId, cause);
 248     }
 249
 250     private EffectiveSchemaContext transformEffective() throws ReactorException {
 251         Preconditions.checkState(finishedPhase == ModelProcessingPhase.EFFECTIVE_MODEL);
 252         final List<DeclaredStatement<?>> rootStatements = new ArrayList<>(sources.size());
 253         final List<EffectiveStatement<?, ?>> rootEffectiveStatements = new ArrayList<>(sources.size());
 254
 255         try {
 256             for (final SourceSpecificContext source : sources) {
 257                 final RootStatementContext<?, ?, ?> root = source.getRoot();
 258                 try {
 259                     rootStatements.add(root.buildDeclared());
 260                     rootEffectiveStatements.add(root.buildEffective());
 261                 } catch (final RuntimeException ex) {
 262                     throw propagateException(source, ex);
 263                 }
 264             }
 265         } finally {
 266             RecursiveObjectLeaker.cleanup();
 267         }
 268
 269         sealMutableStatements();
 270         return new EffectiveSchemaContext(rootStatements, rootEffectiveStatements);
 271     }
 272
 273     private void startPhase(final ModelProcessingPhase phase) {
 274         Preconditions.checkState(Objects.equals(finishedPhase, phase.getPreviousPhase()));
 275         startPhaseFor(phase, sources);
 276         startPhaseFor(phase, libSources);
 277
 278         currentPhase = phase;
 279         LOG.debug("Global phase {} started", phase);
 280     }
 281
 282     private static void startPhaseFor(final ModelProcessingPhase phase, final Set<SourceSpecificContext> sources) {
 283         for (final SourceSpecificContext source : sources) {
 284             source.startPhase(phase);
 285         }
 286     }
 287
 288     private void loadPhaseStatements() throws ReactorException {
 289         Preconditions.checkState(currentPhase != null);
 290         loadPhaseStatementsFor(sources);
 291         loadPhaseStatementsFor(libSources);
 292     }
 293
 294     private void loadPhaseStatementsFor(final Set<SourceSpecificContext> sources) throws ReactorException {
 295         for (final SourceSpecificContext source : sources) {
 296             try {
 297                 source.loadStatements();
 298             } catch (final RuntimeException ex) {
 299                 throw propagateException(source, ex);
 300             }
 301         }
 302     }
 303
 304     private SomeModifiersUnresolvedException addSourceExceptions(final List<SourceSpecificContext> sourcesToProgress) {
 305         boolean addedCause = false;
 306         SomeModifiersUnresolvedException buildFailure = null;
 307         for (final SourceSpecificContext failedSource : sourcesToProgress) {
 308             final Optional<SourceException> optSourceEx = failedSource.failModifiers(currentPhase);
 309             if (!optSourceEx.isPresent()) {
 310                 continue;
 311             }
 312
 313             final SourceException sourceEx = optSourceEx.get();
 314             // Workaround for broken logging implementations which ignore
 315             // suppressed exceptions
 316             final Throwable cause = sourceEx.getCause() != null ? sourceEx.getCause() : sourceEx;
 317             if (LOG.isDebugEnabled()) {
 318                 LOG.error("Failed to parse YANG from source {}", failedSource, sourceEx);
 319             } else {
 320                 LOG.error("Failed to parse YANG from source {}: {}", failedSource, cause.getMessage());
 321             }
 322
 323             final Throwable[] suppressed = sourceEx.getSuppressed();
 324             if (suppressed.length > 0) {
 325                 LOG.error("{} additional errors reported:", suppressed.length);
 326
 327                 int i = 1;
 328                 for (final Throwable t : suppressed) {
 329                     // FIXME: this should be configured in the appender, really
 330                     if (LOG.isDebugEnabled()) {
 331                         LOG.error("Error {}: {}", i, t.getMessage(), t);
 332                     } else {
 333                         LOG.error("Error {}: {}", i, t.getMessage());
 334                     }
 335
 336                     i++;
 337                 }
 338             }
 339
 340             if (!addedCause) {
 341                 addedCause = true;
 342                 final SourceIdentifier sourceId = Utils.createSourceIdentifier(failedSource.getRoot());
 343                 buildFailure = new SomeModifiersUnresolvedException(currentPhase, sourceId, sourceEx);
 344             } else {
 345                 buildFailure.addSuppressed(sourceEx);
 346             }
 347         }
 348         return buildFailure;
 349     }
 350
 351     private void completePhaseActions() throws ReactorException {
 352         Preconditions.checkState(currentPhase != null);
 353         final List<SourceSpecificContext> sourcesToProgress = Lists.newArrayList(sources);
 354         if (!libSources.isEmpty()) {
 355             Preconditions.checkState(currentPhase == ModelProcessingPhase.SOURCE_PRE_LINKAGE,
 356                     "Yang library sources should be empty after ModelProcessingPhase.SOURCE_PRE_LINKAGE, "
 357                             + "but current phase was %s", currentPhase);
 358             sourcesToProgress.addAll(libSources);
 359         }
 360
 361         boolean progressing = true;
 362         while (progressing) {
 363             // We reset progressing to false.
 364             progressing = false;
 365             final Iterator<SourceSpecificContext> currentSource = sourcesToProgress.iterator();
 366             while (currentSource.hasNext()) {
 367                 final SourceSpecificContext nextSourceCtx = currentSource.next();
 368                 try {
 369                     final PhaseCompletionProgress sourceProgress = nextSourceCtx.tryToCompletePhase(currentPhase);
 370                     switch (sourceProgress) {
 371                         case FINISHED:
 372                             currentSource.remove();
 373                             // Fallback to progress, since we were able to make
 374                             // progress in computation
 375                         case PROGRESS:
 376                             progressing = true;
 377                             break;
 378                         case NO_PROGRESS:
 379                             // Noop
 380                             break;
 381                         default:
 382                             throw new IllegalStateException("Unsupported phase progress " + sourceProgress);
 383                     }
 384                 } catch (final RuntimeException ex) {
 385                     throw propagateException(nextSourceCtx, ex);
 386                 }
 387             }
 388         }
 389
 390         if (!libSources.isEmpty()) {
 391             final Set<SourceSpecificContext> requiredLibs = getRequiredSourcesFromLib();
 392             sources.addAll(requiredLibs);
 393             libSources = ImmutableSet.of();
 394             /*
 395              * We want to report errors of relevant sources only, so any others can
 396              * be removed.
 397              */
 398             sourcesToProgress.retainAll(sources);
 399         }
 400
 401         if (!sourcesToProgress.isEmpty()) {
 402             final SomeModifiersUnresolvedException buildFailure = addSourceExceptions(sourcesToProgress);
 403             if (buildFailure != null) {
 404                 throw buildFailure;
 405             }
 406         }
 407     }
 408
 409     private Set<SourceSpecificContext> getRequiredSourcesFromLib() {
 410         Preconditions.checkState(currentPhase == ModelProcessingPhase.SOURCE_PRE_LINKAGE,
 411                 "Required library sources can be collected only in ModelProcessingPhase.SOURCE_PRE_LINKAGE phase,"
 412                         + " but current phase was %s", currentPhase);
 413         final TreeBasedTable<String, Date, SourceSpecificContext> libSourcesTable = TreeBasedTable.create();
 414         for (final SourceSpecificContext libSource : libSources) {
 415             final ModuleIdentifier libSourceIdentifier = Preconditions.checkNotNull(libSource.getRootIdentifier());
 416             libSourcesTable.put(libSourceIdentifier.getName(), libSourceIdentifier.getRevision(), libSource);
 417         }
 418
 419         final Set<SourceSpecificContext> requiredLibs = new HashSet<>();
 420         for (final SourceSpecificContext source : sources) {
 421             collectRequiredSourcesFromLib(libSourcesTable, requiredLibs, source);
 422         }
 423         return requiredLibs;
 424     }
 425
 426     private void collectRequiredSourcesFromLib(
 427             final TreeBasedTable<String, Date, SourceSpecificContext> libSourcesTable,
 428             final Set<SourceSpecificContext> requiredLibs, final SourceSpecificContext source) {
 429         final Collection<ModuleIdentifier> requiredModules = source.getRequiredModules();
 430         for (final ModuleIdentifier requiredModule : requiredModules) {
 431             final SourceSpecificContext libSource = getRequiredLibSource(requiredModule, libSourcesTable);
 432             if (libSource != null && requiredLibs.add(libSource)) {
 433                 collectRequiredSourcesFromLib(libSourcesTable, requiredLibs, libSource);
 434             }
 435         }
 436     }
 437
 438     private static SourceSpecificContext getRequiredLibSource(final ModuleIdentifier requiredModule,
 439             final TreeBasedTable<String, Date, SourceSpecificContext> libSourcesTable) {
 440         return requiredModule.getRevision() == SimpleDateFormatUtil.DEFAULT_DATE_IMP ? getLatestRevision(libSourcesTable
 441                 .row(requiredModule.getName())) : libSourcesTable.get(requiredModule.getName(),
 442                 requiredModule.getRevision());
 443     }
 444
 445     private static SourceSpecificContext getLatestRevision(final SortedMap<Date, SourceSpecificContext> sourceMap) {
 446         return sourceMap != null && !sourceMap.isEmpty() ? sourceMap.get(sourceMap.lastKey()) : null;
 447     }
 448
 449     private void endPhase(final ModelProcessingPhase phase) {
 450         Preconditions.checkState(currentPhase == phase);
 451         finishedPhase = currentPhase;
 452         LOG.debug("Global phase {} finished", phase);
 453     }
 454
 455     Set<SourceSpecificContext> getSources() {
 456         return sources;
 457     }
 458
 459     public Set<YangVersion> getSupportedVersions() {
 460         return supportedVersions;
 461     }
 462
 463     void addMutableStmtToSeal(final MutableStatement mutableStatement) {
 464         mutableStatementsToSeal.add(mutableStatement);
 465     }
 466
 467     void sealMutableStatements() {
 468         for (final MutableStatement mutableStatement : mutableStatementsToSeal) {
 469             mutableStatement.seal();
 470         }
 471         mutableStatementsToSeal.clear();
 472     }
 473 }