features-test/src/main/java/org/opendaylight/odlparent/featuretest/SingleFeatureTest.java

   1 /*
   2  * Copyright (c) 2014 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.odlparent.featuretest;
  10
  11 import static java.util.concurrent.TimeUnit.SECONDS;
  12 import static org.opendaylight.odlparent.featuretest.Constants.ORG_OPENDAYLIGHT_FEATURETEST_FEATURENAME_PROP;
  13 import static org.opendaylight.odlparent.featuretest.Constants.ORG_OPENDAYLIGHT_FEATURETEST_FEATUREVERSION_PROP;
  14 import static org.opendaylight.odlparent.featuretest.Constants.ORG_OPENDAYLIGHT_FEATURETEST_URI_PROP;
  15 import static org.ops4j.pax.exam.CoreOptions.maven;
  16 import static org.ops4j.pax.exam.CoreOptions.mavenBundle;
  17 import static org.ops4j.pax.exam.CoreOptions.when;
  18 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.configureConsole;
  19 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.editConfigurationFilePut;
  20 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.features;
  21 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.karafDistributionConfiguration;
  22 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.keepRuntimeFolder;
  23 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.logLevel;
  24
  25 import com.google.common.collect.ImmutableList;
  26 import java.io.File;
  27 import java.io.IOException;
  28 import java.io.InputStream;
  29 import java.net.URI;
  30 import java.net.URISyntaxException;
  31 import java.net.URL;
  32 import java.util.ArrayList;
  33 import java.util.EnumSet;
  34 import java.util.List;
  35 import java.util.Properties;
  36 import javax.inject.Inject;
  37 import org.apache.karaf.bundle.core.BundleService;
  38 import org.apache.karaf.features.Feature;
  39 import org.apache.karaf.features.FeaturesService;
  40 import org.apache.karaf.features.Repository;
  41 import org.apache.karaf.features.internal.model.Features;
  42 import org.apache.karaf.features.internal.model.JaxbUtil;
  43 import org.eclipse.jdt.annotation.NonNull;
  44 import org.junit.Assert;
  45 import org.junit.Before;
  46 import org.junit.Test;
  47 import org.junit.runner.RunWith;
  48 import org.opendaylight.odlparent.bundlestest.lib.TestBundleDiag;
  49 import org.ops4j.pax.exam.Configuration;
  50 import org.ops4j.pax.exam.CoreOptions;
  51 import org.ops4j.pax.exam.Option;
  52 import org.ops4j.pax.exam.karaf.options.LogLevelOption.LogLevel;
  53 import org.ops4j.pax.exam.options.PropagateSystemPropertyOption;
  54 import org.ops4j.pax.exam.options.extra.VMOption;
  55 import org.osgi.framework.BundleContext;
  56 import org.slf4j.Logger;
  57 import org.slf4j.LoggerFactory;
  58
  59 @RunWith(PerRepoTestRunner.class)
  60 public class SingleFeatureTest {
  61
  62     private static final String MAVEN_REPO_LOCAL = "maven.repo.local";
  63     private static final String ORG_OPS4J_PAX_URL_MVN_LOCAL_REPOSITORY = "org.ops4j.pax.url.mvn.localRepository";
  64     private static final String ORG_OPS4J_PAX_URL_MVN_REPOSITORIES = "org.ops4j.pax.url.mvn.repositories";
  65     private static final String ETC_ORG_OPS4J_PAX_URL_MVN_CFG = "etc/org.ops4j.pax.url.mvn.cfg";
  66     private static final String ETC_ORG_OPS4J_PAX_LOGGING_CFG = "etc/org.ops4j.pax.logging.cfg";
  67
  68     private static final String KEEP_UNPACK_DIRECTORY_PROP = "karaf.keep.unpack";
  69     private static final String PROFILE_PROP = "karaf.featureTest.profile";
  70     private static final String BUNDLES_DIAG_SKIP_PROP = "sft.diag.skip";
  71     private static final String BUNDLES_DIAG_FORCE_PROP = "sft.diag.force";
  72     private static final String BUNDLES_DIAG_TIMEOUT_PROP = "sft.diag.timeout";
  73
  74     private static final String LOG4J_LOGGER_ORG_OPENDAYLIGHT_YANGTOOLS_FEATURETEST =
  75             "log4j.logger.org.opendaylight.odlparent.featuretest";
  76     private static final Logger LOG = LoggerFactory.getLogger(SingleFeatureTest.class);
  77
  78     /*
  79      * File name to add our logging config property too.
  80      */
  81     private static final String ORG_OPS4J_PAX_LOGGING_CFG = "etc/org.ops4j.pax.logging.cfg";
  82
  83     /*
  84      * Default values for karaf distro type, groupId, and artifactId
  85      */
  86     private static final String KARAF_DISTRO_TYPE = "zip";
  87     private static final String KARAF_DISTRO_ARTIFACTID = "apache-karaf";
  88     private static final String KARAF_DISTRO_GROUPID = "org.apache.karaf";
  89
  90     /*
  91      * Property names to override defaults for karaf distro artifactId, groupId, version, and type
  92      */
  93     private static final String KARAF_DISTRO_VERSION_PROP = "karaf.distro.version";
  94     private static final String KARAF_DISTRO_TYPE_PROP = "karaf.distro.type";
  95     private static final String KARAF_DISTRO_ARTIFACTID_PROP = "karaf.distro.artifactId";
  96     private static final String KARAF_DISTRO_GROUPID_PROP = "karaf.distro.groupId";
  97
  98     /**
  99      * Property file used to store the Karaf distribution version.
 100      */
 101     private static final String PROPERTIES_FILENAME = "singlefeaturetest.properties";
 102
 103     /**
 104      * <p>List of Karaf 3.0.4 default maven repositories with snapshot repositories excluded.</p>
 105      * <p>Unfortunately this must be hard-coded since declarative model which uses Options,
 106      * does not allow us to read value, parse it (properties has allways
 107      * problems with lists) and construct replacement string which does
 108      * not contains snapshots.</p>
 109      * <p>When updating Karaf, check this against org.ops4j.pax.url.mvn.cfg in the Karaf distribution.</p>
 110      */
 111     private static final String EXTERNAL_DEFAULT_REPOSITORIES = "http://repo1.maven.org/maven2@id=central, "
 112             + "http://repository.springsource.com/maven/bundles/release@id=spring.ebr.release, "
 113             + "http://repository.springsource.com/maven/bundles/external@id=spring.ebr.external, "
 114             + "http://zodiac.springsource.com/maven/bundles/release@id=gemini ";
 115
 116     @Inject @NonNull
 117     private BundleContext bundleContext;
 118
 119     @Inject @NonNull
 120     private FeaturesService featuresService;
 121
 122     @Inject @NonNull
 123     private BundleService bundleService; // NOT BundleStateService, see checkBundleStatesDiag()
 124
 125     private String karafVersion;
 126     private String karafDistroVersion;
 127
 128     /**
 129      * Returns the required configuration.
 130      *
 131      * @return The Pax Exam configuration.
 132      * @throws IOException if an error occurs.
 133      */
 134     @Configuration
 135     public Option[] config() throws IOException {
 136         return new Option[] {
 137             // TODO: Find a way to inherit memory limits from Maven options.
 138             new VMOption("-Xmx2g"),
 139             new VMOption("-XX:+HeapDumpOnOutOfMemoryError"),
 140             new VMOption("-XX:OnOutOfMemoryError=\"kill -3 %p\""),
 141             // inspired by org.apache.commons.lang.SystemUtils
 142             when(System.getProperty("os.name").toLowerCase().startsWith("linux")).useOptions(
 143                 // This prevents low entropy issues on Linux to affect Java random numbers
 144                 // which can block crypto such as the SSH server in netconf
 145                 // see https://bugs.opendaylight.org/show_bug.cgi?id=6790
 146                 new VMOption("-Djava.security.egd=file:/dev/./urandom")
 147             ),
 148             when(Boolean.getBoolean(PROFILE_PROP)).useOptions(
 149                 new VMOption("-XX:+UnlockCommercialFeatures"),
 150                 new VMOption("-XX:+FlightRecorder"),
 151                 new VMOption("-XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true,dumponexitpath="
 152                                + getNewJFRFile())
 153             ),
 154             getKarafDistroOption(),
 155             when(Boolean.getBoolean(KEEP_UNPACK_DIRECTORY_PROP)).useOptions(keepRuntimeFolder()),
 156             configureConsole().ignoreLocalConsole(),
 157             logLevel(LogLevel.WARN),
 158             mvnLocalRepoOption(),
 159             standardKarafFeatures(),
 160             // Guava is used by this SingleFeatureTest class itself (not, anymore, by by bundles-test),
 161             // in (at least) the BLACKLISTED_BROKEN_FEATURES ImmutableList at the end, so we need to
 162             // add Guava so that this class can run in Karaf under OSGi, as it compiled here
 163             mavenBundle(maven("com.google.guava", "guava").versionAsInProject()),
 164             mavenBundle(maven("org.opendaylight.odlparent", "bundles-test").versionAsInProject()),
 165             editConfigurationFilePut(ORG_OPS4J_PAX_LOGGING_CFG, LOG4J_LOGGER_ORG_OPENDAYLIGHT_YANGTOOLS_FEATURETEST,
 166                     LogLevel.INFO.name()),
 167             editConfigurationFilePut(ETC_ORG_OPS4J_PAX_LOGGING_CFG, "log4j.rootLogger", "INFO, stdout, osgi:*"),
 168              /*
 169               *
 170               * Disables external snapshot repositories.
 171               *
 172               * Pax URL and Karaf by default always search for new version of snapshots
 173               * in all snapshots repository, even if that snapshots does not belong to that
 174               * repository and maven is invoked even with -nsu (no snapshot update)
 175               * or offline mode.
 176               *
 177               * This is also true for OpenDaylight snapshot artefacts - pax url tries
 178               * to resolve them from third-party repositories, even if they are not present
 179               * there - this increases time which takes for features to install.
 180               *
 181               * For more complex projects this actually means several HTTP GETs for each
 182               * snapshot bundle referenced, even if the bundle is already present
 183               * in local maven repository.
 184               *
 185               * In order to speed-up installation and remove unnecessary network traffic,
 186               * which fails for obvious reasons, external snapshot repositories are
 187               * removed.
 188               *
 189               *
 190               */
 191             disableExternalSnapshotRepositories(),
 192             new PropagateSystemPropertyOption(ORG_OPENDAYLIGHT_FEATURETEST_URI_PROP),
 193             new PropagateSystemPropertyOption(ORG_OPENDAYLIGHT_FEATURETEST_FEATURENAME_PROP),
 194             new PropagateSystemPropertyOption(ORG_OPENDAYLIGHT_FEATURETEST_FEATUREVERSION_PROP),
 195             new PropagateSystemPropertyOption(BUNDLES_DIAG_SKIP_PROP),
 196             new PropagateSystemPropertyOption(BUNDLES_DIAG_FORCE_PROP),
 197             new PropagateSystemPropertyOption(BUNDLES_DIAG_TIMEOUT_PROP),
 198             // Needed for Agrona/aeron.io
 199             CoreOptions.systemPackages("com.sun.media.sound", "sun.nio.ch"),
 200         };
 201     }
 202
 203     private String getNewJFRFile() throws IOException {
 204         return File.createTempFile("SingleFeatureTest-Karaf-JavaFlightRecorder", ".jfr").getAbsolutePath();
 205     }
 206
 207     private Option standardKarafFeatures() throws IOException {
 208         String url = maven().groupId("org.apache.karaf.features").artifactId("standard").classifier("features").type(
 209                 "xml").version(getKarafVersion()).getURL();
 210         try {
 211             Features features = JaxbUtil.unmarshal(new URL(url).openStream(), false);
 212             List<String> featureNames = new ArrayList<>();
 213             for (Feature f : features.getFeature()) {
 214                 featureNames.add(f.getName());
 215             }
 216
 217             return features(url, featureNames.toArray(new String[featureNames.size()]));
 218         } catch (IOException e) {
 219             throw new IOException("Could not obtain features from URL: " + url, e);
 220         }
 221     }
 222
 223     private String getKarafVersion() throws IOException {
 224         if (karafVersion == null) {
 225             // We use a properties file to retrieve ${karaf.version}, instead of .versionAsInProject()
 226             // This avoids forcing all users to depend on Karaf in their POMs
 227             Properties singleFeatureTestProps = new Properties();
 228             try (InputStream singleFeatureTestInputStream = Thread.currentThread().getContextClassLoader()
 229                     .getResourceAsStream(PROPERTIES_FILENAME)) {
 230                 if (singleFeatureTestInputStream == null) {
 231                     throw new IOException("Resource not found; expected to be present on current thread classloader: "
 232                             + PROPERTIES_FILENAME);
 233                 }
 234                 singleFeatureTestProps.load(singleFeatureTestInputStream);
 235             }
 236             karafVersion = singleFeatureTestProps.getProperty(KARAF_DISTRO_VERSION_PROP);
 237
 238             LOG.info("Retrieved karafVersion {} from properties file {}", karafVersion, PROPERTIES_FILENAME);
 239         } else {
 240             LOG.info("Retrieved karafVersion {} from system property {}", karafVersion, KARAF_DISTRO_VERSION_PROP);
 241         }
 242
 243         return karafVersion;
 244     }
 245
 246     private String getKarafDistroVersion() throws IOException {
 247         if (karafDistroVersion == null) {
 248             karafDistroVersion = System.getProperty(KARAF_DISTRO_VERSION_PROP);
 249             if (karafDistroVersion == null) {
 250                 karafDistroVersion = getKarafVersion();
 251             } else {
 252                 LOG.info("Retrieved karafDistroVersion {} from system property {}", karafVersion,
 253                         KARAF_DISTRO_VERSION_PROP);
 254             }
 255         }
 256
 257         return karafDistroVersion;
 258     }
 259
 260     /**
 261      * Disables snapshot repositories, which are enabled by default in karaf distribution.
 262      *
 263      * @return Edit Configuration option which removes external snapshot repositories.
 264      */
 265     private static Option disableExternalSnapshotRepositories() {
 266         return editConfigurationFilePut(ETC_ORG_OPS4J_PAX_URL_MVN_CFG, ORG_OPS4J_PAX_URL_MVN_REPOSITORIES,
 267                 EXTERNAL_DEFAULT_REPOSITORIES);
 268     }
 269
 270     protected Option mvnLocalRepoOption() {
 271         String mvnRepoLocal = System.getProperty(MAVEN_REPO_LOCAL, "");
 272         LOG.info("mvnLocalRepo \"{}\"", mvnRepoLocal);
 273         return editConfigurationFilePut(ETC_ORG_OPS4J_PAX_URL_MVN_CFG, ORG_OPS4J_PAX_URL_MVN_LOCAL_REPOSITORY,
 274                 mvnRepoLocal);
 275     }
 276
 277     protected Option getKarafDistroOption() throws IOException {
 278         String groupId = System.getProperty(KARAF_DISTRO_GROUPID_PROP, KARAF_DISTRO_GROUPID);
 279         String artifactId = System.getProperty(KARAF_DISTRO_ARTIFACTID_PROP, KARAF_DISTRO_ARTIFACTID);
 280         String type = System.getProperty(KARAF_DISTRO_TYPE_PROP, KARAF_DISTRO_TYPE);
 281         LOG.info("Using karaf distro {} {} {} {}", groupId, artifactId, getKarafDistroVersion(), type);
 282         return karafDistributionConfiguration()
 283                 .frameworkUrl(
 284                         maven()
 285                                 .groupId(groupId)
 286                                 .artifactId(artifactId)
 287                                 .type(type)
 288                                 .version(getKarafDistroVersion()))
 289                 .name("OpenDaylight")
 290                 .unpackDirectory(new File("target/pax"))
 291                 .useDeployFolder(false);
 292     }
 293
 294     private static URI getRepoUri() throws URISyntaxException {
 295         return new URI(getProperty(ORG_OPENDAYLIGHT_FEATURETEST_URI_PROP));
 296     }
 297
 298     private static String getFeatureName() {
 299         return getProperty(ORG_OPENDAYLIGHT_FEATURETEST_FEATURENAME_PROP);
 300     }
 301
 302     public String getFeatureVersion() {
 303         return getProperty(ORG_OPENDAYLIGHT_FEATURETEST_FEATUREVERSION_PROP);
 304     }
 305
 306     private static String getProperty(final String propName) {
 307         String prop = System.getProperty(propName);
 308         Assert.assertTrue("Missing property :" + propName, prop != null);
 309         return prop;
 310     }
 311
 312     private void checkRepository(final URI repoUri) {
 313         Repository repo = null;
 314         for (Repository r : featuresService.listRepositories()) {
 315             if (r.getURI().equals(repoUri)) {
 316                 repo = r;
 317                 break;
 318             }
 319         }
 320         Assert.assertNotNull("Repository not found: " + repoUri, repo);
 321     }
 322
 323     /**
 324      * Sets the repository up.
 325      *
 326      * @throws Exception if an error occurs.
 327      */
 328     @Before
 329     public void installRepo() throws Exception {
 330         final URI repoUri = getRepoUri();
 331         LOG.info("Attempting to add repository {}", repoUri);
 332         featuresService.addRepository(repoUri);
 333         checkRepository(repoUri);
 334         LOG.info("Successfully loaded repository {}", repoUri);
 335     }
 336
 337     // Give it 10 minutes max as we've seen feature install hang on jenkins.
 338     @Test(timeout = 600000)
 339     public void installFeature() throws Exception {
 340         LOG.info("Attempting to install feature {} {}", getFeatureName(), getFeatureVersion());
 341         featuresService.installFeature(getFeatureName(), getFeatureVersion(),
 342                 EnumSet.of(FeaturesService.Option.NoCleanIfFailure, FeaturesService.Option.PrintExecptionPerFeature));
 343         Feature feature = featuresService.getFeature(getFeatureName(), getFeatureVersion());
 344         Assert.assertNotNull(
 345                 "Attempt to get feature " + getFeatureName() + " " + getFeatureVersion() + "resulted in null", feature);
 346         Assert.assertTrue("Failed to install Feature: " + getFeatureName() + " " + getFeatureVersion(),
 347                 featuresService.isInstalled(feature));
 348         LOG.info("Successfull installed feature {} {}", getFeatureName(), getFeatureVersion());
 349
 350         if (!Boolean.getBoolean(BUNDLES_DIAG_SKIP_PROP)
 351                 && (Boolean.getBoolean(BUNDLES_DIAG_FORCE_PROP)
 352                     || !BLACKLISTED_BROKEN_FEATURES.contains(getFeatureName()))) {
 353             Integer timeOutInSeconds = Integer.getInteger(BUNDLES_DIAG_TIMEOUT_PROP, 5 * 60);
 354             new TestBundleDiag(bundleContext, bundleService).checkBundleDiagInfos(timeOutInSeconds, SECONDS);
 355         } else {
 356             LOG.warn("SKIPPING TestBundleDiag because system property {} is true or feature is blacklisted: {}",
 357                     BUNDLES_DIAG_SKIP_PROP, getFeatureName());
 358         }
 359     }
 360
 361     // TODO remove this when all issues linked to parent https://bugs.opendaylight.org/show_bug.cgi?id=7582 are resolved
 362     private static final List<String> BLACKLISTED_BROKEN_FEATURES = ImmutableList.of(
 363             // integration/distribution/features-test due to DOMRpcService
 364             // see https://bugs.opendaylight.org/show_bug.cgi?id=7595
 365             "odl-integration-all",
 366             // controller/features/mdsal/ due to IllegalStateException: ./configuration/initial/akka.conf is missing
 367             // see https://bugs.opendaylight.org/show_bug.cgi?id=7583
 368             "odl-mdsal-broker-local",
 369             "odl-mdsal-clustering-commons",
 370             "odl-mdsal-distributed-datastore",
 371             "odl-mdsal-remoterpc-connector",
 372             // 1/17 in groupbasedpolicy/features due to NOK org.opendaylight.groupbasedpolicy
 373             // Caused by: org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException
 374             // see https://bugs.opendaylight.org/show_bug.cgi?id=7587
 375             "odl-groupbasedpolicy-ne-location-provider",
 376             // 1/11 in tsdr/features due to (strange) ClassNotFoundException: odlparent.bundlestest
 377             //   .TestBundleDiag (works for all other features; class loading issue in that feature?)
 378             // see https://bugs.opendaylight.org/show_bug.cgi?id=7588
 379             // Check tsdr-verify-carbon on Sandbox before removing this.
 380             "odl-hbaseclient",
 381             // 1/9 in unimgr/features due missing mdsal, similar to issue to odl-integration-all?
 382             // TODO retry after https://bugs.opendaylight.org/show_bug.cgi?id=7595 is fixed
 383             "odl-unimgr-netvirt"
 384     );
 385 }