features4-test/src/main/java/org/opendaylight/odlparent/featuretest/SingleFeatureTest.java

   1 /*
   2  * Copyright © 2014, 2017 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.odlparent.featuretest;
  10
  11 import static java.util.concurrent.TimeUnit.SECONDS;
  12 import static org.opendaylight.odlparent.featuretest.Constants.ORG_OPENDAYLIGHT_FEATURETEST_FEATURENAME_PROP;
  13 import static org.opendaylight.odlparent.featuretest.Constants.ORG_OPENDAYLIGHT_FEATURETEST_FEATUREVERSION_PROP;
  14 import static org.opendaylight.odlparent.featuretest.Constants.ORG_OPENDAYLIGHT_FEATURETEST_URI_PROP;
  15 import static org.ops4j.pax.exam.CoreOptions.maven;
  16 import static org.ops4j.pax.exam.CoreOptions.mavenBundle;
  17 import static org.ops4j.pax.exam.CoreOptions.propagateSystemProperty;
  18 import static org.ops4j.pax.exam.CoreOptions.systemPackages;
  19 import static org.ops4j.pax.exam.CoreOptions.when;
  20 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.configureConsole;
  21 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.editConfigurationFilePut;
  22 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.features;
  23 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.karafDistributionConfiguration;
  24 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.keepRuntimeFolder;
  25 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.logLevel;
  26
  27 import com.google.common.collect.ImmutableList;
  28 import java.io.File;
  29 import java.io.IOException;
  30 import java.io.InputStream;
  31 import java.net.URI;
  32 import java.net.URISyntaxException;
  33 import java.util.List;
  34 import java.util.Properties;
  35 import javax.inject.Inject;
  36 import org.apache.karaf.bundle.core.BundleService;
  37 import org.apache.karaf.features.Feature;
  38 import org.apache.karaf.features.FeaturesService;
  39 import org.apache.karaf.features.Repository;
  40 import org.eclipse.jdt.annotation.NonNull;
  41 import org.junit.Assert;
  42 import org.junit.Before;
  43 import org.junit.Test;
  44 import org.junit.runner.RunWith;
  45 import org.opendaylight.odlparent.bundles4test.TestBundleDiag;
  46 import org.ops4j.pax.exam.Configuration;
  47 import org.ops4j.pax.exam.Option;
  48 import org.ops4j.pax.exam.karaf.options.LogLevelOption.LogLevel;
  49 import org.ops4j.pax.exam.options.MavenUrlReference;
  50 import org.ops4j.pax.exam.options.extra.VMOption;
  51 import org.osgi.framework.BundleContext;
  52 import org.slf4j.Logger;
  53 import org.slf4j.LoggerFactory;
  54
  55 @RunWith(PerRepoTestRunner.class)
  56 public class SingleFeatureTest {
  57
  58     private static final String MAVEN_REPO_LOCAL = "maven.repo.local";
  59     private static final String ORG_OPS4J_PAX_URL_MVN_LOCAL_REPOSITORY = "org.ops4j.pax.url.mvn.localRepository";
  60     private static final String ORG_OPS4J_PAX_URL_MVN_REPOSITORIES = "org.ops4j.pax.url.mvn.repositories";
  61     private static final String ETC_ORG_OPS4J_PAX_URL_MVN_CFG = "etc/org.ops4j.pax.url.mvn.cfg";
  62     private static final String ETC_ORG_OPS4J_PAX_LOGGING_CFG = "etc/org.ops4j.pax.logging.cfg";
  63
  64     private static final String KEEP_UNPACK_DIRECTORY_PROP = "karaf.keep.unpack";
  65     private static final String PROFILE_PROP = "karaf.featureTest.profile";
  66     private static final String BUNDLES_DIAG_SKIP_PROP = "sft.diag.skip";
  67     private static final String BUNDLES_DIAG_FORCE_PROP = "sft.diag.force";
  68     private static final String BUNDLES_DIAG_TIMEOUT_PROP = "sft.diag.timeout";
  69
  70     private static final String LOG4J_LOGGER_ORG_OPENDAYLIGHT_YANGTOOLS_FEATURETEST =
  71             "log4j.logger.org.opendaylight.odlparent.featuretest";
  72     private static final Logger LOG = LoggerFactory.getLogger(SingleFeatureTest.class);
  73
  74     /*
  75      * File name to add our logging config property too.
  76      */
  77     private static final String ORG_OPS4J_PAX_LOGGING_CFG = "etc/org.ops4j.pax.logging.cfg";
  78
  79     /*
  80      * Default values for karaf distro type, groupId, and artifactId
  81      */
  82     private static final String KARAF_DISTRO_TYPE = "zip";
  83     private static final String KARAF_DISTRO_ARTIFACTID = "apache-karaf";
  84     private static final String KARAF_DISTRO_GROUPID = "org.apache.karaf";
  85
  86     /*
  87      * Property names to override defaults for karaf distro artifactId, groupId, version, and type
  88      */
  89     private static final String KARAF_DISTRO_VERSION_PROP = "karaf.distro.version";
  90     private static final String KARAF_DISTRO_TYPE_PROP = "karaf.distro.type";
  91     private static final String KARAF_DISTRO_ARTIFACTID_PROP = "karaf.distro.artifactId";
  92     private static final String KARAF_DISTRO_GROUPID_PROP = "karaf.distro.groupId";
  93
  94     /**
  95      * Property file used to store the Karaf distribution version.
  96      */
  97     private static final String PROPERTIES_FILENAME = "singlefeaturetest.properties";
  98
  99     /**
 100      * <p>List of Karaf 3.0.4 default maven repositories with snapshot repositories excluded.</p>
 101      * <p>Unfortunately this must be hard-coded since declarative model which uses Options,
 102      * does not allow us to read value, parse it (properties has allways
 103      * problems with lists) and construct replacement string which does
 104      * not contains snapshots.</p>
 105      * <p>When updating Karaf, check this against org.ops4j.pax.url.mvn.cfg in the Karaf distribution.</p>
 106      */
 107     private static final String EXTERNAL_DEFAULT_REPOSITORIES = "http://repo1.maven.org/maven2@id=central, "
 108             + "http://repository.springsource.com/maven/bundles/release@id=spring.ebr.release, "
 109             + "http://repository.springsource.com/maven/bundles/external@id=spring.ebr.external, "
 110             + "http://zodiac.springsource.com/maven/bundles/release@id=gemini ";
 111
 112     @Inject @NonNull
 113     private BundleContext bundleContext;
 114
 115     @Inject @NonNull
 116     private FeaturesService featuresService;
 117
 118     @Inject @NonNull
 119     private BundleService bundleService; // NOT BundleStateService, see checkBundleStatesDiag()
 120
 121     private String karafVersion;
 122     private String karafDistroVersion;
 123
 124     /**
 125      * Returns the required configuration.
 126      *
 127      * @return The Pax Exam configuration.
 128      * @throws IOException if an error occurs.
 129      */
 130     @Configuration
 131     public Option[] config() throws IOException {
 132         MavenUrlReference bundleTestRepo = maven()
 133                 .groupId("org.opendaylight.odlparent")
 134                 .artifactId("odl-bundles-test")
 135                 .classifier("features")
 136                 .type("xml")
 137                 .versionAsInProject();
 138
 139         return new Option[] {
 140             // TODO: Find a way to inherit memory limits from Maven options.
 141             new VMOption("-Xmx2g"),
 142             // inspired by org.apache.commons.lang.SystemUtils
 143             when(System.getProperty("os.name").toLowerCase().startsWith("linux")).useOptions(
 144                 // This prevents low entropy issues on Linux to affect Java random numbers
 145                 // which can block crypto such as the SSH server in netconf
 146                 // see https://bugs.opendaylight.org/show_bug.cgi?id=6790
 147                 new VMOption("-Djava.security.egd=file:/dev/./urandom")
 148             ),
 149             when(Boolean.getBoolean(PROFILE_PROP)).useOptions(
 150                 new VMOption("-XX:+UnlockCommercialFeatures"),
 151                 new VMOption("-XX:+FlightRecorder"),
 152                 new VMOption("-XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true,dumponexitpath="
 153                                + getNewJFRFile())
 154             ),
 155             getKarafDistroOption(),
 156             when(Boolean.getBoolean(KEEP_UNPACK_DIRECTORY_PROP)).useOptions(keepRuntimeFolder()),
 157             configureConsole().ignoreLocalConsole(),
 158             logLevel(LogLevel.WARN),
 159             mvnLocalRepoOption(),
 160             features(bundleTestRepo, "odl-bundles-test"),
 161             mavenBundle("org.apache.aries.quiesce", "org.apache.aries.quiesce.api", "1.0.0"),
 162             editConfigurationFilePut(ORG_OPS4J_PAX_LOGGING_CFG, LOG4J_LOGGER_ORG_OPENDAYLIGHT_YANGTOOLS_FEATURETEST,
 163                     LogLevel.INFO.name()),
 164             editConfigurationFilePut("etc/config.properties", "karaf.framework", "equinox"),
 165             editConfigurationFilePut(ETC_ORG_OPS4J_PAX_LOGGING_CFG, "log4j.rootLogger", "INFO, stdout, osgi:*"),
 166              /*
 167               *
 168               * Disables external snapshot repositories.
 169               *
 170               * Pax URL and Karaf by default always search for new version of snapshots
 171               * in all snapshots repository, even if that snapshots does not belong to that
 172               * repository and maven is invoked even with -nsu (no snapshot update)
 173               * or offline mode.
 174               *
 175               * This is also true for OpenDaylight snapshot artefacts - pax url tries
 176               * to resolve them from third-party repositories, even if they are not present
 177               * there - this increases time which takes for features to install.
 178               *
 179               * For more complex projects this actually means several HTTP GETs for each
 180               * snapshot bundle referenced, even if the bundle is already present
 181               * in local maven repository.
 182               *
 183               * In order to speed-up installation and remove unnecessary network traffic,
 184               * which fails for obvious reasons, external snapshot repositories are
 185               * removed.
 186               *
 187               *
 188               */
 189             disableExternalSnapshotRepositories(),
 190             propagateSystemProperty(ORG_OPENDAYLIGHT_FEATURETEST_URI_PROP),
 191             propagateSystemProperty(ORG_OPENDAYLIGHT_FEATURETEST_FEATURENAME_PROP),
 192             propagateSystemProperty(ORG_OPENDAYLIGHT_FEATURETEST_FEATUREVERSION_PROP),
 193             propagateSystemProperty(BUNDLES_DIAG_SKIP_PROP),
 194             propagateSystemProperty(BUNDLES_DIAG_FORCE_PROP),
 195             propagateSystemProperty(BUNDLES_DIAG_TIMEOUT_PROP),
 196             // Needed for Agrona/aeron.io
 197             systemPackages("com.sun.media.sound", "sun.nio.ch"),
 198         };
 199     }
 200
 201     private String getNewJFRFile() throws IOException {
 202         return File.createTempFile("SingleFeatureTest-Karaf-JavaFlightRecorder", ".jfr").getAbsolutePath();
 203     }
 204
 205     private String getKarafVersion() throws IOException {
 206         if (karafVersion == null) {
 207             // We use a properties file to retrieve ${karaf.version}, instead of .versionAsInProject()
 208             // This avoids forcing all users to depend on Karaf in their POMs
 209             Properties singleFeatureTestProps = new Properties();
 210             try (InputStream singleFeatureTestInputStream = Thread.currentThread().getContextClassLoader()
 211                     .getResourceAsStream(PROPERTIES_FILENAME)) {
 212                 if (singleFeatureTestInputStream == null) {
 213                     throw new IOException("Resource not found; expected to be present on current thread classloader: "
 214                             + PROPERTIES_FILENAME);
 215                 }
 216                 singleFeatureTestProps.load(singleFeatureTestInputStream);
 217             }
 218             karafVersion = singleFeatureTestProps.getProperty(KARAF_DISTRO_VERSION_PROP);
 219
 220             LOG.info("Retrieved karafVersion {} from properties file {}", karafVersion, PROPERTIES_FILENAME);
 221         } else {
 222             LOG.info("Retrieved karafVersion {} from system property {}", karafVersion, KARAF_DISTRO_VERSION_PROP);
 223         }
 224
 225         return karafVersion;
 226     }
 227
 228     private String getKarafDistroVersion() throws IOException {
 229         if (karafDistroVersion == null) {
 230             karafDistroVersion = System.getProperty(KARAF_DISTRO_VERSION_PROP);
 231             if (karafDistroVersion == null) {
 232                 karafDistroVersion = getKarafVersion();
 233             } else {
 234                 LOG.info("Retrieved karafDistroVersion {} from system property {}", karafVersion,
 235                         KARAF_DISTRO_VERSION_PROP);
 236             }
 237         }
 238
 239         return karafDistroVersion;
 240     }
 241
 242     /**
 243      * Disables snapshot repositories, which are enabled by default in karaf distribution.
 244      *
 245      * @return Edit Configuration option which removes external snapshot repositories.
 246      */
 247     private static Option disableExternalSnapshotRepositories() {
 248         return editConfigurationFilePut(ETC_ORG_OPS4J_PAX_URL_MVN_CFG, ORG_OPS4J_PAX_URL_MVN_REPOSITORIES,
 249                 EXTERNAL_DEFAULT_REPOSITORIES);
 250     }
 251
 252     protected Option mvnLocalRepoOption() {
 253         String mvnRepoLocal = System.getProperty(MAVEN_REPO_LOCAL, "");
 254         LOG.info("mvnLocalRepo \"{}\"", mvnRepoLocal);
 255         return editConfigurationFilePut(ETC_ORG_OPS4J_PAX_URL_MVN_CFG, ORG_OPS4J_PAX_URL_MVN_LOCAL_REPOSITORY,
 256                 mvnRepoLocal);
 257     }
 258
 259     protected Option getKarafDistroOption() throws IOException {
 260         String groupId = System.getProperty(KARAF_DISTRO_GROUPID_PROP, KARAF_DISTRO_GROUPID);
 261         String artifactId = System.getProperty(KARAF_DISTRO_ARTIFACTID_PROP, KARAF_DISTRO_ARTIFACTID);
 262         String type = System.getProperty(KARAF_DISTRO_TYPE_PROP, KARAF_DISTRO_TYPE);
 263         LOG.info("Using karaf distro {} {} {} {}", groupId, artifactId, getKarafDistroVersion(), type);
 264         return karafDistributionConfiguration()
 265                 .frameworkUrl(
 266                         maven()
 267                                 .groupId(groupId)
 268                                 .artifactId(artifactId)
 269                                 .type(type)
 270                                 .version(getKarafDistroVersion()))
 271                 .name("OpenDaylight")
 272                 .unpackDirectory(new File("target/pax"))
 273                 .useDeployFolder(false);
 274     }
 275
 276     private static URI getRepoUri() throws URISyntaxException {
 277         return new URI(getProperty(ORG_OPENDAYLIGHT_FEATURETEST_URI_PROP));
 278     }
 279
 280     private static String getFeatureName() {
 281         return getProperty(ORG_OPENDAYLIGHT_FEATURETEST_FEATURENAME_PROP);
 282     }
 283
 284     public String getFeatureVersion() {
 285         return getProperty(ORG_OPENDAYLIGHT_FEATURETEST_FEATUREVERSION_PROP);
 286     }
 287
 288     private static String getProperty(final String propName) {
 289         String prop = System.getProperty(propName);
 290         Assert.assertTrue("Missing property :" + propName, prop != null);
 291         return prop;
 292     }
 293
 294     private void checkRepository(final URI repoUri) throws Exception {
 295         Repository repo = null;
 296         for (Repository r : featuresService.listRepositories()) {
 297             if (r.getURI().equals(repoUri)) {
 298                 repo = r;
 299                 break;
 300             }
 301         }
 302         Assert.assertNotNull("Repository not found: " + repoUri, repo);
 303     }
 304
 305     /**
 306      * Sets the repository up.
 307      *
 308      * @throws Exception if an error occurs.
 309      */
 310     @Before
 311     public void installRepo() throws Exception {
 312         final URI repoUri = getRepoUri();
 313         LOG.info("Attempting to add repository {}", repoUri);
 314         featuresService.addRepository(repoUri);
 315         checkRepository(repoUri);
 316         LOG.info("Successfully loaded repository {}", repoUri);
 317     }
 318
 319     // Give it 10 minutes max as we've seen feature install hang on jenkins.
 320     @Test(timeout = 600000)
 321     public void installFeature() throws Exception {
 322         LOG.info("Attempting to install feature {} {}", getFeatureName(), getFeatureVersion());
 323         featuresService.installFeature(getFeatureName(), getFeatureVersion());
 324         Feature feature = featuresService.getFeature(getFeatureName(), getFeatureVersion());
 325         Assert.assertNotNull(
 326                 "Attempt to get feature " + getFeatureName() + " " + getFeatureVersion() + "resulted in null", feature);
 327         Assert.assertTrue("Failed to install Feature: " + getFeatureName() + " " + getFeatureVersion(),
 328                 featuresService.isInstalled(feature));
 329         LOG.info("Successfull installed feature {} {}", getFeatureName(), getFeatureVersion());
 330
 331         if (!Boolean.getBoolean(BUNDLES_DIAG_SKIP_PROP)
 332                 && (Boolean.getBoolean(BUNDLES_DIAG_FORCE_PROP)
 333                     || !BLACKLISTED_BROKEN_FEATURES.contains(getFeatureName()))) {
 334             Integer timeOutInSeconds = Integer.getInteger(BUNDLES_DIAG_TIMEOUT_PROP, 5 * 60);
 335             new TestBundleDiag(bundleContext, bundleService).checkBundleDiagInfos(timeOutInSeconds, SECONDS);
 336         } else {
 337             LOG.warn("SKIPPING TestBundleDiag because system property {} is true or feature is blacklisted: {}",
 338                     BUNDLES_DIAG_SKIP_PROP, getFeatureName());
 339         }
 340     }
 341
 342     // TODO remove this when all issues linked to parent https://bugs.opendaylight.org/show_bug.cgi?id=7582 are resolved
 343     private static final List<String> BLACKLISTED_BROKEN_FEATURES = ImmutableList.of(
 344             // integration/distribution/features-test due to DOMRpcService
 345             // see https://bugs.opendaylight.org/show_bug.cgi?id=7595
 346             "odl-integration-all",
 347             // controller/features/mdsal/ due to IllegalStateException: ./configuration/initial/akka.conf is missing
 348             // see https://bugs.opendaylight.org/show_bug.cgi?id=7583
 349             "odl-mdsal-broker-local",
 350             "odl-mdsal-clustering-commons",
 351             "odl-mdsal-distributed-datastore",
 352             "odl-mdsal-remoterpc-connector",
 353             // 1/17 in groupbasedpolicy/features due to NOK org.opendaylight.groupbasedpolicy
 354             // Caused by: org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException
 355             // see https://bugs.opendaylight.org/show_bug.cgi?id=7587
 356             "odl-groupbasedpolicy-ne-location-provider",
 357             // 1/11 in tsdr/features due to (strange) ClassNotFoundException: odlparent.bundlestest
 358             //   .TestBundleDiag (works for all other features; class loading issue in that feature?)
 359             // see https://bugs.opendaylight.org/show_bug.cgi?id=7588
 360             "odl-hbaseclient",
 361             // 1/9 in unimgr/features due missing mdsal, similar to issue to odl-integration-all?
 362             // TODO retry after https://bugs.opendaylight.org/show_bug.cgi?id=7595 is fixed
 363             "odl-unimgr-netvirt"
 364     );
 365 }