features4-test/src/main/java/org/opendaylight/odlparent/featuretest/SingleFeatureTest.java

   1 /*
   2  * Copyright © 2014, 2017 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.odlparent.featuretest;
  10
  11 import static java.util.concurrent.TimeUnit.SECONDS;
  12 import static org.opendaylight.odlparent.featuretest.Constants.ORG_OPENDAYLIGHT_FEATURETEST_FEATURENAME_PROP;
  13 import static org.opendaylight.odlparent.featuretest.Constants.ORG_OPENDAYLIGHT_FEATURETEST_FEATUREVERSION_PROP;
  14 import static org.opendaylight.odlparent.featuretest.Constants.ORG_OPENDAYLIGHT_FEATURETEST_URI_PROP;
  15 import static org.ops4j.pax.exam.CoreOptions.maven;
  16 import static org.ops4j.pax.exam.CoreOptions.mavenBundle;
  17 import static org.ops4j.pax.exam.CoreOptions.propagateSystemProperty;
  18 import static org.ops4j.pax.exam.CoreOptions.systemPackages;
  19 import static org.ops4j.pax.exam.CoreOptions.when;
  20 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.configureConsole;
  21 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.editConfigurationFilePut;
  22 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.features;
  23 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.karafDistributionConfiguration;
  24 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.keepRuntimeFolder;
  25 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.logLevel;
  26
  27 import com.google.common.collect.ImmutableList;
  28 import java.io.File;
  29 import java.io.IOException;
  30 import java.io.InputStream;
  31 import java.net.URI;
  32 import java.net.URISyntaxException;
  33 import java.util.List;
  34 import java.util.Properties;
  35 import javax.inject.Inject;
  36 import org.apache.karaf.bundle.core.BundleService;
  37 import org.apache.karaf.features.Feature;
  38 import org.apache.karaf.features.FeaturesService;
  39 import org.apache.karaf.features.Repository;
  40 import org.eclipse.jdt.annotation.NonNull;
  41 import org.junit.Assert;
  42 import org.junit.Before;
  43 import org.junit.Test;
  44 import org.junit.runner.RunWith;
  45 import org.opendaylight.odlparent.bundles4test.TestBundleDiag;
  46 import org.ops4j.pax.exam.Configuration;
  47 import org.ops4j.pax.exam.Option;
  48 import org.ops4j.pax.exam.karaf.options.LogLevelOption.LogLevel;
  49 import org.ops4j.pax.exam.options.MavenUrlReference;
  50 import org.ops4j.pax.exam.options.extra.VMOption;
  51 import org.osgi.framework.BundleContext;
  52 import org.slf4j.Logger;
  53 import org.slf4j.LoggerFactory;
  54
  55 @RunWith(PerRepoTestRunner.class)
  56 public class SingleFeatureTest {
  57
  58     private static final String MAVEN_REPO_LOCAL = "maven.repo.local";
  59     private static final String ORG_OPS4J_PAX_URL_MVN_LOCAL_REPOSITORY = "org.ops4j.pax.url.mvn.localRepository";
  60     private static final String ORG_OPS4J_PAX_URL_MVN_REPOSITORIES = "org.ops4j.pax.url.mvn.repositories";
  61     private static final String ETC_ORG_OPS4J_PAX_URL_MVN_CFG = "etc/org.ops4j.pax.url.mvn.cfg";
  62     private static final String ETC_ORG_OPS4J_PAX_LOGGING_CFG = "etc/org.ops4j.pax.logging.cfg";
  63
  64     private static final String KEEP_UNPACK_DIRECTORY_PROP = "karaf.keep.unpack";
  65     private static final String PROFILE_PROP = "karaf.featureTest.profile";
  66     private static final String BUNDLES_DIAG_SKIP_PROP = "sft.diag.skip";
  67     private static final String BUNDLES_DIAG_FORCE_PROP = "sft.diag.force";
  68     private static final String BUNDLES_DIAG_TIMEOUT_PROP = "sft.diag.timeout";
  69
  70     private static final String LOG4J_LOGGER_ORG_OPENDAYLIGHT_YANGTOOLS_FEATURETEST =
  71             "log4j.logger.org.opendaylight.odlparent.featuretest";
  72     private static final Logger LOG = LoggerFactory.getLogger(SingleFeatureTest.class);
  73
  74     /*
  75      * File name to add our logging config property too.
  76      */
  77     private static final String ORG_OPS4J_PAX_LOGGING_CFG = "etc/org.ops4j.pax.logging.cfg";
  78
  79     /*
  80      * Default values for karaf distro type, groupId, and artifactId
  81      */
  82     private static final String KARAF_DISTRO_TYPE = "zip";
  83     private static final String KARAF_DISTRO_ARTIFACTID = "apache-karaf";
  84     private static final String KARAF_DISTRO_GROUPID = "org.apache.karaf";
  85
  86     /*
  87      * Property names to override defaults for karaf distro artifactId, groupId, version, and type
  88      */
  89     private static final String KARAF_DISTRO_VERSION_PROP = "karaf.distro.version";
  90     private static final String KARAF_DISTRO_TYPE_PROP = "karaf.distro.type";
  91     private static final String KARAF_DISTRO_ARTIFACTID_PROP = "karaf.distro.artifactId";
  92     private static final String KARAF_DISTRO_GROUPID_PROP = "karaf.distro.groupId";
  93
  94     /**
  95      * Property file used to store the Karaf distribution version.
  96      */
  97     private static final String PROPERTIES_FILENAME = "singlefeaturetest.properties";
  98
  99     /**
 100      * <p>List of Karaf 3.0.4 default maven repositories with snapshot repositories excluded.</p>
 101      * <p>Unfortunately this must be hard-coded since declarative model which uses Options,
 102      * does not allow us to read value, parse it (properties has allways
 103      * problems with lists) and construct replacement string which does
 104      * not contains snapshots.</p>
 105      * <p>When updating Karaf, check this against org.ops4j.pax.url.mvn.cfg in the Karaf distribution.</p>
 106      */
 107     private static final String EXTERNAL_DEFAULT_REPOSITORIES = "http://repo1.maven.org/maven2@id=central, "
 108             + "http://repository.springsource.com/maven/bundles/release@id=spring.ebr.release, "
 109             + "http://repository.springsource.com/maven/bundles/external@id=spring.ebr.external, "
 110             + "http://zodiac.springsource.com/maven/bundles/release@id=gemini ";
 111
 112     @Inject @NonNull
 113     private BundleContext bundleContext;
 114
 115     @Inject @NonNull
 116     private FeaturesService featuresService;
 117
 118     @Inject @NonNull
 119     private BundleService bundleService; // NOT BundleStateService, see checkBundleStatesDiag()
 120
 121     private String karafVersion;
 122     private String karafDistroVersion;
 123
 124     /**
 125      * Returns the required configuration.
 126      *
 127      * @return The Pax Exam configuration.
 128      * @throws IOException if an error occurs.
 129      */
 130     @Configuration
 131     public Option[] config() throws IOException {
 132         MavenUrlReference bundleTestRepo = maven()
 133                 .groupId("org.opendaylight.odlparent")
 134                 .artifactId("odl-bundles-test")
 135                 .classifier("features")
 136                 .type("xml")
 137                 .versionAsInProject();
 138
 139         return new Option[] {
 140             // TODO: Find a way to inherit memory limits from Maven options.
 141             new VMOption("-Xmx2g"),
 142             new VMOption("-XX:+HeapDumpOnOutOfMemoryError"),
 143             new VMOption("-XX:OnOutOfMemoryError=\"kill -3 %p\""),
 144             // inspired by org.apache.commons.lang.SystemUtils
 145             when(System.getProperty("os.name").toLowerCase().startsWith("linux")).useOptions(
 146                 // This prevents low entropy issues on Linux to affect Java random numbers
 147                 // which can block crypto such as the SSH server in netconf
 148                 // see https://bugs.opendaylight.org/show_bug.cgi?id=6790
 149                 new VMOption("-Djava.security.egd=file:/dev/./urandom")
 150             ),
 151             when(Boolean.getBoolean(PROFILE_PROP)).useOptions(
 152                 new VMOption("-XX:+UnlockCommercialFeatures"),
 153                 new VMOption("-XX:+FlightRecorder"),
 154                 new VMOption("-XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true,dumponexitpath="
 155                                + getNewJFRFile())
 156             ),
 157             getKarafDistroOption(),
 158             when(Boolean.getBoolean(KEEP_UNPACK_DIRECTORY_PROP)).useOptions(keepRuntimeFolder()),
 159             configureConsole().ignoreLocalConsole(),
 160             logLevel(LogLevel.WARN),
 161             mvnLocalRepoOption(),
 162             features(bundleTestRepo, "odl-bundles-test"),
 163             mavenBundle("org.apache.aries.quiesce", "org.apache.aries.quiesce.api", "1.0.0"),
 164             editConfigurationFilePut(ORG_OPS4J_PAX_LOGGING_CFG, LOG4J_LOGGER_ORG_OPENDAYLIGHT_YANGTOOLS_FEATURETEST,
 165                     LogLevel.INFO.name()),
 166             editConfigurationFilePut("etc/config.properties", "karaf.framework", "equinox"),
 167             editConfigurationFilePut(ETC_ORG_OPS4J_PAX_LOGGING_CFG, "log4j.rootLogger", "INFO, stdout, osgi:*"),
 168              /*
 169               *
 170               * Disables external snapshot repositories.
 171               *
 172               * Pax URL and Karaf by default always search for new version of snapshots
 173               * in all snapshots repository, even if that snapshots does not belong to that
 174               * repository and maven is invoked even with -nsu (no snapshot update)
 175               * or offline mode.
 176               *
 177               * This is also true for OpenDaylight snapshot artefacts - pax url tries
 178               * to resolve them from third-party repositories, even if they are not present
 179               * there - this increases time which takes for features to install.
 180               *
 181               * For more complex projects this actually means several HTTP GETs for each
 182               * snapshot bundle referenced, even if the bundle is already present
 183               * in local maven repository.
 184               *
 185               * In order to speed-up installation and remove unnecessary network traffic,
 186               * which fails for obvious reasons, external snapshot repositories are
 187               * removed.
 188               *
 189               *
 190               */
 191             disableExternalSnapshotRepositories(),
 192             propagateSystemProperty(ORG_OPENDAYLIGHT_FEATURETEST_URI_PROP),
 193             propagateSystemProperty(ORG_OPENDAYLIGHT_FEATURETEST_FEATURENAME_PROP),
 194             propagateSystemProperty(ORG_OPENDAYLIGHT_FEATURETEST_FEATUREVERSION_PROP),
 195             propagateSystemProperty(BUNDLES_DIAG_SKIP_PROP),
 196             propagateSystemProperty(BUNDLES_DIAG_FORCE_PROP),
 197             propagateSystemProperty(BUNDLES_DIAG_TIMEOUT_PROP),
 198             // Needed for Agrona/aeron.io
 199             systemPackages("com.sun.media.sound", "sun.nio.ch"),
 200         };
 201     }
 202
 203     private String getNewJFRFile() throws IOException {
 204         return File.createTempFile("SingleFeatureTest-Karaf-JavaFlightRecorder", ".jfr").getAbsolutePath();
 205     }
 206
 207     private String getKarafVersion() throws IOException {
 208         if (karafVersion == null) {
 209             // We use a properties file to retrieve ${karaf.version}, instead of .versionAsInProject()
 210             // This avoids forcing all users to depend on Karaf in their POMs
 211             Properties singleFeatureTestProps = new Properties();
 212             try (InputStream singleFeatureTestInputStream = Thread.currentThread().getContextClassLoader()
 213                     .getResourceAsStream(PROPERTIES_FILENAME)) {
 214                 if (singleFeatureTestInputStream == null) {
 215                     throw new IOException("Resource not found; expected to be present on current thread classloader: "
 216                             + PROPERTIES_FILENAME);
 217                 }
 218                 singleFeatureTestProps.load(singleFeatureTestInputStream);
 219             }
 220             karafVersion = singleFeatureTestProps.getProperty(KARAF_DISTRO_VERSION_PROP);
 221
 222             LOG.info("Retrieved karafVersion {} from properties file {}", karafVersion, PROPERTIES_FILENAME);
 223         } else {
 224             LOG.info("Retrieved karafVersion {} from system property {}", karafVersion, KARAF_DISTRO_VERSION_PROP);
 225         }
 226
 227         return karafVersion;
 228     }
 229
 230     private String getKarafDistroVersion() throws IOException {
 231         if (karafDistroVersion == null) {
 232             karafDistroVersion = System.getProperty(KARAF_DISTRO_VERSION_PROP);
 233             if (karafDistroVersion == null) {
 234                 karafDistroVersion = getKarafVersion();
 235             } else {
 236                 LOG.info("Retrieved karafDistroVersion {} from system property {}", karafVersion,
 237                         KARAF_DISTRO_VERSION_PROP);
 238             }
 239         }
 240
 241         return karafDistroVersion;
 242     }
 243
 244     /**
 245      * Disables snapshot repositories, which are enabled by default in karaf distribution.
 246      *
 247      * @return Edit Configuration option which removes external snapshot repositories.
 248      */
 249     private static Option disableExternalSnapshotRepositories() {
 250         return editConfigurationFilePut(ETC_ORG_OPS4J_PAX_URL_MVN_CFG, ORG_OPS4J_PAX_URL_MVN_REPOSITORIES,
 251                 EXTERNAL_DEFAULT_REPOSITORIES);
 252     }
 253
 254     protected Option mvnLocalRepoOption() {
 255         String mvnRepoLocal = System.getProperty(MAVEN_REPO_LOCAL, "");
 256         LOG.info("mvnLocalRepo \"{}\"", mvnRepoLocal);
 257         return editConfigurationFilePut(ETC_ORG_OPS4J_PAX_URL_MVN_CFG, ORG_OPS4J_PAX_URL_MVN_LOCAL_REPOSITORY,
 258                 mvnRepoLocal);
 259     }
 260
 261     protected Option getKarafDistroOption() throws IOException {
 262         String groupId = System.getProperty(KARAF_DISTRO_GROUPID_PROP, KARAF_DISTRO_GROUPID);
 263         String artifactId = System.getProperty(KARAF_DISTRO_ARTIFACTID_PROP, KARAF_DISTRO_ARTIFACTID);
 264         String type = System.getProperty(KARAF_DISTRO_TYPE_PROP, KARAF_DISTRO_TYPE);
 265         LOG.info("Using karaf distro {} {} {} {}", groupId, artifactId, getKarafDistroVersion(), type);
 266         return karafDistributionConfiguration()
 267                 .frameworkUrl(
 268                         maven()
 269                                 .groupId(groupId)
 270                                 .artifactId(artifactId)
 271                                 .type(type)
 272                                 .version(getKarafDistroVersion()))
 273                 .name("OpenDaylight")
 274                 .unpackDirectory(new File("target/pax"))
 275                 .useDeployFolder(false);
 276     }
 277
 278     private static URI getRepoUri() throws URISyntaxException {
 279         return new URI(getProperty(ORG_OPENDAYLIGHT_FEATURETEST_URI_PROP));
 280     }
 281
 282     private static String getFeatureName() {
 283         return getProperty(ORG_OPENDAYLIGHT_FEATURETEST_FEATURENAME_PROP);
 284     }
 285
 286     public String getFeatureVersion() {
 287         return getProperty(ORG_OPENDAYLIGHT_FEATURETEST_FEATUREVERSION_PROP);
 288     }
 289
 290     private static String getProperty(final String propName) {
 291         String prop = System.getProperty(propName);
 292         Assert.assertTrue("Missing property :" + propName, prop != null);
 293         return prop;
 294     }
 295
 296     private void checkRepository(final URI repoUri) throws Exception {
 297         Repository repo = null;
 298         for (Repository r : featuresService.listRepositories()) {
 299             if (r.getURI().equals(repoUri)) {
 300                 repo = r;
 301                 break;
 302             }
 303         }
 304         Assert.assertNotNull("Repository not found: " + repoUri, repo);
 305     }
 306
 307     /**
 308      * Sets the repository up.
 309      *
 310      * @throws Exception if an error occurs.
 311      */
 312     @Before
 313     public void installRepo() throws Exception {
 314         final URI repoUri = getRepoUri();
 315         LOG.info("Attempting to add repository {}", repoUri);
 316         featuresService.addRepository(repoUri);
 317         checkRepository(repoUri);
 318         LOG.info("Successfully loaded repository {}", repoUri);
 319     }
 320
 321     // Give it 10 minutes max as we've seen feature install hang on jenkins.
 322     @Test(timeout = 600000)
 323     public void installFeature() throws Exception {
 324         LOG.info("Attempting to install feature {} {}", getFeatureName(), getFeatureVersion());
 325         featuresService.installFeature(getFeatureName(), getFeatureVersion());
 326         Feature feature = featuresService.getFeature(getFeatureName(), getFeatureVersion());
 327         Assert.assertNotNull(
 328                 "Attempt to get feature " + getFeatureName() + " " + getFeatureVersion() + "resulted in null", feature);
 329         Assert.assertTrue("Failed to install Feature: " + getFeatureName() + " " + getFeatureVersion(),
 330                 featuresService.isInstalled(feature));
 331         LOG.info("Successfull installed feature {} {}", getFeatureName(), getFeatureVersion());
 332
 333         if (!Boolean.getBoolean(BUNDLES_DIAG_SKIP_PROP)
 334                 && (Boolean.getBoolean(BUNDLES_DIAG_FORCE_PROP)
 335                     || !BLACKLISTED_BROKEN_FEATURES.contains(getFeatureName()))) {
 336             Integer timeOutInSeconds = Integer.getInteger(BUNDLES_DIAG_TIMEOUT_PROP, 5 * 60);
 337             new TestBundleDiag(bundleContext, bundleService).checkBundleDiagInfos(timeOutInSeconds, SECONDS);
 338         } else {
 339             LOG.warn("SKIPPING TestBundleDiag because system property {} is true or feature is blacklisted: {}",
 340                     BUNDLES_DIAG_SKIP_PROP, getFeatureName());
 341         }
 342     }
 343
 344     // TODO remove this when all issues linked to parent https://bugs.opendaylight.org/show_bug.cgi?id=7582 are resolved
 345     private static final List<String> BLACKLISTED_BROKEN_FEATURES = ImmutableList.of(
 346             // integration/distribution/features-test due to DOMRpcService
 347             // see https://bugs.opendaylight.org/show_bug.cgi?id=7595
 348             "odl-integration-all",
 349             // controller/features/mdsal/ due to IllegalStateException: ./configuration/initial/akka.conf is missing
 350             // see https://bugs.opendaylight.org/show_bug.cgi?id=7583
 351             "odl-mdsal-broker-local",
 352             "odl-mdsal-clustering-commons",
 353             "odl-mdsal-distributed-datastore",
 354             "odl-mdsal-remoterpc-connector",
 355             // Karaf 4 specific problems with genius features
 356             // see https://bugs.opendaylight.org/show_bug.cgi?id=8100
 357             "odl-genius",
 358             "odl-genius-api",
 359             "odl-genius-rest",
 360             "odl-genius-ui",
 361             "odl-genius-fcaps-framework",
 362             "odl-genius-fcaps-application",
 363             // 1/17 in groupbasedpolicy/features due to NOK org.opendaylight.groupbasedpolicy
 364             // Caused by: org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException
 365             // see https://bugs.opendaylight.org/show_bug.cgi?id=7587
 366             "odl-groupbasedpolicy-ne-location-provider",
 367             // 1/11 in tsdr/features due to (strange) ClassNotFoundException: odlparent.bundlestest
 368             //   .TestBundleDiag (works for all other features; class loading issue in that feature?)
 369             // see https://bugs.opendaylight.org/show_bug.cgi?id=7588
 370             "odl-hbaseclient",
 371             // 1/9 in unimgr/features due missing mdsal, similar to issue to odl-integration-all?
 372             // TODO retry after https://bugs.opendaylight.org/show_bug.cgi?id=7595 is fixed
 373             "odl-unimgr-netvirt"
 374     );
 375 }