features4-test/src/main/java/org/opendaylight/odlparent/featuretest/SingleFeatureTest.java

   1 /*
   2  * Copyright © 2014, 2017 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.odlparent.featuretest;
  10
  11 import static java.util.concurrent.TimeUnit.SECONDS;
  12 import static org.opendaylight.odlparent.featuretest.Constants.ORG_OPENDAYLIGHT_FEATURETEST_FEATURENAME_PROP;
  13 import static org.opendaylight.odlparent.featuretest.Constants.ORG_OPENDAYLIGHT_FEATURETEST_FEATUREVERSION_PROP;
  14 import static org.opendaylight.odlparent.featuretest.Constants.ORG_OPENDAYLIGHT_FEATURETEST_URI_PROP;
  15 import static org.ops4j.pax.exam.CoreOptions.maven;
  16 import static org.ops4j.pax.exam.CoreOptions.propagateSystemProperty;
  17 import static org.ops4j.pax.exam.CoreOptions.systemPackages;
  18 import static org.ops4j.pax.exam.CoreOptions.when;
  19 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.configureConsole;
  20 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.editConfigurationFilePut;
  21 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.features;
  22 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.karafDistributionConfiguration;
  23 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.keepRuntimeFolder;
  24 import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.logLevel;
  25
  26 import com.google.common.collect.ImmutableList;
  27 import java.io.File;
  28 import java.io.IOException;
  29 import java.io.InputStream;
  30 import java.net.URI;
  31 import java.net.URISyntaxException;
  32 import java.util.List;
  33 import java.util.Properties;
  34 import javax.inject.Inject;
  35 import org.apache.karaf.bundle.core.BundleService;
  36 import org.apache.karaf.features.Feature;
  37 import org.apache.karaf.features.FeaturesService;
  38 import org.apache.karaf.features.Repository;
  39 import org.eclipse.jdt.annotation.NonNull;
  40 import org.junit.Assert;
  41 import org.junit.Before;
  42 import org.junit.Test;
  43 import org.junit.runner.RunWith;
  44 import org.opendaylight.odlparent.bundles4test.TestBundleDiag;
  45 import org.ops4j.pax.exam.Configuration;
  46 import org.ops4j.pax.exam.Option;
  47 import org.ops4j.pax.exam.karaf.options.LogLevelOption.LogLevel;
  48 import org.ops4j.pax.exam.options.MavenUrlReference;
  49 import org.ops4j.pax.exam.options.extra.VMOption;
  50 import org.osgi.framework.BundleContext;
  51 import org.slf4j.Logger;
  52 import org.slf4j.LoggerFactory;
  53
  54 @RunWith(PerRepoTestRunner.class)
  55 public class SingleFeatureTest {
  56
  57     private static final String MAVEN_REPO_LOCAL = "maven.repo.local";
  58     private static final String ORG_OPS4J_PAX_URL_MVN_LOCAL_REPOSITORY = "org.ops4j.pax.url.mvn.localRepository";
  59     private static final String ORG_OPS4J_PAX_URL_MVN_REPOSITORIES = "org.ops4j.pax.url.mvn.repositories";
  60     private static final String ETC_ORG_OPS4J_PAX_URL_MVN_CFG = "etc/org.ops4j.pax.url.mvn.cfg";
  61     private static final String ETC_ORG_OPS4J_PAX_LOGGING_CFG = "etc/org.ops4j.pax.logging.cfg";
  62
  63     private static final String KEEP_UNPACK_DIRECTORY_PROP = "karaf.keep.unpack";
  64     private static final String PROFILE_PROP = "karaf.featureTest.profile";
  65     private static final String BUNDLES_DIAG_SKIP_PROP = "sft.diag.skip";
  66     private static final String BUNDLES_DIAG_FORCE_PROP = "sft.diag.force";
  67     private static final String BUNDLES_DIAG_TIMEOUT_PROP = "sft.diag.timeout";
  68
  69     private static final String LOG4J_LOGGER_ORG_OPENDAYLIGHT_YANGTOOLS_FEATURETEST =
  70             "log4j.logger.org.opendaylight.odlparent.featuretest";
  71     private static final Logger LOG = LoggerFactory.getLogger(SingleFeatureTest.class);
  72
  73     /*
  74      * File name to add our logging config property too.
  75      */
  76     private static final String ORG_OPS4J_PAX_LOGGING_CFG = "etc/org.ops4j.pax.logging.cfg";
  77
  78     /*
  79      * Default values for karaf distro type, groupId, and artifactId
  80      */
  81     private static final String KARAF_DISTRO_TYPE = "zip";
  82     private static final String KARAF_DISTRO_ARTIFACTID = "apache-karaf";
  83     private static final String KARAF_DISTRO_GROUPID = "org.apache.karaf";
  84
  85     /*
  86      * Property names to override defaults for karaf distro artifactId, groupId, version, and type
  87      */
  88     private static final String KARAF_DISTRO_VERSION_PROP = "karaf.distro.version";
  89     private static final String KARAF_DISTRO_TYPE_PROP = "karaf.distro.type";
  90     private static final String KARAF_DISTRO_ARTIFACTID_PROP = "karaf.distro.artifactId";
  91     private static final String KARAF_DISTRO_GROUPID_PROP = "karaf.distro.groupId";
  92
  93     /**
  94      * Property file used to store the Karaf distribution version.
  95      */
  96     private static final String PROPERTIES_FILENAME = "singlefeaturetest.properties";
  97
  98     /**
  99      * <p>List of Karaf 3.0.4 default maven repositories with snapshot repositories excluded.</p>
 100      * <p>Unfortunately this must be hard-coded since declarative model which uses Options,
 101      * does not allow us to read value, parse it (properties has allways
 102      * problems with lists) and construct replacement string which does
 103      * not contains snapshots.</p>
 104      * <p>When updating Karaf, check this against org.ops4j.pax.url.mvn.cfg in the Karaf distribution.</p>
 105      */
 106     private static final String EXTERNAL_DEFAULT_REPOSITORIES = "http://repo1.maven.org/maven2@id=central, "
 107             + "http://repository.springsource.com/maven/bundles/release@id=spring.ebr.release, "
 108             + "http://repository.springsource.com/maven/bundles/external@id=spring.ebr.external, "
 109             + "http://zodiac.springsource.com/maven/bundles/release@id=gemini ";
 110
 111     @Inject @NonNull
 112     private BundleContext bundleContext;
 113
 114     @Inject @NonNull
 115     private FeaturesService featuresService;
 116
 117     @Inject @NonNull
 118     private BundleService bundleService; // NOT BundleStateService, see checkBundleStatesDiag()
 119
 120     private String karafVersion;
 121     private String karafDistroVersion;
 122
 123     /**
 124      * Returns the required configuration.
 125      *
 126      * @return The Pax Exam configuration.
 127      * @throws IOException if an error occurs.
 128      */
 129     @Configuration
 130     public Option[] config() throws IOException {
 131         MavenUrlReference bundleTestRepo = maven()
 132                 .groupId("org.opendaylight.odlparent")
 133                 .artifactId("odl-bundles-test")
 134                 .classifier("features")
 135                 .type("xml")
 136                 .versionAsInProject();
 137
 138         return new Option[] {
 139             // TODO: Find a way to inherit memory limits from Maven options.
 140             new VMOption("-Xmx2g"),
 141             // inspired by org.apache.commons.lang.SystemUtils
 142             when(System.getProperty("os.name").toLowerCase().startsWith("linux")).useOptions(
 143                 // This prevents low entropy issues on Linux to affect Java random numbers
 144                 // which can block crypto such as the SSH server in netconf
 145                 // see https://bugs.opendaylight.org/show_bug.cgi?id=6790
 146                 new VMOption("-Djava.security.egd=file:/dev/./urandom")
 147             ),
 148             when(Boolean.getBoolean(PROFILE_PROP)).useOptions(
 149                 new VMOption("-XX:+UnlockCommercialFeatures"),
 150                 new VMOption("-XX:+FlightRecorder"),
 151                 new VMOption("-XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true,dumponexitpath="
 152                                + getNewJFRFile())
 153             ),
 154             getKarafDistroOption(),
 155             when(Boolean.getBoolean(KEEP_UNPACK_DIRECTORY_PROP)).useOptions(keepRuntimeFolder()),
 156             configureConsole().ignoreLocalConsole(),
 157             logLevel(LogLevel.WARN),
 158             mvnLocalRepoOption(),
 159             features(bundleTestRepo, "odl-bundles-test"),
 160             editConfigurationFilePut(ORG_OPS4J_PAX_LOGGING_CFG, LOG4J_LOGGER_ORG_OPENDAYLIGHT_YANGTOOLS_FEATURETEST,
 161                     LogLevel.INFO.name()),
 162             editConfigurationFilePut(ETC_ORG_OPS4J_PAX_LOGGING_CFG, "log4j.rootLogger", "INFO, stdout, osgi:*"),
 163              /*
 164               *
 165               * Disables external snapshot repositories.
 166               *
 167               * Pax URL and Karaf by default always search for new version of snapshots
 168               * in all snapshots repository, even if that snapshots does not belong to that
 169               * repository and maven is invoked even with -nsu (no snapshot update)
 170               * or offline mode.
 171               *
 172               * This is also true for OpenDaylight snapshot artefacts - pax url tries
 173               * to resolve them from third-party repositories, even if they are not present
 174               * there - this increases time which takes for features to install.
 175               *
 176               * For more complex projects this actually means several HTTP GETs for each
 177               * snapshot bundle referenced, even if the bundle is already present
 178               * in local maven repository.
 179               *
 180               * In order to speed-up installation and remove unnecessary network traffic,
 181               * which fails for obvious reasons, external snapshot repositories are
 182               * removed.
 183               *
 184               *
 185               */
 186             disableExternalSnapshotRepositories(),
 187             propagateSystemProperty(ORG_OPENDAYLIGHT_FEATURETEST_URI_PROP),
 188             propagateSystemProperty(ORG_OPENDAYLIGHT_FEATURETEST_FEATURENAME_PROP),
 189             propagateSystemProperty(ORG_OPENDAYLIGHT_FEATURETEST_FEATUREVERSION_PROP),
 190             propagateSystemProperty(BUNDLES_DIAG_SKIP_PROP),
 191             propagateSystemProperty(BUNDLES_DIAG_FORCE_PROP),
 192             propagateSystemProperty(BUNDLES_DIAG_TIMEOUT_PROP),
 193             // Needed for Agrona/aeron.io
 194             systemPackages("com.sun.media.sound", "sun.nio.ch"),
 195         };
 196     }
 197
 198     private String getNewJFRFile() throws IOException {
 199         return File.createTempFile("SingleFeatureTest-Karaf-JavaFlightRecorder", ".jfr").getAbsolutePath();
 200     }
 201
 202     private String getKarafVersion() {
 203         if (karafVersion == null) {
 204             // We use a properties file to retrieve ${karaf.version}, instead of .versionAsInProject()
 205             // This avoids forcing all users to depend on Karaf in their POMs
 206             Properties singleFeatureTestProps = new Properties();
 207             try (InputStream singleFeatureTestInputStream = Thread.currentThread().getContextClassLoader()
 208                     .getResourceAsStream(PROPERTIES_FILENAME)) {
 209                 singleFeatureTestProps.load(singleFeatureTestInputStream);
 210             } catch (IOException e) {
 211                 LOG.error("Unable to load {} to determine the Karaf version", PROPERTIES_FILENAME, e);
 212             }
 213             karafVersion = singleFeatureTestProps.getProperty(KARAF_DISTRO_VERSION_PROP);
 214
 215             LOG.info("Retrieved karafVersion {} from properties file {}", karafVersion, PROPERTIES_FILENAME);
 216         } else {
 217             LOG.info("Retrieved karafVersion {} from system property {}", karafVersion, KARAF_DISTRO_VERSION_PROP);
 218         }
 219
 220         return karafVersion;
 221     }
 222
 223     private String getKarafDistroVersion() {
 224         if (karafDistroVersion == null) {
 225             karafDistroVersion = System.getProperty(KARAF_DISTRO_VERSION_PROP);
 226             if (karafDistroVersion == null) {
 227                 karafDistroVersion = getKarafVersion();
 228             } else {
 229                 LOG.info("Retrieved karafDistroVersion {} from system property {}", karafVersion,
 230                         KARAF_DISTRO_VERSION_PROP);
 231             }
 232         }
 233
 234         return karafDistroVersion;
 235     }
 236
 237     /**
 238      * Disables snapshot repositories, which are enabled by default in karaf distribution.
 239      *
 240      * @return Edit Configuration option which removes external snapshot repositories.
 241      */
 242     private static Option disableExternalSnapshotRepositories() {
 243         return editConfigurationFilePut(ETC_ORG_OPS4J_PAX_URL_MVN_CFG, ORG_OPS4J_PAX_URL_MVN_REPOSITORIES,
 244                 EXTERNAL_DEFAULT_REPOSITORIES);
 245     }
 246
 247     protected Option mvnLocalRepoOption() {
 248         String mvnRepoLocal = System.getProperty(MAVEN_REPO_LOCAL, "");
 249         LOG.info("mvnLocalRepo \"{}\"", mvnRepoLocal);
 250         return editConfigurationFilePut(ETC_ORG_OPS4J_PAX_URL_MVN_CFG, ORG_OPS4J_PAX_URL_MVN_LOCAL_REPOSITORY,
 251                 mvnRepoLocal);
 252     }
 253
 254     protected Option getKarafDistroOption() {
 255         String groupId = System.getProperty(KARAF_DISTRO_GROUPID_PROP, KARAF_DISTRO_GROUPID);
 256         String artifactId = System.getProperty(KARAF_DISTRO_ARTIFACTID_PROP, KARAF_DISTRO_ARTIFACTID);
 257         String type = System.getProperty(KARAF_DISTRO_TYPE_PROP, KARAF_DISTRO_TYPE);
 258         LOG.info("Using karaf distro {} {} {} {}", groupId, artifactId, getKarafDistroVersion(), type);
 259         return karafDistributionConfiguration()
 260                 .frameworkUrl(
 261                         maven()
 262                                 .groupId(groupId)
 263                                 .artifactId(artifactId)
 264                                 .type(type)
 265                                 .version(getKarafDistroVersion()))
 266                 .name("OpenDaylight")
 267                 .unpackDirectory(new File("target/pax"))
 268                 .useDeployFolder(false);
 269     }
 270
 271     private static URI getRepoUri() throws URISyntaxException {
 272         return new URI(getProperty(ORG_OPENDAYLIGHT_FEATURETEST_URI_PROP));
 273     }
 274
 275     private static String getFeatureName() {
 276         return getProperty(ORG_OPENDAYLIGHT_FEATURETEST_FEATURENAME_PROP);
 277     }
 278
 279     public String getFeatureVersion() {
 280         return getProperty(ORG_OPENDAYLIGHT_FEATURETEST_FEATUREVERSION_PROP);
 281     }
 282
 283     private static String getProperty(final String propName) {
 284         String prop = System.getProperty(propName);
 285         Assert.assertTrue("Missing property :" + propName, prop != null);
 286         return prop;
 287     }
 288
 289     private void checkRepository(final URI repoUri) throws Exception {
 290         Repository repo = null;
 291         for (Repository r : featuresService.listRepositories()) {
 292             if (r.getURI().equals(repoUri)) {
 293                 repo = r;
 294                 break;
 295             }
 296         }
 297         Assert.assertNotNull("Repository not found: " + repoUri, repo);
 298     }
 299
 300     /**
 301      * Sets the repository up.
 302      *
 303      * @throws Exception if an error occurs.
 304      */
 305     @Before
 306     public void installRepo() throws Exception {
 307         final URI repoUri = getRepoUri();
 308         LOG.info("Attempting to add repository {}", repoUri);
 309         featuresService.addRepository(repoUri);
 310         checkRepository(repoUri);
 311         LOG.info("Successfully loaded repository {}", repoUri);
 312     }
 313
 314     // Give it 10 minutes max as we've seen feature install hang on jenkins.
 315     @Test(timeout = 600000)
 316     public void installFeature() throws Exception {
 317         LOG.info("Attempting to install feature {} {}", getFeatureName(), getFeatureVersion());
 318         featuresService.installFeature(getFeatureName(), getFeatureVersion());
 319         Feature feature = featuresService.getFeature(getFeatureName(), getFeatureVersion());
 320         Assert.assertNotNull(
 321                 "Attempt to get feature " + getFeatureName() + " " + getFeatureVersion() + "resulted in null", feature);
 322         Assert.assertTrue("Failed to install Feature: " + getFeatureName() + " " + getFeatureVersion(),
 323                 featuresService.isInstalled(feature));
 324         LOG.info("Successfull installed feature {} {}", getFeatureName(), getFeatureVersion());
 325
 326         if (!Boolean.getBoolean(BUNDLES_DIAG_SKIP_PROP)
 327                 && (Boolean.getBoolean(BUNDLES_DIAG_FORCE_PROP)
 328                     || !BLACKLISTED_BROKEN_FEATURES.contains(getFeatureName()))) {
 329             Integer timeOutInSeconds = Integer.getInteger(BUNDLES_DIAG_TIMEOUT_PROP, 5 * 60);
 330             new TestBundleDiag(bundleContext, bundleService).checkBundleDiagInfos(timeOutInSeconds, SECONDS);
 331         } else {
 332             LOG.warn("SKIPPING TestBundleDiag because system property {} is true or feature is blacklisted: {}",
 333                     BUNDLES_DIAG_SKIP_PROP, getFeatureName());
 334         }
 335     }
 336
 337     // TODO remove this when all issues linked to parent https://bugs.opendaylight.org/show_bug.cgi?id=7582 are resolved
 338     private static final List<String> BLACKLISTED_BROKEN_FEATURES = ImmutableList.of(
 339             // integration/distribution/features-test due to DOMRpcService
 340             // see https://bugs.opendaylight.org/show_bug.cgi?id=7595
 341             "odl-integration-all",
 342             // controller/features/mdsal/ due to IllegalStateException: ./configuration/initial/akka.conf is missing
 343             // see https://bugs.opendaylight.org/show_bug.cgi?id=7583
 344             "odl-mdsal-broker-local",
 345             "odl-mdsal-clustering-commons",
 346             "odl-mdsal-distributed-datastore",
 347             "odl-mdsal-remoterpc-connector",
 348             // 1/17 in groupbasedpolicy/features due to NOK org.opendaylight.groupbasedpolicy
 349             // Caused by: org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException
 350             // see https://bugs.opendaylight.org/show_bug.cgi?id=7587
 351             "odl-groupbasedpolicy-ne-location-provider",
 352             // 1/11 in tsdr/features due to (strange) ClassNotFoundException: odlparent.bundlestest
 353             //   .TestBundleDiag (works for all other features; class loading issue in that feature?)
 354             // see https://bugs.opendaylight.org/show_bug.cgi?id=7588
 355             "odl-hbaseclient",
 356             // 1/9 in unimgr/features due missing mdsal, similar to issue to odl-integration-all?
 357             // TODO retry after https://bugs.opendaylight.org/show_bug.cgi?id=7595 is fixed
 358             "odl-unimgr-netvirt"
 359     );
 360 }