jjb/integration/distribution/distribution-macros.yaml

   1 # JJB macros specific to Integration/Distribution ODL project
   2 ---
   3 - builder:
   4     name: distribution-check-wipe
   5     # Step zero: Wipe file repositories up front.
   6     builders:
   7       - shell: !include-raw-escape: distribution-check-wipe.sh
   8
   9 - builder:
  10     name: distribution-check-build-project
  11     # Step one: Online build of the project, using local repository /tmp/r/ and deploying artifacts to /tmp/n/.
  12     # Ordinary SingleFeatureTest failures are detected in the verify job, so we can use "q" profile here.
  13     # Arguments:
  14     #   pom: Relative path to pom file to use. Typically '$GERRIT_PROJECT/pom.xml'.
  15     builders:
  16       - maven-target:
  17           maven-version: "{mvn-version}"
  18           pom: "{pom}"
  19           goals: |
  20             clean deploy
  21             dependency:tree -DoutputFile=dependency_tree.txt
  22             -Pq
  23             -DaltDeploymentRepository=fake-nexus::default::file:///tmp/n/
  24             {mvn-opts}
  25           java-opts:
  26             - "-Xmx4096m -Dmaven.compile.fork=true"
  27           settings: "integration-settings"
  28           settings-type: cfp
  29           global-settings: "global-settings"
  30           global-settings-type: cfp
  31
  32 - builder:
  33     name: distribution-check-verify-groupid
  34     # Step two: Verify all deployed artifacts belong to the project's groupId.
  35     # This is done by moving the allowed directories out of /tmp/n and checking no files remained there.
  36     # The correct directory is derived from $GERRIT_PROJECT.
  37     # TODO: Verify all deployed artifacts are snapshots.
  38     # Arguments:
  39     #   gerrit-project: Project name as nexus URI part. Typically '$GERRIT_PROJECT'.
  40     builders:
  41       - shell: !include-raw-escape: distribution-check-verify-groupid.sh
  42
  43 - builder:
  44     name: distribution-check-download-deps
  45     # Step three: Online build of integration distribution.
  46     # This step is mainly used for downloading other project artifacts.
  47     # Running SingleFeaturesTest here does not seem to be required, so -Pq is used again.
  48     # Arguments:
  49     #   dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
  50     builders:
  51       - maven-target:
  52           maven-version: "{mvn-version}"
  53           pom: "{dist-pom}"
  54           goals: |
  55             clean install
  56             dependency:tree -DoutputFile=dependency_tree.txt
  57             -Pq
  58             {mvn-opts}
  59           java-opts:
  60             - "-Xmx1024m -Dmaven.compile.fork=true"
  61           settings: "integration-settings"
  62           settings-type: cfp
  63           global-settings: "global-settings"
  64           global-settings-type: cfp
  65
  66 - builder:
  67     name: distribution-check-delete-snapshots
  68     # Step four: Delete snapshot artifacts from the local repository.
  69     # This is critical to detect orphaned artifacts or missing project-internal dependency declarations.
  70     # Also other files related to maven repository resolution are removed,
  71     # and then empty directories are removed, in order to simplify debugging.
  72     builders:
  73       - shell: !include-raw-escape: distribution-delete-snaphot-artifacts.sh
  74
  75 - builder:
  76     name: distribution-check-configure-remotes
  77     # Now the ugly part. It seems that the only way to tell Maven 2+
  78     # which remote repositories to use is via settings.xml file.
  79     # So we create such a file here, but it needs most of odlparent:settings.xml
  80     builders:
  81       - shell: |
  82           echo '
  83           <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
  84             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  85             xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
  86             https://maven.apache.org/xsd/settings-1.0.0.xsd">
  87             <profiles>
  88               <profile>
  89                 <id>opendaylight-release</id>
  90                 <repositories>
  91                   <repository>
  92                     <id>opendaylight-mirror</id>
  93                     <name>opendaylight</name>
  94                     <url>https://nexus.opendaylight.org/content/repositories/public/</url>
  95                     <releases><updatePolicy>never</updatePolicy></releases>
  96                     <snapshots><enabled>false</enabled></snapshots>
  97                   </repository>
  98                 </repositories>
  99                 <pluginRepositories>
 100                   <pluginRepository>
 101                     <id>opendaylight-plugin-mirror</id>
 102                     <name>opendaylight-plugin</name>
 103                     <url>https://nexus.opendaylight.org/content/repositories/public/</url>
 104                     <releases><updatePolicy>never</updatePolicy></releases>
 105                     <snapshots><enabled>false</enabled></snapshots>
 106                   </pluginRepository>
 107                 </pluginRepositories>
 108               </profile>
 109               <profile>
 110                 <id>file-snapshots</id>
 111                 <repositories>
 112                   <repository>
 113                     <id>file-snapshots</id>
 114                     <name>file</name>
 115                     <url>file:///tmp/n/</url>
 116                     <releases><enabled>false</enabled></releases>
 117                   </repository>
 118                 </repositories>
 119                 <pluginRepositories>
 120                   <pluginRepository>
 121                     <id>file-plugin-snapshots</id>
 122                     <name>file-plugin</name>
 123                     <url>file:///tmp/n/</url>
 124                     <releases><enabled>false</enabled></releases>
 125                   </pluginRepository>
 126                 </pluginRepositories>
 127               </profile>
 128             </profiles>
 129             <activeProfiles>
 130               <activeProfile>file-snapshots</activeProfile>
 131               <activeProfile>opendaylight-release</activeProfile>
 132             </activeProfiles>
 133           </settings>
 134           ' > fake_remotes.xml
 135           # Notes: The settings are minimal in order to detect breakage scenarios,
 136           # while allowing for the following quirks:
 137           # * Some plugins seem to have hardcoded repos, for example check-license looks at repository.apache.org
 138           # * Some plugin artifacts (related to surefire) are not downloaded when tests are skipped.
 139           # * populate-local-repo looks at oss.sonatype.org and does not store things (like guava) to /tmp/r
 140
 141 - builder:
 142     name: distribution-check-repeat-project-build
 143     # Step five: Repeat the project build but with the new settings.
 144     # Here, only the project snapshot artifacts deployed to /tmp/n are available,
 145     # which faithfully reproduces conditions in later verify-like job runs.
 146     # We cannot use --offline, because: "Cannot access file (file:///tmp/n) in offline mode".
 147     # SingleFeatureTest is still skipped, as we expect the same results as in the verify job.
 148     # Current content of /tmp/r is restored afterwards to not affect the next step.
 149     # TODO: Could there be missing dependencies which would only affect SFT here?
 150     # Arguments:
 151     #   pom: Relative path to pom file to use. Typically '$GERRIT_PROJECT/pom.xml'.
 152     builders:
 153       - shell: |
 154           cp -r /tmp/r /tmp/t
 155       - maven-target:
 156           maven-version: "{mvn-version}"
 157           pom: "{pom}"
 158           goals: |
 159             clean install
 160             dependency:tree -DoutputFile=dependency_tree.txt
 161             -s fake_remotes.xml
 162             -Pq
 163             {mvn-opts}
 164           java-opts:
 165             - "-Xmx4096m -Dmaven.compile.fork=true"
 166       - shell: |
 167           rm -rf /tmp/r
 168           mv /tmp/t /tmp/r
 169
 170 - builder:
 171     name: distribution-check-repeat-distro-build
 172     # Step six: Repeat the distribution build but with the new settings.
 173     # Here, only the project snapshot artifacts deployed to /tmp/n are available,
 174     # which faithfully reproduces conditions in later verify-like job runs.
 175     # We cannot use --offline, because: "Cannot access file (file:///tmp/n) in offline mode".
 176     # This is where SingleFeatureTest is not skipped.
 177     # TODO: Lower -Dsft.heap.max to 3g or lower when Bug 9218 is fixed.
 178     # Arguments:
 179     #   dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
 180     builders:
 181       - maven-target:
 182           maven-version: "{mvn-version}"
 183           pom: "{dist-pom}"
 184           goals: |
 185             clean install
 186             dependency:tree -DoutputFile=dependency_tree.txt
 187             -s fake_remotes.xml
 188             -Pq
 189             -Dsft.heap.max=4g
 190             -DskipTests=false
 191             {mvn-opts}
 192           java-opts:
 193             - "-Xmx1024m -Dmaven.compile.fork=true"
 194
 195 - builder:
 196     name: distribution-check-warn-9191
 197     # Detect ODL bundles referenced cross-project from features, store them to a file to be archived.
 198     # Do not fail the build, this is just to inform interested contributors.
 199     # TODO: Improve integration archiving macro so that it finds bug_9191 log without us crating it in $WORKSPACE/.
 200     builders:
 201       - shell: |
 202           # No failure in this macro should affect the job result.
 203           set +e
 204           rm -f "$WORKSPACE/bug_9191.log"
 205           pushd "$WORKSPACE/distribution/$KARAF_ARTIFACT/target/assembly/system/org/opendaylight"
 206           for folder in `ls`; do
 207           find "$folder" -name \*features.xml | xargs fgrep -n '<bundle>' > bug_9191.0.log
 208           fgrep org.opendaylight bug_9191.0.log | fgrep -v "org.opendaylight.$folder" >> "$WORKSPACE/bug_9191.log"
 209           rm bug_9191.0.log
 210           done
 211           popd
 212
 213 - builder:
 214     name: distribution-check-warn-9192
 215     # Detect ODL bundles referenced from multiple features within a project, store them to a file to be archived.
 216     # Current implementation will also include Bug 9191 violations,
 217     # presenting an unified diff with lines marked to be removed, among unmarked lines to compare to.
 218     # Do not fail the build, this is just to inform interested contributors.
 219     # TODO: Improve integration archiving macro so that it finds bug_9192 log without us crating it in $WORKSPACE/.
 220     builders:
 221       - shell: |
 222           # No failure in this macro should affect the job result.
 223           set +e
 224           pushd "$WORKSPACE/distribution/$KARAF_ARTIFACT/target/assembly/system/org/opendaylight"
 225           find . -name \*features.xml | xargs fgrep -n '<bundle>' > "$WORKSPACE/bug_9192.0.log"
 226           popd
 227           fgrep org.opendaylight bug_9192.0.log > "$WORKSPACE/bug_9192.1.log"
 228           sort -s -t '<' -k 2 "$WORKSPACE/bug_9192.1.log" > "$WORKSPACE/bug_9192.2.log"
 229           sort -u -s -t '<' -k 2 "$WORKSPACE/bug_9192.1.log" > "$WORKSPACE/bug_9192.3.log"
 230           diff -U 999 -b -t -d "$WORKSPACE/bug_9192.2.log" "$WORKSPACE/bug_9192.3.log" > "$WORKSPACE/bug_9192.diff.log"
 231           rm -f "$WORKSPACE"/bug_9192.?.log
 232
 233 - builder:
 234     name: distribution-check-bootup
 235     # Operation: Locally deploy ODL with all fetures, check it is not broken.
 236     # Required bash variables:
 237     #   KARAF_VERSION: 'karaf4' or 'karaf3'.
 238     #     Use distribution-karaf-version macro to customize.
 239     builders:
 240       - shell: !include-raw-escape: distribution-check-bootup.sh