jjb/integration/distribution/distribution-macros.yaml

   1 # JJB macros specific to Integration/Distribution ODL project
   2 ---
   3
   4 - builder:
   5     name: distribution-deploy-verify
   6     # Operation: Deploy ODL with all fetures and check it is not broken.
   7     # Required bash variables:
   8     #   KARAF_VERSION: 'karaf4' or 'karaf3'.
   9     #     Use distribution-karaf-version macro to customize.
  10     builders:
  11       - integration-detect-variables
  12       - shell: !include-raw: distribution-deploy-verify.sh
  13
  14 - builder:
  15     name: distribution-deploy-offline
  16     # Operation: Deploys ODL on the current system, with no external repo configuration, check it works.
  17     # Required bash variables:
  18     #   KARAF_VERSION: 'karaf4' or 'karaf3'.
  19     #     Use distribution-karaf-version macro to customize.
  20     builders:
  21       - integration-detect-variables
  22       - shell: !include-raw: distribution-deploy-offline.sh
  23
  24 - builder:
  25     name: distribution-check-wipe
  26     # Step zero: Wipe file repositories up front.
  27     builders:
  28       - shell: |
  29           echo "wipe r: the local Maven repository"
  30           rm -rfv /tmp/r
  31           echo "wipe n: the fake remote (Nexus) repository"
  32           rm -rfv /tmp/n
  33           echo "wipe t: the transient repository used in some manipulations"
  34           rm -rfv /tmp/t
  35           echo "create n: multithreaded execution might fail at creating it."
  36           mkdir /tmp/n
  37
  38 - builder:
  39     name: distribution-check-build-project
  40     # Step one: Online build of the project, using local repository /tmp/r/ and deploying artifacts to /tmp/n/.
  41     # Ordinary SingleFeatureTest failures are detected in the verify job, so we can use "q" profile here.
  42     # Arguments:
  43     #   pom: Relative path to pom file to use. Typically '$GERRIT_PROJECT/pom.xml'.
  44     builders:
  45       - maven-target:
  46           maven-version: '{mvn-version}'
  47           pom: '{pom}'
  48           goals: |
  49               clean deploy
  50               dependency:tree -DoutputFile=dependency_tree.txt
  51               -Pq
  52               -DaltDeploymentRepository=fake-nexus::default::file:///tmp/n/
  53               {mvn-opts}
  54           java-opts:
  55             - '-Xmx4096m -XX:MaxPermSize=1024m -Dmaven.compile.fork=true'
  56           settings: 'integration-settings'
  57           settings-type: cfp
  58           global-settings: 'global-settings'
  59           global-settings-type: cfp
  60
  61 - builder:
  62     name: distribution-check-verify-groupid
  63     # Step two: Verify all deployed artifacts belong to the project's groupId.
  64     # This is done by moving the allowed directories out of /tmp/n and checking no files remained there.
  65     # The correct directory is derived from $GERRIT_PROJECT.
  66     # TODO: Verify all deployed artifacts are snapshots.
  67     # Arguments:
  68     #   gerrit-project: Project name as nexus URI part. Typically '$GERRIT_PROJECT'.
  69     builders:
  70       - shell: |
  71           mkdir -p /tmp/t/org/opendaylight/{gerrit-project}
  72           mv /tmp/n/org/opendaylight/{gerrit-project}/* /tmp/t/org/opendaylight/{gerrit-project}/
  73           test -z "`find /tmp/n/ -type f`" || ( echo "ERROR: Mismatched groupId detected (see above)." && false )
  74           rm -rf /tmp/n
  75           mv /tmp/t /tmp/n
  76
  77 - builder:
  78     name: distribution-check-download-deps
  79     # Step three: Online build of integration distribution.
  80     # This step is mainly used for downloading other project artifacts.
  81     # Running SingleFeaturesTest here does not seem to be required, so -Pq is used again.
  82     # Arguments:
  83     #   dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
  84     builders:
  85       - maven-target:
  86           maven-version: '{mvn-version}'
  87           pom: '{dist-pom}'
  88           goals: |
  89               clean install
  90               dependency:tree -DoutputFile=dependency_tree.txt
  91               -Pq
  92               {mvn-opts}
  93           java-opts:
  94             - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true'
  95           settings: 'integration-settings'
  96           settings-type: cfp
  97           global-settings: 'global-settings'
  98           global-settings-type: cfp
  99
 100 - builder:
 101     name: distribution-check-delete-snapshots
 102     # Step four: Delete snapshot artifacts from the local repository.
 103     # This is critical to detect orphaned artifacts or missing project-internal dependency declarations.
 104     # Also other files related to maven repository resolution are removed,
 105     # and then empty directories are removed, in order to simplify debugging.
 106     builders:
 107       - shell: !include-raw-escape: distribution-delete-snaphot-artifacts.sh
 108
 109 - builder:
 110     name: distribution-check-configure-remotes
 111     # Now the ugly part. It seems that the only way to tell Maven 2+
 112     # which remote repositories to use is via settings.xml file.
 113     # So we create such a file here, but it needs most of odlparent:settings.xml
 114     builders:
 115       - shell: |
 116           echo '
 117           <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
 118             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 119             xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
 120             https://maven.apache.org/xsd/settings-1.0.0.xsd">
 121             <profiles>
 122               <profile>
 123                 <id>opendaylight-release</id>
 124                 <repositories>
 125                   <repository>
 126                     <id>opendaylight-mirror</id>
 127                     <name>opendaylight</name>
 128                     <url>https://nexus.opendaylight.org/content/repositories/public/</url>
 129                     <releases><updatePolicy>never</updatePolicy></releases>
 130                     <snapshots><enabled>false</enabled></snapshots>
 131                   </repository>
 132                 </repositories>
 133                 <pluginRepositories>
 134                   <pluginRepository>
 135                     <id>opendaylight-plugin-mirror</id>
 136                     <name>opendaylight-plugin</name>
 137                     <url>https://nexus.opendaylight.org/content/repositories/public/</url>
 138                     <releases><updatePolicy>never</updatePolicy></releases>
 139                     <snapshots><enabled>false</enabled></snapshots>
 140                   </pluginRepository>
 141                 </pluginRepositories>
 142               </profile>
 143               <profile>
 144                 <id>file-snapshots</id>
 145                 <repositories>
 146                   <repository>
 147                     <id>file-snapshots</id>
 148                     <name>file</name>
 149                     <url>file:///tmp/n/</url>
 150                     <releases><enabled>false</enabled></releases>
 151                   </repository>
 152                 </repositories>
 153                 <pluginRepositories>
 154                   <pluginRepository>
 155                     <id>file-plugin-snapshots</id>
 156                     <name>file-plugin</name>
 157                     <url>file:///tmp/n/</url>
 158                     <releases><enabled>false</enabled></releases>
 159                   </pluginRepository>
 160                 </pluginRepositories>
 161               </profile>
 162             </profiles>
 163             <activeProfiles>
 164               <activeProfile>file-snapshots</activeProfile>
 165               <activeProfile>opendaylight-release</activeProfile>
 166             </activeProfiles>
 167           </settings>
 168           ' > fake_remotes.xml
 169           # Notes: The settings are minimal in order to detect breakage scenarios,
 170           # while allowing for the following quirks:
 171           # * Some plugins seem to have hardcoded repos, for example check-license looks at repository.apache.org
 172           # * Some plugin artifacts (related to surefire) are not downloaded when tests are skipped.
 173           # * populate-local-repo looks at oss.sonatype.org and does not store things (like guava) to /tmp/r
 174
 175 - builder:
 176     name: distribution-check-repeat-project-build
 177     # Step five: Repeat the project build but with the new settings.
 178     # Here, only the project snapshot artifacts deployed to /tmp/n are available,
 179     # which faithfully reproduces conditions in later verify-like job runs.
 180     # We cannot use --offline, because: "Cannot access file (file:///tmp/n) in offline mode".
 181     # SingleFeatureTest is still skipped, as we expect the same results as in the verify job.
 182     # Current content of /tmp/r is restored afterwards to not affect the next step.
 183     # TODO: Could there be missing dependencies which would only affect SFT here?
 184     # Arguments:
 185     #   pom: Relative path to pom file to use. Typically '$GERRIT_PROJECT/pom.xml'.
 186     builders:
 187       - shell: |
 188           cp -r /tmp/r /tmp/t
 189       - maven-target:
 190           maven-version: '{mvn-version}'
 191           pom: '{pom}'
 192           goals: |
 193               clean install
 194               dependency:tree -DoutputFile=dependency_tree.txt
 195               -s fake_remotes.xml
 196               -Pq
 197               {mvn-opts}
 198           java-opts:
 199             - '-Xmx4096m -XX:MaxPermSize=1024m -Dmaven.compile.fork=true'
 200       - shell: |
 201           rm -rf /tmp/r
 202           mv /tmp/t /tmp/r
 203
 204 - builder:
 205     name: distribution-check-repeat-distro-build
 206     # Step six: Repeat the distribution build but with the new settings.
 207     # Here, only the project snapshot artifacts deployed to /tmp/n are available,
 208     # which faithfully reproduces conditions in later verify-like job runs.
 209     # We cannot use --offline, because: "Cannot access file (file:///tmp/n) in offline mode".
 210     # This is where SingleFeatureTest is not skipped.
 211     # TODO: Lower -Dsft.heap.max to 3g or lower when Bug 9218 is fixed.
 212     # Arguments:
 213     #   dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
 214     builders:
 215       - maven-target:
 216           maven-version: '{mvn-version}'
 217           pom: '{dist-pom}'
 218           goals: |
 219               clean install
 220               dependency:tree -DoutputFile=dependency_tree.txt
 221               -s fake_remotes.xml
 222               -Pq
 223               -Dsft.heap.max=4g
 224               -DskipTests=false
 225               {mvn-opts}
 226           java-opts:
 227             - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true'
 228
 229 - builder:
 230     name: distribution-check-warn-9191
 231     # Detect ODL bundles referenced cross-project from features, store them to a file to be archived.
 232     # Do not fail the build, this is just to inform interested contributors.
 233     # TODO: Improve integration archiving macro so that it finds bug_9191 log without us crating it in $WORKSPACE/.
 234     builders:
 235       - shell: |
 236           # No failure in this macro should affect the job result.
 237           set +e
 238           rm -f "$WORKSPACE/bug_9191.log"
 239           pushd "$WORKSPACE/distribution/$KARAF_ARTIFACT/target/assembly/system/org/opendaylight"
 240           for folder in `ls`; do
 241           find "$folder" -name \*features.xml | xargs fgrep -n '<bundle>' > bug_9191.0.log
 242           fgrep org.opendaylight bug_9191.0.log | fgrep -v "org.opendaylight.$folder" >> "$WORKSPACE/bug_9191.log"
 243           rm bug_9191.0.log
 244           done
 245           popd
 246
 247 - builder:
 248     name: distribution-check-warn-9192
 249     # Detect ODL bundles referenced from multiple features within a project, store them to a file to be archived.
 250     # Current implementation will also include Bug 9191 violations,
 251     # presenting an unified diff with lines marked to be removed, among unmarked lines to compare to.
 252     # Do not fail the build, this is just to inform interested contributors.
 253     # TODO: Improve integration archiving macro so that it finds bug_9192 log without us crating it in $WORKSPACE/.
 254     builders:
 255       - shell: |
 256           # No failure in this macro should affect the job result.
 257           set +e
 258           pushd "$WORKSPACE/distribution/$KARAF_ARTIFACT/target/assembly/system/org/opendaylight"
 259           find . -name \*features.xml | xargs fgrep -n '<bundle>' > "$WORKSPACE/bug_9192.0.log"
 260           popd
 261           fgrep org.opendaylight bug_9192.0.log > "$WORKSPACE/bug_9192.1.log"
 262           sort -s -t '<' -k 2 "$WORKSPACE/bug_9192.1.log" > "$WORKSPACE/bug_9192.2.log"
 263           sort -u -s -t '<' -k 2 "$WORKSPACE/bug_9192.1.log" > "$WORKSPACE/bug_9192.3.log"
 264           diff -U 999 -b -t -d "$WORKSPACE/bug_9192.2.log" "$WORKSPACE/bug_9192.3.log" > "$WORKSPACE/bug_9192.diff.log"
 265           rm -f "$WORKSPACE"/bug_9192.?.log
 266
 267 - builder:
 268     name: distribution-check-bootup
 269     # Operation: Locally deploy ODL with all fetures, check it is not broken.
 270     # Required bash variables:
 271     #   KARAF_VERSION: 'karaf4' or 'karaf3'.
 272     #     Use distribution-karaf-version macro to customize.
 273     builders:
 274       - shell: !include-raw: distribution-check-bootup.sh