jjb/integration/distribution/distribution-macros.yaml

   1 # JJB macros specific to Integration/Distribution ODL project
   2 ---
   3 - builder:
   4     name: distribution-check-wipe
   5     # Step zero: Wipe file repositories up front.
   6     builders:
   7       - shell: |
   8           echo "wipe r: the local Maven repository"
   9           rm -rfv /tmp/r
  10           echo "wipe n: the fake remote (Nexus) repository"
  11           rm -rfv /tmp/n
  12           echo "wipe t: the transient repository used in some manipulations"
  13           rm -rfv /tmp/t
  14           echo "create n: multithreaded execution might fail at creating it."
  15           mkdir /tmp/n
  16           echo "detecting distribution allowed projects"
  17           # Some allowed projects cannot be detected in distribution because they do not produce features.
  18           ALLOW_PROJECTS=(yangtools mdsal openflowjava)
  19           ALLOW_PROJECTS+=(`grep '<groupId>org.opendaylight.' -Rh distribution \
  20           | sed -e 's%^[ \t]*<groupId>org.opendaylight.%%' \
  21           | sed -e 's%</groupId>%%' | sort -u`)
  22           echo "Allowed projects are ${ALLOW_PROJECTS[@]}"
  23           echo "ALLOW_PROJECTS=${ALLOW_PROJECTS[@]}" > allowed_projects.txt
  24
  25 - builder:
  26     name: distribution-check-build-project
  27     # Step one: Online build of the project, using local repository /tmp/r/ and deploying artifacts to /tmp/n/.
  28     # Ordinary SingleFeatureTest failures are detected in the verify job, so we can use "q" profile here.
  29     # Arguments:
  30     #   pom: Relative path to pom file to use. Typically '$GERRIT_PROJECT/pom.xml'.
  31     builders:
  32       - maven-target:
  33           maven-version: '{mvn-version}'
  34           pom: '{pom}'
  35           goals: |
  36               clean deploy
  37               dependency:tree -DoutputFile=dependency_tree.txt
  38               -Pq -Pmanaged -Punmanaged
  39               -DaltDeploymentRepository=fake-nexus::default::file:///tmp/n/
  40               {mvn-opts}
  41           java-opts:
  42             - '-Xmx4096m -XX:MaxPermSize=1024m -Dmaven.compile.fork=true'
  43           settings: 'integration-settings'
  44           settings-type: cfp
  45           global-settings: 'global-settings'
  46           global-settings-type: cfp
  47
  48 - builder:
  49     name: distribution-check-verify-groupid
  50     # Step two: Verify all deployed artifacts belong to the project's groupId.
  51     # This is done by moving the allowed directories out of /tmp/n and checking no files remained there.
  52     # The correct directory is derived from $GERRIT_PROJECT.
  53     # TODO: Verify all deployed artifacts are snapshots.
  54     # Arguments:
  55     #   gerrit-project: Project name as nexus URI part. Typically '$GERRIT_PROJECT'.
  56     builders:
  57       - shell: |
  58           echo "These are allowed projects: $ALLOW_PROJECTS"
  59           echo "These are distribution pulled projects:"
  60           EXIT_CODE="0"
  61           for PROJECT in `ls /tmp/r/org/opendaylight`; do
  62           echo "checking $PROJECT"
  63           if [[ "$ALLOW_PROJECTS" != *"$PROJECT"* ]]; then
  64           echo "ERROR: Not allowed project $PROJECT pulled"
  65           EXIT_CODE="1"
  66           fi
  67           done
  68           exit $EXIT_CODE
  69           echo "verify project groupId"
  70           mkdir -p /tmp/t/org/opendaylight/{gerrit-project}
  71           mv /tmp/n/org/opendaylight/{gerrit-project}/* /tmp/t/org/opendaylight/{gerrit-project}/
  72           test -z "`find /tmp/n/ -type f`" || ( echo "ERROR: Mismatched groupId detected (see above)." && false )
  73           rm -rf /tmp/n
  74           mv /tmp/t /tmp/n
  75
  76 - builder:
  77     name: distribution-check-download-deps
  78     # Step three: Online build of integration distribution.
  79     # This step is mainly used for downloading other project artifacts.
  80     # Running SingleFeaturesTest here does not seem to be required, so -Pq is used again.
  81     # Arguments:
  82     #   dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
  83     builders:
  84       - maven-target:
  85           maven-version: '{mvn-version}'
  86           pom: '{dist-pom}'
  87           goals: |
  88               clean install
  89               dependency:tree -DoutputFile=dependency_tree.txt
  90               -Pq -Pmanaged -Punmanaged
  91               {mvn-opts}
  92           java-opts:
  93             - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true'
  94           settings: 'integration-settings'
  95           settings-type: cfp
  96           global-settings: 'global-settings'
  97           global-settings-type: cfp
  98
  99 - builder:
 100     name: distribution-check-delete-snapshots
 101     # Step four: Delete snapshot artifacts from the local repository.
 102     # This is critical to detect orphaned artifacts or missing project-internal dependency declarations.
 103     # Also other files related to maven repository resolution are removed,
 104     # and then empty directories are removed, in order to simplify debugging.
 105     builders:
 106       - shell: !include-raw-escape: distribution-delete-snaphot-artifacts.sh
 107
 108 - builder:
 109     name: distribution-check-configure-remotes
 110     # Now the ugly part. It seems that the only way to tell Maven 2+
 111     # which remote repositories to use is via settings.xml file.
 112     # So we create such a file here, but it needs most of odlparent:settings.xml
 113     builders:
 114       - shell: |
 115           echo '
 116           <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
 117             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 118             xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
 119             https://maven.apache.org/xsd/settings-1.0.0.xsd">
 120             <profiles>
 121               <profile>
 122                 <id>opendaylight-release</id>
 123                 <repositories>
 124                   <repository>
 125                     <id>opendaylight-mirror</id>
 126                     <name>opendaylight</name>
 127                     <url>https://nexus.opendaylight.org/content/repositories/public/</url>
 128                     <releases><updatePolicy>never</updatePolicy></releases>
 129                     <snapshots><enabled>false</enabled></snapshots>
 130                   </repository>
 131                 </repositories>
 132                 <pluginRepositories>
 133                   <pluginRepository>
 134                     <id>opendaylight-plugin-mirror</id>
 135                     <name>opendaylight-plugin</name>
 136                     <url>https://nexus.opendaylight.org/content/repositories/public/</url>
 137                     <releases><updatePolicy>never</updatePolicy></releases>
 138                     <snapshots><enabled>false</enabled></snapshots>
 139                   </pluginRepository>
 140                 </pluginRepositories>
 141               </profile>
 142               <profile>
 143                 <id>file-snapshots</id>
 144                 <repositories>
 145                   <repository>
 146                     <id>file-snapshots</id>
 147                     <name>file</name>
 148                     <url>file:///tmp/n/</url>
 149                     <releases><enabled>false</enabled></releases>
 150                   </repository>
 151                 </repositories>
 152                 <pluginRepositories>
 153                   <pluginRepository>
 154                     <id>file-plugin-snapshots</id>
 155                     <name>file-plugin</name>
 156                     <url>file:///tmp/n/</url>
 157                     <releases><enabled>false</enabled></releases>
 158                   </pluginRepository>
 159                 </pluginRepositories>
 160               </profile>
 161             </profiles>
 162             <activeProfiles>
 163               <activeProfile>file-snapshots</activeProfile>
 164               <activeProfile>opendaylight-release</activeProfile>
 165             </activeProfiles>
 166           </settings>
 167           ' > fake_remotes.xml
 168           # Notes: The settings are minimal in order to detect breakage scenarios,
 169           # while allowing for the following quirks:
 170           # * Some plugins seem to have hardcoded repos, for example check-license looks at repository.apache.org
 171           # * Some plugin artifacts (related to surefire) are not downloaded when tests are skipped.
 172           # * populate-local-repo looks at oss.sonatype.org and does not store things (like guava) to /tmp/r
 173
 174 - builder:
 175     name: distribution-check-repeat-project-build
 176     # Step five: Repeat the project build but with the new settings.
 177     # Here, only the project snapshot artifacts deployed to /tmp/n are available,
 178     # which faithfully reproduces conditions in later verify-like job runs.
 179     # We cannot use --offline, because: "Cannot access file (file:///tmp/n) in offline mode".
 180     # SingleFeatureTest is still skipped, as we expect the same results as in the verify job.
 181     # Current content of /tmp/r is restored afterwards to not affect the next step.
 182     # TODO: Could there be missing dependencies which would only affect SFT here?
 183     # Arguments:
 184     #   pom: Relative path to pom file to use. Typically '$GERRIT_PROJECT/pom.xml'.
 185     builders:
 186       - shell: |
 187           cp -r /tmp/r /tmp/t
 188       - maven-target:
 189           maven-version: '{mvn-version}'
 190           pom: '{pom}'
 191           goals: |
 192               clean install
 193               dependency:tree -DoutputFile=dependency_tree.txt
 194               -s fake_remotes.xml
 195               -Pq -Pmanaged -Punmanaged
 196               {mvn-opts}
 197           java-opts:
 198             - '-Xmx4096m -XX:MaxPermSize=1024m -Dmaven.compile.fork=true'
 199       - shell: |
 200           rm -rf /tmp/r
 201           mv /tmp/t /tmp/r
 202
 203 - builder:
 204     name: distribution-check-repeat-distro-build
 205     # Step six: Repeat the distribution build but with the new settings.
 206     # Here, only the project snapshot artifacts deployed to /tmp/n are available,
 207     # which faithfully reproduces conditions in later verify-like job runs.
 208     # We cannot use --offline, because: "Cannot access file (file:///tmp/n) in offline mode".
 209     # This is where SingleFeatureTest is not skipped.
 210     # TODO: Lower -Dsft.heap.max to 3g or lower when Bug 9218 is fixed.
 211     # Arguments:
 212     #   dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
 213     builders:
 214       - maven-target:
 215           maven-version: '{mvn-version}'
 216           pom: '{dist-pom}'
 217           goals: |
 218               clean install
 219               dependency:tree -DoutputFile=dependency_tree.txt
 220               -s fake_remotes.xml
 221               -Pq -Pmananaged -Punmanaged
 222               -Dsft.heap.max=4g
 223               -DskipTests=false
 224               {mvn-opts}
 225           java-opts:
 226             - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true'
 227
 228 - builder:
 229     name: distribution-check-warn-9191
 230     # Detect ODL bundles referenced cross-project from features, store them to a file to be archived.
 231     # Do not fail the build, this is just to inform interested contributors.
 232     # TODO: Improve integration archiving macro so that it finds bug_9191 log without us crating it in $WORKSPACE/.
 233     builders:
 234       - shell: |
 235           # No failure in this macro should affect the job result.
 236           set +e
 237           rm -f "$WORKSPACE/bug_9191.log"
 238           pushd "$WORKSPACE/distribution/$KARAF_ARTIFACT/target/assembly/system/org/opendaylight"
 239           for folder in `ls`; do
 240           find "$folder" -name \*features.xml | xargs fgrep -n '<bundle>' > bug_9191.0.log
 241           fgrep org.opendaylight bug_9191.0.log | fgrep -v "org.opendaylight.$folder" >> "$WORKSPACE/bug_9191.log"
 242           rm bug_9191.0.log
 243           done
 244           popd
 245
 246 - builder:
 247     name: distribution-check-warn-9192
 248     # Detect ODL bundles referenced from multiple features within a project, store them to a file to be archived.
 249     # Current implementation will also include Bug 9191 violations,
 250     # presenting an unified diff with lines marked to be removed, among unmarked lines to compare to.
 251     # Do not fail the build, this is just to inform interested contributors.
 252     # TODO: Improve integration archiving macro so that it finds bug_9192 log without us crating it in $WORKSPACE/.
 253     builders:
 254       - shell: |
 255           # No failure in this macro should affect the job result.
 256           set +e
 257           pushd "$WORKSPACE/distribution/$KARAF_ARTIFACT/target/assembly/system/org/opendaylight"
 258           find . -name \*features.xml | xargs fgrep -n '<bundle>' > "$WORKSPACE/bug_9192.0.log"
 259           popd
 260           fgrep org.opendaylight bug_9192.0.log > "$WORKSPACE/bug_9192.1.log"
 261           sort -s -t '<' -k 2 "$WORKSPACE/bug_9192.1.log" > "$WORKSPACE/bug_9192.2.log"
 262           sort -u -s -t '<' -k 2 "$WORKSPACE/bug_9192.1.log" > "$WORKSPACE/bug_9192.3.log"
 263           diff -U 999 -b -t -d "$WORKSPACE/bug_9192.2.log" "$WORKSPACE/bug_9192.3.log" > "$WORKSPACE/bug_9192.diff.log"
 264           rm -f "$WORKSPACE"/bug_9192.?.log
 265
 266 - builder:
 267     name: distribution-check-bootup
 268     # Operation: Locally deploy ODL with all fetures, check it is not broken.
 269     # Required bash variables:
 270     #   KARAF_VERSION: 'karaf4' or 'karaf3'.
 271     #     Use distribution-karaf-version macro to customize.
 272     builders:
 273       - shell: !include-raw: distribution-check-bootup.sh