jjb/integration/distribution/distribution-macros.yaml

   1 # JJB macros specific to Integration/Distribution ODL project
   2 ---
   3 - builder:
   4     name: distribution-check-wipe
   5     # Step zero: Wipe file repositories up front.
   6     builders:
   7       - shell: |
   8           echo "wipe r: the local Maven repository"
   9           rm -rfv /tmp/r
  10           echo "wipe n: the fake remote (Nexus) repository"
  11           rm -rfv /tmp/n
  12           echo "wipe t: the transient repository used in some manipulations"
  13           rm -rfv /tmp/t
  14           echo "create n: multithreaded execution might fail at creating it."
  15           mkdir /tmp/n
  16           echo "detecting distribution allowed projects"
  17           # Some allowed projects cannot be detected in distribution because they do not produce features.
  18           ALLOW_PROJECTS=(yangtools mdsal openflowjava)
  19           if [[ "$KARAF_VERSION" == "odl" ]]; then
  20           ALLOW_PROJECTS+=(`grep '<groupId>org.opendaylight.' -Rh distribution \
  21           | sed -e 's%^[ \t]*<groupId>org.opendaylight.%%' \
  22           | sed -e 's%</groupId>%%' | sort -u`)
  23           else
  24           # For Managed distro we only look at the features folder
  25           ALLOW_PROJECTS+=(`grep '<groupId>org.opendaylight.' -Rh distribution/features \
  26           | sed -e 's%^[ \t]*<groupId>org.opendaylight.%%' \
  27           | sed -e 's%</groupId>%%' | sort -u`)
  28           fi
  29           echo "Allowed projects are ${ALLOW_PROJECTS[@]}"
  30           echo "ALLOW_PROJECTS=${ALLOW_PROJECTS[@]}" > allowed_projects.txt
  31
  32 - builder:
  33     name: distribution-check-build-project
  34     # Step one: Online build of the project, using local repository /tmp/r/ and deploying artifacts to /tmp/n/.
  35     # Ordinary SingleFeatureTest failures are detected in the verify job, so we can use "q" profile here.
  36     # Arguments:
  37     #   pom: Relative path to pom file to use. Typically '$GERRIT_PROJECT/pom.xml'.
  38     builders:
  39       - maven-target:
  40           maven-version: '{mvn-version}'
  41           pom: '{pom}'
  42           goals: |
  43               clean deploy
  44               dependency:tree -DoutputFile=dependency_tree.txt
  45               -Pq
  46               -DaltDeploymentRepository=fake-nexus::default::file:///tmp/n/
  47               {mvn-opts}
  48           java-opts:
  49             - '-Xmx4096m -XX:MaxPermSize=1024m -Dmaven.compile.fork=true'
  50           settings: 'integration-settings'
  51           settings-type: cfp
  52           global-settings: 'global-settings'
  53           global-settings-type: cfp
  54
  55 - builder:
  56     name: distribution-check-verify-groupid
  57     # Step two: Verify all deployed artifacts belong to the project's groupId.
  58     # This is done by moving the allowed directories out of /tmp/n and checking no files remained there.
  59     # The correct directory is derived from $GERRIT_PROJECT.
  60     # TODO: Verify all deployed artifacts are snapshots.
  61     # Arguments:
  62     #   gerrit-project: Project name as nexus URI part. Typically '$GERRIT_PROJECT'.
  63     builders:
  64       - shell: |
  65           echo "These are allowed projects: $ALLOW_PROJECTS"
  66           echo "These are distribution pulled projects:"
  67           EXIT_CODE="0"
  68           for PROJECT in `ls /tmp/r/org/opendaylight`; do
  69           echo "checking $PROJECT"
  70           if [[ "$ALLOW_PROJECTS" != *"$PROJECT"* ]]; then
  71           echo "ERROR: Not allowed project $PROJECT pulled"
  72           EXIT_CODE="1"
  73           fi
  74           done
  75           exit $EXIT_CODE
  76           echo "verify project groupId"
  77           mkdir -p /tmp/t/org/opendaylight/{gerrit-project}
  78           mv /tmp/n/org/opendaylight/{gerrit-project}/* /tmp/t/org/opendaylight/{gerrit-project}/
  79           test -z "`find /tmp/n/ -type f`" || ( echo "ERROR: Mismatched groupId detected (see above)." && false )
  80           rm -rf /tmp/n
  81           mv /tmp/t /tmp/n
  82
  83 - builder:
  84     name: distribution-check-download-deps
  85     # Step three: Online build of integration distribution.
  86     # This step is mainly used for downloading other project artifacts.
  87     # Running SingleFeaturesTest here does not seem to be required, so -Pq is used again.
  88     # Arguments:
  89     #   dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
  90     builders:
  91       - maven-target:
  92           maven-version: '{mvn-version}'
  93           pom: '{dist-pom}'
  94           goals: |
  95               clean install
  96               dependency:tree -DoutputFile=dependency_tree.txt
  97               -Pq
  98               {mvn-opts}
  99           java-opts:
 100             - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true'
 101           settings: 'integration-settings'
 102           settings-type: cfp
 103           global-settings: 'global-settings'
 104           global-settings-type: cfp
 105
 106 - builder:
 107     name: distribution-check-delete-snapshots
 108     # Step four: Delete snapshot artifacts from the local repository.
 109     # This is critical to detect orphaned artifacts or missing project-internal dependency declarations.
 110     # Also other files related to maven repository resolution are removed,
 111     # and then empty directories are removed, in order to simplify debugging.
 112     builders:
 113       - shell: !include-raw-escape: distribution-delete-snaphot-artifacts.sh
 114
 115 - builder:
 116     name: distribution-check-configure-remotes
 117     # Now the ugly part. It seems that the only way to tell Maven 2+
 118     # which remote repositories to use is via settings.xml file.
 119     # So we create such a file here, but it needs most of odlparent:settings.xml
 120     builders:
 121       - shell: |
 122           echo '
 123           <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
 124             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 125             xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
 126             https://maven.apache.org/xsd/settings-1.0.0.xsd">
 127             <profiles>
 128               <profile>
 129                 <id>opendaylight-release</id>
 130                 <repositories>
 131                   <repository>
 132                     <id>opendaylight-mirror</id>
 133                     <name>opendaylight</name>
 134                     <url>https://nexus.opendaylight.org/content/repositories/public/</url>
 135                     <releases><updatePolicy>never</updatePolicy></releases>
 136                     <snapshots><enabled>false</enabled></snapshots>
 137                   </repository>
 138                 </repositories>
 139                 <pluginRepositories>
 140                   <pluginRepository>
 141                     <id>opendaylight-plugin-mirror</id>
 142                     <name>opendaylight-plugin</name>
 143                     <url>https://nexus.opendaylight.org/content/repositories/public/</url>
 144                     <releases><updatePolicy>never</updatePolicy></releases>
 145                     <snapshots><enabled>false</enabled></snapshots>
 146                   </pluginRepository>
 147                 </pluginRepositories>
 148               </profile>
 149               <profile>
 150                 <id>file-snapshots</id>
 151                 <repositories>
 152                   <repository>
 153                     <id>file-snapshots</id>
 154                     <name>file</name>
 155                     <url>file:///tmp/n/</url>
 156                     <releases><enabled>false</enabled></releases>
 157                   </repository>
 158                 </repositories>
 159                 <pluginRepositories>
 160                   <pluginRepository>
 161                     <id>file-plugin-snapshots</id>
 162                     <name>file-plugin</name>
 163                     <url>file:///tmp/n/</url>
 164                     <releases><enabled>false</enabled></releases>
 165                   </pluginRepository>
 166                 </pluginRepositories>
 167               </profile>
 168             </profiles>
 169             <activeProfiles>
 170               <activeProfile>file-snapshots</activeProfile>
 171               <activeProfile>opendaylight-release</activeProfile>
 172             </activeProfiles>
 173           </settings>
 174           ' > fake_remotes.xml
 175           # Notes: The settings are minimal in order to detect breakage scenarios,
 176           # while allowing for the following quirks:
 177           # * Some plugins seem to have hardcoded repos, for example check-license looks at repository.apache.org
 178           # * Some plugin artifacts (related to surefire) are not downloaded when tests are skipped.
 179           # * populate-local-repo looks at oss.sonatype.org and does not store things (like guava) to /tmp/r
 180
 181 - builder:
 182     name: distribution-check-repeat-project-build
 183     # Step five: Repeat the project build but with the new settings.
 184     # Here, only the project snapshot artifacts deployed to /tmp/n are available,
 185     # which faithfully reproduces conditions in later verify-like job runs.
 186     # We cannot use --offline, because: "Cannot access file (file:///tmp/n) in offline mode".
 187     # SingleFeatureTest is still skipped, as we expect the same results as in the verify job.
 188     # Current content of /tmp/r is restored afterwards to not affect the next step.
 189     # TODO: Could there be missing dependencies which would only affect SFT here?
 190     # Arguments:
 191     #   pom: Relative path to pom file to use. Typically '$GERRIT_PROJECT/pom.xml'.
 192     builders:
 193       - shell: |
 194           cp -r /tmp/r /tmp/t
 195       - maven-target:
 196           maven-version: '{mvn-version}'
 197           pom: '{pom}'
 198           goals: |
 199               clean install
 200               dependency:tree -DoutputFile=dependency_tree.txt
 201               -s fake_remotes.xml
 202               -Pq
 203               {mvn-opts}
 204           java-opts:
 205             - '-Xmx4096m -XX:MaxPermSize=1024m -Dmaven.compile.fork=true'
 206       - shell: |
 207           rm -rf /tmp/r
 208           mv /tmp/t /tmp/r
 209
 210 - builder:
 211     name: distribution-check-repeat-distro-build
 212     # Step six: Repeat the distribution build but with the new settings.
 213     # Here, only the project snapshot artifacts deployed to /tmp/n are available,
 214     # which faithfully reproduces conditions in later verify-like job runs.
 215     # We cannot use --offline, because: "Cannot access file (file:///tmp/n) in offline mode".
 216     # This is where SingleFeatureTest is not skipped.
 217     # TODO: Lower -Dsft.heap.max to 3g or lower when Bug 9218 is fixed.
 218     # Arguments:
 219     #   dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
 220     builders:
 221       - maven-target:
 222           maven-version: '{mvn-version}'
 223           pom: '{dist-pom}'
 224           goals: |
 225               clean install
 226               dependency:tree -DoutputFile=dependency_tree.txt
 227               -s fake_remotes.xml
 228               -Pq
 229               -Dsft.heap.max=4g
 230               -DskipTests=false
 231               {mvn-opts}
 232           java-opts:
 233             - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true'
 234
 235 - builder:
 236     name: distribution-check-warn-9191
 237     # Detect ODL bundles referenced cross-project from features, store them to a file to be archived.
 238     # Do not fail the build, this is just to inform interested contributors.
 239     # TODO: Improve integration archiving macro so that it finds bug_9191 log without us crating it in $WORKSPACE/.
 240     builders:
 241       - shell: |
 242           # No failure in this macro should affect the job result.
 243           set +e
 244           rm -f "$WORKSPACE/bug_9191.log"
 245           pushd "$WORKSPACE/distribution/$KARAF_ARTIFACT/target/assembly/system/org/opendaylight"
 246           for folder in `ls`; do
 247           find "$folder" -name \*features.xml | xargs fgrep -n '<bundle>' > bug_9191.0.log
 248           fgrep org.opendaylight bug_9191.0.log | fgrep -v "org.opendaylight.$folder" >> "$WORKSPACE/bug_9191.log"
 249           rm bug_9191.0.log
 250           done
 251           popd
 252
 253 - builder:
 254     name: distribution-check-warn-9192
 255     # Detect ODL bundles referenced from multiple features within a project, store them to a file to be archived.
 256     # Current implementation will also include Bug 9191 violations,
 257     # presenting an unified diff with lines marked to be removed, among unmarked lines to compare to.
 258     # Do not fail the build, this is just to inform interested contributors.
 259     # TODO: Improve integration archiving macro so that it finds bug_9192 log without us crating it in $WORKSPACE/.
 260     builders:
 261       - shell: |
 262           # No failure in this macro should affect the job result.
 263           set +e
 264           pushd "$WORKSPACE/distribution/$KARAF_ARTIFACT/target/assembly/system/org/opendaylight"
 265           find . -name \*features.xml | xargs fgrep -n '<bundle>' > "$WORKSPACE/bug_9192.0.log"
 266           popd
 267           fgrep org.opendaylight bug_9192.0.log > "$WORKSPACE/bug_9192.1.log"
 268           sort -s -t '<' -k 2 "$WORKSPACE/bug_9192.1.log" > "$WORKSPACE/bug_9192.2.log"
 269           sort -u -s -t '<' -k 2 "$WORKSPACE/bug_9192.1.log" > "$WORKSPACE/bug_9192.3.log"
 270           diff -U 999 -b -t -d "$WORKSPACE/bug_9192.2.log" "$WORKSPACE/bug_9192.3.log" > "$WORKSPACE/bug_9192.diff.log"
 271           rm -f "$WORKSPACE"/bug_9192.?.log
 272
 273 - builder:
 274     name: distribution-check-bootup
 275     # Operation: Locally deploy ODL with all fetures, check it is not broken.
 276     # Required bash variables:
 277     #   KARAF_VERSION: 'karaf4' or 'karaf3'.
 278     #     Use distribution-karaf-version macro to customize.
 279     builders:
 280       - shell: !include-raw: distribution-check-bootup.sh