1 # JJB macros specific to Integration/Distribution ODL project
4 name: distribution-check-wipe
5 # Step zero: Wipe file repositories up front.
8 echo "wipe r: the local Maven repository"
10 echo "wipe n: the fake remote (Nexus) repository"
12 echo "wipe t: the transient repository used in some manipulations"
14 echo "create n: multithreaded execution might fail at creating it."
16 echo "detecting distribution allowed projects"
17 # Some allowed projects cannot be detected in distribution because they do not produce features.
18 ALLOW_PROJECTS=(yangtools mdsal openflowjava)
19 if [[ "$KARAF_VERSION" == "odl" ]]; then
20 ALLOW_PROJECTS+=(`grep '<groupId>org.opendaylight.' -Rh distribution \
21 | sed -e 's%^[ \t]*<groupId>org.opendaylight.%%' \
22 | sed -e 's%</groupId>%%' | sort -u`)
24 # For Managed distro we only look at the features folder
25 ALLOW_PROJECTS+=(`grep '<groupId>org.opendaylight.' -Rh distribution/features \
26 | sed -e 's%^[ \t]*<groupId>org.opendaylight.%%' \
27 | sed -e 's%</groupId>%%' | sort -u`)
29 echo "Allowed projects are ${ALLOW_PROJECTS[@]}"
30 echo "ALLOW_PROJECTS=${ALLOW_PROJECTS[@]}" > allowed_projects.txt
33 name: distribution-check-build-project
34 # Step one: Online build of the project, using local repository /tmp/r/ and deploying artifacts to /tmp/n/.
35 # Ordinary SingleFeatureTest failures are detected in the verify job, so we can use "q" profile here.
37 # pom: Relative path to pom file to use. Typically '$GERRIT_PROJECT/pom.xml'.
40 maven-version: '{mvn-version}'
44 dependency:tree -DoutputFile=dependency_tree.txt
46 -DaltDeploymentRepository=fake-nexus::default::file:///tmp/n/
49 - '-Xmx4096m -Dmaven.compile.fork=true'
50 settings: 'integration-settings'
52 global-settings: 'global-settings'
53 global-settings-type: cfp
56 name: distribution-check-verify-groupid
57 # Step two: Verify all deployed artifacts belong to the project's groupId.
58 # This is done by moving the allowed directories out of /tmp/n and checking no files remained there.
59 # The correct directory is derived from $GERRIT_PROJECT.
60 # TODO: Verify all deployed artifacts are snapshots.
62 # gerrit-project: Project name as nexus URI part. Typically '$GERRIT_PROJECT'.
65 echo "These are allowed projects: $ALLOW_PROJECTS"
66 echo "These are distribution pulled projects:"
68 for PROJECT in `ls /tmp/r/org/opendaylight`; do
69 echo "checking $PROJECT"
70 if [[ "$ALLOW_PROJECTS" != *"$PROJECT"* ]]; then
71 echo "ERROR: Not allowed project $PROJECT pulled"
76 echo "verify project groupId"
77 mkdir -p /tmp/t/org/opendaylight/{gerrit-project}
78 mv /tmp/n/org/opendaylight/{gerrit-project}/* /tmp/t/org/opendaylight/{gerrit-project}/
79 test -z "`find /tmp/n/ -type f`" || ( echo "ERROR: Mismatched groupId detected (see above)." && false )
84 name: distribution-check-download-deps
85 # Step three: Online build of integration distribution.
86 # This step is mainly used for downloading other project artifacts.
87 # Running SingleFeaturesTest here does not seem to be required, so -Pq is used again.
89 # dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
92 maven-version: '{mvn-version}'
96 dependency:tree -DoutputFile=dependency_tree.txt
100 - '-Xmx1024m -Dmaven.compile.fork=true'
101 settings: 'integration-settings'
103 global-settings: 'global-settings'
104 global-settings-type: cfp
107 name: distribution-check-delete-snapshots
108 # Step four: Delete snapshot artifacts from the local repository.
109 # This is critical to detect orphaned artifacts or missing project-internal dependency declarations.
110 # Also other files related to maven repository resolution are removed,
111 # and then empty directories are removed, in order to simplify debugging.
113 - shell: !include-raw-escape: distribution-delete-snaphot-artifacts.sh
116 name: distribution-check-configure-remotes
117 # Now the ugly part. It seems that the only way to tell Maven 2+
118 # which remote repositories to use is via settings.xml file.
119 # So we create such a file here, but it needs most of odlparent:settings.xml
123 <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
124 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
125 xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
126 https://maven.apache.org/xsd/settings-1.0.0.xsd">
129 <id>opendaylight-release</id>
132 <id>opendaylight-mirror</id>
133 <name>opendaylight</name>
134 <url>https://nexus.opendaylight.org/content/repositories/public/</url>
135 <releases><updatePolicy>never</updatePolicy></releases>
136 <snapshots><enabled>false</enabled></snapshots>
141 <id>opendaylight-plugin-mirror</id>
142 <name>opendaylight-plugin</name>
143 <url>https://nexus.opendaylight.org/content/repositories/public/</url>
144 <releases><updatePolicy>never</updatePolicy></releases>
145 <snapshots><enabled>false</enabled></snapshots>
147 </pluginRepositories>
150 <id>file-snapshots</id>
153 <id>file-snapshots</id>
155 <url>file:///tmp/n/</url>
156 <releases><enabled>false</enabled></releases>
161 <id>file-plugin-snapshots</id>
162 <name>file-plugin</name>
163 <url>file:///tmp/n/</url>
164 <releases><enabled>false</enabled></releases>
166 </pluginRepositories>
170 <activeProfile>file-snapshots</activeProfile>
171 <activeProfile>opendaylight-release</activeProfile>
175 # Notes: The settings are minimal in order to detect breakage scenarios,
176 # while allowing for the following quirks:
177 # * Some plugins seem to have hardcoded repos, for example check-license looks at repository.apache.org
178 # * Some plugin artifacts (related to surefire) are not downloaded when tests are skipped.
179 # * populate-local-repo looks at oss.sonatype.org and does not store things (like guava) to /tmp/r
182 name: distribution-check-repeat-project-build
183 # Step five: Repeat the project build but with the new settings.
184 # Here, only the project snapshot artifacts deployed to /tmp/n are available,
185 # which faithfully reproduces conditions in later verify-like job runs.
186 # We cannot use --offline, because: "Cannot access file (file:///tmp/n) in offline mode".
187 # SingleFeatureTest is still skipped, as we expect the same results as in the verify job.
188 # Current content of /tmp/r is restored afterwards to not affect the next step.
189 # TODO: Could there be missing dependencies which would only affect SFT here?
191 # pom: Relative path to pom file to use. Typically '$GERRIT_PROJECT/pom.xml'.
196 maven-version: '{mvn-version}'
200 dependency:tree -DoutputFile=dependency_tree.txt
205 - '-Xmx4096m -Dmaven.compile.fork=true'
211 name: distribution-check-repeat-distro-build
212 # Step six: Repeat the distribution build but with the new settings.
213 # Here, only the project snapshot artifacts deployed to /tmp/n are available,
214 # which faithfully reproduces conditions in later verify-like job runs.
215 # We cannot use --offline, because: "Cannot access file (file:///tmp/n) in offline mode".
216 # This is where SingleFeatureTest is not skipped.
217 # TODO: Lower -Dsft.heap.max to 3g or lower when Bug 9218 is fixed.
219 # dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
222 maven-version: '{mvn-version}'
226 dependency:tree -DoutputFile=dependency_tree.txt
233 - '-Xmx1024m -Dmaven.compile.fork=true'
236 name: distribution-check-warn-9191
237 # Detect ODL bundles referenced cross-project from features, store them to a file to be archived.
238 # Do not fail the build, this is just to inform interested contributors.
239 # TODO: Improve integration archiving macro so that it finds bug_9191 log without us crating it in $WORKSPACE/.
242 # No failure in this macro should affect the job result.
244 rm -f "$WORKSPACE/bug_9191.log"
245 pushd "$WORKSPACE/distribution/$KARAF_ARTIFACT/target/assembly/system/org/opendaylight"
246 for folder in `ls`; do
247 find "$folder" -name \*features.xml | xargs fgrep -n '<bundle>' > bug_9191.0.log
248 fgrep org.opendaylight bug_9191.0.log | fgrep -v "org.opendaylight.$folder" >> "$WORKSPACE/bug_9191.log"
254 name: distribution-check-warn-9192
255 # Detect ODL bundles referenced from multiple features within a project, store them to a file to be archived.
256 # Current implementation will also include Bug 9191 violations,
257 # presenting an unified diff with lines marked to be removed, among unmarked lines to compare to.
258 # Do not fail the build, this is just to inform interested contributors.
259 # TODO: Improve integration archiving macro so that it finds bug_9192 log without us crating it in $WORKSPACE/.
262 # No failure in this macro should affect the job result.
264 pushd "$WORKSPACE/distribution/$KARAF_ARTIFACT/target/assembly/system/org/opendaylight"
265 find . -name \*features.xml | xargs fgrep -n '<bundle>' > "$WORKSPACE/bug_9192.0.log"
267 fgrep org.opendaylight bug_9192.0.log > "$WORKSPACE/bug_9192.1.log"
268 sort -s -t '<' -k 2 "$WORKSPACE/bug_9192.1.log" > "$WORKSPACE/bug_9192.2.log"
269 sort -u -s -t '<' -k 2 "$WORKSPACE/bug_9192.1.log" > "$WORKSPACE/bug_9192.3.log"
270 diff -U 999 -b -t -d "$WORKSPACE/bug_9192.2.log" "$WORKSPACE/bug_9192.3.log" > "$WORKSPACE/bug_9192.diff.log"
271 rm -f "$WORKSPACE"/bug_9192.?.log
274 name: distribution-check-bootup
275 # Operation: Locally deploy ODL with all fetures, check it is not broken.
276 # Required bash variables:
277 # KARAF_VERSION: 'karaf4' or 'karaf3'.
278 # Use distribution-karaf-version macro to customize.
280 - shell: !include-raw: distribution-check-bootup.sh