Do not run packer merge jobs concurrently
[releng/builder.git] / jjb / releng-jobs.yaml
index 52cbc226c2d957a8d39357a766a65624ae1ad883..5c670819831d8f96f6755292ecc7789a62bc7e70 100644 (file)
         - 'builder-merge'
         - 'builder-weekly'
         - 'builder-check-poms'
+        - 'builder-verify-packer'
+        - 'builder-merge-packer-{platforms}-{templates}'
+        # OpenStack Related
+        - 'builder-delete-stale-stacks'
+        # Automation for docs and jobs
+        - 'builder-update-csit-tests'
+        - 'builder-update-image-list'
+
+    platforms:
+        - centos
+        - ubuntu-14.04
+
+    templates:
+        - devstack
+        - devstack-pre-pip-mitaka
+        - devstack-pre-pip-newton
+        - docker
+        - gbp
+        - java-builder
+        - mininet
+        - mininet-ovs-2.3
+        - mininet-ovs-2.5
+        - robot
+
+    exclude:
+        - platforms: centos
+          templates: gbp
+        - platforms: centos
+          templates: mininet
+        - platforms: centos
+          templates: mininet-ovs-2.3
+        - platforms: centos
+          templates: mininet-ovs-2.5
+        - platforms: ubuntu-14.04
+          templates: java-builder
+        - platforms: ubuntu-14.04
+          templates: robot
+        - platforms: ubuntu-14.04
+          templates: devstack-pre-pip-mitaka
+        - platforms: ubuntu-14.04
+          templates: devstack-pre-pip-newton
+        - platforms: ubuntu-14.04
+          templates: devstack
+        - platforms: ubuntu-14.04
+          templates: docker
 
-        # Maven Sites
-        - 'builder-site-publish'
-        - 'builder-site-verify'
-
+    project-type: freestyle
+    archive-artifacts: '**/*.log'
+    build-timeout: '60'
+    # The following two values are only specified here
+    # so that code blocks look similar to other projects.
     project: 'releng/builder'
+    branch: 'master'
 
 - job-template:
     name: builder-verify-jjb
-    node: jjbuilder
+    node: centos7-java-builder-2c-4g
+    concurrent: true
 
     project-type: freestyle
 
-    logrotate:
-        daysToKeep: 7
+    properties:
+        - opendaylight-infra-properties:
+            build-days-to-keep: 14
 
     parameters:
-        - project-parameter:
+        - opendaylight-infra-parameters:
             project: '{project}'
-        - gerrit-parameters:
-            project: '{project}'
-            branch: 'master'
-            refspec: 'refs/heads/master'
+            branch: '{branch}'
+            refspec: 'refs/heads/{branch}'
+            artifacts: '{archive-artifacts}'
 
     scm:
         - gerrit-trigger-scm:
-            credentials-id: '{ssh-credentials}'
             refspec: '$GERRIT_REFSPEC'
             choosing-strategy: 'gerrit'
 
     wrappers:
-        - ssh-agent-credentials:
-            users:
-                - '{ssh-credentials}'
+        - opendaylight-infra-wrappers:
+            build-timeout: '{build-timeout}'
 
     triggers:
-        - gerrit:
-            server-name: 'OpenDaylight'
-            trigger-on:
-                - patchset-created-event:
-                    exclude-drafts: 'false'
-                    exclude-trivial-rebase: 'false'
-                    exclude-no-code-change: 'false'
-                - draft-published-event
-                - comment-added-contains-event:
-                    comment-contains-value: 'recheck'
-            projects:
-              - project-compare-type: 'ANT'
-                project-pattern: 'releng/builder'
-                branches:
-                  - branch-compare-type: 'ANT'
-                    branch-pattern: '**/master'
-                file-paths:
-                    - compare-type: ANT
-                      pattern: jjb/**
-                    - compare-type: ANT
-                      pattern: jjb-templates/**
+        - gerrit-trigger-patch-submitted:
+            server: '{server-name}'
+            project: '{project}'
+            branch: '{branch}'
+            files: 'jjb/**'
 
     builders:
+        - opendaylight-infra-jjbini
         - shell: |
+            virtualenv $WORKSPACE/venv
+            source $WORKSPACE/venv/bin/activate
+            pip install --upgrade pip
             pip freeze
-            python scripts/jjb-autoupdate-project.py
-            jenkins-jobs test --recursive -o archives/job_output jjb/
+            pip install jenkins-job-builder
+            jenkins-jobs -l DEBUG test --recursive -o archives/job_output jjb/
             gzip archives/job_output/*
-
         - releng-check-unicode
+        - verify-gpg-signature
 
     publishers:
         - email-notification:
+            email-recipients: '{email-recipients}'
             email-prefix: '[releng]'
-        - archive-build:
-            maven-version: '{mvn33}'
+        - opendaylight-infra-shiplogs:
+            maven-version: 'mvn33'
 
+# FIXME: Migrate to {project-name}-verify-python-{stream} or delete this entirely,
+#        as there are zero python files (aside docs/conf.py) in releng/builder.
 - job-template:
     name: builder-verify-python
-    node: dynamic_verify
+    node: centos7-java-builder-2c-4g
+    concurrent: true
 
     project-type: freestyle
 
-    logrotate:
-        daysToKeep: 7
-        numToKeep: 10
-        artifactDaysToKeep: -1
-        artifactNumToKeep: -1
+    properties:
+        - opendaylight-infra-properties:
+            build-days-to-keep: 14
 
     parameters:
-        - project-parameter:
-            project: '{project}'
-        - gerrit-parameters:
+        - opendaylight-infra-parameters:
             project: '{project}'
-            branch: 'master'
-            refspec: 'refs/heads/master'
+            branch: '{branch}'
+            refspec: 'refs/heads/{branch}'
+            artifacts: '{archive-artifacts}'
 
     scm:
         - gerrit-trigger-scm:
-            credentials-id: '{ssh-credentials}'
             refspec: '$GERRIT_REFSPEC'
             choosing-strategy: 'gerrit'
 
     wrappers:
-        - ssh-agent-credentials:
-            users:
-                - '{ssh-credentials}'
-        - openstack:
-            single-use: true
+        - opendaylight-infra-wrappers:
+            build-timeout: '{build-timeout}'
 
     triggers:
-        - gerrit:
-            server-name: 'OpenDaylight'
-            trigger-on:
-                - patchset-created-event:
-                    exclude-drafts: 'false'
-                    exclude-trivial-rebase: 'false'
-                    exclude-no-code-change: 'false'
-                - draft-published-event
-                - comment-added-contains-event:
-                    comment-contains-value: 'recheck'
-            projects:
-              - project-compare-type: 'ANT'
-                project-pattern: 'releng/builder'
-                branches:
-                  - branch-compare-type: 'ANT'
-                    branch-pattern: '**/master'
-                file-paths:
-                    - compare-type: ANT
-                      pattern: scripts/**
-
+        - gerrit-trigger-patch-submitted:
+            server: '{server-name}'
+            project: '{project}'
+            branch: '{branch}'
+            files: 'scripts/**'
 
     builders:
+        - verify-gpg-signature
         - shell: |
             virtualenv $WORKSPACE/venv
             source $WORKSPACE/venv/bin/activate
 
     publishers:
         - email-notification:
+            email-recipients: '{email-recipients}'
             email-prefix: '[releng]'
-        - archive-build:
-            maven-version: '{mvn33}'
+        - opendaylight-infra-shiplogs:
+            maven-version: 'mvn33'
 
 - job-template:
     name: 'builder-merge'
-    node: jjbuilder
+    node: centos7-java-builder-2c-4g
 
     # builder-merge job to run JJB update
     #
 
     project-type: freestyle
 
-    logrotate:
-        daysToKeep: 7
-        numToKeep: 3
-        artifactDaysToKeep: -1
-        artifactNumToKeep: -1
+    properties:
+        - opendaylight-infra-properties:
+            build-days-to-keep: 14
 
     parameters:
-        - project-parameter:
-            project: '{project}'
-        - gerrit-parameters:
+        - opendaylight-infra-parameters:
             project: '{project}'
-            branch: 'master'
-            refspec: 'refs/heads/master'
+            branch: '{branch}'
+            refspec: 'refs/heads/{branch}'
+            artifacts: '{archive-artifacts}'
 
     scm:
         - gerrit-trigger-scm:
-            credentials-id: '{ssh-credentials}'
             refspec: ''
             choosing-strategy: 'default'
 
     wrappers:
-        - ssh-agent-credentials:
-            users:
-                - '{ssh-credentials}'
+        - opendaylight-infra-wrappers:
+            build-timeout: '{build-timeout}'
 
     triggers:
         - gerrit:
-            server-name: 'OpenDaylight'
+            server-name: '{server-name}'
             trigger-on:
                 - change-merged-event
                 - comment-added-contains-event:
                 project-pattern: 'releng/builder'
                 branches:
                     - branch-compare-type: 'ANT'
-                      branch-pattern: '**/master'
+                      branch-pattern: '**/{branch}'
                 file-paths:
                     - compare-type: ANT
                       pattern: jjb/**
-                    - compare-type: ANT
-                      pattern: jjb-templates/**
 
     builders:
-        - shell:
-            !include-raw-escape:
-                - include-raw-jjb-update.sh
+        - opendaylight-infra-jjbini
+        - shell: |
+            virtualenv $WORKSPACE/venv
+            source $WORKSPACE/venv/bin/activate
+            pip install --upgrade pip
+            pip freeze
+            pip install jenkins-job-builder
+            jenkins-jobs --version
+            jenkins-jobs update --recursive --delete-old --workers 4 jjb/
 
     publishers:
         - email-notification:
+            email-recipients: '{email-recipients}'
             email-prefix: '[releng]'
-        - archive-build:
-            maven-version: '{mvn33}'
+        - opendaylight-infra-shiplogs:
+            maven-version: 'mvn33'
 
 - job-template:
     name: 'builder-weekly'
-    disabled: true
-    node: jjbuilder
+    disabled: false
+    node: centos7-java-builder-2c-4g
 
     # builder-weekly job to run JJB update weekly
     #
 
     project-type: freestyle
 
-    logrotate:
-        daysToKeep: 7
-        numToKeep: 3
-        artifactDaysToKeep: -1
-        artifactNumToKeep: -1
+    properties:
+        - opendaylight-infra-properties:
+            build-days-to-keep: 14
 
     parameters:
-        - project-parameter:
+        - opendaylight-infra-parameters:
             project: '{project}'
+            branch: '{branch}'
+            refspec: 'refs/heads/{branch}'
+            artifacts: '{archive-artifacts}'
 
     scm:
         - git-scm:
-            credentials-id: '{ssh-credentials}'
-            branch: 'master'
+            branch: '{branch}'
 
     wrappers:
-        - ssh-agent-credentials:
-            users:
-                - '{ssh-credentials}'
+        - opendaylight-infra-wrappers:
+            build-timeout: '{build-timeout}'
 
     triggers:
         - timed: 'H H * * 0'
 
     builders:
+        - opendaylight-infra-jjbini
         - shell: |
-            jenkins-jobs --flush-cache update --recursive jjb/
+            virtualenv $WORKSPACE/venv
+            source $WORKSPACE/venv/bin/activate
+            pip install --upgrade pip
+            pip freeze
+            pip install jenkins-job-builder
+            jenkins-jobs --flush-cache update --recursive --workers 4 jjb/
 
     publishers:
         - email-notification:
+            email-recipients: '{email-recipients}'
             email-prefix: '[releng]'
-        - archive-build:
-            maven-version: '{mvn33}'
+        - opendaylight-infra-shiplogs:
+            maven-version: 'mvn33'
 
 - job-template:
     name: builder-check-poms
-    node: dynamic_verify
+    node: centos7-java-builder-2c-4g
 
     project-type: freestyle
 
-    logrotate:
-        daysToKeep: 1
-        numToKeep: 1
-        artifactDaysToKeep: -1
-        artifactNumToKeep: -1
+    properties:
+        - opendaylight-infra-properties:
+            build-days-to-keep: 14
+
+    parameters:
+        - opendaylight-infra-parameters:
+            project: '{project}'
+            branch: '{branch}'
+            refspec: 'refs/heads/{branch}'
+            artifacts: '{archive-artifacts}'
 
     wrappers:
-        - ssh-agent-credentials:
-            users:
-                - '{ssh-credentials}'
-        - openstack:
-            single-use: true
+        - opendaylight-infra-wrappers:
+            build-timeout: '{build-timeout}'
 
     triggers:
         - timed: 'H H * * 1'
 
     publishers:
         - email-notification:
+            email-recipients: '{email-recipients}'
             email-prefix: '[releng]'
-        - archive-build:
-            maven-version: '{mvn33}'
+        - opendaylight-infra-shiplogs:
+            maven-version: 'mvn33'
 
+- job-template:
+    name: 'builder-delete-stale-stacks'
+    project-type: freestyle
+    node: centos7-java-builder-2c-4g
 
-###
-### Maven Sites
-###
+    properties:
+        - opendaylight-infra-properties:
+            build-days-to-keep: 7
+
+    wrappers:
+        - opendaylight-infra-wrappers:
+            build-timeout: '{build-timeout}'
+
+    triggers:
+        # Attempt to clear up stacks every 30 mins in case we have orphaned stacks
+        - timed: '0,30 * * * *'
+
+    builders:
+        - shell: !include-raw-escape: opendaylight-infra-cleanup-stale-stacks.sh
+
+    publishers:
+        - email-notification:
+            email-recipients: '{email-recipients}'
+            email-prefix: '[releng]'
+        - opendaylight-infra-shiplogs:
+            maven-version: 'mvn33'
 
 - job-template:
-    name: 'builder-site-publish'
-    node: dynamic_merge
-    concurrent: false
-    jdk: 'openjdk8'
+    name: 'builder-update-image-list'
+    project-type: freestyle
+    node: centos7-java-builder-2c-4g
+
+    properties:
+        - opendaylight-infra-properties:
+            build-days-to-keep: 7
+
+    parameters:
+        - opendaylight-infra-parameters:
+            project: '{project}'
+            branch: '{branch}'
+            refspec: 'refs/heads/{branch}'
+            artifacts: '{archive-artifacts}'
+
+    scm:
+        - git-scm:
+            branch: '{branch}'
 
-    # This job's purpose is to publish maven site
+    wrappers:
+        - opendaylight-infra-wrappers:
+            build-timeout: '{build-timeout}'
+        # Listed after to override openstack-infra-wrappers clouds.yaml definition
+        - config-file-provider:
+            files:
+                - file-id: opendaylight-private-clouds-yaml
+                  target: '$HOME/.config/openstack/clouds.yaml'
 
-    project-type: maven
+    triggers:
+        # Update image list every Monday to Friday at 11:00 UTC
+        - timed: '0 11 * * 1-5'
 
-    logrotate:
-        daysToKeep: 7
-        numToKeep: 3
-        artifactDaysToKeep: -1
-        artifactNumToKeep: -1
+    builders:
+        - shell: !include-raw-escape: opendaylight-infra-update-image-list.sh
+        - opendaylight-infra-push-gerrit-patch:
+            project: '{project}'
+            gerrit-topic: 'releng-update-cloud-image-list'
+            gerrit-commit-message: 'Update cloud image list docs'
+
+    publishers:
+        - email-notification:
+            email-recipients: '{email-recipients}'
+            email-prefix: '[releng]'
+        - opendaylight-infra-shiplogs:
+            maven-version: 'mvn33'
+
+- job-template:
+    name: 'builder-verify-packer'
+    project-type: freestyle
+    node: centos7-java-builder-2c-4g
+    concurrent: true
+
+    properties:
+        - opendaylight-infra-properties:
+            build-days-to-keep: 14
 
     parameters:
-        - project-parameter:
+        - opendaylight-infra-parameters:
             project: '{project}'
-        - gerrit-parameters:
+            branch: '{branch}'
+            refspec: 'refs/heads/{branch}'
+            artifacts: '{archive-artifacts}'
+
+    scm:
+        - gerrit-trigger-scm:
+            refspec: '$GERRIT_REFSPEC'
+            choosing-strategy: 'gerrit'
+
+    wrappers:
+        - opendaylight-infra-wrappers:
+            build-timeout: '{build-timeout}'
+
+    triggers:
+        - gerrit-trigger-patch-submitted:
+            server: '{server-name}'
             project: '{project}'
-            branch: 'master'
-            refspec: 'refs/heads/master'
+            branch: '{branch}'
+            files: 'packer/**'
+
+    builders:
+        - packer-validate:
+
+    publishers:
+        - email-notification:
+            email-recipients: '{email-recipients}'
+            email-prefix: '[{project}]'
+        - opendaylight-infra-shiplogs:
+            maven-version: 'mvn33'
+
+- job-template:
+    name: 'builder-merge-packer-{platforms}-{templates}'
+    project-type: freestyle
+    node: centos7-java-builder-2c-4g
+
+    properties:
+        - opendaylight-infra-properties:
+            build-days-to-keep: 14
+
+    parameters:
+        - opendaylight-infra-parameters:
+            project: '{project}'
+            branch: '{branch}'
+            refspec: 'refs/heads/{branch}'
+            artifacts: '{archive-artifacts}'
 
     scm:
         - gerrit-trigger-scm:
-            credentials-id: '{ssh-credentials}'
-            refspec: ''
+            refspec: '$GERRIT_REFSPEC'
+            branches:
+                - '$GERRIT_BRANCH'
             choosing-strategy: 'default'
 
     wrappers:
-        - ssh-agent-credentials:
-            users:
-                - '{ssh-credentials}'
-        - openstack:
-            single-use: true
+        - opendaylight-infra-wrappers:
+            build-timeout: '{build-timeout}'
 
     triggers:
+        - timed: '00 10 1 * *'
         - gerrit:
-            server-name: 'OpenDaylight'
+            server-name: '{server-name}'
             trigger-on:
                 - change-merged-event
                 - comment-added-contains-event:
-                    comment-contains-value: 'republish'
+                    comment-contains-value: 'remerge'
             projects:
               - project-compare-type: 'ANT'
                 project-pattern: 'releng/builder'
                 branches:
                     - branch-compare-type: 'ANT'
-                      branch-pattern: '**/master'
+                      branch-pattern: '**/{branch}'
                 file-paths:
                     - compare-type: ANT
-                      pattern: src/**
+                      pattern: 'packer/vars/{platforms}.json'
+                    - compare-type: ANT
+                      pattern: 'packer/templates/{templates}.json'
+                    - compare-type: ANT
+                      pattern: 'packer/provision/{templates}.sh'
+                    - compare-type: ANT
+                      pattern: 'packer/provision/system_reseal.sh'
+                    - compare-type: ANT
+                      pattern: 'packer/provision/rh-user_data.sh'
                     - compare-type: ANT
-                      pattern: pom.xml
+                      pattern: 'packer/provision/devstack-pre-pip.sh'
+                    - compare-type: ANT
+                      pattern: 'packer/provision/baseline.sh'
 
-    maven:
-        maven-name: '{mvn33}'
-        root-pom: 'pom.xml'
-        goals: 'site:site site:deploy -Dstream=master'
-        maven-opts: '-Xmx1024m'
-        settings: 'releng-builder-settings'
-        global-settings: 'odl-global-settings'
+    builders:
+        - packer-validate:
+        - packer-build:
+            platform: '{platforms}'
+            template: '{templates}'
 
     publishers:
         - email-notification:
-            email-prefix: '[releng]'
-        - archive-build:
-            maven-version: '{mvn33}'
+            email-recipients: '{email-recipients}'
+            email-prefix: '[{project}]'
+        - opendaylight-infra-shiplogs:
+            maven-version: 'mvn33'
 
 - job-template:
-    name: 'builder-site-verify'
-    node: dynamic_verify
-    concurrent: true
-    jdk: 'openjdk8'
+    name: 'builder-update-csit-tests'
+    node: centos7-java-builder-2c-8g
 
-    # This job's purpose is to verify maven site configuration
+    # builder-update-csit-tests job
+    #
+    # This job's purpose is to update csit
+    # tests to run in integration-distribution-test-{stream}
+    # by updating releng-defaults variables: csit-list-carbon,
+    # csit-list-boron, csit-list-beryllium
 
-    project-type: maven
+    project-type: freestyle
 
-    logrotate:
-        daysToKeep: 7
-        numToKeep: 10
-        artifactDaysToKeep: -1
-        artifactNumToKeep: -1
+    properties:
+        - opendaylight-infra-properties:
+            build-days-to-keep: 14
 
     parameters:
-        - project-parameter:
+        - opendaylight-infra-parameters:
             project: '{project}'
-        - gerrit-parameters:
-            project: '{project}'
-            branch: 'master'
-            refspec: 'refs/heads/master'
+            branch: '{branch}'
+            refspec: 'refs/heads/{branch}'
+            artifacts: '{archive-artifacts}'
 
     scm:
         - gerrit-trigger-scm:
-            credentials-id: '{ssh-credentials}'
-            refspec: '$GERRIT_REFSPEC'
-            choosing-strategy: 'gerrit'
+            refspec: ''
+            choosing-strategy: 'default'
 
     wrappers:
-        - ssh-agent-credentials:
-            users:
-                - '{ssh-credentials}'
-        - openstack:
-            single-use: true
+        - opendaylight-infra-wrappers:
+            build-timeout: '{build-timeout}'
 
     triggers:
-        - gerrit:
-            server-name: 'OpenDaylight'
-            trigger-on:
-                - patchset-created-event:
-                    exclude-drafts: 'false'
-                    exclude-trivial-rebase: 'false'
-                    exclude-no-code-change: 'false'
-                - draft-published-event
-                - comment-added-contains-event:
-                    comment-contains-value: 'recheck'
-            projects:
-              - project-compare-type: 'ANT'
-                project-pattern: 'releng/builder'
-                branches:
-                  - branch-compare-type: 'ANT'
-                    branch-pattern: '**/master'
-                file-paths:
-                    - compare-type: ANT
-                      pattern: src/**
-                    - compare-type: ANT
-                      pattern: pom.xml
+        - reverse:
+               jobs: 'builder-merge'
 
-    maven:
-        maven-name: '{mvn33}'
-        root-pom: 'pom.xml'
-        goals: 'site:site -Dstream=master'
-        maven-opts: '-Xmx1024m'
-        settings: 'autorelease-settings'
-        global-settings: 'odl-global-settings'
+    builders:
+        - shell: |
+            git checkout -b auto-update-csit-jobs
+        - integration-list-jobs:
+            blacklist: '{csit-list-blacklist}'
+            search_string: "csit"
+            stream: "carbon"
+            jobs-filename: "csit_jobs_carbon.lst"
+        - integration-list-jobs:
+            blacklist: '{csit-list-blacklist}'
+            search_string: "csit"
+            stream: "boron"
+            jobs-filename: "csit_jobs_boron.lst"
+        - integration-list-jobs:
+            blacklist: '{csit-list-blacklist}'
+            search_string: "csit"
+            stream: "beryllium"
+            jobs-filename: "csit_jobs_beryllium.lst"
+        - integration-replace-block-text:
+            starting-regex: "'csit-list-carbon: >'"
+            ending-regex: "'csit-list-boron: >'"
+            file-with-changes-to-insert: "'csit_jobs_carbon.lst'"
+            file-to-change: "'jjb/releng-defaults.yaml'"
+        - integration-replace-block-text:
+            starting-regex: "'csit-list-boron: >'"
+            ending-regex: "'csit-list-beryllium: >'"
+            file-with-changes-to-insert: "'csit_jobs_boron.lst'"
+            file-to-change: "'jjb/releng-defaults.yaml'"
+        - integration-replace-block-text:
+            starting-regex: "'csit-list-beryllium: >'"
+            ending-regex: "'# CSIT TESTS END SED MARKER'"
+            file-with-changes-to-insert: "'csit_jobs_beryllium.lst'"
+            file-to-change: "'jjb/releng-defaults.yaml'"
+        - shell: |
+            git status
+            git add jjb/releng-defaults.yaml
+        - opendaylight-infra-push-gerrit-patch:
+            project: "releng/builder"
+            gerrit-topic: "auto-update-csit-jobs"
+            gerrit-commit-message: "Auto Update CSIT Jobs to run"
 
     publishers:
         - email-notification:
+            email-recipients: '{email-recipients}'
             email-prefix: '[releng]'
-        - archive-build:
-            maven-version: '{mvn33}'
+        - opendaylight-infra-shiplogs:
+            maven-version: 'mvn33'