From 2ab60b6cb2bfe6c23ce442ccc8ef628a2ac26609 Mon Sep 17 00:00:00 2001 From: Yuxiang Zhu Date: Sep 06 2018 04:17:00 +0000 Subject: CI/CD: add C3I jobs to poll for master branch and PRs on Pagure Two Jenkins jobs are added to poll for master branch changes and new PRs on Pagure. As a temporary solution for triggering pipeline builds automatically when changes on master branch or pull-requests, polling is used before we eventually move to UMB-WebHook triggers. The 2 polling jobs are needed because 1. to keep the logic of polling separated with dev-pipeline job, which could be triggered by UMB-WebHook eventually. 2. Currently Jenkins requires a persist workspace to poll for wildcard branch names. dev-pipeline job runs Jenkins slaves as pods, which are discarded after job runs. 3. We need to run dev-pipline for both pre-merge and post-merge workflows. Polling for PRs and the master branch in a single Jenkins job will *NOT* trigger a master branch build when a fast-forward merge happens to a PR. We can't move the Jenkinsfile to an external file otherwise Jenkins will also poll for the branch where Jenkins file is referenced. --- diff --git a/openshift/pipelines/jobs/waiverdb-polling-for-master.env b/openshift/pipelines/jobs/waiverdb-polling-for-master.env new file mode 100644 index 0000000..63d1edc --- /dev/null +++ b/openshift/pipelines/jobs/waiverdb-polling-for-master.env @@ -0,0 +1,4 @@ +NAME=waiverdb-polling-for-master +PAGURE_POLLING_SCHEDULE="H/5 * * * *" +PAGURE_POLLED_BRANCH=master +DEV_PIPELINE_BC_NAME=waiverdb-dev diff --git a/openshift/pipelines/jobs/waiverdb-polling-for-master.tmpl b/openshift/pipelines/jobs/waiverdb-polling-for-master.tmpl new file mode 100644 index 0000000..3d4e8e5 --- /dev/null +++ b/openshift/pipelines/jobs/waiverdb-polling-for-master.tmpl @@ -0,0 +1 @@ +waiverdb-polling-pagure.yaml diff --git a/openshift/pipelines/jobs/waiverdb-polling-for-prs.env b/openshift/pipelines/jobs/waiverdb-polling-for-prs.env new file mode 100644 index 0000000..702e71a --- /dev/null +++ b/openshift/pipelines/jobs/waiverdb-polling-for-prs.env @@ -0,0 +1,4 @@ +NAME=waiverdb-polling-for-prs +PAGURE_POLLING_FOR_PR=true +PAGURE_POLLING_SCHEDULE="H/5 * * * *" +DEV_PIPELINE_BC_NAME=waiverdb-dev diff --git a/openshift/pipelines/jobs/waiverdb-polling-for-prs.tmpl b/openshift/pipelines/jobs/waiverdb-polling-for-prs.tmpl new file mode 100644 index 0000000..3d4e8e5 --- /dev/null +++ b/openshift/pipelines/jobs/waiverdb-polling-for-prs.tmpl @@ -0,0 +1 @@ +waiverdb-polling-pagure.yaml diff --git a/openshift/pipelines/templates/waiverdb-dev-template.yaml b/openshift/pipelines/templates/waiverdb-dev-template.yaml index ac139ea..d6e2d4d 100644 --- a/openshift/pipelines/templates/waiverdb-dev-template.yaml +++ b/openshift/pipelines/templates/waiverdb-dev-template.yaml @@ -208,4 +208,6 @@ objects: value: "${WAIVERDB_INTEGRATION_TEST_BUILD_CONFIG_NAME}" - name: "WAIVERDB_INTEGRATION_TEST_BUILD_CONFIG_NAMESPACE" value: "${WAIVERDB_INTEGRATION_TEST_BUILD_CONFIG_NAMESPACE}" + - name: BUILD_DISPLAY_RENAME_TO + value: "" jenkinsfilePath: openshift/pipelines/templates/waiverdb-dev.Jenkinsfile diff --git a/openshift/pipelines/templates/waiverdb-dev.Jenkinsfile b/openshift/pipelines/templates/waiverdb-dev.Jenkinsfile index 327a046..d8c90d3 100644 --- a/openshift/pipelines/templates/waiverdb-dev.Jenkinsfile +++ b/openshift/pipelines/templates/waiverdb-dev.Jenkinsfile @@ -80,6 +80,9 @@ pipeline { stage('Prepare') { steps { script { + if (params.BUILD_DISPLAY_RENAME_TO) { + currentBuild.displayName = params.BUILD_DISPLAY_RENAME_TO + } def scmVars = checkout([$class: 'GitSCM', branches: [[name: params.WAIVERDB_GIT_REF]], userRemoteConfigs: [[url: params.WAIVERDB_GIT_REPO]], diff --git a/openshift/pipelines/templates/waiverdb-polling-pagure.yaml b/openshift/pipelines/templates/waiverdb-polling-pagure.yaml new file mode 100644 index 0000000..bca30a1 --- /dev/null +++ b/openshift/pipelines/templates/waiverdb-polling-pagure.yaml @@ -0,0 +1,204 @@ +# Template to produce a new OpenShift pipeline job for polling for Pagure branches or PRs +# +--- +apiVersion: v1 +kind: Template +metadata: + name: waiverdb-polling-to-pagure +labels: + template: waiverdb-polling-to-pagure +parameters: +- name: NAME + displayName: Short unique identifier for the templated instances + description: This field is used to deploy multiple pipelines to one OpenShift project from this template. + required: true + value: waiverdb-polling-to-pagure +- name: PAGURE_REPO_NAME + displayName: Pagure project name + description: // + required: true + value: waiverdb +- name: PAGURE_REPO_IS_FORK + displayName: Is the Pagure repo a fork? + required: true + value: "false" +- name: PAGURE_POLLING_FOR_PR + displayName: set to 'true' to poll for PRs, or 'false' for the master branch + required: true + value: "false" +- name: PAGURE_POLLING_SCHEDULE + displayName: Schedule of polling + description: using cron-style syntax + required: true + value: "H/5 * * * *" +- name: PAGURE_POLLED_BRANCH + displayName: Name of polled branch + required: true + value: "master" +- name: DEV_PIPELINE_BC_NAME + displayName: Name of BuildConfig for starting dev pipeline builds + required: true + value: waiverdb-dev +- name: DEV_PIPELINE_BC_NAMESPACE + displayName: Namespace of BuildConfig for starting dev pipeline builds + required: false +- name: JENKINS_AGENT_IMAGE + displayName: Container image for Jenkins slave pods + required: true + value: docker-registry.engineering.redhat.com/factory2/waiverdb-jenkins-slave:latest +- name: OPENSHIFT_CLOUD_NAME + displayName: Name of OpenShift cloud in Jenkins master configuration + required: true + value: openshift +objects: +- kind: ServiceAccount + apiVersion: v1 + metadata: + name: "${NAME}-jenkins-slave" + labels: + app: "${NAME}" +- kind: RoleBinding + apiVersion: v1 + metadata: + name: "${NAME}-jenkins-slave_edit" + labels: + app: "${NAME}" + subjects: + - kind: ServiceAccount + name: "${NAME}-jenkins-slave" + roleRef: + name: edit +- kind: "BuildConfig" + apiVersion: "v1" + metadata: + name: "${NAME}" + labels: + app: "${NAME}" + spec: + runPolicy: "Serial" + completionDeadlineSeconds: 1800 + strategy: + type: JenkinsPipeline + jenkinsPipelineStrategy: + jenkinsfile: |- + // Don't use external Jenkinsfile here, or Jenkins will also poll on that repo and branch + pipeline { + agent { + kubernetes { + cloud "${OPENSHIFT_CLOUD_NAME}" + label "jenkins-slave-${UUID.randomUUID().toString()}" + serviceAccount "${NAME}-jenkins-slave" + defaultContainer 'jnlp' + yaml """ + apiVersion: v1 + kind: Pod + metadata: + labels: + app: "jenkins-${env.JOB_BASE_NAME}" + factory2-pipeline-kind: "waiverdb-polling-to-pagure-pipeline" + factory2-pipeline-build-number: "${env.BUILD_NUMBER}" + spec: + containers: + - name: jnlp + image: "${JENKINS_AGENT_IMAGE}" + imagePullPolicy: Always + tty: true + resources: + requests: + memory: 256Mi + cpu: 200m + limits: + memory: 384Mi + cpu: 300m + """ + } + } + options { + timestamps() + } + triggers { pollSCM("${PAGURE_POLLING_SCHEDULE}") } + stages { + stage('Prepare') { + agent { label 'master' } + steps { + script { + if (env.PAGURE_REPO_IS_FORK == 'true') { + env.PAGURE_HOME = "https://pagure.io/fork/${PAGURE_REPO_NAME}" + env.GIT_URL = "https://pagure.io/forks/${PAGURE_REPO_NAME}.git" + env.API_PREFIX = "https://pagure.io/api/0/fork/${PAGURE_REPO_NAME}" + } else { + env.PAGURE_HOME = "https://pagure.io/${PAGURE_REPO_NAME}" + env.GIT_URL = "https://pagure.io/${PAGURE_REPO_NAME}.git" + env.API_PREFIX = "https://pagure.io/api/0/${PAGURE_REPO_NAME}" + } + def pollingBranch = env.PAGURE_POLLING_FOR_PR == 'true' ? 'origin/pr/*' : "origin/${PAGURE_POLLED_BRANCH}" + def scmVars = checkout([$class: 'GitSCM', + branches: [[name: pollingBranch]], + userRemoteConfigs: [ + [ + name: 'origin', + url: env.GIT_URL, + refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*', + ], + ], + extensions: [[$class: 'CleanBeforeCheckout']], + ]) + echo "Build on branch=${scmVars.GIT_BRANCH}, commit=${scmVars.GIT_COMMIT}" + env.GIT_COMMIT = scmVars.GIT_COMMIT + // setting build display name + def prefix = 'origin/' + def branch = scmVars.GIT_BRANCH.startsWith(prefix) ? scmVars.GIT_BRANCH.substring(prefix.size()) + : scmVars.GIT_BRANCH // origin/pr/1234 -> pr/1234, origin/master -> master + if (env.PAGURE_POLLING_FOR_PR == 'false' && branch == "${PAGURE_POLLED_BRANCH}") { + echo 'Building master' + currentBuild.displayName = "${PAGURE_POLLED_BRANCH}" + } + else if (env.PAGURE_POLLING_FOR_PR == 'true' && branch ==~ /^pr\/[0-9]+$/) { + prNo = branch.split('/')[-1] + def pagureUrl = "${env.PAGURE_HOME}/pull-request/${prNo}" + def pagureLink = """PR-${prNo}""" + try { + def response = httpRequest "${env.API_PREFIX}/pull-request/${prNo}" + def content = readJSON text: response.content + pagureLink = """${content.title}""" + } catch (Exception e) { + echo 'Error using pagure API:' + echo e.message + // ignoring this... + } + echo "Building PR #${prNo}: ${pagureUrl}" + // FIXME: We are going to pass the display name to the triggered dev pipeline build, + // however OpenShift Pipeline DSL is buggy to handle arguments with special bash characters (like whitespaces, #, etc). + currentBuild.displayName = "PR-${prNo}" + currentBuild.description = pagureLink + } else { // This shouldn't happen. + error("Build is aborted due to unexpected polling trigger actions.") + } + } + } + } + stage('Trigger Dev Build') { + steps { + script { + openshift.withCluster() { + openshift.withProject("${DEV_PIPELINE_BC_NAMESPACE}") { + def bcSelector = openshift.selector('bc', "${DEV_PIPELINE_BC_NAME}") + echo 'Starting a dev pipeline build...' + def isMaster = env.PAGURE_POLLING_FOR_PR != 'true' + def buildSelector = bcSelector.startBuild( + '-e', "WAIVERDB_GIT_REPO=${env.GIT_URL}", + '-e', "WAIVERDB_GIT_REF=${env.GIT_COMMIT}", + '-e', "FORCE_PUBLISH_IMAGE=${isMaster}", + '-e', "FORCE_PUBLISH_DOCS=${isMaster}", + '-e', "WAIVERDB_MAIN_BRANCH=${PAGURE_POLLED_BRANCH}", + '-e', "BUILD_DISPLAY_RENAME_TO=${currentBuild.displayName}", + ) + def buildName = buildSelector.object().metadata.name + echo "Pipeline job build ${buildName} triggered." + } + } + } + } + } + } + }