From a8c5dda1c084a19d36556c0906db2f16776b5ddb Mon Sep 17 00:00:00 2001 From: Pavlina Bortlova Date: Jun 01 2020 07:12:28 +0000 Subject: Merge #72 `Refactor ansible role` --- diff --git a/roles/c3i/defaults/main.yml b/roles/c3i/defaults/main.yml index c99aa35..93e8f54 100644 --- a/roles/c3i/defaults/main.yml +++ b/roles/c3i/defaults/main.yml @@ -15,6 +15,9 @@ c3i_git_repo: https://pagure.io/{{ c3i_component }}.git # Main branch used for triggering post merge workflow. c3i_git_main_branch: master +# Git provider +c3i_git_type: "{{ 'pagure' if 'pagure' in c3i_git_repo else 'gerrit' }}" + # Openshift connection c3i_ocp_token: "{{ lookup('file', '/run/secrets/kubernetes.io/serviceaccount/token', errors='ignore') | default(lookup('pipe', 'oc whoami -t'), true) }}" c3i_ocp_host: https://paas.psi.redhat.com diff --git a/roles/c3i/tasks/build.yml b/roles/c3i/tasks/build.yml index 81f118f..2c482ed 100644 --- a/roles/c3i/tasks/build.yml +++ b/roles/c3i/tasks/build.yml @@ -32,7 +32,8 @@ - name: Generate Jenkins file {{ job_vars.name }} template: - src: build.Jenkinsfile + # gerrit or pagure + src: "build-workflow/build-{{ c3i_git_type }}.Jenkinsfile" dest: "{{ job_vars.name }}.Jenkinsfile" vars: task_var_build_and_test: "{{ lookup('template', c3i_build_and_test_snippet, errors='ignore') | default('') }}" diff --git a/roles/c3i/templates/build-workflow/build-default.groovy b/roles/c3i/templates/build-workflow/build-default.groovy new file mode 100644 index 0000000..8a934e6 --- /dev/null +++ b/roles/c3i/templates/build-workflow/build-default.groovy @@ -0,0 +1,77 @@ +stage('Allocate C3IaaS project') { + steps { + script { + if (env.PR_NO) { + env.PIPELINE_ID = "c3i-{{ c3i_component }}-pr-${env.PR_NO}-git${env.GIT_COMMIT.take(8)}-${currentBuild.id}" + } else { + env.PIPELINE_ID = "c3i-{{ c3i_component }}-git${env.GIT_COMMIT.take(8)}-${currentBuild.id}" + } + echo "Requesting new OpenShift project ${env.PIPELINE_ID}..." + openshift.withCluster() { + openshift.withProject(params.C3IAAS_NAMESPACE) { + c3i.buildAndWait(script: this, objs: "bc/${params.C3IAAS_JOB_NAME}", + '-e', "PROJECT_NAME=${env.PIPELINE_ID}", + '-e', "ADMIN_GROUPS=system:serviceaccounts:${env.TRIGGER_NAMESPACE},system:serviceaccounts:${env.C3IAAS_NAMESPACE}", + '-e', "LIFETIME_IN_MINUTES=${params.C3IAAS_LIFETIME}" + ) + } + } + namespaceLink = """Openshift""" + currentBuild.description = "${currentBuild.description}
${namespaceLink}" + } + } + post { + success { + echo "Allocated project ${env.PIPELINE_ID}" + } + failure { + echo "Failed to allocate ${env.PIPELINE_ID} project" + } + } +} +// run a component snippet to run and test the component +{{ task_var_build_and_test }} + +{% if job_vars.workflow == "postmerge" %} +stage('Push container') { + steps { + script { + dir ("${env.HOME}/.docker") { + // for the OpenShift internal registry + def dockerConfig = readJSON text: '{ "auths": {} }' + dockerConfig.auths['docker-registry.default.svc:5000'] = [ + 'email': '', + 'auth': sh(returnStdout: true, script: 'set +x; echo -n "serviceaccount:$SERVICE_ACCOUNT_TOKEN" | base64 -').trim() + ] + // merging user specified credentials + if (params.CONTAINER_REGISTRY_CREDENTIALS) { + openshift.withCluster() { + def dockerconf = openshift.selector('secret', params.CONTAINER_REGISTRY_CREDENTIALS).object().data['.dockerconfigjson'] + def dockerString = new String(dockerconf.decodeBase64()) + toBeMerged = readJSON text: dockerString + dockerConfig.auths.putAll(toBeMerged.auths) + } + } + // writing to ~/.docker/config.json + writeJSON file: 'config.json', json: dockerConfig + } + env.RESULTING_IMAGE_REPOS.tokenize(',').each { + def sourceImage = it + ":" + env.RESULTING_TAG + def imageName = it.split('/').last() + def destImage = "${env.IMAGE_DESTINATION_NAMESPACE}/${imageName}:${env.IMAGE_TAG}" + // copy between registies + echo "Copying container from ${sourceImage} to ${destImage}" + retry(5) { + sh "skopeo copy --src-cert-dir=/var/run/secrets/kubernetes.io/serviceaccount/ docker://${sourceImage} docker://${destImage}" + } + } + } + } +} +stage('Trigger Tower deployment') { + when { + expression { params.TOWER_TEMPLATE_ID && params.TOWER_SECRET } + } + {% include "trigger-tower-steps.groovy" %} +} +{% endif %} diff --git a/roles/c3i/templates/build-workflow/build-gerrit.Jenkinsfile b/roles/c3i/templates/build-workflow/build-gerrit.Jenkinsfile new file mode 100644 index 0000000..fdadb62 --- /dev/null +++ b/roles/c3i/templates/build-workflow/build-gerrit.Jenkinsfile @@ -0,0 +1,97 @@ +library identifier: "c3i@{{ c3i_lib_branch }}", changelog: false, + retriever: modernSCM([$class: 'GitSCMSource', remote: "{{ c3i_lib_url }}"]) +import static org.apache.commons.lang.StringEscapeUtils.escapeHtml; +pipeline { + // agent + {% if c3i_trigger_update_only %} + {% include "master-agent.groovy" %} + {% elif "job-updater" == job_vars.workflow %} + {% include c3i_default_agent_snippet %} + {% else %} + {% include c3i_build_agent_snippet %} + {% endif %} + + options { + timestamps() + timeout(time: 30, unit: 'MINUTES') + } + environment { + SERVICE_ACCOUNT_TOKEN = readFile(file: '/run/secrets/kubernetes.io/serviceaccount/token').trim() + } + + // git actions premerge/ postmerge for trigger setup + {% if job_vars.workflow in ["premerge", "job-updater"] %} + triggers { + gerrit customUrl: '', + gerritProjects: + [[branches: [[compareType: 'PLAIN', pattern: params.GIT_MAIN_BRANCH ]], + compareType: 'ANT', + disableStrictForbiddenFileVerification: false, + pattern: env.GIT_REPO.split('/')[-1].replaceAll(/.git$/, '')]], + serverName: env.GERRIT_SERVER_NAME, + // there are variables in job-updater.yml and premerge.yml which change the value of trigger + triggerOnEvents: [{{ job_vars.gerrit_trigger }}] + } // triggers + {% endif %} + + // the trigger update happens only on master + {% if c3i_trigger_update_only %} + // runs build which update triggers + {% include "trigger-update-stages.groovy" %} + {% else %} + stages { + stage('Proceeding Gerrit trigger') { + when { + expression { env.GERRIT_REFSPEC } + } + steps { + script { + // figure out how look the message when it is merged + if (env.GERRIT_EVENT_TYPE == "patchset-created") { + env.GIT_REPO_REF = env.GERRIT_REFSPEC + env.PR_NO = env.GERRIT_CHANGE_NUMBER + } + } // script + } // steps + } // stage + stage('Clone git repo') { + steps { + script { + if (!env.GIT_REPO_REF) { + error("This build is not started by a CI message and GIT_REPO_REF is empty. Only configurations were done.") + } + + // FIXME: Due to a bug described in https://issues.jenkins-ci.org/browse/JENKINS-45489 + c3i.clone(repo: params.GIT_REPO, branch: env.GIT_REPO_REF) + env.GIT_COMMIT = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() + echo "Build ${env.GIT_REPO_REF}, commit=${env.GIT_COMMIT}" + } // script + } // steps + } // stage + + {% if "job-updater" == job_vars.workflow %} + {% include "build-workflow/job-updater.groovy" %} + {% else %} + {% include "build-workflow/build-default.groovy" %} + {% endif %} + + } // stages + post { + failure { + script { + // on post-merge workflow failure sending email + if (env.C3I_JOB_WORKFLOW == 'postmerge' && params.MAIL_ADDRESS) { + try { + def recipient = params.MAIL_ADDRESS + def subject = "Jenkins job ${env.JOB_NAME} #${env.BUILD_NUMBER} failed." + def body = "Build URL: ${env.BUILD_URL}" + emailext to: recipient, subject: subject, body: body + } catch (e) { + echo "Error sending email: ${e}" + } + } + } // script + } // failure + } // post + {% endif %} // not trigger_update_only +} // pipeline diff --git a/roles/c3i/templates/build-workflow/build-gerrit.yml b/roles/c3i/templates/build-workflow/build-gerrit.yml new file mode 100644 index 0000000..19854f6 --- /dev/null +++ b/roles/c3i/templates/build-workflow/build-gerrit.yml @@ -0,0 +1,4 @@ +- name: GERRIT_SERVER_NAME + value: {{ c3i_gerrit_server }} +jenkinsfile: | + {% filter indent(width=10) %}{% include "build-workflow/build-gerrit.Jenkinsfile" %}{% endfilter %} diff --git a/roles/c3i/templates/build-workflow/build-pagure.Jenkinsfile b/roles/c3i/templates/build-workflow/build-pagure.Jenkinsfile new file mode 100644 index 0000000..ba0d554 --- /dev/null +++ b/roles/c3i/templates/build-workflow/build-pagure.Jenkinsfile @@ -0,0 +1,208 @@ +library identifier: "c3i@{{ c3i_lib_branch }}", changelog: false, + retriever: modernSCM([$class: 'GitSCMSource', remote: "{{ c3i_lib_url }}"]) +import static org.apache.commons.lang.StringEscapeUtils.escapeHtml; +pipeline { + // agent + {% if c3i_trigger_update_only %} + {% include "master-agent.groovy" %} + {% elif "job-updater" == job_vars.workflow %} + {% include c3i_default_agent_snippet %} + {% else %} + {% include c3i_build_agent_snippet %} + {% endif %} + + options { + timestamps() + timeout(time: 30, unit: 'MINUTES') + } + environment { + SERVICE_ACCOUNT_TOKEN = readFile(file: '/run/secrets/kubernetes.io/serviceaccount/token').trim() + // needed by c3i pagure client + TRIGGER_NAMESPACE = readFile('/run/secrets/kubernetes.io/serviceaccount/namespace').trim() + PAGURE_REPO_NAME = env.GIT_REPO.split('/')[3..-1].join('/').replace('forks/', '').replaceAll(/.git$/, '') + PAGURE_URL = env.GIT_REPO.split('/')[0..2].join('/') + PAGURE_API = "${env.PAGURE_URL}/api/0" + PAGURE_REPO_IS_FORK = "${env.GIT_REPO.contains('/forks/') ? 'true': 'false'}" + } + + // git actions premerge/ postmerge for trigger setup + {% if job_vars.workflow in ["premerge", "job-updater"] %} + triggers { + ciBuildTrigger( + noSquash: true, + providerList: [ + // iterate over list of messages + {% for topic in job_vars.messaging_fedmsg_topics %} + fedmsgSubscriber( + name: params.MESSAGING_FEDMSG_PROVIDER, + overrides: [topic: "{{ topic }}"], + checks: [ + {% if "premerge" == job_vars.workflow %} + [field: '$.pullrequest.project.url_path', expectedValue: params.GIT_REPO.split('/')[3..-1].join('/').replace('forks/', 'fork/').replaceAll(/.git$/, '')], + [field: '$.pullrequest.branch', expectedValue: params.GIT_MAIN_BRANCH], + {% elif "job-updater" == job_vars.workflow %} + [field: '$.repo.url_path', expectedValue: "^${params.GIT_REPO.split('/')[3..-1].join('/').replace('forks/', 'fork/').replaceAll(/.git$/, '')}\$"], + [field: '$.branch', expectedValue: params.GIT_MAIN_BRANCH], + {% endif %} + ] + ), + {% endfor %} + ] + ) + } + {% endif %} + + // the trigger update happens only on master + {% if c3i_trigger_update_only %} + // runs build which update triggers + {% include "trigger-update-stages.groovy" %} + {% else %} + stages { + stage('Proceeding CI_MESSAGE from UMB') { + when { + expression { env.CI_MESSAGE && env.C3I_JOB_WORKFLOW == "premerge"} + } + steps { + script { + def message = readJSON text: params.CI_MESSAGE + env.GIT_REPO_REF = "pull/${message.pullrequest.id}/head" + } // script + } // steps + } // stage + stage('Clone git repo') { + steps { + script { + if (!env.GIT_REPO_REF) { + error("This build is not started by a CI message and GIT_REPO_REF is empty. Only configurations were done.") + } + + // FIXME: Due to a bug described in https://issues.jenkins-ci.org/browse/JENKINS-45489 + c3i.clone(repo: params.GIT_REPO, branch: env.GIT_REPO_REF) + env.GIT_COMMIT = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() + echo "Build ${env.GIT_REPO_REF}, commit=${env.GIT_COMMIT}" + } // script + } // steps + } // stage + stage('Update Build Info') { + steps { + script { + // Set friendly display name and description + def pagure_repo_home = env.GIT_REPO.replace('/forks/', '/fork/').replaceAll(/.git$/,'') + {% if "premerge" == job_vars.workflow %} + // Is the current branch a pull-request? If no, env.PR_NO will be empty. + env.PR_NO = env.GIT_REPO_REF.split('/')[1] // return X from pull/X/head + env.PR_URL = "${pagure_repo_home}/pull-request/${env.PR_NO}" + echo "Building PR #${env.PR_NO}: ${env.PR_URL}" + // NOTE: Old versions of OpenShift Client Jenkins plugin are buggy to handle arguments + // with special bash characters (like whitespaces, #, etc). + // https://bugzilla.redhat.com/show_bug.cgi?id=1625518 + currentBuild.displayName = "PR#${env.PR_NO}" + // To enable HTML syntax in build description, go to `Jenkins/Global Security/Markup Formatter` and select 'Safe HTML'. + def pagureLink = """${currentBuild.displayName}""" + try { + def prInfo = pagure.getPR(env.PR_NO) + pagureLink = """PR#${env.PR_NO}: ${escapeHtml(prInfo.title)}""" + // set PR status to Pending + if (params.PAGURE_API_KEY_SECRET_NAME) + pagure.setBuildStatusOnPR(null, 'Building...') + } catch (Exception e) { + echo "Error using pagure API: ${e}" + } + currentBuild.description = pagureLink + {% else %} + currentBuild.displayName = "${env.GIT_REPO_REF}: ${env.GIT_COMMIT.substring(0, 7)}" + currentBuild.description = """${currentBuild.displayName}""" + if (params.PAGURE_API_KEY_SECRET_NAME) { + try { + pagure.flagCommit('pending', null, 'Building...') + echo "Updated commit ${env.GIT_COMMIT} status to PENDING." + } catch (e) { + echo "Error updating commit ${env.GIT_COMMIT} status to PENDING: ${e}" + } + } + {% endif %} + {% include "get_paas_domain.groovy" %} + } // script + } // steps + } // stage + + {% if "job-updater" == job_vars.workflow %} + {% include "build-workflow/job-updater.groovy" %} + {% else %} + {% include "build-workflow/build-default.groovy" %} + {% endif %} + + } // stages + post { + success { + script { + // on pre-merge workflow success + if (params.PAGURE_API_KEY_SECRET_NAME && env.C3I_JOB_WORKFLOW == "premerge") { + try { + pagure.setBuildStatusOnPR(100, 'Build passed.') + echo "Updated PR #${env.PR_NO} status to PASS." + } catch (e) { + echo "Error updating PR #${env.PR_NO} status to PASS: ${e}" + } + } + // on post-merge workflow success + if (params.PAGURE_API_KEY_SECRET_NAME && env.C3I_JOB_WORKFLOW == 'postmerge') { + try { + pagure.flagCommit('success', 100, 'Build passed.') + echo "Updated commit ${env.GIT_COMMIT} status to PASS." + } catch (e) { + echo "Error updating commit ${env.GIT_COMMIT} status to PASS: ${e}" + } + } + } // script + } // success + failure { + script { + // on pre-merge workflow failure + if (params.PAGURE_API_KEY_SECRET_NAME && env.C3I_JOB_WORKFLOW == 'premerge') { + // updating Pagure PR flag + try { + pagure.setBuildStatusOnPR(0, 'Build failed.') + echo "Updated PR #${env.PR_NO} status to FAILURE." + } catch (e) { + echo "Error updating PR #${env.PR_NO} status to FAILURE: ${e}" + } + // making a comment + try { + pagure.commentOnPR(""" + Build ${env.GIT_COMMIT} [FAILED](${env.BUILD_URL})! + Rebase or make new commits to rebuild. + """.stripIndent(), env.PR_NO) + echo "Comment made." + } catch (e) { + echo "Error making a comment on PR #${env.PR_NO}: ${e}" + } + } + // on post-merge workflow failure + if (env.C3I_JOB_WORKFLOW == 'postmerge') { + // updating Pagure commit flag + if (params.PAGURE_API_KEY_SECRET_NAME) { + try { + pagure.flagCommit('failure', 0, 'Build failed.') + echo "Updated commit ${env.GIT_COMMIT} status to FAILURE." + } catch (e) { + echo "Error updating commit ${env.GIT_COMMIT} status to FAILURE: ${e}" + } + } + // sending email + if (params.MAIL_ADDRESS){ + try { + def recipient = params.MAIL_ADDRESS + def subject = "Jenkins job ${env.JOB_NAME} #${env.BUILD_NUMBER} failed." + def body = "Build URL: ${env.BUILD_URL}" + emailext to: recipient, subject: subject, body: body + } catch (e) { + echo "Error sending email: ${e}" + } + } + } + } // script + } // failure + } // post + {% endif %} +} // pipeline diff --git a/roles/c3i/templates/build-workflow/build-pagure.yml b/roles/c3i/templates/build-workflow/build-pagure.yml new file mode 100644 index 0000000..9e2753b --- /dev/null +++ b/roles/c3i/templates/build-workflow/build-pagure.yml @@ -0,0 +1,15 @@ +- name: PAGURE_DOC_REPO_NAME + value: {{ c3i_pagure_doc_repo_name }} +- name: PAGURE_DOC_SECRET + value: {{ c3i_pagure_doc_secret }} +- name: PAGURE_API_KEY_SECRET_NAME + value: {{ c3i_pagure_api_key_secret }} +- name: MESSAGING_PROVIDER + value: {{ c3i_messaging_provider }} +- name: MESSAGING_FEDMSG_PROVIDER + value: {{ c3i_fedmsg_provider }} +# CI_MESSAGE and MESSAGE_HEADERS are used internally by JMS messaging plugin +- name: CI_MESSAGE +- name: MESSAGE_HEADERS +jenkinsfile: | + {% filter indent(width=10) %}{% include "build-workflow/build-pagure.Jenkinsfile" %}{% endfilter %} diff --git a/roles/c3i/templates/build-workflow/job-updater.groovy b/roles/c3i/templates/build-workflow/job-updater.groovy new file mode 100644 index 0000000..e28da48 --- /dev/null +++ b/roles/c3i/templates/build-workflow/job-updater.groovy @@ -0,0 +1,26 @@ +// Update jenkins files through oc command +// run all jobs from templates/jobs directory +// job updater is on the same level as postmerge/premerge +stage('Update pipeline jobs') { + steps { + script { + dir("{{ c3i_definition_dir }}") { + sh '{{ c3i_definition_update_script }}' + } + } // script + } // steps +} // stage +stage('Update trigger postmerge - build, test and upload container') { + steps { + script { + openshift.withCluster() { + def build = c3i.build(script: this, objs: "bc/${env.POSTMERGE_JOB}") + c3i.waitForBuildStart(script: this, build: build) + def devBuildInfo = build.object() + def downstreamBuildName = devBuildInfo.metadata.name + def downstreamBuildUrl = devBuildInfo.metadata.annotations['openshift.io/jenkins-build-uri'] + echo "Downstream build ${downstreamBuildName}(${downstreamBuildUrl}) started." + } + } // script + } // steps +} // stage diff --git a/roles/c3i/templates/build.Jenkinsfile b/roles/c3i/templates/build.Jenkinsfile deleted file mode 100644 index 33303ec..0000000 --- a/roles/c3i/templates/build.Jenkinsfile +++ /dev/null @@ -1,348 +0,0 @@ -library identifier: "c3i@{{ c3i_lib_branch }}", changelog: false, - retriever: modernSCM([$class: 'GitSCMSource', remote: "{{ c3i_lib_url }}"]) -import static org.apache.commons.lang.StringEscapeUtils.escapeHtml; -pipeline { - {% if c3i_trigger_update_only %} - {% include "master-agent.groovy" %} - {% elif "job-updater" in job_vars.name %} - {% include c3i_default_agent_snippet %} - {% else %} - {% include c3i_build_agent_snippet %} - {% endif %} - - options { - timestamps() - timeout(time: 30, unit: 'MINUTES') - } - environment { - SERVICE_ACCOUNT_TOKEN = readFile(file: '/run/secrets/kubernetes.io/serviceaccount/token').trim() - // needed by c3i pagure client - TRIGGER_NAMESPACE = readFile('/run/secrets/kubernetes.io/serviceaccount/namespace').trim() - PAGURE_REPO_NAME = env.GIT_REPO.split('/')[3..-1].join('/').replace('forks/', '').replaceAll(/.git$/, '') - PAGURE_URL = env.GIT_REPO.split('/')[0..2].join('/') - PAGURE_API = "${env.PAGURE_URL}/api/0" - PAGURE_REPO_IS_FORK = "${env.GIT_REPO.contains('/forks/') ? 'true': 'false'}" - } - {% if "postmerge" not in job_vars.name %} - // pagure - {% if "pagure" in c3i_git_repo %} - triggers { - ciBuildTrigger( - noSquash: true, - providerList: [ - {% if "premerge" in job_vars.name %} - {% for topic in job_vars.messaging_fedmsg_topics %} - fedmsgSubscriber( - name: params.MESSAGING_FEDMSG_PROVIDER, - overrides: [topic: "{{ topic }}"], - checks: [ - [field: '$.pullrequest.project.url_path', expectedValue: params.GIT_REPO.split('/')[3..-1].join('/').replace('forks/', 'fork/').replaceAll(/.git$/, '')], - [field: '$.pullrequest.branch', expectedValue: params.GIT_MAIN_BRANCH], - ] - ), - {% endfor %} - {% elif "job-updater" in job_vars.name %} - {% for topic in job_vars.messaging_fedmsg_topics %} - fedmsgSubscriber( - name: params.MESSAGING_FEDMSG_PROVIDER, - overrides: [topic: "{{ topic }}"], - checks: [ - [field: '$.repo.url_path', expectedValue: "^${params.GIT_REPO.split('/')[3..-1].join('/').replace('forks/', 'fork/').replaceAll(/.git$/, '')}\$"], - [field: '$.branch', expectedValue: params.GIT_MAIN_BRANCH], - ] - ) - {% endfor %} - {% endif %} - ] - ) - } - // gerrit trigger - {% elif "gerrit" in c3i_git_repo %} - triggers { - gerrit customUrl: '', - gerritProjects: - [[branches: [[compareType: 'PLAIN', pattern: params.GIT_MAIN_BRANCH ]], - compareType: 'ANT', - disableStrictForbiddenFileVerification: false, - pattern: env.GIT_REPO.split('/')[-1].replaceAll(/.git$/, '')]], - serverName: env.GERRIT_SERVER_NAME, - // there are variables in job-updater.yml and premerge.yml which change the value of trigger - triggerOnEvents: [{{ job_vars.gerrit_trigger }}] - } - {% endif %} - {% endif %} - {% if c3i_trigger_update_only %} - {% include "trigger-update-stages.groovy" %} - {% else %} - stages { - stage('Proceeding CI_MESSAGE from UMB') { - when { - expression { env.CI_MESSAGE } - } - steps { - script { - def message = readJSON text: params.CI_MESSAGE - if (env.GIT_REPO_REF != params.GIT_MAIN_BRANCH) { - env.GIT_REPO_REF = "pull/${message.pullrequest.id}/head" - } - } - } - } - stage('Proceeding Gerrit trigger') { - when { - expression { env.GERRIT_REFSPEC } - } - steps { - script { - // figure out how look the message when it is merged - if (env.GERRIT_EVENT_TYPE == "patchset-created") { - env.GIT_REPO_REF = env.GERRIT_REFSPEC - env.PR_NO = env.GERRIT_CHANGE_NUMBER - } - } - } - } - stage('Clone git repo') { - steps { - script { - if (!env.GIT_REPO_REF) { - error("This build is not started by a CI message and GIT_REPO_REF is empty. Only configurations were done.") - } - - // FIXME: Due to a bug described in https://issues.jenkins-ci.org/browse/JENKINS-45489 - c3i.clone(repo: params.GIT_REPO, branch: env.GIT_REPO_REF) - env.GIT_COMMIT = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() - echo "Build ${env.GIT_REPO_REF}, commit=${env.GIT_COMMIT}" - } - } - } - // this is not needed for Gerrit - {% if "pagure" in c3i_git_repo %} - stage('Update Build Info') { - steps { - script { - // Set friendly display name and description - def pagure_repo_home = env.GIT_REPO.replace('/forks/', '/fork/').replaceAll(/.git$/,'') - {% if "premerge" in job_vars.name %} - // Is the current branch a pull-request? If no, env.PR_NO will be empty. - env.PR_NO = env.GIT_REPO_REF.split('/')[1] // return X from pull/X/head - env.PR_URL = "${pagure_repo_home}/pull-request/${env.PR_NO}" - echo "Building PR #${env.PR_NO}: ${env.PR_URL}" - // NOTE: Old versions of OpenShift Client Jenkins plugin are buggy to handle arguments - // with special bash characters (like whitespaces, #, etc). - // https://bugzilla.redhat.com/show_bug.cgi?id=1625518 - currentBuild.displayName = "PR#${env.PR_NO}" - // To enable HTML syntax in build description, go to `Jenkins/Global Security/Markup Formatter` and select 'Safe HTML'. - def pagureLink = """${currentBuild.displayName}""" - try { - def prInfo = pagure.getPR(env.PR_NO) - pagureLink = """PR#${env.PR_NO}: ${escapeHtml(prInfo.title)}""" - // set PR status to Pending - if (params.PAGURE_API_KEY_SECRET_NAME) - pagure.setBuildStatusOnPR(null, 'Building...') - } catch (Exception e) { - echo "Error using pagure API: ${e}" - } - currentBuild.description = pagureLink - {% else %} - currentBuild.displayName = "${env.GIT_REPO_REF}: ${env.GIT_COMMIT.substring(0, 7)}" - currentBuild.description = """${currentBuild.displayName}""" - if (params.PAGURE_API_KEY_SECRET_NAME) { - try { - pagure.flagCommit('pending', null, 'Building...') - echo "Updated commit ${env.GIT_COMMIT} status to PENDING." - } catch (e) { - echo "Error updating commit ${env.GIT_COMMIT} status to PENDING: ${e}" - } - } - {% endif %} - {% include "get_paas_domain.groovy" %} - } - } - } - {% endif %} - // Update jenkins files through oc command - // run all jobs from templates/jobs directory - // job updater is on the same lever as postmerge/premerge - {% if "job-updater" in job_vars.name %} - stage('Update pipeline jobs') { - steps { - script { - dir("{{ c3i_definition_dir }}") { - sh '{{ c3i_definition_update_script }}' - } - } - } - } - stage('Trigger postmerge - build, test and upload container') { - steps { - script { - openshift.withCluster() { - def build = c3i.build(script: this, objs: "bc/${env.POSTMERGE_JOB}") - c3i.waitForBuildStart(script: this, build: build) - def devBuildInfo = build.object() - def downstreamBuildName = devBuildInfo.metadata.name - def downstreamBuildUrl = devBuildInfo.metadata.annotations['openshift.io/jenkins-build-uri'] - echo "Downstream build ${downstreamBuildName}(${downstreamBuildUrl}) started." - } - } - } - } - {% else %} - stage('Allocate C3IaaS project') { - steps { - script { - if (env.PR_NO) { - env.PIPELINE_ID = "c3i-{{ c3i_component }}-pr-${env.PR_NO}-git${env.GIT_COMMIT.take(8)}-${currentBuild.id}" - } else { - env.PIPELINE_ID = "c3i-{{ c3i_component }}-git${env.GIT_COMMIT.take(8)}-${currentBuild.id}" - } - echo "Requesting new OpenShift project ${env.PIPELINE_ID}..." - openshift.withCluster() { - openshift.withProject(params.C3IAAS_NAMESPACE) { - c3i.buildAndWait(script: this, objs: "bc/${params.C3IAAS_JOB_NAME}", - '-e', "PROJECT_NAME=${env.PIPELINE_ID}", - '-e', "ADMIN_GROUPS=system:serviceaccounts:${env.TRIGGER_NAMESPACE},system:serviceaccounts:${env.C3IAAS_NAMESPACE}", - '-e', "LIFETIME_IN_MINUTES=${params.C3IAAS_LIFETIME}" - ) - } - } - namespaceLink = """Openshift""" - currentBuild.description = "${currentBuild.description}
${namespaceLink}" - } - } - post { - success { - echo "Allocated project ${env.PIPELINE_ID}" - } - failure { - echo "Failed to allocate ${env.PIPELINE_ID} project" - } - } - } - // run component snippet to run and test the component - {{ task_var_build_and_test }} - - - stage('Push container') { - when { - expression { - return env.GIT_REPO_REF == params.GIT_MAIN_BRANCH - } - } - steps { - script { - dir ("${env.HOME}/.docker") { - // for the OpenShift internal registry - def dockerConfig = readJSON text: '{ "auths": {} }' - dockerConfig.auths['docker-registry.default.svc:5000'] = [ - 'email': '', - 'auth': sh(returnStdout: true, script: 'set +x; echo -n "serviceaccount:$SERVICE_ACCOUNT_TOKEN" | base64 -').trim() - ] - // merging user specified credentials - if (params.CONTAINER_REGISTRY_CREDENTIALS) { - openshift.withCluster() { - def dockerconf = openshift.selector('secret', params.CONTAINER_REGISTRY_CREDENTIALS).object().data['.dockerconfigjson'] - def dockerString = new String(dockerconf.decodeBase64()) - toBeMerged = readJSON text: dockerString - dockerConfig.auths.putAll(toBeMerged.auths) - } - } - // writing to ~/.docker/config.json - writeJSON file: 'config.json', json: dockerConfig - } - env.RESULTING_IMAGE_REPOS.tokenize(',').each { - def sourceImage = it + ":" + env.RESULTING_TAG - def imageName = it.split('/').last() - def destImage = "${env.IMAGE_DESTINATION_NAMESPACE}/${imageName}:${env.IMAGE_TAG}" - // copy between registies - echo "Copying container from ${sourceImage} to ${destImage}" - retry(5) { - sh "skopeo copy --src-cert-dir=/var/run/secrets/kubernetes.io/serviceaccount/ docker://${sourceImage} docker://${destImage}" - } - } - } - } - } - stage('Triggering Tower deployment') { - when { - // env.GIT_REPO_REF == params.GIT_MAIN_BRANCH it means that this is a postmerge - expression { env.GIT_REPO_REF == params.GIT_MAIN_BRANCH && params.TOWER_TEMPLATE_ID && params.TOWER_SECRET } - } - {% include "triggering-tower-steps.groovy" %} - } - } - post { - success { - script { - // on pre-merge workflow success - if (params.PAGURE_API_KEY_SECRET_NAME && env.PR_NO) { - try { - pagure.setBuildStatusOnPR(100, 'Build passed.') - echo "Updated PR #${env.PR_NO} status to PASS." - } catch (e) { - echo "Error updating PR #${env.PR_NO} status to PASS: ${e}" - } - } - // on post-merge workflow success - if (params.PAGURE_API_KEY_SECRET_NAME && !env.PR_NO) { - try { - pagure.flagCommit('success', 100, 'Build passed.') - echo "Updated commit ${env.GIT_COMMIT} status to PASS." - } catch (e) { - echo "Error updating commit ${env.GIT_COMMIT} status to PASS: ${e}" - } - } - } - } - failure { - script { - // on pre-merge workflow failure - if (params.PAGURE_API_KEY_SECRET_NAME && env.PR_NO) { - // updating Pagure PR flag - try { - pagure.setBuildStatusOnPR(0, 'Build failed.') - echo "Updated PR #${env.PR_NO} status to FAILURE." - } catch (e) { - echo "Error updating PR #${env.PR_NO} status to FAILURE: ${e}" - } - // making a comment - try { - pagure.commentOnPR(""" - Build ${env.GIT_COMMIT} [FAILED](${env.BUILD_URL})! - Rebase or make new commits to rebuild. - """.stripIndent(), env.PR_NO) - echo "Comment made." - } catch (e) { - echo "Error making a comment on PR #${env.PR_NO}: ${e}" - } - } - // on post-merge workflow failure - if (!env.PR_NO) { - // updating Pagure commit flag - if (params.PAGURE_API_KEY_SECRET_NAME) { - try { - pagure.flagCommit('failure', 0, 'Build failed.') - echo "Updated commit ${env.GIT_COMMIT} status to FAILURE." - } catch (e) { - echo "Error updating commit ${env.GIT_COMMIT} status to FAILURE: ${e}" - } - } - // sending email - if (params.MAIL_ADDRESS){ - try { - def recipient = params.MAIL_ADDRESS - def subject = "Jenkins job ${env.JOB_NAME} #${env.BUILD_NUMBER} failed." - def body = "Build URL: ${env.BUILD_URL}" - emailext to: recipient, subject: subject, body: body - } catch (e) { - echo "Error sending email: ${e}" - } - } - } - } - } - {% endif %} - } - {% endif %} -} diff --git a/roles/c3i/templates/build.yml b/roles/c3i/templates/build.yml index f636153..0e967b8 100644 --- a/roles/c3i/templates/build.yml +++ b/roles/c3i/templates/build.yml @@ -17,18 +17,10 @@ spec: value: {{ c3i_git_main_branch }} - name: GIT_REPO_REF value: {{ job_vars.git_repo_ref }} - - name: PAGURE_DOC_REPO_NAME - value: {{ c3i_pagure_doc_repo_name }} - - name: PAGURE_DOC_SECRET - value: {{ c3i_pagure_doc_secret }} - - name: PAGURE_API_KEY_SECRET_NAME - value: {{ c3i_pagure_api_key_secret }} - - name: GERRIT_SERVER_NAME - value: {{ c3i_gerrit_server }} - name: OPENSHIFT_CLOUD_NAME value: {{ c3i_cloud_name }} - name: JENKINS_AGENT_IMAGE - value: {{ c3i_workflow_jenkins_image if "job-updater" in job_vars.name else c3i_jenkins_build_agent_image }} + value: {{ c3i_workflow_jenkins_image if "job-updater" == job_vars.workflow else c3i_jenkins_build_agent_image }} - name: JENKINS_AGENT_SERVICE_ACCOUNT value: {{ job_vars.name }}-jenkins-slave - name: IMAGE_DESTINATION_NAMESPACE @@ -37,12 +29,8 @@ spec: value: {{ c3i_container_registry_credentials }} - name: IMAGE_TAG value: {{ c3i_dev_image_tag }} - - name: MESSAGING_PROVIDER - value: {{ c3i_messaging_provider }} - name: POSTMERGE_JOB value: {{ c3i_component }}-postmerge - - name: MESSAGING_FEDMSG_PROVIDER - value: {{ c3i_fedmsg_provider }} - name: MAIL_ADDRESS value: {{ c3i_mail_address }} - name: C3IAAS_NAMESPACE @@ -55,9 +43,8 @@ spec: - name: {{ param.name }} value: {{ param.value }} {% endfor %} - # CI_MESSAGE and MESSAGE_HEADERS are used internally by JMS messaging plugin - - name: CI_MESSAGE - - name: MESSAGE_HEADERS + - name: C3I_JOB_WORKFLOW + value: "{{ job_vars.workflow }}" # Tower variables - name: TOWER_INSTANCE value: {{ c3i_tower_instance }} @@ -65,5 +52,4 @@ spec: value: "{{ c3i_tower_template_id_dev }}" - name: TOWER_SECRET value: {{ c3i_tower_secret }} - jenkinsfile: | - {% filter indent(width=10) %}{% include "templates/build.Jenkinsfile" %}{% endfilter %} + {% filter indent(width=6) %}{% include 'templates/build-workflow/build-' + c3i_git_type + '.yml' %}{% endfilter %} diff --git a/roles/c3i/templates/greenwave-promote.Jenkinsfile b/roles/c3i/templates/greenwave-promote.Jenkinsfile index bd6f5cd..0b9fa8a 100644 --- a/roles/c3i/templates/greenwave-promote.Jenkinsfile +++ b/roles/c3i/templates/greenwave-promote.Jenkinsfile @@ -134,7 +134,7 @@ pipeline { when { expression { env.ALLOW_DEPLOYMENT && params.TOWER_TEMPLATE_ID && params.TOWER_SECRET } } - {% include "triggering-tower-steps.groovy" %} + {% include "trigger-tower-steps.groovy" %} } } {% endif %} diff --git a/roles/c3i/templates/jobs/job-updater.yml b/roles/c3i/templates/jobs/job-updater.yml index eab0393..8eac9b5 100644 --- a/roles/c3i/templates/jobs/job-updater.yml +++ b/roles/c3i/templates/jobs/job-updater.yml @@ -1,5 +1,6 @@ --- name: "{{ c3i_component }}-job-updater" +workflow: job-updater template: build.yml messaging_fedmsg_topics: - io.pagure.prod.pagure.git.receive diff --git a/roles/c3i/templates/jobs/postmerge.yml b/roles/c3i/templates/jobs/postmerge.yml index e1dd015..81f8741 100644 --- a/roles/c3i/templates/jobs/postmerge.yml +++ b/roles/c3i/templates/jobs/postmerge.yml @@ -1,4 +1,5 @@ --- name: "{{ c3i_component }}-postmerge" +workflow: postmerge template: build.yml git_repo_ref: master diff --git a/roles/c3i/templates/jobs/premerge.yml b/roles/c3i/templates/jobs/premerge.yml index 1fe1978..a954090 100644 --- a/roles/c3i/templates/jobs/premerge.yml +++ b/roles/c3i/templates/jobs/premerge.yml @@ -1,5 +1,6 @@ --- name: "{{ c3i_component }}-premerge" +workflow: premerge template: build.yml messaging_fedmsg_topics: - io.pagure.prod.pagure.pull-request.new diff --git a/roles/c3i/templates/trigger-tower-steps.groovy b/roles/c3i/templates/trigger-tower-steps.groovy new file mode 100644 index 0000000..fc3c4c7 --- /dev/null +++ b/roles/c3i/templates/trigger-tower-steps.groovy @@ -0,0 +1,43 @@ +steps { + script { + echo "Curl ${params.TOWER_INSTANCE}/api/v2/job_templates/${params.TOWER_TEMPLATE_ID}/launch/" + openshift.withCluster() { + if ( openshift.selector('secret', params.TOWER_SECRET).exists() ) { + def towerSecretData = openshift.selector('secret', params.TOWER_SECRET).object().data + if ( !towerSecretData.containsKey("USERNAME") || !towerSecretData.containsKey("PASSWORD") ) { + error("There is not USERNAME or PASSWORD in ${params.TOWER_SECRET}") + } + def towerUser = new String(towerSecretData.USERNAME.decodeBase64()) + def towerPassword = new String(towerSecretData.PASSWORD.decodeBase64()) + + def launchUrl = "${params.TOWER_INSTANCE}/api/v2/job_templates/${params.TOWER_TEMPLATE_ID}/launch/" + echo "curl -X POST -u ${towerUser}:***** ${launchUrl}" + def launchOut = sh script: "set +x; curl -k -X POST -u ${towerUser}:${towerPassword} ${launchUrl}", returnStdout: true + def launchOutJson + try { + launchOutJson = readJSON text: launchOut + } catch (Exception e) { + error("Unable parse output: ${launchOut}") + } + if (!launchOutJson.containsKey("created")){ + error("Job was not created from template: ${launchOut}") + } + def finished = launchOutJson.finished + def jobUrl = "${params.TOWER_INSTANCE}${launchOutJson.url}" + def jobOutJson + while(finished == "null"){ + sh "set +x; sleep 5; echo 'checking state of ${launchOutJson.id} job'" + def jobOut = sh script: "set +x; curl -k -u ${towerUser}:${towerPassword} ${jobUrl}", returnStdout: true + jobOutJson = readJSON text: jobOut + finished = jobOutJson.finished + } + if (jobOutJson.failed) { + error("Tower job ${params.TOWER_INSTANCE}/#/jobs/playbook/${jobOutJson.id} execution failed") + } + echo "Job ${params.TOWER_INSTANCE}/#/jobs/playbook/${jobOutJson.id} execution passed" + } else { + error("Secret ${params.TOWER_SECRET} doesn't exist") + } + } + } +} diff --git a/roles/c3i/templates/triggering-tower-steps.groovy b/roles/c3i/templates/triggering-tower-steps.groovy deleted file mode 100644 index fc3c4c7..0000000 --- a/roles/c3i/templates/triggering-tower-steps.groovy +++ /dev/null @@ -1,43 +0,0 @@ -steps { - script { - echo "Curl ${params.TOWER_INSTANCE}/api/v2/job_templates/${params.TOWER_TEMPLATE_ID}/launch/" - openshift.withCluster() { - if ( openshift.selector('secret', params.TOWER_SECRET).exists() ) { - def towerSecretData = openshift.selector('secret', params.TOWER_SECRET).object().data - if ( !towerSecretData.containsKey("USERNAME") || !towerSecretData.containsKey("PASSWORD") ) { - error("There is not USERNAME or PASSWORD in ${params.TOWER_SECRET}") - } - def towerUser = new String(towerSecretData.USERNAME.decodeBase64()) - def towerPassword = new String(towerSecretData.PASSWORD.decodeBase64()) - - def launchUrl = "${params.TOWER_INSTANCE}/api/v2/job_templates/${params.TOWER_TEMPLATE_ID}/launch/" - echo "curl -X POST -u ${towerUser}:***** ${launchUrl}" - def launchOut = sh script: "set +x; curl -k -X POST -u ${towerUser}:${towerPassword} ${launchUrl}", returnStdout: true - def launchOutJson - try { - launchOutJson = readJSON text: launchOut - } catch (Exception e) { - error("Unable parse output: ${launchOut}") - } - if (!launchOutJson.containsKey("created")){ - error("Job was not created from template: ${launchOut}") - } - def finished = launchOutJson.finished - def jobUrl = "${params.TOWER_INSTANCE}${launchOutJson.url}" - def jobOutJson - while(finished == "null"){ - sh "set +x; sleep 5; echo 'checking state of ${launchOutJson.id} job'" - def jobOut = sh script: "set +x; curl -k -u ${towerUser}:${towerPassword} ${jobUrl}", returnStdout: true - jobOutJson = readJSON text: jobOut - finished = jobOutJson.finished - } - if (jobOutJson.failed) { - error("Tower job ${params.TOWER_INSTANCE}/#/jobs/playbook/${jobOutJson.id} execution failed") - } - echo "Job ${params.TOWER_INSTANCE}/#/jobs/playbook/${jobOutJson.id} execution passed" - } else { - error("Secret ${params.TOWER_SECRET} doesn't exist") - } - } - } -}