From 84d130803a805af84e7256a2f3645f078748b038 Mon Sep 17 00:00:00 2001 From: Giulia Naponiello Date: Oct 23 2019 06:56:48 +0000 Subject: Containerize Greenwave Jenkins job So that we can move it to rad-jenkins instance and increase the reliability. JIRA: FACTORY-5029 Signed-off-by: Giulia Naponiello --- diff --git a/Jenkinsfile b/Jenkinsfile index 6e3c7aa..402ab87 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,267 +6,252 @@ import groovy.json.* // 'global' var to store git info def scmVars -// Greenwave RPM dependencies -def installDepsCmd = ''' -sudo dnf -y install \ - python3-dogpile-cache \ - python3-fedmsg \ - python3-fedora-messaging \ - python3-flask \ - python3-prometheus_client \ - python3-PyYAML \ - python3-requests -'''.trim() - try { // massive try{} catch{} around the entire build for failure notifications -node('master'){ - scmVars = checkout scm - scmVars.GIT_BRANCH_NAME = scmVars.GIT_BRANCH.split('/')[-1] // origin/pr/1234 -> 1234 + podTemplate( + label: 'greenwave-jenkins-job', + cloud: 'psi', + containers: [ + containerTemplate( + name: 'jnlp', + image: 'docker-registry.upshift.redhat.com/factory2/greenwave-jenkins-job:prod', + alwaysPullImage true, + ttyEnabled: true, + resourceRequestMemory '400Mi', + resourceLimitMemory '1000Mi' + ), + ] + ){ - // setting build display name - def branch = scmVars.GIT_BRANCH_NAME - if ( branch == 'master' ) { - echo 'Building master' - } - else if (branch ==~ /[0-9]+/) { - def pagureUrl = "https://pagure.io/greenwave/pull-request/${branch}" - def pagureLink = """PR-${branch}""" - try { - def response = httpRequest "https://pagure.io/api/0/greenwave/pull-request/${branch}" - // Note for future use: JsonSlurper() is not serialiazble (returns a LazyMap) and - // therefore we cannot save this back into the global scmVars. We could use - // JsonSlurperClassic() which returns a hash map, but would need to allow this in - // the jenkins script approval. - def content = new JsonSlurper().parseText(response.content) - pagureLink = """${content.title}""" - } catch (Exception e) { - echo 'Error using pagure API:' - echo e.message - // ignoring this... + node('master'){ + scmVars = checkout scm + scmVars.GIT_BRANCH_NAME = scmVars.GIT_BRANCH.split('/')[-1] // origin/pr/1234 -> 1234 + + // setting build display name + def branch = scmVars.GIT_BRANCH_NAME + if ( branch == 'master' ) { + echo 'Building master' + } + else if (branch ==~ /[0-9]+/) { + def pagureUrl = "https://pagure.io/greenwave/pull-request/${branch}" + def pagureLink = """PR-${branch}""" + try { + def response = httpRequest "https://pagure.io/api/0/greenwave/pull-request/${branch}" + // Note for future use: JsonSlurper() is not serialiazble (returns a LazyMap) and + // therefore we cannot save this back into the global scmVars. We could use + // JsonSlurperClassic() which returns a hash map, but would need to allow this in + // the jenkins script approval. + def content = new JsonSlurper().parseText(response.content) + pagureLink = """${content.title}""" + } catch (Exception e) { + echo 'Error using pagure API:' + echo e.message + // ignoring this... + } + echo "Building PR #${branch}: ${pagureUrl}" + currentBuild.displayName = "PR #${branch}" + currentBuild.description = pagureLink + } } - echo "Building PR #${branch}: ${pagureUrl}" - currentBuild.displayName = "PR #${branch}" - currentBuild.description = pagureLink - } -} -timestamps { -node('fedora-29') { - checkout scm - scmVars.GIT_AUTHOR_EMAIL = sh ( - script: 'git --no-pager show -s --format=\'%ae\'', - returnStdout: true - ).trim() + timestamps { + node('fedora-29') { + checkout scm + scmVars.GIT_AUTHOR_EMAIL = sh ( + script: 'git --no-pager show -s --format=\'%ae\'', + returnStdout: true + ).trim() - sh """ - ${installDepsCmd} - sudo dnf -y install python3-flake8 python3-pylint python3-sphinx \ - python3-sphinxcontrib-httpdomain python3-pytest-cov - """ - /* Needed to get the latest /etc/mock/fedora-28-x86_64.cfg */ - sh 'sudo dnf -y update mock-core-configs' - stage('Invoke Flake8') { - sh 'flake8-3' - } - stage('Invoke Pylint') { - sh 'pylint-3 --reports=n greenwave' - } - stage('Run unit tests') { - sh ''' - rm -rf htmlcov coverage.xml - pytest-3 greenwave/tests/ \ - --cov-config .coveragerc --cov=greenwave \ - --cov-report term --cov-report xml --cov-report html - ''' - archiveArtifacts artifacts: 'htmlcov/**,coverage.xml' - step([ - $class: 'CoberturaPublisher', - autoUpdateHealth: false, - autoUpdateStability: false, - coberturaReportFile: 'coverage.xml', - failUnhealthy: false, - failUnstable: false, - maxNumberOfBuilds: 0, - onlyStable: false, - zoomCoverageChart: false - ]) - } - stage('Build Docs') { - sh ''' - sudo dnf install -y \ - python3-sphinx \ - python3-sphinxcontrib-httpdomain - ''' - sh 'DEV=true GREENWAVE_CONFIG=$(pwd)/conf/settings.py.example make -C docs html' - archiveArtifacts artifacts: 'docs/_build/html/**' - } - if (scmVars.GIT_BRANCH == 'origin/master') { - stage('Publish Docs') { - sshagent (credentials: ['pagure-greenwave-deploy-key']) { - sh ''' - mkdir -p ~/.ssh/ - touch ~/.ssh/known_hosts - ssh-keygen -R pagure.io - echo 'pagure.io,140.211.169.204 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC198DWs0SQ3DX0ptu+8Wq6wnZMrXUCufN+wdSCtlyhHUeQ3q5B4Hgto1n2FMj752vToCfNTn9mWO7l2rNTrKeBsELpubl2jECHu4LqxkRVihu5UEzejfjiWNDN2jdXbYFY27GW9zymD7Gq3u+T/Mkp4lIcQKRoJaLobBmcVxrLPEEJMKI4AJY31jgxMTnxi7KcR+U5udQrZ3dzCn2BqUdiN5dMgckr4yNPjhl3emJeVJ/uhAJrEsgjzqxAb60smMO5/1By+yF85Wih4TnFtF4LwYYuxgqiNv72Xy4D/MGxCqkO/nH5eRNfcJ+AJFE7727F7Tnbo4xmAjilvRria/+l' >>~/.ssh/known_hosts - rm -rf docs-on-pagure - git clone ssh://git@pagure.io/docs/greenwave.git docs-on-pagure - rm -r docs-on-pagure/* - cp -r docs/_build/html/* docs-on-pagure/ - cd docs-on-pagure - git add -A . - if [[ "$(git diff --cached --numstat | wc -l)" -eq 0 ]] ; then - exit 0 # No changes, nothing to commit - fi - git config user.name "Jenkins Job" - git config user.email "nobody@redhat.com" - git commit -m 'Automatic commit of docs built by Jenkins job ${env.JOB_NAME} #${env.BUILD_NUMBER}' - git push origin master - ''' + stage('Invoke Flake8') { + sh 'flake8-3' + } + stage('Invoke Pylint') { + sh 'pylint-3 --reports=n greenwave' + } + stage('Run unit tests') { + sh ''' + rm -rf htmlcov coverage.xml + pytest-3 greenwave/tests/ \ + --cov-config .coveragerc --cov=greenwave \ + --cov-report term --cov-report xml --cov-report html + ''' + archiveArtifacts artifacts: 'htmlcov/**,coverage.xml' + step([ + $class: 'CoberturaPublisher', + autoUpdateHealth: false, + autoUpdateStability: false, + coberturaReportFile: 'coverage.xml', + failUnhealthy: false, + failUnstable: false, + maxNumberOfBuilds: 0, + onlyStable: false, + zoomCoverageChart: false + ]) + } + stage('Build Docs') { + sh 'DEV=true GREENWAVE_CONFIG=$(pwd)/conf/settings.py.example make -C docs html' + archiveArtifacts artifacts: 'docs/_build/html/**' + } + if (scmVars.GIT_BRANCH == 'origin/master') { + stage('Publish Docs') { + sshagent (credentials: ['pagure-greenwave-deploy-key']) { + sh ''' + mkdir -p ~/.ssh/ + touch ~/.ssh/known_hosts + ssh-keygen -R pagure.io + echo 'pagure.io,140.211.169.204 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC198DWs0SQ3DX0ptu+8Wq6wnZMrXUCufN+wdSCtlyhHUeQ3q5B4Hgto1n2FMj752vToCfNTn9mWO7l2rNTrKeBsELpubl2jECHu4LqxkRVihu5UEzejfjiWNDN2jdXbYFY27GW9zymD7Gq3u+T/Mkp4lIcQKRoJaLobBmcVxrLPEEJMKI4AJY31jgxMTnxi7KcR+U5udQrZ3dzCn2BqUdiN5dMgckr4yNPjhl3emJeVJ/uhAJrEsgjzqxAb60smMO5/1By+yF85Wih4TnFtF4LwYYuxgqiNv72Xy4D/MGxCqkO/nH5eRNfcJ+AJFE7727F7Tnbo4xmAjilvRria/+l' >>~/.ssh/known_hosts + rm -rf docs-on-pagure + git clone ssh://git@pagure.io/docs/greenwave.git docs-on-pagure + rm -r docs-on-pagure/* + cp -r docs/_build/html/* docs-on-pagure/ + cd docs-on-pagure + git add -A . + if [[ "$(git diff --cached --numstat | wc -l)" -eq 0 ]] ; then + exit 0 # No changes, nothing to commit + fi + git config user.name "Jenkins Job" + git config user.email "nobody@redhat.com" + git commit -m 'Automatic commit of docs built by Jenkins job ${env.JOB_NAME} #${env.BUILD_NUMBER}' + git push origin master + ''' + } + } + } + } + node('docker') { + checkout scm + stage('Build Docker container') { + def appversion = sh(returnStdout: true, script: './get-version.sh').trim() + // Set the derived version in __init__.py + sh """ + sed --regexp-extended --in-place \ + -e "/^__version__ = /c\\__version__ = '${appversion}'" greenwave/__init__.py + """.trim() + /* Git builds will have a version like 0.3.2.dev1+git.3abbb08 following + * the rules in PEP440. But Docker does not let us have + in the tag + * name, so let's munge it here. */ + appversion = appversion.replace('+', '-') + /* Build and push the same image with the same tag to quay.io, but without the cacert. */ + docker.withRegistry( + 'https://quay.io/', + 'quay-io-factory2-builder-sa-credentials') { + def image = docker.build "factory2/greenwave:${appversion}", "." + image.push() + } + /* Save container version for later steps (this is ugly but I can't find anything better...) */ + writeFile file: 'appversion', text: appversion + archiveArtifacts artifacts: 'appversion' + } } - } - } -} -node('docker') { - checkout scm - stage('Build Docker container') { - def appversion = sh(returnStdout: true, script: './get-version.sh').trim() - // Set the derived version in __init__.py - sh """ - sed --regexp-extended --in-place \ - -e "/^__version__ = /c\\__version__ = '${appversion}'" greenwave/__init__.py - """.trim() - /* Git builds will have a version like 0.3.2.dev1+git.3abbb08 following - * the rules in PEP440. But Docker does not let us have + in the tag - * name, so let's munge it here. */ - appversion = appversion.replace('+', '-') - /* Build and push the same image with the same tag to quay.io, but without the cacert. */ - docker.withRegistry( - 'https://quay.io/', - 'quay-io-factory2-builder-sa-credentials') { - def image = docker.build "factory2/greenwave:${appversion}", "." - image.push() - } - /* Save container version for later steps (this is ugly but I can't find anything better...) */ - writeFile file: 'appversion', text: appversion - archiveArtifacts artifacts: 'appversion' - } -} -node('fedora-29') { - checkout scm + node('fedora-29') { + checkout scm - /* Install packages needed by the functional tests. */ - sh 'sudo dnf -y install python3-pytest python3-requests python3-sqlalchemy python3-gunicorn' + def openshiftHost = 'greenwave-test.cloud.paas.psi.redhat.com' + def buildTag = "${env.BUILD_TAG}".replace('jenkins-','') + def waiverdbURL = "waiverdb-test-${buildTag}-web-${openshiftHost}" + def resultsdbURL = "resultsdb-test-${buildTag}-api-${openshiftHost}" - /* Also need to install Greenwave's dependencies, since we are running it - * locally not in Openshift for now. */ - sh installDepsCmd + def resultsdbRepo = 'https://pagure.io/taskotron/resultsdb/raw/develop/f/openshift' + def resultsdbTemplate = 'resultsdb-test-template.yaml' + sh "curl ${resultsdbRepo}/${resultsdbTemplate} > openshift/${resultsdbTemplate}" - def openshiftHost = 'greenwave-test.cloud.paas.psi.redhat.com' - def buildTag = "${env.BUILD_TAG}".replace('jenkins-','') - def waiverdbURL = "waiverdb-test-${buildTag}-web-${openshiftHost}" - def resultsdbURL = "resultsdb-test-${buildTag}-api-${openshiftHost}" + def waiverdbRepo = 'https://pagure.io/waiverdb/raw/master/f/openshift' + def waiverdbTemplate = 'waiverdb-test-template.yaml' + sh "curl ${waiverdbRepo}/${waiverdbTemplate} > openshift/${waiverdbTemplate}" - def resultsdbRepo = 'https://pagure.io/taskotron/resultsdb/raw/develop/f/openshift' - def resultsdbTemplate = 'resultsdb-test-template.yaml' - sh "curl ${resultsdbRepo}/${resultsdbTemplate} > openshift/${resultsdbTemplate}" + stage('Perform functional tests') { + openshift.withCluster('psi') { + openshift.withCredentials('psi-greenwave-test-jenkins-credentials') { + openshift.withProject('greenwave-test') { + def rtemplate = readYaml file: 'openshift/resultsdb-test-template.yaml' + // TODO: move this image to the factory2 project in the docker registry + def resultsdbImage = 'quay.io/factory2/resultsdb:latest' + def resultsdbModels = openshift.process( + rtemplate, + '-p', "TEST_ID=${buildTag}", + '-p', "RESULTSDB_IMAGE=${resultsdbImage}", + '-p', "RESULTSDB_ADDITIONAL_RESULT_OUTCOMES=\"('RUNNING','QUEUED')\"" + ) + def wtemplate = readYaml file: 'openshift/waiverdb-test-template.yaml' + def waiverdbModels = openshift.process( + wtemplate, + '-p', "TEST_ID=${buildTag}", + '-p', 'WAIVERDB_APP_IMAGE=quay.io/factory2/waiverdb:latest', + '-p', "RESULTSDB_API_URL=${resultsdbURL}", + '-p', "WAIVERDB_REPLICAS=1" + ) + def environment_label = "test-${buildTag}" + try { + openshift.create(resultsdbModels) + openshift.create(waiverdbModels) + echo "Waiting for pods with label environment=${environment_label} to become Ready" + def pods = openshift.selector('pods', ['environment': environment_label]) + timeout(10) { + pods.untilEach(5) { + def conds = it.object().status.conditions + for (int i = 0; i < conds.size(); i++) { + if (conds[i].type == 'Ready' && conds[i].status == 'True') { + return true + } + } + return false + } + } - def waiverdbRepo = 'https://pagure.io/waiverdb/raw/master/f/openshift' - def waiverdbTemplate = 'waiverdb-test-template.yaml' - sh "curl ${waiverdbRepo}/${waiverdbTemplate} > openshift/${waiverdbTemplate}" + timeout(15) { // minutes + def route_hostname = waiverdbURL + echo "Fetching CA chain for https://${route_hostname}/" + def ca_chain = sh(returnStdout: true, script: """openssl s_client \ + -connect ${route_hostname}:443 \ + -servername ${route_hostname} -showcerts < /dev/null | \ + awk 'BEGIN {first_cert=1; in_cert=0}; + /BEGIN CERTIFICATE/ { if (first_cert == 1) first_cert = 0; else in_cert = 1 }; + { if (in_cert) print }; + /END CERTIFICATE/ { in_cert = 0 }'""") + writeFile(file: "${env.WORKSPACE}/ca-chain.crt", text: ca_chain) + echo "Wrote CA certificate chain to ${env.WORKSPACE}/ca-chain.crt" - stage('Perform functional tests') { - openshift.withCluster('psi') { - openshift.withCredentials('psi-greenwave-test-jenkins-credentials') { - openshift.withProject('greenwave-test') { - def rtemplate = readYaml file: 'openshift/resultsdb-test-template.yaml' - // TODO: move this image to the factory2 project in the docker registry - def resultsdbImage = 'quay.io/factory2/resultsdb:latest' - def resultsdbModels = openshift.process( - rtemplate, - '-p', "TEST_ID=${buildTag}", - '-p', "RESULTSDB_IMAGE=${resultsdbImage}", - '-p', "RESULTSDB_ADDITIONAL_RESULT_OUTCOMES=\"('RUNNING','QUEUED')\"" - ) - def wtemplate = readYaml file: 'openshift/waiverdb-test-template.yaml' - def waiverdbModels = openshift.process( - wtemplate, - '-p', "TEST_ID=${buildTag}", - '-p', 'WAIVERDB_APP_IMAGE=quay.io/factory2/waiverdb:latest', - '-p', "RESULTSDB_API_URL=${resultsdbURL}", - '-p', "WAIVERDB_REPLICAS=1" - ) - def environment_label = "test-${buildTag}" - try { - openshift.create(resultsdbModels) - openshift.create(waiverdbModels) - echo "Waiting for pods with label environment=${environment_label} to become Ready" - def pods = openshift.selector('pods', ['environment': environment_label]) - timeout(10) { - pods.untilEach(5) { - def conds = it.object().status.conditions - for (int i = 0; i < conds.size(); i++) { - if (conds[i].type == 'Ready' && conds[i].status == 'True') { - return true + withEnv(["GREENWAVE_CONFIG=${env.WORKSPACE}/conf/settings.py.example" + ,"PYTHONPATH=." + ,"REQUESTS_CA_BUNDLE=${env.WORKSPACE}/ca-chain.crt" + ,"WAIVERDB_TEST_URL=https://${waiverdbURL}/" + ,"RESULTSDB_TEST_URL=https://${resultsdbURL}/"]) { + sh 'py.test-3 -v --junitxml=junit-functional-tests.xml functional-tests/' } + junit 'junit-functional-tests.xml' + } //end timeout + } finally { + /* Extract logs for debugging purposes */ + openshift.selector('deploy,pods', ['environment': environment_label]).logs() + /* Tear down everything we just created */ + openshift.selector('dc,deploy,configmap,secret,svc,route', + ['environment': environment_label]).delete() } - return false } } + } + } + } - timeout(15) { // minutes - def route_hostname = waiverdbURL - echo "Fetching CA chain for https://${route_hostname}/" - def ca_chain = sh(returnStdout: true, script: """openssl s_client \ - -connect ${route_hostname}:443 \ - -servername ${route_hostname} -showcerts < /dev/null | \ - awk 'BEGIN {first_cert=1; in_cert=0}; - /BEGIN CERTIFICATE/ { if (first_cert == 1) first_cert = 0; else in_cert = 1 }; - { if (in_cert) print }; - /END CERTIFICATE/ { in_cert = 0 }'""") - writeFile(file: "${env.WORKSPACE}/ca-chain.crt", text: ca_chain) - echo "Wrote CA certificate chain to ${env.WORKSPACE}/ca-chain.crt" - - withEnv(["GREENWAVE_CONFIG=${env.WORKSPACE}/conf/settings.py.example" - ,"PYTHONPATH=." - ,"REQUESTS_CA_BUNDLE=${env.WORKSPACE}/ca-chain.crt" - ,"WAIVERDB_TEST_URL=https://${waiverdbURL}/" - ,"RESULTSDB_TEST_URL=https://${resultsdbURL}/"]) { - sh 'py.test-3 -v --junitxml=junit-functional-tests.xml functional-tests/' + node('docker') { + checkout scm + if (scmVars.GIT_BRANCH == 'origin/master') { + stage('Tag "latest".') { + unarchive mapping: ['appversion': 'appversion'] + def appversion = readFile('appversion').trim() + docker.withRegistry( + 'https://quay.io/', + 'quay-io-factory2-builder-sa-credentials') { + def image = docker.image("factory2/greenwave:${appversion}") + image.push('latest') } - junit 'junit-functional-tests.xml' - } //end timeout - } finally { - /* Extract logs for debugging purposes */ - openshift.selector('deploy,pods', ['environment': environment_label]).logs() - /* Tear down everything we just created */ - openshift.selector('dc,deploy,configmap,secret,svc,route', - ['environment': environment_label]).delete() } } } - } - } -} -node('docker') { - checkout scm - if (scmVars.GIT_BRANCH == 'origin/master') { - stage('Tag "latest".') { - unarchive mapping: ['appversion': 'appversion'] - def appversion = readFile('appversion').trim() - docker.withRegistry( - 'https://quay.io/', - 'quay-io-factory2-builder-sa-credentials') { - def image = docker.image("factory2/greenwave:${appversion}") - image.push('latest') - } - } + } // end of timestamps } -} - -} // end of timestamps } catch (e) { // since the result isn't set until after the pipeline script runs, we must set it here if it fails currentBuild.result = 'FAILURE' @@ -275,7 +260,6 @@ node('docker') { // if result hasn't been set to failure by this point, its a success. def currentResult = currentBuild.result ?: 'SUCCESS' def branch = scmVars.GIT_BRANCH_NAME - // send pass/fail email def SUBJECT = '' if ( branch ==~ /[0-9]+/) { @@ -287,7 +271,6 @@ node('docker') { } else if (currentResult == 'FAILURE') { SUBJECT = "Jenkins job ${env.JOB_NAME} #${env.BUILD_NUMBER} failed." } - def RECIEPENT = scmVars.GIT_AUTHOR_EMAIL if (ownership.job.ownershipEnabled && branch == 'master') { RECIEPENT = ownership.job.primaryOwnerEmail @@ -297,19 +280,16 @@ node('docker') { if (branch ==~ /[0-9]+/){ BODY = BODY + "\nPull Request: https://pagure.io/greenwave/pull-request/${branch}" } - if (SUBJECT != '') { emailext to: RECIEPENT, - subject: SUBJECT, - body: BODY + subject: SUBJECT, + body: BODY } - // update Pagure PR status if (branch ==~ /[0-9]+/) { // PR's will only be numbers on pagure def resultPercent = (currentResult == 'SUCCESS') ? '100' : '0' def resultComment = (currentResult == 'SUCCESS') ? 'Build passed.' : 'Build failed.' def pagureRepo = new URL(scmVars.GIT_URL).getPath() - ~/^\// - ~/.git$/ // https://pagure.io/my-repo.git -> my-repo - withCredentials([string(credentialsId: "${env.PAGURE_API_TOKEN}", variable: 'TOKEN')]) { build job: 'pagure-PR-status-updater', propagate: false, diff --git a/jenkins/Dockerfile b/jenkins/Dockerfile new file mode 100644 index 0000000..ce4fbf8 --- /dev/null +++ b/jenkins/Dockerfile @@ -0,0 +1,32 @@ +FROM docker-registry.upshift.redhat.com/devops-automation/rad-slave-fedora:latest +LABEL description="Jenkins container for Greenwave Jenkins job" \ + summary="Jenkins container for Greenwave Jenkins job" \ + maintainer="Factory 2.0 " + +USER root + +RUN dnf install -y --nodocs --setopt=install_weak_deps=false \ + python3-dogpile-cache \ + python3-fedmsg \ + python3-fedora-messaging \ + python3-flask \ + python3-prometheus_client \ + python3-PyYAML \ + python3-requests \ + python3-flake8 \ + python3-pylint \ + python3-sphinx \ + # documentation + python3-sphinxcontrib-httpdomain \ + python3-pytest-cov \ + # functional tests + python3-pytest \ + python3-requests \ + python3-sqlalchemy \ + python3-gunicorn \ + mock-core-configs && dnf -y clean all + +# Needed to get the latest /etc/mock/fedora-28-x86_64.cfg +RUN dnf -y update mock-core-configs && dnf -y clean all + +USER ${UID}