#1124 First take on CI pipeline
Merged 9 months ago by darknao. Opened 9 months ago by darknao.
darknao/fedora-websites ci_preview  into  main

file added
+51
@@ -0,0 +1,51 @@ 

+ apiVersion: build.openshift.io/v1

+ kind: BuildConfig

+ metadata:

+   labels:

+     build: fedora-websites

+   name: fedora-websites

+ spec:

+   failedBuildsHistoryLimit: 1

+   output:

+     to:

+       kind: ImageStreamTag

+       name: fedora-websites:builder

+   source:

+     git:

+       ref: master

+       uri: https://pagure.io/fedora-websites.git

+     type: Git

+   strategy:

+     dockerStrategy:

+       dockerfilePath: Dockerfile

+     type: Docker

+   successfulBuildsHistoryLimit: 1

+ ---

+ apiVersion: build.openshift.io/v1

+ kind: BuildConfig

+ metadata:

+   labels:

+     build: fedora-websites-tools

+   name: fedora-websites-tools

+ spec:

+   failedBuildsHistoryLimit: 1

+   output:

+     to:

+       kind: ImageStreamTag

+       name: fedora-websites:ci-tools

+   source:

+     dockerfile: |-

+       FROM fedora-websites:builder

+       RUN pip install pytest pytest-flake8 pytest-cov

+   strategy:

+     dockerStrategy:

+       from:

+         kind: ImageStreamTag

+         name: fedora-websites:builder

+   triggers:

+   - imageChange:

+       from:

+         kind: ImageStreamTag

+         name: fedora-websites:builder

+     type: ImageChange

+   successfulBuildsHistoryLimit: 1

@@ -0,0 +1,32 @@ 

+ apiVersion: batch/v1

+ kind: CronJob

+ metadata:

+   name: clean-orphaned

+ spec:

+   concurrencyPolicy: Forbid

+   schedule: "0 0 * * *"

+   successfulJobsHistoryLimit: 1

+   failedJobsHistoryLimit: 1

+   activeDeadlineSeconds: 300

+   startingDeadlineSeconds: 300

+   jobTemplate:

+     spec:

+       backoffLimit: 0

+       template:

+         metadata:

+           labels:

+             parent: "clean-orphaned"

+         spec:

+           containers:

+           - name: clean

+             image: image-registry.openshift-image-registry.svc:5000/openshift/cli

+             command: ["/bin/sh", "-c"]

+             args: 

+             - |

+               for l in $(oc get ep -l app=websites-ci -o go-template='{{range .items}}{{if not .subsets}}{{.metadata.labels.pr}}{{"\n"}}{{end}}{{end}}'); 

+               do 

+                 oc delete po,svc,route -l pr=$l;

+               done

+           restartPolicy: Never

+           startingDeadlineSeconds: 600

+           serviceAccount: jenkins

@@ -0,0 +1,18 @@ 

+ apiVersion: v1

+ kind: Pod

+ metadata:

+   labels:

+     worker: websites-agent

+ spec:

+   containers:

+   - name: jnlp

+     image: image-registry.openshift-image-registry.svc:5000/openshift/jenkins-agent-base

+     args: ['\$(JENKINS_SECRET)', '\$(JENKINS_NAME)']

+   - name: python

+     image: image-registry.openshift-image-registry.svc:5000/fedora-web-ci/fedora-websites:ci-tools

+     imagePullPolicy: Always

+     command:

+     - cat

+     tty: true

+   serviceAccount: jenkins

+   serviceAccountName: jenkins

file added
+192
@@ -0,0 +1,192 @@ 

+ def route

+ def has_built = false

+ 

+ pipeline {

+     agent none

+     parameters {

+         string(name: 'prnumber', defaultValue: '', description: 'Pull-Request ID')

+         string(name: 'project', defaultValue: '', description: 'Pagure Project FullName')

+         choice(

+                choices: [

+                    'pull-request.updated', 

+                    'pull-request.rebased', 

+                    'pull-request.comment.added', 

+                    'pull-request.new', 

+                    'pull-request.closed'

+                ],

+                description: 'fedmsg topic',

+                name: 'topic')

+         string(name: 'last_comment', defaultValue: '', description: 'PR\'s last comment')

+         booleanParam(defaultValue: true, description: 'Post job status on PR?', name: 'notify')

+     }

+     options {

+       skipDefaultCheckout(true)

+       disableConcurrentBuilds()

+     }

+     triggers {

+         GenericTrigger(

+             causeString: 'Triggered by PR#$prnumber on $project', 

+             genericVariables: [

+                 [

+                     defaultValue: '',

+                     key: 'topic',

+                     regexpFilter: '', 

+                     value: '$.topic'

+                 ],

+                 [

+                     defaultValue: '',

+                     key: 'prnumber',

+                     regexpFilter: '', 

+                     value: '$.msg.pullrequest.id'

+                 ],

+                 [

+                     defaultValue: '',

+                     key: 'project',

+                     regexpFilter: '', 

+                     value: '$.msg.project_fullname'

+                 ],

+                 [

+                     defaultValue: '',

+                     key: 'last_comment',

+                     regexpFilter: '', 

+                     value: '$.msg.pullrequest.comments[-1].comment'

+                 ]

+             ],

+             printContributedVariables: true, 

+             printPostContent: true,

+             /* topic of interrest:

+                  pull-request.new

+                  pull-request.comment.added

+                  pull-request.rebased

+                  pull-request.updated

+                  pull-request.closed

+             */

+             regexpFilterExpression: '^pull-request\\.(new|closed|rebased|updated|comment\\.added)$', 

+             regexpFilterText: '$topic', 

+             tokenCredentialId: 'PagureWebHook'

+         )

+     }

+ 

+     stages {

+         stage('Check requirements') {

+             /* run the pipeline only if we get the right topic */

+             when {

+                 beforeAgent true

+                 anyOf {

+                     expression { topic ==~ /^pull-request\.(new|closed|rebased|updated)/ }

+                     allOf {

+                         expression { topic ==~ /^pull-request\.comment\.added/ }

+                         expression { last_comment ==~ /^pretty please pagure-ci rebuild/ }

+                     }

+                 }

+             }

+             stages {

+                 stage('Clean-up previous build') {

+                     /* Remove all existing openshift objects related to this PR */

+                     agent any

+                     steps {

+                       cleanAll(pr: "${prnumber}", project: "${project}")

+                     }

+                 }

+                 stage('Stop here if PR closed') {

+                     /* If pull-request.closed, we don't go further */

+                     when {

+                         expression { topic ==~ /^pull-request\.closed/ }

+                     }

+                     steps {

+                         script {

+                             currentBuild.result = 'ABORTED'

+                             error("PR Closed. Nothing left to do.")

+                         }

+                     }

+                 }

+ 

+ 

+                 /* 

+                 Here we can run pre-build tests like syntax check, formatting and/or unit tests

+                 following example with flake8: 

+                 */

+                 

+                 stage('Pre build tests') {

+                     agent {

+                         kubernetes {

+                             cloud 'openshift'

+                             inheritFrom 'Base'

+                             yamlFile '.ci/jenkins-pod-agent.yml'

+                         }

+                     }

+                     steps {

+                         checkout scm

+                         checkout(

+                             [

+                                 $class: 'GitSCM', 

+                                 branches: [[name: 'origin/pr/${prnumber}']],

+                             ]

+                         )

+                         catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {

+                             container("python"){

+                                 sh "pytest --flake8 --junit-xml=flake8_junit.xml"

+                             }

+                         }

+                     }

+                     post {

+                         always {

+                             container("python"){

+                                 junit('flake8_junit.xml')

+                             }

+                         }

+                     }

+                 }

+ 

+                 stage('Build Websites') {

+                     /* Try to build all websites */

+                     agent any

+                     steps {

+                         script {

+                             buildPod(pr: "${prnumber}", project: "${project}")

+                             has_built = true

+                         }

+                     }

+                 }

+                 stage('Expose Preview') {

+                     /* If last step is a success, we create a route to expose websites preview */

+                     agent any

+                     steps {

+                         script {

+                             route = exposeBuild(pr: "${prnumber}")

+ 			    currentBuild.description = "${route}"

+                         }

+                     }

+                 }

+ 

+                 /* Post-build test goes here

+                    You can use ${route} as target URL

+                 */

+             }

+         }

+     }

+     post { 

+         success {

+             /* Post a success flag on the PR, but only if we've actually built something */

+             node(null){

+                 script {

+                     if (has_built && params.notify) {

+                         updatePr( pr: "${prnumber}", project: "${project}", url: "${route}", success: 100 )

+                     }

+                 }

+             }

+         }

+ 

+         failure {

+             /* Post an error flag on the PR if something has failed */

+             node(null){

+                 script {

+                     if (params.notify) {

+                         updatePr( pr: "${prnumber}", project: "${project}", success: 0 )

+                     }

+                 }

+             }

+         }

+         

+     }

+ }

Right now, this pipeline "only" build all websites, and share a link to preview your PR.
From there, we can add other tasks like pre or post build tasks (linter, unit tests, ...). I've already added a flake8 task as an example.

For now, this pipeline is manual, but it's meant to be automatically triggered on PR creation & update.
This require an admin to add the required hook to this project, which I'll gladly share with you once merged.

note that I've never used Jenkins before. So let me know if you see something wrong that need to be fixed.
If there are no blockers to it, I'll merge it sometime next week.

@siddharthvipul1, you're the first person that comes to my mind when I hear Jenkins. Any pointers on this?

note that I've never used Jenkins before. So let me know if you see something wrong that need to be fixed. If there are no blockers to it, I'll merge it sometime next week.

I have no objections but also not a Jenkins expert. It would be great if we could find someone who is to give a quick look and review.

For now, this pipeline is manual, but it's meant to be automatically triggered on PR creation & update. This require an admin to add the required hook to this project, which I'll gladly share with you once merged.

I can give you Pagure access if you'd like to make this change directly. Or I can make the change myself. Either way!

rebased onto a7c4b21

9 months ago

Alright, let's merge this.
Ping me on irc/matrix to get the required webhook url.

Pull-Request has been merged by darknao

9 months ago