#1326 improve the reliability of the integration tests
Merged 4 years ago by jkaluza. Opened 4 years ago by mikeb.
mikeb/fm-orchestrator optimize-ci  into  master

@@ -56,10 +56,23 @@ 

        steps {

          script {

            // check out specified branch/commit

-           checkout([$class: 'GitSCM',

-             branches: [[name: params.MBS_GIT_REF]],

-             userRemoteConfigs: [[url: params.MBS_GIT_REPO, refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pull/*/head']],

-           ])

+           def srcRef = env.MBS_GIT_REF.startsWith('pull/') ? env.MBS_GIT_REF : "heads/${env.MBS_GIT_REF}"

+           retry(5) {

+             checkout([$class: 'GitSCM',

+               branches: [[name: params.MBS_GIT_REF]],

+               userRemoteConfigs: [

+                 [

+                   name: 'origin',

+                   url: params.MBS_GIT_REPO,

+                   refspec: "+refs/${srcRef}:refs/remotes/origin/${env.MBS_GIT_REF}",

+                 ],

+               ],

+               extensions: [

+                 [$class: 'CleanBeforeCheckout'],

+                 [$class: 'CloneOption', noTags: true, shallow: true, depth: 2, honorRefspec: true],

+               ],

+             ])

+           }

  

            // get current commit ID

            // FIXME: Due to a bug discribed in https://issues.jenkins-ci.org/browse/JENKINS-45489,

@@ -55,11 +55,26 @@ 

            // Don't set ENVIRONMENT_LABEL in the environment block! Otherwise you will get 2 different UUIDs.

            env.ENVIRONMENT_LABEL = "test-${env.TEST_ID}"

  

-           // check out specified branch/commit

-           checkout([$class: 'GitSCM',

-             branches: [[name: params.MBS_GIT_REF]],

-             userRemoteConfigs: [[url: params.MBS_GIT_REPO, refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pull/*/head']],

-           ])

+           def srcRef = params.MBS_GIT_REF.startsWith('pull/') ? params.MBS_GIT_REF : 'heads/master'
mikeb commented 4 years ago

params.MBS_GIT_REF will either be a reference to a pull request in the format pull/<N>/head or it will be a commit hash. We assume that commit hash is on the master branch, which will always be true in a non-test environment.

+           def localRef = params.MBS_GIT_REF.startsWith('pull/') ? params.MBS_GIT_REF : 'master'

+           def cloneDepth = params.MBS_GIT_REF.startsWith('pull/') ? 2 : 10

+           retry(5) {

+             // check out specified branch/commit

+             checkout([$class: 'GitSCM',

+               branches: [[name: params.MBS_GIT_REF]],

+               userRemoteConfigs: [

+                 [

+                   name: 'origin',

+                   url: params.MBS_GIT_REPO,

+                   refspec: "+refs/${srcRef}:refs/remotes/origin/${localRef}",

+                 ],

+               ],

+               extensions: [

+                 [$class: 'CleanBeforeCheckout'],

+                 [$class: 'CloneOption', noTags: true, shallow: true, depth: cloneDepth, honorRefspec: true],

+               ],

+             ])

+           }

  

            // get current commit ID

            // FIXME: Due to a bug discribed in https://issues.jenkins-ci.org/browse/JENKINS-45489,

@@ -90,6 +90,8 @@ 

        jenkinsPipelineStrategy:

          jenkinsfile: |-

            // Don't use external Jenkinsfile here, or Jenkins will also poll on that repo and branch

+           library identifier: 'c3i@master', changelog: false,

+             retriever: modernSCM([$class: 'GitSCMSource', remote: 'https://pagure.io/c3i-library.git'])

            pipeline {

              agent {

                kubernetes {
@@ -131,10 +133,12 @@ 

              environment {

                PIPELINE_NAMESPACE = readFile('/run/secrets/kubernetes.io/serviceaccount/namespace').trim()

                PAGURE_URL = "${PAGURE_URL}"

+               PAGURE_API = "${env.PAGURE_URL}/api/0"

+               PAGURE_REPO_NAME = "${PAGURE_REPO_NAME}"

                PAGURE_REPO_IS_FORK = "${PAGURE_REPO_IS_FORK}"

                PAGURE_POLLING_FOR_PR = "${PAGURE_POLLING_FOR_PR}"

-               PAGURE_REPO_HOME = "${env.PAGURE_URL}${env.PAGURE_REPO_IS_FORK == 'true' ? '/fork' : ''}/${PAGURE_REPO_NAME}"

-               GIT_URL = "${env.PAGURE_URL}/${env.PAGURE_REPO_IS_FORK == 'true' ? 'forks/' : ''}${PAGURE_REPO_NAME}.git"

+               PAGURE_REPO_HOME = "${env.PAGURE_URL}${env.PAGURE_REPO_IS_FORK == 'true' ? '/fork' : ''}/${env.PAGURE_REPO_NAME}"

+               GIT_URL = "${env.PAGURE_URL}/${env.PAGURE_REPO_IS_FORK == 'true' ? 'forks/' : ''}${env.PAGURE_REPO_NAME}.git"

                PREMERGE_JOB_NAME = "${PREMERGE_JOB_NAME}"

                POSTMERGE_JOB_NAME = "${POSTMERGE_JOB_NAME}"

              }
@@ -146,17 +150,24 @@ 

                    script {

                      // checking out the polled branch

                      def polledBranch = env.PAGURE_POLLING_FOR_PR == 'true' ? 'origin/pull/*/head' : "origin/${PAGURE_POLLED_BRANCH}"

-                     def scmVars = checkout([$class: 'GitSCM',

-                       branches: [[name: polledBranch]],

-                       userRemoteConfigs: [

-                         [

-                           name: 'origin',

-                           url: env.GIT_URL,

-                           refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pull/*/head',

+                     def srcRef = env.PAGURE_POLLING_FOR_PR == 'true' ? 'pull/*/head' : "heads/${PAGURE_POLLED_BRANCH}"

+                     def scmVars

+                     retry(5) {

+                       scmVars = checkout([$class: 'GitSCM',

+                         branches: [[name: polledBranch]],

+                         userRemoteConfigs: [

+                           [

+                             name: 'origin',

+                             url: env.GIT_URL,

+                             refspec: "+refs/${srcRef}:refs/remotes/${polledBranch}",

+                           ],

                          ],

-                       ],

-                       extensions: [[$class: 'CleanBeforeCheckout']],

-                     ])

+                         extensions: [

+                           [$class: 'CleanBeforeCheckout'],

+                           [$class: 'CloneOption', noTags: true, shallow: true, depth: 2, honorRefspec: true],

+                         ],

+                       ])

+                     }

                      env.MBS_GIT_COMMIT = scmVars.GIT_COMMIT

                      // setting build display name

                      def prefix = 'origin/'
@@ -169,12 +180,20 @@ 

                        currentBuild.description = """<a href="${env.PAGURE_REPO_HOME}/c/${env.MBS_GIT_COMMIT}">${currentBuild.displayName}</a>"""

                      } else if (env.PAGURE_POLLING_FOR_PR == 'true' && branch ==~ /^pull\/[0-9]+\/head$/) {

                        env.PR_NO = branch.split('/')[1]

-                       env.PR_URL = "${env.PAGURE_REPO_HOME}/pull-request/${env.PR_NO}"

-                       // To HTML syntax in build description, go to `Jenkins/Global Security/Markup Formatter` and select 'Safe HTML'.

-                       def pagureLink = """<a href="${env.PR_URL}">PR#${env.PR_NO}</a>"""

-                       echo "Building PR #${env.PR_NO}: ${env.PR_URL}"

-                       currentBuild.displayName = "PR#${env.PR_NO}"

-                       currentBuild.description = pagureLink

+                       def prInfo = withPagure {

+                         it.getPR(env.PR_NO)

+                       }

+                       if (prInfo.status == 'Open') {

+                         env.PR_URL = "${env.PAGURE_REPO_HOME}/pull-request/${env.PR_NO}"

+                         // To HTML syntax in build description, go to `Jenkins/Global Security/Markup Formatter` and select 'Safe HTML'.

+                         def pagureLink = """<a href="${env.PR_URL}">PR#${env.PR_NO}</a>"""

+                         echo "Building PR #${env.PR_NO}: ${env.PR_URL}"

+                         currentBuild.displayName = "PR#${env.PR_NO}"

+                         currentBuild.description = pagureLink

+                       } else {

+                         echo "Skipping PR#${env.PR_NO} because it is ${prInfo.status}"

+                         env.SKIP = 'true'

+                       }

                      } else { // This shouldn't happen.

                        error("Build is aborted due to unexpected polling trigger actions.")

                      }
@@ -188,17 +207,22 @@ 

                    }

                  }

                  steps {

-                   checkout([$class: 'GitSCM',

-                     branches: [[name: env.MBS_GIT_BRANCH]],

-                     userRemoteConfigs: [

-                       [

-                         name: 'origin',

-                         url: env.GIT_URL,

-                         refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pull/*/head',

+                   retry(5) {

+                     checkout([$class: 'GitSCM',

+                       branches: [[name: env.MBS_GIT_BRANCH]],

+                       userRemoteConfigs: [

+                         [

+                           name: 'origin',

+                           url: env.GIT_URL,

+                           refspec: "+refs/heads/${env.MBS_GIT_BRANCH}:refs/remotes/origin/${env.MBS_GIT_BRANCH}",

+                         ],

+                       ],

+                       extensions: [

+                         [$class: 'CleanBeforeCheckout'],

+                         [$class: 'CloneOption', noTags: true, shallow: true, depth: 2, honorRefspec: true],

                        ],

-                     ],

-                     extensions: [[$class: 'CleanBeforeCheckout']],

-                   ])

+                     ])

+                   }

                    script {

                      dir('openshift/integration/koji/pipelines') {

                        sh '''
@@ -209,6 +233,11 @@ 

                  }

                }

                stage('Build') {

+                 when {

+                   not {

+                     environment name: 'SKIP', value: 'true'

+                   }

+                 }

                  steps {

                    script {

                      openshift.withCluster() {
@@ -231,3 +260,10 @@ 

                }

              }

            }

+           def withPagure(args=[:], cl) {

+             args.apiUrl = env.PAGURE_API

+             args.repo = env.PAGURE_REPO_NAME

+             args.isFork = env.PAGURE_REPO_IS_FORK == 'true'

+             def pagureClient = pagure.client(args)

+             return cl(pagureClient)

+           }

These changes are designed to make the integration tests more reliable and reduce false failures.

The first change causes the polling job to ignore PRs that aren't in the Open state. This is important if job history is lost or Jenkins needs to be redeployed.

The second change configures jobs to only checkout the specific branch they're interested in, and to use a shallow checkout rather than pulling the full git history. The MBS repo has gotten very large, and a lot of jobs were failing during checkout with early EOF. This change should reduce those errors.

The final change wraps all checkouts in retry(5). The infrastructure can sometimes experience temporary DNS or network issues. The retry may help avoid failures in those cases.

params.MBS_GIT_REF will either be a reference to a pull request in the format pull/<N>/head or it will be a commit hash. We assume that commit hash is on the master branch, which will always be true in a non-test environment.

Looks fine to me. @rayson could you please take a look?

Maybe we can add the Pagure checkout procedure to c3i-library?

@rayson Sure, if you file a RFE I'll look into it.

This PR has two +1s and @mikeb said to address rayson's idea in the new RFE. I think it's therefore ready to be merged -> merging.

Commit d041805 fixes this pull-request

Pull-Request has been merged by jkaluza

4 years ago

Pull-Request has been merged by jkaluza

4 years ago