waiverdb

Clone

#298 CI/CD: minor improvements
Merged 5 years ago by rayson. Opened 5 years ago by rayson.
rayson/waiverdb cicd-improve-1  into  master

file modified
+1 -1 
@@ -181,7 +181,7 @@ 
 

                      def template = readYaml file: 'openshift/waiverdb-test-template.yaml'
 

                      def models = openshift.process(template,
 

                              '-p', "TEST_ID=${env.BUILD_TAG}",
 

-                             '-p', "WAIVERDB_APP_VERSION=internal-${appversion}")
 

+                             '-p', "WAIVERDB_APP_IMAGE=quay.io/factory2/waiverdb:internal-${appversion}")
 

                      def environment_label = "test-${env.BUILD_TAG}"
 

                      try {
 

                          def objects = openshift.create(models)

file modified
+1 -1 
@@ -67,7 +67,7 @@ 
 

    displayName: Comma seperated list of container repositories (without tag) to which the built WaiverDB dev image will be pushed
 

    description: OpenShift registries must be prefixed with 'atomic:'
 

    required: false
 

-   value: "quay.io/factory2/waiverdb,atomic:docker-registry.engineering.redhat.com/factory2/waiverdb"
 

+   value: "quay.io/factory2/waiverdb"
 

  - name: CONTAINER_REGISTRY_CREDENTIALS
 

    displayName: Secret name of container registries used for pulling and pushing images
 

    value: factory2-pipeline-registry-credentials

file modified
+5 -2 
@@ -107,6 +107,10 @@ 
 

            env.WAIVERDB_VERSION = versions[0]
 

            env.WAIVERDB_CONTAINER_VERSION = versions[1]
 

            env.TEMP_TAG = env.WAIVERDB_CONTAINER_VERSION + '-jenkins-' + currentBuild.id
 

+ 

+           if (sh(returnStatus: true, script: 'pip3 install --user -r ./requirements.txt' != 0) {
 

+             echo 'WARNING: Failed to install dependencies from requirements.txt.'
 

+           }
 

          }
 

        }
 

      }
 
@@ -144,8 +148,6 @@ 
 

              currentBuild.description = """<a href="${env.PAGURE_REPO_HOME}/c/${env.WAIVERDB_GIT_COMMIT}">${currentBuild.displayName}</a>"""
 

            }
 

          }
 

-         sh 'cp conf/settings.py.example conf/settings.py'
 

-         sh 'pip3 install --user -r ./requirements.txt'
 

        }
 

      }
 

      stage('Run checks') {
 
@@ -165,6 +167,7 @@ 
 

      }
 

      stage('Run unit tests') {
 

        steps {
 

+         sh 'cp conf/settings.py.example conf/settings.py'
 

          // wait for the test datebase to come up
 

          sh 'wait-for-it -s -t 300 127.0.0.1:5432'
 

          // create a database role

file modified
+1 -1 
@@ -88,7 +88,7 @@ 
 

          - name: "IMAGE"
 

            value: "${IMAGE}"
 

          - name: IMAGE_IS_SCRATCH
 

-           value: "false"
 

+           value: "true"
 

          - name: "CONTAINER_REGISTRY_CREDENTIALS"
 

            value: "${CONTAINER_REGISTRY_CREDENTIALS}"
 

          - name: "TEST_ID"

file modified
+14 -12 
@@ -71,8 +71,12 @@ 
 

                if (now.getTime() - creationTime.getTime() < 1000 * 60 * 60)
 

                  continue
 

                echo "Deleting ${objName}..."
 

-               obj.delete()
 

-               echo "Deleted ${objName}"
 

+               try {
 

+                 obj.delete()
 

+                 echo "Deleted ${objName}"
 

+               } catch (e) {
 

+                 echo "Error deleting ${objName}: ${e.message}"
 

+               }
 

              }
 

            }
 

          }
 
@@ -81,22 +85,20 @@ 
 

      stage('Run functional tests') {
 

        environment {
 

          // Jenkins BUILD_TAG could be too long (> 63 characters) for OpenShift to consume
 

-         TEST_ID = "${params.TEST_ID ?: 'jenkins-' + currentBuild.id}"
 

-         ENVIRONMENT_LABEL = "test-${env.TEST_ID}"
 

+         TEST_ID = "${params.TEST_ID ?: 'jenkins-' + currentBuild.id + '-' + UUID.randomUUID().toString().substring(0,7)}"
 

        }
 

        steps {
 

          echo "Container image ${params.IMAGE} will be tested."
 

          script {
 

            openshift.withCluster() {
 

-             def imageTag = (params.IMAGE =~ /(?::(\w[\w.-]{0,127}))?$/)[0][1]
 

-             def imageRepo = imageTag ? params.IMAGE.substring(0, params.IMAGE.length() - imageTag.length() - 1) : params.IMAGE
 

-             env.IMAGE_TAG = imageTag ?: 'latest'
 

+             // Don't set ENVIRONMENT_LABEL in the environment block! Otherwise you will get 2 different UUIDs.
 

+             env.ENVIRONMENT_LABEL = "test-${env.TEST_ID}"
 

              def template = readYaml file: 'openshift/waiverdb-test-template.yaml'
 

              def webPodReplicas = 1 // The current quota in UpShift is agressively limited
 

+             echo "Creating testing environment with TEST_ID=${env.TEST_ID}..."
 

              def models = openshift.process(template,
 

                '-p', "TEST_ID=${env.TEST_ID}",
 

-               '-p', "WAIVERDB_APP_IMAGE_REPO=${imageRepo}",
 

-               '-p', "WAIVERDB_APP_VERSION=${imageTag ?: 'latest'}",
 

+               '-p', "WAIVERDB_APP_IMAGE=${params.IMAGE}",
 

                '-p', "WAIVERDB_REPLICAS=${webPodReplicas}",
 

              )
 

              def objects = openshift.apply(models)
 
@@ -193,7 +195,7 @@ 
 

                "type": "container-image",
 

                "repository": "factory2/waiverdb",
 

                "digest": "${env.IMAGE_DIGEST}",
 

-               "nvr": "waiverdb:${env.IMAGE_TAG}",
 

+               "nvr": "${params.IMAGE}",
 

                "issuer": "c3i-jenkins",
 

                "scratch": ${params.IMAGE_IS_SCRATCH},
 

                "id": "waiverdb@${env.IMAGE_DIGEST}"
 
@@ -204,8 +206,8 @@ 
 

                    "provider": "openshift",
 

                    "architecture": "x86_64"
 

                 }],
 

-             "type": "${params.ENVIRONMENT}",
 

-             "category": "integration",
 

+             "type": "integration",
 

+             "category": "${params.ENVIRONMENT}",
 

              "status": "${currentBuild.result == null || currentBuild.result == 'SUCCESS' ? 'passed':'failed'}",
 

              "xunit": "${env.BUILD_URL}/artifacts/junit-functional-tests.xml",
 

              "generated_at": "${new Date().format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone('UTC'))}",

file modified
+1 
@@ -84,6 +84,7 @@ 
 

        app: "${NAME}"
 

    spec:
 

      runPolicy: "Serial"
 

+     completionDeadlineSeconds: 1800
 

      strategy:
 

        type: JenkinsPipeline
 

        jenkinsPipelineStrategy:

file modified
+5 -9 
@@ -4,7 +4,7 @@ 
 

  # for Postgres data.
 

  #
 

  # To create an environment from the template, process and apply it:
 

- #   oc process -f openshift/waiverdb-test-template.yaml -p TEST_ID=123 -p WAIVERDB_APP_VERSION=0.1.2.dev24-git.94c0119 | oc apply -f -
 

+ #   oc process -f openshift/waiverdb-test-template.yaml -p TEST_ID=123 -p WAIVERDB_APP_IMAGE=quay.io/factory2/waiverdb:0.1.2.dev24-git.94c0119 | oc apply -f -
 

  # To clean up the environment, use a selector on the environment label:
 

  #   oc delete dc,deploy,pod,configmap,secret,svc,route -l environment=test-123
 

  
@@ -20,14 +20,10 @@ 
 

    displayName: Test id
 

    description: Short unique identifier for this test run (e.g. Jenkins job number)
 

    required: true
 

- - name: WAIVERDB_APP_VERSION
 

-   displayName: WaiverDB application version
 

-   description: Python version of the WaiverDB application being tested
 

+ - name: WAIVERDB_APP_IMAGE
 

+   displayName: The image of WaiverDB application being tested
 

    required: true
 

- - name: WAIVERDB_APP_IMAGE_REPO
 

-   displayName: The registry/repo of WaiverDB application being tested
 

-   required: true
 

-   value: docker-registry.engineering.redhat.com/factory2/waiverdb
 

+   value: quay.io/factory2/waiverdb:latest
 

  - name: FLASK_SECRET_KEY
 

    displayName: Flask secret key
 

    generate: expression
 
@@ -223,7 +219,7 @@ 
 

        spec:
 

          containers:
 

          - name: web
 

-           image: "${WAIVERDB_APP_IMAGE_REPO}:${WAIVERDB_APP_VERSION}"
 

+           image: "${WAIVERDB_APP_IMAGE}"
 

            ports:
 

            - containerPort: 8080
 

            volumeMounts:

  1. Append UUID to TEST_ID for integration tests to avoid name conflicts
  2. Set time limit for Pagure polling job to 30 min
  3. Ignore errors of installing dependencies from requirements.txt
  4. Ignore failures of cleaning old test environment deployments
  5. Do not push to the internal registry (we got enough 'Unknown blob' errors)
  6. Fix waiverdb-test-template doesn't support pulling by digests
  7. Change test case name format

I need to temporarily merge it test if the issues we found in the post-merge workflow is resolved. If it is not working, I will revert it.

Pull-Request has been merged by rayson
5 years ago
Metadata
None
No Tags
Changes Summary 7
+1 -1
file changed
Jenkinsfile
+1 -1
file changed
openshift/pipelines/templates/waiverdb-build-template.yaml
+5 -2
file changed
openshift/pipelines/templates/waiverdb-build.Jenkinsfile
+1 -1
file changed
openshift/pipelines/templates/waiverdb-integration-test-template.yaml
+14 -12
file changed
openshift/pipelines/templates/waiverdb-integration-test.Jenkinsfile
+1 -0
file changed
openshift/pipelines/templates/waiverdb-polling-pagure.yaml
+5 -9
file changed
openshift/waiverdb-test-template.yaml
Powered by Pagure 5.13.3
Documentation • File an Issue • About • SSH Hostkey/Fingerprint
© Red Hat, Inc. and others.