#355 test
Closed a year ago by mkovarik. Opened a year ago by mkovarik.
mkovarik/waiverdb playground  into  master

file modified
+16 -5
@@ -4,6 +4,7 @@ 

  TEMPLATES_DIR:=templates

  JOB_PARAM_FILES:=$(wildcard $(JOBS_DIR)/*.env)

  JOBS:=$(patsubst $(JOBS_DIR)/%.env,%,$(JOB_PARAM_FILES))

+ FAIL_ON_ERROR=true

  

  OC_CMD=$(OC) $(OCFLAGS)

  
@@ -19,12 +20,22 @@ 

  	@echo -e "\tOC\t\tUse this oc command"

  	@echo -e "\tOCFLAGS\t\tOptions to append to the oc command arguments"

  install:

- 	@for job in $(JOBS); do \

- 		echo "[PIPELINE] Updating pipeline job \"$${job}\"..." ; \

+ 	@global_params=$$(cat global.env);\

+         for job in $(JOBS); do \

+           echo "[PIPELINE] Updating pipeline job \"$${job}\"..." ; \

  	  template_file=$$(cat ./$(JOBS_DIR)/$${job}.tmpl); \

- 		$(OC_CMD) process --local -f ./$(TEMPLATES_DIR)/$${template_file} \

- 			--param-file ./$(JOBS_DIR)/$${job}.env | $(OC_CMD) apply -f -; \

- 		echo "[PIPELINE] Pipeline job \"$${job}\" updated" ; \

+           template_params=$$(oc process --local --parameters -f ./$(TEMPLATES_DIR)/$${template_file} | tail -n+2 | awk '{print $$1}'); \

+           add_param=""; \

+           for global_param in $${global_params}; do \

+             global_param_name=$$(echo $${global_param} | cut -f1 -d=); \

+             if ! grep -q "^$${global_param_name}=" $(JOBS_DIR)/$${job}.env && echo "$${template_params}" | grep -q "^$${global_param_name}$$"; then \

+               add_param="$${add_param} --param=$${global_param}"; \

+             fi; \

+           done; \

+           $(OC_CMD) process --local -f ./$(TEMPLATES_DIR)/$${template_file} \

+ 	     --param-file ./$(JOBS_DIR)/$${job}.env $${add_param} | $(OC_CMD) apply -f - || \

+              { [ "$(FAIL_ON_ERROR)" == "true" ] && { echo "[PIPELINE] Pipeline job \"$${job}\" update failed"; exit 1; };}; \

+           echo "[PIPELINE] Pipeline job \"$${job}\" updated" ; \

  	done

  uninstall:

  	@for job in $(JOBS); do \

@@ -0,0 +1,10 @@ 

+ JENKINS_AGENT_IMAGE=docker-registry.default.svc:5000/c3i-dev-mkovarik/waiverdb-jenkins-slave:latest

+ WAIVERDB_DEV_IMAGE_DESTINATIONS=docker-registry.default.svc:5000/c3i-dev-mkovarik/waiverdb

+ PAGURE_REPO_IS_FORK=true

+ MAIL_ADDRESS=mkovarik@redhat.com

+ WAIVERDB_GIT_REPO=https://pagure.io/forks/mkovarik/waiverdb.git

+ PROMOTING_DESTINATIONS=docker-registry.default.svc:5000/c3i-dev-mkovarik/waiverdb

+ PAGURE_REPO_NAME=mkovarik/waiverdb

+ BACKEND_INTEGRATION_TEST_JOB_NAMESPACE=c3i-dev-mkovarik

+ SOURCE_CONTAINER_REPO=docker-registry.default.svc:5000/c3i-dev-mkovarik/waiverdb

+ PAGURE_DOC_REPO_NAME=

@@ -1,2 +1,1 @@ 

  NAME=waiverdb-dev-integration-test

- IMAGE=quay.io/factory2/waiverdb:latest

@@ -1,5 +1,4 @@ 

  NAME=waiverdb-greenwave-promote-to-prod

- SOURCE_CONTAINER_REPO=quay.io/factory2/waiverdb

  TARGET_TAG=prod

  DECISION_CONTEXT_REGEX=c3i_promote_stage_to_prod

  MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-waiverdb-promote-to-prod.VirtualTopic.eng.greenwave.decision.update

@@ -1,5 +1,4 @@ 

  NAME=waiverdb-greenwave-promote-to-stage

- SOURCE_CONTAINER_REPO=quay.io/factory2/waiverdb

  TARGET_TAG=stage

  DECISION_CONTEXT_REGEX=c3i_promote_dev_to_stage

  MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-waiverdb-promote-to-stage.VirtualTopic.eng.greenwave.decision.update

@@ -1,3 +1,2 @@ 

  NAME=waiverdb-postmerge

  PAGURE_DOC_REPO_NAME= # Temporarily disable doc push to workaround https://pagure.io/pagure/issue/3919. Remove this line when it is fixed.

- MAIL_ADDRESS=pnt-factory2-devel@redhat.com 

@@ -2,4 +2,3 @@ 

  IMAGE=quay.io/factory2/waiverdb:prod

  ENVIRONMENT=prod

  BACKEND_INTEGRATION_TEST_JOB=factory2-prod-integration-test

- BACKEND_INTEGRATION_TEST_JOB_NAMESPACE=c3i

@@ -1,3 +1,3 @@ 

  NAME=waiverdb-promoting-to-prod

- IMAGE=quay.io/factory2/waiverdb:stage

+ IMAGE=docker-registry.default.svc:5000/c3i-dev-mkovarik/waiverdb:stage

  DEST_TAG=prod

@@ -1,3 +1,3 @@ 

  NAME=waiverdb-promoting-to-stage

- IMAGE=quay.io/factory2/waiverdb:latest

+ IMAGE=docker-registry.default.svc:5000/c3i-dev-mkovarik/waiverdb:latest

  DEST_TAG=stage

@@ -2,4 +2,3 @@ 

  IMAGE=quay.io/factory2/waiverdb:stage

  ENVIRONMENT=stage

  BACKEND_INTEGRATION_TEST_JOB=factory2-stage-integration-test

- BACKEND_INTEGRATION_TEST_JOB_NAMESPACE=c3i

@@ -41,6 +41,11 @@ 

    description: Default WaiverDB Git repo ref in which to run dev tests against

    required: true

    value: master

+ - name: WAIVERDB_GIT_REF_COMMIT

+   displayName: WaiverDB Git repo ref reachable by Jenkins

+   description: Default WaiverDB Git repo ref in which to get Jenkins pipeline file

+   required: true

+   value: master

  - name: WAIVERDB_MAIN_BRANCH

    displayName: Name of the main branch.

    description: If WAIVERDB_MAIN_BRANCH equals WAIVERDB_GIT_REF, this is a post-merge build, otherwise it's a pre-merge build.
@@ -178,8 +183,9 @@ 

      completionDeadlineSeconds: 1800

      source:

        git:

-         uri: "${WAIVERDB_GIT_REPO}"

-         ref: "${WAIVERDB_MAIN_BRANCH}"

+         # Using jenkins variable, not template

+         uri: "$WAIVERDB_GIT_REPO"

+         ref: "$WAIVERDB_GIT_REF_COMMIT"

      strategy:

        type: JenkinsPipeline

        jenkinsPipelineStrategy:
@@ -188,6 +194,8 @@ 

            value: "${WAIVERDB_GIT_REPO}"

          - name: "WAIVERDB_GIT_REF"

            value: "${WAIVERDB_GIT_REF}"

+         - name: "WAIVERDB_GIT_REF_COMMIT"

+           value: "${WAIVERDB_GIT_REF_COMMIT}"

          - name: "JENKINS_AGENT_CLOUD_NAME"

            value: "${JENKINS_AGENT_CLOUD_NAME}"

          - name: "JENKINS_AGENT_IMAGE"

@@ -35,6 +35,8 @@ 

              value: '/tmp/passwd'

            - name: NSS_WRAPPER_GROUP

              value: '/etc/group'

+           - name: HOME

+             value: '/var/lib/jenkins'

            volumeMounts:

            - name: postgresql-socket

              mountPath: /var/run/postgresql
@@ -77,6 +79,7 @@ 

    }

    environment {

      PIPELINE_NAMESPACE = readFile('/run/secrets/kubernetes.io/serviceaccount/namespace').trim()

+     SERVICE_ACCOUNT_TOKEN = readFile(file: '/var/run/secrets/kubernetes.io/serviceaccount/token').trim()

      PIPELINE_USERNAME = sh(returnStdout: true, script: 'id -un').trim()

      PAGURE_API = "${params.PAGURE_URL}/api/0"

      PAGURE_REPO_IS_FORK = "${params.PAGURE_REPO_IS_FORK}"
@@ -334,6 +337,7 @@ 

                    '-e', "WAIVERDB_GIT_REPO=${params.WAIVERDB_GIT_REPO}",

                    '-e', "IMAGE=${env.RESULTING_IMAGE_REPO}:${env.RESULTING_TAG}",

                    '-e', "WAIVERDB_GIT_REF=${env.PR_NO ? env.WAIVERDB_GIT_REF : env.WAIVERDB_GIT_COMMIT}",

+                   '-e', "WAIVERDB_GIT_REF_COMMIT=${env.WAIVERDB_GIT_COMMIT}",

                    '-e', "IMAGE_IS_SCRATCH=${params.WAIVERDB_GIT_REF != params.WAIVERDB_MAIN_BRANCH}",

                  )

                c3i.wait(buildSelector.name())
@@ -360,21 +364,33 @@ 

            def destinations = env.WAIVERDB_DEV_IMAGE_DESTINATIONS ?

              env.WAIVERDB_DEV_IMAGE_DESTINATIONS.split(',') : []

            openshift.withCluster() {

-             def sourceImage = env.RESULTING_IMAGE_REPO + ":" + env.RESULTING_TAG

-             if (env.REGISTRY_CREDENTIALS) {

-                dir ("${env.HOME}/.docker") {

-                     writeFile file:'config.json', text: env.REGISTRY_CREDENTIALS

-                }

+             // Setting up registry credentials

+             dir ("${env.HOME}/.docker") {

+               // for the OpenShift internal registry

+               def dockerConfig = readJSON text: '{ "auths": {} }'

+               dockerConfig.auths['docker-registry.default.svc:5000'] = [

+                 'email': '',

+                 'auth': sh(returnStdout: true, script: 'set +x; echo -n "serviceaccount:$SERVICE_ACCOUNT_TOKEN" | base64 -').trim()

+                 ]

+               // merging user specified credentials

+               if (env.REGISTRY_CREDENTIALS) {

+                 toBeMerged = readJSON text: env.REGISTRY_CREDENTIALS

+                 dockerConfig.auths.putAll(toBeMerged.auths)

+               }

+               // writing to ~/.docker/config.json

+               writeJSON file: 'config.json', json: dockerConfig

              }

+ 

+             def sourceImage = env.RESULTING_IMAGE_REPO + ":" + env.RESULTING_TAG

+ 

              // pull the built image from imagestream

              echo "Pulling container from ${sourceImage}..."

              def registryToken = readFile(file: '/var/run/secrets/kubernetes.io/serviceaccount/token')

-             withEnv(["SOURCE_IMAGE_REF=${sourceImage}", "TOKEN=${registryToken}"]) {

+             withEnv(["SOURCE_IMAGE_REF=${sourceImage}"]) {

                sh '''set -e +x # hide the token from Jenkins console

                mkdir -p _build

                skopeo copy \

                  --src-cert-dir=/var/run/secrets/kubernetes.io/serviceaccount/ \

-                 --src-creds=serviceaccount:"$TOKEN" \

                  docker://"$SOURCE_IMAGE_REF" dir:_build/waiverdb_container

                '''

              }
@@ -392,7 +408,9 @@ 

                    * with "unknown blob" due to a known issue with the registry

                    * storage configuration. So we retry up to 5 times. */

                    retry(5) {

-                     sh 'skopeo copy dir:_build/waiverdb_container "$DEST_IMAGE_REF"'

+                     sh 'skopeo copy \

+                       --dest-cert-dir=/var/run/secrets/kubernetes.io/serviceaccount/ \

+                       dir:_build/waiverdb_container "$DEST_IMAGE_REF"'

                    }

                  }

                }]

@@ -18,6 +18,9 @@ 

          - name: jnlp

            image: "${params.JENKINS_AGENT_IMAGE}"

            imagePullPolicy: Always

+           env:

+           - name: HOME

+             value: "/var/lib/jenkins"

            resources:

              requests:

                memory: 512Mi

@@ -43,6 +43,9 @@ 

          image: ${params.JENKINS_AGENT_IMAGE}

          imagePullPolicy: Always

          tty: true

+         env:

+         - name: HOME

+           value: "/var/lib/jenkins"

          resources:

            requests:

              memory: 256Mi

@@ -20,6 +20,8 @@ 

            imagePullPolicy: Always

            tty: true

            env:

+           - name: HOME

+             value: "/var/lib/jenkins"

            - name: REGISTRY_CREDENTIALS

              valueFrom:

                secretKeyRef:

@@ -27,6 +27,11 @@ 

    description: Default WaiverDB Git repo ref in which to run functional tests against

    required: true

    value: master

+ - name: WAIVERDB_GIT_REF_COMMIT

+   displayName: WaiverDB Git repo ref reachable by Jenkins

+   description: Default WaiverDB Git repo ref in which to get Jenkins pipeline file

+   required: true

+   value: master

  - name: JENKINS_AGENT_IMAGE

    displayName: Container image for Jenkins slave pods

    required: true
@@ -75,8 +80,9 @@ 

      completionDeadlineSeconds: 1800

      source:

        git:

-         uri: "${WAIVERDB_GIT_REPO}"

-         ref: "${WAIVERDB_GIT_REF}"

+         # Using jenkins variable, not template

+         uri: "$WAIVERDB_GIT_REPO"

+         ref: "$WAIVERDB_GIT_REF_COMMIT"

      strategy:

        type: JenkinsPipeline

        jenkinsPipelineStrategy:
@@ -85,6 +91,8 @@ 

            value: "${WAIVERDB_GIT_REPO}"

          - name: "WAIVERDB_GIT_REF"

            value: "${WAIVERDB_GIT_REF}"

+         - name: "WAIVERDB_GIT_REF_COMMIT"

+           value: "${WAIVERDB_GIT_REF_COMMIT}"

          - name: "IMAGE"

            value: "${IMAGE}"

          - name: IMAGE_IS_SCRATCH

@@ -26,6 +26,8 @@ 

                secretKeyRef:

                  name: "${params.CONTAINER_REGISTRY_CREDENTIALS}"

                  key: '.dockerconfigjson'

+           - name: HOME

+             value: "/var/lib/jenkins"

            resources:

              requests:

                memory: 512Mi

@@ -111,6 +111,9 @@ 

                      image: "${JENKINS_AGENT_IMAGE}"

                      imagePullPolicy: Always

                      tty: true

+                     env:

+                       - name: HOME

+                         value: "/var/lib/jenkins"

                      resources:

                        requests:

                          memory: 378Mi
@@ -199,7 +202,8 @@ 

                    script {

                      dir('openshift/pipelines') {

                        sh '''

-                       make install JOBS_DIR="${PIPELINE_UPDATE_JOBS_DIR}"

+                       # service account cannot update rolebinding

+                       make install JOBS_DIR="${PIPELINE_UPDATE_JOBS_DIR}" FAIL_ON_ERROR=false

                        '''

                      }

                    }
@@ -214,6 +218,7 @@ 

                        echo 'Starting a WaiverDB build run...'

                        def devBuild = bcSelector.startBuild(

                          '-e', "WAIVERDB_GIT_REF=${env.WAIVERDB_GIT_BRANCH}",

+                         '-e', "WAIVERDB_GIT_REF_COMMIT=${env.WAIVERDB_GIT_COMMIT}",

                        )

                        devBuild.watch {

                          return !(it.object().status.phase in ["New", "Pending"])

@@ -38,6 +38,9 @@ 

          image: ${params.JENKINS_AGENT_IMAGE}

          imagePullPolicy: Always

          tty: true

+         env:

+         - name: HOME

+           value: "/var/lib/jenkins"

          resources:

            requests:

              memory: 256Mi

no initial comment

Pull-Request has been closed by mkovarik

a year ago
Metadata
Changes Summary 19