#354 C3I: Global parameters for jobs
Closed a year ago by mkovarik. Opened a year ago by mkovarik.
mkovarik/waiverdb c3i_improvements  into  master

file modified
+16 -5
@@ -4,6 +4,7 @@ 

  TEMPLATES_DIR:=templates

  JOB_PARAM_FILES:=$(wildcard $(JOBS_DIR)/*.env)

  JOBS:=$(patsubst $(JOBS_DIR)/%.env,%,$(JOB_PARAM_FILES))

+ FAIL_ON_ERROR=true

  

  OC_CMD=$(OC) $(OCFLAGS)

  
@@ -19,12 +20,22 @@ 

  	@echo -e "\tOC\t\tUse this oc command"

  	@echo -e "\tOCFLAGS\t\tOptions to append to the oc command arguments"

  install:

- 	@for job in $(JOBS); do \

- 		echo "[PIPELINE] Updating pipeline job \"$${job}\"..." ; \

+ 	@global_params=$$(cat global.env);\
mikeb commented a year ago

oc process accepts multiple --param-file options. It will use the first definition of a parameter it finds, and ignore the rest. It may be simpler to just pass --param-file ./$(JOBS_DIR)/global.env at the end of the command-line and remove the manual handling of the global params.

+         for job in $(JOBS); do \

+           echo "[PIPELINE] Updating pipeline job \"$${job}\"..." ; \

  	  template_file=$$(cat ./$(JOBS_DIR)/$${job}.tmpl); \

- 		$(OC_CMD) process --local -f ./$(TEMPLATES_DIR)/$${template_file} \

- 			--param-file ./$(JOBS_DIR)/$${job}.env | $(OC_CMD) apply -f -; \

- 		echo "[PIPELINE] Pipeline job \"$${job}\" updated" ; \

+           template_params=$$(oc process --local --parameters -f ./$(TEMPLATES_DIR)/$${template_file} | tail -n+2 | awk '{print $$1}'); \

+           add_param=""; \

+           for global_param in $${global_params}; do \

+             global_param_name=$$(echo $${global_param} | cut -f1 -d=); \

+             if ! grep -q "^$${global_param_name}=" $(JOBS_DIR)/$${job}.env && echo "$${template_params}" | grep -q "^$${global_param_name}$$"; then \

+               add_param="$${add_param} --param=$${global_param}"; \

+             fi; \

+           done; \

+           $(OC_CMD) process --local -f ./$(TEMPLATES_DIR)/$${template_file} \

+ 	     --param-file ./$(JOBS_DIR)/$${job}.env $${add_param} | $(OC_CMD) apply -f - || \

+              { [ "$(FAIL_ON_ERROR)" == "true" ] && { echo "[PIPELINE] Pipeline job \"$${job}\" update failed"; exit 1; };}; \

+           echo "[PIPELINE] Pipeline job \"$${job}\" updated" ; \

  	done

  uninstall:

  	@for job in $(JOBS); do \

@@ -0,0 +1,8 @@ 

+ WAIVERDB_GIT_REPO=https://pagure.io/waiverdb.git

+ PAGURE_REPO_NAME=waiverdb

+ PAGURE_REPO_IS_FORK=false

+ WAIVERDB_DEV_IMAGE_DESTINATIONS=quay.io/factory2/waiverdb

+ JENKINS_AGENT_IMAGE=docker-registry.engineering.redhat.com/factory2/waiverdb-jenkins-slave:latest

+ MAIL_ADDRESS=pnt-factory2-devel@redhat.com

+ SOURCE_CONTAINER_REPO=quay.io/factory2/waiverdb

+ BACKEND_INTEGRATION_TEST_JOB_NAMESPACE=c3i

@@ -1,2 +1,1 @@ 

  NAME=waiverdb-dev-integration-test

- IMAGE=quay.io/factory2/waiverdb:latest

@@ -1,5 +1,4 @@ 

  NAME=waiverdb-greenwave-promote-to-prod

- SOURCE_CONTAINER_REPO=quay.io/factory2/waiverdb

  TARGET_TAG=prod

  DECISION_CONTEXT_REGEX=c3i_promote_stage_to_prod

  MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-waiverdb-promote-to-prod.VirtualTopic.eng.greenwave.decision.update

@@ -1,5 +1,4 @@ 

  NAME=waiverdb-greenwave-promote-to-stage

- SOURCE_CONTAINER_REPO=quay.io/factory2/waiverdb

  TARGET_TAG=stage

  DECISION_CONTEXT_REGEX=c3i_promote_dev_to_stage

  MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-waiverdb-promote-to-stage.VirtualTopic.eng.greenwave.decision.update

@@ -1,3 +1,2 @@ 

  NAME=waiverdb-postmerge

  PAGURE_DOC_REPO_NAME= # Temporarily disable doc push to workaround https://pagure.io/pagure/issue/3919. Remove this line when it is fixed.

- MAIL_ADDRESS=pnt-factory2-devel@redhat.com 

@@ -2,4 +2,3 @@ 

  IMAGE=quay.io/factory2/waiverdb:prod

  ENVIRONMENT=prod

  BACKEND_INTEGRATION_TEST_JOB=factory2-prod-integration-test

- BACKEND_INTEGRATION_TEST_JOB_NAMESPACE=c3i

@@ -2,4 +2,3 @@ 

  IMAGE=quay.io/factory2/waiverdb:stage

  ENVIRONMENT=stage

  BACKEND_INTEGRATION_TEST_JOB=factory2-stage-integration-test

- BACKEND_INTEGRATION_TEST_JOB_NAMESPACE=c3i

@@ -41,6 +41,11 @@ 

    description: Default WaiverDB Git repo ref in which to run dev tests against

    required: true

    value: master

+ - name: WAIVERDB_GIT_REF_COMMIT

+   displayName: WaiverDB Git repo ref reachable by Jenkins

+   description: Default WaiverDB Git repo ref in which to get Jenkins pipeline file

+   required: true

+   value: master

  - name: WAIVERDB_MAIN_BRANCH

    displayName: Name of the main branch.

    description: If WAIVERDB_MAIN_BRANCH equals WAIVERDB_GIT_REF, this is a post-merge build, otherwise it's a pre-merge build.
@@ -178,8 +183,9 @@ 

      completionDeadlineSeconds: 1800

      source:

        git:

-         uri: "${WAIVERDB_GIT_REPO}"

-         ref: "${WAIVERDB_MAIN_BRANCH}"

+         # Using jenkins variable, not template

+         uri: "$WAIVERDB_GIT_REPO"

+         ref: "$WAIVERDB_GIT_REF_COMMIT"

      strategy:

        type: JenkinsPipeline

        jenkinsPipelineStrategy:
@@ -188,6 +194,8 @@ 

            value: "${WAIVERDB_GIT_REPO}"

          - name: "WAIVERDB_GIT_REF"

            value: "${WAIVERDB_GIT_REF}"

+         - name: "WAIVERDB_GIT_REF_COMMIT"

+           value: "${WAIVERDB_GIT_REF_COMMIT}"

          - name: "JENKINS_AGENT_CLOUD_NAME"

            value: "${JENKINS_AGENT_CLOUD_NAME}"

          - name: "JENKINS_AGENT_IMAGE"

@@ -35,6 +35,8 @@ 

              value: '/tmp/passwd'

            - name: NSS_WRAPPER_GROUP

              value: '/etc/group'

+           - name: HOME

+             value: '/var/lib/jenkins'

            volumeMounts:

            - name: postgresql-socket

              mountPath: /var/run/postgresql
@@ -334,6 +336,7 @@ 

                    '-e', "WAIVERDB_GIT_REPO=${params.WAIVERDB_GIT_REPO}",

                    '-e', "IMAGE=${env.RESULTING_IMAGE_REPO}:${env.RESULTING_TAG}",

                    '-e', "WAIVERDB_GIT_REF=${env.PR_NO ? env.WAIVERDB_GIT_REF : env.WAIVERDB_GIT_COMMIT}",

+                   '-e', "WAIVERDB_GIT_REF_COMMIT=${env.WAIVERDB_GIT_COMMIT}",

                    '-e', "IMAGE_IS_SCRATCH=${params.WAIVERDB_GIT_REF != params.WAIVERDB_MAIN_BRANCH}",

                  )

                c3i.wait(buildSelector.name())

@@ -18,6 +18,9 @@ 

          - name: jnlp

            image: "${params.JENKINS_AGENT_IMAGE}"

            imagePullPolicy: Always

+           env:

+           - name: HOME

+             value: "/var/lib/jenkins"

            resources:

              requests:

                memory: 512Mi

@@ -43,6 +43,9 @@ 

          image: ${params.JENKINS_AGENT_IMAGE}

          imagePullPolicy: Always

          tty: true

+         env:

+         - name: HOME

+           value: "/var/lib/jenkins"

          resources:

            requests:

              memory: 256Mi

@@ -20,6 +20,8 @@ 

            imagePullPolicy: Always

            tty: true

            env:

+           - name: HOME

+             value: "/var/lib/jenkins"

            - name: REGISTRY_CREDENTIALS

              valueFrom:

                secretKeyRef:

@@ -27,6 +27,11 @@ 

    description: Default WaiverDB Git repo ref in which to run functional tests against

    required: true

    value: master

+ - name: WAIVERDB_GIT_REF_COMMIT

+   displayName: WaiverDB Git repo ref reachable by Jenkins

+   description: Default WaiverDB Git repo ref in which to get Jenkins pipeline file

+   required: true

+   value: master

  - name: JENKINS_AGENT_IMAGE

    displayName: Container image for Jenkins slave pods

    required: true
@@ -75,8 +80,9 @@ 

      completionDeadlineSeconds: 1800

      source:

        git:

-         uri: "${WAIVERDB_GIT_REPO}"

-         ref: "${WAIVERDB_GIT_REF}"

+         # Using jenkins variable, not template

+         uri: "$WAIVERDB_GIT_REPO"

+         ref: "$WAIVERDB_GIT_REF_COMMIT"

      strategy:

        type: JenkinsPipeline

        jenkinsPipelineStrategy:
@@ -85,6 +91,8 @@ 

            value: "${WAIVERDB_GIT_REPO}"

          - name: "WAIVERDB_GIT_REF"

            value: "${WAIVERDB_GIT_REF}"

+         - name: "WAIVERDB_GIT_REF_COMMIT"

+           value: "${WAIVERDB_GIT_REF_COMMIT}"

          - name: "IMAGE"

            value: "${IMAGE}"

          - name: IMAGE_IS_SCRATCH

@@ -26,6 +26,8 @@ 

                secretKeyRef:

                  name: "${params.CONTAINER_REGISTRY_CREDENTIALS}"

                  key: '.dockerconfigjson'

+           - name: HOME

+             value: "/var/lib/jenkins"

            resources:

              requests:

                memory: 512Mi

@@ -111,6 +111,9 @@ 

                      image: "${JENKINS_AGENT_IMAGE}"

                      imagePullPolicy: Always

                      tty: true

+                     env:

+                       - name: HOME

+                         value: "/var/lib/jenkins"

                      resources:

                        requests:

                          memory: 378Mi
@@ -199,7 +202,8 @@ 

                    script {

                      dir('openshift/pipelines') {

                        sh '''

-                       make install JOBS_DIR="${PIPELINE_UPDATE_JOBS_DIR}"

+                       # service account cannot update rolebinding

+                       make install JOBS_DIR="${PIPELINE_UPDATE_JOBS_DIR}" FAIL_ON_ERROR=false

                        '''

                      }

                    }
@@ -214,6 +218,7 @@ 

                        echo 'Starting a WaiverDB build run...'

                        def devBuild = bcSelector.startBuild(

                          '-e', "WAIVERDB_GIT_REF=${env.WAIVERDB_GIT_BRANCH}",

+                         '-e', "WAIVERDB_GIT_REF_COMMIT=${env.WAIVERDB_GIT_COMMIT}",

                        )

                        devBuild.watch {

                          return !(it.object().status.phase in ["New", "Pending"])

@@ -38,6 +38,9 @@ 

          image: ${params.JENKINS_AGENT_IMAGE}

          imagePullPolicy: Always

          tty: true

+         env:

+         - name: HOME

+           value: "/var/lib/jenkins"

          resources:

            requests:

              memory: 256Mi

Add global configuration for project
Use global.env file instead of setting global variables for each job.
Job environment file should contain configuration of template and should
be static. File global.env contains variables which are used by multiple
jobs. Typical content of global.env are URLs.

Variables priority:
  1) use variable in job env file
  2) use variable in global config if template contains it
  3) use default variable in yaml template

Installation of jobs using 'make install' will fail with error code
in case of issue with configuration and other jobs are not applied. This
behavior can be disabled by FAIL_ON_ERROR=false

Set HOME for Jenkins slaves
HOME environment variable is to /home/jenkins which is not writable.
Jenkins slave container has default value of HOME set to
/var/lib/jenkins. HOME variable is probably changed by jenkins plugin.

Do premerge testing with Jenkins files from the PR
Varibales for git checkout (WAIVERDB_GIT_REPO, WAIVERDB_GIT_BRANCH)
were evaluated during openshift template applications. Which caused that
PRs were tested with Jenkins file in master branch intead of PR ones.

Openshift BuildConfig is not able to specify git refs to be fetched and
PR refs are not available, to avoid this issue WAIVERDB_GIT_REF_COMMIT
contains SHA of last PR (or can be master branch).

@mkovarik Have you tested the use of $WAIVERDB_GIT_REF_COMMIT? The original problem was that If you specify a sha1 sum as the ref: value in a OpenShift BuildConfig, OpenShift assumes that ref exists on a branch under refs/heads/*. Because PR branches are created under refs/pull/*, OpenShift cannot find the sha1 for a PR, and the build will fail.

oc process accepts multiple --param-file options. It will use the first definition of a parameter it finds, and ignore the rest. It may be simpler to just pass --param-file ./$(JOBS_DIR)/global.env at the end of the command-line and remove the manual handling of the global params.

@mikeb
I have tested $WAIVERDB_GIT_REF_COMMIT (with sha sum) it worked, but I tested it inside forked repository, PR was from different branch of the same git and that worked. Now I can see that it does not work when PR is from different repository...

I was not aware of oc process behavior, but it causes issue with unknown parameters in global.env (not valid for all templates), command fails with it. It's possible to use param --ignore-unknown-parameters but that I don't want to apply on job env file.

Pull-Request has been closed by mkovarik

a year ago