#376 C3I: updates
Closed 8 months ago by mkovarik. Opened 9 months ago by mkovarik.
mkovarik/waiverdb c3i-improvements  into  master

@@ -20,7 +20,7 @@ 

  USER root

  

  RUN ${DNF_CMD} install -y \

-     java-1.8.0-openjdk-headless gettext nss_wrapper git-core \

+     java-1.8.0-openjdk-headless gettext git-core \

      tar gzip skopeo wget make bind-utils \

      origin-clients \

      # Jenkins pipeline 'sh' steps seem to require ps

@@ -39,13 +39,10 @@ 

  

  # Dynamically create a passwd file for non-arbitrary UIDs.

  # Taken from: https://docs.openshift.org/latest/creating_images/guidelines.html#openshift-origin-specific-guidelines

- export USER_ID=$(id -u)

- export GROUP_ID=$(id -g)

- 

- # Skip for root user

- if [ x"$USER_ID" != x"0" ]; then

-     cp /etc/passwd $NSS_WRAPPER_PASSWD

-     echo "jenkins:x:${USER_ID}:${GROUP_ID}:jenkins:${HOME}:/bin/bash" >> $NSS_WRAPPER_PASSWD

+ if ! whoami &> /dev/null; then

+   if [ -w /etc/passwd ]; then

+     echo "${USER_NAME:-default}:x:$(id -u):0:${USER_NAME:-default} user:${HOME}:/sbin/nologin" >> /etc/passwd

+   fi

  fi

  

  if [ $# -eq 1 ]; then

file modified
+2 -2
@@ -21,14 +21,14 @@ 

  install:

  	@for job in $(JOBS); do \

  		echo "[PIPELINE] Updating pipeline job \"$${job}\"..." ; \

- 	  template_file=$$(cat ./$(JOBS_DIR)/$${job}.tmpl); \

+ 	  template_file=$$(awk '/^TEMPLATE=/ {split($$0,a,"="); print a[2]}' ./$(JOBS_DIR)/$${job}.env); \

  		jinja2 ./$(TEMPLATES_DIR)/$${template_file} | $(OC_CMD) process --local -f - \

  			--param-file ./$(JOBS_DIR)/$${job}.env | $(OC_CMD) apply -f -; \

  		echo "[PIPELINE] Pipeline job \"$${job}\" updated" ; \

  	done

  uninstall:

  	@for job in $(JOBS); do \

- 	  template_file=$$(cat ./$(JOBS_DIR)/$${job}.tmpl); \

+ 	  template_file=$$(awk '/^TEMPLATE=/ {split($0,a,"="); print a[2]}' ./$(JOBS_DIR)/$${job}.env); \

  		template_name=$${template_file%.y?ml}; \

  		template_name=$${template_name%-template}; \

  		echo "[PIPELINE] Deleting pipeline job \"$${job}\"..." ; \

@@ -1,4 +1,5 @@ 

  NAME=waiverdb-greenwave-promote-to-prod

+ TEMPLATE=waiverdb-greenwave-trigger.yaml

  SOURCE_CONTAINER_REPO=quay.io/factory2/waiverdb

  TARGET_TAG=prod

  DECISION_CONTEXT_REGEX=c3i_promote_stage_to_prod

@@ -1,1 +0,0 @@ 

- waiverdb-greenwave-trigger.yaml

@@ -1,4 +1,5 @@ 

  NAME=waiverdb-greenwave-promote-to-stage

+ TEMPLATE=waiverdb-greenwave-trigger.yaml

  SOURCE_CONTAINER_REPO=quay.io/factory2/waiverdb

  TARGET_TAG=stage

  DECISION_CONTEXT_REGEX=c3i_promote_dev_to_stage

@@ -1,1 +0,0 @@ 

- waiverdb-greenwave-trigger.yaml

@@ -1,3 +1,4 @@ 

  NAME=waiverdb-polling-for-master

+ TEMPLATE=waiverdb-polling-pagure.yaml

  PAGURE_POLLING_SCHEDULE="H/5 * * * *"

  PAGURE_POLLED_BRANCH=master

@@ -1,1 +0,0 @@ 

- waiverdb-polling-pagure.yaml

@@ -1,3 +1,4 @@ 

  NAME=waiverdb-polling-for-prs

+ TEMPLATE=waiverdb-polling-pagure.yaml

  PAGURE_POLLING_FOR_PR=true

  PAGURE_POLLING_SCHEDULE="H/5 * * * *"

@@ -1,1 +0,0 @@ 

- waiverdb-polling-pagure.yaml

@@ -1,3 +1,4 @@ 

  NAME=waiverdb-postmerge

+ TEMPLATE=waiverdb-build-template.yaml

  PAGURE_DOC_REPO_NAME= # Temporarily disable doc push to workaround https://pagure.io/pagure/issue/3919. Remove this line when it is fixed.

  MAIL_ADDRESS=pnt-factory2-devel@redhat.com 

@@ -1,1 +0,0 @@ 

- waiverdb-build-template.yaml

@@ -1,1 +1,2 @@ 

  NAME=waiverdb-premerge

+ TEMPLATE=waiverdb-build-template.yaml

@@ -1,1 +0,0 @@ 

- waiverdb-build-template.yaml

@@ -1,4 +1,5 @@ 

  NAME=waiverdb-trigger-on-latest-tag

+ TEMPLATE=waiverdb-repotracker-trigger.yaml

  MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-waiverdb-trigger-on-latest-tag.VirtualTopic.eng.repotracker.container.tag.>

  ENVIRONMENT=stage

  TRACKED_TAG=latest

@@ -1,1 +0,0 @@ 

- waiverdb-repotracker-trigger.yaml

@@ -1,4 +1,5 @@ 

  NAME=waiverdb-trigger-on-stage-tag

+ TEMPLATE=waiverdb-repotracker-trigger.yaml

  MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-waiverdb-trigger-on-stage-tag.VirtualTopic.eng.repotracker.container.tag.>

  TRACKED_TAG=stage

  ENVIRONMENT=prod

@@ -1,1 +0,0 @@ 

- waiverdb-repotracker-trigger.yaml

@@ -26,7 +26,7 @@ 

                  '-e', "WAIVERDB_IMAGE=${env.IMAGE}",

                  '-e', "PIPELINE_ID=${env.PIPELINE_ID}",

                  '-e', "PAAS_DOMAIN=${env.PAAS_DOMAIN}",

-                 '-e', "SERVICES_TO_DEPLOY='resultsdb-updater datanommer greenwave resultsdb umb waiverdb datagrepper krb5 ldap koji'",

+                 '-e', "SERVICES_TO_DEPLOY='resultsdb-updater datanommer greenwave resultsdb umb waiverdb datagrepper krb5 ldap koji-hub'",

                  '-e', "TRIGGERED_BY=${env.BUILD_URL}"

                )

              }

@@ -0,0 +1,31 @@ 

+ - name: PAGURE_REPO_NAME

+   displayName: Pagure project name

+   description: <username>/<namespace>/<repo>

+   required: true

+   value: waiverdb

+ - name: PAGURE_REPO_IS_FORK

+   displayName: Is the Pagure repo a fork?

+   required: true

+   value: "false"

+ - name: WAIVERDB_GIT_REPO

+   displayName: WaiverDB Git repo URL

+   description: Default WaiverDB Git repo URL in which to run dev tests against

+   required: true

+   value: "https://pagure.io/waiverdb.git"

+ - name: PAGURE_DOC_REPO_NAME

+   displayName: namespace/project of Pagure doc repo for publishing docs

+   description: If not emptry, docs will be published to the specified Pagure doc repo when this is a post-merge build

+   required: false

+   value: waiverdb

+ - name: PAGURE_DOC_SECRET

+   displayName: Name of the OpenShift SSH secret for publishing docs to Pagure.

+   required: false

+   value: pagure-doc-secret

+ - name: PAGURE_URL

+   displayName: Pagure URL

+   value: https://pagure.io

+ - name: PAGURE_REPO_NAME

+   value: waiverdb

+ - name: PAGURE_API_KEY_SECRET_NAME

+   displayName: Name of Pagure API key secret for updating Pagure pull-request statuses

+   value: 'pagure-api-key'

@@ -31,11 +31,8 @@ 

    description: This field is used to deploy multiple pipelines to one OpenShift project from this template.

    required: true

    value: waiverdb-build

- - name: WAIVERDB_GIT_REPO

-   displayName: WaiverDB Git repo URL

-   description: Default WaiverDB Git repo URL in which to run dev tests against

-   required: true

-   value: "https://pagure.io/waiverdb.git"

+ - name: TEMPLATE

+   description: Placeholder for template generation, not used.

  - name: WAIVERDB_GIT_REF

    displayName: WaiverDB Git repo ref

    description: Default WaiverDB Git repo ref in which to run dev tests against
@@ -54,15 +51,6 @@ 

    displayName: Container image for Jenkins slave pods

    required: true

    value: docker-registry.upshift.redhat.com/factory2/waiverdb-jenkins-slave:latest

- - name: PAGURE_DOC_REPO_NAME

-   displayName: namespace/project of Pagure doc repo for publishing docs

-   description: If not emptry, docs will be published to the specified Pagure doc repo when this is a post-merge build

-   required: false

-   value: waiverdb

- - name: PAGURE_DOC_SECRET

-   displayName: Name of the OpenShift SSH secret for publishing docs to Pagure.

-   required: false

-   value: pagure-doc-secret

  - name: WAIVERDB_DEV_IMAGE_DESTINATIONS

    displayName: Comma seperated list of container repositories (without tag) to which the built WaiverDB dev image will be pushed

    description: OpenShift registries must be prefixed with 'atomic:'
@@ -105,18 +93,9 @@ 

  - name: MESSAGING_PROVIDER

    displayName: Name of the JMS messaging provider

    value: Red Hat UMB

- - name: PAGURE_URL

-   displayName: Pagure URL

-   value: https://pagure.io

- - name: PAGURE_REPO_NAME

-   value: waiverdb

- - name: PAGURE_REPO_IS_FORK

-   value: 'false'

- - name: PAGURE_API_KEY_SECRET_NAME

-   displayName: Name of Pagure API key secret for updating Pagure pull-request statuses

-   value: 'pagure-api-key'

  - name: MAIL_ADDRESS

    displayName: If set, build failure messages to this mail address.

+ {% include "snippets/waiverdb-pagure-parameters.yaml" %}

  {% include "snippets/c3i-library-parameters.yaml" %}

  labels:

    template: waiverdb-build

@@ -22,13 +22,8 @@ 

            imagePullPolicy: Always

            tty: true

            env:

-           # Required by unit tests: Set up NSS Wrapper to generate a fake user name for the random UID assigned by OpenShift

-           - name: LD_PRELOAD

-             value: '/usr/lib64/libnss_wrapper.so'

-           - name: NSS_WRAPPER_PASSWD

-             value: '/tmp/passwd'

-           - name: NSS_WRAPPER_GROUP

-             value: '/etc/group'

+           - name: USER_NAME

+             value: jenkins

            volumeMounts:

            - name: postgresql-socket

              mountPath: /var/run/postgresql

@@ -32,17 +32,18 @@ 

      stage("Message Check and setup") {

        steps {

          script {

-           if (!params.CI_MESSAGE) {

+           if (params.CI_MESSAGE) {

+             def message = readJSON text: params.CI_MESSAGE

+             // Extract the digest of the image to be promoted.

+             // e.g. factory2/waiverdb@sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b

+             //   -> sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b

+             def digest = (message.msg.subject_identifier =~ /@(sha256:\w+)$/)[0][1]

+             // Generate the pull spec of the image

+             // e.g. quay.io/factory2/waiverdb@sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b

+             env.IMAGE = "${params.SOURCE_CONTAINER_REPO}@${digest}"

+           } else if (!params.IMAGE) {

              error("This build is not started by a CI message. Only configurations were done.")

            }

-           def message = readJSON text: params.CI_MESSAGE

-           // Extract the digest of the image to be promoted.

-           // e.g. factory2/waiverdb@sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b

-           //   -> sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b

-           def digest = (message.msg.subject_identifier =~ /@(sha256:\w+)$/)[0][1]

-           // Generate the pull spec of the image

-           // e.g. quay.io/factory2/waiverdb@sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b

-           env.IMAGE = "${params.SOURCE_CONTAINER_REPO}@${digest}"

            echo "Starting promotion of image ${env.IMAGE} to :${params.TARGET_TAG}..."

            // Setting up registry credentials

            dir ("${env.HOME}/.docker") {

@@ -12,6 +12,10 @@ 

    displayName: Short unique identifier for the templated instances

    description: This field is used to deploy multiple pipelines to one OpenShift project from this template.

    value: waiverdb-greenwave-trigger

+ - name: IMAGE

+   displayName: Manual promotion of IMAGE, expects $REPO/$IMAGE:[$TAG|@sha:$DIGEST]

+ - name: TEMPLATE

+   description: Placeholder for template generation, not used.

  - name: DECISION_CONTEXT_REGEX

    displayName: Regex pattern for Greenwave decision context in CI message

    required: true
@@ -92,6 +96,8 @@ 

          env:

          - name: OPENSHIFT_CLOUD_NAME

            value: "${OPENSHIFT_CLOUD_NAME}"

+         - name: IMAGE

+           value: "$IMAGE"

          - name: PROMOTING_DESTINATIONS

            value: "${PROMOTING_DESTINATIONS}"

          - name: CONTAINER_REGISTRY_CREDENTIALS

@@ -13,22 +13,12 @@ 

    description: This field is used to deploy multiple pipelines to one OpenShift project from this template.

    required: true

    value: waiverdb-polling-to-pagure

- - name: PAGURE_REPO_NAME

-   displayName: Pagure project name

-   description: <username>/<namespace>/<repo>

-   required: true

-   value: waiverdb

- - name: PAGURE_REPO_IS_FORK

-   displayName: Is the Pagure repo a fork?

-   required: true

-   value: "false"

+ - name: TEMPLATE

+   description: Placeholder for template generation, not used.

  - name: PAGURE_POLLING_FOR_PR

    displayName: set to 'true' to poll for PRs, or 'false' for the master branch

    required: true

    value: "false"

- - name: PAGURE_URL

-   displayName: Pagure URL

-   value: "https://pagure.io"

  - name: PAGURE_POLLING_SCHEDULE

    displayName: Schedule of polling

    description: using cron-style syntax
@@ -58,6 +48,7 @@ 

    displayName: Name of OpenShift cloud in Jenkins master configuration

    required: true

    value: openshift

+ {% include "snippets/waiverdb-pagure-parameters.yaml" %}

  {% include "snippets/c3i-library-parameters.yaml" %}

  objects:

  - kind: ServiceAccount

@@ -3,7 +3,7 @@ 

    {% include "snippets/default-agent.groovy" %}

    options {

      timestamps()

-     timeout(time: 120, unit: 'MINUTES')

+     timeout(time: 60, unit: 'MINUTES')

      buildDiscarder(logRotator(numToKeepStr: '10'))

    }

    triggers {
@@ -22,15 +22,18 @@ 

      stage("Message Check and setup") {

        steps {

          script {

-           if (!params.CI_MESSAGE) {

+           if (params.CI_MESSAGE) {

+             def message = readJSON text: params.CI_MESSAGE

+             echo "Tag :${message.tag} is ${message.action} in ${message.repo}. New digest: ${message.digest}"

+             env.IMAGE = "${message.repo}@${message.digest}"

+             env.PIPELINE_ID = "c3i-waiverdb-tag-${params.TRACKED_TAG}-${message.digest[-9..-1]}"

+           } else if (params.IMAGE) {

+             env.PIPELINE_ID = "c3i-waiverdb-tag-${params.TRACKED_TAG}-manual-${UUID.randomUUID().toString().substring(0, 4)}"

+           } else {

              error("This build is not started by a CI message. Only configurations were done.")

            }

-           def message = readJSON text: params.CI_MESSAGE

-           echo "Tag :${message.tag} is ${message.action} in ${message.repo}. New digest: ${message.digest}"

-           env.IMAGE = "${message.repo}@${message.digest}"

-           echo "Triggering a job to test if ${env.IMAGE} meets all criteria of desired tag :${message.tag}"

+           echo "Triggering a job to test if ${env.IMAGE} meets all criteria of desired tag :${params.TRACKED_TAG}"

            env.IMAGE_IS_SCRATCH = false

-           env.PIPELINE_ID = "c3i-waiverdb-tag-${message.tag}-${message.digest[-9..-1]}"

          }

        }

      }

@@ -12,9 +12,13 @@ 

    displayName: Short unique identifier for the templated instances

    description: This field is used to deploy multiple pipelines to one OpenShift project from this template.

    value: waiverdb-repotracker-trigger

+ - name: TEMPLATE

+   description: Placeholder for template generation, not used.

  - name: TRACKED_CONTAINER_REPO

    displayName: Container repo to be tracked

    value: "quay.io/factory2/waiverdb"

+ - name: IMAGE

+   displayName: Manual trigger tests of IMAGE, expects $REPO/$IMAGE:[$TAG|@sha:$DIGEST]

  - name: TRACKED_TAG

    displayName: Name of tag to be tracked

    required: true
@@ -89,7 +93,9 @@ 

          - name: "OPENSHIFT_CLOUD_NAME"

            value: "${OPENSHIFT_CLOUD_NAME}"

          - name: "JENKINS_AGENT_IMAGE"

-           value:  "${JENKINS_AGENT_IMAGE}"

+           value: "${JENKINS_AGENT_IMAGE}"

+         - name: "IMAGE"

+           value: "${IMAGE}"

          - name: "JENKINS_AGENT_SERVICE_ACCOUNT"

            value: "${NAME}-jenkins-slave"

          - name: "TRACKED_CONTAINER_REPO"

Fix name of koji-hub service.
Allow to trigger test/promotion without CI msg.
Separate pagure configuration.
Remove dependency on NSS_WRAPPER.
Reduce number of files, define template in env.

@mikeb , @lholecek , @gnaponie - cleaning up, improvements and fixes. This PR requires to rebuild jenkins image after merge.

Pull-Request has been closed by mkovarik

8 months ago
Metadata
Changes Summary 28
+1 -1
file changed
openshift/containers/jenkins-slave/Dockerfile
+4 -7
file changed
openshift/containers/jenkins-slave/jenkins-agent
+2 -2
file changed
openshift/pipelines/Makefile
+1 -0
file changed
openshift/pipelines/jobs/waiverdb-greenwave-promote-to-prod.env
-1
file removed
openshift/pipelines/jobs/waiverdb-greenwave-promote-to-prod.tmpl
+1 -0
file changed
openshift/pipelines/jobs/waiverdb-greenwave-promote-to-stage.env
-1
file removed
openshift/pipelines/jobs/waiverdb-greenwave-promote-to-stage.tmpl
+1 -0
file changed
openshift/pipelines/jobs/waiverdb-polling-for-master.env
-1
file removed
openshift/pipelines/jobs/waiverdb-polling-for-master.tmpl
+1 -0
file changed
openshift/pipelines/jobs/waiverdb-polling-for-prs.env
-1
file removed
openshift/pipelines/jobs/waiverdb-polling-for-prs.tmpl
+1 -0
file changed
openshift/pipelines/jobs/waiverdb-postmerge.env
-1
file removed
openshift/pipelines/jobs/waiverdb-postmerge.tmpl
+1 -0
file changed
openshift/pipelines/jobs/waiverdb-premerge.env
-1
file removed
openshift/pipelines/jobs/waiverdb-premerge.tmpl
+1 -0
file changed
openshift/pipelines/jobs/waiverdb-trigger-on-latest-tag.env
-1
file removed
openshift/pipelines/jobs/waiverdb-trigger-on-latest-tag.tmpl
+1 -0
file changed
openshift/pipelines/jobs/waiverdb-trigger-on-stage-tag.env
-1
file removed
openshift/pipelines/jobs/waiverdb-trigger-on-stage-tag.tmpl
+1 -1
file changed
openshift/pipelines/templates/snippets/waiverdb-full-integration-test.groovy
+31
file added
openshift/pipelines/templates/snippets/waiverdb-pagure-parameters.yaml
+3 -24
file changed
openshift/pipelines/templates/waiverdb-build-template.yaml
+2 -7
file changed
openshift/pipelines/templates/waiverdb-build.Jenkinsfile
+10 -9
file changed
openshift/pipelines/templates/waiverdb-greenwave-trigger.Jenkinsfile
+6 -0
file changed
openshift/pipelines/templates/waiverdb-greenwave-trigger.yaml
+3 -12
file changed
openshift/pipelines/templates/waiverdb-polling-pagure.yaml
+10 -7
file changed
openshift/pipelines/templates/waiverdb-repotracker-trigger.Jenkinsfile
+7 -1
file changed
openshift/pipelines/templates/waiverdb-repotracker-trigger.yaml