From 60f6e7377e772691c73c510ca0b0fc46467ed9ea Mon Sep 17 00:00:00 2001 From: root Date: Oct 02 2019 14:24:08 +0000 Subject: Automatic update of defaults --- diff --git a/containers/README b/containers/README new file mode 100644 index 0000000..ce25a16 --- /dev/null +++ b/containers/README @@ -0,0 +1,2 @@ +# Each directory shall contains a Dockerfile +# The config-update job build those on the k1s hosts using the localhost/k1s tag prefix diff --git a/playbooks/base/pre.yaml b/playbooks/base/pre.yaml index a8035b7..d74eb98 100644 --- a/playbooks/base/pre.yaml +++ b/playbooks/base/pre.yaml @@ -10,7 +10,14 @@ zuul_log_url: "https://fedora.softwarefactory-project.io/logs" - hosts: all - roles: - - prepare-workspace - - role: validate-host - - add-build-sshkey + tasks: + - block: + - import_role: name=validate-host + - import_role: name=prepare-workspace + - import_role: name=add-build-sshkey + when: "ansible_connection != 'kubectl'" + - block: + - import_role: name=prepare-workspace-openshift + - import_role: name=remove-zuul-sshkey + run_once: true + when: "ansible_connection == 'kubectl'" diff --git a/playbooks/config/check.yaml b/playbooks/config/check.yaml index ccc00cb..2a355b3 100644 --- a/playbooks/config/check.yaml +++ b/playbooks/config/check.yaml @@ -69,7 +69,7 @@ no_log: true - name: Check resources changes - shell: env - REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-bundle.crt /usr/bin/managesf-resources remote-validate --remote-gateway {{ gateway_url }} + command: env - /usr/bin/managesf-resources remote-validate --remote-gateway {{ gateway_url }} args: chdir: "{{ config_root }}" diff --git a/playbooks/config/update.yaml b/playbooks/config/update.yaml index 86b4e19..e083511 100644 --- a/playbooks/config/update.yaml +++ b/playbooks/config/update.yaml @@ -3,3 +3,4 @@ - hosts: localhost tasks: - include_tasks: update_local.yaml + - include_tasks: update_tenant.yaml diff --git a/resources/_internal.yaml b/resources/_internal.yaml index 709a2f3..bce8b9f 100644 --- a/resources/_internal.yaml +++ b/resources/_internal.yaml @@ -2,39 +2,22 @@ --- resources: tenants: - fedora-staging: - description: "The fedora-staging tenant." + fedora: + description: "The fedora tenant." url: "https://fedora.softwarefactory-project.io/manage" default-connection: pagure.io tenant-options: zuul/report-build-page: True - connections: - opendev.org: - base-url: "https://opendev.org" - type: git - pagure.io: - base-url: "https://pagure.io" - type: pagure - stg.pagure.io: - base-url: "https://stg.pagure.io" - type: pagure - src.fedoraproject.org: - base-url: "https://src.fedoraproject.org" - type: pagure - amqp: - base-url: "https://fedora.softwarefactory-project.io/amqp" - type: pagure - projects: internal: - tenant: fedora-staging + tenant: fedora description: Internal configuration project source-repositories: - fedora-project-config: zuul/config-project: True - fedora-zuul-jobs - - zuul/zuul-jobs: - connection: opendev.org + - zuul-jobs: + connection: gerrit zuul/include: [job] - zuul/shadow: fedora-zuul-jobs + zuul/shadow: fedora-zuul-jobs.git diff --git a/roles/prepare-workspace-openshift/README.rst b/roles/prepare-workspace-openshift/README.rst new file mode 100644 index 0000000..caa5163 --- /dev/null +++ b/roles/prepare-workspace-openshift/README.rst @@ -0,0 +1,15 @@ +Prepare remote workspaces + +This role can be used instead of the `prepare-workspace` role when the +synchronize module doesn't work with kubectl connection. It copies the +prepared source repos to the pods' cwd using the `oc rsync` command. + +This role is intended to run once before any other role in a Zuul job. +This role requires the origin-clients to be installed. + +**Role Variables** + +.. zuul:rolevar:: openshift_pods + :default: {{ zuul.resources }} + + The dictionary of pod name, pod information to copy the sources to. diff --git a/roles/prepare-workspace-openshift/defaults/main.yaml b/roles/prepare-workspace-openshift/defaults/main.yaml new file mode 100644 index 0000000..fa94895 --- /dev/null +++ b/roles/prepare-workspace-openshift/defaults/main.yaml @@ -0,0 +1 @@ +openshift_pods: "{{ zuul.resources }}" diff --git a/roles/prepare-workspace-openshift/tasks/main.yaml b/roles/prepare-workspace-openshift/tasks/main.yaml new file mode 100644 index 0000000..0d6d50b --- /dev/null +++ b/roles/prepare-workspace-openshift/tasks/main.yaml @@ -0,0 +1,4 @@ +--- +- include_tasks: rsync.yaml + when: item.1.pod is defined + loop: "{{ openshift_pods.items()|list }}" diff --git a/roles/prepare-workspace-openshift/tasks/rsync.yaml b/roles/prepare-workspace-openshift/tasks/rsync.yaml new file mode 100644 index 0000000..c90c4ed --- /dev/null +++ b/roles/prepare-workspace-openshift/tasks/rsync.yaml @@ -0,0 +1,17 @@ +--- +- name: Create src directory + command: > + oc --context "{{ item.1.context }}" + --namespace "{{ item.1.namespace }}" + exec {{ item.1.pod }} mkdir src + delegate_to: localhost + +- name: Copy src repos to the pod + command: > + oc --context "{{ item.1.context }}" + --namespace "{{ item.1.namespace }}" + rsync -q --progress=false + {{ zuul.executor.src_root }}/ + {{ item.1.pod }}:src/ + no_log: true + delegate_to: localhost diff --git a/roles/remove-zuul-sshkey/README.rst b/roles/remove-zuul-sshkey/README.rst new file mode 100644 index 0000000..2c2d3d2 --- /dev/null +++ b/roles/remove-zuul-sshkey/README.rst @@ -0,0 +1,4 @@ +Remove the zuul ssh key + +This role is intended to be run on the Zuul Executor at the start of +every job to prevent access to public Zuul ssh connection. diff --git a/roles/remove-zuul-sshkey/library/sshagent_remove_keys.py b/roles/remove-zuul-sshkey/library/sshagent_remove_keys.py new file mode 100644 index 0000000..b4f6ea6 --- /dev/null +++ b/roles/remove-zuul-sshkey/library/sshagent_remove_keys.py @@ -0,0 +1,126 @@ +# Copyright 2018 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import os +import socket +import struct +import sys +import re + +from ansible.module_utils.basic import AnsibleModule + + +SSH_AGENT_FAILURE = 5 +SSH_AGENT_SUCCESS = 6 +SSH_AGENT_IDENTITIES_ANSWER = 12 + +SSH_AGENTC_REQUEST_IDENTITIES = 11 +SSH_AGENTC_REMOVE_IDENTITY = 18 + + +def unpack_string(data): + (l,) = struct.unpack('!i', data[:4]) + d = data[4:4 + l] + return (d, data[4 + l:]) + + +def pack_string(data): + ret = struct.pack('!i', len(data)) + return ret + data + + +class Agent(object): + def __init__(self): + path = os.environ['SSH_AUTH_SOCK'] + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.sock.connect(path) + + def send(self, message_type, contents): + payload = struct.pack('!ib', len(contents) + 1, message_type) + payload += bytearray(contents) + self.sock.send(payload) + + def recv(self): + buf = b'' + while len(buf) < 5: + buf += self.sock.recv(1) + message_len, message_type = struct.unpack('!ib', buf[:5]) + buf = buf[5:] + while len(buf) < message_len - 1: + buf += self.sock.recv(1) + return message_type, buf + + def list(self): + self.send(SSH_AGENTC_REQUEST_IDENTITIES, b'') + mtype, data = self.recv() + if mtype != SSH_AGENT_IDENTITIES_ANSWER: + raise Exception("Invalid response to list") + (nkeys,) = struct.unpack('!i', data[:4]) + data = data[4:] + keys = [] + for i in range(nkeys): + blob, data = unpack_string(data) + comment, data = unpack_string(data) + keys.append((blob, comment)) + return keys + + def remove(self, blob): + self.send(SSH_AGENTC_REMOVE_IDENTITY, pack_string(blob)) + mtype, data = self.recv() + if mtype != SSH_AGENT_SUCCESS: + raise Exception("Key was not removed") + + +def run(remove): + a = Agent() + keys = a.list() + removed = [] + to_remove = re.compile(remove) + for blob, comment in keys: + if not to_remove.match(comment.decode('utf8')): + continue + a.remove(blob) + removed.append(comment) + return removed + + +def ansible_main(): + module = AnsibleModule( + argument_spec=dict( + remove=dict(required=True, type='str'))) + + removed = run(module.params.get('remove')) + + module.exit_json(changed=(removed != []), + removed=removed) + + +def cli_main(): + parser = argparse.ArgumentParser( + description="Remove ssh keys from agent" + ) + parser.add_argument('remove', nargs='+', + help='regex matching comments of keys to remove') + args = parser.parse_args() + + removed = run(args.remove) + print(removed) + + +if __name__ == '__main__': + if sys.stdin.isatty(): + cli_main() + else: + ansible_main() diff --git a/roles/remove-zuul-sshkey/tasks/main.yaml b/roles/remove-zuul-sshkey/tasks/main.yaml new file mode 100644 index 0000000..e417f58 --- /dev/null +++ b/roles/remove-zuul-sshkey/tasks/main.yaml @@ -0,0 +1,8 @@ +--- +- name: Remove master key from local agent + # The master key has a filename, all others (e.g., per-project keys) + # have "(stdin)" as a comment. + sshagent_remove_keys: + remove: '^(?!\(stdin\)).*' + delegate_to: localhost + run_once: true diff --git a/zuul.d/_jobs-base.yaml b/zuul.d/_jobs-base.yaml index 8ce4f2c..ec953a4 100644 --- a/zuul.d/_jobs-base.yaml +++ b/zuul.d/_jobs-base.yaml @@ -1,90 +1,5 @@ # This file is managed by ansible, do not edit directly --- -- job: - name: base - parent: null - description: The base job. - pre-run: playbooks/base/pre.yaml - post-run: - - playbooks/base/post.yaml - roles: - - zuul: fedora-zuul-jobs - - zuul: zuul/zuul-jobs - timeout: 1800 - attempts: 3 - secrets: - - site_sflogs - nodeset: - nodes: - - name: container - label: cloud-fedora - -- semaphore: - name: semaphore-config-update - max: 1 - -- job: - name: config-check - parent: base - final: true - allowed-projects: - - fedora-project-config - description: Validate the config repo. - run: playbooks/config/check.yaml - post-run: playbooks/config/check-fetch-artifacts.yaml - secrets: - - service_user - vars: - zuul_log_url: "https://fedora.softwarefactory-project.io/logs" - gateway_url: "https://fedora.softwarefactory-project.io" - tenant_config: False - nodeset: - nodes: [] - -- job: - name: config-update - parent: base - final: true - allowed-projects: - - fedora-project-config - description: Deploy config repo update. - run: playbooks/config/update.yaml - secrets: - - site_install_server - semaphore: semaphore-config-update - nodeset: - nodes: [] - -- project: - name: fedora-project-config - check: - jobs: - - config-check - gate: - jobs: - - config-check - post: - jobs: - - config-update - - -- project: - name: fedora-zuul-jobs - check: - jobs: - - linters: - nodeset: - nodes: - - name: container - label: runc-centos - gate: - jobs: - - linters: - nodeset: - nodes: - - name: container - label: runc-centos - - job: name: wait-for-changes-ahead diff --git a/zuul.d/_jobs-openshift.yaml b/zuul.d/_jobs-openshift.yaml index 2c97459..4ddf26a 100644 --- a/zuul.d/_jobs-openshift.yaml +++ b/zuul.d/_jobs-openshift.yaml @@ -1,48 +1,3 @@ # This file is managed by sfconfig, do not edit manually # The Openshift driver is a Tech Preview, use at your own risk... --- -- job: - name: base-openshift-native - parent: null - description: | - A base job that build and deploy a container image using the - project future state. - pre-run: playbooks/openshift/pre.yaml - post-run: - - playbooks/base/post.yaml - roles: - - zuul: fedora-zuul-jobs - - zuul: zuul/zuul-jobs - timeout: 1800 - # Set attempts to 1 until it's working well - attempts: 1 - secrets: - - site_sflogs - nodeset: - nodes: - - name: project - label: openshift-project - vars: - base_image: "python:3.6" - -- job: - name: base-openshift-pod - parent: null - description: | - A base job to spawn a vanilla container and copy the project - future state - pre-run: playbooks/openshift/unprivileged-machine.yaml - post-run: - - playbooks/base/post.yaml - roles: - - zuul: fedora-zuul-jobs - - zuul: zuul/zuul-jobs - # Set attempts to 1 until it's working well - attempts: 1 - secrets: - - site_sflogs - timeout: 1800 - nodeset: - nodes: - - name: pod - label: openshift-pod-fedora diff --git a/zuul.d/_jobs-pages.yaml b/zuul.d/_jobs-pages.yaml index c7518b5..19b270d 100644 --- a/zuul.d/_jobs-pages.yaml +++ b/zuul.d/_jobs-pages.yaml @@ -1,28 +1,2 @@ # This file is managed by ansible, do not edit directly --- -- job: - name: build-and-publish-pages - parent: base - description: Base build and publish pages job - run: playbooks/pages/build.yaml - post-run: - - playbooks/pages/fetch.yaml - - playbooks/pages/publish.yaml - secrets: - - site_pages - protected: true - vars: - src_dir: / - fqdn: fedora.softwarefactory-project.io - -- job: - name: build-pages - parent: base - description: Pages build - run: playbooks/pages/build.yaml - post-run: - - playbooks/pages/fetch.yaml - success-url: 'pages/' - vars: - src_dir: / - diff --git a/zuul.d/_pipelines.yaml b/zuul.d/_pipelines.yaml index c40e6f2..76bc8da 100644 --- a/zuul.d/_pipelines.yaml +++ b/zuul.d/_pipelines.yaml @@ -1,299 +1,43 @@ # This file is managed by ansible, do not edit directly --- - pipeline: - name: check - description: | - Newly uploaded patchsets enter this pipeline to receive an - initial +/-1 Verified vote. - manager: independent - require: - pagure.io: - merged: False - stg.pagure.io: - merged: False - open: True - src.fedoraproject.org: - merged: False - amqp: - merged: False - trigger: - pagure.io: - - event: pg_pull_request - action: comment - comment: (?i)^\s*recheck\s*$ - - event: pg_pull_request - action: - - opened - - changed - stg.pagure.io: - - event: pg_pull_request - action: comment - comment: (?i)^\s*recheck\s*$ - - event: pg_pull_request - action: - - opened - - changed - src.fedoraproject.org: - - event: pg_pull_request - action: comment - comment: (?i)^\s*recheck\s*$ - - event: pg_pull_request - action: - - opened - - changed - amqp: - - event: pg_pull_request - action: comment - comment: (?i)^\s*recheck\s*$ - - event: pg_pull_request - action: - - opened - - changed - start: - pagure.io: - status: 'pending' - status-url: "https://fedora.softwarefactory-project.io/zuul/t/fedora-staging/status.html" - comment: false - stg.pagure.io: - status: 'pending' - status-url: "https://fedora.softwarefactory-project.io/zuul/t/fedora-staging/status.html" - comment: false - src.fedoraproject.org: - status: 'pending' - status-url: "https://fedora.softwarefactory-project.io/zuul/t/fedora-staging/status.html" - comment: false - amqp: - status: 'pending' - status-url: "https://fedora.softwarefactory-project.io/zuul/t/fedora-staging/status.html" - comment: false - success: - pagure.io: - status: 'success' - stg.pagure.io: - status: 'success' - src.fedoraproject.org: - status: 'success' - amqp: - status: 'success' - sqlreporter: - failure: - pagure.io: - status: 'failure' - stg.pagure.io: - status: 'failure' - src.fedoraproject.org: - status: 'failure' - amqp: - status: 'failure' - sqlreporter: - -- pipeline: - name: gate - description: | - Changes that have been approved by core developers are enqueued - in order in this pipeline, and if they pass tests, will be - merged. - success-message: Build succeeded (gate pipeline). - failure-message: | - Build failed (gate pipeline). For information on how to proceed, see - http://docs.openstack.org/infra/manual/developers.html#automated-testing - manager: dependent - precedence: high - post-review: True - require: - pagure.io: - score: 1 - merged: False - status: success - stg.pagure.io: - score: 1 - merged: False - open: True - status: success - tags: gateit - src.fedoraproject.org: - score: 1 - merged: False - status: success - amqp: - score: 1 - merged: False - status: success - trigger: - pagure.io: - - event: pg_pull_request - action: status - status: success - - event: pg_pull_request_review - action: thumbsup - stg.pagure.io: - - event: pg_pull_request - action: status - status: success - - event: pg_pull_request_review - action: thumbsup - - event: pg_pull_request - action: tagged - tag: gateit - src.fedoraproject.org: - - event: pg_pull_request - action: status - status: success - - event: pg_pull_request_review - action: thumbsup - amqp: - - event: pg_pull_request - action: status - status: success - - event: pg_pull_request_review - action: thumbsup - start: - pagure.io: - status: 'pending' - status-url: "https://fedora.softwarefactory-project.io/zuul/t/fedora-staging/status.html" - comment: false - stg.pagure.io: - status: 'pending' - status-url: "https://fedora.softwarefactory-project.io/zuul/t/fedora-staging/status.html" - comment: false - src.fedoraproject.org: - status: 'pending' - status-url: "https://fedora.softwarefactory-project.io/zuul/t/fedora-staging/status.html" - comment: false - amqp: - status: 'pending' - status-url: "https://fedora.softwarefactory-project.io/zuul/t/fedora-staging/status.html" - comment: false - success: - pagure.io: - status: 'success' - merge: true - comment: true - stg.pagure.io: - status: 'success' - merge: true - comment: true - src.fedoraproject.org: - status: 'success' - merge: true - comment: true - amqp: - status: 'success' - merge: true - comment: true - sqlreporter: - failure: - pagure.io: - status: 'failure' - comment: true - stg.pagure.io: - status: 'failure' - comment: true - src.fedoraproject.org: - status: 'failure' - comment: true - amqp: - status: 'failure' - comment: true - sqlreporter: - window-floor: 20 - window-increase-factor: 2 - -- pipeline: - name: post - post-review: true - description: This pipeline runs jobs that operate after each change is merged. - manager: supercedent - precedence: low - trigger: - pagure.io: - - event: pg_push - ref: ^refs/heads/.*$ - stg.pagure.io: - - event: pg_push - ref: ^refs/heads/.*$ - src.fedoraproject.org: - - event: pg_push - ref: ^refs/heads/.*$ - amqp: - - event: pg_push - ref: ^refs/heads/.*$ - success: - sqlreporter: - failure: - smtp: - from: "zuul@fedora.softwarefactory-project.io" - to: "root@localhost" - subject: '[Zuul] Job failed in post pipeline: {change.project}' - sqlreporter: - -- pipeline: - name: tag + name: periodic post-review: true - description: This pipeline runs jobs when a tge git repo is tagged. - manager: supercedent + description: Jobs in this queue are triggered daily. + manager: independent precedence: low trigger: - pagure.io: - - event: pg_push - ref: ^refs/tags/.*$ - stg.pagure.io: - - event: pg_push - ref: ^refs/tags/.*$ - src.fedoraproject.org: - - event: pg_push - ref: ^refs/tags/.*$ + timer: + - time: '0 0 * * *' success: sqlreporter: failure: smtp: from: "zuul@fedora.softwarefactory-project.io" to: "root@localhost" - subject: '[Zuul] Job failed in tag pipeline: {change.project}' + subject: '[Zuul] Job failed in periodic pipeline: {change.project}' sqlreporter: - pipeline: - name: promote - description: | - This pipeline runs jobs that operate after each change is merged - in order to promote artifacts generated in the gate - pipeline. - manager: supercedent - precedence: high - post-review: True - require: - stg.pagure.io: - merged: True - open: False + name: experimental + description: On-demand pipeline for requesting a run against a set of jobs that are not yet gating. Leave review comment of "check experimental" to run jobs in this pipeline. + success-message: Build succeeded (experimental pipeline). + failure-message: Build failed (experimental pipeline). + manager: independent + precedence: normal trigger: - stg.pagure.io: - - event: pg_pull_request - action: closed success: - stg.pagure.io: - # status: 'success' - comment: true sqlreporter: failure: - stg.pagure.io: - # status: 'failure' - comment: true sqlreporter: - pipeline: - name: periodic - post-review: true - description: Jobs in this queue are triggered daily. + name: merge-check + description: > + Each time a change merges, this pipeline verifies that all open changes + on the same project are still mergeable. + failure-message: Build failed (merge-check pipeline). manager: independent + ignore-dependencies: true precedence: low - trigger: - timer: - - time: '0 0 * * *' - success: - sqlreporter: - failure: - smtp: - from: "zuul@fedora.softwarefactory-project.io" - to: "root@localhost" - subject: '[Zuul] Job failed in periodic pipeline: {change.project}' - sqlreporter: + trigger: {}