From 7e6c18d70d01c5bd310301ee51cbae395276aabe Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Apr 08 2022 07:07:13 +0000 Subject: [PATCH 1/4] Improve README.md This change: - adds more details about how to perform changes in FZCI config. - update Makefile --- diff --git a/Makefile b/Makefile index 8b2c1c7..afb976f 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,10 @@ export FZCI = $(shell pwd)/FZCI.dhall/package.dhall define generate_file @echo "Rendering $(1)" - @dhall-to-yaml --generated-comment --file $(1).dhall | python3 -mzuulfmt > $(1).yaml + @dhall-to-yaml --generated-comment --file $(1).dhall | python -mzuulfmt > $(1).yaml endef -all: render install-req - -render: +render: check-tools @dhall hash --file $$FZCI > /dev/null $(call generate_file,../fedora-zuul-jobs-config/zuul.d/jobs) $(call generate_file,../fedora-zuul-jobs/zuul.d/jobs) @@ -15,11 +13,15 @@ render: $(call generate_file,../fedora-zuul-jobs/zuul.d/templates) define clone - @test -d ../$(1)/.git || git clone https://pagure.io/$(1).git ../$(1)/ + @test -d ../$(1)/.git || git clone https://pagure.io/$(1).git ../$(1)/ && \ + pushd ../$(1)/ && git checkout master && git fetch origin && \ + git reset --hard origin/master && popd endef -install-req: - @type -p dhall-to-yaml > /dev/null || sudo dnf install -y dhall-json - @python3 -mzuulfmt --help > /dev/null || python3 -mpip install --user zuulfmt +update-jobs-repos: $(call clone,fedora-zuul-jobs-config) $(call clone,fedora-zuul-jobs) + +check-tools: + @type -p dhall-to-yaml 2> /dev/null || echo "Unable to find dhall-to-yaml in your PATH" + @type -p -zuulfmt 2> /dev/null || "Unable to find zuulfmt in your PATH" diff --git a/README.md b/README.md index 1938a2d..a573733 100644 --- a/README.md +++ b/README.md @@ -4,34 +4,100 @@ Software Factory configuration for the Fedora project. Here are configured: -- resources/: Git repositories to be listened by softwarefactory-project.io's Zuul. -- zuul.d/: Zuul pipelines and base Zuul jobs for Fedora. -- playbooks/: Ansible playbooks used by Zuul jobs. -- tools/: Some addtionnal tooling. -- FZCI.dhall: [Dhall][dhall-lang] description of Fedora Resources to be used in Zuul job configuration. +- resources/: Git repositories to be listened for events by Fedora's Zuul. +- zuul.d/_pipelines.yaml: Zuul pipelines for the Fedora's Zuul jobs. +- playbooks/: Ansible playbooks used by some Fedora's Zuul jobs. +- tools/: Addtionnal tooling. +- FZCI.dhall: [Dhall][dhall-lang] description of Fedora Zuul resources to be used to render Fedora's Zuul jobs configuration. -## Prepare your workspace to edit FZCI Dhall config +The Zuul CI configuration for Fedora is managed through 3 repositories: -### Clone sources +- `fedora-project-config`: this one. +- `fedora-zuul-jobs-config`: A [Zuul config](https://zuul-ci.org/docs/zuul/latest/glossary.html?highlight=trusted#term-config-project) repository to host sensitive Zuul config (e.g. jobs relying on secrets). +- `fedora-zuul-jobs`: A [Zuul untrusted](https://zuul-ci.org/docs/zuul/latest/glossary.html?highlight=trusted#term-untrusted-project) repository to host +non sensitive Zuul config. -Run the following command to install dependencies. + +Most of the Zuul config for Fedora is managed via the Dhall language. The rendering of the +dhall program produces YAML files to be used by Zuul. YAML files are generated into the +three repositories described above. + +## Prepare your workspace to edit Fedora Zuul jobs config + +### Install required tools + +`dhall-to-yaml` and `zuulfmt` are needed to render the Zuul configuration. + +To install `dhall-to-yaml` runs (or adapt to your working environment) + +``` +dnf install -y dhall-json +``` + +To install `zuulfmt` runs (or adapt to your working environment) + +``` +python -mpip install --user zuulfmt +``` + +It is adviced to install the Dhall lang support for your IDE: + +- [VSCode plugins](https://marketplace.visualstudio.com/items?itemName=dhall.vscode-dhall-lsp-server) + +### Clone/Update jobs repositories + +Run the following command to install (and/or reset to the last origin/HEAD) extra repositories. +This prepares `fedora-zuul-jobs-config` and `fedora-zuul-jobs` checkout in the parent directory. ```ShellSession -make install-req +make update-jobs-repos ``` -### Run the rendering +Then: + +- Fork both repositories in your pagure.io user namesapce +- Use `git remote add fork ssh://git@pagure.io/forks//.git` + + +## How to update Zuul jobs config + + +First, ensure repositories are in the right state (all commands below assume working from the master branch): + +``` +git checkout master && git fetch origin && git reset --hard origin/master +make update-jobs-repos +``` + +Second, add the required changes. + +Then run the rendering. ```ShellSession make render ``` -Note that it is adviced to get a support for Dhall Lang into your IDE but at least you must have -installed the dhall commands on you system. +In case of error, update your changes then run again the rendering. + +Finally, create Pull Requests for each repositories that got an update from the rendering. + +For instance: + +``` +pushd ../fedora-zuul-jobs +git status +git add -A . +git commit -m"Add change for ..." +git push fork master:pr-branch-on-fork +``` + +And open the PR on pagure.io. + +## HOWTOs -## Adding support for a Fedora Branch +### Add support for a Fedora Branch -### Adding new branch definition +#### Adding new branch definition - Edit file [branches.dhall](FZCI.dhall/Branches.dhall) in order to add the . - The type checker will help you to fill the places where the handling must @@ -45,7 +111,7 @@ cd fedora-project-config dhall <<< './FZCI.dhall/Branches.dhall' ``` -### Update jobs from fedora-zuul-jobs-config +#### Update jobs from fedora-zuul-jobs-config In fedora-zuul-jobs-config @@ -61,7 +127,7 @@ dhall-to-yaml --generated-comment --file jobs.dhall | python3 -mzuulfmt > jobs.y # or run: make render ``` -### Update jobs from fedora-zuul-jobs +#### Update jobs from fedora-zuul-jobs In fedora-zuul-jobs @@ -75,7 +141,7 @@ dhall-to-yaml --generated-comment --file jobs.dhall | python3 -mzuulfmt > jobs.y # or run: make render ``` -### Push changes +#### Push changes When edits are done on those repos then commits and push changes (as PR) then wait for CI result. From b811b3923b1c7c34e5c42fd4f1c89c5e3d97baef Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Apr 08 2022 07:20:44 +0000 Subject: [PATCH 2/4] Add check-updates make command --- diff --git a/Makefile b/Makefile index afb976f..23aefd7 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ export FZCI = $(shell pwd)/FZCI.dhall/package.dhall define generate_file @echo "Rendering $(1)" - @dhall-to-yaml --generated-comment --file $(1).dhall | python -mzuulfmt > $(1).yaml + @dhall-to-yaml --generated-comment --file $(1).dhall | zuulfmt > $(1).yaml endef render: check-tools @@ -18,10 +18,14 @@ define clone git reset --hard origin/master && popd endef +check-updates: + @pushd ../fedora-zuul-jobs-config && git status && popd + @pushd ../fedora-zuul-jobs && git status && popd + update-jobs-repos: $(call clone,fedora-zuul-jobs-config) $(call clone,fedora-zuul-jobs) check-tools: @type -p dhall-to-yaml 2> /dev/null || echo "Unable to find dhall-to-yaml in your PATH" - @type -p -zuulfmt 2> /dev/null || "Unable to find zuulfmt in your PATH" + @type -p zuulfmt 2> /dev/null || "Unable to find zuulfmt in your PATH" diff --git a/README.md b/README.md index a573733..7df98c5 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,6 @@ The Zuul CI configuration for Fedora is managed through 3 repositories: - `fedora-zuul-jobs`: A [Zuul untrusted](https://zuul-ci.org/docs/zuul/latest/glossary.html?highlight=trusted#term-untrusted-project) repository to host non sensitive Zuul config. - Most of the Zuul config for Fedora is managed via the Dhall language. The rendering of the dhall program produces YAML files to be used by Zuul. YAML files are generated into the three repositories described above. @@ -84,6 +83,8 @@ Finally, create Pull Requests for each repositories that got an update from the For instance: ``` +make check-updates +# This shows a change in fedore-zuul-jobs then pushd ../fedora-zuul-jobs git status git add -A . From d610429564884e563104e61827d967632f014583 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Apr 08 2022 07:31:39 +0000 Subject: [PATCH 3/4] Import fedora-zuul-jobs-config/zuul.d/jobs.dhall --- diff --git a/FZCI.dhall/fedora-zuul-jobs-config/jobs.dhall b/FZCI.dhall/fedora-zuul-jobs-config/jobs.dhall new file mode 100644 index 0000000..e995b34 --- /dev/null +++ b/FZCI.dhall/fedora-zuul-jobs-config/jobs.dhall @@ -0,0 +1,322 @@ +{-| +To compute the jobs.yaml run: make render +-} +let FZCI = ../package.dhall + +let Zuul = FZCI.Zuul + +let Prelude = FZCI.Prelude + +let --| When setting a custom prefix, redirect the output to a new file to preserve the existing job definition + job-prefix = + env:JOB_PREFIX as Text ? "" + +let Branches = FZCI.Branches + +let Arches = FZCI.Arches + +let Nodesets = FZCI.Nodesets + +let KojiBuild = + let Union = < Scratch | Final > + + let eq_def = { Scratch = False, Final = False } + + in { Type = Union + , all = [ Union.Scratch, Union.Final ] + , isScratch = λ(kb : Union) → merge (eq_def ⫽ { Scratch = True }) kb + , show = + λ(kb : Union) → merge { Scratch = "scratch", Final = "final" } kb + } + +let generateRpmBuildJobName + : Arches.Type → KojiBuild.Type → Text + = λ(arch : Arches.Type) → + λ(kbtype : KojiBuild.Type) → + let suffix = + if Arches.isX86_64 arch then "" else "-" ++ Arches.show arch + + let name = + if KojiBuild.isScratch kbtype + then "rpm-scratch-build" ++ suffix + else "rpm-build" + + in job-prefix ++ name + +let Arch = + Arches + ⫽ { extras = + Prelude.List.filter + Arches.Type + (λ(arch : Arches.Type) → Prelude.Bool.not (Arches.isX86_64 arch)) + Arches.fedora + , scratch-job-names = + Prelude.List.map + Arches.Type + Text + ( λ(arch : Arches.Type) → + generateRpmBuildJobName arch KojiBuild.Type.Scratch + ) + } + +let executor_nodeset = Zuul.Nodeset.Inline Zuul.Nodeset.empty + +let default_nodeset = Nodesets.getName Nodesets.Type.Fedora-34-Container + +let check_for_tests = + Zuul.Job::{ + , name = "${job-prefix}check-for-tests" + , description = Some "Check the project has a tests/tests.yml" + , branches = Some Branches.allText + , run = Some "playbooks/rpm/check-for-tests.yaml" + , nodeset = Some executor_nodeset + } + +let check_for_sti_tests = + Zuul.Job::{ + , name = "${job-prefix}check-for-sti-tests" + , description = Some "Check the project has a tests/tests.yml" + , branches = Some Branches.allText + , run = Some "playbooks/rpm/check-for-sti-tests.yaml" + , nodeset = Some executor_nodeset + } + +let check_for_fmf_tests = + Zuul.Job::{ + , name = "${job-prefix}check-for-fmf-tests" + , description = Some "Check the project has a .fmf directory" + , branches = Some Branches.allText + , run = Some "playbooks/rpm/check-for-fmf-tests.yaml" + , nodeset = Some executor_nodeset + } + +let check_for_arches = + Zuul.Job::{ + , name = "${job-prefix}check-for-arches" + , description = Some "Check the packages needs arches builds" + , branches = Some Branches.allText + , run = Some "playbooks/rpm/check-for-arches.yaml" + , vars = Some + ( Zuul.Vars.object + ( toMap + { arch_jobs = + Zuul.Vars.array + ( Prelude.List.map + Text + Zuul.Vars.Type + Zuul.Vars.string + (Arch.scratch-job-names Arch.extras) + ) + , check_for_arches_ignore = Zuul.Vars.bool False + } + ) + ) + , nodeset = Some default_nodeset + } + +let clean_stalled_koji_tasks = + Zuul.Job::{ + , name = "${job-prefix}clean-stalled-koji-tasks" + , protected = Some True + , description = Some "Cancel Koji tasks running for more than 12 hours" + , nodeset = Some default_nodeset + , roles = Some [ { zuul = "zuul-distro-jobs" } ] + , run = Some "playbooks/koji/clean-stalled-koji-tasks.yaml" + , secrets = Some + [ Zuul.Job.Secret::{ name = "krb_keytab", secret = "krb_keytab" } ] + } + +let common_koji_rpm_build = + Zuul.Job::{ + , name = "${job-prefix}common-koji-rpm-build" + , abstract = Some True + , protected = Some True + , description = Some "Base job for RPM build on Fedora Koji" + , timeout = Some 21600 + , nodeset = Some (Nodesets.getName Nodesets.Type.Fedora-35-Container) + , roles = Some [ { zuul = "zuul-distro-jobs" } ] + , run = Some "playbooks/koji/build-ng.yaml" + , secrets = Some + [ Zuul.Job.Secret::{ name = "krb_keytab", secret = "krb_keytab" } ] + } + +let setVars = + λ(target : Text) → + λ(release : Text) → + λ(arch : Text) → + λ(scratch_build : Bool) → + λ(fetch_artifacts : Bool) → + let basevars = + toMap + { fetch_artifacts = Zuul.Vars.bool fetch_artifacts + , scratch_build = Zuul.Vars.bool scratch_build + , target = Zuul.Vars.string target + , release = Zuul.Vars.string release + } + + let vars = + if scratch_build + then basevars + # [ { mapKey = "arches", mapValue = Zuul.Vars.string arch } + ] + else basevars + + in Zuul.Vars.object vars + +let doFetchArtifact + : Arch.Type → KojiBuild.Type → Bool + = λ(arch : Arch.Type) → + λ(kb : KojiBuild.Type) → + Arch.isX86_64 arch && KojiBuild.isScratch kb + +let generateRpmBuildJob = + λ(kbtype : KojiBuild.Type) → + λ(branch : Branches.Type) → + λ(arch : Arch.Type) → + Zuul.Job::{ + , name = generateRpmBuildJobName arch kbtype + , parent = Some (Zuul.Job.getName common_koji_rpm_build) + , final = Some True + , provides = + if Arch.isX86_64 arch && KojiBuild.isScratch kbtype + then Some [ "${job-prefix}repo" ] + else None (List Text) + , dependencies = + if Arch.isX86_64 arch + then None (List Zuul.Job.Dependency.Union) + else Some + [ Zuul.Job.Dependency.Name "${job-prefix}check-for-arches" ] + , branches = Branches.zuulBranch branch + , vars = Some + ( setVars + (Branches.target branch) + (Branches.show branch) + (Arch.show arch) + (KojiBuild.isScratch kbtype) + (doFetchArtifact arch kbtype) + ) + } + +let generateRpmBuildJobs + : KojiBuild.Type → List Zuul.Job.Type + = λ(kbtype : KojiBuild.Type) → + let forBranch = + λ(branch : Branches.Type) → + Prelude.List.map + Arch.Type + Zuul.Job.Type + (generateRpmBuildJob kbtype branch) + ( if KojiBuild.isScratch kbtype + then Branches.arches branch + else [ Arch.Type.X86_64 ] + ) + + in Prelude.List.concatMap + Branches.Type + Zuul.Job.Type + forBranch + Branches.all + +let getTestingFarmJobVars = + λ(branch : Branches.Type) → + λ(ttype : Text) → + let name = + merge + { Rawhide = "Fedora-Rawhide" + , F34 = "Fedora-34" + , F35 = "Fedora-35" + , F36 = "Fedora-36" + , Epel8 = "CentOS-Stream-8" + } + branch + + let distro = + merge + { Rawhide = "fedora-rawhide" + , F34 = "fedora-34" + , F35 = "fedora-35" + , F36 = "fedora-36" + , Epel8 = "centos-stream-8" + } + branch + + in Zuul.Vars.object + [ { mapKey = "compose", mapValue = Zuul.Vars.string name } + , { mapKey = "distro", mapValue = Zuul.Vars.string distro } + , { mapKey = "test_type", mapValue = Zuul.Vars.string ttype } + ] + +let generateSTITestsJob = + λ(branch : Branches.Type) → + Zuul.Job::{ + , name = "rpm-sti-test" + , description = Some "Running sti test on Testing Farm" + , final = Some True + , branches = Branches.zuulBranch branch + , roles = Some + [ { zuul = "zuul-distro-jobs" }, { zuul = "fedora-zuul-jobs" } ] + , timeout = Some 21600 + , nodeset = Some default_nodeset + , requires = Some [ "repo" ] + , secrets = Some + [ Zuul.Job.Secret::{ + , name = "testing_farm_zuul_ci_key" + , secret = "testing_farm_zuul_ci_key" + } + ] + , run = Some "playbooks/testing-farm/run.yaml" + , post-run = Some [ "playbooks/testing-farm/fetch.yaml" ] + , vars = Some (getTestingFarmJobVars branch "sti") + } + +let generateSTITestsJobs = + Prelude.List.map + Branches.Type + Zuul.Job.Type + generateSTITestsJob + Branches.all + +let generateTMTTestsJob = + λ(branch : Branches.Type) → + Zuul.Job::{ + , name = "rpm-tmt-test" + , description = Some "Running fmf test on Testing Farm" + , final = Some True + , branches = Branches.zuulBranch branch + , roles = Some + [ { zuul = "zuul-distro-jobs" }, { zuul = "fedora-zuul-jobs" } ] + , timeout = Some 21600 + , nodeset = Some default_nodeset + , requires = Some [ "repo" ] + , secrets = Some + [ Zuul.Job.Secret::{ + , name = "testing_farm_zuul_ci_key" + , secret = "testing_farm_zuul_ci_key" + } + ] + , run = Some "playbooks/testing-farm/run.yaml" + , post-run = Some [ "playbooks/testing-farm/fetch.yaml" ] + , vars = Some (getTestingFarmJobVars branch "fmf") + } + +let generateTMTTestsJobs = + Prelude.List.map + Branches.Type + Zuul.Job.Type + generateTMTTestsJob + Branches.all + +let Jobs = + [ clean_stalled_koji_tasks + , check_for_tests + , check_for_sti_tests + , check_for_fmf_tests + , check_for_arches + , common_koji_rpm_build + ] + # generateRpmBuildJobs KojiBuild.Type.Scratch + # generateRpmBuildJobs KojiBuild.Type.Final + # generateTMTTestsJobs + # generateSTITestsJobs + +in Zuul.Job.wrap Jobs diff --git a/Makefile b/Makefile index 23aefd7..83690a7 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,10 @@ export FZCI = $(shell pwd)/FZCI.dhall/package.dhall +define generate_file_ng + @echo "Rendering FZCI.dhall/$(1).dhall in $(2).yaml" + @dhall-to-yaml --generated-comment --file FZCI.dhall/$(1).dhall | zuulfmt > $(2).yaml +endef + define generate_file @echo "Rendering $(1)" @dhall-to-yaml --generated-comment --file $(1).dhall | zuulfmt > $(1).yaml @@ -7,7 +12,7 @@ endef render: check-tools @dhall hash --file $$FZCI > /dev/null - $(call generate_file,../fedora-zuul-jobs-config/zuul.d/jobs) + $(call generate_file_ng,fedora-zuul-jobs-config/jobs,../fedora-zuul-jobs-config/zuul.d/jobs) $(call generate_file,../fedora-zuul-jobs/zuul.d/jobs) $(call generate_file,../fedora-zuul-jobs/zuul.d/nodesets) $(call generate_file,../fedora-zuul-jobs/zuul.d/templates) From ae51d9cb2a5e4f08361c651de64bcdbd56d75e80 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Apr 08 2022 07:38:10 +0000 Subject: [PATCH 4/4] Import from fedora-zuul-jobs This change extract fedora-zuul-jobs/zuul.d/*.dhall into this repository to ease dhall code management. --- diff --git a/FZCI.dhall/fedora-zuul-jobs/jobs.dhall b/FZCI.dhall/fedora-zuul-jobs/jobs.dhall new file mode 100644 index 0000000..b2b8672 --- /dev/null +++ b/FZCI.dhall/fedora-zuul-jobs/jobs.dhall @@ -0,0 +1,166 @@ +{-| +To compute the jobs.yaml run: make render +-} +let FZCI = ../package.dhall + +let Zuul = FZCI.Zuul + +let Prelude = FZCI.Prelude + +let Nodesets = FZCI.Nodesets + +let Branches = FZCI.Branches + +let createJobVars + : Text → Zuul.Vars.Type + = λ(url : Text) → + Zuul.Vars.object + [ { mapKey = "repos" + , mapValue = + Zuul.Vars.array + [ Zuul.Vars.object + [ { mapKey = "name" + , mapValue = Zuul.Vars.string "distro-build" + } + , { mapKey = "url", mapValue = Zuul.Vars.string url } + ] + ] + } + ] + +let createRpmInstallTestJob + : Branches.Type → Zuul.Job.Type + = λ(branch : Branches.Type) → + let getNodesetByBranch = + λ(branch : Branches.Type) → + merge + { Rawhide = Nodesets.getName Nodesets.Type.Fedora-Rawhide-VM + , F34 = Nodesets.getName Nodesets.Type.Fedora-34-Container + , F35 = Nodesets.getName Nodesets.Type.Fedora-35-VM-Medium + , F36 = Nodesets.getName Nodesets.Type.Fedora-36-VM + , Epel8 = Nodesets.getName Nodesets.Type.Centos-8-VM + } + branch + + let job = + Zuul.Job::{ + , name = "rpm-install-test" + , description = Some + "Install the built rpms (${Branches.show branch} variant)" + , roles = Some [ { zuul = "zuul-distro-jobs" } ] + , requires = Some [ "repo" ] + , run = Some "playbooks/rpm/rpm-install-test.yaml" + , branches = Branches.zuulBranch branch + , nodeset = Some (getNodesetByBranch branch) + , vars = Some (createJobVars (Branches.kojirepos branch)) + } + + let job = + if Branches.isEpel8 branch + then job ⫽ { pre-run = Some [ "playbooks/epel8/install.yaml" ] } + else job + + in job + +let createRpmTestJob + : Branches.Type → Zuul.Job.Type + = λ(branch : Branches.Type) → + let getNodesetByBranch = + λ(branch : Branches.Type) → + merge + { Rawhide = + Nodesets.getName Nodesets.Type.Fedora-Rawhide-VM-Medium + , F34 = Nodesets.getName Nodesets.Type.Fedora-34-VM-Medium + , F35 = Nodesets.getName Nodesets.Type.Fedora-35-VM-Medium + , F36 = Nodesets.getName Nodesets.Type.Fedora-36-VM-Medium + , Epel8 = Nodesets.getName Nodesets.Type.Centos-8-VM + } + branch + + let job = + Zuul.Job::{ + , name = "rpm-test" + , description = Some + "Install the built rpms and run STI tests (${Branches.show + branch} variant)" + , roles = Some [ { zuul = "zuul-distro-jobs" } ] + , timeout = Some 18000 + , requires = Some [ "repo" ] + , run = Some "playbooks/rpm/rpmtest.yaml" + , post-run = Some [ "playbooks/rpm/rpmtest-fetch-artifacts.yaml" ] + , branches = Branches.zuulBranch branch + , nodeset = Some (getNodesetByBranch branch) + , vars = Some (createJobVars (Branches.kojirepos branch)) + } + + let job = + if Branches.isEpel8 branch + then job ⫽ { pre-run = Some [ "playbooks/epel8/install.yaml" ] } + else job + + in job + +let createRpmInspectJob + : Branches.Type → Zuul.Job.Type + = λ(branch : Branches.Type) → + let job = + Zuul.Job::{ + , name = "rpm-rpminspect" + , description = Some + "Run the rpminspect report tests (${Branches.show + branch} variant)" + , parent = Some "rpminspect" + , requires = Some [ "repo" ] + , branches = Branches.zuulBranch branch + , nodeset = Some + (Nodesets.getName Nodesets.Type.Fedora-34-Container) + , vars = Some + ( Zuul.Vars.object + [ { mapKey = "rpminspect_cmd" + , mapValue = Zuul.Vars.string "rpminspect-fedora" + } + , { mapKey = "check_previous_build_enabled" + , mapValue = Zuul.Vars.bool True + } + , { mapKey = "target" + , mapValue = Zuul.Vars.string (Branches.target branch) + } + ] + ) + } + + in job + +let rpm_linter_job = + Zuul.Job::{ + , name = "rpm-linter" + , parent = Some "repo-rpm-lint" + , requires = Some [ "repo" ] + , nodeset = Some (Nodesets.getName Nodesets.Type.Fedora-34-Container) + , branches = Some Branches.allText + } + +let rpm_install_test_jobs = + Prelude.List.map + Branches.Type + Zuul.Job.Type + createRpmInstallTestJob + Branches.all + +let rpm_test_jobs = + Prelude.List.map Branches.Type Zuul.Job.Type createRpmTestJob Branches.all + +let rpminspect_jobs = + Prelude.List.map + Branches.Type + Zuul.Job.Type + createRpmInspectJob + Branches.all + +let jobs = + rpm_install_test_jobs + # rpm_test_jobs + # rpminspect_jobs + # [ rpm_linter_job ] + +in Zuul.Job.wrap jobs diff --git a/FZCI.dhall/fedora-zuul-jobs/nodesets.dhall b/FZCI.dhall/fedora-zuul-jobs/nodesets.dhall new file mode 100644 index 0000000..5b3caf4 --- /dev/null +++ b/FZCI.dhall/fedora-zuul-jobs/nodesets.dhall @@ -0,0 +1,18 @@ +{-| +To compute the jobs.yaml run: make render +-} +let FZCI = ../package.dhall + +let Prelude = FZCI.Prelude + +let Zuul = FZCI.Zuul + +let Nodesets = FZCI.Nodesets + +in Zuul.Nodeset.wrap + ( Prelude.List.map + Nodesets.Type + Zuul.Nodeset.Type + Nodesets.toNodeset + Nodesets.all + ) diff --git a/FZCI.dhall/fedora-zuul-jobs/templates.dhall b/FZCI.dhall/fedora-zuul-jobs/templates.dhall new file mode 100644 index 0000000..5f30f0c --- /dev/null +++ b/FZCI.dhall/fedora-zuul-jobs/templates.dhall @@ -0,0 +1,143 @@ +{-| +To compute the jobs.yaml run: make render +-} +let FZCI = ../package.dhall + +let Prelude = FZCI.Prelude + +let Zuul = FZCI.Zuul + +let build = + let createSBJobsForArches = + Prelude.List.map + FZCI.Arches.Type + Zuul.Job.Union + ( λ(arch : FZCI.Arches.Type) → + if FZCI.Arches.isX86_64 arch + then Zuul.Job.Name "rpm-scratch-build" + else Zuul.Job.Inline + Zuul.Job::{ + , name = "rpm-scratch-build-${FZCI.Arches.show arch}" + , voting = Some False + } + ) + FZCI.Arches.fedora + + in toMap + { name = Zuul.ProjectTemplate.Name "build" + , check = + Zuul.ProjectTemplate.Pipeline + Zuul.Project.PipelineConfig::{ + , jobs = + [ Zuul.Job.Name "check-for-arches" ] + # createSBJobsForArches + } + } + +let lint = + toMap + { name = Zuul.ProjectTemplate.Name "lint" + , check = + Zuul.ProjectTemplate.Pipeline + Zuul.Project.PipelineConfig::{ + , jobs = + [ Zuul.Job.Inline + Zuul.Job::{ + , name = "rpm-linter" + , dependencies = Some + [ Zuul.Job.Dependency.Name "rpm-scratch-build" ] + } + , Zuul.Job.Inline + Zuul.Job::{ + , name = "rpm-rpminspect" + , voting = Some False + , dependencies = Some + [ Zuul.Job.Dependency.Name "rpm-scratch-build" ] + } + ] + } + } + +let test = + toMap + { name = Zuul.ProjectTemplate.Name "test" + , check = + Zuul.ProjectTemplate.Pipeline + Zuul.Project.PipelineConfig::{ + , jobs = + [ Zuul.Job.Inline + Zuul.Job::{ + , name = "check-for-tests" + , dependencies = Some + [ Zuul.Job.Dependency.Name "rpm-scratch-build" ] + } + , Zuul.Job.Inline + Zuul.Job::{ + , name = "check-for-sti-tests" + , dependencies = Some + [ Zuul.Job.Dependency.Name "rpm-scratch-build" ] + } + , Zuul.Job.Inline + Zuul.Job::{ + , name = "check-for-fmf-tests" + , dependencies = Some + [ Zuul.Job.Dependency.Name "rpm-scratch-build" ] + } + , Zuul.Job.Inline + Zuul.Job::{ + , name = "rpm-install-test" + , dependencies = Some + [ Zuul.Job.Dependency.Name "rpm-scratch-build" ] + } + , Zuul.Job.Inline + Zuul.Job::{ + , name = "rpm-test" + , dependencies = Some + [ Zuul.Job.Dependency.Name "check-for-tests" + , Zuul.Job.Dependency.Name "rpm-scratch-build" + ] + } + , Zuul.Job.Inline + Zuul.Job::{ + , name = "rpm-tmt-test" + , dependencies = Some + [ Zuul.Job.Dependency.Name "check-for-fmf-tests" + , Zuul.Job.Dependency.Name "rpm-scratch-build" + ] + } + , Zuul.Job.Inline + Zuul.Job::{ + , name = "rpm-sti-test" + , dependencies = Some + [ Zuul.Job.Dependency.Name "check-for-sti-tests" + , Zuul.Job.Dependency.Name "rpm-scratch-build" + ] + } + ] + } + } + +let publish = + toMap + { name = Zuul.ProjectTemplate.Name "publish" + , gate = + Zuul.ProjectTemplate.Pipeline + Zuul.Project.PipelineConfig::{ jobs = [ Zuul.Job.Name "noop" ] } + , promote = + Zuul.ProjectTemplate.Pipeline + Zuul.Project.PipelineConfig::{ + , jobs = [ Zuul.Job.Name "rpm-build" ] + } + } + +let testTestsDef = + toMap + { name = Zuul.ProjectTemplate.Name "test-tests-def" + , check = + Zuul.ProjectTemplate.Pipeline + Zuul.Project.PipelineConfig::{ + , jobs = [ Zuul.Job.Name "rpm-test" ] + } + } + +in Zuul.ProjectTemplate.wrap [ build, lint, test, publish, testTestsDef ] diff --git a/Makefile b/Makefile index 83690a7..a20195f 100644 --- a/Makefile +++ b/Makefile @@ -1,21 +1,16 @@ export FZCI = $(shell pwd)/FZCI.dhall/package.dhall -define generate_file_ng +define generate_file @echo "Rendering FZCI.dhall/$(1).dhall in $(2).yaml" @dhall-to-yaml --generated-comment --file FZCI.dhall/$(1).dhall | zuulfmt > $(2).yaml endef -define generate_file - @echo "Rendering $(1)" - @dhall-to-yaml --generated-comment --file $(1).dhall | zuulfmt > $(1).yaml -endef - render: check-tools @dhall hash --file $$FZCI > /dev/null - $(call generate_file_ng,fedora-zuul-jobs-config/jobs,../fedora-zuul-jobs-config/zuul.d/jobs) - $(call generate_file,../fedora-zuul-jobs/zuul.d/jobs) - $(call generate_file,../fedora-zuul-jobs/zuul.d/nodesets) - $(call generate_file,../fedora-zuul-jobs/zuul.d/templates) + $(call generate_file,fedora-zuul-jobs-config/jobs,../fedora-zuul-jobs-config/zuul.d/jobs) + $(call generate_file,fedora-zuul-jobs/jobs,../fedora-zuul-jobs/zuul.d/jobs) + $(call generate_file,fedora-zuul-jobs/nodesets,../fedora-zuul-jobs/zuul.d/nodesets) + $(call generate_file,fedora-zuul-jobs/templates,../fedora-zuul-jobs/zuul.d/templates) define clone @test -d ../$(1)/.git || git clone https://pagure.io/$(1).git ../$(1)/ && \ diff --git a/README.md b/README.md index 7df98c5..d000248 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,13 @@ Second, add the required changes. Then run the rendering. ```ShellSession -make render +[fedora@localhost fedora-project-config]$ make render +/nix/store/h1p77fzddhjwb9fl678p72hl4hg8d5l3-ghc-8.10.7-with-packages/bin/dhall-to-yaml +/home/fedora/bin/zuulfmt +Rendering FZCI.dhall/fedora-zuul-jobs-config/jobs.dhall in ../fedora-zuul-jobs-config/zuul.d/jobs.yaml +Rendering FZCI.dhall/fedora-zuul-jobs/jobs.dhall in ../fedora-zuul-jobs/zuul.d/jobs.yaml +Rendering FZCI.dhall/fedora-zuul-jobs/nodesets.dhall in ../fedora-zuul-jobs/zuul.d/nodesets.yaml +Rendering FZCI.dhall/fedora-zuul-jobs/templates.dhall in ../fedora-zuul-jobs/zuul.d/templates.yaml ``` In case of error, update your changes then run again the rendering.