#188 Ease dhall code management and update README.md
Merged 4 months ago by zuul. Opened 4 months ago by fbo.
fbo/fedora-project-config process  into  master

@@ -0,0 +1,322 @@ 

+ {-|

+ To compute the jobs.yaml run: make render

+ -}

+ let FZCI = ../package.dhall

+ 

+ let Zuul = FZCI.Zuul

+ 

+ let Prelude = FZCI.Prelude

+ 

+ let --| When setting a custom prefix, redirect the output to a new file to preserve the existing job definition

+     job-prefix =

+       env:JOB_PREFIX as Text ? ""

+ 

+ let Branches = FZCI.Branches

+ 

+ let Arches = FZCI.Arches

+ 

+ let Nodesets = FZCI.Nodesets

+ 

+ let KojiBuild =

+       let Union = < Scratch | Final >

+ 

+       let eq_def = { Scratch = False, Final = False }

+ 

+       in  { Type = Union

+           , all = [ Union.Scratch, Union.Final ]

+           , isScratch = λ(kb : Union) → merge (eq_def ⫽ { Scratch = True }) kb

+           , show =

+               λ(kb : Union) → merge { Scratch = "scratch", Final = "final" } kb

+           }

+ 

+ let generateRpmBuildJobName

+     : Arches.Type → KojiBuild.Type → Text

+     = λ(arch : Arches.Type) →

+       λ(kbtype : KojiBuild.Type) →

+         let suffix =

+               if Arches.isX86_64 arch then "" else "-" ++ Arches.show arch

+ 

+         let name =

+               if    KojiBuild.isScratch kbtype

+               then  "rpm-scratch-build" ++ suffix

+               else  "rpm-build"

+ 

+         in  job-prefix ++ name

+ 

+ let Arch =

+         Arches

+       ⫽ { extras =

+             Prelude.List.filter

+               Arches.Type

+               (λ(arch : Arches.Type) → Prelude.Bool.not (Arches.isX86_64 arch))

+               Arches.fedora

+         , scratch-job-names =

+             Prelude.List.map

+               Arches.Type

+               Text

+               ( λ(arch : Arches.Type) →

+                   generateRpmBuildJobName arch KojiBuild.Type.Scratch

+               )

+         }

+ 

+ let executor_nodeset = Zuul.Nodeset.Inline Zuul.Nodeset.empty

+ 

+ let default_nodeset = Nodesets.getName Nodesets.Type.Fedora-34-Container

+ 

+ let check_for_tests =

+       Zuul.Job::{

+       , name = "${job-prefix}check-for-tests"

+       , description = Some "Check the project has a tests/tests.yml"

+       , branches = Some Branches.allText

+       , run = Some "playbooks/rpm/check-for-tests.yaml"

+       , nodeset = Some executor_nodeset

+       }

+ 

+ let check_for_sti_tests =

+       Zuul.Job::{

+       , name = "${job-prefix}check-for-sti-tests"

+       , description = Some "Check the project has a tests/tests.yml"

+       , branches = Some Branches.allText

+       , run = Some "playbooks/rpm/check-for-sti-tests.yaml"

+       , nodeset = Some executor_nodeset

+       }

+ 

+ let check_for_fmf_tests =

+       Zuul.Job::{

+       , name = "${job-prefix}check-for-fmf-tests"

+       , description = Some "Check the project has a .fmf directory"

+       , branches = Some Branches.allText

+       , run = Some "playbooks/rpm/check-for-fmf-tests.yaml"

+       , nodeset = Some executor_nodeset

+       }

+ 

+ let check_for_arches =

+       Zuul.Job::{

+       , name = "${job-prefix}check-for-arches"

+       , description = Some "Check the packages needs arches builds"

+       , branches = Some Branches.allText

+       , run = Some "playbooks/rpm/check-for-arches.yaml"

+       , vars = Some

+           ( Zuul.Vars.object

+               ( toMap

+                   { arch_jobs =

+                       Zuul.Vars.array

+                         ( Prelude.List.map

+                             Text

+                             Zuul.Vars.Type

+                             Zuul.Vars.string

+                             (Arch.scratch-job-names Arch.extras)

+                         )

+                   , check_for_arches_ignore = Zuul.Vars.bool False

+                   }

+               )

+           )

+       , nodeset = Some default_nodeset

+       }

+ 

+ let clean_stalled_koji_tasks =

+       Zuul.Job::{

+       , name = "${job-prefix}clean-stalled-koji-tasks"

+       , protected = Some True

+       , description = Some "Cancel Koji tasks running for more than 12 hours"

+       , nodeset = Some default_nodeset

+       , roles = Some [ { zuul = "zuul-distro-jobs" } ]

+       , run = Some "playbooks/koji/clean-stalled-koji-tasks.yaml"

+       , secrets = Some

+         [ Zuul.Job.Secret::{ name = "krb_keytab", secret = "krb_keytab" } ]

+       }

+ 

+ let common_koji_rpm_build =

+       Zuul.Job::{

+       , name = "${job-prefix}common-koji-rpm-build"

+       , abstract = Some True

+       , protected = Some True

+       , description = Some "Base job for RPM build on Fedora Koji"

+       , timeout = Some 21600

+       , nodeset = Some (Nodesets.getName Nodesets.Type.Fedora-35-Container)

+       , roles = Some [ { zuul = "zuul-distro-jobs" } ]

+       , run = Some "playbooks/koji/build-ng.yaml"

+       , secrets = Some

+         [ Zuul.Job.Secret::{ name = "krb_keytab", secret = "krb_keytab" } ]

+       }

+ 

+ let setVars =

+       λ(target : Text) →

+       λ(release : Text) →

+       λ(arch : Text) →

+       λ(scratch_build : Bool) →

+       λ(fetch_artifacts : Bool) →

+         let basevars =

+               toMap

+                 { fetch_artifacts = Zuul.Vars.bool fetch_artifacts

+                 , scratch_build = Zuul.Vars.bool scratch_build

+                 , target = Zuul.Vars.string target

+                 , release = Zuul.Vars.string release

+                 }

+ 

+         let vars =

+               if    scratch_build

+               then    basevars

+                     # [ { mapKey = "arches", mapValue = Zuul.Vars.string arch }

+                       ]

+               else  basevars

+ 

+         in  Zuul.Vars.object vars

+ 

+ let doFetchArtifact

+     : Arch.Type → KojiBuild.Type → Bool

+     = λ(arch : Arch.Type) →

+       λ(kb : KojiBuild.Type) →

+         Arch.isX86_64 arch && KojiBuild.isScratch kb

+ 

+ let generateRpmBuildJob =

+       λ(kbtype : KojiBuild.Type) →

+       λ(branch : Branches.Type) →

+       λ(arch : Arch.Type) →

+         Zuul.Job::{

+         , name = generateRpmBuildJobName arch kbtype

+         , parent = Some (Zuul.Job.getName common_koji_rpm_build)

+         , final = Some True

+         , provides =

+             if    Arch.isX86_64 arch && KojiBuild.isScratch kbtype

+             then  Some [ "${job-prefix}repo" ]

+             else  None (List Text)

+         , dependencies =

+             if    Arch.isX86_64 arch

+             then  None (List Zuul.Job.Dependency.Union)

+             else  Some

+                     [ Zuul.Job.Dependency.Name "${job-prefix}check-for-arches" ]

+         , branches = Branches.zuulBranch branch

+         , vars = Some

+             ( setVars

+                 (Branches.target branch)

+                 (Branches.show branch)

+                 (Arch.show arch)

+                 (KojiBuild.isScratch kbtype)

+                 (doFetchArtifact arch kbtype)

+             )

+         }

+ 

+ let generateRpmBuildJobs

+     : KojiBuild.Type → List Zuul.Job.Type

+     = λ(kbtype : KojiBuild.Type) →

+         let forBranch =

+               λ(branch : Branches.Type) →

+                 Prelude.List.map

+                   Arch.Type

+                   Zuul.Job.Type

+                   (generateRpmBuildJob kbtype branch)

+                   ( if    KojiBuild.isScratch kbtype

+                     then  Branches.arches branch

+                     else  [ Arch.Type.X86_64 ]

+                   )

+ 

+         in  Prelude.List.concatMap

+               Branches.Type

+               Zuul.Job.Type

+               forBranch

+               Branches.all

+ 

+ let getTestingFarmJobVars =

+       λ(branch : Branches.Type) →

+       λ(ttype : Text) →

+         let name =

+               merge

+                 { Rawhide = "Fedora-Rawhide"

+                 , F34 = "Fedora-34"

+                 , F35 = "Fedora-35"

+                 , F36 = "Fedora-36"

+                 , Epel8 = "CentOS-Stream-8"

+                 }

+                 branch

+ 

+         let distro =

+               merge

+                 { Rawhide = "fedora-rawhide"

+                 , F34 = "fedora-34"

+                 , F35 = "fedora-35"

+                 , F36 = "fedora-36"

+                 , Epel8 = "centos-stream-8"

+                 }

+                 branch

+ 

+         in  Zuul.Vars.object

+               [ { mapKey = "compose", mapValue = Zuul.Vars.string name }

+               , { mapKey = "distro", mapValue = Zuul.Vars.string distro }

+               , { mapKey = "test_type", mapValue = Zuul.Vars.string ttype }

+               ]

+ 

+ let generateSTITestsJob =

+       λ(branch : Branches.Type) →

+         Zuul.Job::{

+         , name = "rpm-sti-test"

+         , description = Some "Running sti test on Testing Farm"

+         , final = Some True

+         , branches = Branches.zuulBranch branch

+         , roles = Some

+           [ { zuul = "zuul-distro-jobs" }, { zuul = "fedora-zuul-jobs" } ]

+         , timeout = Some 21600

+         , nodeset = Some default_nodeset

+         , requires = Some [ "repo" ]

+         , secrets = Some

+           [ Zuul.Job.Secret::{

+             , name = "testing_farm_zuul_ci_key"

+             , secret = "testing_farm_zuul_ci_key"

+             }

+           ]

+         , run = Some "playbooks/testing-farm/run.yaml"

+         , post-run = Some [ "playbooks/testing-farm/fetch.yaml" ]

+         , vars = Some (getTestingFarmJobVars branch "sti")

+         }

+ 

+ let generateSTITestsJobs =

+       Prelude.List.map

+         Branches.Type

+         Zuul.Job.Type

+         generateSTITestsJob

+         Branches.all

+ 

+ let generateTMTTestsJob =

+       λ(branch : Branches.Type) →

+         Zuul.Job::{

+         , name = "rpm-tmt-test"

+         , description = Some "Running fmf test on Testing Farm"

+         , final = Some True

+         , branches = Branches.zuulBranch branch

+         , roles = Some

+           [ { zuul = "zuul-distro-jobs" }, { zuul = "fedora-zuul-jobs" } ]

+         , timeout = Some 21600

+         , nodeset = Some default_nodeset

+         , requires = Some [ "repo" ]

+         , secrets = Some

+           [ Zuul.Job.Secret::{

+             , name = "testing_farm_zuul_ci_key"

+             , secret = "testing_farm_zuul_ci_key"

+             }

+           ]

+         , run = Some "playbooks/testing-farm/run.yaml"

+         , post-run = Some [ "playbooks/testing-farm/fetch.yaml" ]

+         , vars = Some (getTestingFarmJobVars branch "fmf")

+         }

+ 

+ let generateTMTTestsJobs =

+       Prelude.List.map

+         Branches.Type

+         Zuul.Job.Type

+         generateTMTTestsJob

+         Branches.all

+ 

+ let Jobs =

+         [ clean_stalled_koji_tasks

+         , check_for_tests

+         , check_for_sti_tests

+         , check_for_fmf_tests

+         , check_for_arches

+         , common_koji_rpm_build

+         ]

+       # generateRpmBuildJobs KojiBuild.Type.Scratch

+       # generateRpmBuildJobs KojiBuild.Type.Final

+       # generateTMTTestsJobs

+       # generateSTITestsJobs

+ 

+ in  Zuul.Job.wrap Jobs

@@ -0,0 +1,166 @@ 

+ {-|

+ To compute the jobs.yaml run: make render

+ -}

+ let FZCI = ../package.dhall

+ 

+ let Zuul = FZCI.Zuul

+ 

+ let Prelude = FZCI.Prelude

+ 

+ let Nodesets = FZCI.Nodesets

+ 

+ let Branches = FZCI.Branches

+ 

+ let createJobVars

+     : Text → Zuul.Vars.Type

+     = λ(url : Text) →

+         Zuul.Vars.object

+           [ { mapKey = "repos"

+             , mapValue =

+                 Zuul.Vars.array

+                   [ Zuul.Vars.object

+                       [ { mapKey = "name"

+                         , mapValue = Zuul.Vars.string "distro-build"

+                         }

+                       , { mapKey = "url", mapValue = Zuul.Vars.string url }

+                       ]

+                   ]

+             }

+           ]

+ 

+ let createRpmInstallTestJob

+     : Branches.Type → Zuul.Job.Type

+     = λ(branch : Branches.Type) →

+         let getNodesetByBranch =

+               λ(branch : Branches.Type) →

+                 merge

+                   { Rawhide = Nodesets.getName Nodesets.Type.Fedora-Rawhide-VM

+                   , F34 = Nodesets.getName Nodesets.Type.Fedora-34-Container

+                   , F35 = Nodesets.getName Nodesets.Type.Fedora-35-VM-Medium

+                   , F36 = Nodesets.getName Nodesets.Type.Fedora-36-VM

+                   , Epel8 = Nodesets.getName Nodesets.Type.Centos-8-VM

+                   }

+                   branch

+ 

+         let job =

+               Zuul.Job::{

+               , name = "rpm-install-test"

+               , description = Some

+                   "Install the built rpms (${Branches.show branch} variant)"

+               , roles = Some [ { zuul = "zuul-distro-jobs" } ]

+               , requires = Some [ "repo" ]

+               , run = Some "playbooks/rpm/rpm-install-test.yaml"

+               , branches = Branches.zuulBranch branch

+               , nodeset = Some (getNodesetByBranch branch)

+               , vars = Some (createJobVars (Branches.kojirepos branch))

+               }

+ 

+         let job =

+               if    Branches.isEpel8 branch

+               then  job ⫽ { pre-run = Some [ "playbooks/epel8/install.yaml" ] }

+               else  job

+ 

+         in  job

+ 

+ let createRpmTestJob

+     : Branches.Type → Zuul.Job.Type

+     = λ(branch : Branches.Type) →

+         let getNodesetByBranch =

+               λ(branch : Branches.Type) →

+                 merge

+                   { Rawhide =

+                       Nodesets.getName Nodesets.Type.Fedora-Rawhide-VM-Medium

+                   , F34 = Nodesets.getName Nodesets.Type.Fedora-34-VM-Medium

+                   , F35 = Nodesets.getName Nodesets.Type.Fedora-35-VM-Medium

+                   , F36 = Nodesets.getName Nodesets.Type.Fedora-36-VM-Medium

+                   , Epel8 = Nodesets.getName Nodesets.Type.Centos-8-VM

+                   }

+                   branch

+ 

+         let job =

+               Zuul.Job::{

+               , name = "rpm-test"

+               , description = Some

+                   "Install the built rpms and run STI tests (${Branches.show

+                                                                  branch} variant)"

+               , roles = Some [ { zuul = "zuul-distro-jobs" } ]

+               , timeout = Some 18000

+               , requires = Some [ "repo" ]

+               , run = Some "playbooks/rpm/rpmtest.yaml"

+               , post-run = Some [ "playbooks/rpm/rpmtest-fetch-artifacts.yaml" ]

+               , branches = Branches.zuulBranch branch

+               , nodeset = Some (getNodesetByBranch branch)

+               , vars = Some (createJobVars (Branches.kojirepos branch))

+               }

+ 

+         let job =

+               if    Branches.isEpel8 branch

+               then  job ⫽ { pre-run = Some [ "playbooks/epel8/install.yaml" ] }

+               else  job

+ 

+         in  job

+ 

+ let createRpmInspectJob

+     : Branches.Type → Zuul.Job.Type

+     = λ(branch : Branches.Type) →

+         let job =

+               Zuul.Job::{

+               , name = "rpm-rpminspect"

+               , description = Some

+                   "Run the rpminspect report tests (${Branches.show

+                                                         branch} variant)"

+               , parent = Some "rpminspect"

+               , requires = Some [ "repo" ]

+               , branches = Branches.zuulBranch branch

+               , nodeset = Some

+                   (Nodesets.getName Nodesets.Type.Fedora-34-Container)

+               , vars = Some

+                   ( Zuul.Vars.object

+                       [ { mapKey = "rpminspect_cmd"

+                         , mapValue = Zuul.Vars.string "rpminspect-fedora"

+                         }

+                       , { mapKey = "check_previous_build_enabled"

+                         , mapValue = Zuul.Vars.bool True

+                         }

+                       , { mapKey = "target"

+                         , mapValue = Zuul.Vars.string (Branches.target branch)

+                         }

+                       ]

+                   )

+               }

+ 

+         in  job

+ 

+ let rpm_linter_job =

+       Zuul.Job::{

+       , name = "rpm-linter"

+       , parent = Some "repo-rpm-lint"

+       , requires = Some [ "repo" ]

+       , nodeset = Some (Nodesets.getName Nodesets.Type.Fedora-34-Container)

+       , branches = Some Branches.allText

+       }

+ 

+ let rpm_install_test_jobs =

+       Prelude.List.map

+         Branches.Type

+         Zuul.Job.Type

+         createRpmInstallTestJob

+         Branches.all

+ 

+ let rpm_test_jobs =

+       Prelude.List.map Branches.Type Zuul.Job.Type createRpmTestJob Branches.all

+ 

+ let rpminspect_jobs =

+       Prelude.List.map

+         Branches.Type

+         Zuul.Job.Type

+         createRpmInspectJob

+         Branches.all

+ 

+ let jobs =

+         rpm_install_test_jobs

+       # rpm_test_jobs

+       # rpminspect_jobs

+       # [ rpm_linter_job ]

+ 

+ in  Zuul.Job.wrap jobs

@@ -0,0 +1,18 @@ 

+ {-|

+ To compute the jobs.yaml run: make render

+ -}

+ let FZCI = ../package.dhall

+ 

+ let Prelude = FZCI.Prelude

+ 

+ let Zuul = FZCI.Zuul

+ 

+ let Nodesets = FZCI.Nodesets

+ 

+ in  Zuul.Nodeset.wrap

+       ( Prelude.List.map

+           Nodesets.Type

+           Zuul.Nodeset.Type

+           Nodesets.toNodeset

+           Nodesets.all

+       )

@@ -0,0 +1,143 @@ 

+ {-|

+ To compute the jobs.yaml run: make render

+ -}

+ let FZCI = ../package.dhall

+ 

+ let Prelude = FZCI.Prelude

+ 

+ let Zuul = FZCI.Zuul

+ 

+ let build =

+       let createSBJobsForArches =

+             Prelude.List.map

+               FZCI.Arches.Type

+               Zuul.Job.Union

+               ( λ(arch : FZCI.Arches.Type) →

+                   if    FZCI.Arches.isX86_64 arch

+                   then  Zuul.Job.Name "rpm-scratch-build"

+                   else  Zuul.Job.Inline

+                           Zuul.Job::{

+                           , name = "rpm-scratch-build-${FZCI.Arches.show arch}"

+                           , voting = Some False

+                           }

+               )

+               FZCI.Arches.fedora

+ 

+       in  toMap

+             { name = Zuul.ProjectTemplate.Name "build"

+             , check =

+                 Zuul.ProjectTemplate.Pipeline

+                   Zuul.Project.PipelineConfig::{

+                   , jobs =

+                         [ Zuul.Job.Name "check-for-arches" ]

+                       # createSBJobsForArches

+                   }

+             }

+ 

+ let lint =

+       toMap

+         { name = Zuul.ProjectTemplate.Name "lint"

+         , check =

+             Zuul.ProjectTemplate.Pipeline

+               Zuul.Project.PipelineConfig::{

+               , jobs =

+                 [ Zuul.Job.Inline

+                     Zuul.Job::{

+                     , name = "rpm-linter"

+                     , dependencies = Some

+                       [ Zuul.Job.Dependency.Name "rpm-scratch-build" ]

+                     }

+                 , Zuul.Job.Inline

+                     Zuul.Job::{

+                     , name = "rpm-rpminspect"

+                     , voting = Some False

+                     , dependencies = Some

+                       [ Zuul.Job.Dependency.Name "rpm-scratch-build" ]

+                     }

+                 ]

+               }

+         }

+ 

+ let test =

+       toMap

+         { name = Zuul.ProjectTemplate.Name "test"

+         , check =

+             Zuul.ProjectTemplate.Pipeline

+               Zuul.Project.PipelineConfig::{

+               , jobs =

+                 [ Zuul.Job.Inline

+                     Zuul.Job::{

+                     , name = "check-for-tests"

+                     , dependencies = Some

+                       [ Zuul.Job.Dependency.Name "rpm-scratch-build" ]

+                     }

+                 , Zuul.Job.Inline

+                     Zuul.Job::{

+                     , name = "check-for-sti-tests"

+                     , dependencies = Some

+                       [ Zuul.Job.Dependency.Name "rpm-scratch-build" ]

+                     }

+                 , Zuul.Job.Inline

+                     Zuul.Job::{

+                     , name = "check-for-fmf-tests"

+                     , dependencies = Some

+                       [ Zuul.Job.Dependency.Name "rpm-scratch-build" ]

+                     }

+                 , Zuul.Job.Inline

+                     Zuul.Job::{

+                     , name = "rpm-install-test"

+                     , dependencies = Some

+                       [ Zuul.Job.Dependency.Name "rpm-scratch-build" ]

+                     }

+                 , Zuul.Job.Inline

+                     Zuul.Job::{

+                     , name = "rpm-test"

+                     , dependencies = Some

+                       [ Zuul.Job.Dependency.Name "check-for-tests"

+                       , Zuul.Job.Dependency.Name "rpm-scratch-build"

+                       ]

+                     }

+                 , Zuul.Job.Inline

+                     Zuul.Job::{

+                     , name = "rpm-tmt-test"

+                     , dependencies = Some

+                       [ Zuul.Job.Dependency.Name "check-for-fmf-tests"

+                       , Zuul.Job.Dependency.Name "rpm-scratch-build"

+                       ]

+                     }

+                 , Zuul.Job.Inline

+                     Zuul.Job::{

+                     , name = "rpm-sti-test"

+                     , dependencies = Some

+                       [ Zuul.Job.Dependency.Name "check-for-sti-tests"

+                       , Zuul.Job.Dependency.Name "rpm-scratch-build"

+                       ]

+                     }

+                 ]

+               }

+         }

+ 

+ let publish =

+       toMap

+         { name = Zuul.ProjectTemplate.Name "publish"

+         , gate =

+             Zuul.ProjectTemplate.Pipeline

+               Zuul.Project.PipelineConfig::{ jobs = [ Zuul.Job.Name "noop" ] }

+         , promote =

+             Zuul.ProjectTemplate.Pipeline

+               Zuul.Project.PipelineConfig::{

+               , jobs = [ Zuul.Job.Name "rpm-build" ]

+               }

+         }

+ 

+ let testTestsDef =

+       toMap

+         { name = Zuul.ProjectTemplate.Name "test-tests-def"

+         , check =

+             Zuul.ProjectTemplate.Pipeline

+               Zuul.Project.PipelineConfig::{

+               , jobs = [ Zuul.Job.Name "rpm-test" ]

+               }

+         }

+ 

+ in  Zuul.ProjectTemplate.wrap [ build, lint, test, publish, testTestsDef ]

file modified
+19 -13
@@ -1,25 +1,31 @@ 

  export FZCI = $(shell pwd)/FZCI.dhall/package.dhall

  

  define generate_file

- 	@echo "Rendering $(1)"

- 	@dhall-to-yaml --generated-comment --file $(1).dhall | python3 -mzuulfmt > $(1).yaml

+ 	@echo "Rendering FZCI.dhall/$(1).dhall in $(2).yaml"

+ 	@dhall-to-yaml --generated-comment --file FZCI.dhall/$(1).dhall | zuulfmt > $(2).yaml

  endef

  

- all: render install-req

- 

- render:

+ render: check-tools

  	@dhall hash --file $$FZCI > /dev/null

- 	$(call generate_file,../fedora-zuul-jobs-config/zuul.d/jobs)

- 	$(call generate_file,../fedora-zuul-jobs/zuul.d/jobs)

- 	$(call generate_file,../fedora-zuul-jobs/zuul.d/nodesets)

- 	$(call generate_file,../fedora-zuul-jobs/zuul.d/templates)

+ 	$(call generate_file,fedora-zuul-jobs-config/jobs,../fedora-zuul-jobs-config/zuul.d/jobs)

+ 	$(call generate_file,fedora-zuul-jobs/jobs,../fedora-zuul-jobs/zuul.d/jobs)

+ 	$(call generate_file,fedora-zuul-jobs/nodesets,../fedora-zuul-jobs/zuul.d/nodesets)

+ 	$(call generate_file,fedora-zuul-jobs/templates,../fedora-zuul-jobs/zuul.d/templates)

  

  define clone

- 	@test -d ../$(1)/.git || git clone https://pagure.io/$(1).git ../$(1)/

+ 	@test -d ../$(1)/.git || git clone https://pagure.io/$(1).git ../$(1)/ && \

+ 		pushd ../$(1)/ && git checkout master && git fetch origin && \

+ 		git reset --hard origin/master && popd

  endef

  

- install-req:

- 	@type -p dhall-to-yaml > /dev/null || sudo dnf install -y dhall-json

- 	@python3 -mzuulfmt --help > /dev/null || python3 -mpip install --user zuulfmt

+ check-updates:

+ 	@pushd ../fedora-zuul-jobs-config && git status && popd

+ 	@pushd ../fedora-zuul-jobs && git status && popd

+ 

+ update-jobs-repos:

  	$(call clone,fedora-zuul-jobs-config)

  	$(call clone,fedora-zuul-jobs)

+ 

+ check-tools:

+ 	@type -p dhall-to-yaml 2> /dev/null || echo "Unable to find dhall-to-yaml in your PATH"

+ 	@type -p zuulfmt 2> /dev/null || "Unable to find zuulfmt in your PATH"

file modified
+91 -18
@@ -4,34 +4,107 @@ 

  

  Here are configured:

  

- - resources/: Git repositories to be listened by softwarefactory-project.io's Zuul.

- - zuul.d/: Zuul pipelines and base Zuul jobs for Fedora.

- - playbooks/: Ansible playbooks used by Zuul jobs.

- - tools/: Some addtionnal tooling.

- - FZCI.dhall: [Dhall][dhall-lang] description of Fedora Resources to be used in Zuul job configuration.

+ - resources/: Git repositories to be listened for events by Fedora's Zuul.

+ - zuul.d/_pipelines.yaml: Zuul pipelines for the Fedora's Zuul jobs.

+ - playbooks/: Ansible playbooks used by some Fedora's Zuul jobs.

+ - tools/: Addtionnal tooling.

+ - FZCI.dhall: [Dhall][dhall-lang] description of Fedora Zuul resources to be used to render Fedora's Zuul jobs configuration.

  

- ## Prepare your workspace to edit FZCI Dhall config

+ The Zuul CI configuration for Fedora is managed through 3 repositories:

  

- ### Clone sources

+ - `fedora-project-config`: this one.

+ - `fedora-zuul-jobs-config`: A [Zuul config](https://zuul-ci.org/docs/zuul/latest/glossary.html?highlight=trusted#term-config-project) repository to host sensitive Zuul config (e.g. jobs relying on secrets).

+ - `fedora-zuul-jobs`: A [Zuul untrusted](https://zuul-ci.org/docs/zuul/latest/glossary.html?highlight=trusted#term-untrusted-project) repository to host

+ non sensitive Zuul config.

  

- Run the following command to install dependencies.

+ Most of the Zuul config for Fedora is managed via the Dhall language. The rendering of the

+ dhall program produces YAML files to be used by Zuul. YAML files are generated into the

+ three repositories described above.

+ 

+ ## Prepare your workspace to edit Fedora Zuul jobs config

+ 

+ ### Install required tools

+ 

+ `dhall-to-yaml` and `zuulfmt` are needed to render the Zuul configuration.

+ 

+ To install `dhall-to-yaml` runs (or adapt to your working environment)

+ 

+ ```

+ dnf install -y dhall-json

+ ```

+ 

+ To install `zuulfmt` runs (or adapt to your working environment)

+ 

+ ```

+ python -mpip install --user zuulfmt

+ ```

+ 

+ It is adviced to install the Dhall lang support for your IDE:

+ 

+ - [VSCode plugins](https://marketplace.visualstudio.com/items?itemName=dhall.vscode-dhall-lsp-server)

+ 

+ ### Clone/Update jobs repositories

+ 

+ Run the following command to install (and/or reset to the last origin/HEAD) extra repositories.

+ This prepares `fedora-zuul-jobs-config` and `fedora-zuul-jobs` checkout in the parent directory.

  

  ```ShellSession

- make install-req

+ make update-jobs-repos

+ ```

+ 

+ Then:

+ 

+ - Fork both repositories in your pagure.io user namesapce

+ - Use `git remote add fork ssh://git@pagure.io/forks/<username>/<repo>.git`

+ 

+ 

+ ## How to update Zuul jobs config

+ 

+ 

+ First, ensure repositories are in the right state (all commands below assume working from the master branch):

+ 

+ ```

+ git checkout master && git fetch origin && git reset --hard origin/master

+ make update-jobs-repos

  ```

  

- ### Run the rendering

+ Second, add the required changes.

+ 

+ Then run the rendering.

  

  ```ShellSession

- make render

+ [fedora@localhost fedora-project-config]$ make render

+ /nix/store/h1p77fzddhjwb9fl678p72hl4hg8d5l3-ghc-8.10.7-with-packages/bin/dhall-to-yaml

+ /home/fedora/bin/zuulfmt

+ Rendering FZCI.dhall/fedora-zuul-jobs-config/jobs.dhall in ../fedora-zuul-jobs-config/zuul.d/jobs.yaml

+ Rendering FZCI.dhall/fedora-zuul-jobs/jobs.dhall in ../fedora-zuul-jobs/zuul.d/jobs.yaml

+ Rendering FZCI.dhall/fedora-zuul-jobs/nodesets.dhall in ../fedora-zuul-jobs/zuul.d/nodesets.yaml

+ Rendering FZCI.dhall/fedora-zuul-jobs/templates.dhall in ../fedora-zuul-jobs/zuul.d/templates.yaml

+ ```

+ 

+ In case of error, update your changes then run again the rendering. 

+ 

+ Finally, create Pull Requests for each repositories that got an update from the rendering.

+ 

+ For instance:

+ 

  ```

+ make check-updates

+ # This shows a change in fedore-zuul-jobs then

+ pushd ../fedora-zuul-jobs

+ git status

+ git add -A .

+ git commit -m"Add change for ..."

+ git push fork master:pr-branch-on-fork

+ ```

+ 

+ And open the PR on pagure.io.

  

- Note that it is adviced to get a support for Dhall Lang into your IDE but at least you must have

- installed the dhall commands on you system.

+ ## HOWTOs

  

- ## Adding support for a Fedora Branch

+ ### Add support for a Fedora Branch

  

- ### Adding new branch definition

+ #### Adding new branch definition

  

  - Edit file [branches.dhall](FZCI.dhall/Branches.dhall) in order to add the <new-branch>.

  - The type checker will help you to fill the places where the <new-branch> handling must
@@ -45,7 +118,7 @@ 

  dhall <<< './FZCI.dhall/Branches.dhall'

  ```

  

- ### Update jobs from fedora-zuul-jobs-config

+ #### Update jobs from fedora-zuul-jobs-config

  

  In fedora-zuul-jobs-config

  
@@ -61,7 +134,7 @@ 

  # or run: make render

  ```

  

- ### Update jobs from fedora-zuul-jobs

+ #### Update jobs from fedora-zuul-jobs

  

  In fedora-zuul-jobs

  
@@ -75,7 +148,7 @@ 

  # or run: make render

  ```

  

- ### Push changes

+ #### Push changes

  

  When edits are done on those repos then commits and push changes (as PR) then wait

  for CI result.

no initial comment

Build succeeded.

1 new commit added

  • Add check-updates make command
4 months ago

Build succeeded.

1 new commit added

  • Import fedora-zuul-jobs-config/zuul.d/jobs.dhall
4 months ago

Build succeeded.

4 new commits added

  • Import from fedora-zuul-jobs
  • Import fedora-zuul-jobs-config/zuul.d/jobs.dhall
  • Add check-updates make command
  • Improve README.md
4 months ago

Build succeeded.

Metadata Update from @fbo:
- Pull-request tagged with: gateit

4 months ago

Build succeeded (gate pipeline).

Pull-Request has been merged by zuul

4 months ago