From 576cbd287d83e4dcbcc4a3212dd99083063e266f Mon Sep 17 00:00:00 2001 From: Patrik Polakovič Date: Aug 08 2025 14:09:36 +0000 Subject: Reorganize Release Engineering docs structure Signed-off-by: Patrik Polakovič --- diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 3037ff5..76fb9ae 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -35,38 +35,41 @@ *** xref:sysadmin_guide:index.adoc#_standard_operating_procedures[Standard Operation Procedures] *** xref:sysadmin_guide:index.adoc#_howtos[HOWTOs] * xref:release_guide:index.adoc[Release Engineering] -** xref:release_guide:release_process.adoc[Release process] -*** xref:release_guide:create_release_signing_key.adoc[Create Release Signing Keys] -*** xref:release_guide:mass_rebuild.adoc[Mass Rebuild] -*** xref:release_guide:sop_file_ftbfs.adoc[File FTBFS bugs] -*** xref:release_guide:mass_branching_checklist.adoc[Mass Branching Checklist] +** xref:release_guide:philosophy.adoc[Philosophy] +** xref:release_guide:release_process.adoc[Release Process] +*** xref:release_guide:sop_mass_rebuild.adoc[Mass Rebuild] +*** xref:release_guide:sop_file_ftbfs.adoc[File FTBFS Bugs] +*** xref:release_guide:sop_create_release_signing_key.adoc[Create Release Signing Keys] +*** xref:release_guide:sop_re-sign_rawhide_content.adoc[Re-Sign Rawhide Content With Rawhide+1 Key] +*** xref:release_guide:sop_retire_orphaned_packages.adoc[Retire Orphaned and Long-Time FTBFS Rawhide Packages] +*** xref:release_guide:sop_mass_branching_checklist.adoc[Mass Branching Checklist] *** xref:release_guide:sop_mass_branching.adoc[Mass Branching] +*** xref:release_guide:sop_post-branch_freeze.adoc[Post-Branch Freeze] *** xref:release_guide:sop_epel_minor_mass_branching.adoc[EPEL Minor Version Mass Branching] -*** xref:release_guide:beta_freeze.adoc[Beta freeze and updates-testing activation] -*** xref:release_guide:beta_RC_compose.adoc[Beta RC compose] -*** xref:release_guide:beta_release.adoc[Beta release] -*** xref:release_guide:final_freeze.adoc[Final freeze] -*** xref:release_guide:final_RC_compose.adoc[Final RC compose] -*** xref:release_guide:final_release.adoc[Final release] -*** xref:release_guide:release_post_cleanup.adoc[Release Post Cleanup] -*** xref:release_guide:release_eol.adoc[End Of Life] -*** xref:release_guide:torrentrelease.adoc[Torrent releases] -** xref:releng_misc_guide:index.adoc[Releng Miscellaneous guide] -*** xref:releng_misc_guide:sop_openh264[Generating OpenH264 RPMs] -*** xref:releng_misc_guide:sop_eol_change.adoc[Adjust EOLs and SLs on branches] +*** xref:release_guide:sop_beta_freeze.adoc[Beta Freeze] +*** xref:release_guide:sop_beta_RC_compose.adoc[Beta RC Compose] +*** xref:release_guide:sop_beta_release.adoc[Beta Release] +*** xref:release_guide:sop_torrent_releases.adoc[Torrent Releases] +*** xref:release_guide:sop_final_freeze.adoc[Final Freeze] +*** xref:release_guide:sop_final_RC_compose.adoc[Final RC Compose] +*** xref:release_guide:sop_final_release.adoc[Final Release] +*** xref:release_guide:sop_post-release_cleanup.adoc[Post-Release Cleanup] +*** xref:release_guide:sop_release_eol.adoc[End Of Life] +** xref:releng_misc_guide:index.adoc[Miscellaneous SOPs] +*** xref:releng_misc_guide:sop_unretire.adoc[Unretire a Package] +*** xref:releng_misc_guide:sop_package_blocking.adoc[Block a Package] +*** xref:releng_misc_guide:sop_package_unblocking.adoc[Unblock a Package] +*** xref:releng_misc_guide:sop_eol_change.adoc[Adjust EOLs and SLs on Branches] +*** xref:releng_misc_guide:sop_openh264.adoc[Generate OpenH264 RPMs] *** xref:releng_misc_guide:sop_fedora_media_writer.adoc[Fedora Media Writer Building and Signing] *** xref:releng_misc_guide:sop_find_module_info.adoc[Find Module Information] -*** xref:releng_misc_guide:sop_unretire.adoc[Unretire package branch] *** xref:releng_misc_guide:sop_update_critpath.adoc[Update Critpath] -*** xref:releng_misc_guide:sop_remote_dist_git_branches.adoc[Remove dist-git branches] -*** xref:releng_misc_guide:sop_remote_tarballs_dist_git.adoc[Remove tarballs from dist-git branches] -*** xref:releng_misc_guide:sop_package_unblocking.adoc[Package Unblocking] -*** xref:releng_misc_guide:sop_retire_orphaned_packages.adoc[Retire Orphaned Packages] -*** xref:releng_misc_guide:sop_requesting_task_automation_users.adoc[Requesting Automation Users] -*** xref:releng_misc_guide:sop_process_dist_git_requests.adoc[Process fedora-scm-requests tickets] -*** xref:releng_misc_guide:sop_signing_builds.adoc[Sign the packages] -*** xref:releng_misc_guide:sop_clean_amis.adoc[Clean AMIs Process] -*** xref:releng_misc_guide:sop_pushing_updates.adoc[Pushing Updates] -*** xref:releng_misc_guide:sop_package_blocking.adoc[Package Blocking] +*** xref:releng_misc_guide:sop_remote_dist_git_branches.adoc[Remove Dist-Git Branches] +*** xref:releng_misc_guide:sop_remote_tarballs_dist_git.adoc[Remove Tarballs from Dist-Git Repositories] +*** xref:releng_misc_guide:sop_request_an_automation_user.adoc[Request an Automation User] +*** xref:releng_misc_guide:sop_process_dist_git_requests.adoc[Process `fedora-scm-requests` Tickets] +*** xref:releng_misc_guide:sop_signing_builds.adoc[Sign the Packages] +*** xref:releng_misc_guide:sop_clean_amis.adoc[Clean AMIs] +*** xref:releng_misc_guide:sop_pushing_updates.adoc[Push Updates] *** xref:releng_misc_guide:sop_sigul_client_setup.adoc[Sigul Client Setup] -** xref:releng_misc_guide:troubleshooting.adoc[Fedora Release Engineering Troubleshooting Guide] +*** xref:releng_misc_guide:troubleshooting.adoc[General Troubleshooting Guide] diff --git a/modules/release_guide/pages/beta_RC_compose.adoc b/modules/release_guide/pages/beta_RC_compose.adoc deleted file mode 100644 index 022116b..0000000 --- a/modules/release_guide/pages/beta_RC_compose.adoc +++ /dev/null @@ -1,155 +0,0 @@ -include::_partials/attributes.adoc[] - -= Beta Release candidate - -When Quality Engineering (QE) requests a Release Candidate (RC) they do -so by opening an issue in the releng repository on pagure. Release -candidate composes are not currently automated. - -[cols=",,",options="header",] -|=== -|Compose Name |Configuration File |Compose Script -|Beta |fedora-beta.conf |release-candidate.sh -|=== - -== Action - -=== Create the fedora-beta.conf configuration file using the script - -The script can be found https://pagure.io/pungi-fedora/blob/main/f/create-candidate-configs.py[here]. - -There is a mandatory argument to specify the milestone to generate (beta or final) -and optional arguments for the major and minor versions for the label. - -Example usage: -`./create-candidate-configs.py --minor=5 beta` - -=== Review Compose Tags - -. List any pre-existing builds in the current compose tag -+ -[source,subs="attributes+"] -.... -$ koji list-tagged f{branched}-compose -.... -. Verify preexisting builds are in compose tags -+ -The tagged builds from the previous composes should all be present in -the output from the previous step. Consult the request ticket for the -list of builds expected in this output. The request ticket may list -commands to un-tag existing tagged builds and tag the new requested -builds; if so, you can review these commands and run them. Otherwise, -the steps to do this manually are documented below. -+ -[NOTE] -==== -The very first run of a Beta or GA compose should have no builds -listed under the compose tag. It is important to clear preexisting -builds from the compose tag when moving between the Beta and RC -composes. Verify that these builds were removed. - -[source,subs="attributes+"] -.... -$ koji list-tagged f{branched}-compose -$ koji untag-build --all f{branched}-compose [build1 build2 ...] -.... -==== -+ -[NOTE] -==== -The order in which packages are added into the -f{branched}-compose tag matter. If the builds are untagged -erroneously then special attention should be given to adding them back -correctly. -==== -. Add builds specified by QE to the current compose tag -+ -[source,subs="attributes+"] -.... -$ koji tag-build f{branched}-compose [build1 build2 ...] -.... -+ -[NOTE] -==== -These steps may be completed on a local machine as long as the user has -appropriate permissions in the Koji tool. -==== - -=== Running the Compose - -. Composes use a configuration file to construct the compose. -Each compose uses its own configuration. -The `global_release` variable should start from 1.1 and the second number -should increment each time a new compose is created. - -* Beta - `fedora-beta.conf` -. Log into the compose backend -+ -.... -$ ssh compose-x86-01.phx2.fedoraproject.org -.... -. Open a screen session -+ -.... -$ screen -.... -. Obtain the pungi-fedora branch for the current compose -+ -The first time any user account executes a compose the pungi-fedora git -repository must be cloned. The compose candidate script that invokes -pungi should be run from `compose-x86-01.rdu3.fedoraproject.org`. -+ -.... -$ git clone ssh://git@pagure.io/pungi-fedora.git -.... -+ -Enter the pungi-fedora directory. -+ -.... -$ cd pungi-fedora -.... -+ -If the clone step above was not required then fully update the existing -repository checkout from pagure. -+ -[source,subs="attributes+"] -.... -$ git fetch origin -$ git checkout f{branched} -$ git pull origin f{branched} -.... -. Run the compose and redirect the output to a file (in case it's needed later) -+ -[source,subs="attributes+"] -.... -$ sudo ./release-candidate.sh {branched} Beta-#.# >& {branched}_Beta-#.#.out -# Example: -# $ sudo ./release-candidate.sh {branched} Beta-1.3 >& {branched}_Beta-1.3.out -.... -+ -The first argument is the release number. The second argument is the -compose label. The numbering scheme begins with 1.1 and the second -number is incremented after each compose. The first number always -remains at 1 for Fedora composes. -+ -[NOTE] -==== -productmd requires the label version to be in this format (because -both numbers may be incremented for other products that use productmd -metadata, and have different meanings). It is because of this that -composes always start with the number 1 and the second number is -incremented with each compose. -==== -+ -[NOTE] -==== -If the compose fails with a directory missing error, then create the -compose directory with `mkdir /mnt/koji/compose/{branched}` -==== - -=== Verification - -The method for verifying a compose has completed is checking -`/mnt/koji/compose/{branched}/[compose_dir]/STATUS`. Any status -other than DOOMED is OK. - diff --git a/modules/release_guide/pages/beta_freeze.adoc b/modules/release_guide/pages/beta_freeze.adoc deleted file mode 100644 index a841f7d..0000000 --- a/modules/release_guide/pages/beta_freeze.adoc +++ /dev/null @@ -1,135 +0,0 @@ -include::_partials/attributes.adoc[] - -= Beta freeze and updates-testing activation point - -== Description - -This SOP covers two distinct events that happen at the same time. Two weeks after -https://docs.pagure.org/releng/sop_mass_branching.html[Mass Branching], at 14:00 UTC, the Beta freeze is imposed, -and use of updates-testing is enabled for the new release. - -== Action - -=== Making koji changes - -Make the following koji tag changes - -[source,subs="attributes+"] -.... -$ koji remove-tag-inheritance f{branched}-updates-candidate f{branched} -$ koji remove-tag-inheritance f{branched}-updates-testing f{branched} -$ koji remove-tag-inheritance f{branched}-updates-pending f{branched} -$ koji remove-tag-inheritance f{branched}-override f{branched} -$ koji add-tag-inheritance f{branched}-updates-candidate f{branched}-updates -$ koji add-tag-inheritance f{branched}-updates-testing f{branched}-updates -$ koji add-tag-inheritance f{branched}-updates-pending f{branched}-updates -$ koji add-tag-inheritance f{branched}-override f{branched}-updates -$ koji edit-tag --perm=admin f{branched} -.... - -=== Bodhi configuration - -Configure Bodhi to create composes, not to automatically create updates, and to be frozen. -[source,subs="attributes+"] -.... -$ bodhi releases edit --name "F{branched}" --stable-tag f{branched} --testing-repository updates-testing --package-manager dnf --no-create-automatic-updates --composed-by-bodhi --state frozen -.... - -[NOTE] -==== -Add the container and flatpak releases if they weren't already added to -bodhi -==== - -=== Update Ansible vars - -Set the infra ansible https://pagure.io/fedora-infra/ansible/blob/main/f/vars/all/FedoraBranchedBodhi.yaml[_FedoraBranchedBodhi_ variable] to `prebeta` -and the https://pagure.io/fedora-infra/ansible/blob/main/f/vars/all/Frozen.yaml[_NextReleaseFrozen_ variable] to `True`. - -=== Run the playbooks - -.... -$ rbac-playbook openshift-apps/bodhi.yml -$ rbac-playbook groups/bodhi-backend.yml -$ rbac-playbook groups/koji-hub.yml -.... - -=== Send Announcement - -Email *devel-announce* and *test-announce* lists about Beta freeze and updates-testing activation. - -[source,subs="attributes+"] -.... -Hi all, - -Today's an important day on the Fedora Linux {branched} schedule [1], with -several significant cut-offs. First of all, today is the Bodhi -updates-testing activation point [2]. That means that from now all -Fedora Linux {branched} packages must be submitted to updates-testing and pass -the relevant requirements [3] before they will be marked as 'stable' and -moved to the Fedora Repository. - -Today is also the Beta freeze [4]. This means that only packages which -fix accepted blocker or freeze exception bugs [5][6] will be marked as -'stable' and included in the Beta composes. Other builds will remain in -updates-testing until the Beta release is approved, at which point the -Beta freeze is lifted and packages can move to 'stable' as usual until -the Final freeze. - -Today is also the Software String freeze [7], which means that strings -marked for translation in Fedora-translated projects should not now be -changed for Fedora Linux {branched}. - -Finally, today is the 'completion deadline' Change Checkpoint [8], -meaning that Fedora Linux {branched} Changes must now be 'feature complete or -close enough to completion that a majority of its functionality can be -tested'. All tracking bugs should be on ON_QA state or later to reflect -this. - -Regards, -Fedora Release Engineering - -[1] https://fedorapeople.org/groups/schedule/f-{branched}/f-{branched}-key-tasks.html -[2] https://fedoraproject.org/wiki/Updates_Policy#Bodhi_enabling -[3] https://fedoraproject.org/wiki/Updates_Policy#Branched_release -[4] https://fedoraproject.org/wiki/Milestone_freezes -[5] https://fedoraproject.org/wiki/QA:SOP_blocker_bug_process -[6] https://fedoraproject.org/wiki/QA:SOP_freeze_exception_bug_process -[7] https://fedoraproject.org/wiki/ReleaseEngineering/StringFreezePolicy -[8] https://fedoraproject.org/wiki/Changes/Policy - -.... - - -=== Verification - -Compare koji tagging structure with older release - -[source,subs="attributes+"] -.... -$ koji list-tag-inheritance {branched} --reverse -$ koji list-tag-inheritance {current} --reverse -.... - -Compare the bodhi release with older release - -[source,subs="attributes+"] -.... -$ bodhi releases info {branched} -$ bodhi releases info {current} -.... - -Check for other variants like container and flatpaks. - -== During the Beta freeze - -=== Process stable push requests - -During freezes we need to push updates that fix blocker and freeze exception -issues to stable when requested. -QA will file a ticket with the updates to push, which will include the -appropriate command(s). Verify the command(s) and run them. - -== Consider Before Running - -FIXME diff --git a/modules/release_guide/pages/beta_release.adoc b/modules/release_guide/pages/beta_release.adoc deleted file mode 100644 index 47d5e7c..0000000 --- a/modules/release_guide/pages/beta_release.adoc +++ /dev/null @@ -1,88 +0,0 @@ -include::_partials/attributes.adoc[] - -= Fedora Beta Release - -Based on the GO/NOGO meeting if the currnet compose is GO. The following steps need to be taken. - -== Wait for branched compose that matches the beta rc - -QE will typically request a stable push of all the updates that were in the Beta RC that -was signed off on. After that stable push we need to wait for a branched compose with them. -This ensures that the nightly branched has the same content as the beta we are about to release. -This is to allow us to make sure and tag all the right packages and have a common point. - -== Koji changes - -[source,subs="attributes+"] -.... -$ koji clone-tag --all --latest-only f{branched} f{branched}-Beta -.... - -== Stage release to mirrors - -[source,subs="attributes+"] -.... -$ sh scripts/stage-release.sh {branched}_Beta Fedora-{branched}-YYYYMMDD.0 _Beta-1.2 fedora-{branched} 1 -.... - -Please update YYYYMMDD in the above command accordingly. - -[NOTE] -==== -Make sure to grab the directory size usage numbers which is used to send -an email to [.title-ref]#mirror-admin@lists.fedoraproject.org# list. -==== - -== Sync the signed checksums to stage - -We need to sync the signed checksums to /pub/alt/stage/ by running the -following command - -[source,subs="attributes+"] -.... -$ for dir in Cloud Container Everything Kinoite Labs Sericea Server Silverblue Spins Workstation metadata; do sudo -u ftpsync rsync -avhH /mnt/koji/compose/{branched}/Fedora-{branched}-20221105.0/compose/$dir/ /pub/alt/stage/{branched}_RC-1.7/$dir/ --link-dest=/pub/fedora/linux/releases/{branched}/Everything/ --link-dest=/pub/alt/stage/{branched}_RC-1.2/Everything/ --link-dest=/pub/alt/stage/{branched}_RC-1.3/Everything --link-dest=/pub/alt/stage/{branched}_RC-1.4/Everything --link-dest=/pub/alt/stage/{branched}_RC-1.5/Everything --link-dest=/pub/alt/stage/{branched}_RC-1.6/Everything --link-dest=/pub/alt/stage/{branched}_RC-1.7/Everything; done -.... - -[NOTE] -==== -Replace the RC above with the RC that was 'go' -==== - -== Set release in Bodhi to pending - -[source,subs="attributes+"] -.... -$ bodhi releases edit --name F{branched} --state pending -.... - -=== Update Ansible vars - -Set the infra ansible https://pagure.io/fedora-infra/ansible/blob/main/f/vars/all/FedoraBranchedBodhi.yaml[_FedoraBranchedBodhi_ variable] to `postbeta` -and the https://pagure.io/fedora-infra/ansible/blob/main/f/vars/all/Frozen.yaml[_NextReleaseFrozen_ variable] to `False`. - -== Run the playbooks - -.... -$ sudo rbac-playbook groups/releng-compose.yml -$ sudo rbac-playbook groups/bodhi-backend.yml -$ sudo rbac-playbook openshift-apps/bodhi.yml -$ sudo rbac-playbook groups/koji-hub.yml -.... - -== Beta Installation and Cleanup - -This section outlines the steps that should be followed to ensure that older copies of Fedora betas are removed from the servers and that the correct permissions are set on the directories. - -=== Remove older betas - -This cleanup step is to remove the older betas from the `/pub/fedora-secondary/releases/test` and `/pub/fedora/linux/releases/test/` directories. This can be done using the rm command: - -[source,subs="attributes+"] -.... -$ rm -rf /pub/fedora-secondary/releases/test/{branched}_Beta -$ rm -rf /pub/fedora/linux/releases/test/{branched}_Beta -.... - -Note that {branched} should be replaced with the name of the beta release, such as 36_Beta, 37_Beta, etc. - -By following these steps, one can ensure that older copies of Fedora betas are removed from the servers and that the correct permissions are set on the directories. diff --git a/modules/release_guide/pages/create_release_signing_key.adoc b/modules/release_guide/pages/create_release_signing_key.adoc deleted file mode 100644 index e056e0c..0000000 --- a/modules/release_guide/pages/create_release_signing_key.adoc +++ /dev/null @@ -1,484 +0,0 @@ -= Create Release Signing Key - -== Description - -At the beginning of each release under development, a new package signing key is created. This key is used to prove the authenticity of packages built by Fedora and distributed by Fedora. This key will be used to sign all packages for the public test and final releases. - -== Action - -=== Sigul - -Sigul is the signing server that holds our keys. To make use of a new key, it must be created and access to the key granted. The `new-key`, `grant-key-access`, and `change-passphrase` commands are used. - -These are to be running on `bodhi-backend01` machine. - -[source, bash] ----- -$ ssh bodhi-backend01.rdu3.fedoraproject.org ----- - -[source, bash] ----- -$ sigul new-key --help -usage: client.py new-key [options] key - -Add a key - -options: - -h, --help show this help message and exit - --key-admin=USER Initial key administrator - --name-real=NAME_REAL - Real name of key subject - --name-comment=NAME_COMMENT - A comment about the key subject - --name-email=NAME_EMAIL - E-mail of key subject - --expire-date=YYYY-MM-DD - Key expiration date ----- - -[source, bash] ----- -$ sigul grant-key-access --help -usage: client.py grant-key-access key user - -Grant key access to a user - -options: - -h, --help show this help message and exit ----- - -[source, bash] ----- -$ sigul change-passphrase --help -usage: client.py change-passphrase key - -Change key passphrase - -options: - -h, --help show this help message and exit ----- - -For example, to create the Fedora 23 signing key, do the following: - -. Log into a system configured to run the sigul client. -. Create the key using a strong passphrase when prompted: - -[source, bash] ----- -$ sigul new-key --key-admin ausil --key-type gnupg \ - --gnupg-name-real Fedora \ - --gnupg-name-comment 23 \ - --gnupg-name-email fedora-23-primary@fedoraproject.org fedora-23 - ----- - -[source, bash] ----- -For EPEL - -$ sigul new-key --key-admin ausil --key-type gnupg \ - --gnupg-name-real "Fedora EPEL" \ - --gnupg-name-comment 7 \ - --gnupg-name-email epel@fedoraproject.org epel-7 ----- - -. Wait for entropy. This can take several minutes. -. For Fedora, also create the IMA signing key: - -[source, bash] ----- -$ sigul new-key --key-admin ausil --key-type ECC fedora-23-ima ----- - -. Grant key access to Fedora Account holders who will be signing packages and protect it with a temporary passphrase. For example, `CHANGEME`. Do the same with the -ima key for Fedora: - -[source, bash] ----- -$ sigul grant-key-access fedora-23 kevin ----- - -[NOTE] -==== -You can add --key-admin=true here to make a particular user key-admin for the key -==== - -[NOTE] -==== -**IMPORTANT:** Grant access to the autopen user as it's required for robosignatory autosigning and then restart the robosignatory service. -Make sure when granting key access you should use the autosign/robosignatory passphrase. - -==== - -. Provide the key name and temporary passphrase to signers. If they don't respond, revoke access until they are ready to change their passphrase. Signers can change their passphrase using the `change-passphrase` command: - -[source, bash] ----- -$ sigul change-passphrase fedora-23 ----- - -. When your sigul cert expires, run: - -[source, bash] ----- -certutil -d ~/.sigul -D -n sigul-client-cert ----- - -to remove the old cert, then - -[source, bash] ----- -sigul_setup_client ----- - -to add a new one. - -=== Adding Certificates for IMA Keys - -We now need to create an IMA certificate signed by our Sigul CA and get it to the kernel maintainers. This must be done after the IMA key is created. - -[source, bash] ----- -$ sigul -v -v sign-certificate fedorasigulca fedora-41-ima --issuer-certificate-name fedorasigulca --subject-certificate-name fedoraimafourtyone --validity 2y --certificate-type codesigning --subject "CN=Fedora 41 IMA Code-signing cert" > fedora-41-ima.pem ----- - -NOTE: Change the name from `41/fourtyone` to the appropriate release version. - -This .pem file then needs to be converted to a 'der' file (which is what IMA natively uses): - -openssl x509 -inform PEM -in fedora-44-ima.pem -outform DER -out fedora-44-ima.der - -The .der file should be added to fedora-repos package and the kernel should trust it -because it's signed by our CA. - -=== fedora-repos - -The `fedora-repos` package houses a copy of the public key information. This is used by RPM to verify the signature on files encountered. Currently, the `fedora-repos` package has a single key file named after the version of the key and the arch the key is for. - -To continue our example, the file would be named `RPM-GPG-KEY-fedora-27-primary`, which is the primary arch key for Fedora 27. To create this file, use the `get-public-key` command from sigul: - -[source, bash] ----- -$ sigul get-public-key fedora-27 > RPM-GPG-KEY-fedora-27-primary ----- - -Add this file to the repo and update the archmap file for the new release: - -[source, bash] ----- -$ git add RPM-GPG-KEY-fedora-27-primary ----- - -Then make a new `fedora-repos` build for rawhide (`FIXME: this should be its own SOP`). - -==== Web team SOP - -[source, bash] ----- -# from git repo root -cd fedoraproject.org/ -curl $KEYURL > /tmp/newkey -$EDITOR update-gpg-keys # Add key ID of the recently EOL'd version to obsolete_keys -./update-gpg-key /tmp/newkey -gpg static/fedora.gpg # used to verify the new keyring -# it should look something like this: -# pub 4096R/57BBCCBA 2009-07-29 Fedora (12) -# pub 4096R/E8E40FDE 2010-01-19 Fedora (13) -# pub 4096R/97A1071F 2010-07-23 Fedora (14) -# pub 1024D/217521F6 2007-03-02 Fedora EPEL -# sub 2048g/B6610DAF 2007-03-02 [expires: 2017-02-27] -# it must only have the two supported versions of Fedora, rawhide, and EPEL -# also verify that static/$NEWKEY.txt exists -$EDITOR data/content/{keys,verify}.html # see git diff 1840f96~ 1840f96 ----- - - -=== pungi-fedora - -The nightly compose configs come from the pungi-fedora project on https://pagure.io. We need to create a pull request to pull in the new key: - -[source, bash] ----- -$ git clone ssh://git@pagure.io//pungi-fedora.git -$ cd pungi-fedora -$ vim *conf - -$ git commit -m 'Add new key' -$ git push -$ file a Pull Request ----- - -=== Koji - -Koji has a garbage collection utility that finds builds that meet criteria to be removed to save space. Part of that criteria has to do with whether the build has been signed with a key. If the collection utility doesn't know about a key, it will ignore the build. Thus, as we create new keys, we need to inform the utility of these keys, or else builds can pile up. The configuration for the garbage collection lives within Ansible. - -On a clone of the infrastructure Ansible git repo, edit the `roles/koji_hub/templates/koji-gc.conf.j2` file: - -[source, diff] ----- -diff --git a/roles/koji_hub/templates/koji-gc.conf.j2 b/roles/koji_hub/templates/koji-gc.conf.j2 -index 9ecb750..9c48a8e 100644 ---- a/roles/koji_hub/templates/koji-gc.conf.j2 -+++ b/roles/koji_hub/templates/koji-gc.conf.j2 -@@ -35,6 +35,7 @@ key_aliases = - 81B46521 fedora-24 - FDB19C98 fedora-25 - 64DAB85D fedora-26 -+ F5282EE4 fedora-27 - fedora-extras - redhat-beta - fedora-epel - @@ -91,6 +93,7 @@ policy = - sig fedora-24 && age < 12 weeks :: keep - sig fedora-25 && age < 12 weeks :: keep - sig fedora-26 && age < 12 weeks :: keep - + sig fedora-27 && age < 12 weeks :: keep - sig fedora-epel && age < 12 weeks :: keep - sig fedora-epel-6 && age < 12 weeks :: keep - sig fedora-epel-7 && age < 12 weeks :: keep - ----- - -In this case the fedora-epel key was added to the list of key aliases, then -referenced in the list of unprotected_keys, and finally a policy was created -for how long to keep builds signed with this key. - -Once you've made your change commit and push. The buildsystem will pick up -this change the next time puppet refreshes. - -== Verification - -We can verify that the key was created in sigul, the correct users have access -to the key, the key was added to the fedora-release package, that the website -was updated with the right key, that sigulsign_unsigned was properly updated, -and that the key was successfully updated to the public key servers. - -=== Sigul - -Use the ``list-keys`` command to verify that the key was indeed added to sigul: - -``` -$ sigul list-keys -Administrator's password: -fedora-10 -fedora-10-testing -fedora-11 -fedora-12 -fedora-13 -``` - -Our new key should be on the list. This command expects **your** -administrative password. - -Use the ``list-key-users`` command to verify all the signers have access: - -``` -$ sigul list-key-users fedora-13 -Key passphrase: -jkeating -jwboyer -``` - -This command expects **your** key passphrase for the key in question. - -=== fedora-release - -To verify that the key was added to this package correctly, download the latest -build from koji and run rpm2cpio on it, then run gpg on the key file: - -``` -$ koji download-build --arch noarch --latest f27 fedora-repos -fedora-repos-rawhide-27-0.1.noarch.rpm | 7.3 kB 00:00:00 -fedora-repos-27-0.1.noarch.rpm | 87 kB 00:00:00 -$ rpmdev-extract fedora-repos-27-0.1.noarch.rpm -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-27-fedora -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-10-i386 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-10-ppc -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-10-ppc64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-10-primary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-10-x86_64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-11-i386 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-11-ppc -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-11-ppc64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-11-primary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-11-x86_64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-12-i386 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-12-ppc -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-12-ppc64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-12-primary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-12-x86_64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-13-arm -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-13-armhfp -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-13-i386 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-13-mips -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-13-primary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-13-secondary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-13-x86_64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-14-i386 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-14-primary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-14-x86_64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-15-arm -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-15-armhfp -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-15-i386 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-15-ppc -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-15-ppc64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-15-primary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-15-s390 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-15-s390x -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-15-secondary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-15-x86_64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-16-arm -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-16-armhfp -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-16-i386 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-16-ppc -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-16-ppc64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-16-primary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-16-s390 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-16-s390x -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-16-secondary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-16-x86_64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-17-arm -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-17-armhfp -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-17-i386 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-17-ppc -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-17-ppc64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-17-primary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-17-s390 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-17-s390x -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-17-secondary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-17-x86_64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-18-arm -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-18-armhfp -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-18-i386 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-18-ppc -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-18-ppc64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-18-primary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-18-s390 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-18-s390x -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-18-secondary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-18-x86_64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-19-armhfp -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-19-i386 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-19-ppc -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-19-ppc64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-19-primary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-19-s390 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-19-s390x -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-19-secondary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-19-x86_64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-20-armhfp -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-20-i386 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-20-ppc -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-20-ppc64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-20-primary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-20-s390 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-20-s390x -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-20-secondary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-20-x86_64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-21-aarch64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-21-armhfp -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-21-i386 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-21-ppc64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-21-ppc64le -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-21-primary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-21-s390 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-21-s390x -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-21-secondary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-21-x86_64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-22-aarch64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-22-armhfp -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-22-i386 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-22-ppc64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-22-ppc64le -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-22-primary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-22-s390 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-22-s390x -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-22-secondary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-22-x86_64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-23-aarch64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-23-armhfp -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-23-i386 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-23-ppc64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-23-ppc64le -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-23-primary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-23-s390 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-23-s390x -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-23-secondary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-23-x86_64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-24-aarch64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-24-armhfp -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-24-i386 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-24-ppc64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-24-ppc64le -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-24-primary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-24-s390x -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-24-secondary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-24-x86_64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-25-aarch64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-25-armhfp -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-25-i386 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-25-ppc64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-25-ppc64le -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-25-primary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-25-s390x -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-25-secondary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-25-x86_64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-26-aarch64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-26-armhfp -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-26-i386 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-26-ppc64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-26-ppc64le -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-26-primary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-26-s390x -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-26-secondary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-26-x86_64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-27-aarch64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-27-armhfp -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-27-i386 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-27-ppc64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-27-ppc64le -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-27-primary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-27-s390x -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-27-x86_64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-7-primary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-8-i386 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-8-ppc -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-8-ppc64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-8-primary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-8-primary-original -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-8-x86_64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-9-i386 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-9-ia64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-9-ppc -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-9-ppc64 -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-9-primary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-9-primary-original -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-9-secondary -fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-9-x86_64 -fedora-repos-27-0.1.noarch/etc/yum.repos.d -fedora-repos-27-0.1.noarch/etc/yum.repos.d/fedora-cisco-openh264.repo -fedora-repos-27-0.1.noarch/etc/yum.repos.d/fedora-updates-testing.repo -fedora-repos-27-0.1.noarch/etc/yum.repos.d/fedora-updates.repo -fedora-repos-27-0.1.noarch/etc/yum.repos.d/fedora.repo -``` - -``` -$ gpg2 fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-27-primary -pub rsa4096 2017-02-21 [SCE] - 860E19B0AFA800A1751881A6F55E7430F5282EE4 -uid Fedora 27 (27) -pub 4096R/E8E40FDE 2010-01-19 Fedora (13) -``` - -You may wish to do this in a tempoary directory to make cleaning it up easy. - -=== Koji - -Log into koji02.rdu3.fedoraproject.org by way of bastion.fedoraproject.org. - -Verify that ``/etc/koji-gc/koji-gc.conf`` has the new key in it. - -== Consider Before Running - -Nothing at this time. diff --git a/modules/release_guide/pages/final_RC_compose.adoc b/modules/release_guide/pages/final_RC_compose.adoc deleted file mode 100644 index 2b30bea..0000000 --- a/modules/release_guide/pages/final_RC_compose.adoc +++ /dev/null @@ -1,155 +0,0 @@ -include::_partials/attributes.adoc[] - -= Final Release candidate - -When Quality Engineering (QE) requests a Release Candidate (RC) they do -so by opening an issue in the releng repository on pagure. Release -candidate composes are not currently automated. - -[cols=",,",options="header",] -|=== -|Compose Name |Configuration File |Compose Script -|GA |fedora-final.conf |release-candidate.sh -|=== - -== Action - -=== Create the fedora-final.conf configuration file using the script - -The script can be found https://pagure.io/pungi-fedora/blob/main/f/create-candidate-configs.py[here]. - -There is a mandatory argument to specify the milestone to generate (beta or final) -and optional arguments for the major and minor versions for the label. - -Example usage: -`./create-candidate-configs.py --minor=5 final` - -=== Review Compose Tags - -. List any pre-existing builds in the current compose tag -+ -[source,subs="attributes+"] -.... -$ koji list-tagged f{branched}-compose -.... -. Verify preexisting builds are in compose tags -+ -The tagged builds from the previous composes should all be present in -the output from the previous step. Consult the request ticket for the -list of builds expected in this output. The request ticket may list -commands to un-tag existing tagged builds and tag the new requested -builds; if so, you can review these commands and run them. Otherwise, -the steps to do this manually are documented below. -+ -[NOTE] -==== -The very first run of a Beta or GA compose should have no builds -listed under the compose tag. It is important to clear preexisting -builds from the compose tag when moving between the Beta and RC -composes. Verify that these builds were removed. - -[source,subs="attributes+"] -.... -$ koji list-tagged f{branched}-compose -$ koji untag-build --all f{branched}-compose [build1 build2 ...] -.... -==== -+ -[NOTE] -==== -The order in which packages are added into the -f{branched}-compose tag matter. If the builds are untagged -erroneously then special attention should be given to adding them back -correctly. -==== -. Add builds specified by QE to the current compose tag -+ -[source,subs="attributes+"] -.... -$ koji tag-build f{branched}-compose [build1 build2 ...] -.... -+ -[NOTE] -==== -These steps may be completed on a local machine as long as the user has -appropriate permissions in the Koji tool. -==== - -=== Running the Compose - -. Composes use a configuration file to construct the compose. -Each compose uses its own configuration. -The `global_release` variable should start from 1.1 and the second number -should increment each time a new compose is created. - -* GA - `fedora-final.conf` -. Log into the compose backend -+ -.... -$ ssh compose-x86-01.rdu3.fedoraproject.org -.... -. Open a screen session -+ -.... -$ screen -.... -. Obtain the pungi-fedora branch for the current compose -+ -The first time any user account executes a compose the pungi-fedora git -repository must be cloned. The compose candidate script that invokes -pungi should be run from `compose-x86-01.rdu3.fedoraproject.org`. -+ -.... -$ git clone ssh://git@pagure.io/pungi-fedora.git -.... -+ -Enter the pungi-fedora directory. -+ -.... -$ cd pungi-fedora -.... -+ -If the clone step above was not required then fully update the existing -repository checkout from pagure. -+ -[source,subs="attributes+"] -.... -$ git fetch origin -$ git checkout f{branched} -$ git pull origin f{branched} -.... -. Run the compose and redirect the output to a file (in case it's needed later) -+ -[source,subs="attributes+"] -.... -$ sudo ./release-candidate.sh {branched} RC-#.# >& {branched}_RC-#.#.out -# Example: -# $ sudo ./release-candidate.sh {branched} RC-1.3 >& {branched}_RC-1.3.out -.... -+ -The first argument is the release number. The second argument is the -compose label. The numbering scheme begins with 1.1 and the second -number is incremented after each compose. The first number always -remains at 1 for Fedora composes. -+ -[NOTE] -==== -productmd requires the label version to be in this format (because -both numbers may be incremented for other products that use productmd -metadata, and have different meanings). It is because of this that -composes always start with the number 1 and the second number is -incremented with each compose. -==== -+ -[NOTE] -==== -If the compose fails with a directory missing error, then create the -compose directory with `mkdir /mnt/koji/compose/{branched}` -==== - -=== Verification - -The method for verifying a compose has completed is checking -`/mnt/koji/compose/{branched}/[compose_dir]/STATUS`. Any status -other than DOOMED is OK. - diff --git a/modules/release_guide/pages/final_freeze.adoc b/modules/release_guide/pages/final_freeze.adoc deleted file mode 100644 index edcbd9e..0000000 --- a/modules/release_guide/pages/final_freeze.adoc +++ /dev/null @@ -1,100 +0,0 @@ -include::_partials/attributes.adoc[] - -= Fedora Final Freeze - - -== Make sure all retired packages are blocked in koji - -Before the freeze we have to make sure all retired packages have been blocked in koji. -To do this, [run the script](https://pagure.io/releng/blob/main/f/scripts/get_retired_packages.sh) that collects all the retired packages -and outputs them into json files in the [lookaside cache](https://src.fedoraproject.org/lookaside). - -.... -$ ssh pkgs01.fedoraproject.org -$ cd releng/scripts && bash get_retired_packages.sh -.... - -This script will run about 20 minutes. Once finished, run toddler 'koji_block_retired' to block all retired packages in koji. -Web console: https://console-openshift-console.apps.ocp.fedoraproject.org - -.... -oc login --token= --server=https://api.ocp.fedoraproject.org:6443 -oc project poddlers -oc rsh dc/koji_block_retired -python3 /code/toddlers/playtime.py koji_block_retired -.... - - -Before freezing everything we need to do the last stable push in bodhi. - -== Stable push - -[source,subs="attributes+"] -.... -$ sudo -u apache bodhi-push --releases --stable 'f{branched}' -.... - -== Update Ansible vars - -Set the https://pagure.io/fedora-infra/ansible/blob/main/f/vars/all/Frozen.yaml[_NextReleaseFrozen_ variable] to `True`. - -=== Run the playbook - -.... - -$ rbac-playbook groups/koji-hub.yml -.... - -== Update Bodhi release - -[source,subs="attributes+"] -.... -$ bodhi releases edit --name F{branched} --state frozen -.... - -=== Reminder announcement - -Send an email as mentioned below to `devel@`, `devel-announce@`, `test-announce@`, `announce@` lists as information that the final freeze is now active. - -For the reference, you can refer generic https://pagure.io/releng/blob/main/f/mail-templates/06-final-freeze.txt[template] for final freeze from release engineering repo. - -[source,subs="attributes+"] -.... -Hi all, - -Today, YYYY-MM-DD, is an important day on the Fedora Linux {branched} schedule -[1], with significant cut-offs. - -Today we have the Final Freeze [2] which starts at 14:00 UTC. This means -that only packages which fix accepted blocker or freeze exception bugs -[3][4][5] will be marked as 'stable' and included in the Final composes. -Other builds will remain in updates-testing until the Final release is -approved, at which point the Final freeze is lifted and packages can -move to the 'updates' repository. Pending updates will be pushed before -final release as zero day updates. - -Regards, -Fedora Release Engineering - -[1] https://fedorapeople.org/groups/schedule/f-{branched}/f-{branched}-key-tasks.html -[2] https://fedoraproject.org/wiki/Milestone_freezes -[3] https://fedoraproject.org/wiki/QA:SOP_blocker_bug_process -[4] https://fedoraproject.org/wiki/QA:SOP_freeze_exception_bug_process -[5] https://qa.fedoraproject.org/blockerbugs/milestone/f{branched}/final/buglist - -.... - -[NOTE] -==== -Please ensure that the reminder email is sent once the state becomes frozen. -==== - -== During the Final freeze - -=== Process stable push requests - -During freezes we need to push updates that fix blocker and freeze exception -issues to stable when requested. -QA will file a ticket with the updates to push, which will include the -appropriate command(s). Verify the command(s) and run them. - diff --git a/modules/release_guide/pages/final_release.adoc b/modules/release_guide/pages/final_release.adoc deleted file mode 100644 index 1d34b54..0000000 --- a/modules/release_guide/pages/final_release.adoc +++ /dev/null @@ -1,136 +0,0 @@ -include::_partials/attributes.adoc[] - -= Fedora Final Release - -== Koji changes - -[source,subs="attributes+"] -.... -$ koji edit-tag --lock f{branched} -.... - -== Bodhi Changes - -Set the bodhi release to `current` -[source,subs="attributes+"] -.... -$ bodhi releases edit --name F{branched} --state current -$ bodhi releases edit --name F{branched}F --state current -.... - -Set the bodhi stable tag to point to updates instead of base repo -[source,subs="attributes+"] -.... -$ bodhi releases edit --name F{branched} --stable-tag f{branched}-updates -.... - -Set EOL of oldest release to correct date: four weeks after the official release date of {branched}. -[source,subs="attributes+"] -.... -$ bodhi releases edit --name F{old_release} --eol YYYY-MM-DD -$ bodhi releases edit --name F{old_release}C --eol YYYY-MM-DD -$ bodhi releases edit --name F{old_release}F --eol YYYY-MM-DD -.... - -If this date is different from the one in the https://fedorapeople.org/groups/schedule/f-{old_release}/f-{old_release}-key-tasks.html[Fedora schedule], -ask the program manager to update the date in the schedule. -If it is different from the one in F{old_release}'s `/etc/os-release`, https://bugzilla.redhat.com/enter_bug.cgi?product=Fedora&format&component=fedora-release&version=40[file a bug against fedora-release] asking for it to be updated. - -You may also wish to check the current EOL estimates for other releases on Bodhi, the schedule, and fedora-release, -and make sure they are in sync and sensible. - -== Atomic desktops changes - -Run the releng scripts/update_ostree_refs.sh on a compose machine with /mnt/koji/ mounted. - -$ scripts/update_ostree_refs.sh {branched} - -This updates the refs for Atomic desktops to have users follow updates -for the new release rather than being stuck on the GA compose forever. - -== Ansible Changes - -=== Update releng roles updates - -. Set https://pagure.io/fedora-infra/ansible/blob/main/f/vars/all/00-FedoraCycleNumber.yaml[FedoraCycleNumber] to {branched}. - -. Set https://pagure.io/fedora-infra/ansible/blob/main/f/vars/all/FedoraBranchedBodhi.yaml[FedoraBranchedBodhi] to current. - -. Set https://pagure.io/fedora-infra/ansible/blob/main/f/vars/all/FedoraPreviousPrevious.yaml[FedoraPreviousPrevious] to True. - -. Set https://pagure.io/fedora-infra/ansible/blob/main/f/vars/all/FedoraBranched.yaml[FedoraBranched] to False. - -. Set https://pagure.io/fedora-infra/ansible/blob/main/f/vars/all/Frozen.yaml[NextReleaseFrozen] to False. - -Now run the relevant playbooks: - -.... -$ sudo rbac-playbook groups/koji-hub.yml -$ sudo rbac-playbook groups/releng-compose.yml -$ sudo rbac-playbook openshift-apps/bodhi.yml -$ sudo rbac-playbook groups/bodhi-backend.yml -$ sudo rbac-playbook openshift-apps/greenwave.yml -$ sudo rbac-playbook groups/openqa.yml -t testcase_stats -.... - -If you do not have permissions to run a playbook, ask a sysadmin-main member for help. -A sysadmin-qa member can run the openQA playbook. - -For reference, see the https://pagure.io/fedora-infra/ansible/c/7e501dcb9432885c9ef10e78c0418b0d40c85061[Fedora 41 release PR], -though note it is somewhat old and includes some things that are no longer needed, and leaves out some that should be done. - -== Stage Final Release for Mirrors - -To prepare for running the staging script, make sure you have the following information: - -. Release Version: This is the numerical version number of the release, for example, {branched}. -. ComposeID: The ID associated with the Compose, such as "Fedora-{branched}-20160614.0". -. Label: The label used for the location in stage, for example, "Compsoe label for the location in stage 39_RC-1.2." -. Key: The name of the release key, which can be "fedora-{branched}" or "fedora-{branched}-secondary," as examples. -. Prerelease: Set this to 0 or 1 to determine if the release should be placed in a testing environment or not. -. Arch (Optional): For secondary architectures, this parameter can be used to make changes to some internal locations. -+ -[source,subs="attributes+"] -.... -$ scripts/stage-release.sh {branched} Fedora-{branched}-20160614.0 {branched}_RC-1.2 fedora-{branched} 0 -.... -. Sync the release to the Red Hat internal archive following internally -documented - -[NOTE] -==== -Make sure to grab the directory size usage numbers which is used to send -an email to [.title-ref]#mirror-admin@lists.fedoraproject.org# list. -==== - -== Make sure there are no tagging leftovers from bodhi - -Sometimes builds are not properly tagged during the freeze process. -Make sure all builds are properly tagged, for more info about the [issue](https://pagure.io/releng/issue/11775) - -== Verification - -Verification is somewhat difficult as one cannot look at the content via -the web server due to permissions. Typically we ask somebody from the -Infrastructure team to give the tree a second set of eyes. - -Some things to check for: -. are the CHECKSUM files signed? -. are the spins/labs present? - -== Post Release - -=== One Week After the Release - -In the first week after the release MirrorManager still uses the files -at `fedora/linux/development/` and not at -`fedora/linux/releases/` - -Once enough mirrors have picked up the files in the release directory, -run the following playbook from batcave to change the paths in -MirrorManager: - -[source,subs="attributes+"] -.... -$ rbac-playbook -v /srv/web/infra/ansible/playbooks/manual/mirrormanager/move-devel-to-release.yml --extra-vars="version='42'" -.... diff --git a/modules/release_guide/pages/mass_branching_checklist.adoc b/modules/release_guide/pages/mass_branching_checklist.adoc deleted file mode 100644 index 4910a6a..0000000 --- a/modules/release_guide/pages/mass_branching_checklist.adoc +++ /dev/null @@ -1,195 +0,0 @@ -include::_partials/attributes.adoc[] - -= Mass Branching Checklist - -// Merge things from google document to finalize the checklist - -== Description - -At each alpha freeze, a mass branching process is conducted to separate the pending release from `devel/` branch. This allows Rawhide to continue while the pending release undergoes bugfix and polish activities. - -== Action - -=== T-1 Day actions for mass branching - -==== Send Announcement - -* [ ] One day prior to mass branching, send out an announcement notifying the community about the upcoming mass branching. Mention that new Koji builds for Rawhide will be disabled during the process. - -==== Signing keys for rawhide+1 - -Action: add a separate SOP for the same - -==== Repos to Branch - -* [ ] Update the following repositories we need to prepare before the mass branching occurs: (Two Branches needs to be upserted (rawhide needs update, and the branched branch needs to be created) - -==== Updates Archive Yum Repo Seeding - -* [ ] Ask CoreOS team to seed the N paths for the updates-archive yum repo by running the https://pagure.io/releng/archive-repo-manager/blob/main/f/README.md?text=True#_163[steps here]. This is so tests won't start failing after branching. - -==== Pungi-Fedora - -===== Rawhide Branch - -* [ ] Check for all the RELEASE, RELEASE_TITLE, COMPSFILE (all this points to rawhide version) -* [ ] Make sure to update the sigkeys for the latest rawhide version -* [ ] Files to change: - * [ ] nightly.sh - * [ ] cloud-nightly.sh - * [ ] container-nightly.sh - * [ ] fedora-cloud.conf - * [ ] fedora-container.conf - * [ ] fedora.conf - -===== Branched (FN) Branch - -* [ ] Files to change: - * [ ] general.conf `runroot_tag = 'fN-build'` - * [ ] fedora.conf - * [ ] (All rawhide occurrences should be updated to FN version, release_version, global_ksurl, global_version) - * [ ] ostree use fN branch for config (Update "config_branch" to point to fN from main) - * [ ] nightly.sh - * [ ] (All rawhide occurrences should be updated to FN version, and TARGET_DIR needs to be changed to branched from rawhide, RELEASE_TITLE, FROM) - -==== Fedora Kickstarts - -* [ ] ~~Rawhide Branch~~ -* [ ] Branched (FN) Branch - * [ ] fedora-repo.ks (need to comment out the rawhide includes) - * [ ] fedora-iot.ks - * (All rawhide occurrences should be updated to FN version) - * References: https://pagure.io/fedora-kickstarts/c/f2812c2e279c2bcf8b5bb65e264a7b644e2c81e3?branch=f39 - -==== Fedora Comps - -===== Rawhide Branch - -* [ ] Update the Makefile to rawhide version -* [ ] For fn+1 (rawhide) file: Create a comps-fN+1.xml.in (copy it from the last file and update/sync it according to changesets) - - Sync Rawhide with changes from Fedora {branched}: make sure of including the changeset changes occured in the branched repo - -==== Workstation Ostree Config - -===== Rawhide/Main Branch - -* [ ] Add Fedora (rawhide version) repo configs ([[reference](https://pagure.io/workstation-ostree-config/c/53d2a6b48edbb7a24305d4ae99e052aa14354b9f?branch=main)]) - -===== Branched (FN) Branch - -* [ ] Change all the rawhide references to point to fN version (grep for rawhide and replace it with the branched version) - -==== Fedora Release - -===== Rawhide Branch - -* [ ] Need to update the dist_version and Version in the spec file, with a changelog entry - -===== Branched (FN) Branch - -* [ ] Need to update the release_name from rawhide to branched and is_rawhide should be 0 - -==== Fedora Repos - -===== Rawhide Branch - -* [ ] Need to update the dist_version and Version in the spec file, with a changelog entry -* [ ] Check for the presence of rawhide+1 key in the files, if not present add the key to it. for [example](https://src.fedoraproject.org/rpms/fedora-repos/c/53f6d1a7bd0b89ad7bccd95e8b4d52b5774e7420?branch=main). - -===== Branched (FN) Branch - -* [ ] In the branch, update the definition of rawhide_release, and create a changelog entry for [example](https://src.fedoraproject.org/rpms/fedora-repos/c/724c961a79f52abc7d450b1c33ccebed09de6099?branch=f39). - -==== Ansible - -===== Bodhi - -* [ ] Update the Bodhi2 role in the infra ansible repository. This includes updating `koji-sync-listener.py`, `new-updates-sync`, Pungi configs for RPM updates, and Bodhi templates. - -===== Enable Branched Compose - -* [ ] Enable the branched compose in the releng role of the infra ansible repository. - -===== Fedora Branched - -* [ ] Set `FedoraBranched` and `FedoraBranchedBodhi` variables to appropriate values in the infra ansible repository. - -===== Koji Hub - -* [ ] Update the Koji hub configuration to allow side tags for the new Rawhide tag. - -===== Robosignatory - -* [ ] Update the Robosignatory role in the infra ansible repository. Disable branched signing and add the new release. -* [ ] Always check if there is a new artifact that got introduced in the release cycle, if so then we need to handle it more properly with robosignatory and include it in relevant files. (confirm this more with seniors) - -===== Fedora Release - -* [ ] Update the `fedora-release` package in Rawhide and the Branched release, including necessary changes to `fedora-release.spec`. - -===== Fedora Repos - -* [ ] Update the `fedora-repos` package in Rawhide, Branched to receive new GPG keys and updated rawhide symlinks. -* [ ] Update the `fedora-repos` package in remaining stable release branches and updated rawhide symlinks. - -==== Toddlers - -* [ ] Add the new version to the toddlers App configuration, referring to the provided PR for guidance. (For instance: check this: https://pagure.io/fork/humaton/fedora-infra/ansible/c/36b9cb2097fcfc30024f2982d9396cd9e0da7233) - Changes to be done: - ----- -[consumer_config.scm_request_processor.branch_slas.f] -bug_fixes = 'YYYY-MM-DD' -security_fixes = 'YYYY-MM-DD' ----- - -=== T Day actions for mass branching - -(for reviewers here is a checklist that one can copy this on the releng tracker so we can keep track of everything needed in one place) - -==== Merge all the preparatory PRs: - -* [ ] Pungi-Fedora (https://pagure.io/pungi-fedora) -* [ ] Fedora Kickstarts (https://pagure.io/fedora-kickstarts) -* [ ] Fedora Comps (https://pagure.io/fedora-comps) -* [ ] Workstation Ostree Config (https://pagure.io/workstation-ostree-config/) -* [ ] Fedora Release (https://src.fedoraproject.org/rpms/fedora-release) -* [ ] Fedora Repos (https://src.fedoraproject.org/rpms/fedora-repos) -* [ ] Toddlers (https://pagure.io/fedora-infra/toddlers) - - -==== Merging and running ansible changes - -==== Push the Changes - -* [ ] Commit, push, and apply the changes using the corresponding ansible playbooks for various services. - -==== Disable Rawhide Builds in Koji - -* [ ] Configure an outage in Koji to disable Rawhide builds. -* [ ] Cancel all running builds for Rawhide by listing and selecting relevant tasks, and then cancel each task. - -==== Koji - -* [ ] Run the `make-koji-release-tags` script from the releng repository to handle builds from the new branch. - -==== Dist-Git - -* [ ] Create new branches in Git and update gitolite.conf to allow users to push to the new branches. -* [ ] Run the `mass-branching-git.py` script to create new branches based on the file generated by PDC. - -==== Bodhi - -* [ ] Link empty repos and create empty repos as necessary to prepare for the new release. -* [ ] Create rawhide releases in Bodhi using appropriate commands for various types of releases (e.g., standard, container, flatpak). -* [ ] Update MirrorManager to point to the new Rawhide release. -* [ ] Enable autosigning on the Branched release after the compose is completed. -* [ ] Perform ELN-related work, including updating image configurations and scripts. - -==== Fedora Container Base Image - -* [ ] Import new images for Rawhide and update tags for `fedora:rawhide` and `fedora:${RAWHIDE}`. - -==== Update Sync Script - -* [ ] Update the sync script in the releng repository with the new version. diff --git a/modules/release_guide/pages/mass_rebuild.adoc b/modules/release_guide/pages/mass_rebuild.adoc deleted file mode 100644 index beb019d..0000000 --- a/modules/release_guide/pages/mass_rebuild.adoc +++ /dev/null @@ -1,277 +0,0 @@ -include::_partials/attributes.adoc[] - -= Mass Rebuild - -== Description - -Periodically we do mass rebuilds of rawhide during the development -cycle. This SOP will outline the steps necessary to do this. - -== Assumptions - -This assumes that the mass rebuild has already been approved and -scheduled via release engineering and FESCo. Coordinate with -infrastructure as well for any needed koji updates. - -This also assumes that the mass rebuild does not need to be done in -dependency order, and that the mass rebuild does not involve a ABI -change. - -== Considerations - -* The most important thing to keep in mind while doing a mass rebuild is -to communicate clearly what actions are being performed and the status -of the rebuild. -* Check in on scripts frequently to avoid a long stalled command from -adding significant delays in completing the rebuild. -* Check with secondary arches, whether they up-to-date enough with -primary, create rebuild tag and target when they are. It will then take -care of rebuilds of the arch specific packages in appropriate kojis. -* Please make sure you have right ACLs to run the script mention below, one needs to be atleast in sysadmin-releng/releng-team/cvsadmin/sysadmin in order to get root access of the machine if required. - -== Actions - -=== Preparatory Steps - -The following steps may be completed in the weeks leading up to the -scheduled mass rebuild. - -. Create the Mass Rebuild Pagure Issue -+ -____ -Create an issue on the https://pagure.io/releng/issues[Release -Engineering issues page] that points at the schedule for the current -release. - -See https://pagure.io/releng/issue/6898[the Fedora 27 mass rebuild issue -example]. -____ -. Set up the Mass Rebuild Wiki Page -+ -____ -The mass rebuild wiki page should answer the following questions for -maintainers: - -* Why the mass rebuild is happening? -* How to opt out of the mass rebuild? - * Make sure to inform people either on mail, or in the wiki page about the `PKG_SKIP_LIST` which allows packages to be skipped from rebuilding as well as `noautobuild` file that they can add it to there root of a distgit repository and the mass rebuild script will skip it. -* What changes from changesets are taken in account for F{branched}? - -[NOTE,subs="attributes+"] -==== -See https://fedoraproject.org/wiki/Fedora_{branched}_Mass_Rebuild[the Fedora {branched} -Wiki example]. -==== -____ -. Send out the Mass Rebuild Notice -+ -____ -Send out the same information posted on the wiki to the -[.title-ref]#devel-announce@lists.fedoraproject.org# mailing list. Please check if the email is through, if stuck in moderation please ping respective members. - -[NOTE] -==== -See the template in releng repository -https://pagure.io/releng/blob/main/f/mail-templates/01-mass-rebuild-start.txt[template]. -==== -____ -. Create a Tag to Contain the Mass Rebuild -+ -____ -Mass rebuilds require their own tag to contain all related builds. The -example assumes we are doing a rebuild for Fedora {rawhide}. - -[source,subs="attributes+"] -.... -$ koji add-tag f{rawhide}-rebuild --parent f{rawhide} -.... -____ -. Request Package Auto-Signing for New Mass-Rebuild Tag -+ -____ -File a ticket with https://pagure.io/fedora-infrastructure/issues[Fedora -Infrastructure] requesting the new mass-rebuild tag be enabled for -package auto-signing. -____ -. Create the Koji Target for the Mass Rebuild -+ -____ -Using the same [.title-ref]#f{rawhide}-rebuild# tag created in the previous -example: - -[source,subs="attributes+"] -.... -$ koji add-target f{rawhide}-rebuild f{rawhide}-build -.... - -[NOTE] -==== -*koji add-target* _target-name_ _buildroot-tag_ _destination-tag_ -describes the syntax format above. If the _destination-tag_ is not -specified then it will be the same as the _target-name_. -==== -____ -. Update Scripts -+ -____ -The mass rebuild depends on four main scripts from the -https://pagure.io/releng[releng git repository]. Each one requires some -changes in variables for each new mass rebuild cycle. - -* {blank} -+ -mass-rebuild.py:: - ** buildtag - ** targets - ** epoch - ** comment - ** target -* {blank} -+ -find-failures.py:: - ** buildtag - ** desttag - ** epoch -* mass-tag.py -* {blank} -+ -need-rebuild.py:: - ** buildtag - ** target - ** updates - ** epoch -____ - -Change the following items: - -* the build tag, holding tag, and target tag should be updated to -reflect the Fedora release you're building for -* the `epoch` should be updated to the point at which all features that -the mass rebuild is for have landed in the build system (and a newRepo -task completed with those features) -* the comment which is inserted into spec changelogs -* for the references of the above mentioned changes refer to the https://pagure.io/releng/c/f8d89a58efc0b81b9358baadfeb2ba4f52edeeb4?branch=main[F40 mass rebuild PR]. - -=== Starting the Mass Rebuild - -The `mass-rebuild.py` script takes care of: - -* Discovering available packages in koji -* Trimming out packages which have already been rebuilt -* Checking out packages from git -* Bumping the spec file -* Committing the change -* git tagging the change -* Submitting the build request to Koji - -===== -. Connect to the mass-rebuild Machine -+ -____ -.... -$ ssh compose-branched01.rdu3.fedoraproject.org -.... -____ -. Start a terminal multiplexer (this ensures if user gets interrupted due to various reasons, the script can continue in a tmux session) -+ -____ -.... -$ tmux -.... -____ -. Clone or checkout the latest copy of the -https://pagure.io/releng[releng git repository]. -. Run the mass-rebuild.py script from _releng/scripts_ -+ -____ -.... -$ cd path/to/releng_repo/scripts -$ ./mass-rebuild.py 2>&1 | tee ~/massbuild.out -.... -____ -===== - -=== Monitoring Mass Rebuilds - -The community has a very high interest in the status of rebuilds and -many maintainers will want to know if their build failed right away. The -`find-failures.py` and `need-rebuild.py` scripts are designed to update -publicly available URLs for stakeholders to monitor. - -===== -. Connect to a Compose Machine or the monitoring scripts can be run on the same machine where the mass rebuild is happening. -+ -____ -.... -$ ssh compose-x86-02.rdu3.fedoraproject.org / compose-branched01.rdu3.fedoraproject.org:22 -.... -____ -. Start a terminal multiplexer (this ensures if user gets interrupted due to various reasons, the script can continue in a tmux session) -+ -____ -.... -$ tmux -.... -____ -. Clone or checkout the latest copy of the -https://pagure.io/releng[releng git repository] -. {blank} -+ -Set Up the Rebuild Failures Notification Web Site:: - The `find_failures.py` script discovers attempted builds that have - failed. It lists those failed builds and sorts them by package owner. - + - -[source,subs="attributes+"] -.... -$ while true; do ./find_failures.py > f{rawhide}-failures.html && cp f{rawhide}-failures.html /mnt/koji/mass-rebuild/f{rawhide}-failures.html; sleep 600; done -.... -. Start a second pane in the terminal emulator -. {blank} -+ -Set up the Site for Packages that Need Rebuilt:: - The `need-rebuild.py` script discovers packages that have not yet been - rebuilt and generates an html file listing them sorted by package - owner. This gives external stakeholders a rough idea of how much work - is remaining in the mass rebuild. - + - -[source,subs="attributes+"] -.... -$ while true; do ./need-rebuild.py > f{rawhide}-need-rebuild.html && cp f{rawhide}-need-rebuild.html /mnt/koji/mass-rebuild/f{rawhide}-need-rebuild.html; sleep 600; done -.... -===== - -=== Post Mass Rebuild Tasks - -Once the mass rebuild script completes, and all the pending builds have -finished, the builds will need to be tagged. The `mass-tag.py` script -will accomplish this task. The script will: - -* Discover completed builds -* Trim out builds that are older than the latest build for a given -package -* Tag remaining builds into their final destination (without generating -email) - -===== -. Clone or checkout the latest copy of the -https://pagure.io/releng[releng git repository] -. Run the `mass-tag.py` script (requires koji kerberos authentication) -. Remove the koji target for the rebuild: -$ koji remove-target f{rawhide}-rebuild -+ -____ -[source,subs="attributes+"] -.... -$ cd path/to/releng_repo/scripts -$ ./mass-tag.py --source f{rawhide}-rebuild --target f{rawhide} -.... -____ -. Send the final notification to the -_devel-announce@lists.fedoraproject.org_ list -+ -____ -For the contents use this https://pagure.io/releng/blob/main/f/mail-templates/02-mass-rebuild-finished.txt[template]. -____ -===== \ No newline at end of file diff --git a/modules/release_guide/pages/philosophy.adoc b/modules/release_guide/pages/philosophy.adoc index a4be561..5968c96 100644 --- a/modules/release_guide/pages/philosophy.adoc +++ b/modules/release_guide/pages/philosophy.adoc @@ -1,130 +1,149 @@ [[philosophy]] == Fedora Release Engineering Philosophy +=== Spirit of the Document + Being an official part of Fedora means that it is composed and supported -by Fedora Release Engineering (releng). Releng bestows this status on -items that successfully makes its way through our Manufacturing Train -and satisfies all related policies. With this status the item is Open, -Integrated, Reproducible, Auditable, Definable, and Deliverable. (in no -particular order) This document provides definitions for each of those -terms, why they're important, and how we guarantee them. All parts of -Fedora should strive to be meet all parts of being official at all -times. +by Fedora Release Engineering (`releng`). Release Engineering bestows this status on +items that successfully make its way through our Manufacturing Train +and satisfy all related policies. With this status the item is `Open`, +`Integrated`, `Reproducible`, `Auditable`, `Definable`, and `Deliverable` +in no particular order. + +This document provides definitions for each of those +terms, why they are important, and how we guarantee them. + +All parts of Fedora should strive to be meet all parts of being official +at all times. === Open It goes without saying that Fedora is built on -https://fedoraproject.org/wiki/Foundations[the four F's]. In releng we -are no different, we require everything be open, that is open source, -developed in the open, available for all to look at, use and contribute -to. All downstreams should be able to take our tooling and make their -own derivative of Fedora, either by rebuilding everything, perhaps with -different otions or just adding thier own marks and packages and -recomposing. At any time anyone should be able to see how Fedora is put +https://docs.fedoraproject.org/en-US/project/[the four F's]. + +In Release Engineering we are no different, we require everything to be `open`. That is +open source, developed in the open, available for all to look at, use and +contribute to. All downstreams should be able to take our tooling and make +their own derivative of Fedora, either by rebuilding everything, perhaps with +different options or just adding their own marks and packages and +re-composing. + +At any time anyone should be able to see how Fedora is put together and put together their own version of Fedora. === Integrated -Fedora is a huge project with a massive number of ever growing +Fedora is a huge project with a massive number of ever-growing deliverables. This means when we add new deliverables we need to have the composing of them tightly integrated into the process. We ship -Fedora a whole unit, so we need to make it as a whole unit. Any new -tooling we use needs to be consistent with the existing tooling in how -it works. The tooling has to ensure that the output is Reproducible, -Auditable, Definable so that it can be Deliverable. +Fedora a whole unit, so we need to make it as a whole unit. + +*Any new tooling we use must be consistent with the existing tooling in how +it works.* + +The tooling has to ensure that the output is `Reproducible`, +`Auditable`, `Definable` so that it can be `Deliverable`. === Reproducible -A reproducible component is one we can rebuild from scratch at any time +A `reproducible` component is one we can rebuild from scratch at any time with less than a day's worth of effort. It implies we can look up all of the source code, and know exactly what revisions to use from source control. We know exactly what tools are used in the build environment to transform the source code into product content (binaries). We also know how to reproduce the same build environment to ensure the tools behave -like we expect. This aspect is why releng is in the business of +like we expect. This aspect is why Release Engineering is in the business of standardizing on build tools. -Reproducible components are important because they make them a lot +Reproducible components are important because they are a lot easier to maintain. The Security Team takes advantage of this aspect of -an Product. Not knowing how to rebuild a subsystem in a product to apply +a Product. Not knowing how to rebuild a subsystem in a product to apply security fixes, or bug fixes, makes their job much more difficult. It would be a significant risk to provide a product to users that we are -incapable (or merely unsure) of how to build again. Not knowing the -origin of source code is also a significant risk, which is why many of -our build environments are configured to prevent tools from dynamically -downloading content from the internet. - -The combination of Koji and fedpkg is what enables releng to rebuild a -component. fedpkg manages the source code in our dist-git system, and -Koji archives details about the build environment, the tools used, logs, -and of course the binaries themselves. The reproducibility aspect of a -product is the primary reason we require all products be built in Koji -if they seek to be an officially supported part of Fedora. +incapable of building again. Not knowing the origin of source code is also +a significant risk, which is why many of our build environments are configured +to prevent tools from dynamically downloading content from the internet. + +The combination of https://koji.fedoraproject.org[`Koji`] and `fedpkg` is what enables Release Engineering +to rebuild a component. + +`fedpkg` manages the source code in our +https://src.fedoraproject.org[dist-git system] and +`Koji` archives details about the build environment, the tools used, logs, +and of course the binaries themselves. === Auditable -Fedora and Red Hat expect auditable output too, which means releng knows -who built what, when, and where (and how, but that's reproducibility). +Fedora and Red Hat expect `auditable` output too, which means Release Engineering knows +who built what, when, and where (and how, but that is `Reproducibility`). Being able to authoritatively say that something was built within Fedora -by people who have signed the FPCA is important for several reasons. One -big reason is it promotes and fosters accountability within the -comunity. It promotes ownership. Another one is more defensive: in the +by people who have signed the `FPCA` (Fedora Project Contributor Agreement) +is important for several reasons. One +big reason is that it promotes and fosters accountability within the +community. It promotes ownership. Another is more defensive. In the event of a security breach, we have a lot of evidence and data prepared to help us identify what content (if any) was compromised. If a kernel RPM randomly shows up, and we have no records of building it and/or -shipping it, that should raise a lot of alarms pretty quickly! +shipping it, that should raise a lot of alarms very quickly! -Red Hat's Infosec team and Fedora Security care about this aspect +Red Hat's InfoSec Team and Fedora Security Team care about this aspect deeply. We should never be in a position where we cannot definitively answer why a piece of content is available to users. This aspect is also a key part of the verification that is done when Fedora becomes RHEL. All downstream consumers of Fedora expect that they can verify the code and binaries that they consume from Fedora. -Releng tracks this data in 2 systems, 1 of which we own: Koji and Bodhi. -Koji uses ssl certs tied to FAS and bodhi uses FAS for authentication to -provide a strong relationship between a user and the content. Koji -builds the content of course, the Bodhi tracks the bugs, documentation, +Release Engineering tracks this data in 2 systems: `Koji` and `Bodhi`. + +`Koji` uses SSL certificates tied to `Fedora Account System` (FAS) and +`Bodhi` uses FAS for authentication in order to provide a strong relationship +between a user and the content. + +`Koji` builds the content, `Bodhi` tracks the bugs, documentation, and enhancements associated with the content and actually does the -delivery. Bodhi maintains records of what was shipped when and where, +delivery. `Bodhi` maintains records of what was shipped when and where, and who pushed it. === Definable -The ability to define and predict content is necessary as well. It is +The ability to `define` and predict content is necessary as well. It is important to know exactly what was included in a release. It helps protect against shipping content that unnecessarily causes a support burden. This aspect of a Fedora component helps support other aspects -like reproducibility. No need to reproduce software we do not have to -ship, right? Ensuring the product content is lean and trim may sound +like reproducibility. There is no need to reproduce software we do not +ship. Ensuring the product content is lean and trim may sound obvious, but in the world of sprawling RPM dependencies, Maven artifacts, and Ruby gems, it is actually rather easy to include content during the course of a multi-month or multi-year development cycle. -Furthermore, a definable component has the changes made to it vetted and -understood by multiple teams. They are not made in an ad-hoc manor or -without consent from FESCo, QA, releng, and the Product Working Groups +Furthermore, a `definable` component has the changes made to it vetted and +understood by multiple teams. They are not made in an ad-hoc manner or +without consent from FESCo, QA, Release Engineering, and/or the Product Working Groups that contribute at the program level. This reduces change risk to the user, which our users and downstreams like to hear. -Many systems help make components definable. Releng uses Bugzilla, trac, -blocker bugs and bodhi to track additions and changes. +Many systems help make components `definable`. Release Engineering uses +https://bugzilla.redhat.com[`Bugzilla`], +https://src.fedoraproject.org[`dist-git`], +https://bodhi.fedoraproject.org[`Bodhi`], and +https://qa.fedoraproject.org/blockerbugs[`Blocker Bugs`]. === Deliverable -Official parts of Fedora are eligible to be delivered to `/pub/fedora/` -or `/pub/alt/releases` on https://dl.fedoraproject.org/pub/[Fedora +Official parts of Fedora are eligible to be `delivered` to `/pub/fedora/` +or `/pub/alt/releases/` on https://dl.fedoraproject.org/pub/[Fedora Download] and to get https://mirrormanager.fedoraproject.org[mirrorlists] in -https://github.com/fedora-infra/mirrormanager2[mirrormanager]. These -Distribution Platforms are maintained by Fedora Infrastructure and -releng. This is not a feature of the product content itself or how it -was built, but rather how it was delivered to users through releng's +https://github.com/fedora-infra/mirrormanager2[mirrormanager]. + +These `Distribution Platforms` are maintained by Fedora Infrastructure and +Release Engineering. This is not a feature of the product content itself or how it +was built, but rather how it was delivered to users through Release Engineering's processes. Those platforms are geographically replicated by the volunteer mirror network. They provide a reliable and durable service that ensures users can always reach Fedora for updates, even in the -event of a disaster affecting our data center in phx2. +event of a disaster affecting our data center in `rdu3`. + +User support and Fedora Security Team depend on these services. -User support (user mailing lists and [.title-ref]##fedora#) and Fedora -Security team depend on these services. It is vital to that critical -security fixes and updates are always available to users. +*It is vital for critical security fixes and updates to always be available to users.* diff --git a/modules/release_guide/pages/re-sign_rawhide_content.adoc b/modules/release_guide/pages/re-sign_rawhide_content.adoc new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/modules/release_guide/pages/re-sign_rawhide_content.adoc diff --git a/modules/release_guide/pages/release_eol.adoc b/modules/release_guide/pages/release_eol.adoc deleted file mode 100644 index bde5ec9..0000000 --- a/modules/release_guide/pages/release_eol.adoc +++ /dev/null @@ -1,296 +0,0 @@ -include::_partials/attributes.adoc[] - -= End Of Life - -== Description - -Each release of Fedora is maintained as laid out in the -https://docs.fedoraproject.org/en-US/releases/lifecycle/#_maintenance_schedule[maintenance -schedule]. At the conclusion of the maintenance period, a Fedora release -enters `end of life` status. This procedure describes the tasks -necessary to move a release to that status. To get the correct EOL date visit -https://fedorapeople.org/groups/schedule/f-{branched}/f-{branched}-key-tasks.html[release schedule] for the last release. - -== Actions - -=== Reminder announcement - -Around a week before the EOL date, end an email to `devel@`, `devel-announce@`, `test@`, `test-announce@`, `announce@` lists as remainder about the release EOL. -Use the https://pagure.io/releng/blob/main/f/mail-templates/08-end-of-life.txt[template] from release engineering repo. - -[NOTE] -==== -Please ensure that the reminder email is scheduled to be sent one week before the end of life. -==== - -=== `fedora-appstream-metadata` package update - -If necessary, update the -https://src.fedoraproject.org/rpms/fedora-appstream-metadata[`fedora-appstream-metadata`] -package with the new EOL date. Follow the instructions from the repo README. - -Build the package, create an update and make sure it lands in the stable repo -before proceeding. - -This is not necessary if the package already has the correct EOL date. - -=== Koji tasks - -* Disable builds by removing targets - -[source,subs="attributes+"] -.... -koji remove-target f{old_release} -koji remove-target f{old_release}-candidate -koji remove-target f{old_release}-container-candidate -koji remove-target f{old_release}-flatpak-candidate -koji remove-target f{old_release}-infra -koji remove-target f{old_release}-coreos-continuous -koji remove-target f{old_release}-rebuild -.... - -* Remove sidetags - -Remove any existing sidetags from the eol release. - -[source,subs="attributes+"] -.... -for i in `koji list-sidetags --basetag f{old_release}` -do - koji remove-sidetag $i -done -.... - -You need to do this differently because just removing the target of -a sidetag will leave it in a weird state where it cannot be removed. - - -* Purge from disk the signed copies of rpms that are signed with the -EOL'd release key. To achieve this, add the release key to -*koji_cleanup_signed.py* script in https://pagure.io/releng[releng] repo -and run the script on compose-branched01.rdu3.fedoraproject.org: - -.... -tmux -sudo su -cd /root/releng -./scripts/koji_cleanup_signed.py -.... - -[NOTE] -==== -Running this script in tmux or screen is recommended as it takes a long time to run. -==== - -=== Final stable push - -Do a final stable push for the release in Bodhi, following the -xref:releng_misc_guide:sop_pushing_updates.adoc[Pushing Updates SOP]. - -=== Bodhi tasks - -Run the following bodhi commands to set the releases state to -*archived* - -[source,subs="attributes+"] -.... -bodhi releases edit --name "F{old_release}" --state archived -bodhi releases edit --name "F{old_release}C" --state archived -bodhi releases edit --name "F{old_release}F" --state archived -.... - -=== Fedora Infra Ansible Changes - -We need to make some changes and then run several roles in the ansible repo. - -==== Update FedoraPreviousPrevious.yaml in ansible repository - -Edit /vars/all/FedoraPreviousPrevious.yaml and set the value to `False`. - -==== Adjust the EOL for gnome-software - -Edit /roles/pkgdb-proxy/files/pkgdb-gnome-software-collections.json in ansible -and set the release to status "EOL". - -==== Adjust the greenwave gating policy - -Edit /roles/openshift-apps/greenwave/templates/fedora.yaml -and find the policies with \_upgrade_ in their name, near the bottom. -You should see a comment noting these are not run on "the oldest stable release". -The oldest release listed in the `product_versions` section for each of these policies should be one higher than the release you are EOLing. -Remove the lines for these releases, so the oldest release listed is now *two* higher than the release you are EOLing. -You may also remove all lines for the release you are EOLing, but this is not critical. - -==== Remove eol release from robosignatory config - -Edit roles/robosignatory/templates/robosignatory.toml.j2 -and remove stansas that use "fedora-{old_release}" in them. - -==== Run the playbooks - -First, push the above edits. Then run the associated playbooks on _batcave01_: - -.... -sudo ansible-playbook /srv/web/infra/ansible/playbooks/groups/bodhi-backend.yml -sudo ansible-playbook /srv/web/infra/ansible/playbooks/groups/koji-hub.yml -sudo ansible-playbook /srv/web/infra/ansible/playbooks/groups/releng-compose.yml -sudo ansible-playbook /srv/web/infra/ansible/playbooks/groups/proxies.yml -t pkgdb2 -sudo ansible-playbook /srv/web/infra/ansible/playbooks/openshift-apps/bodhi.yml -sudo ansible-playbook /srv/web/infra/ansible/playbooks/openshift-apps/greenwave.yml -.... - -You will need to have someone enable sshd on autosign02 and then run: - -.... -sudo ansible-playbook /srv/web/infra/ansible/playbooks/manual/autosign.yml -.... - -And then someone with the robosignatory passphrase will need to restart it. - -[NOTE] -==== -Another way to run the playbook is using rbac-playbook, in case you don't have sysadmin-main rights or can't become root. -Syntax: sudo rbac-playbook groups/bodhi-backend.yml -==== - -=== Final announcement - -Send the final announcement to `devel@`, `devel-announce@`, `test@`, `test-announce@`, `announce@` lists. -Use the https://pagure.io/releng/blob/main/f/mail-templates/08-end-of-life.txt[template] from release engineering repo. - -=== Other tasks - -==== Remove EOL release from Koschei - -The release going EOL must be removed from Koschei to prevent a flood of failed scratch builds. -See the xref:sysadmin_guide:koschei.adoc#_removing_an_eol_fedora_release[Koschei sysadmin guide] for details. - -==== Remove release from fedfind source of truth - -The Quality team maintains a source of truth for release information used by the fedfind tool -on fedorapeople (people01) in `/srv/groups/qa/metadata/release.json`. -If you have access to this file, you can remove the release going EOL from the file's list of stable releases. -Otherwise, alert the quality team to do so. - -=== Stage the EOL release to archive - -. Log into to bodhi-backend01 and become ftpsync, via root -+ -____ -.... -ssh bodhi-backend01.rdu3.fedoraproject.org -sudo su -su - ftpsync -.... -____ -. Then change into the releases directory. -+ -____ -.... -cd /pub/fedora/linux/releases -.... -____ -. Check to see that the target directory doesn't already exist. -+ -____ -.... -ls /pub/archive/fedora/linux/releases/ -.... -____ -. Do a recursive rsync to update any changes in the trees since the previous copy. -+ -____ -[source,subs="attributes+"] -.... -rsync -avAXSHP ./{old_release}/ /pub/archive/fedora/linux/releases/{old_release}/ -.... -____ -. Similarly, sync updates and updates/testing. -+ -____ -[source,subs="attributes+"] -.... -cd ../updates/ -rsync -avAXSHP {old_release}/ /pub/archive/fedora/linux/updates/{old_release}/ -cd testing -rsync -avAXSHP {old_release}/ /pub/archive/fedora/linux/updates/testing/{old_release}/ -.... -____ -. Do the same with fedora-secondary. -+ -____ -[source,subs="attributes+"] -.... -cd /pub/fedora-secondary/releases/ -rsync -avAXSHP ./{old_release}/ /pub/archive/fedora-secondary/releases/{old_release}/ -cd ../updates/ -rsync -avAXSHP {old_release}/ /pub/archive/fedora-secondary/updates/{old_release}/ -cd testing -rsync -avAXSHP {old_release}/ /pub/archive/fedora-secondary/updates/testing/{old_release}/ -.... -____ -. Announce to the mirror list this has been done and that in 2 weeks you will -move the old trees to archives. - -=== Move the EOL release to archive - -. In two weeks, run the following playbook from batcave: -____ -[source,subs="attributes+"] -.... -sudo ansible-playbook -v /srv/web/infra/ansible/playbooks/manual/mirrormanager/move-to-archive.yml --extra-vars="product='Fedora' version='{old_release}'" -.... -____ -. If there are problems, the postgres DB may have issues and so you need -to get a DBA to update the backend to fix items. -. Wait an hour or so then you can remove the files from the main tree. - -[NOTE] -==== -We need to do a cleanup here for both fedora and fedora-secondary for releases, updates, and updates/testing for the {old_release} release. -==== - -____ -[source,subs="attributes+"] -.... -ssh bodhi-backend01 -cd /pub/fedora/linux/releases/{old_release} -ls -# make sure you have stuff here -rm -rf * -ln ../20/README . -cd /pub/fedora/linux/updates/{old_release} -ls -# make sure you have stuff here -rm -rf * -ln ../20/README . -cd /pub/fedora/linux/updates/testing/{old_release} -ls -# make sure you have stuff here -rm -rf * -ln ../20/README . - -#for fedora-secondary -cd /pub/fedora-secondary/releases/{old_release} -ls -# make sure you have stuff here -rm -rf * -ln ../20/README . -cd /pub/fedora-secondary/updates/{old_release} -ls -# make sure you have stuff here -rm -rf * -ln ../20/README . -cd /pub/fedora-secondary/updates/testing/{old_release} -ls -# make sure you have stuff here -rm -rf * -ln ../20/README . -.... -____ - -== Consider Before Running - -* Resource contention in infrastructure, such as outages -* Extenuating circumstances for specific planned updates, if any -* Send the reminder announcement, if it isn't sent already diff --git a/modules/release_guide/pages/release_post_cleanup.adoc b/modules/release_guide/pages/release_post_cleanup.adoc deleted file mode 100644 index 007b0e6..0000000 --- a/modules/release_guide/pages/release_post_cleanup.adoc +++ /dev/null @@ -1,81 +0,0 @@ -include::_partials/attributes.adoc[] - -= Post-Release Clean-up Documentation - -:page-description: Steps to clean up resources after a Fedora final release, including removing outdated content, updating configurations, and closing issues. - -== Overview - -This document outlines the necessary steps for cleaning up resources after the release of a Fedora version. It includes actions such as removing outdated content from servers, updating mirroring configurations, and closing open issues related to the release process. - -== Removing N_Beta from Torrents - -.Post-release clean-up of torrents -[discrete] -=== Description - -Remove beta versions of the Fedora release from the torrent server to prevent them from being distributed. This helps focus resources on the stable release. - -[discrete] -=== Steps - -. SSH into `torrent01.fedoraproject.org`. -. Navigate to the directory containing the {branched}_Beta torrents. -. Delete the beta torrents for the {branched} version. -. Regenerate the whitelist to exclude these torrents. - -== Update MirrorManager Configuration - -.MirrorManager redirection post-release -[discrete] -=== Description - -After the release, update MirrorManager to redirect requests from the development directory to the releases directory. - -[discrete] -=== Steps - -. Run the following playbook on batcave to update the paths in MirrorManager: -+ -[source,subs="attributes+"] -.... -$ rbac-playbook -v /srv/web/infra/ansible/playbooks/manual/mirrormanager/move-devel-to-release.yml --extra-vars="version='{branched}'" -.... - -== Cleanup of RCs and Unnecessary Composes - -.Cleanup of unnecessary composes and RCs -[discrete] -=== Description - -Remove release candidates and unnecessary composes that are no longer needed to free up storage. - -[discrete] -=== Steps - -. SSH into `bodhi-backend01.rdu3.fedoraproject.org` or any server where Koji is mounted. -. Perform the following: -** Remove all directories related to Beta and Final RCs in `/pub/alt/stage/`. -** Clean up old composes by removing all but the latest in `/mnt/koji/compose/branched/` and `/mnt/koji/compose/{branched}/`. -** Two weeks after release: Remove directories at `/pub/fedora/linux/development/{branched}` and `/pub/fedora-secondary/development/{branched}`. -** After MirrorManager playbook is run: Remove all but rawhide dirs at - `/pub/alt/development/`. - -== Closure of Failed Composes Issues - -.Closure of open issues related to failed composes -[discrete] -=== Description - -Address and close any open issues related to failed composes to maintain the health of the Fedora release process infrastructure. - -[discrete] -=== Steps - -. Use the script available in the Fedora RelEng repository: https://pagure.io/releng/blob/main/f/scripts/misc/failed_composes_cleanup.py -+ -[source,subs="attributes+"] -.... -python3 failed_composes_cleanup.py -.... -. Run the script to automatically identify and close open issues. diff --git a/modules/release_guide/pages/sop_beta_RC_compose.adoc b/modules/release_guide/pages/sop_beta_RC_compose.adoc new file mode 100644 index 0000000..022116b --- /dev/null +++ b/modules/release_guide/pages/sop_beta_RC_compose.adoc @@ -0,0 +1,155 @@ +include::_partials/attributes.adoc[] + += Beta Release candidate + +When Quality Engineering (QE) requests a Release Candidate (RC) they do +so by opening an issue in the releng repository on pagure. Release +candidate composes are not currently automated. + +[cols=",,",options="header",] +|=== +|Compose Name |Configuration File |Compose Script +|Beta |fedora-beta.conf |release-candidate.sh +|=== + +== Action + +=== Create the fedora-beta.conf configuration file using the script + +The script can be found https://pagure.io/pungi-fedora/blob/main/f/create-candidate-configs.py[here]. + +There is a mandatory argument to specify the milestone to generate (beta or final) +and optional arguments for the major and minor versions for the label. + +Example usage: +`./create-candidate-configs.py --minor=5 beta` + +=== Review Compose Tags + +. List any pre-existing builds in the current compose tag ++ +[source,subs="attributes+"] +.... +$ koji list-tagged f{branched}-compose +.... +. Verify preexisting builds are in compose tags ++ +The tagged builds from the previous composes should all be present in +the output from the previous step. Consult the request ticket for the +list of builds expected in this output. The request ticket may list +commands to un-tag existing tagged builds and tag the new requested +builds; if so, you can review these commands and run them. Otherwise, +the steps to do this manually are documented below. ++ +[NOTE] +==== +The very first run of a Beta or GA compose should have no builds +listed under the compose tag. It is important to clear preexisting +builds from the compose tag when moving between the Beta and RC +composes. Verify that these builds were removed. + +[source,subs="attributes+"] +.... +$ koji list-tagged f{branched}-compose +$ koji untag-build --all f{branched}-compose [build1 build2 ...] +.... +==== ++ +[NOTE] +==== +The order in which packages are added into the +f{branched}-compose tag matter. If the builds are untagged +erroneously then special attention should be given to adding them back +correctly. +==== +. Add builds specified by QE to the current compose tag ++ +[source,subs="attributes+"] +.... +$ koji tag-build f{branched}-compose [build1 build2 ...] +.... ++ +[NOTE] +==== +These steps may be completed on a local machine as long as the user has +appropriate permissions in the Koji tool. +==== + +=== Running the Compose + +. Composes use a configuration file to construct the compose. +Each compose uses its own configuration. +The `global_release` variable should start from 1.1 and the second number +should increment each time a new compose is created. + +* Beta - `fedora-beta.conf` +. Log into the compose backend ++ +.... +$ ssh compose-x86-01.phx2.fedoraproject.org +.... +. Open a screen session ++ +.... +$ screen +.... +. Obtain the pungi-fedora branch for the current compose ++ +The first time any user account executes a compose the pungi-fedora git +repository must be cloned. The compose candidate script that invokes +pungi should be run from `compose-x86-01.rdu3.fedoraproject.org`. ++ +.... +$ git clone ssh://git@pagure.io/pungi-fedora.git +.... ++ +Enter the pungi-fedora directory. ++ +.... +$ cd pungi-fedora +.... ++ +If the clone step above was not required then fully update the existing +repository checkout from pagure. ++ +[source,subs="attributes+"] +.... +$ git fetch origin +$ git checkout f{branched} +$ git pull origin f{branched} +.... +. Run the compose and redirect the output to a file (in case it's needed later) ++ +[source,subs="attributes+"] +.... +$ sudo ./release-candidate.sh {branched} Beta-#.# >& {branched}_Beta-#.#.out +# Example: +# $ sudo ./release-candidate.sh {branched} Beta-1.3 >& {branched}_Beta-1.3.out +.... ++ +The first argument is the release number. The second argument is the +compose label. The numbering scheme begins with 1.1 and the second +number is incremented after each compose. The first number always +remains at 1 for Fedora composes. ++ +[NOTE] +==== +productmd requires the label version to be in this format (because +both numbers may be incremented for other products that use productmd +metadata, and have different meanings). It is because of this that +composes always start with the number 1 and the second number is +incremented with each compose. +==== ++ +[NOTE] +==== +If the compose fails with a directory missing error, then create the +compose directory with `mkdir /mnt/koji/compose/{branched}` +==== + +=== Verification + +The method for verifying a compose has completed is checking +`/mnt/koji/compose/{branched}/[compose_dir]/STATUS`. Any status +other than DOOMED is OK. + diff --git a/modules/release_guide/pages/sop_beta_freeze.adoc b/modules/release_guide/pages/sop_beta_freeze.adoc new file mode 100644 index 0000000..a841f7d --- /dev/null +++ b/modules/release_guide/pages/sop_beta_freeze.adoc @@ -0,0 +1,135 @@ +include::_partials/attributes.adoc[] + += Beta freeze and updates-testing activation point + +== Description + +This SOP covers two distinct events that happen at the same time. Two weeks after +https://docs.pagure.org/releng/sop_mass_branching.html[Mass Branching], at 14:00 UTC, the Beta freeze is imposed, +and use of updates-testing is enabled for the new release. + +== Action + +=== Making koji changes + +Make the following koji tag changes + +[source,subs="attributes+"] +.... +$ koji remove-tag-inheritance f{branched}-updates-candidate f{branched} +$ koji remove-tag-inheritance f{branched}-updates-testing f{branched} +$ koji remove-tag-inheritance f{branched}-updates-pending f{branched} +$ koji remove-tag-inheritance f{branched}-override f{branched} +$ koji add-tag-inheritance f{branched}-updates-candidate f{branched}-updates +$ koji add-tag-inheritance f{branched}-updates-testing f{branched}-updates +$ koji add-tag-inheritance f{branched}-updates-pending f{branched}-updates +$ koji add-tag-inheritance f{branched}-override f{branched}-updates +$ koji edit-tag --perm=admin f{branched} +.... + +=== Bodhi configuration + +Configure Bodhi to create composes, not to automatically create updates, and to be frozen. +[source,subs="attributes+"] +.... +$ bodhi releases edit --name "F{branched}" --stable-tag f{branched} --testing-repository updates-testing --package-manager dnf --no-create-automatic-updates --composed-by-bodhi --state frozen +.... + +[NOTE] +==== +Add the container and flatpak releases if they weren't already added to +bodhi +==== + +=== Update Ansible vars + +Set the infra ansible https://pagure.io/fedora-infra/ansible/blob/main/f/vars/all/FedoraBranchedBodhi.yaml[_FedoraBranchedBodhi_ variable] to `prebeta` +and the https://pagure.io/fedora-infra/ansible/blob/main/f/vars/all/Frozen.yaml[_NextReleaseFrozen_ variable] to `True`. + +=== Run the playbooks + +.... +$ rbac-playbook openshift-apps/bodhi.yml +$ rbac-playbook groups/bodhi-backend.yml +$ rbac-playbook groups/koji-hub.yml +.... + +=== Send Announcement + +Email *devel-announce* and *test-announce* lists about Beta freeze and updates-testing activation. + +[source,subs="attributes+"] +.... +Hi all, + +Today's an important day on the Fedora Linux {branched} schedule [1], with +several significant cut-offs. First of all, today is the Bodhi +updates-testing activation point [2]. That means that from now all +Fedora Linux {branched} packages must be submitted to updates-testing and pass +the relevant requirements [3] before they will be marked as 'stable' and +moved to the Fedora Repository. + +Today is also the Beta freeze [4]. This means that only packages which +fix accepted blocker or freeze exception bugs [5][6] will be marked as +'stable' and included in the Beta composes. Other builds will remain in +updates-testing until the Beta release is approved, at which point the +Beta freeze is lifted and packages can move to 'stable' as usual until +the Final freeze. + +Today is also the Software String freeze [7], which means that strings +marked for translation in Fedora-translated projects should not now be +changed for Fedora Linux {branched}. + +Finally, today is the 'completion deadline' Change Checkpoint [8], +meaning that Fedora Linux {branched} Changes must now be 'feature complete or +close enough to completion that a majority of its functionality can be +tested'. All tracking bugs should be on ON_QA state or later to reflect +this. + +Regards, +Fedora Release Engineering + +[1] https://fedorapeople.org/groups/schedule/f-{branched}/f-{branched}-key-tasks.html +[2] https://fedoraproject.org/wiki/Updates_Policy#Bodhi_enabling +[3] https://fedoraproject.org/wiki/Updates_Policy#Branched_release +[4] https://fedoraproject.org/wiki/Milestone_freezes +[5] https://fedoraproject.org/wiki/QA:SOP_blocker_bug_process +[6] https://fedoraproject.org/wiki/QA:SOP_freeze_exception_bug_process +[7] https://fedoraproject.org/wiki/ReleaseEngineering/StringFreezePolicy +[8] https://fedoraproject.org/wiki/Changes/Policy + +.... + + +=== Verification + +Compare koji tagging structure with older release + +[source,subs="attributes+"] +.... +$ koji list-tag-inheritance {branched} --reverse +$ koji list-tag-inheritance {current} --reverse +.... + +Compare the bodhi release with older release + +[source,subs="attributes+"] +.... +$ bodhi releases info {branched} +$ bodhi releases info {current} +.... + +Check for other variants like container and flatpaks. + +== During the Beta freeze + +=== Process stable push requests + +During freezes we need to push updates that fix blocker and freeze exception +issues to stable when requested. +QA will file a ticket with the updates to push, which will include the +appropriate command(s). Verify the command(s) and run them. + +== Consider Before Running + +FIXME diff --git a/modules/release_guide/pages/sop_beta_release.adoc b/modules/release_guide/pages/sop_beta_release.adoc new file mode 100644 index 0000000..47d5e7c --- /dev/null +++ b/modules/release_guide/pages/sop_beta_release.adoc @@ -0,0 +1,88 @@ +include::_partials/attributes.adoc[] + += Fedora Beta Release + +Based on the GO/NOGO meeting if the currnet compose is GO. The following steps need to be taken. + +== Wait for branched compose that matches the beta rc + +QE will typically request a stable push of all the updates that were in the Beta RC that +was signed off on. After that stable push we need to wait for a branched compose with them. +This ensures that the nightly branched has the same content as the beta we are about to release. +This is to allow us to make sure and tag all the right packages and have a common point. + +== Koji changes + +[source,subs="attributes+"] +.... +$ koji clone-tag --all --latest-only f{branched} f{branched}-Beta +.... + +== Stage release to mirrors + +[source,subs="attributes+"] +.... +$ sh scripts/stage-release.sh {branched}_Beta Fedora-{branched}-YYYYMMDD.0 _Beta-1.2 fedora-{branched} 1 +.... + +Please update YYYYMMDD in the above command accordingly. + +[NOTE] +==== +Make sure to grab the directory size usage numbers which is used to send +an email to [.title-ref]#mirror-admin@lists.fedoraproject.org# list. +==== + +== Sync the signed checksums to stage + +We need to sync the signed checksums to /pub/alt/stage/ by running the +following command + +[source,subs="attributes+"] +.... +$ for dir in Cloud Container Everything Kinoite Labs Sericea Server Silverblue Spins Workstation metadata; do sudo -u ftpsync rsync -avhH /mnt/koji/compose/{branched}/Fedora-{branched}-20221105.0/compose/$dir/ /pub/alt/stage/{branched}_RC-1.7/$dir/ --link-dest=/pub/fedora/linux/releases/{branched}/Everything/ --link-dest=/pub/alt/stage/{branched}_RC-1.2/Everything/ --link-dest=/pub/alt/stage/{branched}_RC-1.3/Everything --link-dest=/pub/alt/stage/{branched}_RC-1.4/Everything --link-dest=/pub/alt/stage/{branched}_RC-1.5/Everything --link-dest=/pub/alt/stage/{branched}_RC-1.6/Everything --link-dest=/pub/alt/stage/{branched}_RC-1.7/Everything; done +.... + +[NOTE] +==== +Replace the RC above with the RC that was 'go' +==== + +== Set release in Bodhi to pending + +[source,subs="attributes+"] +.... +$ bodhi releases edit --name F{branched} --state pending +.... + +=== Update Ansible vars + +Set the infra ansible https://pagure.io/fedora-infra/ansible/blob/main/f/vars/all/FedoraBranchedBodhi.yaml[_FedoraBranchedBodhi_ variable] to `postbeta` +and the https://pagure.io/fedora-infra/ansible/blob/main/f/vars/all/Frozen.yaml[_NextReleaseFrozen_ variable] to `False`. + +== Run the playbooks + +.... +$ sudo rbac-playbook groups/releng-compose.yml +$ sudo rbac-playbook groups/bodhi-backend.yml +$ sudo rbac-playbook openshift-apps/bodhi.yml +$ sudo rbac-playbook groups/koji-hub.yml +.... + +== Beta Installation and Cleanup + +This section outlines the steps that should be followed to ensure that older copies of Fedora betas are removed from the servers and that the correct permissions are set on the directories. + +=== Remove older betas + +This cleanup step is to remove the older betas from the `/pub/fedora-secondary/releases/test` and `/pub/fedora/linux/releases/test/` directories. This can be done using the rm command: + +[source,subs="attributes+"] +.... +$ rm -rf /pub/fedora-secondary/releases/test/{branched}_Beta +$ rm -rf /pub/fedora/linux/releases/test/{branched}_Beta +.... + +Note that {branched} should be replaced with the name of the beta release, such as 36_Beta, 37_Beta, etc. + +By following these steps, one can ensure that older copies of Fedora betas are removed from the servers and that the correct permissions are set on the directories. diff --git a/modules/release_guide/pages/sop_create_release_signing_key.adoc b/modules/release_guide/pages/sop_create_release_signing_key.adoc new file mode 100644 index 0000000..e056e0c --- /dev/null +++ b/modules/release_guide/pages/sop_create_release_signing_key.adoc @@ -0,0 +1,484 @@ += Create Release Signing Key + +== Description + +At the beginning of each release under development, a new package signing key is created. This key is used to prove the authenticity of packages built by Fedora and distributed by Fedora. This key will be used to sign all packages for the public test and final releases. + +== Action + +=== Sigul + +Sigul is the signing server that holds our keys. To make use of a new key, it must be created and access to the key granted. The `new-key`, `grant-key-access`, and `change-passphrase` commands are used. + +These are to be running on `bodhi-backend01` machine. + +[source, bash] +---- +$ ssh bodhi-backend01.rdu3.fedoraproject.org +---- + +[source, bash] +---- +$ sigul new-key --help +usage: client.py new-key [options] key + +Add a key + +options: + -h, --help show this help message and exit + --key-admin=USER Initial key administrator + --name-real=NAME_REAL + Real name of key subject + --name-comment=NAME_COMMENT + A comment about the key subject + --name-email=NAME_EMAIL + E-mail of key subject + --expire-date=YYYY-MM-DD + Key expiration date +---- + +[source, bash] +---- +$ sigul grant-key-access --help +usage: client.py grant-key-access key user + +Grant key access to a user + +options: + -h, --help show this help message and exit +---- + +[source, bash] +---- +$ sigul change-passphrase --help +usage: client.py change-passphrase key + +Change key passphrase + +options: + -h, --help show this help message and exit +---- + +For example, to create the Fedora 23 signing key, do the following: + +. Log into a system configured to run the sigul client. +. Create the key using a strong passphrase when prompted: + +[source, bash] +---- +$ sigul new-key --key-admin ausil --key-type gnupg \ + --gnupg-name-real Fedora \ + --gnupg-name-comment 23 \ + --gnupg-name-email fedora-23-primary@fedoraproject.org fedora-23 + +---- + +[source, bash] +---- +For EPEL + +$ sigul new-key --key-admin ausil --key-type gnupg \ + --gnupg-name-real "Fedora EPEL" \ + --gnupg-name-comment 7 \ + --gnupg-name-email epel@fedoraproject.org epel-7 +---- + +. Wait for entropy. This can take several minutes. +. For Fedora, also create the IMA signing key: + +[source, bash] +---- +$ sigul new-key --key-admin ausil --key-type ECC fedora-23-ima +---- + +. Grant key access to Fedora Account holders who will be signing packages and protect it with a temporary passphrase. For example, `CHANGEME`. Do the same with the -ima key for Fedora: + +[source, bash] +---- +$ sigul grant-key-access fedora-23 kevin +---- + +[NOTE] +==== +You can add --key-admin=true here to make a particular user key-admin for the key +==== + +[NOTE] +==== +**IMPORTANT:** Grant access to the autopen user as it's required for robosignatory autosigning and then restart the robosignatory service. +Make sure when granting key access you should use the autosign/robosignatory passphrase. + +==== + +. Provide the key name and temporary passphrase to signers. If they don't respond, revoke access until they are ready to change their passphrase. Signers can change their passphrase using the `change-passphrase` command: + +[source, bash] +---- +$ sigul change-passphrase fedora-23 +---- + +. When your sigul cert expires, run: + +[source, bash] +---- +certutil -d ~/.sigul -D -n sigul-client-cert +---- + +to remove the old cert, then + +[source, bash] +---- +sigul_setup_client +---- + +to add a new one. + +=== Adding Certificates for IMA Keys + +We now need to create an IMA certificate signed by our Sigul CA and get it to the kernel maintainers. This must be done after the IMA key is created. + +[source, bash] +---- +$ sigul -v -v sign-certificate fedorasigulca fedora-41-ima --issuer-certificate-name fedorasigulca --subject-certificate-name fedoraimafourtyone --validity 2y --certificate-type codesigning --subject "CN=Fedora 41 IMA Code-signing cert" > fedora-41-ima.pem +---- + +NOTE: Change the name from `41/fourtyone` to the appropriate release version. + +This .pem file then needs to be converted to a 'der' file (which is what IMA natively uses): + +openssl x509 -inform PEM -in fedora-44-ima.pem -outform DER -out fedora-44-ima.der + +The .der file should be added to fedora-repos package and the kernel should trust it +because it's signed by our CA. + +=== fedora-repos + +The `fedora-repos` package houses a copy of the public key information. This is used by RPM to verify the signature on files encountered. Currently, the `fedora-repos` package has a single key file named after the version of the key and the arch the key is for. + +To continue our example, the file would be named `RPM-GPG-KEY-fedora-27-primary`, which is the primary arch key for Fedora 27. To create this file, use the `get-public-key` command from sigul: + +[source, bash] +---- +$ sigul get-public-key fedora-27 > RPM-GPG-KEY-fedora-27-primary +---- + +Add this file to the repo and update the archmap file for the new release: + +[source, bash] +---- +$ git add RPM-GPG-KEY-fedora-27-primary +---- + +Then make a new `fedora-repos` build for rawhide (`FIXME: this should be its own SOP`). + +==== Web team SOP + +[source, bash] +---- +# from git repo root +cd fedoraproject.org/ +curl $KEYURL > /tmp/newkey +$EDITOR update-gpg-keys # Add key ID of the recently EOL'd version to obsolete_keys +./update-gpg-key /tmp/newkey +gpg static/fedora.gpg # used to verify the new keyring +# it should look something like this: +# pub 4096R/57BBCCBA 2009-07-29 Fedora (12) +# pub 4096R/E8E40FDE 2010-01-19 Fedora (13) +# pub 4096R/97A1071F 2010-07-23 Fedora (14) +# pub 1024D/217521F6 2007-03-02 Fedora EPEL +# sub 2048g/B6610DAF 2007-03-02 [expires: 2017-02-27] +# it must only have the two supported versions of Fedora, rawhide, and EPEL +# also verify that static/$NEWKEY.txt exists +$EDITOR data/content/{keys,verify}.html # see git diff 1840f96~ 1840f96 +---- + + +=== pungi-fedora + +The nightly compose configs come from the pungi-fedora project on https://pagure.io. We need to create a pull request to pull in the new key: + +[source, bash] +---- +$ git clone ssh://git@pagure.io//pungi-fedora.git +$ cd pungi-fedora +$ vim *conf + +$ git commit -m 'Add new key' +$ git push +$ file a Pull Request +---- + +=== Koji + +Koji has a garbage collection utility that finds builds that meet criteria to be removed to save space. Part of that criteria has to do with whether the build has been signed with a key. If the collection utility doesn't know about a key, it will ignore the build. Thus, as we create new keys, we need to inform the utility of these keys, or else builds can pile up. The configuration for the garbage collection lives within Ansible. + +On a clone of the infrastructure Ansible git repo, edit the `roles/koji_hub/templates/koji-gc.conf.j2` file: + +[source, diff] +---- +diff --git a/roles/koji_hub/templates/koji-gc.conf.j2 b/roles/koji_hub/templates/koji-gc.conf.j2 +index 9ecb750..9c48a8e 100644 +--- a/roles/koji_hub/templates/koji-gc.conf.j2 ++++ b/roles/koji_hub/templates/koji-gc.conf.j2 +@@ -35,6 +35,7 @@ key_aliases = + 81B46521 fedora-24 + FDB19C98 fedora-25 + 64DAB85D fedora-26 ++ F5282EE4 fedora-27 + fedora-extras + redhat-beta + fedora-epel + @@ -91,6 +93,7 @@ policy = + sig fedora-24 && age < 12 weeks :: keep + sig fedora-25 && age < 12 weeks :: keep + sig fedora-26 && age < 12 weeks :: keep + + sig fedora-27 && age < 12 weeks :: keep + sig fedora-epel && age < 12 weeks :: keep + sig fedora-epel-6 && age < 12 weeks :: keep + sig fedora-epel-7 && age < 12 weeks :: keep + +---- + +In this case the fedora-epel key was added to the list of key aliases, then +referenced in the list of unprotected_keys, and finally a policy was created +for how long to keep builds signed with this key. + +Once you've made your change commit and push. The buildsystem will pick up +this change the next time puppet refreshes. + +== Verification + +We can verify that the key was created in sigul, the correct users have access +to the key, the key was added to the fedora-release package, that the website +was updated with the right key, that sigulsign_unsigned was properly updated, +and that the key was successfully updated to the public key servers. + +=== Sigul + +Use the ``list-keys`` command to verify that the key was indeed added to sigul: + +``` +$ sigul list-keys +Administrator's password: +fedora-10 +fedora-10-testing +fedora-11 +fedora-12 +fedora-13 +``` + +Our new key should be on the list. This command expects **your** +administrative password. + +Use the ``list-key-users`` command to verify all the signers have access: + +``` +$ sigul list-key-users fedora-13 +Key passphrase: +jkeating +jwboyer +``` + +This command expects **your** key passphrase for the key in question. + +=== fedora-release + +To verify that the key was added to this package correctly, download the latest +build from koji and run rpm2cpio on it, then run gpg on the key file: + +``` +$ koji download-build --arch noarch --latest f27 fedora-repos +fedora-repos-rawhide-27-0.1.noarch.rpm | 7.3 kB 00:00:00 +fedora-repos-27-0.1.noarch.rpm | 87 kB 00:00:00 +$ rpmdev-extract fedora-repos-27-0.1.noarch.rpm +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-27-fedora +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-10-i386 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-10-ppc +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-10-ppc64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-10-primary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-10-x86_64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-11-i386 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-11-ppc +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-11-ppc64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-11-primary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-11-x86_64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-12-i386 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-12-ppc +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-12-ppc64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-12-primary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-12-x86_64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-13-arm +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-13-armhfp +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-13-i386 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-13-mips +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-13-primary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-13-secondary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-13-x86_64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-14-i386 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-14-primary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-14-x86_64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-15-arm +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-15-armhfp +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-15-i386 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-15-ppc +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-15-ppc64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-15-primary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-15-s390 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-15-s390x +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-15-secondary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-15-x86_64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-16-arm +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-16-armhfp +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-16-i386 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-16-ppc +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-16-ppc64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-16-primary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-16-s390 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-16-s390x +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-16-secondary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-16-x86_64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-17-arm +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-17-armhfp +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-17-i386 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-17-ppc +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-17-ppc64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-17-primary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-17-s390 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-17-s390x +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-17-secondary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-17-x86_64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-18-arm +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-18-armhfp +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-18-i386 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-18-ppc +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-18-ppc64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-18-primary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-18-s390 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-18-s390x +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-18-secondary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-18-x86_64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-19-armhfp +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-19-i386 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-19-ppc +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-19-ppc64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-19-primary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-19-s390 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-19-s390x +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-19-secondary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-19-x86_64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-20-armhfp +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-20-i386 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-20-ppc +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-20-ppc64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-20-primary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-20-s390 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-20-s390x +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-20-secondary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-20-x86_64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-21-aarch64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-21-armhfp +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-21-i386 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-21-ppc64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-21-ppc64le +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-21-primary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-21-s390 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-21-s390x +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-21-secondary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-21-x86_64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-22-aarch64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-22-armhfp +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-22-i386 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-22-ppc64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-22-ppc64le +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-22-primary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-22-s390 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-22-s390x +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-22-secondary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-22-x86_64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-23-aarch64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-23-armhfp +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-23-i386 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-23-ppc64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-23-ppc64le +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-23-primary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-23-s390 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-23-s390x +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-23-secondary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-23-x86_64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-24-aarch64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-24-armhfp +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-24-i386 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-24-ppc64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-24-ppc64le +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-24-primary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-24-s390x +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-24-secondary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-24-x86_64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-25-aarch64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-25-armhfp +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-25-i386 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-25-ppc64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-25-ppc64le +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-25-primary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-25-s390x +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-25-secondary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-25-x86_64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-26-aarch64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-26-armhfp +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-26-i386 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-26-ppc64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-26-ppc64le +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-26-primary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-26-s390x +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-26-secondary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-26-x86_64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-27-aarch64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-27-armhfp +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-27-i386 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-27-ppc64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-27-ppc64le +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-27-primary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-27-s390x +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-27-x86_64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-7-primary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-8-i386 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-8-ppc +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-8-ppc64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-8-primary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-8-primary-original +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-8-x86_64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-9-i386 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-9-ia64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-9-ppc +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-9-ppc64 +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-9-primary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-9-primary-original +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-9-secondary +fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-9-x86_64 +fedora-repos-27-0.1.noarch/etc/yum.repos.d +fedora-repos-27-0.1.noarch/etc/yum.repos.d/fedora-cisco-openh264.repo +fedora-repos-27-0.1.noarch/etc/yum.repos.d/fedora-updates-testing.repo +fedora-repos-27-0.1.noarch/etc/yum.repos.d/fedora-updates.repo +fedora-repos-27-0.1.noarch/etc/yum.repos.d/fedora.repo +``` + +``` +$ gpg2 fedora-repos-27-0.1.noarch/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-27-primary +pub rsa4096 2017-02-21 [SCE] + 860E19B0AFA800A1751881A6F55E7430F5282EE4 +uid Fedora 27 (27) +pub 4096R/E8E40FDE 2010-01-19 Fedora (13) +``` + +You may wish to do this in a tempoary directory to make cleaning it up easy. + +=== Koji + +Log into koji02.rdu3.fedoraproject.org by way of bastion.fedoraproject.org. + +Verify that ``/etc/koji-gc/koji-gc.conf`` has the new key in it. + +== Consider Before Running + +Nothing at this time. diff --git a/modules/release_guide/pages/sop_final_RC_compose.adoc b/modules/release_guide/pages/sop_final_RC_compose.adoc new file mode 100644 index 0000000..2b30bea --- /dev/null +++ b/modules/release_guide/pages/sop_final_RC_compose.adoc @@ -0,0 +1,155 @@ +include::_partials/attributes.adoc[] + += Final Release candidate + +When Quality Engineering (QE) requests a Release Candidate (RC) they do +so by opening an issue in the releng repository on pagure. Release +candidate composes are not currently automated. + +[cols=",,",options="header",] +|=== +|Compose Name |Configuration File |Compose Script +|GA |fedora-final.conf |release-candidate.sh +|=== + +== Action + +=== Create the fedora-final.conf configuration file using the script + +The script can be found https://pagure.io/pungi-fedora/blob/main/f/create-candidate-configs.py[here]. + +There is a mandatory argument to specify the milestone to generate (beta or final) +and optional arguments for the major and minor versions for the label. + +Example usage: +`./create-candidate-configs.py --minor=5 final` + +=== Review Compose Tags + +. List any pre-existing builds in the current compose tag ++ +[source,subs="attributes+"] +.... +$ koji list-tagged f{branched}-compose +.... +. Verify preexisting builds are in compose tags ++ +The tagged builds from the previous composes should all be present in +the output from the previous step. Consult the request ticket for the +list of builds expected in this output. The request ticket may list +commands to un-tag existing tagged builds and tag the new requested +builds; if so, you can review these commands and run them. Otherwise, +the steps to do this manually are documented below. ++ +[NOTE] +==== +The very first run of a Beta or GA compose should have no builds +listed under the compose tag. It is important to clear preexisting +builds from the compose tag when moving between the Beta and RC +composes. Verify that these builds were removed. + +[source,subs="attributes+"] +.... +$ koji list-tagged f{branched}-compose +$ koji untag-build --all f{branched}-compose [build1 build2 ...] +.... +==== ++ +[NOTE] +==== +The order in which packages are added into the +f{branched}-compose tag matter. If the builds are untagged +erroneously then special attention should be given to adding them back +correctly. +==== +. Add builds specified by QE to the current compose tag ++ +[source,subs="attributes+"] +.... +$ koji tag-build f{branched}-compose [build1 build2 ...] +.... ++ +[NOTE] +==== +These steps may be completed on a local machine as long as the user has +appropriate permissions in the Koji tool. +==== + +=== Running the Compose + +. Composes use a configuration file to construct the compose. +Each compose uses its own configuration. +The `global_release` variable should start from 1.1 and the second number +should increment each time a new compose is created. + +* GA - `fedora-final.conf` +. Log into the compose backend ++ +.... +$ ssh compose-x86-01.rdu3.fedoraproject.org +.... +. Open a screen session ++ +.... +$ screen +.... +. Obtain the pungi-fedora branch for the current compose ++ +The first time any user account executes a compose the pungi-fedora git +repository must be cloned. The compose candidate script that invokes +pungi should be run from `compose-x86-01.rdu3.fedoraproject.org`. ++ +.... +$ git clone ssh://git@pagure.io/pungi-fedora.git +.... ++ +Enter the pungi-fedora directory. ++ +.... +$ cd pungi-fedora +.... ++ +If the clone step above was not required then fully update the existing +repository checkout from pagure. ++ +[source,subs="attributes+"] +.... +$ git fetch origin +$ git checkout f{branched} +$ git pull origin f{branched} +.... +. Run the compose and redirect the output to a file (in case it's needed later) ++ +[source,subs="attributes+"] +.... +$ sudo ./release-candidate.sh {branched} RC-#.# >& {branched}_RC-#.#.out +# Example: +# $ sudo ./release-candidate.sh {branched} RC-1.3 >& {branched}_RC-1.3.out +.... ++ +The first argument is the release number. The second argument is the +compose label. The numbering scheme begins with 1.1 and the second +number is incremented after each compose. The first number always +remains at 1 for Fedora composes. ++ +[NOTE] +==== +productmd requires the label version to be in this format (because +both numbers may be incremented for other products that use productmd +metadata, and have different meanings). It is because of this that +composes always start with the number 1 and the second number is +incremented with each compose. +==== ++ +[NOTE] +==== +If the compose fails with a directory missing error, then create the +compose directory with `mkdir /mnt/koji/compose/{branched}` +==== + +=== Verification + +The method for verifying a compose has completed is checking +`/mnt/koji/compose/{branched}/[compose_dir]/STATUS`. Any status +other than DOOMED is OK. + diff --git a/modules/release_guide/pages/sop_final_freeze.adoc b/modules/release_guide/pages/sop_final_freeze.adoc new file mode 100644 index 0000000..edcbd9e --- /dev/null +++ b/modules/release_guide/pages/sop_final_freeze.adoc @@ -0,0 +1,100 @@ +include::_partials/attributes.adoc[] + += Fedora Final Freeze + + +== Make sure all retired packages are blocked in koji + +Before the freeze we have to make sure all retired packages have been blocked in koji. +To do this, [run the script](https://pagure.io/releng/blob/main/f/scripts/get_retired_packages.sh) that collects all the retired packages +and outputs them into json files in the [lookaside cache](https://src.fedoraproject.org/lookaside). + +.... +$ ssh pkgs01.fedoraproject.org +$ cd releng/scripts && bash get_retired_packages.sh +.... + +This script will run about 20 minutes. Once finished, run toddler 'koji_block_retired' to block all retired packages in koji. +Web console: https://console-openshift-console.apps.ocp.fedoraproject.org + +.... +oc login --token= --server=https://api.ocp.fedoraproject.org:6443 +oc project poddlers +oc rsh dc/koji_block_retired +python3 /code/toddlers/playtime.py koji_block_retired +.... + + +Before freezing everything we need to do the last stable push in bodhi. + +== Stable push + +[source,subs="attributes+"] +.... +$ sudo -u apache bodhi-push --releases --stable 'f{branched}' +.... + +== Update Ansible vars + +Set the https://pagure.io/fedora-infra/ansible/blob/main/f/vars/all/Frozen.yaml[_NextReleaseFrozen_ variable] to `True`. + +=== Run the playbook + +.... + +$ rbac-playbook groups/koji-hub.yml +.... + +== Update Bodhi release + +[source,subs="attributes+"] +.... +$ bodhi releases edit --name F{branched} --state frozen +.... + +=== Reminder announcement + +Send an email as mentioned below to `devel@`, `devel-announce@`, `test-announce@`, `announce@` lists as information that the final freeze is now active. + +For the reference, you can refer generic https://pagure.io/releng/blob/main/f/mail-templates/06-final-freeze.txt[template] for final freeze from release engineering repo. + +[source,subs="attributes+"] +.... +Hi all, + +Today, YYYY-MM-DD, is an important day on the Fedora Linux {branched} schedule +[1], with significant cut-offs. + +Today we have the Final Freeze [2] which starts at 14:00 UTC. This means +that only packages which fix accepted blocker or freeze exception bugs +[3][4][5] will be marked as 'stable' and included in the Final composes. +Other builds will remain in updates-testing until the Final release is +approved, at which point the Final freeze is lifted and packages can +move to the 'updates' repository. Pending updates will be pushed before +final release as zero day updates. + +Regards, +Fedora Release Engineering + +[1] https://fedorapeople.org/groups/schedule/f-{branched}/f-{branched}-key-tasks.html +[2] https://fedoraproject.org/wiki/Milestone_freezes +[3] https://fedoraproject.org/wiki/QA:SOP_blocker_bug_process +[4] https://fedoraproject.org/wiki/QA:SOP_freeze_exception_bug_process +[5] https://qa.fedoraproject.org/blockerbugs/milestone/f{branched}/final/buglist + +.... + +[NOTE] +==== +Please ensure that the reminder email is sent once the state becomes frozen. +==== + +== During the Final freeze + +=== Process stable push requests + +During freezes we need to push updates that fix blocker and freeze exception +issues to stable when requested. +QA will file a ticket with the updates to push, which will include the +appropriate command(s). Verify the command(s) and run them. + diff --git a/modules/release_guide/pages/sop_final_release.adoc b/modules/release_guide/pages/sop_final_release.adoc new file mode 100644 index 0000000..1d34b54 --- /dev/null +++ b/modules/release_guide/pages/sop_final_release.adoc @@ -0,0 +1,136 @@ +include::_partials/attributes.adoc[] + += Fedora Final Release + +== Koji changes + +[source,subs="attributes+"] +.... +$ koji edit-tag --lock f{branched} +.... + +== Bodhi Changes + +Set the bodhi release to `current` +[source,subs="attributes+"] +.... +$ bodhi releases edit --name F{branched} --state current +$ bodhi releases edit --name F{branched}F --state current +.... + +Set the bodhi stable tag to point to updates instead of base repo +[source,subs="attributes+"] +.... +$ bodhi releases edit --name F{branched} --stable-tag f{branched}-updates +.... + +Set EOL of oldest release to correct date: four weeks after the official release date of {branched}. +[source,subs="attributes+"] +.... +$ bodhi releases edit --name F{old_release} --eol YYYY-MM-DD +$ bodhi releases edit --name F{old_release}C --eol YYYY-MM-DD +$ bodhi releases edit --name F{old_release}F --eol YYYY-MM-DD +.... + +If this date is different from the one in the https://fedorapeople.org/groups/schedule/f-{old_release}/f-{old_release}-key-tasks.html[Fedora schedule], +ask the program manager to update the date in the schedule. +If it is different from the one in F{old_release}'s `/etc/os-release`, https://bugzilla.redhat.com/enter_bug.cgi?product=Fedora&format&component=fedora-release&version=40[file a bug against fedora-release] asking for it to be updated. + +You may also wish to check the current EOL estimates for other releases on Bodhi, the schedule, and fedora-release, +and make sure they are in sync and sensible. + +== Atomic desktops changes + +Run the releng scripts/update_ostree_refs.sh on a compose machine with /mnt/koji/ mounted. + +$ scripts/update_ostree_refs.sh {branched} + +This updates the refs for Atomic desktops to have users follow updates +for the new release rather than being stuck on the GA compose forever. + +== Ansible Changes + +=== Update releng roles updates + +. Set https://pagure.io/fedora-infra/ansible/blob/main/f/vars/all/00-FedoraCycleNumber.yaml[FedoraCycleNumber] to {branched}. + +. Set https://pagure.io/fedora-infra/ansible/blob/main/f/vars/all/FedoraBranchedBodhi.yaml[FedoraBranchedBodhi] to current. + +. Set https://pagure.io/fedora-infra/ansible/blob/main/f/vars/all/FedoraPreviousPrevious.yaml[FedoraPreviousPrevious] to True. + +. Set https://pagure.io/fedora-infra/ansible/blob/main/f/vars/all/FedoraBranched.yaml[FedoraBranched] to False. + +. Set https://pagure.io/fedora-infra/ansible/blob/main/f/vars/all/Frozen.yaml[NextReleaseFrozen] to False. + +Now run the relevant playbooks: + +.... +$ sudo rbac-playbook groups/koji-hub.yml +$ sudo rbac-playbook groups/releng-compose.yml +$ sudo rbac-playbook openshift-apps/bodhi.yml +$ sudo rbac-playbook groups/bodhi-backend.yml +$ sudo rbac-playbook openshift-apps/greenwave.yml +$ sudo rbac-playbook groups/openqa.yml -t testcase_stats +.... + +If you do not have permissions to run a playbook, ask a sysadmin-main member for help. +A sysadmin-qa member can run the openQA playbook. + +For reference, see the https://pagure.io/fedora-infra/ansible/c/7e501dcb9432885c9ef10e78c0418b0d40c85061[Fedora 41 release PR], +though note it is somewhat old and includes some things that are no longer needed, and leaves out some that should be done. + +== Stage Final Release for Mirrors + +To prepare for running the staging script, make sure you have the following information: + +. Release Version: This is the numerical version number of the release, for example, {branched}. +. ComposeID: The ID associated with the Compose, such as "Fedora-{branched}-20160614.0". +. Label: The label used for the location in stage, for example, "Compsoe label for the location in stage 39_RC-1.2." +. Key: The name of the release key, which can be "fedora-{branched}" or "fedora-{branched}-secondary," as examples. +. Prerelease: Set this to 0 or 1 to determine if the release should be placed in a testing environment or not. +. Arch (Optional): For secondary architectures, this parameter can be used to make changes to some internal locations. ++ +[source,subs="attributes+"] +.... +$ scripts/stage-release.sh {branched} Fedora-{branched}-20160614.0 {branched}_RC-1.2 fedora-{branched} 0 +.... +. Sync the release to the Red Hat internal archive following internally +documented + +[NOTE] +==== +Make sure to grab the directory size usage numbers which is used to send +an email to [.title-ref]#mirror-admin@lists.fedoraproject.org# list. +==== + +== Make sure there are no tagging leftovers from bodhi + +Sometimes builds are not properly tagged during the freeze process. +Make sure all builds are properly tagged, for more info about the [issue](https://pagure.io/releng/issue/11775) + +== Verification + +Verification is somewhat difficult as one cannot look at the content via +the web server due to permissions. Typically we ask somebody from the +Infrastructure team to give the tree a second set of eyes. + +Some things to check for: +. are the CHECKSUM files signed? +. are the spins/labs present? + +== Post Release + +=== One Week After the Release + +In the first week after the release MirrorManager still uses the files +at `fedora/linux/development/` and not at +`fedora/linux/releases/` + +Once enough mirrors have picked up the files in the release directory, +run the following playbook from batcave to change the paths in +MirrorManager: + +[source,subs="attributes+"] +.... +$ rbac-playbook -v /srv/web/infra/ansible/playbooks/manual/mirrormanager/move-devel-to-release.yml --extra-vars="version='42'" +.... diff --git a/modules/release_guide/pages/sop_mass_branching_checklist.adoc b/modules/release_guide/pages/sop_mass_branching_checklist.adoc new file mode 100644 index 0000000..4910a6a --- /dev/null +++ b/modules/release_guide/pages/sop_mass_branching_checklist.adoc @@ -0,0 +1,195 @@ +include::_partials/attributes.adoc[] + += Mass Branching Checklist + +// Merge things from google document to finalize the checklist + +== Description + +At each alpha freeze, a mass branching process is conducted to separate the pending release from `devel/` branch. This allows Rawhide to continue while the pending release undergoes bugfix and polish activities. + +== Action + +=== T-1 Day actions for mass branching + +==== Send Announcement + +* [ ] One day prior to mass branching, send out an announcement notifying the community about the upcoming mass branching. Mention that new Koji builds for Rawhide will be disabled during the process. + +==== Signing keys for rawhide+1 + +Action: add a separate SOP for the same + +==== Repos to Branch + +* [ ] Update the following repositories we need to prepare before the mass branching occurs: (Two Branches needs to be upserted (rawhide needs update, and the branched branch needs to be created) + +==== Updates Archive Yum Repo Seeding + +* [ ] Ask CoreOS team to seed the N paths for the updates-archive yum repo by running the https://pagure.io/releng/archive-repo-manager/blob/main/f/README.md?text=True#_163[steps here]. This is so tests won't start failing after branching. + +==== Pungi-Fedora + +===== Rawhide Branch + +* [ ] Check for all the RELEASE, RELEASE_TITLE, COMPSFILE (all this points to rawhide version) +* [ ] Make sure to update the sigkeys for the latest rawhide version +* [ ] Files to change: + * [ ] nightly.sh + * [ ] cloud-nightly.sh + * [ ] container-nightly.sh + * [ ] fedora-cloud.conf + * [ ] fedora-container.conf + * [ ] fedora.conf + +===== Branched (FN) Branch + +* [ ] Files to change: + * [ ] general.conf `runroot_tag = 'fN-build'` + * [ ] fedora.conf + * [ ] (All rawhide occurrences should be updated to FN version, release_version, global_ksurl, global_version) + * [ ] ostree use fN branch for config (Update "config_branch" to point to fN from main) + * [ ] nightly.sh + * [ ] (All rawhide occurrences should be updated to FN version, and TARGET_DIR needs to be changed to branched from rawhide, RELEASE_TITLE, FROM) + +==== Fedora Kickstarts + +* [ ] ~~Rawhide Branch~~ +* [ ] Branched (FN) Branch + * [ ] fedora-repo.ks (need to comment out the rawhide includes) + * [ ] fedora-iot.ks + * (All rawhide occurrences should be updated to FN version) + * References: https://pagure.io/fedora-kickstarts/c/f2812c2e279c2bcf8b5bb65e264a7b644e2c81e3?branch=f39 + +==== Fedora Comps + +===== Rawhide Branch + +* [ ] Update the Makefile to rawhide version +* [ ] For fn+1 (rawhide) file: Create a comps-fN+1.xml.in (copy it from the last file and update/sync it according to changesets) + - Sync Rawhide with changes from Fedora {branched}: make sure of including the changeset changes occured in the branched repo + +==== Workstation Ostree Config + +===== Rawhide/Main Branch + +* [ ] Add Fedora (rawhide version) repo configs ([[reference](https://pagure.io/workstation-ostree-config/c/53d2a6b48edbb7a24305d4ae99e052aa14354b9f?branch=main)]) + +===== Branched (FN) Branch + +* [ ] Change all the rawhide references to point to fN version (grep for rawhide and replace it with the branched version) + +==== Fedora Release + +===== Rawhide Branch + +* [ ] Need to update the dist_version and Version in the spec file, with a changelog entry + +===== Branched (FN) Branch + +* [ ] Need to update the release_name from rawhide to branched and is_rawhide should be 0 + +==== Fedora Repos + +===== Rawhide Branch + +* [ ] Need to update the dist_version and Version in the spec file, with a changelog entry +* [ ] Check for the presence of rawhide+1 key in the files, if not present add the key to it. for [example](https://src.fedoraproject.org/rpms/fedora-repos/c/53f6d1a7bd0b89ad7bccd95e8b4d52b5774e7420?branch=main). + +===== Branched (FN) Branch + +* [ ] In the branch, update the definition of rawhide_release, and create a changelog entry for [example](https://src.fedoraproject.org/rpms/fedora-repos/c/724c961a79f52abc7d450b1c33ccebed09de6099?branch=f39). + +==== Ansible + +===== Bodhi + +* [ ] Update the Bodhi2 role in the infra ansible repository. This includes updating `koji-sync-listener.py`, `new-updates-sync`, Pungi configs for RPM updates, and Bodhi templates. + +===== Enable Branched Compose + +* [ ] Enable the branched compose in the releng role of the infra ansible repository. + +===== Fedora Branched + +* [ ] Set `FedoraBranched` and `FedoraBranchedBodhi` variables to appropriate values in the infra ansible repository. + +===== Koji Hub + +* [ ] Update the Koji hub configuration to allow side tags for the new Rawhide tag. + +===== Robosignatory + +* [ ] Update the Robosignatory role in the infra ansible repository. Disable branched signing and add the new release. +* [ ] Always check if there is a new artifact that got introduced in the release cycle, if so then we need to handle it more properly with robosignatory and include it in relevant files. (confirm this more with seniors) + +===== Fedora Release + +* [ ] Update the `fedora-release` package in Rawhide and the Branched release, including necessary changes to `fedora-release.spec`. + +===== Fedora Repos + +* [ ] Update the `fedora-repos` package in Rawhide, Branched to receive new GPG keys and updated rawhide symlinks. +* [ ] Update the `fedora-repos` package in remaining stable release branches and updated rawhide symlinks. + +==== Toddlers + +* [ ] Add the new version to the toddlers App configuration, referring to the provided PR for guidance. (For instance: check this: https://pagure.io/fork/humaton/fedora-infra/ansible/c/36b9cb2097fcfc30024f2982d9396cd9e0da7233) + Changes to be done: + +---- +[consumer_config.scm_request_processor.branch_slas.f] +bug_fixes = 'YYYY-MM-DD' +security_fixes = 'YYYY-MM-DD' +---- + +=== T Day actions for mass branching + +(for reviewers here is a checklist that one can copy this on the releng tracker so we can keep track of everything needed in one place) + +==== Merge all the preparatory PRs: + +* [ ] Pungi-Fedora (https://pagure.io/pungi-fedora) +* [ ] Fedora Kickstarts (https://pagure.io/fedora-kickstarts) +* [ ] Fedora Comps (https://pagure.io/fedora-comps) +* [ ] Workstation Ostree Config (https://pagure.io/workstation-ostree-config/) +* [ ] Fedora Release (https://src.fedoraproject.org/rpms/fedora-release) +* [ ] Fedora Repos (https://src.fedoraproject.org/rpms/fedora-repos) +* [ ] Toddlers (https://pagure.io/fedora-infra/toddlers) + + +==== Merging and running ansible changes + +==== Push the Changes + +* [ ] Commit, push, and apply the changes using the corresponding ansible playbooks for various services. + +==== Disable Rawhide Builds in Koji + +* [ ] Configure an outage in Koji to disable Rawhide builds. +* [ ] Cancel all running builds for Rawhide by listing and selecting relevant tasks, and then cancel each task. + +==== Koji + +* [ ] Run the `make-koji-release-tags` script from the releng repository to handle builds from the new branch. + +==== Dist-Git + +* [ ] Create new branches in Git and update gitolite.conf to allow users to push to the new branches. +* [ ] Run the `mass-branching-git.py` script to create new branches based on the file generated by PDC. + +==== Bodhi + +* [ ] Link empty repos and create empty repos as necessary to prepare for the new release. +* [ ] Create rawhide releases in Bodhi using appropriate commands for various types of releases (e.g., standard, container, flatpak). +* [ ] Update MirrorManager to point to the new Rawhide release. +* [ ] Enable autosigning on the Branched release after the compose is completed. +* [ ] Perform ELN-related work, including updating image configurations and scripts. + +==== Fedora Container Base Image + +* [ ] Import new images for Rawhide and update tags for `fedora:rawhide` and `fedora:${RAWHIDE}`. + +==== Update Sync Script + +* [ ] Update the sync script in the releng repository with the new version. diff --git a/modules/release_guide/pages/sop_mass_rebuild.adoc b/modules/release_guide/pages/sop_mass_rebuild.adoc new file mode 100644 index 0000000..beb019d --- /dev/null +++ b/modules/release_guide/pages/sop_mass_rebuild.adoc @@ -0,0 +1,277 @@ +include::_partials/attributes.adoc[] + += Mass Rebuild + +== Description + +Periodically we do mass rebuilds of rawhide during the development +cycle. This SOP will outline the steps necessary to do this. + +== Assumptions + +This assumes that the mass rebuild has already been approved and +scheduled via release engineering and FESCo. Coordinate with +infrastructure as well for any needed koji updates. + +This also assumes that the mass rebuild does not need to be done in +dependency order, and that the mass rebuild does not involve a ABI +change. + +== Considerations + +* The most important thing to keep in mind while doing a mass rebuild is +to communicate clearly what actions are being performed and the status +of the rebuild. +* Check in on scripts frequently to avoid a long stalled command from +adding significant delays in completing the rebuild. +* Check with secondary arches, whether they up-to-date enough with +primary, create rebuild tag and target when they are. It will then take +care of rebuilds of the arch specific packages in appropriate kojis. +* Please make sure you have right ACLs to run the script mention below, one needs to be atleast in sysadmin-releng/releng-team/cvsadmin/sysadmin in order to get root access of the machine if required. + +== Actions + +=== Preparatory Steps + +The following steps may be completed in the weeks leading up to the +scheduled mass rebuild. + +. Create the Mass Rebuild Pagure Issue ++ +____ +Create an issue on the https://pagure.io/releng/issues[Release +Engineering issues page] that points at the schedule for the current +release. + +See https://pagure.io/releng/issue/6898[the Fedora 27 mass rebuild issue +example]. +____ +. Set up the Mass Rebuild Wiki Page ++ +____ +The mass rebuild wiki page should answer the following questions for +maintainers: + +* Why the mass rebuild is happening? +* How to opt out of the mass rebuild? + * Make sure to inform people either on mail, or in the wiki page about the `PKG_SKIP_LIST` which allows packages to be skipped from rebuilding as well as `noautobuild` file that they can add it to there root of a distgit repository and the mass rebuild script will skip it. +* What changes from changesets are taken in account for F{branched}? + +[NOTE,subs="attributes+"] +==== +See https://fedoraproject.org/wiki/Fedora_{branched}_Mass_Rebuild[the Fedora {branched} +Wiki example]. +==== +____ +. Send out the Mass Rebuild Notice ++ +____ +Send out the same information posted on the wiki to the +[.title-ref]#devel-announce@lists.fedoraproject.org# mailing list. Please check if the email is through, if stuck in moderation please ping respective members. + +[NOTE] +==== +See the template in releng repository +https://pagure.io/releng/blob/main/f/mail-templates/01-mass-rebuild-start.txt[template]. +==== +____ +. Create a Tag to Contain the Mass Rebuild ++ +____ +Mass rebuilds require their own tag to contain all related builds. The +example assumes we are doing a rebuild for Fedora {rawhide}. + +[source,subs="attributes+"] +.... +$ koji add-tag f{rawhide}-rebuild --parent f{rawhide} +.... +____ +. Request Package Auto-Signing for New Mass-Rebuild Tag ++ +____ +File a ticket with https://pagure.io/fedora-infrastructure/issues[Fedora +Infrastructure] requesting the new mass-rebuild tag be enabled for +package auto-signing. +____ +. Create the Koji Target for the Mass Rebuild ++ +____ +Using the same [.title-ref]#f{rawhide}-rebuild# tag created in the previous +example: + +[source,subs="attributes+"] +.... +$ koji add-target f{rawhide}-rebuild f{rawhide}-build +.... + +[NOTE] +==== +*koji add-target* _target-name_ _buildroot-tag_ _destination-tag_ +describes the syntax format above. If the _destination-tag_ is not +specified then it will be the same as the _target-name_. +==== +____ +. Update Scripts ++ +____ +The mass rebuild depends on four main scripts from the +https://pagure.io/releng[releng git repository]. Each one requires some +changes in variables for each new mass rebuild cycle. + +* {blank} ++ +mass-rebuild.py:: + ** buildtag + ** targets + ** epoch + ** comment + ** target +* {blank} ++ +find-failures.py:: + ** buildtag + ** desttag + ** epoch +* mass-tag.py +* {blank} ++ +need-rebuild.py:: + ** buildtag + ** target + ** updates + ** epoch +____ + +Change the following items: + +* the build tag, holding tag, and target tag should be updated to +reflect the Fedora release you're building for +* the `epoch` should be updated to the point at which all features that +the mass rebuild is for have landed in the build system (and a newRepo +task completed with those features) +* the comment which is inserted into spec changelogs +* for the references of the above mentioned changes refer to the https://pagure.io/releng/c/f8d89a58efc0b81b9358baadfeb2ba4f52edeeb4?branch=main[F40 mass rebuild PR]. + +=== Starting the Mass Rebuild + +The `mass-rebuild.py` script takes care of: + +* Discovering available packages in koji +* Trimming out packages which have already been rebuilt +* Checking out packages from git +* Bumping the spec file +* Committing the change +* git tagging the change +* Submitting the build request to Koji + +===== +. Connect to the mass-rebuild Machine ++ +____ +.... +$ ssh compose-branched01.rdu3.fedoraproject.org +.... +____ +. Start a terminal multiplexer (this ensures if user gets interrupted due to various reasons, the script can continue in a tmux session) ++ +____ +.... +$ tmux +.... +____ +. Clone or checkout the latest copy of the +https://pagure.io/releng[releng git repository]. +. Run the mass-rebuild.py script from _releng/scripts_ ++ +____ +.... +$ cd path/to/releng_repo/scripts +$ ./mass-rebuild.py 2>&1 | tee ~/massbuild.out +.... +____ +===== + +=== Monitoring Mass Rebuilds + +The community has a very high interest in the status of rebuilds and +many maintainers will want to know if their build failed right away. The +`find-failures.py` and `need-rebuild.py` scripts are designed to update +publicly available URLs for stakeholders to monitor. + +===== +. Connect to a Compose Machine or the monitoring scripts can be run on the same machine where the mass rebuild is happening. ++ +____ +.... +$ ssh compose-x86-02.rdu3.fedoraproject.org / compose-branched01.rdu3.fedoraproject.org:22 +.... +____ +. Start a terminal multiplexer (this ensures if user gets interrupted due to various reasons, the script can continue in a tmux session) ++ +____ +.... +$ tmux +.... +____ +. Clone or checkout the latest copy of the +https://pagure.io/releng[releng git repository] +. {blank} ++ +Set Up the Rebuild Failures Notification Web Site:: + The `find_failures.py` script discovers attempted builds that have + failed. It lists those failed builds and sorts them by package owner. + + + +[source,subs="attributes+"] +.... +$ while true; do ./find_failures.py > f{rawhide}-failures.html && cp f{rawhide}-failures.html /mnt/koji/mass-rebuild/f{rawhide}-failures.html; sleep 600; done +.... +. Start a second pane in the terminal emulator +. {blank} ++ +Set up the Site for Packages that Need Rebuilt:: + The `need-rebuild.py` script discovers packages that have not yet been + rebuilt and generates an html file listing them sorted by package + owner. This gives external stakeholders a rough idea of how much work + is remaining in the mass rebuild. + + + +[source,subs="attributes+"] +.... +$ while true; do ./need-rebuild.py > f{rawhide}-need-rebuild.html && cp f{rawhide}-need-rebuild.html /mnt/koji/mass-rebuild/f{rawhide}-need-rebuild.html; sleep 600; done +.... +===== + +=== Post Mass Rebuild Tasks + +Once the mass rebuild script completes, and all the pending builds have +finished, the builds will need to be tagged. The `mass-tag.py` script +will accomplish this task. The script will: + +* Discover completed builds +* Trim out builds that are older than the latest build for a given +package +* Tag remaining builds into their final destination (without generating +email) + +===== +. Clone or checkout the latest copy of the +https://pagure.io/releng[releng git repository] +. Run the `mass-tag.py` script (requires koji kerberos authentication) +. Remove the koji target for the rebuild: +$ koji remove-target f{rawhide}-rebuild ++ +____ +[source,subs="attributes+"] +.... +$ cd path/to/releng_repo/scripts +$ ./mass-tag.py --source f{rawhide}-rebuild --target f{rawhide} +.... +____ +. Send the final notification to the +_devel-announce@lists.fedoraproject.org_ list ++ +____ +For the contents use this https://pagure.io/releng/blob/main/f/mail-templates/02-mass-rebuild-finished.txt[template]. +____ +===== \ No newline at end of file diff --git a/modules/release_guide/pages/sop_post-branch_freeze.adoc b/modules/release_guide/pages/sop_post-branch_freeze.adoc new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/modules/release_guide/pages/sop_post-branch_freeze.adoc diff --git a/modules/release_guide/pages/sop_post-release_cleanup.adoc b/modules/release_guide/pages/sop_post-release_cleanup.adoc new file mode 100644 index 0000000..007b0e6 --- /dev/null +++ b/modules/release_guide/pages/sop_post-release_cleanup.adoc @@ -0,0 +1,81 @@ +include::_partials/attributes.adoc[] + += Post-Release Clean-up Documentation + +:page-description: Steps to clean up resources after a Fedora final release, including removing outdated content, updating configurations, and closing issues. + +== Overview + +This document outlines the necessary steps for cleaning up resources after the release of a Fedora version. It includes actions such as removing outdated content from servers, updating mirroring configurations, and closing open issues related to the release process. + +== Removing N_Beta from Torrents + +.Post-release clean-up of torrents +[discrete] +=== Description + +Remove beta versions of the Fedora release from the torrent server to prevent them from being distributed. This helps focus resources on the stable release. + +[discrete] +=== Steps + +. SSH into `torrent01.fedoraproject.org`. +. Navigate to the directory containing the {branched}_Beta torrents. +. Delete the beta torrents for the {branched} version. +. Regenerate the whitelist to exclude these torrents. + +== Update MirrorManager Configuration + +.MirrorManager redirection post-release +[discrete] +=== Description + +After the release, update MirrorManager to redirect requests from the development directory to the releases directory. + +[discrete] +=== Steps + +. Run the following playbook on batcave to update the paths in MirrorManager: ++ +[source,subs="attributes+"] +.... +$ rbac-playbook -v /srv/web/infra/ansible/playbooks/manual/mirrormanager/move-devel-to-release.yml --extra-vars="version='{branched}'" +.... + +== Cleanup of RCs and Unnecessary Composes + +.Cleanup of unnecessary composes and RCs +[discrete] +=== Description + +Remove release candidates and unnecessary composes that are no longer needed to free up storage. + +[discrete] +=== Steps + +. SSH into `bodhi-backend01.rdu3.fedoraproject.org` or any server where Koji is mounted. +. Perform the following: +** Remove all directories related to Beta and Final RCs in `/pub/alt/stage/`. +** Clean up old composes by removing all but the latest in `/mnt/koji/compose/branched/` and `/mnt/koji/compose/{branched}/`. +** Two weeks after release: Remove directories at `/pub/fedora/linux/development/{branched}` and `/pub/fedora-secondary/development/{branched}`. +** After MirrorManager playbook is run: Remove all but rawhide dirs at + `/pub/alt/development/`. + +== Closure of Failed Composes Issues + +.Closure of open issues related to failed composes +[discrete] +=== Description + +Address and close any open issues related to failed composes to maintain the health of the Fedora release process infrastructure. + +[discrete] +=== Steps + +. Use the script available in the Fedora RelEng repository: https://pagure.io/releng/blob/main/f/scripts/misc/failed_composes_cleanup.py ++ +[source,subs="attributes+"] +.... +python3 failed_composes_cleanup.py +.... +. Run the script to automatically identify and close open issues. diff --git a/modules/release_guide/pages/sop_release_eol.adoc b/modules/release_guide/pages/sop_release_eol.adoc new file mode 100644 index 0000000..bde5ec9 --- /dev/null +++ b/modules/release_guide/pages/sop_release_eol.adoc @@ -0,0 +1,296 @@ +include::_partials/attributes.adoc[] + += End Of Life + +== Description + +Each release of Fedora is maintained as laid out in the +https://docs.fedoraproject.org/en-US/releases/lifecycle/#_maintenance_schedule[maintenance +schedule]. At the conclusion of the maintenance period, a Fedora release +enters `end of life` status. This procedure describes the tasks +necessary to move a release to that status. To get the correct EOL date visit +https://fedorapeople.org/groups/schedule/f-{branched}/f-{branched}-key-tasks.html[release schedule] for the last release. + +== Actions + +=== Reminder announcement + +Around a week before the EOL date, end an email to `devel@`, `devel-announce@`, `test@`, `test-announce@`, `announce@` lists as remainder about the release EOL. +Use the https://pagure.io/releng/blob/main/f/mail-templates/08-end-of-life.txt[template] from release engineering repo. + +[NOTE] +==== +Please ensure that the reminder email is scheduled to be sent one week before the end of life. +==== + +=== `fedora-appstream-metadata` package update + +If necessary, update the +https://src.fedoraproject.org/rpms/fedora-appstream-metadata[`fedora-appstream-metadata`] +package with the new EOL date. Follow the instructions from the repo README. + +Build the package, create an update and make sure it lands in the stable repo +before proceeding. + +This is not necessary if the package already has the correct EOL date. + +=== Koji tasks + +* Disable builds by removing targets + +[source,subs="attributes+"] +.... +koji remove-target f{old_release} +koji remove-target f{old_release}-candidate +koji remove-target f{old_release}-container-candidate +koji remove-target f{old_release}-flatpak-candidate +koji remove-target f{old_release}-infra +koji remove-target f{old_release}-coreos-continuous +koji remove-target f{old_release}-rebuild +.... + +* Remove sidetags + +Remove any existing sidetags from the eol release. + +[source,subs="attributes+"] +.... +for i in `koji list-sidetags --basetag f{old_release}` +do + koji remove-sidetag $i +done +.... + +You need to do this differently because just removing the target of +a sidetag will leave it in a weird state where it cannot be removed. + + +* Purge from disk the signed copies of rpms that are signed with the +EOL'd release key. To achieve this, add the release key to +*koji_cleanup_signed.py* script in https://pagure.io/releng[releng] repo +and run the script on compose-branched01.rdu3.fedoraproject.org: + +.... +tmux +sudo su +cd /root/releng +./scripts/koji_cleanup_signed.py +.... + +[NOTE] +==== +Running this script in tmux or screen is recommended as it takes a long time to run. +==== + +=== Final stable push + +Do a final stable push for the release in Bodhi, following the +xref:releng_misc_guide:sop_pushing_updates.adoc[Pushing Updates SOP]. + +=== Bodhi tasks + +Run the following bodhi commands to set the releases state to +*archived* + +[source,subs="attributes+"] +.... +bodhi releases edit --name "F{old_release}" --state archived +bodhi releases edit --name "F{old_release}C" --state archived +bodhi releases edit --name "F{old_release}F" --state archived +.... + +=== Fedora Infra Ansible Changes + +We need to make some changes and then run several roles in the ansible repo. + +==== Update FedoraPreviousPrevious.yaml in ansible repository + +Edit /vars/all/FedoraPreviousPrevious.yaml and set the value to `False`. + +==== Adjust the EOL for gnome-software + +Edit /roles/pkgdb-proxy/files/pkgdb-gnome-software-collections.json in ansible +and set the release to status "EOL". + +==== Adjust the greenwave gating policy + +Edit /roles/openshift-apps/greenwave/templates/fedora.yaml +and find the policies with \_upgrade_ in their name, near the bottom. +You should see a comment noting these are not run on "the oldest stable release". +The oldest release listed in the `product_versions` section for each of these policies should be one higher than the release you are EOLing. +Remove the lines for these releases, so the oldest release listed is now *two* higher than the release you are EOLing. +You may also remove all lines for the release you are EOLing, but this is not critical. + +==== Remove eol release from robosignatory config + +Edit roles/robosignatory/templates/robosignatory.toml.j2 +and remove stansas that use "fedora-{old_release}" in them. + +==== Run the playbooks + +First, push the above edits. Then run the associated playbooks on _batcave01_: + +.... +sudo ansible-playbook /srv/web/infra/ansible/playbooks/groups/bodhi-backend.yml +sudo ansible-playbook /srv/web/infra/ansible/playbooks/groups/koji-hub.yml +sudo ansible-playbook /srv/web/infra/ansible/playbooks/groups/releng-compose.yml +sudo ansible-playbook /srv/web/infra/ansible/playbooks/groups/proxies.yml -t pkgdb2 +sudo ansible-playbook /srv/web/infra/ansible/playbooks/openshift-apps/bodhi.yml +sudo ansible-playbook /srv/web/infra/ansible/playbooks/openshift-apps/greenwave.yml +.... + +You will need to have someone enable sshd on autosign02 and then run: + +.... +sudo ansible-playbook /srv/web/infra/ansible/playbooks/manual/autosign.yml +.... + +And then someone with the robosignatory passphrase will need to restart it. + +[NOTE] +==== +Another way to run the playbook is using rbac-playbook, in case you don't have sysadmin-main rights or can't become root. +Syntax: sudo rbac-playbook groups/bodhi-backend.yml +==== + +=== Final announcement + +Send the final announcement to `devel@`, `devel-announce@`, `test@`, `test-announce@`, `announce@` lists. +Use the https://pagure.io/releng/blob/main/f/mail-templates/08-end-of-life.txt[template] from release engineering repo. + +=== Other tasks + +==== Remove EOL release from Koschei + +The release going EOL must be removed from Koschei to prevent a flood of failed scratch builds. +See the xref:sysadmin_guide:koschei.adoc#_removing_an_eol_fedora_release[Koschei sysadmin guide] for details. + +==== Remove release from fedfind source of truth + +The Quality team maintains a source of truth for release information used by the fedfind tool +on fedorapeople (people01) in `/srv/groups/qa/metadata/release.json`. +If you have access to this file, you can remove the release going EOL from the file's list of stable releases. +Otherwise, alert the quality team to do so. + +=== Stage the EOL release to archive + +. Log into to bodhi-backend01 and become ftpsync, via root ++ +____ +.... +ssh bodhi-backend01.rdu3.fedoraproject.org +sudo su +su - ftpsync +.... +____ +. Then change into the releases directory. ++ +____ +.... +cd /pub/fedora/linux/releases +.... +____ +. Check to see that the target directory doesn't already exist. ++ +____ +.... +ls /pub/archive/fedora/linux/releases/ +.... +____ +. Do a recursive rsync to update any changes in the trees since the previous copy. ++ +____ +[source,subs="attributes+"] +.... +rsync -avAXSHP ./{old_release}/ /pub/archive/fedora/linux/releases/{old_release}/ +.... +____ +. Similarly, sync updates and updates/testing. ++ +____ +[source,subs="attributes+"] +.... +cd ../updates/ +rsync -avAXSHP {old_release}/ /pub/archive/fedora/linux/updates/{old_release}/ +cd testing +rsync -avAXSHP {old_release}/ /pub/archive/fedora/linux/updates/testing/{old_release}/ +.... +____ +. Do the same with fedora-secondary. ++ +____ +[source,subs="attributes+"] +.... +cd /pub/fedora-secondary/releases/ +rsync -avAXSHP ./{old_release}/ /pub/archive/fedora-secondary/releases/{old_release}/ +cd ../updates/ +rsync -avAXSHP {old_release}/ /pub/archive/fedora-secondary/updates/{old_release}/ +cd testing +rsync -avAXSHP {old_release}/ /pub/archive/fedora-secondary/updates/testing/{old_release}/ +.... +____ +. Announce to the mirror list this has been done and that in 2 weeks you will +move the old trees to archives. + +=== Move the EOL release to archive + +. In two weeks, run the following playbook from batcave: +____ +[source,subs="attributes+"] +.... +sudo ansible-playbook -v /srv/web/infra/ansible/playbooks/manual/mirrormanager/move-to-archive.yml --extra-vars="product='Fedora' version='{old_release}'" +.... +____ +. If there are problems, the postgres DB may have issues and so you need +to get a DBA to update the backend to fix items. +. Wait an hour or so then you can remove the files from the main tree. + +[NOTE] +==== +We need to do a cleanup here for both fedora and fedora-secondary for releases, updates, and updates/testing for the {old_release} release. +==== + +____ +[source,subs="attributes+"] +.... +ssh bodhi-backend01 +cd /pub/fedora/linux/releases/{old_release} +ls +# make sure you have stuff here +rm -rf * +ln ../20/README . +cd /pub/fedora/linux/updates/{old_release} +ls +# make sure you have stuff here +rm -rf * +ln ../20/README . +cd /pub/fedora/linux/updates/testing/{old_release} +ls +# make sure you have stuff here +rm -rf * +ln ../20/README . + +#for fedora-secondary +cd /pub/fedora-secondary/releases/{old_release} +ls +# make sure you have stuff here +rm -rf * +ln ../20/README . +cd /pub/fedora-secondary/updates/{old_release} +ls +# make sure you have stuff here +rm -rf * +ln ../20/README . +cd /pub/fedora-secondary/updates/testing/{old_release} +ls +# make sure you have stuff here +rm -rf * +ln ../20/README . +.... +____ + +== Consider Before Running + +* Resource contention in infrastructure, such as outages +* Extenuating circumstances for specific planned updates, if any +* Send the reminder announcement, if it isn't sent already diff --git a/modules/release_guide/pages/sop_retire_orphaned_packages.adoc b/modules/release_guide/pages/sop_retire_orphaned_packages.adoc new file mode 100644 index 0000000..8dac570 --- /dev/null +++ b/modules/release_guide/pages/sop_retire_orphaned_packages.adoc @@ -0,0 +1,86 @@ +include::_partials/attributes.adoc[] + +== Retire Orphaned Packages + +=== Description + +Every release prior to the +https://fedoraproject.org/wiki/Schedule[Feature Freeze/Branching] +Release Engineering retires +https://fedoraproject.org/wiki/Orphaned_package_that_need_new_maintainers[orphaned +packages]. This keeps out unowned software and prevents future problems +down the road. + +=== Action + +The orphan process takes place in stages: + +. Detecting a list of orphans and the dependencies that will be broken +if the orphans are removed. +. Sending the list of potential orphans to devel@lists.fedoraproject.org +for community review and removal from the orphan list. +. Retriring packages nobody wants to adopt. + +==== Detecting Orphans + +A script called `find_unblocked_orphans.py` assists in the detection +process. It should be run on a machine that has `koji` and +`python-fedora` installed. It runs without options and takes a while to +complete. + +`find_unblocked_orphans.py` is available in the +https://pagure.io/releng[Release Engineering git repository] + +==== Announcing Packages to be retired + +`find_unblocked_orphans.py` outputs text to stdout on the command line +in a form suitable for the body of an email message. + +.... +$ ./find-unblocked-orphans.py > email-message +.... + +Email the output to the development list +(`devel@lists.fedodraproject.org`) at least a month before the feature +freeze, send mails with updated lists as necessary. This gives +maintainers an opportunity to pick up orphans that are important to them +or are required by other packages. + +==== Retiring Orphans + +Once maintainers have been given an opportunity to pick up orphaned +packages, the remaining +https://fedoraproject.org/wiki/How_to_remove_a_package_at_end_of_life[packages +are retired] + +===== Bugs + +This procedure probably leaves open bugs for the retired packages +behind. It is not within the scope of release engineering to take care +of these. If bugs are closed, only bugs targeted at Rawhide should be +affected, since other branches might still be maintained. + +=== Verification + +To verify that the packages were blocked correctly we can use the +`latest-pkg` `koji` action. + +[source,subs="attributes+"] +.... +$ koji latest-pkg dist-f{branched} wdm +.... + +This should return nothing, as the `wdm` package is blocked. + +=== Consider Before Running + +Generally we retire anything that doesn't leave broken dependencies. If +there are orphans whose removal would result in broken dependencies a +second warning should be sent to `devel@lists.fedoraproject.org` and to +`-owner@fedoraproject.org` for each dependent package. + +Allow another couple of days for maintainers to take notice and fix +these package so the package repository can be maintained without broken +dependencies or needing to the package. It is not good to have broken +package dependencies in our package repositories so every effort should +be made to find owners or to fix the broken dependencies. diff --git a/modules/release_guide/pages/sop_torrent_releases.adoc b/modules/release_guide/pages/sop_torrent_releases.adoc new file mode 100644 index 0000000..49a8c3c --- /dev/null +++ b/modules/release_guide/pages/sop_torrent_releases.adoc @@ -0,0 +1,152 @@ += Torrent Releases Infrastructure SOP + + +http://torrent.fedoraproject.org/ is our master torrent server for +Fedora distribution. It runs out of ibiblio. + + +== Contact Information + +Owner:: + Fedora Infrastructure Team +Contact:: + #fedora-admin, sysadmin-torrent group +Location:: + ibiblio +Servers:: + torrent.fedoraproject.org +Purpose:: + Provides the torrent master server for Fedora distribution + +== Torrent Release + +When you want to add a new torrent to the tracker at +http://torrent.fedoraproject.org you need to take the following +steps to have it listed correctly: + +[arabic] +. login to _torrent01.fedoraproject.org_. If you are unable to do so please +contact the fedora infrastructure group about access. This procedure +requires membership in the torrentadmin group. +. Change the group ID to torrentadmin ++ +.... +newgrp torrentadmin +.... +. Remove everything from the working directory `/srv/torrent/new/fedora/` ++ +.... +rm -r /srv/torrent/new/fedora/* +.... +. rsync all the iso's from ibiblio ++ +.... +rsync -avhHP4 rsync://download-ib01.fedoraproject.org/fedora-buffet0/fedora/linux/releases/test/_/*/*/iso/ /srv/torrent/new/fedora/ +.... +. cd into `/srv/torrent/new/fedora/` to change the directory structure ++ +.... +cd /srv/torrent/new/fedora/ +.... +. Directories should be created by removing Label in the iso's name ++ +.... +for iso in $(ls *iso); do dest=$(echo $iso|sed -e 's|-