#10982 Add script for executing the FESCo SIG Policy
Merged 2 years ago by kevin. Opened 2 years ago by decathorpe.
Unknown source main  into  main

@@ -0,0 +1,24 @@

+ This is free and unencumbered software released into the public domain.

+ 

+ Anyone is free to copy, modify, publish, use, compile, sell, or

+ distribute this software, either in source code form or as a compiled

+ binary, for any purpose, commercial or non-commercial, and by any

+ means.

+ 

+ In jurisdictions that recognize copyright laws, the author or authors

+ of this software dedicate any and all copyright interest in the

+ software to the public domain. We make this dedication for the benefit

+ of the public at large and to the detriment of our heirs and

+ successors. We intend this dedication to be an overt act of

+ relinquishment in perpetuity of all present and future rights to this

+ software under copyright law.

+ 

+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,

+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF

+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.

+ IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR

+ OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,

+ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR

+ OTHER DEALINGS IN THE SOFTWARE.

+ 

+ For more information, please refer to <https://unlicense.org>

@@ -0,0 +1,176 @@

+ #!/usr/bin/python3

+ 

+ # sig_policy.py

+ # =============

+ #

+ # This script enacts the FESCo SIG Policy as documented here:

+ # https://docs.fedoraproject.org/en-US/fesco/SIG_policy/

+ #

+ # Author: Fabio Valentini <decathorpe@gmail.org>

+ # SPDX-License-Identifier: Unlicense

+ 

+ import argparse

+ import os

+ import sys

+ 

+ import requests

+ 

+ # (name of SIG group, ACL, package name filter)

+ POLICY = [

+     # Rust SIG: https://pagure.io/fesco/fesco-docs/pull-request/66

+     ("rust-sig", "commit", lambda x: x.startswith("rust-")),

+ ]

+ 

+ PAGURE_DIST_GIT_DATA_URL = "https://src.fedoraproject.org/extras/pagure_bz.json"

+ 

+ VALID_ACLS = ["ticket", "commit", "admin"]

+ 

+ 

+ def get_package_data() -> dict[str, list[str]]:

+     """

+     Download the latest cached mapping from source package name -> list of

+     (co)maintainers from pagure-dist-git.

+ 

+     Raises an exception if the HTTP GET request failed, or if the data is not

+     valid JSON in the expected format.

+     """

+ 

+     ret = requests.get(PAGURE_DIST_GIT_DATA_URL)

+     ret.raise_for_status()

+ 

+     data = ret.json()

+     rpms = data["rpms"]

+ 

+     return rpms

+ 

+ 

+ def add_package_acl(package: str, group: str, acl: str, token: str):

+     """

+     Send an HTTP POST request to the pagure API endpoint for modifying ACLs on

+     a project.

+ 

+     Raises an exception if an HTTP error status was returned, or if the network

+     request failed for other reasons.

+     """

+ 

+     if acl not in VALID_ACLS:

+         raise ValueError(f"Not a valid ACL: {acl}")

+ 

+     url = f"https://src.fedoraproject.org/api/0/rpms/{package}/git/modifyacls"

+ 

+     payload = {

+         "user_type": "group",

+         "name": group,

+         "acl": acl,

+     }

+ 

+     headers = {

+         "Authorization": f"token {token}",

+         "Content-Type": "application/x-www-form-urlencoded",

+     }

+ 

+     response = requests.post(url, data=payload, headers=headers)

+     response.raise_for_status()

+ 

+ 

+ def main() -> int:

+     cli = argparse.ArgumentParser()

+     cli.add_argument(

+         "--dry-run",

+         "-n",

+         dest="dry",

+         action="store_true",

+         help="print results but do not modify any data",

+     )

+     cli.add_argument(

+         "--api-token",

+         dest="token",

+         action="store",

+         default=None,

+         help="API token for src.fedoraproject.org (overrides PAGURE_API_TOKEN)",

+     )

+     args = cli.parse_args()

+ 

+     token = args.token or os.environ.get("PAGURE_API_TOKEN")

+     if not token:

+         print("PAGURE_API_TOKEN environment variable not set.", file=sys.stderr)

+         return 1

print("PAGURE_API_TOKEN environment variable not set.", file=sys.stderr)
return 1

You can just do sys.exit("PAGURE_API_TOKEN environment variable not set.")

+ 

+     try:

+         packages = get_package_data()

+ 

+     except IOError as ex:

+         print("Failed to fetch data from pagure-dist-git:", file=sys.stderr)

+         print(ex, file=sys.stderr)

+         return 1

+ 

+     # keep track of failed requests

+     failures = dict()

+ 

+     for (group, acl, filtr) in POLICY:

+         print(f"Processing group: {group}")

+ 

+         # keep track of candidate packages

+         candidates = []

+ 

+         for (package, maintainers) in packages.items():

+             # check if the package matches the filter set by the policy

+             if not filtr(package):

+                 continue

+ 

+             # check if the package is already retired on all branches

+             if maintainers == ["orphan"]:

+                 continue

+ 

+             # check if the package already has the group as co-maintainer

+             # FIXME: this cannot check whether the ACL is present but too low

+             if f"@{group}" not in maintainers:

+                 candidates.append(package)

+ 

+         if not candidates:

+             print(f"No pending actions for group {group!r}.")

+             print()

+             continue

+ 

+         # keep track of failed requests

+         failed = []

+ 

+         for candidate in candidates:

+             print(f"- add {group!r} with {acl!r} ACL to {candidate!r}")

+ 

+             if not args.dry:

+                 try:

+                     add_package_acl(candidate, group, acl, token)

+                 except Exception as ex:

+                     print(ex, file=sys.stderr)

+                     failed.append(candidate)

+ 

+         if failed:

+             failures[group] = failed

+ 

+         print()

+ 

+     if not failures:

+         print("Finished successfully.")

+         return 0

+ 

+     print("Finished with errors:")

+     for (group, failed) in failures:

+         for package in failed:

+             print(f"- failed to add {group!r} group to package {package!r}")

+ 

+     return 1

+ 

+ 

+ if __name__ == "__main__":

+     try:

+         exit(main())

exit is a wrapper intended for interactive use. Use sys.exit instead.

+ 

+     except KeyboardInterrupt:

+         print("Cancelled.")

print("Cancelled.")

This should go to stderr too.

+         exit(0)

+ 

+     except Exception as e:

+         print(e, file=sys.stderr)
    print(e, file=sys.stderr)

sys.exit(str(e)) as usual.

+         exit(1)

+ 

Initial version of the script to enforce the new SIG Policy, which was approved by FESCo:
https://docs.fedoraproject.org/en-US/fesco/SIG_policy/

The "--dry-run" option just prints what the script would do without making any actual HTTP POST requests to change package settings.

Adding new rules to the script should be fairly easy, for example:

  • ("r-maint-sig", "commit", lambda x: x.startswith("R-"))
  • ("go-sig", "commit", lambda x: x.startswith("golang-") or x in GOLANG_PACKAGES), with GOLANG_PACKAGES set to a list of additional packages

I have tested the script with the "--dry-run" option only, because I do not have the necessary privileges to actually run it.

Signed-off-by: There's nothing to sign off here. :shrug:

seems ok from a quick glance... perhaps we should put these under a fesco-policy/ directory instead of just under scripts/sig-policy ?

I guess it doesn't matter too much.

Can we setup a time when I can run the script and you are available in case of any issues to help fix things?

I wanted to add it to the fesco repo, but that one doesn't accept PRs for whatever reason, so I submitted it here. I can move it to a fesco subdirectory, if that helps.

@churchyard volunteered to run it regularly, i.e. when running the other scripts for fesco policies or the orphaned packages report. I should be able to respond to questions quickly between 0700 and 2200 UTC. But I intentionally made the code quite defensive, so if there's an error along the way for any of the packages, it should fail gracefully, continue processing, and print all steps that failed at the end.

This type of filtering will not work for Go. We need to be able to query for packages that BuildRequire go-rpm-macros and/or golang(*).

This is why it allows an arbitrary function as filter ...

You could add a function that runs the repoquery for the packages you want, store that list, and then check packages against that list. I have thought of this when I wrote the script. If you want, I can do the necessary implementation for cases like this later.

rebased onto e52a1c2c5bacc8c9b31d79b7943dd3bb62a5eb26

2 years ago

I made some small adjustments and moved the sig-policy directory from /scripts/ into /scripts/fesco/.

I wanted to add it to the fesco repo, but that one doesn't accept PRs for whatever reason, so I submitted it here. I can move it to a fesco subdirectory, if that helps.

The fesco repo has no contents. It is only meant for tickets.

I think here is the appropriate home for this.

print("PAGURE_API_TOKEN environment variable not set.", file=sys.stderr)
return 1

You can just do sys.exit("PAGURE_API_TOKEN environment variable not set.")

exit is a wrapper intended for interactive use. Use sys.exit instead.

print("Cancelled.")

This should go to stderr too.

    print(e, file=sys.stderr)

sys.exit(str(e)) as usual.

No. I don't like sys.exit(string). It creates additional, non-obvious return points in the script. Using print and return 1 is consistent with the other code, and ensures that the control flow is predictable.

rebased onto 1d637a9

2 years ago

Lets go ahead and merge and run...

Pull-Request has been merged by kevin

2 years ago

Seems to have run just fine:

Processing group: rust-sig                                                                            
- add 'rust-sig' with 'commit' ACL to 'rust-arrayvec0.5'
- add 'rust-sig' with 'commit' ACL to 'rust-atomic-traits' 
- add 'rust-sig' with 'commit' ACL to 'rust-aws-nitro-enclaves-cose'
- add 'rust-sig' with 'commit' ACL to 'rust-blsctl'   
- add 'rust-sig' with 'commit' ACL to 'rust-ciborium'
- add 'rust-sig' with 'commit' ACL to 'rust-ciborium-io'
- add 'rust-sig' with 'commit' ACL to 'rust-ciborium-ll'
- add 'rust-sig' with 'commit' ACL to 'rust-clap_generate'
- add 'rust-sig' with 'commit' ACL to 'rust-clap_generate_fig'
- add 'rust-sig' with 'commit' ACL to 'rust-clircle'    
- add 'rust-sig' with 'commit' ACL to 'rust-concolor-query'  
- add 'rust-sig' with 'commit' ACL to 'rust-cryptoki'      
- add 'rust-sig' with 'commit' ACL to 'rust-cryptoki-sys'       
- add 'rust-sig' with 'commit' ACL to 'rust-cty'                                                      
- add 'rust-sig' with 'commit' ACL to 'rust-dbus-codegen' 
- add 'rust-sig' with 'commit' ACL to 'rust-dbus-crossroads'
- add 'rust-sig' with 'commit' ACL to 'rust-derivative'
- add 'rust-sig' with 'commit' ACL to 'rust-gag'                                                      
- add 'rust-sig' with 'commit' ACL to 'rust-helvum'         
- add 'rust-sig' with 'commit' ACL to 'rust-is_ci'
- add 'rust-sig' with 'commit' ACL to 'rust-is_debug'
- add 'rust-sig' with 'commit' ACL to 'rust-josekit'
- add 'rust-sig' with 'commit' ACL to 'rust-js-sys'
- add 'rust-sig' with 'commit' ACL to 'rust-json5'
- add 'rust-sig' with 'commit' ACL to 'rust-keccak'
- add 'rust-sig' with 'commit' ACL to 'rust-krunvm'
- add 'rust-sig' with 'commit' ACL to 'rust-lebe'
- add 'rust-sig' with 'commit' ACL to 'rust-libnotcurses-sys'
- add 'rust-sig' with 'commit' ACL to 'rust-libspa'
- add 'rust-sig' with 'commit' ACL to 'rust-libspa-sys'
- add 'rust-sig' with 'commit' ACL to 'rust-nanorand'
- add 'rust-sig' with 'commit' ACL to 'rust-normpath'
- add 'rust-sig' with 'commit' ACL to 'rust-nu-engine'
- add 'rust-sig' with 'commit' ACL to 'rust-nu-parser'
- add 'rust-sig' with 'commit' ACL to 'rust-nu-protocol'
- add 'rust-sig' with 'commit' ACL to 'rust-oid'
- add 'rust-sig' with 'commit' ACL to 'rust-openssl-kdf'
- add 'rust-sig' with 'commit' ACL to 'rust-ouroboros0.9'
- add 'rust-sig' with 'commit' ACL to 'rust-ouroboros_macro0.9'
- add 'rust-sig' with 'commit' ACL to 'rust-owo-colors'
- add 'rust-sig' with 'commit' ACL to 'rust-parsec-client'
- add 'rust-sig' with 'commit' ACL to 'rust-parsec-interface'
- add 'rust-sig' with 'commit' ACL to 'rust-passwd'
- add 'rust-sig' with 'commit' ACL to 'rust-phf0.8'
- add 'rust-sig' with 'commit' ACL to 'rust-phf_generator0.8'
- add 'rust-sig' with 'commit' ACL to 'rust-phf_macros0.8'
- add 'rust-sig' with 'commit' ACL to 'rust-phf_shared0.8'
- add 'rust-sig' with 'commit' ACL to 'rust-picky-asn1'
- add 'rust-sig' with 'commit' ACL to 'rust-picky-asn1-der'
- add 'rust-sig' with 'commit' ACL to 'rust-picky-asn1-x509'
- add 'rust-sig' with 'commit' ACL to 'rust-pipewire'
- add 'rust-sig' with 'commit' ACL to 'rust-pipewire-sys'
- add 'rust-sig' with 'commit' ACL to 'rust-pkcs11'
- add 'rust-sig' with 'commit' ACL to 'rust-primal-bit'
- add 'rust-sig' with 'commit' ACL to 'rust-primal-check'
- add 'rust-sig' with 'commit' ACL to 'rust-process_control'
- add 'rust-sig' with 'commit' ACL to 'rust-prost'
- add 'rust-sig' with 'commit' ACL to 'rust-prost-build'
- add 'rust-sig' with 'commit' ACL to 'rust-prost-derive'
- add 'rust-sig' with 'commit' ACL to 'rust-prost-types'
- add 'rust-sig' with 'commit' ACL to 'rust-psa-crypto'
- add 'rust-sig' with 'commit' ACL to 'rust-psa-crypto-sys'
- add 'rust-sig' with 'commit' ACL to 'rust-rsa'
- add 'rust-sig' with 'commit' ACL to 'rust-sd-notify'
- add 'rust-sig' with 'commit' ACL to 'rust-secrecy'
- add 'rust-sig' with 'commit' ACL to 'rust-serde_with'
- add 'rust-sig' with 'commit' ACL to 'rust-sha3'
- add 'rust-sig' with 'commit' ACL to 'rust-shadow-rs'
- add 'rust-sig' with 'commit' ACL to 'rust-signal'
- add 'rust-sig' with 'commit' ACL to 'rust-simple_asn1'
- add 'rust-sig' with 'commit' ACL to 'rust-starship-battery'
- add 'rust-sig' with 'commit' ACL to 'rust-supports-color'
- add 'rust-sig' with 'commit' ACL to 'rust-supports-hyperlinks'
- add 'rust-sig' with 'commit' ACL to 'rust-supports-unicode'
- add 'rust-sig' with 'commit' ACL to 'rust-tss-esapi-sys'
- add 'rust-sig' with 'commit' ACL to 'rust-universal-hash'
- add 'rust-sig' with 'commit' ACL to 'rust-version'
- add 'rust-sig' with 'commit' ACL to 'rust-versions'
- add 'rust-sig' with 'commit' ACL to 'rust-wezterm-dynamic'

Finished successfully.

Great, looks like it finished without errors, and I also don't see any obvious errors.

I should be able to add rules for other SIGs pretty easily :)