From fc566e4b31b143f367f9708f96faa9a3d8fd1f4c Mon Sep 17 00:00:00 2001 From: Brendan Reilly Date: Oct 30 2019 13:12:24 +0000 Subject: [PATCH 1/3] 1.19 release --- diff --git a/docs/source/conf.py b/docs/source/conf.py index 8281521..5a291a4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -54,9 +54,9 @@ copyright = u'2017, Mike McLean, Mike B, Dennis Gilmore, Mathieu Bridon, Ian McL # built documents. # # The short X.Y version. -version = '1.18' +version = '1.19' # The full version, including alpha/beta/rc tags. -release = '1.18.1' +release = '1.19.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/migrating_to_1.19.rst b/docs/source/migrating_to_1.19.rst new file mode 100644 index 0000000..e61d42c --- /dev/null +++ b/docs/source/migrating_to_1.19.rst @@ -0,0 +1,36 @@ +Migrating to Koji 1.19 +====================== + +.. + reStructured Text formatted + +You should consider the following changes when migrating to 1.19: + +DB Updates +---------- + +This release has a few schema changes: + + * A new ``tag_package_owners`` table + * A new ``user_krb_principals`` table + * Updates to the data in the ``archivetypes`` table + * Updates to the data in the ``permissions`` table + * The ``content_generator`` table now enforces unique strings in the ``names`` field + +As in previous releases, we provide a migration script that updates the +database. + +:: + + # psql koji koji 9.4 is required to run this script. + + +Other changes +------------- + +There are numerous other changes in 1.19 that should not have a direct impact +on migration. For details see: +:doc:`release_notes_1.19` diff --git a/docs/source/migrations.rst b/docs/source/migrations.rst index f594d52..71857c5 100644 --- a/docs/source/migrations.rst +++ b/docs/source/migrations.rst @@ -5,6 +5,7 @@ Migrations .. toctree:: :maxdepth: 1 + migrating_to_1.19 migrating_to_1.18 migrating_to_1.17 migrating_to_1.16 diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst index dfbf632..5f5c0f2 100644 --- a/docs/source/release_notes.rst +++ b/docs/source/release_notes.rst @@ -5,6 +5,7 @@ Release Notes .. toctree:: :maxdepth: 1 + release_notes_1.19 release_notes_1.18.1 release_notes_1.18 release_notes_1.17 diff --git a/docs/source/release_notes_1.19.rst b/docs/source/release_notes_1.19.rst new file mode 100644 index 0000000..bd68cc0 --- /dev/null +++ b/docs/source/release_notes_1.19.rst @@ -0,0 +1,416 @@ +Koji 1.19.0 Release notes +========================= + + +Migrating from Koji 1.18 +------------------------ + +For details on migrating see :doc:`migrating_to_1.19` + + + +Security Fixes +-------------- + +**GSSAPI authentication checks kerberos principal** + +| PR: https://pagure.io/koji/pull-request/1419 + +When using GSSAPI authentication the user's kerberos principal will be checked +for their username to avoid a potential username and kerberos principal mismatch. + + + +Client Changes +-------------- + +**Add user edit** + +| PR: https://pagure.io/koji/pull-request/902 +| PR: https://pagure.io/koji/pull-request/1701 +| PR: https://pagure.io/koji/pull-request/1713 + +A new ``edit-user`` command and API call was added, allowing for user rename, +and changing, adding, or removing the kerberos principal of a user. + + +**Add remove group** + +| PR: https://pagure.io/koji/pull-request/923 + +A new ``remove-group`` command was added, allowing the removal of a group +from a tag. It uses the existing ``groupListRemove`` API call. + + +**Query builds per chunks in prune-signed-builds** + +| PR: https://pagure.io/koji/pull-request/1589 + +For bigger installations querying all builds can cause the hub to run out of memory. +``prune-signed-builds`` now queries these in 50k chunks. + + +**Show inheritance flags in list-tag-inheritance output** + +| PR: https://pagure.io/koji/pull-request/1120 + +While not often used, tag inheritance can be modified with a few different options (e.g. maxdepth). +These options are shown in the ``taginfo`` display, but not the ``list-tag-inheritance`` display. +This change adds basic indicators to the latter. + + +**Return usage information in make-task** + +| PR: https://pagure.io/koji/pull-request/1157 + +``make-task`` now returns usage information if no arguments are provided. + + +**Clarify clone-tag usage** + +| PR: https://pagure.io/koji/pull-request/1623 + +The ``clone-tag`` help text now clarifies that the destination tag will be created +if it does not already exist. + + +**Add option check for list-signed** + +| PR: https://pagure.io/koji/pull-request/1631 + +The ``list-signed`` command will now fail if no options are provided. + + + +Library Changes +--------------- + +**Consolidate config reading style** + +| PR: https://pagure.io/koji/pull-request/1296 + +Changes have been made to make configuration handling more consistent. + +With this new implementation: + +* ``read_config_files`` is extended with a strict option and directory support +* ``ConfigParser`` is used for all invokings except kojixmlrpc and ``kojid`` +* ``RawConfigParser`` is used for ``kojid`` + + +**list_archive_files handles multi-type builds** + +| PR: https://pagure.io/koji/pull-request/1508 + +If ``list_archive_files`` is provided a build with multiple archive types it now correctly +handles them instead of failing. + + +**Disallow archive imports that don't match build type** + +| PR: https://pagure.io/koji/pull-request/1627 +| PR: https://pagure.io/koji/pull-request/1633 + +The ``importArchive`` call now refuses to proceed if the build does not have the given type. + + +**Add listCG RPC** + +| PR: https://pagure.io/koji/pull-request/1160 + +``listCGs`` has been added to list new content generator records. + +The purpose of this change is to make it easier for administrators to determine what +content generators are present and what user accounts have access to those. + + +**Add method to cancel CG reservations** + +| PR: https://pagure.io/koji/pull-request/1662 + +The new ``CGRefundBuild`` call allows CGs to cancel build reservations, such as in the case +of a failing build. + + +**Allow ClientSession objects to get cleaned up by the garbage collector** + +| PR: https://pagure.io/koji/pull-request/1653 + +This change ensures ``koji.ClientSession`` objects are destroyed once their requests are complete. + + +**Add missing package list check** + +| PR: https://pagure.io/koji/pull-request/1244 +| PR: https://pagure.io/koji/pull-request/1702 + +The ``host.tagBuild`` method was missing a check to ensure the package was actually listed in the +destination tag. This should now be checked as expected. + + +**Increase buildReferences SQL performance** + +| PR: https://pagure.io/koji/pull-request/1675 + +The performance for ``build_references`` has been improved. + + +**ensuredir does not duplicate directories** + +| PR: https://pagure.io/koji/pull-request/1197 + +``koji.ensuredir`` no longer creates duplicate directories if provided a path ending in a +forward slash. + + +**Warn users if buildroot uses yum instead of dnf** + +| PR: https://pagure.io/koji/pull-request/1595 + +This change sets the mock config ``dnf_warning`` to True for buildroots using yum. + + +**Tag permission can be used for tagBuildBypass and untagBuildBypass** + +| PR: https://pagure.io/koji/pull-request/1685 + +The ``tag`` permission can now be used in place of admin to call ``tagBuildBypass`` +and ``untagBuildBypass``. Admin is still required to use the ``--force`` option. + + +**Rework update of reserved builds** + +| PR: https://pagure.io/koji/pull-request/1621 + +This change reworks and simplifies the code that updates reserved build entries for cg imports. +It removes redundancy with checks in ``prep_build`` and avoids duplicate ``*BuildStateChange`` +callbacks. + + +**Use correct top limit for randint** + +| PR: https://pagure.io/koji/pull-request/1612 + +The top limit for ``randint`` has been set to 255 from 256 to prevent ``generate_token`` from +creating unneccesarily long tokens. + + +**Add strict option to getRPMFile** + +| PR: https://pagure.io/koji/pull-request/1068 + +``getRPMFile`` now has a ``strict`` option, failing when the RPM or filename does not exist. + + +**Stricter groupListRemove** + +| PR: https://pagure.io/koji/pull-request/1173 +| PR: https://pagure.io/koji/pull-request/1678 + +``groupListRemove`` now returns an error if the provided group does not exist for the tag. + + +**Clarified docs for build.extra.source** + +| PR: https://pagure.io/koji/pull-request/1677 + +The usage for ``build.extra.source`` has now been clarified in the ``getBuild`` call. + + +**Use bytes for debug string** + +| PR: https://pagure.io/koji/pull-request/1657 + +This change fixes debug output for Python 3. + + +**Removed host.repoAddRPM call** + +| PR: https://pagure.io/koji/pull-request/1680 + +The ``host.repoAddRPM`` call has been removed because it was unused and broken. + + + +Web UI Changes +-------------- + +**Made difference between Builds and Tags sections more clear** + +| PR: https://pagure.io/koji/pull-request/1676 + +The search page results for packages now has a clearer delineation between builds and tags. + + + +Builder Changes +--------------- + +**Use preferred arch when builder provides multiple** + +| PR: https://pagure.io/koji/pull-request/1684 + +When using ExclusiveArch for noarch builds the build task will now use the +arch specified instead of randomly picking from the arches the builder provides. + +This change adds a ``preferred_arch`` parameter to ``find_arch``. + + +**Log insufficient disk space location** + +| PR: https://pagure.io/koji/pull-request/1523 + +When ``kojid`` fails due to insufficient disk space, the directory which needs more +disk space is now included as part of the log message. + + +**Allow builder to attempt krb if gssapi is available** + +| PR: https://pagure.io/koji/pull-request/1613 + +``kojid`` will now use ``requests_kebreros`` for kerberos authentication when available. + + +**Add support for new mock exit codes** + +| PR: https://pagure.io/koji/pull-request/1682 + +``kojid`` now expects mock exit code 10 for failed builds (previously 1). + + +**Fix kickstart uploads for Python 3** + +| PR: https://pagure.io/koji/pull-request/1618 + +This change fixes the file handling of kickstarts for Python 3. + + + +System Changes +-------------- + +**Package ownership changes do not trigger repo regens** + +| PR: https://pagure.io/koji/pull-request/1473 +| PR: https://pagure.io/koji/pull-request/1643 + +Changing tag or package owners no longer cause repo regeneration. A new +``tag_package_owners`` table has been added for this purpose. + + +**Support multiple realms by kerberos auth** + +| PR: https://pagure.io/koji/pull-request/1648 +| PR: https://pagure.io/koji/pull-request/1696 +| PR: https://pagure.io/koji/pull-request/1701 + +This change adds a new table ``user_krb_principals`` which tracks a list of ``krb_principals`` +for each user instead of the previous one-to-one mapping. In addition: + +* all APIs related to user or krb principals are changed +* ``userinfo`` of ``getUser`` will contain a new list ``krb_principals`` + * ``krb_principals`` will contain all available principals if ``krb_princs=True`` +* there is a new hub option ``AllowedKrbRealms`` to indicate which realms are allowed +* there is a new client option ``krb_server_realm`` to allow krbV login to set server realm + * Previously same as client principal realm before, supported by all clients +* ``QueryProcessor`` has a new queryOpt ``group``, which is used to generate ``GROUP BY`` section + * By default, this feature is disabled by arg ``enable_group=False`` + + +**Added cronjob for sessions table maintenance** + +| PR: https://pagure.io/koji/pull-request/1492 + +The sessions table is now periodically cleaned up via script (handled by cron by default). +Without this the sessions table can grow large enough to affect Koji performance. + + +**Added basic email template for koji-gc** + +| PR: https://pagure.io/koji/pull-request/1430 + +The email message koji-gc uses has been moved to ``/etc/koji-gc/email.tpl`` for +easier customization. + + +**Add all permissions to database** + +| PR: https://pagure.io/koji/pull-request/1681 + +Permissions previously missing from schema have been added, including ``dist-repo``, ``host``, +``image-import``, ``sign``, ``tag``, and ``target``. + + +**Add new CoreOS artifact types** + +| PR: https://pagure.io/koji/pull-request/1616 + +This change adds the new CoreOS artifact types ``iso-compressed``, ``vhd-compressed``, +``vhdx-compressed``, and ``vmdk-compressed`` to the database. + + +**Enforce unique content generator names in database** + +| PR: https://pagure.io/koji/pull-request/1159 + +Set a uniqueness constraint on the content generator name in the database. +Prior to this change, we were only enforcing this in the hub application layer. +Configure this in postgres for safety. + + +**Fix typo preventing VM builds** + +| PR: https://pagure.io/koji/pull-request/1666 + +This change fixes the options passed to ``verifyChecksum`` which was preventing VM builds. + + +**Fix verifyChecksum for non-output files** + +| PR: https://pagure.io/koji/pull-request/1670 + +``verifyChecksum`` now accepts files under the build requires path as well as the output path. +Other paths can be added as needed. + + +**Set f30+ python-devel default** + +| PR: https://pagure.io/koji/pull-request/1683 + +When installed on a Fedora 30+ host with Python 2 support, Koji will now require +``python2-devel`` instead of ``python-devel``. + + +**Handle sys.exc_clear for Python 3** + +| PR: https://pagure.io/koji/pull-request/1642 + +The method ``sys.exc_clear`` does not exist in Python 3, so it has been escaped for those instances. + + +**Remove deprecated koji.util.relpath** + +| PR: https://pagure.io/koji/pull-request/1458 + +``koji.util.relpath`` was deprecated in 1.16 and has been removed from 1.19. + + +**Remove deprecated BuildRoot.uploadDir** + +| PR: https://pagure.io/koji/pull-request/1511 + +``BuildRoot.uploadDir`` was deprecated in 1.18 and has been removed from 1.19. + + +**Remove deprecated koji_cli.lib_unique_path** + +| PR: https://pagure.io/koji/pull-request/1512 + +``koji_cli.lib_unique_path`` was deprecated in 1.17 and has been removed from 1.19. + + +**Deprecation of sha1_constructor and md5_constructor** + +| PR: https://pagure.io/koji/pull-request/1490 + +``sha1_constructor`` and ``md5_constructor`` have been deprecated in favor of ``hashlib``. diff --git a/koji.spec b/koji.spec index 12e29b7..47aae90 100644 --- a/koji.spec +++ b/koji.spec @@ -81,7 +81,7 @@ %define release %{baserelease} %endif Name: koji -Version: 1.18.1 +Version: 1.19.0 Release: %{release}%{?dist} License: LGPLv2 and GPLv2+ # the included arch lib from yum's rpmUtils is GPLv2+ @@ -727,6 +727,70 @@ fi %endif %changelog +* Wed Oct 30 2019 Brendan Reilly - 1.19.0-1 +- PR#1713: cli: fix typo in edit-user cmd +- PR#1662: CGUninitBuild for cancelling CG reservations +- PR#1681: add all used permissions to db +- PR#1702: fix log message to show package name +- PR#1682: mostly only mock exit code 10 ends in build.log +- PR#1694: doc: change user creating sql for kerberos auth +- PR#1706: fix test for RHEL6 +- PR#1701: fix user operations typos +- PR#1296: extract read_config_files util for config parsing +- PR#1670: verifyChecksum fails for non-output files +- PR#1492: bundle db maintenance script to hub +- PR#1160: hub: new listCGs RPC +- PR#1120: Show inheritance flags in list-tag-inheritance output +- PR#1683: in f30+ python-devel defaults to python3 +- PR#1685: Tag permission can be used for un/tagBuildBypass +- PR#902: Added editUser api call +- PR#1684: use preferred arch if there is more options +- PR#1700: README: fix bullet indentation +- PR#1159: enforce unique content generator names in database +- PR#1699: remove references to PythonOption +- PR#923: Remove Groups CLI Call +- PR#1696: fix typo in createUser +- PR#1419: checking kerberos prinicipal instead of username in GSSAPI authentication +- PR#1648: support multiple realms by kerberos auth +- PR#1657: Use bytes for debug string +- PR#1068: hub: [getRPMFile] add strict behavior +- PR#1631: check options for list-signed +- PR#1688: clarify fixed/affected versions in cve announcement +- PR#1687: Docs updates for CVE-2019-17109 +- PR#1686: Fix for CVE-2019-17109 +- PR#1680: drop unused host.repoAddRPM call +- PR#1666: Fix typo preventing vm builds +- PR#1677: docs for build.extra.source +- PR#1675: Subselect gives better performance +- PR#1642: Handle sys.exc_clear in Python 3 +- PR#1157: cli: [make-task] raise readable error when no args +- PR#1678: swapped values in message +- PR#1676: Made difference between Builds and Tags sections more clear +- PR#1173: hub: [groupListRemove] raise Error when no group for tag +- PR#1197: [lib] ensuredir: normalize directory and don't throw error when dir exists +- PR#1244: hub: add missing package list check +- PR#1523: builder: log insufficent disk space location +- PR#1616: docs/schema-upgrade-1.18-1.19.sql/schema.sql: additional CoreOS artifact types. +- PR#1643: fix schema.sql introduced by moving owner from tag_packages to another table +- PR#1589: query builds per chunks in prune-signed-builds +- PR#1653: Allow ClientSession objects to get cleaned up by the garbage collector +- PR#1473: move tag/package owners to separate table +- PR#1430: koji-gc: Added basic email template +- PR#1633: Fix lookup_name usage + tests +- PR#1627: Don't allow archive imports that don't match build type +- PR#1618: write binary data to ks file +- PR#1623: Extend help message to clarify clone-tag usage +- PR#1621: rework update of reserved builds +- PR#1508: fix btype lookup in list_archive_files() +- PR#1223: Unit test download_file +- PR#1613: Allow builder to attempt krb if gssapi is available +- PR#1612: use right top limit +- PR#1595: enable dnf_warning in mock config +- PR#1458: remove deprecated koji.util.relpath +- PR#1511: remove deprecated BuildRoot.uploadDir() +- PR#1512: remove deprecated koji_cli.lib_unique_path +- PR#1490: deprecate sha1/md5_constructor from koji.util + * Fri Aug 9 2019 Mike McLean - 1.18.0-1 - PR#1606: pull owner from correct place - PR#1602: copy updated policy for reserved cg builds diff --git a/setup.py b/setup.py index 8029da9..21c789e 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def get_install_requires(): setup( name="koji", - version="1.18.1", + version="1.19.0", description=("Koji is a system for building and tracking RPMS. The base" " package contains shared libraries and the command-line" " interface."), From 6a57c7db213c3ffc18bbef44fd26492bcc778ace Mon Sep 17 00:00:00 2001 From: Brendan Reilly Date: Oct 30 2019 13:12:24 +0000 Subject: [PATCH 2/3] Updated migration doc --- diff --git a/docs/source/migrating_to_1.19.rst b/docs/source/migrating_to_1.19.rst index e61d42c..8578320 100644 --- a/docs/source/migrating_to_1.19.rst +++ b/docs/source/migrating_to_1.19.rst @@ -25,9 +25,6 @@ database. # psql koji koji 9.4 is required to run this script. - - Other changes ------------- From 4de3f6fa0f6ba9f48865c9adc3fe8e95317d9864 Mon Sep 17 00:00:00 2001 From: Brendan Reilly Date: Oct 30 2019 13:13:32 +0000 Subject: [PATCH 3/3] Updated changelog --- diff --git a/koji.spec b/koji.spec index 47aae90..659e886 100644 --- a/koji.spec +++ b/koji.spec @@ -728,6 +728,7 @@ fi %changelog * Wed Oct 30 2019 Brendan Reilly - 1.19.0-1 += PR#1720: backward-compatible db conversion - PR#1713: cli: fix typo in edit-user cmd - PR#1662: CGUninitBuild for cancelling CG reservations - PR#1681: add all used permissions to db