From 37bcb83dede6aba033f82556d9b5c9e1bd6f7530 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Aug 14 2019 08:37:39 +0000 Subject: [PATCH 1/6] bundle db maintenance script to hub Fixes: https://pagure.io/koji/issue/1478 --- diff --git a/docs/source/server_howto.rst b/docs/source/server_howto.rst index b070b94..02482d4 100644 --- a/docs/source/server_howto.rst +++ b/docs/source/server_howto.rst @@ -497,6 +497,28 @@ there is no password manipulation support exposed through the koji tools. The sql commands you need to use vary by authentication mechanism. +Maintaining database +-------------------- + +For now, there is one table which needs periodical cleanup. As postgres doesn't +have any mechanism for this, we need to do it via some other mechanism. Default +handling is done by cron, but can be substituted by anything else (Ansible +tower, etc.) + +Script is by default installed on hub as `/usr/sbin/koji-sweepd-db`. On systemd +systems it also has corresponding `koji-sweep-db` service and timer. Note, that +timer is not enabled by default, so you need to run usual `systemctl` commands. +Cron files are not distributed, but are pretty easy to set up, if you prefer +one. + +If you don't want to use this script, be sure to run following SQL with +appropriate age setting. Default value of one day should be ok for normal +deployments. + +:: + + DELETE FROM sessions WHERE update_time < now() - '1 day'::interval; + Set User/Password Authentication ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/koji.spec b/koji.spec index 05eefe3..f8b8910 100644 --- a/koji.spec +++ b/koji.spec @@ -450,7 +450,7 @@ done make DESTDIR=$RPM_BUILD_ROOT PYTHON=%{__python3} %{?install_opt} install # alter python interpreter in koji CLI scripts='%{_bindir}/koji %{_sbindir}/kojid %{_sbindir}/kojira %{_sbindir}/koji-shadow - %{_sbindir}/koji-gc %{_sbindir}/kojivmd' + %{_sbindir}/koji-gc %{_sbindir}/kojivmd %{_sbindir}/koji-sweep-db' for fn in $scripts ; do sed -i 's|#!/usr/bin/python2|#!/usr/bin/python3|' $RPM_BUILD_ROOT$fn done @@ -534,6 +534,11 @@ rm -rf $RPM_BUILD_ROOT %dir /etc/koji-hub %config(noreplace) /etc/koji-hub/hub.conf %dir /etc/koji-hub/hub.conf.d +%{_sbindir}/koji-sweep-db +%if %{use_systemd} +%{_unitdir}/koji-sweep-db.service +%{_unitdir}/koji-sweep-db.timer +%endif %if 0%{py2_support} > 1 %files -n python2-%{name}-hub diff --git a/util/Makefile b/util/Makefile index 5ea8bde..92adfee 100644 --- a/util/Makefile +++ b/util/Makefile @@ -1,4 +1,4 @@ -BINFILES = kojira koji-gc koji-shadow +BINFILES = kojira koji-gc koji-shadow koji-sweep-db SYSTEMDSYSTEMUNITDIR = $(shell pkg-config systemd --variable=systemdsystemunitdir) TYPE = systemd @@ -29,6 +29,8 @@ _install: install-systemd: _install mkdir -p $(DESTDIR)$(SYSTEMDSYSTEMUNITDIR) install -p -m 644 kojira.service $(DESTDIR)$(SYSTEMDSYSTEMUNITDIR) + install -p -m 644 koji-sweep-db.service $(DESTDIR)$(SYSTEMDSYSTEMUNITDIR) + install -p -m 644 koji-sweep-db.timer $(DESTDIR)$(SYSTEMDSYSTEMUNITDIR) install-sysv: _install mkdir -p $(DESTDIR)/etc/rc.d/init.d diff --git a/util/koji-sweep-db b/util/koji-sweep-db new file mode 100755 index 0000000..806cadd --- /dev/null +++ b/util/koji-sweep-db @@ -0,0 +1,70 @@ +#!/usr/bin/python2 + +import os + +from optparse import OptionParser +from six.moves.configparser import RawConfigParser + +import koji.db + +def clean_sessions(cursor): + q = " FROM sessions WHERE update_time < now() - '1 day'::interval" + if options.verbose: + cursor.execute("SELECT COUNT(*) " + q) + rows = cursor.fetchall()[0][0] + print("Deleting %d sessions" % rows) + + cursor.execute("DELETE " + q) + + +if __name__ == "__main__": + global options + parser = OptionParser("%prog cleans koji database") + parser.add_option('-v', '--verbose', action="store_true", help="Be verbose") + parser.add_option('-c', '--conf', default='/etc/koji-hub/hub.conf', + action='store', help="Path to koji's hub.conf") + options, args = parser.parse_args() + + if not os.path.exists(options.conf): + parser.error("Config file doesn't exist") + + config = RawConfigParser() + config.read(options.conf) + + cfgmap = [ + #option, type, default + ['DBName', 'string', None], + ['DBUser', 'string', None], + ['DBHost', 'string', None], + ['DBhost', 'string', None], # alias for backwards compatibility + ['DBPort', 'integer', None], + ['DBPass', 'string', None], + ] + + opts = {} + for name, dtype, default in cfgmap: + key = ('hub', name) + if config and config.has_option(*key): + if dtype == 'integer': + opts[name] = config.getint(*key) + elif dtype == 'boolean': + opts[name] = config.getboolean(*key) + else: + opts[name] = config.get(*key) + continue + opts[name] = default + if opts['DBHost'] is None: + opts['DBHost'] = opts['DBhost'] + + + koji.db.provideDBopts(database=opts["DBName"], + user=opts["DBUser"], + password=opts.get("DBPass", None), + host=opts.get("DBHost", None), + port=opts.get("DBPort", None)) + + conn = koji.db.connect() + cursor = conn.cursor() + + clean_sessions(cursor) + conn.commit() diff --git a/util/koji-sweep-db.service b/util/koji-sweep-db.service new file mode 100644 index 0000000..af80afb --- /dev/null +++ b/util/koji-sweep-db.service @@ -0,0 +1,10 @@ +[Unit] +Description=Daily maintenance script for koji db +Documentation=https://pagure.io/docs/koji/ + +[Service] +Type=simple +ExecStart=/usr/sbin/koji-sweep-db + +[Install] +WantedBy=multi-user.target diff --git a/util/koji-sweep-db.timer b/util/koji-sweep-db.timer new file mode 100644 index 0000000..281c352 --- /dev/null +++ b/util/koji-sweep-db.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Daily maintenance script for koji db +Documentation=https://pagure.io/docs/koji/ + +[Timer] +OnCalendar=daily +Accuracy=1h +Persistent=true +Unit + +[Install] +WantedBy=timers.target From 8ae76401b7b0304fb5df4c98ca93fe3d13ef7fd2 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Aug 14 2019 08:37:39 +0000 Subject: [PATCH 2/6] remove Install section --- diff --git a/util/koji-sweep-db.service b/util/koji-sweep-db.service index af80afb..fb492a8 100644 --- a/util/koji-sweep-db.service +++ b/util/koji-sweep-db.service @@ -5,6 +5,3 @@ Documentation=https://pagure.io/docs/koji/ [Service] Type=simple ExecStart=/usr/sbin/koji-sweep-db - -[Install] -WantedBy=multi-user.target From 18809889845b7f4dfc6efb88467c8ab10145bf22 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Aug 14 2019 08:37:39 +0000 Subject: [PATCH 3/6] improve docs --- diff --git a/docs/source/server_howto.rst b/docs/source/server_howto.rst index 02482d4..6dc341f 100644 --- a/docs/source/server_howto.rst +++ b/docs/source/server_howto.rst @@ -507,12 +507,14 @@ tower, etc.) Script is by default installed on hub as `/usr/sbin/koji-sweepd-db`. On systemd systems it also has corresponding `koji-sweep-db` service and timer. Note, that -timer is not enabled by default, so you need to run usual `systemctl` commands. -Cron files are not distributed, but are pretty easy to set up, if you prefer -one. +timer is not enabled by default, so you need to run usual `systemctl` commands: + +:: + + systemctl enable --now koji-sweep-db.timer If you don't want to use this script, be sure to run following SQL with -appropriate age setting. Default value of one day should be ok for normal +appropriate age setting. Default value of one day should be ok for most deployments. :: From eedc00632ad5c0d63238a8b035e1f8c3f5ab5d09 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Aug 14 2019 08:37:39 +0000 Subject: [PATCH 4/6] remove typo --- diff --git a/util/koji-sweep-db.timer b/util/koji-sweep-db.timer index 281c352..7aa21e6 100644 --- a/util/koji-sweep-db.timer +++ b/util/koji-sweep-db.timer @@ -6,7 +6,6 @@ Documentation=https://pagure.io/docs/koji/ OnCalendar=daily Accuracy=1h Persistent=true -Unit [Install] WantedBy=timers.target From 5ced473a5fecb11803d32fe10b140a17ba572a25 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Aug 14 2019 08:37:39 +0000 Subject: [PATCH 5/6] Add vaccum to sessions cleanup --- diff --git a/docs/source/server_howto.rst b/docs/source/server_howto.rst index 6dc341f..a1e630f 100644 --- a/docs/source/server_howto.rst +++ b/docs/source/server_howto.rst @@ -515,11 +515,13 @@ timer is not enabled by default, so you need to run usual `systemctl` commands: If you don't want to use this script, be sure to run following SQL with appropriate age setting. Default value of one day should be ok for most -deployments. +deployments. As there will be tons of freed records, additional VACUUM can be +handy. :: DELETE FROM sessions WHERE update_time < now() - '1 day'::interval; + VACUUM ANALYZE sessions; Set User/Password Authentication ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/util/koji-sweep-db b/util/koji-sweep-db index 806cadd..39e305f 100755 --- a/util/koji-sweep-db +++ b/util/koji-sweep-db @@ -7,7 +7,7 @@ from six.moves.configparser import RawConfigParser import koji.db -def clean_sessions(cursor): +def clean_sessions(cursor, vacuum): q = " FROM sessions WHERE update_time < now() - '1 day'::interval" if options.verbose: cursor.execute("SELECT COUNT(*) " + q) @@ -15,6 +15,8 @@ def clean_sessions(cursor): print("Deleting %d sessions" % rows) cursor.execute("DELETE " + q) + if vacuum: + cursor.execute("VACUUM ANALYZE sessions") if __name__ == "__main__": @@ -23,6 +25,9 @@ if __name__ == "__main__": parser.add_option('-v', '--verbose', action="store_true", help="Be verbose") parser.add_option('-c', '--conf', default='/etc/koji-hub/hub.conf', action='store', help="Path to koji's hub.conf") + parser.add_option('--no-vacuum', action="store_false", dest="vacuum", + default=True, + help="Don't run vacuum on affected tables") options, args = parser.parse_args() if not os.path.exists(options.conf): @@ -66,5 +71,5 @@ if __name__ == "__main__": conn = koji.db.connect() cursor = conn.cursor() - clean_sessions(cursor) + clean_sessions(cursor, options.vacuum) conn.commit() From 36097b2ea6deec5fb6b5af1297a960cff6ea7dd7 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Aug 14 2019 08:56:44 +0000 Subject: [PATCH 6/6] clean build_reservations table --- diff --git a/docs/source/content_generators.rst b/docs/source/content_generators.rst index fcbe31d..4acbd8c 100644 --- a/docs/source/content_generators.rst +++ b/docs/source/content_generators.rst @@ -115,6 +115,8 @@ Metadata will be provided by the Content Generator as a JSON file. There is a proposal of the :doc:`Content Generator Metadata ` format available for review. +.. _cg_api: + API === diff --git a/docs/source/server_howto.rst b/docs/source/server_howto.rst index a1e630f..59834d4 100644 --- a/docs/source/server_howto.rst +++ b/docs/source/server_howto.rst @@ -518,11 +518,19 @@ appropriate age setting. Default value of one day should be ok for most deployments. As there will be tons of freed records, additional VACUUM can be handy. -:: +.. code-block:: sql - DELETE FROM sessions WHERE update_time < now() - '1 day'::interval; + DELETE FROM sessions WHERE update_time < NOW() - '1 day'::interval; VACUUM ANALYZE sessions; +Optionally (if you're using :ref:`reservation API ` for +content generators), you could want to run also reservation cleanup: + +.. code-block:: sql + + DELETE FROM build_reservations WHERE update_time < NOW() - '1 day'::interval; + VACUUM ANALYZE build_reservations; + Set User/Password Authentication ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/util/koji-sweep-db b/util/koji-sweep-db index 39e305f..8a83df2 100755 --- a/util/koji-sweep-db +++ b/util/koji-sweep-db @@ -19,6 +19,18 @@ def clean_sessions(cursor, vacuum): cursor.execute("VACUUM ANALYZE sessions") +def clean_reservations(cursor, vacuum): + q = " FROM build_reservations WHERE create_time < now() - '1 day'::interval" + if options.verbose: + cursor.execute("SELECT COUNT(*) " + q) + rows = cursor.fetchall()[0][0] + print("Deleting %d build reservations" % rows) + + cursor.execute("DELETE " + q) + if vacuum: + cursor.execute("VACUUM ANALYZE build_reservations") + + if __name__ == "__main__": global options parser = OptionParser("%prog cleans koji database") @@ -72,4 +84,5 @@ if __name__ == "__main__": cursor = conn.cursor() clean_sessions(cursor, options.vacuum) + clean_reservations(cursor, options.vacuum) conn.commit()