| |
@@ -0,0 +1,171 @@
|
| |
+ #!/bin/python3
|
| |
+ # -*- coding: utf-8 -*-
|
| |
+ #
|
| |
+ # prune-atomic-ostree-repos.py - A utility to prune our atomic ostree repos.
|
| |
+ #
|
| |
+ # Copyright (C) 2015 Red Hat, Inc.
|
| |
+ # SPDX-License-Identifier: GPL-2.0+
|
| |
+ #
|
| |
+ # Authors:
|
| |
+ # Dusty Mabe <dusty@dustymabe.com>
|
| |
+
|
| |
+ import argparse
|
| |
+ import logging
|
| |
+ import os
|
| |
+ import subprocess
|
| |
+ import sys
|
| |
+
|
| |
+
|
| |
+ logging.basicConfig(level=logging.INFO)
|
| |
+ logger = logging.getLogger('prune-atomic-ostree-repos')
|
| |
+
|
| |
+ # The location of our two unified ostree repos
|
| |
+ ATOMICCOMPOSEREPO = '/mnt/koji/compose/atomic/repo'
|
| |
+ ATOMICPRODREPO = '/mnt/koji/atomic/repo'
|
| |
+
|
| |
+ # The # of commits to retain in the compose repo for each branch
|
| |
+ COMPOSE_REPO_POLICY_DEPTH = '5'
|
| |
+
|
| |
+ # The policy for each ref in the prod repo. None means "keep all"
|
| |
+ PROD_REF_POLICIES = {
|
| |
+ # rawhide atomic host
|
| |
+ 'fedora/rawhide/x86_64/atomic-host': "30 days ago",
|
| |
+ 'fedora/rawhide/aarch64/atomic-host': "30 days ago",
|
| |
+ 'fedora/rawhide/ppc64le/atomic-host': "30 days ago",
|
| |
+
|
| |
+ # rawhide atomic workstation
|
| |
+ 'fedora/rawhide/x86_64/workstation': "30 days ago",
|
| |
+
|
| |
+ # f28 atomic host
|
| |
+ 'fedora/28/x86_64/atomic-host': None,
|
| |
+ 'fedora/28/aarch64/atomic-host': None,
|
| |
+ 'fedora/28/ppc64le/atomic-host': None,
|
| |
+ # f28 atomic host (updates)
|
| |
+ 'fedora/28/x86_64/updates/atomic-host': None,
|
| |
+ 'fedora/28/aarch64/updates/atomic-host': None,
|
| |
+ 'fedora/28/ppc64le/updates/atomic-host': None,
|
| |
+ # f28 atomic host (testing)
|
| |
+ 'fedora/28/x86_64/testing/atomic-host': "30 days ago",
|
| |
+ 'fedora/28/aarch64/testing/atomic-host': "30 days ago",
|
| |
+ 'fedora/28/ppc64le/testing/atomic-host': "30 days ago",
|
| |
+
|
| |
+ # f28 atomic workstation
|
| |
+ 'fedora/28/x86_64/workstation': None,
|
| |
+ # f28 atomic workstation (updates)
|
| |
+ 'fedora/28/x86_64/updates/workstation': None,
|
| |
+ # f28 atomic workstation (testing)
|
| |
+ 'fedora/28/x86_64/testing/workstation': "30 days ago",
|
| |
+ }
|
| |
+
|
| |
+ # Beneath this is code, no config needed here
|
| |
+ def run_command(cmd):
|
| |
+ logger.info('Running %s', cmd)
|
| |
+ try:
|
| |
+ output = subprocess.check_output(cmd,
|
| |
+ stderr=subprocess.STDOUT,
|
| |
+ shell=False)
|
| |
+ except subprocess.CalledProcessError as e:
|
| |
+ logger.error(e.stdout.decode('utf-8'))
|
| |
+ raise # re-raise CalledProcessError
|
| |
+
|
| |
+ return output
|
| |
+
|
| |
+
|
| |
+ def prune_compose_repo(test=False):
|
| |
+
|
| |
+ # prune the compose repo
|
| |
+ logger.info('Pruning the compose repo %s to %s depth' %
|
| |
+ (ATOMICCOMPOSEREPO, COMPOSE_REPO_POLICY_DEPTH))
|
| |
+ cmd = ['ostree', 'prune', '--repo', ATOMICCOMPOSEREPO,
|
| |
+ '--depth', COMPOSE_REPO_POLICY_DEPTH, '--refs-only']
|
| |
+
|
| |
+ if test:
|
| |
+ cmd.append('--no-prune')
|
| |
+
|
| |
+ output = run_command(cmd)
|
| |
+ logger.info('####################################')
|
| |
+ logger.info('\n' + output.decode('utf-8'))
|
| |
+ logger.info('####################################')
|
| |
+
|
| |
+
|
| |
+ def prune_prod_repo(test=False):
|
| |
+
|
| |
+ # Error out if any refs exist that aren't defined in
|
| |
+ # the policy
|
| |
+ cmd = ['ostree', 'refs', '--repo', ATOMICPRODREPO]
|
| |
+ output = run_command(cmd)
|
| |
+ prodrefs = output.decode('utf-8').splitlines()
|
| |
+ for ref in prodrefs:
|
| |
+ if ref not in PROD_REF_POLICIES:
|
| |
+ msg = 'ref %s in repo %s but no policy defined' % (ref, ATOMICPRODREPO)
|
| |
+ logger.error(msg)
|
| |
+ raise Exception(msg)
|
| |
+
|
| |
+ # prune each branch in the policy with specified value
|
| |
+ for ref,policy in PROD_REF_POLICIES.items():
|
| |
+ if ref not in prodrefs:
|
| |
+ logger.warn('policy defined for ref %s but ref not in repo %s' %
|
| |
+ (ref, ATOMICPRODREPO))
|
| |
+ continue
|
| |
+
|
| |
+ if policy is None:
|
| |
+ logger.info('Skipping ref %s in repo %s. Policy is to keep all commits' %
|
| |
+ (ref, ATOMICPRODREPO))
|
| |
+ continue
|
| |
+
|
| |
+ logger.info('Pruning the %s ref in repo %s to %s' %
|
| |
+ (ref, ATOMICPRODREPO, policy))
|
| |
+ cmd = ['ostree', 'prune', '--repo', ATOMICPRODREPO,
|
| |
+ '--only-branch', ref, '--refs-only',
|
| |
+ '--keep-younger-than', policy]
|
| |
+ if test:
|
| |
+ cmd.append('--no-prune')
|
| |
+ output = run_command(cmd)
|
| |
+ logger.info('####################################')
|
| |
+ logger.info('\n' + output.decode('utf-8'))
|
| |
+ logger.info('####################################')
|
| |
+
|
| |
+ # XXX this function should not be used yet as there are a few
|
| |
+ # bugs to work out:
|
| |
+ # https://github.com/ostreedev/ostree/issues/1479
|
| |
+ # https://github.com/ostreedev/ostree/issues/1481
|
| |
+ def prune_prod_repo_deltas(test=False):
|
| |
+
|
| |
+ logger.info("prune_prod_repo_deltas: this function is disabled until the"
|
| |
+ "following bugs are fixed:")
|
| |
+ logger.info("https://github.com/ostreedev/ostree/issues/1479")
|
| |
+ logger.info("https://github.com/ostreedev/ostree/issues/1481")
|
| |
+ return
|
| |
+
|
| |
+ cmd = ['ostree', 'prune', '--repo', ATOMICPRODREPO,
|
| |
+ '--refs-only', '--static-deltas-only']
|
| |
+ if test:
|
| |
+ cmd.append('--no-prune')
|
| |
+ output = run_command(cmd)
|
| |
+ logger.info('####################################')
|
| |
+ logger.info('\n' + output.decode('utf-8'))
|
| |
+ logger.info('####################################')
|
| |
+
|
| |
+ def main():
|
| |
+
|
| |
+ parser = argparse.ArgumentParser()
|
| |
+ parser.add_argument("--test",
|
| |
+ help="Don't actually prune", action='store_true')
|
| |
+ args = parser.parse_args()
|
| |
+
|
| |
+ # set args.test = True for now. Once we have proved
|
| |
+ # everything out in prod and the logs look good we'll
|
| |
+ # delete this code.
|
| |
+ args.test = True
|
| |
+
|
| |
+ prune_compose_repo(args.test)
|
| |
+ prune_prod_repo(args.test)
|
| |
+
|
| |
+ # XXX this function should not be used yet as there are a few
|
| |
+ # bugs to work out:
|
| |
+ # https://github.com/ostreedev/ostree/issues/1479
|
| |
+ # https://github.com/ostreedev/ostree/issues/1481
|
| |
+ #prune_prod_repo_deltas(args.test)
|
| |
+
|
| |
+ if __name__ == '__main__':
|
| |
+ main()
|
| |
This could have stayed prod_ref - PEP standard for names. I know it's a nit, but, the other change made the difference clear already for us.