PR#950: Drop databases from repo - pungi

pungi

#950 Drop databases from repo

Merged 5 years ago by lsedlar. Opened 5 years ago by lsedlar.

lsedlar/pungi no-db-for-comps-repo into master

createrepo: Allow disabling SQLite database

Lubomír Sedlář • 5 years ago

b4e746a

init: Drop database from comps repo

Lubomír Sedlář • 5 years ago

1afb709

doc/configuration.rst

file modified

		`@@ -412,6 +412,12 @@`
		`createrepo_num_workers`
		(int) -- how many concurrent ``createrepo`` workers to run. Value defaults to 3.

		`+ createrepo_database`
		`+ (bool) -- whether to create SQLite database as part of the repodata. This`
		`+ is only useful as an optimization for clients using Yum to consume to the`
		`+ repo. Default value depends on gather backend. For DNF it's turned off, for`
		+ Yum the default is ``True``.
		`+`
		`product_id = None`
		(:ref:`scm_dict <scm_support>`) -- If specified, it should point to a
		directory with certificates ``<variant_uid>-<arch>-*.pem``. This

pungi/checks.py

file modified

		`@@ -627,6 +627,9 @@`
		`"type": "number",`
		`"default": 3,`
		`},`
		`+ "createrepo_database": {`
		`+ "type": "boolean",`
		`+ },`
		`"repoclosure_strictness": _variant_arch_mapping({`
		`"type": "string",`
		`"default": "lenient",`

pungi/compose.py

file modified

+10

		`@@ -201,6 +201,16 @@`
		`def config_dir(self):`
		`return os.path.dirname(self.conf._open_file or "")`

		`+ @property`
		`+ def should_create_yum_database(self):`
		`+ """Explicit configuration trumps all. Otherwise check gather backend`
		`+ and only create it for Yum.`
		`+ """`
		`+ config = self.conf.get('createrepo_database')`
		`+ if config is not None:`
		`+ return config`
		`+ return self.conf['gather_backend'] == 'yum'`
		`+`
		`def read_variants(self):`
		`# TODO: move to phases/init ?`
		`variants_file = self.paths.work.variants_file(arch="global")`

pungi/phases/createrepo.py

file modified

+3 -1

		`@@ -163,7 +163,9 @@`
		`comps_path = None`
		`if compose.has_comps and pkg_type == "rpm":`
		`comps_path = compose.paths.work.comps(arch=arch, variant=variant)`
		`- cmd = repo.get_createrepo_cmd(repo_dir, update=True, database=True, skip_stat=True,`
		`+ cmd = repo.get_createrepo_cmd(repo_dir, update=True,`
		`+ database=compose.should_create_yum_database,`
		`+ skip_stat=True,`
		`pkglist=file_list, outputdir=repo_dir,`
		`workers=compose.conf["createrepo_num_workers"],`
		`groupfile=comps_path, update_md_path=repo_dir_arch,`

pungi/phases/init.py

file modified

+4 -2

		`@@ -149,10 +149,12 @@`
		`compose.log_warning("[SKIP ] %s" % msg)`
		`else:`
		`compose.log_info("[BEGIN] %s" % msg)`
		`- cmd = repo.get_createrepo_cmd(comps_repo, update=True, database=True, skip_stat=True,`
		`+ cmd = repo.get_createrepo_cmd(comps_repo, database=False,`
		`outputdir=comps_repo, groupfile=comps_path,`
		`checksum=createrepo_checksum)`
		`- run(cmd, logfile=compose.paths.log.log_file(arch, "comps_repo"), show_cmd=True)`
		`+ logfile = 'comps_repo-%s' % variant if variant else 'comps_repo'`
		`+ run(cmd, logfile=compose.paths.log.log_file(arch, logfile),`
		`+ show_cmd=True)`
		`compose.log_info("[DONE ] %s" % msg)`

tests/helpers.py

file modified

		`@@ -149,6 +149,7 @@`
		`self.attempt_deliverable = mock.Mock()`
		`self.fail_deliverable = mock.Mock()`
		`self.require_deliverable = mock.Mock()`
		`+ self.should_create_yum_database = True`

		`def setup_optional(self):`
		`self.all_variants['Server-optional'] = MockVariant(`

tests/test_compose.py

file modified

+18

		`@@ -481,6 +481,24 @@`

		`self.assertTrue(self.compose.notifier.send.call_count, 1)`

		`+ def test_no_database_with_dnf_backend(self):`
		`+ self.compose.conf['gather_backend'] = 'dnf'`
		`+ self.assertFalse(self.compose.should_create_yum_database)`
		`+`
		`+ def test_no_database_with_dnf_backend_config_override(self):`
		`+ self.compose.conf['gather_backend'] = 'dnf'`
		`+ self.compose.conf['createrepo_database'] = True`
		`+ self.assertTrue(self.compose.should_create_yum_database)`
		`+`
		`+ def test_no_database_with_yum_backend(self):`
		`+ self.compose.conf['gather_backend'] = 'yum'`
		`+ self.assertTrue(self.compose.should_create_yum_database)`
		`+`
		`+ def test_no_database_with_yum_backend_config_override(self):`
		`+ self.compose.conf['gather_backend'] = 'yum'`
		`+ self.compose.conf['createrepo_database'] = False`
		`+ self.assertFalse(self.compose.should_create_yum_database)`
		`+`

		`if __name__ == "__main__":`
		`unittest.main()`

tests/test_createrepophase.py

file modified

+32

		`@@ -153,6 +153,38 @@`

		`@mock.patch('pungi.phases.createrepo.run')`
		`@mock.patch('pungi.phases.createrepo.CreaterepoWrapper')`
		`+ def test_variant_repo_rpms_without_database(self, CreaterepoWrapperCls, run):`
		`+ compose = DummyCompose(self.topdir, {`
		`+ 'createrepo_checksum': 'sha256',`
		`+ })`
		`+ compose.should_create_yum_database = False`
		`+ compose.DEBUG = False`
		`+ compose.has_comps = False`
		`+`
		`+ repo = CreaterepoWrapperCls.return_value`
		`+ copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json'))`
		`+`
		`+ create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm')`
		`+`
		`+ list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf'`
		`+ self.assertEqual(CreaterepoWrapperCls.mock_calls[0],`
		`+ mock.call(createrepo_c=True))`
		`+ self.assertItemsEqual(`
		`+ repo.get_createrepo_cmd.mock_calls,`
		`+ [mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256',`
		`+ database=False, groupfile=None, workers=3,`
		`+ outputdir=self.topdir + '/compose/Server/x86_64/os',`
		`+ pkglist=list_file, skip_stat=True, update=True,`
		`+ update_md_path=self.topdir + '/work/x86_64/repo',`
		`+ deltas=False, oldpackagedirs=None, use_xz=False)])`
		`+ self.assertItemsEqual(`
		`+ repo.get_modifyrepo_cmd.mock_calls,`
		`+ [])`
		`+ with open(list_file) as f:`
		`+ self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')`
		`+`
		`+ @mock.patch('pungi.phases.createrepo.run')`
		`+ @mock.patch('pungi.phases.createrepo.CreaterepoWrapper')`
		`def test_variant_repo_source(self, CreaterepoWrapperCls, run):`
		`compose = DummyCompose(self.topdir, {`
		`'createrepo_checksum': 'sha256',`

tests/test_initphase.py

file modified

+19 -1

		`@@ -142,12 +142,30 @@`
		`[mock.call(['createrepo_c', self.topdir + '/work/x86_64/comps_repo',`
		`'--outputdir=%s/work/x86_64/comps_repo' % self.topdir,`
		`'--groupfile=%s/work/x86_64/comps/comps-x86_64.xml' % self.topdir,`
		`- '--update', '--skip-stat', '--database', '--checksum=sha256',`
		`+ '--update', '--no-database', '--checksum=sha256',`
		`'--unique-md-filenames'],`
		`logfile=self.topdir + '/logs/x86_64/comps_repo.x86_64.log',`
		`show_cmd=True)])`

		`@mock.patch('pungi.phases.init.run')`
		`+ def test_run_with_variant(self, run):`
		`+ compose = DummyCompose(self.topdir, {`
		`+ 'createrepo_checksum': 'sha256',`
		`+ })`
		`+ compose.DEBUG = False`
		`+`
		`+ init.create_comps_repo(compose, 'x86_64', compose.variants['Server'])`
		`+`
		`+ self.assertEqual(run.mock_calls,`
		`+ [mock.call(['createrepo_c', self.topdir + '/work/x86_64/comps_repo_Server',`
		`+ '--outputdir=%s/work/x86_64/comps_repo_Server' % self.topdir,`
		`+ '--groupfile=%s/work/x86_64/comps/comps-Server.x86_64.xml' % self.topdir,`
		`+ '--update', '--no-database', '--checksum=sha256',`
		`+ '--unique-md-filenames'],`
		`+ logfile=self.topdir + '/logs/x86_64/comps_repo-Server.x86_64.log',`
		`+ show_cmd=True)])`
		`+`
		`+ @mock.patch('pungi.phases.init.run')`
		`def test_run_in_debug(self, run):`
		`compose = DummyCompose(self.topdir, {`
		`'createrepo_checksum': 'sha256',`

lsedlar commented 5 years ago

For internal comps repos created in init phase the databases are gone for good. They are not needed there, since the repo is empty anyway. A test is added for the variant specific comps repo.

For actual shipped repos there is now a configuration option. The default depends on gather backend: for DNF composes we don't create the database (but can turn it on if wanted), for Yum composes they are created (but can be disabled).

rebased onto 88d81a03223ae506ce0e39fde4540109422bedc9

5 years ago