From 0255c4d79ade4e3ab35fe4356456deb953ddb07b Mon Sep 17 00:00:00 2001 From: Brian Stinson Date: Nov 08 2018 19:59:41 +0000 Subject: [PATCH 1/9] start on a module for differing package source layouts Signed-off-by: Brian Stinson --- diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py index 6877e80..9dc60b2 100644 --- a/pyrpkg/__init__.py +++ b/pyrpkg/__init__.py @@ -47,6 +47,7 @@ from .gitignore import GitIgnore from pyrpkg.lookaside import CGILookasideCache from pyrpkg.sources import SourcesFile from pyrpkg.utils import cached_property, log_result, find_me +from pyrpkg.layout import Layout PY26 = sys.version_info < (2, 7, 0) @@ -241,6 +242,10 @@ class Commands(object): self._path = value @property + def layout(self): + return Layout.load(self.path) + + @property def kojisession(self): """This property ensures the kojisession attribute""" @@ -821,16 +826,17 @@ class Commands(object): self._distvar, self._distval = osver.split('-') self._distval = self._distval.replace('.', '_') self._disttag = 'el%s' % self._distval - self._rpmdefines = ["--define '_sourcedir %s'" % self.path, - "--define '_specdir %s'" % self.path, - "--define '_builddir %s'" % self.path, - "--define '_srcrpmdir %s'" % self.path, - "--define '_rpmdir %s'" % self.path, + self._rpmdefines = ["--define '_sourcedir %s'" % self.layout.sourcedir, + "--define '_specdir %s'" % self.layout.specdir, + "--define '_builddir %s'" % self.layout.builddir, + "--define '_srcrpmdir %s'" % self.layout.srcrpmdir, + "--define '_rpmdir %s'" % self.layout.rpmdir, "--define 'dist .%s'" % self._disttag, "--define '%s %s'" % (self._distvar, self._distval.split('_')[0]), # int and float this to remove the decimal "--define '%s 1'" % self._disttag] + self.log.debug("RPMDefines: %s" % self._rpmdefines) @property def spec(self): @@ -846,11 +852,11 @@ class Commands(object): deadpackage = False # Get a list of files in the path we're looking at - files = os.listdir(self.path) + files = os.listdir(self.layout.specdir) # Search the files for the first one that ends with ".spec" for f in files: if f.endswith('.spec') and not f.startswith('.'): - self._spec = f + self._spec = os.path.join(self.layout.specdir, f) return if f == 'dead.package': deadpackage = True @@ -2818,7 +2824,7 @@ class Commands(object): digests. """ - self.srpmname = os.path.join(self.path, + self.srpmname = os.path.join(self.layout.srcrpmdir, "%s-%s-%s.src.rpm" % (self.repo_name, self.ver, self.rel)) diff --git a/pyrpkg/layout.py b/pyrpkg/layout.py new file mode 100644 index 0000000..bf04c1c --- /dev/null +++ b/pyrpkg/layout.py @@ -0,0 +1,66 @@ +# Copyright (c) 2018 - Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + +"""Work with different source layouts + +This module contains configuration objects that correspond to differnet +dist-git source layouts. Distributions like Fedora and RHEL use the standard +dist-git layout while CentOS uses the Exploded SRPM layout. +""" + +import logging +import os + +log = logging.getLogger(__name__) + + +class Layout(object): + subclasses = {} + + @classmethod + def subclass(cls, sources_filename_substring): + def decorator(subcls): + cls.subclasses[sources_filename_substring] = subcls + return subcls + return decorator + + @classmethod + def load(cls, path=os.getcwd()): + files = os.listdir(path) + + for substring_key in cls.subclasses: + for filename in files: + if substring_key in filename: + return cls.subclasses[substring_key](path) + + +@Layout.subclass('sources') +class DistGitLayout(Layout): + def __init__(self, rpkg_path): + self.site_name = 'fedpkg' + self.sourcedir = rpkg_path + self.specdir = rpkg_path + self.builddir = rpkg_path + self.rpmdir = rpkg_path + self.srcrpmdir = rpkg_path + log.debug("Subclass: DistGitLayout") + log.debug("Site: %s", self.site_name) + + +@Layout.subclass('.metadata') +class ExplodedSRPMLayout(object): + + def __init__(self, rpkg_path): + self.sourcedir = os.path.join(rpkg_path, "SOURCES") + self.specdir = os.path.join(rpkg_path, "SPECS") + self.builddir = os.path.join(rpkg_path, "BUILD") + self.rpmdir = os.path.join(rpkg_path, "RPMS") + self.srcrpmdir = os.path.join(rpkg_path, "SRPMS") + self.site_name = 'centpkg' + log.debug("Subclass: ExplodedSRPMLayout") + log.debug("Site: %s", self.site_name) From 479bbe9ef2c426325bf5a2492e5e0bf29d65eb52 Mon Sep 17 00:00:00 2001 From: Brian Stinson Date: Nov 09 2018 04:41:34 +0000 Subject: [PATCH 2/9] check for dead.package sooner, before we start looking for a SRPM Signed-off-by: Brian Stinson --- diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py index 9dc60b2..47dc158 100644 --- a/pyrpkg/__init__.py +++ b/pyrpkg/__init__.py @@ -849,7 +849,11 @@ class Commands(object): def load_spec(self): """This sets the spec attribute""" - deadpackage = False + # For all layouts, check for dead.pkg in the root of the package dir + files = os.listdir(self.path) + for f in files: + if f == 'dead.package': + raise rpkgError('No spec file found. This package is retired') # Get a list of files in the path we're looking at files = os.listdir(self.layout.specdir) @@ -858,10 +862,6 @@ class Commands(object): if f.endswith('.spec') and not f.startswith('.'): self._spec = os.path.join(self.layout.specdir, f) return - if f == 'dead.package': - deadpackage = True - if deadpackage: - raise rpkgError('No spec file found. This package is retired') else: raise rpkgError('No spec file found.') From ef8d7770553b3e120618a86481e81882461cdb8d Mon Sep 17 00:00:00 2001 From: Brian Stinson Date: Nov 09 2018 04:41:34 +0000 Subject: [PATCH 3/9] provide a blank Layout for when the directory is not populated yet Signed-off-by: Brian Stinson --- diff --git a/pyrpkg/layout.py b/pyrpkg/layout.py index bf04c1c..7a321ed 100644 --- a/pyrpkg/layout.py +++ b/pyrpkg/layout.py @@ -22,6 +22,14 @@ log = logging.getLogger(__name__) class Layout(object): subclasses = {} + def __init__(self, rpkg_path): + self.site_name = '' + self.sourcedir = '' + self.specdir = '' + self.builddir = '' + self.rpmdir = '' + self.srcrpmdir = '' + @classmethod def subclass(cls, sources_filename_substring): def decorator(subcls): @@ -37,6 +45,7 @@ class Layout(object): for filename in files: if substring_key in filename: return cls.subclasses[substring_key](path) + return cls(path) @Layout.subclass('sources') @@ -53,7 +62,7 @@ class DistGitLayout(Layout): @Layout.subclass('.metadata') -class ExplodedSRPMLayout(object): +class ExplodedSRPMLayout(Layout): def __init__(self, rpkg_path): self.sourcedir = os.path.join(rpkg_path, "SOURCES") From bbdadd6a05ec1e9a12c8e3ccebf3554254a48b03 Mon Sep 17 00:00:00 2001 From: Brian Stinson Date: Nov 09 2018 04:41:34 +0000 Subject: [PATCH 4/9] the specfile methods give a full path now, update the tests Signed-off-by: Brian Stinson --- diff --git a/tests/test_cli.py b/tests/test_cli.py index 318e671..d02896a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -920,7 +920,7 @@ class TestGimmeSpec(CliTestCase): cli.gimmespec() output = sys.stdout.getvalue().strip() - self.assertEqual('docpkg.spec', output) + self.assertEqual(os.path.join(self.cloned_repo_path,'docpkg.spec'), output) class TestClean(CliTestCase): diff --git a/tests/test_commands.py b/tests/test_commands.py index d7b20cc..9d72844 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -400,7 +400,7 @@ class TestProperties(CommandTestCase): def test_spec(self): cmd = self.make_commands() - self.assertEqual('docpkg.spec', cmd.spec) + self.assertEqual(os.path.join(self.cloned_repo_path,'docpkg.spec'), cmd.spec) def test_no_spec_as_it_is_deadpackage(self): with patch('os.listdir', return_value=['dead.package']): From a2fad8750c035bf68e1b232bddb8befc7c4c187b Mon Sep 17 00:00:00 2001 From: Brian Stinson Date: Nov 09 2018 04:41:34 +0000 Subject: [PATCH 5/9] flake8 fixes Signed-off-by: Brian Stinson --- diff --git a/tests/test_cli.py b/tests/test_cli.py index d02896a..26aebdb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -920,7 +920,7 @@ class TestGimmeSpec(CliTestCase): cli.gimmespec() output = sys.stdout.getvalue().strip() - self.assertEqual(os.path.join(self.cloned_repo_path,'docpkg.spec'), output) + self.assertEqual(os.path.join(self.cloned_repo_path, 'docpkg.spec'), output) class TestClean(CliTestCase): diff --git a/tests/test_commands.py b/tests/test_commands.py index 9d72844..b40a399 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -400,7 +400,7 @@ class TestProperties(CommandTestCase): def test_spec(self): cmd = self.make_commands() - self.assertEqual(os.path.join(self.cloned_repo_path,'docpkg.spec'), cmd.spec) + self.assertEqual(os.path.join(self.cloned_repo_path, 'docpkg.spec'), cmd.spec) def test_no_spec_as_it_is_deadpackage(self): with patch('os.listdir', return_value=['dead.package']): From 9006c93961426383477a10058971b7ce19142e97 Mon Sep 17 00:00:00 2001 From: Brian Stinson Date: Nov 15 2018 04:17:30 +0000 Subject: [PATCH 6/9] add some tests Signed-off-by: Brian Stinson --- diff --git a/tests/test_layout.py b/tests/test_layout.py new file mode 100644 index 0000000..1841b74 --- /dev/null +++ b/tests/test_layout.py @@ -0,0 +1,80 @@ +import os +import shutil +import tempfile +import unittest + +from pyrpkg.layout import Layout + +class EmptyLayoutTestCase(unittest.TestCase): + def setUp(self): + self.workdir = tempfile.mkdtemp(prefix='rpkg-tests.') + self.layout = Layout.load(self.workdir) + + def tearDown(self): + shutil.rmtree(self.workdir) + + + def test_empty_layout_sets_sourcedir_to_cwd(self): + self.assertEqual(self.layout.sourcedir, self.workdir) + + def test_empty_layout_sets_specdir_to_cwd(self): + self.assertEqual(self.layout.specdir, self.workdir) + + def test_empty_layout_sets_builddir_to_cwd(self): + self.assertEqual(self.layout.builddir, self.workdir) + + def test_empty_layout_sets_rpmdir_to_cwd(self): + self.assertEqual(self.layout.rpmdir, self.workdir) + + def test_distgit_layout_sets_srcpmdir_to_cwd(self): + self.assertEqual(self.layout.srcrpmdir, self.workdir) + +class DistGitLayoutTestCase(unittest.TestCase): + def setUp(self): + self.workdir = tempfile.mkdtemp(prefix='rpkg-tests.') + self.file = os.path.join(self.workdir,'sources') + open(self.file, 'a').close() + self.layout = Layout.load(self.workdir) + + def tearDown(self): + shutil.rmtree(self.workdir) + + def test_distgit_layout_sets_sourcedir_to_cwd(self): + self.assertEqual(self.layout.sourcedir, self.workdir) + + def test_distgit_layout_sets_specdir_to_cwd(self): + self.assertEqual(self.layout.specdir, self.workdir) + + def test_distgit_layout_sets_builddir_to_cwd(self): + self.assertEqual(self.layout.builddir, self.workdir) + + def test_distgit_layout_sets_rpmdir_to_cwd(self): + self.assertEqual(self.layout.rpmdir, self.workdir) + + def test_distgit_layout_sets_srcpmdir_to_cwd(self): + self.assertEqual(self.layout.srcrpmdir, self.workdir) + +class ExplodedSRPMLayoutTestCase(unittest.TestCase): + def setUp(self): + self.workdir = tempfile.mkdtemp(prefix='rpkg-tests.') + self.file = os.path.join(self.workdir,'.test.metadata') + open(self.file, 'a').close() + self.layout = Layout.load(self.workdir) + + def tearDown(self): + shutil.rmtree(self.workdir) + + def test_distgit_layout_sets_sourcedir_to_SOURCES(self): + self.assertEqual(self.layout.sourcedir, os.path.join(self.workdir, 'SOURCES')) + + def test_distgit_layout_sets_specdir_to_SPECS(self): + self.assertEqual(self.layout.specdir, os.path.join(self.workdir, 'SPECS')) + + def test_distgit_layout_sets_builddir_to_BUILD(self): + self.assertEqual(self.layout.builddir, os.path.join(self.workdir, 'BUILD')) + + def test_distgit_layout_sets_rpmdir_to_RPMS(self): + self.assertEqual(self.layout.rpmdir, os.path.join(self.workdir, 'RPMS')) + + def test_distgit_layout_sets_srpmdir_to_SRPMS(self): + self.assertEqual(self.layout.srcrpmdir, os.path.join(self.workdir, 'SRPMS')) From 1cdf62ae7a1687c51d3b7d9f6d21aca7d72803d4 Mon Sep 17 00:00:00 2001 From: Brian Stinson Date: Nov 15 2018 04:17:30 +0000 Subject: [PATCH 7/9] make the DistGit properties the default if a layout cannot be detected Signed-off-by: Brian Stinson --- diff --git a/pyrpkg/layout.py b/pyrpkg/layout.py index 7a321ed..50ca05b 100644 --- a/pyrpkg/layout.py +++ b/pyrpkg/layout.py @@ -23,12 +23,12 @@ class Layout(object): subclasses = {} def __init__(self, rpkg_path): - self.site_name = '' - self.sourcedir = '' - self.specdir = '' - self.builddir = '' - self.rpmdir = '' - self.srcrpmdir = '' + self.sourcedir = rpkg_path + self.specdir = rpkg_path + self.builddir = rpkg_path + self.rpmdir = rpkg_path + self.srcrpmdir = rpkg_path + self.sources_file_template = 'sources' @classmethod def subclass(cls, sources_filename_substring): @@ -57,6 +57,7 @@ class DistGitLayout(Layout): self.builddir = rpkg_path self.rpmdir = rpkg_path self.srcrpmdir = rpkg_path + self.sources_file_template = 'sources' log.debug("Subclass: DistGitLayout") log.debug("Site: %s", self.site_name) @@ -71,5 +72,8 @@ class ExplodedSRPMLayout(Layout): self.rpmdir = os.path.join(rpkg_path, "RPMS") self.srcrpmdir = os.path.join(rpkg_path, "SRPMS") self.site_name = 'centpkg' + # If we call str.format() from the command object with `self` as the + # parameter we can can get any of the attributes (including properties) + self.sources_file_template = '.{0.repo_name}.metadata' log.debug("Subclass: ExplodedSRPMLayout") log.debug("Site: %s", self.site_name) From 5e7f6d770bf8e73cd26d4b9ad229f2832c142725 Mon Sep 17 00:00:00 2001 From: Brian Stinson Date: Nov 15 2018 04:17:30 +0000 Subject: [PATCH 8/9] In upload, we actually want the relative path to the file not the basename we want a path relative to the root so that we know where to download it to. In the dist-git layout this works out to be the basename, but if the file exists in a subdirectory that won't be enough. Signed-off-by: Brian Stinson --- diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py index 47dc158..20186d0 100644 --- a/pyrpkg/__init__.py +++ b/pyrpkg/__init__.py @@ -2756,7 +2756,7 @@ class Commands(object): for f in files: # TODO: Skip empty file needed? file_hash = self.lookasidecache.hash_file(f) - file_basename = os.path.basename(f) + file_basename = f.replace(self.path + '/', '') try: sourcesf.add_entry(self.lookasidehash, file_basename, @@ -2783,7 +2783,7 @@ class Commands(object): sourcesf.write() gitignore.write() - self.repo.index.add(['sources', '.gitignore']) + self.repo.index.add([sourcesf.sourcesfile, '.gitignore']) def prep(self, arch=None, builddir=None, buildrootdir=None): """Run ``rpmbuild -bp`` From ed2ba38121dd20825311e4f9cf5b81f63865eaba Mon Sep 17 00:00:00 2001 From: Brian Stinson Date: Nov 15 2018 04:17:30 +0000 Subject: [PATCH 9/9] we can get the sources file from the layout now Signed-off-by: Brian Stinson --- diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py index 20186d0..b32d180 100644 --- a/pyrpkg/__init__.py +++ b/pyrpkg/__init__.py @@ -1073,7 +1073,7 @@ class Commands(object): @property def sources_filename(self): - return os.path.join(self.path, 'sources') + return os.path.join(self.path, self.layout.sources_file_template.format(self)) @property def osbs_config_filename(self):