From f8658add6d44db44aab3677b3890a219b4f9506a Mon Sep 17 00:00:00 2001 From: Adam Miller Date: May 25 2017 08:04:05 +0000 Subject: [PATCH 1/3] Allow container builds from any namespace Previously rpkg enforced a "%s-docker-candidate" koji tag for any container-build such that "%s" was the DistGit branch unless there was a target override passed. This patch allows for the DistGit namespace to be inherited into the koji tag making it "%s-%s-candidate" such that "%s-%s" % (DistGitBranch, DistGitNamespace). Signed-off-by: Adam Miller --- diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py index 3a1d544..8e22310 100644 --- a/pyrpkg/__init__.py +++ b/pyrpkg/__init__.py @@ -606,6 +606,34 @@ class Commands(object): ' Use --module-name.') @property + def ns(self): + """This property provides the namespace of the module""" + + if not self._ns: + self.load_ns() + return self._ns + + def load_ns(self): + """Loads the namespace""" + + try: + if self.distgit_namespaced: + if self.push_url: + parts = urllib.parse.urlparse(self.push_url) + + path_parts = [p for p in parts.path.split("/") if p] + if len(path_parts) == 1: + path_parts.insert(0, "rpms") + ns = path_parts[-2] + + self._ns = ns + else: + self._ns = None + self.log.info("Could not find ns, distgit is not namespaced") + except rpkgError: + self.log.warning('Failed to get ns from Git url or pushurl') + + @property def ns_module_name(self): """This property ensures the module attribute""" @@ -2507,11 +2535,17 @@ class Commands(object): git_branch = self.branch_merge user = self.user component = self.module_name - docker_target = self.target + container_target = self.target if not target_override: # Translate the build target into a docker target, # but only if --target wasn't specified on the command-line - docker_target = '%s-docker-candidate' % self.target.split('-candidate')[0] + if self.distgit_namespaced: + # Allow for any namespace, not just "docker" + container_target = '%s-%s-candidate' % \ + (self.target.split('-candidate')[0], self.ns) + else: + container_target = '%s-docker-candidate' % \ + self.target.split('-candidate')[0] build = osbs.create_build( git_uri=git_uri, @@ -2519,7 +2553,7 @@ class Commands(object): git_branch=git_branch, user=user, component=component, - target=docker_target, + target=container_target, architecture="x86_64", yum_repourls=yum_repourls ) @@ -2552,11 +2586,17 @@ class Commands(object): nowait=False): # check if repo is dirty and all commits are pushed self.check_repo() - docker_target = self.target + container_target = self.target if not target_override: # Translate the build target into a docker target, # but only if --target wasn't specified on the command-line - docker_target = '%s-docker-candidate' % self.target.split('-candidate')[0] + if self.distgit_namespaced: + # Allow for any namespace, not just "docker" + container_target = '%s-%s-candidate' % \ + (self.target.split('-candidate')[0], self.ns) + else: + container_target = '%s-docker-candidate' % \ + self.target.split('-candidate')[0] koji_session_backup = (self.build_client, self.kojiconfig) (self.build_client, self.kojiconfig) = (build_client, kojiconfig) @@ -2565,9 +2605,9 @@ class Commands(object): if "buildContainer" not in self.kojisession.system.listMethods(): raise RuntimeError("Kojihub instance does not support buildContainer") - build_target = self.kojisession.getBuildTarget(docker_target) + build_target = self.kojisession.getBuildTarget(container_target) if not build_target: - msg = "Unknown build target: %s" % docker_target + msg = "Unknown build target: %s" % container_target self.log.error(msg) raise UnknownTargetError(msg) else: @@ -2587,7 +2627,7 @@ class Commands(object): task_opts[key] = opts[key] priority = opts.get("priority", None) task_id = self.kojisession.buildContainer(source, - docker_target, + container_target, task_opts, priority=priority) self.log.info('Created task: %s', task_id) From be0f10833a016c1389783aa113c769992b20f41c Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: May 25 2017 08:04:05 +0000 Subject: [PATCH 2/3] Add a separate property for namespace Instead of having module_name and ns_module_name, we now only explicitly store the bare module name and namespace. The old ns_module_name property is still available. Signed-off-by: Lubomír Sedlář --- diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py index 8e22310..2cdd28d 100644 --- a/pyrpkg/__init__.py +++ b/pyrpkg/__init__.py @@ -126,8 +126,8 @@ class Commands(object): self._mockconfig = None # The name of the cloned module self._module_name = None - # The distgit namespaced name of the cloned module - self._ns_module_name = None + # The dist git namespace + self._ns = None # The name of the module from spec file self._module_name_spec = None # The rpm name-version-release of the cloned module @@ -204,7 +204,7 @@ class Commands(object): self._push_url = None self._branch_remote = None self._repo = None - self._ns_module_name = None + self._ns = None self._path = value @property @@ -613,6 +613,10 @@ class Commands(object): self.load_ns() return self._ns + @ns.setter + def ns(self, ns): + self._ns = ns + def load_ns(self): """Loads the namespace""" @@ -635,37 +639,10 @@ class Commands(object): @property def ns_module_name(self): - """This property ensures the module attribute""" - - if not self._ns_module_name: - self.load_ns_module_name() - return self._ns_module_name - - @ns_module_name.setter - def ns_module_name(self, ns_module_name): - self._ns_module_name = ns_module_name - - def load_ns_module_name(self): - """Loads a package module.""" - - try: - if self.push_url: - parts = urllib.parse.urlparse(self.push_url) - - if self.distgit_namespaced: - path_parts = [p for p in parts.path.split("/") if p] - if len(path_parts) == 1: - path_parts.insert(0, "rpms") - ns_module_name = "/".join(path_parts[-2:]) - else: - ns_module_name = posixpath.basename(parts.path) - - if ns_module_name.endswith('.git'): - ns_module_name = ns_module_name[:-len('.git')] - self._ns_module_name = ns_module_name - return - except rpkgError: - self.log.warning('Failed to get ns_module_name from Git url or pushurl') + if self.distgit_namespaced: + return '%s/%s' % (self.ns, self.module_name) + else: + return self.module_name @property def nvr(self): diff --git a/pyrpkg/cli.py b/pyrpkg/cli.py index 4a08f05..5d405d4 100644 --- a/pyrpkg/cli.py +++ b/pyrpkg/cli.py @@ -141,12 +141,12 @@ class cliClient(object): if self.args.module_name: # Module name was specified via argument if '/' not in self.args.module_name: - # No slash, assume rpms namespace self._cmd.module_name = self.args.module_name - self._cmd.ns_module_name = 'rpms/%s' % self.args.module_name + if dg_namespaced: + # No slash, assume rpms namespace + self._cmd.ns = 'rpms' else: - self._cmd.ns_module_name = self.args.module_name - _, self._cmd.module_name = self.args.module_name.rsplit('/', 1) + self._cmd.ns, self._cmd.module_name = self.args.module_name.rsplit('/', 1) self._cmd.password = self.args.password self._cmd.runas = self.args.runas self._cmd.debug = self.args.debug diff --git a/tests/fixtures/rpkg-ns.conf b/tests/fixtures/rpkg-ns.conf new file mode 100644 index 0000000..e3c49e1 --- /dev/null +++ b/tests/fixtures/rpkg-ns.conf @@ -0,0 +1,12 @@ +[rpkg] +lookaside = http://localhost/repo/pkgs +lookasidehash = md5 +lookaside_cgi = https://localhost/repo/pkgs/upload.cgi +gitbaseurl = ssh://%(user)s@localhost/%(module)s +anongiturl = git://localhost/%(module)s +branchre = f\d$|f\d\d$|el\d$|olpc\d$|master$ +kojiconfig = /etc/koji.conf +build_client = koji +clone_config = + bz.default-component %(module)s +distgit_namespaced = True diff --git a/tests/test_cli.py b/tests/test_cli.py index bbc5810..816002c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -40,9 +40,9 @@ package demo for testing class CliTestCase(CommandTestCase): - def new_cli(self): + def new_cli(self, cfg=None): config = configparser.SafeConfigParser() - config.read(config_file) + config.read(cfg or config_file) client = pyrpkg.cli.cliClient(config, name='rpkg') client.setupLogging(pyrpkg.log) @@ -70,31 +70,44 @@ class CliTestCase(CommandTestCase): class TestModuleNameOption(CliTestCase): - def get_cmd(self, module_name): + def get_cmd(self, module_name, cfg=None): cmd = ['rpkg', '--path', self.cloned_repo_path, '--module-name', module_name, 'verrel'] with patch('sys.argv', new=cmd): - cli = self.new_cli() + cli = self.new_cli(cfg=cfg) return cli.cmd - def test_just_module_name(self): + def test_non_namespaced(self): cmd = self.get_cmd('foo') self.assertEqual(cmd._module_name, 'foo') - self.assertEqual(cmd._ns_module_name, 'rpms/foo') + self.assertEqual(cmd.ns_module_name, 'foo') + + def test_just_module_name(self): + cmd = self.get_cmd( + 'foo', + os.path.join(os.path.dirname(__file__), 'fixtures', 'rpkg-ns.conf')) + self.assertEqual(cmd._module_name, 'foo') + self.assertEqual(cmd.ns_module_name, 'rpms/foo') def test_explicit_default(self): - cmd = self.get_cmd('rpms/foo') + cmd = self.get_cmd( + 'rpms/foo', + os.path.join(os.path.dirname(__file__), 'fixtures', 'rpkg-ns.conf')) self.assertEqual(cmd._module_name, 'foo') - self.assertEqual(cmd._ns_module_name, 'rpms/foo') + self.assertEqual(cmd.ns_module_name, 'rpms/foo') def test_with_namespace(self): - cmd = self.get_cmd('container/foo') + cmd = self.get_cmd( + 'container/foo', + os.path.join(os.path.dirname(__file__), 'fixtures', 'rpkg-ns.conf')) self.assertEqual(cmd._module_name, 'foo') - self.assertEqual(cmd._ns_module_name, 'container/foo') + self.assertEqual(cmd.ns_module_name, 'container/foo') def test_with_nested_namespace(self): - cmd = self.get_cmd('user/project/foo') + cmd = self.get_cmd( + 'user/project/foo', + os.path.join(os.path.dirname(__file__), 'fixtures', 'rpkg-ns.conf')) self.assertEqual(cmd._module_name, 'foo') - self.assertEqual(cmd._ns_module_name, 'user/project/foo') + self.assertEqual(cmd.ns_module_name, 'user/project/foo') class TestClog(CliTestCase): diff --git a/tests/test_commands.py b/tests/test_commands.py index 891b29a..7c5fdb6 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -428,8 +428,9 @@ class TestProperties(CommandTestCase): ) for push_url, expected_ns_module_name in tests: cmd._push_url = push_url - cmd.load_ns_module_name() - self.assertEqual(expected_ns_module_name, cmd._ns_module_name) + cmd.load_ns() + cmd.load_module_name() + self.assertEqual(expected_ns_module_name, cmd.ns_module_name) cmd.distgit_namespaced = True tests = ( @@ -440,8 +441,9 @@ class TestProperties(CommandTestCase): ) for push_url, expected_ns_module_name in tests: cmd._push_url = push_url - cmd.load_ns_module_name() - self.assertEqual(expected_ns_module_name, cmd._ns_module_name) + cmd.load_ns() + cmd.load_module_name() + self.assertEqual(expected_ns_module_name, cmd.ns_module_name) class TestNamespaced(CommandTestCase): From 4010bef4f4a8f354201b8917b3f59919c4a9d5fb Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: May 25 2017 08:04:05 +0000 Subject: [PATCH 3/3] Allow overriding container build target by downstream Signed-off-by: Lubomír Sedlář --- diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py index 2cdd28d..0d6918b 100644 --- a/pyrpkg/__init__.py +++ b/pyrpkg/__init__.py @@ -142,6 +142,8 @@ class Commands(object): self._spec = None # The build target within the buildsystem self._target = target + # The build target for containers within the buildsystem + self._container_build_target = target # The top url to our build server self._topurl = None # The user to use or discover @@ -805,6 +807,17 @@ class Commands(object): self._target = '%s-candidate' % self.branch_merge @property + def container_build_target(self): + """This property ensures the target for container builds.""" + if not self._container_build_target: + self.load_container_build_target() + return self._container_build_target + + def load_container_build_target(self): + """This creates a target based on git branch and namespace.""" + self._container_build_target = '%s-%s-candidate' % (self.branch_merge, self.ns) + + @property def topurl(self): """This property ensures the topurl attribute""" @@ -2512,17 +2525,7 @@ class Commands(object): git_branch = self.branch_merge user = self.user component = self.module_name - container_target = self.target - if not target_override: - # Translate the build target into a docker target, - # but only if --target wasn't specified on the command-line - if self.distgit_namespaced: - # Allow for any namespace, not just "docker" - container_target = '%s-%s-candidate' % \ - (self.target.split('-candidate')[0], self.ns) - else: - container_target = '%s-docker-candidate' % \ - self.target.split('-candidate')[0] + container_target = self.target if target_override else self.container_build_target build = osbs.create_build( git_uri=git_uri, @@ -2563,17 +2566,7 @@ class Commands(object): nowait=False): # check if repo is dirty and all commits are pushed self.check_repo() - container_target = self.target - if not target_override: - # Translate the build target into a docker target, - # but only if --target wasn't specified on the command-line - if self.distgit_namespaced: - # Allow for any namespace, not just "docker" - container_target = '%s-%s-candidate' % \ - (self.target.split('-candidate')[0], self.ns) - else: - container_target = '%s-docker-candidate' % \ - self.target.split('-candidate')[0] + container_target = self.target if target_override else self.container_build_target koji_session_backup = (self.build_client, self.kojiconfig) (self.build_client, self.kojiconfig) = (build_client, kojiconfig)