From d995585a9b772c8279abfda8cfe44740584c77b3 Mon Sep 17 00:00:00 2001 From: Jakub Kadlčík Date: Oct 23 2018 01:48:48 +0000 Subject: [PATCH 1/6] [frontend][rpmbuild] preprocess repo URLs on frontend Although the `build_config` API structure needs to be a bit more complicated, there are several benefits of preprocessing repo URLs on frontend. - The copr-rpmbuild doesn't need to know `backend_url` - Multiple clients can use the build config from API without needing to preprocess the URLs by themselves - Only frontend will know the ?priority syntax. --- diff --git a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_build_chroots.py b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_build_chroots.py index 5884974..ad7eeab 100644 --- a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_build_chroots.py +++ b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_build_chroots.py @@ -18,9 +18,8 @@ def to_dict(build_chroot): def build_config(build_chroot): config = generate_build_config(build_chroot.build.copr, build_chroot.name) - copr_chroot = CoprChrootsLogic.get_by_name_safe(build_chroot.build.copr, build_chroot.name) return { - "additional_repos": generate_additional_repos(copr_chroot), + "repos": config.get("repos"), "additional_packages": config.get("additional_packages"), "use_bootstrap_container": config.get("use_bootstrap_container"), "with_opts": config.get("with_opts"), diff --git a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_project_chroots.py b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_project_chroots.py index 7dfab4c..58ff1e1 100644 --- a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_project_chroots.py +++ b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_project_chroots.py @@ -1,7 +1,7 @@ import flask from . import query_params, get_copr, file_upload, GET, PUT from .json2form import get_form_compatible_data -from coprs.helpers import generate_additional_repos +from coprs.helpers import generate_additional_repos, generate_build_config from coprs.views.misc import api_login_required from coprs.views.apiv3_ns import apiv3_ns from coprs.logic.complex_logic import ComplexLogic @@ -24,10 +24,11 @@ def to_dict(project_chroot): def to_build_config_dict(project_chroot): + config = generate_build_config(project_chroot.copr, project_chroot.name) return { "chroot": project_chroot.name, + "repos": config["repos"], "additional_packages": (project_chroot.buildroot_pkgs or "").split(), - "additional_repos": generate_additional_repos(project_chroot), "use_bootstrap_container": project_chroot.copr.use_bootstrap_container, "enable_net": project_chroot.copr.enable_net, "with_opts": str_to_list(project_chroot.with_opts), diff --git a/rpmbuild/copr_rpmbuild/helpers.py b/rpmbuild/copr_rpmbuild/helpers.py index e01119b..5f7e334 100644 --- a/rpmbuild/copr_rpmbuild/helpers.py +++ b/rpmbuild/copr_rpmbuild/helpers.py @@ -236,48 +236,6 @@ def parse_copr_name(name): return ownername, projectname -def generate_repo_name(repo_url): - """ based on url, generate repo name """ - repo_url = re.sub("[^a-zA-Z0-9]", '_', repo_url) - repo_url = re.sub("(__*)", '_', repo_url) - repo_url = re.sub("(_*$)|^_*", '', repo_url) - return repo_url - - -def preprocess_repo_url(repo_url, chroot, backend_base_url): - """ - Expands variables and sanitize repo url to be used for mock config - """ - parsed_url = urlparse(repo_url) - - if parsed_url.scheme == "copr": - user = parsed_url.netloc - prj = parsed_url.path.split("/")[1] - repo_url = "/".join([ - backend_base_url, - "results", user, prj, chroot - ]) + "/" - - repo_url = repo_url.replace("$chroot", chroot) - repo_url = repo_url.replace("$distname", chroot.rsplit("-", 2)[0]) - return repo_url - - -def get_additional_repo_configs(repo_urls, chroot, backend_base_url): - repo_configs = [] - - for repo_url in repo_urls: - repo_name = generate_repo_name(repo_url) - repo_config = { - "id": repo_name, - "url": preprocess_repo_url(repo_url, chroot, backend_base_url), - "name": "Additional repo {0}".format(repo_name), - } - repo_configs.append(repo_config) - - return repo_configs - - def dump_live_log(logfile): filter_continuing_lines = r"sed 's/.*\x0D\([^\x0a]\)/\1/g' --unbuffered" tee_output = "tee -a {0}".format(pipes.quote(logfile)) diff --git a/rpmbuild/main.py b/rpmbuild/main.py index 22f3e42..12f817a 100755 --- a/rpmbuild/main.py +++ b/rpmbuild/main.py @@ -24,7 +24,7 @@ except ImportError: from copr_rpmbuild import providers from copr_rpmbuild.builders.mock import MockBuilder from copr_rpmbuild.helpers import read_config, extract_srpm, locate_srpm, \ - SourceType, parse_copr_name, get_additional_repo_configs, \ + SourceType, parse_copr_name, \ copr_chroot_to_task_id, dump_live_log try: @@ -214,11 +214,6 @@ def get_task(args, config, build_config_url_path=None, task_id=None): 'srpm_build_method': args.srpm_build_method, }) - # temporary due to transition to using api3 instead of /backend/ interface - if not task.get('repos') and task.get('additional_repos'): - task['repos'] = get_additional_repo_configs( - task['additional_repos'], args.chroot, config.get('main', 'backend_url')) - return task From 89ebf2efb01804be2fcb076fec93c3e7baebeeb8 Mon Sep 17 00:00:00 2001 From: Jakub Kadlčík Date: Nov 13 2018 20:16:09 +0000 Subject: [PATCH 2/6] [frontend] dont remove additional_repos list We can't break backward compatibility like that --- diff --git a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_build_chroots.py b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_build_chroots.py index ad7eeab..d372790 100644 --- a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_build_chroots.py +++ b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_build_chroots.py @@ -18,8 +18,10 @@ def to_dict(build_chroot): def build_config(build_chroot): config = generate_build_config(build_chroot.build.copr, build_chroot.name) + copr_chroot = CoprChrootsLogic.get_by_name_safe(build_chroot.build.copr, build_chroot.name) return { "repos": config.get("repos"), + "additional_repos": generate_additional_repos(copr_chroot), "additional_packages": config.get("additional_packages"), "use_bootstrap_container": config.get("use_bootstrap_container"), "with_opts": config.get("with_opts"), diff --git a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_project_chroots.py b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_project_chroots.py index 58ff1e1..e7d0c4d 100644 --- a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_project_chroots.py +++ b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_project_chroots.py @@ -28,6 +28,7 @@ def to_build_config_dict(project_chroot): return { "chroot": project_chroot.name, "repos": config["repos"], + "additional_repos": generate_additional_repos(project_chroot), "additional_packages": (project_chroot.buildroot_pkgs or "").split(), "use_bootstrap_container": project_chroot.copr.use_bootstrap_container, "enable_net": project_chroot.copr.enable_net, From 85fce1cd2d07d50f0253cc93aeb8aed84c30366b Mon Sep 17 00:00:00 2001 From: Jakub Kadlčík Date: Nov 13 2018 20:16:09 +0000 Subject: [PATCH 3/6] [frontend] refactor repo_id property --- diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index 288061c..c165f62 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -376,10 +376,7 @@ class Copr(db.Model, helpers.Serializer, CoprSearchRelatedData): @property def repo_id(self): - if self.is_a_group_project: - return "group_{}-{}".format(self.group.name, self.main_dir.name) - else: - return "{}-{}".format(self.user.name, self.main_dir.name) + return "-".join([self.owner_name.replace("@", "group_"), self.name]) @property def modules_url(self): From 4e6571e88e4fb7bb0cfe06ed97fa422f3302cae2 Mon Sep 17 00:00:00 2001 From: Jakub Kadlčík Date: Nov 13 2018 20:16:09 +0000 Subject: [PATCH 4/6] [frontend][rpmbuild] provide repo_id in project chroot build config --- diff --git a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_project_chroots.py b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_project_chroots.py index e7d0c4d..1bfe7ad 100644 --- a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_project_chroots.py +++ b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_project_chroots.py @@ -26,6 +26,7 @@ def to_dict(project_chroot): def to_build_config_dict(project_chroot): config = generate_build_config(project_chroot.copr, project_chroot.name) return { + "repo_id": "_".join([config["project_id"], project_chroot.name]), "chroot": project_chroot.name, "repos": config["repos"], "additional_repos": generate_additional_repos(project_chroot), diff --git a/rpmbuild/copr_rpmbuild/helpers.py b/rpmbuild/copr_rpmbuild/helpers.py index 5f7e334..7c1a2b9 100644 --- a/rpmbuild/copr_rpmbuild/helpers.py +++ b/rpmbuild/copr_rpmbuild/helpers.py @@ -223,12 +223,6 @@ def build_srpm(srcdir, destdir): run_cmd(cmd) -def copr_chroot_to_task_id(copr, chroot): - copr_token = re.sub('@', 'group_', copr) - copr_token = re.sub('/', '-', copr_token) - return copr_token +'-'+chroot - - def parse_copr_name(name): m = re.match(r"([^/]+)/(.*)", name) ownername = m.group(1) diff --git a/rpmbuild/main.py b/rpmbuild/main.py index 12f817a..1e1c06a 100755 --- a/rpmbuild/main.py +++ b/rpmbuild/main.py @@ -24,8 +24,7 @@ except ImportError: from copr_rpmbuild import providers from copr_rpmbuild.builders.mock import MockBuilder from copr_rpmbuild.helpers import read_config, extract_srpm, locate_srpm, \ - SourceType, parse_copr_name, \ - copr_chroot_to_task_id, dump_live_log + SourceType, parse_copr_name, dump_live_log try: from urllib.parse import urlparse, urljoin @@ -203,6 +202,9 @@ def get_task(args, config, build_config_url_path=None, task_id=None): if args.chroot: task['chroot'] = args.chroot + if args.copr: + task['task_id'] = task['repo_id'] + if args.submode == 'scm': task['source_type'] = SourceType.SCM task['source_json'].update({ @@ -250,11 +252,13 @@ def build_rpm(args, config): if not args.chroot: raise RuntimeError("Missing --chroot parameter") + task_id = None + build_config_url_path = None + if args.build_id: task_id = "-".join([args.build_id, args.chroot]) build_config_url_path = urljoin("/backend/get-build-task/", task_id) elif args.copr: - task_id = copr_chroot_to_task_id(args.copr, args.chroot) ownername, projectname = parse_copr_name(args.copr) get_params = { 'ownername': ownername, @@ -263,9 +267,6 @@ def build_rpm(args, config): } build_config_url_path = ("/api_3/project-chroot/build-config?" + urlencode(get_params)) - else: - task_id = None - build_config_url_path = None task = get_task(args, config, build_config_url_path, task_id) log_task(task) @@ -291,11 +292,13 @@ def dump_configs(args, config): if not args.chroot: raise RuntimeError("Missing --chroot parameter") + task_id = None + build_config_url_path = None + if args.build_id: task_id = "-".join([args.build_id, args.chroot]) build_config_url_path = urljoin("/backend/get-build-task/", task_id) elif args.copr: - task_id = copr_chroot_to_task_id(args.copr, args.chroot) ownername, projectname = parse_copr_name(args.copr) get_params = { 'ownername': ownername, @@ -304,9 +307,6 @@ def dump_configs(args, config): } build_config_url_path = ("/api_3/project-chroot/build-config?" + urlencode(get_params)) - else: - task_id = None - build_config_url_path = None task = get_task(args, config, build_config_url_path, task_id) log_task(task) From a815f4788359788e9e48f890a9ede8573458ba49 Mon Sep 17 00:00:00 2001 From: Jakub Kadlčík Date: Nov 13 2018 20:16:09 +0000 Subject: [PATCH 5/6] [frontend][rpmbuild] rename repos 'url' attribute to 'baseurl' --- diff --git a/frontend/coprs_frontend/coprs/helpers.py b/frontend/coprs_frontend/coprs/helpers.py index ae78814..1eef8b0 100644 --- a/frontend/coprs_frontend/coprs/helpers.py +++ b/frontend/coprs_frontend/coprs/helpers.py @@ -509,14 +509,14 @@ def generate_build_config(copr, chroot_id): repos = [{ "id": "copr_base", - "url": copr.repo_url + "/{}/".format(chroot_id), + "baseurl": copr.repo_url + "/{}/".format(chroot_id), "name": "Copr repository", }] if not copr.auto_createrepo: repos.append({ "id": "copr_base_devel", - "url": copr.repo_url + "/{}/devel/".format(chroot_id), + "baseurl": copr.repo_url + "/{}/devel/".format(chroot_id), "name": "Copr buildroot", }) @@ -526,7 +526,7 @@ def generate_build_config(copr, chroot_id): params = parse_repo_params(repo) repo_view = { "id": generate_repo_name(repo), - "url": pre_process_repo_url(chroot_id, repo), + "baseurl": pre_process_repo_url(chroot_id, repo), "name": "Additional repo " + generate_repo_name(repo), } repo_view.update(params) diff --git a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py index 024d10c..afd8da8 100755 --- a/frontend/coprs_frontend/coprs/views/api_ns/api_general.py +++ b/frontend/coprs_frontend/coprs/views/api_ns/api_general.py @@ -1074,4 +1074,8 @@ def copr_build_config(copr, chroot): if not output['build_config']: raise LegacyApiError('Chroot not found.') + # To preserve backwards compatibility, repos needs to have the `url` attribute + for repo in output["build_config"]["repos"]: + repo["url"] = repo["baseurl"] + return flask.jsonify(output) diff --git a/rpmbuild/mock.cfg.j2 b/rpmbuild/mock.cfg.j2 index 805ff5f..a240dbc 100644 --- a/rpmbuild/mock.cfg.j2 +++ b/rpmbuild/mock.cfg.j2 @@ -27,7 +27,7 @@ config_opts['{{ pkg_manager_conf }}.conf'] += """ {% for repo in repos %} [{{ repo["id"] }}] name='{{ repo["name"] }}' -baseurl={{ repo["url"] }} +baseurl={{ repo["baseurl"] }} gpgcheck=0 enabled=1 skip_if_unavailable=1 From f1ee83cc7221e7c4be24dab37aec0daacd30a093 Mon Sep 17 00:00:00 2001 From: Jakub Kadlčík Date: Nov 13 2018 20:16:09 +0000 Subject: [PATCH 6/6] [frontend][rpmbuild] let mock rootdir generation on clients --- diff --git a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_project_chroots.py b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_project_chroots.py index 1bfe7ad..e7d0c4d 100644 --- a/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_project_chroots.py +++ b/frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_project_chroots.py @@ -26,7 +26,6 @@ def to_dict(project_chroot): def to_build_config_dict(project_chroot): config = generate_build_config(project_chroot.copr, project_chroot.name) return { - "repo_id": "_".join([config["project_id"], project_chroot.name]), "chroot": project_chroot.name, "repos": config["repos"], "additional_repos": generate_additional_repos(project_chroot), diff --git a/rpmbuild/copr_rpmbuild/helpers.py b/rpmbuild/copr_rpmbuild/helpers.py index 7c1a2b9..5f7e334 100644 --- a/rpmbuild/copr_rpmbuild/helpers.py +++ b/rpmbuild/copr_rpmbuild/helpers.py @@ -223,6 +223,12 @@ def build_srpm(srcdir, destdir): run_cmd(cmd) +def copr_chroot_to_task_id(copr, chroot): + copr_token = re.sub('@', 'group_', copr) + copr_token = re.sub('/', '-', copr_token) + return copr_token +'-'+chroot + + def parse_copr_name(name): m = re.match(r"([^/]+)/(.*)", name) ownername = m.group(1) diff --git a/rpmbuild/main.py b/rpmbuild/main.py index 1e1c06a..8be26a6 100755 --- a/rpmbuild/main.py +++ b/rpmbuild/main.py @@ -24,7 +24,7 @@ except ImportError: from copr_rpmbuild import providers from copr_rpmbuild.builders.mock import MockBuilder from copr_rpmbuild.helpers import read_config, extract_srpm, locate_srpm, \ - SourceType, parse_copr_name, dump_live_log + SourceType, parse_copr_name, dump_live_log, copr_chroot_to_task_id try: from urllib.parse import urlparse, urljoin @@ -203,7 +203,7 @@ def get_task(args, config, build_config_url_path=None, task_id=None): task['chroot'] = args.chroot if args.copr: - task['task_id'] = task['repo_id'] + task['task_id'] = copr_chroot_to_task_id(args.copr, args.chroot) if args.submode == 'scm': task['source_type'] = SourceType.SCM