copr / copr

Clone

#2352 Add support for external repositories for custom build method
Merged a year ago by praiskup. Opened 2 years ago by mfocko.
copr/ mfocko/copr external-repos-for-custom-build  into  main

file modified
+5 -1 
@@ -407,7 +407,7 @@ 
 

              'script': ''.join(args.script.readlines()),
 

          }
 

          for arg in ['script_chroot', 'script_builddeps',
 

-                     'script_resultdir']:
 

+                     'script_resultdir', 'script_repos']:
 

              data[arg] = getattr(args, arg)
 

          return self.process_build(args, self.client.build_proxy.create_from_custom, data)
 

  
@@ -1304,6 +1304,10 @@ 
 

              '--script-resultdir',
 

              help='where SCRIPT generates the result, relatively to script\'s '
 

                   '$PWD (defaults to \'.\')')
 

+     parser_custom_args_parent.add_argument(
 

+             '--script-repos',
 

+             help="space separated string of additional repo urls for script dependencies"
 

+     )
 

  

      #########################################################
 

      ###                    Build options                  ###

file modified
+6 
@@ -1041,6 +1041,11 @@ 
 

          "Build dependencies",
 

          validators=[wtforms.validators.Optional()])
 

  

+     repos = wtforms.TextAreaField(
 

+         "External repositories for build dependencies",
 

+         validators=[UrlRepoListValidator()],
 

+         filters=[StringListFilter()])
 

+ 

      chroot = wtforms.SelectField(
 

          'Mock chroot',
 

          choices=[],
 
@@ -1077,6 +1082,7 @@ 
 

              "chroot": self.chroot.data,
 

              "builddeps": self.builddeps.data,
 

              "resultdir": self.resultdir.data,
 

+             "repos": self.repos.data,
 

          })

file modified
+4 -1 
@@ -534,7 +534,8 @@ 
 

  

      @classmethod
 

      def create_new_from_custom(cls, user, copr, script, script_chroot=None, script_builddeps=None,
 

-                                script_resultdir=None, chroot_names=None, copr_dirname=None, **kwargs):
 

+                                script_resultdir=None, script_repos=None, chroot_names=None,
 

+                                copr_dirname=None, **kwargs):
 

          """
 

          :type user: models.User
 

          :type copr: models.Copr
 
@@ -542,6 +543,7 @@ 
 

          :type script_chroot: str
 

          :type script_builddeps: str
 

          :type script_resultdir: str
 

+         :type script_repos: str
 

          :type chroot_names: List[str]
 

          :rtype: models.Build
 

          """
 
@@ -551,6 +553,7 @@ 
 

              'chroot': script_chroot,
 

              'builddeps': script_builddeps,
 

              'resultdir': script_resultdir,
 

+             'repos': script_repos,
 

          }
 

  

          return cls.create_new(user, copr, source_type, json.dumps(source_dict),

file modified
+1 
@@ -12,6 +12,7 @@ 
 

          form.chroot,
 

          placeholder="fedora-latest-x86_64",
 

          info="what chroot to run the <strong>script</strong> in") }}
 

+   {{ render_field(form.repos, rows=5, cols=50, placeholder='Optional - URL to additional yum repos, which can be used during build. Space separated. This should be baseurl from .repo file. E.g.: http://copr-be.cloud.fedoraproject.org/results/rhughes/f20-gnome-3-12/fedora-$releasever-$basearch/') }}
 

    {{ render_field(form.builddeps, placeholder="Optional - space-separated list of packages",
 

          info="packages that the <strong>script</strong> requires for its execution" ) }}
 

    {{ render_field(form.resultdir, placeholder="Optional - directory where SCRIPT generates sources",

file modified
+1 
@@ -296,6 +296,7 @@ 
 

              form.chroot.data,
 

              form.builddeps.data,
 

              form.resultdir.data,
 

+             form.repos.data,
 

              **options,
 

          )
 

      return process_creating_new_build(copr, form, create_new_build)

file modified
+4 
@@ -206,6 +206,10 @@ 
 

              "project_name": task.copr_name,
 

              "project_dirname": task.copr_dirname,
 

              "appstream": bool(task.copr.appstream),
 

+             "repos": BuildConfigLogic.get_additional_repo_views(
 

+                 task.source_json_dict.get('repos', "").split(),
 

+                 chroot,
 

+             ),
 

          })
 

  

      except Exception as err:

file modified
+6 -2 
@@ -264,8 +264,8 @@ 
 

          return self._create(endpoint, data, buildopts=buildopts)
 

  

      def create_from_custom(self, ownername, projectname, script, script_chroot=None,
 

-                            script_builddeps=None, script_resultdir=None, buildopts=None,
 

-                            project_dirname=None):
 

+                            script_builddeps=None, script_resultdir=None, script_repos=None,
 

+                            buildopts=None, project_dirname=None):
 

          """
 

          Create a build from custom script.
 

  
@@ -277,6 +277,9 @@ 
 

          :param script_builddeps: [optional] list of script's dependencies
 

          :param script_resultdir: [optional] where script generates results
 

              (relative to cwd)
 

+         :parm script_repos: [optional] external repositories containing script's
 

+             dependencies
 

+         :param buildopts: http://python-copr.readthedocs.io/en/latest/client_v3/build_options.html
 

          :param str project_dirname:
 

          :return: Munch
 

          """
 
@@ -289,6 +292,7 @@ 
 

              "builddeps": script_builddeps,
 

              "resultdir": script_resultdir,
 

              "project_dirname": project_dirname,
 

+             "repos": script_repos,
 

          }
 

          return self._create(endpoint, data, buildopts=buildopts)

file modified
+2 
@@ -243,6 +243,7 @@ 
 

  install -m 644 mock.cfg.j2 %{buildroot}%{_sysconfdir}/copr-rpmbuild/mock.cfg.j2
 

  install -m 644 rpkg.conf.j2 %{buildroot}%{_sysconfdir}/copr-rpmbuild/rpkg.conf.j2
 

  install -m 644 mock-source-build.cfg.j2 %{buildroot}%{_sysconfdir}/copr-rpmbuild/
 

+ install -m 644 mock-custom-build.cfg.j2 %{buildroot}%{_sysconfdir}/copr-rpmbuild/
 

  

  cat <<EOF > %buildroot%mock_config_overrides/README
 

  Contents of this directory is used by %_bindir/copr-update-builder script.
 
@@ -307,6 +308,7 @@ 
 

  %config(noreplace) %{_sysconfdir}/copr-rpmbuild/mock.cfg.j2
 

  %config(noreplace) %{_sysconfdir}/copr-rpmbuild/rpkg.conf.j2
 

  %config(noreplace) %{_sysconfdir}/copr-rpmbuild/mock-source-build.cfg.j2
 

+ %config(noreplace) %{_sysconfdir}/copr-rpmbuild/mock-custom-build.cfg.j2
 

  

  %files -n copr-builder
 

  %license LICENSE

file modified
+2 -1 
@@ -18,10 +18,11 @@ 
 

      # pylint: disable=too-many-instance-attributes
 

      _safe_resultdir = None
 

  

-     def __init__(self, source_dict, config, macros=None):
 

+     def __init__(self, source_dict, config, macros=None, task=None):
 

          self.source_dict = source_dict
 

          self.config = config
 

          self.request = SafeRequest(log=log)
 

+         self.task = task
 

  

          # Additional macros that should be defined in the buildroot
 

          self.macros = macros or {}

file modified
+14 -12 
@@ -2,8 +2,10 @@ 
 

  import logging
 

  import shutil
 

  

+ from jinja2 import Environment, FileSystemLoader
 

  from copr_rpmbuild import helpers
 

  from .base import Provider
 

+ from ..helpers import CONF_DIRS
 

  

  

  log = logging.getLogger("__main__")
 
@@ -12,6 +14,7 @@ 
 

  class CustomProvider(Provider):
 

      chroot = 'fedora-rawhide-x86_64'
 

      builddeps = None
 

+     repos = None
 

      file_script = None
 

      inner_resultdir = None
 

      inner_workdir = '/workdir'
 
@@ -24,6 +27,7 @@ 
 

          self.chroot = source_json.get('chroot')
 

          self.inner_resultdir = source_json.get('resultdir')
 

          self.builddeps = source_json.get('builddeps')
 

+         self.repos = self.task.get('repos')
 

          self.timeout = source_json.get("timeout", 3600)
 

  

          if 'hook_data' in source_json:
 
@@ -37,18 +41,16 @@ 
 

              script.write(source_json['script'])
 

  

      def render_mock_config_template(self, *_args):
 

-         template = "include('/etc/mock/{0}.cfg')\n".format(self.chroot)
 

-         template += "config_opts['rpmbuild_networking'] = True\n"
 

-         template += "config_opts['use_host_resolv'] = True\n"
 

- 

-         # Important e.g. to keep '/script' file available across several
 

-         # /bin/mock calls (when tmpfs_enable is on).
 

-         template += "config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True\n"
 

- 

-         for key, value in self.macros.items():
 

-             template += ("config_opts['macros']['{0}'] = '{1}'\n"
 

-                          .format(key, value))
 

-         return template
 

+         """
 

+         Return a mock config (as a string) for a specific task
 

+         """
 

+         jinja_env = Environment(loader=FileSystemLoader(CONF_DIRS))
 

+         template = jinja_env.get_template("mock-custom-build.cfg.j2")
 

+         return template.render(
 

+             chroot=self.chroot,
 

+             repos=self.repos,
 

+             macros=self.macros,
 

+         )
 

  

      def produce_srpm(self):
 

          mock_config_file = self.generate_mock_config("mock-config.cfg")

file modified
+1 -1 
@@ -160,7 +160,7 @@ 
 

      try:
 

          macros = macros_for_task(task, config)
 

          clazz = providers.factory(task["source_type"])
 

-         provider = clazz(task["source_json"], config, macros=macros)
 

+         provider = clazz(task["source_json"], config, macros=macros, task=task)
 

          provider.produce_srpm()
 

          provider.copy_insecure_results()
 

      finally:

file added
+36 
@@ -0,0 +1,36 @@ 
 

+ include('/etc/mock/{{ chroot }}.cfg')
 

+ config_opts['rpmbuild_networking'] = True
 

+ config_opts['use_host_resolv'] = True
 

+ 

+ # Important e.g. to keep '/script' file available across several
 

+ # /bin/mock calls (when tmpfs_enable is on).
 

+ config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True
 

+ 

+ {%- for key, value in macros.items() %}
 

+ config_opts['macros']['{{ key }}'] = '{{ value }}'
 

+ {%- endfor %}
 

+ 

+ {% if repos %}
 

+ config_opts[f"{config_opts.package_manager}.conf"] += """
 

+ {% for repo in repos %}
 

+ [{{ repo["id"] }}]
 

+ name='{{ repo["name"] }}'
 

+ baseurl={{ repo["baseurl"] }}
 

+ gpgcheck=0
 

+ enabled=1
 

+ skip_if_unavailable=0
 

+ metadata_expire=0
 

+ cost=1
 

+ best=1
 

+ 

+ {%- if "priority" in repo %}
 

+ priority={{ repo["priority"] }}
 

+ {%- endif %}
 

+ 

+ {%- if "module_hotfixes" in repo %}
 

+ module_hotfixes={{ repo["module_hotfixes"] }}
 

+ {% endif -%}
 

+ 

+ {% endfor %}
 

+ """
 

+ {% endif %}

  • python
    • Add script_repos parameter that allows specifying external repositories
      that can be used for installing script's build dependencies.
    • Add missing buildopts to the docstring.
  • frontend
    • Pass external repos acquired from the API endpoint.
    • Add text area for external repos when triggering custom build from the
      web view.
  • rpmbuild
    • Add all external repositories passed in to the custom build method to
      the mock config.
  • cli - added one more argument for script repos

Fixes #530

Build succeeded.

Do you mean a jinja template, like we have e.g. the mock.cfg.j2? I don't disagree ;)

A small wish - would you mind adding a copr-cli command line option for this as well?

Thank you for the PR, @mfocko

Metadata Update from @praiskup:
- Pull-request tagged with: wip
2 years ago

rebased onto bb7e022
2 years ago

Build succeeded.

I'll add it to the cli by the noon tomorrow.

Tried it testing locally, but somehow I cannot get that new field to show up, if I change any of the ones that are already there, the changes take effect. /o\

1 new commit added

  • cli: add parameter for custom method repos
2 years ago

Build succeeded.

Build succeeded.

5 new commits added

  • cli: add parameter for custom method repos
  • rpmbuild: switch to template for custom method
  • frontend: expand repos for custom SRPM
  • frontend: process external repos for custom build
  • python: add external repos for custom build
2 years ago

Build succeeded.

5 new commits added

  • cli: add parameter for custom method repos
  • rpmbuild: switch to template for custom method
  • frontend: expand repos for custom SRPM
  • frontend: process external repos for custom build
  • python: add external repos for custom build
2 years ago

Build succeeded.

Metadata Update from @praiskup:
- Pull-request untagged with: wip
2 years ago

This looks good to me. Thank you!

1 new commit added

  • rpmbuild: use repos from task for custom method
2 years ago

Build succeeded.

1 new commit added

  • rpmbuild: add new template to the specfile
2 years ago

Build succeeded.

There was an issue with rpmbuild processing repos and I've also forgotten to add the template for custom SRPM method to the specfile, it should be fixed now.

Locally seems to work fine.

+1

Pull-Request has been merged by praiskup
a year ago
Metadata
None
No Tags
Changes Summary 12
+5 -1
file changed
cli/copr_cli/main.py
+6 -0
file changed
frontend/coprs_frontend/coprs/forms.py
+4 -1
file changed
frontend/coprs_frontend/coprs/logic/builds_logic.py
+1 -0
file changed
frontend/coprs_frontend/coprs/templates/coprs/detail/_method_forms.html
+1 -0
file changed
frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_builds.py
+4 -0
file changed
frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py
+6 -2
file changed
python/copr/v3/proxies/build.py
+2 -0
file changed
rpmbuild/copr-rpmbuild.spec
+2 -1
file changed
rpmbuild/copr_rpmbuild/providers/base.py
+14 -12
file changed
rpmbuild/copr_rpmbuild/providers/custom.py
+1 -1
file changed
rpmbuild/main.py
+36
file added
rpmbuild/mock-custom-build.cfg.j2
Powered by Pagure 5.13.3
Documentation • File an Issue • About • SSH Hostkey/Fingerprint
© Red Hat, Inc. and others.