#313 [rpmbuild] add --dump-configs option
Merged 5 years ago by clime. Opened 5 years ago by clime.

@@ -51,18 +51,11 @@ 

          self.resultdir = resultdir

          self.config = config

          self.logfile = self.config.get("main", "logfile")

-         log.info(self.__dict__)

  

      def run(self):

-         configdir = os.path.join(self.resultdir, "configs")

-         os.makedirs(configdir)

-         shutil.copy2("/etc/mock/site-defaults.cfg", configdir)

-         shutil.copy2("/etc/mock/{0}.cfg".format(self.chroot), configdir)

-         cfg = self.render_config_template()

-         with open(os.path.join(configdir, "child.cfg"), "w") as child:

-             child.write(cfg)

- 

          open(self.logfile, 'w').close() # truncate logfile

+         configdir = os.path.join(self.resultdir, "configs")

+         self.prepare_configs(configdir)

  

          spec = locate_spec(self.sourcedir)

          shutil.copy(spec, self.resultdir)
@@ -71,6 +64,20 @@ 

          srpm = locate_srpm(self.resultdir)

          self.produce_rpm(srpm, configdir, self.resultdir)

  

+     def prepare_configs(self, configdir):

+         site_config_path = os.path.join(configdir, "site-defaults.cfg")

+         mock_config_path = os.path.join(configdir, "{0}.cfg".format(self.chroot))

+         child_config_path = os.path.join(configdir, "child.cfg")

+ 

+         os.makedirs(configdir, exist_ok=True)

+         shutil.copy2("/etc/mock/site-defaults.cfg", site_config_path)

+         shutil.copy2("/etc/mock/{0}.cfg".format(self.chroot), mock_config_path)

+         cfg = self.render_config_template()

+         with open(child_config_path, "w") as child:

+             child.write(cfg)

+ 

+         return [child_config_path, mock_config_path, site_config_path]

+ 

      def render_config_template(self):

          jinja_env = Environment(loader=FileSystemLoader(CONF_DIRS))

          template = jinja_env.get_template("mock.cfg.j2")

file modified
+59 -35
@@ -70,6 +70,7 @@ 

      product = shared_parser.add_mutually_exclusive_group()

      product.add_argument("--rpm", action="store_true", help="Build rpms. This is the default action.")

      product.add_argument("--srpm", action="store_true", help="Build srpm.")

+     product.add_argument("--dump-configs", action="store_true", help="Only dump configs, without actual building.")

      #product.add_argument("--tgz", action="store_true", help="Make tar.gz with build sources, spec and patches.")

  

      base_parser = argparse.ArgumentParser(description="COPR building tool.", parents=[shared_parser])
@@ -132,7 +133,14 @@ 

      try:

          fcntl.lockf(lockfd, fcntl.LOCK_EX, 1)

          init(args, config)

-         action = build_srpm if args.srpm else build_rpm

+ 

+         if args.dump_configs:

+             action = dump_configs

+         elif args.srpm:

+             action = build_srpm

+         else:

+             action = build_rpm

+ 

          action(args, config)

      except (RuntimeError, OSError):

          log.exception("")
@@ -152,22 +160,6 @@ 

          os.makedirs(resultdir)

  

  

- def task_merge_args(task, args):

-     if args.chroot:

-         task['chroot'] = args.chroot

- 

-     if args.submode == 'scm':

-         task["source_type"] = SourceType.SCM

-         task["source_json"].update({

-             'clone_url': args.clone_url,

-             'committish': args.committish,

-             'subdirectory': args.subdirectory,

-             'spec': args.spec,

-             'type': args.type,

-             'srpm_build_method': args.srpm_build_method,

-         })

- 

- 

  def produce_srpm(task, config, resultdir):

      """

      create tmpdir to allow --private-users=pick with make_srpm
@@ -183,21 +175,42 @@ 

              shutil.copy(os.path.join(tmpdir_abspath, item), resultdir)

  

  

- def build_srpm(args, config):

+ def get_task(args, config, task_url_path=None):

+     task = {'source_type': None, 'source_json': {}}

+ 

+     if task_url_path:

+         task.update(get_vanilla_task(task_url_path, config))

+ 

      if args.chroot:

-         raise RuntimeError("--chroot option is not supported with --srpm")

+         task['chroot'] = args.chroot

  

-     resultdir = config.get("main", "resultdir")

+     if args.submode == 'scm':

+         task['source_type'] = SourceType.SCM

+         task['source_json'].update({

+             'clone_url': args.clone_url,

+             'committish': args.committish,

+             'subdirectory': args.subdirectory,

+             'spec': args.spec,

+             'type': args.type,

+             'srpm_build_method': args.srpm_build_method,

+         })

  

-     task = {'source_type': None, 'source_json': {}}

-     if args.build_id:

-         task.update(get_task("/backend/get-srpm-build-task/", args.build_id, config))

+     return task

  

-     task_merge_args(task, args)

  

+ def log_task(task):

      pp = pprint.PrettyPrinter(width=120)

      log.info("Task:\n"+pp.pformat(task)+'\n')

  

+ 

+ def build_srpm(args, config):

+     if args.chroot:

+         raise RuntimeError("--chroot option is not supported with --srpm")

+ 

+     task = get_task(args, config, urljoin("/backend/get-srpm-build-task/", args.build_id))

+     log_task(task)

+ 

+     resultdir = config.get("main", "resultdir")

      produce_srpm(task, config, resultdir)

  

      log.info("Output: {}".format(
@@ -211,15 +224,9 @@ 

      if not args.chroot:

          raise RuntimeError("Missing --chroot parameter")

  

-     task = {'source_type': None, 'source_json': {}}

-     if args.build_id:

-         task_id = "-".join([args.build_id, args.chroot])

-         task.update(get_task("/backend/get-build-task/", task_id, config))

- 

-     task_merge_args(task, args)

- 

-     pp = pprint.PrettyPrinter(width=120)

-     log.info("Task:\n"+pp.pformat(task)+'\n')

+     task_id = "-".join([args.build_id, args.chroot])

+     task = get_task(args, config, urljoin("/backend/get-build-task/", task_id))

+     log_task(task)

  

      sourcedir = tempfile.mkdtemp()

      scm_provider = providers.ScmProvider(task["source_json"], sourcedir, config)
@@ -238,9 +245,26 @@ 

      shutil.rmtree(sourcedir)

  

  

- def get_task(endpoint, id, config):

+ def dump_configs(args, config):

+     if not args.chroot:

+         raise RuntimeError("Missing --chroot parameter")

+ 

+     task_id = "-".join([args.build_id, args.chroot])

+     task = get_task(args, config, urljoin("/backend/get-build-task/", task_id))

+     log_task(task)

+ 

+     resultdir = config.get("main", "resultdir")

+     builder = MockBuilder(task, None, resultdir, config)

+ 

+     configdir = os.path.join(resultdir, "configs")

+     config_paths = builder.prepare_configs(configdir)

+     for config_path in config_paths:

+         log.info("Wrote: "+config_path)

+ 

+ 

+ def get_vanilla_task(task_url_path, config):

      try:

-         url = urljoin(urljoin(config.get("main", "frontend_url"), endpoint), id)

+         url = urljoin(config.get("main", "frontend_url"), task_url_path)

          response = requests.get(url)

          task = response.json()

          task["source_json"] = json.loads(task["source_json"])

This PR implements:

copr-rpmbuild --dump-configs

functionality. E.g.

 copr-rpmbuild  --dump-configs --chroot fedora-27-x86_64 --build-id 766668

which dumps all the mock configs used for the build into /var/lib/copr-rpmbuild/results/configs/.

I will likely add some unittests + review but feel free to review/comment anytime, please.

This is not an alternative to copr mock-config. I'd like to respect Mirek, and wait
for the meeting for more discussion.

This "just" dump config, which would copr-rpmbuild use when I run it on my machine. What I thought this should be doing is to dump configs, which would Copr use when the task is run in Copr.
In other words, if Copr have some change in site-defaults.cfg and I run copr-rpmbuild --dump-configs then I will not get this change.

I'm not usually interested to see full dump, OTOH. Especially that copr uses too bleeding edge optoins. I'm mostly interested in additional build deps, and copr repositories so I can install build-deps in local mock.

Also, e.g. the tmpfs config is often unrealistic for my boxes :-( so the 1:1 configuration from copr backend result directory is counterproductive for daily packager's business; I have to edit it to make things work.

This is not an alternative to copr mock-config. I'd like to respect Mirek, and wait
for the meeting for more discussion.

Please, describe why it is not an alternative to copr mock-config. What's missing to make it an alternative?

I dislike re-iterating again :-(

I really use copr mock-config as a trivial daily helper to develop packages, usually the workflow looks like this:

$ copr mock-config my-copr chroot > my.cfg
$ mock -r ./my.cfg --init # (and keep the cache as long as reasonably possible)
$ mock --shell, mock --install, cp, ...

I have my site-defaults.cfg and fedora-- configurations tweaked appropriately to my development box, so mostly I appreciate mock configuration that only adds the additional repositories (sometimes, in SCLs, I need "additional_pkgs"). I can do this basically everywhere el6+ (el6 requires quick C&P because there's missing %include support in mock).

Alternative would be if copr-rpmbuild had:
1. no dependencies other than mock
2. was installable el6+
3. generated the same format of mock config

IOW, I basically need a trivial tool that converts

  "build_config": {
    "additional_packages": [], 
    "chroot": "fedora-rawhide-x86_64", 
    "project_id": "praiskup-ping", 
    "repos": [
      {
        "id": "copr_base", 
        "name": "Copr repository", 
        "url": "https://copr-be.cloud.fedoraproject.org/results/praiskup/ping/fedora-rawhide-x86_64/"
      }
    ], 

The rest is not interesting for me:

    "use_bootstrap_container": true, 
    "with_opts": [], 
    "without_opts": []

(with/without is something which should never be set in project/mock configuration, and it's some bug that I got use_bootstrap_container enabled in several of my projects)

Merging so I have a starting point for e.g.:

 copr-rpmbuild  --dump-configs --chroot fedora-28-x86_64 --copr @copr/copr

Pull-Request has been merged by clime

5 years ago

I veto this, as others have right to veto, too.

And I neither see single +1 in this PR, so it is clear sign of pull-request policy disrespect.