#2197 rpmbuild: define copr-specific macros also for SRPM builds
Merged 3 months ago by praiskup. Opened 3 months ago by frostyx.
copr/ frostyx/copr srpm-macros  into  main

@@ -0,0 +1,35 @@ 

+ # There is probably no way to assert that a macro is defined so we need to be

+ # creative. Many failures because of missing macros would happen when building

+ # an RPM package. In order to test that macros are defined when building a SRPM,

+ # we can for example have a conditional preamble.

+ 

+ %if 0%{?copr_username:1} && 0%{?copr_projectname:1}

+ Name:       pkg-with-macros

+ Version:    1.0

+ Release:    1%{?dist}

+ Summary:    Testing spec file

+ License:    GPLv2

+ URL:        https://pagure.io/copr/copr

+ %endif

+ 

+ 

+ %description

+ Test spec file with macros

+ 

+ %files

+ 

+ %build

+ 

+ %check

+ 

+ # Make sure that macros are defined when building the actual RPM package

+ %if 0%{?copr_username:1} && 0%{?copr_projectname:1}

+ # It's okay, macros are defined

+ %else

+ exit 1

+ %endif

+ 

+ 

+ %changelog

+ * Thu May 12 2022 Jakub Kadlcik <jkadlcik@redhat.com> - 1.0-1

+ - Initial version

@@ -0,0 +1,43 @@ 

+ #! /bin/bash

+ #

+ # Copyright (c) 2022 Red Hat, Inc.

+ #

+ # This program is free software: you can redistribute it and/or

+ # modify it under the terms of the GNU General Public License as

+ # published by the Free Software Foundation, either version 2 of

+ # the License, or (at your option) any later version.

+ #

+ # This program is distributed in the hope that it will be

+ # useful, but WITHOUT ANY WARRANTY; without even the implied

+ # warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

+ # PURPOSE.  See the GNU General Public License for more details.

+ #

+ # You should have received a copy of the GNU General Public License

+ # along with this program. If not, see http://www.gnu.org/licenses/.

+ 

+ 

+ # Include Beaker environment

+ . /usr/share/beakerlib/beakerlib.sh || exit 1

+ 

+ # Load config settings

+ HERE=$(dirname "$(realpath "$0")")

+ source "$HERE/config"

+ source "$HERE/helpers"

+ 

+ 

+ rlJournalStart

+     rlPhaseStartSetup

+         setup_checks

+         setupProjectName "PackageWithMacros"

+     rlPhaseEnd

+ 

+     rlPhaseStartTest

+         rlRun "copr-cli create --chroot $CHROOT $PROJECT"

+         rlRun -s "copr-cli build $PROJECT files/pkg-with-macros.spec "

+     rlPhaseEnd

+ 

+     rlPhaseStartCleanup

+         cleanProject

+     rlPhaseEnd

+ rlJournalPrintText

+ rlJournalEnd

@@ -6,7 +6,14 @@ 

  import subprocess

  

  from jinja2 import Environment, FileSystemLoader

- from ..helpers import locate_spec, locate_srpm, CONF_DIRS, get_mock_uniqueext, GentlyTimeoutedPopen

+ from ..helpers import (

+     locate_spec,

+     locate_srpm,

+     CONF_DIRS,

+     get_mock_uniqueext,

+     GentlyTimeoutedPopen,

+     macros_for_task,

+ )

  

  log = logging.getLogger("__main__")

  
@@ -33,12 +40,7 @@ 

          self.copr_projectname = task.get("project_name")

          self.modules = task.get("modules")

          self.isolation = task.get("isolation")

-         rpm_vendor_copr_name = self.config.get("main", "rpm_vendor_copr_name")

-         self.vendor = "{0} - {1} {2}".format(

-             rpm_vendor_copr_name,

-             "group" if self.copr_username.startswith("@") else "user",

-             self.copr_username,

-         )

+         self.macros = macros_for_task(task, config)

          self.uniqueext = get_mock_uniqueext()

  

      def run(self):
@@ -72,15 +74,18 @@ 

      def render_config_template(self):

          jinja_env = Environment(loader=FileSystemLoader(CONF_DIRS))

          template = jinja_env.get_template("mock.cfg.j2")

-         return template.render(chroot=self.chroot, task_id=self.task_id, buildroot_pkgs=self.buildroot_pkgs,

-                                enable_net=self.enable_net,

-                                bootstrap=self.bootstrap,

-                                bootstrap_image=self.bootstrap_image,

-                                repos=self.repos,

-                                copr_username=self.copr_username, copr_projectname=self.copr_projectname,

-                                modules=self.module_setup_commands,

-                                copr_build_id=self.build_id,

-                                vendor=self.vendor, isolation=self.isolation)

+         return template.render(

+             chroot=self.chroot,

+             buildroot_pkgs=self.buildroot_pkgs,

+             enable_net=self.enable_net,

+             bootstrap=self.bootstrap,

+             bootstrap_image=self.bootstrap_image,

+             repos=self.repos,

+             modules=self.module_setup_commands,

+             copr_build_id=self.build_id,

+             isolation=self.isolation,

+             macros=self.macros,

+         )

  

      def produce_srpm(self, spec, sources, resultdir):

          cmd = MOCK_CALL + [

@@ -1,8 +1,6 @@ 

  import errno

  import logging

- import munch

  import subprocess

- import rpm

  import glob

  import os

  import sys
@@ -11,8 +9,13 @@ 

  import datetime

  import pipes

  from threading import Timer

+ from collections import OrderedDict

+ import rpm

+ import munch

  

  from six.moves.urllib.parse import urlparse

+ from copr_common.enums import BuildSourceEnum

+ 

  

  log = logging.getLogger("__main__")

  
@@ -317,3 +320,43 @@ 

  

          checkout_cmd = ['git', 'checkout', committish]

          run_cmd(checkout_cmd, cwd=repo_path)

+ 

+ 

+ def macros_for_task(task, config):

+     """

+     Based on a task definition, generate RPM macros that should be defined in

+     the buildroot. Macros are simply returned as a `dict` and it is up to the

+     caller to pass them into Mock via `--define`, dump `~/.rpmmacros`, etc.

+ 

+     Each macro is to be defined with %-sign at the begining.

+     """

+     # We use OdrderedDict just to be sure that we generate the macros in the

+     # same order. It doesn't really matter in production, but is easier to test.

+     macros = OrderedDict({

+         "%copr_username": task["project_owner"],

+         "%copr_projectname": task["project_name"],

+     })

+ 

+     rpm_vendor_copr_name = config.get("main", "rpm_vendor_copr_name", fallback=None)

+     if rpm_vendor_copr_name:

+         vendor = "{0} - {1} {2}".format(

+             rpm_vendor_copr_name,

+             "group" if task["project_owner"].startswith("@") else "user",

+             task["project_owner"],

+         )

+         macros["%vendor"] = vendor

+ 

+     task_id = task.get("task_id")

+     if task_id:

+         macros["%buildtag"] = ".copr" + re.sub("-.*", "", task_id)

+ 

+     if not task["chroot"]:

+         macros["%_disable_source_fetch"] = "0"

+ 

+     protocols_str = config.get("main", "enabled_source_protocols", fallback=None)

+     if task["source_type"] != BuildSourceEnum.upload and protocols_str:

+         protocols_list = string2list(protocols_str)

+         protocols = ",".join(["+" + protocol for protocol in protocols_list])

+         macros["%__urlhelper_localopts"] = "--proto -all,{0}".format(protocols)

+ 

+     return macros

@@ -7,7 +7,6 @@ 

  

  from copr_common.request import SafeRequest

  

- from ..helpers import string2list

  

  log = logging.getLogger("__main__")

  
@@ -16,11 +15,14 @@ 

      # pylint: disable=too-many-instance-attributes

      _safe_resultdir = None

  

-     def __init__(self, source_dict, config):

+     def __init__(self, source_dict, config, macros=None):

          self.source_dict = source_dict

          self.config = config

          self.request = SafeRequest(log=log)

  

+         # Additional macros that should be defined in the buildroot

+         self.macros = macros or {}

+ 

          # Where we should produce output, everything there gets copied to

          # backend once build ends!

          self.real_resultdir = config.get("main", "resultdir")
@@ -104,10 +106,8 @@ 

      def create_rpmmacros(self):

          path = os.path.join(self.workdir, ".rpmmacros")

          with open(path, "w") as rpmmacros:

-             rpmmacros.write("%_disable_source_fetch 0\n")

-             enabled_protocols = string2list(self.config.get("main", "enabled_source_protocols"))

-             rpmmacros.write("%__urlhelper_localopts --proto -all,{0}\n"

-                             .format(','.join(["+"+protocol for protocol in enabled_protocols])))

+             for key, value in self.macros.items():

+                 rpmmacros.write("{0} {1}\n".format(key, value))

We don't add %-sign here, while ...

That's because we already define the macros with %-sign in macros_for_task function

  

      def produce_srpm(self):

          """

@@ -25,8 +25,11 @@ 

          spec_path = self.save_spec()

          cmd = ["mock", "-r", "/etc/copr-rpmbuild/mock-source-build.cfg",

                 "--buildsrpm", "--spec", spec_path,

-                "--define", "_disable_source_fetch 0",

                 "--resultdir", self.resultdir]

+ 

+         for key, value in self.macros.items():

+             cmd += ["--define", "{0} {1}".format(key, value)]

+ 

          return run_cmd(cmd, cwd=self.workdir)

  

      def download_srpm(self):

file modified
+9 -7
@@ -22,7 +22,7 @@ 

  from copr_rpmbuild.builders.mock import MockBuilder

  from copr_rpmbuild.automation import run_automation_tools

  from copr_rpmbuild.helpers import read_config, \

-      parse_copr_name, dump_live_log, copr_chroot_to_task_id

+      parse_copr_name, dump_live_log, copr_chroot_to_task_id, macros_for_task

  from six.moves.urllib.parse import urlparse, urljoin, urlencode

  

  log = logging.getLogger(__name__)
@@ -158,8 +158,9 @@ 

      Use *Provider() classes to create source RPM in config.get("resultdir")

      """

      try:

-         provider = providers.factory(task["source_type"])(

-             task["source_json"], config)

+         macros = macros_for_task(task, config)

+         clazz = providers.factory(task["source_type"])

+         provider = clazz(task["source_json"], config, macros=macros)

          provider.produce_srpm()

          provider.copy_insecure_results()

      finally:
@@ -246,10 +247,11 @@ 

      log_task(task)

  

      try:

-         distgit = providers.DistGitProvider(

-             {"clone_url": task["git_repo"], "committish": task["git_hash"]},

-             config,

-         )

+         source_json = {

+             "clone_url": task["git_repo"],

+             "committish": task["git_hash"],

+         }

+         distgit = providers.DistGitProvider(source_json, config)

  

          # Just clone and download sources, don't create source RPM (aka

          # produce_srpm).  We want to create the source RPM using Mock

file modified
+3 -5
@@ -7,7 +7,6 @@ 

  {% if buildroot_pkgs %}

  config_opts['chroot_additional_packages'] = '{{ buildroot_pkgs| join(" ") }}'

  {% endif %}

- config_opts['macros']['%vendor'] = '{{ vendor }}'

  {% if enable_net %}

  config_opts['rpmbuild_networking'] = True

  config_opts['use_host_resolv'] = True
@@ -19,10 +18,9 @@ 

  config_opts['isolation'] = '{{ isolation }}'

  {%- endif %}

  

- config_opts['macros']['%copr_username'] = '{{ copr_username }}'

- config_opts['macros']['%copr_projectname'] = '{{ copr_projectname }}'

- # Build-system's (or build) ID

- config_opts['macros']['%buildtag'] = '.copr{{ copr_build_id }}'

+ {%- for key, value in macros.items() %}

+ config_opts['macros']['{{ key }}'] = '{{ value }}'

+ {%- endfor %}

  

  {%- if bootstrap == "on" %}

  config_opts['use_bootstrap'] = True

file modified
+17 -2
@@ -1,5 +1,7 @@ 

  import os

+ from copr_common.enums import BuildSourceEnum

  from copr_rpmbuild.providers.base import Provider

+ from copr_rpmbuild.helpers import macros_for_task

  from . import TestCase

  

  try:
@@ -18,14 +20,27 @@ 

      @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open())

      @mock.patch('copr_rpmbuild.providers.base.os.mkdir')

      def test_create_rpmmacros(self, mock_mkdir, mock_open):

-         provider = Provider(self.source_json, self.config)

+         task = {

+             "task_id": "123",

+             "chroot": None,

+             "project_owner": "@copr",

+             "project_name": "copr-dev",

+             "source_type": BuildSourceEnum.scm,

+         }

+         macros = macros_for_task(task, self.config)

+         provider = Provider(self.source_json, self.config, macros)

          rpmmacros = mock.MagicMock()

          mock_open.return_value = rpmmacros

          provider.create_rpmmacros()

          mock_open.assert_called_with("{0}/.rpmmacros".format(provider.workdir), "w")

          calls = [

              mock.call.__enter__().write('%_disable_source_fetch 0\n'),

-             mock.call.__enter__().write('%__urlhelper_localopts --proto -all,+https,+ftps\n'),

+             mock.call.__enter__().write(

+                 '%__urlhelper_localopts --proto -all,+https,+ftps\n'),

+             mock.call.__enter__().write('%copr_username @copr\n'),

+             mock.call.__enter__().write('%copr_projectname copr-dev\n'),

+             mock.call.__enter__().write('%buildtag .copr123\n'),

+             mock.call.__enter__().write('%vendor Unknown Copr - group @copr\n'),

          ]

          rpmmacros.assert_has_calls(calls, any_order=True)

  

file modified
+7 -2
@@ -22,8 +22,13 @@ 

      resultdir = None

      workspace = None

  

-     task = {"source_type": BuildSourceEnum.upload,

-             "source_json": {"url": "http://foo.ex/somepackage.spec"}}

+     task = {

+         "chroot": None,

+         "source_type": BuildSourceEnum.upload,

+         "source_json": {"url": "http://foo.ex/somepackage.spec"},

+         "project_owner": "u1",

+         "project_name": "p1",

+     }

  

      def auto_test_setup(self):

          self.config_basic_dirs()

file modified
+1 -3
@@ -152,16 +152,14 @@ 

  

  config_opts['chroot_additional_packages'] = 'pkg1 pkg2 pkg3'

  

- config_opts['macros']['%vendor'] = 'Copr Testsuite - group @copr'

  

  config_opts['rpmbuild_networking'] = True

  config_opts['use_host_resolv'] = True

  

  config_opts['isolation'] = 'nspawn'

- 

  config_opts['macros']['%copr_username'] = '@copr'

  config_opts['macros']['%copr_projectname'] = 'copr-dev'

- # Build-system's (or build) ID

+ config_opts['macros']['%vendor'] = 'Copr Testsuite - group @copr'

  config_opts['macros']['%buildtag'] = '.copr10'

  

  """  # TODO: make the output nicer

file modified
+4 -2
@@ -42,14 +42,16 @@ 

      @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open())

      @mock.patch('copr_rpmbuild.providers.base.os.mkdir')

      def test_produce_srpm(self, mock_mkdir, mock_open, run_cmd, mock_get):

-         provider = UrlProvider(self.source_json, self.config)

+         macros = {"_disable_source_fetch": 0}

+         provider = UrlProvider(self.source_json, self.config, macros)

          provider.produce_srpm()

          args = [

              'mock', '-r', '/etc/copr-rpmbuild/mock-source-build.cfg',

              '--buildsrpm',

              '--spec', '{0}/somepackage.spec'.format(provider.workdir),

+             '--resultdir', self.config.get("main", "resultdir"),

              '--define', '_disable_source_fetch 0',

-             '--resultdir', self.config.get("main", "resultdir")]

+         ]

          run_cmd.assert_called_with(args, cwd=provider.workdir)

  

      @mock.patch('copr_common.request.SafeRequest.get')

Metadata Update from @frostyx:
- Pull-request tagged with: needs-tests, wip

3 months ago

Build succeeded.

1 new commit added

  • rpmbuild: define copr-specific macros also for SRPM builds
3 months ago

Build succeeded.

2 new commits added

  • rpmbuild: define copr-specific macros also for SRPM builds
  • rpmbuild: define copr-specific macros also for SRPM builds
3 months ago

Build succeeded.

I added one more commit. It is a separate commit in case you don't like the change, so we can easily drop it. I will squash them before merging.

There were macro definitions at too many places, so I tried to unify them. Ideally, we
would run every provider within Mock and have just one way of defining
macros but that's not a small change.

Instead, I picked all the custom macros and moved them to one
function. It is still up to the providers/builders how the macros will
be defined (using --define, generating ~/.rpmmacros, etc) but the list
of the macros and their values are always the same.

Metadata Update from @frostyx:
- Pull-request untagged with: needs-tests

3 months ago

... here it is without %-sign. Seems weird, perhaps it is worth documenting what is the expected format of the "macros" argumentL?

I like the idea, thanks!

That's because we already define the macros with %-sign in macros_for_task function

Ah, good catch, thanks. This is actually a left-over that is not used. Just a couple of lines below, there is the macros variable defined again. Let me just remove this line.

rebased onto d54d3cd

3 months ago

Build succeeded.

Metadata Update from @frostyx:
- Pull-request untagged with: wip

3 months ago

Added a couple of suggested fixes, squashed the commits together, and fixed the EPEL7 build.

Commit cef22ba fixes this pull-request

Pull-Request has been merged by praiskup

3 months ago

Commit 50c7f73 fixes this pull-request

Pull-Request has been merged by praiskup

3 months ago