#2218 Add support for pyp2spec generator
Merged 2 years ago by praiskup. Opened 2 years ago by frostyx.
copr/ frostyx/copr pyp2spec  into  main

@@ -0,0 +1,61 @@ 

+ #! /bin/bash

+ #

+ # Copyright (c) 2022 Red Hat, Inc.

+ #

+ # This program is free software: you can redistribute it and/or

+ # modify it under the terms of the GNU General Public License as

+ # published by the Free Software Foundation, either version 2 of

+ # the License, or (at your option) any later version.

+ #

+ # This program is distributed in the hope that it will be

+ # useful, but WITHOUT ANY WARRANTY; without even the implied

+ # warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

+ # PURPOSE.  See the GNU General Public License for more details.

+ #

+ # You should have received a copy of the GNU General Public License

+ # along with this program. If not, see http://www.gnu.org/licenses/.

+ 

+ 

+ # Include Beaker environment

+ . /usr/share/beakerlib/beakerlib.sh || exit 1

+ 

+ # Load config settings

+ HERE=$(dirname "$(realpath "$0")")

+ source "$HERE/config"

+ source "$HERE/helpers"

+ 

+ PYPI_PACKAGE="motionpaint"

+ PACKAGE="test_package_pypi"

+ 

+ 

+ rlJournalStart

+     rlPhaseStartSetup

+         setup_checks

+         setupProjectName "Pyp2spec"

+     rlPhaseEnd

+ 

+     rlPhaseStartTest

+         rlRun "copr-cli create --chroot $CHROOT $PROJECT"

+ 

+         # Submit pyp2spec build directly

+         rlRun "copr-cli buildpypi $PROJECT --packagename $PYPI_PACKAGE --spec-generator pyp2spec"

+ 

+         # Create pyp2spec package

+         rlRun "copr-cli add-package-pypi $PROJECT --name $PACKAGE --packagename $PYPI_PACKAGE --spec-generator pyp2spec"

+ 

+         # Build the package

+         rlRun -s "copr-cli build-package --name $PACKAGE $PROJECT -r $CHROOT --nowait"

+         rlRun "parse_build_id"

+         rlRun "copr watch-build $BUILD_ID"

+ 

+         # rlRun "copr-cli download-build $BUILD_ID"

+         rlRun "wget $BACKEND_URL/results/$PROJECT/srpm-builds/$(printf %08d "$BUILD_ID")/builder-live.log.gz"

+         rlRun "gzip -fd builder-live.log.gz"

+         rlRun "head builder-live.log -n 50 |grep spec_generator |grep pyp2spec"

+     rlPhaseEnd

+ 

+     rlPhaseStartCleanup

+         cleanProject

+     rlPhaseEnd

+ rlJournalPrintText

+ rlJournalEnd

file modified
+1 -1
@@ -6,7 +6,7 @@ 

  %global with_python2 1

  %endif

  

- %global min_python_copr_version 1.120.2.dev

+ %global min_python_copr_version 1.120.3.dev

  

  Name:       copr-cli

  Version:    1.101

file modified
+11
@@ -351,6 +351,7 @@ 

          data = {

              "pypi_package_name": args.packagename,

              "pypi_package_version": args.packageversion,

+             "spec_generator": args.spec_generator,

              "spec_template": args.spec_template,

              "python_versions": args.pythonversions,

          }
@@ -725,6 +726,7 @@ 

              "package_name": args.name,

              "pypi_package_name": args.packagename,

              "pypi_package_version": args.packageversion,

+             "spec_generator": args.spec_generator,

              "spec_template": args.spec_template,

              "python_versions": args.pythonversions,

              "max_builds": args.max_builds,
@@ -1214,6 +1216,15 @@ 

                                           help="For what Python versions to build (by default: 3 2)")

      parser_pypi_args_optional.add_argument("--packageversion", metavar = "PYPIVERSION",

                                           help="Version of the PyPI package to be built (by default latest)")

+ 

+     parser_pypi_args_optional.add_argument(

+         "--spec-generator",

+         dest="spec_generator",

+         help="Tool for generating specfile from a PyPI package",

+         choices=["pyp2rpm", "pyp2spec"],

+         default="pyp2rpm",

+     )

+ 

      parser_pypi_args_optional.add_argument("--template", "-t", dest="spec_template",

                                           help="Spec template to be used to build srpm with pyp2rpm")

  

@@ -872,6 +872,13 @@ 

              wtforms.validators.Optional(),

          ])

  

+     spec_generator = wtforms.SelectField(

+         "Spec generator",

+         choices=[

+             ("pyp2rpm", "pyp2rpm"),

+             ("pyp2spec", "pyp2spec"),

+         ], default="pyp2rpm")

+ 

      spec_template = wtforms.SelectField(

          "Spec template",

          choices=[
@@ -895,6 +902,7 @@ 

          return json.dumps({

              "pypi_package_name": self.pypi_package_name.data,

              "pypi_package_version": self.pypi_package_version.data,

+             "spec_generator": self.spec_generator.data,

              "spec_template": self.spec_template.data,

              "python_versions": self.python_versions.data

          })

@@ -502,8 +502,10 @@ 

          return cls.create_new(user, copr, source_type, source_json, chroot_names, copr_dirname=copr_dirname, **build_options)

  

      @classmethod

-     def create_new_from_pypi(cls, user, copr, pypi_package_name, pypi_package_version, spec_template,

-                              python_versions, chroot_names=None, copr_dirname=None, **build_options):

+     def create_new_from_pypi(cls, user, copr, pypi_package_name,

+                              pypi_package_version, spec_generator,

+                              spec_template, python_versions, chroot_names=None,

+                              copr_dirname=None, **build_options):

Shouldn't you move spec_generator right before the build_options argument, and provide some default value? The API compatibility hit would be much smaller..

          """

          :type user: models.User

          :type copr: models.Copr
@@ -518,6 +520,7 @@ 

          source_type = helpers.BuildSourceEnum("pypi")

          source_json = json.dumps({"pypi_package_name": pypi_package_name,

                                    "pypi_package_version": pypi_package_version,

+                                   "spec_generator": spec_generator,

Same here

                                    "spec_template": spec_template,

                                    "python_versions": python_versions})

          return cls.create_new(user, copr, source_type, source_json, chroot_names, copr_dirname=copr_dirname, **build_options)

@@ -72,6 +72,12 @@ 

        <label for="python_versions-0">3</label>&nbsp;&nbsp;&nbsp;

        <input id="python_versions-1" name="python_versions" value="2" type="checkbox" {% if '2' in field.data %}checked="checked"{% endif %}>

        <label for="python_versions-1">2</label>

+ 

+       <p class="help-block">

+         <small class="text-muted pficon pficon-info"></small>

+         Limited to <strong>pyp2rpm</strong> spec generator

+       </p>

+ 

      </div>

      {% if field.errors %}

        {% for error in field.errors %}

@@ -152,12 +152,45 @@ 

  {% macro copr_build_form_pypi(form, view, copr) %}

    {{ copr_build_form_begin(form, view, copr) }}

  

-     {{ source_description('This method uses <a href="https://github.com/fedora-python/pyp2rpm">pyp2rpm</a> to create the RPM for you automatically from PyPI - the Python Package Index. Please provide the package name.')}}

+     {{ source_description(

+          'This method uses '

+          '<a href="https://github.com/fedora-python/pyp2rpm">pyp2rpm</a> '

+          'or <a href="https://github.com/befeleme/pyp2spec">pyp2spec</a> '

+          'to create the RPM for you automatically from PyPI - '

+          'the Python Package Index. Please provide the package name.'

+        )

+     }}

  

      {{ render_field(form.pypi_package_name, placeholder="Package name in the Python Package Index.") }}

      {{ render_field(form.pypi_package_version, placeholder="Optional - Version of the package PyPI") }}

-     {{ render_field(form.spec_template, placeholder="Distribution specific spec template") }}

-     {{ render_pypi_python_versions_field(form.python_versions) }}

+ 

+     {{ render_field(

+          form.spec_generator,

+          info="Tool for generating specfile from a PyPI package. The options "

+                "are full-featured <strong>pyp2rpm</strong> with cross "

+                "distribution support, and <strong>pyp2spec</strong> that is "

+                "being actively developed and considered to be the future."

+        )

+     }}

+ 

+     {# End the previous instructions box #}

+       </div>

+     </div>

+ 

+     <div class="panel panel-default">

+       <div class="panel-heading">

+         <h3 class="panel-title">

+           {{ counter('instructions') }}. Options specific for pyp2rpm

+         </h3>

+       </div>

+       <div class="panel-body">

+         {{ render_field(

+                form.spec_template,

+                placeholder="Distribution specific spec template",

+                info="Limited to <strong>pyp2rpm</strong> spec generator")

+         }}

+ 

+         {{ render_pypi_python_versions_field(form.python_versions) }}

  

    {{ copr_build_form_end(form, view, copr) }}

  {% endmacro %}

@@ -52,13 +52,21 @@ 

    <dd>{{ source_json_dict["pypi_package_version"] }}</dd>

    {% endif %}

  

-   {% if source_json_dict["spec_template"] %}

-   <dt>Spec template</dt>

-   <dd>{{ source_json_dict["spec_template"] }}</dd>

+   {% if source_json_dict["spec_generator"] %}

+   <dt>Spec generator</dt>

+   <dd>{{ source_json_dict["spec_generator"] }}</dd>

    {% endif %}

  

-   <dt>for Python</dt>

-   <dd>{{ ', '.join(source_json_dict["python_versions"]).strip(', ') }}</dd>

+   {# These options are not available for pyp2spec #}

+   {% if source_json_dict["spec_generator"] == "pyp2rpm" %}

+     {% if source_json_dict["spec_template"] %}

+     <dt>Spec template</dt>

+     <dd>{{ source_json_dict["spec_template"] }}</dd>

+     {% endif %}

+ 

+     <dt>for Python</dt>

+     <dd>{{ ', '.join(source_json_dict["python_versions"]).strip(', ') }}</dd>

+     {% endif %}

    {% endif %}

  

    {% if source_type_text == "rubygems" %}

@@ -89,8 +89,34 @@ 

  {% macro copr_package_form_pypi(form, view, copr, package) %}

      {{ copr_package_form_begin(form, view, copr, package) }}

      {{ render_field(form.pypi_package_name, placeholder="Package name in the Python Package Index.") }}

-     {{ render_field(form.spec_template, placeholder="Spec template to be used for generating srpm.") }}

-     {{ render_pypi_python_versions_field(form.python_versions) }}

+ 

+     {{ render_field(

+          form.spec_generator,

+          info="Tool for generating specfile from a PyPI package. The options "

+                "are full-featured <strong>pyp2rpm</strong> with cross "

+                "distribution support, and <strong>pyp2spec</strong> that is "

+                "being actively developed and considered to be the future."

+        )

+     }}

+ 

+     {# End the previous instructions box #}

+       </div>

+     </div>

+ 

+     <div class="panel panel-default">

+       <div class="panel-heading">

+         <h3 class="panel-title">

+           {{ counter('instructions') }}. Options specific for pyp2rpm

+         </h3>

+       </div>

+       <div class="panel-body">

+         {{ render_field(

+                form.spec_template,

+                placeholder="Distribution specific spec template",

+                info="Limited to <strong>pyp2rpm</strong> spec generator")

+         }}

+         {{ render_pypi_python_versions_field(form.python_versions) }}

+ 

      {{ render_generic_pkg_form(form) }}

      {{ render_anitya_autorebuild(form) }}

      {{ copr_package_form_end(form, package, 'rubygems') }}

@@ -256,6 +256,7 @@ 

              copr,

              form.pypi_package_name.data,

              form.pypi_package_version.data,

+             form.spec_generator.data,

              form.spec_template.data,

              form.python_versions.data,

              **options,

@@ -291,6 +291,7 @@ 

              copr,

              form.pypi_package_name.data,

              form.pypi_package_version.data,

+             form.spec_generator.data,

              form.spec_template.data,

              form.python_versions.data,

              form.selected_chroots,

@@ -119,9 +119,16 @@ 

          self.spec_template = source_json['spec_template']

  

      def build(self, copr, new_updated_version):

-         return BuildsLogic.create_new_from_pypi(copr.user, copr, self.name, new_updated_version,

-                                                 self.spec_template, self.python_versions,

-                                                 chroot_names=None)

+         return BuildsLogic.create_new_from_pypi(

+             copr.user,

+             copr,

+             self.name,

+             new_updated_version,

+             "pyp2rpm",

+             self.spec_template,

+             self.python_versions,

+             chroot_names=None

+         )

  

  

  def package_from_source(backend, source_json):

@@ -212,8 +212,10 @@ 

          return self._create(endpoint, data, buildopts=buildopts)

  

  

-     def create_from_pypi(self, ownername, projectname, pypi_package_name, pypi_package_version=None,

-                          spec_template='', python_versions=None, buildopts=None, project_dirname=None):

+     def create_from_pypi(self, ownername, projectname, pypi_package_name,

+                          pypi_package_version=None, spec_template='',

+                          python_versions=None, buildopts=None,

+                          project_dirname=None, spec_generator=None):

          """

          Create a build from PyPI - https://pypi.org/

  
@@ -225,6 +227,7 @@ 

          :param list python_versions: list of python versions to build for

          :param buildopts: http://python-copr.readthedocs.io/en/latest/client_v3/build_options.html

          :param str project_dirname:

+         :param str spec_generator: what tool should be used for spec generation

          :return: Munch

          """

          endpoint = "/build/create/pypi"
@@ -233,6 +236,7 @@ 

              "projectname": projectname,

              "pypi_package_name": pypi_package_name,

              "pypi_package_version": pypi_package_version,

+             "spec_generator": spec_generator,

              "spec_template": spec_template,

              "python_versions": python_versions or [3, 2],

              "project_dirname": project_dirname,

file modified
+1 -1
@@ -9,7 +9,7 @@ 

  %endif

  

  Name:       python-copr

- Version:    1.120.2.dev

+ Version:    1.120.3.dev

  Release:    1%{?dist}

  Summary:    Python interface for Copr

  

@@ -80,6 +80,7 @@ 

  Suggests: tito

  Suggests: rubygem-gem2rpm

  Suggests: pyp2rpm

+ Suggests: pyp2spec

  %endif

  

  %description
@@ -116,6 +117,7 @@ 

  Requires: openssh-clients

  Requires: podman

  Requires: pyp2rpm

+ Requires: pyp2spec

  # We need %%pypi_source defined, which is in 3-29+

  Requires: python-srpm-macros >= 3-29

  Requires: rpkg

@@ -8,6 +8,7 @@ 

  

  from copr_common.request import SafeRequest

  from copr_rpmbuild.helpers import CONF_DIRS

+ from copr_rpmbuild.helpers import run_cmd

  

  

  log = logging.getLogger("__main__")
@@ -136,3 +137,17 @@ 

          RESULTDIR.  Each method needs to override this one.

          """

          raise NotImplementedError

+ 

+     def build_srpm_from_spec(self, spec_path):

+         """

+         Generate a SRPM package for a locally stored spec file

+         """

+         mock_config_file = self.generate_mock_config()

+         cmd = ["mock", "-r", mock_config_file,

+                "--buildsrpm", "--spec", spec_path,

+                "--resultdir", self.resultdir]

+ 

+         for key, value in self.macros.items():

+             cmd += ["--define", "{0} {1}".format(key, value)]

+ 

+         return run_cmd(cmd, cwd=self.workdir)

@@ -1,3 +1,4 @@ 

+ import os

  import logging

  from ..helpers import run_cmd

  from .base import Provider
@@ -10,19 +11,30 @@ 

          source_json = self.source_dict

          self.pypi_package_version = source_json["pypi_package_version"]

          self.pypi_package_name = source_json["pypi_package_name"]

+         self.spec_generator = source_json.get("spec_generator", "pyp2rpm")

          self.spec_template = source_json.get("spec_template", '')

          self.python_versions = source_json["python_versions"] or []

  

      def tool_presence_check(self):

+         if self.spec_generator not in ["pyp2rpm", "pyp2spec"]:

+             msg = "Unsupported tool: {0}".format(self.spec_generator)

+             raise RuntimeError(msg)

+ 

          try:

-             run_cmd(["which", "pyp2rpm"])

+             run_cmd(["which", self.spec_generator])

          except RuntimeError as err:

-             log.error("Please, install pyp2rpm.")

+             log.error("Please, install `%s'.", self.spec_generator)

              raise err

  

      def produce_srpm(self):

          self.tool_presence_check()

  

+         if self.spec_generator == "pyp2rpm":

+             self._produce_srpm_pyp2rpm()

+         else:

+             self._produce_srpm_pyp2spec()

+ 

+     def _produce_srpm_pyp2rpm(self):

          cmd = ["pyp2rpm", self.pypi_package_name, "-t", self.spec_template,

                 "--srpm", "-d", self.resultdir]

  
@@ -36,3 +48,18 @@ 

              cmd += ["-v", self.pypi_package_version]

  

          return run_cmd(cmd)

+ 

+     def _produce_srpm_pyp2spec(self):

+         os.chdir(self.resultdir)

+ 

+         try:

+             cmd = ["pyp2spec", self.pypi_package_name]

+             run_cmd(cmd)

+         except RuntimeError as err:

+             log.error("Unable to generate spec for `%s'",

+                       self.pypi_package_name)

+             raise err

+ 

+         spec = "python-{0}.spec".format(self.pypi_package_name)

+         spec = os.path.join(self.resultdir, spec)

+         self.build_srpm_from_spec(spec)

@@ -3,7 +3,6 @@ 

  

  from six.moves.urllib.parse import urlparse

  

- from copr_rpmbuild.helpers import run_cmd

  from copr_rpmbuild.providers.base import Provider

  

  log = logging.getLogger("__main__")
@@ -21,18 +20,6 @@ 

              spec.write(response.text)

          return path

  

-     def build_srpm_from_spec(self):

-         mock_config_file = self.generate_mock_config()

-         spec_path = self.save_spec()

-         cmd = ["mock", "-r", mock_config_file,

-                "--buildsrpm", "--spec", spec_path,

-                "--resultdir", self.resultdir]

- 

-         for key, value in self.macros.items():

-             cmd += ["--define", "{0} {1}".format(key, value)]

- 

-         return run_cmd(cmd, cwd=self.workdir)

- 

      def download_srpm(self):

          basename = os.path.basename(self.parsed_url.path)

          filename = os.path.join(self.resultdir, basename)
@@ -48,7 +35,8 @@ 

  

      def produce_srpm(self):

          if self.parsed_url.path.endswith(".spec"):

-             return self.build_srpm_from_spec()

+             spec_path = self.save_spec()

+             return self.build_srpm_from_spec(spec_path)

          if self.parsed_url.path.endswith(".src.rpm"):

              return self.download_srpm()

          raise RuntimeError("Url is not a path to .spec nor .src.rpm file")

file modified
+1 -1
@@ -38,7 +38,7 @@ 

          self.assertEqual(provider.url, "http://foo.ex/somepackage.spec")

  

      @mock.patch('copr_common.request.SafeRequest.get')

-     @mock.patch("copr_rpmbuild.providers.spec.run_cmd")

+     @mock.patch("copr_rpmbuild.providers.base.run_cmd")

      @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open())

      @mock.patch("copr_rpmbuild.providers.spec.UrlProvider.create_rpmmacros")

      @mock.patch("copr_rpmbuild.providers.spec.UrlProvider.generate_mock_config")

Build succeeded.

Metadata Update from @praiskup:
- Pull-request tagged with: wip

2 years ago

I like it. The common and specific options are clearly delimited. +1

1 new commit added

  • wip
2 years ago

Build succeeded.

rebased onto 5bf73fc410dbdc1276b47e92e428c65c8fa4709d

2 years ago

Metadata Update from @frostyx:
- Pull-request untagged with: wip

2 years ago

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci

rebased onto f7d9e7168ad2e2f0c58dd377123440dc7e8dff15

2 years ago

rebased onto fffdb50036db8db3ddf51f456926e5340ff0ca3d

2 years ago

rebased onto 7f450e78ef1d0a55f159277b04be790f0f64c4f1

2 years ago

rebased onto 7f450e78ef1d0a55f159277b04be790f0f64c4f1

2 years ago

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci

rebased onto 5d85a8b939fee0cf188a6f58b72f2870b21dc892

2 years ago

Build succeeded.

rpmbuild seems to fail, is this material ready for review?

rebased onto 896ea01089d5cddc6a1c52a27fad0a2ad0125124

2 years ago

Build succeeded.

rpmbuild seems to fail, is this material ready for review?

Ah, my bad. Tests fixed, and ready for review.

Shouldn't you move spec_generator right before the build_options argument, and provide some default value? The API compatibility hit would be much smaller..

Probably just OCD problem, but the intent of the "The rest of the options" paragraph feels weird, too much on the left side. I'd probably put comments for each of the fields separately, or wrap the pyp2rpm part into it's own section (like "3. pyp2rpm-only options, ignored by pyp2spec").
pyp2spec.png

Otherwise LGTM, very nice!

rebased onto dfbcdd4eafcaf2ab2135e85c8c5f09f6e3ddca02

2 years ago

Build succeeded.

rebased onto 8fec0c17dd7a02aa93734d7f405b23ee2abade42

2 years ago

1 new commit added

  • frontend: define the response variable outside the try block
2 years ago

rebased onto c0cb1db8510be410eea5ff967861637ebde0acaa

2 years ago

Shouldn't you move spec_generator right before the build_options argument, and provide some default value? The API compatibility hit would be much smaller..
Same here

I think it doesn't really matter here because BuildsLogic doesn't face users.
The only place where this IMHO makes sense is in the python-copr function, so I fixed it there. Good catch, @praiskup

Probably just OCD problem, but the intent of the "The rest of the options" paragraph feels weird, too much on the left side. I'd probably put comments for each of the fields separately, or wrap the pyp2rpm part into it's own section (like "3. pyp2rpm-only options, ignored by pyp2spec").

Updated. What do you think of this?

Screenshot_2022-07-25_23-39-44.png

I wasn't sure what the title should be. Maybe "Options limited to pyp2rpm" would be better?

I added one commit, that is unrelated to this feature but fixes the copr-cli tests, that were failing. The copr-cli build still fails because it pulls the wrong version of python3-copr from repos.

I wasn't sure what the title should be. Maybe "Options limited to pyp2rpm" would be better?

Dunno :D The left alignment of this form is IMVHO overall wrong and makes an ugly feeling.
So don't worry, we can make it prettier at some other occasion (some style re-format or something).

I think it doesn't really matter here because BuildsLogic doesn't face users.

There's still the PyPIPackage.build() call which wasn't updated by this PR, and collides with the actual create_from_pypi method notation.

rebased onto d454dd6aa22c0a5ec7a08dc27e66c86765af9b7c

2 years ago

Merge Failed.

This change or one of its cross-repo dependencies was unable to be automatically merged with the current state of its repository. Please rebase the change and upload a new patchset.
Warning:
Error merging pagure.io/copr/copr for 2218,d48886117a0e3139b35002404173ca75b178b0da

Merge Failed.

This change or one of its cross-repo dependencies was unable to be automatically merged with the current state of its repository. Please rebase the change and upload a new patchset.
Warning:
Error merging pagure.io/copr/copr for 2218,d48886117a0e3139b35002404173ca75b178b0da

rebased onto f4e078c6ce57bd93cd0eb60d02523382b7c7e0c4

2 years ago

Merge Failed.

This change or one of its cross-repo dependencies was unable to be automatically merged with the current state of its repository. Please rebase the change and upload a new patchset.
Warning:
Error merging pagure.io/copr/copr for 2218,0a98923a006db012157cd670727e830bd9c3c817

rebased onto dfaf3344d3c912627e5045655341a5be180f50a6

2 years ago

Build succeeded.

rebased onto 5ad4e261211160efa90484e1f0a27c359fd7c52a

2 years ago

Build succeeded.

rebased onto 568f31e

2 years ago

Pull-Request has been merged by praiskup

2 years ago

Build succeeded.