#185 Custom build source method
Merged 9 months ago by clime. Opened 11 months ago by praiskup.
copr/ praiskup/copr custom-source-method  into  master

@@ -0,0 +1,49 @@ 

+ #! /bin/sh -x

+ 

+ set -e

+ 

+ generate_specfile()

+ {

+     test -n "$DESTDIR" && mkdir -p "$DESTDIR"

+ 

+     test -n "$BUILDDEPS" && {

+         for i in $BUILDDEPS; do

+             rpm -q $i

+         done

+     }

+ 

+     if ${HOOK_PAYLOAD-false}; then

+         test -f hook_payload

+         test "$(cat hook_payload)" = "{\"a\": \"b\"}"

+     else

+         ! test -f hook_payload

+     fi

+ 

+ cat > "${DESTDIR-.}"/quick-package.spec <<\EOF

+ Name:           quick-package

+ Version:        0

+ Release:        0%{?dist}

+ Summary:        dummy package

+ License:        GPL

+ URL:            http://example.com/

+ 

+ %{!?_pkgdocdir: %global _pkgdocdir %{_docdir}/%{name}-%{version}}

+ 

+ %description

+ nothing

+ 

+ 

+ %install

+ mkdir -p $RPM_BUILD_ROOT/%{_pkgdocdir}

+ echo "this does nothing" > $RPM_BUILD_ROOT/%{_pkgdocdir}/README

+ 

+ 

+ %files

+ %doc %{_pkgdocdir}/README

+ 

+ 

+ %changelog

+ * Thu Jun 05 2014 Pavel Raiskup <praiskup@redhat.com> - 0-1

+ - does nothing!

+ EOF

+ }

@@ -0,0 +1,216 @@ 

+ #!/bin/bash

+ 

+ . /usr/bin/rhts-environment.sh || exit 1

+ . /usr/share/beakerlib/beakerlib.sh || exit 1

+ 

+ export RESULTDIR=`mktemp -d`

+ 

+ export TESTPATH="$( builtin cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

+ 

+ export IN=$TESTPATH/action-tasks.json

+ export OUT=$TESTPATH/action-results.out.json

+ 

+ if [[ ! $FRONTEND_URL ]]; then

+     FRONTEND_URL="http://copr-fe-dev.cloud.fedoraproject.org"

+ fi

+ 

+ 

+ NAME_VAR="TEST$(date +%s)"

+ 

+ 

+ parse_build_id()

+ {

+    local id

+    id=$(grep 'Created builds:' "$rlRun_LOG" | sed 's/.* //')

+    test -n "$id" || return 1

+    export BUILD_ID=$id

+ }

+ 

+ cleanup_resultdir ()

+ (

+     rm -rf "$RESULTDIR/*"

+ )

+ 

+ check_resultdir ()

+ (

+     set -e

+     cd "$RESULTDIR/fedora-rawhide-x86_64"

+     # @FIXME

+     #test -f "$RESULTDIR"/script

+     for i in $FILES; do

+         echo "checking that $i exists in resultdir"

+         test -f "$i"

+     done

+ 

+     NV=$1

+ 

+     echo "checking that only one srpm exists"

+     set -- *.src.rpm

+     test 1 -eq "$#"

+ 

+     echo "checking that srpm version is fine"

+     case $1 in

+         $NV*.src.rpm) ;; # OK

+         *) false ;;

+     esac

+ )

+ 

+ check_http_status ()

+ {

+    grep "HTTP/1.1 $1" "$rlRun_LOG"

+ }

+ 

+ quick_package_script ()

+ {

+     cp "$TESTPATH/files/quick-package.sh" script

+     echo "$1" >> script

+ }

+ 

+ 

+ rlJournalStart

+     rlPhaseStartTest Test

+     rlRun "export WORKDIR=\`mktemp -d\`"

+     rlRun "test -n \"\$WORKDIR\""

+ 

+     rlRun 'cd "$WORKDIR"'

+     rlRun 'echo workdir: $WORKDIR'

+ 

+ 

+     rlLogInfo "Create the project $PROJECT"

+     PROJECT=custom-1-$NAME_VAR

+     rlRun 'copr-cli create "$PROJECT" --chroot fedora-rawhide-x86_64'

+ 

+ 

+     rlLogInfo "Test add-package && build"

+     rlRun 'cleanup_resultdir'

+     rlRun 'quick_package_script generate_specfile'

+     rlRun 'copr add-package-custom "$PROJECT" \

+         --name quick-package \

+         --script script \

+         --script-chroot fedora-rawhide-x86_64'

+     rlRun -s 'copr build-package "$PROJECT" --name quick-package --nowait'

+     rlRun 'parse_build_id'

+     rlRun 'copr watch-build $BUILD_ID'

+     rlRun 'copr download-build $BUILD_ID --dest $RESULTDIR'

+     rlRun 'FILES="success" check_resultdir quick-package-0-0'

+ 

+ 

+     rlLogInfo "Test edit-package && --resultdir"

+     rlRun 'cleanup_resultdir'

+     rlRun 'quick_package_script "DESTDIR=rrr generate_specfile"'

+     rlRun 'copr edit-package-custom "$PROJECT" \

+         --name quick-package \

+         --script script \

+         --script-chroot fedora-rawhide-x86_64'

+     rlRun -s 'copr build-package "$PROJECT" --name quick-package --nowait'

+     rlRun 'parse_build_id'

+     # Should fail, there's no spec file in expected resultdir=.

+     rlRun 'copr watch-build $BUILD_ID' 4

+ 

+     rlRun 'copr edit-package-custom "$PROJECT" \

+         --name quick-package \

+         --script script \

+         --script-resultdir rrr \

+         --script-chroot fedora-rawhide-x86_64'

+     rlRun -s 'copr build-package "$PROJECT" --name quick-package --nowait'

+     rlRun 'parse_build_id'

+     rlRun 'copr watch-build $BUILD_ID'

+     rlRun 'copr download-build $BUILD_ID --dest $RESULTDIR'

+     rlRun 'FILES="success" check_resultdir quick-package-0-0'

+ 

+ 

+     rlLogInfo "Test that builddeps get propagated"

+     builddeps="automake autoconf spax"

+     rlRun 'cleanup_resultdir'

+     rlRun 'quick_package_script "BUILDDEPS=xxx generate_specfile"'

+     rlRun 'copr edit-package-custom "$PROJECT" \

+         --name quick-package \

+         --script script \

+         --script-builddeps "$builddeps" \

+         --script-chroot fedora-rawhide-x86_64'

+     rlRun -s 'copr build-package "$PROJECT" --name quick-package --nowait'

+     rlRun 'parse_build_id'

+     # Invalid BUILDDEPS value, should fail

+     rlRun 'copr watch-build $BUILD_ID' 4

+ 

+     rlRun 'quick_package_script "BUILDDEPS=\"$builddeps\" generate_specfile"'

+     rlRun 'copr edit-package-custom "$PROJECT" \

+         --name quick-package \

+         --script script \

+         --script-builddeps "$builddeps" \

+         --script-chroot fedora-rawhide-x86_64'

+     rlRun -s 'copr build-package "$PROJECT" --name quick-package --nowait'

+     rlRun 'parse_build_id'

+     # Invalid BUILDDEPS value, should fail

+     rlRun 'copr watch-build $BUILD_ID'

+     rlRun 'copr download-build $BUILD_ID --dest $RESULTDIR'

+     rlRun 'FILES="success" check_resultdir quick-package-0-0'

+ 

+ 

+     rlLogInfo "check that hook_payload get's created"

+     rlRun 'cleanup_resultdir'

+     rlRun 'quick_package_script "HOOK_PAYLOAD=: generate_specfile"'

+     rlRun 'copr edit-package-custom "$PROJECT" \

+         --name quick-package \

+         --script script \

+         --script-chroot fedora-rawhide-x86_64 \

+         --webhook-rebuild on'

+     rlRun -s 'copr build-package "$PROJECT" --name quick-package --nowait'

+     rlRun 'parse_build_id'

+     rlLogInfo "Still should fail, since this build is not triggered by webhook."

+     rlRun 'copr watch-build $BUILD_ID' 4

+ 

+     trigger_url="$FRONTEND_URL/webhooks/custom/$copr_id/webhook_secret/quick-package/"

+     rlRun -s 'curl -I "$trigger_url"' 0 # GET can't work

+     rlRun 'check_http_status 405'

+ 

+     content_type_option=' -H "Content-Type: application/json"'

+     data_option=' --data '\''{"a": "b"}'\'

+ 

+     rlLogInfo "full cmd would be: curl -X POST $content_type_option $data_option $trigger_url"

+     rlRun "build_id=\$(curl -X POST $data_option \"$trigger_url\")" 0

+     rlLogInfo "Still fails since the POST data are not json"

+     rlRun 'copr watch-build $BUILD_ID' 4

+ 

+     rlLogInfo "Still fails since the POST data are not json"

+     rlRun "build_id=\$(curl -X POST $content_type_option $data_option \"$trigger_url\")" 0

+     rlLogInfo "Should succeed finally"

+     # @FIXME

+     # rlRun 'copr watch-build $build_id'

+     # rlRun 'copr download-build $build_id --dest $RESULTDIR'

+     # rlRun 'FILES="success" check_resultdir quick-package-0-0'

+ 

+ 

+     rlLogInfo "basic buildcustom command, with fedora-latest-x86_64 chroot (default)"

+     rlRun 'cleanup_resultdir'

+     rlRun 'quick_package_script "generate_specfile"'

+     rlRun -s "copr buildcustom $PROJECT --script script --nowait"

+     rlRun 'parse_build_id'

+     rlRun 'copr watch-build $BUILD_ID'

+     rlRun 'copr download-build $BUILD_ID --dest $RESULTDIR'

+     rlRun 'FILES="success" check_resultdir quick-package-0-0'

+ 

+ 

+     rlLogInfo "buildcustom with --builddeps"

+     builddeps='postgresql-devel'

+     rlRun 'cleanup_resultdir'

+     rlRun "quick_package_script 'BUILDDEPS=\"$builddeps\" generate_specfile'"

+     rlRun -s "copr buildcustom $PROJECT --script script --script-builddeps \"$builddeps\" --nowait"

+     rlRun 'parse_build_id'

+     rlRun 'copr watch-build $BUILD_ID'

+     rlRun 'copr download-build $BUILD_ID --dest $RESULTDIR'

+     rlRun 'FILES="success" check_resultdir quick-package-0-0'

+ 

+ 

+     rlLogInfo "buildcustom with --builddeps and --resultdir"

+     destdir=abc

+     rlRun 'cleanup_resultdir'

+     rlRun "quick_package_script 'BUILDDEPS=\"$builddeps\" DESTDIR=$destdir generate_specfile'"

+     rlRun -s "copr buildcustom $PROJECT --script script --script-resultdir=$destdir --script-builddeps \"$builddeps\" --nowait"

+     rlRun 'parse_build_id'

+     rlRun 'copr watch-build $BUILD_ID'

+     rlRun 'copr download-build $BUILD_ID --dest $RESULTDIR'

+     rlRun 'FILES="success" check_resultdir quick-package-0-0'

+ 

+     rlPhaseEnd

+ rlJournalEnd

file modified
+71 -0

@@ -270,6 +270,21 @@ 

          return self.process_build(args, self.client.create_new_build_rubygems, data)

  

      @requires_api_auth

+     def action_build_custom(self, args):

+         """

+         Method called when 'buildcustom' has been selected by the user.

+ 

+         :param args: argparse arguments provided by the user

+         """

+         data = {

+             'script': ''.join(args.script.readlines()),

+         }

+         for arg in ['script_chroot', 'script_builddeps',

+                 'script_resultdir']:

+             data[arg] = getattr(args, arg)

+         return self.process_build(args, self.client.create_new_build_custom, data)

+ 

+     @requires_api_auth

      def action_build_distgit(self, args):

          """

          Method called when the 'buildfedpkg' action has been selected by the user.

@@ -556,6 +571,23 @@ 

              result = self.client.edit_package_rubygems(ownername=ownername, projectname=projectname, **data)

          print(result.message)

  

+     @requires_api_auth

+     def action_add_or_edit_package_custom(self, args):

+         ownername, projectname = parse_name(args.copr)

+         data = {

+             "package_name": args.name,

+             "script": ''.join(args.script.readlines()),

+             "script_chroot": args.script_chroot,

+             "script_builddeps": args.script_builddeps,

+             "script_resultdir": args.script_resultdir,

+             "webhook_rebuild": ON_OFF_MAP[args.webhook_rebuild],

+         }

+         if args.create:

+             result = self.client.add_package_custom(ownername=ownername, projectname=projectname, **data)

+         else:

+             result = self.client.edit_package_custom(ownername=ownername, projectname=projectname, **data)

+         print(result.message)

+ 

      def action_list_packages(self, args):

          ownername, projectname = parse_name(args.copr)

          data = {

@@ -810,6 +842,22 @@ 

      parser_distgit_args_parent.add_argument("--branch", metavar="BRANCH", dest="branch",

                                               help="Specify branch to be used")

  

+     parser_custom_args_parent = argparse.ArgumentParser(add_help=False)

+     parser_custom_args_parent.add_argument(

+             '--script', required=True,

+             type=argparse.FileType('r'),

+             help='text file (script) to be used to prepare the sources')

+     parser_custom_args_parent.add_argument(

+             '--script-chroot',

+             help='mock chroot to build sources for the SRPM in')

+     parser_custom_args_parent.add_argument(

+             '--script-builddeps',

+             help='space separated list of packages needed to build the sources')

+     parser_custom_args_parent.add_argument(

+             '--script-resultdir',

+             help='where SCRIPT generates the result, relatively to script\'s '

+                  '$PWD (defaults to \'.\')')

+ 

      #########################################################

      ###                    Build options                  ###

      #########################################################

@@ -846,6 +894,13 @@ 

                                                    help="Build gem from rubygems.org to a specified copr")

      parser_build_rubygems.set_defaults(func="action_build_rubygems")

  

+     # create the parser for the "buildcustom" command

+     parser_build_custom = subparsers.add_parser(

+             "buildcustom",

+             parents=[parser_custom_args_parent, parser_build_parent],

+             help="Build packages from SRPM generated by custom script")

+     parser_build_custom.set_defaults(func="action_build_custom")

+ 

      # create the parser for the "buildfedpkg" command

      parser_build_distgit = subparsers.add_parser("buildfedpkg", parents=[parser_distgit_args_parent, parser_build_parent],

                                                    help="DEPRECATED. Use SCM source type instead.")

@@ -991,6 +1046,22 @@ 

                                                           parents=[parser_rubygems_args_parent, parser_add_or_edit_package_parent])

      parser_edit_package_rubygems.set_defaults(func="action_add_or_edit_package_rubygems", create=False)

  

+     # Custom build method - edit/create package

+     parser_add_package_custom = subparsers.add_parser(

+             "add-package-custom",

+             help="Creates a new package where sources are built by custom script",

+             parents=[parser_custom_args_parent, parser_add_or_edit_package_parent])

+     parser_add_package_custom.set_defaults(

+             func="action_add_or_edit_package_custom",

+             create=True)

+     parser_edit_package_custom = subparsers.add_parser(

+             "edit-package-custom",

+             help="Edits an existing Custom package",

+             parents=[parser_custom_args_parent, parser_add_or_edit_package_parent])

+     parser_edit_package_custom.set_defaults(

+             func="action_add_or_edit_package_custom",

+             create=False)

+ 

      # package listing

      parser_list_packages = subparsers.add_parser("list-packages",

                                                   help="Returns list of packages in the given copr")

@@ -0,0 +1,130 @@ 

+ .. _custom_source_method:

+ 

+ Coustom source method

+ =====================

+ 

+ Build sources (for SRPM) by user-defined script.

+ 

+ The idea behind the script is simple, when the script is run - it's only

+ mandatory output is specfile, plus optionally any other file needed to

+ successfully build a source RPM from that spec file (usually tarball(s),

+ patches, etc.).  By default we expect that the script generates the files in

+ current working directory (resultdir='.').

+ 

+ Having turing-complete powers and Internet access - the script can do basically

+ anything to get all the source pieces together.  The only limitation is that it

+ is executed under non-privileged user (the script is executed in mock chroot,

+ under 'mockbuild' user).  This brings one major obstacle that you can not

+ install any RPM packages from within the script; if you *need* to have some

+ packages pre-installed, you need to specify them "declaratively" as a list of

+ (srpm)build-dependencies.  The reasons for this design are that (a) it is easier

+ and and safer to develop scripts which don't require admin access, (b) it is

+ convenient to write "portable" scripts (even though the script is executed in

+ rpm-based mock chroot, the script itself can be easily written/tested on e.g.

+ Gentoo) and (c) it follows the usual workflows of maintainers (install packages

+ under root, and work the rest of the day as non-root) and mock workflow.

+ 

+ 

+ Required configuration for custom method

+ ----------------------------------------

+ 

+ Basically you only have to specify **script** and **chroot** parameter.

+ 

+ - **script** - scipt file content;  written in any scripting language, but pay

+   attention to specify shebang properly, e.g. `#! /bin/sh` for (posix) shell

+   scripts.  Also note that the interpreter might not be available by default,

+   you might have to request its installation via **builddeps** argument).

+ 

+ - **chroot** - (mock) chroot where the script is executed.  By default, the

+   `fedora-latest-x86_64` chroot is used, which represents the latest stable

+   or `branched <https://fedoraproject.org/wiki/Releases/Branched>`_ Fedora

+   version available in Copr at the time of the build request (e.g.

+   `fedora-27-x86_64` when `fedora-rawhide-x86_64` represents Fedora 28).

+ 

+ 

+ Optional parameters

+ -------------------

+ 

+ - **builddeps** - space-separated list of packages which are pre-installed into

+   the build **chroot** (before the script is executed).

+ 

+ - **resultdir** - where the **script** generates its output. By default, it is

+   assumed to be current working directory.

+ 

+ 

+ Webhook support

+ ---------------

+ 

+ The only useful webhook for custom source method is the custom web-hook.

+ Because unlike other methods, custom method implementation doesn't itself pay

+ attention to webhook payload (json data used e.g. by GitLab to indicate what

+ type of event triggered the webhok call) nor there's any particular "clone url".

+ 

+ With custom webhook, the payload parsing/analysis is left to the **script** (in

+ other words it is user's responsibility).  For that purpose custom webhook

+ handler dumps the webhook payload (if any) into file `$PWD/hook_payload` file

+ (from the **script** POV).

+ 

+ For example, if GitLab's *merge-request* event from *contributor/project.git* to

+ *owner/repo.git* "calls" the custom webhook for package *foo*, the **script**

+ has to parse the `hook_payload` file to detect *the fact* that

+ *contributor/project.git* should be cloned (instead of *owner/repo.git*) to

+ generate the sources.

+ 

+ Since this all is in user's hands, it is not technically incorrect to have empty

+ hook payload, e.g. it is valid to call `curl -X POST <THE_CUSTOM_HOOK_URL>` to

+ trigger the custom source build method.

+ 

+ 

+ Examples

+ --------

+ 

+ - Trivial example (only spec file)::

+ 

+     $ cat script

+     #! /bin/sh -x

+     curl https://praiskup.fedorapeople.org/quick-package.spec -O

+ 

+     $ copr add-package-custom PROJECT \

+             --name quick-package \

+             --script script

+ 

+     $ copr build-package --name quick-package PROJECT # trigger the build

+ 

+ 

+ - Simple example with Python package with git submodules and in-tree sources::

+ 

+     $ cat script

+     #! /bin/sh

+ 

+     mkdir -p results

+     resultdir=$(readlink -f results)

+ 

+     set -x # verbose output

+     set -e # fail the whole script if some command fails

+ 

+     # obtain the source code

+     git clone https://github.com/praiskup/resalloc --recursive --depth 1

+     cd resalloc

+ 

+     # 1. generate source tarball into resultdir

+     python setup.py sdist -d "$resultdir"

+ 

+     # 2. copy the spec file into resultdir, change the release number so each build

+     #    has unique name-version-release triplet

+     cd rpm

+     release='~'$(date +"%Y%m%d_%H%M%S")

+     sed "s/\(^Release:[[:space:]]*[[:digit:]]\+\)/\1$release/" resalloc.spec \

+         > "$resultdir"/resalloc.spec

+ 

+     # 3. copy other sources

+     cp *.service "$resultdir"

+ 

+     $ copr add-package-custom PROJECT \

+             --name resalloc \

+             --script script \

+             --script-resultdir results \

+             --script-builddeps 'git' \

+             --script-chroot fedora-rawhide-x86_64

+ 

+     $ copr build-package --name resalloc PROJECT # trigger the build

file modified
+8 -0

@@ -115,6 +115,14 @@ 

  here is `gem2rpm <https://github.com/fedora-ruby/gem2rpm>`_.

  

  

+ Custom (script)

+ ^^^^^^^^^^^^^^^

+ 

+ This source type uses a user-defined script to generate sources (which are later

+ used to create SRPM).  For more info, have a look at

+ :ref:`custom_source_method`.

+ 

+ 

  GitHub Webhooks

  ---------------

  

@@ -220,6 +220,7 @@ 

          "scm": "Build from an SCM repository",

          "pypi": "Build from PyPI",

          "rubygems": "Build from RubyGems",

+         "custom": "Custom build method",

      }

  

      return description_map.get(state, "")

@@ -41,6 +41,8 @@ 

          return PackageFormTito # deprecated

      elif source_type_text == 'mock_scm':

          return PackageFormMock # deprecated

+     elif source_type_text == "custom":

+         return PackageFormCustom

      else:

          raise exceptions.UnknownSourceTypeException("Invalid source type")

  

@@ -574,6 +576,60 @@ 

          })

  

  

+ class PackageFormCustom(BasePackageForm):

+     script = wtforms.TextAreaField(

+         "Script",

+         validators=[

+             wtforms.validators.DataRequired(),

+             wtforms.validators.Length(

+                 max=4096,

+                 message="Maximum script size is 4kB"),

+         ],

+     )

+ 

+     builddeps = wtforms.StringField(

+         "Build dependencies",

+         validators=[wtforms.validators.Optional()])

+ 

+     chroot = wtforms.SelectField(

+         'Mock chroot',

+         choices=[],

+         default='fedora-latest-x86_64',

+     )

+ 

+     resultdir = wtforms.StringField(

+         "Result directory",

+         validators=[wtforms.validators.Optional()])

+ 

+     def __init__(self, *args, **kwargs):

+         super(PackageFormCustom, self).__init__(*args, **kwargs)

+         chroot_objects = models.MockChroot.query.filter(models.MockChroot.is_active).all()

+ 

+         chroots = [c.name for c in chroot_objects]

+         chroots.sort()

+         chroots = [(name, name) for name in chroots]

+ 

+         arches = set()

+         for ch in chroot_objects:

+             if ch.os_release == 'fedora':

+                 arches.add(ch.arch)

+ 

+         self.chroot.choices = []

+         if arches:

+             self.chroot.choices += [('fedora-latest-' + l, 'fedora-latest-' + l) for l in arches]

+ 

+         self.chroot.choices += chroots

+ 

+     @property

+     def source_json(self):

+         return json.dumps({

+             "script": self.script.data,

+             "chroot": self.chroot.data,

+             "builddeps": self.builddeps.data,

+             "resultdir": self.resultdir.data,

+         })

+ 

+ 

  class RebuildAllPackagesFormFactory(object):

      def __new__(cls, active_chroots, package_names):

          form_cls = BaseBuildFormFactory(active_chroots, FlaskForm)

@@ -677,6 +733,11 @@ 

          return form

  

  

+ class BuildFormCustomFactory(object):

+     def __new__(cls, active_chroots):

+         return BaseBuildFormFactory(active_chroots, PackageFormCustom)

+ 

+ 

  class BuildFormUrlFactory(object):

      def __new__(cls, active_chroots):

          form = BaseBuildFormFactory(active_chroots, FlaskForm)

@@ -114,6 +114,7 @@ 

              "pypi": 5, # package_name, version, python_versions

              "rubygems": 6, # gem_name

              "scm": 8, # type, clone_url, committish, subdirectory, spec, srpm_build_method

+             "custom": 9, # user-provided script to build sources

             }

  

  # The same enum is also in distgit's helpers.py

@@ -483,6 +483,31 @@ 

          return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)

  

      @classmethod

+     def create_new_from_custom(cls, user, copr,

+             script, script_chroot=None, script_builddeps=None,

+             script_resultdir=None, chroot_names=None, **kwargs):

+         """

+         :type user: models.User

+         :type copr: models.Copr

+         :type script: str

+         :type script_chroot: str

+         :type script_builddeps: str

+         :type script_resultdir: str

+         :type chroot_names: List[str]

+         :rtype: models.Build

+         """

+         source_type = helpers.BuildSourceEnum("custom")

+         source_dict = {

+             'script': script,

+             'chroot': script_chroot,

+             'builddeps': script_builddeps,

+             'resultdir': script_resultdir,

+         }

+ 

+         return cls.create_new(user, copr, source_type, json.dumps(source_dict),

+                 chroot_names, **kwargs)

+ 

+     @classmethod

      def create_new_from_upload(cls, user, copr, f_uploader, orig_filename,

                                 chroot_names=None, **build_options):

          """

@@ -505,6 +505,17 @@ 

                        db.Index('build_order', "is_background", "id"),

                        db.Index('build_filter', "source_type", "canceled"))

  

+     def __init__(self, *args, **kwargs):

+         if kwargs.get('source_type') == helpers.BuildSourceEnum("custom"):

+             source_dict = json.loads(kwargs['source_json'])

+             if 'fedora-latest' in source_dict['chroot']:

+                 arch = source_dict['chroot'].split('-')[2]

+                 source_dict['chroot'] = \

+                     MockChroot.latest_fedora_branched_chroot(arch=arch).name

+             kwargs['source_json'] = json.dumps(source_dict)

+ 

+         super(Build, self).__init__(*args, **kwargs)

+ 

      id = db.Column(db.Integer, primary_key=True)

      # single url to the source rpm, should not contain " ", "\n", "\t"

      pkgs = db.Column(db.Text)

@@ -834,6 +845,16 @@ 

      distgit_branch = db.relationship("DistGitBranch",

              backref=db.backref("chroots"))

  

+     @classmethod

+     def latest_fedora_branched_chroot(cls, arch='x86_64'):

+         return (cls.query

+                 .filter(cls.is_active == True)

+                 .filter(cls.os_release == 'fedora')

+                 .filter(cls.os_version != 'rawhide')

+                 .filter(cls.arch == arch)

+                 .order_by(cls.os_version.desc())

+                 .first())

+ 

      @property

      def name(self):

          """

@@ -1,4 +1,5 @@ 

  {% from "_helpers.html" import render_field, render_form_errors, copr_url, render_pypi_python_versions_field, render_additional_build_options, render_srpm_build_method_box %}

+ {% from "coprs/detail/_method_forms.html" import copr_method_form_fileds_custom %}

  

  {# This file contains forms for the "New Build" action

  

@@ -156,6 +157,14 @@ 

  {% endmacro %}

  

  

+ {% macro copr_build_form_custom(form, view, copr) %}

+   {{ copr_build_form_begin(form, view, copr) }}

+   {{ source_description('Provide custom script to build sources.')}}

+   {{ copr_method_form_fileds_custom(form) }}

+   {{ copr_build_form_end(form, view, copr) }}

+ {% endmacro %}

+ 

+ 

  {% macro copr_build_form_rebuild(form, view, copr, build) %}

    {{ copr_build_form_begin(form, view, copr, build, hide_panels=True) }}

    {{ copr_build_form_end(form, view, copr, hide_panels=True) }}

@@ -0,0 +1,22 @@ 

+ {% from "_helpers.html" import render_field %}

+ {% macro copr_method_form_fileds_custom(form) %}

+   {{ render_field(

+         form.script,

+         rows=5,

+         style='font-family: monospace;',

+         placeholder="""#! /bin/sh -x

+ curl https://example.com/package.spec -O

+ curl https://example.com/tarball.tar.gz -O

+ """,

+         info="write a script that generates spec and sources... (Internet ON, non-root UID)"

+   )}}

+   {{ render_field(

+         form.chroot,

+         placeholder="fedora-latest-x86_64",

+         info="what chroot to run the <strong>script</strong> in") }}

+   {{ render_field(form.builddeps, placeholder="Optional - space-separated list of packages",

+         info="packages that the <strong>script</strong> requires for its execution" ) }}

+   {{ render_field(form.resultdir, placeholder="Optional - directory where SCRIPT generates sources",

+                   info="path relative to the <strong>script</strong>'s current

+                   working directory") }}

+ {% endmacro %}

@@ -1,4 +1,5 @@ 

  {% from "_helpers.html" import render_field, render_form_errors, copr_url, render_pypi_python_versions_field, render_additional_build_options, render_srpm_build_method_box %}

+ {% from "coprs/detail/_method_forms.html" import copr_method_form_fileds_custom %}

  

  {% macro copr_package_form_begin(form, view, copr, package) %}

    {{ render_form_errors(form) }}

@@ -86,6 +87,15 @@ 

    {{ copr_package_form_end(form, package, 'rubygems') }}

  {% endmacro %}

  

+ 

+ {% macro copr_package_form_custom(form, view, copr, package) %}

+   {{ copr_package_form_begin(form, view, copr, package) }}

+   {{ copr_method_form_fileds_custom(form) }}

+   {{ render_webhook_rebuild(form) }}

+   {{ copr_package_form_end(form, package, 'custom') }}

+ {% endmacro %}

+ 

+ 

  {% macro copr_package_form_scm(form, view, copr, package) %}

    {{ copr_package_form_begin(form, view, copr, package) }}

  

@@ -3,6 +3,7 @@ 

     copr_package_form_scm,

     copr_package_form_pypi,

     copr_package_form_rubygems,

+    copr_package_form_custom,

  with context %}

  

  

@@ -20,6 +21,7 @@ 

      {{ nav_element("scm", "SCM", copr_url(view, copr, source_type_text="scm", **kwargs)) }}

      {{ nav_element("pypi", "PyPI", copr_url(view, copr, source_type_text="pypi", **kwargs)) }}

      {{ nav_element("rubygems", "RubyGems", copr_url(view, copr, source_type_text="rubygems", **kwargs)) }}

+     {{ nav_element("custom", "Custom", copr_url(view, copr, source_type_text="custom", **kwargs)) }}

    </ul>

  {% endmacro %}

  

@@ -33,6 +35,9 @@ 

    {% elif source_type_text == "rubygems" %}

      {{ copr_package_form_rubygems(form_rubygems, view, copr, package) }}

  

+   {% elif source_type_text == "custom" %}

+     {{ copr_package_form_custom(form_custom, view, copr, package) }}

+ 

    {% else %}

      <h3>Wrong source type</h3>

    {% endif %}

@@ -38,6 +38,7 @@ 

        {{ nav_element("scm", "SCM", copr_url('coprs_ns.copr_add_build_scm', copr)) }}

        {{ nav_element("pypi", "PyPI", copr_url('coprs_ns.copr_add_build_pypi', copr)) }}

        {{ nav_element("rubygems", "RubyGems", copr_url('coprs_ns.copr_add_build_rubygems', copr)) }}

+       {{ nav_element("custom", "Custom", copr_url('coprs_ns.copr_add_build_custom', copr)) }}

      </ul>

    </div>

  </div>

@@ -0,0 +1,11 @@ 

+ {% extends "coprs/detail/add_build.html" %}

+ 

+ {% from "coprs/detail/_builds_forms.html" import copr_build_form_custom with context %}

+ 

+ {% set add_build_tab = "custom" %}

+ 

+ {% block build_form %}

+ 

+ {{ copr_build_form_custom(form, view, copr) }}

+ 

+ {% endblock %}

@@ -65,6 +65,20 @@ 

            <li> Select to trigger on <code>Repository Push</code>. </li>

            <li> Click the <code>Save</code> button. </li>

          </ol>

+ 

+         <h3> Custom webhook </h3>

+         <div class="well well-sm">

+             {{ custom_url }}

+         </div>

+         <h4> How to use it: </h4>

+         <p> Use the GitLab/GitHub steps above (when needed), or simply </p>

+         <p>

+           <div class="well well-sm">

+             $ curl -X POST {{ custom_url }}

+           </div>

+           Note that the package of name 'PACKAGE_NAME' must exist within this

+           project, and that the 'POST' http method must be specified.

+         </p>

      </div>

  </div>

  

@@ -483,6 +483,25 @@ 

      return process_creating_new_build(copr, form, create_new_build)

  

  

+ @api_ns.route("/coprs/<username>/<coprname>/new_build_custom/", methods=["POST"])

+ @api_login_required

+ @api_req_with_copr

+ def copr_new_build_custom(copr):

+     form = forms.BuildFormCustomFactory(copr.active_chroots)(csrf_enabled=False)

+     def create_new_build():

+         return BuildsLogic.create_new_from_custom(

+             flask.g.user,

+             copr,

+             form.script.data,

+             form.chroot.data,

+             form.builddeps.data,

+             form.resultdir.data,

+             chroot_names=form.selected_chroots,

+             background=form.background.data,

+         )

+     return process_creating_new_build(copr, form, create_new_build)

+ 

+ 

  @api_ns.route("/coprs/<username>/<coprname>/new_build_scm/", methods=["POST"])

  @api_login_required

  @api_req_with_copr

@@ -5,6 +5,7 @@ 

  import shutil

  import tempfile

  

+ from functools import wraps

  from werkzeug import secure_filename

  

  from coprs import app

@@ -351,6 +352,52 @@ 

      form = forms.BuildFormRubyGemsFactory(copr.active_chroots)()

      return process_new_build(copr, form, factory, render_add_build_rubygems, add_view, url_on_success)

  

+ ############################### Custom builds ###############################

+ 

+ @coprs_ns.route("/g/<group_name>/<coprname>/new_build_custom/", methods=["POST"])

+ @coprs_ns.route("/<username>/<coprname>/new_build_custom/", methods=["POST"])

+ @login_required

+ @req_with_copr

+ def copr_new_build_custom(copr):

+     """ Handle the build request and redirect back. """

+ 

+     # TODO: parametric decorator for this view && url_on_success

+     view = 'coprs_ns.copr_new_build_custom'

+     url_on_success = helpers.copr_url('coprs_ns.copr_add_build_custom', copr)

+ 

+     def factory(**build_options):

+         BuildsLogic.create_new_from_custom(

+             flask.g.user,

+             copr,

+             form.script.data,

+             form.chroot.data,

+             form.builddeps.data,

+             form.resultdir.data,

+             chroot_names=form.selected_chroots,

+             **build_options

+         )

+ 

+     form = forms.BuildFormCustomFactory(copr.active_chroots)()

+ 

+     return process_new_build(copr, form, factory, render_add_build_custom,

+                              view, url_on_success)

+ 

+ 

+ 

+ @coprs_ns.route("/g/<group_name>/<coprname>/add_build_custom/")

+ @coprs_ns.route("/<username>/<coprname>/add_build_custom/")

+ @login_required

+ @req_with_copr

+ def copr_add_build_custom(copr, form=None):

+     return render_add_build_custom(copr, form,

+                                    'coprs_ns.copr_new_build_custom')

+ 

+ def render_add_build_custom(copr, form, view, package=None):

+     if not form:

+         form = forms.BuildFormCustomFactory(copr.active_chroots)()

+     return flask.render_template("coprs/detail/add_build/custom.html",

+                                  copr=copr, form=form, view=view)

+ 

  

  ################################ Upload builds ################################

  

@@ -422,10 +422,15 @@ 

                    copr.id,

                    copr.webhook_secret)

  

+     custom_url = "https://{}/webhooks/custom/{}/{}/".format(

+                   app.config["PUBLIC_COPR_HOSTNAME"],

+                   copr.id,

+                   copr.webhook_secret) + "<PACKAGE_NAME>/"

+ 

      return flask.render_template(

          "coprs/detail/settings/webhooks.html",

          copr=copr, bitbucket_url=bitbucket_url, github_url=github_url,

-         gitlab_url=gitlab_url)

+         gitlab_url=gitlab_url, custom_url=custom_url)

  

  

  @coprs_ns.route("/g/<group_name>/<coprname>/webhooks/")

@@ -8,7 +8,7 @@ 

  from coprs import helpers

  from coprs.models import Package, Build

  from coprs.views.coprs_ns import coprs_ns

- from coprs.views.coprs_ns.coprs_builds import render_add_build_scm, render_add_build_pypi

+ from coprs.views.coprs_ns.coprs_builds import render_add_build_scm, render_add_build_pypi, render_add_build_custom

  from coprs.views.misc import login_required, page_not_found, req_with_copr, req_with_copr

  from coprs.logic.complex_logic import ComplexLogic

  from coprs.logic.packages_logic import PackagesLogic

@@ -107,6 +107,10 @@ 

          form = forms.BuildFormPyPIFactory

          f = render_add_build_pypi

          view_suffix = "_pypi"

+     elif package.source_type_text == "custom":

+         form = forms.BuildFormCustomFactory

+         f = render_add_build_custom

+         view_suffix = "_custom"

      else:

          flask.flash("Package {} has not the default source which is required for rebuild. Please configure some source"

                      .format(package_name, copr.full_name))

@@ -127,6 +131,7 @@ 

          "scm": forms.PackageFormScm(),

          "pypi": forms.PackageFormPyPI(),

          "rubygems": forms.PackageFormRubyGems(),

+         "custom": forms.PackageFormCustom(),

      }

  

      if "form" in kwargs:

@@ -135,7 +140,8 @@ 

      return flask.render_template("coprs/detail/add_package.html", copr=copr, package=None,

                                   source_type_text=source_type_text, view="coprs_ns.copr_new_package",

                                   form_scm=form["scm"], form_pypi=form["pypi"],

-                                  form_rubygems=form["rubygems"])

+                                  form_rubygems=form["rubygems"],

+                                  form_custom=form['custom'])

  

  

  @coprs_ns.route("/<username>/<coprname>/package/new/<source_type_text>", methods=["POST"])

@@ -167,6 +173,7 @@ 

          "scm": forms.PackageFormScm,

          "pypi": forms.PackageFormPyPI,

          "rubygems": forms.PackageFormRubyGems,

+         "custom": forms.PackageFormCustom,

      }

      form = {k: v(formdata=None) for k, v in form_classes.items()}

  

@@ -178,7 +185,8 @@ 

      return flask.render_template("coprs/detail/edit_package.html", package=package, copr=copr,

                                   source_type_text=source_type_text, view="coprs_ns.copr_edit_package",

                                   form_scm=form["scm"], form_pypi=form["pypi"],

-                                  form_rubygems=form["rubygems"])

+                                  form_rubygems=form["rubygems"],

+                                  form_custom=form['custom'])

  

  

  @coprs_ns.route("/<username>/<coprname>/package/<package_name>/edit/<source_type_text>", methods=["POST"])

@@ -226,8 +226,8 @@ 

              shutil.rmtree(self.tmp)

  

  

- @webhooks_ns.route("/custom/<uuid>/<copr_id>/", methods=["POST"])

- @webhooks_ns.route("/custom/<uuid>/<copr_id>/<package_name>/", methods=["POST"])

+ @webhooks_ns.route("/custom/<copr_id>/<uuid>/", methods=["POST"])

+ @webhooks_ns.route("/custom/<copr_id>/<uuid>/<package_name>/", methods=["POST"])

  @copr_id_and_uuid_required

  @package_name_required

  @skip_invalid_calls

@@ -6,7 +6,7 @@ 

  

  class TestCustomWebhook(CoprsTestCase):

      def custom_post(self, data, token, copr_id, package_name=None):

-         url = "/webhooks/custom/{uuid}/{copr_id}/"

+         url = "/webhooks/custom/{copr_id}/{uuid}/"

          url = url.format(uuid=token, copr_id=copr_id)

          if package_name:

              url = "{0}{1}/".format(url, package_name)

file modified
+82 -0

@@ -69,6 +69,7 @@ 

  SOURCE_TYPE_PYPI = 'pypi'

  SOURCE_TYPE_RUBYGEMS = 'rubygems'

  SOURCE_TYPE_SCM = 'scm'

+ SOURCE_TYPE_CUSTOM = 'custom'

  

  class CoprClient(UnicodeMixin):

      """ Main interface to the copr service

@@ -588,6 +589,47 @@ 

          return self.process_creating_new_build(projectname, data, api_endpoint, username,

                                                 chroots, background=background)

  

+ 

+     def create_new_build_custom(self, projectname,

+             script, script_chroot=None, script_builddeps=None,

+             script_resultdir=None,

+             username=None, timeout=None, memory=None, chroots=None,

+             background=False, progress_callback=None):

+         """ Creates new build with Custom source build method.

+ 

+             :param projectname: name of Copr project (without user namespace)

+             :param script: script to execute to generate sources

+             :param script_chroot: [optional] what chroot to use to generate

+                 sources (defaults to fedora-latest-x86_64)

+             :param script_builddeps: [optional] list of script's dependencies

+             :param script_resultdir: [optional] where script generates results

+                 (relative to cwd)

+             :param username: [optional] use alternative username

+             :param timeout: [optional] build timeout

+             :param memory: [optional] amount of required memory for build process

+             :param chroots: [optional] build only with given chroots

+             :param background: [optional] mark the build as a background job.

+             :param progress_callback: [optional] a function that received a

+             MultipartEncoderMonitor instance for each chunck of uploaded data

+ 

+             :return: :py:class:`~.responses.CoprResponse` with additional fields:

+ 

+                 - **builds_list**: list of :py:class:`~.responses.BuildWrapper`

+         """

+         data = {

+             "memory_reqs": memory,

+             "timeout": timeout,

+             "script": script,

+             "chroot": script_chroot,

+             "builddeps": script_builddeps,

+             "resultdir": script_resultdir,

+         }

+ 

+         api_endpoint = "new_build_custom"

+         return self.process_creating_new_build(projectname, data, api_endpoint, username,

+                                                chroots, background=background)

+ 

+ 

      def create_new_build_distgit(self, projectname, clone_url, branch=None, username=None,

                                timeout=None, memory=None, chroots=None, background=False, progress_callback=None):

          """ Creates new build from a dist-git repository

@@ -818,6 +860,46 @@ 

          })

          return response

  

+     def edit_package_custom(self, package_name, projectname,

+             script, script_chroot=None, script_builddeps=None,

+             script_resultdir=None,

+             ownername=None, webhook_rebuild=None):

+ 

+         request_url = self.get_package_edit_url(ownername, projectname,

+                 package_name, SOURCE_TYPE_CUSTOM)

+ 

+         data = {

+             "package_name": package_name,

+             "script": script,

+             "builddeps": script_builddeps,

+             "resultdir": script_resultdir,

+             "chroot": script_chroot,

+         }

+         if webhook_rebuild != None:

+             data['webhook_rebuild'] = 'y' if webhook_rebuild else ''

+ 

+         response = self.process_package_action(request_url, ownername, projectname, data)

+         return response

+ 

+     def add_package_custom(self, package_name, projectname,

+             script, script_chroot=None, script_builddeps=None,

+             script_resultdir=None,

+             ownername=None, webhook_rebuild=None):

+ 

+         request_url = self.get_package_add_url(ownername, projectname,

+                 SOURCE_TYPE_CUSTOM)

+         response = self.process_package_action(request_url, ownername,

+                 projectname, data={

+                     "package_name": package_name,

+                     "script": script,

+                     "builddeps": script_builddeps,

+                     "resultdir": script_resultdir,

+                     "chroot": script_chroot,

+                     "webhook_rebuild": 'y' if webhook_rebuild else '',

+                 },

+         )

+         return response

+ 

      def process_package_action(self, request_url, ownername, projectname, data, fetch_functor=None):

          if not ownername:

              ownername = self.username

@@ -0,0 +1,107 @@ 

+ #! /bin/env python

+ 

+ import os

+ import subprocess

+ import argparse

+ import logging

+ import pipes

+ 

+ DEFAULT_WORKDIR = '/builddir/copr-sources-custom'

+ 

+ logging.basicConfig(level=logging.DEBUG)

+ log = logging.getLogger(__name__)

+ 

+ description = """

+ Generate spec + patches + tarballs using custom script in mock chroot.

+ """.strip()

+ 

+ parser = argparse.ArgumentParser(

+         description = description

+ )

+ parser.add_argument(

+         "-r", "--mock-config", "--root",

+         dest='config',

+         required=True,

+         help="mock config reference (alternative to '-r/--root' option in mock)"

+ )

+ parser.add_argument(

+         "--script",

+         required=True,

+         type=argparse.FileType('r'),

+         help="file to be copyied into mock chroot and executed"

+ )

+ parser.add_argument(

+         "--hook-payload-file",

+         type=argparse.FileType('r'),

+         help="file with 'webhook payload content', it is going to be copied "

+              "into working directory as 'hook_payload' file",

+ )

+ parser.add_argument(

+         "--builddeps",

+         help="space separated list of build dependencies (packages)"

+ )

+ parser.add_argument(

+         "--resultdir",

+         default=".",

+         help="directory were the SCRIPT generates the sources within mock chroot"

+ )

+ parser.add_argument(

+         "--workdir",

+         default=DEFAULT_WORKDIR,

+         help="execute the SCRIPT from with CWD=WORKDIR (within mock chroot)"

+ )

+ 

+ 

+ def run_cmd(command, **kwargs):

+     log.info("running command: " + ' '.join([pipes.quote(x) for x in command]))

+     return subprocess.check_call(command, **kwargs)

+ 

+ 

+ if __name__ == "__main__":

+     args = parser.parse_args()

+ 

+     user = 'mockbuild'

+ 

+     # Where the script is run from within mock chroot.

+     workdir = os.path.normpath(args.workdir)

+ 

+     # Where the script's result will be copied from, by default

+     # equal to workdir.

+     resultdir = os.path.normpath(os.path.join(workdir, args.resultdir))

+ 

+     log.info("Working in '{0}'".format(workdir))

+     log.info("Results should be created in '{0}'".format(resultdir))

+ 

+     mock = ['mock', '-r', args.config]

+ 

+     run_cmd(mock + ['--init'])

+ 

+     if args.builddeps:

+         pkgs = args.builddeps.split()

+         run_cmd(mock + ['--install'] + pkgs)

+ 

+     run_cmd(mock + ['--shell', 'cat - > /script'], stdin=args.script)

+ 

+     setup_cmd = 'set -ex; chmod a+x /script;'

+ 

+     prep_dir_template = ' rm -rf {d}; mkdir -p {d}; chown {user} {d};'

+     setup_cmd += prep_dir_template.format(

+         user=user,

+         d=pipes.quote(workdir),

+     )

+ 

+     run_cmd(mock + ['--chroot', setup_cmd])

+ 

+     if args.hook_payload_file:

+         payload_file_inner = "{0}/hook_payload".format(pipes.quote(workdir))

+         run_cmd(mock + ['--shell', 'cat - > ' + payload_file_inner],

+                 stdin=args.hook_payload_file)

+         run_cmd(mock + ['--shell', 'chmod a+r ' + payload_file_inner])

+ 

+     cmd = 'set -xe ; cd {workdir} ; {env} /script'.format(

+         workdir=pipes.quote(workdir),

+         env='COPR_RESULTDIR=' + pipes.quote(resultdir),

+     )

+     cmd = pipes.quote(cmd)

+ 

+     run_cmd(mock + ['--chroot', 'su - {0} -c {1}'.format(user, cmd)])

file modified
+2 -0

@@ -59,6 +59,7 @@ 

  

  install -d %{buildroot}%{_mandir}/man1

  install -p -m 644 man/copr-rpmbuild.1 %{buildroot}/%{_mandir}/man1/

+ install -p -m 755 bin/copr-sources-custom %buildroot%_bindir

  

  %py3_install

  

@@ -68,6 +69,7 @@ 

  %{python3_sitelib}/*

  

  %{_bindir}/copr-rpmbuild

+ %{_bindir}/copr-sources-custom

  %{_mandir}/man1/copr-rpmbuild.1*

  

  %dir %attr(0775, root, mock) %{_sharedstatedir}/copr-rpmbuild

@@ -23,6 +23,7 @@ 

      PYPI = 5

      RUBYGEMS = 6

      SCM = 8

+     CUSTOM = 9

  

  

  def cmd_debug(result):

@@ -189,3 +190,26 @@ 

          subprocess.check_output(cmd, shell=True)

      finally:

          os.chdir(cwd)

+ 

+ 

+ def build_srpm(srcdir, destdir):

+     cmd = [

+         'rpmbuild', '-bs',

+         '--define', '_sourcedir ' + srcdir,

+         '--define', '_rpmdir '    + srcdir,

+         '--define', '_builddir '  + srcdir,

+         '--define', '_specdir '   + srcdir,

+         '--define', '_srcrpmdir ' + destdir,

+     ]

+ 

+     specfiles = glob.glob(os.path.join(srcdir, '*.spec'))

+     if len(specfiles) == 0:

+         raise RuntimeError("no spec file available")

+ 

+     if len(specfiles) > 1:

+         raise RuntimeError("too many specfiles: {0}".format(

+             ', '.join(specfiles)

+         ))

+ 

+     cmd += [specfiles[0]]

+     run_cmd(cmd)

@@ -3,6 +3,7 @@ 

  from .pypi import PyPIProvider

  from .spec import SpecUrlProvider

  from .scm import ScmProvider

+ from .custom import CustomProvider

  

  

  __all__ = [RubyGemsProvider, PyPIProvider,

@@ -17,6 +18,7 @@ 

              SourceType.RUBYGEMS: RubyGemsProvider,

              SourceType.PYPI: PyPIProvider,

              SourceType.SCM: ScmProvider,

+             SourceType.CUSTOM: CustomProvider,

          }[source_type]

      except KeyError:

          raise RuntimeError("No provider associated with this source type")

@@ -0,0 +1,94 @@ 

+ import os

+ import json

+ import logging

+ import shutil

+ import tempfile

+ import subprocess

+ import glob

+ import requests

+ 

+ from copr_rpmbuild import helpers

+ from .base import Provider

+ 

+ 

+ log = logging.getLogger("__main__")

+ 

+ 

+ class CustomProvider(Provider):

+     chroot = 'fedora-rawhide-x86_64'

+     builddeps = None

+     file_script = None

+     inner_resultdir = None

+     inner_workdir = '/workdir'

+     hook_payload_url = None

+ 

+     workdir = None

+ 

+     def __init__(self, source_json, outdir, config):

+         super(CustomProvider, self).__init__(source_json, outdir, config)

+ 

+         self.outdir = outdir

+         self.chroot = source_json.get('chroot')

+         self.inner_resultdir = source_json.get('resultdir')

+         self.builddeps = source_json.get('builddeps')

+ 

+         if 'hook_data' in source_json:

+             self.hook_payload_url = "{server}/tmp/{tmp}/hook_payload".format(

+                 server=config.get("main", "frontend_url"),

+                 tmp=source_json['tmp'],

+             )

+ 

+         self.workdir = outdir

+         self.file_script = os.path.join(self.workdir, 'script')

+         with open(self.file_script, 'w') as script:

+             script.write(source_json['script'])

+ 

+ 

+     def produce_srpm(self):

+         mock_config_file = os.path.join(self.outdir, 'mock-config.cfg')

+ 

+         with open(mock_config_file, 'w') as f:

+             # Enable network.

+             f.write("include('/etc/mock/{0}.cfg')\n".format(self.chroot))

+             f.write("config_opts['rpmbuild_networking'] = True\n")

+             f.write("config_opts['use_host_resolv'] = True\n")

+ 

+         cmd = [

+             'unbuffer',

+             'copr-sources-custom',

+             '--workdir', self.inner_workdir,

+             '--mock-config', mock_config_file,

+             '--script', self.file_script,

+         ]

+         if self.builddeps:

+             cmd += ['--builddeps', self.builddeps]

+ 

+         if self.hook_payload_url:

+             chunk_size = 1024

+             hook_payload_file = os.path.join(self.outdir, 'hook_payload')

+             response = requests.get(self.hook_payload_url, stream=True)

+             response.raise_for_status()

+ 

+             with open(hook_payload_file, 'wb') as payload_file:

+                 for chunk in response.iter_content(chunk_size):

+                     payload_file.write(chunk)

+ 

+             cmd += ['--hook-payload-file', hook_payload_file]

+ 

+         inner_resultdir = self.inner_workdir

+         if self.inner_resultdir:

+             # User wishes to re-define resultdir.

+             cmd += ['--resultdir', self.inner_resultdir]

+             inner_resultdir = os.path.normpath(os.path.join(

+                 self.inner_workdir, self.inner_resultdir))

+ 

+         # prepare the sources

+         helpers.run_cmd(cmd)

+ 

+         # copy the sources out into workdir

+         mock = ['mock', '-r', mock_config_file]

+ 

+         srpm_srcdir = os.path.join(self.workdir, 'srcdir')

+         helpers.run_cmd(mock + ['--copyout', inner_resultdir, srpm_srcdir])

+         helpers.build_srpm(srpm_srcdir, self.outdir)

+         shutil.rmtree(srpm_srcdir)

Missing things to be done:

[-] generate mock config (postponed to next PR)
[x] create the script from /get-srpm-build-task/ info provided by frontend
[x] (different semantics now) call the copr-sources-custom script (with --resultdir=$(mktemp -d))
[x] copy out the results from build chroot (mock --copy-out)
[x] run rpmbuild -bs (or something similar to get the srpm, this is long-term undesirable and redundant step since the sources could be imported to distgit without "wrapping" to src.rpm)
[x] api/cli is needed (not a blocker in the first step, where webui should be enough)
[x] webui is needed
[x] documentation (webhook payload parser example once 'generate mock config' is implemented)
[x] custom webhook + docs (PR#194)
[x] specfile fixes for installation of rpmbuild package
[x] copr-rpmbuild needs to download the hook_payload, and "export" that somehow
[x] beaker tests

Thank you for this PR/suggestion. At a first glance, it seems very similar to the make_srpm method for SCM source type and I am curious about the added value here. Also note that there will raise an issue with auto-rebuilding for this Custom Source Type. With SCM, we compare clone URLs in the incoming event with what's configured for a package to know if the event relates to the package or not. That approach is not possible here (due to missing Clone URL info). The last objection is that it is unclear what should be the output of the script.

I am curious about the added value here.

This method will work for every package out there, without ever touching existing
git repo or creating new one, just for the CI case.

.. That approach is not possible here (due to missing Clone URL info).

With this method, you need to put 'git' into minimal buildroot dependencies, and you
are responsible for cloning the repo yourselves.

The last objection is that it is unclear what should be the output of the script.

That's something which needs to be properly documented, but the output of this
script is content of %_sourcedir (tarball + spec file + maybe patches).

This method will work for every package out there, without ever touching existing
git repo or creating new one, just for the CI case.

Creating a new repo just for CI case is, I think, fine.

With this method, you need to put 'git' into minimal buildroot dependencies, and you
are responsible for cloning the repo yourselves.

You probably haven't read or misunderstood my argument about auto-rebuilding.

Creating a new repo just for CI case is, I think, fine.

I agree that it about preferences, but I prefer not to create additional repo and
not touching upstream repos or as minimally as possible.

You probably haven't read or misunderstood my argument about auto-rebuilding.

You are right. Can you elaborate? Why you need to compare clone urls?
The web-hook is not bound to particular package rebuild?

Please take another look at this WIP, there's big TODO list, but at least to understand the workflow idea, feedback needed.

rebased onto 69f36c80d3f79473f4ae965c79042b0c753c451b

10 months ago

Please don't add some extra command-line tools to the copr-rpmbuild package. All the functionality needed to implement the custom method should be present in the CustomProvider below. With this "concept" of extra script, copr-backend needs to know the build type (srpm/scm/custom) to invoke the right tool. It currently does get that information but it probly won't in the future to minimize data transfer between frontend and backend. It will only get what's important to schedule/allocate the job properly (arch, owner, task_id). Also, the backend would need to parse the source_json structure, we want only builder doing that.

Oh, I see you are invoking this script from the CustomProvider. I will read the whole thing first.

The actual task (job definition for the srpm build) needs to be fully specified by task["source_type"] and task["source_json"]. All the required information to build the srpm should be placed there.

Why are you creating another "local" executable. Is it possible not to do that?

You shouldn't need to obtain the mock config. The frontend's chroot build config (that gets translated into the respective mock config on builders) is only relevant for rpm builds, not for srpm builds. If you want a user to be able to provide a mock configuration (like the builddeps), then this configuration should be fully specified in the custom package or build definition (unless being always the same) and also be dedicated for the srpm build itself. The rpm build afterwards should follow the rules of the other rpm builds. Why? It's according to the already existing source types and the way srpms are created for them. I think the same rules should apply for all the dynamically created srpms.

I can keep this for my own debugging purposes locally, if that matters, any consequences out of curiosity? My "pros" is that it is just convenience wrapper which really simplifies testing of the software directly from git repo...

You shouldn't need to obtain the mock config. The frontend's chroot build config (that gets translated into the respective mock config on builders) is only relevant for rpm builds, not for srpm builds.

This is somewhat crucial motivating point for me though. I need to be able to on-demand update software in chroot, and I really want to let that affect the srpm build -- e.g. if I want to devel some universal web-hook-parser library, going through official fedora repos would be too long shot. Typical usecase for me: I may well try to "fix" or develop autotools packages only in my copr repo (which are used to produce tarballs through custom method ...).

If you want a user to be able to provide a mock configuration (like the builddeps), then this configuration should be fully specified in the custom package or build definition (unless being always the same) and also be dedicated for the srpm build itself.

The 'builddeps' parameter is dedicated to srpm build only, however... I don't take that as "configuration", but rather step to do to get the srpm built (maybe instead of specifying biuld-deps, I could specify procedural "root-prepare-command").

If you speak about "additional_repos" or "additional_packages" "configuration" --
configured per-project/per-chroot, it really is mock configuration I requested globally to build against -- and thus I want to prepare the srpm against that configuration, too.

The rpm build afterwards should follow the rules of the other rpm builds. Why?
It's according to the already existing source types and the way srpms are created for them. I think the same rules should apply for all the dynamically created srpms.

I guess I am missing the point here, can you please elaborate?

The point is that I need the copr id in question; to generate appropriate mock config.. Do you suggest to hard-code this once more into source_json?

Well, there is the provider interface and you should stick to it. Otherwise, you are breaking the interface, which is bad. It's as simple as that.

Does that mean "yes"? :) I don't feel it is clean. There's alternative option to pre-generate the mock configuration in advance -- before calling the provider.. but it would just complicate things.

Fwiw, consider that you develop /bin/rpkg... and you want to build it in copr and check that srpm builds fine with the re-built versoin --- for this reason I think wrapping other methods into mock chroot job would be useful, too.

Of course, we can also change the providers' interface... (is that what you meant)? Btw. that's why ask for advice since I have no significant preference...

We don't want to change the provider interface. source_type and source_json fully define the srpm build and any new method should stick to it. I am afraid you will need to update your requirements on this new method.

We don't want to change the provider interface. source_type and source_json fully define the srpm build and any new method should stick to it

Agreed. Though additional provider.prepare(task) call (usually unimplemented wouldn't be that bad).

I am afraid you will need to update your requirements on this new method.

So how? If you agree that I need that information, I'm interested to see what requirement should I update.

rebased onto 7911d8017d449353f010942b6f230f7e23283648

10 months ago

The requirement that chroot (and project) settings in COPR configuration influence the srpm build. There are two good options even without that. First is to extend the make_srpm method and give it the third argument: payload. In the make_srpm script, you can do everything you need to. And note that you can have just one Git repository with srpm generation scripts for all your maintained packages. For that you can use the subdirectory setting to invoke the correct script on the builder (i.e. you set the subdirectory for a given package to point to the directory where the respective srpm generation script can be found in your repo). The second option is giving structure to your custom script with one of the sections being e.g. "additional packages".

The requirement that chroot (and project) settings in COPR configuration influence the srpm build.

It is certainly what I want, however - same as I want to build (other) rpms against the buildroot, I want to build the srpms the same way. Please don't make me to relax this particular requirement :-/ that was something I was wishing about long time before I started doing the work and it is driving my work here :-)

I see the issue here (getting mock profile config), but nice solution will come with #185, can we agree on that? Until then, let's consider this WIP only. Btw., koji devels - after many years - now deal with the same issue (how to generate the mock config for specific target properly), while they realized that they want to generate slightly different mock config on fedora than on rhel (dnf vs yum issues).

nice solution will come with #185,

I meant #179 (I can not edit the comments, for some reason).

Sorry, but chroot and project settings are only intended for rpm building. It's already done like this for the other options to dynamically create the srpm in COPR. There are two good options to make your use-case possible so, please, stick to one them. In https://pagure.io/copr/copr/pull-request/179, you are just forcing a bad coding practice.

rebased onto a314121159a9913aaa7cb6bda35430bdc406f618

10 months ago

Sorry, but chroot and project settings are only intended for rpm building.

Who has/had this intention? It doesn't feel correct ...

It's already done like this for the other options to dynamically create the srpm in COPR.

How this is related to the new method - we can design/define/document anything differently in the new method, can't we?

In https://pagure.io/copr/copr/pull-request/179, you are just forcing a bad coding practice.

Self-confident, IMPOV statement which makes me sad again. Let that on the random reader where the true is.... but in #179 I was trying to only find a way to de-duplicate some efforts. You know how much time we spent on finding the best way towards this... The cause for