#1238 python, cli, frontend: module dist-git instance choice
Merged 4 years ago by praiskup. Opened 4 years ago by praiskup.

file modified
+8 -2
@@ -698,9 +698,11 @@

          ownername, projectname = self.parse_name(args.copr)

  

          if args.yaml:

-             module = self.client.module_proxy.build_from_file(ownername, projectname, args.yaml)

+             module = self.client.module_proxy.build_from_file(

+                     ownername, projectname, args.yaml, distgit=args.distgit)

          else:

-             module = self.client.module_proxy.build_from_url(ownername, projectname, args.url)

+             module = self.client.module_proxy.build_from_url(

+                     ownername, projectname, args.url, distgit=args.distgit)

          print("Created module {0}".format(module.nsv))

  

      def action_permissions_edit(self, args):
@@ -1208,6 +1210,10 @@

      parser_build_module_mmd_source.add_argument("--url", help="SCM with modulemd file in yaml format")

      parser_build_module_mmd_source.add_argument("--yaml", help="Path to modulemd file in yaml format")

      parser_build_module.set_defaults(func="action_build_module")

+     parser_build_module.add_argument(

+         "--distgit",

+         help="Dist-git instance to build against, e.g. 'fedora'"

+     )

  

  

      #########################################################

file modified
+24
@@ -697,3 +697,27 @@

      })

  

      main.main(argv=["list-chroots"])

+ 

+ 

+ @pytest.mark.parametrize('patch_name', ['build_from_file', 'build_from_url'])

+ @pytest.mark.parametrize('args', [[], ['--distgit', 'test']])

+ @mock.patch('copr_cli.main.config_from_file', return_value=mock_config)

+ def test_module_build(config, patch_name, args):

+     argv = ["build-module", "testproject"]

+ 

+     if patch_name == 'build_from_file':

+         argv += ["--yaml", "non-existing.yaml"]

+     else:

+         argv += ["--url", "http://example.com/test.yml"]

+ 

+     module = 'copr.v3.proxies.module'

+     with mock.patch('{0}.ModuleProxy.{1}'.format(module, patch_name)) as p:

+         main.main(argv=argv + args)

+ 

+         assert len(p.call_args_list) == 1

+         kwargs = p.call_args_list[0][1]

+         assert 'distgit' in kwargs

+         if args:

+             assert kwargs['distgit'] == args[1]

+         else:

+             assert kwargs['distgit'] is None

@@ -0,0 +1,34 @@

+ """

+ add dist_git_instance table

+ 

+ Revision ID: 2561c13a3556

+ Revises: d230af5e05d8

+ Create Date: 2020-01-30 14:49:07.698008

+ """

+ 

+ import sqlalchemy as sa

+ from alembic import op

+ 

+ 

+ revision = '2561c13a3556'

+ down_revision = 'd230af5e05d8'

+ 

+ def upgrade():

+     new_table = op.create_table('dist_git_instance',

+     sa.Column('id', sa.Integer(), nullable=False),

+     sa.Column('name', sa.String(length=50), nullable=False),

+     sa.Column('clone_url', sa.String(length=100), nullable=False),

+     sa.Column('clone_package_uri', sa.String(length=100), nullable=False),

+     sa.Column('priority', sa.Integer(), nullable=False),

+     sa.PrimaryKeyConstraint('id'),

+     sa.UniqueConstraint('name')

+     )

+     op.bulk_insert(new_table, [{

+         'name': 'fedora',

+         'clone_url': 'https://src.fedoraproject.org',

+         'clone_package_uri': 'rpms/{pkgname}.git',

+         'priority': 100,

+     }])

+ 

+ def downgrade():

+     op.drop_table('dist_git_instance')

@@ -19,6 +19,7 @@

  from coprs import models

  from coprs.logic.coprs_logic import CoprsLogic, MockChrootsLogic

  from coprs.logic.users_logic import UsersLogic

+ from coprs.logic.dist_git_logic import DistGitLogic

  from coprs import exceptions

  

  
@@ -940,10 +941,41 @@

      build = wtforms.BooleanField("build", default=True, false_values=FALSE_VALUES)

  

  

- class ModuleBuildForm(FlaskForm):

-     modulemd = FileField("modulemd")

-     scmurl = wtforms.StringField()

-     branch = wtforms.StringField()

+ class DistGitValidator(object):

+     def __call__(self, form, field):

+         if field.data not in form.distgit_choices:

+             message = "DistGit ID must be one of: {}".format(

+                 ", ".join(form.distgit_choices))

+             raise wtforms.ValidationError(message)

+ 

+ 

+ class NoneFilter():

+     def __init__(self, default):

+         self.default = default

+ 

+     def __call__(self, value):

+         if value in [None, 'None']:

+             return self.default

+         return value

+ 

+ 

+ def get_module_build_form(*args, **kwargs):

We seem not to use the arguments. Do we need to wrap the ModuleBuildForm with this function then?

We don't (yet?).

But yes, we need it ... as we need the other "Factories". You need to
fill the class fields from database query ... and if not wrapped by this method,
the fields would be filled too soon (at the time python processes
the class, and at that time e.g. isn't clear whether the db exists at
all) or too late (in constructor, the class was already read by python
and adding fields later raises UnboundField exception, or something
ugly as that).

the fields would be filled too soon

True, I didn't realize that. +1 for function then.

+     class ModuleBuildForm(FlaskForm):

+         distgit_choices = [x.id for x in DistGitLogic.ordered().all()]

+         distgit_default = distgit_choices[0]

+ 

+         modulemd = FileField("modulemd")

+         scmurl = wtforms.StringField()

+         branch = wtforms.StringField()

+ 

+         distgit = wtforms.SelectField(

+             'Build against DistGit instance',

+             choices=[(x, x) for x in distgit_choices],

+             validators=[DistGitValidator()],

+             filters=[NoneFilter(distgit_default)],

+         )

+ 

+     return ModuleBuildForm(*args, **kwargs)

  

  

  class PagureIntegrationForm(FlaskForm):

@@ -0,0 +1,17 @@

+ from sqlalchemy import desc

+ from coprs import models

+ 

+ 

+ class DistGitLogic:

+     @classmethod

+     def ordered(cls):

+         'get the default dist git instance object'

+         query = models.DistGitInstance.query

+         return query.order_by(desc('priority'), 'name')

+ 

+     @classmethod

+     def get_with_default(cls, distgit_name=None):

+         if distgit_name is None:

+             return cls.ordered().limit(1).one()

+         query = models.DistGitInstance.query.filter_by(name=distgit_name)

+         return query.one()

@@ -9,6 +9,7 @@

  from coprs import db

  from coprs import exceptions

  from coprs.logic import builds_logic

+ from coprs.logic.dist_git_logic import DistGitLogic

  from wtforms import ValidationError

  

  import gi
@@ -82,11 +83,12 @@

  

  

  class ModuleBuildFacade(object):

-     def __init__(self, user, copr, yaml, filename=None):

+     def __init__(self, user, copr, yaml, filename=None, distgit_name=None):

          self.user = user

          self.copr = copr

          self.yaml = yaml

          self.filename = filename

+         self.distgit = DistGitLogic.get_with_default(distgit_name)

  

          self.modulemd = ModulesLogic.yaml2modulemd(yaml)

          ModulesLogic.set_defaults_for_optional_params(self.modulemd, filename=filename)
@@ -179,12 +181,8 @@

      def get_clone_url(self, pkgname, rpm):

          if rpm.peek_repository():

              return rpm.peek_repository()

-         return self.default_distgit.format(pkgname=pkgname)

  

-     @property

-     def default_distgit(self):

-         # @TODO move to better place

-         return "https://src.fedoraproject.org/rpms/{pkgname}"

+         return self.distgit.package_clone_url(pkgname)

  

  

  class ModulemdGenerator(object):

@@ -9,6 +9,7 @@

  from sqlalchemy import outerjoin

  from sqlalchemy.ext.associationproxy import association_proxy

  from sqlalchemy.orm import column_property, validates

+ from sqlalchemy.event import listens_for

  from six.moves.urllib.parse import urljoin

  from libravatar import libravatar_url

  import zlib
@@ -1667,3 +1668,36 @@

      waiting = db.Column(db.Integer)

      success = db.Column(db.Integer)

      failed = db.Column(db.Integer)

+ 

+ 

+ class DistGitInstance(db.Model):

+     """ Dist-git instances, e.g. Fedora/CentOS/RHEL/ """

+ 

+     # numeric id, not used ATM

+     id = db.Column(db.Integer, primary_key=True)

+ 

+     # case sensitive identificator, e.g. 'fedora'

+     name = db.Column(db.String(50), nullable=False, unique=True)

+ 

+     # e.g. 'https://src.fedoraproject.org'

+     clone_url = db.Column(db.String(100), nullable=False)

+ 

+     # e.g. 'rpms/{pkgname}', needs to contain {pkgname} to be expanded later

+     clone_package_uri = db.Column(db.String(100), nullable=False)

+ 

+     # for UI form ordering, higher number means higher priority

+     priority = db.Column(db.Integer, default=100, nullable=False)

+ 

+     def package_clone_url(self, pkgname):

+         url = '/'.join([self.clone_url, self.clone_package_uri])

+         return url.format(pkgname=pkgname)

+ 

+ 

+ @listens_for(DistGitInstance.__table__, 'after_create')

+ def insert_fedora_distgit(*args, **kwargs):

I don't know why my comment disappeared:

I didn't know, that default table values can be created like this. Thanks for learning :-). However, we already have this in migration, so I guess there is no reason for this code?

+     db.session.add(DistGitInstance(

+         name="fedora",

+         clone_url="https://src.fedoraproject.org",

+         clone_package_uri="rpms/{pkgname}",

+     ))

+     db.session.commit()

@@ -21,14 +21,15 @@

  @file_upload()

  def build_module(ownername, projectname):

      copr = get_copr(ownername, projectname)

-     form = forms.ModuleBuildForm(meta={'csrf': False})

+     form = forms.get_module_build_form(meta={'csrf': False})

      if not form.validate_on_submit():

          raise BadRequest(form.errors)

  

      facade = None

      try:

          mod_info = ModuleProvider.from_input(form.modulemd.data or form.scmurl.data)

-         facade = ModuleBuildFacade(flask.g.user, copr, mod_info.yaml, mod_info.filename)

+         facade = ModuleBuildFacade(flask.g.user, copr, mod_info.yaml,

+                                    mod_info.filename, form.distgit.data)

          module = facade.submit_build()

          db.session.commit()

          return flask.jsonify(to_dict(module))

@@ -16,6 +16,7 @@

  from coprs import models

  from coprs import cache

  from coprs.logic.coprs_logic import BranchesLogic

+ from coprs.logic.dist_git_logic import DistGitLogic

  

  from unittest import mock

  
@@ -71,6 +72,10 @@

          for tbl in reversed(self.db.metadata.sorted_tables):

              self.db.engine.execute(tbl.delete())

  

+         # This table has after_create() hook with default data in models.py, so

+         # we actually need to drop it so the setup_method() re-creates the data

+         self.db.engine.execute('drop table dist_git_instance')

+ 

          self.rmodel_TSE_coprs_general_patcher.stop()

          self.app.config = self.original_config.copy()

          cache.clear()
@@ -567,6 +572,23 @@

          self.batch3 = models.Batch()

          self.batch4 = models.Batch()

  

+     @pytest.fixture

+     def f_other_distgit(self):

+         self.dg0 = DistGitLogic.get_with_default()

+         self.dg1 = models.DistGitInstance(

+             name='test',

+             clone_url='git://example.com',

+             clone_package_uri='some/uri/{pkgname}/git',

+             priority='80',

+         )

+         self.dg2 = models.DistGitInstance(

+             name='prioritized',

+             clone_url='git://prio.com',

+             clone_package_uri='some/other/uri/{pkgname}/git',

+             priority='120',

+         )

+         self.db.session.add_all([self.dg1, self.dg2])

+ 

      def request_rest_api_with_auth(self, url,

                                     login=None, token=None,

                                     content=None, method="GET",

@@ -127,6 +127,28 @@

          assert {chroot.name for chroot in self.c1.active_chroots} == set(fedora_chroots)

          assert set(facade.platform_chroots) == {"fedora-22-i386", "fedora-22-x86_64"}

  

+     def test_distgit_option(self, f_users, f_coprs, f_mock_chroots_many,

+                             f_builds, f_modules, f_other_distgit, f_db):

+         pkg1 = Modulemd.ComponentRpm(name="foo", rationale="foo package")

+         generator = ModulemdGenerator(name="testmodule", stream="master",

+                                       version=123, summary="some summary")

+ 

+         # check default distgit

+         facade = ModuleBuildFacade(self.u1, self.c1, generator.generate(),

+                                    "testmodule.yaml")

+         facade.add_builds({"foo": pkg1}, self.m1)

+         assert facade.distgit.id == self.dg2.id

+         assert facade.get_clone_url('foo', pkg1) == \

+             'git://prio.com/some/other/uri/foo/git'

+ 

+         # check explicit distgit

+         facade = ModuleBuildFacade(self.u1, self.c1, generator.generate(),

+                                    "testmodule.yaml", distgit_name='test')

+         facade.add_builds({"foo": pkg1}, self.m1)

+         assert facade.distgit == self.dg1

+         assert facade.get_clone_url('foo', pkg1) == \

+             'git://example.com/some/uri/foo/git'

+ 

  

  class TestModulemdGenerator(CoprsTestCase):

      config = {"DIST_GIT_URL": "http://distgiturl.org"}

@@ -0,0 +1,66 @@

+ import os

+ import mock

+ import pytest

+ import tempfile

+ import shutil

+ from json import loads

+ from copr.v3 import ModuleProxy

+ 

+ 

+ class TestModuleProxy(object):

+     config_auth = {

+         "copr_url": "http://copr",

+         "login": "test_api_login",

+         "token": "test_api_token",

+     }

+ 

+     def setup_method(self, method):

+         self.tmpdir = tempfile.mkdtemp(prefix='test-python-copr-modules')

+         self.yaml_file = os.path.join(self.tmpdir, 'test.yaml')

+         with open(self.yaml_file, 'w') as f:

+             f.write("")

+ 

+     def teardown_method(self, method):

+         shutil.rmtree(self.tmpdir)

+ 

+     @pytest.mark.parametrize('distgit_opt', [None, 'fedora'])

+     @mock.patch('copr.v3.requests.requests.request')

+     def test_module_dist_git_choice_url(self, request, distgit_opt):

+         proxy = ModuleProxy(self.config_auth)

+         proxy.build_from_url('owner', 'project', 'http://test.yaml',

+                              distgit=distgit_opt)

+ 

+         assert len(request.call_args_list) == 1

+         call = request.call_args_list[0]

+         kwargs = call[1]

+         json = kwargs['json']

+         if distgit_opt is None:

+             assert 'distgit' not in json

+         else:

+             assert json['distgit'] == distgit_opt

+ 

+         assert json['scmurl'] == 'http://test.yaml'

+ 

+     @pytest.mark.parametrize('distgit_opt', [None, 'fedora'])

+     @mock.patch('copr.v3.requests.requests.request')

+     def test_module_dist_git_choice_upload(self, request, distgit_opt):

+         proxy = ModuleProxy(self.config_auth)

+         proxy.build_from_file('owner', 'project',

+                               self.yaml_file,

+                               distgit=distgit_opt)

+ 

+         assert len(request.call_args_list) == 1

+         call = request.call_args_list[0]

+         kwargs = call[1]

+         json = kwargs['json']

+ 

+         assert json is None

+ 

+         # ('json', 'jsonstr', 'application/json')

+         json_encoded = kwargs['data'].encoder.fields['json']

+         json = loads(json_encoded[1])

+ 

+         if distgit_opt is None:

+             assert json is None

+         else:

+             assert json['distgit'] == distgit_opt

@@ -7,7 +7,8 @@

  

  class ModuleProxy(BaseProxy):

  

-     def build_from_url(self, ownername, projectname, url, branch="master"):

+     def build_from_url(self, ownername, projectname, url, branch="master",

+                        distgit=None):

          """

          Build a module from a URL pointing to a modulemd YAML file

  
@@ -26,12 +27,14 @@

              "scmurl": url,

              "branch": branch,

          }

+         if distgit is not None:

+             data["distgit"] = distgit

I think you can send it unconditionally, it should be filtered when None

Sure, but I somewhat dislike sending over network more than I have to. I
originally wanted to send distgit = None, but this variant seems more
nice to me (considering that we should support both variants on frontend
side anyways).

No hard requirement/preference on my side on this particular thing
though... so you tell :)

          request = Request(endpoint, api_base_url=self.api_base_url, method=POST,

                            params=params, data=data, auth=self.auth)

          response = request.send()

          return munchify(response)

  

-     def build_from_file(self, ownername, projectname, path):

+     def build_from_file(self, ownername, projectname, path, distgit=None):

          """

          Build a module from a local modulemd YAML file

  
@@ -49,7 +52,10 @@

          files = {

              "modulemd": (os.path.basename(f.name), f, "application/x-rpm")

          }

+         data = None

+         if distgit is not None:

+             data = {"distgit": distgit}

          request = FileRequest(endpoint, api_base_url=self.api_base_url, method=POST,

-                               params=params, files=files, auth=self.auth)

+                               params=params, files=files, data=data, auth=self.auth)

Have you tested this? I am not sure whether data somehow doesn't collide with files. But otherwise looks good.

Yes, tested -- even unit tested.

          response = request.send()

          return munchify(response)

Before, any module build from provided modulemd.yaml cloned packages
from Fedora's dist-git (if otherwise unspecified per-package in
modulemd.yaml).

Newly, there's a commandline option/api option to pick a concrete
instance of dist-git. New dist-git instances can be added dynamically
by manual edit of 'dist_git_instance' table.

The reason for this patch is that the modulemd files that are stored in
dist-git usually don't self-reference the dist-git instance they are
stored in. This caused issues in Red Hat internal copr instance where
we don't self-reference internal dist-git. Now, when properly
configured, we can do copr build-module ... --distgit redhat.

Relates: rhbz#1771699

Metadata Update from @praiskup:
- Pull-request tagged with: needs-work

4 years ago

Metadata Update from @praiskup:
- Pull-request untagged with: needs-work
- Pull-request tagged with: needs-tests

4 years ago

rebased onto 487d77eb14d505fc6f59dd0aff609145291a01b8

4 years ago

rebased onto ee488881a2443c4586aad2c4727d6c60c689b001

4 years ago

rebased onto 6557bf74ff59be46ddc30bd3d8f2f3d0d1b472d0

4 years ago

Metadata Update from @praiskup:
- Pull-request untagged with: needs-tests

4 years ago

rebased onto e771776ea7e94157b02c390f1f2db0a52d812569

4 years ago

Metadata Update from @praiskup:
- Request assigned

4 years ago

Please, have a numeric primary key. I think it is never a bad idea to have one.

We seem not to use the arguments. Do we need to wrap the ModuleBuildForm with this function then?

This is IMHO weird comment style. AFAIK comments are # foo and docstrings """ foo """.

Hmm, I was about to argue that this would be pointless ... adding another
unique key (numeric) when we need the unique key on string. But perhaps
we don't need primary key at all here, only unique one (named name, e.g.).
WDYT?

I didn't know, that default table values can be created like this. Thanks for learning :-). However, we already have this in migration, so I guess there is no reason for this code?

We don't (yet?).

But yes, we need it ... as we need the other "Factories". You need to
fill the class fields from database query ... and if not wrapped by this method,
the fields would be filled too soon (at the time python processes
the class, and at that time e.g. isn't clear whether the db exists at
all) or too late (in constructor, the class was already read by python
and adding fields later raises UnboundField exception, or something
ugly as that).

I think you can send it unconditionally, it should be filtered when None

This is doc string, we should put this into guidelines then if you
dislike saving the additional """ two lines ....

This is valid doc string as any str() instance at the beginning of the method, afaik.

# comment is different animal :)

Have you tested this? I am not sure whether data somehow doesn't collide with files. But otherwise looks good.

But perhaps we don't need primary key at all here

I must disagree, having a primary key is the first normal form. Honestly, I would prefer

sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=50), nullable=False),
sa.PrimaryKeyConstraint('id')
sa.UniqueConstraint('name')

I have never in my life regretted that "Ah man, why did I create that integer primary key". I wish I could say the same the other way around.

the fields would be filled too soon

True, I didn't realize that. +1 for function then.

True, In python3 it doesn't matter whether you use class Foo or class Foo(object), it will create a new-style class in both cases. Having (object) is therefore a necessity only when using both python2 and 3. So I guess, we don't need it here.

I would argue that it is a valid docstring. There is a PEP 257 -- Docstring Conventions and it doesn't mention such case. See One-line Docstrings

Triple quotes are used even though the string fits on one line. This makes it easy to later expand it.

Meh, the same document says:

A docstring is a string literal that occurs as the first statement in a module, function, class, or method definition. Such a docstring becomes the __doc__ special attribute of that object.

It definitely is a valid docstring, and then it is matter of taste ... I'm ok to change it
and never add it again, but we should put this into guidelines. Are you OK to take
a look at this?

I won't waste more our time here, and do what you prefer, rewriting this
PR will cost less efforts...

But this is again material for project guidelines/best practices.

I dislike doing something which doesn't make sense (in this case two
indexes, for no purpose - nothing which can not be migrated later);

if we had this case in guidelines - I would have to comply to them,
and I could not regret my PR decisions in future (because it was a
guideline which made me to do that, not my random ad-hoc wrong
opinion).

I would stick to """, we have it all of our codebase, so ...

but we should put this into guidelines. Are you OK to take a look at this?

I can write some proposal into https://docs.pagure.org/copr.copr/developer_documentation.html

Thanks! ... I'd slightly prefer CONTRIBUTE file in git root, and perhaps include it into ./doc somehow (if you wish, or just reference it from ./doc). The file is somewhat standard
place where every random contributor looks first.

On Thursday, January 30, 2020 1:39:20 PM CET Jakub Kadl=C4=8D=C3=ADk wrote:

I can write some proposal into
https://docs.pagure.org/copr.copr/developer_documentation.html

I filled this for now, we can put there more topics ... e.g. the
'class Class(object)' thing:
https://pagure.io/copr/copr/issue/1242

I thought that as well, and then I had to add this ... (for testsuite, and for
freshly created copr instances).

Sure, but I somewhat dislike sending over network more than I have to. I
originally wanted to send distgit = None, but this variant seems more
nice to me (considering that we should support both variants on frontend
side anyways).

No hard requirement/preference on my side on this particular thing
though... so you tell :)

Yes, tested -- even unit tested.

rebased onto 15124ec7564411cc191db67d748eec8dc5eb9036

4 years ago

Updated, please take another look.

I don't know why my comment disappeared:

I didn't know, that default table values can be created like this. Thanks for learning :-). However, we already have this in migration, so I guess there is no reason for this code?

But yea, otherwise looks good to me. Feel free to merge.

I don't know why my comment disappeared:

I didn't know, that default table values can be created like this. Thanks for
learning :-). However, we already have this in migration, so I guess there is
no reason for this code?

This was meant to be my answer:

I thought that as well, and then I had to add this ... (for testsuite, and for
freshly created copr instances).

IOW there are situations when only
frontend/coprs_frontend/commands/create_db.py:create_db() is called, without
executing migrations. This could be fixed as separate thing I guess.

rebased onto be56551

4 years ago

Pull-Request has been merged by praiskup

4 years ago
Metadata