#3 Allow passing buildrequires to MBS.get_modules_with_requires
Merged 4 months ago by cqi. Opened 4 months ago by cqi.
cqi/ursa-major remove-module  into  master

@@ -0,0 +1,67 @@ 

+ {

+   "items": [

+     {

+       "state_name": "ready",

+       "stream": "rhel-8.0",

+       "owner": "qwan",

+       "id": 604,

+       "state_reason": null,

+       "time_completed": "2018-05-07T10:25:38Z",

+       "build_context": "3044bea5ac56e79502c45b08536a51d5a9e0a88e",

+       "state": 5,

+       "version": "20180507102412",

+       "koji_tag": "module-e5c3306aa4e9db10",

+       "scmurl": "git://git.example.com/modules/testmodule?#d8cf5ee02cbbd31615bba3b47c85cca6a8ae54a8",

+       "rebuild_strategy": "only-changed",

+       "runtime_context": "3044bea5ac56e79502c45b08536a51d5a9e0a88e",

+       "state_url": "/module-build-service/1/module-builds/604",

+       "component_builds": [

+           13220,

+           13221,

+           13222

+       ],

+       "name": "testmodule",

+       "time_submitted": "2018-05-07T10:24:45Z",

+       "time_modified": "2018-05-07T10:25:55Z",

+       "modulemd": "document: modulemd\nversion: 1\ndata:\n  name: testmodule\n  stream: rhel-8.0\n  version: 20180507102412\n  context: c2c572ec\n  summary: A test module in all its beautiful beauty.\n  description: This module demonstrates how to write simple modulemd files And can\n    be used for testing the build and release pipeline.\n  license:\n    module:\n    - MIT\n  dependencies:\n    buildrequires:\n      platform: el8\n    requires:\n      platform: el8\n  references:\n    community: https://fedoraproject.org/wiki/Modularity\n    documentation: https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules\n    tracker: https://taiga.fedorainfracloud.org/project/modularity\n  xmd:\n    mbs:\n      buildrequires:\n        platform:\n          filtered_rpms: []\n          ref: virtual\n          stream: el8\n          version: '1'\n      commit: d8cf5ee02cbbd31615bba3b47c85cca6a8ae54a8\n      requires:\n        platform:\n          filtered_rpms: []\n          ref: virtual\n          stream: el8\n          version: '1'\n      rpms:\n        acl:\n          ref: a3d9dd4e406ef5e8e50b80277156cb6759462cf3\n        attr:\n          ref: 0ec2a94e4af3bcd17fead0318bf7c1d2235992e2\n        tar:\n          ref: 5a03229e4ff9450dd0cdd524cb3490a69b21c76b\n      scmurl: git://git.example.com/modules/testmodule?#d8cf5ee02cbbd31615bba3b47c85cca6a8ae54a8\n  profiles:\n    default:\n      rpms:\n      - tangerine\n  api:\n    rpms:\n    - perl-Tangerine\n    - tangerine\n  components:\n    rpms:\n      attr:\n        rationale: A build dependency of acl\n        repository: git://git.example.com/rpms/attr\n        ref: rhel-8.0\n        cache: http://git.example.com/repo/pkgs/attr\n      tar:\n        rationale: The main component of this module\n        buildorder: 2\n        repository: git://git.example.com/rpms/tar\n        ref: rhel-8.0\n        cache: http://git.example.com/repo/pkgs/tar\n      acl:\n        rationale: A build dependency of tar\n        buildorder: 1\n        repository: git://git.example.com/rpms/acl\n        ref: rhel-8.0\n        cache: http://git.example.com/repo/pkgs/acl\n",

+       "context": "c2c572ec"

+     },

+     {

+       "comment": "is for commenting this second fake module build info. Please not that, this module build buildrequires el8.1.0, that is different from the above one which has buildrequires el8.",

+       "state_name": "ready",

+       "stream": "rhel-8.0",

+       "owner": "qwan",

+       "id": 605,

+       "state_reason": null,

+       "time_completed": "2018-05-07T10:25:38Z",

+       "build_context": "3044bea5ac56e79502c45b08536a51d5a9e0a88e",

+       "state": 5,

+       "version": "20180507102412",

+       "koji_tag": "module-e5c3306aa4e9db10",

+       "scmurl": "git://git.example.com/modules/testmodule?#d8cf5ee02cbbd31615bba3b47c85cca6a8ae54a8",

+       "rebuild_strategy": "only-changed",

+       "runtime_context": "3044bea5ac56e79502c45b08536a51d5a9e0a88e",

+       "state_url": "/module-build-service/1/module-builds/604",

+       "component_builds": [

+           13220,

+           13221,

+           13222

+       ],

+       "name": "testmodule",

+       "time_submitted": "2018-05-07T10:24:45Z",

+       "time_modified": "2018-05-07T10:25:55Z",

+       "modulemd": "document: modulemd\nversion: 1\ndata:\n  name: testmodule\n  stream: rhel-8.0\n  version: 20180507102412\n  context: c2c572ec\n  summary: A test module in all its beautiful beauty.\n  description: This module demonstrates how to write simple modulemd files And can\n    be used for testing the build and release pipeline.\n  license:\n    module:\n    - MIT\n  dependencies:\n    buildrequires:\n      platform: el8.1.0\n    requires:\n      platform: el8\n  references:\n    community: https://fedoraproject.org/wiki/Modularity\n    documentation: https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules\n    tracker: https://taiga.fedorainfracloud.org/project/modularity\n  xmd:\n    mbs:\n      buildrequires:\n        platform:\n          filtered_rpms: []\n          ref: virtual\n          stream: el8\n          version: '1'\n      commit: d8cf5ee02cbbd31615bba3b47c85cca6a8ae54a8\n      requires:\n        platform:\n          filtered_rpms: []\n          ref: virtual\n          stream: el8\n          version: '1'\n      rpms:\n        acl:\n          ref: a3d9dd4e406ef5e8e50b80277156cb6759462cf3\n        attr:\n          ref: 0ec2a94e4af3bcd17fead0318bf7c1d2235992e2\n        tar:\n          ref: 5a03229e4ff9450dd0cdd524cb3490a69b21c76b\n      scmurl: git://git.example.com/modules/testmodule?#d8cf5ee02cbbd31615bba3b47c85cca6a8ae54a8\n  profiles:\n    default:\n      rpms:\n      - tangerine\n  api:\n    rpms:\n    - perl-Tangerine\n    - tangerine\n  components:\n    rpms:\n      attr:\n        rationale: A build dependency of acl\n        repository: git://git.example.com/rpms/attr\n        ref: rhel-8.0\n        cache: http://git.example.com/repo/pkgs/attr\n      tar:\n        rationale: The main component of this module\n        buildorder: 2\n        repository: git://git.example.com/rpms/tar\n        ref: rhel-8.0\n        cache: http://git.example.com/repo/pkgs/tar\n      acl:\n        rationale: A build dependency of tar\n        buildorder: 1\n        repository: git://git.example.com/rpms/acl\n        ref: rhel-8.0\n        cache: http://git.example.com/repo/pkgs/acl\n",

+       "context": "c2c572ec"

+     }

+   ],

+   "meta": {

+     "prev": null,

+     "last": null,

+     "next": null,

+     "total": 1,

+     "per_page": 10,

+     "first": "https://mbs.example.com/module-build-service/1/module-builds/?per_page=10&page=1&name=testmodule",

+     "pages": 1,

+     "page": 1

+   }

+ }

file modified
+32

@@ -108,3 +108,35 @@ 

  

          mbs = MBS('http://mbs.example.com')

          self.assertFalse(mbs.module_has_requires(604, {'platform': 'f28'}))

+ 

+     def test_get_modules_with_requires(self):

+         mbs_resp_data = self.load_json_from_file('test_get_modules_with_requires.json')

+         mbs = MBS('http://mbs.example.com')

+ 

+         # name, stream, requires, buildrequires, expected module build ids

+         test_matrix = (

+             ('testmodule', 'rhel-8.0', None, None, [604, 605]),

+             ('testmodule', 'rhel-8.0', {'platform': 'el8'}, None, [604, 605]),

+             ('testmodule', 'rhel-8.0', None, {'platform': 'el8'}, [604]),

+             ('testmodule', 'rhel-8.0', {'platform': 'el8'}, {'platform': 'el8'}, [604]),

+             ('testmodule', 'rhel-8.0', {'platform': 'f30'}, None, []),

+             ('testmodule', 'rhel-8.0', {'platform': 'f30'}, {'platform': 'f30'}, []),

+             ('testmodule', 'rhel-8.0', None, {'platform': 'f30'}, []),

+         )

+ 

+         for name, stream, requires, buildrequires, expected_build_ids in test_matrix:

+             with mock.patch('ursa_major.mbs.requests.get') as get:

+                 get.return_value = MockResponse(mbs_resp_data, 200)

+ 

+                 module_builds = mbs.get_modules_with_requires(

+                     name=name,

+                     stream=stream,

+                     requires=requires,

+                     buildrequires=buildrequires)

+ 

+                 get.assert_called_once_with(

+                     mbs.module_builds_api_url,

+                     params={'name': name, 'stream': stream, 'verbose': 'true'})

+ 

+                 build_ids = sorted(build_info['id'] for build_info in module_builds)

+                 assert expected_build_ids == build_ids

@@ -145,7 +145,7 @@ 

          requires = module_config.get('requires', {})

          if requires:

              modules = self.mbs.get_modules_with_requires(

-                 requires, name=name, stream=stream, state=5)

+                 requires=requires, name=name, stream=stream, state=5)

          else:

              modules = self.mbs.get_modules(

                  auto_next_page=True, name=name, stream=stream, state=5)

file modified
+22 -7

@@ -28,7 +28,7 @@ 

  except ImportError:

      import urlparse

  

- from ursa_major.utils import load_mmd, mmd_has_requires

+ from ursa_major.utils import load_mmd, mmd_has_requires, mmd_has_buildrequires

  

  

  class MBS(object):

@@ -93,15 +93,25 @@ 

          mmd = self.get_module_mmd(build_id)

          return mmd_has_requires(mmd, requires)

  

-     def get_modules_with_requires(self, requires, **kwargs):

+     def get_modules_with_requires(self, **kwargs):

          """

          Query MBS modules with specified query parameters.

  

-         :param requires: requires in a module

-         :param kwargs: query parameters in keyword arguments for

-                        listing module builds

-         :return: a list of modules

+         :param dict kwargs: query parameters in keyword arguments for listing

+             module builds. Modules can be filtered by either requires or

+             buildrequires or both by passing ``requires`` and ``buildrequires``,

+             which accepts a mapping containing requirements, for example,

+             ``{"platform": "f30"}``.

+         :return: a list of mappings each of them presents a module returned

+             from MBS without modified.

+         :rtype: list[dict]

          """

+         # MBS does not support filtering module builds based on requires and

+         # buildrequires. Pop either of them and filter modules later by

+         # ourselves.

+         requires = kwargs.pop('requires', None)

+         buildrequires = kwargs.pop('buildrequires', None)

+ 

          # we need modulemd in result, so set verbose to true

          kwargs.update({'verbose': 'true'})

          modules = self.get_modules(auto_next_page=True, **kwargs)

@@ -109,6 +119,11 @@ 

          matched = []

          for module in modules:

              mmd = load_mmd(module['modulemd'])

-             if mmd_has_requires(mmd, requires):

+             b = True

+             if requires:

+                 b = b and mmd_has_requires(mmd, requires)

+             if buildrequires:

+                 b = b and mmd_has_buildrequires(mmd, buildrequires)

+             if b:

                  matched.append(module)

          return matched