#261 Add experimental /api/2/composes to clean-up and rationalize the REST API.
Opened 4 months ago by jkaluza. Modified 3 days ago
jkaluza/odcs api_v2  into  master

@@ -30,6 +30,7 @@ 

      RAW_CONFIG = 5

      BUILD = 6

      PUNGI_COMPOSE = 7

+     ODCS_V2_TEMPLATE = 8

  

  

  PUNGI_SOURCE_TYPE_NAMES = {

@@ -49,6 +50,8 @@ 

      # Generates compose using the same set of RPMs as in existing 3rd party

      # Pungi compose.

      "pungi_compose": PungiSourceType.PUNGI_COMPOSE,

+     # Source type defining the ODCS v2 API input format.

+     "odcs_v2_template": PungiSourceType.ODCS_V2_TEMPLATE,

  }

  

  INVERSE_PUNGI_SOURCE_TYPE_NAMES = {

file added
+142

@@ -0,0 +1,142 @@ 

+ ========================

+ ODCS REST API version 2

+ ========================

+ 

+ POST /api/2/composes

+ ====================

+ 

+ Generates new ODCS compose. Accepts JSON in following format:

+ 

+ .. code:: json

+ 

+     {

+         "sources": [

+             {...},

+             {...}

+         ]

+         "sigkeys": ["sigkey1", "sigkey2"],

+         "flags": ["flag1", "flag2"],

+         "arches": ["x86_64", "i686"],

+         "multilib_method": ["runtime", "devel"],

+         "multilib_arches": ["x86_64"],

+         "lookaside_repos": ["http://localhost/repo/$basearch"],

+         "module_defaults_url": "git://localhost/module-defaults",

+         "module_defaults_commit": "master",

+     }

+ 

+ Only the ``sources`` object is strictly required. All the other objects are optional.

+ 

+ The ``sources`` object contains a list of dictionaries defining the sources of RPMs for compose. There are following types of ``sources``:

+ 

+ **Source: rpm_koji_tags**

+ 

+ Includes the RPMs found in Koji builds tagged in particular Koji tags.

+ 

+ .. code:: json

+ 

+     {

+         "type": "rpm_koji_tags",

+         "tags": ["f26"],

+         "rpms": ["mod_ssl", "logrotate"]

+     }

+ 

+ Possible options:

+ 

+ - **tags** (required) - List of Koji tags.

+ - **rpms** (optional) - List of RPM names from the Koji builds in a tag which should be include in a compose. If not set, all RPMs are included.

+ 

+ **Source: module_koji_tags**

+ 

+ Includes the Modules found in Koji module builds tagged in particular Koji tags.

+ 

+ .. note::

+ 

+   Because of implementation details, this currently cannot be combined with ``module_koji_builds``.

+ 

+ .. code:: json

+ 

+     {

+         "type": "module_koji_tags",

+         "tags": ["f26"],

+     }

+ 

+ Possible options:

+ 

+ - **tags** (required) - List of Koji tags.

+ 

+ **Source: rpm_koji_builds**

+ 

+ Includes the Koji builds.

+ 

+ .. code:: json

+ 

+     {

+         "type": "rpm_koji_builds",

+         "builds": ["httpd-2.4.11-1", "logroatte-3.8.0-1"],

+         "rpms": ["mod_ssl", "logrotate"]

+     }

+ 

+ Possible options:

+ 

+ - **builds** (required) - List of Koji builds.

+ - **rpms** (optional) - List of RPM names from the Koji builds which should be include in a compose. If not set, all RPMs are included.

+ 

+ **Source: module_koji_builds**

+ 

+ Includes the Koji builds.

+ 

+ .. note::

+ 

+   Because of implementation details, this currently cannot be combined with ``module_koji_tags``.

+ 

+ .. code:: json

+ 

+     {

+         "type": "module_koji_builds",

+         "builds": ["testmodule:stream", "httpd:24:1"],

+     }

+ 

+ Possible options:

+ 

+ - **builds** (required) - List of modules defined as ``name:stream``, ``name:stream:version`` or ``name:stream:version:context``.

+ 

+ **Source: pulp_content_sets**

+ 

+ Generates the compose with .repo file containing links to existing Pulp repositories defined by list of content-sets.

+ 

+ .. note::

+ 

+   This cannot be combined with any other source type.

+ 

+ .. code:: json

+ 

+     {

+         "type": "pulp_content_sets",

+         "content_sets": ["content-set-1", "content-set-2"],

+     }

+ 

+ Possible options:

+ 

+ - **content_sets** (required) - List of content-sets.

Optional: Can we provide more info here?

+ 

+ **Source: pungi_raw_config**

+ 

+ Generates the compose using the raw Pungi configuration. To use the raw Pungi configuration, it is needed to configure the URL to git repository with the configuration server-side.

+ 

+ .. note::

+ 

+   This cannot be combined with any other source type.

+ 

+ .. code:: json

+ 

+     {

+         "type": "pungi_raw_config",

+         "name": "raw_config_name",

+         "commit": "master",

Optional: How about renaming this to commit_source to make it more intuitive?

+     }

+ 

+ Possible options:

+ 

+ - **name** (required) - Name of the raw configuration as configured in ODCS server.

+ - **commit** (required) - Commit hash or branch name defining the version of raw config.

+ 

file modified
+6 -2

@@ -37,7 +37,11 @@ 

  pkgset_source = 'koji'

  

  {%- if config.koji_tag %}

- pkgset_koji_tag = '{{ config.koji_tag }}'

+ pkgset_koji_tag = [

+ {%- for tag in config.koji_tag %}

+     '{{ tag }}',

+ {%- endfor %}

+ ]

  {%- else %}

  pkgset_koji_tag = ""

  {%- endif %}

@@ -68,7 +72,7 @@ 

  {%- endif %}

  

  

- {%- if config.source_type_str in ["tag", "build"] and not config.packages %}

+ {%- if config.include_additional_packages %}

  # In case no package is requested, include all of them.

  additional_packages = [

      ('^Temporary$', {

@@ -97,6 +97,7 @@ 

        - source_types - compose.source_type

        - sources - compose.source

        - arches - compose.arches

+       - v2_source_types - source types from the ODCS_V2_TEMPLATE `sources`.

  

      The decision whether the user is allowed or not is done based on

      conf.allowed_clients value.

@@ -157,11 +158,15 @@ 

          iterator = enumerate(dict_or_list)

      for k, v in iterator:

          if isinstance(v, dict):

-             # Allow only dict with "source" key name in first level of

-             # json object.

-             if level != 0 or k not in ["source"]:

+             # The dict is only allowed in these cases:

+             #   1) It is the root dict (`level == 0`).

+             #   2) It is the child of "source" sub-dict.

+             #   3) It is the included in list in "sources" dict.

+             # Every other use is disallowed and raises an exception.

+             if ((level != 0 or k not in ["source"]) and

+                     (level != 1 or last_dict_key != "sources")):

                  raise ValueError(

-                     "Only 'source' key is allowed to contain dict.")

+                     "Key \"%s\" is not allowed to contain dict." % k)

              validate_json_data(v, level + 1, k)

          elif isinstance(v, list):

              validate_json_data(v, level + 1, k)

file modified
+81 -34

@@ -307,6 +307,54 @@ 

      return koji_session.tagChangedSinceEvent(koji_event, tags)

  

  

+ def _get_koji_event(tag):

+     """

+     Returns the actual Koji event id. In case we have already generated

+     compose from the input Koji `tag` and it have not changed since

+     that time, this method will return the same Koji event id as used

+     in the already generated compose. That way, such compose can be

+     reused.

+     """

+     global LAST_EVENTS_CACHE

+     koji_session = create_koji_session()

+     if tag not in LAST_EVENTS_CACHE:

+         event_id = int(koji_session.getLastEvent()['id'])

+     elif tag_changed(koji_session, tag, LAST_EVENTS_CACHE[tag]):

+         event_id = int(koji_session.getLastEvent()['id'])

+     else:

+         event_id = LAST_EVENTS_CACHE[tag]

+         log.info('Reuse koji event %s to generate compose from tag %s',

+                  event_id, tag)

+     # event_id could be a new koji event ID. Cache it for next potential

+     # reuse for same tag.

+     LAST_EVENTS_CACHE[tag] = event_id

+     return event_id

+ 

+ 

+ def _resolve_modules(compose, modules):

+     """

+     Takes the list of `modules` in n:s, n:s:v or n:s:v:c format and

+     resolves them to n:s:v:c. Depending on the compose flags, also

+     resolve the dependencies between modules and pull them in.

+     """

+     # Resolve the latest release of modules which do not have the release

+     # string defined in the compose.source.

+     mbs = odcs.server.mbs.MBS(conf)

+ 

+     specified_mbs_modules = []

+     for module in modules:

+         specified_mbs_modules += mbs.get_latest_modules(module)

+ 

+     expand = not compose.flags & COMPOSE_FLAGS["no_deps"]

+     new_mbs_modules = mbs.validate_module_list(specified_mbs_modules, expand=expand)

+ 

+     uids = sorted(

+         "{name}:{stream}:{version}:{context}".format(**m)

+         for m in new_mbs_modules

+         if m['name'] not in conf.base_module_names)

+     return uids

+ 

+ 

  def resolve_compose(compose):

      """

      Resolves various general compose values to the real ones. For example:

@@ -323,44 +371,15 @@ 

          revision = e.find("{http://linux.duke.edu/metadata/repo}revision").text

          compose.koji_event = int(revision)

      elif compose.source_type == PungiSourceType.KOJI_TAG:

-         global LAST_EVENTS_CACHE

-         koji_session = create_koji_session()

          # If compose.koji_event is set, it means that we are regenerating

          # previous compose and we have to respect the previous koji_event to

          # get the same results.

          if not compose.koji_event:

-             if compose.source not in LAST_EVENTS_CACHE:

-                 event_id = int(koji_session.getLastEvent()['id'])

-             elif tag_changed(koji_session,

-                              compose.source,

-                              LAST_EVENTS_CACHE[compose.source]):

-                 event_id = int(koji_session.getLastEvent()['id'])

-             else:

-                 event_id = LAST_EVENTS_CACHE[compose.source]

-                 log.info('Reuse koji event %s to generate compose %s from source %s',

-                          event_id, compose.id, compose.source)

-             compose.koji_event = event_id

-             # event_id could be a new koji event ID. Cache it for next potential

-             # reuse for same tag.

-             LAST_EVENTS_CACHE[compose.source] = event_id

+             compose.koji_event = _get_koji_event(compose.source)

      elif compose.source_type == PungiSourceType.MODULE:

- 

          # Resolve the latest release of modules which do not have the release

          # string defined in the compose.source.

-         mbs = odcs.server.mbs.MBS(conf)

-         modules = compose.source.split(" ")

- 

-         specified_mbs_modules = []

-         for module in modules:

-             specified_mbs_modules += mbs.get_latest_modules(module)

- 

-         expand = not compose.flags & COMPOSE_FLAGS["no_deps"]

-         new_mbs_modules = mbs.validate_module_list(specified_mbs_modules, expand=expand)

- 

-         uids = sorted(

-             "{name}:{stream}:{version}:{context}".format(**m)

-             for m in new_mbs_modules

-             if m['name'] not in conf.base_module_names)

+         uids = _resolve_modules(compose, compose.source.split(" "))

          compose.source = ' '.join(uids)

      elif compose.source_type == PungiSourceType.PUNGI_COMPOSE:

          external_compose = PungiCompose(compose.source)

@@ -391,6 +410,12 @@ 

              for rpm_nevra in rpms:

                  packages.add(productmd.common.parse_nvra(rpm_nevra)['name'])

          compose.packages = " ".join(packages)

+     elif compose.source_type == PungiSourceType.ODCS_V2_TEMPLATE:

+         if (compose.modular_koji_tags or compose.rpm_koji_tags) and not compose.koji_event:

+             compose.koji_event = _get_koji_event(compose.source)

+         if compose.module_builds:

+             nsvcs = _resolve_modules(compose, compose.module_builds.split(" "))

+             compose.module_builds = ' '.join(nsvcs)

  

  

  def get_reusable_compose(compose):

@@ -507,6 +532,24 @@ 

                        old_compose)

              continue

  

+         rpm_koji_tags = set(compose.rpm_koji_tags.split(" ")) \

+             if compose.rpm_koji_tags else set()

+         old_rpm_koji_tags = set(old_compose.rpm_koji_tags.split(" ")) \

+             if old_compose.rpm_koji_tags else set()

+         if rpm_koji_tags != old_rpm_koji_tags:

+             log.debug("%r: Cannot reuse %r - rpm_koji_tags not same", compose,

+                       old_compose)

+             continue

+ 

+         module_builds = set(compose.module_builds.split(" ")) \

+             if compose.module_builds else set()

+         old_module_builds = set(old_compose.module_builds.split(" ")) \

+             if old_compose.module_builds else set()

+         if module_builds != old_module_builds:

+             log.debug("%r: Cannot reuse %r - module_builds not same", compose,

+                       old_compose)

+             continue

+ 

          module_defaults_url = compose.module_defaults_url

          old_module_defaults_url = old_compose.module_defaults_url

          if module_defaults_url != old_module_defaults_url:

@@ -526,7 +569,8 @@ 

                        old_compose)

              continue

  

-         if compose.source_type == PungiSourceType.KOJI_TAG:

+         if (compose.source_type == PungiSourceType.KOJI_TAG or

+                 compose.rpm_koji_tags or compose.modular_koji_tags):

              # For KOJI_TAG compose, check that all the inherited tags by our

              # Koji tag have not changed since previous old_compose.

              koji_session = create_koji_session()

@@ -680,14 +724,17 @@ 

                                      builds=builds, flags=compose.flags,

                                      lookaside_repos=compose.lookaside_repos,

                                      modular_koji_tags=compose.modular_koji_tags,

-                                     module_defaults_url=compose.module_defaults_url)

+                                     module_defaults_url=compose.module_defaults_url,

+                                     rpm_koji_tags=compose.rpm_koji_tags,

+                                     module_builds=compose.module_builds)

              if compose.flags & COMPOSE_FLAGS["no_deps"]:

                  pungi_cfg.gather_method = "nodeps"

              if compose.flags & COMPOSE_FLAGS["no_inheritance"]:

                  pungi_cfg.pkgset_koji_inherit = False

  

          koji_event = None

-         if compose.source_type == PungiSourceType.KOJI_TAG:

+         if compose.source_type in [PungiSourceType.KOJI_TAG,

+                                    PungiSourceType.ODCS_V2_TEMPLATE]:

              koji_event = compose.koji_event

  

          old_compose = None

file modified
+6 -1

@@ -199,7 +199,12 @@ 

              'desc': 'Number of concurrent Pungi processes.'},

          'allowed_source_types': {

              'type': list,

-             'default': ["tag", "module", "build"],

+             'default': ["tag", "module", "build", "odcs_v2_template"],

+             'desc': 'Allowed source types.'},

+         'allowed_v2_source_types': {

+             'type': list,

+             'default': ["rpm_koji_tags", "module_koji_tags", "rpm_koji_builds",

+                         "module_koji_builds"],

              'desc': 'Allowed source types.'},

          'allowed_flags': {

              'type': list,

@@ -0,0 +1,28 @@ 

+ """empty message

+ 

+ Revision ID: 983a8eef6e82

+ Revises: e186faabdafe

+ Create Date: 2019-03-06 09:08:28.448908

+ 

+ """

+ 

+ # revision identifiers, used by Alembic.

+ revision = '983a8eef6e82'

+ down_revision = 'e186faabdafe'

+ 

+ from alembic import op

+ import sqlalchemy as sa

+ 

+ 

+ def upgrade():

+     # ### commands auto generated by Alembic - please adjust! ###

+     op.add_column('composes', sa.Column('module_builds', sa.String(), nullable=True))

+     op.add_column('composes', sa.Column('rpm_koji_tags', sa.String(), nullable=True))

+     # ### end Alembic commands ###

+ 

+ 

+ def downgrade():

+     # ### commands auto generated by Alembic - please adjust! ###

+     op.drop_column('composes', 'rpm_koji_tags')

+     op.drop_column('composes', 'module_builds')

+     # ### end Alembic commands ###

file modified
+13 -1

@@ -142,13 +142,19 @@ 

      multilib_method = db.Column(db.Integer)

      # White-space separated lookaside repository URLs.

      lookaside_repos = db.Column(db.String, nullable=True)

+     # Used by v2 ODCS API to store the list of Koji tags which would be normally

+     # stored in the 'source'.

+     rpm_koji_tags = db.Column(db.String, nullable=True)

+     # Module builds

+     module_builds = db.Column(db.String, nullable=True)

  

      @classmethod

      def create(cls, session, owner, source_type, source, results,

                 seconds_to_live, packages=None, flags=0, sigkeys=None,

                 koji_event=None, arches=None, multilib_arches=None,

                 multilib_method=None, builds=None, lookaside_repos=None,

-                modular_koji_tags=None, module_defaults_url=None):

+                modular_koji_tags=None, module_defaults_url=None,

+                rpm_koji_tags=None, module_builds=None):

          now = datetime.utcnow()

          compose = cls(

              owner=owner,

@@ -169,6 +175,8 @@ 

              lookaside_repos=lookaside_repos,

              modular_koji_tags=modular_koji_tags,

              module_defaults_url=module_defaults_url,

+             rpm_koji_tags=rpm_koji_tags,

+             module_builds=module_builds,

          )

          session.add(compose)

          return compose

@@ -203,6 +211,8 @@ 

              lookaside_repos=compose.lookaside_repos,

              modular_koji_tags=compose.modular_koji_tags,

              module_defaults_url=compose.module_defaults_url,

+             rpm_koji_tags=compose.rpm_koji_tags,

+             module_builds=compose.module_builds,

          )

          session.add(compose)

          return compose

@@ -333,6 +343,8 @@ 

              'lookaside_repos': self.lookaside_repos,

              'modular_koji_tags': self.modular_koji_tags,

              'module_defaults_url': self.module_defaults_url,

+             'rpm_koji_tags': self.rpm_koji_tags,

+             'module_builds': self.module_builds,

          }

  

      @staticmethod

file modified
+62 -9

@@ -32,6 +32,7 @@ 

  import random

  import string

  from productmd.composeinfo import ComposeInfo

+ from collections import namedtuple

  

  import odcs.server.utils

  from odcs.server import conf, log, db

@@ -112,11 +113,12 @@ 

  

  

  class PungiConfig(BasePungiConfig):

-     def __init__(self, release_name, release_version, source_type, source,

+     def __init__(self, release_name, release_version, source_type, source=None,

                   packages=None, arches=None, sigkeys=None, results=0,

                   multilib_arches=None, multilib_method=0, builds=None,

                   flags=0, lookaside_repos=None, modular_koji_tags=None,

-                  module_defaults_url=None):

+                  module_defaults_url=None, rpm_koji_tags=None,

+                  module_builds=None):

          self.release_name = release_name

          self.release_version = release_version

          self.bootable = False

@@ -127,6 +129,10 @@ 

          self.pkgset_koji_inherit = True

          self.lookaside_repos = lookaside_repos.split(" ") if lookaside_repos else []

          self.include_devel_modules = []

+         self.module_builds = module_builds.split(" ") if module_builds else []

+         self.koji_module_tags = None

+         self.module_defaults_url = []

+         self.include_additional_packages = False

          if arches:

              self.arches = arches

          else:

@@ -154,18 +160,21 @@ 

          if source_type == PungiSourceType.KOJI_TAG:

              self.koji_module_tags = modular_koji_tags.split(" ") if modular_koji_tags else []

              self.module_defaults_url = module_defaults_url.split(" ") if module_defaults_url else []

-             self.koji_tag = source

+             self.koji_tag = [source]

              self.gather_source = "comps"

              if self.koji_module_tags:

                  self.gather_method = "hybrid"

              else:

                  self.gather_method = "deps"

+             if not self.packages:

+                 self.include_additional_packages = True

          elif source_type == PungiSourceType.MODULE:

              self.koji_tag = None

              self.gather_source = "module"

              self.gather_method = "nodeps"

  

-             self._sort_out_devel_modules()

+             new_source = self._sort_out_devel_modules(self.source.split(" "))

+             self.source = " ".join(new_source)

  

              if self.packages:

                  raise ValueError("Exact packages cannot be set for MODULE "

@@ -176,19 +185,55 @@ 

              self.gather_source = "comps"

              self.gather_method = "deps"

              self.koji_tag = None

+             if source_type == PungiSourceType.BUILD and not self.packages:

+                 self.include_additional_packages = True

+         elif source_type == PungiSourceType.ODCS_V2_TEMPLATE:

+             self.koji_tag = rpm_koji_tags.split(" ") if rpm_koji_tags else []

+             self.koji_module_tags = modular_koji_tags.split(" ") if modular_koji_tags else []

+ 

+             ODCSv2SourceTypes = namedtuple(

+                 "ODCSv2SourceTypes",

+                 ["rpm_koji_tags", "module_koji_tags", "rpm_koji_builds", "module_koji_builds"])

+             s = ODCSv2SourceTypes(

+                 self.koji_tag, self.koji_module_tags, self.builds, self.module_builds)

+ 

+             # Karnaugh map ftw! We need to define something for all the possible combinations.

+             if s.module_koji_tags and s.module_koji_builds:

+                 raise ValueError(

+                     'The "module_koji_builds" source_type and "module_koji_tags" '

+                     'cannot be used together')

+             elif ((s.rpm_koji_tags or s.rpm_koji_builds) and

+                   not s.module_koji_tags and not s.module_koji_builds):

+                 # Compose just from Koji RPM tag or Koji RPM builds.

+                 self.gather_source = "comps"

+                 self.gather_method = "deps"

+                 if not self.packages:

+                     self.include_additional_packages = True

+             elif s.module_koji_tags or (s.rpm_koji_tags and s.module_koji_builds):

+                 # Compose just from modular Koji tag, or from normal Koji tag + some modules.

+                 self.gather_source = "comps"

+                 self.gather_method = "hybrid"

+                 self.module_defaults_url = module_defaults_url.split(" ") if module_defaults_url else []

+             elif s.module_koji_builds:

+                 # Compose just from modules.

+                 self.gather_source = "module"

+                 self.gather_method = "nodeps"

+                 self.module_defaults_url = module_defaults_url.split(" ") if module_defaults_url else []

+                 self.module_builds = self._sort_out_devel_modules(self.module_builds)

+             else:

+                 raise ValueError("Unsupported combination of sources %s." % s)

          else:

              raise ValueError("Unknown source_type %r" % source_type)

  

          self.check_deps = bool(flags & COMPOSE_FLAGS["check_deps"])

  

-     def _sort_out_devel_modules(self):

+     def _sort_out_devel_modules(self, modules):

          """

          Helper method filtering out "-devel" modules from `self.source`

          and adding them to `include_devel_modules` list.

          """

-         source_list = self.source.split(" ")

          new_source = []

-         for nsvc in source_list:

+         for nsvc in modules:

              n, s, v, c = nsvc.split(":")

  

              # It does not have -devel suffix, so it is not -devel module.

@@ -199,12 +244,12 @@ 

              # If it is -devel module, there must exist the non-devel

              # counterpart.

              non_devel_nsvc = ":".join([n[:-len("-devel")], s, v, c])

-             if non_devel_nsvc not in source_list:

+             if non_devel_nsvc not in modules:

                  new_source.append(nsvc)

                  continue

  

              self.include_devel_modules.append(":".join([n, s]))

-         self.source = " ".join(new_source)

+         return new_source

  

      @property

      def source_type_str(self):

@@ -252,6 +297,14 @@ 

                  tmp_variant.add_group(comps.Group('odcs-group', 'odcs-group', 'ODCS compose default group'))

              if self.koji_module_tags:

                  tmp_variant.add_module(comps.Module("*"))

+         elif self.source_type == PungiSourceType.ODCS_V2_TEMPLATE:

+             if self.packages:

+                 tmp_variant.add_group(comps.Group('odcs-group', 'odcs-group', 'ODCS compose default group'))

+             if self.koji_module_tags:

+                 tmp_variant.add_module(comps.Module("*"))

+             elif self.module_builds:

+                 for module in self.module_builds:

+                     tmp_variant.add_module(comps.Module(module))

  

          odcs_product.add_variant(tmp_variant)

  

file modified
+248 -39

@@ -49,45 +49,51 @@ 

      CELERY_AVAILABLE = False

  

  

- api_v1 = {

-     'composes': {

-         'url': '/api/1/composes/',

-         'options': {

-             'defaults': {'id': None},

-             'methods': ['GET'],

-         }

-     },

-     'compose': {

-         'url': '/api/1/composes/<int:id>',

-         'options': {

-             'methods': ['GET'],

-         }

-     },

-     'composes_post': {

-         'url': '/api/1/composes/',

-         'options': {

-             'methods': ['POST'],

-         }

-     },

-     'compose_regenerate': {

-         'url': '/api/1/composes/<int:id>',

-         'options': {

-             'methods': ['PATCH'],

-         }

-     },

-     'composes_delete': {

-         'url': '/api/1/composes/<int:id>',

-         'options': {

-             'methods': ['DELETE'],

-         }

-     },

-     'about': {

-         'url': '/api/1/about/',

-         'options': {

-             'methods': ['GET']

-         }

-     },

- }

+ def _generate_api_dict(version):

+     api_dict = {

+         'composes_%d' % version: {

+             'url': '/api/%d/composes/' % version,

+             'options': {

+                 'defaults': {'id': None},

+                 'methods': ['GET'],

+             }

+         },

+         'compose_%d' % version: {

+             'url': '/api/%d/composes/<int:id>' % version,

+             'options': {

+                 'methods': ['GET'],

+             }

+         },

+         'composes_post_%d' % version: {

+             'url': '/api/%d/composes/' % version,

+             'options': {

+                 'methods': ['POST'],

+             }

+         },

+         'compose_regenerate_%d' % version: {

+             'url': '/api/%d/composes/<int:id>' % version,

+             'options': {

+                 'methods': ['PATCH'],

+             }

+         },

+         'composes_delete_%d' % version: {

+             'url': '/api/%d/composes/<int:id>' % version,

+             'options': {

+                 'methods': ['DELETE'],

+             }

+         },

+         'about_%d' % version: {

+             'url': '/api/%d/about/' % version,

+             'options': {

+                 'methods': ['GET']

+             }

+         },

+     }

+     return api_dict

+ 

+ 

+ api_v1 = _generate_api_dict(1)

+ api_v2 = _generate_api_dict(2)

  

  

  class ODCSAPI(MethodView):

@@ -404,6 +410,189 @@ 

              raise NotFound('No such compose found.')

  

  

+ class ODCSAPIv2(ODCSAPI):

+ 

+     def _set_sources(self, compose, data):

+         """

+         Parses the `sources` section of POST request and sets the matching

+         attributes in `compose`.

+         """

+         sources = data.get("sources", None)

+         if not isinstance(sources, list):

+             raise ValueError('The "sources" list must be set.')

+ 

+         compose_rpms = set()

+         source_types = set()

+         for source in sources:

+             source_type = source.get("type", None)

+             if not source_type:

+                 raise ValueError('The "type" is missing in source entry.')

+ 

+             if source_type in source_types:

+                 raise ValueError('The "%s" source type define multiple times.' % source_type)

+             source_types.add(source_type)

+ 

+             if source_type == "rpm_koji_tags":

+                 tags = source.get("tags", None)

+                 if not isinstance(tags, list):

+                     raise ValueError('The "tags" list must be set for "rpm_koji_tags" source type.')

+                 tags = list(set(tags))

+                 compose.rpm_koji_tags = " ".join(tags)

+                 compose.source_type = PungiSourceType.ODCS_V2_TEMPLATE

+ 

+                 rpms = source.get("rpms", None)

+                 if isinstance(rpms, list):

+                     compose_rpms = compose_rpms.union(set(rpms))

+             elif source_type == "module_koji_tags":

+                 tags = source.get("tags", None)

+                 if not isinstance(tags, list):

+                     raise ValueError('The "tags" list must be set for "module_koji_tags" source type.')

+                 tags = list(set(tags))

+                 compose.modular_koji_tags = " ".join(tags)

+                 compose.source_type = PungiSourceType.ODCS_V2_TEMPLATE

+             elif source_type == "rpm_koji_builds":

+                 builds = source.get("builds", None)

+                 if not isinstance(builds, list):

+                     raise ValueError('The "builds" list must be set for "rpm_koji_builds" source type.')

+                 builds = sorted(list(set(builds)))

+                 compose.builds = " ".join(builds)

+                 compose.source_type = PungiSourceType.ODCS_V2_TEMPLATE

+ 

+                 rpms = source.get("rpms", None)

+                 if isinstance(rpms, list):

+                     compose_rpms = compose_rpms.union(set(rpms))

+             elif source_type == "module_koji_builds":

+                 builds = source.get("builds", None)

+                 if not isinstance(builds, list):

+                     raise ValueError('The "builds" list must be set for "module_koji_builds" source type.')

+                 builds = sorted(list(set(builds)))

+                 compose.module_builds = " ".join(builds)

+                 compose.source_type = PungiSourceType.ODCS_V2_TEMPLATE

+ 

+                 for module_str in builds:

+                     nsvc = module_str.split(":")

+                     if len(nsvc) < 2:

+                         raise ValueError(

+                             'Module definition must be in "n:s", "n:s:v" or '

+                             '"n:s:v:c" format, but got %s' % module_str)

+                     if nsvc[0] in conf.base_module_names:

+                         raise ValueError(

+                             "ODCS currently cannot create compose with base "

+                             "modules, but %s was requested." % nsvc[0])

+             elif source_type == "pulp_content_sets":

+                 content_sets = source.get("content_sets", None)

+                 if not isinstance(content_sets, list):

+                     raise ValueError('The "content_sets" list must be set for "pulp_content_sets" source type.')

+                 compose.source = " ".join(content_sets)

+                 compose.source_type = PungiSourceType.PULP

+             elif source_type == "pungi_raw_config":

+                 name = source.get("name", None)

+                 if not name:

+                     raise ValueError('The "name" must be set for "pungi_raw_config" source type.')

+                 commit = source.get("commit", None)

+                 if not commit:

+                     raise ValueError('The "commit" must be set for "pungi_raw_config" source type.')

+                 compose.source = "%s#%s" % (name, commit)

+                 compose.source_type = PungiSourceType.RAW_CONFIG

+ 

+                 if name not in conf.raw_config_urls:

+                     raise ValueError(

+                         'Source "%s" does not exist in server configuration.' %

+                         name)

+             else:

+                 raise ValueError('Unknown source type "%s".' % source_type)

+ 

+         if compose_rpms:

+             compose.packages = " ".join(compose_rpms)

+ 

+         if "pulp_content_sets" in source_types and len(source_types) != 1:

+             raise ValueError('The "pulp_content_sets" source_type must not be combined with others.')

+         if "pungi_raw_config" in source_types and len(source_types) != 1:

+             raise ValueError('The "pungi_raw_config" source_type must not be combined with others.')

+         if "module_koji_builds" in source_types and "module_koji_tags" in source_types:

+             raise ValueError(

+                 'The "module_koji_builds" source_type and "module_koji_tags" '

+                 'cannot be used together')

+ 

+         if compose.source_type == PungiSourceType.ODCS_V2_TEMPLATE:

+             compose.source = ""

+ 

+         return source_types

+ 

+     def _str_enum_list_to_int(self, data, key, default, enum):

+         ret = 0

+         for name in data.get(key, default):

+             if name not in enum:

+                 raise ValueError('Unknown %s value "%s".' % (key, name))

+             ret |= enum[name]

+         return ret

+ 

+     def _set_global_data(self, compose, data):

+         """

+         Parses the global fields in the POST request and sets the matching

+         `compose` attributes.

+         """

+         compose.sigkeys = " ".join(data.get("sigkeys", conf.sigkeys))

+         compose.flags = self._str_enum_list_to_int(

+             data, "flags", [], COMPOSE_FLAGS)

+         compose.results = self._str_enum_list_to_int(

+             data, "results", ["repository"], COMPOSE_RESULTS) | COMPOSE_RESULTS["repository"]

+         compose.arches = " ".join(data.get("arches", conf.arches))

+         compose.multilib_arches = " ".join(data.get("multilib_arches", []))

+         compose.multilib_method = self._str_enum_list_to_int(

+             data, "multilib_method", ["none"], MULTILIB_METHODS)

+         compose.lookaside_repos = " ".join(data.get("lookaside_repos", []))

+ 

+         module_defaults_url = data.get("module_defaults_url", None)

+         module_defaults_commit = data.get("module_defaults_commit", None)

+         # The "^" operator is logical XOR.

+         if bool(module_defaults_url) ^ bool(module_defaults_commit):

+             raise ValueError(

+                 'The "module_defaults_url" and "module_defaults_commit" '

+                 'must be used together.')

+         elif module_defaults_url and module_defaults_commit:

+             compose.module_defaults_url = "%s %s" % (module_defaults_url, module_defaults_commit)

+         return compose

+ 

+     @login_required

+     @require_scopes('new-compose')

+     @requires_role('allowed_clients')

+     def post(self):

+         data = request.get_json(force=True)

+         if not data:

+             raise ValueError('No JSON POST data submitted')

+ 

+         validate_json_data(data)

+ 

+         now = datetime.datetime.utcnow()

+         owner = self._get_compose_owner()

+         compose = Compose(owner=owner, time_submitted=now, state="wait")

+         seconds_to_live = self._get_seconds_to_live(data)

+         compose.time_to_expire = now + datetime.timedelta(seconds=seconds_to_live)

+ 

+         source_types = self._set_sources(compose, data)

+         compose = self._set_global_data(compose, data)

+ 

+         raise_if_input_not_allowed(

+             source_types=compose.source_type, sources=compose.source, results=compose.results,

+             flags=compose.flags, arches=compose.arches, v2_source_types=source_types)

+ 

+         db.session.add(compose)

+         # Flush is needed, because we use `before_commit` SQLAlchemy event to

+         # send message and before_commit can be called before flush and

+         # therefore the compose ID won't be set.

+         db.session.flush()

+         db.session.commit()

+ 

+         if CELERY_AVAILABLE and conf.celery_broker_url:

+             if compose.source_type == PungiSourceType.PULP:

+                 generate_pulp_compose.delay(compose.id)

+             else:

+                 generate_pungi_compose.delay(compose.id)

+ 

+         return jsonify(compose.json()), 200

+ 

+ 

  class AboutAPI(MethodView):

      def get(self):

          json = {'version': version}

@@ -438,4 +627,24 @@ 

              raise ValueError("Unhandled API key: %s." % key)

  

  

+ def register_api_v2():

+     """ Registers version 2 of ODCS API. """

+     composes_view = ODCSAPIv2.as_view('composes')

+     about_view = AboutAPI.as_view('about')

+     for key, val in api_v2.items():

+         if key.startswith("compose"):

+             app.add_url_rule(val['url'],

+                              endpoint=key,

+                              view_func=composes_view,

+                              **val['options'])

+         elif key.startswith("about"):

+             app.add_url_rule(val['url'],

+                              endpoint=key,

+                              view_func=about_view,

+                              **val['options'])

+         else:

+             raise ValueError("Unhandled API key: %s." % key)

+ 

+ 

  register_api_v1()

+ register_api_v2()

file modified
+3 -1

@@ -71,7 +71,9 @@ 

                           'multilib_method': 0,

                           'lookaside_repos': None,

                           'modular_koji_tags': None,

-                          'module_defaults_url': None}

+                          'module_defaults_url': None,

+                          'rpm_koji_tags': None,

+                          'module_builds': None}

          self.assertEqual(c.json(), expected_json)

  

      def test_create_copy(self):

file modified
+113 -2

@@ -46,9 +46,20 @@ 

      def setUp(self):

          super(TestPungiConfig, self).setUp()

  

+         _, self.mock_path = tempfile.mkstemp()

+         template_path = os.path.abspath(

+             os.path.join(test_dir, "../conf/pungi.conf"))

+         shutil.copy2(template_path, self.mock_path)

+         self.pungi_conf_path_patcher = patch(

+             "odcs.server.pungi.conf.pungi_conf_path", self.mock_path)

+         self.pungi_conf_path_patcher.start()

+ 

      def tearDown(self):

          super(TestPungiConfig, self).tearDown()

  

+         os.unlink(self.mock_path)

+         self.pungi_conf_path_patcher.stop()

+ 

      def _load_pungi_cfg(self, cfg):

          conf = PyConfigParser()

          conf.load_from_string(cfg)

@@ -264,7 +275,7 @@ 

  

              template = pungi_cfg.get_pungi_config()

              cfg = self._load_pungi_cfg(template)

-             self.assertEqual(cfg["pkgset_koji_tag"], 'f26')

+             self.assertEqual(cfg["pkgset_koji_tag"], ['f26'])

              self.assertEqual(cfg["additional_packages"],

                               [('^Temporary$', {'*': ['*']})])

  

@@ -281,7 +292,7 @@ 

  

              template = pungi_cfg.get_pungi_config()

              cfg = self._load_pungi_cfg(template)

-             self.assertEqual(cfg["pkgset_koji_tag"], 'f26')

+             self.assertEqual(cfg["pkgset_koji_tag"], ['f26'])

              self.assertTrue("additional_packages" not in cfg)

  

      def test_get_pungi_conf_lookaside_repos(self):

@@ -319,6 +330,106 @@ 

                  {"Temporary": ["foo-devel:1"]})

              self.assertEqual(pungi_cfg.source, "foo:1:1:1 bar-devel:1:1:1")

  

+     def test_v2_rpm_koji_tags(self):

+         pungi_cfg = PungiConfig(

+             "MBS-512", "1", PungiSourceType.ODCS_V2_TEMPLATE,

+             rpm_koji_tags="f26")

+         template = pungi_cfg.get_pungi_config()

+         cfg = self._load_pungi_cfg(template)

+         self.assertEqual(cfg["pkgset_koji_tag"], ["f26"])

+         self.assertEqual(cfg["gather_method"], "deps")

+         self.assertEqual(cfg["gather_source"], "comps")

+ 

+     def test_v2_module_koji_tags(self):

+         pungi_cfg = PungiConfig(

+             "MBS-512", "1", PungiSourceType.ODCS_V2_TEMPLATE,

+             modular_koji_tags="f26")

+         template = pungi_cfg.get_pungi_config()

+         cfg = self._load_pungi_cfg(template)

+         self.assertEqual(cfg["pkgset_koji_tag"], "")

+         self.assertEqual(cfg["pkgset_koji_module_tag"], ["f26"])

+         self.assertEqual(cfg["gather_method"], "hybrid")

+         self.assertEqual(cfg["gather_source"], "comps")

+ 

+     def test_v2_module_koji_builds(self):

+         pungi_cfg = PungiConfig(

+             "MBS-512", "1", PungiSourceType.ODCS_V2_TEMPLATE,

+             module_builds="test:master:1:1 test:foo:1:1")

+         template = pungi_cfg.get_pungi_config()

+         cfg = self._load_pungi_cfg(template)

+         self.assertEqual(cfg["pkgset_koji_tag"], "")

+         self.assertEqual(cfg["gather_method"], "nodeps")

+         self.assertEqual(cfg["gather_source"], "module")

+ 

+         variants = pungi_cfg.get_variants_config()

+ 

+         self.assertTrue(variants.find("<module>") != -1)

+         self.assertTrue(variants.find("test:master:1:1") != -1)

+         self.assertTrue(variants.find("test:foo:1:1") != -1)

+ 

+     def test_v2_rpm_koji_builds(self):

+         pungi_cfg = PungiConfig(

+             "MBS-512", "1", PungiSourceType.ODCS_V2_TEMPLATE,

+             builds=["foo-1-1", "bar-1-1"])

+         template = pungi_cfg.get_pungi_config()

+         cfg = self._load_pungi_cfg(template)

+         self.assertEqual(cfg["pkgset_koji_tag"], "")

+         self.assertEqual(cfg["gather_method"], "deps")

+         self.assertEqual(cfg["gather_source"], "comps")

+ 

+         self.assertEqual(set(cfg["pkgset_koji_builds"]),

+                          set(["foo-1-1", "bar-1-1"]))

+         self.assertEqual(cfg["additional_packages"],

+                          [(u'^Temporary$', {u'*': [u'*']})])

+ 

+     def test_v2_rpm_koji_builds_rpm_koji_tags(self):

+         pungi_cfg = PungiConfig(

+             "MBS-512", "1", PungiSourceType.ODCS_V2_TEMPLATE,

+             rpm_koji_tags="f26",

+             builds=["foo-1-1", "bar-1-1"])

+         template = pungi_cfg.get_pungi_config()

+         cfg = self._load_pungi_cfg(template)

+         self.assertEqual(cfg["pkgset_koji_tag"], ["f26"])

+         self.assertEqual(cfg["gather_method"], "deps")

+         self.assertEqual(cfg["gather_source"], "comps")

+ 

+         self.assertEqual(set(cfg["pkgset_koji_builds"]),

+                          set(["foo-1-1", "bar-1-1"]))

+         self.assertEqual(cfg["additional_packages"],

+                          [(u'^Temporary$', {u'*': [u'*']})])

+ 

+     def test_v2_rpm_koji_builds_module_koji_tags(self):

+         pungi_cfg = PungiConfig(

+             "MBS-512", "1", PungiSourceType.ODCS_V2_TEMPLATE,

+             modular_koji_tags="f26",

+             builds=["foo-1-1", "bar-1-1"])

+         template = pungi_cfg.get_pungi_config()

+         cfg = self._load_pungi_cfg(template)

+         self.assertEqual(cfg["pkgset_koji_module_tag"], ["f26"])

+         self.assertEqual(cfg["gather_method"], "hybrid")

+         self.assertEqual(cfg["gather_source"], "comps")

+ 

+         self.assertEqual(set(cfg["pkgset_koji_builds"]),

+                          set(["foo-1-1", "bar-1-1"]))

+ 

+     def test_v2_rpm_koji_builds_module_koji_builds(self):

+         pungi_cfg = PungiConfig(

+             "MBS-512", "1", PungiSourceType.ODCS_V2_TEMPLATE,

+             module_builds="n:s:v:c",

+             builds=["foo-1-1", "bar-1-1"])

+         template = pungi_cfg.get_pungi_config()

+         cfg = self._load_pungi_cfg(template)

+         self.assertEqual(cfg["gather_method"], "nodeps")

+         self.assertEqual(cfg["gather_source"], "module")

+ 

+         self.assertEqual(set(cfg["pkgset_koji_builds"]),

+                          set(["foo-1-1", "bar-1-1"]))

+ 

+         variants = pungi_cfg.get_variants_config()

+ 

+         self.assertTrue(variants.find("<module>") != -1)

+         self.assertTrue(variants.find("n:s:v:c") != -1)

+ 

  

  class TestPungi(unittest.TestCase):

  

file modified
+10 -2

@@ -113,6 +113,10 @@ 

                  },

                  'dev2': {

                      'source_types': ['module', 'raw_config']

+                 },

+                 'dev3': {

+                     'source_types': ['module', 'raw_config', 'odcs_v2_template'],

+                     'v2_source_types': ['pungi_raw_config']

                  }

              }

          }

@@ -307,7 +311,9 @@ 

                           'multilib_method': 0,

                           'lookaside_repos': '',

                           'modular_koji_tags': None,

-                          'module_defaults_url': None}

+                          'module_defaults_url': None,

+                          'rpm_koji_tags': None,

+                          'module_builds': None}

          self.assertEqual(data, expected_json)

  

          db.session.expire_all()

@@ -1083,7 +1089,9 @@ 

                           'multilib_method': 0,

                           'lookaside_repos': '',

                           'modular_koji_tags': None,

-                          'module_defaults_url': None}

+                          'module_defaults_url': None,

+                          'rpm_koji_tags': None,

+                          'module_builds': None}

          self.assertEqual(data, expected_json)

  

          db.session.expire_all()

@@ -0,0 +1,730 @@ 

+ # Copyright (c) 2019  Red Hat, Inc.

+ #

+ # Permission is hereby granted, free of charge, to any person obtaining a copy

+ # of this software and associated documentation files (the "Software"), to deal

+ # in the Software without restriction, including without limitation the rights

+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

+ # copies of the Software, and to permit persons to whom the Software is

+ # furnished to do so, subject to the following conditions:

+ #

+ # The above copyright notice and this permission notice shall be included in all

+ # copies or substantial portions of the Software.

+ #

+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

+ # SOFTWARE.

+ #

+ # Written by Jan Kaluza <jkaluza@redhat.com>

+ 

+ import json

+ 

+ from datetime import datetime, timedelta

+ 

+ import flask

+ 

+ from freezegun import freeze_time

+ from mock import patch, PropertyMock

+ 

+ import odcs.server.auth

+ 

+ from odcs.server import conf, db

+ from odcs.server.models import Compose

+ from odcs.common.types import (COMPOSE_STATES, COMPOSE_RESULTS, COMPOSE_FLAGS,

+                                MULTILIB_METHODS)

+ from odcs.server.pungi import PungiSourceType

+ 

+ from .test_views import ViewBaseTest

+ 

+ 

+ class TestViews(ViewBaseTest):

+     maxDiff = None

+ 

+     def setUp(self):

+         super(TestViews, self).setUp()

+         self.oidc_base_namespace = patch.object(conf, 'oidc_base_namespace',

+                                                 new='http://example.com/')

+         self.oidc_base_namespace.start()

+ 

+     def tearDown(self):

+         self.oidc_base_namespace.stop()

+         super(TestViews, self).tearDown()

+ 

+     def setup_test_data(self):

+         self.initial_datetime = datetime(year=2016, month=1, day=1,

+                                          hour=0, minute=0, second=0)

+         with freeze_time(self.initial_datetime):

+             self.c1 = Compose.create(

+                 db.session, "unknown", PungiSourceType.MODULE, "testmodule:master",

+                 COMPOSE_RESULTS["repository"], 60)

+             self.c2 = Compose.create(

+                 db.session, "me", PungiSourceType.KOJI_TAG, "f26",

+                 COMPOSE_RESULTS["repository"], 60)

+             db.session.add(self.c1)

+             db.session.add(self.c2)

+             db.session.commit()

+ 

+     def test_submit_invalid_json(self):

+         with self.test_request_context(user='dev'):

+             flask.g.oidc_scopes = [

+                 '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')

+             ]

+ 

+             rv = self.client.post('/api/2/composes/', data="{")

+             data = json.loads(rv.get_data(as_text=True))

+ 

+         self.assertEqual(rv.status, '400 BAD REQUEST')

+         self.assertEqual(data["error"], "Bad Request")

+         self.assertEqual(data["status"], 400)

+         self.assertTrue(data["message"].find("Failed to decode JSON object") != -1)

+ 

+     def test_submit_build(self):

+         with self.test_request_context(user='dev'):

+             flask.g.oidc_scopes = [

+                 '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')

+             ]

+ 

+             rv = self.client.post('/api/2/composes/', data=json.dumps(

+                 {'sources': [{'type': 'module_koji_builds', 'builds': ['testmodule:master']}]}))

+             data = json.loads(rv.get_data(as_text=True))

+ 

+         expected_json = {'source_type': 8, 'state': 0, 'time_done': None,

+                          'state_name': 'wait',

+                          'state_reason': None,

+                          'source': u'',

+                          'owner': u'dev',

+                          'result_repo': 'http://localhost/odcs/latest-odcs-%d-1/compose/Temporary' % data['id'],

+                          'result_repofile': 'http://localhost/odcs/latest-odcs-%d-1/compose/Temporary/odcs-%d.repo' % (data['id'], data['id']),

+                          'time_submitted': data["time_submitted"], 'id': data['id'],

+                          'time_removed': None,

+                          'removed_by': None,

+                          'time_to_expire': data["time_to_expire"],

+                          'flags': [],

+                          'results': ['repository'],

+                          'sigkeys': '',

+                          'koji_event': None,

+                          'koji_task_id': None,

+                          'packages': None,

+                          'builds': None,

+                          'arches': 'x86_64',

+                          'multilib_arches': '',

+                          'multilib_method': 0,

+                          'lookaside_repos': '',

+                          'modular_koji_tags': None,

+                          'module_defaults_url': None,

+                          'rpm_koji_tags': None,

+                          'module_builds': 'testmodule:master'}

+         self.assertEqual(data, expected_json)

+ 

+         db.session.expire_all()

+         c = db.session.query(Compose).filter(Compose.id == 1).one()

+         self.assertEqual(c.state, COMPOSE_STATES["wait"])

+ 

+     def test_submit_build_no_packages(self):

+         with self.test_request_context(user='dev'):

+             flask.g.oidc_scopes = [

+                 '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')

+             ]

+ 

+             rv = self.client.post('/api/2/composes/', data=json.dumps(

+                 {'sources': [{'type': 'rpm_koji_tags', 'tags': ['f26']}],

+                  'flags': ['no_deps']}))

+             data = json.loads(rv.get_data(as_text=True))

+ 

+         self.assertEqual(data["state_name"], "wait")

+ 

+         db.session.expire_all()

+         c = db.session.query(Compose).filter(Compose.id == 1).one()

+         self.assertEqual(c.state, COMPOSE_STATES["wait"])

+         self.assertEqual(c.packages, None)

+ 

+     def test_submit_build_nodeps(self):

+         with self.test_request_context(user='dev'):

+             flask.g.oidc_scopes = [

+                 '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')

+             ]

+ 

+             rv = self.client.post('/api/2/composes/', data=json.dumps(

+                 {'sources': [{'type': 'rpm_koji_tags', 'tags': ['f26'],

+                               'packages': ['ed']}],

+                  'flags': ['no_deps']}))

+             data = json.loads(rv.get_data(as_text=True))

+ 

+         self.assertEqual(data['flags'], ['no_deps'])

+ 

+         db.session.expire_all()

+         c = db.session.query(Compose).filter(Compose.id == 3).one()

+         self.assertEqual(c.state, COMPOSE_STATES["wait"])

+         self.assertEqual(c.flags, COMPOSE_FLAGS["no_deps"])

+ 

+     def test_submit_build_noinheritance(self):

+         with self.test_request_context(user='dev'):

+             flask.g.oidc_scopes = [

+                 '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')

+             ]

+ 

+             rv = self.client.post('/api/2/composes/', data=json.dumps(

+                 {'sources': [{'type': 'rpm_koji_tags', 'tags': ['f26'],

+                               'packages': ['ed']}],

+                  'flags': ['no_inheritance']}))

+             data = json.loads(rv.get_data(as_text=True))

+ 

+         self.assertEqual(data['flags'], ['no_inheritance'])

+ 

+         db.session.expire_all()

+         c = db.session.query(Compose).filter(Compose.id == 3).one()

+         self.assertEqual(c.state, COMPOSE_STATES["wait"])

+         self.assertEqual(c.flags, COMPOSE_FLAGS["no_inheritance"])

+ 

+     def test_submit_build_boot_iso(self):

+         with self.test_request_context(user='dev'):

+             flask.g.oidc_scopes = [

+                 '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')

+             ]

+ 

+             rv = self.client.post('/api/2/composes/', data=json.dumps(

+                 {'sources': [{'type': 'rpm_koji_tags', 'tags': ['f26'], 'packages': ['ed']}],

+                  'results': ['boot.iso']}))

+             data = json.loads(rv.get_data(as_text=True))

+ 

+         self.assertEqual(set(data['results']), set(['repository', 'boot.iso']))

+ 

+         db.session.expire_all()

+         c = db.session.query(Compose).filter(Compose.id == 3).one()

+         self.assertEqual(c.state, COMPOSE_STATES["wait"])

+         self.assertEqual(

+             c.results,

+             COMPOSE_RESULTS["boot.iso"] | COMPOSE_RESULTS["repository"])

+ 

+     def test_submit_build_sigkeys(self):

+         with self.test_request_context(user='dev'):

+             flask.g.oidc_scopes = [

+                 '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')

+             ]

+ 

+             rv = self.client.post('/api/2/composes/', data=json.dumps(

+                 {'sources': [{'type': 'rpm_koji_tags', 'tags': ['f26'],

+                               'packages': ['ed']}],

+                  'sigkeys': ["123", "456"]}))

+             data = json.loads(rv.get_data(as_text=True))

+ 

+         self.assertEqual(data['sigkeys'], '123 456')

+ 

+         db.session.expire_all()

+         c = db.session.query(Compose).filter(Compose.id == 1).one()

+         self.assertEqual(c.state, COMPOSE_STATES["wait"])

+ 

+     @patch.object(odcs.server.config.Config, 'sigkeys', new_callable=PropertyMock)

+     def test_submit_build_default_sigkeys(self, sigkeys):

+         with self.test_request_context(user='dev'):

+             sigkeys.return_value = ["x", "y"]

+             flask.g.oidc_scopes = [

+                 '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')

+             ]

+ 

+             rv = self.client.post('/api/2/composes/', data=json.dumps(

+                 {'sources': [{'type': 'rpm_koji_tags', 'tags': ['f26'], 'packages': ['ed']}]}))

+             data = json.loads(rv.get_data(as_text=True))

+ 

+         self.assertEqual(data['sigkeys'], 'x y')

+ 

+         db.session.expire_all()

+         c = db.session.query(Compose).filter(Compose.id == 1).one()

+         self.assertEqual(c.state, COMPOSE_STATES["wait"])

+ 

+     def test_submit_build_arches(self):

+         with self.test_request_context(user='dev'):

+             flask.g.oidc_scopes = [

+                 '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')

+             ]

+ 

+             rv = self.client.post('/api/2/composes/', data=json.dumps(

+                 {'sources': [{'type': 'rpm_koji_tags', 'tags': ['f26'], 'packages': ['ed']}],

+                  'arches': ["ppc64", "s390"]}))

+             data = json.loads(rv.get_data(as_text=True))

+ 

+         self.assertEqual(data['arches'], 'ppc64 s390')

+ 

+         db.session.expire_all()

+         c = db.session.query(Compose).filter(Compose.id == 1).one()

+         self.assertEqual(c.state, COMPOSE_STATES["wait"])

+ 

+     def test_submit_build_multilib_arches(self):

+         with self.test_request_context(user='dev'):

+             flask.g.oidc_scopes = [

+                 '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')

+             ]

+ 

+             rv = self.client.post('/api/2/composes/', data=json.dumps(

+                 {'sources': [{'type': 'rpm_koji_tags', 'tags': ['f26'], 'packages': ['ed']}],

+                  'arches': ["ppc64", "s390"], 'multilib_arches': ["x86_64", "ppc64le"]}))

+             data = json.loads(rv.get_data(as_text=True))

+ 

+         self.assertEqual(data['multilib_arches'], 'x86_64 ppc64le')

+ 

+         db.session.expire_all()

+         c = db.session.query(Compose).filter(Compose.id == 1).one()

+         self.assertEqual(c.state, COMPOSE_STATES["wait"])

+ 

+     def test_submit_build_multilib_method(self):

+         with self.test_request_context(user='dev'):

+             flask.g.oidc_scopes = [

+                 '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')

+             ]

+ 

+             rv = self.client.post('/api/2/composes/', data=json.dumps(

+                 {'sources': [{'type': 'rpm_koji_tags', 'tags': ['f26'], 'packages': ['ed']}],

+                  'arches': ["ppc64", "s390"], 'multilib_method': ["runtime", "devel"]}))

+             data = json.loads(rv.get_data(as_text=True))

+ 

+         self.assertEqual(data['multilib_method'],

+                          MULTILIB_METHODS["runtime"] | MULTILIB_METHODS["devel"])

+ 

+         db.session.expire_all()

+         c = db.session.query(Compose).filter(Compose.id == 1).one()

+         self.assertEqual(c.state, COMPOSE_STATES["wait"])

+ 

+     def test_submit_build_multilib_method_unknown(self):

+         with self.test_request_context(user='dev'):

+             flask.g.oidc_scopes = [

+                 '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')

+             ]

+ 

+             rv = self.client.post('/api/2/composes/', data=json.dumps(

+                 {'sources': [{'type': 'rpm_koji_tags', 'tags': ['f26'], 'packages': ['ed']}],

+                  'arches': ["ppc64", "s390"], 'multilib_method': ["foo", "devel"]}))

+             data = json.loads(rv.get_data(as_text=True))

+ 

+         self.assertEqual(

+             data['message'], 'Unknown multilib_method value "foo".')

+ 

+     def test_submit_build_modular_koji_tags(self):

+         with self.test_request_context(user='dev'):

+             flask.g.oidc_scopes = [

+                 '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')

+             ]

+ 

+             rv = self.client.post('/api/2/composes/', data=json.dumps(

+                 {'sources': [{'type': 'rpm_koji_tags', 'tags': ['f26']},

+                              {'type': 'module_koji_tags', 'tags': ['f26-modules']}]}))

+             data = json.loads(rv.get_data(as_text=True))

+ 

+         self.assertEqual(data['modular_koji_tags'], "f26-modules")

+ 

+         db.session.expire_all()

+         c = db.session.query(Compose).filter(Compose.id == 1).one()

+         self.assertEqual(c.state, COMPOSE_STATES["wait"])

+ 

+     def test_submit_build_module_defaults_url(self):

+         with self.test_request_context(user='dev'):

+             flask.g.oidc_scopes = [

+                 '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')

+             ]

+ 

+             rv = self.client.post('/api/2/composes/', data=json.dumps(

+                 {'sources': [{'type': 'rpm_koji_tags', 'tags': ['f26']}],

+                  'module_defaults_url': 'git://localhost.tld/x.git',

+                  'module_defaults_commit': 'master'}))

+             data = json.loads(rv.get_data(as_text=True))

+ 

+         self.assertEqual(data['module_defaults_url'], 'git://localhost.tld/x.git master')

+ 

+         db.session.expire_all()

+         c = db.session.query(Compose).filter(Compose.id == 1).one()

+         self.assertEqual(c.state, COMPOSE_STATES["wait"])

+ 

+     def test_submit_build_module_defaults_url_no_branch(self):

+         with self.test_request_context(user='dev'):

+             flask.g.oidc_scopes = [

+                 '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')

+             ]

+ 

+             rv = self.client.post('/api/2/composes/', data=json.dumps(

+                 {'sources': [{'type': 'rpm_koji_tags', 'tags': ['f26']}],

+                  'module_defaults_url': 'git://localhost.tld/x.git'}))

+             data = json.loads(rv.get_data(as_text=True))

+             self.assertEqual(data['status'], 400)

+             self.assertEqual(data['error'], 'Bad Request')

+             self.assertEqual(data['message'],

+                              'The "module_defaults_url" and "module_defaults_commit" '

+                              'must be used together.')

+ 

+     def test_submit_build_duplicate_sources(self):

+         with self.test_request_context(user='dev'):

+             flask.g.oidc_scopes = [

+                 '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')

+             ]

+ 

+             rv = self.client.post('/api/2/composes/', data=json.dumps(

+                 {'sources': [{'type': 'module_koji_builds',

+                               'builds': ['foo:x', 'foo:x', 'foo:y']}]}))

+             data = json.loads(rv.get_data(as_text=True))

+ 

+         self.assertEqual(data['module_builds'].count("foo:x"), 1)

+         self.assertEqual(data['module_builds'].count("foo:y"), 1)

+