#293 Get and set architectures.
Merged a year ago by ralph. Opened a year ago by ralph.

file modified
+13

@@ -205,6 +205,19 @@ 

              'type': bool,

              'default': False,

              'desc': 'Whether to make a scratch build to rebuild the image.'},

+         'supply_arch_overrides': {

+             'type': bool,

+             'default': False,

+             'desc': 'Determines whether or not to supply architecture overrides to OSBS.',

+         },

+         'manifest_v2_arch_map': {

+             'type': dict,

+             'default': {

+                 # Someday, somebody please tell me why these names are different.

+                 'amd64': 'x86_64',

+                 'arm64': 'aarch64',

+             },

+             'desc': 'A map of manifest api v2 architectures to brew architectures.'},

          'dry_run': {

              'type': bool,

              'default': False,

@@ -391,7 +391,8 @@ 

  

      def build_container(self, scm_url, branch, target,

                          repo_urls=None, isolated=False,

-                         release=None, koji_parent_build=None):

+                         release=None, koji_parent_build=None,

+                         arch_override=None):

          """

          Build a container in Koji.

  

@@ -407,8 +408,8 @@ 

                  profile=conf.koji_profile, logger=log,

                  dry_run=self.dry_run) as service:

              log.info('Building container from source: %s, '

-                      'release=%r, parent=%r, target=%r',

-                      scm_url, release, koji_parent_build, target)

+                      'release=%r, parent=%r, target=%r, arch=%r',

+                      scm_url, release, koji_parent_build, target, arch_override)

  

              return service.build_container(scm_url,

                                             branch,

@@ -417,6 +418,7 @@ 

                                             isolated=isolated,

                                             release=release,

                                             koji_parent_build=koji_parent_build,

+                                            arch_override=arch_override,

                                             scratch=conf.koji_container_scratch_build)

  

      @fail_artifact_build_on_handler_exception

@@ -447,6 +449,10 @@ 

          target = args["target"]

          parent = args["parent"]

  

+         # If set to None, then OSBS defaults to using the arches

+         # of the build tag associated with the target.

+         arches = args.get("arches")

+ 

          if not build.rebuilt_nvr and build.original_nvr:

              build.rebuilt_nvr = get_rebuilt_nvr(

                  build.type, build.original_nvr)

@@ -461,7 +467,8 @@ 

  

          return self.build_container(

              scm_url, branch, target, repo_urls=repo_urls,

-             isolated=True, release=release, koji_parent_build=parent)

+             isolated=True, release=release, koji_parent_build=parent,

+             arch_override=arches)

  

      def odcs_get_compose(self, compose_id):

          """

file modified
+4 -1

@@ -139,7 +139,8 @@ 

  

      def build_container(self, source_url, branch, target,

                          scratch=None, repo_urls=None, isolated=False,

-                         release=None, koji_parent_build=None):

+                         release=None, koji_parent_build=None,

+                         arch_override=None):

          """Build container by buildContainer"""

  

          build_target = target

@@ -154,6 +155,8 @@ 

              build_opts['isolated'] = True

          if koji_parent_build:

              build_opts['koji_parent_build'] = koji_parent_build

+         if arch_override:

+             build_opts['arch_override'] = arch_override

          if release:

              build_opts['release'] = release

  

file modified
+73 -2

@@ -21,6 +21,7 @@ 

  #

  # Written by Chenxiong Qi <cqi@redhat.com>

  #            Jan Kaluza <jkaluza@redhat.com>

+ #            Ralph Bean <rbean@redhat.com>

  

  import yaml

  import json

@@ -171,8 +172,14 @@ 

          return dockerfile[0]

  

      def _get_default_additional_data(self):

-         return {"repository": None, "commit": None, "target": None,

-                 "git_branch": None, "error": None}

+         return {

+             "repository": None,

+             "commit": None,

+             "target": None,

+             "git_branch": None,

+             "error": None,

+             "arches": None,

+         }

  

      @region.cache_on_arguments()

      def _get_additional_data_from_koji(self, nvr):

@@ -240,8 +247,72 @@ 

                  raise KojiLookupError(

                      "Cannot find valid source of Koji build %r" % build)

  

+             data['arches'] = self._get_architectures_from_registry(nvr, build)

+ 

          return data

  

+     def _get_architectures_from_registry(self, nvr, build):

+         """ Determine the architectures of the build by reading the manifest """

+ 

+         # First thing, check our feature flag.  This feature won't work if the OSBS instance we're

+         # working with doesn't support supply arch_overrides, so, if our configuration says "don't

+         # supply arch overrides to OSBS" then return None.

+         if not conf.supply_arch_overrides:

+             return None

+ 

+         # If the image doesn't have the digest metadata we need, then we can

+         # assume it is an old single-arch build.  But, if it does have a digest then carefully

+         # query the registry for it to extract the list of arches produced last time.

+         if 'extra' not in build:

+             return 'x86_64'

+         if 'image' not in build['extra']:

+             return 'x86_64'

+         if 'index' not in build['extra']['image']:

+             return 'x86_64'

+ 

+         index = build['extra']['image']['index']

+         manifest_list = 'application/vnd.docker.distribution.manifest.list.v2+json'

+         digest = index.get('digests', {}).get(manifest_list)

+ 

+         if not digest:

+             return 'x86_64'

+ 

+         # If it has a digest, then it must have a pull url.

+         registry_urls = [url for url in index['pull'] if digest in url]

+         if not registry_urls:

+             raise KojiLookupError(

+                 "Could not find pull url for Koji build %r %r" % (nvr, digest))

+ 

+         url = registry_urls[0].split(digest)[0].strip('@')

+         response = requests.get(url, headers=dict(Accept=manifest_list))

+         if not response.ok:

+             raise KojiLookupError(

+                 "Could not pull manifest list from %s for %r: %r" % (url, nvr, response))

+ 

+         try:

+             data = response.json()

+         except ValueError as e:

+             raise KojiLookupError(

+                 "Manifest list response for %r was not json: %r %s" % (nvr, e, url))

+ 

+         if 'manifests' not in data:

+             raise KojiLookupError(

+                 "Manifest list response for %r was malformed: %s" % (nvr, url, data))

+ 

+         # Extract the list of arches, as written

+         manifests = data['manifests']

+         arches = [

+             manifest['platform']['architecture']

+             for manifest in manifests

+             if 'platform' in manifest and 'architecture' in manifest['platform']

+         ]

+         # But!  Convert some arch values into ones familiar to Brew.

+         # Notably, turn amd64 into x86_64.

+         arches = [conf.manifest_v2_arch_map.get(arch, arch) for arch in arches]

+ 

+         # Finally, return the list, joined.

+         return ','.join(arches)

+ 

      @region.cache_on_arguments()

      def _get_additional_data_from_distgit(self, repository, branch, commit):

          """

file modified
+81

@@ -26,8 +26,11 @@ 

  from mock import call, patch, Mock, mock_open

  from six.moves import http_client

  

+ import freshmaker

+ 

  from freshmaker.lightblue import ContainerImage

  from freshmaker.lightblue import ContainerRepository

+ from freshmaker.lightblue import KojiLookupError

  from freshmaker.lightblue import LightBlue

  from freshmaker.lightblue import LightBlueRequestError

  from freshmaker.lightblue import LightBlueSystemError

@@ -964,6 +967,7 @@ 

                                   "target": "target2",

                                   "git_branch": "mybranch",

                                   "error": None,

+                                  "arches": None,

                                   "brew": {

                                       "completion_date": u"20170421T04:27:51.000-0400",

                                       "build": "package-name-2-4-12.10",

@@ -1633,3 +1637,80 @@ 

          # We expect each image to be rebuilt just once.

          expected = [[deps[3]], [deps[2]], [deps[1]], [deps[0]]]

          self.assertEqual(batches, expected)

+ 

+ 

+ class TestArchitecturesFromRegistry(helpers.FreshmakerTestCase):

+ 

+     def setUp(self):

+         super(TestArchitecturesFromRegistry, self).setUp()

+         self.build = {

+             'extra': {

+                 'container_koji_task_id': 16879260,

+                 'image': {

+                     'autorebuild': False,

+                     'help': None,

+                     'index': {

+                         'digests': {

+                             'application/vnd.docker.distribution.manifest.list.v2+json': 'sha256:252084580dd052fd6d16b5eb25397cf2396c69ec485ba34692577ebd25693fa7'

+                         },

+                         'pull': [

+                             'blue-pulp-smocker01.sledmat.com:8888/devtools/rust-toolset-7-rhel7@sha256:252084580dd052fd6d16b5eb25397cf2396c69ec485ba34692577ebd25693fa7',

+                             'blue-pulp-smocker01.sledmat.com:8888/devtools/rust-toolset-7-rhel7:1.26.2-4',

+                         ],

+                         'tags': ['latest', '1.26.2', '1.26.2-4']

+                     },

+                     'isolated': False,

+                     'media_types': [

+                         'application/json',

+                         'application/vnd.docker.distribution.manifest.list.v2+json',

+                         'application/vnd.docker.distribution.manifest.v1+json',

+                         'application/vnd.docker.distribution.manifest.v2+json'],

+                     'parent_build_id': 714853,

+                 },

+                 'submitter': 'osbs'

+             },

+         }

+ 

+     def test_feature_flag(self):

+         """ Unless configured, the method should always return None. """

+         image = ContainerImage.create({"brew": {"build": "nvr"}})

+         result = image._get_architectures_from_registry('foo', 'whatever')

+         self.assertEqual(result, None)

+ 

+     @patch.object(freshmaker.conf, 'supply_arch_overrides', new=True)

+     def test_premature_exit(self):

+         image = ContainerImage.create({"brew": {"build": "nvr"}})

+         result = image._get_architectures_from_registry('foo', 'whatever')

+         self.assertEqual(result, 'x86_64')

+ 

+     @patch.object(freshmaker.conf, 'supply_arch_overrides', new=True)

+     @patch('freshmaker.lightblue.requests')

+     def test_happy_path(self, requests):

+         image = ContainerImage.create({"brew": {"build": "nvr"}})

+         requests.get.return_value.json.return_value = {

+             "manifests": [

+                 {"platform": {"architecture": "amd64"}},

+                 {"platform": {"architecture": "s390x"}},

+                 {"platform": {"architecture": "ppc64le"}},

+             ]

+         }

+         result = image._get_architectures_from_registry("foo", self.build)

+         self.assertEqual(result, 'x86_64,s390x,ppc64le')

+         requests.get.assert_called_once_with(

+             'blue-pulp-smocker01.sledmat.com:8888/devtools/rust-toolset-7-rhel7',

+             headers={'Accept': 'application/vnd.docker.distribution.manifest.list.v2+json'},

+         )

+ 

+     @patch.object(freshmaker.conf, 'supply_arch_overrides', new=True)

+     @patch('freshmaker.lightblue.requests')

+     def test_invalid_json(self, requests):

+         image = ContainerImage.create({"brew": {"build": "nvr"}})

+         requests.get.return_value.json.side_effect = ValueError

+         self.assertRaises(KojiLookupError, image._get_architectures_from_registry, "foo", self.build)

+ 

+     @patch.object(freshmaker.conf, 'supply_arch_overrides', new=True)

+     @patch('freshmaker.lightblue.requests')

+     def test_bad_status(self, requests):

+         image = ContainerImage.create({"brew": {"build": "nvr"}})

+         requests.get.return_value.ok = False

+         self.assertRaises(KojiLookupError, image._get_architectures_from_registry, "foo", self.build)

This fixes a bug we ran into today as a part of multi-arch migrations.

First, describing the bug:

When building an image, OSBS looks at the architectures set in koji on the
'build' tag associated with the koji target of the requested build. This is
fine, normally. When a set of builds is ready to "go multi-arch", we just
start adding architectures to their build tag and suddenly, new builds produce
multiarch images.

When freshmaker comes along, it tries to rebuild based on the last shipped
image and reuses the parent image and the koji target of the last shipped
image. Now, the last shipped image's parent is single-arch (from a few months
ago), but the koji target now has a koji tag that specifies multiple
architectures. OSBS fails our build because it cannot build a multiarch build
on a single arch parent.

The fix here is to look up the architectures produced by the last-shipped build
and simply copy those into our request for the new build (just like we do with
the scm url and koji target). The architectures of the last-shipped build are
pulled from the manifest list api in the registry associated with koji.

There is a corresponding change in OSBS that allows Freshmaker to supply the
list of architectures which should be used for the new build here:
https://github.com/release-engineering/koji-containerbuild/pull/99

We're going from 1 to N architectures now, but in the future we'll make moves
again to go from N to N+1 architectures which warrants the complexity
introduced in this change.

This could use some additional test cases, but I'm out of steam for the night. Will revisit tomorrow to add those.

There's also:

'arm64': 'aarch64'

Would it be simpler to iterate over the archives of type image (I think) in Koji to figure out which arches? The benefit is that you won't need to query the registry.

One thing to note is that this change must be deployed after the corresponding osbs changes. Otherwise, all Freshmaker builds will fail with "arch-override is only allowed for scratch builds".

Will add. Thanks! metaxor may need this fix too.

Yes, but folks online last night all seemed to agreed that parsing the archive filenames was unsatisfactory.

Another approach would be to pull the metadata.json file from brew with all of the content generator data. It would have some relevant info.

I'm inclined to proceed with this approach (unless you see something scary in here w.r.t. a quay migration).

1 new commit added

  • Add arm64.
a year ago

1 new commit added

  • Feature flag: disable arch overrides by default.
a year ago

One thing to note is that this change must be deployed after the corresponding osbs changes.

I added a feature flag for it in 3e9a70d.

(unless you see something scary in here w.r.t. a quay migration)

Nope! Looks good as is. Staying away from metadata.json is actually a good idea :)

1 new commit added

  • Add tests for architecture discovery.
a year ago

OK - tests are added. This is ready for "final" review.

The feature flag has been added, so we can merge and deploy this before OSBS's arch_override feature is in place.

:thumbsup:

As usual, loved the explanation of the PR :)

Pull-Request has been merged by ralph

a year ago