Learn more about these different git repos.
Other Git URLs
I have been trying now for a couple of weeks to build fedora-toolbox:36 in Koji but it fails now with KeyError: 'release'.
KeyError: 'release'
https://koji.fedoraproject.org/koji/taskinfo?taskID=83110679
2022-02-21 04:53:14,482 - osbs.core - DEBUG - before modification: b'{"kind":"Build","apiVersion":"build.openshift.io/v1","metadata":{"name":"fedora-toolbox-f36-28a3d-13","namespace":"osbs-fedora","selfLink":"/apis/build.openshift.io/v1/namespaces/osbs-fedora/builds/fedora-toolbox-f36-28a3d-13","uid":"29b2614d-92d2-11ec-94ad-525400de6908","resourceVersion":"84118909","creationTimestamp":"2022-02-21T04:53:08Z","labels":{"buildconfig":"fedora-toolbox-f36-28a3d","git-branch":"f36","git-full-repo":"httpssrc.fedoraproject.orgcontainerfedora-toolbox.git","git-repo-name":"fedora-toolbox","is_autorebuild":"false","koji-task-id":"83110679","openshift.io/build-config.name":"fedora-toolbox-f36-28a3d","openshift.io/build.start-policy":"Serial","original-koji-task-id":"83110679"},"annotations":{"openshift.io/build-config.name":"fedora-toolbox-f36-28a3d","openshift.io/build.number":"13","openshift.io/build.pod-name":"fedora-toolbox-f36-28a3d-13-build"},"ownerReferences":[{"apiVersion":"build.openshift.io/v1","kind":"BuildConfig","name":"fedora-toolbox-f36-28a3d","uid":"92197e79-8ae7-11ec-808a-525400de6908","controller":true}]},"spec":{"serviceAccount":"builder","source":{"type":"Git","git":{"uri":"https://src.fedoraproject.org/container/fedora-toolbox.git","ref":"b2d0ebbad6324c6d9b1c5f8028839c1c7649305a"}},"strategy":{"type":"Custom","customStrategy":{"from":{"kind":"DockerImage","name":"buildroot:latest"},"env":[{"name":"REACTOR_CONFIG","valueFrom":{"configMapKeyRef":{"name":"reactor-config-map","key":"config.yaml"}}},{"name":"USER_PARAMS","value":"{\\"arrangement_version\\": 6, \\"base_image\\": \\"registry.fedoraproject.org/fedora:36\\", \\"build_from\\": \\"image:buildroot:latest\\", \\"build_image\\": \\"buildroot:latest\\", \\"build_json_dir\\": \\"/usr/share/osbs/\\", \\"build_type\\": \\"orchestrator\\", \\"component\\": \\"fedora-toolbox\\", \\"customize_conf\\": \\"worker_customize.json\\", \\"git_branch\\": \\"f36\\", \\"git_commit_depth\\": 1, \\"git_ref\\": \\"b2d0ebbad6324c6d9b1c5f8028839c1c7649305a\\", \\"git_uri\\": \\"https://src.fedoraproject.org/container/fedora-toolbox.git\\", \\"image_tag\\": \\"petersen/fedora-toolbox:f36-container-candidate-24778-20220221045307\\", \\"imagestream_name\\": \\"registry.fedoraproject.org-fedora-toolbox\\", \\"kind\\": \\"build_user_params\\", \\"koji_target\\": \\"f36-container-candidate\\", \\"koji_task_id\\": 83110679, \\"name\\": \\"fedora-toolbox-f36-28a3d\\", \\"orchestrator_deadline\\": 4, \\"platforms\\": [\\"x86_64\\", \\"aarch64\\"], \\"reactor_config_map\\": \\"reactor-config-map\\", \\"trigger_imagestreamtag\\": \\"registry.fedoraproject.org-fedora:36\\", \\"user\\": \\"petersen\\", \\"worker_deadline\\": 3}"},{"name":"OPENSHIFT_CUSTOM_BUILD_BASE_IMAGE","value":"buildroot:latest"}],"exposeDockerSocket":true,"secrets":[{"secretSource":{"name":"x86-64-orchestrator"},"mountPath":"/var/run/secrets/atomic-reactor/x86-64-orchestrator"},{"secretSource":{"name":"client-config-secret"},"mountPath":"/var/run/secrets/atomic-reactor/client-config-secret"},{"secretSource":{"name":"aarch64-orchestrator"},"mountPath":"/var/run/secrets/atomic-reactor/aarch64-orchestrator"},{"secretSource":{"name":"odcs-oidc-secret"},"mountPath":"/var/run/secrets/atomic-reactor/odcs-oidc-secret"},{"secretSource":{"name":"v2-registry-dockercfg"},"mountPath":"/var/run/secrets/atomic-reactor/v2-registry-dockercfg"}]}},"output":{"to":{"kind":"DockerImage","name":"petersen/fedora-toolbox:f36-container-candidate-24778-20220221045307"},"pushSecret":{"name":"builder-dockercfg-zmvzd"}},"resources":{},"postCommit":{},"completionDeadlineSeconds":14400,"nodeSelector":null,"triggeredBy":[{"message":"Image change","imageChangeBuild":{"imageID":"registry.fedoraproject.org/fedora@sha256:a82f104b2f2f0e9678c24bc9660020e603d536b2083981cce00e9e590aad017e","fromRef":{"kind":"ImageStreamTag","name":"registry.fedoraproject.org-fedora:36"}}}]},"status":{"phase":"Running","startTimestamp":"2022-02-21T04:53:08Z","outputDockerImageReference":"petersen/fedora-toolbox:f36-container-candidate-24778-20220221045307","config":{"kind":"BuildConfig","namespace":"osbs-fedora","name":"fedora-toolbox-f36-28a3d"},"output":{}}}\n' 2022-02-21 04:53:14,505 - osbs.http - DEBUG - cleaning up 2022-02-21 04:53:14,506 platform:- - atomic_reactor.plugin - DEBUG - plugin 'store_metadata_in_osv3' finished in 0s 2022-02-21 04:53:14,507 platform:- - atomic_reactor.plugin - DEBUG - running plugin 'remove_built_image' 2022-02-21 04:53:14,507 platform:- - atomic_reactor.plugin - INFO - running plugin instance with args: '{}' 2022-02-21 04:53:14,507 platform:- - atomic_reactor.plugin - DEBUG - plugin 'remove_built_image' finished in 0s 2022-02-21 04:53:14,507 platform:- - atomic_reactor.plugin - DEBUG - running plugin 'remove_worker_metadata' 2022-02-21 04:53:14,507 platform:- - atomic_reactor.plugin - INFO - running plugin instance with args: '{}' 2022-02-21 04:53:14,507 platform:- - atomic_reactor.plugin - DEBUG - plugin 'remove_worker_metadata' finished in 0s 2022-02-21 04:53:14,514 platform:- - atomic_reactor.inner - INFO - Dockerfile used for build: None 2022-02-21 04:53:14,514 platform:- - atomic_reactor.inner - ERROR - image build failed: plugin 'pull_base_image' raised an exception: KeyError: 'release' Traceback (most recent call last): File "/usr/lib/python3.9/site-packages/atomic_reactor/plugin.py", line 268, in run plugin_response = plugin_instance.run() File "/usr/lib/python3.9/site-packages/atomic_reactor/plugins/pre_pull_base_image.py", line 107, in run self._validate_platforms_in_image(image) File "/usr/lib/python3.9/site-packages/atomic_reactor/plugins/pre_pull_base_image.py", line 331, in _validate_platforms_in_image manifest_list = self._get_manifest_list(image) File "/usr/lib/python3.9/site-packages/atomic_reactor/plugins/pre_pull_base_image.py", line 300, in _get_manifest_list release = config_blob['config']['Labels']['release'] KeyError: 'release' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/bin/atomic-reactor", line 33, in <module> sys.exit(load_entry_point('atomic-reactor==1.6.47', 'console_scripts', 'atomic-reactor')()) File "/usr/lib/python3.9/site-packages/atomic_reactor/cli/main.py", line 318, in run cli.run() File "/usr/lib/python3.9/site-packages/atomic_reactor/cli/main.py", line 300, in run args.func(args) File "/usr/lib/python3.9/site-packages/atomic_reactor/cli/main.py", line 98, in cli_inside_build build_inside(input_method=args.input, input_args=args.input_arg, File "/usr/lib/python3.9/site-packages/atomic_reactor/inner.py", line 615, in build_inside build_result = dbw.build_docker_image() File "/usr/lib/python3.9/site-packages/atomic_reactor/inner.py", line 492, in build_docker_image prebuild_runner.run() File "/usr/lib/python3.9/site-packages/atomic_reactor/plugin.py", line 309, in run raise PluginFailedException(msg) atomic_reactor.plugin.PluginFailedException: plugin 'pull_base_image' raised an exception: KeyError: 'release'
It might be getting slightly further than before (or not I am not sure) previously I was seeing an error like "unknown image fedora:36".
Somewhat similarly for Rawhide fedora-toolbox container-build's I am getting:
https://koji.fedoraproject.org/koji/taskinfo?taskID=83111189
83111189 buildContainer (noarch): open (buildvm-x86-05.iad2.fedoraproject.org) -> FAILED: Fault: <Fault 1: 'Traceback (most recent call last):\n File "/usr/lib/python3.10/site-packages/osbs/api.py", line 617, in _create_build_config_and_build\n self.get_image_stream_tag_with_retry(tag_id).json()\n File "/usr/lib/python3.10/site-packages/osbs/api.py", line 71, in catch_exceptions\n return func(*args, **kwargs)\n File "/usr/lib/python3.10/site-packages/osbs/api.py", line 1203, in get_image_stream_tag_with_retry\n return self.os.get_image_stream_tag_with_retry(tag_id)\n File "/usr/lib/python3.10/site-packages/osbs/utils/__init__.py", line 562, in retry\n return retry_func.go(func, *args, **kwargs)\n File "/usr/lib/python3.10/site-packages/osbs/utils/__init__.py", line 619, in go\n return func(*args, **kwargs)\n File "/usr/lib/python3.10/site-packages/osbs/core.py", line 883, in get_image_stream_tag_with_retry\n check_response(response, log_level=logging.DEBUG)\n File "/usr/lib/python3.10/site-packages/osbs/core.py", line 68, in check_response\n raise OsbsResponseException(message=content, status_code=response.status_code)\nosbs.exceptions.OsbsResponseException: b\'{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"imagestreamtags.image.openshift.io \\\\"registry.fedoraproject.org-fedora:37\\\\" not found","reason":"NotFound","details":{"name":"registry.fedoraproject.org-fedora:37","group":"image.openshift.io","kind":"imagestreamtags"},"code":404}\\n\'\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File "/usr/lib/python3.10/site-packages/koji/daemon.py", line 1458, in runTask\n response = (handler.run(),)\n File "/usr/lib/python3.10/site-packages/koji/tasks.py", line 335, in run\n return koji.util.call_with_argcheck(self.handler, self.params, self.opts)\n File "/usr/lib/python3.10/site-packages/koji/util.py", line 271, in call_with_argcheck\n return func(*args, **kwargs)\n File "/usr/lib/koji-builder-plugins/builder_containerbuild.py", line 1029, in handler\n result = self.createContainer(**kwargs)\n File "/usr/lib/koji-builder-plugins/builder_containerbuild.py", line 816, in createContainer\n build_response = create_method(**orchestrator_create_build_args)\n File "/usr/lib/python3.10/site-packages/osbs/api.py", line 71, in catch_exceptions\n return func(*args, **kwargs)\n File "/usr/lib/python3.10/site-packages/osbs/api.py", line 974, in create_orchestrator_build\n return self._do_create_prod_build(**kwargs)\n File "/usr/lib/python3.10/site-packages/osbs/api.py", line 819, in _do_create_prod_build\n response = self._create_build_config_and_build(build_request)\n File "/usr/lib/python3.10/site-packages/osbs/api.py", line 621, in _create_build_config_and_build\n raise OsbsException(\'Provided base image does not exist: \'\nosbs.exceptions.OsbsException: Provided base image does not exist: registry.fedoraproject.org/fedora:37\n'>
The rawhide builds have already been failing for months, sadly.
I can open a separate ticket for that if it is better.
adding a note. This is causing some CI to fail in Fedora CoreOS: https://github.com/coreos/fedora-coreos-tracker/issues/1103
Metadata Update from @mohanboddu: - Issue tagged with: medium-gain, medium-trouble, ops
I opened a separate ticket now about the missing fedora:37 tag in #10671.
fedora:37
(But noting that the last successful fedora-toolbox:36 rawhide build was 2021-10-13.)
Another data-point is that we noticed that fedpkg container-build --scratch succeeds for f36.
fedpkg container-build --scratch
Now is fedora:37 has been fixed fedora-toolbox:37 is failing in the same way as 36:
https://koji.fedoraproject.org/koji/taskinfo?taskID=83536309
"/var/run/secrets/atomic-reactor/aarch64-orchestrator"},{"secretSource":{"name":"x86-64-orchestrator"},"mountPath":"/var/run/secrets/atomic-reactor/x86-64-orchestrator"},{"secretSource":{"name":"v2-registry-dockercfg"},"mountPath":"/var/run/secrets/atomic-reactor/v2-registry-dockercfg"}]}},"output":{"to":{"kind":"DockerImage","name":"petersen/fedora-toolbox:rawhide-container-candidate-88055-20220302050114"},"pushSecret":{"name":"builder-dockercfg-zmvzd"}},"resources":{},"postCommit":{},"completionDeadlineSeconds":14400,"nodeSelector":null,"triggeredBy":[{"message":"Image change","imageChangeBuild":{"imageID":"registry.fedoraproject.org/fedora@sha256:3c4b3d1a84439c11f50518a58f52e728d76615b40bb3a984414baa7d4bbe4f6b","fromRef":{"kind":"ImageStreamTag","name":"registry.fedoraproject.org-fedora:37"}}}]},"status":{"phase":"Running","startTimestamp":"2022-03-02T05:01:15Z","outputDockerImageReference":"petersen/fedora-toolbox:rawhide-container-candidate-88055-20220302050114","config":{"kind":"BuildConfig","namespace":"osbs-fedora","name":"fedora-toolbox-rawhide-342e8"},"output":{}}}\n' 2022-03-02 05:01:21,092 - osbs.http - DEBUG - cleaning up 2022-03-02 05:01:21,093 platform:- - atomic_reactor.plugin - DEBUG - plugin 'store_metadata_in_osv3' finished in 0s 2022-03-02 05:01:21,093 platform:- - atomic_reactor.plugin - DEBUG - running plugin 'remove_built_image' 2022-03-02 05:01:21,093 platform:- - atomic_reactor.plugin - INFO - running plugin instance with args: '{}' 2022-03-02 05:01:21,093 platform:- - atomic_reactor.plugin - DEBUG - plugin 'remove_built_image' finished in 0s 2022-03-02 05:01:21,093 platform:- - atomic_reactor.plugin - DEBUG - running plugin 'remove_worker_metadata' 2022-03-02 05:01:21,093 platform:- - atomic_reactor.plugin - INFO - running plugin instance with args: '{}' 2022-03-02 05:01:21,093 platform:- - atomic_reactor.plugin - DEBUG - plugin 'remove_worker_metadata' finished in 0s 2022-03-02 05:01:21,102 platform:- - atomic_reactor.inner - INFO - Dockerfile used for build: None 2022-03-02 05:01:21,102 platform:- - atomic_reactor.inner - ERROR - image build failed: plugin 'pull_base_image' raised an exception: KeyError: 'release' Traceback (most recent call last): File "/usr/lib/python3.9/site-packages/atomic_reactor/plugin.py", line 268, in run plugin_response = plugin_instance.run() File "/usr/lib/python3.9/site-packages/atomic_reactor/plugins/pre_pull_base_image.py", line 107, in run self._validate_platforms_in_image(image) File "/usr/lib/python3.9/site-packages/atomic_reactor/plugins/pre_pull_base_image.py", line 331, in _validate_platforms_in_image manifest_list = self._get_manifest_list(image) File "/usr/lib/python3.9/site-packages/atomic_reactor/plugins/pre_pull_base_image.py", line 300, in _get_manifest_list release = config_blob['config']['Labels']['release'] KeyError: 'release' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/bin/atomic-reactor", line 33, in <module> sys.exit(load_entry_point('atomic-reactor==1.6.47', 'console_scripts', 'atomic-reactor')()) File "/usr/lib/python3.9/site-packages/atomic_reactor/cli/main.py", line 318, in run cli.run() File "/usr/lib/python3.9/site-packages/atomic_reactor/cli/main.py", line 300, in run args.func(args) File "/usr/lib/python3.9/site-packages/atomic_reactor/cli/main.py", line 98, in cli_inside_build build_inside(input_method=args.input, input_args=args.input_arg, File "/usr/lib/python3.9/site-packages/atomic_reactor/inner.py", line 615, in build_inside build_result = dbw.build_docker_image() File "/usr/lib/python3.9/site-packages/atomic_reactor/inner.py", line 492, in build_docker_image prebuild_runner.run() File "/usr/lib/python3.9/site-packages/atomic_reactor/plugin.py", line 309, in run raise PluginFailedException(msg) atomic_reactor.plugin.PluginFailedException: plugin 'pull_base_image' raised an exception: KeyError: 'release'
Is it just a configuration issue for this release key?
release
We have a theory that it might be caused by this commit: https://src.fedoraproject.org/container/fedora-toolbox/c/40a5f298f9ff884a7c6b764793887b12ea08d6ff
we don't actually support autorebuild and it might be messing up because it can't figure out the release.
Can you try reverting that and see if it builds?
credit to @darknao for the theory, which seems pretty plausible to me.
Ah right that would certainly explain it - since I don't think any build succeeded since it was added :( - thank you!
Thank you so much - I have already built fedora-toolbox:36 and fedora-toolbox:37 now with that change.
Thanks @darknao for catching the problem and Kevin
Metadata Update from @petersen: - Issue close_status updated to: Fixed - Issue status updated to: Closed (was: Open)
Thank you @darknao and @kevin for your help. Finally I can see F36 toolbox built on koji.
Log in to comment on this ticket.