#1527 Add integration test for reuse components
Merged 4 years ago by csomh. Opened 4 years ago by ejegrova.
ejegrova/fm-orchestrator test_reuse_components  into  master

@@ -51,3 +51,13 @@ 

      build_id_reused: 1235

      module: testmodule

      branch: reuse-all-components-branch

+   reuse_components:

+     # Build id to be reused for this test, it will be used to set a baseline.

+     baseline_build_id: 1234

+     build_id: 1235

+     module: testmodule

+     branch: reuse-components-build-branch

+     # Name of package that will be changed and will be rebuild in the build.

+     package: acl

+     # Name of the branch which is going to be built for the specified package.

+     component_branch: private-test-reuse-components

@@ -0,0 +1,55 @@ 

+ # -*- coding: utf-8 -*-

+ # SPDX-License-Identifier: MIT

+ 

+ import utils

+ 

+ 

+ def test_reuse_components(test_env, repo, koji):

+     """

+     Bump the commit of one of the components that MBS uses.

+     Bump the commit of the same testmodule that was mentioned in the preconditions.

+     Submit a testmodule build again with `fedpkg module-build -w --optional

+         rebuild_strategy=only-changed` (this is the default rebuild strategy).

+ 

+     Checks:

+     * Verify all the components were reused except for the component that had their commit change.

+     * Verify that the component with the changed commit was rebuilt.

+     """

+     repo.bump()

+     baseline_build = utils.Build(test_env["packaging_utility"], test_env["mbs_api"])

+     baseline_build.run(

+         "--watch",

+         "--optional",

+         "rebuild_strategy=all",

+         reuse=test_env["testdata"]["reuse_components"].get("baseline_build_id"),

+     )

+ 

+     package = test_env["testdata"]["reuse_components"].get("package")

+     component = utils.Component(

+         package,

+         test_env["testdata"]["reuse_components"].get("component_branch")

+     )

+     component.clone(test_env["packaging_utility"])

+     component.bump()

+ 

+     repo.bump()

+     build = utils.Build(test_env["packaging_utility"], test_env["mbs_api"])

+     build.run(

+         "--watch",

+         "--optional",

+         "rebuild_strategy=only-changed",

+         reuse=test_env["testdata"]["reuse_components"].get("build_id"),

+     )

+ 

+     comp_task_ids_base = baseline_build.component_task_ids()

+     comp_task_ids = build.component_task_ids()

+     comp_task_ids_base.pop('module-build-macros')

+     comp_task_ids.pop('module-build-macros')

+     changed_package_base_task_id = comp_task_ids_base.pop(package)

+     changed_package_task_id = comp_task_ids.pop(package)

+     assert changed_package_base_task_id != changed_package_task_id

+     assert comp_task_ids == comp_task_ids_base

+ 

+     assert build.components(package=package)[0]['state_name'] == 'COMPLETE'

+     state_reason = build.components(package=package)[0]['state_reason']

+     assert state_reason != "Reused component from previous module build"

file modified
+56 -11
@@ -9,10 +9,11 @@ 

  import koji

  import yaml

  import requests

+ import tempfile

  import sh

  

  our_sh = sh(_out=sys.stdout, _err=sys.stderr, _tee=True)

- from our_sh import Command, git  # noqa

+ from our_sh import Command, git, pushd  # noqa

  

  

  class Koji:
@@ -244,13 +245,21 @@ 

  

          :param string state: Koji build state the components should be in

          :param int batch: the number of the batch the components should be in

-         :param string: name of component (package):

+         :param string package: name of component (package):

          :return: List of components packages

          :rtype: list of strings

          """

          components = self.components(state, batch, package)

          return [item["package"] for item in components]

  

+     def component_task_ids(self):

+         """Dictionary containing all names of packages from build and appropriate task ids

+ 

+         :return: Dictionary containing name of packages and their task id

+         :rtype: dict

+         """

+         return {comp["package"]: comp["task_id"] for comp in self.components()}

+ 

      def batches(self):

          """

          Components of the module build separated in sets according to batches
@@ -268,18 +277,10 @@ 

  

          return batches

  

-     def component_task_ids(self):

-         """Dictionary containing all names of packages from build and appropriate task ids

- 

-             :return: Dictionary containing name of packages and their task id

-             :rtype: dict

-         """

-         return {comp["package"]: comp["task_id"] for comp in self.components()}

- 

      def wait_for_koji_task_id(self, package, batch, timeout=300, sleep=10):

          """Wait until the component is submitted to Koji (has a task_id)

  

-         :param string: name of component (package)

+         :param string package: name of component (package)

          :param int batch: the number of the batch the components should be in

          :param int timeout: time in seconds

          :param int sleep: time in seconds
@@ -325,3 +326,47 @@ 

              ):

                  return True

          return False

+ 

+ 

+ class Component:

+     """Wrapper class to work with git repositories of components

+ 

+     :attribute string module_name: name of the module stored in this repo

+     :attribute string branch: branch of the git repo that will be used

+     :attribute TemporaryDirectory _clone_dir: directory where is the clone of the repo

+     """

+     def __init__(self, module_name, branch):

+         self._module_name = module_name

+         self._branch = branch

+         self._clone_dir = None

+ 

+     def __del__(self):

+         self._clone_dir.cleanup()

+ 

+     def clone(self, packaging_utility):

+         """Clone the git repo of the component to be used by the test in a temporary location

+ 

+         Directory of the clone is stored in self._clone_dir.

+         :param string packaging_utility: packaging utility as defined in test.env.yaml

+         """

+         tempdir = tempfile.TemporaryDirectory()

+         args = [

+             "--branch",

+             self._branch,

+             f'rpms/{self._module_name}',

+             tempdir.name

+         ]

+         packaging_util = Command(packaging_utility)

+         packaging_util("clone", *args)

+         self._clone_dir = tempdir

+ 

+     def bump(self):

+         """Create a "bump" commit and push it in git"""

+         args = [

+             "--allow-empty",

+             "-m",

+             "Bump"

+         ]

+         with pushd(self._clone_dir.name):

+             git("commit", *args)

+             git("push")

no initial comment

Build #645 failed (commit: b68bd53bc689fea04aeb8727202df7b9389e679e).
Rebase or make new commits to rebuild.

rebased onto 433c21abf4e9b4186982d3ed54b05fd2aec8eee5

4 years ago

Build #646 failed (commit: 433c21abf4e9b4186982d3ed54b05fd2aec8eee5).
Rebase or make new commits to rebuild.

rebased onto bcf6b70d428d13a36dbc37c8e45b62c213fe03ac

4 years ago

Build #649 failed (commit: bcf6b70d428d13a36dbc37c8e45b62c213fe03ac).
Rebase or make new commits to rebuild.

rebased onto bf5588dd226723806fa68c1a9d3800701df02b4d

4 years ago

rebased onto 02d9b515a483cfe4b155106b9a7430530f438ffa

4 years ago

Build #652 failed (commit: bf5588dd226723806fa68c1a9d3800701df02b4d).
Rebase or make new commits to rebuild.

rebased onto 9f98edef1ffda940dd8b96591c8681f2a2d9087a

4 years ago

rebased onto f83fb43f8a64aa60d399a579adbd4e4cb7d9a476

4 years ago

Build #653 failed (commit: f83fb43f8a64aa60d399a579adbd4e4cb7d9a476).
Rebase or make new commits to rebuild.

Build #654 failed (commit: f83fb43f8a64aa60d399a579adbd4e4cb7d9a476).
Rebase or make new commits to rebuild.

rebased onto 0dd63271c16edf1130671f72309d93de91a1101a

4 years ago

Build #655 failed (commit: 0dd63271c16edf1130671f72309d93de91a1101a).
Rebase or make new commits to rebuild.

1 new commit added

  • Bump
4 years ago

Build #660 failed (commit: 146fba14d6ebf74bc82353ffff54bc9b02130b02).
Rebase or make new commits to rebuild.

1 new commit added

  • Bump
4 years ago

Build #661 failed (commit: 823fd58eb1dad7843c185bbdc940c5e308d91985).
Rebase or make new commits to rebuild.

rebased onto fd5286bbcd92ba459b0694e254e5ccd9fba5cfb7

4 years ago

Build #668 failed (commit: fd5286bbcd92ba459b0694e254e5ccd9fba5cfb7).
Rebase or make new commits to rebuild.

rebased onto 5ae1f216d67ab18f07dcea8a6e0cf0d66b4e5b7f

4 years ago

Build #671 failed (commit: 5ae1f216d67ab18f07dcea8a6e0cf0d66b4e5b7f).
Rebase or make new commits to rebuild.

rebased onto 61c2be83b63041688c3d961aa06177edc8c438ab

4 years ago

rebased onto 9710b7da7a3b789623d6fa3c58baf53d9d5659de

4 years ago

Build #673 failed (commit: 9710b7da7a3b789623d6fa3c58baf53d9d5659de).
Rebase or make new commits to rebuild.

Build #674 failed (commit: 9710b7da7a3b789623d6fa3c58baf53d9d5659de).
Rebase or make new commits to rebuild.

This method doesn't seem to be used anywhere. Would it be safe to remove it, and possible also redo the change which renames components to component_names?

Edit: pagure doesn't render it nicely, but this has a followup.

build_baseline -> baseline_build
build_id_baseline -> baseline_build_id

Sounds more English :)

This method doesn't seem to be used anywhere. Would it be safe to remove it, and possible also redo the change which renames components to component_names?

Oh, sorry, I see now where this should have been used, but for some reason it wasn't.

Still, I think filtering is not required atm, so components could return just all the component objects (ie: return self.modulemd["data"]["components"]["rpms"]).

(Though I have the feeling that we are missing the facts that "components" can have other types than "rpms". But we can leave fixing this ambiguity for later.)

The baseline build is not a scratch build, so we need to bump the commit before starting that. This repo.bump() should be called before calling baseline_build.run() above.

Then it should be called one more time before build.run() bellow.

It would be more correct to get the component information from the modulemd file stored in the repo, than from the koji build.

This can safely be kept as git("push") only, without any additional arguments.

This could be moved to Component.__del__, so that users don't have to remember to call it in order for the cleanup to happen.

module-build-macros is expected to be built every time, so it's safe use pop() without a default value and get a KeyError if module-build-macros happens to be missing.

You could use dict comprehension to get to the same mapping with less work:

{comp["package"]: comp["task_id"] for comp in self.components()}

Cloning could be done with package_utility, which would also take care of the ssh parts, and will figure out the correct URL to clone from. This way the corresponding parts could be removed from the test.

I'm planning to update the repo fixture the same way.

fedpkg/rhpkg clone --branch {branch} rpms/{component} {tempdir}

Bumping would still use git, though.

rebased onto bffac10f1f7430f5f0fe10fc3fa122933206144c

4 years ago

Build #682 failed (commit: bffac10f1f7430f5f0fe10fc3fa122933206144c).
Rebase or make new commits to rebuild.

rebased onto 0b16973400c402fc231011bbef9d3c812afdb80d

4 years ago

@csomh I did the changes, hopefully all of them.
Repo.components are as it was previously, I wanted to filter that for url but then I find out the url is only in the modulemd file produced by koji and forgot to change it back. Cloning via packaging utility solved all that.

Let's stay consistent and use sh.Command instead :)

rebased onto 93fc0a827084b88c41f0b62b416782498ce6e5d8

4 years ago

rebased onto 5ce2a8c0be4a500fb43006e58c65c3bcca864764

4 years ago

rebased onto b7fa4a79ed882aeb3d53db105f60948a32a99baf

4 years ago

rebased onto f61f7af

4 years ago

Pull-Request has been merged by csomh

4 years ago
Metadata
Flags
jenkins
success (100%)
Build #1398 successful (commit: f61f7afb)
4 years ago
jenkins
success (100%)
Build #1394 successful (commit: b7fa4a79)
4 years ago
jenkins
failure
Build #1393 failed (commit: 5ce2a8c0)
4 years ago
jenkins
failure
Build #1392 failed (commit: 93fc0a82)
4 years ago
jenkins
success (100%)
Build #1384 successful (commit: 0b169734)
4 years ago
c3i-jenkins
success (100%)
Build #684 successful (commit: 0b169734)
4 years ago
jenkins
failure
Build #1382 failed (commit: bffac10f)
4 years ago
c3i-jenkins
failure
Build #682 failed (commit: bffac10f)
4 years ago
jenkins
success (100%)
Build #1377 successful (commit: 9710b7da)
4 years ago
jenkins
failure
Build #1376 failed (commit: 9710b7da)
4 years ago
c3i-jenkins
failure
Build #674 failed (commit: 9710b7da)
4 years ago
c3i-jenkins
failure
Build #671 failed (commit: 5ae1f216)
4 years ago
jenkins
failure
Build #1372 failed (commit: fd5286bb)
4 years ago
c3i-jenkins
failure
Build #668 failed (commit: fd5286bb)
4 years ago
jenkins
failure
Build #1365 failed (commit: 823fd58e)
4 years ago
c3i-jenkins
failure
Build #661 failed (commit: 823fd58e)
4 years ago
c3i-jenkins
failure
Build #660 failed (commit: 146fba14)
4 years ago
jenkins
failure
Build #1364 failed (commit: 146fba14)
4 years ago
jenkins
failure
Build #1358 failed (commit: 0dd63271)
4 years ago
c3i-jenkins
failure
Build #655 failed (commit: 0dd63271)
4 years ago
jenkins
failure
Build #1357 failed (commit: f83fb43f)
4 years ago
jenkins
failure
Build #1356 failed (commit: f83fb43f)
4 years ago
jenkins
failure
Build #1355 failed (commit: f83fb43f)
4 years ago
c3i-jenkins
failure
Build #654 failed (commit: f83fb43f)
4 years ago
c3i-jenkins
failure
Build #652 failed (commit: bf5588dd)
4 years ago
jenkins
failure
Build #1354 failed (commit: bf5588dd)
4 years ago
jenkins
failure
Build #1350 failed (commit: bcf6b70d)
4 years ago
c3i-jenkins
failure
Build #649 failed (commit: bcf6b70d)
4 years ago
c3i-jenkins
failure
Build #646 failed (commit: 433c21ab)
4 years ago
jenkins
failure
Build #1347 failed (commit: 433c21ab)
4 years ago
jenkins
failure
Build #1346 failed (commit: b68bd53b)
4 years ago
c3i-jenkins
failure
Build #645 failed (commit: b68bd53b)
4 years ago