#1200 DONT MERGE: pre-release testing
Closed 2 years ago by praiskup. Opened 4 years ago by praiskup.
Unknown source pre-release  into  main

@@ -116,6 +116,7 @@

          project_dirnames = data["project_dirnames"]

          chroots = data["chroots"]

          appstream = data["appstream"]

+         devel = data["devel"]

  

          result = ActionResult.SUCCESS

  
@@ -132,7 +133,8 @@

                  except FileExistsError:

                      pass

  

-                 if not call_copr_repo(repo, appstream=appstream, logger=self.log):

+                 if not call_copr_repo(repo, appstream=appstream, devel=devel,

+                                       logger=self.log):

                      result = ActionResult.FAILURE

  

          return result

@@ -31,7 +31,7 @@

  

  MAX_HOST_ATTEMPTS = 3

  MAX_SSH_ATTEMPTS = 5

- MIN_BUILDER_VERSION = "0.51.1.dev"

+ MIN_BUILDER_VERSION = "0.54.1.dev"

  CANCEL_CHECK_PERIOD = 5

  

  MESSAGES = {

@@ -8,7 +8,7 @@

  from copr_common.request import SafeRequest, RequestError

  from copr_backend.exceptions import FrontendClientException

  

- MIN_FE_BE_API = 2

+ MIN_FE_BE_API = 3

  

  class FrontendClient:

      """

file modified
+4 -2
@@ -761,7 +761,8 @@

              "ownername": "foo",

              "projectname": "bar",

              "appstream": True,

-             "project_dirnames": ["bar"]

+             "project_dirnames": ["bar"],

+             "devel": False,

          })

          self.opts.destdir = tmp_dir

  
@@ -794,7 +795,8 @@

              "ownername": "foo",

              "projectname": "bar",

              "appstream": True,

-             "project_dirnames": ["bar"]

+             "project_dirnames": ["bar"],

+             "devel": False,

          })

          self.opts.destdir = tmp_dir

          test_action = Action.create_from(

@@ -6,6 +6,13 @@

  source "$HERE/config"

  source "$HERE/helpers"

  

+ check_repo()

+ {

+     url=$1/repodata/repomd.xml

+     rlLog "Checking $url"

+     curl --fail "$url" >/dev/null

+ }

+ 

  rlJournalStart

      rlPhaseStartSetup

          setup_checks
@@ -16,11 +23,39 @@

      rlPhaseEnd

  

      rlPhaseStartTest

-         rlRun "copr-cli build $PROJECT_F $HELLO"

-         rlRun "curl --silent $BACKEND_URL/results/$PROJECT_F/$CHROOT/devel/repodata/ | grep \"404.*Not Found\"" 0

+         while true; do

+             success=:

+             for url in \

+                 $BACKEND_URL/results/$PROJECT_T/$CHROOT/devel \

+                 $BACKEND_URL/results/$PROJECT_T/$CHROOT \

+                 $BACKEND_URL/results/$PROJECT_F/$CHROOT ;

+             do

+                 # all those must be created ^^^

+                 check_repo "$url" && continue

+                 success=false

+                 break

+             done

+             $success && break

+             seconds=5

+             rlLog "The repositories are not prepared, waiting ${seconds}s more"

+             sleep "$seconds"

+         done

  

+         rlRun "copr-cli build $PROJECT_F $HELLO"

          rlRun "copr-cli build $PROJECT_T $HELLO"

-         rlRun "curl --silent $BACKEND_URL/results/$PROJECT_T/$CHROOT/devel/repodata/ | grep -E \"404.*Not Found\"" 1

+ 

+         rlLog "The devel repo must not exist in $PROJECT_F, till we flip the config"

+         rlRun "check_repo $BACKEND_URL/results/$PROJECT_F/$CHROOT/devel" 22

+ 

+         rlRun "copr-cli modify --disable_createrepo true $PROJECT_F"

+         while true; do

+             repo=$BACKEND_URL/results/$PROJECT_F/$CHROOT/devel

+             check_repo "$repo" && break

+             seconds=5

+             rlLog "The repo $repo is not prepared yet, waiting ${seconds}s more"

+             sleep "$seconds"

+         done

+ 

      rlPhaseEnd

  

      rlPhaseStartCleanup

file modified
+19 -21
@@ -24,27 +24,25 @@

  

  The following table describes situations when we execute ``/bin/createrepo_c``.

  

- +----------------------------------+------------------------------+--------------------------+--------------------------+

- | situation                        | what copr dirs               | what chroots             | devel/normal repo        |

- +==================================+==============================+==========================+==========================+

- | new Project created              | main                         | all enabled              | ACR reflected            |

- +----------------------------------+------------------------------+--------------------------+--------------------------+

- | new chroot(s) enabled            | all                          | only related chroots     | ACR reflected            |

- +----------------------------------+------------------------------+--------------------------+--------------------------+

- | new (PR) copr dir                | the one just created         | all enabled              | ACR reflected            |

- +----------------------------------+------------------------------+--------------------------+--------------------------+

- | ACR disabled                     | all                          | all enabled              | devel                    |

- +----------------------------------+------------------------------+--------------------------+--------------------------+

- | ACR enable                       | all                          | all enabled              | normal                   |

- +----------------------------------+------------------------------+--------------------------+--------------------------+

- | post-build                       | affected                     | only related chroots     | ACR reflected            |

- +----------------------------------+------------------------------+--------------------------+--------------------------+

- | manual createrepo event          | all                          | all enabled              | non-devel only           |

- +----------------------------------+------------------------------+--------------------------+--------------------------+

- | delete build event               | affected                     | only related chroots     | both for prod packages   |

- +----------------------------------+------------------------------+--------------------------+--------------------------+

- | prunerepo script                 | affected                     | affected                 | normal                   |

- +----------------------------------+------------------------------+--------------------------+--------------------------+

+ +----------------------------------+------------------------------+--------------------------+---------------------------+

+ | situation                        | what copr dirs               | what chroots             | devel/normal repo         |

+ +==================================+==============================+==========================+===========================+

+ | new Project created              | main                         | all                      | normal + devel if not ACR |

+ +----------------------------------+------------------------------+--------------------------+---------------------------+

+ | new chroot(s) added              | all                          | only new chroots         | Normal + devel if not ACR |

+ +----------------------------------+------------------------------+--------------------------+---------------------------+

+ | new CoprDir added (pull-request) | the one just created         | all                      | Normal + devel if not ACR |

+ +----------------------------------+------------------------------+--------------------------+---------------------------+

+ | ACR toggle                       | all                          | all                      | normal if ACR else devel  |

+ +----------------------------------+------------------------------+--------------------------+---------------------------+

+ | post-build                       | affected                     | affected                 | normal if ACR else devel  |

+ +----------------------------------+------------------------------+--------------------------+---------------------------+

+ | manual createrepo event          | all                          | all                      | normal                    |

+ +----------------------------------+------------------------------+--------------------------+---------------------------+

+ | delete build event               | affected                     | affected                 | both                      |

+ +----------------------------------+------------------------------+--------------------------+---------------------------+

+ | prunerepo script                 | affected                     | affected                 | normal                    |

+ +----------------------------------+------------------------------+--------------------------+---------------------------+

  

  The ``copr dirs`` can be either the default project directory (e.g.

  ``jsmith/coolproject``), or pull-request directories (e.g.

@@ -77,7 +77,8 @@

          return action

  

      @classmethod

-     def send_createrepo(cls, copr, dirnames=None, chroots=None):

+     def send_createrepo(cls, copr, dirnames=None, chroots=None, devel=None,

+                         priority=None):

          """

          Create a new createrepo Action in queue for given copr.

  
@@ -87,6 +88,11 @@

              otherwise the createrepo action is run in all dirnames.

          :param chroots: List of MockChroot *names*.  Optional, by default

              createrepo is run in all active chroots.

+         :param devel: If devel=True, the createrepo action is run only in the

+             devel/ subdirectory, if devel=False the createrepo is run in the

+             "production" directory.  When devel=None, depending on the current

+             auto_createrepo settings, both production and devel might be

+             regenerated.

          """

          possible_dirnames = [copr_dir.name for copr_dir in copr.dirs]

          if not dirnames:
@@ -109,15 +115,28 @@

              "chroots": chroots,

              "appstream": copr.appstream,

          }

-         action = models.Action(

-             action_type=ActionTypeEnum("createrepo"),

-             object_type="repository",

-             object_id=0,

-             data=json.dumps(data_dict),

-             created_on=int(time.time()),

-         )

-         db.session.add(action)

-         return action

+ 

+         run_in = set()

+         if devel is None:

+             run_in.add('prod')

+             if not copr.auto_createrepo:

+                 run_in.add('dev')

+         else:

+             run_in.add('dev' if devel else 'prod')

+ 

+         for run in run_in:

+             data_dict["devel"] = run == "dev"

+             action = models.Action(

+                 action_type=ActionTypeEnum("createrepo"),

+                 object_type="repository",

+                 object_id=0,

+                 data=json.dumps(data_dict),

+                 created_on=int(time.time()),

+             )

+             if priority is not None:

+                 action.priority = priority

+             db.session.add(action)

+ 

  

      @classmethod

      def send_delete_copr(cls, copr):

@@ -796,8 +796,6 @@

  

          copr_dir = None

          if copr_dirname:

-             if not copr_dirname.startswith(copr.name+':') and copr_dirname != copr.name:

-                 raise MalformedArgumentException("Copr dirname not starting with copr name.")

              copr_dir = coprs_logic.CoprDirsLogic.get_or_create(copr, copr_dirname)

  

          chroots = cls._chroots_to_add(package, chroots, package_chroots_subset)

@@ -11,7 +11,7 @@

  from sqlalchemy import not_

  from sqlalchemy import desc

  from sqlalchemy.event import listens_for

- from sqlalchemy.orm.attributes import NEVER_SET

+ from sqlalchemy.orm.attributes import NEVER_SET, NO_VALUE

  from sqlalchemy.orm.exc import NoResultFound

  from sqlalchemy.orm.attributes import get_history

  
@@ -567,17 +567,24 @@

          return coprdir

  

      @classmethod

-     def get_or_create(cls, copr, dirname, main=False):

+     def get_or_create(cls, copr, dirname):

+         """

+         Create a CoprDir on-demand, e.g. before pull-request builds is

+         submitted.  We don't create the "main" CoprDirs here (those are created

+         when a new project is created.

+         """

          copr_dir = cls.get_by_copr_safe(copr, dirname)

- 

          if copr_dir:

              return copr_dir

  

-         copr_dir = models.CoprDir(

-             name=dirname, copr=copr, main=main)

+         if not dirname.startswith(copr.name+':'):

+             raise MalformedArgumentException(

+                 "Copr dirname must start with '{}:' prefix".format(

+                 copr.name,

+             ))

  

+         copr_dir = models.CoprDir(name=dirname, copr=copr, main=False)

          ActionsLogic.send_createrepo(copr, dirnames=[dirname])

- 

          db.session.add(copr_dir)

          return copr_dir

  
@@ -754,12 +761,17 @@

  @listens_for(models.Copr.auto_createrepo, 'set')

  def on_auto_createrepo_change(target_copr, value_acr, old_value_acr, initiator):

      """ Emit createrepo action when auto_createrepo re-enabled"""

-     if old_value_acr == NEVER_SET:

-         #  created new copr, not interesting

+     if old_value_acr in [NEVER_SET, NO_VALUE]:

+         # Created a new copr.  We handle the createrepo actions within

+         # CoprsLogic.add() and the CoprChrootsLogic.new_from_names() called

+         # inside.

+         return

+ 

+     if old_value_acr == value_acr:

+         # no change

          return

-     if not old_value_acr and value_acr:

-         #  re-enabled

-         ActionsLogic.send_createrepo(target_copr)

+ 

+     ActionsLogic.send_createrepo(target_copr, devel=not value_acr)

  

  

  class BranchesLogic(object):
@@ -836,8 +848,7 @@

              db.session.add(

                  models.CoprChroot(copr=copr, mock_chroot=mock_chroot))

  

-         action = ActionsLogic.send_createrepo(copr)

-         action.priority = ActionPriorityEnum("highest")

+         ActionsLogic.send_createrepo(copr, priority=ActionPriorityEnum("highest"))

  

      @classmethod

      def create_chroot(cls, user, copr, mock_chroot, buildroot_pkgs=None, repos=None, comps=None, comps_name=None,

@@ -276,6 +276,6 @@

      This function will regenerate all repository metadata for a project.

      """

      with db_session_scope():

-         ActionsLogic.send_createrepo(copr)

+         ActionsLogic.send_createrepo(copr, devel=False)

  

      return flask.jsonify(to_dict(copr))

@@ -27,7 +27,7 @@

      do something new with the protocol.  On the Backend/builder side we can

      setup the version according to our needs.

      """

-     response.headers['Copr-FE-BE-API-Version'] = '2'

+     response.headers['Copr-FE-BE-API-Version'] = '3'

      return response

  

  

@@ -754,7 +754,7 @@

              "You are not allowed to recreate repository metadata of copr with id {}.".format(copr_id), "error")

          return flask.redirect(url_for_copr_details(copr))

  

-     actions_logic.ActionsLogic.send_createrepo(copr)

+     actions_logic.ActionsLogic.send_createrepo(copr, devel=False)

      db.session.commit()

  

      flask.flash("Repository metadata in all directories will be regenerated...", "success")

@@ -101,6 +101,37 @@

  

          # make sure no initial build was submitted

          assert self.models.Build.query.first() is None

+         # one createrepo action generated

+         actions = ActionsLogic.get_many().all()

+         assert len(actions) == 2

+         for action in actions:

+             if action.action_type == ActionTypeEnum("createrepo"):

+                 assert json.loads(actions[0].data)["devel"] is False

+ 

+     @TransactionDecorator("u1")

+     @pytest.mark.usefixtures("f_users", "f_mock_chroots", "f_db")

+     def test_copr_new_ACR_OFF(self):

+         r = self.test_client.post(

+             "/coprs/{0}/new/".format(self.u1.name),

+             data={

+                 "name": "foo",

+                 "chroots": ["fedora-rawhide-i386"],

+                 "arches": ["i386"],

+                 "disable_createrepo": True,

+             },

+             follow_redirects=True)

+ 

+         assert self.models.Copr.query\

+             .order_by(desc(models.Copr.created_on))\

+             .filter(self.models.Copr.name == "foo").first()

+         assert self.success_string.encode("utf-8") in r.data

+ 

+         # make sure no initial build was submitted

+         assert self.models.Build.query.first() is None

+ 

+         actions = ActionsLogic.get_many().filter_by(action_type=3).order_by('id').all()

+         assert {True, False} == {json.loads(action.data)["devel"]

+                                  for action in actions}

  

      @TransactionDecorator("u3")

      def test_copr_new_exists_for_another_user(self, f_users, f_coprs,
@@ -424,8 +455,8 @@

          assert len(mock_chroots) == 1

  

      @TransactionDecorator("u1")

-     def test_re_enable_auto_createrepo_produce_action(

-             self, f_users, f_coprs, f_mock_chroots, f_db):

+     @pytest.mark.usefixtures("f_users", "f_coprs", "f_mock_chroots", "f_db")

+     def test_changed_ACR_produces_action(self):

  

          self.db.session.add_all(

              [self.u1, self.c1, self.mc1, self.mc2, self.mc3])
@@ -434,13 +465,15 @@

          coprname = self.c1.name

          copr_id = self.c1.id

          chroot = self.mc1.name

+         chroots = {self.mc1.name, self.mc2.name, self.mc3.name}

  

-         # 1.ensure ACR enabled

- 

+         # 1. Ensure ACR is enabled

          self.db.session.commit()

          c1_actual = CoprsLogic.get(self.u1.name, self.c1.name).one()

          assert c1_actual.auto_createrepo

-         # 1. disabling ACR

+         assert len(ActionsLogic.get_many().all()) == 0

+ 

+         # 2. Disabling ACR (generates createrepo action in devel/

          self.test_client.post(

              "/coprs/{0}/{1}/update/".format(username, coprname),

              data={"name": coprname, "chroots": [chroot], "id": copr_id,
@@ -448,33 +481,52 @@

              follow_redirects=True

          )

          self.db.session.commit()

- 

-         # check current status

          c1_actual = CoprsLogic.get(username, coprname).one()

          assert not c1_actual.auto_createrepo

-         # no actions issued before

-         assert len(ActionsLogic.get_many().all()) == 0

+         assert len(ActionsLogic.get_many().all()) == 1

+         action = ActionsLogic.get_many().one()

+         handled_ids = {action.id}

  

-         # 2. enabling ACR

+         expected_data = {

+             "ownername": "user1",

+             "projectname": "foocopr",

+             "project_dirnames": ["foocopr"],

+             "chroots": ["fedora-18-x86_64"],

+             "appstream": True,

+             "devel": True,

+         }

+ 

+         assert json.loads(action.data) == expected_data

+ 

+         # 3. Re-enable ACR, and enable two new chroots

          self.test_client.post(

              "/coprs/{0}/{1}/update/".format(username, coprname),

-             data={"name": coprname, "chroots": [chroot], "id": copr_id,

+             data={"name": coprname, "chroots": list(chroots), "id": copr_id,

                    "disable_createrepo": "false"},

              follow_redirects=True

          )

          self.db.session.commit()

- 

          c1_actual = CoprsLogic.get(username, coprname).one()

- 

-         # ACR enabled

          assert c1_actual.auto_createrepo

-         # added action

-         assert len(ActionsLogic.get_many().all()) > 0

-         action = ActionsLogic.get_many(action_type=ActionTypeEnum("createrepo")).one()

- 

-         data_dict = json.loads(action.data)

-         assert data_dict["ownername"] == username

-         assert data_dict["projectname"] == coprname

+         actions = ActionsLogic.get_many().all()

+         assert len(actions) == 3

+ 

+         expected_chroots = chroots

+ 

+         for action in ActionsLogic.get_many():

+             if action.id in handled_ids:

+                 continue

+             expected_data["devel"] = False

+             # TODO: the form re-sets appstream to False for None value

+             expected_data["appstream"] = False

+             data = json.loads(action.data)

+             expected_data["chroots"] = data["chroots"]

+             for chroot in data["chroots"]:

+                 assert chroot in expected_chroots

+                 expected_chroots.remove(chroot)

+             assert data == expected_data

+         # createrepo was created in all the three chroots

+         assert len(expected_chroots) == 0

  

  

  class TestCoprApplyForPermissions(CoprsTestCase):
@@ -1101,6 +1153,7 @@

              "projectname": "test",

              "project_dirnames": ["test"],

              "appstream": True,

+             "devel": False,

          }

          def _expected(action, chroots):

              template["chroots"] = chroots

file modified
+1 -1
@@ -20,7 +20,7 @@

  %{expand: %%global latest_requires_packages %1 %%{?latest_requires_packages}}

  

  Name:    copr-rpmbuild

- Version: 0.54

+ Version: 0.54.1.dev

  Summary: Run COPR build tasks

  Release: 1%{?dist}

  URL: https://pagure.io/copr/copr

file modified
-4
@@ -49,11 +49,7 @@

  baseurl={{ repo["baseurl"] }}

  gpgcheck=0

  enabled=1

- {%- if repo['id'] == 'copr_base_devel' %}

- skip_if_unavailable=1

- {%- else %}

  skip_if_unavailable=0

- {%- endif %}

  metadata_expire=0

  cost=1

  best=1

This PR exists only to easily build the Copr packages via Copr CI, so they can be installed on our devel instance - and beaker tested.

We usually merge other pull requests here (those that are expected to be merged before the upcoming release).

4 new commits added

  • backend: put build-ID.log file to resultdir
  • backend: call call_copr_repo if initial createrepo failed
  • backend: RedisLogHandler bugfix
  • backend: copr-repo: createrepo_c --outputdir must imply --baseurl
4 years ago

4 new commits added

  • backend: put build-ID.log file to resultdir
  • backend: call call_copr_repo if initial createrepo failed
  • backend: RedisLogHandler bugfix
  • backend: copr-repo: createrepo_c --outputdir must imply --baseurl
4 years ago

rebased onto 5b0a795d433cd3f458f4cd2c5c10951f395be3cb

4 years ago

Pull-Request has been closed by praiskup

4 years ago

Pull-Request has been reopened by praiskup

4 years ago

rebased onto bd474fa0da71cd17de5fd2476e0d5bff7c1927dc

4 years ago

Pull-Request has been closed by praiskup

4 years ago

Pull-Request has been reopened by praiskup

2 years ago

rebased onto da84af50e6cb01bb523eb9d7bf4915cd49059bab

2 years ago

Build succeeded.

6 new commits added

  • frontend: large project modification timeout fix
  • backend, frontend: initial createrepo in devel
  • cli: more detailed "Request is not in JSON format" error
  • frontend: "Rebuild All" form to respect chroot denylist
  • frontend: "rebuild all packages" from successful builds
  • frontend: homepage redesign
2 years ago

Build succeeded.

Metadata Update from @frostyx:
- Pull-request tagged with: blocked

2 years ago

I added the blocked tag to make the PR look more scary and avoid accidental merging :-)

rebased onto 2804be6df59aa1cd2184c48b1f30397db6dd0e96

2 years ago

Build succeeded.

Build succeeded.

Pull-Request has been closed by praiskup

2 years ago

Pull-Request has been reopened by praiskup

2 years ago

rebased onto 103dd5c

2 years ago

Build succeeded.

rebased onto 7ae1bee

2 years ago

Build succeeded.

Pull-Request has been closed by praiskup

2 years ago