copr / copr

Clone

#2108 More detailed info for the large build (background) queue
Merged 2 years ago by praiskup. Opened 2 years ago by praiskup.
Unknown source overview  into  main

file modified
+18 -11 
@@ -278,9 +278,9 @@
 

          return query
 

  

      @classmethod
 

-     def _todo_states(cls, for_backend):
 

+     def _todo_states(cls, data_type):
 

          """
 

-         When "for_backend" is False, we only return builds which are in
 

+         When data_type is not "for_backend", we only return builds which are in
 

          "pending" state.  That queries are used by the Frontend Web-UI/API and
 

          that's what is the end-user interested in when looking at pending tasks.
 

  
@@ -290,20 +290,20 @@
 

          tasks too (otherwise they stay in starting/running states forever).
 

          """
 

          todo_states = ["pending"]
 

-         if for_backend:
 

+         if data_type == "for_backend":
 

              todo_states += ["starting", "running"]
 

          return [StatusEnum(x) for x in todo_states]
 

  

      @classmethod
 

-     def get_pending_srpm_build_tasks(cls, background=None, for_backend=False):
 

+     def get_pending_srpm_build_tasks(cls, background=None, data_type=None):
 

          query = (
 

              models.Build.query
 

              .join(models.Copr)
 

              .filter(models.Build.canceled == false())
 

-             .filter(models.Build.source_status.in_(cls._todo_states(for_backend)))
 

+             .filter(models.Build.source_status.in_(cls._todo_states(data_type)))
 

              .order_by(models.Build.is_background.asc(), models.Build.id.asc())
 

          )
 

-         if for_backend:
 

+         if data_type in ["for_backend", "overview"]:
 

              query = query.options(
 

                  load_only("is_background", "source_type", "source_json",
 

                            "submitted_by"),
 
@@ -320,7 +320,7 @@
 

          return query
 

  

      @classmethod
 

-     def get_pending_build_tasks(cls, background=None, for_backend=False):
 

+     def get_pending_build_tasks(cls, background=None, data_type=None):
 

          """
 

          Get list of BuildChroot objects that are to be (re)processed.
 

          """
 
@@ -329,16 +329,23 @@
 

              models.BuildChroot.query
 

              .join(models.Build)
 

              .join(models.CoprDir)
 

+         )
 

+ 

+         # We need the Package information for the /get-pending-srpm-task/ and
 

+         # /get-build-task/ routes, but anywhere else.
 

+         if data_type is None:
 

              # TODO: BuildChroot objects should be self-standing.  The thing is
 

              # that this is racy -- Package reference provides some build
 

              # configuration which can be changed in the middle of the
 

              # BuildChroot processing.
 

-             .join(models.Package, models.Package.id == models.Build.package_id)
 

-             .filter(models.Build.canceled == false())
 

-             .filter(models.BuildChroot.status.in_(cls._todo_states(for_backend)))
 

+             query = query.join(models.Package, models.Package.id == models.Build.package_id)
 

+ 

+         query = (
 

+             query.filter(models.Build.canceled == false())
 

+             .filter(models.BuildChroot.status.in_(cls._todo_states(data_type)))
 

              .order_by(models.Build.is_background.asc(), models.Build.id.asc()))
 

  

-         if for_backend:
 

+         if data_type in ["for_backend", "overview"]:
 

              query = query.options(
 

                  joinedload('build').load_only("is_background", "submitted_by",
 

                                                "source_json", "source_type")

file modified
+3 
@@ -49,6 +49,9 @@
 

      {% endif %}
 

      in the {{ state_of_tasks }} state.
 

  </h2>
 

+ {% if state_of_tasks == "pending" %}
 

+ <p>See more <a href="{{ url_for('status_ns.pending_all') }}">detailed</a> statistics.</p>
 

+ {% endif %}
 

  <p>{{ build_state_text(state_of_tasks) }} - {{ state_of_tasks|build_state_description }}</p>
 

  {{ status_info(type=state_of_tasks, tasks=tasks) }}
 

  {{ initialize_datatables(order="desc") }}

file added
+93 
@@ -0,0 +1,93 @@
 

+ {% extends "status.html" %}
 

+ 

+ {% block status_breadcrumb %}
 

+ <li> Overview </li>
 

+ {% endblock %}
 

+ 

+ {% block status_body %}
 

+ {% set right_column_width = "160px" %}
 

+ 

+ <h1>Full job queue statistics</h1>
 

+ <p>
 

+ The normal "Pending" stats provide info only about the normal (or "foreground")
 

+ tasks, but the project maintainers can submit also the background jobs (with
 

+ lower priority) that are processed only when there's enough computation power
 

+ left in the Copr cloud that is not taken by normal tasks.  This page provides
 

+ some statistics related to the queue, mostly useful for the system
 

+ administrators (for queue analysis).
 

+ </p>
 

+ <p>
 

+ This page is cached for <strong>{{ cache_seconds }} seconds</strong>.
 

+ </p>
 

+ 

+ <h2>All jobs</h2>
 

+ 

+ <p>There's crurrently <strong>{{ stats["background"].total() }}</strong> jobs.</p>
 

+ 

+ <h2>Sort pending tasks per ownership</h2>
 

+ <div class="table-responsive">
 

+ <table class="table table-bordered table-hover table-treegrid">
 

+ <thead>
 

+   <th></th>
 

+   <th style="width: {{ right_column_width }};">Count</th>
 

+ </thead>
 

+ <tbody>
 

+ {% for owner_name, full_count in stats.owners.most_common() %}
 

+ <tr id="owner_name_{{ owner_name }}" class="collapsed">
 

+   <td class="treegrid-node">
 

+     <span class="icon node-icon fa fa-tasks"></span>
 

+     {{ owner_name }}
 

+   </td>
 

+   <td>{{ full_count }}</td>
 

+ </tr>
 

+ {% for stattype in ["chroots", "projects", "background"] %}
 

+ <tr id="owner_name_{{ owner_name }}_{{ stattype }}"
 

+   data-parent="#owner_name_{{ owner_name }}"
 

+   class="collapsed"
 

+ >
 

+   <td class="treegrid-node">{{ stattype }}</td>
 

+   <td> {{ stats.owners_details[owner_name][stattype].total() }}  </td>
 

+ </tr>
 

+ {% for key, count in stats.owners_details[owner_name][stattype].most_common() %}
 

+ <tr id="owner_name_{{ owner_name }}_{{ stattype }}_{{ key }}"
 

+   data-parent="#owner_name_{{ owner_name }}_{{ stattype }}"
 

+   class="collapsed"
 

+ >
 

+   <td class="treegrid-node">{{ key}}</td>
 

+   <td>{{ count }}</td>
 

+ </tr>
 

+ {% endfor %}
 

+ {% endfor %}
 

+ {% endfor %}
 

+ </tbody>
 

+ </table>
 

+ 

+ 

+ {% for stattype in ["projects", "chroots", "background"] %}
 

+ <h2>Queue by {{ stattype }}</h2>
 

+ <table class="table table-bordered table-hover">
 

+ <thead>
 

+   <th></th>
 

+   <th style="width: {{ right_column_width }};">Count</th>
 

+ </thead>
 

+ <tbody>
 

+ {% for key, count in stats[stattype].most_common() %}
 

+ <tr id="owner_name_{{ owner_name }}_{{ stattype }}_{{ key }}"
 

+   data-parent="#owner_name_{{ owner_name }}_{{ stattype }}"
 

+ >
 

+   <td>{{ key }}</td>
 

+   <td>{{ count }}</td>
 

+ </tr>
 

+ {% endfor %}
 

+ </tbody>
 

+ </table>
 

+ {% endfor %}
 

+ 

+ 

+ </div>
 

+ <script type="text/javascript">
 

+   $(document).ready(function() {
 

+     $('.table-treegrid').treegrid({ });
 

+   });
 

+ </script>
 

+ {% endblock %}

file modified
+3 -2 
@@ -305,12 +305,13 @@
 

          cache.add(build.batch)
 

          return not build.blocked
 

  

+     args = {"data_type": "for_backend"}
 

      build_records = (
 

          [get_srpm_build_record(task, for_backend=True)
 

-          for task in BuildsLogic.get_pending_srpm_build_tasks(for_backend=True)
 

+          for task in BuildsLogic.get_pending_srpm_build_tasks(**args)
 

           if build_ready(task)] +
 

          [get_build_record(task, for_backend=True)
 

-          for task in BuildsLogic.get_pending_build_tasks(for_backend=True)
 

+          for task in BuildsLogic.get_pending_build_tasks(**args)
 

           if build_ready(task.build)]
 

      )

file modified
+71 -1 
@@ -1,12 +1,16 @@
 

- import flask
 

+ from collections import Counter, defaultdict
 

  from time import time
 

  

+ import flask
 

+ 

  from copr_common.enums import StatusEnum
 

  from coprs.views.status_ns import status_ns
 

  from coprs.logic import batches_logic
 

  from coprs.logic import builds_logic
 

  from coprs.logic import complex_logic
 

+ from coprs import cache
 

  

+ PENDING_ALL_CACHE_SECONDS = 2*60
 

  

  @status_ns.context_processor
 

  def inject_common_blueprint_variables():
 
@@ -29,6 +33,72 @@
 

      tasks.extend(srpm_tasks)
 

      return render_status("pending", tasks=tasks, bg_tasks_cnt=bg_tasks_cnt)
 

  

+ @status_ns.route("/pending/all/")
 

+ @cache.memoize(timeout=PENDING_ALL_CACHE_SECONDS)
 

+ def pending_all():
 

+     """
 

+     Provide the overview for _all_ the pending jobs.
 

+     """
 

+ 

+     # Get "for_backend" type of data, which are much cheaper to get.  This page
 

+     # is for admins, to analyze the build queue (to allow us to understand best
 

+     # what backend sees).
 

+     rpm_tasks = builds_logic.BuildsLogic.get_pending_build_tasks(data_type="overview")
 

+ 

+     owner_stats = Counter()
 

+     owner_substats = defaultdict(lambda: {
 

+         "projects": Counter(),
 

+         "chroots": Counter(),
 

+         "background": Counter(),
 

+     })
 

+ 

+     project_stats = Counter()
 

+     background_stats = Counter()
 

+     chroot_stats = Counter()
 

+ 

+     def _calc_task(owner_name, project_name, chroot_name, background):
 

+         owner_stats[owner_name] += 1
 

+         owner = owner_substats[owner_name]
 

+         owner["projects"][project_name] += 1
 

+         owner["chroots"][chroot_name] += 1
 

+         owner["background"][background] += 1
 

+         project_stats[project_name] += 1
 

+         chroot_stats[chroot_name] += 1
 

+         background_stats[background] += 1
 

+ 

+ 

+     for task in rpm_tasks:
 

+         _calc_task(
 

+             task.build.copr.owner.name,
 

+             task.build.copr.full_name,
 

+             task.mock_chroot.name,
 

+             task.build.is_background,
 

+         )
 

+ 

+     srpm_tasks = builds_logic.BuildsLogic.get_pending_srpm_build_tasks(data_type="overview").all()
 

+     for task in srpm_tasks:
 

+         _calc_task(
 

+             task.copr.owner.name,
 

+             task.copr.full_name,
 

+             "srpm-builds",
 

+             task.is_background,
 

+         )
 

+ 

+     calculated_stats = {
 

+         "owners": owner_stats,
 

+         "owners_details": owner_substats,
 

+         "projects": project_stats,
 

+         "chroots": chroot_stats,
 

+         "background": background_stats,
 

+     }
 

+ 

+     return flask.render_template(
 

+         "status_overview.html",
 

+         stats=calculated_stats,
 

+         state_of_tasks="pending",
 

+         cache_seconds=PENDING_ALL_CACHE_SECONDS,
 

+     )
 

+ 

  

  @status_ns.route("/running/")
 

  def running():

file modified
+2 -2 
@@ -151,7 +151,7 @@
 

                  build_chroot.ended_on = None
 

  

          self.db.session.commit()
 

-         data = BuildsLogic.get_pending_build_tasks(for_backend=True).all()
 

+         data = BuildsLogic.get_pending_build_tasks(data_type="for_backend").all()
 

  

          assert len(data) == 2
 

          assert set([data[0], data[1]]) == set([self.b1_bc[0], self.b2_bc[0]])
 
@@ -178,7 +178,7 @@
 

          models.Build.query.get(2).source_status = StatusEnum("starting")
 

          models.Build.query.get(3).source_status = StatusEnum("running")
 

          assert len(BuildsLogic.get_pending_srpm_build_tasks().all()) == 1
 

-         assert len(BuildsLogic.get_pending_srpm_build_tasks(for_backend=True).all()) == 3
 

+         assert len(BuildsLogic.get_pending_srpm_build_tasks(data_type="for_backend").all()) == 3
 

  

      def test_delete_build_exceptions(
 

              self, f_users, f_coprs, f_mock_chroots, f_builds, f_db):

no initial comment

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci

recheck

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci

recheck
[copr-build]

rebased onto 13839597bfbeee43986a5bee6c9dc236b3812534
2 years ago

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci

rebased onto b74b8109e297e1a75dadecaf188dbc72ea2eb394
2 years ago

Build succeeded.

Should we drop this then?
Or can you please add a comment on what is stopping us?

Oh man, this is so much better than I even imagined.
Really good job @praiskup.

Just one question. Is it still possible to see the table listing all the pending builds?

It's a leftover comment, I just dumped the idea here before implementing it (aka the second commit).

rebased onto b776350ad816b9fc8e7faa4c5c3fb618051255a6
2 years ago

Build succeeded.

Is it still possible to see the table listing all the pending builds?

Hmm. No such page has been implemented so far, I'm afraid?
The /pending/ page shows just the non-background jobs, and this PR just
added "the link to the new page" there - nothing else.

I actually tried to edit the /pending/ page to provide all the jobs,
but it is too expensive to print (not only the database and the flask
calculations, but also browser has a lot of work drawing the giant
table... we would have to switch from table to some floating div
solution, etc...

PTAL

Hmm. No such page has been implemented so far, I'm afraid?

Sorry, I meant a page with the list of all non-background jobs.

+1

rebased onto 8ffe371
2 years ago

Build succeeded.

Pull-Request has been merged by praiskup
2 years ago
Metadata
None
No Tags
Changes Summary 6
+18 -11
file changed
frontend/coprs_frontend/coprs/logic/builds_logic.py
+3 -0
file changed
frontend/coprs_frontend/coprs/templates/status.html
+93
file added
frontend/coprs_frontend/coprs/templates/status_overview.html
+3 -2
file changed
frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py
+71 -1
file changed
frontend/coprs_frontend/coprs/views/status_ns/status_general.py
+2 -2
file changed
frontend/coprs_frontend/tests/test_logic/test_builds_logic.py
Powered by Pagure 5.14.1
Documentation • File an Issue • About • SSH Hostkey/Fingerprint
© Red Hat, Inc. and others.