#1764 Re-enable cgit on copr-dist-git
Merged a year ago by praiskup. Opened a year ago by praiskup.
copr/ praiskup/copr copr-dist-git-fixes  into  main

file modified
+1
@@ -1,2 +1,3 @@ 

  .coverage

  .cache

+ dist

@@ -10,3 +10,14 @@ 

  log_dir=/tmp/copr-dist-git

  

  per_task_log_dir=/var/lib/copr-dist-git/per-task-logs/

+ 

+ # Copr administrator is responsible for referencing this file in /etc/cgitrc as

+ # include=CGIT_CACHE_FILE.  This file is automatically updated anytime a new

+ # package is imported into dist-git (plus periodically by a monthly cron job).

+ #cgit_cache_file=/var/cache/cgit/repo-configuration.rc

+ 

+ # Intermediate file created when generating cgit_cache_file

+ #cgit_cache_list_file=/var/cache/cgit/repo-subdirs.list

+ 

+ # Lock file used to atomically work with cgit_cache_file

+ #cgit_cache_lock_file=/var/cache/cgit/copr-repo.lock

@@ -1,3 +1,7 @@ 

  #!/usr/bin/sh

  

  runuser -c 'find /var/lib/copr-dist-git/per-task-logs -name *.log -mtime +30 -delete' - copr-dist-git

+ 

+ # From time to time assure that the CGIT caches are consistent, and that the

+ # ownership of cache files is correct (run this as root).

+ /usr/bin/copr-dist-git-refresh-cgit

@@ -8,7 +8,6 @@ 

  User=copr-dist-git

  Group=packager

  AmbientCapabilities=CAP_SETGID

- Environment="PYTHONPATH=/usr/share/copr/"

  ExecStart=/usr/bin/python3 /usr/bin/importer_runner.py

  

  [Install]

file modified
+7 -3
@@ -27,6 +27,8 @@ 

  Recommends: logrotate

  Requires: systemd

  Requires: httpd

+ Requires: coreutils

+ Requires: crudini

  Requires: dist-git

  Requires: python3-copr-common

  Requires: python3-requests
@@ -55,6 +57,8 @@ 

  

  

  %build

+ %py3_build

+ 

  

  %pre

  getent group packager >/dev/null || groupadd -r packager
@@ -65,6 +69,7 @@ 

  /usr/bin/passwd -l copr-dist-git >/dev/null

  

  %install

+ %py3_install

  

  install -d %{buildroot}%{_datadir}/copr/dist_git

  install -d %{buildroot}%{_sysconfdir}/copr
@@ -74,17 +79,14 @@ 

  install -d %{buildroot}%{_var}/log/copr-dist-git

  install -d %{buildroot}%{_tmpfilesdir}

  install -d %{buildroot}%{_sharedstatedir}/copr-dist-git

- install -d %{buildroot}%{_bindir}/

  install -d %{buildroot}%{_sysconfdir}/cron.monthly

  

  install -p -m 755 conf/cron.monthly/copr-dist-git %{buildroot}%{_sysconfdir}/cron.monthly/copr-dist-git

  

- cp -a dist_git/* %{buildroot}%{_datadir}/copr/dist_git

  cp -a conf/copr-dist-git.conf.example %{buildroot}%{_sysconfdir}/copr/copr-dist-git.conf

  cp -a conf/httpd/copr-dist-git.conf %{buildroot}%{_sysconfdir}/httpd/conf.d/copr-dist-git.conf

  cp -a conf/tmpfiles.d/* %{buildroot}/%{_tmpfilesdir}

  cp -a copr-dist-git.service %{buildroot}%{_unitdir}/

- cp -a run/* %{buildroot}%{_bindir}/

  

  cp -a conf/logrotate %{buildroot}%{_sysconfdir}/logrotate.d/copr-dist-git

  
@@ -110,6 +112,8 @@ 

  

  %files

  %license LICENSE

+ %python3_sitelib/copr_dist_git

+ %python3_sitelib/copr_dist_git*egg-info

  

  %{_bindir}/*

  %dir %{_datadir}/copr

dist-git/copr_dist_git/__init__.py dist-git/dist_git/__init__.py
file renamed
file was moved with no change to the file
dist-git/copr_dist_git/exceptions.py dist-git/dist_git/exceptions.py
file renamed
file was moved with no change to the file
dist-git/copr_dist_git/helpers.py dist-git/dist_git/helpers.py
file renamed
+12 -2
@@ -116,8 +116,18 @@ 

              cp, "dist-git", "pool_busy_sleep_time", 0.5, mode="float"

          )

  

-         opts.cgit_pkg_list_location = _get_conf(

-             cp, "dist-git", "cgit_pkg_list_location", "/var/lib/copr-dist-git/cgit_pkg_list"

+         # The Copr administrator is responsible for configuring /etc/cgitrc to

+         # include this file.

+         opts.cgit_cache_file = _get_conf(

+             cp, "dist-git", "cgit_cache_file", "/var/cache/cgit/repo-configuration.rc"

+         )

+ 

+         opts.cgit_cache_list_file = _get_conf(

+             cp, "dist-git", "cgit_cache_list_file", "/var/cache/cgit/repo-subdirs.list"

+         )

+ 

+         opts.cgit_cache_lock_file = _get_conf(

+             cp, "dist-git", "cgit_cache_lock_file", "/var/cache/cgit/copr-repo.lock"

          )

  

          opts.lookaside_location = _get_conf(

dist-git/copr_dist_git/import_task.py dist-git/dist_git/import_task.py
file renamed
file was moved with no change to the file
dist-git/copr_dist_git/importer.py dist-git/dist_git/importer.py
file renamed
file was moved with no change to the file
dist-git/copr_dist_git/package_import.py dist-git/dist_git/package_import.py
file renamed
+9 -3
@@ -88,12 +88,14 @@ 

          log.debug("nothing to commit into branch '{0}'".format(new_branch))

  

  

- def refresh_cgit_listing(opts):

+ def refresh_cgit_listing(reponame=None):

      """

      Refresh cgit repository list. See cgit docs for more information.

      """

      try:

-         cmd = ["/usr/share/copr/dist_git/bin/cgit_pkg_list", opts.cgit_pkg_list_location]

+         cmd = ["copr-dist-git-refresh-cgit"]

+         if reponame:

+             cmd += [reponame]

          subprocess.check_output(cmd, stderr=subprocess.STDOUT, encoding='utf-8')

      except OSError as e:

          log.error(str(e))
@@ -109,9 +111,11 @@ 

      :param str branches: branch names to be created inside that repo

      """

      log.info("make sure repos exist: {}".format(reponame))

+     brand_new_package = False

      try:

          cmd = ["/usr/share/dist-git/setup_git_package", reponame]

          subprocess.check_output(cmd, stderr=subprocess.STDOUT, encoding='utf-8')

+         brand_new_package = True

      except subprocess.CalledProcessError as e:

          log.error("cmd: {}, rc: {}, msg: {}"

                    .format(cmd, e.returncode, e.output.strip()))
@@ -120,6 +124,9 @@ 

          else:

              raise PackageImportException(e.output)

  

+     if brand_new_package:

+         refresh_cgit_listing(reponame)

+ 

      for branch in branches:

          try:

              cmd = ["/usr/share/dist-git/mkbranch", branch, reponame]
@@ -239,7 +246,6 @@ 

  

      os.chdir(oldpath)

      shutil.rmtree(repo_dir)

-     refresh_cgit_listing(opts)

  

      return munch.Munch(

          branch_commits=branch_commits,

dist-git/copr_dist_git/process_pool.py dist-git/dist_git/process_pool.py
file renamed
file was moved with no change to the file
@@ -1,24 +0,0 @@ 

- #!/bin/sh

- 

- # This simple script lists out the current pkgs git repos to a file.

- # This speeds up cgit as it doesn't have to recurse into all dirs

- # Looking for git repos.

- 

- destination=/var/lib/copr-dist-git/cgit_pkg_list

- 

- if [ -n "$1" ]

- then

-   destination=$1

- fi

- 

- newfile=`mktemp`

- 

- # TODO: we should read the 'SetEnv GIT_PROJECT_ROOT /var/lib/dist-git/git'

- # instead of hardcoding it here

- cd /var/lib/dist-git/git

- 

- # We expect <user>/<copr>/<package>.git directories.

- find -maxdepth 3 -mindepth 3 -type d -printf '%P\n' > "$newfile"

- cp -fZ "$newfile" "$destination"

- rm "$newfile"

- chmod 644 "$destination"

@@ -0,0 +1,16 @@ 

+ #! /usr/bin/python3

+ 

+ """

+ Read the Copr DistGit configuration file, and print the desired config option to

+ stdout.  Alternative to crudini parser, though we want to read also the defaults

+ from ConfigReader.

+ """

+ 

+ import sys

+ from copr_dist_git.helpers import ConfigReader

+ 

+ if __name__ == "__main__":

+     reader = ConfigReader()

+     opts = reader.read()

+     requested_option = sys.argv[1]

+     print(opts[requested_option])

@@ -0,0 +1,106 @@ 

+ #!/usr/bin/sh

+ 

+ # This script generates a "cgit_cache_file" configuration file to be included

+ # into the CGIT config /etc/cgitrc.  We execute this as 'copr-dist-git' user

+ # from copr-dist-git service, and as root (full scan) from cron job.

+ 

+ cachefile=$(copr-dist-git-config cgit_cache_file)

+ listfile=$(copr-dist-git-config cgit_cache_list_file)

+ lockfile=$(copr-dist-git-config cgit_cache_lock_file)

+ reposdir=$(crudini --get /etc/dist-git/dist-git.conf dist-git gitroot_dir)

+ ownership=copr-dist-git:apache

+ subdir=$1

+ 

+ single_repo ()

+ {

+     # Note that we used to use /var/www/cgi-bin/cgit --scan-path=/..../ to

+     # generate the output, but since our directories are too large and deep

+     # nowadays it is too expensive, and cgit isn't anyhow optimized -- when

+     # --scan-path argument is used the full directory scan is done, and the

+     # project-list= argument in CGIT_CONFIG file is entirely ignored (see the

+     # difference between scan_projects() and scan_tree()):

+     # https://git.zx2c4.com/cgit/tree/cgit.c?id=bd6f5683f6cde4212364354b3139c1d521f40f39#n1001

+     subpath=$1

+ 

+     case $subpath in

+         *.git) ;;

+         *) subpath=$subpath.git ;;

+     esac

+ cat <<EOF

+ 

+ repo.url=$subpath

+ repo.name=$subpath

+ repo.path=/$reposdir/$subpath/

+ repo.owner=

+ repo.desc=Unnamed repository; edit this file 'description' to name the repository.

+ repo.section=

+ repo.enable-blame=0

+ repo.enable-commit-graph=0

+ repo.enable-log-filecount=0

+ repo.enable-log-linecount=0

+ repo.enable-remote-branches=0

+ repo.enable-subject-links=0

+ repo.enable-html-serving=0

+ repo.hide=0

+ repo.ignore=0

+ 

+ EOF

+ }

+ 

+ tempfile_from ()

+ {

+     base=$(basename "$1")

+     mktemp "/tmp/$base-XXXXXXXX"

I am not sure if this wasn't meant to be a timestamp or something instead of XXXXXXXX

I didn't want to care, it's just:

$ mktemp /tmp/aaa-XXXXXXXX
/tmp/aaa-kEMg0Esv

I can move this to date/timestamp, though such value is more predictable ... (can be attacked).

+ }

+ 

+ if test ! -f "$lockfile"; then

+     touch "$lockfile"

+     chown $ownership "$lockfile"

+ fi

+ 

+ if test -z "$subdir"; then

+     # Full run, it takes some time to go through all the repos so work with

+     # a separate file, and without lock to not block others.

+     newlistfile=$(tempfile_from "$listfile")

+     find "$reposdir" -maxdepth 3 -mindepth 3 -type d -printf '%P\n' | sort > "$newlistfile"

+ fi

+ 

+ (

+     # !! Commands executed under lock, make this block FAST !!

+     set -e

+     flock 9

+ 

+     if test -n "$subdir"; then

+         # only one repo is being added, catenate to the existing config files

+         echo "$subdir" >> "$listfile"

+         single_repo "$subdir" >> "$cachefile"

+         exit 0

+     fi

+ 

+     # Generating the new cache file shouldn't take terribly long, though just in

+     # case -- generate it in a separate file name to not break the background

+     # httpd cgit cgi processes serving users.

+     newcachefile=$(tempfile_from "$cachefile")

+ 

+     # Some background copr-dist-git-workers might change the $listfile while

+     # we were traversing the directory tree (see above).  So we can not just

+     # rely on the "$newlistfile" - so we rather use both the old and new list

+     # files.  The file doesn't exist for the first run, so ignore failures.

+     cat "$listfile" >> "$newlistfile" || :

+     sort < "$newlistfile" | uniq | tee "$listfile" | \

+     while read -r line; do

+       single_repo "$line"

+     done >> "$newcachefile"

+ 

+     cp -fZ "$newcachefile" "$cachefile"

+     rm "$newcachefile" "$newlistfile"

+     chmod 644 "$cachefile"

+     chown "$ownership" "$cachefile" "$listfile"

+ 

+ ) 9>"$lockfile"

+ 

+ # fix lock file so copr-dist-git user can lock as well

+ case $(stat -c '%U:%G' "$lockfile"):$(id -u -n) in

+     "$ownership:root") ;;

+     *) chown "$ownership" "$lockfile"

+ esac

@@ -5,8 +5,8 @@ 

  import sys

  import logging

  

- from dist_git.helpers import ConfigReader

- from dist_git.importer import Importer

+ from copr_dist_git.helpers import ConfigReader

+ from copr_dist_git.importer import Importer

  

  log = logging.getLogger(__name__)

  

file added
+31
@@ -0,0 +1,31 @@ 

+ #!/usr/bin/python

+ 

+ """\

+ COPR is lightweight build system. It allows you to create new project in WebUI,

+ and submit new builds and COPR will create yum repository from latest builds.

+ 

+ This package provides the DistGit component."""

+ 

+ import os

+ from setuptools import setup, find_packages

+ 

+ 

+ __description__ = "Copr DistGit component."

+ __author__ = "Copr Team"

+ __author_email__ = "copr-team@redhat.com"

+ __url__ = "https://pagure.io/copr/copr"

+ 

+ setup(

+     name='copr-dist-git',

+     version="0.49",

+     description=__description__,

+     long_description=__doc__,

+     author=__author__,

+     author_email=__author_email__,

+     url=__url__,

+     license='GPLv2+',

+     packages=find_packages(exclude=('tests*',)),

+     scripts=[os.path.join("run", p) for p in os.listdir("run")],

+     include_package_data=True,

+     zip_safe=False,

+ )

file modified
+2 -2
@@ -5,8 +5,8 @@ 

  import time

  import json

  

- from dist_git import importer

- from dist_git import import_task

+ from copr_dist_git import importer

+ from copr_dist_git import import_task

  

  class Base(object):

  

@@ -12,14 +12,14 @@ 

  from munch import Munch

  from subprocess import check_output

  

- from dist_git.package_import import import_package

+ from copr_dist_git.package_import import import_package

  

  def scriptdir():

      return os.path.dirname(os.path.realpath(__file__))

  

  @pytest.yield_fixture

  def mc_setup_git_repo():

-     with mock.patch("{}.setup_git_repo".format('dist_git.package_import')) as handle:

+     with mock.patch("{}.setup_git_repo".format('copr_dist_git.package_import')) as handle:

          yield handle

  

  

@@ -13,7 +13,7 @@ 

  from unittest.mock import MagicMock

  

  

- MODULE_REF = 'dist_git.importer'

+ MODULE_REF = 'copr_dist_git.importer'

  

  

  @pytest.yield_fixture

@@ -7,9 +7,9 @@ 

  from unittest import mock

  from unittest.mock import MagicMock

  

- from dist_git import process_pool

+ from copr_dist_git import process_pool

  

- MODULE_REF = 'dist_git.process_pool'

+ MODULE_REF = 'copr_dist_git.process_pool'

  

  

  class TestWorker(object):

@@ -8,12 +8,17 @@ 

  

  from base import Base

  

- from dist_git.package_import import my_upload_fabric, import_package, setup_git_repo

+ from copr_dist_git.package_import import (

+     import_package,

+     my_upload_fabric,

+     refresh_cgit_listing,

+     setup_git_repo,

+ )

  

  from unittest import mock

  from unittest.mock import MagicMock

  

- MODULE_REF = 'dist_git.package_import'

+ MODULE_REF = 'copr_dist_git.package_import'

  

  

  @pytest.yield_fixture
@@ -116,14 +121,21 @@ 

          reponame = 'foo'

          branches = ['f25', 'f26']

          setup_git_repo(reponame, branches)

-         assert mc_subprocess_check_output.has_calls([

-             mock.call(['/usr/share/dist-git/setup_git_package', 'foo']),

-             mock.call(['/usr/share/dist-git/mkbranch', 'f25', 'foo']),

-             mock.call(['/usr/share/dist-git/mkbranch', 'f26', 'foo']),

-         ])

- 

-     def refresh_cgit_listing(self, mc_subprocess_check_output):

-         refresh_cgit_listing(self.opts)

-         assert mc_subprocess_check_output.has_calls([

-             mock.call(["/usr/share/copr/dist_git/bin/cgit_pkg_list", self.opts.cgit_pkg_list_location])

-         ])

+         mc_subprocess_check_output.assert_has_calls([

+             mock.call(['/usr/share/dist-git/setup_git_package', 'foo'],

+                       stderr=-2, encoding='utf-8'),

+             mock.call(['/usr/share/dist-git/mkbranch', 'f25', 'foo'],

+                       stderr=-2, encoding='utf-8'),

+             mock.call(['/usr/share/dist-git/mkbranch', 'f26', 'foo'],

+                       stderr=-2, encoding='utf-8'),

+             mock.call(['copr-dist-git-refresh-cgit', 'foo'],

+                       stderr=-2, encoding='utf-8'),

+         ], any_order=True)

+ 

+ 

+     def test_refresh_cgit_listing(self, mc_subprocess_check_output):

+         refresh_cgit_listing()

+         mc_subprocess_check_output.assert_has_calls([

+             mock.call(['copr-dist-git-refresh-cgit'],

+                       stderr=-2, encoding='utf-8'),

+         ], any_order=True)

no initial comment

Metadata Update from @praiskup:
- Pull-request tagged with: wip

a year ago

Build succeeded.

rebased onto 0c9a814011789418aabde0e60c3406dece6f171c

a year ago

Build succeeded.

1 new commit added

  • fix 1
a year ago

Build succeeded.

1 new commit added

  • fix 2
a year ago

Build succeeded.

2 new commits added

  • dist-git: optimize cgit cache file generator
  • dist-git: move everything to Python path
a year ago

Metadata Update from @praiskup:
- Pull-request untagged with: wip

a year ago

Do we want to enhance this configuration? Please take a look.

Build succeeded.

2 new commits added

  • dist-git: optimize cgit cache file generator
  • dist-git: move everything to Python path
a year ago

Build succeeded.

2 new commits added

  • dist-git: optimize cgit cache file generator
  • dist-git: move everything to Python path
a year ago

Build succeeded.

(deployed on staging for now, if anyone wants to give it a try)

Metadata Update from @praiskup:
- Request assigned

a year ago

I am not sure if this wasn't meant to be a timestamp or something instead of XXXXXXXX

I am sorry the review took so long. Just two nits, otherwise it looks good to me. Also, +1 to moving the code into copr_dist_git.

I didn't want to care, it's just:

$ mktemp /tmp/aaa-XXXXXXXX
/tmp/aaa-kEMg0Esv

I can move this to date/timestamp, though such value is more predictable ... (can be attacked).

rebased onto 92f1388

a year ago

Build succeeded.

Pull-Request has been merged by praiskup

a year ago