- %global tests_version 1

- Requires:   prunerepo

- from subprocess import Popen, PIPE

- from shlex import split

- from setproctitle import getproctitle, setproctitle

- from oslo_concurrency import lockutils

- from .exceptions import CreateRepoError

- 

- 

- def run_cmd_unsafe(comm_str, lock_name, lock_path="/var/lock/copr-backend"):

-     # log.info("Running command: {}".format(comm_str))

-     comm = split(comm_str)

-     title = getproctitle()

-     try:

-         # TODO change this to logger

-         setproctitle(title + " [locked] in createrepo")

-         with lockutils.lock(name=lock_name, external=True, lock_path=lock_path):

-             cmd = Popen(comm, stdout=PIPE, stderr=PIPE, encoding="utf-8")

-             out, err = cmd.communicate()

-     except Exception as err:

-         raise CreateRepoError(msg="Failed to execute: {}".format(err), cmd=comm_str)

-     setproctitle(title)

- 

-     if cmd.returncode != 0:

-         raise CreateRepoError(msg="exit code != 0",

-                               cmd=comm_str, exit_code=cmd.returncode,

-                               stdout=out, stderr=err)

-     return out

- 

- 

- def createrepo_unsafe(path, dest_dir=None, base_url=None):

-     """

-         Run createrepo_c on the given path

- 

-         Warning! This function doesn't check user preferences.

-         In most cases use `createrepo(...)`

- 

-     :param string path: target location to create repo

-     :param str dest_dir: [optional] relative to path location for repomd,

-             in most cases you should also provide base_url.

-     :param str base_url: optional parameter for createrepo_c, "--baseurl"

- 

-     :return tuple: (return_code,  stdout, stderr)

-     """

- 

-     comm = ['/usr/bin/createrepo_c', '--database', '--ignore-lock', '--local-sqlite',

-             '--cachedir', '/tmp/', '--workers', '8']

-     if os.path.exists(path + '/repodata/repomd.xml'):

-         comm.append("--update")

-     if "epel-5" in path or "rhel-5" in path:

-         # this is because rhel-5 doesn't know sha256

-         comm.extend(['-s', 'sha', '--checksum', 'md5'])

- 

-     if dest_dir:

-         dest_dir_path = os.path.join(path, dest_dir)

-         comm.extend(['--outputdir', dest_dir_path])

-         if not os.path.exists(dest_dir_path):

-             os.makedirs(dest_dir_path)

- 

-     if base_url:

-         comm.extend(['--baseurl', base_url])

- 

-     mb_comps_xml_path = os.path.join(path, "comps.xml")

-     if os.path.exists(mb_comps_xml_path):

-         comm.extend(['--groupfile', mb_comps_xml_path, '--keep-all-metadata'])

- 

-     comm.append(path)

- 

-     return run_cmd_unsafe(" ".join(map(str, comm)), os.path.join(path, "createrepo.lock"))

- 

- 

- APPDATA_CMD_TEMPLATE = \

-     """/usr/bin/timeout --kill-after=240 180 \

- /usr/bin/appstream-builder \

- --temp-dir={packages_dir}/tmp \

- --cache-dir={packages_dir}/cache \

- --packages-dir={packages_dir} \

- --output-dir={packages_dir}/appdata \

- --basename=appstream \

- --include-failed \

- --min-icon-size=48 \

- --veto-ignore=missing-parents \

- --enable-hidpi \

- --origin={username}/{projectname}

- """

- INCLUDE_APPSTREAM = \

-     """/usr/bin/modifyrepo_c \

- --no-compress \

- {packages_dir}/appdata/appstream.xml.gz \

- {packages_dir}/repodata

- """

- INCLUDE_ICONS = \

-     """/usr/bin/modifyrepo_c \

- --no-compress \

- {packages_dir}/appdata/appstream-icons.tar.gz \

- {packages_dir}/repodata

- """

- 

- 

- def add_appdata(path, username, projectname, lock=None):

-     out = ""

- 

-     # We need to have a possibility to disable an appstream builder for some projects

-     # because it doesn't properly scale up for a large ammount of packages

-     parent_dir = os.path.dirname(os.path.normpath(path))

- 

-     # We don't generate appstream metadata for anything else than the main

-     # directory.  If we reconsidered this in future, we would have to check the

-     # file ../../<main_dir>/.disable-appstream for existance.

-     if ":" in os.path.basename(parent_dir):

-         return out

- 

-     if os.path.exists(os.path.join(parent_dir, ".disable-appstream")):

-         return out

- 

-     kwargs = {

-         "packages_dir": path,

-         "username": username,

-         "projectname": projectname

-     }

-     try:

-         out += "\n" + run_cmd_unsafe(

-             APPDATA_CMD_TEMPLATE.format(**kwargs), os.path.join(path, "createrepo.lock"))

- 

-         if os.path.exists(os.path.join(path, "appdata", "appstream.xml.gz")):

-             out += "\n" + run_cmd_unsafe(

-                 INCLUDE_APPSTREAM.format(**kwargs), os.path.join(path, "createrepo.lock"))

- 

-         if os.path.exists(os.path.join(path, "appdata", "appstream-icons.tar.gz")):

-             out += "\n" + run_cmd_unsafe(

-                 INCLUDE_ICONS.format(**kwargs), os.path.join(path, "createrepo.lock"))

- 

-         # appstream builder provide strange access rights to result dir

-         # fix them, so that lighttpd could serve appdata dir

-         out += "\n" + run_cmd_unsafe("chmod -R +rX {packages_dir}/appdata"

-                                      .format(**kwargs), os.path.join(path, "createrepo.lock"))

-     except CreateRepoError as err:

-         err.stdout = out + "\nLast command\n" + err.stdout

-         raise

-     return out

- 

- 

- def createrepo(path, username, projectname, devel=False, base_url=None):

-     """

-     Creates repodata.  Depending on the "auto_createrepo" parameter it either

-     creates the repodata directory in `path`, or in `path/devel`.

- 

-     :param path: directory with rpms

-     :param username: copr project owner username

-     :param projectname: copr project name

-     :param devel: create the repository in 'devel' subdirectory

-     :param base_url: base_url to access rpms independently of repomd location

- 

-     :return: tuple(returncode, stdout, stderr) produced by `createrepo_c`

-     """

-     # TODO: add means of logging

-     if not devel:

-         out_cr = createrepo_unsafe(path)

-         out_ad = add_appdata(path, username, projectname)

-         return "\n".join([out_cr, out_ad])

- 

-     # Automatic createrepo disabled.  Even so, we still need to createrepo in

-     # special "devel" directory so we can later build packages against it.

-     return createrepo_unsafe(path, base_url=base_url, dest_dir="devel")

- 

- 

-     def __init__(self, dirname, full, add, delete, log,

- 

-             return (full, add, delete)

-         return (full, add, delete)

- def call_copr_repo(directory, devel=False, add=None, delete=None, timeout=None,

-     (batch_full, batch_add, batch_delete) = batch.options()

- #!/usr/bin/python3

- 

- import logging

- import os

- 

- import sys

- import pwd

- 

- logging.basicConfig(

-     filename="/var/log/copr-backend/copr_createrepo.log",

-     format='[%(asctime)s][%(levelname)6s]: %(message)s',

-     level=logging.INFO)

- log = logging.getLogger(__name__)

- 

- 

- from copr_backend.createrepo import createrepo

- from copr_backend.helpers import SortedOptParser, BackendConfigReader

- 

- 

- def main(args):

- 

-     parser = SortedOptParser(

-         "copr_create_repo")

- 

-     parser.add_option("-u", "--user", dest="user",

-                       help="copr project owner username")

- 

-     parser.add_option("-p", "--project", dest="project",

-                       help="copr project name")

- 

-     parser.add_option("-f", "--front_url", dest="front_url",

-                       help="copr frontend url")

- 

-     cli_opts, args = parser.parse_args(args)

- 

-     if not cli_opts.user:

-         print("No user was specified, exiting", file=sys.stderr)

-         sys.exit(1)

- 

-     if not cli_opts.project:

-         print("No project was specified, exiting", file=sys.stderr)

-         sys.exit(1)

- 

-     opts = BackendConfigReader().read()

- 

-     front_url = cli_opts.front_url or opts.frontend_base_url

-     project_path = os.path.join(opts.destdir, cli_opts.user, cli_opts.project)

- 

-     log.info("start processing {}/{}".format(cli_opts.user, cli_opts.project))

-     for subdir in os.listdir(project_path):

-         if os.path.isdir(subdir):

-             path = os.path.join(project_path, subdir)

-             log.info("entering dir: {}".format(subdir))

-             createrepo(path=path, front_url=front_url,

-                        username=cli_opts.user, projectname=cli_opts.project)

-             log.info("done dir: {}".format(subdir))

-     log.info("finished processing {}/{}".format(cli_opts.user, cli_opts.project))

- 

- if __name__ == "__main__":

-     """

-         Provides cli interface for conditional execution of createrepo_c

-               depending on user setting `auto_createrepo`

-     """

-     if pwd.getpwuid(os.getuid())[0] != "copr":

-         print("This script should be executed under the `copr` user")

-         sys.exit(1)

-     else:

-         main(sys.argv[1:])

- 

- 

- #with open("coprs_to_cr.txt") as inp:

- #    with open("cr_cmd_list.txt", "w") as outp:

- #        for line in inp:

- #            copr, user = line.strip().split("/")

- #            outp.write("{} -u {} -p {}\n".format("/usr/bin/copr_createrepo.py", copr, user))

- from copr_backend.helpers import BackendConfigReader

- from copr_backend.createrepo import createrepo_unsafe, add_appdata

-         log.info("> > Running createrepo_unsafe for {}".format(dir_path))

-         createrepo_unsafe(dir_path)

- 

-         log.info("> > Running add_appdata for {}".format(dir_path))

-         add_appdata(dir_path, owner, coprname)

- from copr_backend.helpers import uses_devel_repo, get_persistent_status, get_auto_prune_status

- from copr_backend.createrepo import createrepo

- def run_prunerepo(chroot_path, username, projectname, projectdir, sub_dir_name, prune_days):

-         cmd = ['prunerepo', '--verbose', '--days', str(prune_days), '--nocreaterepo', chroot_path]

-         stdout = runcmd(cmd)

-         LOG.info("Prunerepo stdout:\n%s", stdout)

-         createrepo(path=chroot_path, username=username,

-                    projectname=projectname)

-                                   (chroot_path, username, projectname,

- from copr_backend.helpers import (BackendConfigReader, create_file_logger,

-                              uses_devel_repo)

- from copr_backend.sign import get_pubkey, sign_rpms_in_dir, create_user_keys

- from copr_backend.exceptions import CoprSignNoKeyError

- from copr_backend.createrepo import createrepo

- 

- 

- def check_signed_rpms_in_pkg_dir(pkg_dir, user, project, chroot, chroot_dir, opts, devel):

- 

-         base_url = "/".join([opts.results_baseurl, user,

-                              project, chroot])

-         createrepo(

-             path=chroot_dir,

-             base_url=base_url,

-             username=user,

-             projectname=project,

-             devel=devel,

-         )

- 

-                                                 chroot, chroot_path, opts,

- import copy

- import tarfile

- import time

- import pytest

- from unittest import mock

- from unittest.mock import MagicMock

-     createrepo,

-     createrepo_unsafe,

-     run_cmd_unsafe,

- from copr_backend.exceptions import CreateRepoError

- 

- import testlib

- from testlib import assert_logs_exist, AsyncCreaterepoRequestFactory

- @mock.patch('copr_backend.createrepo.createrepo_unsafe')

- @mock.patch('copr_backend.createrepo.add_appdata')

- def test_createrepo_conditional_true(mc_add_appdata, mc_create_unsafe):

-     mc_create_unsafe.return_value = ""

-     mc_add_appdata.return_value = ""

- 

-     createrepo(path="/tmp/", username="foo", projectname="bar")

-     mc_create_unsafe.reset_mock()

- 

-     createrepo(path="/tmp/", username="foo", projectname="bar")

-     mc_create_unsafe.reset_mock()

- 

- 

- @mock.patch('copr_backend.createrepo.createrepo_unsafe')

- def test_createrepo_conditional_false(mc_create_unsafe):

-     base_url = "http://example.com/repo/"

-     createrepo(path="/tmp/", username="foo", projectname="bar", base_url=base_url, devel=True)

-     assert mc_create_unsafe.call_args == mock.call('/tmp/', dest_dir='devel', base_url=base_url)

- 

- 

- @pytest.yield_fixture

- def mc_popen():

-     with mock.patch('copr_backend.createrepo.Popen') as handle:

-         yield handle

- 

- 

- @pytest.yield_fixture

- def mc_run_cmd_unsafe():

-     with mock.patch('copr_backend.createrepo.run_cmd_unsafe') as handle:

-         yield handle

- 

- 

- class TestCreaterepo(object):

-     def setup_method(self, method):

-         self.tmp_dir_name = self.make_temp_dir()

-         self.test_time = time.time()

-         self.base_url = "http://example.com/repo/"

- 

-         self.username = "foo"

-         self.projectname = "bar"

- 

-     # def unpack_resource(self, resource_name):

-     #     if self.tmp_dir_name is None:

-     #         self.make_temp_dir()

-     #

-     #     src_path = os.path.join(os.path.dirname(__file__),

-     #                             "_resources", resource_name)

-     #

-     #     with tarfile.open(src_path, "r:gz") as tfile:

-     #         tfile.extractall(os.path.join(self.tmp_dir_name, "old_dir"))

- 

-     def teardown_method(self, method):

-         self.rm_tmp_dir()

- 

-     def rm_tmp_dir(self):

-         if self.tmp_dir_name:

-             shutil.rmtree(self.tmp_dir_name)

-             self.tmp_dir_name = None

- 

-     def make_temp_dir(self):

-         root_tmp_dir = tempfile.gettempdir()

-         subdir = "test_createrepo_{}".format(time.time())

-         self.tmp_dir_name = os.path.join(root_tmp_dir, subdir)

-         os.mkdir(self.tmp_dir_name)

-         return self.tmp_dir_name

- 

-     #def test_add_appdata(self, mc_run_cmd_unsafe):

-     #    todo: implement, need to test behaviour with/withou produced appstream files

-     #    for lock in [None, MagicMock()]:

-     #        add_appdata(self.tmp_dir_name, self.username, self.projectname, lock=lock)

-     #        print mc_run_cmd_unsafe.call_args_list

-     #        mc_run_cmd_unsafe.reset()

- 

-     def test_run_cmd_unsafe_ok(self, mc_popen):

-         cmd = "foo --bar"

-         mc_popen.return_value.communicate.return_value = ("stdout", "stderr")

-         mc_popen.return_value.returncode = 0

- 

-         assert run_cmd_unsafe(cmd, self.tmp_dir_name, lock_path="/tmp") == "stdout"

-         mc_popen.reset()

- 

-     def test_run_cmd_unsafe_err_popen(self, mc_popen):

-         cmd = "foo --bar"

-         mc_popen.side_effect = IOError()

- 

-         with pytest.raises(CreateRepoError) as err:

-             run_cmd_unsafe(cmd, self.tmp_dir_name, lock_path="/tmp") == "stdout"

- 

-         assert err.value.cmd == cmd

-         assert mc_popen.call_args[0][0] == ["foo", "--bar"]

-         mc_popen.reset()

- 

-     def test_run_cmd_unsafe_err_return_code(self, mc_popen):

-         cmd = "foo --bar"

-         mc_popen.return_value.communicate.return_value = ("stdout", "stderr")

-         mc_popen.return_value.returncode = 1

- 

- 

-         with pytest.raises(CreateRepoError) as err:

-             run_cmd_unsafe(cmd, self.tmp_dir_name, lock_path="/tmp") == "stdout"

- 

-         assert err.value.cmd == cmd

-         assert err.value.stdout == "stdout"

-         assert err.value.stderr == "stderr"

-         assert err.value.exit_code == 1

-         assert mc_popen.call_args[0][0] == ["foo", "--bar"]

-         mc_popen.reset()

- 

-     def test_run_cmd_unsafe_err_communicate(self, mc_popen):

-         cmd = "foo --bar"

-         mc_handle = MagicMock()

-         mc_popen.return_value = MagicMock()

-         mc_handle.returncode = 0

-         mc_handle.side_effect = RuntimeError()

- 

-         with pytest.raises(CreateRepoError) as err:

-             run_cmd_unsafe(cmd, self.tmp_dir_name, lock_path="/tmp") == "stdout"

- 

-         assert err.value.cmd == cmd

-         assert mc_popen.call_args[0][0] == ["foo", "--bar"]

-         mc_popen.reset()

- 

-     def test_createrepo_generated_commands_existing_repodata(self, mc_run_cmd_unsafe):

-         path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")

-         expected_epel_5 = ('/usr/bin/createrepo_c --database --ignore-lock --local-sqlite --cachedir /tmp/ --workers 8 '

-                            '--update -s sha --checksum md5 ' + path_epel_5)

-         path_fedora = os.path.join(self.tmp_dir_name, "fedora-21")

-         expected_fedora = ('/usr/bin/createrepo_c --database --ignore-lock --local-sqlite --cachedir /tmp/ --workers 8 '

-                            '--update ' + path_fedora)

-         for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]:

-             os.makedirs(path)

- 

-             repo_path = os.path.join(path, "repodata")

-             os.makedirs(repo_path)

-             with open(os.path.join(repo_path, "repomd.xml"), "w") as handle:

-                 handle.write("1")

- 

-             createrepo_unsafe(path)

-             assert mc_run_cmd_unsafe.call_args[0][0] == expected

- 

-     def test_createrepo_generated_commands_comps_xml(self, mc_run_cmd_unsafe):

-         path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")

-         path_fedora = os.path.join(self.tmp_dir_name, "fedora-21")

-         for path in [path_epel_5, path_fedora]:

-             for add_comps in [True, False]:

-                 os.makedirs(path)

- 

-                 comps_path = os.path.join(path, "comps.xml")

-                 if add_comps:

-                     with open(comps_path, "w") as handle:

-                         handle.write("1")

- 

-                 repo_path = os.path.join(path, "repodata")

-                 os.makedirs(repo_path)

-                 with open(os.path.join(repo_path, "repomd.xml"), "w") as handle:

-                     handle.write("1")

- 

-                 createrepo_unsafe(path)

-                 if add_comps:

-                     assert "--groupfile" in mc_run_cmd_unsafe.call_args[0][0]

-                 else:

-                     assert "--groupfile" not in mc_run_cmd_unsafe.call_args[0][0]

-                 mc_run_cmd_unsafe.mock_reset()

-                 shutil.rmtree(path, ignore_errors=True)

- 

-     def test_createrepo_devel_generated_commands_existing_repodata(self, mc_run_cmd_unsafe):

-         path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")

-         expected_epel_5 = ("/usr/bin/createrepo_c --database --ignore-lock --local-sqlite --cachedir /tmp/ --workers 8 "

-                            "-s sha --checksum md5 "

-                            "--outputdir " + os.path.join(path_epel_5, "devel") + " "

-                            "--baseurl " + self.base_url + " " + path_epel_5)

-         path_fedora = os.path.join(self.tmp_dir_name, "fedora-21")

-         expected_fedora = ("/usr/bin/createrepo_c --database --ignore-lock --local-sqlite --cachedir /tmp/ --workers 8 "

-                            "--outputdir " + os.path.join(path_fedora, "devel") + " "

-                            "--baseurl " + self.base_url + " " + path_fedora)

-         for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]:

-             os.makedirs(path)

- 

-             repo_path = os.path.join(path, "devel", "repodata")

-             os.makedirs(repo_path)

-             with open(os.path.join(repo_path, "repomd.xml"), "w") as handle:

-                 handle.write("1")

- 

-             createrepo_unsafe(path, base_url=self.base_url, dest_dir="devel")

-             assert mc_run_cmd_unsafe.call_args[0][0] == expected

- 

-     def test_createrepo_devel_generated_commands(self, mc_run_cmd_unsafe):

-         path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")

-         expected_epel_5 = ("/usr/bin/createrepo_c --database --ignore-lock --local-sqlite --cachedir /tmp/ --workers 8 "

-                            "-s sha --checksum md5 "

-                            "--outputdir " + os.path.join(path_epel_5, "devel") + " "

-                            "--baseurl " + self.base_url + " " + path_epel_5)

-         path_fedora = os.path.join(self.tmp_dir_name, "fedora-21")

-         expected_fedora = ("/usr/bin/createrepo_c --database --ignore-lock --local-sqlite --cachedir /tmp/ --workers 8 "

-                            "--outputdir " + os.path.join(path_fedora, "devel") + " "

-                            "--baseurl " + self.base_url + " " + path_fedora)

-         for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]:

-             os.makedirs(path)

- 

-             createrepo_unsafe(path, base_url=self.base_url, dest_dir="devel")

-             assert os.path.exists(os.path.join(path, "devel"))

-             assert mc_run_cmd_unsafe.call_args[0][0] == expected

-             # assert mc_popen.call_args == mock.call(expected, stderr=-1, stdout=-1)

- 

-     def test_createrepo_devel_creates_folder(self, mc_run_cmd_unsafe):

-         path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")

-         path_fedora = os.path.join(self.tmp_dir_name, "fedora-21")

- 

-         for path in [path_epel_5, path_fedora]:

-             os.makedirs(path)

- 

-             createrepo_unsafe(path, base_url=self.base_url, dest_dir="devel")

-             assert os.path.exists(os.path.join(path, "devel"))

- 

-     def _prep_batched_repo(self, some_dir, full=False, add=None, delete=None):