- import yaml

- from freshmaker.utils import sorted_by_nvr, get_distgit_files, get_distgit_url

-     @region.cache_on_arguments()

-     def _get_additional_data_from_distgit(self, repository, branch, commit):

-         """

-         Finds out information about this image in distgit and returns a dict

-         with following keys:

- 

-         - "generate_pulp_repos" - True when Freshmaker needs to generate Pulp

-             repos using ODCS itself (it means it is not done by OSBS).

-         - "content_sets" - List of x86_64 content_sets as defined in

-             content_sets.yml. We care only about x86_64, because to build

-             non-x86_64 image, OSBS will generate the Pulp repos and therefore

-             we don't need content_sets in Freshmaker.

-         """

-         nvr = self.nvr

-         data = {"generate_pulp_repos": False,

-                 "content_sets": []}

- 

-         if not repository or not branch or not commit:

-             log.warning("%s: Cannot get additional data from distgit.", nvr)

-             return data

-         if "/" in repository:

-             namespace, name = repository.split("/")

-         else:

-             namespace = "rpms"

-             name = repository

- 

-         try:

-             repo_url = get_distgit_url(namespace, name, ssh=False)

-             files = get_distgit_files(

-                 repo_url, commit, ["content_sets.yml", "container.yaml"],

-                 logger=log)

-         except OSError as e:

-             self.log_error("Error while fetching dist-git repo files: %s" % e)

-             return data

- 

-         content_sets_data = files["content_sets.yml"]

-         container_data = files["container.yaml"]

- 

-         if content_sets_data is None:

-             log.debug("%s: Should generate Pulp repo, content_sets.yml does "

-                       "not exist.", nvr)

-             data["generate_pulp_repos"] = True

-             return data

- 

-         try:

-             content_sets_yaml = yaml.safe_load(content_sets_data)

-         except Exception as err:

-             log.exception(err)

-             data["generate_pulp_repos"] = True

-             return data

- 

-         if not content_sets_yaml:

-             log.warning("%s: Should generate Pulp repo, content_sets.yml is "

-                         "empty" % nvr)

-             data["generate_pulp_repos"] = True

-             return data

- 

-         for content_sets in content_sets_yaml.values():

-             data["content_sets"] += content_sets

- 

-         if container_data is None:

-             log.debug("%s: Should generate Pulp repo, container.yaml does not "

-                       "exist.", nvr)

-             data["generate_pulp_repos"] = True

-             return data

- 

-         container_yaml = yaml.safe_load(container_data)

- 

-         if (not container_yaml or "compose" not in container_yaml or

-                 "pulp_repos" not in container_yaml["compose"] or

-                 not container_yaml["compose"]["pulp_repos"]):

-             log.debug("%s: Should generate Pulp repo, pulp_repos not "

-                       "enabled in containery.yaml.", nvr)

-             data["generate_pulp_repos"] = True

-             return data

- 

-         # This is workaround until OSBS-5919 is fixed.

-         if "arches" in self and self["arches"] == "x86_64":

-             data["generate_pulp_repos"] = True

- 

-         return data

- 

-         data = self._get_additional_data_from_distgit(

-             self["repository"], self["git_branch"], self["commit"])

-         self["generate_pulp_repos"] = data["generate_pulp_repos"]

- 

-         # Prefer content_sets from content_sets.yml, because it contains

-         # content_sets for all architectures.

-         if data["content_sets"]:

-             self["content_sets"] = data["content_sets"]

-             self["content_sets_source"] = "distgit"

-             log.info("Container image %s uses following content sets: %r",

-                      self.nvr, data["content_sets"])

-             return

-         # In case content_sets cannot be get from content_sets.yml and also

-         # are not set directly in this ContainerImage, try to get them from

-         # children image.

-             for k, v in groupby(sorted_images, key=lambda item: item.nvr):

-                 images.append(next(v))

- import getpass

- import os

- import tarfile

- import io

- def get_distgit_url(namespace, name, ssh=True, user=None):

-     """

-     Returns the dist-git repository URL.

- 

-     :param str namespace: Namespace in which the repository is located, for

-         example "rpms", "containers", "modules", ...

-     :param str name: Name of the repository inside the namespace.

-     :param bool ssh: indicate whether SSH auth will be used when fetching the

-         files. Default is True.

-     :param str user: If set, overrides the default user for SSH auth.

-         Otherwise, username specified in config ``git_user`` will be used.

-     :return: The dist-git repository URL.

-     :rtype: str

-     """

-     if ssh:

-         if user is None:

-             if hasattr(conf, 'git_user'):

-                 user = conf.git_user

-             else:

-                 user = getpass.getuser()

-         repo_url = conf.git_ssh_base_url % user

-     else:

-         repo_url = conf.git_base_url

- 

-     repo_url = os.path.join(repo_url, namespace, name)

-     return repo_url

- 

- 

- @retry(logger=log)

- def get_distgit_files(repo_url, commit_or_branch, files, logger=None):

-     """

-     Fetches the `files` from dist-git repository defined by `namespace`,

-     `name` and `commit_or_branch` and returns them.

- 

-     This is much faster than cloning the dist-git repository and should be

-     preferred method to get the files from dist-git in case the full clone

-     of repository is not needed.

- 

-     :param str repo_url: the repository URL.

-     :param str commit_or_branch: Commit hash or branch name.

-     :param list[str] files: List of files to fetch.

-     :param freshmaker.log logger: Logger instance.

-     :return: Dictionary with file name as key and file content as value.

-         If the file does not exist in a dist-git repo, None is used as value.

-     :rtype: dict[str, str or None]

-     """

-     # Use the "git archive" to get the files in tarball and then extract

-     # them and return in dict. We need to go file by file, because the

-     # "git archive" would fail completely in case any file does not exist

-     # in the git repo.

-     ret = {}

-     for f in files:

-         try:

-             cmd = ['git', 'archive', '--remote=%s' % repo_url,

-                    commit_or_branch, f]

-             tar_data = _run_command(cmd, logger=logger, log_output=False)

-             with io.BytesIO(tar_data.encode()) as tar_bytes:

-                 with tarfile.open(fileobj=tar_bytes) as tar:

-                     for member in tar.getmembers():

-                         with tar.extractfile(member) as fd:

-                             ret[member.name] = fd.read().decode()

-         except OSError as e:

-             if "path not found" in str(e):

-                 ret[os.path.basename(f)] = None

-             else:

-                 raise

- 

-     return ret

- 

- 

- from freshmaker.utils import sorted_by_nvr, get_distgit_url

- from freshmaker import log

- class TestGetAdditionalDataFromDistGit(helpers.FreshmakerTestCase):

- 

-     def setUp(self):

-         super(TestGetAdditionalDataFromDistGit, self).setUp()

- 

-         self.patcher = helpers.Patcher(

-             "freshmaker.lightblue.")

-         self.get_distgit_files = self.patcher.patch("get_distgit_files")

-         self.get_distgit_files.return_value = {

-             "content_sets.yml": None,

-             "container.yaml": None,

-         }

- 

-     def tearDown(self):

-         super(TestGetAdditionalDataFromDistGit, self).tearDown()

-         self.patcher.unpatch_all()

- 

-     def test_generate(self):

-         self.get_distgit_files.return_value = {

-             "content_sets.yml": "x86_64:\n  - content_set",

-             "container.yaml": "compose:\n  pulp_repos: True",

-         }

- 

-         image = ContainerImage.create({"brew": {"build": "nvr"}})

-         ret = image._get_additional_data_from_distgit(

-             "rpms/foo-docker", "branch", "commit")

-         self.assertEqual(ret["generate_pulp_repos"], False)

- 

-         repo_url = get_distgit_url('rpms', 'foo-docker', ssh=False)

-         self.get_distgit_files.assert_called_once_with(

-             repo_url, "commit",

-             ["content_sets.yml", "container.yaml"], logger=log)

- 

-     def test_generate_os_error(self):

-         self.get_distgit_files.side_effect = OSError(

-             "Got an error (128) from git: fatal: reference is not a tree: "

-             "4d42e2009cec70d871c65de821396cd750d523f1")

- 

-         image = ContainerImage.create({"brew": {"build": "nvr"}})

-         ret = image._get_additional_data_from_distgit(

-             "rpms/foo-docker", "branch", "commit")

-         self.assertEqual(ret["generate_pulp_repos"], False)

- 

-         self.assertEqual(

-             image["error"],

-             "Error while fetching dist-git repo files: Got an error (128) from git: "

-             "fatal: reference is not a tree: 4d42e2009cec70d871c65de821396cd750d523f1")

- 

-         repo_url = get_distgit_url('rpms', 'foo-docker', ssh=False)

-         self.get_distgit_files.assert_called_once_with(

-             repo_url, 'commit',

-             ["content_sets.yml", "container.yaml"], logger=log)

- 

-     def test_generate_no_namespace(self):

-         self.get_distgit_files.return_value = {

-             "content_sets.yml": "x86_64:\n  - content_set",

-             "container.yaml": "compose:\n  pulp_repos: True",

-         }

- 

-         image = ContainerImage.create({"brew": {"build": "nvr"}})

-         ret = image._get_additional_data_from_distgit(

-             "foo-docker", "branch", "commit")

-         self.assertEqual(ret["generate_pulp_repos"], False)

- 

-         repo_url = get_distgit_url('rpms', 'foo-docker', ssh=False)

-         self.get_distgit_files.assert_called_once_with(

-             repo_url, "commit",

-             ["content_sets.yml", "container.yaml"], logger=log)

- 

-     def test_generate_no_pulp_repos(self):

-         self.get_distgit_files.return_value = {

-             "content_sets.yml": "x86_64:\n  - content_set",

-             "container.yaml": "compose:\n  pulp_repos_x: True",

-         }

- 

-         image = ContainerImage.create({"brew": {"build": "nvr"}})

-         ret = image._get_additional_data_from_distgit(

-             "rpms/foo-docker", "branch", "commit")

-         self.assertEqual(ret["generate_pulp_repos"], True)

- 

-     def test_generate_pulp_repos_false(self):

-         self.get_distgit_files.return_value = {

-             "content_sets.yml": "x86_64:\n  - content_set",

-             "container.yaml": "compose:\n  pulp_repos: False",

-         }

- 

-         image = ContainerImage.create({"brew": {"build": "nvr"}})

-         ret = image._get_additional_data_from_distgit(

-             "rpms/foo-docker", "branch", "commit")

-         self.assertEqual(ret["generate_pulp_repos"], True)

- 

-     def test_generate_no_content_sets_yml(self):

-         self.get_distgit_files.return_value = {

-             "content_sets.yml": None,

-             "container.yaml": "compose:\n  pulp_repos: False",

-         }

- 

-         image = ContainerImage.create({"brew": {"build": "nvr"}})

-         ret = image._get_additional_data_from_distgit(

-             "rpms/foo-docker", "branch", "commit")

-         self.assertEqual(ret["generate_pulp_repos"], True)

- 

-     def test_generate_no_container_yaml(self):

-         self.get_distgit_files.return_value = {

-             "content_sets.yml": "x86_64:\n  - content_set",

-             "container.yaml": None,

-         }

- 

-         image = ContainerImage.create({"brew": {"build": "nvr"}})

-         ret = image._get_additional_data_from_distgit(

-             "rpms/foo-docker", "branch", "commit")

-         self.assertEqual(ret["generate_pulp_repos"], True)

- 

-     def test_generate_content_sets_yml_empty(self):

-         self.get_distgit_files.return_value = {

-             "content_sets.yml": "",

-             "container.yaml": "compose:\n  pulp_repos: False",

-         }

- 

-         image = ContainerImage.create({"brew": {"build": "nvr"}})

-         ret = image._get_additional_data_from_distgit(

-             "rpms/foo-docker", "branch", "commit")

-         self.assertEqual(ret["generate_pulp_repos"], True)

- 

-     def test_generate_container_yaml_empty(self):

-         self.get_distgit_files.return_value = {

-             "content_sets.yml": "x86_64:\n  - content_set",

-             "container.yaml": "",

-         }

- 

-         image = ContainerImage.create({"brew": {"build": "nvr"}})

-         ret = image._get_additional_data_from_distgit(

-             "rpms/foo-docker", "branch", "commit")

-         self.assertEqual(ret["generate_pulp_repos"], True)

- 

- 

-         self.get_distgit_files = self.patcher.patch("get_distgit_files")

-         self.get_distgit_files.return_value = {

-             "content_sets.yml": None,

-             "container.yaml": None,

-         }

-         self.get_distgit_files = self.patcher.patch("get_distgit_files")

-         self.get_distgit_files.return_value = {

-             "content_sets.yml": None,

-             "container.yaml": None,

-         }

- import shutil

- import tempfile

- 

- from freshmaker.utils import get_rebuilt_nvr, sorted_by_nvr, get_distgit_files

- 

- 

- class TestGetDistGitFiles(object):

-     """Test get_distgit_files"""

- 

-     @classmethod

-     def setup_class(cls):

-         import os

-         import subprocess

-         cls.repo_dir = tempfile.mkdtemp()

-         with open(os.path.join(cls.repo_dir, 'a.txt'), 'w') as f:

-             f.write('hello')

-         with open(os.path.join(cls.repo_dir, 'b.txt'), 'w') as f:

-             f.write('world')

-         git_cmds = [

-             ['git', 'init'],

-             ['git', 'add', 'a.txt', 'b.txt'],

-             ['git', 'config', 'user.name', 'tester'],

-             ['git', 'config', 'user.email', 'tester@localhost'],

-             ['git', 'commit', '-m', 'initial commit for test'],

-         ]

-         for cmd in git_cmds:

-             subprocess.check_call(cmd, cwd=cls.repo_dir)

- 

-         cls.repo_url = 'file://' + cls.repo_dir

- 

-     @classmethod

-     def teardown_class(cls):

-         shutil.rmtree(cls.repo_dir)

- 

-     @pytest.mark.parametrize('files,expected', [

-         [['a.txt'], {'a.txt': 'hello'}],

-         [['a.txt', 'b.txt'], {'a.txt': 'hello', 'b.txt': 'world'}],

-     ])

-     def test_get_files(self, files, expected):

-         result = get_distgit_files(self.repo_url, 'master', files)

-         assert expected == result

- 

-     @patch('freshmaker.utils._run_command')

-     def test_error_path_not_found(self, run_command):

-         run_command.side_effect = OSError('path not found')

-         result = get_distgit_files(self.repo_url, 'master', ['a.txt'])

-         assert {'a.txt': None} == result

- 

-     @patch('freshmaker.utils._run_command')

-     @patch('time.sleep')

-     def test_unhandled_error_occurs(self, sleep, run_command):

-         run_command.side_effect = ValueError

-         with pytest.raises(ValueError):

-             get_distgit_files(self.repo_url, 'master', ['a.txt'])