#100 Add tool to generate a repo file for a compose
Merged 4 years ago by lsedlar. Opened 4 years ago by lsedlar.
lsedlar/compose-utils repo-tool  into  master

file modified
+3
@@ -45,6 +45,9 @@ 

  **compose-latest-symlink**

      create a symbolic link with a well-known name to the given compose

  

+ **compose-write-repo-file**

+     create a `.repo` file pointing to repositories in a compose

+ 

  

  Related tools

  -------------

@@ -0,0 +1,14 @@ 

+ #!/usr/bin/env python

+ # -*- encoding: utf-8 -*-

+ 

+ import os

+ import sys

+ 

+ here = sys.path[0]

+ if here != "/usr/bin":

+     sys.path[0] = os.path.dirname(here)

+ 

+ from compose_utils import repo_file

+ 

+ if __name__ == '__main__':

+     repo_file.main()

@@ -0,0 +1,157 @@ 

+ # -*- encoding: utf-8 -*-

+ 

+ from __future__ import print_function

+ 

+ import argparse

+ import os

+ import sys

+ 

+ import productmd

+ 

+ REPO = """\

+ [{name}]

+ named = {name}

+ baseurl = {baseurl}

+ enabled = {enabled}

+ gpgcheck = 0

+ """

+ 

+ CONTENT_TYPES = {

+     "repository": "-rpms",

+     "debug_repository": "-debuginfo-rpms",

+     "source_repository": "-source-rpms",

+ }

+ 

+ 

+ def translate(path, strip_prefix, url_prefix):

+     """Remove prefix from path and replace it with url_prefix."""

+     if not path.startswith(strip_prefix):

+         raise RuntimeError("{0} does not start with {1}.".format(path, strip_prefix))

+     return url_prefix + path[len(strip_prefix) :]

+ 

+ 

+ def emit(compose, opts, variant, output, content_type="repository"):

+     """Print configuration for a single repository into output."""

+     paths = getattr(variant.paths, content_type)

+     release = compose.info.release

+     bp = compose.info.base_product

+     name = opts.name_pattern.format(

+         release_id=compose.info.release_id,

+         release_short=compose.info.release.short,

+         release_name=release.name,

+         release_version=release.version,

+         release_type=release.type,

+         release_major_version=release.major_version,

+         base_product_short=bp.short if release.is_layered else None,

+         base_product_name=bp.name if release.is_layered else None,

+         base_product_version=bp.version if release.is_layered else None,

+         base_product_type=bp.type if release.is_layered else None,

+         compose_id=compose.info.compose.id,

+         compose_type=compose.info.compose.type,

+         compose_date=compose.info.compose.date,

+         compose_respin=compose.info.compose.respin,

+         label=compose.info.compose.label,

+         variant=variant.uid,

+         arch=opts.arch,

+     )

+     name += CONTENT_TYPES[content_type]

+     enabled = "1" if not opts.disabled or variant.uid not in opts.disabled else "0"

+ 

+     if opts.arch:

+         if opts.arch not in paths:

+             raise RuntimeError(

+                 "Variant {0} does not have arch {1}.".format(variant.uid, opts.arch)

+             )

+         url = paths[opts.arch]

+     elif paths:

+         arch, path = list(paths.items())[0]

+         url = path.replace(arch, "$basearch")

+     else:

+         # No paths...

+         return

+ 

+     baseurl = os.path.join(compose.compose_path, url)

+     if opts.translate:

+         baseurl = translate(baseurl, *opts.translate)

+     else:

+         baseurl = "file://" + baseurl

+ 

+     content = REPO.format(name=name, enabled=enabled, baseurl=baseurl)

+     print(content, file=output)

+ 

+ 

+ def run(opts, output):

+     compose = productmd.Compose(os.path.realpath(opts.COMPOSE))

+ 

+     def by_uid(variant):

+         return variant.uid

+ 

+     for variant in sorted(compose.info.variants.variants.values(), key=by_uid):

+         if opts.variant and variant.uid not in opts.variant:

+             # Variant is excluded

+             continue

+ 

+         emit(compose, opts, variant, output)

+         if opts.include_debuginfo:

+             emit(compose, opts, variant, output, content_type="debug_repository")

+ 

+         if opts.include_source:

+             emit(compose, opts, variant, output, content_type="source_repository")

+ 

+ 

+ def parse_mapping(value):

+     return value.split(",", 1)

+ 

+ 

+ def main(args=None):

+     parser = argparse.ArgumentParser()

+     parser.add_argument("COMPOSE", help="Compose to work with.")

+     parser.add_argument(

+         "--arch", help="Hardcode given architecture instead of using $basearch."

+     )

+     parser.add_argument(

+         "--include-debuginfo",

+         action="store_true",

+         help="Create entries for debuginfo repos as well.",

+     )

+     parser.add_argument(

+         "--include-source",

+         action="store_true",

+         help="Create entries for source repos as well.",

+     )

+     parser.add_argument(

+         "--variant",

+         action="append",

+         help="Include only these variants. May be used multiple times.",

+     )

+     parser.add_argument(

+         "--disabled",

+         metavar="VARIANT",

+         action="append",

+         help="Disable this variant by default. May be used multiple times.",

+     )

+     parser.add_argument(

+         "-o",

+         "--output",

+         default="/dev/stdout",

+         help="Path for output file. Defaults to stdout.",

+     )

+     parser.add_argument(

+         "--translate",

+         metavar="PATH_PREFIX,URL_PREFIX",

+         help="How to generate URLs.",

+         type=parse_mapping,

+     )

+     parser.add_argument(

+         "--name-pattern",

+         default="{release_short}-{release_version}-{variant}",

+         help="Pattern for repository names.",

+     )

+     opts = parser.parse_args(args)

+ 

+     try:

+         with open(opts.output, "w") as output:

+             run(opts, output)

+     except RuntimeError as exc:

+         print(str(exc), file=sys.stderr)

+         sys.exit(1)

@@ -0,0 +1,66 @@ 

+ .TH compose-write-repo-file 1

+ .SH NAME

+ compose-write-repo-file \- write .repo file pointing to a compose

+ .SH SYNOPSIS

+ .B compose-write-repo-file

+ [\fIOPTIONS\fR...]

+ \fICOMPOSE_PATH\fR

+ .SH DESCRIPTION

+ .B compose-write-repo-file

+ loads metadata from a compose and creates a YUM/DNF configuration file with

+ entries for repositories in the compose.

+ .SH OPTIONS

+ .TP

+ .BR \-h ", " \-\-help

+ Print help and exit.

+ .TP

+ .BR \-\-arch = \fIARCH\fR

+ Use URLs with this hardcoded architecture. By default \fI$basearch\fR is used.

+ .TP

+ .BR \-\-include\-debuginfo

+ Create entries for debuginfo repositories.

+ .TP

+ .BR \-\-include\-source

+ Create entries for source repositories.

+ .TP

+ .BR \-\-variant = \fIVARIANT\fR

+ Create entries only for listed variants. Can be used multiple times. Variants

+ that do not include repositories are skipped automatically.

+ .TP

+ .BR \-\-disabled = \fIVARIANT\fR

+ Make entries for this variant disabled. By default all repositories are created

+ as enabled.

+ .TP

+ .BR \-\-output = \fIFILE_PATH\fR

+ Write output to this file instead of to stdout.

+ .TP

+ .BR \-\-translate = \fIPATH_PREFIX,URL_PREFIX\fR

+ By default local filepaths are created. With this option it is possible to

+ generate URLs. The path prefix will be stripped from path, and replaced with

+ URL prefix.

+ .TP

+ .BR \-\-name\-pattern = \fIPATTERN\fR

+ Customize name for the repositories. These fragments are available:

+ .sp

+ .RS

+ - \fIrelease_id\fR, \fIrelease_short\fR, \fIrelease_name\fR, \fIrelease_version\fR,

+ \fIrelease_type\fR, \fIrelease_major_version\fR

+ .br

+ - \fIbase_product_short\fR, \fIbase_product_name\fR, \fIbase_product_version\fR,

+ \fIbase_product_type\fR (these are only available for layered product composes)

+ .br

+ - \fIcompose_id\fR, \fIcompose_type\fR, \fIcompose_date\fR, \fIcompose_respin\fR,

+ \fIlabel\fR

+ .br

+ - \fIvariant\fR (should always be used unless a single variant is processed)

+ .br

+ - \fIarch\fR (only makes sense if \fB\-\-arch\fR is used)

+ .sp

+ The default is \fI{release_short}-{release_version}-{variant}\fR

+ .SH EXIT CODE

+ Exit code is always 0 on success and non-zero if there was a problem loading

+ the compose.

+ .SH BUGS

+ Please report bugs at

+ .br

+ https://pagure.io/compose-utils/issues

file modified
+2
@@ -25,6 +25,7 @@ 

          'bin/compose-partial-copy',

          'bin/compose-print-essentials',

          'bin/compose-report-package-moves',

+         'bin/compose-write-repo-file',

      ],

      install_requires=[

          'productmd>=1.0',
@@ -42,6 +43,7 @@ 

              'doc/compose-partial-copy.1',

              'doc/compose-print-essentials.1',

              'doc/compose-report-package-moves.1',

+             'doc/compose-write-repo-file.1',

          ]),

      ],

      include_package_data=True,

@@ -0,0 +1,6 @@ 

+ [DP-1.0-Client-rpms]

+ named = DP-1.0-Client-rpms

+ baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Client/$basearch/os

+ enabled = 1

+ gpgcheck = 0

+ 

@@ -0,0 +1,24 @@ 

+ [DP-1.0-Client-rpms]

+ named = DP-1.0-Client-rpms

+ baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Client/$basearch/os

+ enabled = 1

+ gpgcheck = 0

+ 

+ [DP-1.0-Client-debuginfo-rpms]

+ named = DP-1.0-Client-debuginfo-rpms

+ baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Client/$basearch/debug/tree

+ enabled = 1

+ gpgcheck = 0

+ 

+ [DP-1.0-Server-rpms]

+ named = DP-1.0-Server-rpms

+ baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Server/$basearch/os

+ enabled = 1

+ gpgcheck = 0

+ 

+ [DP-1.0-Server-debuginfo-rpms]

+ named = DP-1.0-Server-debuginfo-rpms

+ baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Server/$basearch/debug/tree

+ enabled = 1

+ gpgcheck = 0

+ 

@@ -0,0 +1,12 @@ 

+ [DP-1.0-Client-rpms]

+ named = DP-1.0-Client-rpms

+ baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Client/$basearch/os

+ enabled = 1

+ gpgcheck = 0

+ 

+ [DP-1.0-Server-rpms]

+ named = DP-1.0-Server-rpms

+ baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Server/$basearch/os

+ enabled = 1

+ gpgcheck = 0

+ 

empty or binary file added
@@ -0,0 +1,24 @@ 

+ [DP-1.0-Client-rpms]

+ named = DP-1.0-Client-rpms

+ baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Client/$basearch/os

+ enabled = 1

+ gpgcheck = 0

+ 

+ [DP-1.0-Client-source-rpms]

+ named = DP-1.0-Client-source-rpms

+ baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Client/source/tree

+ enabled = 1

+ gpgcheck = 0

+ 

+ [DP-1.0-Server-rpms]

+ named = DP-1.0-Server-rpms

+ baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Server/$basearch/os

+ enabled = 1

+ gpgcheck = 0

+ 

+ [DP-1.0-Server-source-rpms]

+ named = DP-1.0-Server-source-rpms

+ baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Server/source/tree

+ enabled = 1

+ gpgcheck = 0

+ 

@@ -0,0 +1,12 @@ 

+ [DP-1.0-Client-rpms]

+ named = DP-1.0-Client-rpms

+ baseurl = http://example.com/composes/DP-1.0-20160315.t.0/compose/Client/$basearch/os

+ enabled = 1

+ gpgcheck = 0

+ 

+ [DP-1.0-Server-rpms]

+ named = DP-1.0-Server-rpms

+ baseurl = http://example.com/composes/DP-1.0-20160315.t.0/compose/Server/$basearch/os

+ enabled = 1

+ gpgcheck = 0

+ 

@@ -0,0 +1,12 @@ 

+ [DP-1.0-Client-rpms]

+ named = DP-1.0-Client-rpms

+ baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Client/x86_64/os

+ enabled = 1

+ gpgcheck = 0

+ 

+ [DP-1.0-Server-rpms]

+ named = DP-1.0-Server-rpms

+ baseurl = file:///composes/DP-1.0-20160315.t.0/compose/Server/x86_64/os

+ enabled = 1

+ gpgcheck = 0

+ 

@@ -0,0 +1,102 @@ 

+ # -*- encoding: utf-8 -*-

+ 

+ import difflib

+ import os

+ import tempfile

+ 

+ try:

+     import unittest2 as unittest

+ except ImportError:

+     import unittest

+ 

+ import mock

+ from six import StringIO

+ 

+ from .helpers import get_compose_path, get_fixture

+ 

+ from compose_utils import repo_file

+ 

+ 

+ class TestRepoFile(unittest.TestCase):

+     def setUp(self):

+         _, self.temp_file = tempfile.mkstemp(prefix="repo-file-")

+ 

+     def tearDown(self):

+         os.remove(self.temp_file)

+ 

+     def assertFilesEqual(self, fn1, fn2, strip=None):

+         with open(fn1, "r") as f1:

+             lines1 = f1.readlines()

+         with open(fn2, "r") as f2:

+             lines2 = f2.readlines()

+             if strip:

+                 lines2 = [line.replace(strip, "") for line in lines2]

+         diff = "".join(

+             difflib.unified_diff(lines1, lines2, fromfile="EXPECTED", tofile="ACTUAL")

+         )

+         self.assertEqual(diff, "", "Files differ:\n" + diff)

+ 

+     def success_run(self, args, compose_id, expected):

+         repo_file.main(

+             args + ["--output", self.temp_file, get_compose_path(compose_id)]

+         )

+         strip = os.path.dirname(__file__) if "--translate" not in args else None

+         self.assertFilesEqual(get_fixture(expected), self.temp_file, strip=strip)

+ 

+     def test_no_filter(self):

+         self.success_run([], "DP-1.0-20160315.t.0", "default.repo")

+ 

+     def test_translate(self):

+         self.success_run(

+             ["--translate", os.path.dirname(__file__) + ",http://example.com"],

+             "DP-1.0-20160315.t.0",

+             "translate.repo",

+         )

+ 

+     def test_with_source(self):

+         self.success_run(["--include-source"], "DP-1.0-20160315.t.0", "sources.repo")

+ 

+     def test_with_debuginfo(self):

+         self.success_run(

+             ["--include-debuginfo"], "DP-1.0-20160315.t.0", "debuginfo.repo"

+         )

+ 

+     def test_skip_variant(self):

+         self.success_run(["--variant=Client"], "DP-1.0-20160315.t.0", "client.repo")

+ 

+     def test_hardcode_arch(self):

+         self.success_run(["--arch=x86_64"], "DP-1.0-20160315.t.0", "x86_64.repo")

+ 

+     def test_no_paths(self):

+         self.success_run([], "DP-1.0-20181012.t.0", "empty.repo")

+ 

+     def test_bad_hardcode_arch(self):

+         with mock.patch("sys.stderr", new_callable=StringIO) as mock_err:

+             with self.assertRaises(SystemExit):

+                 repo_file.main(

+                     [

+                         "--output",

+                         self.temp_file,

+                         "--arch=ppc64le",

+                         get_compose_path("DP-1.0-20160315.t.0"),

+                     ]

+                 )

+         self.assertIn(

+             "Variant Client does not have arch ppc64le.\n", mock_err.getvalue()

+         )

+ 

+     def test_bad_translate(self):

+         with mock.patch("sys.stderr", new_callable=StringIO) as mock_err:

+             with self.assertRaises(SystemExit):

+                 repo_file.main(

+                     [

+                         "--output",

+                         self.temp_file,

+                         get_compose_path("DP-1.0-20160315.t.0"),

+                         "--translate",

+                         "/foo/bar,http://example.com",

+                     ]

+                 )

+         self.assertIn(

+             "/Client/$basearch/os does not start with /foo/bar.\n", mock_err.getvalue()

+         )

file modified
+1 -1
@@ -18,7 +18,7 @@ 

  [flake8]

  exclude = doc,*.pyc,*.py~,*.in,*.spec,*.sh,*.rst

  filename = *.py

- ignore = E501,E402,W503

+ ignore = E501,E402,W503,E203

  

  [run]

  omit = tests/*