From 18658a656e178065f24df273125c9034f79af070 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Jun 12 2017 14:21:13 +0000 Subject: Improve README, add way how to test compose without /mnt/koji using the local repository --- diff --git a/README.md b/README.md index 345e780..a7881c4 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,46 @@ # On Demand Compose Service -## Testing +Currently, there is no API which can be used by other services or even people to generate temporary composes with limited content. There is increasing need for such composes from various reasons: +To rebuild Docker images automatically with updated packages (for example when there is OpenSSL security update), we need a repository containing the updated packages, so we can point Koji to take the packages from these repositories without waiting for the packages to appear in the official public repository. +To test modules right after the build, QA team needs a compose containing the built module together with all the modules this module depends on, so QA team is able to install the module and run the tests. + +Furthermore: + +In the mid-/long-term, Fedora releng would like to generate the main compose from the smaller composes generated by the ODCS, so the composing would be faster. +Current composes are also not event-based. For example in Fedora, the composes are built even when nothing changed on input side of a compose. + + +## Unit-testing ``` $ tox -e py27,py35,flake8 ``` + +## Function testing + +You can test ODCS by generating compose form the `./tests/repo` repository using following commands: + +``` +$ ./create_sqlite_db +$ ./start_frontend_from_here +``` + +And in another terminal, submit a request to frontend: + +``` +$ ./submit_test_compose repo `pwd`/tests/repo ed +{ + "id": 1, + "owner": "Unknown", + "result_repo": null, + "source": "/home/hanzz/code/fedora-modularization/odcs/tests/repo", + "source_type": 3, + "state": 0, + "state_name": "wait", + "time_done": null, + "time_removed": null, + "time_submitted": "2017-06-12T14:18:19Z" +} +``` + +You should then see the Pungi process generating the compose and once it's done, the resulting compose in `./test_composes/latest-Unknown-1/compose/Temporary` directory. diff --git a/conf/config.py b/conf/config.py index 94d48a6..c06bafb 100644 --- a/conf/config.py +++ b/conf/config.py @@ -1,4 +1,4 @@ -from os import path +from os import path, mkdir # FIXME: workaround for this moment till confdir, dbdir (installdir etc.) are @@ -44,6 +44,11 @@ class DevConfiguration(BaseConfiguration): # Global network-related values, in seconds NET_TIMEOUT = 5 NET_RETRY_INTERVAL = 1 + TARGET_DIR = path.join(dbdir, "test_composes") + try: + mkdir(TARGET_DIR) + except: + pass class TestConfiguration(BaseConfiguration): diff --git a/odcs/pungi.py b/odcs/pungi.py index 28b013f..fe5c955 100644 --- a/odcs/pungi.py +++ b/odcs/pungi.py @@ -31,6 +31,7 @@ import odcs.utils class PungiSourceType: KOJI_TAG = 1 MODULE = 2 + REPO = 3 class PungiConfig(object): @@ -50,22 +51,26 @@ class PungiConfig(object): self.arches = arches else: self.arches = conf.arches - self.packages = packages + self.packages = packages or [] if source_type == PungiSourceType.KOJI_TAG: self.koji_tag = source - self.gather_source = "json" + self.gather_source = "comps" self.gather_method = "deps" elif source_type == PungiSourceType.MODULE: # We have to set koji_tag to something even when we are not using # it. - self.koji_tag = "not-used" + self.koji_tag = None self.gather_source = "module" self.gather_method = "nodeps" if self.packages: raise ValueError("Exact packages cannot be set for MODULE " "source type.") + elif source_type == PungiSourceType.REPO: + self.gather_source = "comps" + self.gather_method = "deps" + self.koji_tag = None else: raise ValueError("Unknown source_type %r" % source_type) @@ -78,20 +83,34 @@ class PungiConfig(object): if not self.sigkeys: return "sigkeys = [None]\n" + def _get_pkgset(self): + ret = "" + if self.source_type == PungiSourceType.REPO: + ret += "pkgset_source = 'repos'\n" + ret += "pkgset_repos = {\n" + for arch in self.arches: + ret += "'%s': [\n" % arch + ret += " '%s',\n" % self.source + ret += "],\n" + ret += "}\n" + else: + ret += "pkgset_source = 'koji'\n" + ret += "pkgset_koji_tag = '%s'\n" % self.koji_tag + ret += "pkgset_koji_inherit = False\n" + + return ret + def get_comps_config(self): if self.source_type == PungiSourceType.MODULE: return "" - if not self.packages: - return "" - cfg = """ odcs-group odcs-group - ODCS compose default group/description> + ODCS compose default group false true @@ -103,6 +122,7 @@ class PungiConfig(object): cfg += """ + """ return cfg @@ -130,7 +150,7 @@ class PungiConfig(object): cfg += " odcs-group\n" cfg += " " cfg += """ - + """ return cfg @@ -160,17 +180,15 @@ pdc_insecure = {pdc_insecure} pdc_develop = {pdc_develop} # PKGSET -pkgset_source = 'koji' - -# PKGSET - KOJI -pkgset_koji_tag = '{koji_tag}' -pkgset_koji_inherit = False +{pkgset} filter_system_release_packages = False # GATHER gather_source = '{gather_source}' gather_method = '{gather_method}' +{comps_file} +comps_file = 'comps.xml' check_deps = False greedy_method = 'build' @@ -194,8 +212,9 @@ koji_profile = '{koji_profile}' release_short=self.release_name[:16], bootable=self._get_bootable(), sigkeys=self._get_sigkeys(), pdc_url=self.pdc_url, pdc_insecure=self.pdc_insecure, pdc_develop=self.pdc_develop, - koji_tag=self.koji_tag, gather_source=self.gather_source, - gather_method=self.gather_method, koji_profile=self.koji_profile) + pkgset=self._get_pkgset(), gather_source=self.gather_source, + gather_method=self.gather_method, koji_profile=self.koji_profile, + comps_file = "comps_file = 'comps.xml'" if self.source_type != PungiSourceType.REPO else "") return cfg diff --git a/odcs/views.py b/odcs/views.py index f9aef88..144f5be 100644 --- a/odcs/views.py +++ b/odcs/views.py @@ -60,7 +60,7 @@ def generate_compose(compose_id): log.info("%r: Starting compose generation", compose) pungi_cfg = PungiConfig(compose.owner, "1", compose.source_type, - compose.source) + compose.source, packages=compose.packages.split(" ")) pungi = Pungi(pungi_cfg) pungi.run() @@ -106,6 +106,8 @@ class ODCSAPI(MethodView): source_type = PungiSourceType.MODULE elif source_type == "tag": source_type = PungiSourceType.KOJI_TAG + elif source_type == "repo": + source_type = PungiSourceType.REPO else: err = "Unknown source_type %s" % source_type log.error(err) @@ -123,9 +125,14 @@ class ODCSAPI(MethodView): seconds_to_live = max(int(seconds_to_live), conf.max_seconds_to_live) + packages = None + if "packages" in data: + packages = data["packages"] + compose = Compose.create( db.session, owner, source_type, source, - COMPOSE_RESULTS["repository"], seconds_to_live) + COMPOSE_RESULTS["repository"], seconds_to_live, + ' '.join(packages)) db.session.add(compose) db.session.commit() diff --git a/submit_test_compose b/submit_test_compose index 4be5600..bdd1847 100755 --- a/submit_test_compose +++ b/submit_test_compose @@ -1,3 +1,9 @@ #!/bin/bash +if [ $# -gt 2 ] +then + curl --data "{\"source_type\": \"$1\", \"source\": \"$2\", \"packages\": [\"$3\"]}" localhost:5005/odcs/1/composes/ + exit 0 +fi + curl --data "{\"source_type\": \"$1\", \"source\": \"$2\"}" localhost:5005/odcs/1/composes/ diff --git a/tests/repo/ed-1.14.1-2.module_fd8ca23e.src.rpm b/tests/repo/ed-1.14.1-2.module_fd8ca23e.src.rpm new file mode 100644 index 0000000..c15ee6c Binary files /dev/null and b/tests/repo/ed-1.14.1-2.module_fd8ca23e.src.rpm differ diff --git a/tests/repo/ed-1.14.1-2.module_fd8ca23e.x86_64.rpm b/tests/repo/ed-1.14.1-2.module_fd8ca23e.x86_64.rpm new file mode 100644 index 0000000..53563fd Binary files /dev/null and b/tests/repo/ed-1.14.1-2.module_fd8ca23e.x86_64.rpm differ diff --git a/tests/repo/ed-debuginfo-1.14.1-2.module_fd8ca23e.x86_64.rpm b/tests/repo/ed-debuginfo-1.14.1-2.module_fd8ca23e.x86_64.rpm new file mode 100644 index 0000000..cfbc3b3 Binary files /dev/null and b/tests/repo/ed-debuginfo-1.14.1-2.module_fd8ca23e.x86_64.rpm differ diff --git a/tests/repo/module-build-macros-0.1-1.module_fd8ca23e.noarch.rpm b/tests/repo/module-build-macros-0.1-1.module_fd8ca23e.noarch.rpm new file mode 100644 index 0000000..e24f48c Binary files /dev/null and b/tests/repo/module-build-macros-0.1-1.module_fd8ca23e.noarch.rpm differ diff --git a/tests/repo/module-build-macros-0.1-1.module_fd8ca23e.src.rpm b/tests/repo/module-build-macros-0.1-1.module_fd8ca23e.src.rpm new file mode 100644 index 0000000..3fd23d1 Binary files /dev/null and b/tests/repo/module-build-macros-0.1-1.module_fd8ca23e.src.rpm differ diff --git a/tests/repo/repodata/028b753de97c22923bffe2159dcb478cdfa919376e21a68a1aa6ca40fba06f45-primary.sqlite.bz2 b/tests/repo/repodata/028b753de97c22923bffe2159dcb478cdfa919376e21a68a1aa6ca40fba06f45-primary.sqlite.bz2 new file mode 100644 index 0000000..5f81a34 Binary files /dev/null and b/tests/repo/repodata/028b753de97c22923bffe2159dcb478cdfa919376e21a68a1aa6ca40fba06f45-primary.sqlite.bz2 differ diff --git a/tests/repo/repodata/1879db5a1658329be62225ebf1b40cbf8e5d522abd2fdd86157d565c83e1ddc2-modules.yaml.gz b/tests/repo/repodata/1879db5a1658329be62225ebf1b40cbf8e5d522abd2fdd86157d565c83e1ddc2-modules.yaml.gz new file mode 100644 index 0000000..acbc37a Binary files /dev/null and b/tests/repo/repodata/1879db5a1658329be62225ebf1b40cbf8e5d522abd2fdd86157d565c83e1ddc2-modules.yaml.gz differ diff --git a/tests/repo/repodata/2c3a353ba9e47c3e9c8be6c2e25c50fcc11e184a727c645f3f08b66aa1a1654d-other.xml.gz b/tests/repo/repodata/2c3a353ba9e47c3e9c8be6c2e25c50fcc11e184a727c645f3f08b66aa1a1654d-other.xml.gz new file mode 100644 index 0000000..e7f96d4 Binary files /dev/null and b/tests/repo/repodata/2c3a353ba9e47c3e9c8be6c2e25c50fcc11e184a727c645f3f08b66aa1a1654d-other.xml.gz differ diff --git a/tests/repo/repodata/635015f39babdbcdcca3a516576bdcafb57fb77be3b7919cf85363ec558fb8fe-other.sqlite.bz2 b/tests/repo/repodata/635015f39babdbcdcca3a516576bdcafb57fb77be3b7919cf85363ec558fb8fe-other.sqlite.bz2 new file mode 100644 index 0000000..e9f65a4 Binary files /dev/null and b/tests/repo/repodata/635015f39babdbcdcca3a516576bdcafb57fb77be3b7919cf85363ec558fb8fe-other.sqlite.bz2 differ diff --git a/tests/repo/repodata/77e6eb5a69d53a9d56f32b3154832d4eff6e2de518c268102d2c6848cffa3e1e-filelists.xml.gz b/tests/repo/repodata/77e6eb5a69d53a9d56f32b3154832d4eff6e2de518c268102d2c6848cffa3e1e-filelists.xml.gz new file mode 100644 index 0000000..2c2930d Binary files /dev/null and b/tests/repo/repodata/77e6eb5a69d53a9d56f32b3154832d4eff6e2de518c268102d2c6848cffa3e1e-filelists.xml.gz differ diff --git a/tests/repo/repodata/92ec6ea92c2b01e232c69539450c2f6ae0e4b31ccbad0f3a9678a4fc126b9fc0-filelists.sqlite.bz2 b/tests/repo/repodata/92ec6ea92c2b01e232c69539450c2f6ae0e4b31ccbad0f3a9678a4fc126b9fc0-filelists.sqlite.bz2 new file mode 100644 index 0000000..d55ef74 Binary files /dev/null and b/tests/repo/repodata/92ec6ea92c2b01e232c69539450c2f6ae0e4b31ccbad0f3a9678a4fc126b9fc0-filelists.sqlite.bz2 differ diff --git a/tests/repo/repodata/ca15f74c9060791c17a93315bc8d2df4fd6a5d3e7c8eace70d82d023562a669e-primary.xml.gz b/tests/repo/repodata/ca15f74c9060791c17a93315bc8d2df4fd6a5d3e7c8eace70d82d023562a669e-primary.xml.gz new file mode 100644 index 0000000..f6eccc8 Binary files /dev/null and b/tests/repo/repodata/ca15f74c9060791c17a93315bc8d2df4fd6a5d3e7c8eace70d82d023562a669e-primary.xml.gz differ diff --git a/tests/repo/repodata/repomd.xml b/tests/repo/repodata/repomd.xml new file mode 100644 index 0000000..23f3b4d --- /dev/null +++ b/tests/repo/repodata/repomd.xml @@ -0,0 +1,63 @@ + + + 1496834159 + + ca15f74c9060791c17a93315bc8d2df4fd6a5d3e7c8eace70d82d023562a669e + eedbc9af8362a1dc40bfeb4e3f2a44506e9695c72ce7296ba6081148c4555eec + + 1496834159 + 1884 + 14292 + + + 77e6eb5a69d53a9d56f32b3154832d4eff6e2de518c268102d2c6848cffa3e1e + 1a82b840da85128850450a8be42a1614bff1877f1b7247ceefdff9317bbb25c3 + + 1496834159 + 827 + 5250 + + + 2c3a353ba9e47c3e9c8be6c2e25c50fcc11e184a727c645f3f08b66aa1a1654d + afdb78f98c7bc17b2ca268f3e880770df504dce40a7fd6cb1ff001dd8e4a2d09 + + 1496834159 + 1015 + 12204 + + + 028b753de97c22923bffe2159dcb478cdfa919376e21a68a1aa6ca40fba06f45 + 799edcb8eaec4bd2983f3d17803234d31de88a38dfe6f579b31349c75dd35a41 + + 1496834159 + 4133 + 114688 + 10 + + + 92ec6ea92c2b01e232c69539450c2f6ae0e4b31ccbad0f3a9678a4fc126b9fc0 + 650c538311e917bf1226490ecf4c8e54f5483a6461d6921b5cb404df33b525a4 + + 1496834159 + 2263 + 28672 + 10 + + + 635015f39babdbcdcca3a516576bdcafb57fb77be3b7919cf85363ec558fb8fe + 4b543633422f88fdd89825f8d7f872f5b8752ab0f829fc37f9e0951d79b6c6d2 + + 1496834159 + 2722 + 32768 + 10 + + + 1879db5a1658329be62225ebf1b40cbf8e5d522abd2fdd86157d565c83e1ddc2 + 958657781cf1b2a9bd841580e035912b0c86ade90728b825cf5abd049a3d8b58 + + 1496834160 + 750 + 1529 + +