From e32ffa57ad2ab9a6e9586b7e7f931d3174846828 Mon Sep 17 00:00:00 2001 From: Owen W. Taylor Date: Aug 03 2018 04:01:55 +0000 Subject: Initial import --- diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..0e2e870 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length=100 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6080949 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +/*.egg-info/ +__pycache__ +codehilite.css +README.html +out/* +dist diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..01b8850 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,43 @@ +FROM fedora:28 + +RUN dnf -y update +RUN dnf -y install \ + httpd \ + mod_ssl \ + openssl \ + python3 \ + python3-click \ + python3-fedmsg \ + python3-pip \ + python3-requests \ + python3-yaml \ + python3-six \ + python3-www-authenticate && \ + dnf clean all + +ADD services/index/config.yaml /etc/regindexer/config.yaml +RUN mkdir -p -m 0755 /var/lib/regindexer/index + +ADD services/index/trigger-reindex.sh /usr/local/bin + +# Disable standard endpoints +RUN rm /etc/fedmsg.d/endpoints.py +# Disable checking for signed messages +RUN sed -i \ + 's@validate_signatures=True@validate_signatures=False@' \ + /etc/fedmsg.d/ssl.py + +# Use a central relay +RUN sed -i \ + 's@tcp://127.0.0.1:4001@tcp://fedmsg-relay:9940@; s@tcp://127.0.0.1:2003@tcp://fedmsg-relay:2003@' \ + /etc/fedmsg.d/relay.py + +ADD . /tmp/regindexer +RUN cd /tmp/regindexer && pip3 install --no-deps --upgrade . + +ADD fedmsg.d/regindexer.py /etc/fedmsg.d +RUN sed -i \ + "s@'regindexer.consumer.enabled': False@'regindexer.consumer.enabled': True@" \ + /etc/fedmsg.d/regindexer.py + +CMD ["/usr/bin/fedmsg-hub-3"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1138ddf --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2016 Red Hat, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..e951d12 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +include config-example.yaml +include fedmsg.d/regindexer.py +include LICENSE diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4d44f90 --- /dev/null +++ b/Makefile @@ -0,0 +1,30 @@ +all: + +reset-data: + docker-compose down || true + docker volume rm regindexer_registry || true + +trust-local: + docker-compose exec frontend cat /etc/pki/tls/certs/regindexer_ca.crt > regindexer.crt + sudo sh -c 'cp regindexer.crt /etc/pki/ca-trust/source/anchors/ && update-ca-trust' + sudo sh -c 'grep -l registry.local.fishsoup.net /etc/hosts > /dev/null || echo "127.0.0.1 registry.local.fishsoup.net" >> /etc/hosts' + rm -f regindexer.crt + +untrust-local: + sudo sh -c 'rm /etc/pki/ca-trust/source/anchors/regindexer.crt && update-ca-trust' + sudo sh -c 'sed -i /registry.local.fishsoup.net/d /etc/hosts' + +trigger-reindex: + docker-compose exec index trigger-reindex.sh + +README.html: README.md codehilite.css Makefile + ( echo '' && \ + markdown_py-3 -x codehilite -x partial_gfm -o html5 $< && \ + echo '' \ + ) > $@.tmp && \ + mv $@.tmp $@ || rm -f $@.tmp + +codehilite.css: + pygmentize -S default -f html > codehilite.css + +.PHONY: reset-local trust-local untrust-local diff --git a/README.md b/README.md new file mode 100644 index 0000000..8b82c14 --- /dev/null +++ b/README.md @@ -0,0 +1,117 @@ +regindexer +========== +regindexer is a tool for creating an index of a container registry. It can be run manually from the command line, or can run as a fedmsg consumer, rebuilding the index when it sees messages from Bodhi. + +regindexer creates a static index or indexes and always scans the entire registry. [Flagstate](https://github.com/owtaylor/flagstate) is a more advanced (and also trickier to deploy) version of the registry-indexing concept - it maintains a incrementally updated database of the registry metadata and supports flexible queries against the database. regindexer and Flagstate share the same output JSON format. See https://github.com/owtaylor/flagstate/blob/master/docs/protocol.md#responses for a description of the format. + +One thing that regindexer supports that Flagstate doesn't currently is icon extraction: if `extract_icons` is true in the config file, then when a `org.freedesktop.appstream.icon-64` or +`org.freedesktop.appstream.icon-128` annotations is found and the contents point to a data URI, then the icon is stored in the configured icon directory (with a content-addressed path), and the data: URI converted to a URI pointing to the icon. + +Installation +============ +To install: + +``` sh +python3 setup.py install +``` + +this installs the code, and the `regindexer` script. + +Usage +===== + +The indexer is run manually as: + +``` +regindexer [-c/--config=CONFIG_FILE] [-v/--verbose] index +``` + +The config file has a default location of `/etc/regindexer/config.yaml` and has the following structure: + +``` yaml +# Directory to extract icons into, resolved with the index as a base +icons_dir: /var/lib/regindexer/icons +# Public URI relative to the index file URIs for the icon directory +icons_uri: /icons/ +indexes: + flatpak_amd64: + # Local location where to write the index JSON + output: /var/lib/regindexer/flatpak-amd64.json + # Registry to index + registry: https://1.2.3.4:5000 + # Public URI to the registry, to be included in the output index, + # resolved with the index as a base + registry_public: https://registry.example.com:5000 + # Tags of images to index (* and ? are supported for globbing) + tags: ['latest', 'latest-*'] + # Only index images with the specified annotations + required_annotations: ['org.flatpak.body'] + # Only index images for the specified architectures + architectures: ['amd64'] + # Whether to extract icons into the icons directory + output: /home/otaylor/tmp/flatpak.json + all: + output: /var/lib/regindexer/all.json + registry: https://1.2.3.4:5000 + registry_public: https://registry.example.com:5000 + tags: ['latest'] +``` + +fedmsg +====== + +To enable the fedmsg listener, you'll need to create `/etc/fedmsg.d/regindexer.py` with contents like: + +``` python +config = { + 'regindexer.consumer.enabled': True, + 'regindexer.config_file': '/etc/regindexer/config.yaml', + 'logging': { + 'loggers': { + 'regindexer': { + "level": "INFO", + "propagate": False, + "handlers": ["console"], + }, + 'regindexer.consumer': { + "level": "INFO", + "propagate": False, + "handlers": ["console"], + } + } + } +} +``` + +and then start the fedmsg-hub service on your machine. The consumer will then listen for messages `org.fedoraproject..bodhi.mashtask.complete` where `` comes from the global fedmsg `environment` config key (see `/etc/fedmsg.d/base.py`), with a content type of `container` or `flatpak`, and rebuild the index. The configured output locations must be writable by the fedmsg user or group. + +Development environment +======================= + +The distribution contains a docker-compose environment that sets up: + + * a registry with test data + * a local fedmsg bus + * a regindexer index triggerered off of fedmsg + * a HTTP frontend that exports the registry and index in a combined web heirarchy + +To start it up and load the teset data, do: + +``` +docker-compose build && docker compose up +``` + +The index and registry will be available at http://localhost:7080 and https://localhost:7443 ; to test access via https with valid certificates, `make trust-local` will adjust `/etc/pki/ca-trust/source/anchors/` and `/etc/hosts` so that https://registry.local.fishsoup.net:7443 works. You can find a generated index at: https://https://registry.local.fishsoup.net:7443/index/all.json + +To trigger a reindex via fedmsg, use `make trigger-reindex`. + +You can also run regindexer from a Python virtualenv, either against a remote registry, or against the registry from the docker-compose setup. + +``` +$ virtualenv-3 ~/.virtualenvs/regindexer +$ . ~/.virtualenvs/regindexer/bin/active +$ pip install -e . + +# Read from docker-compose registry and writes output into out/ +$ regindexer -v -c config-devel.yaml index +``` diff --git a/config-devel.yaml b/config-devel.yaml new file mode 100644 index 0000000..b3e9093 --- /dev/null +++ b/config-devel.yaml @@ -0,0 +1,25 @@ +icons_dir: out/icons/ +icons_uri: /app-icons/ +indexes: + flatpak: + output: out/flatpak.json + registry: http://localhost:7000 + tags: ['latest', 'latest-*'] + required_annotations: ['org.flatpak.body'] + extract_icons: True + flatpak_amd64: + output: out/flatpak-amd64.json + registry: http://localhost:7000 + tags: ['latest', 'latest-*'] + required_annotations: ['org.flatpak.body'] + architectures: ['amd64'] + extract_icons: True + all: + output: out/all.json + registry: http://localhost:7000 + tags: ['latest'] + all_amd64: + output: out/all-amd64.json + registry: http://localhost:7000 + tags: ['latest'] + architectures: ['amd64'] diff --git a/config-example.yaml b/config-example.yaml new file mode 100644 index 0000000..2da5f06 --- /dev/null +++ b/config-example.yaml @@ -0,0 +1,25 @@ +icons_dir: /var/lib/regindexer/icons +icons_uri: /app-icons/ +indexes: +# flatpak: +# output: /var/lib/regindexer/flatpak.json +# registry: https://registry.fedoraproject.org +# tags: ['latest', 'latest-*'] +# required_annotations: ['org.flatpak.body'] +# extract_icons: True +# flatpak_amd64: +# output: /var/lib/regindexer/flatpak-amd64.json +# registry: https://registry.fedoraproject.org +# tags: ['latest', 'latest-*'] +# required_annotations: ['org.flatpak.body'] +# extract_icons: True +# architectures: ['amd64'] +# all: +# output: /var/lib/regindexer/all.json +# registry: https://registry.fedoraproject.org +# tags: ['latest'] +# all_amd64: +# output: /var/lib/regindexer/all.json +# registry: https://registry.fedoraproject.org +# tags: ['latest'] +# architectures: ['amd64'] diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..06794b2 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,36 @@ +version: '2.1' + +volumes: + index: + registry: + tls: + +services: + registry: + build: services/registry + ports: + - 7000:5000 + volumes: + - registry:/var/lib/registry:z + - ./services/registry/config.yml:/etc/docker/registry/config.yml:z + + frontend: + build: services/frontend + ports: + - 7080:8080 + - 7443:443 + volumes: + - index:/var/lib/regindexer/index + - ${TEST_DATA:-./test-data}:/mnt/test-data:z + - tls:/etc/pki/tls:z + + fedmsg-relay: + build: services/fedmsg-relay + ports: + - 7003:2003 + - 7940:9940 + + index: + build: . + volumes: + - index:/var/lib/regindexer/index diff --git a/fedmsg.d/regindexer.py b/fedmsg.d/regindexer.py new file mode 100644 index 0000000..8a547f3 --- /dev/null +++ b/fedmsg.d/regindexer.py @@ -0,0 +1,18 @@ +config = { + 'regindexer.consumer.enabled': False, + 'regindexer.config_file': '/etc/regindexer/config.yaml', + 'logging': { + 'loggers': { + 'regindexer': { + "level": "INFO", + "propagate": False, + "handlers": ["console"], + }, + 'regindexer.consumer': { + "level": "INFO", + "propagate": False, + "handlers": ["console"], + } + } + } +} diff --git a/out/.dummy b/out/.dummy new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/out/.dummy diff --git a/regindexer/cli.py b/regindexer/cli.py new file mode 100644 index 0000000..ffe2317 --- /dev/null +++ b/regindexer/cli.py @@ -0,0 +1,30 @@ +import click +import logging + +from regindexer.indexer import Indexer + + +@click.group() +@click.option('-c', '--config', metavar='CONFIG_FILE', + help='Path to config file', + default='/etc/regindexer/config.yaml') +@click.option('-v', '--verbose', is_flag=True, + help='Show verbose debugging output') +@click.pass_context +def cli(ctx, config, verbose): + ctx.obj = { + 'config': config + } + + if verbose: + logging.basicConfig(level=logging.INFO) + else: + logging.basicConfig(level=logging.WARNING) + + +@cli.command(name="index") +@click.pass_context +def index(ctx): + """Rebuild indexes""" + indexer = Indexer(config=ctx.obj['config']) + indexer.index() diff --git a/regindexer/config.py b/regindexer/config.py new file mode 100644 index 0000000..1b4165a --- /dev/null +++ b/regindexer/config.py @@ -0,0 +1,33 @@ +import yaml + +from regindexer.tag_pattern import TagPattern + + +class IndexConfig(object): + def __init__(self, name, attrs): + self.name = name + self.output = attrs['output'] + self.registry = attrs['registry'] + self.registry_public = attrs.get('registry_public', self.registry) + self.tags = [TagPattern(x) for x in attrs['tags']] + self.required_annotations = attrs.get('required_annotations', []) + self.architectures = attrs.get('architectures', None) + self.extract_icons = attrs.get('extract_icons', False) + + def __repr__(self): + return 'IndexConfig(%r)' % self.__dict__ + + +class Config(object): + def __init__(self, path): + self.indexes = [] + with open(path, 'r') as f: + yml = yaml.safe_load(f) + self.icons_dir = yml.get('icons_dir', None) + self.icons_uri = yml.get('icons_uri', None) + if not self.icons_uri.endswith('/'): + self.icons_uri = self.icons_uri + '/' + indexes = yml.get('indexes') + if indexes: + for name, attrs in indexes.items(): + self.indexes.append(IndexConfig(name, attrs)) diff --git a/regindexer/consumer.py b/regindexer/consumer.py new file mode 100644 index 0000000..dfc7213 --- /dev/null +++ b/regindexer/consumer.py @@ -0,0 +1,50 @@ +import logging + +from fedmsg.consumers import FedmsgConsumer +import moksha.hub.reactor +import six.moves.queue as queue + +from regindexer.indexer import Indexer + +log = logging.getLogger('regindexer.consumer') + + +class Consumer(FedmsgConsumer): + config_key = 'regindexer.consumer.enabled' + + def __init__(self, hub): + super(Consumer, self).__init__(hub) + + prefix = hub.config.get('topic_prefix') + env = hub.config.get('environment') + self.topic = [ + prefix + '.' + env + '.bodhi.mashtask.complete', + ] + + # Need to do this explicitly to avoid triggering the Initialization task + if not hub.config.get(self.config_key, False): + return + + self.index_queue = queue.Queue() + + config_file = hub.config.get('regindexer.config_file', + '/etc/regindexer/config.yaml') + self.indexer = Indexer(config_file) + + moksha.hub.reactor.reactor.callInThread(self._do_index_work) + self.index_queue.put('Initialization') + + def consume(self, raw_msg): + topic = raw_msg['topic'] + msg = raw_msg['body']['msg'] + + if msg.get('ctype') in ('container', 'flatpak'): + log.info("Got %s message for ctype=%s", topic, msg['ctype']) + # trigger rebuilding the index + self.index_queue.put('Bodhi mash, ctype="%s"' % msg['ctype']) + + def _do_index_work(self): + while True: + reason = self.index_queue.get() + log.info("Rebuilding index for: %s", reason) + self.indexer.index() diff --git a/regindexer/indexer.py b/regindexer/indexer.py new file mode 100644 index 0000000..2f464c5 --- /dev/null +++ b/regindexer/indexer.py @@ -0,0 +1,328 @@ +from regindexer.config import Config + +import base64 +import hashlib +import logging +import json +import requests +import os +import sys +from urllib.parse import urljoin + +log = logging.getLogger('regindexer') + + +MEDIA_TYPE_MANIFEST_V2 = 'application/vnd.docker.distribution.manifest.v2+json' +MEDIA_TYPE_LIST_V2 = 'application/vnd.docker.distribution.manifest.list.v2+json' +MEDIA_TYPE_OCI = 'application/vnd.oci.image.manifest.v1+json' +MEDIA_TYPE_OCI_INDEX = 'application/vnd.oci.image.index.v1+json' + +MANIFEST_HEADERS = { + 'Accept': ', '.join(( + MEDIA_TYPE_MANIFEST_V2, + MEDIA_TYPE_LIST_V2, + MEDIA_TYPE_OCI, + MEDIA_TYPE_OCI_INDEX, + )) +} + +DATA_URI_PREFIX = 'data:image/png;base64,' + + +class DownloadInfo(object): + def __init__(self, digest, content_type, json): + self.digest = digest + self.content_type = content_type + self.json = json + + def __repr__(self): + return 'DownloadInfo(%r)' % self.__dict__ + + +class RegistryQuery(object): + def __init__(self, registry, verbose=False): + self.registry = registry + self.verbose = verbose + self.session = requests.Session() + + def _get(self, relative_url, headers=None): + return self.session.get(self.registry + relative_url, + headers=headers) + + def get_manifest(self, name, ref): + log.info("Querying %s:%s", name, ref) + + url = '/v2/{}/manifests/{}'.format(name, ref) + response = self._get(url, headers=MANIFEST_HEADERS) + if response.status_code != 200: + log.warning("Could not download %s:%s", name, ref) + return None + + return DownloadInfo(response.headers['Docker-Content-Digest'], + response.headers['Content-Type'], + response.json()) + + def get_blob(self, name, digest): + url = '/v2/{}/blobs/{}'.format(name, digest) + response = self._get(url) + if response.status_code != 200: + return None + + return DownloadInfo(response.headers['Docker-Content-Digest'], + response.headers['Content-Type'], + response.json()) + + def iterate_images(self, tag_patterns): + repo_names = self._get('/v2/_catalog').json()['repositories'] + + for name in repo_names: + url = '/v2/{}/tags/list'.format(name) + tags = self._get(url).json()['tags'] + matches = set() + + for tag in tags: + for pat in tag_patterns: + if pat.matches(tag): + matches.add(tag) + + manifests = {} + tags = {} + for tag in matches: + manifest_info = self.get_manifest(name, tag) + if not manifest_info: + continue + + manifests[manifest_info.digest] = manifest_info + tags.setdefault(manifest_info.digest, []).append(tag) + + for digest in manifests: + yield name, manifests[digest], tags[digest] + + +class IconStore(object): + def __init__(self, icons_dir, icons_uri): + self.icons_dir = icons_dir + self.icons_uri = icons_uri + + if not os.path.exists(icons_dir): + # Create only one level + os.mkdir(icons_dir) + + self.old_icons = {} + for subdir in os.listdir(icons_dir): + for filename in os.listdir(os.path.join(icons_dir, subdir)): + self.old_icons[(subdir, filename)] = True + + self.icons = {} + + def store(self, uri): + if not uri.startswith(DATA_URI_PREFIX): + return None + + decoded = base64.b64decode(uri[len(DATA_URI_PREFIX):]) + + h = hashlib.sha256() + h.update(decoded) + digest = h.hexdigest() + subdir = digest[:2] + filename = digest[2:] + '.png' + + key = (subdir, filename) + if key in self.icons: + pass + elif key in self.old_icons: + self.icons[key] = True + else: + if not os.path.exists(os.path.join(self.icons_dir, subdir)): + os.mkdir(os.path.join(self.icons_dir, subdir)) + fullpath = os.path.join(self.icons_dir, subdir, filename) + log.info("Storing icon: %s", fullpath) + with open(os.path.join(self.icons_dir, subdir, filename), 'wb') as f: + f.write(decoded) + self.icons[key] = True + + return urljoin(self.icons_uri, subdir + '/' + filename) + + def clean(self): + for key in self.old_icons: + if key not in self.icons: + subdir, filename = key + fullpath = os.path.join(self.icons_dir, subdir, filename) + os.unlink(fullpath) + log.info("Removing icon: %s", fullpath) + + +class Index(object): + def __init__(self, conf, icon_store=None): + if conf.extract_icons and icon_store is None: + raise RuntimeError("extract_icons is set, but no icons_dir is configured") + + self.config = conf + self.icon_store = icon_store + self.repos = {} + + def extract_icon(self, annotations, key): + if not self.config.extract_icons: + return + + value = annotations.get(key) + if value is None: + return + + uri = self.icon_store.store(value) + if uri is not None: + annotations[key] = uri + + def make_image(self, query, name, manifest_info, tags=None): + config_info = query.get_blob(name, manifest_info.json['config']['digest']) + if not config_info: + log.warning("Failed to download config json") + return None + + arch = config_info.json['architecture'] + os = config_info.json['os'] + + if self.config.architectures: + if arch not in self.config.architectures: + return None + + if manifest_info.content_type == MEDIA_TYPE_OCI: + annotations = manifest_info.json['annotations'] + else: + annotations = {} + + for annotation in self.config.required_annotations: + if annotation not in annotations: + return None + + self.extract_icon(annotations, 'org.freedesktop.appstream.icon-64') + self.extract_icon(annotations, 'org.freedesktop.appstream.icon-128') + + image = { + 'Digest': manifest_info.digest, + 'MediaType': manifest_info.content_type, + 'OS': os, + 'Architecture': arch, + 'Labels': {}, + } + + image['Annotations'] = annotations + + labels = config_info.json['config'].get('Labels') + if labels is None: + labels = {} + image['Labels'] = labels + + if tags: + image['Tags'] = tags + + return image + + def make_list(self, query, name, list_info, tags=None): + images = [] + for manifest in list_info.json['manifests']: + manifest_info = query.get_manifest(name, manifest['digest']) + + if not manifest_info: + log.warning("Failed to download manifest {}".format(manifest['digest']), + file=sys.stderr) + continue + + if manifest_info.content_type in (MEDIA_TYPE_OCI, MEDIA_TYPE_MANIFEST_V2): + image = self.make_image(query, name, manifest_info) + if image: + images.append(image) + + if len(images) == 0: + return None + + image_list = { + 'Digest': list_info.digest, + 'MediaType': list_info.content_type, + 'Images': images, + } + + if tags: + image_list['Tags'] = tags + + return image_list + + def add_image_or_list(self, query, name, info, tags): + if name not in self.repos: + self.repos[name] = { + "Name": name, + "Images": [], + "Lists": [], + } + + repo = self.repos[name] + + if info.content_type in (MEDIA_TYPE_OCI, MEDIA_TYPE_MANIFEST_V2): + image = self.make_image(query, name, info, tags=tags) + if image: + repo["Images"].append(image) + elif info.content_type in (MEDIA_TYPE_OCI_INDEX, MEDIA_TYPE_LIST_V2): + image_list = self.make_list(query, name, info, tags=tags) + if image_list: + repo["Lists"].append(image_list) + else: + log.info("{}/{}: not an OCI image or image index".format(name, tags)) + + def write(self): + with open(self.config.output, 'w') as f: + filtered_repos = (v for v in self.repos.values() if v['Images'] or v['Lists']) + sorted_repos = sorted(filtered_repos, key=lambda r: r['Name']) + for repo in sorted_repos: + repo["Images"].sort(key=lambda x: x["Tags"]) + repo["Lists"].sort(key=lambda x: x["Tags"]) + + json.dump({ + 'Registry': self.config.registry_public, + 'Results': sorted_repos, + }, f, sort_keys=True, indent=4, ensure_ascii=False) + log.info("Wrote %s", self.config.output) + + +class Indexer(object): + def __init__(self, config): + self.conf = Config(config) + + def index(self): + if not self.conf.indexes: + log.warning("No indexes configured") + return + + icon_store = None + if self.conf.icons_dir is not None: + if self.conf.icons_uri is None: + raise RuntimeError("icons_dir is configured, but not icons_uri") + + icon_store = IconStore(self.conf.icons_dir, self.conf.icons_uri) + + indexes_by_registry = {} + for index_config in self.conf.indexes: + indexes = indexes_by_registry.setdefault(index_config.registry, []) + indexes.append(Index(index_config, icon_store=icon_store)) + + for registry, indexes in indexes_by_registry.items(): + query = RegistryQuery(registry) + + tag_patterns = set() + for index in indexes: + tag_patterns.update(index.config.tags) + + for repo, image, tags in query.iterate_images(tag_patterns): + for index in indexes: + matches_tag = False + for tag in tags: + for tag_pattern in index.config.tags: + if tag_pattern.matches(tag): + matches_tag = True + if matches_tag: + index.add_image_or_list(query, repo, image, tags) + + for index in indexes: + index.write() + + if icon_store is not None: + icon_store.clean() diff --git a/regindexer/tag_pattern.py b/regindexer/tag_pattern.py new file mode 100644 index 0000000..6942351 --- /dev/null +++ b/regindexer/tag_pattern.py @@ -0,0 +1,46 @@ +import re + + +def to_re(m): + if m.group(0) == '*': + return '.*' + elif m.group(0) == '?': + return '.' + else: + return re.escape(m.group(0)) + + +class TagPattern(object): + def __init__(self, raw): + self.raw = raw + + as_re = re.sub(r'\*|\?|[^*?]*', to_re, raw) + '$' + self.compiled = re.compile(as_re) + + def __eq__(self, other): + return self.raw == other.raw + + def __hash__(self): + return hash(self.raw) + + def matches(self, tag): + return self.compiled.match(tag) is not None + + def __repr__(self): + return 'TagPattern({!r})'.format(self.raw) + + +if __name__ == '__main__': + tp = TagPattern('latest') + assert not tp.matches('late') + assert tp.matches('latest') + tp = TagPattern('?atest') + assert tp.matches('latest') + assert not tp.matches('llatest') + tp = TagPattern('?ate*') + assert tp.matches('latest') + assert tp.matches('latester') + assert not tp.matches('') + + s = {TagPattern(x) for x in ['latest', 'lat*', 'latest']} + assert len(s) == 2 diff --git a/services/fedmsg-relay/Dockerfile b/services/fedmsg-relay/Dockerfile new file mode 100644 index 0000000..6834038 --- /dev/null +++ b/services/fedmsg-relay/Dockerfile @@ -0,0 +1,24 @@ +FROM fedora:28 + +RUN dnf -y update +RUN dnf -y install \ + python3-fedmsg \ + && \ + dnf clean all + +# Disable standard endpoints +RUN rm /etc/fedmsg.d/endpoints.py +# Disable checking for signed messages +RUN sed -i \ + 's@validate_signatures=True@validate_signatures=False@' \ + /etc/fedmsg.d/ssl.py + +# Listen non-locally, so that we can relay messages for other hosts in the docker-compose cluster +RUN sed -i \ + 's@tcp://127.0.0.1:4001@tcp://0.0.0.0:9940@; s@tcp://127.0.0.1:2003@tcp://0.0.0.0:2003@' \ + /etc/fedmsg.d/relay.py + +EXPOSE 2003 +EXPOSE 9940 + +CMD ["/usr/bin/fedmsg-relay-3"] diff --git a/services/frontend/Dockerfile b/services/frontend/Dockerfile new file mode 100644 index 0000000..4b78ccb --- /dev/null +++ b/services/frontend/Dockerfile @@ -0,0 +1,31 @@ +FROM fedora:28 + +RUN dnf -y update +RUN dnf -y install \ + httpd \ + mod_ssl \ + openssl \ + python3 \ + python3-pip \ + python3-requests \ + python3-yaml \ + python3-six \ + python3-www-authenticate && \ + dnf clean all + +ADD \ + check-for-data.py \ + create-test-data.sh \ + entrypoint.sh \ + generate-cert.sh \ + registry_copy.py \ + /usr/local/bin/ + +ADD frontend.conf /etc/httpd/conf.d +RUN sed -i s/^SSLCertificate/#^SSLCertificate/ /etc/httpd/conf.d/ssl.conf + +VOLUME /etc/pki/tls + +EXPOSE 80 +EXPOSE 443 +CMD ["/usr/local/bin/entrypoint.sh"] diff --git a/services/frontend/check-for-data.py b/services/frontend/check-for-data.py new file mode 100755 index 0000000..f1fcdce --- /dev/null +++ b/services/frontend/check-for-data.py @@ -0,0 +1,10 @@ +#!/usr/bin/python3 + +import requests +import sys + +repositories = requests.get(sys.argv[1] + '/v2/_catalog').json()['repositories'] +if len(repositories) > 0: + sys.exit(0) +else: + sys.exit(1) diff --git a/services/frontend/config.yaml b/services/frontend/config.yaml new file mode 100644 index 0000000..39fd351 --- /dev/null +++ b/services/frontend/config.yaml @@ -0,0 +1,29 @@ +icons_dir: /var/lib/regindexer/icons/ +icons_uri: /app-icons/ +indexes: + flatpak: + output: /var/lib/regindexer/index/flatpak.json + registry: http://registry:5000 + registry_public: https://registry.local.fishsoup.net:7443 + tags: ['latest', 'latest-*'] + required_annotations: ['org.flatpak.body'] + extract_icons: True + flatpak_amd64: + output: /var/lib/regindexer/index/flatpak-amd64.json + registry: http://registry:5000 + registry_public: https://registry.local.fishsoup.net:7443 + tags: ['latest', 'latest-*'] + required_annotations: ['org.flatpak.body'] + architectures: ['amd64'] + extract_icons: True + all: + output: /var/lib/regindexer/index/all.json + registry: http://registry:5000 + registry_public: https://registry.local.fishsoup.net:7443 + tags: ['latest', 'latest-*'] + all_amd64: + output: /var/lib/regindexer/index/all-amd64.json + registry: http://registry:5000 + registry_public: https://registry.local.fishsoup.net:7443 + tags: ['latest'] + architectures: ['amd64'] diff --git a/services/frontend/create-test-data.sh b/services/frontend/create-test-data.sh new file mode 100755 index 0000000..c6c2389 --- /dev/null +++ b/services/frontend/create-test-data.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e -x + +cd /mnt/test-data +while read src arch dest ; do + arch_arg= + if [ "$arch" != "*" ] ; then + arch_arg="--arch=$arch" + fi + registry_copy.py --dest-tls-verify=false $src $arch_arg docker:registry:5000/$dest +done < contents diff --git a/services/frontend/entrypoint.sh b/services/frontend/entrypoint.sh new file mode 100755 index 0000000..c0b95e7 --- /dev/null +++ b/services/frontend/entrypoint.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +if ! [ -e /etc/pki/tls/certs/regindexer.crt ] ; then + generate-cert.sh +fi + +echo -n "Waiting for registry ... " +while ! curl -s http://registry:5000/v2 > /dev/null ; do + sleep 1 +done +echo "started" + +if ! check-for-data.py http://registry:5000 ; then + create-test-data.sh +fi + +exec httpd -DNO_DETACH -DFOREGROUND diff --git a/services/frontend/frontend.conf b/services/frontend/frontend.conf new file mode 100644 index 0000000..6973161 --- /dev/null +++ b/services/frontend/frontend.conf @@ -0,0 +1,49 @@ +ServerName registry.local.fishsoup.net + +ErrorLog /dev/stderr +TransferLog /dev/stdout + +ProxyPass "/v2" "http://registry:5000/v2" +ProxyPassReverse "/v2" "http://registry:5000/v2" + +Alias "/index/" "/var/lib/regindexer/index/" +Alias "/app-icons/" "/var/lib/regindexer/icons/" + + + Options +FollowSymLinks + + ExpiresActive on + ExpiresDefault "access plus 30 minutes" + + RewriteEngine on + RewriteBase /index/ + + RewriteCond "&%{QUERY_STRING}" &annotation:org.flatpak.ref:exists=1 + RewriteCond "&%{QUERY_STRING}" &architecture=([^&]+) + RewriteRule "static" flatpak-%1.json [L] + + RewriteCond "&%{QUERY_STRING}" &architecture=([^&]+) + RewriteRule "static" all-%1.json [L] + + RewriteCond "&%{QUERY_STRING}" &annotation:org.flatpak.ref:exists=1 + RewriteRule "static" flatpak.json [L] + + RewriteRule "static" all.json [L] + + AllowOverride None + Options +Indexes + Require all granted + + + + ExpiresActive on + ExpiresDefault "access plus 1 year" + + AllowOverride None + Options +Indexes + Require all granted + + +SSLEngine on +SSLCertificateFile /etc/pki/tls/certs/regindexer.crt +SSLCertificateKeyFile /etc/pki/tls/private/regindexer.key diff --git a/services/frontend/generate-cert.sh b/services/frontend/generate-cert.sh new file mode 100755 index 0000000..668440b --- /dev/null +++ b/services/frontend/generate-cert.sh @@ -0,0 +1,59 @@ +#!/bin/bash +set -e + +work=$(mktemp -d) +cleanup() { + rm -rf $work +} +trap cleanup EXIT + +cd $work + +# Generate private keys +openssl genrsa -out regindexer_ca.key 2048 +openssl genrsa -out regindexer.key 2048 + +# Generate CSRs +cat > ca.config < cert.config < /dev/null ; do + sleep 1 +done +echo "started" + +if ! check-for-data.py http://registry:5000 ; then + create-test-data.sh +fi + +exec httpd -DNO_DETACH -DFOREGROUND diff --git a/services/index/trigger-reindex.sh b/services/index/trigger-reindex.sh new file mode 100755 index 0000000..55da89e --- /dev/null +++ b/services/index/trigger-reindex.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo '{"ctype": "flatpak"}' | fedmsg-logger-3 --modname=bodhi --topic="mashtask.complete" --json-input diff --git a/services/registry/Dockerfile b/services/registry/Dockerfile new file mode 100644 index 0000000..d419607 --- /dev/null +++ b/services/registry/Dockerfile @@ -0,0 +1,9 @@ +FROM fedora:28 + +RUN dnf -y update +RUN dnf -y install docker-distribution && dnf clean all + +VOLUME ["/var/lib/registry"] +EXPOSE 5000 +ENTRYPOINT ["registry"] +CMD ["serve", "/etc/docker/registry/config.yml"] diff --git a/services/registry/config.yml b/services/registry/config.yml new file mode 100644 index 0000000..ac559e2 --- /dev/null +++ b/services/registry/config.yml @@ -0,0 +1,29 @@ +version: 0.1 +log: + fields: + service: registry +storage: + cache: + blobdescriptor: inmemory + filesystem: + rootdirectory: /var/lib/registry + delete: + enabled: true +# notifications: +# endpoints: +# - name: index +# url: http://index:8088/events +# headers: +# Authorization: [Bearer OPEN_SESAME] +# timeout: 1s +# threshold: 10 +# backoff: 1s +http: + addr: :5000 + headers: + X-Content-Type-Options: [nosniff] +health: + storagedriver: + enabled: true + interval: 10s + threshold: 3 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..a2c5741 --- /dev/null +++ b/setup.py @@ -0,0 +1,24 @@ +from setuptools import setup + +setup(name='regindexer', + version='0.1', + description='fedmsg-hub plugin to maintain a static registry index', + url='https://pagure.io/regindexer', + author='Owen Taylor', + author_email='otaylor@redhat.com', + license='MIT', + packages=['regindexer'], + include_package_data=True, + install_requires=[ + 'click', + 'requests', + 'PyYAML', + ], + entry_points= { + 'console_scripts': [ + 'regindexer=regindexer.cli:cli' + ], + 'moksha.consumer': [ + "regindexer = regindexer.consumer:Consumer", + ], + }) diff --git a/test-data/.gitignore b/test-data/.gitignore new file mode 100644 index 0000000..851c8ab --- /dev/null +++ b/test-data/.gitignore @@ -0,0 +1 @@ +busybox-manifest-list.json diff --git a/test-data/README-test-data.md b/test-data/README-test-data.md new file mode 100644 index 0000000..35c3f54 --- /dev/null +++ b/test-data/README-test-data.md @@ -0,0 +1,9 @@ +Test Data +========= + +`banner-flatpak-oci/` is a [Flatpak](http://www.flatpak.org) of the +[Banner](http://software.cedar-solutions.com/utilities.html) utility, exported +to an [OCI image](https://github.com/opencontainers/image-spec). For GPL +compliance, the complete source code can be found in `banner-1.3.4-6.fc26.src.rpm`. +The OCI image was generated using [https://github.com/owtaylor/osbs-box]. + diff --git a/test-data/banner-1.3.4-6.fc26.src.rpm b/test-data/banner-1.3.4-6.fc26.src.rpm new file mode 100644 index 0000000..37e435a Binary files /dev/null and b/test-data/banner-1.3.4-6.fc26.src.rpm differ diff --git a/test-data/banner-flatpak-oci-no-media-type/blobs/sha256/c813d7ba2a00449b7fc5f5aaf3cbf6c32c683c1d500513a8ca60270f7efb0f08 b/test-data/banner-flatpak-oci-no-media-type/blobs/sha256/c813d7ba2a00449b7fc5f5aaf3cbf6c32c683c1d500513a8ca60270f7efb0f08 new file mode 100644 index 0000000..07fd18c --- /dev/null +++ b/test-data/banner-flatpak-oci-no-media-type/blobs/sha256/c813d7ba2a00449b7fc5f5aaf3cbf6c32c683c1d500513a8ca60270f7efb0f08 @@ -0,0 +1,16 @@ +{ + "created" : "2017-09-07T19:54:23Z", + "architecture" : "arm64", + "os" : "linux", + "config" : { + "Memory" : 0, + "MemorySwap" : 0, + "CpuShares" : 0 + }, + "rootfs" : { + "type" : "layers", + "diff_ids" : [ + "sha256:5ae3d7cb247bd8998455618133183d4bdb036a9aea5269c13b338e1765da2c27" + ] + } +} \ No newline at end of file diff --git a/test-data/banner-flatpak-oci-no-media-type/blobs/sha256/caa78176c374f7ffc54139355fda25544fa2a46edb5c35643926d6d8246d09f6 b/test-data/banner-flatpak-oci-no-media-type/blobs/sha256/caa78176c374f7ffc54139355fda25544fa2a46edb5c35643926d6d8246d09f6 new file mode 100644 index 0000000..a53b4ff --- /dev/null +++ b/test-data/banner-flatpak-oci-no-media-type/blobs/sha256/caa78176c374f7ffc54139355fda25544fa2a46edb5c35643926d6d8246d09f6 @@ -0,0 +1,31 @@ +{ + "schemaVersion" : 2, + "config" : { + "mediaType" : "application/vnd.oci.image.config.v1+json", + "digest" : "sha256:c813d7ba2a00449b7fc5f5aaf3cbf6c32c683c1d500513a8ca60270f7efb0f08", + "size" : 314 + }, + "layers" : [ + { + "mediaType" : "application/vnd.oci.image.layer.v1.tar+gzip", + "digest" : "sha256:e0504937d72e87cd26bbcda0545b7ae383373098ff8c25b536bd107584ef4e50", + "size" : 6007 + } + ], + "annotations" : { + "org.flatpak.commit-metadata.xa.ref" : "YXBwL2NvbS5jZWRhcl9zb2x1dGlvbnMuQmFubmVyL3g4Nl82NC9tYXN0ZXIAAHM=", + "org.flatpak.body" : "Name: com.cedar_solutions.Banner\nArch: x86_64\nBranch: master\nBuilt with: Flatpak 0.9.8\n", + "org.flatpak.commit-metadata.xa.metadata" : "W0FwcGxpY2F0aW9uXQpuYW1lPWNvbS5jZWRhcl9zb2x1dGlvbnMuQmFubmVyCnJ1bnRpbWU9b3JnLmZlZG9yYXByb2plY3QuTWluaW1hbFBsYXRmb3JtL3g4Nl82NC8yNgpzZGs9b3JnLmZlZG9yYXByb2plY3QuTWluaW1hbFBsYXRmb3JtL3g4Nl82NC8yNgpjb21tYW5kPWJhbm5lcgoAAHM=", + "org.flatpak.commit-metadata.ostree.ref-binding" : "YXBwL2NvbS5jZWRhcl9zb2x1dGlvbnMuQmFubmVyL3g4Nl82NC9tYXN0ZXIALQBhcw==", + "org.flatpak.download-size" : "6007", + "org.flatpak.commit-metadata.xa.download-size" : "AAAAAAAAFvIAdA==", + "org.flatpak.commit-metadata.xa.installed-size" : "AAAAAAAAQAAAdA==", + "org.flatpak.subject" : "Export com.cedar_solutions.Banner", + "org.flatpak.installed-size" : "16384", + "org.flatpak.commit" : "3eddff0efa5d3ace9e28834f5d1e5a042913ae82473ed13d33508173f425687c", + "org.flatpak.metadata" : "[Application]\nname=com.cedar_solutions.Banner\nruntime=org.fedoraproject.MinimalPlatform/x86_64/26\nsdk=org.fedoraproject.MinimalPlatform/x86_64/26\ncommand=banner\n", + "org.opencontainers.image.ref.name" : "app/com.cedar_solutions.Banner/x86_64/master", + "org.flatpak.timestamp" : "1504814063", + "org.flatpak.commit-metadata.ostree.collection-binding" : "AABz" + } +} \ No newline at end of file diff --git a/test-data/banner-flatpak-oci-no-media-type/blobs/sha256/e0504937d72e87cd26bbcda0545b7ae383373098ff8c25b536bd107584ef4e50 b/test-data/banner-flatpak-oci-no-media-type/blobs/sha256/e0504937d72e87cd26bbcda0545b7ae383373098ff8c25b536bd107584ef4e50 new file mode 100644 index 0000000..5260be4 Binary files /dev/null and b/test-data/banner-flatpak-oci-no-media-type/blobs/sha256/e0504937d72e87cd26bbcda0545b7ae383373098ff8c25b536bd107584ef4e50 differ diff --git a/test-data/banner-flatpak-oci-no-media-type/index.json b/test-data/banner-flatpak-oci-no-media-type/index.json new file mode 100644 index 0000000..baf7662 --- /dev/null +++ b/test-data/banner-flatpak-oci-no-media-type/index.json @@ -0,0 +1,17 @@ +{ + "schemaVersion" : 2, + "manifests" : [ + { + "mediaType" : "application/vnd.oci.image.manifest.v1+json", + "digest" : "sha256:caa78176c374f7ffc54139355fda25544fa2a46edb5c35643926d6d8246d09f6", + "size" : 1875, + "annotations" : { + "org.opencontainers.image.ref.name" : "latest" + }, + "platform" : { + "architecture" : "amd64", + "os" : "linux" + } + } + ] +} diff --git a/test-data/banner-flatpak-oci-no-media-type/oci-layout b/test-data/banner-flatpak-oci-no-media-type/oci-layout new file mode 100644 index 0000000..21b1439 --- /dev/null +++ b/test-data/banner-flatpak-oci-no-media-type/oci-layout @@ -0,0 +1 @@ +{"imageLayoutVersion": "1.0.0"} \ No newline at end of file diff --git a/test-data/banner-flatpak-oci-single/blobs/sha256/8125ee777fe53c1405acbef2d6ab6b309ef0ff0157f2c11056723abebcf4182d b/test-data/banner-flatpak-oci-single/blobs/sha256/8125ee777fe53c1405acbef2d6ab6b309ef0ff0157f2c11056723abebcf4182d new file mode 100644 index 0000000..805ac46 --- /dev/null +++ b/test-data/banner-flatpak-oci-single/blobs/sha256/8125ee777fe53c1405acbef2d6ab6b309ef0ff0157f2c11056723abebcf4182d @@ -0,0 +1,32 @@ +{ + "schemaVersion" : 2, + "mediaType" : "application/vnd.oci.image.manifest.v1+json", + "config" : { + "mediaType" : "application/vnd.oci.image.config.v1+json", + "digest" : "sha256:c813d7ba2a00449b7fc5f5aaf3cbf6c32c683c1d500513a8ca60270f7efb0f08", + "size" : 314 + }, + "layers" : [ + { + "mediaType" : "application/vnd.oci.image.layer.v1.tar+gzip", + "digest" : "sha256:e0504937d72e87cd26bbcda0545b7ae383373098ff8c25b536bd107584ef4e50", + "size" : 6007 + } + ], + "annotations" : { + "org.flatpak.commit-metadata.xa.ref" : "YXBwL2NvbS5jZWRhcl9zb2x1dGlvbnMuQmFubmVyL3g4Nl82NC9tYXN0ZXIAAHM=", + "org.flatpak.body" : "Name: com.cedar_solutions.Banner\nArch: x86_64\nBranch: master\nBuilt with: Flatpak 0.9.8\n", + "org.flatpak.commit-metadata.xa.metadata" : "W0FwcGxpY2F0aW9uXQpuYW1lPWNvbS5jZWRhcl9zb2x1dGlvbnMuQmFubmVyCnJ1bnRpbWU9b3JnLmZlZG9yYXByb2plY3QuTWluaW1hbFBsYXRmb3JtL3g4Nl82NC8yNgpzZGs9b3JnLmZlZG9yYXByb2plY3QuTWluaW1hbFBsYXRmb3JtL3g4Nl82NC8yNgpjb21tYW5kPWJhbm5lcgoAAHM=", + "org.flatpak.commit-metadata.ostree.ref-binding" : "YXBwL2NvbS5jZWRhcl9zb2x1dGlvbnMuQmFubmVyL3g4Nl82NC9tYXN0ZXIALQBhcw==", + "org.flatpak.download-size" : "6007", + "org.flatpak.commit-metadata.xa.download-size" : "AAAAAAAAFvIAdA==", + "org.flatpak.commit-metadata.xa.installed-size" : "AAAAAAAAQAAAdA==", + "org.flatpak.subject" : "Export com.cedar_solutions.Banner", + "org.flatpak.installed-size" : "16384", + "org.flatpak.commit" : "3eddff0efa5d3ace9e28834f5d1e5a042913ae82473ed13d33508173f425687c", + "org.flatpak.metadata" : "[Application]\nname=com.cedar_solutions.Banner\nruntime=org.fedoraproject.MinimalPlatform/x86_64/26\nsdk=org.fedoraproject.MinimalPlatform/x86_64/26\ncommand=banner\n", + "org.opencontainers.image.ref.name" : "app/com.cedar_solutions.Banner/x86_64/master", + "org.flatpak.timestamp" : "1504814063", + "org.flatpak.commit-metadata.ostree.collection-binding" : "AABz" + } +} \ No newline at end of file diff --git a/test-data/banner-flatpak-oci-single/blobs/sha256/c813d7ba2a00449b7fc5f5aaf3cbf6c32c683c1d500513a8ca60270f7efb0f08 b/test-data/banner-flatpak-oci-single/blobs/sha256/c813d7ba2a00449b7fc5f5aaf3cbf6c32c683c1d500513a8ca60270f7efb0f08 new file mode 100644 index 0000000..07fd18c --- /dev/null +++ b/test-data/banner-flatpak-oci-single/blobs/sha256/c813d7ba2a00449b7fc5f5aaf3cbf6c32c683c1d500513a8ca60270f7efb0f08 @@ -0,0 +1,16 @@ +{ + "created" : "2017-09-07T19:54:23Z", + "architecture" : "arm64", + "os" : "linux", + "config" : { + "Memory" : 0, + "MemorySwap" : 0, + "CpuShares" : 0 + }, + "rootfs" : { + "type" : "layers", + "diff_ids" : [ + "sha256:5ae3d7cb247bd8998455618133183d4bdb036a9aea5269c13b338e1765da2c27" + ] + } +} \ No newline at end of file diff --git a/test-data/banner-flatpak-oci-single/blobs/sha256/e0504937d72e87cd26bbcda0545b7ae383373098ff8c25b536bd107584ef4e50 b/test-data/banner-flatpak-oci-single/blobs/sha256/e0504937d72e87cd26bbcda0545b7ae383373098ff8c25b536bd107584ef4e50 new file mode 100644 index 0000000..5260be4 Binary files /dev/null and b/test-data/banner-flatpak-oci-single/blobs/sha256/e0504937d72e87cd26bbcda0545b7ae383373098ff8c25b536bd107584ef4e50 differ diff --git a/test-data/banner-flatpak-oci-single/index.json b/test-data/banner-flatpak-oci-single/index.json new file mode 100644 index 0000000..fcc354c --- /dev/null +++ b/test-data/banner-flatpak-oci-single/index.json @@ -0,0 +1,14 @@ +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 1937, + "digest": "sha256:8125ee777fe53c1405acbef2d6ab6b309ef0ff0157f2c11056723abebcf4182d", + "platform": { + "architecture": "arm64", + "os": "linux" + } + } + ] +} diff --git a/test-data/banner-flatpak-oci-single/oci-layout b/test-data/banner-flatpak-oci-single/oci-layout new file mode 100644 index 0000000..60304f2 --- /dev/null +++ b/test-data/banner-flatpak-oci-single/oci-layout @@ -0,0 +1 @@ +{"imageLayoutVersion": "1.0.0"} diff --git a/test-data/banner-flatpak-oci/blobs/sha256/8125ee777fe53c1405acbef2d6ab6b309ef0ff0157f2c11056723abebcf4182d b/test-data/banner-flatpak-oci/blobs/sha256/8125ee777fe53c1405acbef2d6ab6b309ef0ff0157f2c11056723abebcf4182d new file mode 100644 index 0000000..805ac46 --- /dev/null +++ b/test-data/banner-flatpak-oci/blobs/sha256/8125ee777fe53c1405acbef2d6ab6b309ef0ff0157f2c11056723abebcf4182d @@ -0,0 +1,32 @@ +{ + "schemaVersion" : 2, + "mediaType" : "application/vnd.oci.image.manifest.v1+json", + "config" : { + "mediaType" : "application/vnd.oci.image.config.v1+json", + "digest" : "sha256:c813d7ba2a00449b7fc5f5aaf3cbf6c32c683c1d500513a8ca60270f7efb0f08", + "size" : 314 + }, + "layers" : [ + { + "mediaType" : "application/vnd.oci.image.layer.v1.tar+gzip", + "digest" : "sha256:e0504937d72e87cd26bbcda0545b7ae383373098ff8c25b536bd107584ef4e50", + "size" : 6007 + } + ], + "annotations" : { + "org.flatpak.commit-metadata.xa.ref" : "YXBwL2NvbS5jZWRhcl9zb2x1dGlvbnMuQmFubmVyL3g4Nl82NC9tYXN0ZXIAAHM=", + "org.flatpak.body" : "Name: com.cedar_solutions.Banner\nArch: x86_64\nBranch: master\nBuilt with: Flatpak 0.9.8\n", + "org.flatpak.commit-metadata.xa.metadata" : "W0FwcGxpY2F0aW9uXQpuYW1lPWNvbS5jZWRhcl9zb2x1dGlvbnMuQmFubmVyCnJ1bnRpbWU9b3JnLmZlZG9yYXByb2plY3QuTWluaW1hbFBsYXRmb3JtL3g4Nl82NC8yNgpzZGs9b3JnLmZlZG9yYXByb2plY3QuTWluaW1hbFBsYXRmb3JtL3g4Nl82NC8yNgpjb21tYW5kPWJhbm5lcgoAAHM=", + "org.flatpak.commit-metadata.ostree.ref-binding" : "YXBwL2NvbS5jZWRhcl9zb2x1dGlvbnMuQmFubmVyL3g4Nl82NC9tYXN0ZXIALQBhcw==", + "org.flatpak.download-size" : "6007", + "org.flatpak.commit-metadata.xa.download-size" : "AAAAAAAAFvIAdA==", + "org.flatpak.commit-metadata.xa.installed-size" : "AAAAAAAAQAAAdA==", + "org.flatpak.subject" : "Export com.cedar_solutions.Banner", + "org.flatpak.installed-size" : "16384", + "org.flatpak.commit" : "3eddff0efa5d3ace9e28834f5d1e5a042913ae82473ed13d33508173f425687c", + "org.flatpak.metadata" : "[Application]\nname=com.cedar_solutions.Banner\nruntime=org.fedoraproject.MinimalPlatform/x86_64/26\nsdk=org.fedoraproject.MinimalPlatform/x86_64/26\ncommand=banner\n", + "org.opencontainers.image.ref.name" : "app/com.cedar_solutions.Banner/x86_64/master", + "org.flatpak.timestamp" : "1504814063", + "org.flatpak.commit-metadata.ostree.collection-binding" : "AABz" + } +} \ No newline at end of file diff --git a/test-data/banner-flatpak-oci/blobs/sha256/c813d7ba2a00449b7fc5f5aaf3cbf6c32c683c1d500513a8ca60270f7efb0f08 b/test-data/banner-flatpak-oci/blobs/sha256/c813d7ba2a00449b7fc5f5aaf3cbf6c32c683c1d500513a8ca60270f7efb0f08 new file mode 100644 index 0000000..07fd18c --- /dev/null +++ b/test-data/banner-flatpak-oci/blobs/sha256/c813d7ba2a00449b7fc5f5aaf3cbf6c32c683c1d500513a8ca60270f7efb0f08 @@ -0,0 +1,16 @@ +{ + "created" : "2017-09-07T19:54:23Z", + "architecture" : "arm64", + "os" : "linux", + "config" : { + "Memory" : 0, + "MemorySwap" : 0, + "CpuShares" : 0 + }, + "rootfs" : { + "type" : "layers", + "diff_ids" : [ + "sha256:5ae3d7cb247bd8998455618133183d4bdb036a9aea5269c13b338e1765da2c27" + ] + } +} \ No newline at end of file diff --git a/test-data/banner-flatpak-oci/blobs/sha256/e0504937d72e87cd26bbcda0545b7ae383373098ff8c25b536bd107584ef4e50 b/test-data/banner-flatpak-oci/blobs/sha256/e0504937d72e87cd26bbcda0545b7ae383373098ff8c25b536bd107584ef4e50 new file mode 100644 index 0000000..5260be4 Binary files /dev/null and b/test-data/banner-flatpak-oci/blobs/sha256/e0504937d72e87cd26bbcda0545b7ae383373098ff8c25b536bd107584ef4e50 differ diff --git a/test-data/banner-flatpak-oci/index.json b/test-data/banner-flatpak-oci/index.json new file mode 100644 index 0000000..fbe7add --- /dev/null +++ b/test-data/banner-flatpak-oci/index.json @@ -0,0 +1,18 @@ +{ + "schemaVersion" : 2, + "mediaType" : "application/vnd.oci.image.index.v1+json", + "manifests" : [ + { + "mediaType" : "application/vnd.oci.image.manifest.v1+json", + "digest" : "sha256:8125ee777fe53c1405acbef2d6ab6b309ef0ff0157f2c11056723abebcf4182d", + "size" : 1937, + "annotations" : { + "org.opencontainers.image.ref.name" : "latest" + }, + "platform" : { + "architecture" : "amd64", + "os" : "linux" + } + } + ] +} diff --git a/test-data/banner-flatpak-oci/oci-layout b/test-data/banner-flatpak-oci/oci-layout new file mode 100644 index 0000000..21b1439 --- /dev/null +++ b/test-data/banner-flatpak-oci/oci-layout @@ -0,0 +1 @@ +{"imageLayoutVersion": "1.0.0"} \ No newline at end of file diff --git a/test-data/contents b/test-data/contents new file mode 100644 index 0000000..3653468 --- /dev/null +++ b/test-data/contents @@ -0,0 +1,4 @@ +docker:docker.io/library/busybox:latest * test.list/busybox:latest +docker:docker.io/library/busybox:latest amd64 test/busybox:latest +dir:banner-flatpak-oci * flatpak.list/banner:latest +dir:banner-flatpak-oci amd64 flatpak/banner:latest diff --git a/test-data/create-test-data.sh b/test-data/create-test-data.sh new file mode 100755 index 0000000..f78178c --- /dev/null +++ b/test-data/create-test-data.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -e -x + +while read src arch dest ; do + arch_arg= + if [ "$arch" != "*" ] ; then + arch_arg="--arch=$arch" + fi + ./registry_copy.py --dest-tls-verify=false $src $arch_arg docker:localhost:5000/$dest +done < contents diff --git a/test-data/registry_copy.py b/test-data/registry_copy.py new file mode 100755 index 0000000..ad32e37 --- /dev/null +++ b/test-data/registry_copy.py @@ -0,0 +1,559 @@ +#!/usr/bin/python3 + +""" +Copyright (c) 2017 Red Hat, Inc +All rights reserved. + +This software may be modified and distributed under the terms +of the BSD license. See the LICENSE file for details. +""" + + +from __future__ import unicode_literals +import argparse +import json +import logging +import os +import requests +from requests.exceptions import SSLError, ConnectionError +from requests.auth import AuthBase +import re +import shutil +import tempfile +from urllib.parse import urlparse, urlunparse, urlencode +import www_authenticate + +import sys + +logger = logging.getLogger('registry_copy') +logging.basicConfig(level=logging.INFO) + +MEDIA_TYPE_MANIFEST_V2 = 'application/vnd.docker.distribution.manifest.v2+json' +MEDIA_TYPE_LIST_V2 = 'application/vnd.docker.distribution.manifest.list.v2+json' +MEDIA_TYPE_OCI = 'application/vnd.oci.image.manifest.v1+json' +MEDIA_TYPE_OCI_INDEX = 'application/vnd.oci.image.index.v1+json' + + +def registry_hostname(registry): + """ + Strip a reference to a registry to just the hostname:port + """ + if registry.startswith('http:') or registry.startswith('https:'): + return urlparse(registry).netloc + else: + return registry + + +class Config(object): + def __init__(self, path): + self.json_secret_path = path + try: + with open(self.json_secret_path) as fp: + self.json_secret = json.load(fp) + except Exception: + msg = "failed to read registry secret" + logger.error(msg, exc_info=True) + raise RuntimeError(msg) + + def get_credentials(self, docker_registry): + # For maximal robustness we check the host:port of the passed in + # registry against the host:port of the items in the secret. This is + # somewhat similar to what the Docker CLI does. + # + docker_registry = registry_hostname(docker_registry) + try: + return self.json_secret[docker_registry] + except KeyError: + for reg, creds in self.json_secret.items(): + if registry_hostname(reg) == docker_registry: + return creds + + logger.warn('%s not found in %s', docker_registry, self.json_secret_path) + return {} + + +class DirectorySpec(object): + def __init__(self, directory): + self.directory = directory + + def get_endpoint(self): + return DirectoryEndpoint(self.directory) + + +class RegistrySpec(object): + def __init__(self, registry, repo, tag, creds, tls_verify): + if registry == 'docker.io': + self.registry = 'registry-1.docker.io' + else: + self.registry = registry + self.repo = repo + self.tag = tag + self.creds = creds + self.tls_verify = tls_verify + + def get_session(self): + return RegistrySession(self.registry, insecure = not self.tls_verify, + creds=self.creds) + + def get_endpoint(self): + return RegistryEndpoint(self) + + +def parse_spec(parser, spec, creds, tls_verify): + if spec.startswith('dir:'): + _, directory = spec.split(':', 1) + if creds: + parser.print_usage() + parser.exit("Credentials can't be specified for a directory") + return DirectorySpec(directory) + elif spec.startswith('docker:'): + _, rest = spec.split(':', 1) + + parts = rest.split('/', 1) + if len(parts) == 1: + parser.exit("Registry specification should be docker:REGISTRY