From e8477663134cdf0c28df185f380c4d60fa4db637 Mon Sep 17 00:00:00 2001 From: Robert Mayr Date: Jan 10 2017 21:21:35 +0000 Subject: Merge branch 'alt-cloud' --- diff --git a/alt.fedoraproject.org/build/atomic_vars.py b/alt.fedoraproject.org/build/atomic_vars.py new file mode 100755 index 0000000..7c9f677 --- /dev/null +++ b/alt.fedoraproject.org/build/atomic_vars.py @@ -0,0 +1,237 @@ +#!/usr/bin/python +""" Return the results of the atomic release engine from datagrepper. + +For the Two-Week Atomic Change (F23) + +Deps: $ sudo dnf install python-requests + +Author: Ralph Bean +License: LGPLv2+ +""" + +from __future__ import print_function + +import collections +import functools +import json +import logging +import os +import sys +import socket + +from datetime import datetime, timedelta + +import dateutil.relativedelta +import dateutil.tz +import dogpile.cache +import requests + +log = logging.getLogger("atomic_vars") + +try: + sys.path.append('../build.d') + + import globalvar +except ImportError: + log.error("Unable to import globalvar") + sys.exit(1) + +base_url = 'https://apps.fedoraproject.org/datagrepper/raw' +topic = "org.fedoraproject.prod.releng.atomic.twoweek.complete" + +UTC = dateutil.tz.tzutc() + +session = requests.session() + +cache = dogpile.cache.make_region().configure( + "dogpile.cache.dbm", + # 'make clean' does not remove this cache, but we let the values expire + # once every this many seconds (once a day) + expiration_time=86400, + arguments={ + "filename": os.path.join(os.getcwd(), 'build/atomic.cache') + }, +) + +# Are we running in fedora-infra or on someone's laptop? +hostname = socket.gethostname() +if '.phx2.fedoraproject.org' in hostname: + DL_URL_PREFIX = 'http://dl.phx2.fedoraproject.org' +else: + DL_URL_PREFIX = 'https://dl.fedoraproject.org' + +download_fpo = 'https://download.fedoraproject.org' + + +def get_page(page, pages): + """ Retrieve the JSON for a particular page of datagrepper results """ + log.debug("Getting page %i of %s", page, pages) + params = dict( + start=1441402109, # the timestamp of when we first started doing this + topic=topic, + page=page, + rows_per_page=1, + ) + response = session.get(base_url, params=params) + if not bool(response): + raise IOError("Failed to talk to %r %r" % (response.url, response)) + return response.json() + + +# A list of fedmsg messages ideas that were produced erroneously. +# We don't want to use them, so ban them from our results. +blacklist = [ + '2016-dd05c4b7-958b-439f-90d6-e5ca0af2197c', + '2016-b2a2eb00-acef-4a1f-bc6a-ad5aa9d81eee', + '2016-0307f681-1eae-4aeb-9126-8a43b7a378e2', +] + +def get_messages(target): + """ Generator that yields messages from datagrepper """ + + # Get the first page + data = get_page(1, 'unknown') + for message in data['raw_messages']: + if message['msg_id'] in blacklist: + continue + if target in json.dumps(message): + yield message + + more = functools.partial(get_page, pages=data['pages']) + + # Get all subsequent pages (if there are any...) + for page in range(1, data['pages']): + data = more(page + 1) + + for message in data['raw_messages']: + if message['msg_id'] in blacklist: + continue + if target in json.dumps(message): + yield message + + +def make_templates(curr_atomic_id, next_atomic_id): + return [ + # As things stand now, we only do two-week-atomic stuff for the current + # stable release. + (curr_atomic_id, '', ''), + + # If we ever move to doing pre-release versions as well, just uncomment + # the following line and it should all work. We leave it commented out + # now because querying datagrepper for pre-release results that are not + # there is much more slow than querying for something that exists. + #(next_atomic_id, 'pre_atomic_', 'pre_'), + ] + + +# We cache this guy on disk so we don't hit datagrepper over and over. +@cache.cache_on_arguments() +def collect(curr_atomic_id, next_atomic_id): + results = collections.defaultdict(dict) + + # This is the information needed to provide "latest" download targets that + # redirect to the actual mirrormanager url via htpasswd file + results['release']['redir_map'] = collections.defaultdict(dict) + + for idx, composedate_prefix, iso_size_prefix in make_templates(curr_atomic_id, next_atomic_id): + + log.info("Looking for latest atomic release for %s" % idx) + # Get the *latest* atomic release information. + messages = get_messages('-%s-' % idx) + try: + message = messages.next() + except StopIteration: + log.warn("Couldn't find any two-week-atomic content for %r" % idx) + continue + + # Parse the composedate out of the image_name + image_name = message['msg']['atomic_qcow2']['image_name'] + composedate = '.'.join(image_name.split('-')[-1].split('.')[:-2]) + log.info(" Found composedate: %s" % composedate) + results['release'][composedate_prefix + 'atomic_composedate'] = composedate + + # Save the timestamp so we can compute the age later, off-cache. + results['release'][composedate_prefix + 'atomic_ts'] = message['timestamp'] + + # Get the sizes of the isos in megabytes. To do this, we need... + # A mapping between what the release-engine tool calls each artifact, + # and what we call them. + mapping = { + 'atomic_qcow2': 'atomic_qcow2_cloud', + 'atomic_raw': 'atomic_raw_cloud', + 'atomic_vagrant_libvirt': 'atomic_libvag_cloud', + 'atomic_vagrant_virtualbox': 'atomic_VBvag_cloud', + } + for key, entry in message['msg'].items(): + # There are some other keys in there we don't care about. + if not key.startswith('atomic_'): + continue + + # Do an HTTP HEAD to find the size of the file in megabytes + url = entry['image_url'] + download_url = entry['image_url'] + if not url.startswith('http'): + url = DL_URL_PREFIX + url + download_url = download_fpo + entry['image_url'] + response = requests.head(url) + if not bool(response): + log.error("Failed to HEAD %s for size. %r" % (url, response)) + continue + + length = int(response.headers['content-length']) / 1000000 + + # Provide the download URL + url_key = mapping[key] + "_url" + results['release'][url_key] = download_url + + # Provide the redirect rule mapping + img_filename = download_url.split('/')[-1] + results['release']['redir_map'][key] = {} + results['release']['redir_map'][key]['redirect'] = download_url + results['release']['redir_map'][key]['filename'] = img_filename + + # Figure out which of our vars we're going to set, and set it + iso_size_key = iso_size_prefix + mapping[key] + results['iso_size'][iso_size_key] = str(length) + + # Special case for Atomic ISO latest redirect rule mapping because it's + # not included in fedmsg data + atomic_iso_filename = "Fedora-Atomic-ostree-x86_64-{}-{}.iso".format( + globalvar.release['curr_id'], + results['release'][composedate_prefix + 'atomic_composedate'] + ) + results['release']['redir_map']['atomic_iso'] = {} + results['release']['redir_map']['atomic_iso']['redirect'] = \ + globalvar.path['download_atomic'] + "/stable/Fedora-Atomic-" + \ + globalvar.release['curr_id'] + '-' + \ + results['release'][composedate_prefix + 'atomic_composedate'] + \ + "/Atomic/x86_64/iso/{}".format(atomic_iso_filename) + results['release']['redir_map']['atomic_iso']['filename'] = atomic_iso_filename + + return results + + +# Note, this is *not* cached, since we need to update it frequently. +def update_age(release): + """ Is it old and stale? + + We aim to produce new atomic releases every two weeks at minimum. If we're + older than two weeks, we should put up a warning on the websites. Here we + just compute a flag that gets checked in the template. If this latest + release if younger than two weeks, call it "fresh". If it is older than + two weeks, it is no longer fresh. + http://taiga.cloud.fedoraproject.org/project/acarter-fedora-docker-atomic-tooling/us/31 + """ + + results = collections.defaultdict(dict) + templates = make_templates(release['curr_atomic_id'], release['next_atomic_id']) + for idx, composedate_prefix, iso_size_prefix in templates: + two_weeks_ago = datetime.now(UTC) - timedelta(days=14) + timestamp = release[composedate_prefix + 'atomic_ts'] + latest = datetime.fromtimestamp(timestamp, UTC) + freshness = bool(latest >= two_weeks_ago) + relative_delta = datetime.now(UTC) - latest + casual_delta = relative_delta.days + results['release'][composedate_prefix + 'atomic_freshness'] = freshness + results['release'][composedate_prefix + 'atomic_age'] = casual_delta + return results diff --git a/alt.fedoraproject.org/build/fedimg_vars.py b/alt.fedoraproject.org/build/fedimg_vars.py new file mode 100755 index 0000000..576a30f --- /dev/null +++ b/alt.fedoraproject.org/build/fedimg_vars.py @@ -0,0 +1,159 @@ +#!/usr/bin/python +""" Return the AMIs uploaded by fedimg for a given set of release vars. + +Search datagrepper to find the results. + +Deps: $ sudo dnf install python-requests + +Author: Ralph Bean +License: LGPLv2+ +""" + +from __future__ import print_function + +import collections +import functools +import logging +import os + +import requests +import dogpile.cache + +log = logging.getLogger('fedimg_vars') + +base_url = 'https://apps.fedoraproject.org/datagrepper/raw' +topic = "org.fedoraproject.prod.fedimg.image.upload" + +session = requests.session() + +cache = dogpile.cache.make_region().configure( + "dogpile.cache.dbm", + # 'make clean' does not remove this cache, but we let the values expire + # once every this many seconds (once a day) + expiration_time=86400, + arguments={ + "filename": os.path.join(os.getcwd(), 'build/amis.cache') + }, +) + +def get_page(page, pages, target): + """ Retrieve the JSON for a particular page of datagrepper results """ + log.debug("Getting page %i of %s", page, pages) + response = session.get(base_url, params=dict( + topic=topic, + page=page, + contains=target, + rows_per_page=100, + )) + return response.json() + + +def get_messages(target): + """ Generator that yields messages from datagrepper """ + + # Get the first page + data = get_page(1, 'unknown', target) + for message in data['raw_messages']: + yield message + + more = functools.partial(get_page, pages=data['pages'], target=target) + + # Get all subsequent pages (if there are any...) + for page in range(1, data['pages']): + data = more(page + 1) + + for message in data['raw_messages']: + yield message + +# We cache this guy on disk for 500s +@cache.cache_on_arguments() +def collect(release): + results = collections.defaultdict(dict) + + # 1 - transform release vars into an image name we want to query for + templates = [ + # The F22 released AMIs uploads didn't appear to go through fedimg, so + # we can't use this scheme for them. Stuff for F23 should all go that + # route though, so we can hopefully switch over soon. + ("Fedora-Cloud-Base-{curr_cloud_AMI_id}-{atomic_composedate}.x86_64", { + 'HVM_base_AMI': lambda e: e.get('virt_type') == 'hvm' and e.get('vol_type') == 'standard', + 'GP2_HVM_base_AMI': lambda e: e.get('virt_type') == 'hvm' and e.get('vol_type') == 'gp2', + 'PV_base_AMI': lambda e: e.get('virt_type') == 'paravirtual' and e.get('vol_type') == 'standard', + 'GP2_PV_base_AMI': lambda e: e.get('virt_type') == 'paravirtual' and e.get('vol_type') == 'gp2', + }), + #("Fedora-Atomic-{curr_cloud_AMI_id}-{atomic_composedate}.x86_64", { + # 'HVM_atomic_AMI': lambda e: e.get('virt_type') == 'hvm' and e.get('vol_type') == 'standard', + # 'GP2_HVM_atomic_AMI': lambda e: e.get('virt_type') == 'hvm' and e.get('vol_type') == 'gp2', + #}), + #("Fedora-Cloud-Base-{next_cloud_AMI_id}_{curr_cloud_AMI_state}-{RC_pre_gold}.x86_64", { + # 'pre_HVM_base_AMI': lambda e: e.get('virt_type') == 'hvm' and e.get('vol_type') == 'standard', + # 'pre_GP2_HVM_base_AMI': lambda e: e.get('virt_type') == 'hvm' and e.get('vol_type') == 'gp2', + # 'pre_PV_base_AMI': lambda e: e.get('virt_type') == 'paravirtual' and e.get('vol_type') == 'standard', + # 'pre_GP2_PV_base_AMI': lambda e: e.get('virt_type') == 'paravirtual' and e.get('vol_type') == 'gp2', + #}), + #("Fedora-Cloud-Atomic-{next_cloud_AMI_id}_{curr_cloud_AMI_state}-{manual_pre_cloud_AMI_atomic_composedate}.x86_64", { + # 'pre_HVM_atomic_AMI': lambda e: e.get('virt_type') == 'hvm' and e.get('vol_type') == 'standard', + # 'pre_GP2_HVM_atomic_AMI': lambda e: e.get('virt_type') == 'hvm' and e.get('vol_type') == 'gp2', + #}), + ] + + for template, buckets in templates: + # 2 - Build an intermediary dict + intermediary = collections.OrderedDict() + target = template.format(**release) + log.info("Looking for AMIs for %s" % target) + + messages = get_messages(target) + for message in messages: + key = message['msg']['image_name'] + if not key in intermediary: + intermediary[key] = [] + intermediary[key].append(message['msg']) + + if not intermediary: + log.warn("No AMIs found for %s" % target) + continue + + # What would this even mean? + assert len(intermediary) < 2, "Impossible. Got more than one target." + + uploads = intermediary[target] + + # 3- transform intermediary representation into results + for name, matches in buckets.items(): + for upload in uploads: + if matches(upload['extra']): + ami = upload['extra']['id'] + # The region looks like "EC2 (REGION)", so we strip stuff. + region = upload['destination'][5:-1] + results[name][region] = ami + + return results + + +def sanity_check(globalvar, collected_fedimg_vars): + """ This is a sanity check just to make sure the datagrepper code is not + way off from what we had hand-typed before. + + Eventually, remove this. + """ + + names = [ + 'pre_HVM_base_AMI', + 'pre_GP2_HVM_base_AMI', + 'pre_PV_base_AMI', + 'pre_GP2_PV_base_AMI', + 'pre_HVM_atomic_AMI', + 'pre_GP2_HVM_atomic_AMI', + ] + for name in names: + handtyped = getattr(globalvar, name) + collected = collected_fedimg_vars[name] + + for key in handtyped: + if not key in collected: + log.warn("collected %r is missing %r" % (name, key)) + + for key in collected: + if not key in handtyped: + log.warn("handtyped %r is missing %r" % (name, key)) diff --git a/alt.fedoraproject.org/data/content/cloud/index.html b/alt.fedoraproject.org/data/content/cloud/index.html new file mode 100644 index 0000000..cce5fc7 --- /dev/null +++ b/alt.fedoraproject.org/data/content/cloud/index.html @@ -0,0 +1,651 @@ + + + + + ${_('Fedora Cloud')} + + + + + + + + + + +
+
+
+
+

${Markup(_('Fedora %(rel)s Cloud Base images')) % {'rel':global_variables.release['curr_id']}}

+
+
+
+
+

${_('Fedora Cloud Base images are for creating general purpose virtual machines (VMs). You can use the Qcow 2 image for use with Openstack or the compressed raw image. If you are not sure what to use, try the raw image.')}

+
+
+
+
+
+ +
+
+

${_('Cloud Base raw')}

+

${_('Download')} | ${_('Verify')}

+
+
+
+
+ +
+
+

${_('Cloud Base qcow2')}

+

${_('Download')} | ${_('Verify')}

+
+
+
+
+
+ + + + +
+ + + + + + +
+
+
+
+

${Markup(_('Fedora %(rel)s Cloud Base Images for Vagrant')) % {'rel':global_variables.release['curr_id']}}

+
+
+
+
+

${_('These images are Vagrant Boxes images for deployment using Vagrant. If you\'re using Vagrant on Mac OS X or Windows, the VirtualBox image is likely the image you\'ll want to use. If you\'re using Vagrant on Fedora, use the libvirt/KVM image.')}

+
+
+
+
+
+ +
+
+

${_('VirtualBox image')}

+

${_('Download')} | ${_('Verify')}

+
+
+
+
+ +
+
+

${_('libvirt/KVM image')}

+

${_('Download')} | ${_('Verify')}

+
+
+
+
+
+ + + + +
+ + + + + + + +
+
+
+
+

${Markup(_('Fedora %(rel)s Cloud Base Images for Amazon Public Cloud')) % {'rel':global_variables.release['curr_id']}}

+
+
+
+
+

${_('The links below will provide you listings of available AMIs by region with buttons to launch them in your Amazon Web Services account. AMI IDs for usage with the AWS Console or command-line tools are also provided.')}

+

${_('All images are available in two formats: a GP2 format which uses faster SSD storage, and a standard format which is more suitable if you have infrequently accessed data and you want to keep storage cost low.')}

+

${_('If you know you need a paravirtual format image for EC2, use the Paravirtual (PV) format. If you\'re unsure which image to choose, use the Hardware Virtual Machine (HVM) format. It can provide the same or better performance than PV images.')}

+
+
+
+
+
+ +
+
+

${_('GP2 HVM AMIs')}

+

${_('Click to launch')}

+ +
+
+
+
+ +
+
+

${_('Standard HVM AMIs')}

+

${_('Click to launch')}

+ +
+
+
+
+ +
+
+

${_('GP2 PV AMIs')}

+

${_('Click to launch')}

+ +
+
+
+
+ +
+
+

${_('Standard PV AMIs')}

+

${_('Click to launch')}

+ +
+
+
+
+
+ +
+ + + + +
+
+
+
+ +
+
+
+
+ + + +
+
+
+
+

${_('Support Resources')}

+
+
+
+
+
+

${_('Documentation')}

+

${Markup(_('A wiki page with information about Fedora Cloud and its images is available <a href="%s">here</a>.') % ('https://fedoracloud.readthedocs.io/en/latest/whatis.html'))}

+
+
+
+
+

${_('Email')}

+

${Markup(_('Users and developers are also available on the mailing list at <a href="%s">cloud@lists.fedoraproject.org</a>.') % ('https://admin.fedoraproject.org/mailman/listinfo/cloud'))}

+
+
+
+
+

${_('Chat')}

+

${Markup(_('Users and developers are available in the <a href="%s">#fedora-cloud</a> IRC channel on <strong>irc.freenode.org</strong> for real-time chat.') % ('http://webchat.freenode.net/?channels=fedora-cloud'))}

+
+
+
+
+
+
+ + + + + + diff --git a/alt.fedoraproject.org/data/content/index.html b/alt.fedoraproject.org/data/content/index.html index bea85b7..82ef37e 100644 --- a/alt.fedoraproject.org/data/content/index.html +++ b/alt.fedoraproject.org/data/content/index.html @@ -219,6 +219,25 @@ + +
+
+
+
+

${_('Cloud Base Images')}

+
+
+
+
+

${_('These are standard Cloud Base images.')}

+
+
+

${_('Download')}

+
+
+
+
+