From d073d013bd70befde8d3e7bdb999f5eabe46a74f Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: May 26 2018 20:45:25 +0000 Subject: initial f29/rawhide configs Signed-off-by: Peter Robinson --- diff --git a/fedora-iot.conf b/fedora-iot.conf new file mode 100644 index 0000000..38a99fe --- /dev/null +++ b/fedora-iot.conf @@ -0,0 +1,335 @@ +# PRODUCT INFO +release_name = 'Fedora-IoT' +release_short = 'Fedora-IoT' +release_version = '29' +release_is_layered = False +# Let's not build install media during this run. For the image builds +# and OSTree installer we'll use inputs from other f29 runs. +skip_phases = ["buildinstall", "productimg", "extra_files"] +ostree_installer_overwrite = True + +# GENERAL SETTINGS +bootable = True +comps_file = { + 'scm': 'git', + 'repo': 'https://pagure.io/fedora-comps.git', + 'branch': 'master', + 'file': 'comps-f29.xml', + 'command': 'make comps-f29.xml' +} +variants_file='variants-fedora.xml' +sigkeys = ['9DB62FB1'] +# limit tree architectures +# if undefined, all architectures from variants.xml will be included +# tree_arches = ['aarch64', 'armhfp', 'x86_64'] +# limit tree variants +# if undefined, all variants from variants.xml will be included +tree_variants = ['IoT'] +hashed_directories = True +# RUNROOT settings +runroot = True +runroot_channel = 'compose' +runroot_tag = 'f29-build' +# PKGSET +pkgset_source = 'koji' # koji, repos + +# PKGSET - REPOS +# pkgset_repos format: {arch: [repo1_url, repo2_url, ...]} +# pkgset_repos = {} + +# PKGSET - KOJI +#pkgset_koji_tag = 'f29-atomic' +pkgset_koji_tag = 'f29-iot' +pkgset_koji_inherit = False + +filter_system_release_packages = False + +# GATHER +gather_source = 'comps' +gather_method = 'deps' +gather_profiler = True +check_deps = False +greedy_method = 'build' +# fomat: [(variant_uid_regex, {arch|*: [repos]})] +# gather_lookaside_repos = [] +# GATHER - JSON +# format: {variant_uid: {arch: package: [arch1, arch2, None (for any arch)]}} +#gather_source_mapping = '/path/to/mapping.json' +# CREATEREPO +createrepo_c = True +createrepo_checksum = 'sha256' +# CHECKSUMS +media_checksums = ['sha256'] +media_checksum_one_file = True +media_checksum_base_filename = 'Fedora-%(variant)s-%(version)s-%(date)s.%(respin)s-%(arch)s' +#jigdo +create_jigdo = False + +# BUILDINSTALL +buildinstall_method = 'lorax' +buildinstall_skip = [ + ('^Modular$', { + '*': True + }), +] + +# Enables macboot on x86_64 for all variants and disables upgrade image building +# # everywhere. +lorax_options = [ + ('^.*$', { + 'x86_64': { + 'nomacboot': False + }, + '*': { + 'noupgrade': True + } + }) +] + +# We just grab everything, since everything that's gather'ed is meant as an +# override. +additional_packages = [ + ('^IoT$$', { + '*': [ + '*', + ], + }), +] +multilib = [ + ('^Everything$', { + 'x86_64': ['devel', 'runtime'], + 's390x': ['devel', 'runtime'] + }) +] +filter_packages = [] +# format: {arch|*: [packages]} +multilib_blacklist = { + '*': ['kernel', 'kernel-PAE*', 'kernel*debug*', + 'dmraid-devel', 'kdeutils-devel', 'mkinitrd-devel', + 'php-devel', 'java-*', + 'httpd-devel', 'tomcat-native', 'php*', 'httpd', + 'krb5-server', 'krb5-server-ldap', 'mod_*', 'ghc-*' + ], +} +# format: {arch|*: [packages]} +multilib_whitelist = { + '*': ['libgnat', 'wine', 'lmms-vst', 'nspluginwrapper', + 'libflashsupport', 'valgrind', 'perl-libs', 'redhat-lsb', + 'yaboot', 'syslinux-extlinux-nonlinux', 'syslinux-nonlinux', + 'syslinux-tftpboot', 'nosync', '*-static', 'apitrace-libs', + 'fakeroot-libs', 'postgresql-odbc', 'mysql-connector-odbc', + 'fakechroot-libs','mesa-vdpau-drivers', 'p11-kit-trust', + 'mariadb-connector-odbc', 'compiler-rt' + ], +} +createiso_skip = [ + ('^Workstation$', { + '*': True, + 'src': True + }), + + ('^Server$', { + 'src': True + }), + + ('^Everything$', { + '*': True, + 'src': True + }), + + ('^Modular$', { + '*': True, + 'src': True + }), + ] + +# fomat: [(variant_uid_regex, {arch|*: [scm_dicts]})] +#extra_files = [ +# ('^(Server|Workstation)$', { +# '*': [ +# { +# 'scm': 'rpm', +# 'repo': 'fedora-release-%(variant_id_lower)s', +# 'branch': None, +# 'file': [ +# '/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-release', +# ], +# 'target': '', +# }, +# ], +# }), +#] +# Image name respecting Fedora's image naming policy +image_name_format = 'Fedora-%(variant)s-%(disc_type)s-%(arch)s-%(version)s-%(date)s.%(respin)s.iso' +# # Use the same format for volume id +image_volid_formats = [ + 'Fedora-%(variant)s-%(disc_type)s-%(arch)s-%(version)s' + ] +# No special handling for layered products, use same format as for regular images +image_volid_layered_product_formats = [] +# Used by Pungi to replace 'Cloud' with 'C' (etc.) in ISO volume IDs. +# There is a hard 32-character limit on ISO volume IDs, so we use +# these to try and produce short enough but legible IDs. Note this is +# duplicated in Koji for live images, as livemedia-creator does not +# allow Pungi to tell it what volume ID to use. Note: +# https://fedoraproject.org/wiki/User:Adamwill/Draft_fedora_image_naming_policy +# '-Workstation-' is a temporary workaround. See +# https://pagure.io/pungi-fedora/pull-request/525 +volume_id_substitutions = { + 'Beta': 'B', + 'Rawhide': 'rawh', + 'Astronomy_KDE': 'AstK', + 'AtomicHost': 'AH', + 'AtomicWorkstation': 'AW', + 'Cinnamon': 'Cinn', + 'Cloud': 'C', + 'Design_suite': 'Dsgn', + 'Electronic_Lab': 'Elec', + 'Everything': 'E', + 'Games': 'Game', + 'Images': 'img', + 'IoT': 'iot', + 'Jam_KDE': 'Jam', + 'MATE_Compiz': 'MATE', + # Note https://pagure.io/pungi-fedora/issue/533 + 'Python-Classroom': 'Clss', + 'Python_Classroom': 'Clss', + 'Robotics': 'Robo', + 'Scientific_KDE': 'SciK', + 'Security': 'Sec', + 'Server': 'S', + '-Workstation-': '-WS-', +} + +disc_types = { + 'boot': 'netinst', + 'live': 'Live', +} + +translate_paths = [ + ('/mnt/koji/compose/', 'http://kojipkgs.fedoraproject.org/compose/'), +] + +# These will be inherited by live_media, live_images and image_build +global_ksurl = 'git+https://pagure.io/fedora-kickstarts.git' +global_release = '!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN' +global_version = '29' +# live_images ignores this in favor of live_target +global_target = 'f29' +image_build = { + '^IoT$': [ + { + 'image-build': { + 'format': [('raw-xz','raw.xz')], + 'name': 'Fedora-IoT', + 'kickstart': 'fedora-iot.ks', + 'distro': 'Fedora-22', + 'disk_size': 6, + 'arches': ['aarch64', 'x86_64'], + 'install_tree_from': "https://kojipkgs.fedoraproject.org/compose/29/latest-Fedora-29/compose/Everything/$arch/os/", + 'subvariant': 'IoT', + } + } + ], +} + +ostree = [ + ("^IoT$", { + "aarch64": { + "version": "!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN", + "treefile": "fedora-iot.json", + "config_url": "https://pagure.io/fedora-iot/atomic.git", + "config_branch": "f29", + "repo": [ + "https://kojipkgs.fedoraproject.org/compose/rawhide/latest-Fedora-Rawhide/compose/Everything/$basearch/os/" + ] + "tag_ref": False, + "ostree_repo": "/mnt/koji/compose/iot/repo/", + 'failable': ['*'], + }, + "armhfp": { + "version": "!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN", + "treefile": "fedora-iot.json", + "config_url": "https://pagure.io/fedora-iot/atomic.git", + "config_branch": "f29", + "repo": [ + "https://kojipkgs.fedoraproject.org/compose/rawhide/latest-Fedora-Rawhide/compose/Everything/$basearch/os/" + ] + "tag_ref": False, + "ostree_repo": "/mnt/koji/compose/iot/repo/", + 'failable': ['*'], + }, + "x86_64": { + "version": "!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN", + "treefile": "fedora-iot.json", + "config_url": "https://pagure.io/fedora-iot/atomic.git", + "config_branch": "f29", + "repo": [ + "https://kojipkgs.fedoraproject.org/compose/rawhide/latest-Fedora-Rawhide/compose/Everything/$basearch/os/" + ] + "tag_ref": False, + "ostree_repo": "/mnt/koji/compose/iot/repo/", + 'failable': ['*'], + } + }) +] + +ostree_installer = [ + ("^IoT$", { + "x86_64": { + "repo": [ + "https://kojipkgs.fedoraproject.org/compose/rawhide/latest-Fedora-Rawhide/compose/Everything/$arch/os/" + ], + "release": None, + "rootfs_size": "4", + "add_template": ["ostree-based-installer/lorax-configure-repo.tmpl", + "ostree-based-installer/lorax-embed-repo.tmpl"], + "add_template_var": [ + "ostree_install_repo=https://kojipkgs.fedoraproject.org/compose/iot/repo/", + "ostree_update_repo=https://dl.fedoraproject.org/iot/repo/", + "ostree_osname=fedora-iot", + "ostree_install_ref=fedora/29/x86_64/iot", + "ostree_update_ref=fedora/29/x86_64/iot", + ], + 'template_repo': 'https://pagure.io/fedora-lorax-templates.git', + 'template_branch': 'f29', + }, + "aarch64": { + "repo": [ + "https://kojipkgs.fedoraproject.org/compose/rawhide/latest-Fedora-Rawhide/compose/Everything/$arch/os/" + ], + "release": None, + "rootfs_size": "4", + "add_template": ["ostree-based-installer/lorax-configure-repo.tmpl", + "ostree-based-installer/lorax-embed-repo.tmpl"], + "add_template_var": [ + "ostree_install_repo=https://kojipkgs.fedoraproject.org/compose/iot/repo/", + "ostree_update_repo=https://dl.fedoraproject.org/iot/repo/", + "ostree_osname=fedora-iot", + "ostree_install_ref=fedora/29/aarch64/iot", + "ostree_update_ref=fedora/29/aarch64/iot", + ], + 'template_repo': 'https://pagure.io/fedora-lorax-templates.git', + 'template_branch': 'f29', + }, + "armhfp": { + "repo": [ + "https://kojipkgs.fedoraproject.org/compose/rawhide/latest-Fedora-Rawhide/compose/Everything/$arch/os/" + ], + "release": None, + "rootfs_size": "4", + "add_template": ["ostree-based-installer/lorax-configure-repo.tmpl", + "ostree-based-installer/lorax-embed-repo.tmpl"], + "add_template_var": [ + "ostree_install_repo=https://kojipkgs.fedoraproject.org/compose/iot/repo/", + "ostree_update_repo=https://dl.fedoraproject.org/iot/repo/" + "ostree_osname=fedora-iot", + "ostree_install_ref=fedora/29/armhfp/iot", + "ostree_update_ref=fedora/29/armhfp/iot", + ], + 'template_repo': 'https://pagure.io/fedora-lorax-templates.git', + 'template_branch': 'f29', + } + }) +] +koji_profile = 'compose_koji' diff --git a/nightly.sh b/nightly.sh new file mode 100755 index 0000000..a63c4a1 --- /dev/null +++ b/nightly.sh @@ -0,0 +1,113 @@ +#!/bin/sh + +export LC_ALL=C + +CONFIG="fedora-iot.conf" +TARGET_DIR="/mnt/koji/compose/iot" +NIGHTLY="--nightly" +SKIP_PHASES="--skip-phase=productimg" +DEST=$(pwd) +DATE=$(date "+%Y%m%d") +SHORT="Fedora-IoT" +RELEASE="29" +RELEASE_TITLE="29" +COMPSFILE="comps-f29.xml" +TMPDIR=`mktemp -d /tmp/$RELEASE.$DATE.XXXX` +TOMAIL="iot@lists.fedoraproject.org" +FROM="Fedora IoT Report " +RSYNCPREFIX="sudo -u ftpsync" +RSYNCTARGET="/pub/fedora/linux/development/$RELEASE" +RSYNCALTTARGET="/pub/alt/development/$RELEASE" +RSYNCSECTARGET="/pub/fedora-secondary/development/$RELEASE" +ATOMICSRCREPO="/mnt/koji/compose/atomic/repo/" +ATOMICDESTREPO="/mnt/koji/atomic/repo/" +OLDCOMPOSE_ID=$(cat $TARGET_DIR/latest-$SHORT-$RELEASE_TITLE/COMPOSE_ID) + +# assume a releng dir is a git checkout of the releng repo +# if it does not exist clone it +if [ -d releng ]; then + pushd releng + git pull --rebase + popd +else + git clone https://pagure.io/releng.git +fi + +# Set up our fedmsg function, using the releng repo definition +#FEDMSG_MODNAME="compose" +#FEDMSG_CERTPREFIX="releng" +. ./releng/scripts/fedmsg-functions.sh + +# Announce that we are starting, even though we don't know the compose_id yet.. +#fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s"}' "$RELEASE" "$ARCH" "$SHORT") +#send_fedmsg "${fedmsg_json_start}" ${RELEASE} start + +#pushd $TMPDIR +#git clone https://pagure.io/fedora-comps.git && { +# pushd fedora-comps +# make "${COMPSFILE}" +# cp "${COMPSFILE}" $DEST/ +# popd +#} +#popd + +CMD="pungi-koji --notification-script=/usr/bin/true --notification-script=pungi-wait-for-signed-ostree-handler --config=$CONFIG --old-composes=$TARGET_DIR $OLD_COMPOSES_DIR $NIGHTLY $SKIP_PHASES" + +if [ -z "$COMPOSE_ID" ]; then + CMD="$CMD --target-dir=$TARGET_DIR" +else + CMD="$CMD --debug-mode --compose-dir=$TARGET_DIR/$COMPOSE_ID" +fi + +time $CMD "$@" +if [ "$?" != "0" ]; then + exit 1 +fi + +NEWCOMPOSE_ID=$(cat $TARGET_DIR/latest-$SHORT-$RELEASE_TITLE/COMPOSE_ID) +SHORTCOMPOSE_ID=$(echo $NEWCOMPOSE_ID|sed -e 's|Fedora-.*-||g') + +# Set this to use later for a few items include depcheck +DESTDIR=$TARGET_DIR/$NEWCOMPOSE_ID +# Public URLs the synced compose will wind up at, we put them in fedmsgs +LOCATION="https://dl.fedoraproject.org$RSYNCTARGET" +ALT_LOCATION="https://dl.fedoraproject.org$RSYNCALTTARGET" +SECONDARY_LOCATION="https://dl.fedoraproject.org$RSYNCSECTARGET" +# Update fedmsg template +# fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s", "location": "%s", "alt_location": "%s", "secondary_location": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID", "$LOCATION", "$ALT_LOCATION", "$SECONDARY_LOCATION") +# fedmsg_json_done=$(printf '{"log": "done", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s", "location": "%s", "alt_location": "%s", "secondary_location": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID", "$LOCATION", "$ALT_LOCATION", "$SECONDARY_LOCATION") + +# Fix permissions on the grub efi files and fonts (they're 0600) +chmod -R go+r $DESTDIR/compose/*/*/os/EFI/ + +#compose-changelog -p $DESTDIR/logs/ $TARGET_DIR/$OLDCOMPOSE_ID/ $DESTDIR/ + +# Tell interested persons that the rsync is starting (zomg!) +#send_fedmsg "${fedmsg_json_start}" ${RELEASE} rsync.start + + +# sync over atomic host/workstation to the unified ostree repo +#for arch in x86_64 aarch64 ppc64le; do +# ostree pull-local --repo=$ATOMICDESTREPO $ATOMICSRCREPO --depth=-1 "fedora/29/${arch}/atomic-host" +#done +#for arch in x86_64; do +# ostree pull-local --repo=$ATOMICDESTREPO $ATOMICSRCREPO --depth=-1 "fedora/29/${arch}/workstation" +#done +#ostree summary -u --repo=$ATOMICDESTREPO # update summary file + +# Tell interested persons that the rsync is done. +#send_fedmsg "${fedmsg_json_done}" ${RELEASE} rsync.complete + +# Tell everyone by fedmsg about the compose +#send_fedmsg "${fedmsg_json_done}" ${RELEASE} complete + +# Tell everyone by email about the compose +# "$DESTDIR/logs/depcheck" lets not cat out depcheck for now as it does +# not understand rich dependencies +SUBJECT='Fedora '$RELEASE' compose report: '$SHORTCOMPOSE_ID' changes' +#for tomail in $TOMAIL ; do +# cat $DESTDIR/logs/*verbose | \ +# mutt -e "set from=\"$FROM\"" -e 'set envelope_from=yes' -s "$SUBJECT" $tomail +#done + +find $TARGET_DIR -xdev -depth -maxdepth 2 -mtime +14 -exec rm -rf {} \; diff --git a/twoweek-nightly.sh b/twoweek-nightly.sh new file mode 100755 index 0000000..0a60643 --- /dev/null +++ b/twoweek-nightly.sh @@ -0,0 +1,63 @@ +#!/bin/sh +export LC_ALL=C +LABEL=$1 +# Remove the label from arguments. It gets special treatment. Other arguments +# to the script are passed to pungi-koji directly. +shift +CONFIG="fedora-iot.conf" +TARGET_DIR="/mnt/koji/compose/iot" +NIGHTLY="" +DEST=$(pwd) +DATE=$(date "+%Y%m%d") +# the Pungi 'shortname', which we will include in fedmsgs for disambiguation +SHORT="Fedora-IoT" +RELEASE="29" +RELEASE_TITLE="29" +COMPSFILE="comps-f29.xml" +TMPDIR=`mktemp -d /tmp/$RELEASE.$DATE.XXXX` +TOMAIL="iot@lists.fedoraproject.org" +FROM="Fedora IoT Report " +RSYNCPREFIX="sudo -u ftpsync" +RSYNCTARGET="/pub/alt/iot/testing/$SHORTCOMPOSE_ID" +OLDCOMPOSE_ID=$(cat $TARGET_DIR/latest-$SHORT-$RELEASE_TITLE/COMPOSE_ID) + +# assume a releng dir is a git checkout of the releng repo +# if it does not exist clone it +if [ -d releng ]; then + pushd releng + git pull --rebase + popd +else + git clone https://pagure.io/releng.git +fi +# Set up our fedmsg function, using the releng repo definition +fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s"}' "$RELEASE" "$ARCH" "$SHORT") +FEDMSG_MODNAME="compose" +FEDMSG_CERTPREFIX="releng" +. ./releng/scripts/fedmsg-functions.sh +# Announce that we are starting... +send_fedmsg "${fedmsg_json_start}" ${RELEASE} start +CMD="pungi-koji --notification-script=/usr/bin/pungi-fedmsg-notification --notification-script=pungi-wait-for-signed-ostree-handler --config=$CONFIG --old-composes=$TARGET_DIR $OLD_COMPOSES_DIR $NIGHTLY --label=$LABEL" +if [ -z "$COMPOSE_ID" ]; then + CMD="$CMD --target-dir=$TARGET_DIR" +else + CMD="$CMD --debug-mode --compose-dir=$TARGET_DIR/$COMPOSE_ID" +fi +time $CMD "$@" +if [ "$?" != "0" ]; then + exit 1 +fi + +NEWCOMPOSE_ID=$(cat $TARGET_DIR/latest-$SHORT-$RELEASE_TITLE/COMPOSE_ID) +SHORTCOMPOSE_ID=$(echo $NEWCOMPOSE_ID|sed -e 's|Fedora-.*-||g') + +DESTDIR=$TARGET_DIR/$NEWCOMPOSE_ID +# Public URL the synced compose will wind up at, we put it in fedmsgs +LOCATION="https://dl.fedoraproject.org$RSYNCTARGET" +# Update fedmsg template +fedmsg_json_start=$(printf '{"log": "start", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s", "location": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID", "$LOCATION") +fedmsg_json_done=$(printf '{"log": "done", "branch": "%s", "arch": "%s", "short": "%s", "compose_id": "%s", "location": "%s"}' "$RELEASE" "$ARCH" "$SHORT" "$NEWCOMPOSE_ID" "$LOCATION") + +# Tell everyone by fedmsg about the compose +send_fedmsg "${fedmsg_json_done}" ${RELEASE} complete + diff --git a/variants-fedora.xml b/variants-fedora.xml new file mode 100644 index 0000000..8586282 --- /dev/null +++ b/variants-fedora.xml @@ -0,0 +1,11 @@ + + + + + + aarch64 + armhfp + x86_64 + + +