#598 Fedora Minimal Compose for OpenQA testing
Closed 3 years ago by humaton. Opened 6 years ago by mohanboddu.
mohanboddu/pungi-fedora minimal-compose  into  f28

@@ -0,0 +1,273 @@ 

+ # PRODUCT INFO

+ release_name = 'Fedora-Minimal-Compose'

+ release_short = 'Fedora-Minimal-Compose'

+ release_version = 'Rawhide'

+ release_is_layered = False

+ 

+ # GENERAL SETTINGS

+ comps_file = {

+     'scm': 'git',

+     'repo': 'https://pagure.io/fedora-comps.git',

+     'branch': 'master', 

+     'file': 'comps-rawhide.xml',

+     'command': 'make comps-rawhide.xml'

+ }

+ 

+ variants_file='variants-fedora.xml'

+ sigkeys = ['9570FF31']

+ 

+ # limit tree architectures

+ # if undefined, all architectures from variants.xml will be included

+ tree_arches = ['x86_64',]

+ 

+ # limit tree variants

+ # if undefined, all variants from variants.xml will be included

+ tree_variants = ['Server']

+ 

+ hashed_directories = True

+ 

+ # RUNROOT settings

+ runroot_method = 'koji'

+ runroot_channel = 'compose'

+ runroot_tag = 'f33-build'

+ 

+ # PKGSET

+ pkgset_source = 'koji' # koji, repos

+ 

+ # PKGSET - REPOS

+ # pkgset_repos format: {arch: [repo1_url, repo2_url, ...]}

+ # pkgset_repos = {}

+ 

+ # PKGSET - KOJI

+ pkgset_koji_tag = 'f33-minimal-compose'

+ pkgset_koji_inherit = True

+ 

+ filter_system_release_packages = False

+ 

+ # GATHER

+ gather_method = {

+     '^.*': {                # For all variants

+         'comps': 'deps',    # resolve dependencies for packages from comps file

+         'module': 'nodeps', # but not for packages from modules

+     }

+ }

+ gather_backend = 'dnf'

+ gather_profiler = True

+ check_deps = False

+ greedy_method = 'build'

+ 

+ repoclosure_backend = 'dnf'

+ 

+ # fomat: [(variant_uid_regex, {arch|*: [repos]})]

+ # gather_lookaside_repos = []

+ 

+ # GATHER - JSON

+ # format: {variant_uid: {arch: package: [arch1, arch2, None (for any arch)]}}

+ #gather_source_mapping = '/path/to/mapping.json'

+ 

+ 

+ # CREATEREPO

+ createrepo_deltas = False

+ createrepo_database = True

+ createrepo_use_xz = True

+ createrepo_extra_args = ['--zck', '--zck-dict-dir=/usr/share/fedora-repo-zdicts/rawhide']

+ createrepo_num_workers = 10

+ 

+ # CHECKSUMS

+ media_checksums = ['sha256']

+ media_checksum_one_file = True

+ media_checksum_base_filename = '%(release_short)s-%(variant)s-%(version)s-%(arch)s-%(date)s%(type_suffix)s.%(respin)s'

+ #jigdo

+ create_jigdo = False

+ 

+ # CREATEISO

+ iso_hfs_ppc64le_compatible = False

+ 

+ # BUILDINSTALL

+ buildinstall_method = 'lorax'

+ buildinstall_skip = [

+     ('^Modular$', {

+         '*': True

+     }),

+     ('^Everything$', {

+         'i386': True

+     }),

+ ]

+ 

+ # Enables macboot on x86_64 for all variants and disables upgrade image building

+ # # everywhere.

+ lorax_options = [

+   ('^.*$', {

+      'x86_64': {

+          'nomacboot': False

+      },

+      '*': {

+          'noupgrade': True,

+          'rootfs_size': 3

+      }

+   })

+ ]

+ 

+ #extra_packages = [

+ #    '/mnt/packages/foo*',

+ #]

+ 

+ 

+ # fomat: [(variant_uid_regex, {arch|*: [packages]})]

+ additional_packages = [

+     ('^(Server|Everything)$', {

+         '*': [

+             'kernel*',

+             'dracut.*',

+             'autocorr-*',

+             'eclipse-nls',

+             'eclipse-nls-*',

+             'hunspell-*',

+             'hyphen-*',

+             'kde-l10n-*',

+             'libreoffice-langpack-*',

+             'man-pages-*',

+             'mythes-*',

+         ],

+     }),

+ 

+     ('^Everything$', {

+         '*': [

+             '*',

+         ],

+     }),

+ 

+     ('^Server$', {

+         '*': [

+ 

+         ],

+     }),

+ 

+ ]

+ 

+ multilib = [

+     ('^Everything$', {

+         'x86_64': ['devel', 'runtime'],

+     })

+ ]

+ 

+ filter_packages = [

+     ("^.*$", {"*": ["glibc32", "libgcc32"]}),

+     ('(Server)$', {

+         '*': [

+         'kernel*debug*',

+         'kernel-kdump*',

+         'kernel-tools*',

+         'syslog-ng*',

+         'astronomy-bookmarks',

+         'generic*',

+         'GConf2-dbus*',

+         'bluez-gnome',

+         'java-11-openjdk',

+         'community-mysql*',

+         'jruby*',

+         'gimp-help-*',

+         ]

+     }),

+ ]

+ 

+ 

+ # format: {arch|*: [packages]}

+ multilib_blacklist = {

+     '*': ['kernel', 'kernel-PAE*', 'kernel*debug*',

+         'dmraid-devel', 'kdeutils-devel', 'mkinitrd-devel',

+         'php-devel', 'java-*', 'bash-devel',

+         'httpd-devel', 'tomcat-native', 'php*', 'httpd',

+         'krb5-server', 'krb5-server-ldap', 'mod_*', 'ghc-*'

+     ],

+ }

+ 

+ 

+ # format: {arch|*: [packages]}

+ multilib_whitelist = {

+     '*': ['libgnat', 'wine', 'lmms-vst', 'nspluginwrapper',

+         'libflashsupport', 'valgrind', 'perl-libs', 'redhat-lsb',

+         'yaboot', 'syslinux-extlinux-nonlinux', 'syslinux-nonlinux',

+         'syslinux-tftpboot', 'nosync', '*-static', 'apitrace-libs',

+         'fakeroot-libs', 'postgresql-odbc', 'mysql-connector-odbc',

+         'fakechroot-libs','mesa-vdpau-drivers', 'p11-kit-trust',

+         'mariadb-connector-odbc', 'compiler-rt',

+         'nvidia-query-resource-opengl-lib',

+         'ibus-libs', 'ibus-gtk2', 'ibus-gtk3',

+         'glib-networking'

+     ],

+ }

+ 

+ createiso_skip = [

+         ('^Server$', {

+             'src': True

+         }),

+ 

+         ('^Everything$', {

+             '*': True,

+             'src': True

+         }),

+ 

+         ('^Modular$', {

+             '*': True,

+             'src': True

+         }),

+     ]

+ 

+ # Image name respecting Fedora's image naming policy

+ image_name_format = '%(release_short)s-%(variant)s-%(disc_type)s-%(arch)s-%(version)s-%(date)s%(type_suffix)s.%(respin)s.iso'

+ # # Use the same format for volume id

+ image_volid_formats = [

+      '%(release_short)s-%(variant)s-%(disc_type)s-%(arch)s-%(version)s'

+      ]

+ # No special handling for layered products, use same format as for regular images

+ image_volid_layered_product_formats = []

+ # Replace 'Cloud' with 'C' in volume id etc.

+ volume_id_substitutions = {

+                  'Beta': 'B',

+               'Rawhide': 'rawh',

+         'Astronomy_KDE': 'AstK',

+            'Silverblue': 'SB',

+              'Cinnamon': 'Cinn',

+                 'Cloud': 'C',

+            'Comp_Neuro': 'CNr',

+          'Design_suite': 'Dsgn',

+        'Electronic_Lab': 'Elec',

+            'Everything': 'E',

+                 'Games': 'Game',

+                'Images': 'img',

+               'Jam_KDE': 'Jam',

+           'MATE_Compiz': 'MATE',

+      # Note https://pagure.io/pungi-fedora/issue/533

+      'Python-Classroom': 'Clss',

+      'Python_Classroom': 'Clss',

+              'Robotics': 'Robo',

+        'Scientific_KDE': 'SciK',

+              'Security': 'Sec',

+                'Server': 'S',

+         '-Workstation-': '-WS-',

+ }

+ 

+ disc_types = {

+     'boot': 'netinst',

+     'live': 'Live',

+ }

+ 

+ translate_paths = [

+    ('/mnt/koji/compose/', 'https://kojipkgs.fedoraproject.org/compose/'),

+ ]

+ 

+ # These will be inherited by live_media, live_images and image_build

+ global_ksurl = 'git+https://pagure.io/fedora-kickstarts.git?#HEAD'

+ global_release = '!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN'

+ global_version = 'Rawhide'

+ # live_images ignores this in favor of live_target

+ global_target = 'f33'

+ 

+ 

+ 

+ live_target = 'f33'

+ live_images_no_rename = True

+ 

+ 

+ koji_profile = 'compose_koji'

Fixes #6746

When an update is created for certain pkg builds, QA wants to have
a server image to be composed and copied to OpenQA stagging server.
Then QA can automatically run certain tests on the compose with the
updates in it.

Signed-off-by: Mohan Boddu mboddu@redhat.com

i'd still like to have this compose be called the anaconda compose... we have an atomic compose we have a cloud compose, etc.. To me the barebones test for sanity checking the state of a branch (f28 rawhide, etc) is that anaconda can be built and used.

I agree with @dustymabe that the compose should be named after our intended purpose.

I would rather we use installer as the name for more clarity for those who may not realize that the installer software is called anaconda.

I agree with @dustymabe that the compose should be named after our intended purpose.

thanks

I would rather we use installer as the name for more clarity for those who may not realize that the installer software is called anaconda.

I'm +1 for installer too, but do have a slight preference for anaconda. Either one works.

rebased onto 20b2ef3fdc332b185515ea1f8fab191ef5c5856e

6 years ago

I am not a big fan of either one, as the usage can be extended to other packages which are not installer specific.

I am calling this as Fedora-Minimal compose for now, what do you think?

From a long term perspective, I don't think we should ever extend this past the installer.

Anything we add will slow down our eventual desired end result for them and, if we find more use cases later, we can add them at that time.

Let's not plan for complexity that may never exist.

Tracking other packages is not going to slow down the compose process, probably adding more composes which means more space (we have to gate on that, not starting a compose if an existing one is running) but it wont slow down the compose process at all.

I guess the concern here is answering the question "what is this compose for?". I really think the name should reflect what it is for and I don't really think this should be a compose that gets extended for things that land outside of the installer scope.

What I'm saying is that the scope should be installer only or otherwise very well defined.

@dustymabe When we first got the request, it was definitely for installer related packages(not all of them, but some), but then it might be extended to other packages.

So, I asked @adamwill to come up with a name for it.

Right. I think what me and @kellin are saying is that it would be nice to have this work appropriately scoped to just the installer and not some not well defined compose that people end up throwing more and more stuff in to.

Maybe we should ask @adamwill for input on this topic/discussion.

True, even if we call it as "Fedora-Installer" compose, we are not going to test each and every installer related package and if we call it something else that doesn't mean we will test any random package.

Its a set of packages that QA is defined/interested in(starting with anaconda).

@dustymabe this is a more general problem than just 'the installer', though. There are certainly other potential reasons we might want a compose to test an update. In my head, what this 'thing' basically is is "a compose containing the bits from update (FOO) for the purpose of testing update (FOO) in ways which require a compose to test", if that clarifies things at all. Right now, the major "way which requires a compose to test" that we're thinking about is "installer tests", but there could be others.

@adamwill, I understand. One possible Other Use I have for this compose is a bit of a one off, but after GA it is useful for Atomic to be able to respin anaconda for a fix if needed. If we had an anaconda compose then we just tag a new package and run the compose by hand and use the output of that.

Barring that special use case.. Maybe we call this the Fedora-minimal-qa compose or Fedora-minimal-test compose?

I dunno, it's a tricky naming problem in general. I suggested Fedora-Updatetest...but that only makes sense in terms of what it's for, it doesn't really make sense in terms of what's in the compose.

I suppose the question is really this: if we decided we wanted a less minimal compose profile for update testing in future, would we:

1) edit this one
2) keep this as a minimal compose profile, and create a new less-minimal profile

if 1), then 'Fedora-Updatetest' or so makes more sense. If 2), then a name like 'Fedora-Minimal' makes more sense...

rebased onto 4061c77fea27fd8287132c4555dbf5b3400e370f

6 years ago

This requires f28-update-test-compose tag (https://pagure.io/pungi-fedora/pull-request/598#_1,50) which inherits from f28 tag.

We use f28-update-test-compose tag to tag the builds that QA wants to test and everything else will be pulled from the stable content(f28 tag).

Didn't we talk about this before and note that you'll need to be careful with lifecycle management of that tag? Somehow you're going to have to ensure that each time we run the 'minimal' compose the tag contains exactly the packages from the update we're producing a compose for, and no other packages - so you're going to have to keep tagging things in and out of it...

@adamwill Yes, when we automate this entire process, every time you request a compose with certain set of builds to test, it will untag everything inside this tag and tag everything you just requested.

And who ever running it manually until then needs to be careful and remember to do untagging and tagging until everything is automated.

hmm. is this not going to be automated? At least for anaconda testing I feel like it should run once a day, as well as on-demand (i.e. when someone from anaconda has pushed a new package and wants a new compose to run)

rebased onto 88d493b

4 years ago

Hi, this PR was opened for an old fedora-release that is EOL now. If it is still relevant could you please reopen it to the current release?

Pull-Request has been closed by humaton

3 years ago

Well, we still have the use case, but before work started here I implemented it differently (by having openQA build the images it needs to test itself). This means we're duplicating work to some extent, but it got things going, and there were still going to be some awkward little issues with actually using this compose (including coming up with some kind of way to note what update each compose actually contained, in a way openQA could consume).

So I think as long as that's working we can leave this on the back burner.

Metadata