#4 Initial version of the ci.fmf examples
Merged 5 years ago by psss. Opened 5 years ago by psss.

@@ -0,0 +1,1 @@ 

+ 1

@@ -0,0 +1,19 @@ 

+ # Basic structure of artifacts and related testsets

+ /test:

+     /pull-request:

+         /pep:

+             summary: All code must comply with the PEP8 style guide

+         /lint:

+             summary: Run pylint to catch common problems (no gating)

+     /build:

+         /smoke:

+             summary: Basic smoke test (Tier1)

+         /features:

+             summary: Verify important features

+     /update:

+         /basic:

+             summary: Run all Tier1, Tier2 and Tier3 tests

+         /security:

+             summary: Security tests (extra job to get quick results)

+         /integration:

+             summary: Integration tests with related components

file added
+30
@@ -0,0 +1,30 @@ 

+ # Turn on gating for selected testsets (defined inline)

+ /test:

+     /pull-request:

+         /pep:

+             summary: All code must comply with the PEP8 style guide

+             # Do not allow ugly code to be merged into master

+             gate:

+                 - merge-pull-request

+         /lint:

+             summary: Run pylint to catch common problems (no gating)

+     /build:

+         /smoke:

+             summary: Basic smoke test (Tier1)

+             # Basic smoke test is used by three gates

+             gate:

+                 - merge-pull-request

+                 - add-build-to-update

+                 - add-build-to-compose

+         /features:

+             summary: Verify important features

+     /update:

+         # This enables the 'release-update' gate for all three testsets

+         gate:

+             - release-update

+         /basic:

+             summary: Run all Tier1, Tier2 and Tier3 tests

+         /security:

+             summary: Security tests (extra job to get quick results)

+         /integration:

+             summary: Integration tests with related components

@@ -0,0 +1,43 @@ 

+ # Turn on gating for selected testsets (separate gate section)

+ /test:

+     /pull-request:

+         /pep:

+             summary: All code must comply with the PEP8 style guide

+         /lint:

+             summary: Run pylint to catch common problems (no gating)

+     /build:

+         /smoke:

+             summary: Basic smoke test (Tier1)

+         /features:

+             summary: Verify important features

+     /update:

+         /basic:

+             summary: Run all Tier1, Tier2 and Tier3 tests

+         /security:

+             summary: Security tests (extra job to get quick results)

+         /integration:

+             summary: Integration tests with related components

+ 

+ /gate:

+     # Gate blocking the pull request merge into master

+     /merge-pull-request:

+         test:

+             - /test/pull-request/pep

+             - /test/build/smoke

+ 

+     # Gate blocking the build to be added to an update/erratum

+     /add-build-to-update:

+         test:

+             - /test/build/smoke

+ 

+     # Gate blocking the build to be added to compose

+     /add-build-to-compose:

+         test:

+             - /test/build/smoke

+ 

+     # Gate blocking release of an update/erratum

+     /release-update:

+         test:

+             - /test/update/basic

+             - /test/update/security

+             - /test/update/integration

file added
+32
@@ -0,0 +1,32 @@ 

+ # Multiple tool support for test discovery & execution

+ /test:

+     /build:

+         # Standard Test Inteface

+         /sti:

+             execute:

+                 tool: sti

+                 playbooks:

+                     - 'tests.yml'

+                 tags:

+                     - 'classic'

+         # Simple shell script

+         /shell:

+             execute:

+                 tool: shell

+                 path: 'tests'

+                 test: './test.sh'

+                 environment:

+                     PYTHON: 'python2'

+         # Workflow Tomorrow

+         /wow:

+             execute:

+                 tool: wow

+                 options: '--plan 1234 --tags Tier1'

+         # Flexible Metadata Format + Restraint

+         /fmf:

+             discover:

+                 tool: fmf

+                 filter: 'tier: 1, 2'

+                 repository: https://src.fedoraproject.org/tests/python

+             execute:

+                 tool: restraint

@@ -0,0 +1,59 @@ 

+ # Workflow steps: discover, provision, prepare, execute, report

+ /test:

+     # Discover relevant tests

+     discover:

+         tool: fmf

+     # System requirements

+     provision:

+         memory:

+             min: '1 GB'

+         arch:

+             - 'x86_64'

+     # Machine configuration

+     prepare:

+         ansible:

+             - 'setup.yml'

+     # Test execution

+     execute:

+         tool: restraint

+     # Result reporting

+     report:

+         contact: email@address.org

+ 

+     # Build testing

+     /build:

+         /smoke:

+             summary: Basic smoke test (Tier1)

+             discover+:

+                 filter: 'tier: 1'

+         /features:

+             summary: Verify important features

+             discover+:

+                 filter: 'tag: functional'

+ 

+     # Errata testing

+     /update:

+         provision+:

+             arch:

+                 - x86_64

+                 - ppc64

+                 - s390x

+         /basic:

+             summary: Run all Tier1, Tier2 and Tier3 tests

+             discover+:

+                 filter: 'tier: 1, 2, 3'

+         /security:

+             summary: Security tests (extra job to get quick results)

+             discover+:

+                 filter: 'tag: security'

+         /integration:

+             summary: Integration tests with related components

+             discover+:

+                 filter: 'tag: integration'

+             provision+:

+                 memory:

+                     min: '4 GB'

+                     max: '8 GB'

+             prepare:

+                 ansible:

+                     - 'integration.yml'

This is an initial set of examples demonstrating how the
Extensible Configuration for CI could look like. The files
illustrate different aspects of the concept.

In production everything would be stored under a single "ci"
parent, that is saved in the ci.fmf file or scattered on the
filesystem as defined by the Flexible Metadata Format.

Here's an example output of fmf show for the workflow example:

/ci/test/build/smoke
discover: {u'filter': u'tier: 1', u'tool': u'fmf'}
execute: {u'tool': u'restraint'}
prepare: {u'ansible': [u'setup.yml']}
provision: {u'arch': [u'x86_64'], u'memory': u'1 GB'}
report: {u'contact': u'email@address.org'}
summary: Basic smoke test (Tier1)

/ci/test/build/features
discover: {u'filter': u'tag: functional', u'tool': u'fmf'}
execute: {u'tool': u'restraint'}
prepare: {u'ansible': [u'setup.yml']}
provision: {u'arch': [u'x86_64'], u'memory': u'1 GB'}
report: {u'contact': u'email@address.org'}
summary: Verify important features

/ci/test/update/basic
discover: {u'filter': u'tier: 1, 2, 3', u'tool': u'fmf'}
execute: {u'tool': u'restraint'}
prepare: {u'ansible': [u'setup.yml']}
provision: {u'arch': [u'x86_64', u'ppc64', u's390x'], u'memory': u'1 GB'}
report: {u'contact': u'email@address.org'}
summary: Run all Tier1, Tier2 and Tier3 tests

/ci/test/update/integration
discover: {u'filter': u'tag: integration', u'tool': u'fmf'}
execute: {u'tool': u'restraint'}
prepare: {u'ansible': [u'integration.yml']}
provision: {u'arch': [u'x86_64', u'ppc64', u's390x'], u'memory': u'4 GB'}
report: {u'contact': u'email@address.org'}
summary: Integration tests with related components

/ci/test/update/security
discover: {u'filter': u'tag: security', u'tool': u'fmf'}
execute: {u'tool': u'restraint'}
prepare: {u'ansible': [u'setup.yml']}
provision: {u'arch': [u'x86_64', u'ppc64', u's390x'], u'memory': u'1 GB'}
report: {u'contact': u'email@address.org'}
summary: Security tests (extra job to get quick results)

It nicely demonstrates how fmf features like inheritance and
attribute merging can prevent unnecessary data duplication.

+1 - the examples follow the ideas and the structure we discussed, looks good to me.

I'd just like to point out to other readers that these examples are not final - e.g. fields like provision will need a bit more work to support constraints users put on the testing systems these days. I've spent few hours digging into schema Beaker machine filtering provides, and I believe we're going to need a bit more tools than just "memory: 1 GB" :) But that doesn't invalidate the overall structure of the examples.

Bear with me, please - I am aware this is just an example, I just want to make you aware of what's running through my head these days :)

I was checking out a way Beaker handles "machine filtering" - a complex set of XML rules, including conditions (<and>, <or>), and a vast list of dimensions and properties one can check. I also checked how this works in Travis, for example (it doesn't, one size fits all...). Another input into our thinking would be possible backends and what they are able to deliver when asked for details, possibly leading to a small common set of properties based on the most common use cases.

For example, both OpenStack and beaker can provide machines with a different number of CPUs and amounts of RAM. I guess this would be true for many other provisioners. It'd make sense to start with these, and support general minimal/maximal/absolute constraints for these two dimensions, e.g.

  ...
  provision:
    # the box must have at least 2 CPUs and at least 4 GB RAM
    memory:
      min: 4 GB
      # or "max: 8GB", or both of them to specify range, or "equal"...
    cpu:
      min: 2
      # or "max" or ...

  ...
  provision:
    # I'm happy with a box with 4 GB RAM, or with a box with at least 2 CPUs
    any:
      memory:
        min: 4 GB
      cpu:
        min: 2

My idea is that whatever code stands between this configuration and the actual provisioner, translates these rules into something the provisioner understands. When asking beaker for machines, this would become an XML snippet for <hostFilter>, when dealing with OpenStack, the code would pick an instance flavor, when running QEMU, the code would translate these rules into standard-inventory-qcow2 options, when... I'd really like to support a generic way, separated from the actual provisioning backed - the idea of typing "give me a box with at least 4 GB of RAM" in ten different ways for ten different backends my test might be executed on gives me goosebumbs :)

There are many other properties of cpu in Beaker (e.g. one can request machines of specific CPU family or a machine with specific CPU flag) and other objects as well (e.g. disk and its properties) - when in an extensible way, these less frequent properties don't really have be supported from the beginning, focusing on common use cases would be better, leaving door opened for the additions.

@happz I like the any attribute idea :) sounds reasonable to me about proposing min/max values instead of some exact value

rebased onto 7831f4b

5 years ago

Thanks for the review, guys. I've updated the workflow example with memory attribute supporting min and max values. As for the any keyword I'd leave that for some future discussion (more detailed and backed up by tangible use cases). I'd like to keep the initial set of examples as simple as possible. Going to merge.

Metadata Update from @psss:
- Request assigned

5 years ago

Pull-Request has been merged by psss

5 years ago

@psss Sure, any, min, max and so on were just to illustrate a concept.

Metadata Update from @psss:
- Pull-request tagged with: ci metadata

5 years ago