#10 Add validation script for module defaults
Merged 6 years ago by mohanboddu. Opened 6 years ago by sgallagh.
releng/ sgallagh/fedora-module-defaults validation  into  master

file modified
+1 -1
@@ -1,2 +1,2 @@ 

  check:

- 	/usr/bin/true

+ 	@python3 tests/validate.py

file added
+8
@@ -0,0 +1,8 @@ 

+ document: modulemd-defaults

+ version: 1

+ data:

+     module: samplemodule

+     profiles:

+         6: [default]

+         8: [default]

+         9: [default]

empty or binary file added
file added
+22
@@ -0,0 +1,22 @@ 

+ # This file lists repo contents that should not be validated by the CI system

+ 

+ # Skip the CI integration bits

+ .cico.pipeline

+ run_tests.sh

+ Makefile

+ 

+ # Skip the README file

+ README.md

+ 

+ # Skip the validator files

+ tests/exclusions.txt

+ tests/validate.py

+ 

+ # Comment out the following when testing the validator

+ tests/diffname.yaml

+ tests/empty.yaml

+ tests/intents.yaml

+ tests/missingstream.yaml

+ tests/module.yaml

+ tests/nodejs.yaml

+ 

file added
+18
@@ -0,0 +1,18 @@ 

+ document: modulemd-defaults

+ version: 1

+ data:

+     module: intents

+     stream: x.y

+     profiles:

+         'x.y': []

+         bar: [baz, snafu]

+     intents:

+         desktop:

+             stream: y.z

+             profiles:

+                 'y.z': [blah]

+                 'x.y': [other]

+         server:

+             stream: x.y

+             profiles:

+                 'x.y': []

@@ -0,0 +1,5 @@ 

+ document: modulemd-defaults

+ version: 1

+ data:

+     module: missingstream

+     stream: 2.5 

\ No newline at end of file

file added
+51
@@ -0,0 +1,51 @@ 

+ ---

+ document: modulemd

+ version: 2

+ data:

+   summary: Javascript runtime

+   description: >-

+     Node.js is a platform built on Chrome''s JavaScript runtime for easily building

+     fast, scalable network applications. Node.js uses an event-driven, non-blocking

+     I/O model that makes it lightweight and efficient, perfect for data-intensive

+     real-time applications that run across distributed devices.

+   license:

+     module:

+     - MIT

+   dependencies:

+   - buildrequires:

+       platform: []

+     requires:

+       platform: []

+   references:

+     community: http://nodejs.org

+     documentation: http://nodejs.org/en/docs

+     tracker: https://github.com/nodejs/node/issues

+   profiles:

+     default:

+       rpms:

+       - nodejs

+       - npm

+     development:

+       rpms:

+       - nodejs

+       - nodejs-devel

+       - npm

+     minimal:

+       rpms:

+       - nodejs

+   api:

+     rpms:

+     - nodejs

+     - nodejs-devel

+     - npm

+   components:

+     rpms:

+       libuv:

+         rationale: Platform abstraction layer for Node.js

+         ref: 1.20

+         buildorder: 0

+       nodejs:

+         rationale: Javascript runtime and npm package manager.

+         ref: 10

+         buildorder: 10

+ ...

file added
+8
@@ -0,0 +1,8 @@ 

+ document: modulemd-defaults

+ version: 1

+ data:

+     module: nodejs

+     profiles:

+         6: [different]

+         8: [default]

+         9: [default]

file added
+140
@@ -0,0 +1,140 @@ 

+ #!/usr/bin/python3

+ 

+ 

+ import gi

+ import git

+ import os

+ import sys

+ 

+ from gi.repository import GLib

+ gi.require_version('Modulemd', '1.0')

+ from gi.repository import Modulemd

+ 

+ def do_validate(filename):

+     # Valid filenames must end in ".yaml" to be properly included by Pungi

+     if not filename.endswith(".yaml"):

+         print("%s does not end with .yaml. It will not be included by "

+               "Pungi. If this file does not contain defaults, it should "

+               "be added to the tests/exclusions.txt file." % filename,

+               file=sys.stderr,)

+         return False, None

+ 

+     # The files must parse correctly

+     try:

+         (objects, failures) = Modulemd.objects_from_file_ext(filename)

+     except GLib.Error as e:

+         print(e.message, file=sys.stderr)

+         return False, None

+ 

+     if (failures):

+         for failure in failures:

+             print("Failed subdocument (%s): \n%s\n", failure.message,

+                   failure.doc, file=sys.stderr)

+         return False, None

+ 

+     # There must be exactly one object per file

+     if len(objects) != 1:

+         print("There must be exactly one subdocument", file=sys.stderr)

+         return False, None

+ 

+     # The files must have exactly one Modulemd.Defaults object

+     try:

+         (default, def_failures) = \

+             Modulemd.Defaults.new_from_file_ext (filename)

+     except GLib.Error as e:

+         print(e.message, file=sys.stderr)

+         return False, None

+ 

+     # Filenames must match their contents

+     expected_name = os.path.basename(filename).rsplit('.', maxsplit=1)[0]

+ 

+     if expected_name != default.props.module_name:

+         print("Module name \"%s\" doesn't match filename \"%s.yaml\"" % (

+               default.props.module_name, expected_name),

+               file=sys.stderr)

+         return False, None

+ 

+     if default.props.default_stream:

+         # Default streams must also appear in the profiles list

+         if not default.props.default_stream in default.props.profile_defaults:

+             print("Stream '%s' missing from profiles" % (

+                   default.props.default_stream),

+                   file=sys.stderr)

+             return False, None

+ 

+     # Modules in Fedora must not specify "Intents"

+     try:

+         if (default.props.intents):

+             print ('Module name "%s" includes the intents section, which is '

+                    'not currently permitted in Fedora.' % (

+                        default.props.module_name),

+                    file=sys.stderr)

+             return False, None

+     except AttributeError:

+         # The available version of libmodulemd does not support intents

+         pass

+ 

+     print("%s is valid" % filename)

+     return (True, default)

+ 

+ 

+ def main():

+     result = os.EX_OK

+ 

+     script_dir = os.path.dirname(os.path.realpath(__file__))

+ 

+     # Get the repo we're running in

+     repo = git.Repo(script_dir,

+                     search_parent_directories=True)

+ 

+     # Get the list of files in this repository

+     files = [x for (x, y) in repo.index.entries.keys()]

+ 

+     # Get the list of excluded files

+     exclusions = []

+     with open(os.path.join(script_dir, "exclusions.txt")) as f:

+         for line in f:

+             stripped_line = line.strip()

+             if not stripped_line or stripped_line.startswith("#"):

+                 # Ignore comments and empty lines

+                 continue

+             exclusions.append(line.strip())

+ 

+     # Validate all of the files

+     prioritizer = Modulemd.Prioritizer()

+     for file in files:

+         excluded = False

+         for excl in exclusions:

+             if (file.startswith(excl)):

+                 excluded = True

+                 break

+         if not excluded:

+             (valid, obj) = do_validate(file)

+             if not valid:

+                 print("%s failed to validate" % file, file=sys.stderr)

+                 result = os.EX_DATAERR

+             else:

+                 try:

+                     prioritizer.add([obj,], 0)

+                 except GLib.Error as e:

+                     print("Could not merge %s with other defaults: %s" % (

+                           file, e.message))

+                     result = os.EX_DATAERR

+ 

+     if result == os.EX_DATAERR:

+         return result

+ 

+     # For sanity's sake, also do a merge of all the defaults to make sure no

+     # conflicts arise that weren't detected by the above tests. This should be

+     # impossible.

+     try:

+         prioritizer.resolve()

+     except GLib.Error as e:

+         print("Could not merge all defaults: %s" % e.message)

+         result = os.EX_DATAERR

+ 

+ 

+     return result

+ 

+ if __name__ == "__main__":

+     sys.exit(main()) 

\ No newline at end of file

This script should be run on all pull-requests to ensure that no invalid module defaults get merged.

This patch is a precursor to adding CI for pull-requests on this repository. It is being submitted so the validation tests can be reviewed in parallel with the CI work.

Signed-off-by: Stephen Gallagher sgallagh@redhat.com

1 new commit added

  • Add test for intents
6 years ago

1 new commit added

  • start on a pipeline
6 years ago

pretty please pagure-ci rebuild

pretty please pagure-ci rebuild

pretty please pagure-ci rebuild

OK, the test integration now works, finally. Please review and merge, @mohanboddu

rebased onto d05be2a

6 years ago

Hmm, belay that. New errors showing up now.

pretty please pagure-ci rebuild

pretty please pagure-ci rebuild

OK, we just needed to get a couple packages added to the Jenkins node. This is now ready for review.

Pull-Request has been merged by mohanboddu

6 years ago