From c44fa5653192818d14afaa6e72f50e02bf7792ed Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Mar 05 2018 13:52:32 +0000 Subject: Add support for standard test interface The changes in this commit are: - change the file we discover from 'runtask.yml' to 'tests.yml' - check for existance of a repo before attempting to clone it - removal of checks and tests for the scheduling part of old forumulae Fixes https://pagure.io/taskotron/taskotron-trigger/issue/39 --- diff --git a/docs/base-design.rst b/docs/base-design.rst index 6519a94..f552b6d 100644 --- a/docs/base-design.rst +++ b/docs/base-design.rst @@ -78,8 +78,7 @@ only if the package name is in the list of critpath packages. And in the last case, instead of having the task-name hardcoded, the trigger is able to search a git repo of given url (here it is inferred from the data) -for ``runtask.yml`` (the Taskotron formula) files, and schedules those as jobs, -using runners. +for ``tests.yml`` file, and schedules that as a job, using runners. Runners diff --git a/jobtriggers/jobtrigger.py b/jobtriggers/jobtrigger.py index 445f2cc..09b9e08 100644 --- a/jobtriggers/jobtrigger.py +++ b/jobtriggers/jobtrigger.py @@ -71,6 +71,12 @@ class JobTrigger(object): repo = discover['repo'] branch = discover.get('branch', 'master').strip() or 'master' fallback = discover.get('fallback_branch', '').strip() + + # make sure that the repo exists before we try to clone it + if not utils.git_repo_exists(repo): + self.log.warn("Repo {} does not exist".format(repo)) + continue + repo_dir, branch = utils.clone_repo(repo, branch, fallback) # append base_dir, if specified diff --git a/jobtriggers/utils.py b/jobtriggers/utils.py index 08b0df0..d2e73a3 100644 --- a/jobtriggers/utils.py +++ b/jobtriggers/utils.py @@ -5,6 +5,7 @@ import csv import datetime import shutil import koji +import urllib2 import yaml @@ -29,14 +30,10 @@ def parse_yaml_from_file(filename): def _should_schedule(formula_file, item_type): - formula = parse_yaml_from_file(formula_file) - try: - run = formula['scheduling']['run'] - except KeyError: - run = True - - return run and item_type in formula['input']['args'] + # FIXME: it's not clear whether we want to keep this feature as we move + # away from custom YAML formulae. Leaving this in as a stub for now. + return True def get_distgit_branch(release): matches = re.findall(r'\.(fc\d{1,2})\.?', release) @@ -119,8 +116,8 @@ def _discover_formulae(directory, recursive=False): directory = directory.strip().rstrip('/') for dirpath, dirs, files in os.walk(directory): - if 'runtask.yml' in files: - formula = os.path.join(dirpath, 'runtask.yml') + if 'tests.yml' in files: + formula = os.path.join(dirpath, 'tests.yml') # remove the base directory path formulae.append(formula[len(directory)+1:]) if not recursive: @@ -140,3 +137,16 @@ def discover_tasks(directory, item_type, recursive=False): tasks = [f for f in formulae if _should_schedule(os.path.join(directory, f), item_type)] return tasks + +def git_repo_exists(repo): + """Attempts to open a connection to the repo parameter. Returns False if + there were issues in connecting, otherwise returns True""" + try: + ret = urllib2.urlopen(repo) + except urllib2.URLError: + return False + + if ret.code != 200: + return False + + return True diff --git a/setup.py b/setup.py index ab64adc..9bd8205 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ moksha_consumer = ( setup( name='jobtriggers', - version='0.4.9', + version='0.5.0', description='triggering jobs via fedmsg', author='Tim Flink', author_email='tflink@fedoraproject.org', diff --git a/taskotron-trigger.spec b/taskotron-trigger.spec index 3428f0d..bbbaaf8 100644 --- a/taskotron-trigger.spec +++ b/taskotron-trigger.spec @@ -1,7 +1,7 @@ Name: taskotron-trigger # NOTE: if you update version, *make sure* to also update `setup.py` -Version: 0.4.9 -Release: 2%{?dist} +Version: 0.5.0 +Release: 1%{?dist} Summary: Triggering Taskotron jobs via fedmsg License: GPLv2+ @@ -86,6 +86,9 @@ install -d %{buildroot}%{_sharedstatedir}/taskotron-trigger/ %config(noreplace) %{_sysconfdir}/logrotate.d/taskotron-trigger %changelog +* Mon Mar 05 2018 Kamil Páral - 0.5.0-1 +- support Standard Test Interface tasks instead of custom libtaskotron formulae + * Fri Feb 02 2018 Frantisek Zatloukal - 0.4.9-2 - Update Python 2 dependency declarations to new packaging standards (See https://fedoraproject.org/wiki/FinalizingFedoraSwitchtoPython3) diff --git a/testing/test_jobtrigger.py b/testing/test_jobtrigger.py index 8b785e8..20bc34f 100644 --- a/testing/test_jobtrigger.py +++ b/testing/test_jobtrigger.py @@ -51,6 +51,11 @@ class TestJobtrigger(): jobtrigger.config.valid_arches = self.ref_default_arches + # short-circuit the logic to check if a repo exists + stub_repoexist = mock.Mock(return_value=True) + monkeypatch.setattr(utils, 'git_repo_exists', stub_repoexist) + + def test_load_rules(self): template = """$in: [${foo}, ${in}]""" data = {"foo": "bar", "in": "out"} @@ -168,3 +173,23 @@ class TestJobtrigger(): runner_calls = self.helper.runner.trigger_job.calls() assert len(runner_calls) == 0 + + def test_discover_repo_exists(self, monkeypatch): + """Make sure that when a repo does not exist, a clone is not attempted""" + + stub_clone_repo = mock.Mock(return_value=('/tmp/path_to_repo', 'master')) + monkeypatch.setattr(utils, 'clone_repo', stub_clone_repo) + + stub_repo_exist = mock.Mock(return_value=False) + monkeypatch.setattr(utils, 'git_repo_exists', stub_repo_exist) + + mock__load_rules = mock.Mock(return_value=self.ref_rules) + monkeypatch.setattr(self.helper, '_load_rules', mock__load_rules) + + rule = {'do': [{'discover': {'repo': 'http://bogus.repo/${name}.git'}}], + 'when': {'message_type': 'DiscoverTasks'}} + + self.helper._tasks_from_rule(rule, self.ref_message_data) + + stub_repo_exist.assert_called() + stub_clone_repo.assert_not_called() diff --git a/testing/test_koji_build_trigger.py b/testing/test_koji_build_trigger.py index 415f9d5..a8ed649 100644 --- a/testing/test_koji_build_trigger.py +++ b/testing/test_koji_build_trigger.py @@ -74,6 +74,10 @@ class TestKojiBuildCompletedJobConsumer(): stub_get_session = mock.MagicMock(return_value=stub_session) monkeypatch.setattr(utils, '_get_koji_session', stub_get_session) + # short-circuit the logic to check if a repo exists + stub_repoexist = mock.Mock(return_value=True) + monkeypatch.setattr(utils, 'git_repo_exists', stub_repoexist) + def _create_msg(self, ref_instance, ref_new, ref_name, ref_release, ref_version, ref_owner): msg = Munch(body='{"i": 1,\ "msg": {"build_id": 221146,\ diff --git a/testing/test_utils.py b/testing/test_utils.py index 9f7f2f5..3fa5987 100644 --- a/testing/test_utils.py +++ b/testing/test_utils.py @@ -46,42 +46,6 @@ class TestUtils(): with mock.patch('__builtin__.open', mocked_open): assert utils._should_schedule('mocked_filename', 'koji_build') - def test_check__should_schedule_false(self): - file_data = ''' - scheduling: - run: False - - input: - args: - - koji_build - ''' - mocked_open = mock.mock_open(read_data=file_data) - - with mock.patch('__builtin__.open', mocked_open): - assert not utils._should_schedule('mocked_filename', 'koji_build') - - def test_check__should_schedule_default(self): - file_data = ''' - input: - args: - - koji_build - ''' - mocked_open = mock.mock_open(read_data=file_data) - - with mock.patch('__builtin__.open', mocked_open): - assert utils._should_schedule('mocked_filename', 'koji_build') - - def test_check__should_schedule_false_item_type(self): - file_data = ''' - input: - args: - - koji_tag - ''' - mocked_open = mock.mock_open(read_data=file_data) - - with mock.patch('__builtin__.open', mocked_open): - assert not utils._should_schedule('mocked_filename', 'koji_build') - def test_clone_repo(self, monkeypatch): mock_path_exists = mock.Mock(return_value=False) mock_check_output = mock.Mock(return_value='') @@ -131,19 +95,19 @@ class TestUtils(): def test__discover_formulae(self, monkeypatch): def mock_os_walk(directory): - yield (directory, ['task_1', 'nontask'], ['runtask.yml']) + yield (directory, ['task_1', 'nontask'], ['tests.yml']) yield ('%s/nontask' % directory, [], []) - yield ('%s/task_1' % directory, [], ['runtask.yml']) + yield ('%s/task_1' % directory, [], ['tests.yml']) monkeypatch.setattr(os, 'walk', mock_os_walk) formulae = utils._discover_formulae(' /path/to/ ', recursive=True) assert len(formulae) == 2 - assert formulae == ['runtask.yml', 'task_1/runtask.yml'] + assert formulae == ['tests.yml', 'task_1/tests.yml'] formulae = utils._discover_formulae('/path/to') assert len(formulae) == 1 - assert formulae == ['runtask.yml'] + assert formulae == ['tests.yml'] def test_discover_tasks(self, monkeypatch): mock__discover_formulae = mock.Mock(return_value=['task1.yml', 'task2.yml'])