From 0fa102feb9a55bc5a2f5f6717d9107568b7b1e59 Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Sep 21 2017 20:19:04 +0000 Subject: [PATCH 1/2] Adding support for standard test roles. Fixes #39 The changes in this commit are: - change the file we discover from 'runtask.yml' to 'tests.yml' - check for existance of a repo before attempting to clone it - removal of checks and tests for the scheduling part of old forumulae --- diff --git a/jobtriggers/jobtrigger.py b/jobtriggers/jobtrigger.py index 445f2cc..037cb4a 100644 --- a/jobtriggers/jobtrigger.py +++ b/jobtriggers/jobtrigger.py @@ -71,6 +71,12 @@ class JobTrigger(object): repo = discover['repo'] branch = discover.get('branch', 'master').strip() or 'master' fallback = discover.get('fallback_branch', '').strip() + + # make sure that the repo exists before we try to clone it + if not utils.git_repo_exists(repo): + self.log.info("Repo {} does not exist".format(repo)) + continue + repo_dir, branch = utils.clone_repo(repo, branch, fallback) # append base_dir, if specified diff --git a/jobtriggers/utils.py b/jobtriggers/utils.py index 08b0df0..a9c858e 100644 --- a/jobtriggers/utils.py +++ b/jobtriggers/utils.py @@ -5,6 +5,7 @@ import csv import datetime import shutil import koji +import urllib2 import yaml @@ -29,14 +30,10 @@ def parse_yaml_from_file(filename): def _should_schedule(formula_file, item_type): - formula = parse_yaml_from_file(formula_file) - try: - run = formula['scheduling']['run'] - except KeyError: - run = True - - return run and item_type in formula['input']['args'] + # it's not clear whether we want to keep this feature as we move away from + # custom YAML formulae. Leaving this in as a stub for now. + return True def get_distgit_branch(release): matches = re.findall(r'\.(fc\d{1,2})\.?', release) @@ -119,8 +116,8 @@ def _discover_formulae(directory, recursive=False): directory = directory.strip().rstrip('/') for dirpath, dirs, files in os.walk(directory): - if 'runtask.yml' in files: - formula = os.path.join(dirpath, 'runtask.yml') + if 'tests.yml' in files: + formula = os.path.join(dirpath, 'tests.yml') # remove the base directory path formulae.append(formula[len(directory)+1:]) if not recursive: @@ -137,6 +134,22 @@ def discover_tasks(directory, item_type, recursive=False): Only formulae that are relevant for the item_type are returned. """ formulae = _discover_formulae(directory, recursive) + pprint(formulae) tasks = [f for f in formulae if _should_schedule(os.path.join(directory, f), item_type)] return tasks + +def git_repo_exists(repo): + """Attempts to open a connection to the repo parameter. Returns False if + there were issues in connecting, otherwise returns True""" + try: + ret = urllib2.urlopen(repo) + except urllib2.URLError: + return False + + if ret.code != 200: + return False + + return True + + diff --git a/testing/test_jobtrigger.py b/testing/test_jobtrigger.py index 8b785e8..20bc34f 100644 --- a/testing/test_jobtrigger.py +++ b/testing/test_jobtrigger.py @@ -51,6 +51,11 @@ class TestJobtrigger(): jobtrigger.config.valid_arches = self.ref_default_arches + # short-circuit the logic to check if a repo exists + stub_repoexist = mock.Mock(return_value=True) + monkeypatch.setattr(utils, 'git_repo_exists', stub_repoexist) + + def test_load_rules(self): template = """$in: [${foo}, ${in}]""" data = {"foo": "bar", "in": "out"} @@ -168,3 +173,23 @@ class TestJobtrigger(): runner_calls = self.helper.runner.trigger_job.calls() assert len(runner_calls) == 0 + + def test_discover_repo_exists(self, monkeypatch): + """Make sure that when a repo does not exist, a clone is not attempted""" + + stub_clone_repo = mock.Mock(return_value=('/tmp/path_to_repo', 'master')) + monkeypatch.setattr(utils, 'clone_repo', stub_clone_repo) + + stub_repo_exist = mock.Mock(return_value=False) + monkeypatch.setattr(utils, 'git_repo_exists', stub_repo_exist) + + mock__load_rules = mock.Mock(return_value=self.ref_rules) + monkeypatch.setattr(self.helper, '_load_rules', mock__load_rules) + + rule = {'do': [{'discover': {'repo': 'http://bogus.repo/${name}.git'}}], + 'when': {'message_type': 'DiscoverTasks'}} + + self.helper._tasks_from_rule(rule, self.ref_message_data) + + stub_repo_exist.assert_called() + stub_clone_repo.assert_not_called() diff --git a/testing/test_koji_build_trigger.py b/testing/test_koji_build_trigger.py index 415f9d5..a8ed649 100644 --- a/testing/test_koji_build_trigger.py +++ b/testing/test_koji_build_trigger.py @@ -74,6 +74,10 @@ class TestKojiBuildCompletedJobConsumer(): stub_get_session = mock.MagicMock(return_value=stub_session) monkeypatch.setattr(utils, '_get_koji_session', stub_get_session) + # short-circuit the logic to check if a repo exists + stub_repoexist = mock.Mock(return_value=True) + monkeypatch.setattr(utils, 'git_repo_exists', stub_repoexist) + def _create_msg(self, ref_instance, ref_new, ref_name, ref_release, ref_version, ref_owner): msg = Munch(body='{"i": 1,\ "msg": {"build_id": 221146,\ diff --git a/testing/test_utils.py b/testing/test_utils.py index 9f7f2f5..3fa5987 100644 --- a/testing/test_utils.py +++ b/testing/test_utils.py @@ -46,42 +46,6 @@ class TestUtils(): with mock.patch('__builtin__.open', mocked_open): assert utils._should_schedule('mocked_filename', 'koji_build') - def test_check__should_schedule_false(self): - file_data = ''' - scheduling: - run: False - - input: - args: - - koji_build - ''' - mocked_open = mock.mock_open(read_data=file_data) - - with mock.patch('__builtin__.open', mocked_open): - assert not utils._should_schedule('mocked_filename', 'koji_build') - - def test_check__should_schedule_default(self): - file_data = ''' - input: - args: - - koji_build - ''' - mocked_open = mock.mock_open(read_data=file_data) - - with mock.patch('__builtin__.open', mocked_open): - assert utils._should_schedule('mocked_filename', 'koji_build') - - def test_check__should_schedule_false_item_type(self): - file_data = ''' - input: - args: - - koji_tag - ''' - mocked_open = mock.mock_open(read_data=file_data) - - with mock.patch('__builtin__.open', mocked_open): - assert not utils._should_schedule('mocked_filename', 'koji_build') - def test_clone_repo(self, monkeypatch): mock_path_exists = mock.Mock(return_value=False) mock_check_output = mock.Mock(return_value='') @@ -131,19 +95,19 @@ class TestUtils(): def test__discover_formulae(self, monkeypatch): def mock_os_walk(directory): - yield (directory, ['task_1', 'nontask'], ['runtask.yml']) + yield (directory, ['task_1', 'nontask'], ['tests.yml']) yield ('%s/nontask' % directory, [], []) - yield ('%s/task_1' % directory, [], ['runtask.yml']) + yield ('%s/task_1' % directory, [], ['tests.yml']) monkeypatch.setattr(os, 'walk', mock_os_walk) formulae = utils._discover_formulae(' /path/to/ ', recursive=True) assert len(formulae) == 2 - assert formulae == ['runtask.yml', 'task_1/runtask.yml'] + assert formulae == ['tests.yml', 'task_1/tests.yml'] formulae = utils._discover_formulae('/path/to') assert len(formulae) == 1 - assert formulae == ['runtask.yml'] + assert formulae == ['tests.yml'] def test_discover_tasks(self, monkeypatch): mock__discover_formulae = mock.Mock(return_value=['task1.yml', 'task2.yml']) From 400aa06f17ff8f434c27247fd01b0184ddea7a4c Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Sep 22 2017 03:23:22 +0000 Subject: [PATCH 2/2] removing debug pprint that shouldn't be there --- diff --git a/jobtriggers/utils.py b/jobtriggers/utils.py index a9c858e..4a07e06 100644 --- a/jobtriggers/utils.py +++ b/jobtriggers/utils.py @@ -134,7 +134,6 @@ def discover_tasks(directory, item_type, recursive=False): Only formulae that are relevant for the item_type are returned. """ formulae = _discover_formulae(directory, recursive) - pprint(formulae) tasks = [f for f in formulae if _should_schedule(os.path.join(directory, f), item_type)] return tasks