From 75bdc08a72f1f44d8fd236c1f5428c0af9ffe682 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 07 2017 01:10:32 +0000 Subject: [PATCH 1/12] Adds Analyzer as a new kiskadee model. - Creates the analyzers on database when kiskadee initialize. - Reads the available analyzers from kiskadee.conf --- diff --git a/kiskadee/__init__.py b/kiskadee/__init__.py index dfc82f4..aa8844f 100644 --- a/kiskadee/__init__.py +++ b/kiskadee/__init__.py @@ -28,6 +28,10 @@ import configparser import logging import sys +from kiskadee.model import Analyzer, Base +import kiskadee.model +import kiskadee.database + __version__ = '0.1.dev0' _my_path = os.path.dirname(os.path.realpath(__file__)) @@ -104,3 +108,24 @@ else: _debug.setFormatter(formatter) logger.setLevel(logging.DEBUG) logger.addHandler(_debug) +logger.addHandler(_warning) +logger.addHandler(_info) + +database = kiskadee.database.Database() +engine = database.engine +session = database.session +Base.metadata.create_all(engine) +Base.metadata.bind = engine + +def create_analyzers(): + list_of_analyzers = dict(kiskadee.config._sections["analyzers"]) + for name, version in list_of_analyzers.items(): + if not (session.query(Analyzer).filter(Analyzer.name == name). + filter(Analyzer.version == version).first()): + new_analyzer = kiskadee.model.Analyzer() + new_analyzer.name = name + new_analyzer.version = version + session.add(new_analyzer) + session.commit() + +create_analyzers() diff --git a/kiskadee/model.py b/kiskadee/model.py index 607f41d..05ca6d9 100644 --- a/kiskadee/model.py +++ b/kiskadee/model.py @@ -1,5 +1,7 @@ """This module provides kiskadee database model.""" +import kiskadee.database +import kiskadee.model from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import Column, Integer, UnicodeText, UniqueConstraint,\ Sequence, Unicode, ForeignKey, Boolean, orm @@ -46,8 +48,28 @@ class Version(Base): Sequence('versions_id_seq', optional=True), primary_key=True) number = Column(Unicode(100), nullable=False) package_id = Column(Integer, ForeignKey('packages.id'), nullable=False) - has_analysis = Column(Boolean) - analysis = Column(UnicodeText) + analysis = orm.relationship('Analysis', backref='versions') __table_args__ = ( UniqueConstraint('number', 'package_id'), ) + +class Analyzer(Base): + """Abstraction of a static analyzer.""" + + __tablename__ = 'analyzers' + id = Column(Integer, + Sequence('analyzers_id_seq', optional=True), primary_key=True) + name = Column(Unicode(255), nullable=False, unique=True) + version = Column(Unicode(255), nullable=True) + analysis = orm.relationship('Analysis', backref='analyzers') + + +class Analysis(Base): + """Abstraction of a package analysis.""" + + __tablename__ = 'analysis' + id = Column(Integer, + Sequence('analysis_id_seq', optional=True), primary_key=True) + version_id = Column(Integer, ForeignKey('versions.id'), nullable=False) + analyzer_id = Column(Integer, ForeignKey('analyzers.id'), nullable=False) + raw = Column(UnicodeText) diff --git a/util/kiskadee.conf b/util/kiskadee.conf index 323b284..eb48c03 100644 --- a/util/kiskadee.conf +++ b/util/kiskadee.conf @@ -37,3 +37,7 @@ target = example description = SAMATE Juliet test suite analyzers = cppcheck flawfinder active = yes + +[analyzers] +cppcheck = 1.0.0 +flawfinder = 1.0.0 From e5ba52b5b212cae8c6b316c57bcc7f8e7b79951d Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 07 2017 01:10:32 +0000 Subject: [PATCH 2/12] Save analysis after run analyzer. --- diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py index 09ed83e..efea517 100644 --- a/kiskadee/monitor.py +++ b/kiskadee/monitor.py @@ -36,7 +36,7 @@ class Monitor: the analysis will never be performed. You can use thee decorator `@kiskadee.queue.package_enqueuer` to easiliy enqueue a package. """ - database = kiskadee.database.Database() + database = kiskadee.database self.engine = database.engine self.session = database.session Base.metadata.create_all(self.engine) @@ -86,8 +86,7 @@ class Monitor: _package = Package(name=pkg['name'], plugin_id=_plugin.id) _version = Version(number=pkg['version'], - package_id=_package.id, - has_analysis=False) + package_id=_package.id) _package.versions.append(_version) self.session.add(_package) kiskadee.logger.debug("Saving package in db: {}".format(str(pkg))) @@ -106,8 +105,7 @@ class Monitor: if(pkg['plugin'].Plugin(). compare_versions(pkg['version'], current_pkg_version) == 1): _new_version = Version(number=pkg['version'], - package_id=_pkg.id, - has_analysis=False) + package_id=_pkg.id) _pkg.versions.append(_new_version) self.session.add(_pkg) self.session.commit() diff --git a/kiskadee/runner.py b/kiskadee/runner.py index 9bd458f..f083a9e 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -10,21 +10,18 @@ import kiskadee.util import kiskadee.converter running = True - +database = kiskadee.database +engine = database.engine +session = database.session +kiskadee.model.Base.metadata.create_all(engine) +kiskadee.model.Base.metadata.bind = engine def runner(): """Run static analyzers. - Continuously dequeue packages from `analyses_queue` and call the :func:`analyze` method, passing the dequeued package. After the analysis, updates the status of this package on the database. """ - kiskadee.logger.debug("Starting runner component") - database = kiskadee.database.Database() - engine = database.engine - session = database.session - kiskadee.model.Base.metadata.create_all(engine) - kiskadee.model.Base.metadata.bind = engine while running: if not kiskadee.queue.is_empty(): kiskadee.logger.debug('RUNNER: dequeuing...') @@ -33,22 +30,7 @@ def runner(): % (package['name'], package['version'], package['plugin'].__name__)) - analysis_reports = analyze(package) - if analysis_reports: - kiskadee.logger.debug('RUNNER: Saving analysis %s' % - str(package)) - all_analyses = '\n'.join(analysis_reports) - pkg = (session.query(kiskadee.model.Package). - filter(kiskadee.model.Package.name == package['name']). - first()) - pkg.versions[-1].has_analysis = True - pkg.versions[-1].analysis = all_analyses - session.add(pkg) - session.commit() - kiskadee.logger.debug('RUNNER: DONE running analysis') - else: - kiskadee.logger.debug('RUNNER: Something went wrong') - kiskadee.logger.debug('RUNNER: could not generate analysis') + analyze(package) def analyze(package): @@ -67,6 +49,9 @@ def analyze(package): 'source...'.format(package['name']) ) compressed_source = plugin.get_sources(package) + pkg = (session.query(kiskadee.model.Package). + filter(kiskadee.model.Package.name == package['name']). + first()) if compressed_source: kiskadee.logger.debug('ANALYSIS: Downloaded!') kiskadee.logger.debug('ANALYSIS: Unpacking...') @@ -80,18 +65,37 @@ def analyze(package): kiskadee.logger.debug('ANALYSIS: running %s ...' % analyzer) try: analysis = kiskadee.analyzers.run(analyzer, path) - firehose_report = kiskadee.converter.to_firehose( - analysis, analyzer - ) - reports.append(str(firehose_report)) + firehose_report = kiskadee.converter.to_firehose(analysis, + analyzer) + analysed_version = pkg.versions[-1].id kiskadee.logger.debug( - 'ANALYSIS: DONE running %s' % analyzer + "Saving analysis of {} on {} version {}" + .format(analyzer, pkg.name, pkg.versions[-1].number) + ) + self._save_source_analysis( + analysed_version, firehose_report, analyzer ) - except: kiskadee.logger.debug( - 'ERROR: Could not run analysis inside container' + 'ANALYSIS: DONE {} analysis'.format(analyzer) ) + except Exception as err: + kiskadee.logger.debug('RUNNER: could not generate analysis') + kiskadee.logger.debug(err) # TODO: remove compressed/uncompressed files after the analysis - return reports else: kiskadee.logger.debug('RUNNER: invalid source dict') + +def _save_source_analysis(self, version_id, analysis, analyzer): + _analysis = kiskadee.model.Analysis() + try: + _analyzer = session.query(Analyzers).\ + filter(Analyzer.name == analyzer).first() + _analysis.analyzer_id = _analyzer.id + _analysis.version_id = version_id + _analysis.raw = analysis + session.add(_analysis) + except Exception as err: + kiskadee.logger.debug( + "The required analyzer was not registered in kiskadee" + ) + kiskadee.logger.debug(err) From 407c97c0adff0d0e4d2bf9a42f9cf09bab6474d1 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 07 2017 01:10:32 +0000 Subject: [PATCH 3/12] Fix how the analysis has been saved. --- diff --git a/kiskadee/__init__.py b/kiskadee/__init__.py index aa8844f..17b2b30 100644 --- a/kiskadee/__init__.py +++ b/kiskadee/__init__.py @@ -108,8 +108,6 @@ else: _debug.setFormatter(formatter) logger.setLevel(logging.DEBUG) logger.addHandler(_debug) -logger.addHandler(_warning) -logger.addHandler(_info) database = kiskadee.database.Database() engine = database.engine diff --git a/kiskadee/converter.py b/kiskadee/converter.py index 3bf16ae..b24819e 100644 --- a/kiskadee/converter.py +++ b/kiskadee/converter.py @@ -33,7 +33,7 @@ def to_firehose(bytes_input, analyzer): if (analyzer_module): with open(file_to_parse, 'r') as f: analysis_instance = analyzer_module.parse_file(f) - firehose_tree = analysis_instance.to_xml_bytes() + firehose_tree = str(analysis_instance.to_xml_bytes()) shutil.rmtree(tempdir) diff --git a/kiskadee/runner.py b/kiskadee/runner.py index f083a9e..f665508 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -13,8 +13,6 @@ running = True database = kiskadee.database engine = database.engine session = database.session -kiskadee.model.Base.metadata.create_all(engine) -kiskadee.model.Base.metadata.bind = engine def runner(): """Run static analyzers. @@ -33,6 +31,7 @@ def runner(): analyze(package) + def analyze(package): """Run each analyzer on a package. @@ -69,12 +68,13 @@ def analyze(package): analyzer) analysed_version = pkg.versions[-1].id kiskadee.logger.debug( - "Saving analysis of {} on {} version {}" + "Saving analysis of {} on package {}-{}" .format(analyzer, pkg.name, pkg.versions[-1].number) ) - self._save_source_analysis( + _save_source_analysis( analysed_version, firehose_report, analyzer ) + session.commit() kiskadee.logger.debug( 'ANALYSIS: DONE {} analysis'.format(analyzer) ) @@ -85,11 +85,11 @@ def analyze(package): else: kiskadee.logger.debug('RUNNER: invalid source dict') -def _save_source_analysis(self, version_id, analysis, analyzer): +def _save_source_analysis(version_id, analysis, analyzer): _analysis = kiskadee.model.Analysis() try: - _analyzer = session.query(Analyzers).\ - filter(Analyzer.name == analyzer).first() + _analyzer = session.query(kiskadee.model.Analyzer).\ + filter(kiskadee.model.Analyzer.name == analyzer).first() _analysis.analyzer_id = _analyzer.id _analysis.version_id = version_id _analysis.raw = analysis From a1facae36c86e2114b69a6fc8b5057d196cbc95c Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 07 2017 01:11:00 +0000 Subject: [PATCH 4/12] Refactors analyze method. - The analyze method was to big, and was difficult to read. This commit breaks the method in other two, in order to facilitate it's behavior. - Creates _path_to_uncompressed_source method, responsible to download the source, uncompress it and return it's absolute path. --- diff --git a/kiskadee/runner.py b/kiskadee/runner.py index f665508..1395385 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -20,72 +20,63 @@ def runner(): :func:`analyze` method, passing the dequeued package. After the analysis, updates the status of this package on the database. """ + kiskadee.logger.debug('Starting runner component') while running: if not kiskadee.queue.is_empty(): kiskadee.logger.debug('RUNNER: dequeuing...') - package = kiskadee.queue.dequeue_analysis() + source_to_analysis = kiskadee.queue.dequeue_analysis() kiskadee.logger.debug('RUNNER: dequeued %s-%s from %s' - % (package['name'], - package['version'], - package['plugin'].__name__)) - analyze(package) + % (source_to_analysis['name'], + source_to_analysis['version'], + source_to_analysis['plugin'].__name__)) + source_name = source_to_analysis['name'] + pkg = ( + session.query( kiskadee.model.Package) + .filter(kiskadee.model.Package.name == source_name) + .first() + ) + analyze(pkg, source_to_analysis) +def analyze(pkg, source_to_analysis): + """Run each analyzer on a source_to_analysis. - -def analyze(package): - """Run each analyzer on a package. - - The package dict is in the queue. The keys are: + The source_to_analysis dict is in the queue. The keys are: plugin: the plugin module itself name: the package name version: the package version path: plugin default path for packages return: list with firehose reports """ - plugin = package['plugin'].Plugin() - kiskadee.logger.debug( - 'ANALYSIS: Downloading {} ' - 'source...'.format(package['name']) - ) - compressed_source = plugin.get_sources(package) - pkg = (session.query(kiskadee.model.Package). - filter(kiskadee.model.Package.name == package['name']). - first()) - if compressed_source: - kiskadee.logger.debug('ANALYSIS: Downloaded!') - kiskadee.logger.debug('ANALYSIS: Unpacking...') - reports = [] - path = tempfile.mkdtemp() - shutil.unpack_archive(compressed_source, path) - with kiskadee.util.chdir(path): - kiskadee.logger.debug('ANALYSIS: Unpacked!') - analyzers = plugin.analyzers() - for analyzer in analyzers: - kiskadee.logger.debug('ANALYSIS: running %s ...' % analyzer) - try: - analysis = kiskadee.analyzers.run(analyzer, path) - firehose_report = kiskadee.converter.to_firehose(analysis, + plugin = source_to_analysis['plugin'].Plugin() + source_path = _path_to_uncompressed_source(source_to_analysis, plugin) + + if source_path is None: + return None + + with kiskadee.helpers.chdir(source_path): + analyzers = plugin.analyzers() + for analyzer in analyzers: + kiskadee.logger.debug('ANALYSIS: running {} ...'.format(analyzer)) + try: + analysis = kiskadee.analyzers.run(analyzer, source_path) + firehose_report = kiskadee.converter.to_firehose(analysis, analyzer) - analysed_version = pkg.versions[-1].id - kiskadee.logger.debug( - "Saving analysis of {} on package {}-{}" - .format(analyzer, pkg.name, pkg.versions[-1].number) - ) - _save_source_analysis( - analysed_version, firehose_report, analyzer - ) - session.commit() - kiskadee.logger.debug( - 'ANALYSIS: DONE {} analysis'.format(analyzer) - ) - except Exception as err: - kiskadee.logger.debug('RUNNER: could not generate analysis') - kiskadee.logger.debug(err) - # TODO: remove compressed/uncompressed files after the analysis - else: - kiskadee.logger.debug('RUNNER: invalid source dict') + kiskadee.logger.debug( + "Saving analysis of {} on package {}-{}" + .format(analyzer, pkg.name, pkg.versions[-1].number) + ) + _save_source_analysis(pkg, firehose_report, analyzer) + session.commit() + kiskadee.logger.debug( + 'ANALYSIS: DONE {} analysis'.format(analyzer) + ) + except Exception as err: + kiskadee.logger.debug('RUNNER: could not generate analysis') + kiskadee.logger.debug(err) + # TODO: remove compressed/uncompressed files after the analysis -def _save_source_analysis(version_id, analysis, analyzer): +def _save_source_analysis(package, analysis, analyzer): + version_id = package.versions[-1].id _analysis = kiskadee.model.Analysis() try: _analyzer = session.query(kiskadee.model.Analyzer).\ @@ -99,3 +90,21 @@ def _save_source_analysis(version_id, analysis, analyzer): "The required analyzer was not registered in kiskadee" ) kiskadee.logger.debug(err) + + +def _path_to_uncompressed_source(package, plugin): + kiskadee.logger.debug( + 'ANALYSIS: Downloading {} ' + 'source...'.format(package['name']) + ) + compressed_source = plugin.get_sources(package) + if compressed_source: + kiskadee.logger.debug('ANALYSIS: Downloaded!') + kiskadee.logger.debug('ANALYSIS: Unpacking...') + path = tempfile.mkdtemp() + shutil.unpack_archive(compressed_source, path) + kiskadee.logger.debug('ANALYSIS: Unpacked!') + return path + else: + kiskadee.logger.debug('RUNNER: invalid compressed source') + return None From b8f44632b31034fa3469b88ffb4a19733d70a13c Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 07 2017 01:11:31 +0000 Subject: [PATCH 5/12] Decrease the analyze complexity. - Now we iterate over the plugin's analyzers in a specific method. This will help us to implement more atomic tests, and will decrease the complexity of the analyze method. The only objective of the analyze method is to call the wrapper to Docker, and run a static analyzer on some valid path. --- diff --git a/kiskadee/__init__.py b/kiskadee/__init__.py index 17b2b30..256ee2c 100644 --- a/kiskadee/__init__.py +++ b/kiskadee/__init__.py @@ -115,7 +115,7 @@ session = database.session Base.metadata.create_all(engine) Base.metadata.bind = engine -def create_analyzers(): +def _create_analyzers(): list_of_analyzers = dict(kiskadee.config._sections["analyzers"]) for name, version in list_of_analyzers.items(): if not (session.query(Analyzer).filter(Analyzer.name == name). @@ -126,4 +126,4 @@ def create_analyzers(): session.add(new_analyzer) session.commit() -create_analyzers() +_create_analyzers() diff --git a/kiskadee/runner.py b/kiskadee/runner.py index 1395385..ca0b9bb 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -14,6 +14,7 @@ database = kiskadee.database engine = database.engine session = database.session + def runner(): """Run static analyzers. Continuously dequeue packages from `analyses_queue` and call the @@ -29,15 +30,28 @@ def runner(): % (source_to_analysis['name'], source_to_analysis['version'], source_to_analysis['plugin'].__name__)) - source_name = source_to_analysis['name'] - pkg = ( - session.query( kiskadee.model.Package) - .filter(kiskadee.model.Package.name == source_name) - .first() + + _call_analyzers(source_to_analysis) + + +def _call_analyzers(source_to_analysis): + plugin = source_to_analysis['plugin'].Plugin() + source_path = _path_to_uncompressed_source( + source_to_analysis, plugin ) - analyze(pkg, source_to_analysis) + analyzers = plugin.analyzers() + for analyzer in analyzers: + firehose_report = _analyze( + source_to_analysis, analyzer, source_path + ) + _save_source_analysis( + source_to_analysis, firehose_report, analyzer + ) + + session.commit() + -def analyze(pkg, source_to_analysis): +def _analyze(source_to_analysis, analyzer, source_path): """Run each analyzer on a source_to_analysis. The source_to_analysis dict is in the queue. The keys are: @@ -47,27 +61,16 @@ def analyze(pkg, source_to_analysis): path: plugin default path for packages return: list with firehose reports """ - plugin = source_to_analysis['plugin'].Plugin() - source_path = _path_to_uncompressed_source(source_to_analysis, plugin) - if source_path is None: return None with kiskadee.helpers.chdir(source_path): - analyzers = plugin.analyzers() - for analyzer in analyzers: kiskadee.logger.debug('ANALYSIS: running {} ...'.format(analyzer)) try: analysis = kiskadee.analyzers.run(analyzer, source_path) firehose_report = kiskadee.converter.to_firehose(analysis, analyzer) kiskadee.logger.debug( - "Saving analysis of {} on package {}-{}" - .format(analyzer, pkg.name, pkg.versions[-1].number) - ) - _save_source_analysis(pkg, firehose_report, analyzer) - session.commit() - kiskadee.logger.debug( 'ANALYSIS: DONE {} analysis'.format(analyzer) ) except Exception as err: @@ -75,7 +78,23 @@ def analyze(pkg, source_to_analysis): kiskadee.logger.debug(err) # TODO: remove compressed/uncompressed files after the analysis -def _save_source_analysis(package, analysis, analyzer): + +def _save_source_analysis(source_to_analysis, analysis, analyzer): + + if analysis is None: + return None + + source_name = source_to_analysis['name'] + source_version = source_to_analysis['version'] + + kiskadee.logger.debug( + "Saving analysis of {} on package {}-{}" + .format(analyzer, source_name, source_version) + ) + package = ( + session.query(kiskadee.model.Package) + .filter(kiskadee.model.Package.name == source_name).first() + ) version_id = package.versions[-1].id _analysis = kiskadee.model.Analysis() try: From 8ca21cc05959777b4a8c72f9bb69119fec76ab68 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 07 2017 01:11:33 +0000 Subject: [PATCH 6/12] Fix test_run_analyzer test. - This test it's a simple exemple of how the analyze method should be called now. Following the unique responsability principle, the analyze method receive the analyzer to use, the path to the source, and the meta informations about the package. --- diff --git a/kiskadee/runner.py b/kiskadee/runner.py index ca0b9bb..f37d015 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -73,6 +73,7 @@ def _analyze(source_to_analysis, analyzer, source_path): kiskadee.logger.debug( 'ANALYSIS: DONE {} analysis'.format(analyzer) ) + return firehose_report except Exception as err: kiskadee.logger.debug('RUNNER: could not generate analysis') kiskadee.logger.debug(err) diff --git a/kiskadee/tests/test_analyzers.py b/kiskadee/tests/test_analyzers.py index d9fd3f7..fa0c64c 100644 --- a/kiskadee/tests/test_analyzers.py +++ b/kiskadee/tests/test_analyzers.py @@ -1,8 +1,8 @@ import os from unittest import TestCase -from kiskadee.runner import analyze -import kiskadee.plugins.debian +from kiskadee.runner import _analyze, _path_to_uncompressed_source +import kiskadee.plugins.example class TestAnalyzers(TestCase): @@ -16,18 +16,14 @@ class TestAnalyzers(TestCase): def test_run_analyzer(self): - def mock_get_sources(arg1, arg2): - base_path = os.path.dirname(os.getcwd()) - return ''.join([base_path, - '/kiskadee/kiskadee/tests/test_source/' - 'test_source.tar.gz']) - - kiskadee.plugins.debian.Plugin.get_sources = mock_get_sources - self.deb_pkg = {'name': 'test', - 'version': '1.0.0', - 'plugin': kiskadee.plugins.debian - } - - result = analyze(self.deb_pkg) - self.assertTrue(isinstance(result, list)) - self.assertTrue(len(result) == 1) + source_to_analysis = { + 'name': 'test', + 'version': '1.0.0', + 'plugin': kiskadee.plugins.example + } + + source_path = _path_to_uncompressed_source( + source_to_analysis, kiskadee.plugins.example.Plugin() + ) + firehose_report = _analyze(self.deb_pkg, "cppcheck", source_path) + self.assertIsNotNone(firehose_report) From 5b6aa31d2eac06f6a7ed6ceb65b2d0a01a885ec5 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 07 2017 01:11:33 +0000 Subject: [PATCH 7/12] Fix tests --- diff --git a/kiskadee/__init__.py b/kiskadee/__init__.py index 256ee2c..9e62996 100644 --- a/kiskadee/__init__.py +++ b/kiskadee/__init__.py @@ -115,6 +115,7 @@ session = database.session Base.metadata.create_all(engine) Base.metadata.bind = engine + def _create_analyzers(): list_of_analyzers = dict(kiskadee.config._sections["analyzers"]) for name, version in list_of_analyzers.items(): @@ -126,4 +127,5 @@ def _create_analyzers(): session.add(new_analyzer) session.commit() + _create_analyzers() diff --git a/kiskadee/model.py b/kiskadee/model.py index 05ca6d9..f071742 100644 --- a/kiskadee/model.py +++ b/kiskadee/model.py @@ -1,10 +1,8 @@ """This module provides kiskadee database model.""" -import kiskadee.database -import kiskadee.model from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import Column, Integer, UnicodeText, UniqueConstraint,\ - Sequence, Unicode, ForeignKey, Boolean, orm + Sequence, Unicode, ForeignKey, orm Base = declarative_base() @@ -53,6 +51,7 @@ class Version(Base): UniqueConstraint('number', 'package_id'), ) + class Analyzer(Base): """Abstraction of a static analyzer.""" diff --git a/kiskadee/runner.py b/kiskadee/runner.py index f37d015..ae1dcb1 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -17,6 +17,7 @@ session = database.session def runner(): """Run static analyzers. + Continuously dequeue packages from `analyses_queue` and call the :func:`analyze` method, passing the dequeued package. After the analysis, updates the status of this package on the database. diff --git a/kiskadee/tests/test_analyzers.py b/kiskadee/tests/test_analyzers.py index fa0c64c..cf1d146 100644 --- a/kiskadee/tests/test_analyzers.py +++ b/kiskadee/tests/test_analyzers.py @@ -1,4 +1,3 @@ -import os from unittest import TestCase from kiskadee.runner import _analyze, _path_to_uncompressed_source diff --git a/kiskadee/tests/test_model.py b/kiskadee/tests/test_model.py index c81a5a8..5ef7db6 100644 --- a/kiskadee/tests/test_model.py +++ b/kiskadee/tests/test_model.py @@ -11,14 +11,22 @@ class TestModel(TestCase): Session = sessionmaker(bind=self.engine) self.session = Session() model.Base.metadata.create_all(self.engine) + self.analyzer = model.Analyzer(name="cppcheck", version="1.0.0") self.plugin = model.Plugin(name='kiskadee-plugin', target='university') self.package = model.Package(name='python-kiskadee') - self.version = model.Version(number='1.0-rc1', has_analysis=False) + self.version = model.Version(number='1.0-rc1') self.plugin.packages.append(self.package) self.package.versions.append(self.version) self.session.add(self.package) self.session.add(self.plugin) self.session.add(self.version) + self.session.add(self.analyzer) + + self.analysis = model.Analysis( + analyzer_id=self.analyzer.id, + version_id=self.version.id, + raw="" + ) self.session.commit() def tearDown(self): @@ -44,7 +52,7 @@ class TestModel(TestCase): self.assertEqual(len(plugins), 2) def test_add_version_without_package(self): - version = model.Version(number='3.1', has_analysis=False) + version = model.Version(number='3.1') self.session.add(version) with self.assertRaises(exc.IntegrityError): self.session.commit() @@ -64,8 +72,8 @@ class TestModel(TestCase): self.session.commit() def test_unique_version_for_package(self): - package_version_1 = model.Version(number='1.0', has_analysis=False) - package_version_2 = model.Version(number='1.0', has_analysis=False) + package_version_1 = model.Version(number='1.0') + package_version_2 = model.Version(number='1.0') self.package.versions.append(package_version_1) self.package.versions.append(package_version_2) with self.assertRaises(exc.IntegrityError): From 3369e6ffaa47c1f126994a2d58495b915536983c Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 07 2017 01:11:33 +0000 Subject: [PATCH 8/12] Move _create_analyzers method to runner. --- diff --git a/kiskadee/__init__.py b/kiskadee/__init__.py index 9e62996..4cf9997 100644 --- a/kiskadee/__init__.py +++ b/kiskadee/__init__.py @@ -28,7 +28,7 @@ import configparser import logging import sys -from kiskadee.model import Analyzer, Base +from kiskadee.model import Base import kiskadee.model import kiskadee.database @@ -114,18 +114,3 @@ engine = database.engine session = database.session Base.metadata.create_all(engine) Base.metadata.bind = engine - - -def _create_analyzers(): - list_of_analyzers = dict(kiskadee.config._sections["analyzers"]) - for name, version in list_of_analyzers.items(): - if not (session.query(Analyzer).filter(Analyzer.name == name). - filter(Analyzer.version == version).first()): - new_analyzer = kiskadee.model.Analyzer() - new_analyzer.name = name - new_analyzer.version = version - session.add(new_analyzer) - session.commit() - - -_create_analyzers() diff --git a/kiskadee/runner.py b/kiskadee/runner.py index ae1dcb1..0f57153 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -8,6 +8,7 @@ import kiskadee.model import kiskadee.database import kiskadee.util import kiskadee.converter +from kiskadee.model import Analyzer running = True database = kiskadee.database @@ -23,6 +24,7 @@ def runner(): updates the status of this package on the database. """ kiskadee.logger.debug('Starting runner component') + _create_analyzers(session) while running: if not kiskadee.queue.is_empty(): kiskadee.logger.debug('RUNNER: dequeuing...') @@ -129,3 +131,15 @@ def _path_to_uncompressed_source(package, plugin): else: kiskadee.logger.debug('RUNNER: invalid compressed source') return None + + +def _create_analyzers(_session): + list_of_analyzers = dict(kiskadee.config._sections["analyzers"]) + for name, version in list_of_analyzers.items(): + if not (_session.query(Analyzer).filter(Analyzer.name == name). + filter(Analyzer.version == version).first()): + new_analyzer = kiskadee.model.Analyzer() + new_analyzer.name = name + new_analyzer.version = version + _session.add(new_analyzer) + _session.commit() diff --git a/kiskadee/tests/test_analyzers.py b/kiskadee/tests/test_analyzers.py index cf1d146..9ad1a1a 100644 --- a/kiskadee/tests/test_analyzers.py +++ b/kiskadee/tests/test_analyzers.py @@ -2,11 +2,20 @@ from unittest import TestCase from kiskadee.runner import _analyze, _path_to_uncompressed_source import kiskadee.plugins.example +from kiskadee.runner import _create_analyzers +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from kiskadee import model class TestAnalyzers(TestCase): def setUp(self): + self.engine = create_engine('sqlite:///:memory:') + Session = sessionmaker(bind=self.engine) + self.session = Session() + model.Base.metadata.create_all(self.engine) + _create_analyzers(self.session) self.plugin = kiskadee.plugins.debian.Plugin() self.deb_pkg = {'name': 'test', 'version': '1.0.0', diff --git a/kiskadee/tests/test_model.py b/kiskadee/tests/test_model.py index 5ef7db6..48e4d30 100644 --- a/kiskadee/tests/test_model.py +++ b/kiskadee/tests/test_model.py @@ -1,8 +1,10 @@ from unittest import TestCase -from kiskadee import model from sqlalchemy import create_engine, exc from sqlalchemy.orm import sessionmaker +from kiskadee import model +from kiskadee.runner import _create_analyzers + class TestModel(TestCase): @@ -11,7 +13,7 @@ class TestModel(TestCase): Session = sessionmaker(bind=self.engine) self.session = Session() model.Base.metadata.create_all(self.engine) - self.analyzer = model.Analyzer(name="cppcheck", version="1.0.0") + _create_analyzers(self.session) self.plugin = model.Plugin(name='kiskadee-plugin', target='university') self.package = model.Package(name='python-kiskadee') self.version = model.Version(number='1.0-rc1') @@ -20,10 +22,9 @@ class TestModel(TestCase): self.session.add(self.package) self.session.add(self.plugin) self.session.add(self.version) - self.session.add(self.analyzer) self.analysis = model.Analysis( - analyzer_id=self.analyzer.id, + analyzer_id=1, version_id=self.version.id, raw="" ) diff --git a/kiskadee/tests/test_monitor.py b/kiskadee/tests/test_monitor.py index 965c550..205a13a 100644 --- a/kiskadee/tests/test_monitor.py +++ b/kiskadee/tests/test_monitor.py @@ -7,6 +7,7 @@ from kiskadee.queue import enqueue_package from kiskadee.model import Package, Plugin import kiskadee.queue import kiskadee.plugins.debian +from kiskadee.runner import _create_analyzers class TestMonitor(TestCase): @@ -17,6 +18,7 @@ class TestMonitor(TestCase): Session = sessionmaker(bind=self.monitor.engine) self.monitor.session = Session() model.Base.metadata.create_all(self.monitor.engine) + _create_analyzers(self.monitor.session) self.pkg1 = {'name': 'curl', 'version': '7.52.1-5', 'plugin': kiskadee.plugins.debian, From 20dcf0716a98e09f8d1e1f728bafbe6c4fb33a8f Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 07 2017 01:11:33 +0000 Subject: [PATCH 9/12] Remove database inicialization from __init__.py - This was triggering in our tests requisitions to the postgres database. Now the tests do not depends of the development database, but is using sqlite to run the tests. - This commit also moves the process of create a session to the database module. --- diff --git a/kiskadee/__init__.py b/kiskadee/__init__.py index 4cf9997..dfc82f4 100644 --- a/kiskadee/__init__.py +++ b/kiskadee/__init__.py @@ -28,10 +28,6 @@ import configparser import logging import sys -from kiskadee.model import Base -import kiskadee.model -import kiskadee.database - __version__ = '0.1.dev0' _my_path = os.path.dirname(os.path.realpath(__file__)) @@ -108,9 +104,3 @@ else: _debug.setFormatter(formatter) logger.setLevel(logging.DEBUG) logger.addHandler(_debug) - -database = kiskadee.database.Database() -engine = database.engine -session = database.session -Base.metadata.create_all(engine) -Base.metadata.bind = engine diff --git a/kiskadee/database.py b/kiskadee/database.py index f66c705..6f07372 100644 --- a/kiskadee/database.py +++ b/kiskadee/database.py @@ -1,6 +1,7 @@ """Provide kiskadee Database operations.""" import kiskadee +from kiskadee.model import Base from sqlalchemy import create_engine, orm @@ -10,6 +11,8 @@ class Database: def __init__(self): """Return a Database object with SQLAlchemy session and engine.""" self.engine = self._create_engine() + Base.metadata.create_all(self.engine) + Base.metadata.bind = self.engine self.session = self._create_session(self.engine) def _create_engine(self): diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py index efea517..838c889 100644 --- a/kiskadee/monitor.py +++ b/kiskadee/monitor.py @@ -10,7 +10,7 @@ import time import kiskadee.database import kiskadee.runner import kiskadee.queue -from kiskadee.model import Package, Plugin, Version, Base +from kiskadee.model import Package, Plugin, Version RUNNING = True @@ -18,10 +18,9 @@ RUNNING = True class Monitor: """Provide kiskadee monitoring objects.""" - def __init__(self): + def __init__(self, _session): """Return a non initialized Monitor.""" - self.engine = None - self.session = None + self.session = _session def initialize(self): """Start all threads related to the monitoring process. @@ -36,11 +35,6 @@ class Monitor: the analysis will never be performed. You can use thee decorator `@kiskadee.queue.package_enqueuer` to easiliy enqueue a package. """ - database = kiskadee.database - self.engine = database.engine - self.session = database.session - Base.metadata.create_all(self.engine) - Base.metadata.bind = self.engine _start(self.monitor) plugins = kiskadee.load_plugins() for plugin in plugins: @@ -148,9 +142,9 @@ def _start(module, joinable=False, timeout=None): def daemon(): """Entry point to the monitor module.""" # TODO: improve with start/stop system - monitor = Monitor() + session = kiskadee.database.Database().session + monitor = Monitor(session) p = Process(target=monitor.initialize()) p.daemon = True p.start() p.join() - # cleanup goes here diff --git a/kiskadee/runner.py b/kiskadee/runner.py index 0f57153..da8c398 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -11,9 +11,6 @@ import kiskadee.converter from kiskadee.model import Analyzer running = True -database = kiskadee.database -engine = database.engine -session = database.session def runner(): @@ -24,6 +21,7 @@ def runner(): updates the status of this package on the database. """ kiskadee.logger.debug('Starting runner component') + session = kiskadee.database.Database().session _create_analyzers(session) while running: if not kiskadee.queue.is_empty(): @@ -34,10 +32,10 @@ def runner(): source_to_analysis['version'], source_to_analysis['plugin'].__name__)) - _call_analyzers(source_to_analysis) + _call_analyzers(source_to_analysis, session) -def _call_analyzers(source_to_analysis): +def _call_analyzers(source_to_analysis, session): plugin = source_to_analysis['plugin'].Plugin() source_path = _path_to_uncompressed_source( source_to_analysis, plugin @@ -48,7 +46,7 @@ def _call_analyzers(source_to_analysis): source_to_analysis, analyzer, source_path ) _save_source_analysis( - source_to_analysis, firehose_report, analyzer + source_to_analysis, firehose_report, analyzer, session ) session.commit() @@ -83,7 +81,7 @@ def _analyze(source_to_analysis, analyzer, source_path): # TODO: remove compressed/uncompressed files after the analysis -def _save_source_analysis(source_to_analysis, analysis, analyzer): +def _save_source_analysis(source_to_analysis, analysis, analyzer, session): if analysis is None: return None diff --git a/kiskadee/tests/test_monitor.py b/kiskadee/tests/test_monitor.py index 205a13a..115d1c7 100644 --- a/kiskadee/tests/test_monitor.py +++ b/kiskadee/tests/test_monitor.py @@ -13,11 +13,11 @@ from kiskadee.runner import _create_analyzers class TestMonitor(TestCase): def setUp(self): - self.monitor = Monitor() - self.monitor.engine = create_engine('sqlite:///:memory:') - Session = sessionmaker(bind=self.monitor.engine) - self.monitor.session = Session() - model.Base.metadata.create_all(self.monitor.engine) + self.engine = create_engine('sqlite:///:memory:') + Session = sessionmaker(bind=self.engine) + session = Session() + self.monitor = Monitor(session) + model.Base.metadata.create_all(self.engine) _create_analyzers(self.monitor.session) self.pkg1 = {'name': 'curl', 'version': '7.52.1-5', @@ -39,7 +39,7 @@ class TestMonitor(TestCase): def tearDown(self): # model.metadata.drop_all(self.engine) - model.Base.metadata.drop_all(self.monitor.engine) + model.Base.metadata.drop_all(self.engine) def test_dequeue_package(self): enqueue_package(self.pkg1) From 02ec6d7a0a6b3a679093523cde6dbe0c41f05017 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 07 2017 01:11:33 +0000 Subject: [PATCH 10/12] Docker could not start a container. - This is related with selinux permissions. This post helped me to solve this issue https://goo.gl/KXGQhr. - close #23 --- diff --git a/kiskadee/analyzers.py b/kiskadee/analyzers.py index e3875f5..7cf8020 100644 --- a/kiskadee/analyzers.py +++ b/kiskadee/analyzers.py @@ -10,7 +10,7 @@ def run(analyzer, sources): `sources` is the absolute path for the uncompressed package. Returns a analysis results. """ - volume = {sources: {'bind': '/src', 'mode': 'ro'}} + volume = {sources: {'bind': '/src', 'mode': 'Z'}} client = docker.from_env(version='auto') return client.containers.run(analyzer, '/src', volumes=volume, stdout=True, stderr=True, tty=True) From 90413932661bd05e0132fd2ca0c92332f1727943 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 07 2017 01:11:33 +0000 Subject: [PATCH 11/12] Test a package creation. - Now a package can have several versions, and each version can have several analysis from different analyzers. - Rename test_analyzers.py to test_runner.py - Test _save_source_analysis - Test _path_to_uncompressed_source. - Save several analysis to a same package version. - close #13 --- diff --git a/kiskadee/runner.py b/kiskadee/runner.py index da8c398..93e82e2 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -118,18 +118,19 @@ def _path_to_uncompressed_source(package, plugin): 'ANALYSIS: Downloading {} ' 'source...'.format(package['name']) ) - compressed_source = plugin.get_sources(package) - if compressed_source: - kiskadee.logger.debug('ANALYSIS: Downloaded!') - kiskadee.logger.debug('ANALYSIS: Unpacking...') - path = tempfile.mkdtemp() - shutil.unpack_archive(compressed_source, path) - kiskadee.logger.debug('ANALYSIS: Unpacked!') - return path - else: + try: + compressed_source = plugin.get_sources(package) + except Exception as err: kiskadee.logger.debug('RUNNER: invalid compressed source') return None + kiskadee.logger.debug('ANALYSIS: Downloaded!') + kiskadee.logger.debug('ANALYSIS: Unpacking...') + path = tempfile.mkdtemp() + shutil.unpack_archive(compressed_source, path) + kiskadee.logger.debug('ANALYSIS: Unpacked!') + return path + def _create_analyzers(_session): list_of_analyzers = dict(kiskadee.config._sections["analyzers"]) diff --git a/kiskadee/tests/test_analyzers.py b/kiskadee/tests/test_analyzers.py deleted file mode 100644 index 9ad1a1a..0000000 --- a/kiskadee/tests/test_analyzers.py +++ /dev/null @@ -1,37 +0,0 @@ -from unittest import TestCase - -from kiskadee.runner import _analyze, _path_to_uncompressed_source -import kiskadee.plugins.example -from kiskadee.runner import _create_analyzers -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from kiskadee import model - - -class TestAnalyzers(TestCase): - - def setUp(self): - self.engine = create_engine('sqlite:///:memory:') - Session = sessionmaker(bind=self.engine) - self.session = Session() - model.Base.metadata.create_all(self.engine) - _create_analyzers(self.session) - self.plugin = kiskadee.plugins.debian.Plugin() - self.deb_pkg = {'name': 'test', - 'version': '1.0.0', - 'plugin': kiskadee.plugins.debian - } - - def test_run_analyzer(self): - - source_to_analysis = { - 'name': 'test', - 'version': '1.0.0', - 'plugin': kiskadee.plugins.example - } - - source_path = _path_to_uncompressed_source( - source_to_analysis, kiskadee.plugins.example.Plugin() - ) - firehose_report = _analyze(self.deb_pkg, "cppcheck", source_path) - self.assertIsNotNone(firehose_report) diff --git a/kiskadee/tests/test_model.py b/kiskadee/tests/test_model.py index 48e4d30..552c4e0 100644 --- a/kiskadee/tests/test_model.py +++ b/kiskadee/tests/test_model.py @@ -79,3 +79,78 @@ class TestModel(TestCase): self.package.versions.append(package_version_2) with self.assertRaises(exc.IntegrityError): self.session.commit() + + def test_compose_kiskadee_source(self): + _analyzer = self.session.query(model.Analyzer)\ + .filter(model.Analyzer.name == "cppcheck").first() + package = model.Package( + name='bla', + plugin_id=self.plugin.id + ) + package_version = model.Version( + number='1.0.1', + package_id=package.id + ) + + package_analysis = model.Analysis( + raw="<>", + analyzer_id=_analyzer.id, + version_id=package_version.id + ) + + self.plugin.packages.append(package) + package.versions.append(package_version) + package_version.analysis.append(package_analysis) + + self.assertEqual(package.versions[0].analysis[0].raw, "<>") + + def test_save_several_analysis(self): + + _analyzer1 = ( + self.session.query(model.Analyzer) + .filter(model.Analyzer.name == "cppcheck").first() + ) + _analyzer2 = ( + self.session.query(model.Analyzer) + .filter(model.Analyzer.name == "flawfinder").first() + ) + + package = model.Package( + name='bla', + plugin_id=self.plugin.id + ) + package_version = model.Version( + number='1.0.1', + package_id=package.id + ) + + self.plugin.packages.append(package) + package.versions.append(package_version) + + self.session.add(package) + self.session.add(package_version) + self.session.commit() + + package_analysis1 = model.Analysis( + raw="<>", + analyzer_id=_analyzer1.id, + version_id=package_version.id + ) + package_analysis2 = model.Analysis( + raw="><", + analyzer_id=_analyzer2.id, + version_id=package_version.id + ) + + self.session.add(package_analysis1) + self.session.add(package_analysis2) + self.session.commit() + + saved_package = ( + self.session.query(model.Package) + .filter(model.Package.name == 'bla').first() + ) + analysis = saved_package.versions[-1].analysis + self.assertEqual(len(analysis), 2) + self.assertEqual(analysis[0].raw, "<>") + self.assertEqual(analysis[1].raw, "><") diff --git a/kiskadee/tests/test_runner.py b/kiskadee/tests/test_runner.py new file mode 100644 index 0000000..e6bd6fb --- /dev/null +++ b/kiskadee/tests/test_runner.py @@ -0,0 +1,108 @@ +from unittest import TestCase + +from kiskadee.runner import _analyze, _path_to_uncompressed_source +from kiskadee.runner import _save_source_analysis +import kiskadee.plugins.example +from kiskadee.runner import _create_analyzers +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from kiskadee import model + + +class TestAnalyzers(TestCase): + + def setUp(self): + self.engine = create_engine('sqlite:///:memory:') + Session = sessionmaker(bind=self.engine) + self.session = Session() + model.Base.metadata.create_all(self.engine) + _create_analyzers(self.session) + self.plugin = kiskadee.plugins.debian.Plugin() + self.deb_pkg = {'name': 'test', + 'version': '1.0.0', + 'plugin': kiskadee.plugins.debian + } + self.plugin = model.Plugin(name='kiskadee-plugin', target='university') + self.session.add(self.plugin) + self.session.commit() + + def test_run_analyzer(self): + + source_to_analysis = { + 'name': 'test', + 'version': '1.0.0', + 'plugin': kiskadee.plugins.example + } + + source_path = _path_to_uncompressed_source( + source_to_analysis, kiskadee.plugins.example.Plugin() + ) + firehose_report = _analyze(self.deb_pkg, "cppcheck", source_path) + self.assertIsNotNone(firehose_report) + + def test_save_source_analysis(self): + + source_to_analysis = { + 'name': 'test', + 'version': '1.0.0', + 'plugin': kiskadee.plugins.example + } + + package = model.Package( + name='test', + plugin_id=self.plugin.id + ) + + package_version = model.Version( + number='1.0.0', + package_id=package.id + ) + + package.versions.append(package_version) + + self.session.add(package) + self.session.add(package_version) + self.session.commit() + + firehose_report = "<>" + _save_source_analysis( + source_to_analysis, + firehose_report, + "cppcheck", + self.session + ) + + saved_analysis = ( + self.session.query(model.Analysis) + .filter(model.Analysis.raw == firehose_report).first() + ) + + self.assertIsNotNone(saved_analysis) + + def test_path_to_uncompressed_source(self): + + source_to_analysis = { + 'name': 'test', + 'version': '1.0.0', + 'plugin': kiskadee.plugins.example + } + + source_path = _path_to_uncompressed_source( + source_to_analysis, kiskadee.plugins.example.Plugin() + ) + + self.assertIsNotNone(source_path) + + def test_invalid_path_to_uncompressed_source(self): + + source_to_analysis = { + 'name': 'test', + 'version': '1.0.0', + 'plugin': kiskadee.plugins.example + } + + source_path = _path_to_uncompressed_source( + source_to_analysis, None + ) + + self.assertIsNone(source_path) From d8a493d6d152c71e537bc498c460ef043b0d41e2 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Jul 07 2017 01:14:05 +0000 Subject: [PATCH 12/12] Add documentation for new runner methods. - Use util module instead of helpers. --- diff --git a/kiskadee/runner.py b/kiskadee/runner.py index 93e82e2..a7f4fab 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -22,7 +22,7 @@ def runner(): """ kiskadee.logger.debug('Starting runner component') session = kiskadee.database.Database().session - _create_analyzers(session) + create_analyzers(session) while running: if not kiskadee.queue.is_empty(): kiskadee.logger.debug('RUNNER: dequeuing...') @@ -32,40 +32,50 @@ def runner(): source_to_analysis['version'], source_to_analysis['plugin'].__name__)) - _call_analyzers(source_to_analysis, session) + call_analyzers(source_to_analysis, session) -def _call_analyzers(source_to_analysis, session): - plugin = source_to_analysis['plugin'].Plugin() - source_path = _path_to_uncompressed_source( - source_to_analysis, plugin - ) - analyzers = plugin.analyzers() - for analyzer in analyzers: - firehose_report = _analyze( - source_to_analysis, analyzer, source_path - ) - _save_source_analysis( - source_to_analysis, firehose_report, analyzer, session - ) +def call_analyzers(source_to_analysis, session): + """Iterate over the package analyzers. - session.commit() + For each analyzer defined to analysis the source, call + the function :func:`analyze`, passing the source dict, the analyzer + to run the analysis, and the path to a compressed source. + """ + plugin = source_to_analysis['plugin'].Plugin() + source_path = _path_to_uncompressed_source( + source_to_analysis, plugin + ) + analyzers = plugin.analyzers() + for analyzer in analyzers: + firehose_report = analyze( + source_to_analysis, analyzer, source_path + ) + _save_source_analysis( + source_to_analysis, firehose_report, analyzer, session + ) + + session.commit() -def _analyze(source_to_analysis, analyzer, source_path): +def analyze(source_to_analysis, analyzer, source_path): """Run each analyzer on a source_to_analysis. - The source_to_analysis dict is in the queue. The keys are: - plugin: the plugin module itself - name: the package name - version: the package version - path: plugin default path for packages - return: list with firehose reports + The `source_to_analysis` dict is in the queue. The keys are: + - plugin: the plugin module itself + - name: the package name + - version: the package version + - path: plugin default path for packages + - return: list with firehose reports + The `analyzer` is the name of a static analyzer already created on the + database. + The `source_path` is the absolute path to a compressed source, returned + by the :func:`_path_to_uncompressed_source`. """ if source_path is None: return None - with kiskadee.helpers.chdir(source_path): + with kiskadee.util.chdir(source_path): kiskadee.logger.debug('ANALYSIS: running {} ...'.format(analyzer)) try: analysis = kiskadee.analyzers.run(analyzer, source_path) @@ -132,7 +142,13 @@ def _path_to_uncompressed_source(package, plugin): return path -def _create_analyzers(_session): +def create_analyzers(_session): + """Create the analyzers on database. + + The kiskadee analyzers are defined on the section `analyzers` of the + kiskadee.conf file. The `_session` argument represents a sqlalchemy + session. + """ list_of_analyzers = dict(kiskadee.config._sections["analyzers"]) for name, version in list_of_analyzers.items(): if not (_session.query(Analyzer).filter(Analyzer.name == name). diff --git a/kiskadee/tests/test_model.py b/kiskadee/tests/test_model.py index 552c4e0..4179ac6 100644 --- a/kiskadee/tests/test_model.py +++ b/kiskadee/tests/test_model.py @@ -3,7 +3,7 @@ from sqlalchemy import create_engine, exc from sqlalchemy.orm import sessionmaker from kiskadee import model -from kiskadee.runner import _create_analyzers +from kiskadee.runner import create_analyzers class TestModel(TestCase): @@ -13,7 +13,7 @@ class TestModel(TestCase): Session = sessionmaker(bind=self.engine) self.session = Session() model.Base.metadata.create_all(self.engine) - _create_analyzers(self.session) + create_analyzers(self.session) self.plugin = model.Plugin(name='kiskadee-plugin', target='university') self.package = model.Package(name='python-kiskadee') self.version = model.Version(number='1.0-rc1') diff --git a/kiskadee/tests/test_monitor.py b/kiskadee/tests/test_monitor.py index 115d1c7..e59bdd1 100644 --- a/kiskadee/tests/test_monitor.py +++ b/kiskadee/tests/test_monitor.py @@ -1,13 +1,14 @@ from unittest import TestCase -from kiskadee.monitor import Monitor -from kiskadee import model from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker + +from kiskadee import model +from kiskadee.monitor import Monitor from kiskadee.queue import enqueue_package from kiskadee.model import Package, Plugin import kiskadee.queue import kiskadee.plugins.debian -from kiskadee.runner import _create_analyzers +from kiskadee.runner import create_analyzers class TestMonitor(TestCase): @@ -18,7 +19,7 @@ class TestMonitor(TestCase): session = Session() self.monitor = Monitor(session) model.Base.metadata.create_all(self.engine) - _create_analyzers(self.monitor.session) + create_analyzers(self.monitor.session) self.pkg1 = {'name': 'curl', 'version': '7.52.1-5', 'plugin': kiskadee.plugins.debian, diff --git a/kiskadee/tests/test_runner.py b/kiskadee/tests/test_runner.py index e6bd6fb..36aa29c 100644 --- a/kiskadee/tests/test_runner.py +++ b/kiskadee/tests/test_runner.py @@ -1,9 +1,8 @@ from unittest import TestCase -from kiskadee.runner import _analyze, _path_to_uncompressed_source -from kiskadee.runner import _save_source_analysis +from kiskadee.runner import analyze, _path_to_uncompressed_source +from kiskadee.runner import _save_source_analysis, create_analyzers import kiskadee.plugins.example -from kiskadee.runner import _create_analyzers from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from kiskadee import model @@ -16,7 +15,7 @@ class TestAnalyzers(TestCase): Session = sessionmaker(bind=self.engine) self.session = Session() model.Base.metadata.create_all(self.engine) - _create_analyzers(self.session) + create_analyzers(self.session) self.plugin = kiskadee.plugins.debian.Plugin() self.deb_pkg = {'name': 'test', 'version': '1.0.0', @@ -37,7 +36,7 @@ class TestAnalyzers(TestCase): source_path = _path_to_uncompressed_source( source_to_analysis, kiskadee.plugins.example.Plugin() ) - firehose_report = _analyze(self.deb_pkg, "cppcheck", source_path) + firehose_report = analyze(self.deb_pkg, "cppcheck", source_path) self.assertIsNotNone(firehose_report) def test_save_source_analysis(self):