From 82c91dff9dd6126fe927363d75f9e2c7121e3183 Mon Sep 17 00:00:00 2001
From: David Carlos <ddavidcarlos1392@gmail.com>
Date: Mar 16 2018 02:29:46 +0000
Subject: [PATCH 1/17] Use a migration to create kiskadee fetchers.


---

diff --git a/alembic/versions/50988af48b09_create_fetcher_table.py b/alembic/versions/50988af48b09_create_fetcher_table.py
index 54fd6c4..33e90a3 100644
--- a/alembic/versions/50988af48b09_create_fetcher_table.py
+++ b/alembic/versions/50988af48b09_create_fetcher_table.py
@@ -30,7 +30,7 @@ def upgrade():
             primary_key=True
         ),
         sa.Column('name', sa.Unicode(255), nullable=False, unique=True),
-        sa.Column('target', sa.Unicode(255), nullable=True),
+        sa.Column('source', sa.Unicode(255), nullable=True),
         sa.Column('description', sa.UnicodeText)
     )
 
diff --git a/alembic/versions/cc4887b7c3da_create_fetchers_on_database.py b/alembic/versions/cc4887b7c3da_create_fetchers_on_database.py
index bd88705..01ec98c 100644
--- a/alembic/versions/cc4887b7c3da_create_fetchers_on_database.py
+++ b/alembic/versions/cc4887b7c3da_create_fetchers_on_database.py
@@ -23,7 +23,7 @@ def upgrade():
     fetchers_table = table('fetchers',
             column('id', Integer),
             column('name', String),
-            column('target', String),
+            column('source', String),
             column('description', String)
             )
 
@@ -35,7 +35,7 @@ def upgrade():
                 [
                     {
                         'id': idx, 'name': _fetcher.name,
-                        'target': _fetcher.config['target'],
+                        'source': _fetcher.config['source'],
                         'description': _fetcher.config['description']
                         }
                     ]
diff --git a/util/kiskadee.conf b/util/kiskadee.conf
index c2dc351..5eb6ec8 100644
--- a/util/kiskadee.conf
+++ b/util/kiskadee.conf
@@ -19,7 +19,7 @@ port = 5432
 dbname = kiskadee_test
 
 [debian_fetcher]
-target = http://ftp.us.debian.org/debian
+source = http://ftp.us.debian.org/debian
 description = A fetcher to monitor Debian Repositories
 release = sid
 meta = Sources.gz
@@ -28,13 +28,13 @@ analyzers = cppcheck flawfinder
 active = no
 
 [juliet_fetcher]
-target = juliet
+source = juliet
 description = SAMATE Juliet test suite
 analyzers = cppcheck
 active = no
 
 [anitya_fetcher]
-target = release-monitoring.org
+source = release-monitoring.org
 description = A fetcher to monitor Anitya packages
 analyzers = cppcheck flawfinder
 active = no
@@ -42,7 +42,7 @@ zmq_port = 5556
 zmq_topic = anitya
 
 [example_fetcher]
-target = example
+source = example
 description = SAMATE Juliet test suite
 analyzers = cppcheck flawfinder clanganalyzer frama_c
 active = yes

From 3b7f7a559f35f47187071def6f892796011dea8b Mon Sep 17 00:00:00 2001
From: David Carlos <ddavidcarlos1392@gmail.com>
Date: Mar 16 2018 02:29:46 +0000
Subject: [PATCH 2/17] Refactors queue.py module.


  - Add a new decorator to check when a queue is empty.
  - Create subpackage to unit tests.
  - Change Queues methods to static methods.

---

diff --git a/kiskadee/fetchers/example.py b/kiskadee/fetchers/example.py
index 02eb9b2..a4c1593 100644
--- a/kiskadee/fetchers/example.py
+++ b/kiskadee/fetchers/example.py
@@ -9,7 +9,8 @@ class Fetcher(kiskadee.fetchers.Fetcher):
         """Use sources from test suite."""
         return 'kiskadee/tests/test_source/test_source.tar.gz'
 
-    def watch(self, **kwargs):
+    @kiskadee.queue.enqueue_project
+    def watch(self):
         """There is no proper API to inspect new example versions.
         It should not matter, since example will not receive updates.
         """
diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py
index ef3fb77..fbf68bc 100644
--- a/kiskadee/monitor.py
+++ b/kiskadee/monitor.py
@@ -53,9 +53,32 @@ class Monitor:
             if pkg:
                 self._send_to_runner(pkg)
             time.sleep(2)
-            analyzed_project = self.kiskadee_queue.dequeue_result()
+            analyzed_project = self.queues.dequeue_result()
             self._save_analyzed_project(analyzed_project)
 
+    def dequeue_package(self):
+        """Dequeue packages from packages_queue."""
+        if not kiskadee.queue.packages_queue.empty():
+            pkg = kiskadee.queue.packages_queue.get()
+            kiskadee.logger.debug(
+                    "MONITOR: Dequed Package: {}_{}"
+                    .format(pkg["name"], pkg["version"])
+                )
+            return pkg
+        return {}
+
+    # Use directly dequeue_result from kiskadee.queue
+    #def dequeue_result(self):
+    #    """Dequeue analyzed packages from result_queue."""
+    #    if not self.kiskadee_queue.results_empty():
+    #        pkg = self.kiskadee_queue.dequeue_result()
+    #        kiskadee.logger.debug(
+    #                "MONITOR: Dequed result for package : {}-{}"
+    #                .format(pkg["name"], pkg["version"])
+    #            )
+    #        return pkg
+    #    return {}
+
     def _send_to_runner(self, pkg):
         _name = pkg['fetcher'].split('.')[-1]
         _fetcher = self._query(Fetcher).filter_by(name = _name).first()
@@ -80,8 +103,8 @@ class Monitor:
                     self.kiskadee_queue.enqueue_analysis(pkg)
 
     # Move this to model.py
-    def _save_analyzed_project(self, data):
-        if not data:
+    def _save_analyzed_project(self, project):
+        if not pkg:
             return {}
         project = self._query(Package).filter_by(name = data['name']).first()
         if not project:
@@ -93,7 +116,7 @@ class Monitor:
             self._save_analysis(data, analyzer, result, project.versions[-1])
 
     # Move this to model.py
-    def _update_project(self, package, pkg):
+    def _update_pkg(self, package, pkg):
 
         if(package.versions[-1].number == pkg['version']):
             return package
@@ -115,7 +138,7 @@ class Monitor:
             return None
 
     # Move this to model.py
-    def _save_project(self, pkg):
+    def _save_pkg(self, pkg):
         homepage = None
         if ('meta' in pkg) and ('homepage' in pkg['meta']):
             homepage = pkg['meta']['homepage']
@@ -230,11 +253,11 @@ def daemon():
     runner = Runner()
     monitor_process = Process(
             target=monitor.monitor,
-            args=(queues,)
+            args=(queues)
         )
     runner_process = Process(
             target=runner.runner,
-            args=(queues,)
+            args=(queues)
         )
     monitor_process.start()
     runner_process.start()
diff --git a/kiskadee/queue.py b/kiskadee/queue.py
index 214b631..a6835ea 100644
--- a/kiskadee/queue.py
+++ b/kiskadee/queue.py
@@ -1,6 +1,5 @@
 """Provide kiskadee queues and operations on them."""
 import time
-import dill
 from multiprocessing import Queue
 
 import kiskadee
@@ -24,10 +23,10 @@ def enqueue_project(fetcher_watch):
     """
     def wrapper(*args, **kwargs):
         project = fetcher_watch(*args, **kwargs)
-        kiskadee.queue.queues.enqueue_project(project)
+        kiskadee.queue.Queues.enqueue_project(project)
         fetcher = project['fetcher'].name
         kiskadee.logger.debug(
-                "{} fetcher: sending package {}_{} for monitor"
+                "{} fetcher: Sending package {}_{} for monitor"
                 .format(fetcher, package['name'], package['version'])
             )
         time.sleep(2)
@@ -41,14 +40,15 @@ class Queues():
     """Provide kiskadee queues objects."""
 
     @staticmethod
-    def enqueue_analysis(project_to_analysis):
+    def enqueue_analysis(analysis):
         """Put a analysis on the analysis queue."""
-        analysis.put(project_to_analysis)
+        analysis.put(analysis)
 
     @staticmethod
+    @empty_queue
     def dequeue_analysis():
         """Get a analysis from the analysis queue."""
-        return analysis.get()
+        return analysis.get(timeout=1)
 
     @staticmethod
     def enqueue_result(result):

From 6e7026dbbba646eeb875dd57e4ad4925d013018c Mon Sep 17 00:00:00 2001
From: David Carlos <ddavidcarlos1392@gmail.com>
Date: Mar 16 2018 02:30:27 +0000
Subject: [PATCH 3/17] Use mock to run api tests.


  - We should use mock to test everything that interacts with database.
  This will avoid some database dependecy problems when runing kiskadee tests.

---

diff --git a/alembic/versions/50988af48b09_create_fetcher_table.py b/alembic/versions/50988af48b09_create_fetcher_table.py
index 33e90a3..54fd6c4 100644
--- a/alembic/versions/50988af48b09_create_fetcher_table.py
+++ b/alembic/versions/50988af48b09_create_fetcher_table.py
@@ -30,7 +30,7 @@ def upgrade():
             primary_key=True
         ),
         sa.Column('name', sa.Unicode(255), nullable=False, unique=True),
-        sa.Column('source', sa.Unicode(255), nullable=True),
+        sa.Column('target', sa.Unicode(255), nullable=True),
         sa.Column('description', sa.UnicodeText)
     )
 
diff --git a/alembic/versions/cc4887b7c3da_create_fetchers_on_database.py b/alembic/versions/cc4887b7c3da_create_fetchers_on_database.py
index 01ec98c..bd88705 100644
--- a/alembic/versions/cc4887b7c3da_create_fetchers_on_database.py
+++ b/alembic/versions/cc4887b7c3da_create_fetchers_on_database.py
@@ -23,7 +23,7 @@ def upgrade():
     fetchers_table = table('fetchers',
             column('id', Integer),
             column('name', String),
-            column('source', String),
+            column('target', String),
             column('description', String)
             )
 
@@ -35,7 +35,7 @@ def upgrade():
                 [
                     {
                         'id': idx, 'name': _fetcher.name,
-                        'source': _fetcher.config['source'],
+                        'target': _fetcher.config['target'],
                         'description': _fetcher.config['description']
                         }
                     ]
diff --git a/kiskadee/tests/api/test_api.py b/kiskadee/tests/api/test_api.py
index e88fee5..0b83d1a 100644
--- a/kiskadee/tests/api/test_api.py
+++ b/kiskadee/tests/api/test_api.py
@@ -1,8 +1,10 @@
 import json
 import unittest
+from sqlalchemy.orm import sessionmaker
 from unittest.mock import MagicMock
 
 import kiskadee
+from kiskadee.runner import Runner
 from kiskadee.monitor import Monitor
 import kiskadee.api.app
 import kiskadee.fetchers.example
diff --git a/kiskadee/tests/integration/test_monitor.py b/kiskadee/tests/integration/test_monitor.py
index 67cd7c2..aa95be6 100644
--- a/kiskadee/tests/integration/test_monitor.py
+++ b/kiskadee/tests/integration/test_monitor.py
@@ -3,8 +3,8 @@ from sqlalchemy.orm import sessionmaker
 
 from kiskadee import model
 from kiskadee.monitor import Monitor
-from kiskadee.queue import Queues
-from kiskadee.model import Package, create_analyzers, Report, Analysis
+from kiskadee.queue import packages_queue
+from kiskadee.model import Package, Fetcher, create_analyzers, Report, Analysis
 import kiskadee.queue
 import kiskadee.fetchers.debian
 import kiskadee.fetchers.anitya
@@ -18,13 +18,12 @@ class MonitorTestCase(unittest.TestCase):
         Session = sessionmaker(bind=self.engine)
         self.session = Session()
         self.monitor = Monitor(self.session)
-        self.queues = Queues()
         model.Base.metadata.create_all(self.engine)
         create_analyzers(self.session)
         self.pkg1 = {
                 'name': 'curl',
                 'version': '7.52.1-5',
-                'fetcher': kiskadee.fetchers.debian.__name__,
+                'fetcher': kiskadee.fetchers.debian,
                 'meta': {'directory': 'pool/main/c/curl'},
                 'results': {
                     'cppcheck': '<>',
@@ -33,7 +32,7 @@ class MonitorTestCase(unittest.TestCase):
 
         self.pkg2 = {'name': 'urlscan',
                      'version': '0.8.2',
-                     'fetcher': kiskadee.fetchers.debian.__name__,
+                     'fetcher': kiskadee.fetchers.debian,
                      'meta': {'directory': 'pool/main/u/urlscan'},
                      'results': {
                             'cppcheck': '<>',
@@ -42,7 +41,7 @@ class MonitorTestCase(unittest.TestCase):
 
         self.pkg3 = {'name': 'curl',
                      'version': '7.52.2-5',
-                     'fetcher': kiskadee.fetchers.debian.__name__,
+                     'fetcher': kiskadee.fetchers.debian,
                      'meta': {'directory': 'pool/main/c/curl'},
                      'results': {
                             'cppcheck': '<>',
@@ -50,7 +49,7 @@ class MonitorTestCase(unittest.TestCase):
                      'fetcher_id': 1}
         self.pkg4 = {'name': 'urlanitya',
                      'version': '0.11',
-                     'fetcher': kiskadee.fetchers.anitya.__name__,
+                     'fetcher': kiskadee.fetchers.anitya,
                      'meta': {
                             'backend': 'gitHub',
                             'homepage': 'https://github.com/GesielFreitas/Cros'
@@ -79,14 +78,31 @@ class MonitorTestCase(unittest.TestCase):
         fetcher = kiskadee.fetchers.debian.Fetcher()
         self.assertEqual(fetcher.name, 'debian')
 
+    def test_save_some_fetcher(self):
+        fetcher = kiskadee.fetchers.debian.Fetcher()
+        self.monitor._save_fetcher(fetcher)
+        _fetchers = self.monitor.session.query(Fetcher).all()
+        self.assertEqual(len(_fetchers), 1)
+        self.assertEqual(_fetchers[0].name, 'debian')
+        self.assertEqual(_fetchers[0].description,
+                         fetcher.config['description'])
+
     def test_save_package(self):
         self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher())
-        self.monitor._save_analyzed_project(self.pkg1)
-        self.monitor._save_analyzed_project(self.pkg2)
+        packages_queue.put(self.pkg1)
+        packages_queue.put(self.pkg2)
+
+        _pkg = self.monitor.dequeue_package()
+        self.monitor._save_analyzed_pkg(_pkg)
+        _pkgs = self.monitor.session.query(Package).all()
+        self.assertEqual(len(_pkgs), 1)
+        self.assertEqual(_pkgs[0].name, _pkg['name'])
+
+        _pkg = self.monitor.dequeue_package()
+        self.monitor._save_analyzed_pkg(_pkg)
         _pkgs = self.monitor.session.query(Package).all()
         self.assertEqual(len(_pkgs), 2)
-        self.assertEqual(_pkgs[0].name, self.pkg1['name'])
-        self.assertEqual(_pkgs[1].name, self.pkg2['name'])
+        self.assertEqual(_pkgs[1].name, _pkg['name'])
 
     def test_save_reports(self):
         _fetcher = model.Fetcher(
@@ -126,21 +142,21 @@ class MonitorTestCase(unittest.TestCase):
 
     def test_save_version(self):
         self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher())
-        self.monitor._save_analyzed_project(self.pkg1)
+        self.monitor._save_analyzed_pkg(self.pkg1)
         _pkgs = self.monitor.session.query(Package).all()
         _version = _pkgs[0].versions[0].number
         self.assertEqual(_version, self.pkg1['version'])
 
     def test_update_version(self):
         self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher())
-        self.queues.enqueue_project(self.pkg1)
-        self.queues.enqueue_project(self.pkg3)
+        packages_queue.put(self.pkg1)
+        packages_queue.put(self.pkg3)
 
-        _pkg = self.queues.dequeue_project()
-        self.monitor._save_analyzed_project(_pkg)
+        _pkg = self.monitor.dequeue_package()
+        self.monitor._save_analyzed_pkg(_pkg)
 
-        _pkg = self.queues.dequeue_project()
-        self.monitor._save_analyzed_project(_pkg)
+        _pkg = self.monitor.dequeue_package()
+        self.monitor._save_analyzed_pkg(_pkg)
 
         _pkgs = self.monitor.session.query(Package).all()
         self.assertEqual(len(_pkgs), 1)
@@ -155,9 +171,11 @@ class MonitorTestCase(unittest.TestCase):
 
     def test_save_package_anitya(self):
         self.monitor._save_fetcher(kiskadee.fetchers.anitya.Fetcher())
-        self.queues.enqueue_project(self.pkg4)
-        _pkg = self.queues.dequeue_project()
-        self.monitor._save_analyzed_project(_pkg)
+        packages_queue.put(self.pkg4)
+
+        _pkg = self.monitor.dequeue_package()
+        self.monitor._save_analyzed_pkg(_pkg)
+
         _pkgs = self.monitor.session.query(Package).all()
         self.assertEqual(len(_pkgs), 1)
         self.assertEqual(_pkgs[0].homepage, _pkg['meta']['homepage'])
diff --git a/kiskadee/tests/integration/test_runner.py b/kiskadee/tests/integration/test_runner.py
index f9a2e16..0ad1e0a 100644
--- a/kiskadee/tests/integration/test_runner.py
+++ b/kiskadee/tests/integration/test_runner.py
@@ -1,8 +1,3 @@
-"""
-This integration tests needs docker engine be running and accessible
-by the user that is executing the tests. It also needs selinux running in
-a permissive mode (setenforce 0)
-"""
 import unittest
 import tempfile
 
@@ -11,11 +6,12 @@ import kiskadee.fetchers.example
 import kiskadee.fetchers.debian
 from sqlalchemy.orm import sessionmaker
 from kiskadee import model
-from kiskadee.queue import Queues
+from kiskadee.queue import KiskadeeQueue
 from kiskadee.database import Database
 
 
 class AnalyzersTestCase(unittest.TestCase):
+
     def setUp(self):
         self.engine = Database('db_test').engine
         Session = sessionmaker(bind=self.engine)
@@ -25,15 +21,16 @@ class AnalyzersTestCase(unittest.TestCase):
         self.fetcher = kiskadee.fetchers.debian.Fetcher()
         self.deb_pkg = {'name': 'test',
                         'version': '1.0.0',
-                        'fetcher': kiskadee.fetchers.debian.__name__
+                        'fetcher': kiskadee.fetchers.debian.Fetcher()
                         }
         self.fetcher = model.Fetcher(
                 name='kiskadee-fetcher2', target='university'
             )
         self.session.add(self.fetcher)
         self.session.commit()
+        kiskadee_queue = KiskadeeQueue()
         self.runner = Runner()
-        self.runner.queues = Queues()
+        self.runner.kiskadee_queue = kiskadee_queue
 
     def tearDown(self):
         self.session.close()
@@ -57,11 +54,12 @@ class AnalyzersTestCase(unittest.TestCase):
         source_to_analysis = {
                 'name': 'test',
                 'version': '1.0.0',
-                'fetcher': kiskadee.fetchers.example.__name__
+                'fetcher': kiskadee.fetchers.example.Fetcher()
         }
 
         self.runner.call_analyzers(source_to_analysis)
-        analyzed_pkg = self.runner.queues.dequeue_result()
+        analyzed_pkg = self.runner.kiskadee_queue.dequeue_result()
+
         self.assertEqual(analyzed_pkg['name'], source_to_analysis['name'])
         self.assertIn('cppcheck', analyzed_pkg['results'])
         self.assertIn('flawfinder', analyzed_pkg['results'])
@@ -95,6 +93,5 @@ class AnalyzersTestCase(unittest.TestCase):
 
         self.assertIsNone(source_path)
 
-
 if __name__ == '__main__':
     unittest.main()
diff --git a/kiskadee/tests/plugins/test_plugins.py b/kiskadee/tests/plugins/test_plugins.py
index 9fe8262..030da9a 100644
--- a/kiskadee/tests/plugins/test_plugins.py
+++ b/kiskadee/tests/plugins/test_plugins.py
@@ -70,7 +70,7 @@ class DebianFetcherTestCase(unittest.TestCase):
         self.debian_fetcher._queue_sources_gz_pkgs(temp_dir)
         shutil.rmtree(temp_dir)
 
-        some_pkg = kiskadee.queue.Queues.dequeue_project()
+        some_pkg = kiskadee.queue.packages_queue.get()
         self.assertTrue(isinstance(some_pkg, dict))
         self.assertIn('name', some_pkg)
         self.assertIn('version', some_pkg)
@@ -190,8 +190,8 @@ class TestAnityaFetcher(unittest.TestCase):
 
     def test_create_package_dict(self):
 
-        self.anitya_fetcher.project_to_enqueue(self.msg)
-        _dict = kiskadee.queue.Queues().dequeue_project()
+        self.anitya_fetcher._create_package_dict(self.msg)
+        _dict = kiskadee.queue.packages_queue.get()
         self.assertEqual(_dict['name'], 'urlscan')
         self.assertEqual(_dict['version'], '0.8.5')
         self.assertEqual(_dict['meta']['backend'], 'GitHub')
@@ -199,7 +199,7 @@ class TestAnityaFetcher(unittest.TestCase):
                 _dict['meta']['homepage'],
                 'https://github.com/firecat53/urlscan'
         )
-        self.assertEqual(_dict['fetcher'], 'kiskadee.fetchers.anitya')
+        self.assertEqual(_dict['fetcher'].name, 'anitya')
 
 
 if __name__ == '__main__':
diff --git a/kiskadee/tests/test_api.py b/kiskadee/tests/test_api.py
deleted file mode 100644
index 71a58fb..0000000
--- a/kiskadee/tests/test_api.py
+++ /dev/null
@@ -1,118 +0,0 @@
-import json
-import unittest
-from sqlalchemy.orm import sessionmaker
-
-import kiskadee
-from kiskadee.runner import Runner
-from kiskadee.monitor import Monitor
-import kiskadee.api.app
-import kiskadee.fetchers.example
-
-
-class ApiTestCase(unittest.TestCase):
-
-    def setUp(self):
-        kiskadee.api.app.kiskadee.testing = True
-        self.engine = kiskadee.database.Database('db_test').engine
-        Session = sessionmaker(bind=self.engine)
-        self.session = Session()
-        self.app = kiskadee.api.app.kiskadee.test_client()
-        kiskadee.model.create_analyzers(self.session)
-        kiskadee.model.Base.metadata.create_all(self.engine)
-        kiskadee.model.create_analyzers(self.session)
-        fetcher = kiskadee.model.Fetcher(
-                name='kiskadee-fetcher', target='university'
-        )
-        pkg = kiskadee.model.Package(
-                name='kiskadee-package', fetcher_id=1
-        )
-        version = kiskadee.model.Version(
-                number='7.23', package_id=1
-        )
-        analysis = kiskadee.model.Analysis(
-                version_id=1, analyzer_id=1,
-                raw={
-                    'results': [
-                        {'severity': 'warning'},
-                        {'severity': 'style'},
-                        {'severity': 'error'}
-                    ]
-                }
-        )
-        report = kiskadee.model.Report(
-            analysis_id=1
-        )
-        self.session.add(fetcher)
-        self.session.add(pkg)
-        self.session.add(version)
-        self.session.add(analysis)
-        self.session.add(report)
-        self.session.commit()
-        self.runner = Runner()
-        self.monitor = Monitor(self.session)
-        self.runner.kiskadee_queue = kiskadee.queue.KiskadeeQueue()
-
-    def tearDown(self):
-        self.session.close()
-        kiskadee.model.Base.metadata.drop_all()
-
-    def test_get_fetchers(self):
-        def mock_kiskadee_db_session():
-            return self.session
-
-        kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session
-        response = self.app.get("/fetchers")
-        self.assertIn("fetchers", json.loads(response.data.decode("utf-8")))
-
-    def test_get_activated_fetcher(self):
-
-        def mock_kiskadee_db_session():
-            return self.session
-
-        kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session
-        response = self.app.get("/fetchers")
-        response_as_json = json.loads(response.data.decode("utf-8"))
-        fetcher_name = response_as_json["fetchers"][0]["name"]
-        self.assertEqual("kiskadee-fetcher", fetcher_name)
-
-    def test_get_analysis_as_json(self):
-        source_to_analysis = {
-                'name': 'test',
-                'version': '1.0.0',
-                'fetcher_id': '1',
-                'fetcher': kiskadee.fetchers.example.Fetcher()
-        }
-
-        def mock_kiskadee_db_session():
-            return self.session
-
-        self.runner.call_analyzers(source_to_analysis)
-        analyzed_pkg = self.runner.kiskadee_queue.dequeue_result()
-        self.monitor._save_analyzed_pkg(analyzed_pkg)
-        kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session
-        response = self.app.get("/analysis/test/1.0.0")
-        response_data = json.loads(response.data.decode("utf-8"))
-        self.assertTrue(len(response_data) >= 1)
-
-    def test_get_analysis_results(self):
-        def mock_kiskadee_db_session():
-            return self.session
-
-        kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session
-        response = self.app.get("/analysis/kiskadee-package/7.23/1/results")
-        response_data = json.loads(response.data.decode("utf-8"))
-        self.assertIn("analysis_results", response_data)
-        self.assertTrue(len(response_data["analysis_results"]) > 0)
-
-    def test_get_analysis_reports(self):
-        def mock_kiskadee_db_session():
-            return self.session
-
-        kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session
-        response = self.app.get("/analysis/kiskadee-package/7.23/1/reports")
-        response_data = json.loads(response.data.decode("utf-8"))
-        self.assertIn("analysis_report", response_data)
-
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/kiskadee/tests/test_model.py b/kiskadee/tests/test_model.py
deleted file mode 100644
index 4a56835..0000000
--- a/kiskadee/tests/test_model.py
+++ /dev/null
@@ -1,179 +0,0 @@
-import unittest
-from sqlalchemy import exc
-from sqlalchemy.orm import sessionmaker
-
-from kiskadee import model
-from kiskadee.database import Database
-
-
-class ModelTestCase(unittest.TestCase):
-
-    def setUp(self):
-        self.engine = Database('db_test').engine
-        Session = sessionmaker(bind=self.engine)
-        self.session = Session()
-        model.Base.metadata.create_all(self.engine)
-        model.create_analyzers(self.session)
-        self.fetcher = model.Fetcher(
-              name='kiskadee-fetcher', target='university'
-            )
-        self.package = model.Package(name='python-kiskadee')
-        self.version = model.Version(number='1.0-rc1')
-        self.fetcher.packages.append(self.package)
-        self.package.versions.append(self.version)
-        self.session.add(self.package)
-        self.session.add(self.fetcher)
-        self.session.add(self.version)
-
-        self.analysis = model.Analysis(
-                analyzer_id=1,
-                version_id=1,
-                raw=""
-                )
-        self.session.add(self.analysis)
-        self.report = model.Report(
-                analysis_id=1,
-        )
-        self.session.add(self.report)
-        self.session.commit()
-
-    def tearDown(self):
-        self.session.close()
-        model.Base.metadata.drop_all()
-
-    def test_query_fetcher(self):
-        fetchers = self.session.query(model.Fetcher).all()
-        self.assertEqual(fetchers, [self.fetcher])
-
-    def test_query_package(self):
-        packages = self.session.query(model.Package).all()
-        self.assertEqual(packages, [self.package])
-
-    def test_query_version(self):
-        versions = self.session.query(model.Version).all()
-        self.assertEqual(versions, [self.version])
-
-    def test_query_report(self):
-        reports = self.session.query(model.Report).all()
-        self.assertEqual(reports, [self.report])
-
-    def test_add_fetcher(self):
-        fetchers = self.session.query(model.Fetcher).all()
-        self.assertEqual(len(fetchers), 1)
-        self.session.add(model.Fetcher(name='foo', target='bar'))
-        fetchers = self.session.query(model.Fetcher).all()
-        self.assertEqual(len(fetchers), 2)
-
-    def test_add_version_without_package(self):
-        version = model.Version(number='3.1')
-        self.session.add(version)
-        with self.assertRaises(exc.IntegrityError):
-            self.session.commit()
-
-    def test_add_report_without_analysis(self):
-        report = model.Report(
-        )
-        self.session.add(report)
-        with self.assertRaises(exc.IntegrityError):
-            self.session.commit()
-
-    def test_add_package_without_fetcher(self):
-        package = model.Package(name='foo-bar')
-        self.session.add(package)
-        with self.assertRaises(exc.IntegrityError):
-            self.session.commit()
-
-    def test_unique_package_in_fetcher(self):
-        package_1 = model.Package(name='foo-bar')
-        package_2 = model.Package(name='foo-bar')
-        self.fetcher.packages.append(package_1)
-        self.fetcher.packages.append(package_2)
-        with self.assertRaises(exc.IntegrityError):
-            self.session.commit()
-
-    def test_unique_version_for_package(self):
-        package_version_1 = model.Version(number='1.0')
-        package_version_2 = model.Version(number='1.0')
-        self.package.versions.append(package_version_1)
-        self.package.versions.append(package_version_2)
-        with self.assertRaises(exc.IntegrityError):
-            self.session.commit()
-
-    def test_compose_kiskadee_source(self):
-        _analyzer = self.session.query(model.Analyzer)\
-                    .filter(model.Analyzer.name == "cppcheck").first()
-        package = model.Package(
-                name='bla',
-                fetcher_id=self.fetcher.id
-                )
-        package_version = model.Version(
-                number='1.0.1',
-                package_id=package.id
-                )
-
-        package_analysis = model.Analysis(
-                raw="<>",
-                analyzer_id=_analyzer.id,
-                version_id=package_version.id
-                )
-
-        self.fetcher.packages.append(package)
-        package.versions.append(package_version)
-        package_version.analysis.append(package_analysis)
-
-        self.assertEqual(package.versions[0].analysis[0].raw, "<>")
-
-    def test_save_several_analysis(self):
-
-        _analyzer1 = (
-                self.session.query(model.Analyzer)
-                .filter(model.Analyzer.name == "cppcheck").first()
-                )
-        _analyzer2 = (
-                self.session.query(model.Analyzer)
-                .filter(model.Analyzer.name == "flawfinder").first()
-                )
-
-        package = model.Package(
-                name='bla',
-                fetcher_id=self.fetcher.id
-                )
-        package_version = model.Version(
-                number='1.0.1',
-                package_id=package.id
-                )
-
-        self.fetcher.packages.append(package)
-        package.versions.append(package_version)
-
-        self.session.add(package)
-        self.session.add(package_version)
-        self.session.commit()
-
-        package_analysis1 = model.Analysis(
-                raw="<>",
-                analyzer_id=_analyzer1.id,
-                version_id=package_version.id
-                )
-        package_analysis2 = model.Analysis(
-                raw="><",
-                analyzer_id=_analyzer2.id,
-                version_id=package_version.id
-                )
-
-        self.session.add(package_analysis1)
-        self.session.add(package_analysis2)
-        self.session.commit()
-
-        saved_package = (
-                self.session.query(model.Package)
-                .filter(model.Package.name == 'bla').first()
-                )
-        analysis = saved_package.versions[-1].analysis
-        self.assertEqual(len(analysis), 2)
-        self.assertEqual(analysis[0].raw, "<>")
-        self.assertEqual(analysis[1].raw, "><")
-
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/kiskadee/tests/test_monitor.py b/kiskadee/tests/test_monitor.py
deleted file mode 100644
index 2a759fa..0000000
--- a/kiskadee/tests/test_monitor.py
+++ /dev/null
@@ -1,190 +0,0 @@
-import unittest
-from sqlalchemy.orm import sessionmaker
-
-from kiskadee import model
-from kiskadee.monitor import Monitor
-from kiskadee.queue import packages_queue
-from kiskadee.model import Package, Fetcher, create_analyzers, Report, Analysis
-import kiskadee.queue
-import kiskadee.fetchers.debian
-import kiskadee.fetchers.anitya
-from kiskadee.database import Database
-
-
-class MonitorTestCase(unittest.TestCase):
-
-    def setUp(self):
-        self.engine = Database('db_test').engine
-        Session = sessionmaker(bind=self.engine)
-        self.session = Session()
-        self.monitor = Monitor(self.session)
-        model.Base.metadata.create_all(self.engine)
-        create_analyzers(self.session)
-        self.pkg1 = {
-                'name': 'curl',
-                'version': '7.52.1-5',
-                'fetcher': kiskadee.fetchers.debian,
-                'meta': {'directory': 'pool/main/c/curl'},
-                'results': {
-                    'cppcheck': '<>',
-                    'flawfinder': '><'},
-                'fetcher_id': 1}
-
-        self.pkg2 = {'name': 'urlscan',
-                     'version': '0.8.2',
-                     'fetcher': kiskadee.fetchers.debian,
-                     'meta': {'directory': 'pool/main/u/urlscan'},
-                     'results': {
-                            'cppcheck': '<>',
-                            'flawfinder': '><'},
-                     'fetcher_id': 1}
-
-        self.pkg3 = {'name': 'curl',
-                     'version': '7.52.2-5',
-                     'fetcher': kiskadee.fetchers.debian,
-                     'meta': {'directory': 'pool/main/c/curl'},
-                     'results': {
-                            'cppcheck': '<>',
-                            'flawfinder': '><'},
-                     'fetcher_id': 1}
-        self.pkg4 = {'name': 'urlanitya',
-                     'version': '0.11',
-                     'fetcher': kiskadee.fetchers.anitya,
-                     'meta': {
-                            'backend': 'gitHub',
-                            'homepage': 'https://github.com/GesielFreitas/Cros'
-                        },
-                     'results': {
-                            'cppcheck': '<>',
-                            'flawfinder': '><'},
-                     'fetcher_id': 1}
-        self.analysis = {
-                'analyzer_id': 1,
-                'id': 1,
-                'raw': {
-                    'results': [
-                        {'severity': 'warning'},
-                        {'severity': 'style'},
-                        {'severity': 'error'}
-                    ]
-                }
-            }
-
-    def tearDown(self):
-        self.session.close()
-        model.Base.metadata.drop_all()
-
-    def test_dequeue_package(self):
-        packages_queue.put(self.pkg1)
-        _pkg = self.monitor.dequeue_package()
-        self.assertTrue(isinstance(_pkg, dict))
-
-    def test_return_fetcher_name(self):
-        fetcher = kiskadee.fetchers.debian.Fetcher()
-        self.assertEqual(fetcher.name, 'debian')
-
-    def test_save_some_fetcher(self):
-        fetcher = kiskadee.fetchers.debian.Fetcher()
-        self.monitor._save_fetcher(fetcher)
-        _fetchers = self.monitor.session.query(Fetcher).all()
-        self.assertEqual(len(_fetchers), 1)
-        self.assertEqual(_fetchers[0].name, 'debian')
-        self.assertEqual(_fetchers[0].description,
-                         fetcher.config['description'])
-
-    def test_save_package(self):
-        self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher())
-        packages_queue.put(self.pkg1)
-        packages_queue.put(self.pkg2)
-
-        _pkg = self.monitor.dequeue_package()
-        self.monitor._save_analyzed_pkg(_pkg)
-        _pkgs = self.monitor.session.query(Package).all()
-        self.assertEqual(len(_pkgs), 1)
-        self.assertEqual(_pkgs[0].name, _pkg['name'])
-
-        _pkg = self.monitor.dequeue_package()
-        self.monitor._save_analyzed_pkg(_pkg)
-        _pkgs = self.monitor.session.query(Package).all()
-        self.assertEqual(len(_pkgs), 2)
-        self.assertEqual(_pkgs[1].name, _pkg['name'])
-
-    def test_save_reports(self):
-        _fetcher = model.Fetcher(
-                name='kiskadee-fetcher', target='university'
-            )
-        _pkg = model.Package(
-                name='kiskadee-package', fetcher_id=1
-            )
-        _version = model.Version(
-                number='1.0-rc1', package_id=1
-                )
-        _raw = {
-            'results': [
-                {'severity': 'warning'},
-                {'severity': 'style'},
-                {'severity': 'error'}
-            ]
-        }
-        _analysis = Analysis(
-                version_id=1,
-                analyzer_id=1,
-                id=1,
-                raw=_raw
-            )
-        self.session.add(_fetcher)
-        self.session.add(_pkg)
-        self.session.add(_version)
-        self.session.add(_analysis)
-        self.session.commit()
-        _dict_analysis = {
-                'results': _raw['results'],
-                'id': 1
-            }
-        self.monitor._save_reports(_dict_analysis, self.pkg1, 'cppcheck')
-        _report = self.monitor.session.query(Report).all()
-        self.assertEqual(len(_report), 1)
-
-    def test_save_version(self):
-        self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher())
-        self.monitor._save_analyzed_pkg(self.pkg1)
-        _pkgs = self.monitor.session.query(Package).all()
-        _version = _pkgs[0].versions[0].number
-        self.assertEqual(_version, self.pkg1['version'])
-
-    def test_update_version(self):
-        self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher())
-        packages_queue.put(self.pkg1)
-        packages_queue.put(self.pkg3)
-
-        _pkg = self.monitor.dequeue_package()
-        self.monitor._save_analyzed_pkg(_pkg)
-
-        _pkg = self.monitor.dequeue_package()
-        self.monitor._save_analyzed_pkg(_pkg)
-
-        _pkgs = self.monitor.session.query(Package).all()
-        self.assertEqual(len(_pkgs), 1)
-
-        _pkg_versions = self.monitor._query(Package).\
-            filter(Package.name == _pkg['name']).first().versions
-
-        _first_version = _pkg_versions[0].number
-        _current_version = _pkg_versions[-1].number
-        self.assertEqual(self.pkg1['version'], _first_version)
-        self.assertEqual(_pkg['version'], _current_version)
-
-    def test_save_package_anitya(self):
-        self.monitor._save_fetcher(kiskadee.fetchers.anitya.Fetcher())
-        packages_queue.put(self.pkg4)
-
-        _pkg = self.monitor.dequeue_package()
-        self.monitor._save_analyzed_pkg(_pkg)
-
-        _pkgs = self.monitor.session.query(Package).all()
-        self.assertEqual(len(_pkgs), 1)
-        self.assertEqual(_pkgs[0].homepage, _pkg['meta']['homepage'])
-
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/kiskadee/tests/test_plugins.py b/kiskadee/tests/test_plugins.py
deleted file mode 100644
index 873581a..0000000
--- a/kiskadee/tests/test_plugins.py
+++ /dev/null
@@ -1,207 +0,0 @@
-import unittest
-import os
-import shutil
-import tempfile
-import zmq
-import threading
-import time
-
-import kiskadee.queue
-import kiskadee
-
-
-class FetchersTestCase(unittest.TestCase):
-    def test_loading(self):
-        _config = kiskadee.config
-        _config['debian_fetcher'] = {'active': 'no'}
-        _config['juliet_fetcher'] = {'active': 'yes'}
-        _config['example_fetcher'] = {'active': 'yes'}
-        kiskadee.config = _config
-        fetchers = kiskadee.load_fetchers()
-        for fetcher in fetchers:
-            name_index = len(fetcher.__name__.split('.')) - 1
-            name = fetcher.__name__.split('.')[name_index]
-            self.assertTrue(name != 'debian')
-        kiskadee.config['example_fetcher'] = {
-                'target': 'example',
-                'description': 'SAMATE Juliet test suite',
-                'analyzers': 'cppcheck flawfinder',
-                'active': 'yes'
-            }
-
-
-class DebianFetcherTestCase(unittest.TestCase):
-
-    def setUp(self):
-        import kiskadee.fetchers.debian
-        self.debian_fetcher = kiskadee.fetchers.debian.Fetcher()
-        self.data = self.debian_fetcher.config
-
-    def _download_sources_gz(self):
-        tmp_path = tempfile.gettempdir()
-        path = tempfile.mkdtemp(dir=tmp_path)
-        source = 'kiskadee/tests/test_source/Sources.gz'
-        shutil.copy2(source, path)
-        return path
-
-    def test_mount_sources_gz_url(self):
-        mirror = self.data['target']
-        release = self.data['release']
-        url = self.debian_fetcher._sources_gz_url()
-        expected_url = "%s/dists/%s/main/source/Sources.gz" % (mirror, release)
-        self.assertEqual(url, expected_url)
-
-    def test_uncompress_sources_gz(self):
-        tmp_path = tempfile.gettempdir()
-        temp_dir = tempfile.mkdtemp(dir=tmp_path)
-        self.debian_fetcher._download_sources_gz = self._download_sources_gz
-        temp_dir = self.debian_fetcher._download_sources_gz()
-        self.debian_fetcher._uncompress_gz(temp_dir)
-        files = os.listdir(temp_dir)
-        shutil.rmtree(temp_dir)
-        self.assertTrue('Sources' in files)
-
-    def test_enqueue_a_valid_pkg(self):
-        tmp_path = tempfile.gettempdir()
-        temp_dir = tempfile.mkdtemp(dir=tmp_path)
-        self.debian_fetcher._download_sources_gz = self._download_sources_gz
-        temp_dir = self.debian_fetcher._download_sources_gz()
-        self.debian_fetcher._uncompress_gz(temp_dir)
-        self.debian_fetcher._queue_sources_gz_pkgs(temp_dir)
-        shutil.rmtree(temp_dir)
-
-        some_pkg = kiskadee.queue.packages_queue.get()
-        self.assertTrue(isinstance(some_pkg, dict))
-        self.assertIn('name', some_pkg)
-        self.assertIn('version', some_pkg)
-        self.assertIn('fetcher', some_pkg)
-        self.assertIn('meta', some_pkg)
-        self.assertIn('directory', some_pkg['meta'])
-
-    def test_mount_dsc_url(self):
-        expected_dsc_url = ("http://ftp.us.debian.org" +
-                            "/debian/pool/main/0/0ad/0ad_0.0.21-2.dsc")
-        sample_package = {'name': '0ad',
-                          'version': '0.0.21-2',
-                          'meta': {'directory': 'pool/main/0/0ad'}}
-        url = self.debian_fetcher._dsc_url(sample_package)
-        self.assertEqual(expected_dsc_url, url)
-
-    def test_compare_gt_version(self):
-        new = '1.1.1'
-        old = '1.1.0'
-        result = self.debian_fetcher.compare_versions(new, old)
-        self.assertTrue(result)
-
-    def test_compare_smallest_version(self):
-        new = '8.5-2'
-        old = '8.6-0'
-        result = self.debian_fetcher.compare_versions(new, old)
-        self.assertFalse(result)
-
-    def test_compare_equal_version(self):
-        new = '3.3.3-0'
-        old = '3.3.3-0'
-        result = self.debian_fetcher.compare_versions(new, old)
-        self.assertFalse(result)
-
-
-class TestAnityaFetcher(unittest.TestCase):
-
-    def setUp(self):
-        import kiskadee.fetchers.anitya
-        self.anitya_fetcher = kiskadee.fetchers.anitya.Fetcher()
-
-        self.msg = "anitya {'body':{'msg':{'project':{name: 'urlscan',"\
-                   "'version':'0.8.5','backend':'GitHub',"\
-                   "'homepage':'https://github.com/firecat53/urlscan'}}}}"
-
-        self.msg1 = "{'body':{'msg':{'project':{name: 'urlscan',"\
-                    "'version':'0.8.5','backend':'GitHub',"\
-                    "'homepage':'https://github.com/firecat53/urlscan'}}}}"
-
-    def test_connect_to_zmq(self):
-
-        def zmq_server():
-            context = zmq.Context()
-            socket = context.socket(zmq.PUB)
-            socket.bind("tcp://*:7776")
-
-        zmq_server()
-        socket = self.anitya_fetcher._connect_to_zmq("7776", "anitya")
-        self.assertIsNotNone(socket)
-
-    def test_receive_msg_from_zmq(self):
-        """definitely this is not a unit test, but is important to kiskadee
-        be able to interact correctly with ZeroMQ.
-        We need to define other test levels to kiskadee asap.
-        When we do that, we can move integration tests
-        to a proper place. For now we will maintain this test here"""
-
-        def zmq_server():
-            context = zmq.Context()
-            socket = context.socket(zmq.PUB)
-            socket.bind("tcp://*:7776")
-            time.sleep(1)
-            socket.send_string("%s" % (self.msg))
-            time.sleep(1)
-
-        def receive_msg_from_server():
-            client_socket = self.anitya_fetcher._connect_to_zmq(
-                    "7776", "anitya")
-            if client_socket:
-                response = client_socket.recv_string()
-                results[0] = response[response.find(" ")+1::]
-            else:
-                results[0] = "invalid"
-
-        results = [None]
-
-        client_as_thread = threading.Thread(target=receive_msg_from_server)
-        server_as_thread = threading.Thread(
-                target=zmq_server)
-
-        server_as_thread.start()
-        client_as_thread.start()
-        server_as_thread.join()
-        self.assertEqual(self.msg1, results[0])
-
-    def test_compare_versions(self):
-        is_greater = self.anitya_fetcher.compare_versions('0.8.5-2', '0.8.5-1')
-        self.assertTrue(is_greater)
-
-    def test_load_backend(self):
-        backend = self.anitya_fetcher._load_backend('github')
-        self.assertIsNotNone(backend)
-
-    def test_not_load_backend(self):
-        backend = self.anitya_fetcher._load_backend('foo')
-        self.assertEqual(backend, {})
-
-    def test_get_sources(self):
-
-        def mock_github(self, fetcher, source_data, path):
-            return 'kiskadee/tests/test_source/Sources.gz'
-
-        kiskadee.fetchers.anitya.Backends.github = mock_github
-        source_data = {'meta': {'backend': 'GitHub'}}
-        source_path = self.anitya_fetcher.get_sources(source_data)
-        self.assertEqual(source_path,
-                         mock_github("self", "fetcher", "foo", "bla"))
-
-    def test_create_package_dict(self):
-
-        self.anitya_fetcher._create_package_dict(self.msg)
-        _dict = kiskadee.queue.packages_queue.get()
-        self.assertEqual(_dict['name'], 'urlscan')
-        self.assertEqual(_dict['version'], '0.8.5')
-        self.assertEqual(_dict['meta']['backend'], 'GitHub')
-        self.assertEqual(
-                _dict['meta']['homepage'],
-                'https://github.com/firecat53/urlscan'
-        )
-        self.assertEqual(_dict['fetcher'].name, 'anitya')
-
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/kiskadee/tests/test_queue.py b/kiskadee/tests/test_queue.py
deleted file mode 100644
index 19abf96..0000000
--- a/kiskadee/tests/test_queue.py
+++ /dev/null
@@ -1,23 +0,0 @@
-import unittest
-import kiskadee.queue
-import kiskadee.fetchers.example
-
-
-class QueueTestCase(unittest.TestCase):
-
-    def test_enqueue_wrapper_pkg(self):
-
-        def my_dict():
-            return {
-                    'name': 'bar',
-                    'fetcher': kiskadee.fetchers.example.Fetcher(),
-                    'version': '1.0.0'
-                   }
-
-        enque_my_dict = kiskadee.queue.package_enqueuer(my_dict)
-        enque_my_dict()
-        self.assertTrue(isinstance(kiskadee.queue.packages_queue.get(), dict))
-
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/kiskadee/tests/test_report.py b/kiskadee/tests/test_report.py
deleted file mode 100644
index 10cb936..0000000
--- a/kiskadee/tests/test_report.py
+++ /dev/null
@@ -1,50 +0,0 @@
-import unittest
-
-from sqlalchemy.orm import sessionmaker
-from kiskadee.report import CppcheckReport, FlawfinderReport
-from kiskadee.database import Database
-
-
-class ReportTestCase(unittest.TestCase):
-
-    def setUp(self):
-        self.engine = Database('db_test').engine
-        Session = sessionmaker(bind=self.engine)
-        self.session = Session()
-
-    def tearDown(self):
-        self.session.close()
-
-    def test_compute_cppcheck_reports(self):
-        _reports = [
-                {'severity': 'warning'},
-                {'severity': 'error'},
-                {'severity': 'style'},
-                {'some-attribute': None}
-            ]
-        _cpp_reporter = CppcheckReport(_reports)
-        result = _cpp_reporter._compute_reports('cppcheck')
-        self.assertEqual(len(result.keys()), 3)
-        self.assertEqual(result['warning'], 1)
-        self.assertEqual(result['style'], 1)
-        self.assertEqual(result['error'], 1)
-
-    def test_compute_flawfinder_reports(self):
-        _reports = [
-                {'severity': '5'},
-                {'severity': '4'},
-                {'severity': '3'},
-                {'some-attribute': None}
-            ]
-        _flawfinder_reporter = FlawfinderReport(_reports)
-        result = _flawfinder_reporter._compute_reports('flawfinder')
-        self.assertEqual(len(result.keys()), 5)
-        self.assertEqual(result['severity_5'], 1)
-        self.assertEqual(result['severity_4'], 1)
-        self.assertEqual(result['severity_3'], 1)
-        self.assertEqual(result['severity_2'], 0)
-        self.assertEqual(result['severity_1'], 0)
-
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/kiskadee/tests/test_runner.py b/kiskadee/tests/test_runner.py
deleted file mode 100644
index 3776e89..0000000
--- a/kiskadee/tests/test_runner.py
+++ /dev/null
@@ -1,98 +0,0 @@
-import unittest
-import tempfile
-
-from kiskadee.runner import Runner
-import kiskadee.fetchers.example
-import kiskadee.fetchers.debian
-from sqlalchemy.orm import sessionmaker
-from kiskadee import model
-from kiskadee.queue import KiskadeeQueue
-from kiskadee.database import Database
-
-
-class AnalyzersTestCase(unittest.TestCase):
-
-    def setUp(self):
-        self.engine = Database('db_test').engine
-        Session = sessionmaker(bind=self.engine)
-        self.session = Session()
-        model.Base.metadata.create_all(self.engine)
-        model.create_analyzers(self.session)
-        self.fetcher = kiskadee.fetchers.debian.Fetcher()
-        self.deb_pkg = {'name': 'test',
-                        'version': '1.0.0',
-                        'fetcher': kiskadee.fetchers.debian.Fetcher()
-                        }
-        self.fetcher = model.Fetcher(
-                name='kiskadee-fetcher', target='university'
-            )
-        self.session.add(self.fetcher)
-        self.session.commit()
-        kiskadee_queue = KiskadeeQueue()
-        self.runner = Runner()
-        self.runner.kiskadee_queue = kiskadee_queue
-
-    def tearDown(self):
-        self.session.close()
-        model.Base.metadata.drop_all()
-
-    def test_run_analyzer(self):
-
-        source_to_analysis = {
-                'name': 'test',
-                'version': '1.0.0',
-                'fetcher': kiskadee.fetchers.example.Fetcher()
-        }
-
-        source_path = self.runner._path_to_uncompressed_source(
-                source_to_analysis, kiskadee.fetchers.example.Fetcher()
-            )
-        firehose_report = self.runner.analyze("cppcheck", source_path)
-        self.assertIsNotNone(firehose_report)
-
-    def test_generate_a_firehose_report(self):
-        source_to_analysis = {
-                'name': 'test',
-                'version': '1.0.0',
-                'fetcher': kiskadee.fetchers.example.Fetcher()
-        }
-
-        self.runner.call_analyzers(source_to_analysis)
-        analyzed_pkg = self.runner.kiskadee_queue.dequeue_result()
-
-        self.assertEqual(analyzed_pkg['name'], source_to_analysis['name'])
-        self.assertIn('cppcheck', analyzed_pkg['results'])
-        self.assertIn('flawfinder', analyzed_pkg['results'])
-
-    def test_path_to_uncompressed_source(self):
-
-        source_to_analysis = {
-                'name': 'test',
-                'version': '1.0.0',
-                'fetcher': kiskadee.fetchers.example
-        }
-
-        source_path = self.runner._path_to_uncompressed_source(
-                source_to_analysis, kiskadee.fetchers.example.Fetcher()
-        )
-        tmp_path = tempfile.gettempdir()
-        self.assertTrue(source_path.find(tmp_path) >= 0)
-        self.assertIsNotNone(source_path)
-
-    def test_invalid_path_to_uncompressed_source(self):
-
-        source_to_analysis = {
-                'name': 'test',
-                'version': '1.0.0',
-                'fetcher': kiskadee.fetchers.example
-        }
-
-        source_path = self.runner._path_to_uncompressed_source(
-                source_to_analysis, None
-        )
-
-        self.assertIsNone(source_path)
-
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/util/kiskadee.conf b/util/kiskadee.conf
index 5eb6ec8..c2dc351 100644
--- a/util/kiskadee.conf
+++ b/util/kiskadee.conf
@@ -19,7 +19,7 @@ port = 5432
 dbname = kiskadee_test
 
 [debian_fetcher]
-source = http://ftp.us.debian.org/debian
+target = http://ftp.us.debian.org/debian
 description = A fetcher to monitor Debian Repositories
 release = sid
 meta = Sources.gz
@@ -28,13 +28,13 @@ analyzers = cppcheck flawfinder
 active = no
 
 [juliet_fetcher]
-source = juliet
+target = juliet
 description = SAMATE Juliet test suite
 analyzers = cppcheck
 active = no
 
 [anitya_fetcher]
-source = release-monitoring.org
+target = release-monitoring.org
 description = A fetcher to monitor Anitya packages
 analyzers = cppcheck flawfinder
 active = no
@@ -42,7 +42,7 @@ zmq_port = 5556
 zmq_topic = anitya
 
 [example_fetcher]
-source = example
+target = example
 description = SAMATE Juliet test suite
 analyzers = cppcheck flawfinder clanganalyzer frama_c
 active = yes

From f307ccd458a2ffa1fb2f13561bd4fafa9cc16dcc Mon Sep 17 00:00:00 2001
From: David Carlos <ddavidcarlos1392@gmail.com>
Date: Mar 16 2018 02:30:29 +0000
Subject: [PATCH 4/17] Group tests into subpackages.


  - This will enable us to run each test suite separately.

---

diff --git a/kiskadee/fetchers/anitya.py b/kiskadee/fetchers/anitya.py
index 771fbb9..589d59c 100644
--- a/kiskadee/fetchers/anitya.py
+++ b/kiskadee/fetchers/anitya.py
@@ -93,7 +93,6 @@ class Fetcher(kiskadee.fetchers.Fetcher):
                         }
                 }
                 super().watch(**kiskadee_project)
-            
 
     def _event_to_dict(self, msg):
         msg = msg[msg.find(" ")+1::]
diff --git a/kiskadee/fetchers/example.py b/kiskadee/fetchers/example.py
index a4c1593..02eb9b2 100644
--- a/kiskadee/fetchers/example.py
+++ b/kiskadee/fetchers/example.py
@@ -9,8 +9,7 @@ class Fetcher(kiskadee.fetchers.Fetcher):
         """Use sources from test suite."""
         return 'kiskadee/tests/test_source/test_source.tar.gz'
 
-    @kiskadee.queue.enqueue_project
-    def watch(self):
+    def watch(self, **kwargs):
         """There is no proper API to inspect new example versions.
         It should not matter, since example will not receive updates.
         """
diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py
index fbf68bc..ef3fb77 100644
--- a/kiskadee/monitor.py
+++ b/kiskadee/monitor.py
@@ -53,32 +53,9 @@ class Monitor:
             if pkg:
                 self._send_to_runner(pkg)
             time.sleep(2)
-            analyzed_project = self.queues.dequeue_result()
+            analyzed_project = self.kiskadee_queue.dequeue_result()
             self._save_analyzed_project(analyzed_project)
 
-    def dequeue_package(self):
-        """Dequeue packages from packages_queue."""
-        if not kiskadee.queue.packages_queue.empty():
-            pkg = kiskadee.queue.packages_queue.get()
-            kiskadee.logger.debug(
-                    "MONITOR: Dequed Package: {}_{}"
-                    .format(pkg["name"], pkg["version"])
-                )
-            return pkg
-        return {}
-
-    # Use directly dequeue_result from kiskadee.queue
-    #def dequeue_result(self):
-    #    """Dequeue analyzed packages from result_queue."""
-    #    if not self.kiskadee_queue.results_empty():
-    #        pkg = self.kiskadee_queue.dequeue_result()
-    #        kiskadee.logger.debug(
-    #                "MONITOR: Dequed result for package : {}-{}"
-    #                .format(pkg["name"], pkg["version"])
-    #            )
-    #        return pkg
-    #    return {}
-
     def _send_to_runner(self, pkg):
         _name = pkg['fetcher'].split('.')[-1]
         _fetcher = self._query(Fetcher).filter_by(name = _name).first()
@@ -103,8 +80,8 @@ class Monitor:
                     self.kiskadee_queue.enqueue_analysis(pkg)
 
     # Move this to model.py
-    def _save_analyzed_project(self, project):
-        if not pkg:
+    def _save_analyzed_project(self, data):
+        if not data:
             return {}
         project = self._query(Package).filter_by(name = data['name']).first()
         if not project:
@@ -116,7 +93,7 @@ class Monitor:
             self._save_analysis(data, analyzer, result, project.versions[-1])
 
     # Move this to model.py
-    def _update_pkg(self, package, pkg):
+    def _update_project(self, package, pkg):
 
         if(package.versions[-1].number == pkg['version']):
             return package
@@ -138,7 +115,7 @@ class Monitor:
             return None
 
     # Move this to model.py
-    def _save_pkg(self, pkg):
+    def _save_project(self, pkg):
         homepage = None
         if ('meta' in pkg) and ('homepage' in pkg['meta']):
             homepage = pkg['meta']['homepage']
@@ -253,11 +230,11 @@ def daemon():
     runner = Runner()
     monitor_process = Process(
             target=monitor.monitor,
-            args=(queues)
+            args=(queues,)
         )
     runner_process = Process(
             target=runner.runner,
-            args=(queues)
+            args=(queues,)
         )
     monitor_process.start()
     runner_process.start()
diff --git a/kiskadee/queue.py b/kiskadee/queue.py
index a6835ea..214b631 100644
--- a/kiskadee/queue.py
+++ b/kiskadee/queue.py
@@ -1,5 +1,6 @@
 """Provide kiskadee queues and operations on them."""
 import time
+import dill
 from multiprocessing import Queue
 
 import kiskadee
@@ -23,10 +24,10 @@ def enqueue_project(fetcher_watch):
     """
     def wrapper(*args, **kwargs):
         project = fetcher_watch(*args, **kwargs)
-        kiskadee.queue.Queues.enqueue_project(project)
+        kiskadee.queue.queues.enqueue_project(project)
         fetcher = project['fetcher'].name
         kiskadee.logger.debug(
-                "{} fetcher: Sending package {}_{} for monitor"
+                "{} fetcher: sending package {}_{} for monitor"
                 .format(fetcher, package['name'], package['version'])
             )
         time.sleep(2)
@@ -40,15 +41,14 @@ class Queues():
     """Provide kiskadee queues objects."""
 
     @staticmethod
-    def enqueue_analysis(analysis):
+    def enqueue_analysis(project_to_analysis):
         """Put a analysis on the analysis queue."""
-        analysis.put(analysis)
+        analysis.put(project_to_analysis)
 
     @staticmethod
-    @empty_queue
     def dequeue_analysis():
         """Get a analysis from the analysis queue."""
-        return analysis.get(timeout=1)
+        return analysis.get()
 
     @staticmethod
     def enqueue_result(result):
diff --git a/kiskadee/tests/api/test_api.py b/kiskadee/tests/api/test_api.py
index 0b83d1a..e88fee5 100644
--- a/kiskadee/tests/api/test_api.py
+++ b/kiskadee/tests/api/test_api.py
@@ -1,10 +1,8 @@
 import json
 import unittest
-from sqlalchemy.orm import sessionmaker
 from unittest.mock import MagicMock
 
 import kiskadee
-from kiskadee.runner import Runner
 from kiskadee.monitor import Monitor
 import kiskadee.api.app
 import kiskadee.fetchers.example
diff --git a/kiskadee/tests/integration/test_monitor.py b/kiskadee/tests/integration/test_monitor.py
index aa95be6..67cd7c2 100644
--- a/kiskadee/tests/integration/test_monitor.py
+++ b/kiskadee/tests/integration/test_monitor.py
@@ -3,8 +3,8 @@ from sqlalchemy.orm import sessionmaker
 
 from kiskadee import model
 from kiskadee.monitor import Monitor
-from kiskadee.queue import packages_queue
-from kiskadee.model import Package, Fetcher, create_analyzers, Report, Analysis
+from kiskadee.queue import Queues
+from kiskadee.model import Package, create_analyzers, Report, Analysis
 import kiskadee.queue
 import kiskadee.fetchers.debian
 import kiskadee.fetchers.anitya
@@ -18,12 +18,13 @@ class MonitorTestCase(unittest.TestCase):
         Session = sessionmaker(bind=self.engine)
         self.session = Session()
         self.monitor = Monitor(self.session)
+        self.queues = Queues()
         model.Base.metadata.create_all(self.engine)
         create_analyzers(self.session)
         self.pkg1 = {
                 'name': 'curl',
                 'version': '7.52.1-5',
-                'fetcher': kiskadee.fetchers.debian,
+                'fetcher': kiskadee.fetchers.debian.__name__,
                 'meta': {'directory': 'pool/main/c/curl'},
                 'results': {
                     'cppcheck': '<>',
@@ -32,7 +33,7 @@ class MonitorTestCase(unittest.TestCase):
 
         self.pkg2 = {'name': 'urlscan',
                      'version': '0.8.2',
-                     'fetcher': kiskadee.fetchers.debian,
+                     'fetcher': kiskadee.fetchers.debian.__name__,
                      'meta': {'directory': 'pool/main/u/urlscan'},
                      'results': {
                             'cppcheck': '<>',
@@ -41,7 +42,7 @@ class MonitorTestCase(unittest.TestCase):
 
         self.pkg3 = {'name': 'curl',
                      'version': '7.52.2-5',
-                     'fetcher': kiskadee.fetchers.debian,
+                     'fetcher': kiskadee.fetchers.debian.__name__,
                      'meta': {'directory': 'pool/main/c/curl'},
                      'results': {
                             'cppcheck': '<>',
@@ -49,7 +50,7 @@ class MonitorTestCase(unittest.TestCase):
                      'fetcher_id': 1}
         self.pkg4 = {'name': 'urlanitya',
                      'version': '0.11',
-                     'fetcher': kiskadee.fetchers.anitya,
+                     'fetcher': kiskadee.fetchers.anitya.__name__,
                      'meta': {
                             'backend': 'gitHub',
                             'homepage': 'https://github.com/GesielFreitas/Cros'
@@ -78,31 +79,14 @@ class MonitorTestCase(unittest.TestCase):
         fetcher = kiskadee.fetchers.debian.Fetcher()
         self.assertEqual(fetcher.name, 'debian')
 
-    def test_save_some_fetcher(self):
-        fetcher = kiskadee.fetchers.debian.Fetcher()
-        self.monitor._save_fetcher(fetcher)
-        _fetchers = self.monitor.session.query(Fetcher).all()
-        self.assertEqual(len(_fetchers), 1)
-        self.assertEqual(_fetchers[0].name, 'debian')
-        self.assertEqual(_fetchers[0].description,
-                         fetcher.config['description'])
-
     def test_save_package(self):
         self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher())
-        packages_queue.put(self.pkg1)
-        packages_queue.put(self.pkg2)
-
-        _pkg = self.monitor.dequeue_package()
-        self.monitor._save_analyzed_pkg(_pkg)
-        _pkgs = self.monitor.session.query(Package).all()
-        self.assertEqual(len(_pkgs), 1)
-        self.assertEqual(_pkgs[0].name, _pkg['name'])
-
-        _pkg = self.monitor.dequeue_package()
-        self.monitor._save_analyzed_pkg(_pkg)
+        self.monitor._save_analyzed_project(self.pkg1)
+        self.monitor._save_analyzed_project(self.pkg2)
         _pkgs = self.monitor.session.query(Package).all()
         self.assertEqual(len(_pkgs), 2)
-        self.assertEqual(_pkgs[1].name, _pkg['name'])
+        self.assertEqual(_pkgs[0].name, self.pkg1['name'])
+        self.assertEqual(_pkgs[1].name, self.pkg2['name'])
 
     def test_save_reports(self):
         _fetcher = model.Fetcher(
@@ -142,21 +126,21 @@ class MonitorTestCase(unittest.TestCase):
 
     def test_save_version(self):
         self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher())
-        self.monitor._save_analyzed_pkg(self.pkg1)
+        self.monitor._save_analyzed_project(self.pkg1)
         _pkgs = self.monitor.session.query(Package).all()
         _version = _pkgs[0].versions[0].number
         self.assertEqual(_version, self.pkg1['version'])
 
     def test_update_version(self):
         self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher())
-        packages_queue.put(self.pkg1)
-        packages_queue.put(self.pkg3)
+        self.queues.enqueue_project(self.pkg1)
+        self.queues.enqueue_project(self.pkg3)
 
-        _pkg = self.monitor.dequeue_package()
-        self.monitor._save_analyzed_pkg(_pkg)
+        _pkg = self.queues.dequeue_project()
+        self.monitor._save_analyzed_project(_pkg)
 
-        _pkg = self.monitor.dequeue_package()
-        self.monitor._save_analyzed_pkg(_pkg)
+        _pkg = self.queues.dequeue_project()
+        self.monitor._save_analyzed_project(_pkg)
 
         _pkgs = self.monitor.session.query(Package).all()
         self.assertEqual(len(_pkgs), 1)
@@ -171,11 +155,9 @@ class MonitorTestCase(unittest.TestCase):
 
     def test_save_package_anitya(self):
         self.monitor._save_fetcher(kiskadee.fetchers.anitya.Fetcher())
-        packages_queue.put(self.pkg4)
-
-        _pkg = self.monitor.dequeue_package()
-        self.monitor._save_analyzed_pkg(_pkg)
-
+        self.queues.enqueue_project(self.pkg4)
+        _pkg = self.queues.dequeue_project()
+        self.monitor._save_analyzed_project(_pkg)
         _pkgs = self.monitor.session.query(Package).all()
         self.assertEqual(len(_pkgs), 1)
         self.assertEqual(_pkgs[0].homepage, _pkg['meta']['homepage'])
diff --git a/kiskadee/tests/integration/test_runner.py b/kiskadee/tests/integration/test_runner.py
index 0ad1e0a..f9a2e16 100644
--- a/kiskadee/tests/integration/test_runner.py
+++ b/kiskadee/tests/integration/test_runner.py
@@ -1,3 +1,8 @@
+"""
+This integration tests needs docker engine be running and accessible
+by the user that is executing the tests. It also needs selinux running in
+a permissive mode (setenforce 0)
+"""
 import unittest
 import tempfile
 
@@ -6,12 +11,11 @@ import kiskadee.fetchers.example
 import kiskadee.fetchers.debian
 from sqlalchemy.orm import sessionmaker
 from kiskadee import model
-from kiskadee.queue import KiskadeeQueue
+from kiskadee.queue import Queues
 from kiskadee.database import Database
 
 
 class AnalyzersTestCase(unittest.TestCase):
-
     def setUp(self):
         self.engine = Database('db_test').engine
         Session = sessionmaker(bind=self.engine)
@@ -21,16 +25,15 @@ class AnalyzersTestCase(unittest.TestCase):
         self.fetcher = kiskadee.fetchers.debian.Fetcher()
         self.deb_pkg = {'name': 'test',
                         'version': '1.0.0',
-                        'fetcher': kiskadee.fetchers.debian.Fetcher()
+                        'fetcher': kiskadee.fetchers.debian.__name__
                         }
         self.fetcher = model.Fetcher(
                 name='kiskadee-fetcher2', target='university'
             )
         self.session.add(self.fetcher)
         self.session.commit()
-        kiskadee_queue = KiskadeeQueue()
         self.runner = Runner()
-        self.runner.kiskadee_queue = kiskadee_queue
+        self.runner.queues = Queues()
 
     def tearDown(self):
         self.session.close()
@@ -54,12 +57,11 @@ class AnalyzersTestCase(unittest.TestCase):
         source_to_analysis = {
                 'name': 'test',
                 'version': '1.0.0',
-                'fetcher': kiskadee.fetchers.example.Fetcher()
+                'fetcher': kiskadee.fetchers.example.__name__
         }
 
         self.runner.call_analyzers(source_to_analysis)
-        analyzed_pkg = self.runner.kiskadee_queue.dequeue_result()
-
+        analyzed_pkg = self.runner.queues.dequeue_result()
         self.assertEqual(analyzed_pkg['name'], source_to_analysis['name'])
         self.assertIn('cppcheck', analyzed_pkg['results'])
         self.assertIn('flawfinder', analyzed_pkg['results'])
@@ -93,5 +95,6 @@ class AnalyzersTestCase(unittest.TestCase):
 
         self.assertIsNone(source_path)
 
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/kiskadee/tests/plugins/test_plugins.py b/kiskadee/tests/plugins/test_plugins.py
index 030da9a..9fe8262 100644
--- a/kiskadee/tests/plugins/test_plugins.py
+++ b/kiskadee/tests/plugins/test_plugins.py
@@ -70,7 +70,7 @@ class DebianFetcherTestCase(unittest.TestCase):
         self.debian_fetcher._queue_sources_gz_pkgs(temp_dir)
         shutil.rmtree(temp_dir)
 
-        some_pkg = kiskadee.queue.packages_queue.get()
+        some_pkg = kiskadee.queue.Queues.dequeue_project()
         self.assertTrue(isinstance(some_pkg, dict))
         self.assertIn('name', some_pkg)
         self.assertIn('version', some_pkg)
@@ -190,8 +190,8 @@ class TestAnityaFetcher(unittest.TestCase):
 
     def test_create_package_dict(self):
 
-        self.anitya_fetcher._create_package_dict(self.msg)
-        _dict = kiskadee.queue.packages_queue.get()
+        self.anitya_fetcher.project_to_enqueue(self.msg)
+        _dict = kiskadee.queue.Queues().dequeue_project()
         self.assertEqual(_dict['name'], 'urlscan')
         self.assertEqual(_dict['version'], '0.8.5')
         self.assertEqual(_dict['meta']['backend'], 'GitHub')
@@ -199,7 +199,7 @@ class TestAnityaFetcher(unittest.TestCase):
                 _dict['meta']['homepage'],
                 'https://github.com/firecat53/urlscan'
         )
-        self.assertEqual(_dict['fetcher'].name, 'anitya')
+        self.assertEqual(_dict['fetcher'], 'kiskadee.fetchers.anitya')
 
 
 if __name__ == '__main__':
diff --git a/playbook/roles/common/tasks/main.yml b/playbook/roles/common/tasks/main.yml
index 5c17a0c..edcdba0 100644
--- a/playbook/roles/common/tasks/main.yml
+++ b/playbook/roles/common/tasks/main.yml
@@ -12,4 +12,5 @@
     - postgresql-server
     - postgresql-contrib
     - libselinux-python
+    - devscripts
   ignore_errors: yes

From 8bd0fcfba03a489d25ce33954d41a80ba93c6446 Mon Sep 17 00:00:00 2001
From: David Carlos <ddavidcarlos1392@gmail.com>
Date: Mar 16 2018 02:30:29 +0000
Subject: [PATCH 5/17] Add make rules to run tests.


---

diff --git a/Makefile b/Makefile
index 6483460..cffabee 100644
--- a/Makefile
+++ b/Makefile
@@ -5,9 +5,21 @@ help:
 	@printf "Available targets: check, analyzers, clean\n\n"
 
 check:
-	coverage run --omit="lib/*","setup.py","kiskadee/tests/*",".eggs/*",".venv/*" ./setup.py test
+	coverage run --omit="lib/*","setup.py","kiskadee/tests/*",".eggs/*",".venv/*","/usr/*" -m unittest kiskadee/tests/*/*.py
 	coverage html
 
+check_units:
+	python3 -m unittest kiskadee/tests/units/*.py
+
+check_integration:
+	python3 -m unittest kiskadee/tests/integration/*.py
+
+check_api:
+	python3 -m unittest kiskadee/tests/api/*.py
+
+check_plugins:
+	python3 -m unittest kiskadee/tests/plugins/*.py
+
 analyzers:
 	docker ps 2> /dev/null; \
 	if [ $$? -ne 1 ]; then \

From 0c8e2977ccd6fe4b4d48340a75c92dddb06e4d23 Mon Sep 17 00:00:00 2001
From: David Carlos <ddavidcarlos1392@gmail.com>
Date: Mar 16 2018 02:30:29 +0000
Subject: [PATCH 6/17] Minor fixes to run ansible deploy.


  - Add some documentation to run deploy locally.

---

diff --git a/Vagrantfile b/Vagrantfile
index 24daa82..41fa18a 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -29,7 +29,8 @@ Vagrant.configure(2) do |config|
     # https://developer.fedoraproject.org/tools/vagrant/vagrant-nfs.html
     # After enable nfs-server, you will need to start the service.
     core.vm.synced_folder "./", "/home/vagrant/kiskadee", type: "nfs", nfs_udp: false
-    core.vm.network "public_network", ip: ips[:core]
+    core.vm.network "public_network", ip: ips[:core], 
+    :bridge => 'virbr0', :dev => 'virbr0'
     core.vm.provider "libvirt" do |v|
       v.memory = 2048
     end
diff --git a/doc/installing.rst b/doc/installing.rst
index d5c850c..946e2c3 100644
--- a/doc/installing.rst
+++ b/doc/installing.rst
@@ -1,7 +1,24 @@
 Installing kiskadee
 ===================
 
-Nothing to show here yet :(
+Development
+-----------
+To install locally run:
 
-We are still packaging kiskadee and writing proper ansible playbooks for fully
-automated deployment :)
+First, make shure that ansible will be able to login on the vm. ansible will
+use the root user to do this, so you will have to add your public ssh
+key inside the root ~/.ssh/authorized_keys file. You can also create the host
+user inside the vm, in order to be able to test the vm access with the ping
+command.
+
+To check if ansible can access the machine:
+
+.. code-block:: bash
+
+    ansible -i playbook/hosts.local all  -m ping
+
+To deploy kiskadee locally:
+
+.. code-block:: bash
+
+    ansible-playbook  -c paramiko -i playbook/hosts.local playbook/local.yml
diff --git a/playbook/hosts.local b/playbook/hosts.local
index 2b078eb..05f11e9 100644
--- a/playbook/hosts.local
+++ b/playbook/hosts.local
@@ -1,5 +1,5 @@
 [kiskadee-core]
-core-host ansible_host=10.18.0.115
+core-host ansible_host=192.168.121.6 ansible_user=root
 
 [kiskadee-frontend]
 frontend-host ansible_host=10.18.0.116
diff --git a/playbook/roles/kiskadee/tasks/main.yml b/playbook/roles/kiskadee/tasks/main.yml
index c335027..02907e5 100644
--- a/playbook/roles/kiskadee/tasks/main.yml
+++ b/playbook/roles/kiskadee/tasks/main.yml
@@ -1,15 +1,3 @@
-- name: Create kiskadee group
-  group:
-    name: kiskadee
-    state: present
-
-- name: Create kiskadee user
-  user:
-    name: kiskadee
-    groups: kiskadee, wheel
-    append: yes
-    shell: /bin/bash
-
 # TODO: do not turn selinux off. Add rules instead
 - name: Turn off selinux
   command: setenforce 0

From c1ff5c8e30eeec589aab3bdf92fc4bc59a767d67 Mon Sep 17 00:00:00 2001
From: David Carlos <ddavidcarlos1392@gmail.com>
Date: Mar 16 2018 02:30:29 +0000
Subject: [PATCH 7/17] Move code from monitor to model class.


  - All code that was responsible to save data on the database were
  moved to model.py, this made monitor class more cohesive.

---

diff --git a/Vagrantfile b/Vagrantfile
index 41fa18a..617b439 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -29,8 +29,9 @@ Vagrant.configure(2) do |config|
     # https://developer.fedoraproject.org/tools/vagrant/vagrant-nfs.html
     # After enable nfs-server, you will need to start the service.
     core.vm.synced_folder "./", "/home/vagrant/kiskadee", type: "nfs", nfs_udp: false
-    core.vm.network "public_network", ip: ips[:core], 
+    core.vm.network "public_network", ip: ips[:core],
     :bridge => 'virbr0', :dev => 'virbr0'
+    core.vm.network "forwarded_port", guest: 5000, host: 5000
     core.vm.provider "libvirt" do |v|
       v.memory = 2048
     end
diff --git a/kiskadee/model.py b/kiskadee/model.py
index 6aa3233..4e4dff2 100644
--- a/kiskadee/model.py
+++ b/kiskadee/model.py
@@ -3,7 +3,10 @@
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import Column, Integer, UnicodeText, UniqueConstraint,\
                        Sequence, Unicode, ForeignKey, orm, JSON
+import json
+
 import kiskadee
+from kiskadee.report import CppcheckReport, FlawfinderReport
 
 Base = declarative_base()
 
@@ -27,6 +30,47 @@ class Package(Base):
             UniqueConstraint('name', 'fetcher_id'),
             )
 
+    @staticmethod
+    def save(session, data):
+        homepage = None
+        if ('meta' in data) and ('homepage' in data['meta']):
+            homepage = data['meta']['homepage']
+
+        _package = Package(
+                name=data['name'],
+                homepage=homepage,
+                fetcher_id=data['fetcher_id']
+                )
+        session.add(_package)
+        session.commit()
+        _version = Version(number=data['version'],
+                           package_id=_package.id)
+        session.add(_version)
+        session.commit()
+        return _package
+
+    @staticmethod
+    def update(session, project, data):
+
+        if(project.versions[-1].number == data['version']):
+            return project
+        try:
+            _new_version = Version(
+                    number=data['version'],
+                    project_id=project.id
+                    )
+            project.versions.append(_new_version)
+            session.add(project)
+            session.commit()
+            kiskadee.logger.debug(
+                    "MONITOR: Sending project {}_{}"
+                    "for analysis".format(data['name'], data['version'])
+                    )
+            return project
+        except ValueError:
+            kiskadee.logger.debug("MONITOR: Could not compare versions")
+            return None
+
 
 class Fetcher(Base):
     """kiskadee fetcher abstraction."""
@@ -64,6 +108,23 @@ class Analyzer(Base):
     version = Column(Unicode(255), nullable=True)
     analysis = orm.relationship('Analysis', backref='analyzers')
 
+    @staticmethod
+    def create_analyzers(_session):
+        """Create the analyzers on database.
+
+        The kiskadee analyzers are defined on the section `analyzers` of the
+        kiskadee.conf file. The `_session` argument represents a sqlalchemy
+        session.
+        """
+        list_of_analyzers = dict(kiskadee.config._sections["analyzers"])
+        for _name, _version in list_of_analyzers.items():
+            if not _session.query(Analyzer)\
+                            .filter_by(name = _name, version = _version )\
+                            .first():
+                new_analyzer = kiskadee.model.Analyzer(name = _name,
+                                                       version = _version)
+                _session.add(new_analyzer)
+        _session.commit()
 
 class Analysis(Base):
     """Abstraction of a package analysis."""
@@ -77,10 +138,44 @@ class Analysis(Base):
     report = orm.relationship('Report',
                               uselist=False, back_populates='analysis')
 
+    @staticmethod
+    def save(session, data, analyzer, result, version):
+        _analysis = kiskadee.model.Analysis()
+        try:
+            _analyzer = session.query(kiskadee.model.Analyzer).\
+                    filter_by(name = analyzer).first()
+            _analysis.analyzer_id = _analyzer.id
+            _analysis.version_id = version.id
+            _analysis.raw = json.loads(result)
+            session.add(_analysis)
+            session.commit()
+            dict_analysis = {
+                    'results': _analysis.raw['results'],
+                    'id': _analysis.id
+                }
+            Report.save(session, dict_analysis, data, _analyzer.name)
+            kiskadee.logger.debug(
+                    "MONITOR: Saved analysis done by {} for package: {}-{}"
+                    .format(analyzer, data["name"], data["version"])
+                )
+            return
+        except Exception as err:
+            kiskadee.logger.debug(
+                    "MONITOR: The required analyzer was " +
+                    "not registered in kiskadee"
+                )
+            kiskadee.logger.debug(err)
+            return None
+
 
 class Report(Base):
     """Abstraction of a analysis report."""
 
+    REPORTERS = {
+        'cppcheck': CppcheckReport,
+        'flawfinder': FlawfinderReport
+    }
+
     __tablename__ = 'reports'
     id = Column(Integer,
                 Sequence('reports_id_seq', optional=True), primary_key=True)
@@ -88,20 +183,34 @@ class Report(Base):
     results = Column(JSON)
     analysis = orm.relationship('Analysis', back_populates='report')
 
+    @staticmethod
+    def save(session, analysis, data, analyzer_name):
+        try:
+            results = analysis['results']
+            analyzer_report = Report.REPORTERS[analyzer_name](results)
+            _reports = Report()
+            _reports.results = json.dumps(
+                    analyzer_report
+                    ._compute_reports(analyzer_name)
+                )
+            _reports.analysis_id = analysis['id']
+            session.add(_reports)
+            session.commit()
+            kiskadee.logger.debug(
+                    "MONITOR: Saved analysis reports for {} package"
+                    .format(data["name"])
+                )
+        except KeyError as key:
+            kiskadee.logger.debug(
+                    "ERROR: There's no reporter " +
+                    "to get reports from {} analyzer. ".format(key) +
+                    "Make shure to import or implement them."
+                )
+        except Exception as err:
+            kiskadee.logger.debug(
+                    "MONITOR: Failed to get analysis reports to {} package"
+                    .format(data["name"])
+                )
+            kiskadee.logger.debug(err)
+        return
 
-def create_analyzers(_session):
-    """Create the analyzers on database.
-
-    The kiskadee analyzers are defined on the section `analyzers` of the
-    kiskadee.conf file. The `_session` argument represents a sqlalchemy
-    session.
-    """
-    list_of_analyzers = dict(kiskadee.config._sections["analyzers"])
-    for name, version in list_of_analyzers.items():
-        if not (_session.query(Analyzer).filter(Analyzer.name == name).
-                filter(Analyzer.version == version).first()):
-            new_analyzer = kiskadee.model.Analyzer()
-            new_analyzer.name = name
-            new_analyzer.version = version
-            _session.add(new_analyzer)
-    _session.commit()
diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py
index ef3fb77..20afeef 100644
--- a/kiskadee/monitor.py
+++ b/kiskadee/monitor.py
@@ -7,22 +7,15 @@ import threading
 from multiprocessing import Process
 import time
 import os
-import json
 import importlib
 
 import kiskadee.database
-from kiskadee.report import CppcheckReport, FlawfinderReport
 from kiskadee.runner import Runner
 import kiskadee.queue
-from kiskadee.model import Package, Fetcher, Version, Report
+from kiskadee.model import Package, Fetcher, Version, Report, Analysis
 
 RUNNING = True
 
-REPORTERS = {
-    'cppcheck': CppcheckReport,
-    'flawfinder': FlawfinderReport
-}
-
 class Monitor:
     """Provide kiskadee monitoring objects."""
 
@@ -51,163 +44,42 @@ class Monitor:
             pkg = self.kiskadee_queue.dequeue_project()
 
             if pkg:
-                self._send_to_runner(pkg)
+                self.send_to_runner(pkg)
             time.sleep(2)
             analyzed_project = self.kiskadee_queue.dequeue_result()
-            self._save_analyzed_project(analyzed_project)
+            self.save_analyzed_project(analyzed_project)
 
-    def _send_to_runner(self, pkg):
-        _name = pkg['fetcher'].split('.')[-1]
+    def send_to_runner(self, data):
+        _name = data['fetcher'].split('.')[-1]
         _fetcher = self._query(Fetcher).filter_by(name = _name).first()
-        _package = (
-                self._query(Package)
-                .filter(Package.name == pkg['name']).first()
-            )
+        _package = self._query(Package).filter_by(name = data['name']).first()
 
         if _fetcher:
-            pkg["fetcher_id"] = _fetcher.id
+            data["fetcher_id"] = _fetcher.id
             if not _package:
                 kiskadee.logger.debug(
                         "MONITOR: Sending package {}_{} "
-                        " for analysis".format(pkg['name'], pkg['version'])
+                        " for analysis".format(data['name'], data['version'])
                 )
-                self.kiskadee_queue.enqueue_analysis(pkg)
+                self.kiskadee_queue.enqueue_analysis(data)
             else:
-                new_version = pkg['version']
+                new_version = data['version']
                 analysed_version = _package.versions[-1].number
-                fetcher = importlib.import_module(pkg['fetcher']).Fetcher()
+                fetcher = importlib.import_module(data['fetcher']).Fetcher()
                 if (fetcher.compare_versions(new_version, analysed_version)):
-                    self.kiskadee_queue.enqueue_analysis(pkg)
+                    self.kiskadee_queue.enqueue_analysis(data)
 
-    # Move this to model.py
-    def _save_analyzed_project(self, data):
+    def save_analyzed_project(self, data):
         if not data:
             return {}
         project = self._query(Package).filter_by(name = data['name']).first()
         if not project:
-            project = self._save_project(data)
+            project = Package.save(self.session, data)
         if project:
-            project = self._update_project(project, data)
+            project = Package.update(self.session, project, data)
 
         for analyzer, result in data['results'].items():
-            self._save_analysis(data, analyzer, result, project.versions[-1])
-
-    # Move this to model.py
-    def _update_project(self, package, pkg):
-
-        if(package.versions[-1].number == pkg['version']):
-            return package
-        try:
-            _new_version = Version(
-                    number=pkg['version'],
-                    package_id=package.id
-                    )
-            package.versions.append(_new_version)
-            self.session.add(package)
-            self.session.commit()
-            kiskadee.logger.debug(
-                    "MONITOR: Sending package {}_{}"
-                    "for analysis".format(pkg['name'], pkg['version'])
-                    )
-            return package
-        except ValueError:
-            kiskadee.logger.debug("MONITOR: Could not compare versions")
-            return None
-
-    # Move this to model.py
-    def _save_project(self, pkg):
-        homepage = None
-        if ('meta' in pkg) and ('homepage' in pkg['meta']):
-            homepage = pkg['meta']['homepage']
-
-        _package = Package(
-                name=pkg['name'],
-                homepage=homepage,
-                fetcher_id=pkg['fetcher_id']
-                )
-        self.session.add(_package)
-        self.session.commit()
-        _version = Version(number=pkg['version'],
-                           package_id=_package.id)
-        self.session.add(_version)
-        self.session.commit()
-        return _package
-
-    # Move this to model.py
-    def _save_reports(self, analysis, pkg, analyzer_name):
-        try:
-            results = analysis['results']
-            analyzer_report = REPORTERS[analyzer_name](results)
-            _reports = Report()
-            _reports.results = json.dumps(
-                    analyzer_report
-                    ._compute_reports(analyzer_name)
-                )
-            _reports.analysis_id = analysis['id']
-            self.session.add(_reports)
-            self.session.commit()
-            kiskadee.logger.debug(
-                    "MONITOR: Saved analysis reports for {} package"
-                    .format(pkg["name"])
-                )
-        except KeyError as key:
-            kiskadee.logger.debug(
-                    "ERROR: There's no reporter " +
-                    "to get reports from {} analyzer. ".format(key) +
-                    "Make shure to import or implement them."
-                )
-        except Exception as err:
-            kiskadee.logger.debug(
-                    "MONITOR: Failed to get analysis reports to {} package"
-                    .format(pkg["name"])
-                )
-            kiskadee.logger.debug(err)
-        return
-
-    # Move this to model.py
-    def _save_analysis(self, pkg, analyzer, result, version):
-        _analysis = kiskadee.model.Analysis()
-        try:
-            _analyzer = self._query(kiskadee.model.Analyzer).\
-                    filter_by(name = analyzer).first()
-            _analysis.analyzer_id = _analyzer.id
-            _analysis.version_id = version.id
-            _analysis.raw = json.loads(result)
-            self.session.add(_analysis)
-            self.session.commit()
-            dict_analysis = {
-                    'results': _analysis.raw['results'],
-                    'id': _analysis.id
-                }
-            self._save_reports(dict_analysis, pkg, _analyzer.name)
-            kiskadee.logger.debug(
-                    "MONITOR: Saved analysis done by {} for package: {}-{}"
-                    .format(analyzer, pkg["name"], pkg["version"])
-                )
-            return
-        except Exception as err:
-            kiskadee.logger.debug(
-                    "MONITOR: The required analyzer was " +
-                    "not registered in kiskadee"
-                )
-            kiskadee.logger.debug(err)
-            return None
-
-    # Move this to model.py
-    def _save_fetcher(self, fetcher):
-        name = fetcher.name
-        kiskadee.logger.debug(
-                "MONITOR: Saving {} fetcher in database".format(name)
-            )
-        if not self.session.query(Fetcher)\
-                .filter(Fetcher.name == name).first():
-            _fetcher = Fetcher(
-                    name=name,
-                    target=fetcher.config['target'],
-                    description=fetcher.config['description']
-                )
-            self.session.add(_fetcher)
-            self.session.commit()
+            Analysis.save(self.session, data, analyzer, result, project.versions[-1])
 
     def _query(self, arg):
         return self.session.query(arg)
diff --git a/kiskadee/queue.py b/kiskadee/queue.py
index 214b631..959be72 100644
--- a/kiskadee/queue.py
+++ b/kiskadee/queue.py
@@ -1,6 +1,5 @@
 """Provide kiskadee queues and operations on them."""
 import time
-import dill
 from multiprocessing import Queue
 
 import kiskadee
diff --git a/kiskadee/runner.py b/kiskadee/runner.py
index 5f7e764..8fd8de7 100644
--- a/kiskadee/runner.py
+++ b/kiskadee/runner.py
@@ -30,7 +30,7 @@ class Runner:
         kiskadee.logger.debug('Starting runner subprocess')
         kiskadee.logger.debug('runner PID: {}'.format(os.getpid()))
         session = kiskadee.database.Database().session
-        kiskadee.model.create_analyzers(session)
+        kiskadee.model.Analyzer.create_analyzers(session)
         self.queues = queues
         while RUNNING:
             kiskadee.logger.debug('RUNNER: Waiting to dequeue'\
diff --git a/kiskadee/tests/integration/test_monitor.py b/kiskadee/tests/integration/test_monitor.py
index 67cd7c2..6d64edf 100644
--- a/kiskadee/tests/integration/test_monitor.py
+++ b/kiskadee/tests/integration/test_monitor.py
@@ -4,7 +4,7 @@ from sqlalchemy.orm import sessionmaker
 from kiskadee import model
 from kiskadee.monitor import Monitor
 from kiskadee.queue import Queues
-from kiskadee.model import Package, create_analyzers, Report, Analysis
+from kiskadee.model import Package, Analyzer, Report, Analysis
 import kiskadee.queue
 import kiskadee.fetchers.debian
 import kiskadee.fetchers.anitya
@@ -20,7 +20,7 @@ class MonitorTestCase(unittest.TestCase):
         self.monitor = Monitor(self.session)
         self.queues = Queues()
         model.Base.metadata.create_all(self.engine)
-        create_analyzers(self.session)
+        Analyzer.create_analyzers(self.session)
         self.pkg1 = {
                 'name': 'curl',
                 'version': '7.52.1-5',
@@ -81,8 +81,8 @@ class MonitorTestCase(unittest.TestCase):
 
     def test_save_package(self):
         self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher())
-        self.monitor._save_analyzed_project(self.pkg1)
-        self.monitor._save_analyzed_project(self.pkg2)
+        self.monitor.save_analyzed_project(self.pkg1)
+        self.monitor.save_analyzed_project(self.pkg2)
         _pkgs = self.monitor.session.query(Package).all()
         self.assertEqual(len(_pkgs), 2)
         self.assertEqual(_pkgs[0].name, self.pkg1['name'])
@@ -126,7 +126,7 @@ class MonitorTestCase(unittest.TestCase):
 
     def test_save_version(self):
         self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher())
-        self.monitor._save_analyzed_project(self.pkg1)
+        self.monitor.save_analyzed_project(self.pkg1)
         _pkgs = self.monitor.session.query(Package).all()
         _version = _pkgs[0].versions[0].number
         self.assertEqual(_version, self.pkg1['version'])
@@ -137,10 +137,10 @@ class MonitorTestCase(unittest.TestCase):
         self.queues.enqueue_project(self.pkg3)
 
         _pkg = self.queues.dequeue_project()
-        self.monitor._save_analyzed_project(_pkg)
+        self.monitor.save_analyzed_project(_pkg)
 
         _pkg = self.queues.dequeue_project()
-        self.monitor._save_analyzed_project(_pkg)
+        self.monitor.save_analyzed_project(_pkg)
 
         _pkgs = self.monitor.session.query(Package).all()
         self.assertEqual(len(_pkgs), 1)
@@ -157,7 +157,7 @@ class MonitorTestCase(unittest.TestCase):
         self.monitor._save_fetcher(kiskadee.fetchers.anitya.Fetcher())
         self.queues.enqueue_project(self.pkg4)
         _pkg = self.queues.dequeue_project()
-        self.monitor._save_analyzed_project(_pkg)
+        self.monitor.save_analyzed_project(_pkg)
         _pkgs = self.monitor.session.query(Package).all()
         self.assertEqual(len(_pkgs), 1)
         self.assertEqual(_pkgs[0].homepage, _pkg['meta']['homepage'])
diff --git a/kiskadee/tests/integration/test_runner.py b/kiskadee/tests/integration/test_runner.py
index f9a2e16..d2f08df 100644
--- a/kiskadee/tests/integration/test_runner.py
+++ b/kiskadee/tests/integration/test_runner.py
@@ -21,7 +21,7 @@ class AnalyzersTestCase(unittest.TestCase):
         Session = sessionmaker(bind=self.engine)
         self.session = Session()
         model.Base.metadata.create_all(self.engine)
-        model.create_analyzers(self.session)
+        model.Analyzer.create_analyzers(self.session)
         self.fetcher = kiskadee.fetchers.debian.Fetcher()
         self.deb_pkg = {'name': 'test',
                         'version': '1.0.0',
diff --git a/kiskadee/tests/units/test_model.py b/kiskadee/tests/units/test_model.py
index 4a56835..aba56e7 100644
--- a/kiskadee/tests/units/test_model.py
+++ b/kiskadee/tests/units/test_model.py
@@ -13,7 +13,7 @@ class ModelTestCase(unittest.TestCase):
         Session = sessionmaker(bind=self.engine)
         self.session = Session()
         model.Base.metadata.create_all(self.engine)
-        model.create_analyzers(self.session)
+        model.Analyzer.create_analyzers(self.session)
         self.fetcher = model.Fetcher(
               name='kiskadee-fetcher', target='university'
             )

From 0e3f6ca2bda1aff43a2ef755a3df339d16500f09 Mon Sep 17 00:00:00 2001
From: David Carlos <ddavidcarlos1392@gmail.com>
Date: Mar 16 2018 02:30:29 +0000
Subject: [PATCH 8/17] Make Monitor code more readable and maintainable.


  - Add more useful tests to monitor.

---

diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py
index 20afeef..12eaad1 100644
--- a/kiskadee/monitor.py
+++ b/kiskadee/monitor.py
@@ -1,6 +1,6 @@
 """Provide kiskadee monitoring capabilities.
 
-kiskadee monitors repositories checking for new package versions to be
+kiskadee monitors repositories checking for new project versions to be
 analyzed. This module provides such capabilities.
 """
 import threading
@@ -19,55 +19,57 @@ RUNNING = True
 class Monitor:
     """Provide kiskadee monitoring objects."""
 
-    def __init__(self, _session):
+    def __init__(self, _session, queues):
         """Return a non initialized Monitor."""
         self.session = _session
-        self.kiskadee_queue = None
+        self.kiskadee_queue = queues
 
-    def monitor(self, queues):
-        """Dequeue packages and check if they need to be analyzed.
-
-        The packages are dequeued from the `package_queue`. When a package
-        needs to be analyzed, this package is enqueued in the `analyses_queue`
-        so the runner component can trigger an analysis.  Each fetcher must
-        enqueue its packages in the `packages_queue`.
-        """
+    def start(self):
         kiskadee.logger.debug('kiskadee PID: {}'.format(os.getppid()))
         kiskadee.logger.debug('Starting monitor subprocess')
         kiskadee.logger.debug('monitor PID: {}'.format(os.getpid()))
 
         for fetcher in kiskadee.load_fetchers():
-            _start_fetcher(fetcher.Fetcher().watch)
+            thread = Monitor._start_fetcher(fetcher.Fetcher().watch)
 
         while RUNNING:
-            self.kiskadee_queue = queues
-            pkg = self.kiskadee_queue.dequeue_project()
-
-            if pkg:
-                self.send_to_runner(pkg)
-            time.sleep(2)
-            analyzed_project = self.kiskadee_queue.dequeue_result()
+            new_project = self.dequeue_from_fetchers()
+            self.send_project_to_runner(new_project)
+            analyzed_project = self.dequeue_analysis_from_runner()
             self.save_analyzed_project(analyzed_project)
 
-    def send_to_runner(self, data):
-        _name = data['fetcher'].split('.')[-1]
-        _fetcher = self._query(Fetcher).filter_by(name = _name).first()
-        _package = self._query(Package).filter_by(name = data['name']).first()
+    def dequeue_project_from_fetchers(self):
+        new_project = self.kiskadee_queue.dequeue_project()
+        return new_project if new_project else {}
 
-        if _fetcher:
-            data["fetcher_id"] = _fetcher.id
-            if not _package:
+    def dequeue_analysis_from_runner(self):
+        return self.kiskadee_queue.dequeue_result()
+
+    def send_project_to_runner(self, data):
+        if data:
+            fetcher, project = self.get_fetcher_and_project(data)
+            data["fetcher_id"] = fetcher.id if fetcher else ''
+            if not project:
                 kiskadee.logger.debug(
-                        "MONITOR: Sending package {}_{} "
+                        "MONITOR: Sending project {}_{} "
                         " for analysis".format(data['name'], data['version'])
                 )
                 self.kiskadee_queue.enqueue_analysis(data)
-            else:
-                new_version = data['version']
-                analysed_version = _package.versions[-1].number
-                fetcher = importlib.import_module(data['fetcher']).Fetcher()
-                if (fetcher.compare_versions(new_version, analysed_version)):
-                    self.kiskadee_queue.enqueue_analysis(data)
+            elif self.not_analyzed_project_version(project, data):
+                self.kiskadee_queue.enqueue_analysis(data)
+
+    def get_fetcher_and_project(self, data):
+        fetcher_name = data['fetcher'].split('.')[-1]
+        project_name = data['name']
+        fetcher = self._query(Fetcher).filter_by(name = fetcher_name).first()
+        project = self._query(Package).filter_by(name = project_name).first()
+        return fetcher, project
+
+    def not_analyzed_project_version(self, project, data):
+        project_version = data['version']
+        analysed_version = project.versions[-1].number
+        fetcher = importlib.import_module(data['fetcher']).Fetcher()
+        fetcher.compare_versions(project_version, analysed_version)
 
     def save_analyzed_project(self, data):
         if not data:
@@ -85,12 +87,13 @@ class Monitor:
         return self.session.query(arg)
 
 
-def _start_fetcher(module, joinable=False, timeout=None):
-    module_as_a_thread = threading.Thread(target=module)
-    module_as_a_thread.daemon = True
-    module_as_a_thread.start()
-    if joinable or timeout:
-        module_as_a_thread.join(timeout)
+    @staticmethod
+    def _start_fetcher(module, joinable=False, timeout=None):
+        module_as_a_thread = threading.Thread(target=module)
+        module_as_a_thread.daemon = True
+        module_as_a_thread.start()
+        if joinable or timeout:
+            module_as_a_thread.join(timeout)
 
 
 def daemon():
@@ -98,11 +101,10 @@ def daemon():
     # TODO: improve with start/stop system
     queues = kiskadee.queue.Queues()
     session = kiskadee.database.Database().session
-    monitor = Monitor(session)
+    monitor = Monitor(session, queues)
     runner = Runner()
     monitor_process = Process(
-            target=monitor.monitor,
-            args=(queues,)
+            target=monitor.start,
         )
     runner_process = Process(
             target=runner.runner,
diff --git a/kiskadee/tests/api/test_api.py b/kiskadee/tests/api/test_api.py
index e88fee5..7c8d021 100644
--- a/kiskadee/tests/api/test_api.py
+++ b/kiskadee/tests/api/test_api.py
@@ -50,10 +50,10 @@ class ApiTestCase(unittest.TestCase):
         Monitor(kiskadee.api.app.kiskadee_db_session())
         db_session = kiskadee.api.app.kiskadee_db_session()
         db_session.query(kiskadee.model.Package)\
-                  .filter_by().id = MagicMock(side_effect=side_effect)
+                  .filter_by().id = MagicMock()
 
         db_session.query(kiskadee.model.Version)\
-                  .filter_by().id = MagicMock(side_effect=side_effect)
+                  .filter_by().id = MagicMock()
         db_session.query().options()\
                   .filter().all = MagicMock(return_value=[analysis])
 
diff --git a/kiskadee/tests/integration/test_monitor.py b/kiskadee/tests/integration/test_monitor.py
index 6d64edf..eae9829 100644
--- a/kiskadee/tests/integration/test_monitor.py
+++ b/kiskadee/tests/integration/test_monitor.py
@@ -1,27 +1,36 @@
 import unittest
+from unittest.mock import MagicMock
 from sqlalchemy.orm import sessionmaker
 
 from kiskadee import model
 from kiskadee.monitor import Monitor
 from kiskadee.queue import Queues
-from kiskadee.model import Package, Analyzer, Report, Analysis
+from kiskadee.model import Package, Analyzer, Report, Analysis, Fetcher
 import kiskadee.queue
 import kiskadee.fetchers.debian
 import kiskadee.fetchers.anitya
+import kiskadee.fetchers.example
 from kiskadee.database import Database
 
 
 class MonitorTestCase(unittest.TestCase):
 
     def setUp(self):
-        self.engine = Database('db_test').engine
-        Session = sessionmaker(bind=self.engine)
-        self.session = Session()
-        self.monitor = Monitor(self.session)
-        self.queues = Queues()
-        model.Base.metadata.create_all(self.engine)
-        Analyzer.create_analyzers(self.session)
-        self.pkg1 = {
+        def mocked_models(klass):
+            class Package:
+                def all(self):
+                    [self.pkg1, self.pkg2, self.pkg3, self.pkg4]
+            klass()
+
+        self.example_fetcher = kiskadee.fetchers.example.Fetcher()
+        self.session = MagicMock()
+
+        queues = Queues()
+        self.monitor = Monitor(self.session, queues)
+
+        self.session.query = MagicMock(side_effect=mocked_models)
+
+        self.data1 = {
                 'name': 'curl',
                 'version': '7.52.1-5',
                 'fetcher': kiskadee.fetchers.debian.__name__,
@@ -31,34 +40,34 @@ class MonitorTestCase(unittest.TestCase):
                     'flawfinder': '><'},
                 'fetcher_id': 1}
 
-        self.pkg2 = {'name': 'urlscan',
-                     'version': '0.8.2',
-                     'fetcher': kiskadee.fetchers.debian.__name__,
-                     'meta': {'directory': 'pool/main/u/urlscan'},
-                     'results': {
-                            'cppcheck': '<>',
-                            'flawfinder': '><'},
-                     'fetcher_id': 1}
-
-        self.pkg3 = {'name': 'curl',
-                     'version': '7.52.2-5',
-                     'fetcher': kiskadee.fetchers.debian.__name__,
-                     'meta': {'directory': 'pool/main/c/curl'},
-                     'results': {
-                            'cppcheck': '<>',
-                            'flawfinder': '><'},
-                     'fetcher_id': 1}
-        self.pkg4 = {'name': 'urlanitya',
-                     'version': '0.11',
-                     'fetcher': kiskadee.fetchers.anitya.__name__,
-                     'meta': {
-                            'backend': 'gitHub',
-                            'homepage': 'https://github.com/GesielFreitas/Cros'
-                        },
-                     'results': {
-                            'cppcheck': '<>',
-                            'flawfinder': '><'},
-                     'fetcher_id': 1}
+        self.data2 = {'name': 'urlscan',
+                'version': '0.8.2',
+                'fetcher': kiskadee.fetchers.debian.__name__,
+                'meta': {'directory': 'pool/main/u/urlscan'},
+                'results': {
+                    'cppcheck': '<>',
+                    'flawfinder': '><'},
+                'fetcher_id': 1}
+
+        self.data3 = {'name': 'curl',
+                'version': '7.52.2-5',
+                'fetcher': kiskadee.fetchers.debian.__name__,
+                'meta': {'directory': 'pool/main/c/curl'},
+                'results': {
+                    'cppcheck': '<>',
+                    'flawfinder': '><'},
+                'fetcher_id': 1}
+        self.data4 = {'name': 'urlanitya',
+                'version': '0.11',
+                'fetcher': kiskadee.fetchers.anitya.__name__,
+                'meta': {
+                    'backend': 'gitHub',
+                    'homepage': 'https://github.com/GesielFreitas/Cros'
+                    },
+                'results': {
+                    'cppcheck': '<>',
+                    'flawfinder': '><'},
+                'fetcher_id': 1}
         self.analysis = {
                 'analyzer_id': 1,
                 'id': 1,
@@ -67,101 +76,50 @@ class MonitorTestCase(unittest.TestCase):
                         {'severity': 'warning'},
                         {'severity': 'style'},
                         {'severity': 'error'}
-                    ]
+                        ]
+                    }
                 }
-            }
-
-    def tearDown(self):
-        self.session.close()
-        model.Base.metadata.drop_all()
-
-    def test_return_fetcher_name(self):
-        fetcher = kiskadee.fetchers.debian.Fetcher()
-        self.assertEqual(fetcher.name, 'debian')
-
-    def test_save_package(self):
-        self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher())
-        self.monitor.save_analyzed_project(self.pkg1)
-        self.monitor.save_analyzed_project(self.pkg2)
-        _pkgs = self.monitor.session.query(Package).all()
-        self.assertEqual(len(_pkgs), 2)
-        self.assertEqual(_pkgs[0].name, self.pkg1['name'])
-        self.assertEqual(_pkgs[1].name, self.pkg2['name'])
-
-    def test_save_reports(self):
-        _fetcher = model.Fetcher(
-                name='kiskadee-fetcher3', target='university'
-            )
-        _pkg = model.Package(
-                name='kiskadee-package', fetcher_id=1
-            )
-        _version = model.Version(
-                number='1.0-rc1', package_id=1
+
+    def test_dequeue_project_from_fetcher(self):
+        self.example_fetcher.watch()
+        monitored_project = self.monitor.dequeue_project_from_fetchers()
+        self.assertIn("version", monitored_project)
+        self.assertIn("name", monitored_project)
+        self.assertIn("fetcher", monitored_project)
+        self.assertEqual(monitored_project['version'], '0.1')
+
+    def test_run_fetchers_as_threads(self):
+        Monitor._start_fetcher(self.example_fetcher.watch)
+        Monitor._start_fetcher(self.example_fetcher.watch)
+        first_monitored_project = self.monitor.dequeue_project_from_fetchers()
+        second_monitored_project = self.monitor.dequeue_project_from_fetchers()
+        self.assertIn("version", first_monitored_project)
+        self.assertIn("version", second_monitored_project)
+        self.assertIn("name", first_monitored_project)
+        self.assertIn("name", second_monitored_project)
+        self.assertIn("fetcher", first_monitored_project)
+        self.assertIn("fetcher", second_monitored_project)
+
+    def test_send_project_to_runner(self):
+        fetcher = kiskadee.model.Fetcher(name='example')
+        project = kiskadee.model.Package(name='project1',
+                fetcher_id=fetcher.id)
+        self.monitor.get_fetcher_and_project = MagicMock(
+                return_value=[fetcher, {}]
+                )
+        self.monitor.not_analyzed_project_version = MagicMock(
+                return_value=True
                 )
-        _raw = {
-            'results': [
-                {'severity': 'warning'},
-                {'severity': 'style'},
-                {'severity': 'error'}
-            ]
-        }
-        _analysis = Analysis(
-                version_id=1,
-                analyzer_id=1,
-                id=1,
-                raw=_raw
-            )
-        self.session.add(_fetcher)
-        self.session.add(_pkg)
-        self.session.add(_version)
-        self.session.add(_analysis)
-        self.session.commit()
-        _dict_analysis = {
-                'results': _raw['results'],
-                'id': 1
-            }
-        self.monitor._save_reports(_dict_analysis, self.pkg1, 'cppcheck')
-        _report = self.monitor.session.query(Report).all()
-        self.assertEqual(len(_report), 1)
-
-    def test_save_version(self):
-        self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher())
-        self.monitor.save_analyzed_project(self.pkg1)
-        _pkgs = self.monitor.session.query(Package).all()
-        _version = _pkgs[0].versions[0].number
-        self.assertEqual(_version, self.pkg1['version'])
-
-    def test_update_version(self):
-        self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher())
-        self.queues.enqueue_project(self.pkg1)
-        self.queues.enqueue_project(self.pkg3)
-
-        _pkg = self.queues.dequeue_project()
-        self.monitor.save_analyzed_project(_pkg)
-
-        _pkg = self.queues.dequeue_project()
-        self.monitor.save_analyzed_project(_pkg)
-
-        _pkgs = self.monitor.session.query(Package).all()
-        self.assertEqual(len(_pkgs), 1)
-
-        _pkg_versions = self.monitor._query(Package).\
-            filter(Package.name == _pkg['name']).first().versions
-
-        _first_version = _pkg_versions[0].number
-        _current_version = _pkg_versions[-1].number
-        self.assertEqual(self.pkg1['version'], _first_version)
-        self.assertEqual(_pkg['version'], _current_version)
-
-    def test_save_package_anitya(self):
-        self.monitor._save_fetcher(kiskadee.fetchers.anitya.Fetcher())
-        self.queues.enqueue_project(self.pkg4)
-        _pkg = self.queues.dequeue_project()
-        self.monitor.save_analyzed_project(_pkg)
-        _pkgs = self.monitor.session.query(Package).all()
-        self.assertEqual(len(_pkgs), 1)
-        self.assertEqual(_pkgs[0].homepage, _pkg['meta']['homepage'])
-
-
-if __name__ == '__main__':
-    unittest.main()
+        self.monitor.send_project_to_runner(self.data1)
+        self.assertEqual(self.monitor.kiskadee_queue.dequeue_analysis(), self.data1)
+
+    @unittest.skip("will implement")
+    def test_receive_analysis_from_runner(self):
+        pass
+
+    @unittest.skip("will implement")
+    def test_save_analysed_runner_project(self):
+        pass
+
+        if __name__ == '__main__':
+            unittest.main()

From a3165c2264df0a7163b0e67fb0fe302a95ed38c0 Mon Sep 17 00:00:00 2001
From: David Carlos <ddavidcarlos1392@gmail.com>
Date: Mar 16 2018 02:30:30 +0000
Subject: [PATCH 9/17] WIP: Refactoring runner code.


---

diff --git a/kiskadee/fetchers/__init__.py b/kiskadee/fetchers/__init__.py
index 29b5cf3..b61511b 100644
--- a/kiskadee/fetchers/__init__.py
+++ b/kiskadee/fetchers/__init__.py
@@ -93,6 +93,9 @@ class Fetcher(abc.ABC):
             download_path = os.path.join(path, file_name)
             with open(download_path, 'wb') as info:
                 info.write(data)
+            kiskadee.logger.debug(
+                    "Downloaded {} source in {} path".format(file_name, path)
+                    )
             return download_path
         except Exception as err:
             kiskadee.logger.debug(
diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py
index 12eaad1..9490b2a 100644
--- a/kiskadee/monitor.py
+++ b/kiskadee/monitor.py
@@ -22,41 +22,35 @@ class Monitor:
     def __init__(self, _session, queues):
         """Return a non initialized Monitor."""
         self.session = _session
-        self.kiskadee_queue = queues
+        self.queues = queues
 
     def start(self):
-        kiskadee.logger.debug('kiskadee PID: {}'.format(os.getppid()))
-        kiskadee.logger.debug('Starting monitor subprocess')
-        kiskadee.logger.debug('monitor PID: {}'.format(os.getpid()))
+        kiskadee.logger.debug('Monitor PID: {}'.format(os.getpid()))
 
         for fetcher in kiskadee.load_fetchers():
             thread = Monitor._start_fetcher(fetcher.Fetcher().watch)
 
         while RUNNING:
-            new_project = self.dequeue_from_fetchers()
+            new_project = self.dequeue_project_from_fetchers()
             self.send_project_to_runner(new_project)
             analyzed_project = self.dequeue_analysis_from_runner()
             self.save_analyzed_project(analyzed_project)
 
     def dequeue_project_from_fetchers(self):
-        new_project = self.kiskadee_queue.dequeue_project()
+        new_project = self.queues.dequeue_project()
         return new_project if new_project else {}
 
     def dequeue_analysis_from_runner(self):
-        return self.kiskadee_queue.dequeue_result()
+        return self.queues.dequeue_result()
 
     def send_project_to_runner(self, data):
         if data:
             fetcher, project = self.get_fetcher_and_project(data)
             data["fetcher_id"] = fetcher.id if fetcher else ''
             if not project:
-                kiskadee.logger.debug(
-                        "MONITOR: Sending project {}_{} "
-                        " for analysis".format(data['name'], data['version'])
-                )
-                self.kiskadee_queue.enqueue_analysis(data)
-            elif self.not_analyzed_project_version(project, data):
-                self.kiskadee_queue.enqueue_analysis(data)
+                self.queues.enqueue_analysis(data)
+            elif self.is_a_new_project_version(project, data):
+                self.queues.enqueue_analysis(data)
 
     def get_fetcher_and_project(self, data):
         fetcher_name = data['fetcher'].split('.')[-1]
@@ -65,7 +59,7 @@ class Monitor:
         project = self._query(Package).filter_by(name = project_name).first()
         return fetcher, project
 
-    def not_analyzed_project_version(self, project, data):
+    def is_a_new_project_version(self, project, data):
         project_version = data['version']
         analysed_version = project.versions[-1].number
         fetcher = importlib.import_module(data['fetcher']).Fetcher()
@@ -102,13 +96,12 @@ def daemon():
     queues = kiskadee.queue.Queues()
     session = kiskadee.database.Database().session
     monitor = Monitor(session, queues)
-    runner = Runner()
+    runner = Runner(session, queues)
     monitor_process = Process(
             target=monitor.start,
         )
     runner_process = Process(
             target=runner.runner,
-            args=(queues,)
         )
     monitor_process.start()
     runner_process.start()
diff --git a/kiskadee/queue.py b/kiskadee/queue.py
index 959be72..672958a 100644
--- a/kiskadee/queue.py
+++ b/kiskadee/queue.py
@@ -42,17 +42,33 @@ class Queues():
     @staticmethod
     def enqueue_analysis(project_to_analysis):
         """Put a analysis on the analysis queue."""
+        log_msg = "Sending project {}_{} for analysis"\
+                .format(project_to_analysis['name'],
+                        project_to_analysis['version'])
+        kiskadee.logger.debug(log_msg)
         analysis.put(project_to_analysis)
 
     @staticmethod
     def dequeue_analysis():
         """Get a analysis from the analysis queue."""
-        return analysis.get()
+        project_to_analysis = analysis.get()
+        kiskadee.logger.debug(
+                'RUNNER: deqeued {}-{} from {}'
+                .format(project_to_analysis['name'],
+                        project_to_analysis['version'],
+                        project_to_analysis['fetcher'])
+            )
+        return project_to_analysis
 
     @staticmethod
-    def enqueue_result(result):
+    def enqueue_result(project):
         """Put a result on the results queue."""
-        results.put(result)
+        kiskadee.logger.debug(
+                "RUNNER: Sending {}-{} to Monitor"
+                .format(project["name"],
+                        project["version"])
+            )
+        results.put(project)
 
     @staticmethod
     @empty_queue
diff --git a/kiskadee/runner.py b/kiskadee/runner.py
index 8fd8de7..607fdef 100644
--- a/kiskadee/runner.py
+++ b/kiskadee/runner.py
@@ -16,9 +16,12 @@ RUNNING = True
 class Runner:
     """Provide kiskadee runner objects."""
 
-    def __init__(self):
+    def __init__(self, session, queues):
         """Return a non initialized Runner."""
-        self.queues = None
+        self.queues = queues
+        self.session = session
+        kiskadee.model.Analyzer.create_analyzers(self.session)
+        self.fetcher = None
 
     def runner(self, queues):
         """Run static analyzers.
@@ -27,52 +30,69 @@ class Runner:
         :func:`analyze` method, passing the dequeued package.
         After the analysis, updates the status of this package on the database.
         """
-        kiskadee.logger.debug('Starting runner subprocess')
-        kiskadee.logger.debug('runner PID: {}'.format(os.getpid()))
-        session = kiskadee.database.Database().session
-        kiskadee.model.Analyzer.create_analyzers(session)
-        self.queues = queues
+        kiskadee.logger.debug('Runner PID: {}'.format(os.getpid()))
         while RUNNING:
             kiskadee.logger.debug('RUNNER: Waiting to dequeue'\
                                   ' project to analysis...')
-            source_to_analysis = self.queues.dequeue_analysis()
-            kiskadee.logger.debug(
-                    'RUNNER: deqeued {}-{} from {}'
-                    .format(source_to_analysis['name'],
-                            source_to_analysis['version'],
-                            source_to_analysis['fetcher'])
+            project_to_analysis = self.queues.dequeue_analysis()
+            self.call_analyzers(project_to_analysis)
+
+    def load_project_fetcher(self, project_to_analysis):
+        try:
+        return importlib.import_module(
+                project_to_analysis['fetcher']
+                ).Fetcher()
+        except ModuleNotFoundError:
+            kiskadee.logger.debug("Fetcher {} could not be loaded"\
+                    .format(project_to_analysis['fetcher'])
                 )
-            self.call_analyzers(source_to_analysis)
+            return {}
+
+    def uncompress_project_code(self, compressed_source):
+        dir_to_unpack_source = tempfile.mkdtemp()
+        try:
+            shutil.unpack_archive(compressed_source,
+                    dir_to_unpack_source)
+            return dir_to_unpack_source
+        except Exception as err:
+            kiskadee.logger.debug('Could not unpack project source')
+            kiskadee.logger.debug(err)
+            return {}
+
+    def run_analysis(self, analyzers, project_to_analysis):
+        project_to_analysis['results'] = {}
+        for analyzer in analyzers:
+            firehose_report = self.analyze(analyzer, source_path)
+            if firehose_report:
+                project_to_analysis['results'][analyzer] = firehose_report
+        return project_to_analysis
 
-    def call_analyzers(self, source_to_analysis):
+
+    def clean_temporary_directory(self, dir):
+        # not delete the source code used on tests.
+        if not compressed_source_path .find("kiskadee/tests") > -1:
+            shutil.rmtree(os.path.dirname(dir))
+
+    def call_analyzers(self, project_to_analysis):
         """Iterate over the package analyzers.
 
         For each analyzer defined to analysis the source, call
         the function :func:`analyze`, passing the source dict, the analyzer
         to run the analysis, and the path to a compressed source.
         """
-        fetcher = importlib.import_module(source_to_analysis['fetcher']).Fetcher()
-        source_path = self._path_to_uncompressed_source(
-                source_to_analysis, fetcher
-        )
+        self.fetcher = self.load_project_fetcher(project_to_analysis)
+        source_path = self.prepare_to_get_project_code(project_to_analysis)
         if not source_path:
             return None
 
         analyzers = fetcher.analyzers()
-        source_to_analysis['results'] = {}
-        for analyzer in analyzers:
-            firehose_report = self.analyze(analyzer, source_path)
-            if firehose_report:
-                source_to_analysis['results'][analyzer] = firehose_report
+        analysis_result = self.run_analysis(analyzers, project_to_analysis)
+        self.enqueue_analysis_to_monitor(analysis_result)
+        self.clean_temporary_directory(source_path)
 
-        if source_to_analysis['results']:
-            kiskadee.logger.debug(
-                    "RUNNER: Sending {}-{} to Monitor"
-                    .format(source_to_analysis["name"],
-                            source_to_analysis["version"])
-                )
-            self.queues.enqueue_result(source_to_analysis)
-        shutil.rmtree(source_path)
+    def enqueue_analysis_to_monitor(self, analysis_result):
+        if analysis_result['results']:
+            self.queues.enqueue_result(analysis_result)
 
     def analyze(self, analyzer, source_path):
         """Run each analyzer on some sorce code.
@@ -101,52 +121,18 @@ class Runner:
             kiskadee.logger.debug(err)
             return None
 
-    def _path_to_uncompressed_source(self, package, fetcher):
+    def prepare_to_get_project_code(self, project):
 
-        if not (fetcher and package):
+        if not (self.fetcher and project):
             return None
 
-        kiskadee.logger.debug(
-                'ANALYSIS: Downloading {} '
-                'source...'.format(package['name'])
-        )
-
-        compressed_source = fetcher.get_sources(package)
-
-        if compressed_source:
-            kiskadee.logger.debug(
-                    'ANALYSIS: Downloaded {} source in {} path'
-                    .format(
-                        package['name'],
-                        os.path.dirname(compressed_source)
-                    )
-                )
-            tmp_path = tempfile.gettempdir()
-            uncompressed_source_path = tempfile.mkdtemp(dir=tmp_path)
-            try:
-                shutil.unpack_archive(
-                        compressed_source,
-                        uncompressed_source_path
+        compressed_source_path = self.fetcher.get_sources(project)
+        if compressed_source_path :
+            uncompressed_source_path = uncompress_project_code(
+                    compressed_source_path
                     )
-                kiskadee.logger.debug(
-                        'ANALYSIS: Unpacking {} source in {} path'
-                        .format(package['name'], uncompressed_source_path)
-                        )
-                # not delete the source code used on tests.
-                if not compressed_source.find("kiskadee/tests") > -1:
-                    shutil.rmtree(os.path.dirname(compressed_source))
-                    kiskadee.logger.debug(
-                            'ANALYSIS: Remove {} temp directory'
-                            .format(os.path.dirname(compressed_source))
-                        )
-                kiskadee.logger.debug(
-                        'ANALYSIS: Unpacked {} source'.format(package['name'])
-                        )
-                return uncompressed_source_path
-            except Exception as err:
-                kiskadee.logger.debug('Something went wrong')
-                kiskadee.logger.debug(err)
-                return None
+            self.clean_temporary_directory(os.path.dirname(compressed_source_path))
+            return uncompressed_source_path
         else:
             kiskadee.logger.debug('RUNNER: invalid compressed source')
             return None
diff --git a/kiskadee/tests/integration/test_monitor.py b/kiskadee/tests/integration/test_monitor.py
index eae9829..3018563 100644
--- a/kiskadee/tests/integration/test_monitor.py
+++ b/kiskadee/tests/integration/test_monitor.py
@@ -40,46 +40,6 @@ class MonitorTestCase(unittest.TestCase):
                     'flawfinder': '><'},
                 'fetcher_id': 1}
 
-        self.data2 = {'name': 'urlscan',
-                'version': '0.8.2',
-                'fetcher': kiskadee.fetchers.debian.__name__,
-                'meta': {'directory': 'pool/main/u/urlscan'},
-                'results': {
-                    'cppcheck': '<>',
-                    'flawfinder': '><'},
-                'fetcher_id': 1}
-
-        self.data3 = {'name': 'curl',
-                'version': '7.52.2-5',
-                'fetcher': kiskadee.fetchers.debian.__name__,
-                'meta': {'directory': 'pool/main/c/curl'},
-                'results': {
-                    'cppcheck': '<>',
-                    'flawfinder': '><'},
-                'fetcher_id': 1}
-        self.data4 = {'name': 'urlanitya',
-                'version': '0.11',
-                'fetcher': kiskadee.fetchers.anitya.__name__,
-                'meta': {
-                    'backend': 'gitHub',
-                    'homepage': 'https://github.com/GesielFreitas/Cros'
-                    },
-                'results': {
-                    'cppcheck': '<>',
-                    'flawfinder': '><'},
-                'fetcher_id': 1}
-        self.analysis = {
-                'analyzer_id': 1,
-                'id': 1,
-                'raw': {
-                    'results': [
-                        {'severity': 'warning'},
-                        {'severity': 'style'},
-                        {'severity': 'error'}
-                        ]
-                    }
-                }
-
     def test_dequeue_project_from_fetcher(self):
         self.example_fetcher.watch()
         monitored_project = self.monitor.dequeue_project_from_fetchers()
@@ -107,19 +67,12 @@ class MonitorTestCase(unittest.TestCase):
         self.monitor.get_fetcher_and_project = MagicMock(
                 return_value=[fetcher, {}]
                 )
-        self.monitor.not_analyzed_project_version = MagicMock(
+        self.monitor.is_a_new_project_version = MagicMock(
                 return_value=True
                 )
         self.monitor.send_project_to_runner(self.data1)
-        self.assertEqual(self.monitor.kiskadee_queue.dequeue_analysis(), self.data1)
-
-    @unittest.skip("will implement")
-    def test_receive_analysis_from_runner(self):
-        pass
-
-    @unittest.skip("will implement")
-    def test_save_analysed_runner_project(self):
-        pass
+        self.assertEqual(self.monitor.queues.dequeue_analysis(),
+                self.data1)
 
         if __name__ == '__main__':
             unittest.main()

From 3d27c83831686244ae98130d20beb0b7d186717c Mon Sep 17 00:00:00 2001
From: David Carlos <ddavidcarlos1392@gmail.com>
Date: Mar 16 2018 02:30:30 +0000
Subject: [PATCH 10/17] reimplement runner integration tests


---

diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py
index e39e133..f484bc4 100644
--- a/kiskadee/api/app.py
+++ b/kiskadee/api/app.py
@@ -46,14 +46,10 @@ def package_analysis_overview(pkg_name, version):
     db_session = kiskadee_db_session()
 
     #TODO: This can be a simple inner join between package, version and analysis
-    package_id = (
-            db_session.query(Package).filter_by(name = pkg_name).id
-        )
-    version_id = (
-            db_session.query(Version)
-            .filter(Version.number == version)
+    package_id = db_session.query(Package).filter_by(name = pkg_name).id
+    version_id = db_session.query(Version)\
+            .filter(Version.number == version)\
             .filter(Version.package_id == package_id).first().id
-        )
     analysis = (
             db_session.query(Analysis)
             .options(
@@ -62,8 +58,6 @@ def package_analysis_overview(pkg_name, version):
             .filter(Analysis.version_id == version_id)
             .all()
         )
-    kiskadee.logger.debug("haaaaaaa")
-    kiskadee.logger.debug(analysis)
     analysis_schema = AnalysisSchema(many=True, exclude=['raw', 'report'])
     data, errors = analysis_schema.dump(analysis)
     return jsonify(data)
@@ -76,10 +70,7 @@ def package_analysis_overview(pkg_name, version):
 def analysis_results(pkg_name, version, analysis_id):
     """Get the analysis results from a specific analyzer."""
     db_session = kiskadee_db_session()
-    analysis = (
-            db_session.query(Analysis)
-            .get(analysis_id)
-        )
+    analysis = db_session.query(Analysis).get(analysis_id)
     analysis_schema = AnalysisSchema(only=['raw'])
     data, errors = analysis_schema.dump(analysis)
     response = data['raw']['results']
@@ -93,26 +84,20 @@ def analysis_results(pkg_name, version, analysis_id):
 def analysis_reports(pkg_name, version, analysis_id):
     """Get the analysis reports from a specific analyzer."""
     db_session = kiskadee_db_session()
-    analysis = (
-            db_session.query(Analysis)
-            .get(analysis_id)
-        )
-    analysis_schema = AnalysisSchema(only=['report'])
-    data, errors = analysis_schema.dump(analysis)
+    analysis = db_session.query(Analysis).get(analysis_id)
+    data, errors = AnalysisSchema(only=['report']).dump(analysis)
     report = data['report']
-    if (report is not None) and\
-        ('results' in report.keys()) and\
-            report['results'] is not None:
-        report['results'] = json\
-            .loads(report['results'])
-    return jsonify({'analysis_report': report})
+    try:
+        report['results'] = json.loads(report['results'])
+        return jsonify({'analysis_report': report})
+    except Exception as err:
+        return jsonify({'analysis_report': {}})
 
 
 def kiskadee_db_session():
     """Return a kiskadee database session."""
     return Database().session
 
-
 def main():
     """Initialize the kiskadee API."""
     kiskadee.run('0.0.0.0')
diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py
index 9490b2a..0fc6755 100644
--- a/kiskadee/monitor.py
+++ b/kiskadee/monitor.py
@@ -23,6 +23,7 @@ class Monitor:
         """Return a non initialized Monitor."""
         self.session = _session
         self.queues = queues
+        kiskadee.model.Analyzer.create_analyzers(self.session)
 
     def start(self):
         kiskadee.logger.debug('Monitor PID: {}'.format(os.getpid()))
@@ -96,7 +97,7 @@ def daemon():
     queues = kiskadee.queue.Queues()
     session = kiskadee.database.Database().session
     monitor = Monitor(session, queues)
-    runner = Runner(session, queues)
+    runner = Runner(queues)
     monitor_process = Process(
             target=monitor.start,
         )
diff --git a/kiskadee/runner.py b/kiskadee/runner.py
index 607fdef..dfe02a0 100644
--- a/kiskadee/runner.py
+++ b/kiskadee/runner.py
@@ -16,12 +16,11 @@ RUNNING = True
 class Runner:
     """Provide kiskadee runner objects."""
 
-    def __init__(self, session, queues):
+    def __init__(self, queues):
         """Return a non initialized Runner."""
         self.queues = queues
-        self.session = session
-        kiskadee.model.Analyzer.create_analyzers(self.session)
         self.fetcher = None
+        self.project = None
 
     def runner(self, queues):
         """Run static analyzers.
@@ -34,59 +33,50 @@ class Runner:
         while RUNNING:
             kiskadee.logger.debug('RUNNER: Waiting to dequeue'\
                                   ' project to analysis...')
-            project_to_analysis = self.queues.dequeue_analysis()
-            self.call_analyzers(project_to_analysis)
+            self.project = self.queues.dequeue_analysis()
+            self.call_analyzers()
 
-    def load_project_fetcher(self, project_to_analysis):
+    def import_project_fetcher(self):
         try:
-        return importlib.import_module(
-                project_to_analysis['fetcher']
-                ).Fetcher()
+            return importlib.import_module(
+                    self.project['fetcher']
+                    ).Fetcher()
         except ModuleNotFoundError:
             kiskadee.logger.debug("Fetcher {} could not be loaded"\
-                    .format(project_to_analysis['fetcher'])
+                    .format(self.project['fetcher'])
                 )
             return {}
 
-    def uncompress_project_code(self, compressed_source):
-        dir_to_unpack_source = tempfile.mkdtemp()
-        try:
-            shutil.unpack_archive(compressed_source,
-                    dir_to_unpack_source)
-            return dir_to_unpack_source
-        except Exception as err:
-            kiskadee.logger.debug('Could not unpack project source')
-            kiskadee.logger.debug(err)
-            return {}
-
-    def run_analysis(self, analyzers, project_to_analysis):
-        project_to_analysis['results'] = {}
+    def run_analysis(self, analyzers, source_path):
+        self.project['results'] = {}
         for analyzer in analyzers:
             firehose_report = self.analyze(analyzer, source_path)
             if firehose_report:
-                project_to_analysis['results'][analyzer] = firehose_report
-        return project_to_analysis
+                self.project['results'][analyzer] = firehose_report
+        return self.project
 
 
-    def clean_temporary_directory(self, dir):
+    def clean_temporary_directory(self, temp_dir):
         # not delete the source code used on tests.
-        if not compressed_source_path .find("kiskadee/tests") > -1:
-            shutil.rmtree(os.path.dirname(dir))
+        if not temp_dir.find("kiskadee/tests") > -1:
+            shutil.rmtree(temp_dir)
 
-    def call_analyzers(self, project_to_analysis):
+    def call_analyzers(self):
         """Iterate over the package analyzers.
 
         For each analyzer defined to analysis the source, call
         the function :func:`analyze`, passing the source dict, the analyzer
         to run the analysis, and the path to a compressed source.
         """
-        self.fetcher = self.load_project_fetcher(project_to_analysis)
-        source_path = self.prepare_to_get_project_code(project_to_analysis)
+        self.fetcher = self.import_project_fetcher()
+        source_path = self.get_project_code_path()
         if not source_path:
             return None
 
-        analyzers = fetcher.analyzers()
-        analysis_result = self.run_analysis(analyzers, project_to_analysis)
+        analyzers = self.fetcher.analyzers()
+        analysis_result = self.run_analysis(
+                analyzers, source_path
+                )
         self.enqueue_analysis_to_monitor(analysis_result)
         self.clean_temporary_directory(source_path)
 
@@ -121,14 +111,14 @@ class Runner:
             kiskadee.logger.debug(err)
             return None
 
-    def prepare_to_get_project_code(self, project):
+    def get_project_code_path(self):
 
-        if not (self.fetcher and project):
+        if not (self.fetcher and self.project):
             return None
 
-        compressed_source_path = self.fetcher.get_sources(project)
+        compressed_source_path = self.fetcher.get_sources(self.project)
         if compressed_source_path :
-            uncompressed_source_path = uncompress_project_code(
+            uncompressed_source_path = self.uncompress_project_code(
                     compressed_source_path
                     )
             self.clean_temporary_directory(os.path.dirname(compressed_source_path))
@@ -136,3 +126,15 @@ class Runner:
         else:
             kiskadee.logger.debug('RUNNER: invalid compressed source')
             return None
+
+    def uncompress_project_code(self, compressed_source):
+        dir_to_unpack_source = tempfile.mkdtemp()
+        try:
+            shutil.unpack_archive(compressed_source,
+                    dir_to_unpack_source)
+            return dir_to_unpack_source
+        except Exception as err:
+            kiskadee.logger.debug('Could not unpack project source')
+            kiskadee.logger.debug(err)
+            return {}
+
diff --git a/kiskadee/tests/api/test_api.py b/kiskadee/tests/api/test_api.py
index 7c8d021..1ef5b8a 100644
--- a/kiskadee/tests/api/test_api.py
+++ b/kiskadee/tests/api/test_api.py
@@ -17,6 +17,7 @@ class ApiTestCase(unittest.TestCase):
 
     def setUp(self):
         self.fetcher = kiskadee.model.Fetcher(name='kiskadee-fetcher')
+        self.queues = kiskadee.queue.Queues()
 
     def test_get_fetchers(self):
         kiskadee.api.app.kiskadee_db_session = MagicMock()
@@ -39,15 +40,8 @@ class ApiTestCase(unittest.TestCase):
                 raw='analysis result: foo'
                 )
 
-        def side_effect(klass):
-            mocked_objects = {
-                    'Package': package,
-                    'Version': version,
-                    }
-            return mocked_objects[klass.__name__]
-
         kiskadee.api.app.kiskadee_db_session = MagicMock()
-        Monitor(kiskadee.api.app.kiskadee_db_session())
+        Monitor(kiskadee.api.app.kiskadee_db_session(), self.queues)
         db_session = kiskadee.api.app.kiskadee_db_session()
         db_session.query(kiskadee.model.Package)\
                   .filter_by().id = MagicMock()
diff --git a/kiskadee/tests/integration/test_monitor.py b/kiskadee/tests/integration/test_monitor.py
index 3018563..2369499 100644
--- a/kiskadee/tests/integration/test_monitor.py
+++ b/kiskadee/tests/integration/test_monitor.py
@@ -27,7 +27,6 @@ class MonitorTestCase(unittest.TestCase):
 
         queues = Queues()
         self.monitor = Monitor(self.session, queues)
-
         self.session.query = MagicMock(side_effect=mocked_models)
 
         self.data1 = {
diff --git a/kiskadee/tests/integration/test_runner.py b/kiskadee/tests/integration/test_runner.py
index d2f08df..3cd646f 100644
--- a/kiskadee/tests/integration/test_runner.py
+++ b/kiskadee/tests/integration/test_runner.py
@@ -4,97 +4,54 @@ by the user that is executing the tests. It also needs selinux running in
 a permissive mode (setenforce 0)
 """
 import unittest
-import tempfile
+import re
+from unittest.mock import MagicMock
 
-from kiskadee.runner import Runner
+import kiskadee.queue
+import kiskadee.runner
+import kiskadee.monitor
 import kiskadee.fetchers.example
-import kiskadee.fetchers.debian
-from sqlalchemy.orm import sessionmaker
-from kiskadee import model
-from kiskadee.queue import Queues
-from kiskadee.database import Database
+import kiskadee.analyzers
 
+class RunnerTestCase(unittest.TestCase):
 
-class AnalyzersTestCase(unittest.TestCase):
-    def setUp(self):
-        self.engine = Database('db_test').engine
-        Session = sessionmaker(bind=self.engine)
-        self.session = Session()
-        model.Base.metadata.create_all(self.engine)
-        model.Analyzer.create_analyzers(self.session)
-        self.fetcher = kiskadee.fetchers.debian.Fetcher()
-        self.deb_pkg = {'name': 'test',
-                        'version': '1.0.0',
-                        'fetcher': kiskadee.fetchers.debian.__name__
-                        }
-        self.fetcher = model.Fetcher(
-                name='kiskadee-fetcher2', target='university'
-            )
-        self.session.add(self.fetcher)
-        self.session.commit()
-        self.runner = Runner()
-        self.runner.queues = Queues()
-
-    def tearDown(self):
-        self.session.close()
-        model.Base.metadata.drop_all()
-
-    def test_run_analyzer(self):
-
-        source_to_analysis = {
-                'name': 'test',
-                'version': '1.0.0',
-                'fetcher': kiskadee.fetchers.example.Fetcher()
-        }
+    """Docstring for AnalyzersTestCase. """
 
-        source_path = self.runner._path_to_uncompressed_source(
-                source_to_analysis, kiskadee.fetchers.example.Fetcher()
-            )
-        firehose_report = self.runner.analyze("cppcheck", source_path)
-        self.assertIsNotNone(firehose_report)
+    def setUp(self):
+        self.queues = kiskadee.queue.Queues()
+        self.runner = kiskadee.runner.Runner(self.queues)
 
-    def test_generate_a_firehose_report(self):
-        source_to_analysis = {
+        self.project = {
                 'name': 'test',
                 'version': '1.0.0',
                 'fetcher': kiskadee.fetchers.example.__name__
         }
 
-        self.runner.call_analyzers(source_to_analysis)
-        analyzed_pkg = self.runner.queues.dequeue_result()
-        self.assertEqual(analyzed_pkg['name'], source_to_analysis['name'])
-        self.assertIn('cppcheck', analyzed_pkg['results'])
-        self.assertIn('flawfinder', analyzed_pkg['results'])
-
-    def test_path_to_uncompressed_source(self):
-
-        source_to_analysis = {
-                'name': 'test',
-                'version': '1.0.0',
-                'fetcher': kiskadee.fetchers.example
-        }
-
-        source_path = self.runner._path_to_uncompressed_source(
-                source_to_analysis, kiskadee.fetchers.example.Fetcher()
-        )
-        tmp_path = tempfile.gettempdir()
-        self.assertTrue(source_path.find(tmp_path) >= 0)
-        self.assertIsNotNone(source_path)
-
-    def test_invalid_path_to_uncompressed_source(self):
-
-        source_to_analysis = {
-                'name': 'test',
-                'version': '1.0.0',
-                'fetcher': kiskadee.fetchers.example
-        }
-
-        source_path = self.runner._path_to_uncompressed_source(
-                source_to_analysis, None
-        )
-
-        self.assertIsNone(source_path)
-
+    def tearDown(self):
+        """TODO: to be defined1. """
+
+    def test_run_analyzers(self):
+        self.runner.project = self.project
+        self.runner.call_analyzers()
+        result = self.queues.dequeue_result()
+        self.assertEqual(result['name'], 'test')
+        self.assertIn('results', result)
+
+    def test_run_a_single_analyzer(self):
+        self.runner.project = self.project
+        self.runner.fetcher = self.runner.import_project_fetcher()
+        source_path = self.runner.get_project_code_path()
+        analyzer = self.runner.fetcher.analyzers()[0]
+        self.assertIsNotNone(re.search('.*cppcheck.*', analyzer))
+        self.assertIsNone(re.search('.*pylint.*', analyzer))
+
+    def test_analysis_an_incoming_monitor_project(self):
+        self.session = MagicMock()
+        self.session.query().filter_by().first = MagicMock(return_value={})
+        self.monitor = kiskadee.monitor.Monitor(self.session, self.queues)
+        self.monitor.send_project_to_runner(self.project)
+        incoming_project = self.runner.queues.dequeue_analysis()
+        self.assertEqual(self.project, incoming_project)
 
 if __name__ == '__main__':
     unittest.main()

From e86e9b14bff02935fcadb4758c4350816dcf6862 Mon Sep 17 00:00:00 2001
From: David Carlos <ddavidcarlos1392@gmail.com>
Date: Mar 16 2018 02:30:30 +0000
Subject: [PATCH 11/17] Fix remaining tests.


---

diff --git a/Makefile b/Makefile
index cffabee..fcc2240 100644
--- a/Makefile
+++ b/Makefile
@@ -2,22 +2,22 @@
 SHELL:=/bin/bash
 
 help:
-	@printf "Available targets: check, analyzers, clean\n\n"
+	@printf "Available targets: test, analyzers, clean\n\n"
 
-check:
+test:
 	coverage run --omit="lib/*","setup.py","kiskadee/tests/*",".eggs/*",".venv/*","/usr/*" -m unittest kiskadee/tests/*/*.py
 	coverage html
 
-check_units:
+test_units:
 	python3 -m unittest kiskadee/tests/units/*.py
 
-check_integration:
+test_integration:
 	python3 -m unittest kiskadee/tests/integration/*.py
 
-check_api:
+test_api:
 	python3 -m unittest kiskadee/tests/api/*.py
 
-check_plugins:
+test_plugins:
 	python3 -m unittest kiskadee/tests/plugins/*.py
 
 analyzers:
diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py
index f484bc4..9eb7b3a 100644
--- a/kiskadee/api/app.py
+++ b/kiskadee/api/app.py
@@ -14,13 +14,13 @@ kiskadee = Flask(__name__)
 
 CORS(kiskadee)
 
+db = Database()
 
 @kiskadee.route('/fetchers')
 def index():
     """Get the list of available fetchers."""
     if request.method == 'GET':
-        db_session = kiskadee_db_session()
-        fetchers = db_session.query(Fetcher).all()
+        fetchers = db.session.query(Fetcher).all()
         fetcher_schema = FetcherSchema(many=True)
         result = fetcher_schema.dump(fetchers)
         return jsonify({'fetchers': result.data})
@@ -30,8 +30,7 @@ def index():
 def packages():
     """Get the list of analyzed packages."""
     if request.method == 'GET':
-        db_session = kiskadee_db_session()
-        packages = db_session.query(Package).all()
+        packages = db.session.query(Package).all()
         package_schema = PackageSchema(
             many=True,
             exclude=['versions.analysis', 'versions.package_id']
@@ -42,16 +41,14 @@ def packages():
 
 @kiskadee.route('/analysis/<pkg_name>/<version>', methods=['GET'])
 def package_analysis_overview(pkg_name, version):
-    """Get the a analysis list of some package version."""
-    db_session = kiskadee_db_session()
+    """Get a analysis list of some package version."""
 
     #TODO: This can be a simple inner join between package, version and analysis
-    package_id = db_session.query(Package).filter_by(name = pkg_name).id
-    version_id = db_session.query(Version)\
-            .filter(Version.number == version)\
-            .filter(Version.package_id == package_id).first().id
+    _package_id = db.filter_by_name(Package, pkg_name).id
+    version_id = db.session.query(Version)\
+            .filter_by(number = version, package_id = _package_id ).first().id
     analysis = (
-            db_session.query(Analysis)
+            db.session.query(Analysis)
             .options(
                 eagerload(Analysis.analyzers, innerjoin=True)
             )
@@ -69,8 +66,7 @@ def package_analysis_overview(pkg_name, version):
 )
 def analysis_results(pkg_name, version, analysis_id):
     """Get the analysis results from a specific analyzer."""
-    db_session = kiskadee_db_session()
-    analysis = db_session.query(Analysis).get(analysis_id)
+    analysis = db.get(Analysis, analysis_id)
     analysis_schema = AnalysisSchema(only=['raw'])
     data, errors = analysis_schema.dump(analysis)
     response = data['raw']['results']
@@ -83,8 +79,7 @@ def analysis_results(pkg_name, version, analysis_id):
 )
 def analysis_reports(pkg_name, version, analysis_id):
     """Get the analysis reports from a specific analyzer."""
-    db_session = kiskadee_db_session()
-    analysis = db_session.query(Analysis).get(analysis_id)
+    analysis = db.get(Analysis, analysis_id)
     data, errors = AnalysisSchema(only=['report']).dump(analysis)
     report = data['report']
     try:
@@ -93,11 +88,6 @@ def analysis_reports(pkg_name, version, analysis_id):
     except Exception as err:
         return jsonify({'analysis_report': {}})
 
-
-def kiskadee_db_session():
-    """Return a kiskadee database session."""
-    return Database().session
-
 def main():
     """Initialize the kiskadee API."""
     kiskadee.run('0.0.0.0')
diff --git a/kiskadee/database.py b/kiskadee/database.py
index 945a601..96cefe4 100644
--- a/kiskadee/database.py
+++ b/kiskadee/database.py
@@ -1,9 +1,10 @@
 """Provide kiskadee Database operations."""
 
-import kiskadee
-from kiskadee.model import Base
 from sqlalchemy import create_engine, orm
 
+import kiskadee
+from kiskadee.model import Base
+from kiskadee.model import Package, Fetcher, Version, Report, Analysis
 
 class Database:
     """kiskadee Database class."""
@@ -22,6 +23,11 @@ class Database:
         DBSession = orm.sessionmaker(bind=engine)
         return DBSession()
 
+    def filter_by_name(self, model, _name):
+        return self.session.query(model).filter_by(name = _name).first()
+
+    def get(self, model, id):
+        return self.session.query(model).get(id)
 
 def get_database_uri(db):
     """Return the Database URI of the current session."""
diff --git a/kiskadee/model.py b/kiskadee/model.py
index 4e4dff2..acbd5a0 100644
--- a/kiskadee/model.py
+++ b/kiskadee/model.py
@@ -31,7 +31,7 @@ class Package(Base):
             )
 
     @staticmethod
-    def save(session, data):
+    def save(db, data):
         homepage = None
         if ('meta' in data) and ('homepage' in data['meta']):
             homepage = data['meta']['homepage']
@@ -41,16 +41,16 @@ class Package(Base):
                 homepage=homepage,
                 fetcher_id=data['fetcher_id']
                 )
-        session.add(_package)
-        session.commit()
+        db.session.add(_package)
+        db.session.commit()
         _version = Version(number=data['version'],
                            package_id=_package.id)
-        session.add(_version)
-        session.commit()
+        db.session.add(_version)
+        db.session.commit()
         return _package
 
     @staticmethod
-    def update(session, project, data):
+    def update(db, project, data):
 
         if(project.versions[-1].number == data['version']):
             return project
@@ -60,8 +60,8 @@ class Package(Base):
                     project_id=project.id
                     )
             project.versions.append(_new_version)
-            session.add(project)
-            session.commit()
+            db.session.add(project)
+            db.session.commit()
             kiskadee.logger.debug(
                     "MONITOR: Sending project {}_{}"
                     "for analysis".format(data['name'], data['version'])
@@ -109,7 +109,7 @@ class Analyzer(Base):
     analysis = orm.relationship('Analysis', backref='analyzers')
 
     @staticmethod
-    def create_analyzers(_session):
+    def create_analyzers(db):
         """Create the analyzers on database.
 
         The kiskadee analyzers are defined on the section `analyzers` of the
@@ -118,13 +118,13 @@ class Analyzer(Base):
         """
         list_of_analyzers = dict(kiskadee.config._sections["analyzers"])
         for _name, _version in list_of_analyzers.items():
-            if not _session.query(Analyzer)\
+            if not db.session.query(Analyzer)\
                             .filter_by(name = _name, version = _version )\
                             .first():
                 new_analyzer = kiskadee.model.Analyzer(name = _name,
                                                        version = _version)
-                _session.add(new_analyzer)
-        _session.commit()
+                db.session.add(new_analyzer)
+        db.session.commit()
 
 class Analysis(Base):
     """Abstraction of a package analysis."""
@@ -139,21 +139,21 @@ class Analysis(Base):
                               uselist=False, back_populates='analysis')
 
     @staticmethod
-    def save(session, data, analyzer, result, version):
+    def save(db, data, analyzer, result, version):
         _analysis = kiskadee.model.Analysis()
         try:
-            _analyzer = session.query(kiskadee.model.Analyzer).\
+            _analyzer = db.session.query(kiskadee.model.Analyzer).\
                     filter_by(name = analyzer).first()
             _analysis.analyzer_id = _analyzer.id
             _analysis.version_id = version.id
             _analysis.raw = json.loads(result)
-            session.add(_analysis)
-            session.commit()
+            db.session.add(_analysis)
+            db.session.commit()
             dict_analysis = {
                     'results': _analysis.raw['results'],
                     'id': _analysis.id
                 }
-            Report.save(session, dict_analysis, data, _analyzer.name)
+            Report.save(db, dict_analysis, data, _analyzer.name)
             kiskadee.logger.debug(
                     "MONITOR: Saved analysis done by {} for package: {}-{}"
                     .format(analyzer, data["name"], data["version"])
@@ -184,7 +184,7 @@ class Report(Base):
     analysis = orm.relationship('Analysis', back_populates='report')
 
     @staticmethod
-    def save(session, analysis, data, analyzer_name):
+    def save(db, analysis, data, analyzer_name):
         try:
             results = analysis['results']
             analyzer_report = Report.REPORTERS[analyzer_name](results)
@@ -194,8 +194,8 @@ class Report(Base):
                     ._compute_reports(analyzer_name)
                 )
             _reports.analysis_id = analysis['id']
-            session.add(_reports)
-            session.commit()
+            db.session.add(_reports)
+            db.session.commit()
             kiskadee.logger.debug(
                     "MONITOR: Saved analysis reports for {} package"
                     .format(data["name"])
diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py
index 0fc6755..7f485c2 100644
--- a/kiskadee/monitor.py
+++ b/kiskadee/monitor.py
@@ -19,11 +19,11 @@ RUNNING = True
 class Monitor:
     """Provide kiskadee monitoring objects."""
 
-    def __init__(self, _session, queues):
+    def __init__(self, db, queues):
         """Return a non initialized Monitor."""
-        self.session = _session
+        self.db = db
         self.queues = queues
-        kiskadee.model.Analyzer.create_analyzers(self.session)
+        kiskadee.model.Analyzer.create_analyzers(self.db)
 
     def start(self):
         kiskadee.logger.debug('Monitor PID: {}'.format(os.getpid()))
@@ -56,8 +56,8 @@ class Monitor:
     def get_fetcher_and_project(self, data):
         fetcher_name = data['fetcher'].split('.')[-1]
         project_name = data['name']
-        fetcher = self._query(Fetcher).filter_by(name = fetcher_name).first()
-        project = self._query(Package).filter_by(name = project_name).first()
+        fetcher = self.db.filter_by_name(Fetcher, fetcher_name)
+        project = self.db.filter_by_name(Package, project_name)
         return fetcher, project
 
     def is_a_new_project_version(self, project, data):
@@ -69,17 +69,14 @@ class Monitor:
     def save_analyzed_project(self, data):
         if not data:
             return {}
-        project = self._query(Package).filter_by(name = data['name']).first()
+        project = self.db.filter_by_name(Package, data['name'])
         if not project:
-            project = Package.save(self.session, data)
+            project = Package.save(self.db, data)
         if project:
-            project = Package.update(self.session, project, data)
+            project = Package.update(self.db, project, data)
 
         for analyzer, result in data['results'].items():
-            Analysis.save(self.session, data, analyzer, result, project.versions[-1])
-
-    def _query(self, arg):
-        return self.session.query(arg)
+            Analysis.save(self.db, data, analyzer, result, project.versions[-1])
 
 
     @staticmethod
@@ -95,8 +92,8 @@ def daemon():
     """Entry point to the monitor module."""
     # TODO: improve with start/stop system
     queues = kiskadee.queue.Queues()
-    session = kiskadee.database.Database().session
-    monitor = Monitor(session, queues)
+    db = kiskadee.database.Database()
+    monitor = Monitor(db, queues)
     runner = Runner(queues)
     monitor_process = Process(
             target=monitor.start,
diff --git a/kiskadee/runner.py b/kiskadee/runner.py
index dfe02a0..0dee08a 100644
--- a/kiskadee/runner.py
+++ b/kiskadee/runner.py
@@ -22,7 +22,7 @@ class Runner:
         self.fetcher = None
         self.project = None
 
-    def runner(self, queues):
+    def runner(self):
         """Run static analyzers.
 
         Continuously dequeue packages from `analyses_queue` and call the
diff --git a/kiskadee/tests/__init__.py b/kiskadee/tests/__init__.py
index 2165cb9..296e655 100644
--- a/kiskadee/tests/__init__.py
+++ b/kiskadee/tests/__init__.py
@@ -1 +1,13 @@
 """kiskadee test suite."""
+from kiskadee import model
+
+
+def clean_test_db(db, metadata):
+    """Clean test database to run tests independently."""
+    #  The number of openend sessions is limited by the operational system.
+    # If we open too much connections, the database will lock the tests
+    # execution.
+    db.session.close_all()
+    metadata.drop_all(db.engine)
+    metadata.create_all(db.engine)
+    model.Analyzer.create_analyzers(db)
diff --git a/kiskadee/tests/api/test_api.py b/kiskadee/tests/api/test_api.py
index 1ef5b8a..cff7f1b 100644
--- a/kiskadee/tests/api/test_api.py
+++ b/kiskadee/tests/api/test_api.py
@@ -3,9 +3,9 @@ import unittest
 from unittest.mock import MagicMock
 
 import kiskadee
-from kiskadee.monitor import Monitor
 import kiskadee.api.app
 import kiskadee.fetchers.example
+import kiskadee.database
 
 
 class ApiTestCase(unittest.TestCase):
@@ -14,16 +14,14 @@ class ApiTestCase(unittest.TestCase):
     def setUpClass(cls):
         kiskadee.api.app.kiskadee.testing = True
         cls.app = kiskadee.api.app.kiskadee.test_client()
+        cls.db = kiskadee.api.app.db = MagicMock()
 
     def setUp(self):
         self.fetcher = kiskadee.model.Fetcher(name='kiskadee-fetcher')
         self.queues = kiskadee.queue.Queues()
 
     def test_get_fetchers(self):
-        kiskadee.api.app.kiskadee_db_session = MagicMock()
-        kiskadee.api.app.kiskadee_db_session().query().all = MagicMock(
-                return_value=[self.fetcher]
-                )
+        self.db.session.query().all = MagicMock(return_value=[self.fetcher])
         response = self.app.get("/fetchers")
         response_data = json.loads(response.data.decode("utf-8"))
         self.assertIn("fetchers", response_data)
@@ -33,23 +31,18 @@ class ApiTestCase(unittest.TestCase):
             )
 
     def test_get_analysis_as_json(self):
-        package = kiskadee.model.Package(name='mocked-package', id=1)
-        version = kiskadee.model.Version(number='1.0.0', id=1, package_id=1)
         analysis = kiskadee.model.Analysis(
                 id=1, analyzer_id=1, version_id=1,
                 raw='analysis result: foo'
                 )
 
-        kiskadee.api.app.kiskadee_db_session = MagicMock()
-        Monitor(kiskadee.api.app.kiskadee_db_session(), self.queues)
-        db_session = kiskadee.api.app.kiskadee_db_session()
-        db_session.query(kiskadee.model.Package)\
-                  .filter_by().id = MagicMock()
-
-        db_session.query(kiskadee.model.Version)\
-                  .filter_by().id = MagicMock()
-        db_session.query().options()\
-                  .filter().all = MagicMock(return_value=[analysis])
+        self.db.filter_by_name = MagicMock()
+        self.db.session.query(kiskadee.model.Package)\
+            .filter_by().id = MagicMock()
+        self.db.session.query(kiskadee.model.Version)\
+            .filter_by().id = MagicMock()
+        self.db.session.query().options()\
+            .filter().all = MagicMock(return_value=[analysis])
 
         response = self.app.get("/analysis/mocked-package/1.0.0")
         response_data = json.loads(response.data.decode("utf-8"))
@@ -61,10 +54,7 @@ class ApiTestCase(unittest.TestCase):
                 id=1, analyzer_id=1, version_id=1,
                 raw={'results': 'first analysis results'}
                 )
-
-        kiskadee.api.app.kiskadee_db_session().query().get = MagicMock(
-                return_value=analysis
-                )
+        self.db.get = MagicMock(return_value=analysis)
         response = self.app.get("/analysis/kiskadee-package/7.23/1/results")
         response_data = json.loads(response.data.decode("utf-8"))
         self.assertIn("analysis_results", response_data)
diff --git a/kiskadee/tests/integration/test_monitor.py b/kiskadee/tests/integration/test_monitor.py
index 2369499..9691ca2 100644
--- a/kiskadee/tests/integration/test_monitor.py
+++ b/kiskadee/tests/integration/test_monitor.py
@@ -1,16 +1,13 @@
 import unittest
 from unittest.mock import MagicMock
-from sqlalchemy.orm import sessionmaker
 
-from kiskadee import model
 from kiskadee.monitor import Monitor
 from kiskadee.queue import Queues
-from kiskadee.model import Package, Analyzer, Report, Analysis, Fetcher
+from kiskadee.model import Package, Fetcher
 import kiskadee.queue
 import kiskadee.fetchers.debian
 import kiskadee.fetchers.anitya
 import kiskadee.fetchers.example
-from kiskadee.database import Database
 
 
 class MonitorTestCase(unittest.TestCase):
@@ -23,11 +20,10 @@ class MonitorTestCase(unittest.TestCase):
             klass()
 
         self.example_fetcher = kiskadee.fetchers.example.Fetcher()
-        self.session = MagicMock()
+        self.db = MagicMock()
 
         queues = Queues()
-        self.monitor = Monitor(self.session, queues)
-        self.session.query = MagicMock(side_effect=mocked_models)
+        self.monitor = Monitor(self.db, queues)
 
         self.data1 = {
                 'name': 'curl',
@@ -60,9 +56,8 @@ class MonitorTestCase(unittest.TestCase):
         self.assertIn("fetcher", second_monitored_project)
 
     def test_send_project_to_runner(self):
-        fetcher = kiskadee.model.Fetcher(name='example')
-        project = kiskadee.model.Package(name='project1',
-                fetcher_id=fetcher.id)
+        fetcher = Fetcher(name='example')
+        Package(name='project1', fetcher_id=fetcher.id)
         self.monitor.get_fetcher_and_project = MagicMock(
                 return_value=[fetcher, {}]
                 )
@@ -70,8 +65,7 @@ class MonitorTestCase(unittest.TestCase):
                 return_value=True
                 )
         self.monitor.send_project_to_runner(self.data1)
-        self.assertEqual(self.monitor.queues.dequeue_analysis(),
-                self.data1)
+        self.assertEqual(self.monitor.queues.dequeue_analysis(), self.data1)
 
         if __name__ == '__main__':
             unittest.main()
diff --git a/kiskadee/tests/integration/test_runner.py b/kiskadee/tests/integration/test_runner.py
index 3cd646f..f8ed25b 100644
--- a/kiskadee/tests/integration/test_runner.py
+++ b/kiskadee/tests/integration/test_runner.py
@@ -3,6 +3,7 @@ This integration tests needs docker engine be running and accessible
 by the user that is executing the tests. It also needs selinux running in
 a permissive mode (setenforce 0)
 """
+
 import unittest
 import re
 from unittest.mock import MagicMock
@@ -13,6 +14,7 @@ import kiskadee.monitor
 import kiskadee.fetchers.example
 import kiskadee.analyzers
 
+
 class RunnerTestCase(unittest.TestCase):
 
     """Docstring for AnalyzersTestCase. """
@@ -40,18 +42,19 @@ class RunnerTestCase(unittest.TestCase):
     def test_run_a_single_analyzer(self):
         self.runner.project = self.project
         self.runner.fetcher = self.runner.import_project_fetcher()
-        source_path = self.runner.get_project_code_path()
+        self.runner.get_project_code_path()
         analyzer = self.runner.fetcher.analyzers()[0]
         self.assertIsNotNone(re.search('.*cppcheck.*', analyzer))
         self.assertIsNone(re.search('.*pylint.*', analyzer))
 
     def test_analysis_an_incoming_monitor_project(self):
-        self.session = MagicMock()
-        self.session.query().filter_by().first = MagicMock(return_value={})
-        self.monitor = kiskadee.monitor.Monitor(self.session, self.queues)
+        self.db = MagicMock()
+        self.db.filter_by_name = MagicMock(return_value={})
+        self.monitor = kiskadee.monitor.Monitor(self.db, self.queues)
         self.monitor.send_project_to_runner(self.project)
         incoming_project = self.runner.queues.dequeue_analysis()
         self.assertEqual(self.project, incoming_project)
 
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/kiskadee/tests/units/test_model.py b/kiskadee/tests/units/test_model.py
index aba56e7..f542aaa 100644
--- a/kiskadee/tests/units/test_model.py
+++ b/kiskadee/tests/units/test_model.py
@@ -1,87 +1,85 @@
 import unittest
 from sqlalchemy import exc
-from sqlalchemy.orm import sessionmaker
 
+import kiskadee
 from kiskadee import model
 from kiskadee.database import Database
 
 
 class ModelTestCase(unittest.TestCase):
 
+    @classmethod
+    def setUpClass(cls):
+        cls.db = Database('db_test')
+
     def setUp(self):
-        self.engine = Database('db_test').engine
-        Session = sessionmaker(bind=self.engine)
-        self.session = Session()
-        model.Base.metadata.create_all(self.engine)
-        model.Analyzer.create_analyzers(self.session)
+        kiskadee.tests.clean_test_db(self.db, model.Base.metadata)
+        self.package = model.Package(name='python-kiskadee')
+        self.version = model.Version(number='1.0-rc1')
         self.fetcher = model.Fetcher(
               name='kiskadee-fetcher', target='university'
             )
-        self.package = model.Package(name='python-kiskadee')
-        self.version = model.Version(number='1.0-rc1')
+        self.db.session.add(self.fetcher)
+        self.db.session.commit()
         self.fetcher.packages.append(self.package)
         self.package.versions.append(self.version)
-        self.session.add(self.package)
-        self.session.add(self.fetcher)
-        self.session.add(self.version)
+        self.db.session.add(self.package)
+        self.db.session.add(self.fetcher)
+        self.db.session.add(self.version)
 
         self.analysis = model.Analysis(
                 analyzer_id=1,
                 version_id=1,
                 raw=""
                 )
-        self.session.add(self.analysis)
+        self.db.session.add(self.analysis)
         self.report = model.Report(
                 analysis_id=1,
         )
-        self.session.add(self.report)
-        self.session.commit()
-
-    def tearDown(self):
-        self.session.close()
-        model.Base.metadata.drop_all()
+        self.db.session.add(self.report)
+        self.db.session.commit()
 
     def test_query_fetcher(self):
-        fetchers = self.session.query(model.Fetcher).all()
+        fetchers = self.db.session.query(model.Fetcher).all()
         self.assertEqual(fetchers, [self.fetcher])
 
     def test_query_package(self):
-        packages = self.session.query(model.Package).all()
+        packages = self.db.session.query(model.Package).all()
         self.assertEqual(packages, [self.package])
 
     def test_query_version(self):
-        versions = self.session.query(model.Version).all()
+        versions = self.db.session.query(model.Version).all()
         self.assertEqual(versions, [self.version])
 
     def test_query_report(self):
-        reports = self.session.query(model.Report).all()
+        reports = self.db.session.query(model.Report).all()
         self.assertEqual(reports, [self.report])
 
     def test_add_fetcher(self):
-        fetchers = self.session.query(model.Fetcher).all()
+        fetchers = self.db.session.query(model.Fetcher).all()
         self.assertEqual(len(fetchers), 1)
-        self.session.add(model.Fetcher(name='foo', target='bar'))
-        fetchers = self.session.query(model.Fetcher).all()
+        self.db.session.add(model.Fetcher(name='foo', target='bar'))
+        fetchers = self.db.session.query(model.Fetcher).all()
         self.assertEqual(len(fetchers), 2)
 
     def test_add_version_without_package(self):
         version = model.Version(number='3.1')
-        self.session.add(version)
+        self.db.session.add(version)
         with self.assertRaises(exc.IntegrityError):
-            self.session.commit()
+            self.db.session.commit()
 
     def test_add_report_without_analysis(self):
         report = model.Report(
         )
-        self.session.add(report)
+        self.db.session.add(report)
         with self.assertRaises(exc.IntegrityError):
-            self.session.commit()
+            self.db.session.commit()
 
     def test_add_package_without_fetcher(self):
         package = model.Package(name='foo-bar')
-        self.session.add(package)
+        self.db.session.add(package)
         with self.assertRaises(exc.IntegrityError):
-            self.session.commit()
+            self.db.session.commit()
 
     def test_unique_package_in_fetcher(self):
         package_1 = model.Package(name='foo-bar')
@@ -89,7 +87,7 @@ class ModelTestCase(unittest.TestCase):
         self.fetcher.packages.append(package_1)
         self.fetcher.packages.append(package_2)
         with self.assertRaises(exc.IntegrityError):
-            self.session.commit()
+            self.db.session.commit()
 
     def test_unique_version_for_package(self):
         package_version_1 = model.Version(number='1.0')
@@ -97,10 +95,10 @@ class ModelTestCase(unittest.TestCase):
         self.package.versions.append(package_version_1)
         self.package.versions.append(package_version_2)
         with self.assertRaises(exc.IntegrityError):
-            self.session.commit()
+            self.db.session.commit()
 
     def test_compose_kiskadee_source(self):
-        _analyzer = self.session.query(model.Analyzer)\
+        _analyzer = self.db.session.query(model.Analyzer)\
                     .filter(model.Analyzer.name == "cppcheck").first()
         package = model.Package(
                 name='bla',
@@ -126,11 +124,11 @@ class ModelTestCase(unittest.TestCase):
     def test_save_several_analysis(self):
 
         _analyzer1 = (
-                self.session.query(model.Analyzer)
+                self.db.session.query(model.Analyzer)
                 .filter(model.Analyzer.name == "cppcheck").first()
                 )
         _analyzer2 = (
-                self.session.query(model.Analyzer)
+                self.db.session.query(model.Analyzer)
                 .filter(model.Analyzer.name == "flawfinder").first()
                 )
 
@@ -146,9 +144,9 @@ class ModelTestCase(unittest.TestCase):
         self.fetcher.packages.append(package)
         package.versions.append(package_version)
 
-        self.session.add(package)
-        self.session.add(package_version)
-        self.session.commit()
+        self.db.session.add(package)
+        self.db.session.add(package_version)
+        self.db.session.commit()
 
         package_analysis1 = model.Analysis(
                 raw="<>",
@@ -161,12 +159,12 @@ class ModelTestCase(unittest.TestCase):
                 version_id=package_version.id
                 )
 
-        self.session.add(package_analysis1)
-        self.session.add(package_analysis2)
-        self.session.commit()
+        self.db.session.add(package_analysis1)
+        self.db.session.add(package_analysis2)
+        self.db.session.commit()
 
         saved_package = (
-                self.session.query(model.Package)
+                self.db.session.query(model.Package)
                 .filter(model.Package.name == 'bla').first()
                 )
         analysis = saved_package.versions[-1].analysis
diff --git a/kiskadee/tests/units/test_runner.py b/kiskadee/tests/units/test_runner.py
new file mode 100644
index 0000000..502c969
--- /dev/null
+++ b/kiskadee/tests/units/test_runner.py
@@ -0,0 +1,18 @@
+import unittest
+
+import kiskadee.fetchers.example
+import kiskadee.runner
+
+
+class RunnerTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.queues = kiskadee.queue.Queues()
+        self.runner = kiskadee.runner.Runner(self.queues)
+
+    def test_path_to_uncompressed_source(self):
+
+        self.runner.fetcher = kiskadee.fetchers.example.Fetcher()
+        self.runner.project = {'name': 'test'}
+        uncompressed_source_path = self.runner.get_project_code_path()
+        self.assertIsNotNone(uncompressed_source_path)

From 13684e95bc6d04ff51586ee860f951a671eba13b Mon Sep 17 00:00:00 2001
From: David Carlos <ddavidcarlos1392@gmail.com>
Date: Mar 16 2018 02:30:30 +0000
Subject: [PATCH 12/17] Fix Jenkinsfile


---

diff --git a/Jenkinsfile b/Jenkinsfile
index 9d1ce46..341728d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -6,7 +6,9 @@ pipeline {
     stage('Create Virtualenv') {
       steps {
         sh 'virtualenv -p /usr/bin/python3 .'
-	sh 'source bin/activate && pip install -e .'
+	      sh 'source bin/activate && pip install -e .'
+	      sh "source bin/activate && pip install fedmsg[consumers]"
+	      sh "source bin/activate && pip install -r test_requirements.txt"
       }
     }
 
@@ -18,7 +20,7 @@ pipeline {
 
     stage('Run Tests') {
       steps {
-        sh "source bin/activate && pip install coverage && make check"
+        sh "source bin/activate && make test"
       }
 	post {
 		success {

From f4675510a6c1b8b0624edb7f664bf35396082ed1 Mon Sep 17 00:00:00 2001
From: David Carlos <ddavidcarlos1392@gmail.com>
Date: Mar 16 2018 02:30:30 +0000
Subject: [PATCH 13/17] Add initializer module.


  - Removes from monitor the responsibility of manage kiskadee process.
  Now this will be done by the initializer module.

---

diff --git a/kiskadee/initializer.py b/kiskadee/initializer.py
new file mode 100644
index 0000000..c39304d
--- /dev/null
+++ b/kiskadee/initializer.py
@@ -0,0 +1,22 @@
+"""Initializes kiskadee components.
+
+This module will run monitor and runner as processes.
+"""
+
+from multiprocessing import Process
+
+import kiskadee.model
+import kiskadee.queue
+import kiskadee.database
+from kiskadee.runner import Runner
+from kiskadee.monitor import Monitor
+
+def init():
+    queues = kiskadee.queue.Queues()
+    db = kiskadee.database.Database()
+    monitor = Monitor(db, queues)
+    runner = Runner(queues)
+    Process(target=monitor.run,).start()
+    runner_process = Process(target=runner.run,)
+    runner_process.start()
+    runner_process.join()
diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py
index 7f485c2..1996a0b 100644
--- a/kiskadee/monitor.py
+++ b/kiskadee/monitor.py
@@ -4,15 +4,12 @@ kiskadee monitors repositories checking for new project versions to be
 analyzed. This module provides such capabilities.
 """
 import threading
-from multiprocessing import Process
 import time
 import os
 import importlib
 
-import kiskadee.database
-from kiskadee.runner import Runner
-import kiskadee.queue
-from kiskadee.model import Package, Fetcher, Version, Report, Analysis
+import kiskadee
+from kiskadee.model import Package, Fetcher, Version, Report, Analysis, Analyzer
 
 RUNNING = True
 
@@ -23,9 +20,9 @@ class Monitor:
         """Return a non initialized Monitor."""
         self.db = db
         self.queues = queues
-        kiskadee.model.Analyzer.create_analyzers(self.db)
+        Analyzer.create_analyzers(self.db)
 
-    def start(self):
+    def run(self):
         kiskadee.logger.debug('Monitor PID: {}'.format(os.getpid()))
 
         for fetcher in kiskadee.load_fetchers():
@@ -86,21 +83,3 @@ class Monitor:
         module_as_a_thread.start()
         if joinable or timeout:
             module_as_a_thread.join(timeout)
-
-
-def daemon():
-    """Entry point to the monitor module."""
-    # TODO: improve with start/stop system
-    queues = kiskadee.queue.Queues()
-    db = kiskadee.database.Database()
-    monitor = Monitor(db, queues)
-    runner = Runner(queues)
-    monitor_process = Process(
-            target=monitor.start,
-        )
-    runner_process = Process(
-            target=runner.runner,
-        )
-    monitor_process.start()
-    runner_process.start()
-    runner_process.join()
diff --git a/kiskadee/runner.py b/kiskadee/runner.py
index 0dee08a..ea9ef60 100644
--- a/kiskadee/runner.py
+++ b/kiskadee/runner.py
@@ -22,7 +22,7 @@ class Runner:
         self.fetcher = None
         self.project = None
 
-    def runner(self):
+    def run(self):
         """Run static analyzers.
 
         Continuously dequeue packages from `analyses_queue` and call the
@@ -56,7 +56,7 @@ class Runner:
         return self.project
 
 
-    def clean_temporary_directory(self, temp_dir):
+    def rmdtemp(self, temp_dir):
         # not delete the source code used on tests.
         if not temp_dir.find("kiskadee/tests") > -1:
             shutil.rmtree(temp_dir)
@@ -78,7 +78,7 @@ class Runner:
                 analyzers, source_path
                 )
         self.enqueue_analysis_to_monitor(analysis_result)
-        self.clean_temporary_directory(source_path)
+        self.rmdtemp(source_path)
 
     def enqueue_analysis_to_monitor(self, analysis_result):
         if analysis_result['results']:
@@ -121,7 +121,7 @@ class Runner:
             uncompressed_source_path = self.uncompress_project_code(
                     compressed_source_path
                     )
-            self.clean_temporary_directory(os.path.dirname(compressed_source_path))
+            self.rmdtemp(os.path.dirname(compressed_source_path))
             return uncompressed_source_path
         else:
             kiskadee.logger.debug('RUNNER: invalid compressed source')
diff --git a/setup.py b/setup.py
index 03a13f3..9a74169 100644
--- a/setup.py
+++ b/setup.py
@@ -44,7 +44,7 @@ setup(
     packages=find_packages(),
     include_package_data=False,
     entry_points={'console_scripts': [
-            'kiskadee = kiskadee.monitor:daemon',
+        'kiskadee = kiskadee.initializer:init',
             'kiskadee_api = kiskadee.api.app:main',],
             'moksha.consumer': (
                 'anityaconsumer = kiskadee.fetchers.anitya:AnityaConsumer')},

From 0431fc7de415ce8e25d60819f068d55c17e5aff1 Mon Sep 17 00:00:00 2001
From: David Carlos <ddavidcarlos1392@gmail.com>
Date: Mar 16 2018 02:30:30 +0000
Subject: [PATCH 14/17] Fix logging messages.


---

diff --git a/kiskadee/fetchers/__init__.py b/kiskadee/fetchers/__init__.py
index b61511b..21a6fd6 100644
--- a/kiskadee/fetchers/__init__.py
+++ b/kiskadee/fetchers/__init__.py
@@ -49,12 +49,8 @@ class Fetcher(abc.ABC):
         if not project:
             raise NotImplementedError('fetcher must call parent watch method ')
 
-        kiskadee.queue.Queues().enqueue_project(project)
-        fetcher = project['fetcher']
-        kiskadee.logger.debug(
-                "{} fetcher: sending package {}_{} for monitor"
-                .format(fetcher, project['name'], project['version'])
-            )
+        fetcher = project['fetcher'].split('.')[-1]
+        kiskadee.queue.Queues().enqueue_project(project, fetcher)
 
     @abc.abstractmethod
     def compare_versions(self, new, old):
diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py
index 1996a0b..58f1493 100644
--- a/kiskadee/monitor.py
+++ b/kiskadee/monitor.py
@@ -23,20 +23,20 @@ class Monitor:
         Analyzer.create_analyzers(self.db)
 
     def run(self):
-        kiskadee.logger.debug('Monitor PID: {}'.format(os.getpid()))
+        kiskadee.logger.debug('MONITOR PID: {}'.format(os.getpid()))
 
         for fetcher in kiskadee.load_fetchers():
             thread = Monitor._start_fetcher(fetcher.Fetcher().watch)
 
         while RUNNING:
+            kiskadee.logger.debug('MONITOR STATE: Idle'.format(os.getpid()))
             new_project = self.dequeue_project_from_fetchers()
             self.send_project_to_runner(new_project)
             analyzed_project = self.dequeue_analysis_from_runner()
             self.save_analyzed_project(analyzed_project)
 
     def dequeue_project_from_fetchers(self):
-        new_project = self.queues.dequeue_project()
-        return new_project if new_project else {}
+        return self.queues.dequeue_project()
 
     def dequeue_analysis_from_runner(self):
         return self.queues.dequeue_result()
@@ -45,10 +45,12 @@ class Monitor:
         if data:
             fetcher, project = self.get_fetcher_and_project(data)
             data["fetcher_id"] = fetcher.id if fetcher else ''
-            if not project:
-                self.queues.enqueue_analysis(data)
-            elif self.is_a_new_project_version(project, data):
-                self.queues.enqueue_analysis(data)
+            if not self.is_a_new_project_version(project, data):
+                kiskadee.logger.debug('MONITOR STATE: Project {} already'\
+                        'analyzed'.format(data['name']))
+                return
+            self.queues.enqueue_analysis(data)
+
 
     def get_fetcher_and_project(self, data):
         fetcher_name = data['fetcher'].split('.')[-1]
@@ -58,10 +60,13 @@ class Monitor:
         return fetcher, project
 
     def is_a_new_project_version(self, project, data):
-        project_version = data['version']
-        analysed_version = project.versions[-1].number
-        fetcher = importlib.import_module(data['fetcher']).Fetcher()
-        fetcher.compare_versions(project_version, analysed_version)
+        if project:
+            project_version = data['version']
+            analysed_version = project.versions[-1].number
+            fetcher = importlib.import_module(data['fetcher']).Fetcher()
+            return fetcher.compare_versions(project_version, analysed_version)
+        else:
+            return True
 
     def save_analyzed_project(self, data):
         if not data:
diff --git a/kiskadee/queue.py b/kiskadee/queue.py
index 672958a..bbb7128 100644
--- a/kiskadee/queue.py
+++ b/kiskadee/queue.py
@@ -4,34 +4,6 @@ from multiprocessing import Queue
 
 import kiskadee
 
-def empty_queue(func):
-    """Decorator that checks if retrieve data from a
-    queue will trigger a empty exception.
-    """
-    def wrapper(*args, **kwargs):
-        try:
-            return func(*args, **kwargs)
-        except Exception:
-            return {}
-    return wrapper
-
-def enqueue_project(fetcher_watch):
-    """Decorator to be used with watch fetchers method. Every project returned
-    by the watch method, will be enqueued on the queue projects.
-    :arg1: watch method
-    :returns: a wrapper variable
-    """
-    def wrapper(*args, **kwargs):
-        project = fetcher_watch(*args, **kwargs)
-        kiskadee.queue.queues.enqueue_project(project)
-        fetcher = project['fetcher'].name
-        kiskadee.logger.debug(
-                "{} fetcher: sending package {}_{} for monitor"
-                .format(fetcher, package['name'], package['version'])
-            )
-        time.sleep(2)
-    return wrapper
-
 analysis = Queue()
 results = Queue()
 projects = Queue()
@@ -42,7 +14,7 @@ class Queues():
     @staticmethod
     def enqueue_analysis(project_to_analysis):
         """Put a analysis on the analysis queue."""
-        log_msg = "Sending project {}_{} for analysis"\
+        log_msg = "MONITOR STATE: Sending project {}-{} for analysis"\
                 .format(project_to_analysis['name'],
                         project_to_analysis['version'])
         kiskadee.logger.debug(log_msg)
@@ -52,11 +24,12 @@ class Queues():
     def dequeue_analysis():
         """Get a analysis from the analysis queue."""
         project_to_analysis = analysis.get()
+        fetcher = project_to_analysis ['fetcher'].split('.')[-1]
         kiskadee.logger.debug(
-                'RUNNER: deqeued {}-{} from {}'
+                'RUNNER STATE: dequeued {}-{} from {}'
                 .format(project_to_analysis['name'],
                         project_to_analysis['version'],
-                        project_to_analysis['fetcher'])
+                        fetcher)
             )
         return project_to_analysis
 
@@ -64,25 +37,39 @@ class Queues():
     def enqueue_result(project):
         """Put a result on the results queue."""
         kiskadee.logger.debug(
-                "RUNNER: Sending {}-{} to Monitor"
+                "RUNNER STATE: Sending {}-{} to Monitor"
                 .format(project["name"],
                         project["version"])
             )
         results.put(project)
 
     @staticmethod
-    @empty_queue
     def dequeue_result():
         """Get a result from the results queue."""
-        return results.get(timeout=1)
+        result = results.get()
+        kiskadee.logger.debug(
+                "MONITOR STATE: Pick Up analyzed project"
+                .format(result["name"],
+                        result["version"])
+            )
+        return result
 
     @staticmethod
-    def enqueue_project(project):
+    def enqueue_project(project, fetcher):
         """Put a result on the results queue."""
+        kiskadee.logger.debug(
+                "FETCHER {}: sending package {}-{} for monitor"
+                .format(fetcher, project['name'], project['version'])
+            )
         projects.put(project)
 
     @staticmethod
-    @empty_queue
     def dequeue_project():
         """Get a result from the results queue."""
-        return projects.get(timeout=1)
+        project = projects.get()
+        kiskadee.logger.debug(
+                "MONITOR STATE: Pick Up monitored project."
+                .format(project["name"],
+                        project["version"])
+            )
+        return project
diff --git a/kiskadee/report.py b/kiskadee/report.py
index 10aa7cc..1afc770 100644
--- a/kiskadee/report.py
+++ b/kiskadee/report.py
@@ -36,7 +36,6 @@ class CppcheckReport(Report):
 
     def _compute_reports(self, analyzer):
         """Compute every report type for cppcheck analyzer."""
-        print(analyzer)
         result_dict = {
                 'warning': 0,
                 'error': 0,
diff --git a/kiskadee/runner.py b/kiskadee/runner.py
index ea9ef60..d1fa040 100644
--- a/kiskadee/runner.py
+++ b/kiskadee/runner.py
@@ -1,4 +1,4 @@
-"""Run each static analyzer in each package marked for analysis."""
+"""Run each static analyzer in each project marked for analysis."""
 
 import shutil
 import tempfile
@@ -25,14 +25,13 @@ class Runner:
     def run(self):
         """Run static analyzers.
 
-        Continuously dequeue packages from `analyses_queue` and call the
-        :func:`analyze` method, passing the dequeued package.
-        After the analysis, updates the status of this package on the database.
+        Continuously dequeue projects from `analyses_queue` and call the
+        :func:`analyze` method, passing the dequeued project as argument.
+        After the analysis, send the result back to monitor.
         """
-        kiskadee.logger.debug('Runner PID: {}'.format(os.getpid()))
+        kiskadee.logger.debug('RUNNER PID: {}'.format(os.getpid()))
         while RUNNING:
-            kiskadee.logger.debug('RUNNER: Waiting to dequeue'\
-                                  ' project to analysis...')
+            kiskadee.logger.debug('RUNNER STATE: Idle.')
             self.project = self.queues.dequeue_analysis()
             self.call_analyzers()
 
@@ -42,8 +41,8 @@ class Runner:
                     self.project['fetcher']
                     ).Fetcher()
         except ModuleNotFoundError:
-            kiskadee.logger.debug("Fetcher {} could not be loaded"\
-                    .format(self.project['fetcher'])
+            kiskadee.logger.debug("RUNNER STATE: Fetcher {}\
+                    could not be loaded".format(self.project['fetcher'])
                 )
             return {}
 
@@ -95,19 +94,20 @@ class Runner:
         if source_path is None:
             return None
 
-        kiskadee.logger.debug('ANALYSIS: running {} ...'.format(analyzer))
+        kiskadee.logger.debug('RUNNER STATE: analysing with {} ...'\
+                .format(analyzer))
         try:
             analysis = kiskadee.analyzers.run(analyzer, source_path)
             firehose_report = kiskadee.converter.to_firehose(
                     analysis, analyzer
                 )
             kiskadee.logger.debug(
-                    'ANALYSIS: DONE {} analysis'
+                    'RUNNER STATE: DONE {} analysis'
                     .format(analyzer)
                 )
             return firehose_report
         except Exception as err:
-            kiskadee.logger.debug('RUNNER: could not generate analysis')
+            kiskadee.logger.debug('RUNNER STATE: could not generate analysis')
             kiskadee.logger.debug(err)
             return None
 
@@ -134,7 +134,8 @@ class Runner:
                     dir_to_unpack_source)
             return dir_to_unpack_source
         except Exception as err:
-            kiskadee.logger.debug('Could not unpack project source')
+            kiskadee.logger.debug('RUNNER STATE: Could not unpack'\
+                    'project source')
             kiskadee.logger.debug(err)
             return {}
 

From e405dc01d44923d3f9e693f90ff4f1cd3b1698b6 Mon Sep 17 00:00:00 2001
From: David Carlos <ddavidcarlos1392@gmail.com>
Date: Mar 16 2018 02:30:30 +0000
Subject: [PATCH 15/17] Rename Package model to Project.


  - Kiskadee monitors repositories like github, gitlab, bitbucket, but
  also can monitors linux distributions as Debian. It's better to call
  the source code as Project, instead of Package, because not always we
  will be dealing with packages, but we will always dealing with
  projects.

---

diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py
index 9eb7b3a..eae5017 100644
--- a/kiskadee/api/app.py
+++ b/kiskadee/api/app.py
@@ -4,8 +4,8 @@ from flask import request
 from flask_cors import CORS
 
 from kiskadee.database import Database
-from kiskadee.model import Package, Fetcher, Version, Analysis
-from kiskadee.api.serializers import PackageSchema, FetcherSchema,\
+from kiskadee.model import Project, Fetcher, Version, Analysis
+from kiskadee.api.serializers import ProjectSchema, FetcherSchema,\
         AnalysisSchema
 import json
 from sqlalchemy.orm import eagerload
@@ -30,8 +30,8 @@ def index():
 def packages():
     """Get the list of analyzed packages."""
     if request.method == 'GET':
-        packages = db.session.query(Package).all()
-        package_schema = PackageSchema(
+        packages = db.session.query(Project).all()
+        package_schema = ProjectSchema(
             many=True,
             exclude=['versions.analysis', 'versions.package_id']
         )
@@ -44,7 +44,7 @@ def package_analysis_overview(pkg_name, version):
     """Get a analysis list of some package version."""
 
     #TODO: This can be a simple inner join between package, version and analysis
-    _package_id = db.filter_by_name(Package, pkg_name).id
+    _package_id = db.filter_by_name(Project, pkg_name).id
     version_id = db.session.query(Version)\
             .filter_by(number = version, package_id = _package_id ).first().id
     analysis = (
diff --git a/kiskadee/api/serializers.py b/kiskadee/api/serializers.py
index e7168e9..b6d027c 100644
--- a/kiskadee/api/serializers.py
+++ b/kiskadee/api/serializers.py
@@ -1,7 +1,7 @@
 """Provide objects to serialize the kiskadee models."""
 
 from marshmallow import Schema, fields
-from kiskadee.model import Package, Fetcher, Analysis, Version,\
+from kiskadee.model import Project, Fetcher, Analysis, Version,\
         Report, Analyzer
 
 
@@ -49,7 +49,7 @@ class AnalysisSchema(Schema):
 
 
 class VersionSchema(Schema):
-    """Provide a serializer to the Package model."""
+    """Provide a serializer to the Project model."""
 
     id = fields.Int()
     number = fields.Str()
@@ -57,7 +57,7 @@ class VersionSchema(Schema):
     analysis = fields.Nested(AnalysisSchema, many=True)
 
     def make_object(self, data):
-        """Serialize a Package object."""
+        """Serialize a Project object."""
         print('MAKING OBJECT FROM', data)
         return Version(**data)
 
@@ -76,8 +76,8 @@ class FetcherSchema(Schema):
         return Fetcher(**data)
 
 
-class PackageSchema(Schema):
-    """Provide a serializer to the Package model."""
+class ProjectSchema(Schema):
+    """Provide a serializer to the Project model."""
 
     id = fields.Int()
     name = fields.Str()
@@ -85,6 +85,6 @@ class PackageSchema(Schema):
     versions = fields.Nested(VersionSchema, many=True)
 
     def make_object(self, data):
-        """Serialize a Package object."""
+        """Serialize a Project object."""
         print('MAKING OBJECT FROM', data)
-        return Package(**data)
+        return Project(**data)
diff --git a/kiskadee/database.py b/kiskadee/database.py
index 96cefe4..2c7edf5 100644
--- a/kiskadee/database.py
+++ b/kiskadee/database.py
@@ -4,7 +4,7 @@ from sqlalchemy import create_engine, orm
 
 import kiskadee
 from kiskadee.model import Base
-from kiskadee.model import Package, Fetcher, Version, Report, Analysis
+from kiskadee.model import Project, Fetcher, Version, Report, Analysis
 
 class Database:
     """kiskadee Database class."""
diff --git a/kiskadee/fetchers/debian.py b/kiskadee/fetchers/debian.py
index 1cd1604..9f66aa2 100644
--- a/kiskadee/fetchers/debian.py
+++ b/kiskadee/fetchers/debian.py
@@ -1,4 +1,4 @@
-"""Fetcher to monitor Debian Project Packages."""
+"""Fetcher to monitor Debian Packages."""
 
 import os
 import tempfile
@@ -16,14 +16,14 @@ RUNNING = True
 
 
 class Fetcher(kiskadee.fetchers.Fetcher):
-    """Fetcher to monitor Debian Project Packages."""
+    """Fetcher to monitor Debian Packages."""
 
     def watch(self):
         """Start the monitoring process for Debian Repositories.
 
-        Each project monitored by the fetcher will be
+        Each package monitored by the fetcher will be
         queued by calling the watch parent method,
-        passing the project data as argument.
+        passing it as argument.
         """
         kiskadee.logger.debug("Starting Debian fetcher")
         while RUNNING:
@@ -74,10 +74,10 @@ class Fetcher(kiskadee.fetchers.Fetcher):
         sources = os.path.join(path, 'Sources')
         with open(sources) as sources_file:
             for src in Sources.iter_paragraphs(sources_file):
-                project = self.project_to_enqueue(src)
-                super().watch(**project)
+                package = self.package_to_enqueue(src)
+                super().watch(**package)
 
-    def project_to_enqueue(self, src):
+    def package_to_enqueue(self, src):
         return {'name': src["Package"],
                 'version': self._parse_version(src["Version"]),
                 'fetcher': __name__,
diff --git a/kiskadee/model.py b/kiskadee/model.py
index acbd5a0..83302b9 100644
--- a/kiskadee/model.py
+++ b/kiskadee/model.py
@@ -11,21 +11,21 @@ from kiskadee.report import CppcheckReport, FlawfinderReport
 Base = declarative_base()
 
 
-class Package(Base):
-    """Software packages abstraction.
+class Project(Base):
+    """Software projects abstraction.
 
-    A software package is the source code for a software project. It may be
+    A project is the source code stored on some repository. It may be
     upstream's distribution or the sources provided by some other source, like
     a linux distribution.
     """
 
-    __tablename__ = 'packages'
+    __tablename__ = 'projects'
     id = Column(Integer,
-                Sequence('packages_id_seq', optional=True), primary_key=True)
+                Sequence('projects_id_seq', optional=True), primary_key=True)
     name = Column(Unicode(255), nullable=False)
     homepage = Column(Unicode(255), nullable=True)
     fetcher_id = Column(Integer, ForeignKey('fetchers.id'), nullable=False)
-    versions = orm.relationship('Version', backref='packages')
+    versions = orm.relationship('Version', backref='projects')
     __table_args__ = (
             UniqueConstraint('name', 'fetcher_id'),
             )
@@ -36,18 +36,18 @@ class Package(Base):
         if ('meta' in data) and ('homepage' in data['meta']):
             homepage = data['meta']['homepage']
 
-        _package = Package(
+        _project = Project(
                 name=data['name'],
                 homepage=homepage,
                 fetcher_id=data['fetcher_id']
                 )
-        db.session.add(_package)
+        db.session.add(_project)
         db.session.commit()
         _version = Version(number=data['version'],
-                           package_id=_package.id)
+                           project_id=_project.id)
         db.session.add(_version)
         db.session.commit()
-        return _package
+        return _project
 
     @staticmethod
     def update(db, project, data):
@@ -81,20 +81,20 @@ class Fetcher(Base):
     name = Column(Unicode(255), nullable=False, unique=True)
     target = Column(Unicode(255), nullable=True)
     description = Column(UnicodeText)
-    packages = orm.relationship('Package', backref='fetchers')
+    projects = orm.relationship('Project', backref='fetchers')
 
 
 class Version(Base):
-    """Abstraction of a package version."""
+    """Abstraction of a project version."""
 
     __tablename__ = 'versions'
     id = Column(Integer,
                 Sequence('versions_id_seq', optional=True), primary_key=True)
     number = Column(Unicode(100), nullable=False)
-    package_id = Column(Integer, ForeignKey('packages.id'), nullable=False)
+    project_id = Column(Integer, ForeignKey('projects.id'), nullable=False)
     analysis = orm.relationship('Analysis', backref='versions')
     __table_args__ = (
-            UniqueConstraint('number', 'package_id'),
+            UniqueConstraint('number', 'project_id'),
             )
 
 
@@ -127,7 +127,7 @@ class Analyzer(Base):
         db.session.commit()
 
 class Analysis(Base):
-    """Abstraction of a package analysis."""
+    """Abstraction of a project analysis."""
 
     __tablename__ = 'analysis'
     id = Column(Integer,
@@ -155,7 +155,7 @@ class Analysis(Base):
                 }
             Report.save(db, dict_analysis, data, _analyzer.name)
             kiskadee.logger.debug(
-                    "MONITOR: Saved analysis done by {} for package: {}-{}"
+                    "MONITOR: Saved analysis done by {} for project: {}-{}"
                     .format(analyzer, data["name"], data["version"])
                 )
             return
@@ -197,7 +197,7 @@ class Report(Base):
             db.session.add(_reports)
             db.session.commit()
             kiskadee.logger.debug(
-                    "MONITOR: Saved analysis reports for {} package"
+                    "MONITOR: Saved analysis reports for {} project"
                     .format(data["name"])
                 )
         except KeyError as key:
@@ -208,7 +208,7 @@ class Report(Base):
                 )
         except Exception as err:
             kiskadee.logger.debug(
-                    "MONITOR: Failed to get analysis reports to {} package"
+                    "MONITOR: Failed to get analysis reports to {} project"
                     .format(data["name"])
                 )
             kiskadee.logger.debug(err)
diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py
index 58f1493..d88a085 100644
--- a/kiskadee/monitor.py
+++ b/kiskadee/monitor.py
@@ -9,7 +9,7 @@ import os
 import importlib
 
 import kiskadee
-from kiskadee.model import Package, Fetcher, Version, Report, Analysis, Analyzer
+from kiskadee.model import Project, Fetcher, Version, Report, Analysis, Analyzer
 
 RUNNING = True
 
@@ -56,7 +56,7 @@ class Monitor:
         fetcher_name = data['fetcher'].split('.')[-1]
         project_name = data['name']
         fetcher = self.db.filter_by_name(Fetcher, fetcher_name)
-        project = self.db.filter_by_name(Package, project_name)
+        project = self.db.filter_by_name(Project, project_name)
         return fetcher, project
 
     def is_a_new_project_version(self, project, data):
@@ -71,11 +71,11 @@ class Monitor:
     def save_analyzed_project(self, data):
         if not data:
             return {}
-        project = self.db.filter_by_name(Package, data['name'])
+        project = self.db.filter_by_name(Project, data['name'])
         if not project:
-            project = Package.save(self.db, data)
+            project = Project.save(self.db, data)
         if project:
-            project = Package.update(self.db, project, data)
+            project = Project.update(self.db, project, data)
 
         for analyzer, result in data['results'].items():
             Analysis.save(self.db, data, analyzer, result, project.versions[-1])
diff --git a/kiskadee/tests/api/test_api.py b/kiskadee/tests/api/test_api.py
index cff7f1b..919ae5f 100644
--- a/kiskadee/tests/api/test_api.py
+++ b/kiskadee/tests/api/test_api.py
@@ -37,7 +37,7 @@ class ApiTestCase(unittest.TestCase):
                 )
 
         self.db.filter_by_name = MagicMock()
-        self.db.session.query(kiskadee.model.Package)\
+        self.db.session.query(kiskadee.model.Project)\
             .filter_by().id = MagicMock()
         self.db.session.query(kiskadee.model.Version)\
             .filter_by().id = MagicMock()
diff --git a/kiskadee/tests/integration/test_monitor.py b/kiskadee/tests/integration/test_monitor.py
index 9691ca2..f4acf72 100644
--- a/kiskadee/tests/integration/test_monitor.py
+++ b/kiskadee/tests/integration/test_monitor.py
@@ -3,7 +3,7 @@ from unittest.mock import MagicMock
 
 from kiskadee.monitor import Monitor
 from kiskadee.queue import Queues
-from kiskadee.model import Package, Fetcher
+from kiskadee.model import Project, Fetcher
 import kiskadee.queue
 import kiskadee.fetchers.debian
 import kiskadee.fetchers.anitya
@@ -14,7 +14,7 @@ class MonitorTestCase(unittest.TestCase):
 
     def setUp(self):
         def mocked_models(klass):
-            class Package:
+            class Project:
                 def all(self):
                     [self.pkg1, self.pkg2, self.pkg3, self.pkg4]
             klass()
@@ -57,7 +57,7 @@ class MonitorTestCase(unittest.TestCase):
 
     def test_send_project_to_runner(self):
         fetcher = Fetcher(name='example')
-        Package(name='project1', fetcher_id=fetcher.id)
+        Project(name='project1', fetcher_id=fetcher.id)
         self.monitor.get_fetcher_and_project = MagicMock(
                 return_value=[fetcher, {}]
                 )
diff --git a/kiskadee/tests/units/test_model.py b/kiskadee/tests/units/test_model.py
index f542aaa..b0aafa7 100644
--- a/kiskadee/tests/units/test_model.py
+++ b/kiskadee/tests/units/test_model.py
@@ -14,7 +14,7 @@ class ModelTestCase(unittest.TestCase):
 
     def setUp(self):
         kiskadee.tests.clean_test_db(self.db, model.Base.metadata)
-        self.package = model.Package(name='python-kiskadee')
+        self.package = model.Project(name='python-kiskadee')
         self.version = model.Version(number='1.0-rc1')
         self.fetcher = model.Fetcher(
               name='kiskadee-fetcher', target='university'
@@ -44,7 +44,7 @@ class ModelTestCase(unittest.TestCase):
         self.assertEqual(fetchers, [self.fetcher])
 
     def test_query_package(self):
-        packages = self.db.session.query(model.Package).all()
+        packages = self.db.session.query(model.Project).all()
         self.assertEqual(packages, [self.package])
 
     def test_query_version(self):
@@ -76,14 +76,14 @@ class ModelTestCase(unittest.TestCase):
             self.db.session.commit()
 
     def test_add_package_without_fetcher(self):
-        package = model.Package(name='foo-bar')
+        package = model.Project(name='foo-bar')
         self.db.session.add(package)
         with self.assertRaises(exc.IntegrityError):
             self.db.session.commit()
 
     def test_unique_package_in_fetcher(self):
-        package_1 = model.Package(name='foo-bar')
-        package_2 = model.Package(name='foo-bar')
+        package_1 = model.Project(name='foo-bar')
+        package_2 = model.Project(name='foo-bar')
         self.fetcher.packages.append(package_1)
         self.fetcher.packages.append(package_2)
         with self.assertRaises(exc.IntegrityError):
@@ -100,7 +100,7 @@ class ModelTestCase(unittest.TestCase):
     def test_compose_kiskadee_source(self):
         _analyzer = self.db.session.query(model.Analyzer)\
                     .filter(model.Analyzer.name == "cppcheck").first()
-        package = model.Package(
+        package = model.Project(
                 name='bla',
                 fetcher_id=self.fetcher.id
                 )
@@ -132,7 +132,7 @@ class ModelTestCase(unittest.TestCase):
                 .filter(model.Analyzer.name == "flawfinder").first()
                 )
 
-        package = model.Package(
+        package = model.Project(
                 name='bla',
                 fetcher_id=self.fetcher.id
                 )
@@ -164,8 +164,8 @@ class ModelTestCase(unittest.TestCase):
         self.db.session.commit()
 
         saved_package = (
-                self.db.session.query(model.Package)
-                .filter(model.Package.name == 'bla').first()
+                self.db.session.query(model.Project)
+                .filter(model.Project.name == 'bla').first()
                 )
         analysis = saved_package.versions[-1].analysis
         self.assertEqual(len(analysis), 2)

From cfff0dba40f0a82e56d3670d57c2a0bc639e4b46 Mon Sep 17 00:00:00 2001
From: David Carlos <ddavidcarlos1392@gmail.com>
Date: Mar 16 2018 03:38:34 +0000
Subject: [PATCH 16/17] Update documentation.


  - fix typos.
  - minor fix on anitya tests.
  - create method to save project analysis on Monitor.

---

diff --git a/README.md b/README.md
index 816aa6b..5e1ff50 100644
--- a/README.md
+++ b/README.md
@@ -11,6 +11,14 @@ For more information on the project, installation or development
 environment setup, refer to the full [kiskadee
 documentation](https://docs.pagure.org/kiskadee).
 
+To build the documentation locally:
+
+  pip install -U sphinx sphinx\_rtd\_theme
+  cd doc
+  make html
+
+Open the index.html file, inside the _build/html directory.
+
 ## Repositories
 
 kiskadee core and API development are hosted at [pagure](https://pagure.io/kiskadee).
diff --git a/alembic/versions/0920061a363d_rename_package_table_to_project.py b/alembic/versions/0920061a363d_rename_package_table_to_project.py
new file mode 100644
index 0000000..335101d
--- /dev/null
+++ b/alembic/versions/0920061a363d_rename_package_table_to_project.py
@@ -0,0 +1,25 @@
+"""rename package table to project.
+
+Revision ID: 0920061a363d
+Revises: cc4887b7c3da
+Create Date: 2018-03-13 17:48:05.506903
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '0920061a363d'
+down_revision = 'cc4887b7c3da'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.rename_table('packages', 'projects')
+    op.alter_column('versions', 'package_id', new_column_name='project_id')
+
+
+def downgrade():
+    op.rename_table('projects', 'packages')
diff --git a/doc/_static/kiskadee_arch.png b/doc/_static/kiskadee_arch.png
index 767a459..6d8fc35 100644
Binary files a/doc/_static/kiskadee_arch.png and b/doc/_static/kiskadee_arch.png differ
diff --git a/doc/architecture.rst b/doc/architecture.rst
index 0a8ed5b..7d56cbb 100644
--- a/doc/architecture.rst
+++ b/doc/architecture.rst
@@ -37,7 +37,7 @@ The class defines the following public behaviors:
 
 
 .. autoclass:: kiskadee.monitor.Monitor()
-    :members: monitor
+    :members:
 
 Runner - Module to run static analysis
 --------------------------------------
diff --git a/doc/fetchers.rst b/doc/fetchers.rst
index 98aaaeb..08d6089 100644
--- a/doc/fetchers.rst
+++ b/doc/fetchers.rst
@@ -21,30 +21,34 @@ A simple example of a kiskadee fetcher
 
 .. code-block:: python
 
-    import kiskadee
-    import sys
-    import kiskadee.queue
-    class Fetcher(kiskadee.fetchers.Fetcher):
-        def get_sources(self, source_data):
-            return 'kiskadee/tests/test_source/test_source.tar.gz'
-
-        @kiskadee.queue.package_enqueuer
-        def watch(self):
-            """There is no proper API to inspect new example versions.
-            It should not matter, since example will not receive updates.
-            """
-            example = {}
-            example['fetcher'] = sys.modules[__name__]
-            example['version'] = '0.1'
-            example['name'] = 'example'
-            return example
-
-        def compare_versions(self, new, old):
-            """Example has only one version
-
-            This method does not matter here, let's just pass
-            """
-            return 0
+    """Example kiskadee fetcher implementation."""
+
+        import kiskadee
+
+        class Fetcher(kiskadee.fetchers.Fetcher):
+            """Example kiskadee fetcher implementation."""
+
+            def get_sources(self, source_data):
+                """Use sources from test suite."""
+                return 'kiskadee/tests/test_source/test_source.tar.gz'
+
+            def watch(self, **kwargs):
+                """There is no proper API to inspect new example versions.
+                It should not matter, since example will not receive updates.
+                """
+                example = {}
+                example['fetcher'] = __name__
+                example['version'] = '0.1'
+                example['name'] = 'example'
+                super().watch(**example)
+
+            def compare_versions(self, new, old):
+                """Compare package versions.
+
+                Example has only one version. This method does not matter here, let's
+                just pass.
+                """
+                return 0
 
 List of kiskadee fetchers
 ----------------------------
diff --git a/doc/installing.rst b/doc/installing.rst
index 946e2c3..252cc24 100644
--- a/doc/installing.rst
+++ b/doc/installing.rst
@@ -3,11 +3,9 @@ Installing kiskadee
 
 Development
 -----------
-To install locally run:
-
 First, make shure that ansible will be able to login on the vm. ansible will
 use the root user to do this, so you will have to add your public ssh
-key inside the root ~/.ssh/authorized_keys file. You can also create the host
+key inside the root **~/.ssh/authorized_keys** file. You can also create the host
 user inside the vm, in order to be able to test the vm access with the ping
 command.
 
@@ -22,3 +20,9 @@ To deploy kiskadee locally:
 .. code-block:: bash
 
     ansible-playbook  -c paramiko -i playbook/hosts.local playbook/local.yml
+
+Production
+----------
+
+We are still working on a rpm package to install kiskadee on production mode.
+You can check the latest build `here <https://copr.fedorainfracloud.org/coprs/davidcarlos/kiskadee/>`_.
diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py
index eae5017..3714c17 100644
--- a/kiskadee/api/app.py
+++ b/kiskadee/api/app.py
@@ -26,27 +26,27 @@ def index():
         return jsonify({'fetchers': result.data})
 
 
-@kiskadee.route('/packages')
-def packages():
-    """Get the list of analyzed packages."""
+@kiskadee.route('/projects')
+def projects():
+    """Get the list of analyzed projects."""
     if request.method == 'GET':
-        packages = db.session.query(Project).all()
-        package_schema = ProjectSchema(
+        projects = db.session.query(Project).all()
+        project_schema = ProjectSchema(
             many=True,
-            exclude=['versions.analysis', 'versions.package_id']
+            exclude=['versions.analysis', 'versions.project_id']
         )
-        data, errors = package_schema.dump(packages)
-        return jsonify({'packages': data})
+        data, errors = project_schema.dump(projects)
+        return jsonify({'projects': data})
 
 
-@kiskadee.route('/analysis/<pkg_name>/<version>', methods=['GET'])
-def package_analysis_overview(pkg_name, version):
-    """Get a analysis list of some package version."""
+@kiskadee.route('/analysis/<project_name>/<version>', methods=['GET'])
+def project_analysis_overview(project_name, version):
+    """Get a analysis list of some project version."""
 
-    #TODO: This can be a simple inner join between package, version and analysis
-    _package_id = db.filter_by_name(Project, pkg_name).id
+    #TODO: This can be a simple inner join between project, version and analysis
+    _project_id = db.filter_by_name(Project, project_name).id
     version_id = db.session.query(Version)\
-            .filter_by(number = version, package_id = _package_id ).first().id
+            .filter_by(number = version, project_id = _project_id ).first().id
     analysis = (
             db.session.query(Analysis)
             .options(
diff --git a/kiskadee/model.py b/kiskadee/model.py
index 83302b9..2e861df 100644
--- a/kiskadee/model.py
+++ b/kiskadee/model.py
@@ -139,7 +139,9 @@ class Analysis(Base):
                               uselist=False, back_populates='analysis')
 
     @staticmethod
-    def save(db, data, analyzer, result, version):
+    def save(db, analyzer, result, project):
+        version = project.versions[-1]
+        name = project.name
         _analysis = kiskadee.model.Analysis()
         try:
             _analyzer = db.session.query(kiskadee.model.Analyzer).\
@@ -153,10 +155,10 @@ class Analysis(Base):
                     'results': _analysis.raw['results'],
                     'id': _analysis.id
                 }
-            Report.save(db, dict_analysis, data, _analyzer.name)
+            Report.save(db, dict_analysis, _analyzer.name, name)
             kiskadee.logger.debug(
                     "MONITOR: Saved analysis done by {} for project: {}-{}"
-                    .format(analyzer, data["name"], data["version"])
+                    .format(analyzer, name, version.number)
                 )
             return
         except Exception as err:
@@ -184,7 +186,7 @@ class Report(Base):
     analysis = orm.relationship('Analysis', back_populates='report')
 
     @staticmethod
-    def save(db, analysis, data, analyzer_name):
+    def save(db, analysis, analyzer_name, project_name):
         try:
             results = analysis['results']
             analyzer_report = Report.REPORTERS[analyzer_name](results)
@@ -198,7 +200,7 @@ class Report(Base):
             db.session.commit()
             kiskadee.logger.debug(
                     "MONITOR: Saved analysis reports for {} project"
-                    .format(data["name"])
+                    .format(project_name)
                 )
         except KeyError as key:
             kiskadee.logger.debug(
@@ -209,7 +211,7 @@ class Report(Base):
         except Exception as err:
             kiskadee.logger.debug(
                     "MONITOR: Failed to get analysis reports to {} project"
-                    .format(data["name"])
+                    .format(project_name)
                 )
             kiskadee.logger.debug(err)
         return
diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py
index d88a085..710127a 100644
--- a/kiskadee/monitor.py
+++ b/kiskadee/monitor.py
@@ -23,10 +23,14 @@ class Monitor:
         Analyzer.create_analyzers(self.db)
 
     def run(self):
+        """Run Monitor process. This method resumes Monitor behavior. It dequeue
+        projects from fetchers, check if the project is valid and if so send it
+        to Runner. When Runner ends a analysis, the Monitor dequeue the analysis
+        and save it on database. """
         kiskadee.logger.debug('MONITOR PID: {}'.format(os.getpid()))
 
         for fetcher in kiskadee.load_fetchers():
-            thread = Monitor._start_fetcher(fetcher.Fetcher().watch)
+            thread = Monitor.start_fetcher(fetcher.Fetcher().watch)
 
         while RUNNING:
             kiskadee.logger.debug('MONITOR STATE: Idle'.format(os.getpid()))
@@ -47,7 +51,7 @@ class Monitor:
             data["fetcher_id"] = fetcher.id if fetcher else ''
             if not self.is_a_new_project_version(project, data):
                 kiskadee.logger.debug('MONITOR STATE: Project {} already'\
-                        'analyzed'.format(data['name']))
+                        ' analyzed'.format(data['name']))
                 return
             self.queues.enqueue_analysis(data)
 
@@ -74,15 +78,18 @@ class Monitor:
         project = self.db.filter_by_name(Project, data['name'])
         if not project:
             project = Project.save(self.db, data)
-        if project:
+        else:
             project = Project.update(self.db, project, data)
+        self.save_project_analysis(data, project)
 
+    def save_project_analysis(self, data, project):
         for analyzer, result in data['results'].items():
-            Analysis.save(self.db, data, analyzer, result, project.versions[-1])
+            Analysis.save(self.db, analyzer, result, project)
 
 
     @staticmethod
-    def _start_fetcher(module, joinable=False, timeout=None):
+    def start_fetcher(module, joinable=False, timeout=None):
+        """Start a fetcher as a thread of the Monitor process."""
         module_as_a_thread = threading.Thread(target=module)
         module_as_a_thread.daemon = True
         module_as_a_thread.start()
diff --git a/kiskadee/queue.py b/kiskadee/queue.py
index bbb7128..0d4956f 100644
--- a/kiskadee/queue.py
+++ b/kiskadee/queue.py
@@ -55,12 +55,13 @@ class Queues():
         return result
 
     @staticmethod
-    def enqueue_project(project, fetcher):
+    def enqueue_project(project, fetcher=None):
         """Put a result on the results queue."""
-        kiskadee.logger.debug(
-                "FETCHER {}: sending package {}-{} for monitor"
-                .format(fetcher, project['name'], project['version'])
-            )
+        if fetcher:
+            kiskadee.logger.debug(
+                    "FETCHER {}: sending package {}-{} for monitor"
+                    .format(fetcher, project['name'], project['version'])
+                )
         projects.put(project)
 
     @staticmethod
diff --git a/kiskadee/runner.py b/kiskadee/runner.py
index d1fa040..e245552 100644
--- a/kiskadee/runner.py
+++ b/kiskadee/runner.py
@@ -27,7 +27,7 @@ class Runner:
 
         Continuously dequeue projects from `analyses_queue` and call the
         :func:`analyze` method, passing the dequeued project as argument.
-        After the analysis, send the result back to monitor.
+        After the analysis, send the result back to Monitor.
         """
         kiskadee.logger.debug('RUNNER PID: {}'.format(os.getpid()))
         while RUNNING:
@@ -89,7 +89,7 @@ class Runner:
         The `analyzer` is the name of a static analyzer already created on the
         database.
         The `source_path` is the directory to a uncompressed source, returned
-        by the :func:`_path_to_uncompressed_source`.
+        by the :func:`get_project_code_path`.
         """
         if source_path is None:
             return None
@@ -112,7 +112,8 @@ class Runner:
             return None
 
     def get_project_code_path(self):
-
+        """ Returns a string, representing the path of the uncompressed
+        project source."""
         if not (self.fetcher and self.project):
             return None
 
diff --git a/kiskadee/tests/integration/test_monitor.py b/kiskadee/tests/integration/test_monitor.py
index f4acf72..1017889 100644
--- a/kiskadee/tests/integration/test_monitor.py
+++ b/kiskadee/tests/integration/test_monitor.py
@@ -44,8 +44,8 @@ class MonitorTestCase(unittest.TestCase):
         self.assertEqual(monitored_project['version'], '0.1')
 
     def test_run_fetchers_as_threads(self):
-        Monitor._start_fetcher(self.example_fetcher.watch)
-        Monitor._start_fetcher(self.example_fetcher.watch)
+        Monitor.start_fetcher(self.example_fetcher.watch)
+        Monitor.start_fetcher(self.example_fetcher.watch)
         first_monitored_project = self.monitor.dequeue_project_from_fetchers()
         second_monitored_project = self.monitor.dequeue_project_from_fetchers()
         self.assertIn("version", first_monitored_project)
diff --git a/kiskadee/tests/plugins/test_plugins.py b/kiskadee/tests/plugins/test_plugins.py
index 9fe8262..d9f532d 100644
--- a/kiskadee/tests/plugins/test_plugins.py
+++ b/kiskadee/tests/plugins/test_plugins.py
@@ -180,13 +180,15 @@ class TestAnityaFetcher(unittest.TestCase):
 
     def test_get_sources(self):
 
-        def mock_github(self, source_data, path):
+        def mock_github(self, fetcher, source_data, path):
             return 'kiskadee/tests/test_source/Sources.gz'
 
         kiskadee.fetchers.anitya.Backends.github = mock_github
         source_data = {'meta': {'backend': 'GitHub'}}
         source_path = self.anitya_fetcher.get_sources(source_data)
-        self.assertEqual(source_path, mock_github("self", "foo", "bla"))
+        self.assertEqual(source_path, mock_github(
+            "self", "example", "foo", "bla")
+            )
 
     def test_create_package_dict(self):
 
diff --git a/kiskadee/tests/test_style.py b/kiskadee/tests/test_style.py
deleted file mode 100644
index acf5c15..0000000
--- a/kiskadee/tests/test_style.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# Thanks bodhi
-# https://github.com/fedora-infra/bodhi/blob/develop/bodhi/tests/test_style.py
-
-import os
-import subprocess
-import unittest
-
-KISKADEE_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
-
-
-class TestStyle(unittest.TestCase):
-    def test_with_flake8(self):
-        """Enforce PEP-8"""
-        flake8_command = ['flake8', KISKADEE_PATH]
-        self.assertEqual(subprocess.call(flake8_command), 0)
-
-    def test_with_pydocstyle(self):
-        """Enforce PEP-257"""
-        pydocstyle_command = ['pydocstyle', KISKADEE_PATH]
-        self.assertEqual(subprocess.call(pydocstyle_command), 0)
diff --git a/kiskadee/tests/units/test_model.py b/kiskadee/tests/units/test_model.py
index b0aafa7..6fda0be 100644
--- a/kiskadee/tests/units/test_model.py
+++ b/kiskadee/tests/units/test_model.py
@@ -14,16 +14,16 @@ class ModelTestCase(unittest.TestCase):
 
     def setUp(self):
         kiskadee.tests.clean_test_db(self.db, model.Base.metadata)
-        self.package = model.Project(name='python-kiskadee')
+        self.project = model.Project(name='python-kiskadee')
         self.version = model.Version(number='1.0-rc1')
         self.fetcher = model.Fetcher(
               name='kiskadee-fetcher', target='university'
             )
         self.db.session.add(self.fetcher)
         self.db.session.commit()
-        self.fetcher.packages.append(self.package)
-        self.package.versions.append(self.version)
-        self.db.session.add(self.package)
+        self.fetcher.projects.append(self.project)
+        self.project.versions.append(self.version)
+        self.db.session.add(self.project)
         self.db.session.add(self.fetcher)
         self.db.session.add(self.version)
 
@@ -43,9 +43,9 @@ class ModelTestCase(unittest.TestCase):
         fetchers = self.db.session.query(model.Fetcher).all()
         self.assertEqual(fetchers, [self.fetcher])
 
-    def test_query_package(self):
-        packages = self.db.session.query(model.Project).all()
-        self.assertEqual(packages, [self.package])
+    def test_query_project(self):
+        projects = self.db.session.query(model.Project).all()
+        self.assertEqual(projects, [self.project])
 
     def test_query_version(self):
         versions = self.db.session.query(model.Version).all()
@@ -62,7 +62,7 @@ class ModelTestCase(unittest.TestCase):
         fetchers = self.db.session.query(model.Fetcher).all()
         self.assertEqual(len(fetchers), 2)
 
-    def test_add_version_without_package(self):
+    def test_add_version_without_project(self):
         version = model.Version(number='3.1')
         self.db.session.add(version)
         with self.assertRaises(exc.IntegrityError):
@@ -75,51 +75,51 @@ class ModelTestCase(unittest.TestCase):
         with self.assertRaises(exc.IntegrityError):
             self.db.session.commit()
 
-    def test_add_package_without_fetcher(self):
-        package = model.Project(name='foo-bar')
-        self.db.session.add(package)
+    def test_add_project_without_fetcher(self):
+        project = model.Project(name='foo-bar')
+        self.db.session.add(project)
         with self.assertRaises(exc.IntegrityError):
             self.db.session.commit()
 
-    def test_unique_package_in_fetcher(self):
-        package_1 = model.Project(name='foo-bar')
-        package_2 = model.Project(name='foo-bar')
-        self.fetcher.packages.append(package_1)
-        self.fetcher.packages.append(package_2)
+    def test_unique_project_in_fetcher(self):
+        project_1 = model.Project(name='foo-bar')
+        project_2 = model.Project(name='foo-bar')
+        self.fetcher.projects.append(project_1)
+        self.fetcher.projects.append(project_2)
         with self.assertRaises(exc.IntegrityError):
             self.db.session.commit()
 
-    def test_unique_version_for_package(self):
-        package_version_1 = model.Version(number='1.0')
-        package_version_2 = model.Version(number='1.0')
-        self.package.versions.append(package_version_1)
-        self.package.versions.append(package_version_2)
+    def test_unique_version_for_project(self):
+        project_version_1 = model.Version(number='1.0')
+        project_version_2 = model.Version(number='1.0')
+        self.project.versions.append(project_version_1)
+        self.project.versions.append(project_version_2)
         with self.assertRaises(exc.IntegrityError):
             self.db.session.commit()
 
     def test_compose_kiskadee_source(self):
         _analyzer = self.db.session.query(model.Analyzer)\
                     .filter(model.Analyzer.name == "cppcheck").first()
-        package = model.Project(
+        project = model.Project(
                 name='bla',
                 fetcher_id=self.fetcher.id
                 )
-        package_version = model.Version(
+        project_version = model.Version(
                 number='1.0.1',
-                package_id=package.id
+                project_id=project.id
                 )
 
-        package_analysis = model.Analysis(
+        project_analysis = model.Analysis(
                 raw="<>",
                 analyzer_id=_analyzer.id,
-                version_id=package_version.id
+                version_id=project_version.id
                 )
 
-        self.fetcher.packages.append(package)
-        package.versions.append(package_version)
-        package_version.analysis.append(package_analysis)
+        self.fetcher.projects.append(project)
+        project.versions.append(project_version)
+        project_version.analysis.append(project_analysis)
 
-        self.assertEqual(package.versions[0].analysis[0].raw, "<>")
+        self.assertEqual(project.versions[0].analysis[0].raw, "<>")
 
     def test_save_several_analysis(self):
 
@@ -132,42 +132,42 @@ class ModelTestCase(unittest.TestCase):
                 .filter(model.Analyzer.name == "flawfinder").first()
                 )
 
-        package = model.Project(
+        project = model.Project(
                 name='bla',
                 fetcher_id=self.fetcher.id
                 )
-        package_version = model.Version(
+        project_version = model.Version(
                 number='1.0.1',
-                package_id=package.id
+                project_id=project.id
                 )
 
-        self.fetcher.packages.append(package)
-        package.versions.append(package_version)
+        self.fetcher.projects.append(project)
+        project.versions.append(project_version)
 
-        self.db.session.add(package)
-        self.db.session.add(package_version)
+        self.db.session.add(project)
+        self.db.session.add(project_version)
         self.db.session.commit()
 
-        package_analysis1 = model.Analysis(
+        project_analysis1 = model.Analysis(
                 raw="<>",
                 analyzer_id=_analyzer1.id,
-                version_id=package_version.id
+                version_id=project_version.id
                 )
-        package_analysis2 = model.Analysis(
+        project_analysis2 = model.Analysis(
                 raw="><",
                 analyzer_id=_analyzer2.id,
-                version_id=package_version.id
+                version_id=project_version.id
                 )
 
-        self.db.session.add(package_analysis1)
-        self.db.session.add(package_analysis2)
+        self.db.session.add(project_analysis1)
+        self.db.session.add(project_analysis2)
         self.db.session.commit()
 
-        saved_package = (
+        saved_project = (
                 self.db.session.query(model.Project)
                 .filter(model.Project.name == 'bla').first()
                 )
-        analysis = saved_package.versions[-1].analysis
+        analysis = saved_project.versions[-1].analysis
         self.assertEqual(len(analysis), 2)
         self.assertEqual(analysis[0].raw, "<>")
         self.assertEqual(analysis[1].raw, "><")

From 1f08e5104c7bb015d01dac3aa224857a249ea6f8 Mon Sep 17 00:00:00 2001
From: naiieandrade <andradenaiara@hotmail.com>
Date: May 30 2018 14:23:51 +0000
Subject: [PATCH 17/17] Create API documentantion


---

diff --git a/flasgger/__init__.py b/flasgger/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/flasgger/__init__.py
diff --git a/flasgger/api.py b/flasgger/api.py
new file mode 100644
index 0000000..f2edcd9
--- /dev/null
+++ b/flasgger/api.py
@@ -0,0 +1,210 @@
+from flask import Flask, jsonify
+from flask_restful import Api, Resource
+
+from flasgger import Swagger
+
+
+app = Flask(__name__)
+api = Api(app)
+app.config['SWAGGER'] = {
+    'title': 'Kiskadee Api',
+    'uiversion': 2
+}
+swag = Swagger(app)
+
+
+class GetFetchersView(Resource):
+
+    def get(self):
+        """
+        Example endpoint returning a list of fetchers
+        List fetchers
+        ---
+        tags:
+          - fetchers
+        responses:
+          200:
+            description: Returns a list of fetchers
+            schema:
+              id: Fetchers
+              properties:
+                fetchers:
+                  type: array
+                  items:
+                    $ref: '#/definitions/Fetcher'
+            examples:
+              fetchers: [{'description': 'A fetcher to monitor Anitya packages', 'id': 1, 'name': 'anitya', 'target': 'release-monitoring.org'}]
+        """
+        data = {
+            "fetchers": [
+                {'description': 'A fetcher to monitor Anitya packages',
+                 'id': 1,
+                 'name': 'anitya',
+                 'target': 'release-monitoring.org'},
+                {"description": "SAMATE Juliet test suite",
+                 "id": 2,
+                 'name': 'example',
+                 'target': 'example'},
+            ]
+        }
+        return jsonify(data)
+
+
+class GetAnalysisProjectView(Resource):
+
+    def get(self, project, version):
+        """
+        Example endpoint returning a analyze from a project
+        Analizes the project version
+        ---
+        tags:
+          - analysis
+        parameters:
+          - name: project
+            in: path
+            description: Name of project
+            required: true
+            type: string
+            default: bodhi
+          - name: version
+            in: path
+            description: Version of project
+            required: true
+            type: string
+            default: 3.5.0
+        responses:
+          200:
+            description: Returns a analyze from a project
+        """
+        data = {
+            '': [
+                {'analyzer_id': 2,
+                 'analyzers': {'name': "flawfinder",
+                               'version': "1.0.0"},
+                 'id': 4,
+                 'version_id': 3},
+            ]
+        }
+        return jsonify(data)
+
+
+class GetResultsProjectView(Resource):
+
+    def get(self, project, version, analysis_id):
+        """
+        Example endpoint returning a results from a project version
+        Shows the results of the project version
+        ---
+        tags:
+          - analysis
+        parameters:
+          - name: project
+            in: path
+            description: Name of project
+            required: true
+            type: string
+            default: bodhi
+          - name: version
+            in: path
+            description: Version of project
+            required: true
+            type: string
+            default: 3.5.0
+          - name: analysis_id
+            in: path
+            description: Analysis ID
+            required: true
+            type: integer
+            default: 2
+
+        responses:
+          200:
+            description: Returns a results from a project analyze
+        """
+        data = {
+            'analysis_results': [
+                {'custmofields': 'null',
+                    'cwe': 'null',
+                    'location': {'file': {'abspath': 'null',
+                                          'givenpath': "deepin-menu-2014.2.3/dbus.h",
+                                          'hash_': 'null'},
+                                 'function': 'null',
+                                 'point': {'column': 0,
+                                           'line': 44},
+                                 'range': 'null'},
+                    'message': {'text': 'Class \'ManagerAdaptor\' has a constructor with 1 argument that is not explicit.'},
+                    'notes': {'text': 'Class \'ManagerAdaptor\' has a constructor with 1 argument that is not explicit. Such constructors should in general be explicit for type safety reasons. Using the explicit keyword in the constructor means some mistakes when using the class can be avoided.'},
+                    'severity': 'style',
+                    'testeid': 'noExplicitConstructor',
+                    'trace': 'null',
+                    'type': 'Issue'},
+            ]
+        }
+        return jsonify(data)
+
+
+class GetReportsProjectView(Resource):
+
+    def get(self, project, version, analysis_id):
+        """
+        Example endpoint returning a reports from a project version
+        Shows the results of the project version
+        ---
+        tags:
+          - analysis
+        parameters:
+          - name: project
+            in: path
+            description: Name of project
+            required: true
+            type: string
+            default: bodhi
+          - name: version
+            in: path
+            description: Version of project
+            required: true
+            type: string
+            default: 3.5.0
+          - name: analysis_id
+            in: path
+            description: Analysis ID
+            required: true
+            type: integer
+            default: 2
+
+        responses:
+          200:
+            description: Returns a reports from a project analyze
+        """
+        data = {
+            'analysis_reports': [
+                {'analysis_id': 4,
+                    'id': 3,
+                    'results': {'severity_1': 0,
+                                'severity_2': 0,
+                                'severity_3': 0,
+                                'severity_4': 0,
+                                'severity_5': 0},
+                 },
+            ]
+        }
+        return jsonify(data)
+
+
+api.add_resource(GetFetchersView, '/fetchers')
+api.add_resource(GetAnalysisProjectView, '/analysis/<string:project>/<string:version>')
+api.add_resource(GetResultsProjectView, '/analysis/<string:project>/<string:version>/<int:analysis_id>/results')
+api.add_resource(GetReportsProjectView, '/analysis/<string:project>/<string:version>/<int:analysis_id>/reports')
+
+
+@app.route("/")
+def hello():
+    return """
+    <h1>Welcome Kiskadee API</h1>
+    The API documentation you can see on
+    <a href="/apidocs">Api docs</a>
+    """
+
+
+if __name__ == "__main__":
+    app.run(debug=True)
diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py
index 3714c17..e4de90e 100644
--- a/kiskadee/api/app.py
+++ b/kiskadee/api/app.py
@@ -39,7 +39,7 @@ def projects():
         return jsonify({'projects': data})
 
 
-@kiskadee.route('/analysis/<project_name>/<version>', methods=['GET'])
+@kiskadee.route('/analysis/<path:pkg_name>/<version>', methods=['GET'])
 def project_analysis_overview(project_name, version):
     """Get a analysis list of some project version."""
 
@@ -61,7 +61,7 @@ def project_analysis_overview(project_name, version):
 
 
 @kiskadee.route(
-    '/analysis/<pkg_name>/<version>/<analysis_id>/results',
+    '/analysis/<path:pkg_name>/<version>/<analysis_id>/results',
     methods=['GET']
 )
 def analysis_results(pkg_name, version, analysis_id):
@@ -74,7 +74,7 @@ def analysis_results(pkg_name, version, analysis_id):
 
 
 @kiskadee.route(
-    '/analysis/<pkg_name>/<version>/<analysis_id>/reports',
+    '/analysis/<path:pkg_name>/<version>/<analysis_id>/reports',
     methods=['GET']
 )
 def analysis_reports(pkg_name, version, analysis_id):
diff --git a/requirements.txt b/requirements.txt
index f1039cf..af0b039 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 psutil
-psycopg2
+psycopg2-binary
 firehose>=0.5
 sqlalchemy
 chardet
@@ -13,3 +13,5 @@ Flask-Restless
 marshmallow
 alembic
 python-debian
+flasgger
+flask_restful

