From afb3c06c5173fad38e2783d47f6a926f14ca013a Mon Sep 17 00:00:00 2001 From: David Carlos Date: Apr 08 2018 20:50:18 +0000 Subject: [PATCH 1/18] Merge branch 'master' of ssh://pagure.io/kiskadee --- diff --git a/README.md b/README.md index 816aa6b..b1fcc51 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,11 @@ available [here](https://www.ime.usp.br/~athoscr/files/ranking_data.tgz). Join us on IRC: #kiskadee @ freenode +Mailing list: `kiskadee@googlegroups.com` + +You can subscribe to our mailing lists by sending an email to +`kiskadee+subscribe@googlegroups.com`. + ## License Copyright (C) 2017 the AUTHORS (see the AUTHORS file) diff --git a/doc/development.rst b/doc/development.rst index 53c6d5f..2230c10 100644 --- a/doc/development.rst +++ b/doc/development.rst @@ -1,10 +1,10 @@ kiskadee development ==================== -This section is a guide for new developers willing to -set up a development environemnt to start contributing to -kiskadee development. If you have any doubt, please contact -us on IRC in #kiskadee at freenode. +This section is a guide for new developers willing to set up a development +environemnt to start contributing to kiskadee development. If you have any +doubt, please contact us on IRC in #kiskadee at freenode or join the +development discussion in our mailing list at kiskadee@googlegroups.com. **kiskadee development only suports python versions >= 3. Assume such versions for all commands run along this documentation.** From 3b0a5f79493f1610f4595703205f6c99e88c8400 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Apr 08 2018 20:50:59 +0000 Subject: [PATCH 2/18] Use a migration to create kiskadee fetchers. --- diff --git a/alembic/versions/50988af48b09_create_fetcher_table.py b/alembic/versions/50988af48b09_create_fetcher_table.py index 54fd6c4..33e90a3 100644 --- a/alembic/versions/50988af48b09_create_fetcher_table.py +++ b/alembic/versions/50988af48b09_create_fetcher_table.py @@ -30,7 +30,7 @@ def upgrade(): primary_key=True ), sa.Column('name', sa.Unicode(255), nullable=False, unique=True), - sa.Column('target', sa.Unicode(255), nullable=True), + sa.Column('source', sa.Unicode(255), nullable=True), sa.Column('description', sa.UnicodeText) ) diff --git a/alembic/versions/cc4887b7c3da_create_fetchers_on_database.py b/alembic/versions/cc4887b7c3da_create_fetchers_on_database.py index bd88705..01ec98c 100644 --- a/alembic/versions/cc4887b7c3da_create_fetchers_on_database.py +++ b/alembic/versions/cc4887b7c3da_create_fetchers_on_database.py @@ -23,7 +23,7 @@ def upgrade(): fetchers_table = table('fetchers', column('id', Integer), column('name', String), - column('target', String), + column('source', String), column('description', String) ) @@ -35,7 +35,7 @@ def upgrade(): [ { 'id': idx, 'name': _fetcher.name, - 'target': _fetcher.config['target'], + 'source': _fetcher.config['source'], 'description': _fetcher.config['description'] } ] diff --git a/util/kiskadee.conf b/util/kiskadee.conf index c2dc351..5eb6ec8 100644 --- a/util/kiskadee.conf +++ b/util/kiskadee.conf @@ -19,7 +19,7 @@ port = 5432 dbname = kiskadee_test [debian_fetcher] -target = http://ftp.us.debian.org/debian +source = http://ftp.us.debian.org/debian description = A fetcher to monitor Debian Repositories release = sid meta = Sources.gz @@ -28,13 +28,13 @@ analyzers = cppcheck flawfinder active = no [juliet_fetcher] -target = juliet +source = juliet description = SAMATE Juliet test suite analyzers = cppcheck active = no [anitya_fetcher] -target = release-monitoring.org +source = release-monitoring.org description = A fetcher to monitor Anitya packages analyzers = cppcheck flawfinder active = no @@ -42,7 +42,7 @@ zmq_port = 5556 zmq_topic = anitya [example_fetcher] -target = example +source = example description = SAMATE Juliet test suite analyzers = cppcheck flawfinder clanganalyzer frama_c active = yes From 1f3f91bd6291ee2433f69200cdf643747c175da2 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Apr 08 2018 20:50:59 +0000 Subject: [PATCH 3/18] Refactors queue.py module. - Add a new decorator to check when a queue is empty. - Create subpackage to unit tests. - Change Queues methods to static methods. --- diff --git a/kiskadee/fetchers/example.py b/kiskadee/fetchers/example.py index 02eb9b2..a4c1593 100644 --- a/kiskadee/fetchers/example.py +++ b/kiskadee/fetchers/example.py @@ -9,7 +9,8 @@ class Fetcher(kiskadee.fetchers.Fetcher): """Use sources from test suite.""" return 'kiskadee/tests/test_source/test_source.tar.gz' - def watch(self, **kwargs): + @kiskadee.queue.enqueue_project + def watch(self): """There is no proper API to inspect new example versions. It should not matter, since example will not receive updates. """ diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py index ef3fb77..fbf68bc 100644 --- a/kiskadee/monitor.py +++ b/kiskadee/monitor.py @@ -53,9 +53,32 @@ class Monitor: if pkg: self._send_to_runner(pkg) time.sleep(2) - analyzed_project = self.kiskadee_queue.dequeue_result() + analyzed_project = self.queues.dequeue_result() self._save_analyzed_project(analyzed_project) + def dequeue_package(self): + """Dequeue packages from packages_queue.""" + if not kiskadee.queue.packages_queue.empty(): + pkg = kiskadee.queue.packages_queue.get() + kiskadee.logger.debug( + "MONITOR: Dequed Package: {}_{}" + .format(pkg["name"], pkg["version"]) + ) + return pkg + return {} + + # Use directly dequeue_result from kiskadee.queue + #def dequeue_result(self): + # """Dequeue analyzed packages from result_queue.""" + # if not self.kiskadee_queue.results_empty(): + # pkg = self.kiskadee_queue.dequeue_result() + # kiskadee.logger.debug( + # "MONITOR: Dequed result for package : {}-{}" + # .format(pkg["name"], pkg["version"]) + # ) + # return pkg + # return {} + def _send_to_runner(self, pkg): _name = pkg['fetcher'].split('.')[-1] _fetcher = self._query(Fetcher).filter_by(name = _name).first() @@ -80,8 +103,8 @@ class Monitor: self.kiskadee_queue.enqueue_analysis(pkg) # Move this to model.py - def _save_analyzed_project(self, data): - if not data: + def _save_analyzed_project(self, project): + if not pkg: return {} project = self._query(Package).filter_by(name = data['name']).first() if not project: @@ -93,7 +116,7 @@ class Monitor: self._save_analysis(data, analyzer, result, project.versions[-1]) # Move this to model.py - def _update_project(self, package, pkg): + def _update_pkg(self, package, pkg): if(package.versions[-1].number == pkg['version']): return package @@ -115,7 +138,7 @@ class Monitor: return None # Move this to model.py - def _save_project(self, pkg): + def _save_pkg(self, pkg): homepage = None if ('meta' in pkg) and ('homepage' in pkg['meta']): homepage = pkg['meta']['homepage'] @@ -230,11 +253,11 @@ def daemon(): runner = Runner() monitor_process = Process( target=monitor.monitor, - args=(queues,) + args=(queues) ) runner_process = Process( target=runner.runner, - args=(queues,) + args=(queues) ) monitor_process.start() runner_process.start() diff --git a/kiskadee/queue.py b/kiskadee/queue.py index 214b631..a6835ea 100644 --- a/kiskadee/queue.py +++ b/kiskadee/queue.py @@ -1,6 +1,5 @@ """Provide kiskadee queues and operations on them.""" import time -import dill from multiprocessing import Queue import kiskadee @@ -24,10 +23,10 @@ def enqueue_project(fetcher_watch): """ def wrapper(*args, **kwargs): project = fetcher_watch(*args, **kwargs) - kiskadee.queue.queues.enqueue_project(project) + kiskadee.queue.Queues.enqueue_project(project) fetcher = project['fetcher'].name kiskadee.logger.debug( - "{} fetcher: sending package {}_{} for monitor" + "{} fetcher: Sending package {}_{} for monitor" .format(fetcher, package['name'], package['version']) ) time.sleep(2) @@ -41,14 +40,15 @@ class Queues(): """Provide kiskadee queues objects.""" @staticmethod - def enqueue_analysis(project_to_analysis): + def enqueue_analysis(analysis): """Put a analysis on the analysis queue.""" - analysis.put(project_to_analysis) + analysis.put(analysis) @staticmethod + @empty_queue def dequeue_analysis(): """Get a analysis from the analysis queue.""" - return analysis.get() + return analysis.get(timeout=1) @staticmethod def enqueue_result(result): From 834bea2a94f3114c4ee78e780162e55b67843894 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Apr 08 2018 20:50:59 +0000 Subject: [PATCH 4/18] Use mock to run api tests. - We should use mock to test everything that interacts with database. This will avoid some database dependecy problems when runing kiskadee tests. --- diff --git a/alembic/versions/50988af48b09_create_fetcher_table.py b/alembic/versions/50988af48b09_create_fetcher_table.py index 33e90a3..54fd6c4 100644 --- a/alembic/versions/50988af48b09_create_fetcher_table.py +++ b/alembic/versions/50988af48b09_create_fetcher_table.py @@ -30,7 +30,7 @@ def upgrade(): primary_key=True ), sa.Column('name', sa.Unicode(255), nullable=False, unique=True), - sa.Column('source', sa.Unicode(255), nullable=True), + sa.Column('target', sa.Unicode(255), nullable=True), sa.Column('description', sa.UnicodeText) ) diff --git a/alembic/versions/cc4887b7c3da_create_fetchers_on_database.py b/alembic/versions/cc4887b7c3da_create_fetchers_on_database.py index 01ec98c..bd88705 100644 --- a/alembic/versions/cc4887b7c3da_create_fetchers_on_database.py +++ b/alembic/versions/cc4887b7c3da_create_fetchers_on_database.py @@ -23,7 +23,7 @@ def upgrade(): fetchers_table = table('fetchers', column('id', Integer), column('name', String), - column('source', String), + column('target', String), column('description', String) ) @@ -35,7 +35,7 @@ def upgrade(): [ { 'id': idx, 'name': _fetcher.name, - 'source': _fetcher.config['source'], + 'target': _fetcher.config['target'], 'description': _fetcher.config['description'] } ] diff --git a/kiskadee/tests/api/test_api.py b/kiskadee/tests/api/test_api.py index e88fee5..0b83d1a 100644 --- a/kiskadee/tests/api/test_api.py +++ b/kiskadee/tests/api/test_api.py @@ -1,8 +1,10 @@ import json import unittest +from sqlalchemy.orm import sessionmaker from unittest.mock import MagicMock import kiskadee +from kiskadee.runner import Runner from kiskadee.monitor import Monitor import kiskadee.api.app import kiskadee.fetchers.example diff --git a/kiskadee/tests/integration/test_monitor.py b/kiskadee/tests/integration/test_monitor.py index 67cd7c2..aa95be6 100644 --- a/kiskadee/tests/integration/test_monitor.py +++ b/kiskadee/tests/integration/test_monitor.py @@ -3,8 +3,8 @@ from sqlalchemy.orm import sessionmaker from kiskadee import model from kiskadee.monitor import Monitor -from kiskadee.queue import Queues -from kiskadee.model import Package, create_analyzers, Report, Analysis +from kiskadee.queue import packages_queue +from kiskadee.model import Package, Fetcher, create_analyzers, Report, Analysis import kiskadee.queue import kiskadee.fetchers.debian import kiskadee.fetchers.anitya @@ -18,13 +18,12 @@ class MonitorTestCase(unittest.TestCase): Session = sessionmaker(bind=self.engine) self.session = Session() self.monitor = Monitor(self.session) - self.queues = Queues() model.Base.metadata.create_all(self.engine) create_analyzers(self.session) self.pkg1 = { 'name': 'curl', 'version': '7.52.1-5', - 'fetcher': kiskadee.fetchers.debian.__name__, + 'fetcher': kiskadee.fetchers.debian, 'meta': {'directory': 'pool/main/c/curl'}, 'results': { 'cppcheck': '<>', @@ -33,7 +32,7 @@ class MonitorTestCase(unittest.TestCase): self.pkg2 = {'name': 'urlscan', 'version': '0.8.2', - 'fetcher': kiskadee.fetchers.debian.__name__, + 'fetcher': kiskadee.fetchers.debian, 'meta': {'directory': 'pool/main/u/urlscan'}, 'results': { 'cppcheck': '<>', @@ -42,7 +41,7 @@ class MonitorTestCase(unittest.TestCase): self.pkg3 = {'name': 'curl', 'version': '7.52.2-5', - 'fetcher': kiskadee.fetchers.debian.__name__, + 'fetcher': kiskadee.fetchers.debian, 'meta': {'directory': 'pool/main/c/curl'}, 'results': { 'cppcheck': '<>', @@ -50,7 +49,7 @@ class MonitorTestCase(unittest.TestCase): 'fetcher_id': 1} self.pkg4 = {'name': 'urlanitya', 'version': '0.11', - 'fetcher': kiskadee.fetchers.anitya.__name__, + 'fetcher': kiskadee.fetchers.anitya, 'meta': { 'backend': 'gitHub', 'homepage': 'https://github.com/GesielFreitas/Cros' @@ -79,14 +78,31 @@ class MonitorTestCase(unittest.TestCase): fetcher = kiskadee.fetchers.debian.Fetcher() self.assertEqual(fetcher.name, 'debian') + def test_save_some_fetcher(self): + fetcher = kiskadee.fetchers.debian.Fetcher() + self.monitor._save_fetcher(fetcher) + _fetchers = self.monitor.session.query(Fetcher).all() + self.assertEqual(len(_fetchers), 1) + self.assertEqual(_fetchers[0].name, 'debian') + self.assertEqual(_fetchers[0].description, + fetcher.config['description']) + def test_save_package(self): self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher()) - self.monitor._save_analyzed_project(self.pkg1) - self.monitor._save_analyzed_project(self.pkg2) + packages_queue.put(self.pkg1) + packages_queue.put(self.pkg2) + + _pkg = self.monitor.dequeue_package() + self.monitor._save_analyzed_pkg(_pkg) + _pkgs = self.monitor.session.query(Package).all() + self.assertEqual(len(_pkgs), 1) + self.assertEqual(_pkgs[0].name, _pkg['name']) + + _pkg = self.monitor.dequeue_package() + self.monitor._save_analyzed_pkg(_pkg) _pkgs = self.monitor.session.query(Package).all() self.assertEqual(len(_pkgs), 2) - self.assertEqual(_pkgs[0].name, self.pkg1['name']) - self.assertEqual(_pkgs[1].name, self.pkg2['name']) + self.assertEqual(_pkgs[1].name, _pkg['name']) def test_save_reports(self): _fetcher = model.Fetcher( @@ -126,21 +142,21 @@ class MonitorTestCase(unittest.TestCase): def test_save_version(self): self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher()) - self.monitor._save_analyzed_project(self.pkg1) + self.monitor._save_analyzed_pkg(self.pkg1) _pkgs = self.monitor.session.query(Package).all() _version = _pkgs[0].versions[0].number self.assertEqual(_version, self.pkg1['version']) def test_update_version(self): self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher()) - self.queues.enqueue_project(self.pkg1) - self.queues.enqueue_project(self.pkg3) + packages_queue.put(self.pkg1) + packages_queue.put(self.pkg3) - _pkg = self.queues.dequeue_project() - self.monitor._save_analyzed_project(_pkg) + _pkg = self.monitor.dequeue_package() + self.monitor._save_analyzed_pkg(_pkg) - _pkg = self.queues.dequeue_project() - self.monitor._save_analyzed_project(_pkg) + _pkg = self.monitor.dequeue_package() + self.monitor._save_analyzed_pkg(_pkg) _pkgs = self.monitor.session.query(Package).all() self.assertEqual(len(_pkgs), 1) @@ -155,9 +171,11 @@ class MonitorTestCase(unittest.TestCase): def test_save_package_anitya(self): self.monitor._save_fetcher(kiskadee.fetchers.anitya.Fetcher()) - self.queues.enqueue_project(self.pkg4) - _pkg = self.queues.dequeue_project() - self.monitor._save_analyzed_project(_pkg) + packages_queue.put(self.pkg4) + + _pkg = self.monitor.dequeue_package() + self.monitor._save_analyzed_pkg(_pkg) + _pkgs = self.monitor.session.query(Package).all() self.assertEqual(len(_pkgs), 1) self.assertEqual(_pkgs[0].homepage, _pkg['meta']['homepage']) diff --git a/kiskadee/tests/integration/test_runner.py b/kiskadee/tests/integration/test_runner.py index f9a2e16..0ad1e0a 100644 --- a/kiskadee/tests/integration/test_runner.py +++ b/kiskadee/tests/integration/test_runner.py @@ -1,8 +1,3 @@ -""" -This integration tests needs docker engine be running and accessible -by the user that is executing the tests. It also needs selinux running in -a permissive mode (setenforce 0) -""" import unittest import tempfile @@ -11,11 +6,12 @@ import kiskadee.fetchers.example import kiskadee.fetchers.debian from sqlalchemy.orm import sessionmaker from kiskadee import model -from kiskadee.queue import Queues +from kiskadee.queue import KiskadeeQueue from kiskadee.database import Database class AnalyzersTestCase(unittest.TestCase): + def setUp(self): self.engine = Database('db_test').engine Session = sessionmaker(bind=self.engine) @@ -25,15 +21,16 @@ class AnalyzersTestCase(unittest.TestCase): self.fetcher = kiskadee.fetchers.debian.Fetcher() self.deb_pkg = {'name': 'test', 'version': '1.0.0', - 'fetcher': kiskadee.fetchers.debian.__name__ + 'fetcher': kiskadee.fetchers.debian.Fetcher() } self.fetcher = model.Fetcher( name='kiskadee-fetcher2', target='university' ) self.session.add(self.fetcher) self.session.commit() + kiskadee_queue = KiskadeeQueue() self.runner = Runner() - self.runner.queues = Queues() + self.runner.kiskadee_queue = kiskadee_queue def tearDown(self): self.session.close() @@ -57,11 +54,12 @@ class AnalyzersTestCase(unittest.TestCase): source_to_analysis = { 'name': 'test', 'version': '1.0.0', - 'fetcher': kiskadee.fetchers.example.__name__ + 'fetcher': kiskadee.fetchers.example.Fetcher() } self.runner.call_analyzers(source_to_analysis) - analyzed_pkg = self.runner.queues.dequeue_result() + analyzed_pkg = self.runner.kiskadee_queue.dequeue_result() + self.assertEqual(analyzed_pkg['name'], source_to_analysis['name']) self.assertIn('cppcheck', analyzed_pkg['results']) self.assertIn('flawfinder', analyzed_pkg['results']) @@ -95,6 +93,5 @@ class AnalyzersTestCase(unittest.TestCase): self.assertIsNone(source_path) - if __name__ == '__main__': unittest.main() diff --git a/kiskadee/tests/plugins/test_plugins.py b/kiskadee/tests/plugins/test_plugins.py index 9fe8262..030da9a 100644 --- a/kiskadee/tests/plugins/test_plugins.py +++ b/kiskadee/tests/plugins/test_plugins.py @@ -70,7 +70,7 @@ class DebianFetcherTestCase(unittest.TestCase): self.debian_fetcher._queue_sources_gz_pkgs(temp_dir) shutil.rmtree(temp_dir) - some_pkg = kiskadee.queue.Queues.dequeue_project() + some_pkg = kiskadee.queue.packages_queue.get() self.assertTrue(isinstance(some_pkg, dict)) self.assertIn('name', some_pkg) self.assertIn('version', some_pkg) @@ -190,8 +190,8 @@ class TestAnityaFetcher(unittest.TestCase): def test_create_package_dict(self): - self.anitya_fetcher.project_to_enqueue(self.msg) - _dict = kiskadee.queue.Queues().dequeue_project() + self.anitya_fetcher._create_package_dict(self.msg) + _dict = kiskadee.queue.packages_queue.get() self.assertEqual(_dict['name'], 'urlscan') self.assertEqual(_dict['version'], '0.8.5') self.assertEqual(_dict['meta']['backend'], 'GitHub') @@ -199,7 +199,7 @@ class TestAnityaFetcher(unittest.TestCase): _dict['meta']['homepage'], 'https://github.com/firecat53/urlscan' ) - self.assertEqual(_dict['fetcher'], 'kiskadee.fetchers.anitya') + self.assertEqual(_dict['fetcher'].name, 'anitya') if __name__ == '__main__': diff --git a/kiskadee/tests/test_api.py b/kiskadee/tests/test_api.py deleted file mode 100644 index 71a58fb..0000000 --- a/kiskadee/tests/test_api.py +++ /dev/null @@ -1,118 +0,0 @@ -import json -import unittest -from sqlalchemy.orm import sessionmaker - -import kiskadee -from kiskadee.runner import Runner -from kiskadee.monitor import Monitor -import kiskadee.api.app -import kiskadee.fetchers.example - - -class ApiTestCase(unittest.TestCase): - - def setUp(self): - kiskadee.api.app.kiskadee.testing = True - self.engine = kiskadee.database.Database('db_test').engine - Session = sessionmaker(bind=self.engine) - self.session = Session() - self.app = kiskadee.api.app.kiskadee.test_client() - kiskadee.model.create_analyzers(self.session) - kiskadee.model.Base.metadata.create_all(self.engine) - kiskadee.model.create_analyzers(self.session) - fetcher = kiskadee.model.Fetcher( - name='kiskadee-fetcher', target='university' - ) - pkg = kiskadee.model.Package( - name='kiskadee-package', fetcher_id=1 - ) - version = kiskadee.model.Version( - number='7.23', package_id=1 - ) - analysis = kiskadee.model.Analysis( - version_id=1, analyzer_id=1, - raw={ - 'results': [ - {'severity': 'warning'}, - {'severity': 'style'}, - {'severity': 'error'} - ] - } - ) - report = kiskadee.model.Report( - analysis_id=1 - ) - self.session.add(fetcher) - self.session.add(pkg) - self.session.add(version) - self.session.add(analysis) - self.session.add(report) - self.session.commit() - self.runner = Runner() - self.monitor = Monitor(self.session) - self.runner.kiskadee_queue = kiskadee.queue.KiskadeeQueue() - - def tearDown(self): - self.session.close() - kiskadee.model.Base.metadata.drop_all() - - def test_get_fetchers(self): - def mock_kiskadee_db_session(): - return self.session - - kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session - response = self.app.get("/fetchers") - self.assertIn("fetchers", json.loads(response.data.decode("utf-8"))) - - def test_get_activated_fetcher(self): - - def mock_kiskadee_db_session(): - return self.session - - kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session - response = self.app.get("/fetchers") - response_as_json = json.loads(response.data.decode("utf-8")) - fetcher_name = response_as_json["fetchers"][0]["name"] - self.assertEqual("kiskadee-fetcher", fetcher_name) - - def test_get_analysis_as_json(self): - source_to_analysis = { - 'name': 'test', - 'version': '1.0.0', - 'fetcher_id': '1', - 'fetcher': kiskadee.fetchers.example.Fetcher() - } - - def mock_kiskadee_db_session(): - return self.session - - self.runner.call_analyzers(source_to_analysis) - analyzed_pkg = self.runner.kiskadee_queue.dequeue_result() - self.monitor._save_analyzed_pkg(analyzed_pkg) - kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session - response = self.app.get("/analysis/test/1.0.0") - response_data = json.loads(response.data.decode("utf-8")) - self.assertTrue(len(response_data) >= 1) - - def test_get_analysis_results(self): - def mock_kiskadee_db_session(): - return self.session - - kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session - response = self.app.get("/analysis/kiskadee-package/7.23/1/results") - response_data = json.loads(response.data.decode("utf-8")) - self.assertIn("analysis_results", response_data) - self.assertTrue(len(response_data["analysis_results"]) > 0) - - def test_get_analysis_reports(self): - def mock_kiskadee_db_session(): - return self.session - - kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session - response = self.app.get("/analysis/kiskadee-package/7.23/1/reports") - response_data = json.loads(response.data.decode("utf-8")) - self.assertIn("analysis_report", response_data) - - -if __name__ == '__main__': - unittest.main() diff --git a/kiskadee/tests/test_model.py b/kiskadee/tests/test_model.py deleted file mode 100644 index 4a56835..0000000 --- a/kiskadee/tests/test_model.py +++ /dev/null @@ -1,179 +0,0 @@ -import unittest -from sqlalchemy import exc -from sqlalchemy.orm import sessionmaker - -from kiskadee import model -from kiskadee.database import Database - - -class ModelTestCase(unittest.TestCase): - - def setUp(self): - self.engine = Database('db_test').engine - Session = sessionmaker(bind=self.engine) - self.session = Session() - model.Base.metadata.create_all(self.engine) - model.create_analyzers(self.session) - self.fetcher = model.Fetcher( - name='kiskadee-fetcher', target='university' - ) - self.package = model.Package(name='python-kiskadee') - self.version = model.Version(number='1.0-rc1') - self.fetcher.packages.append(self.package) - self.package.versions.append(self.version) - self.session.add(self.package) - self.session.add(self.fetcher) - self.session.add(self.version) - - self.analysis = model.Analysis( - analyzer_id=1, - version_id=1, - raw="" - ) - self.session.add(self.analysis) - self.report = model.Report( - analysis_id=1, - ) - self.session.add(self.report) - self.session.commit() - - def tearDown(self): - self.session.close() - model.Base.metadata.drop_all() - - def test_query_fetcher(self): - fetchers = self.session.query(model.Fetcher).all() - self.assertEqual(fetchers, [self.fetcher]) - - def test_query_package(self): - packages = self.session.query(model.Package).all() - self.assertEqual(packages, [self.package]) - - def test_query_version(self): - versions = self.session.query(model.Version).all() - self.assertEqual(versions, [self.version]) - - def test_query_report(self): - reports = self.session.query(model.Report).all() - self.assertEqual(reports, [self.report]) - - def test_add_fetcher(self): - fetchers = self.session.query(model.Fetcher).all() - self.assertEqual(len(fetchers), 1) - self.session.add(model.Fetcher(name='foo', target='bar')) - fetchers = self.session.query(model.Fetcher).all() - self.assertEqual(len(fetchers), 2) - - def test_add_version_without_package(self): - version = model.Version(number='3.1') - self.session.add(version) - with self.assertRaises(exc.IntegrityError): - self.session.commit() - - def test_add_report_without_analysis(self): - report = model.Report( - ) - self.session.add(report) - with self.assertRaises(exc.IntegrityError): - self.session.commit() - - def test_add_package_without_fetcher(self): - package = model.Package(name='foo-bar') - self.session.add(package) - with self.assertRaises(exc.IntegrityError): - self.session.commit() - - def test_unique_package_in_fetcher(self): - package_1 = model.Package(name='foo-bar') - package_2 = model.Package(name='foo-bar') - self.fetcher.packages.append(package_1) - self.fetcher.packages.append(package_2) - with self.assertRaises(exc.IntegrityError): - self.session.commit() - - def test_unique_version_for_package(self): - package_version_1 = model.Version(number='1.0') - package_version_2 = model.Version(number='1.0') - self.package.versions.append(package_version_1) - self.package.versions.append(package_version_2) - with self.assertRaises(exc.IntegrityError): - self.session.commit() - - def test_compose_kiskadee_source(self): - _analyzer = self.session.query(model.Analyzer)\ - .filter(model.Analyzer.name == "cppcheck").first() - package = model.Package( - name='bla', - fetcher_id=self.fetcher.id - ) - package_version = model.Version( - number='1.0.1', - package_id=package.id - ) - - package_analysis = model.Analysis( - raw="<>", - analyzer_id=_analyzer.id, - version_id=package_version.id - ) - - self.fetcher.packages.append(package) - package.versions.append(package_version) - package_version.analysis.append(package_analysis) - - self.assertEqual(package.versions[0].analysis[0].raw, "<>") - - def test_save_several_analysis(self): - - _analyzer1 = ( - self.session.query(model.Analyzer) - .filter(model.Analyzer.name == "cppcheck").first() - ) - _analyzer2 = ( - self.session.query(model.Analyzer) - .filter(model.Analyzer.name == "flawfinder").first() - ) - - package = model.Package( - name='bla', - fetcher_id=self.fetcher.id - ) - package_version = model.Version( - number='1.0.1', - package_id=package.id - ) - - self.fetcher.packages.append(package) - package.versions.append(package_version) - - self.session.add(package) - self.session.add(package_version) - self.session.commit() - - package_analysis1 = model.Analysis( - raw="<>", - analyzer_id=_analyzer1.id, - version_id=package_version.id - ) - package_analysis2 = model.Analysis( - raw="><", - analyzer_id=_analyzer2.id, - version_id=package_version.id - ) - - self.session.add(package_analysis1) - self.session.add(package_analysis2) - self.session.commit() - - saved_package = ( - self.session.query(model.Package) - .filter(model.Package.name == 'bla').first() - ) - analysis = saved_package.versions[-1].analysis - self.assertEqual(len(analysis), 2) - self.assertEqual(analysis[0].raw, "<>") - self.assertEqual(analysis[1].raw, "><") - - -if __name__ == '__main__': - unittest.main() diff --git a/kiskadee/tests/test_monitor.py b/kiskadee/tests/test_monitor.py deleted file mode 100644 index 2a759fa..0000000 --- a/kiskadee/tests/test_monitor.py +++ /dev/null @@ -1,190 +0,0 @@ -import unittest -from sqlalchemy.orm import sessionmaker - -from kiskadee import model -from kiskadee.monitor import Monitor -from kiskadee.queue import packages_queue -from kiskadee.model import Package, Fetcher, create_analyzers, Report, Analysis -import kiskadee.queue -import kiskadee.fetchers.debian -import kiskadee.fetchers.anitya -from kiskadee.database import Database - - -class MonitorTestCase(unittest.TestCase): - - def setUp(self): - self.engine = Database('db_test').engine - Session = sessionmaker(bind=self.engine) - self.session = Session() - self.monitor = Monitor(self.session) - model.Base.metadata.create_all(self.engine) - create_analyzers(self.session) - self.pkg1 = { - 'name': 'curl', - 'version': '7.52.1-5', - 'fetcher': kiskadee.fetchers.debian, - 'meta': {'directory': 'pool/main/c/curl'}, - 'results': { - 'cppcheck': '<>', - 'flawfinder': '><'}, - 'fetcher_id': 1} - - self.pkg2 = {'name': 'urlscan', - 'version': '0.8.2', - 'fetcher': kiskadee.fetchers.debian, - 'meta': {'directory': 'pool/main/u/urlscan'}, - 'results': { - 'cppcheck': '<>', - 'flawfinder': '><'}, - 'fetcher_id': 1} - - self.pkg3 = {'name': 'curl', - 'version': '7.52.2-5', - 'fetcher': kiskadee.fetchers.debian, - 'meta': {'directory': 'pool/main/c/curl'}, - 'results': { - 'cppcheck': '<>', - 'flawfinder': '><'}, - 'fetcher_id': 1} - self.pkg4 = {'name': 'urlanitya', - 'version': '0.11', - 'fetcher': kiskadee.fetchers.anitya, - 'meta': { - 'backend': 'gitHub', - 'homepage': 'https://github.com/GesielFreitas/Cros' - }, - 'results': { - 'cppcheck': '<>', - 'flawfinder': '><'}, - 'fetcher_id': 1} - self.analysis = { - 'analyzer_id': 1, - 'id': 1, - 'raw': { - 'results': [ - {'severity': 'warning'}, - {'severity': 'style'}, - {'severity': 'error'} - ] - } - } - - def tearDown(self): - self.session.close() - model.Base.metadata.drop_all() - - def test_dequeue_package(self): - packages_queue.put(self.pkg1) - _pkg = self.monitor.dequeue_package() - self.assertTrue(isinstance(_pkg, dict)) - - def test_return_fetcher_name(self): - fetcher = kiskadee.fetchers.debian.Fetcher() - self.assertEqual(fetcher.name, 'debian') - - def test_save_some_fetcher(self): - fetcher = kiskadee.fetchers.debian.Fetcher() - self.monitor._save_fetcher(fetcher) - _fetchers = self.monitor.session.query(Fetcher).all() - self.assertEqual(len(_fetchers), 1) - self.assertEqual(_fetchers[0].name, 'debian') - self.assertEqual(_fetchers[0].description, - fetcher.config['description']) - - def test_save_package(self): - self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher()) - packages_queue.put(self.pkg1) - packages_queue.put(self.pkg2) - - _pkg = self.monitor.dequeue_package() - self.monitor._save_analyzed_pkg(_pkg) - _pkgs = self.monitor.session.query(Package).all() - self.assertEqual(len(_pkgs), 1) - self.assertEqual(_pkgs[0].name, _pkg['name']) - - _pkg = self.monitor.dequeue_package() - self.monitor._save_analyzed_pkg(_pkg) - _pkgs = self.monitor.session.query(Package).all() - self.assertEqual(len(_pkgs), 2) - self.assertEqual(_pkgs[1].name, _pkg['name']) - - def test_save_reports(self): - _fetcher = model.Fetcher( - name='kiskadee-fetcher', target='university' - ) - _pkg = model.Package( - name='kiskadee-package', fetcher_id=1 - ) - _version = model.Version( - number='1.0-rc1', package_id=1 - ) - _raw = { - 'results': [ - {'severity': 'warning'}, - {'severity': 'style'}, - {'severity': 'error'} - ] - } - _analysis = Analysis( - version_id=1, - analyzer_id=1, - id=1, - raw=_raw - ) - self.session.add(_fetcher) - self.session.add(_pkg) - self.session.add(_version) - self.session.add(_analysis) - self.session.commit() - _dict_analysis = { - 'results': _raw['results'], - 'id': 1 - } - self.monitor._save_reports(_dict_analysis, self.pkg1, 'cppcheck') - _report = self.monitor.session.query(Report).all() - self.assertEqual(len(_report), 1) - - def test_save_version(self): - self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher()) - self.monitor._save_analyzed_pkg(self.pkg1) - _pkgs = self.monitor.session.query(Package).all() - _version = _pkgs[0].versions[0].number - self.assertEqual(_version, self.pkg1['version']) - - def test_update_version(self): - self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher()) - packages_queue.put(self.pkg1) - packages_queue.put(self.pkg3) - - _pkg = self.monitor.dequeue_package() - self.monitor._save_analyzed_pkg(_pkg) - - _pkg = self.monitor.dequeue_package() - self.monitor._save_analyzed_pkg(_pkg) - - _pkgs = self.monitor.session.query(Package).all() - self.assertEqual(len(_pkgs), 1) - - _pkg_versions = self.monitor._query(Package).\ - filter(Package.name == _pkg['name']).first().versions - - _first_version = _pkg_versions[0].number - _current_version = _pkg_versions[-1].number - self.assertEqual(self.pkg1['version'], _first_version) - self.assertEqual(_pkg['version'], _current_version) - - def test_save_package_anitya(self): - self.monitor._save_fetcher(kiskadee.fetchers.anitya.Fetcher()) - packages_queue.put(self.pkg4) - - _pkg = self.monitor.dequeue_package() - self.monitor._save_analyzed_pkg(_pkg) - - _pkgs = self.monitor.session.query(Package).all() - self.assertEqual(len(_pkgs), 1) - self.assertEqual(_pkgs[0].homepage, _pkg['meta']['homepage']) - - -if __name__ == '__main__': - unittest.main() diff --git a/kiskadee/tests/test_plugins.py b/kiskadee/tests/test_plugins.py deleted file mode 100644 index 873581a..0000000 --- a/kiskadee/tests/test_plugins.py +++ /dev/null @@ -1,207 +0,0 @@ -import unittest -import os -import shutil -import tempfile -import zmq -import threading -import time - -import kiskadee.queue -import kiskadee - - -class FetchersTestCase(unittest.TestCase): - def test_loading(self): - _config = kiskadee.config - _config['debian_fetcher'] = {'active': 'no'} - _config['juliet_fetcher'] = {'active': 'yes'} - _config['example_fetcher'] = {'active': 'yes'} - kiskadee.config = _config - fetchers = kiskadee.load_fetchers() - for fetcher in fetchers: - name_index = len(fetcher.__name__.split('.')) - 1 - name = fetcher.__name__.split('.')[name_index] - self.assertTrue(name != 'debian') - kiskadee.config['example_fetcher'] = { - 'target': 'example', - 'description': 'SAMATE Juliet test suite', - 'analyzers': 'cppcheck flawfinder', - 'active': 'yes' - } - - -class DebianFetcherTestCase(unittest.TestCase): - - def setUp(self): - import kiskadee.fetchers.debian - self.debian_fetcher = kiskadee.fetchers.debian.Fetcher() - self.data = self.debian_fetcher.config - - def _download_sources_gz(self): - tmp_path = tempfile.gettempdir() - path = tempfile.mkdtemp(dir=tmp_path) - source = 'kiskadee/tests/test_source/Sources.gz' - shutil.copy2(source, path) - return path - - def test_mount_sources_gz_url(self): - mirror = self.data['target'] - release = self.data['release'] - url = self.debian_fetcher._sources_gz_url() - expected_url = "%s/dists/%s/main/source/Sources.gz" % (mirror, release) - self.assertEqual(url, expected_url) - - def test_uncompress_sources_gz(self): - tmp_path = tempfile.gettempdir() - temp_dir = tempfile.mkdtemp(dir=tmp_path) - self.debian_fetcher._download_sources_gz = self._download_sources_gz - temp_dir = self.debian_fetcher._download_sources_gz() - self.debian_fetcher._uncompress_gz(temp_dir) - files = os.listdir(temp_dir) - shutil.rmtree(temp_dir) - self.assertTrue('Sources' in files) - - def test_enqueue_a_valid_pkg(self): - tmp_path = tempfile.gettempdir() - temp_dir = tempfile.mkdtemp(dir=tmp_path) - self.debian_fetcher._download_sources_gz = self._download_sources_gz - temp_dir = self.debian_fetcher._download_sources_gz() - self.debian_fetcher._uncompress_gz(temp_dir) - self.debian_fetcher._queue_sources_gz_pkgs(temp_dir) - shutil.rmtree(temp_dir) - - some_pkg = kiskadee.queue.packages_queue.get() - self.assertTrue(isinstance(some_pkg, dict)) - self.assertIn('name', some_pkg) - self.assertIn('version', some_pkg) - self.assertIn('fetcher', some_pkg) - self.assertIn('meta', some_pkg) - self.assertIn('directory', some_pkg['meta']) - - def test_mount_dsc_url(self): - expected_dsc_url = ("http://ftp.us.debian.org" + - "/debian/pool/main/0/0ad/0ad_0.0.21-2.dsc") - sample_package = {'name': '0ad', - 'version': '0.0.21-2', - 'meta': {'directory': 'pool/main/0/0ad'}} - url = self.debian_fetcher._dsc_url(sample_package) - self.assertEqual(expected_dsc_url, url) - - def test_compare_gt_version(self): - new = '1.1.1' - old = '1.1.0' - result = self.debian_fetcher.compare_versions(new, old) - self.assertTrue(result) - - def test_compare_smallest_version(self): - new = '8.5-2' - old = '8.6-0' - result = self.debian_fetcher.compare_versions(new, old) - self.assertFalse(result) - - def test_compare_equal_version(self): - new = '3.3.3-0' - old = '3.3.3-0' - result = self.debian_fetcher.compare_versions(new, old) - self.assertFalse(result) - - -class TestAnityaFetcher(unittest.TestCase): - - def setUp(self): - import kiskadee.fetchers.anitya - self.anitya_fetcher = kiskadee.fetchers.anitya.Fetcher() - - self.msg = "anitya {'body':{'msg':{'project':{name: 'urlscan',"\ - "'version':'0.8.5','backend':'GitHub',"\ - "'homepage':'https://github.com/firecat53/urlscan'}}}}" - - self.msg1 = "{'body':{'msg':{'project':{name: 'urlscan',"\ - "'version':'0.8.5','backend':'GitHub',"\ - "'homepage':'https://github.com/firecat53/urlscan'}}}}" - - def test_connect_to_zmq(self): - - def zmq_server(): - context = zmq.Context() - socket = context.socket(zmq.PUB) - socket.bind("tcp://*:7776") - - zmq_server() - socket = self.anitya_fetcher._connect_to_zmq("7776", "anitya") - self.assertIsNotNone(socket) - - def test_receive_msg_from_zmq(self): - """definitely this is not a unit test, but is important to kiskadee - be able to interact correctly with ZeroMQ. - We need to define other test levels to kiskadee asap. - When we do that, we can move integration tests - to a proper place. For now we will maintain this test here""" - - def zmq_server(): - context = zmq.Context() - socket = context.socket(zmq.PUB) - socket.bind("tcp://*:7776") - time.sleep(1) - socket.send_string("%s" % (self.msg)) - time.sleep(1) - - def receive_msg_from_server(): - client_socket = self.anitya_fetcher._connect_to_zmq( - "7776", "anitya") - if client_socket: - response = client_socket.recv_string() - results[0] = response[response.find(" ")+1::] - else: - results[0] = "invalid" - - results = [None] - - client_as_thread = threading.Thread(target=receive_msg_from_server) - server_as_thread = threading.Thread( - target=zmq_server) - - server_as_thread.start() - client_as_thread.start() - server_as_thread.join() - self.assertEqual(self.msg1, results[0]) - - def test_compare_versions(self): - is_greater = self.anitya_fetcher.compare_versions('0.8.5-2', '0.8.5-1') - self.assertTrue(is_greater) - - def test_load_backend(self): - backend = self.anitya_fetcher._load_backend('github') - self.assertIsNotNone(backend) - - def test_not_load_backend(self): - backend = self.anitya_fetcher._load_backend('foo') - self.assertEqual(backend, {}) - - def test_get_sources(self): - - def mock_github(self, fetcher, source_data, path): - return 'kiskadee/tests/test_source/Sources.gz' - - kiskadee.fetchers.anitya.Backends.github = mock_github - source_data = {'meta': {'backend': 'GitHub'}} - source_path = self.anitya_fetcher.get_sources(source_data) - self.assertEqual(source_path, - mock_github("self", "fetcher", "foo", "bla")) - - def test_create_package_dict(self): - - self.anitya_fetcher._create_package_dict(self.msg) - _dict = kiskadee.queue.packages_queue.get() - self.assertEqual(_dict['name'], 'urlscan') - self.assertEqual(_dict['version'], '0.8.5') - self.assertEqual(_dict['meta']['backend'], 'GitHub') - self.assertEqual( - _dict['meta']['homepage'], - 'https://github.com/firecat53/urlscan' - ) - self.assertEqual(_dict['fetcher'].name, 'anitya') - - -if __name__ == '__main__': - unittest.main() diff --git a/kiskadee/tests/test_queue.py b/kiskadee/tests/test_queue.py deleted file mode 100644 index 19abf96..0000000 --- a/kiskadee/tests/test_queue.py +++ /dev/null @@ -1,23 +0,0 @@ -import unittest -import kiskadee.queue -import kiskadee.fetchers.example - - -class QueueTestCase(unittest.TestCase): - - def test_enqueue_wrapper_pkg(self): - - def my_dict(): - return { - 'name': 'bar', - 'fetcher': kiskadee.fetchers.example.Fetcher(), - 'version': '1.0.0' - } - - enque_my_dict = kiskadee.queue.package_enqueuer(my_dict) - enque_my_dict() - self.assertTrue(isinstance(kiskadee.queue.packages_queue.get(), dict)) - - -if __name__ == '__main__': - unittest.main() diff --git a/kiskadee/tests/test_report.py b/kiskadee/tests/test_report.py deleted file mode 100644 index 10cb936..0000000 --- a/kiskadee/tests/test_report.py +++ /dev/null @@ -1,50 +0,0 @@ -import unittest - -from sqlalchemy.orm import sessionmaker -from kiskadee.report import CppcheckReport, FlawfinderReport -from kiskadee.database import Database - - -class ReportTestCase(unittest.TestCase): - - def setUp(self): - self.engine = Database('db_test').engine - Session = sessionmaker(bind=self.engine) - self.session = Session() - - def tearDown(self): - self.session.close() - - def test_compute_cppcheck_reports(self): - _reports = [ - {'severity': 'warning'}, - {'severity': 'error'}, - {'severity': 'style'}, - {'some-attribute': None} - ] - _cpp_reporter = CppcheckReport(_reports) - result = _cpp_reporter._compute_reports('cppcheck') - self.assertEqual(len(result.keys()), 3) - self.assertEqual(result['warning'], 1) - self.assertEqual(result['style'], 1) - self.assertEqual(result['error'], 1) - - def test_compute_flawfinder_reports(self): - _reports = [ - {'severity': '5'}, - {'severity': '4'}, - {'severity': '3'}, - {'some-attribute': None} - ] - _flawfinder_reporter = FlawfinderReport(_reports) - result = _flawfinder_reporter._compute_reports('flawfinder') - self.assertEqual(len(result.keys()), 5) - self.assertEqual(result['severity_5'], 1) - self.assertEqual(result['severity_4'], 1) - self.assertEqual(result['severity_3'], 1) - self.assertEqual(result['severity_2'], 0) - self.assertEqual(result['severity_1'], 0) - - -if __name__ == '__main__': - unittest.main() diff --git a/kiskadee/tests/test_runner.py b/kiskadee/tests/test_runner.py deleted file mode 100644 index 3776e89..0000000 --- a/kiskadee/tests/test_runner.py +++ /dev/null @@ -1,98 +0,0 @@ -import unittest -import tempfile - -from kiskadee.runner import Runner -import kiskadee.fetchers.example -import kiskadee.fetchers.debian -from sqlalchemy.orm import sessionmaker -from kiskadee import model -from kiskadee.queue import KiskadeeQueue -from kiskadee.database import Database - - -class AnalyzersTestCase(unittest.TestCase): - - def setUp(self): - self.engine = Database('db_test').engine - Session = sessionmaker(bind=self.engine) - self.session = Session() - model.Base.metadata.create_all(self.engine) - model.create_analyzers(self.session) - self.fetcher = kiskadee.fetchers.debian.Fetcher() - self.deb_pkg = {'name': 'test', - 'version': '1.0.0', - 'fetcher': kiskadee.fetchers.debian.Fetcher() - } - self.fetcher = model.Fetcher( - name='kiskadee-fetcher', target='university' - ) - self.session.add(self.fetcher) - self.session.commit() - kiskadee_queue = KiskadeeQueue() - self.runner = Runner() - self.runner.kiskadee_queue = kiskadee_queue - - def tearDown(self): - self.session.close() - model.Base.metadata.drop_all() - - def test_run_analyzer(self): - - source_to_analysis = { - 'name': 'test', - 'version': '1.0.0', - 'fetcher': kiskadee.fetchers.example.Fetcher() - } - - source_path = self.runner._path_to_uncompressed_source( - source_to_analysis, kiskadee.fetchers.example.Fetcher() - ) - firehose_report = self.runner.analyze("cppcheck", source_path) - self.assertIsNotNone(firehose_report) - - def test_generate_a_firehose_report(self): - source_to_analysis = { - 'name': 'test', - 'version': '1.0.0', - 'fetcher': kiskadee.fetchers.example.Fetcher() - } - - self.runner.call_analyzers(source_to_analysis) - analyzed_pkg = self.runner.kiskadee_queue.dequeue_result() - - self.assertEqual(analyzed_pkg['name'], source_to_analysis['name']) - self.assertIn('cppcheck', analyzed_pkg['results']) - self.assertIn('flawfinder', analyzed_pkg['results']) - - def test_path_to_uncompressed_source(self): - - source_to_analysis = { - 'name': 'test', - 'version': '1.0.0', - 'fetcher': kiskadee.fetchers.example - } - - source_path = self.runner._path_to_uncompressed_source( - source_to_analysis, kiskadee.fetchers.example.Fetcher() - ) - tmp_path = tempfile.gettempdir() - self.assertTrue(source_path.find(tmp_path) >= 0) - self.assertIsNotNone(source_path) - - def test_invalid_path_to_uncompressed_source(self): - - source_to_analysis = { - 'name': 'test', - 'version': '1.0.0', - 'fetcher': kiskadee.fetchers.example - } - - source_path = self.runner._path_to_uncompressed_source( - source_to_analysis, None - ) - - self.assertIsNone(source_path) - - -if __name__ == '__main__': - unittest.main() diff --git a/util/kiskadee.conf b/util/kiskadee.conf index 5eb6ec8..c2dc351 100644 --- a/util/kiskadee.conf +++ b/util/kiskadee.conf @@ -19,7 +19,7 @@ port = 5432 dbname = kiskadee_test [debian_fetcher] -source = http://ftp.us.debian.org/debian +target = http://ftp.us.debian.org/debian description = A fetcher to monitor Debian Repositories release = sid meta = Sources.gz @@ -28,13 +28,13 @@ analyzers = cppcheck flawfinder active = no [juliet_fetcher] -source = juliet +target = juliet description = SAMATE Juliet test suite analyzers = cppcheck active = no [anitya_fetcher] -source = release-monitoring.org +target = release-monitoring.org description = A fetcher to monitor Anitya packages analyzers = cppcheck flawfinder active = no @@ -42,7 +42,7 @@ zmq_port = 5556 zmq_topic = anitya [example_fetcher] -source = example +target = example description = SAMATE Juliet test suite analyzers = cppcheck flawfinder clanganalyzer frama_c active = yes From b11670b3c8cb41e961301f9153c9e462f93b353f Mon Sep 17 00:00:00 2001 From: David Carlos Date: Apr 08 2018 20:50:59 +0000 Subject: [PATCH 5/18] Group tests into subpackages. - This will enable us to run each test suite separately. --- diff --git a/kiskadee/fetchers/anitya.py b/kiskadee/fetchers/anitya.py index 771fbb9..589d59c 100644 --- a/kiskadee/fetchers/anitya.py +++ b/kiskadee/fetchers/anitya.py @@ -93,7 +93,6 @@ class Fetcher(kiskadee.fetchers.Fetcher): } } super().watch(**kiskadee_project) - def _event_to_dict(self, msg): msg = msg[msg.find(" ")+1::] diff --git a/kiskadee/fetchers/example.py b/kiskadee/fetchers/example.py index a4c1593..02eb9b2 100644 --- a/kiskadee/fetchers/example.py +++ b/kiskadee/fetchers/example.py @@ -9,8 +9,7 @@ class Fetcher(kiskadee.fetchers.Fetcher): """Use sources from test suite.""" return 'kiskadee/tests/test_source/test_source.tar.gz' - @kiskadee.queue.enqueue_project - def watch(self): + def watch(self, **kwargs): """There is no proper API to inspect new example versions. It should not matter, since example will not receive updates. """ diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py index fbf68bc..ef3fb77 100644 --- a/kiskadee/monitor.py +++ b/kiskadee/monitor.py @@ -53,32 +53,9 @@ class Monitor: if pkg: self._send_to_runner(pkg) time.sleep(2) - analyzed_project = self.queues.dequeue_result() + analyzed_project = self.kiskadee_queue.dequeue_result() self._save_analyzed_project(analyzed_project) - def dequeue_package(self): - """Dequeue packages from packages_queue.""" - if not kiskadee.queue.packages_queue.empty(): - pkg = kiskadee.queue.packages_queue.get() - kiskadee.logger.debug( - "MONITOR: Dequed Package: {}_{}" - .format(pkg["name"], pkg["version"]) - ) - return pkg - return {} - - # Use directly dequeue_result from kiskadee.queue - #def dequeue_result(self): - # """Dequeue analyzed packages from result_queue.""" - # if not self.kiskadee_queue.results_empty(): - # pkg = self.kiskadee_queue.dequeue_result() - # kiskadee.logger.debug( - # "MONITOR: Dequed result for package : {}-{}" - # .format(pkg["name"], pkg["version"]) - # ) - # return pkg - # return {} - def _send_to_runner(self, pkg): _name = pkg['fetcher'].split('.')[-1] _fetcher = self._query(Fetcher).filter_by(name = _name).first() @@ -103,8 +80,8 @@ class Monitor: self.kiskadee_queue.enqueue_analysis(pkg) # Move this to model.py - def _save_analyzed_project(self, project): - if not pkg: + def _save_analyzed_project(self, data): + if not data: return {} project = self._query(Package).filter_by(name = data['name']).first() if not project: @@ -116,7 +93,7 @@ class Monitor: self._save_analysis(data, analyzer, result, project.versions[-1]) # Move this to model.py - def _update_pkg(self, package, pkg): + def _update_project(self, package, pkg): if(package.versions[-1].number == pkg['version']): return package @@ -138,7 +115,7 @@ class Monitor: return None # Move this to model.py - def _save_pkg(self, pkg): + def _save_project(self, pkg): homepage = None if ('meta' in pkg) and ('homepage' in pkg['meta']): homepage = pkg['meta']['homepage'] @@ -253,11 +230,11 @@ def daemon(): runner = Runner() monitor_process = Process( target=monitor.monitor, - args=(queues) + args=(queues,) ) runner_process = Process( target=runner.runner, - args=(queues) + args=(queues,) ) monitor_process.start() runner_process.start() diff --git a/kiskadee/queue.py b/kiskadee/queue.py index a6835ea..214b631 100644 --- a/kiskadee/queue.py +++ b/kiskadee/queue.py @@ -1,5 +1,6 @@ """Provide kiskadee queues and operations on them.""" import time +import dill from multiprocessing import Queue import kiskadee @@ -23,10 +24,10 @@ def enqueue_project(fetcher_watch): """ def wrapper(*args, **kwargs): project = fetcher_watch(*args, **kwargs) - kiskadee.queue.Queues.enqueue_project(project) + kiskadee.queue.queues.enqueue_project(project) fetcher = project['fetcher'].name kiskadee.logger.debug( - "{} fetcher: Sending package {}_{} for monitor" + "{} fetcher: sending package {}_{} for monitor" .format(fetcher, package['name'], package['version']) ) time.sleep(2) @@ -40,15 +41,14 @@ class Queues(): """Provide kiskadee queues objects.""" @staticmethod - def enqueue_analysis(analysis): + def enqueue_analysis(project_to_analysis): """Put a analysis on the analysis queue.""" - analysis.put(analysis) + analysis.put(project_to_analysis) @staticmethod - @empty_queue def dequeue_analysis(): """Get a analysis from the analysis queue.""" - return analysis.get(timeout=1) + return analysis.get() @staticmethod def enqueue_result(result): diff --git a/kiskadee/tests/api/test_api.py b/kiskadee/tests/api/test_api.py index 0b83d1a..e88fee5 100644 --- a/kiskadee/tests/api/test_api.py +++ b/kiskadee/tests/api/test_api.py @@ -1,10 +1,8 @@ import json import unittest -from sqlalchemy.orm import sessionmaker from unittest.mock import MagicMock import kiskadee -from kiskadee.runner import Runner from kiskadee.monitor import Monitor import kiskadee.api.app import kiskadee.fetchers.example diff --git a/kiskadee/tests/integration/test_monitor.py b/kiskadee/tests/integration/test_monitor.py index aa95be6..67cd7c2 100644 --- a/kiskadee/tests/integration/test_monitor.py +++ b/kiskadee/tests/integration/test_monitor.py @@ -3,8 +3,8 @@ from sqlalchemy.orm import sessionmaker from kiskadee import model from kiskadee.monitor import Monitor -from kiskadee.queue import packages_queue -from kiskadee.model import Package, Fetcher, create_analyzers, Report, Analysis +from kiskadee.queue import Queues +from kiskadee.model import Package, create_analyzers, Report, Analysis import kiskadee.queue import kiskadee.fetchers.debian import kiskadee.fetchers.anitya @@ -18,12 +18,13 @@ class MonitorTestCase(unittest.TestCase): Session = sessionmaker(bind=self.engine) self.session = Session() self.monitor = Monitor(self.session) + self.queues = Queues() model.Base.metadata.create_all(self.engine) create_analyzers(self.session) self.pkg1 = { 'name': 'curl', 'version': '7.52.1-5', - 'fetcher': kiskadee.fetchers.debian, + 'fetcher': kiskadee.fetchers.debian.__name__, 'meta': {'directory': 'pool/main/c/curl'}, 'results': { 'cppcheck': '<>', @@ -32,7 +33,7 @@ class MonitorTestCase(unittest.TestCase): self.pkg2 = {'name': 'urlscan', 'version': '0.8.2', - 'fetcher': kiskadee.fetchers.debian, + 'fetcher': kiskadee.fetchers.debian.__name__, 'meta': {'directory': 'pool/main/u/urlscan'}, 'results': { 'cppcheck': '<>', @@ -41,7 +42,7 @@ class MonitorTestCase(unittest.TestCase): self.pkg3 = {'name': 'curl', 'version': '7.52.2-5', - 'fetcher': kiskadee.fetchers.debian, + 'fetcher': kiskadee.fetchers.debian.__name__, 'meta': {'directory': 'pool/main/c/curl'}, 'results': { 'cppcheck': '<>', @@ -49,7 +50,7 @@ class MonitorTestCase(unittest.TestCase): 'fetcher_id': 1} self.pkg4 = {'name': 'urlanitya', 'version': '0.11', - 'fetcher': kiskadee.fetchers.anitya, + 'fetcher': kiskadee.fetchers.anitya.__name__, 'meta': { 'backend': 'gitHub', 'homepage': 'https://github.com/GesielFreitas/Cros' @@ -78,31 +79,14 @@ class MonitorTestCase(unittest.TestCase): fetcher = kiskadee.fetchers.debian.Fetcher() self.assertEqual(fetcher.name, 'debian') - def test_save_some_fetcher(self): - fetcher = kiskadee.fetchers.debian.Fetcher() - self.monitor._save_fetcher(fetcher) - _fetchers = self.monitor.session.query(Fetcher).all() - self.assertEqual(len(_fetchers), 1) - self.assertEqual(_fetchers[0].name, 'debian') - self.assertEqual(_fetchers[0].description, - fetcher.config['description']) - def test_save_package(self): self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher()) - packages_queue.put(self.pkg1) - packages_queue.put(self.pkg2) - - _pkg = self.monitor.dequeue_package() - self.monitor._save_analyzed_pkg(_pkg) - _pkgs = self.monitor.session.query(Package).all() - self.assertEqual(len(_pkgs), 1) - self.assertEqual(_pkgs[0].name, _pkg['name']) - - _pkg = self.monitor.dequeue_package() - self.monitor._save_analyzed_pkg(_pkg) + self.monitor._save_analyzed_project(self.pkg1) + self.monitor._save_analyzed_project(self.pkg2) _pkgs = self.monitor.session.query(Package).all() self.assertEqual(len(_pkgs), 2) - self.assertEqual(_pkgs[1].name, _pkg['name']) + self.assertEqual(_pkgs[0].name, self.pkg1['name']) + self.assertEqual(_pkgs[1].name, self.pkg2['name']) def test_save_reports(self): _fetcher = model.Fetcher( @@ -142,21 +126,21 @@ class MonitorTestCase(unittest.TestCase): def test_save_version(self): self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher()) - self.monitor._save_analyzed_pkg(self.pkg1) + self.monitor._save_analyzed_project(self.pkg1) _pkgs = self.monitor.session.query(Package).all() _version = _pkgs[0].versions[0].number self.assertEqual(_version, self.pkg1['version']) def test_update_version(self): self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher()) - packages_queue.put(self.pkg1) - packages_queue.put(self.pkg3) + self.queues.enqueue_project(self.pkg1) + self.queues.enqueue_project(self.pkg3) - _pkg = self.monitor.dequeue_package() - self.monitor._save_analyzed_pkg(_pkg) + _pkg = self.queues.dequeue_project() + self.monitor._save_analyzed_project(_pkg) - _pkg = self.monitor.dequeue_package() - self.monitor._save_analyzed_pkg(_pkg) + _pkg = self.queues.dequeue_project() + self.monitor._save_analyzed_project(_pkg) _pkgs = self.monitor.session.query(Package).all() self.assertEqual(len(_pkgs), 1) @@ -171,11 +155,9 @@ class MonitorTestCase(unittest.TestCase): def test_save_package_anitya(self): self.monitor._save_fetcher(kiskadee.fetchers.anitya.Fetcher()) - packages_queue.put(self.pkg4) - - _pkg = self.monitor.dequeue_package() - self.monitor._save_analyzed_pkg(_pkg) - + self.queues.enqueue_project(self.pkg4) + _pkg = self.queues.dequeue_project() + self.monitor._save_analyzed_project(_pkg) _pkgs = self.monitor.session.query(Package).all() self.assertEqual(len(_pkgs), 1) self.assertEqual(_pkgs[0].homepage, _pkg['meta']['homepage']) diff --git a/kiskadee/tests/integration/test_runner.py b/kiskadee/tests/integration/test_runner.py index 0ad1e0a..f9a2e16 100644 --- a/kiskadee/tests/integration/test_runner.py +++ b/kiskadee/tests/integration/test_runner.py @@ -1,3 +1,8 @@ +""" +This integration tests needs docker engine be running and accessible +by the user that is executing the tests. It also needs selinux running in +a permissive mode (setenforce 0) +""" import unittest import tempfile @@ -6,12 +11,11 @@ import kiskadee.fetchers.example import kiskadee.fetchers.debian from sqlalchemy.orm import sessionmaker from kiskadee import model -from kiskadee.queue import KiskadeeQueue +from kiskadee.queue import Queues from kiskadee.database import Database class AnalyzersTestCase(unittest.TestCase): - def setUp(self): self.engine = Database('db_test').engine Session = sessionmaker(bind=self.engine) @@ -21,16 +25,15 @@ class AnalyzersTestCase(unittest.TestCase): self.fetcher = kiskadee.fetchers.debian.Fetcher() self.deb_pkg = {'name': 'test', 'version': '1.0.0', - 'fetcher': kiskadee.fetchers.debian.Fetcher() + 'fetcher': kiskadee.fetchers.debian.__name__ } self.fetcher = model.Fetcher( name='kiskadee-fetcher2', target='university' ) self.session.add(self.fetcher) self.session.commit() - kiskadee_queue = KiskadeeQueue() self.runner = Runner() - self.runner.kiskadee_queue = kiskadee_queue + self.runner.queues = Queues() def tearDown(self): self.session.close() @@ -54,12 +57,11 @@ class AnalyzersTestCase(unittest.TestCase): source_to_analysis = { 'name': 'test', 'version': '1.0.0', - 'fetcher': kiskadee.fetchers.example.Fetcher() + 'fetcher': kiskadee.fetchers.example.__name__ } self.runner.call_analyzers(source_to_analysis) - analyzed_pkg = self.runner.kiskadee_queue.dequeue_result() - + analyzed_pkg = self.runner.queues.dequeue_result() self.assertEqual(analyzed_pkg['name'], source_to_analysis['name']) self.assertIn('cppcheck', analyzed_pkg['results']) self.assertIn('flawfinder', analyzed_pkg['results']) @@ -93,5 +95,6 @@ class AnalyzersTestCase(unittest.TestCase): self.assertIsNone(source_path) + if __name__ == '__main__': unittest.main() diff --git a/kiskadee/tests/plugins/test_plugins.py b/kiskadee/tests/plugins/test_plugins.py index 030da9a..9fe8262 100644 --- a/kiskadee/tests/plugins/test_plugins.py +++ b/kiskadee/tests/plugins/test_plugins.py @@ -70,7 +70,7 @@ class DebianFetcherTestCase(unittest.TestCase): self.debian_fetcher._queue_sources_gz_pkgs(temp_dir) shutil.rmtree(temp_dir) - some_pkg = kiskadee.queue.packages_queue.get() + some_pkg = kiskadee.queue.Queues.dequeue_project() self.assertTrue(isinstance(some_pkg, dict)) self.assertIn('name', some_pkg) self.assertIn('version', some_pkg) @@ -190,8 +190,8 @@ class TestAnityaFetcher(unittest.TestCase): def test_create_package_dict(self): - self.anitya_fetcher._create_package_dict(self.msg) - _dict = kiskadee.queue.packages_queue.get() + self.anitya_fetcher.project_to_enqueue(self.msg) + _dict = kiskadee.queue.Queues().dequeue_project() self.assertEqual(_dict['name'], 'urlscan') self.assertEqual(_dict['version'], '0.8.5') self.assertEqual(_dict['meta']['backend'], 'GitHub') @@ -199,7 +199,7 @@ class TestAnityaFetcher(unittest.TestCase): _dict['meta']['homepage'], 'https://github.com/firecat53/urlscan' ) - self.assertEqual(_dict['fetcher'].name, 'anitya') + self.assertEqual(_dict['fetcher'], 'kiskadee.fetchers.anitya') if __name__ == '__main__': diff --git a/playbook/roles/common/tasks/main.yml b/playbook/roles/common/tasks/main.yml index 5c17a0c..edcdba0 100644 --- a/playbook/roles/common/tasks/main.yml +++ b/playbook/roles/common/tasks/main.yml @@ -12,4 +12,5 @@ - postgresql-server - postgresql-contrib - libselinux-python + - devscripts ignore_errors: yes From 39f367fa621ce1081d5f4db35445b82e6eb1e524 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Apr 08 2018 20:50:59 +0000 Subject: [PATCH 6/18] Add make rules to run tests. --- diff --git a/Makefile b/Makefile index 6483460..cffabee 100644 --- a/Makefile +++ b/Makefile @@ -5,9 +5,21 @@ help: @printf "Available targets: check, analyzers, clean\n\n" check: - coverage run --omit="lib/*","setup.py","kiskadee/tests/*",".eggs/*",".venv/*" ./setup.py test + coverage run --omit="lib/*","setup.py","kiskadee/tests/*",".eggs/*",".venv/*","/usr/*" -m unittest kiskadee/tests/*/*.py coverage html +check_units: + python3 -m unittest kiskadee/tests/units/*.py + +check_integration: + python3 -m unittest kiskadee/tests/integration/*.py + +check_api: + python3 -m unittest kiskadee/tests/api/*.py + +check_plugins: + python3 -m unittest kiskadee/tests/plugins/*.py + analyzers: docker ps 2> /dev/null; \ if [ $$? -ne 1 ]; then \ From 935e3d7ddfcaf737976b4f20997787015bda05b0 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Apr 08 2018 20:50:59 +0000 Subject: [PATCH 7/18] Minor fixes to run ansible deploy. - Add some documentation to run deploy locally. --- diff --git a/Vagrantfile b/Vagrantfile index 24daa82..41fa18a 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -29,7 +29,8 @@ Vagrant.configure(2) do |config| # https://developer.fedoraproject.org/tools/vagrant/vagrant-nfs.html # After enable nfs-server, you will need to start the service. core.vm.synced_folder "./", "/home/vagrant/kiskadee", type: "nfs", nfs_udp: false - core.vm.network "public_network", ip: ips[:core] + core.vm.network "public_network", ip: ips[:core], + :bridge => 'virbr0', :dev => 'virbr0' core.vm.provider "libvirt" do |v| v.memory = 2048 end diff --git a/doc/installing.rst b/doc/installing.rst index d5c850c..946e2c3 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -1,7 +1,24 @@ Installing kiskadee =================== -Nothing to show here yet :( +Development +----------- +To install locally run: -We are still packaging kiskadee and writing proper ansible playbooks for fully -automated deployment :) +First, make shure that ansible will be able to login on the vm. ansible will +use the root user to do this, so you will have to add your public ssh +key inside the root ~/.ssh/authorized_keys file. You can also create the host +user inside the vm, in order to be able to test the vm access with the ping +command. + +To check if ansible can access the machine: + +.. code-block:: bash + + ansible -i playbook/hosts.local all -m ping + +To deploy kiskadee locally: + +.. code-block:: bash + + ansible-playbook -c paramiko -i playbook/hosts.local playbook/local.yml diff --git a/playbook/hosts.local b/playbook/hosts.local index 2b078eb..05f11e9 100644 --- a/playbook/hosts.local +++ b/playbook/hosts.local @@ -1,5 +1,5 @@ [kiskadee-core] -core-host ansible_host=10.18.0.115 +core-host ansible_host=192.168.121.6 ansible_user=root [kiskadee-frontend] frontend-host ansible_host=10.18.0.116 diff --git a/playbook/roles/kiskadee/tasks/main.yml b/playbook/roles/kiskadee/tasks/main.yml index c335027..02907e5 100644 --- a/playbook/roles/kiskadee/tasks/main.yml +++ b/playbook/roles/kiskadee/tasks/main.yml @@ -1,15 +1,3 @@ -- name: Create kiskadee group - group: - name: kiskadee - state: present - -- name: Create kiskadee user - user: - name: kiskadee - groups: kiskadee, wheel - append: yes - shell: /bin/bash - # TODO: do not turn selinux off. Add rules instead - name: Turn off selinux command: setenforce 0 From 764052a713a57247989723a1c329c492e2c553c1 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Apr 08 2018 20:51:00 +0000 Subject: [PATCH 8/18] Move code from monitor to model class. - All code that was responsible to save data on the database were moved to model.py, this made monitor class more cohesive. --- diff --git a/Vagrantfile b/Vagrantfile index 41fa18a..617b439 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -29,8 +29,9 @@ Vagrant.configure(2) do |config| # https://developer.fedoraproject.org/tools/vagrant/vagrant-nfs.html # After enable nfs-server, you will need to start the service. core.vm.synced_folder "./", "/home/vagrant/kiskadee", type: "nfs", nfs_udp: false - core.vm.network "public_network", ip: ips[:core], + core.vm.network "public_network", ip: ips[:core], :bridge => 'virbr0', :dev => 'virbr0' + core.vm.network "forwarded_port", guest: 5000, host: 5000 core.vm.provider "libvirt" do |v| v.memory = 2048 end diff --git a/kiskadee/model.py b/kiskadee/model.py index 6aa3233..4e4dff2 100644 --- a/kiskadee/model.py +++ b/kiskadee/model.py @@ -3,7 +3,10 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import Column, Integer, UnicodeText, UniqueConstraint,\ Sequence, Unicode, ForeignKey, orm, JSON +import json + import kiskadee +from kiskadee.report import CppcheckReport, FlawfinderReport Base = declarative_base() @@ -27,6 +30,47 @@ class Package(Base): UniqueConstraint('name', 'fetcher_id'), ) + @staticmethod + def save(session, data): + homepage = None + if ('meta' in data) and ('homepage' in data['meta']): + homepage = data['meta']['homepage'] + + _package = Package( + name=data['name'], + homepage=homepage, + fetcher_id=data['fetcher_id'] + ) + session.add(_package) + session.commit() + _version = Version(number=data['version'], + package_id=_package.id) + session.add(_version) + session.commit() + return _package + + @staticmethod + def update(session, project, data): + + if(project.versions[-1].number == data['version']): + return project + try: + _new_version = Version( + number=data['version'], + project_id=project.id + ) + project.versions.append(_new_version) + session.add(project) + session.commit() + kiskadee.logger.debug( + "MONITOR: Sending project {}_{}" + "for analysis".format(data['name'], data['version']) + ) + return project + except ValueError: + kiskadee.logger.debug("MONITOR: Could not compare versions") + return None + class Fetcher(Base): """kiskadee fetcher abstraction.""" @@ -64,6 +108,23 @@ class Analyzer(Base): version = Column(Unicode(255), nullable=True) analysis = orm.relationship('Analysis', backref='analyzers') + @staticmethod + def create_analyzers(_session): + """Create the analyzers on database. + + The kiskadee analyzers are defined on the section `analyzers` of the + kiskadee.conf file. The `_session` argument represents a sqlalchemy + session. + """ + list_of_analyzers = dict(kiskadee.config._sections["analyzers"]) + for _name, _version in list_of_analyzers.items(): + if not _session.query(Analyzer)\ + .filter_by(name = _name, version = _version )\ + .first(): + new_analyzer = kiskadee.model.Analyzer(name = _name, + version = _version) + _session.add(new_analyzer) + _session.commit() class Analysis(Base): """Abstraction of a package analysis.""" @@ -77,10 +138,44 @@ class Analysis(Base): report = orm.relationship('Report', uselist=False, back_populates='analysis') + @staticmethod + def save(session, data, analyzer, result, version): + _analysis = kiskadee.model.Analysis() + try: + _analyzer = session.query(kiskadee.model.Analyzer).\ + filter_by(name = analyzer).first() + _analysis.analyzer_id = _analyzer.id + _analysis.version_id = version.id + _analysis.raw = json.loads(result) + session.add(_analysis) + session.commit() + dict_analysis = { + 'results': _analysis.raw['results'], + 'id': _analysis.id + } + Report.save(session, dict_analysis, data, _analyzer.name) + kiskadee.logger.debug( + "MONITOR: Saved analysis done by {} for package: {}-{}" + .format(analyzer, data["name"], data["version"]) + ) + return + except Exception as err: + kiskadee.logger.debug( + "MONITOR: The required analyzer was " + + "not registered in kiskadee" + ) + kiskadee.logger.debug(err) + return None + class Report(Base): """Abstraction of a analysis report.""" + REPORTERS = { + 'cppcheck': CppcheckReport, + 'flawfinder': FlawfinderReport + } + __tablename__ = 'reports' id = Column(Integer, Sequence('reports_id_seq', optional=True), primary_key=True) @@ -88,20 +183,34 @@ class Report(Base): results = Column(JSON) analysis = orm.relationship('Analysis', back_populates='report') + @staticmethod + def save(session, analysis, data, analyzer_name): + try: + results = analysis['results'] + analyzer_report = Report.REPORTERS[analyzer_name](results) + _reports = Report() + _reports.results = json.dumps( + analyzer_report + ._compute_reports(analyzer_name) + ) + _reports.analysis_id = analysis['id'] + session.add(_reports) + session.commit() + kiskadee.logger.debug( + "MONITOR: Saved analysis reports for {} package" + .format(data["name"]) + ) + except KeyError as key: + kiskadee.logger.debug( + "ERROR: There's no reporter " + + "to get reports from {} analyzer. ".format(key) + + "Make shure to import or implement them." + ) + except Exception as err: + kiskadee.logger.debug( + "MONITOR: Failed to get analysis reports to {} package" + .format(data["name"]) + ) + kiskadee.logger.debug(err) + return -def create_analyzers(_session): - """Create the analyzers on database. - - The kiskadee analyzers are defined on the section `analyzers` of the - kiskadee.conf file. The `_session` argument represents a sqlalchemy - session. - """ - list_of_analyzers = dict(kiskadee.config._sections["analyzers"]) - for name, version in list_of_analyzers.items(): - if not (_session.query(Analyzer).filter(Analyzer.name == name). - filter(Analyzer.version == version).first()): - new_analyzer = kiskadee.model.Analyzer() - new_analyzer.name = name - new_analyzer.version = version - _session.add(new_analyzer) - _session.commit() diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py index ef3fb77..20afeef 100644 --- a/kiskadee/monitor.py +++ b/kiskadee/monitor.py @@ -7,22 +7,15 @@ import threading from multiprocessing import Process import time import os -import json import importlib import kiskadee.database -from kiskadee.report import CppcheckReport, FlawfinderReport from kiskadee.runner import Runner import kiskadee.queue -from kiskadee.model import Package, Fetcher, Version, Report +from kiskadee.model import Package, Fetcher, Version, Report, Analysis RUNNING = True -REPORTERS = { - 'cppcheck': CppcheckReport, - 'flawfinder': FlawfinderReport -} - class Monitor: """Provide kiskadee monitoring objects.""" @@ -51,163 +44,42 @@ class Monitor: pkg = self.kiskadee_queue.dequeue_project() if pkg: - self._send_to_runner(pkg) + self.send_to_runner(pkg) time.sleep(2) analyzed_project = self.kiskadee_queue.dequeue_result() - self._save_analyzed_project(analyzed_project) + self.save_analyzed_project(analyzed_project) - def _send_to_runner(self, pkg): - _name = pkg['fetcher'].split('.')[-1] + def send_to_runner(self, data): + _name = data['fetcher'].split('.')[-1] _fetcher = self._query(Fetcher).filter_by(name = _name).first() - _package = ( - self._query(Package) - .filter(Package.name == pkg['name']).first() - ) + _package = self._query(Package).filter_by(name = data['name']).first() if _fetcher: - pkg["fetcher_id"] = _fetcher.id + data["fetcher_id"] = _fetcher.id if not _package: kiskadee.logger.debug( "MONITOR: Sending package {}_{} " - " for analysis".format(pkg['name'], pkg['version']) + " for analysis".format(data['name'], data['version']) ) - self.kiskadee_queue.enqueue_analysis(pkg) + self.kiskadee_queue.enqueue_analysis(data) else: - new_version = pkg['version'] + new_version = data['version'] analysed_version = _package.versions[-1].number - fetcher = importlib.import_module(pkg['fetcher']).Fetcher() + fetcher = importlib.import_module(data['fetcher']).Fetcher() if (fetcher.compare_versions(new_version, analysed_version)): - self.kiskadee_queue.enqueue_analysis(pkg) + self.kiskadee_queue.enqueue_analysis(data) - # Move this to model.py - def _save_analyzed_project(self, data): + def save_analyzed_project(self, data): if not data: return {} project = self._query(Package).filter_by(name = data['name']).first() if not project: - project = self._save_project(data) + project = Package.save(self.session, data) if project: - project = self._update_project(project, data) + project = Package.update(self.session, project, data) for analyzer, result in data['results'].items(): - self._save_analysis(data, analyzer, result, project.versions[-1]) - - # Move this to model.py - def _update_project(self, package, pkg): - - if(package.versions[-1].number == pkg['version']): - return package - try: - _new_version = Version( - number=pkg['version'], - package_id=package.id - ) - package.versions.append(_new_version) - self.session.add(package) - self.session.commit() - kiskadee.logger.debug( - "MONITOR: Sending package {}_{}" - "for analysis".format(pkg['name'], pkg['version']) - ) - return package - except ValueError: - kiskadee.logger.debug("MONITOR: Could not compare versions") - return None - - # Move this to model.py - def _save_project(self, pkg): - homepage = None - if ('meta' in pkg) and ('homepage' in pkg['meta']): - homepage = pkg['meta']['homepage'] - - _package = Package( - name=pkg['name'], - homepage=homepage, - fetcher_id=pkg['fetcher_id'] - ) - self.session.add(_package) - self.session.commit() - _version = Version(number=pkg['version'], - package_id=_package.id) - self.session.add(_version) - self.session.commit() - return _package - - # Move this to model.py - def _save_reports(self, analysis, pkg, analyzer_name): - try: - results = analysis['results'] - analyzer_report = REPORTERS[analyzer_name](results) - _reports = Report() - _reports.results = json.dumps( - analyzer_report - ._compute_reports(analyzer_name) - ) - _reports.analysis_id = analysis['id'] - self.session.add(_reports) - self.session.commit() - kiskadee.logger.debug( - "MONITOR: Saved analysis reports for {} package" - .format(pkg["name"]) - ) - except KeyError as key: - kiskadee.logger.debug( - "ERROR: There's no reporter " + - "to get reports from {} analyzer. ".format(key) + - "Make shure to import or implement them." - ) - except Exception as err: - kiskadee.logger.debug( - "MONITOR: Failed to get analysis reports to {} package" - .format(pkg["name"]) - ) - kiskadee.logger.debug(err) - return - - # Move this to model.py - def _save_analysis(self, pkg, analyzer, result, version): - _analysis = kiskadee.model.Analysis() - try: - _analyzer = self._query(kiskadee.model.Analyzer).\ - filter_by(name = analyzer).first() - _analysis.analyzer_id = _analyzer.id - _analysis.version_id = version.id - _analysis.raw = json.loads(result) - self.session.add(_analysis) - self.session.commit() - dict_analysis = { - 'results': _analysis.raw['results'], - 'id': _analysis.id - } - self._save_reports(dict_analysis, pkg, _analyzer.name) - kiskadee.logger.debug( - "MONITOR: Saved analysis done by {} for package: {}-{}" - .format(analyzer, pkg["name"], pkg["version"]) - ) - return - except Exception as err: - kiskadee.logger.debug( - "MONITOR: The required analyzer was " + - "not registered in kiskadee" - ) - kiskadee.logger.debug(err) - return None - - # Move this to model.py - def _save_fetcher(self, fetcher): - name = fetcher.name - kiskadee.logger.debug( - "MONITOR: Saving {} fetcher in database".format(name) - ) - if not self.session.query(Fetcher)\ - .filter(Fetcher.name == name).first(): - _fetcher = Fetcher( - name=name, - target=fetcher.config['target'], - description=fetcher.config['description'] - ) - self.session.add(_fetcher) - self.session.commit() + Analysis.save(self.session, data, analyzer, result, project.versions[-1]) def _query(self, arg): return self.session.query(arg) diff --git a/kiskadee/queue.py b/kiskadee/queue.py index 214b631..959be72 100644 --- a/kiskadee/queue.py +++ b/kiskadee/queue.py @@ -1,6 +1,5 @@ """Provide kiskadee queues and operations on them.""" import time -import dill from multiprocessing import Queue import kiskadee diff --git a/kiskadee/runner.py b/kiskadee/runner.py index 5f7e764..8fd8de7 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -30,7 +30,7 @@ class Runner: kiskadee.logger.debug('Starting runner subprocess') kiskadee.logger.debug('runner PID: {}'.format(os.getpid())) session = kiskadee.database.Database().session - kiskadee.model.create_analyzers(session) + kiskadee.model.Analyzer.create_analyzers(session) self.queues = queues while RUNNING: kiskadee.logger.debug('RUNNER: Waiting to dequeue'\ diff --git a/kiskadee/tests/integration/test_monitor.py b/kiskadee/tests/integration/test_monitor.py index 67cd7c2..6d64edf 100644 --- a/kiskadee/tests/integration/test_monitor.py +++ b/kiskadee/tests/integration/test_monitor.py @@ -4,7 +4,7 @@ from sqlalchemy.orm import sessionmaker from kiskadee import model from kiskadee.monitor import Monitor from kiskadee.queue import Queues -from kiskadee.model import Package, create_analyzers, Report, Analysis +from kiskadee.model import Package, Analyzer, Report, Analysis import kiskadee.queue import kiskadee.fetchers.debian import kiskadee.fetchers.anitya @@ -20,7 +20,7 @@ class MonitorTestCase(unittest.TestCase): self.monitor = Monitor(self.session) self.queues = Queues() model.Base.metadata.create_all(self.engine) - create_analyzers(self.session) + Analyzer.create_analyzers(self.session) self.pkg1 = { 'name': 'curl', 'version': '7.52.1-5', @@ -81,8 +81,8 @@ class MonitorTestCase(unittest.TestCase): def test_save_package(self): self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher()) - self.monitor._save_analyzed_project(self.pkg1) - self.monitor._save_analyzed_project(self.pkg2) + self.monitor.save_analyzed_project(self.pkg1) + self.monitor.save_analyzed_project(self.pkg2) _pkgs = self.monitor.session.query(Package).all() self.assertEqual(len(_pkgs), 2) self.assertEqual(_pkgs[0].name, self.pkg1['name']) @@ -126,7 +126,7 @@ class MonitorTestCase(unittest.TestCase): def test_save_version(self): self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher()) - self.monitor._save_analyzed_project(self.pkg1) + self.monitor.save_analyzed_project(self.pkg1) _pkgs = self.monitor.session.query(Package).all() _version = _pkgs[0].versions[0].number self.assertEqual(_version, self.pkg1['version']) @@ -137,10 +137,10 @@ class MonitorTestCase(unittest.TestCase): self.queues.enqueue_project(self.pkg3) _pkg = self.queues.dequeue_project() - self.monitor._save_analyzed_project(_pkg) + self.monitor.save_analyzed_project(_pkg) _pkg = self.queues.dequeue_project() - self.monitor._save_analyzed_project(_pkg) + self.monitor.save_analyzed_project(_pkg) _pkgs = self.monitor.session.query(Package).all() self.assertEqual(len(_pkgs), 1) @@ -157,7 +157,7 @@ class MonitorTestCase(unittest.TestCase): self.monitor._save_fetcher(kiskadee.fetchers.anitya.Fetcher()) self.queues.enqueue_project(self.pkg4) _pkg = self.queues.dequeue_project() - self.monitor._save_analyzed_project(_pkg) + self.monitor.save_analyzed_project(_pkg) _pkgs = self.monitor.session.query(Package).all() self.assertEqual(len(_pkgs), 1) self.assertEqual(_pkgs[0].homepage, _pkg['meta']['homepage']) diff --git a/kiskadee/tests/integration/test_runner.py b/kiskadee/tests/integration/test_runner.py index f9a2e16..d2f08df 100644 --- a/kiskadee/tests/integration/test_runner.py +++ b/kiskadee/tests/integration/test_runner.py @@ -21,7 +21,7 @@ class AnalyzersTestCase(unittest.TestCase): Session = sessionmaker(bind=self.engine) self.session = Session() model.Base.metadata.create_all(self.engine) - model.create_analyzers(self.session) + model.Analyzer.create_analyzers(self.session) self.fetcher = kiskadee.fetchers.debian.Fetcher() self.deb_pkg = {'name': 'test', 'version': '1.0.0', diff --git a/kiskadee/tests/units/test_model.py b/kiskadee/tests/units/test_model.py index 4a56835..aba56e7 100644 --- a/kiskadee/tests/units/test_model.py +++ b/kiskadee/tests/units/test_model.py @@ -13,7 +13,7 @@ class ModelTestCase(unittest.TestCase): Session = sessionmaker(bind=self.engine) self.session = Session() model.Base.metadata.create_all(self.engine) - model.create_analyzers(self.session) + model.Analyzer.create_analyzers(self.session) self.fetcher = model.Fetcher( name='kiskadee-fetcher', target='university' ) From 3a0078f8f2636fb7ec58c665556fc52d64be2d2f Mon Sep 17 00:00:00 2001 From: David Carlos Date: Apr 08 2018 20:51:00 +0000 Subject: [PATCH 9/18] Make Monitor code more readable and maintainable. - Add more useful tests to monitor. --- diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py index 20afeef..12eaad1 100644 --- a/kiskadee/monitor.py +++ b/kiskadee/monitor.py @@ -1,6 +1,6 @@ """Provide kiskadee monitoring capabilities. -kiskadee monitors repositories checking for new package versions to be +kiskadee monitors repositories checking for new project versions to be analyzed. This module provides such capabilities. """ import threading @@ -19,55 +19,57 @@ RUNNING = True class Monitor: """Provide kiskadee monitoring objects.""" - def __init__(self, _session): + def __init__(self, _session, queues): """Return a non initialized Monitor.""" self.session = _session - self.kiskadee_queue = None + self.kiskadee_queue = queues - def monitor(self, queues): - """Dequeue packages and check if they need to be analyzed. - - The packages are dequeued from the `package_queue`. When a package - needs to be analyzed, this package is enqueued in the `analyses_queue` - so the runner component can trigger an analysis. Each fetcher must - enqueue its packages in the `packages_queue`. - """ + def start(self): kiskadee.logger.debug('kiskadee PID: {}'.format(os.getppid())) kiskadee.logger.debug('Starting monitor subprocess') kiskadee.logger.debug('monitor PID: {}'.format(os.getpid())) for fetcher in kiskadee.load_fetchers(): - _start_fetcher(fetcher.Fetcher().watch) + thread = Monitor._start_fetcher(fetcher.Fetcher().watch) while RUNNING: - self.kiskadee_queue = queues - pkg = self.kiskadee_queue.dequeue_project() - - if pkg: - self.send_to_runner(pkg) - time.sleep(2) - analyzed_project = self.kiskadee_queue.dequeue_result() + new_project = self.dequeue_from_fetchers() + self.send_project_to_runner(new_project) + analyzed_project = self.dequeue_analysis_from_runner() self.save_analyzed_project(analyzed_project) - def send_to_runner(self, data): - _name = data['fetcher'].split('.')[-1] - _fetcher = self._query(Fetcher).filter_by(name = _name).first() - _package = self._query(Package).filter_by(name = data['name']).first() + def dequeue_project_from_fetchers(self): + new_project = self.kiskadee_queue.dequeue_project() + return new_project if new_project else {} - if _fetcher: - data["fetcher_id"] = _fetcher.id - if not _package: + def dequeue_analysis_from_runner(self): + return self.kiskadee_queue.dequeue_result() + + def send_project_to_runner(self, data): + if data: + fetcher, project = self.get_fetcher_and_project(data) + data["fetcher_id"] = fetcher.id if fetcher else '' + if not project: kiskadee.logger.debug( - "MONITOR: Sending package {}_{} " + "MONITOR: Sending project {}_{} " " for analysis".format(data['name'], data['version']) ) self.kiskadee_queue.enqueue_analysis(data) - else: - new_version = data['version'] - analysed_version = _package.versions[-1].number - fetcher = importlib.import_module(data['fetcher']).Fetcher() - if (fetcher.compare_versions(new_version, analysed_version)): - self.kiskadee_queue.enqueue_analysis(data) + elif self.not_analyzed_project_version(project, data): + self.kiskadee_queue.enqueue_analysis(data) + + def get_fetcher_and_project(self, data): + fetcher_name = data['fetcher'].split('.')[-1] + project_name = data['name'] + fetcher = self._query(Fetcher).filter_by(name = fetcher_name).first() + project = self._query(Package).filter_by(name = project_name).first() + return fetcher, project + + def not_analyzed_project_version(self, project, data): + project_version = data['version'] + analysed_version = project.versions[-1].number + fetcher = importlib.import_module(data['fetcher']).Fetcher() + fetcher.compare_versions(project_version, analysed_version) def save_analyzed_project(self, data): if not data: @@ -85,12 +87,13 @@ class Monitor: return self.session.query(arg) -def _start_fetcher(module, joinable=False, timeout=None): - module_as_a_thread = threading.Thread(target=module) - module_as_a_thread.daemon = True - module_as_a_thread.start() - if joinable or timeout: - module_as_a_thread.join(timeout) + @staticmethod + def _start_fetcher(module, joinable=False, timeout=None): + module_as_a_thread = threading.Thread(target=module) + module_as_a_thread.daemon = True + module_as_a_thread.start() + if joinable or timeout: + module_as_a_thread.join(timeout) def daemon(): @@ -98,11 +101,10 @@ def daemon(): # TODO: improve with start/stop system queues = kiskadee.queue.Queues() session = kiskadee.database.Database().session - monitor = Monitor(session) + monitor = Monitor(session, queues) runner = Runner() monitor_process = Process( - target=monitor.monitor, - args=(queues,) + target=monitor.start, ) runner_process = Process( target=runner.runner, diff --git a/kiskadee/tests/api/test_api.py b/kiskadee/tests/api/test_api.py index e88fee5..7c8d021 100644 --- a/kiskadee/tests/api/test_api.py +++ b/kiskadee/tests/api/test_api.py @@ -50,10 +50,10 @@ class ApiTestCase(unittest.TestCase): Monitor(kiskadee.api.app.kiskadee_db_session()) db_session = kiskadee.api.app.kiskadee_db_session() db_session.query(kiskadee.model.Package)\ - .filter_by().id = MagicMock(side_effect=side_effect) + .filter_by().id = MagicMock() db_session.query(kiskadee.model.Version)\ - .filter_by().id = MagicMock(side_effect=side_effect) + .filter_by().id = MagicMock() db_session.query().options()\ .filter().all = MagicMock(return_value=[analysis]) diff --git a/kiskadee/tests/integration/test_monitor.py b/kiskadee/tests/integration/test_monitor.py index 6d64edf..eae9829 100644 --- a/kiskadee/tests/integration/test_monitor.py +++ b/kiskadee/tests/integration/test_monitor.py @@ -1,27 +1,36 @@ import unittest +from unittest.mock import MagicMock from sqlalchemy.orm import sessionmaker from kiskadee import model from kiskadee.monitor import Monitor from kiskadee.queue import Queues -from kiskadee.model import Package, Analyzer, Report, Analysis +from kiskadee.model import Package, Analyzer, Report, Analysis, Fetcher import kiskadee.queue import kiskadee.fetchers.debian import kiskadee.fetchers.anitya +import kiskadee.fetchers.example from kiskadee.database import Database class MonitorTestCase(unittest.TestCase): def setUp(self): - self.engine = Database('db_test').engine - Session = sessionmaker(bind=self.engine) - self.session = Session() - self.monitor = Monitor(self.session) - self.queues = Queues() - model.Base.metadata.create_all(self.engine) - Analyzer.create_analyzers(self.session) - self.pkg1 = { + def mocked_models(klass): + class Package: + def all(self): + [self.pkg1, self.pkg2, self.pkg3, self.pkg4] + klass() + + self.example_fetcher = kiskadee.fetchers.example.Fetcher() + self.session = MagicMock() + + queues = Queues() + self.monitor = Monitor(self.session, queues) + + self.session.query = MagicMock(side_effect=mocked_models) + + self.data1 = { 'name': 'curl', 'version': '7.52.1-5', 'fetcher': kiskadee.fetchers.debian.__name__, @@ -31,34 +40,34 @@ class MonitorTestCase(unittest.TestCase): 'flawfinder': '><'}, 'fetcher_id': 1} - self.pkg2 = {'name': 'urlscan', - 'version': '0.8.2', - 'fetcher': kiskadee.fetchers.debian.__name__, - 'meta': {'directory': 'pool/main/u/urlscan'}, - 'results': { - 'cppcheck': '<>', - 'flawfinder': '><'}, - 'fetcher_id': 1} - - self.pkg3 = {'name': 'curl', - 'version': '7.52.2-5', - 'fetcher': kiskadee.fetchers.debian.__name__, - 'meta': {'directory': 'pool/main/c/curl'}, - 'results': { - 'cppcheck': '<>', - 'flawfinder': '><'}, - 'fetcher_id': 1} - self.pkg4 = {'name': 'urlanitya', - 'version': '0.11', - 'fetcher': kiskadee.fetchers.anitya.__name__, - 'meta': { - 'backend': 'gitHub', - 'homepage': 'https://github.com/GesielFreitas/Cros' - }, - 'results': { - 'cppcheck': '<>', - 'flawfinder': '><'}, - 'fetcher_id': 1} + self.data2 = {'name': 'urlscan', + 'version': '0.8.2', + 'fetcher': kiskadee.fetchers.debian.__name__, + 'meta': {'directory': 'pool/main/u/urlscan'}, + 'results': { + 'cppcheck': '<>', + 'flawfinder': '><'}, + 'fetcher_id': 1} + + self.data3 = {'name': 'curl', + 'version': '7.52.2-5', + 'fetcher': kiskadee.fetchers.debian.__name__, + 'meta': {'directory': 'pool/main/c/curl'}, + 'results': { + 'cppcheck': '<>', + 'flawfinder': '><'}, + 'fetcher_id': 1} + self.data4 = {'name': 'urlanitya', + 'version': '0.11', + 'fetcher': kiskadee.fetchers.anitya.__name__, + 'meta': { + 'backend': 'gitHub', + 'homepage': 'https://github.com/GesielFreitas/Cros' + }, + 'results': { + 'cppcheck': '<>', + 'flawfinder': '><'}, + 'fetcher_id': 1} self.analysis = { 'analyzer_id': 1, 'id': 1, @@ -67,101 +76,50 @@ class MonitorTestCase(unittest.TestCase): {'severity': 'warning'}, {'severity': 'style'}, {'severity': 'error'} - ] + ] + } } - } - - def tearDown(self): - self.session.close() - model.Base.metadata.drop_all() - - def test_return_fetcher_name(self): - fetcher = kiskadee.fetchers.debian.Fetcher() - self.assertEqual(fetcher.name, 'debian') - - def test_save_package(self): - self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher()) - self.monitor.save_analyzed_project(self.pkg1) - self.monitor.save_analyzed_project(self.pkg2) - _pkgs = self.monitor.session.query(Package).all() - self.assertEqual(len(_pkgs), 2) - self.assertEqual(_pkgs[0].name, self.pkg1['name']) - self.assertEqual(_pkgs[1].name, self.pkg2['name']) - - def test_save_reports(self): - _fetcher = model.Fetcher( - name='kiskadee-fetcher3', target='university' - ) - _pkg = model.Package( - name='kiskadee-package', fetcher_id=1 - ) - _version = model.Version( - number='1.0-rc1', package_id=1 + + def test_dequeue_project_from_fetcher(self): + self.example_fetcher.watch() + monitored_project = self.monitor.dequeue_project_from_fetchers() + self.assertIn("version", monitored_project) + self.assertIn("name", monitored_project) + self.assertIn("fetcher", monitored_project) + self.assertEqual(monitored_project['version'], '0.1') + + def test_run_fetchers_as_threads(self): + Monitor._start_fetcher(self.example_fetcher.watch) + Monitor._start_fetcher(self.example_fetcher.watch) + first_monitored_project = self.monitor.dequeue_project_from_fetchers() + second_monitored_project = self.monitor.dequeue_project_from_fetchers() + self.assertIn("version", first_monitored_project) + self.assertIn("version", second_monitored_project) + self.assertIn("name", first_monitored_project) + self.assertIn("name", second_monitored_project) + self.assertIn("fetcher", first_monitored_project) + self.assertIn("fetcher", second_monitored_project) + + def test_send_project_to_runner(self): + fetcher = kiskadee.model.Fetcher(name='example') + project = kiskadee.model.Package(name='project1', + fetcher_id=fetcher.id) + self.monitor.get_fetcher_and_project = MagicMock( + return_value=[fetcher, {}] + ) + self.monitor.not_analyzed_project_version = MagicMock( + return_value=True ) - _raw = { - 'results': [ - {'severity': 'warning'}, - {'severity': 'style'}, - {'severity': 'error'} - ] - } - _analysis = Analysis( - version_id=1, - analyzer_id=1, - id=1, - raw=_raw - ) - self.session.add(_fetcher) - self.session.add(_pkg) - self.session.add(_version) - self.session.add(_analysis) - self.session.commit() - _dict_analysis = { - 'results': _raw['results'], - 'id': 1 - } - self.monitor._save_reports(_dict_analysis, self.pkg1, 'cppcheck') - _report = self.monitor.session.query(Report).all() - self.assertEqual(len(_report), 1) - - def test_save_version(self): - self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher()) - self.monitor.save_analyzed_project(self.pkg1) - _pkgs = self.monitor.session.query(Package).all() - _version = _pkgs[0].versions[0].number - self.assertEqual(_version, self.pkg1['version']) - - def test_update_version(self): - self.monitor._save_fetcher(kiskadee.fetchers.debian.Fetcher()) - self.queues.enqueue_project(self.pkg1) - self.queues.enqueue_project(self.pkg3) - - _pkg = self.queues.dequeue_project() - self.monitor.save_analyzed_project(_pkg) - - _pkg = self.queues.dequeue_project() - self.monitor.save_analyzed_project(_pkg) - - _pkgs = self.monitor.session.query(Package).all() - self.assertEqual(len(_pkgs), 1) - - _pkg_versions = self.monitor._query(Package).\ - filter(Package.name == _pkg['name']).first().versions - - _first_version = _pkg_versions[0].number - _current_version = _pkg_versions[-1].number - self.assertEqual(self.pkg1['version'], _first_version) - self.assertEqual(_pkg['version'], _current_version) - - def test_save_package_anitya(self): - self.monitor._save_fetcher(kiskadee.fetchers.anitya.Fetcher()) - self.queues.enqueue_project(self.pkg4) - _pkg = self.queues.dequeue_project() - self.monitor.save_analyzed_project(_pkg) - _pkgs = self.monitor.session.query(Package).all() - self.assertEqual(len(_pkgs), 1) - self.assertEqual(_pkgs[0].homepage, _pkg['meta']['homepage']) - - -if __name__ == '__main__': - unittest.main() + self.monitor.send_project_to_runner(self.data1) + self.assertEqual(self.monitor.kiskadee_queue.dequeue_analysis(), self.data1) + + @unittest.skip("will implement") + def test_receive_analysis_from_runner(self): + pass + + @unittest.skip("will implement") + def test_save_analysed_runner_project(self): + pass + + if __name__ == '__main__': + unittest.main() From 58aa9b1ef1b985b04534b144a8b4bf9554fa7661 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Apr 08 2018 20:51:00 +0000 Subject: [PATCH 10/18] WIP: Refactoring runner code. --- diff --git a/kiskadee/fetchers/__init__.py b/kiskadee/fetchers/__init__.py index 29b5cf3..b61511b 100644 --- a/kiskadee/fetchers/__init__.py +++ b/kiskadee/fetchers/__init__.py @@ -93,6 +93,9 @@ class Fetcher(abc.ABC): download_path = os.path.join(path, file_name) with open(download_path, 'wb') as info: info.write(data) + kiskadee.logger.debug( + "Downloaded {} source in {} path".format(file_name, path) + ) return download_path except Exception as err: kiskadee.logger.debug( diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py index 12eaad1..9490b2a 100644 --- a/kiskadee/monitor.py +++ b/kiskadee/monitor.py @@ -22,41 +22,35 @@ class Monitor: def __init__(self, _session, queues): """Return a non initialized Monitor.""" self.session = _session - self.kiskadee_queue = queues + self.queues = queues def start(self): - kiskadee.logger.debug('kiskadee PID: {}'.format(os.getppid())) - kiskadee.logger.debug('Starting monitor subprocess') - kiskadee.logger.debug('monitor PID: {}'.format(os.getpid())) + kiskadee.logger.debug('Monitor PID: {}'.format(os.getpid())) for fetcher in kiskadee.load_fetchers(): thread = Monitor._start_fetcher(fetcher.Fetcher().watch) while RUNNING: - new_project = self.dequeue_from_fetchers() + new_project = self.dequeue_project_from_fetchers() self.send_project_to_runner(new_project) analyzed_project = self.dequeue_analysis_from_runner() self.save_analyzed_project(analyzed_project) def dequeue_project_from_fetchers(self): - new_project = self.kiskadee_queue.dequeue_project() + new_project = self.queues.dequeue_project() return new_project if new_project else {} def dequeue_analysis_from_runner(self): - return self.kiskadee_queue.dequeue_result() + return self.queues.dequeue_result() def send_project_to_runner(self, data): if data: fetcher, project = self.get_fetcher_and_project(data) data["fetcher_id"] = fetcher.id if fetcher else '' if not project: - kiskadee.logger.debug( - "MONITOR: Sending project {}_{} " - " for analysis".format(data['name'], data['version']) - ) - self.kiskadee_queue.enqueue_analysis(data) - elif self.not_analyzed_project_version(project, data): - self.kiskadee_queue.enqueue_analysis(data) + self.queues.enqueue_analysis(data) + elif self.is_a_new_project_version(project, data): + self.queues.enqueue_analysis(data) def get_fetcher_and_project(self, data): fetcher_name = data['fetcher'].split('.')[-1] @@ -65,7 +59,7 @@ class Monitor: project = self._query(Package).filter_by(name = project_name).first() return fetcher, project - def not_analyzed_project_version(self, project, data): + def is_a_new_project_version(self, project, data): project_version = data['version'] analysed_version = project.versions[-1].number fetcher = importlib.import_module(data['fetcher']).Fetcher() @@ -102,13 +96,12 @@ def daemon(): queues = kiskadee.queue.Queues() session = kiskadee.database.Database().session monitor = Monitor(session, queues) - runner = Runner() + runner = Runner(session, queues) monitor_process = Process( target=monitor.start, ) runner_process = Process( target=runner.runner, - args=(queues,) ) monitor_process.start() runner_process.start() diff --git a/kiskadee/queue.py b/kiskadee/queue.py index 959be72..672958a 100644 --- a/kiskadee/queue.py +++ b/kiskadee/queue.py @@ -42,17 +42,33 @@ class Queues(): @staticmethod def enqueue_analysis(project_to_analysis): """Put a analysis on the analysis queue.""" + log_msg = "Sending project {}_{} for analysis"\ + .format(project_to_analysis['name'], + project_to_analysis['version']) + kiskadee.logger.debug(log_msg) analysis.put(project_to_analysis) @staticmethod def dequeue_analysis(): """Get a analysis from the analysis queue.""" - return analysis.get() + project_to_analysis = analysis.get() + kiskadee.logger.debug( + 'RUNNER: deqeued {}-{} from {}' + .format(project_to_analysis['name'], + project_to_analysis['version'], + project_to_analysis['fetcher']) + ) + return project_to_analysis @staticmethod - def enqueue_result(result): + def enqueue_result(project): """Put a result on the results queue.""" - results.put(result) + kiskadee.logger.debug( + "RUNNER: Sending {}-{} to Monitor" + .format(project["name"], + project["version"]) + ) + results.put(project) @staticmethod @empty_queue diff --git a/kiskadee/runner.py b/kiskadee/runner.py index 8fd8de7..607fdef 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -16,9 +16,12 @@ RUNNING = True class Runner: """Provide kiskadee runner objects.""" - def __init__(self): + def __init__(self, session, queues): """Return a non initialized Runner.""" - self.queues = None + self.queues = queues + self.session = session + kiskadee.model.Analyzer.create_analyzers(self.session) + self.fetcher = None def runner(self, queues): """Run static analyzers. @@ -27,52 +30,69 @@ class Runner: :func:`analyze` method, passing the dequeued package. After the analysis, updates the status of this package on the database. """ - kiskadee.logger.debug('Starting runner subprocess') - kiskadee.logger.debug('runner PID: {}'.format(os.getpid())) - session = kiskadee.database.Database().session - kiskadee.model.Analyzer.create_analyzers(session) - self.queues = queues + kiskadee.logger.debug('Runner PID: {}'.format(os.getpid())) while RUNNING: kiskadee.logger.debug('RUNNER: Waiting to dequeue'\ ' project to analysis...') - source_to_analysis = self.queues.dequeue_analysis() - kiskadee.logger.debug( - 'RUNNER: deqeued {}-{} from {}' - .format(source_to_analysis['name'], - source_to_analysis['version'], - source_to_analysis['fetcher']) + project_to_analysis = self.queues.dequeue_analysis() + self.call_analyzers(project_to_analysis) + + def load_project_fetcher(self, project_to_analysis): + try: + return importlib.import_module( + project_to_analysis['fetcher'] + ).Fetcher() + except ModuleNotFoundError: + kiskadee.logger.debug("Fetcher {} could not be loaded"\ + .format(project_to_analysis['fetcher']) ) - self.call_analyzers(source_to_analysis) + return {} + + def uncompress_project_code(self, compressed_source): + dir_to_unpack_source = tempfile.mkdtemp() + try: + shutil.unpack_archive(compressed_source, + dir_to_unpack_source) + return dir_to_unpack_source + except Exception as err: + kiskadee.logger.debug('Could not unpack project source') + kiskadee.logger.debug(err) + return {} + + def run_analysis(self, analyzers, project_to_analysis): + project_to_analysis['results'] = {} + for analyzer in analyzers: + firehose_report = self.analyze(analyzer, source_path) + if firehose_report: + project_to_analysis['results'][analyzer] = firehose_report + return project_to_analysis - def call_analyzers(self, source_to_analysis): + + def clean_temporary_directory(self, dir): + # not delete the source code used on tests. + if not compressed_source_path .find("kiskadee/tests") > -1: + shutil.rmtree(os.path.dirname(dir)) + + def call_analyzers(self, project_to_analysis): """Iterate over the package analyzers. For each analyzer defined to analysis the source, call the function :func:`analyze`, passing the source dict, the analyzer to run the analysis, and the path to a compressed source. """ - fetcher = importlib.import_module(source_to_analysis['fetcher']).Fetcher() - source_path = self._path_to_uncompressed_source( - source_to_analysis, fetcher - ) + self.fetcher = self.load_project_fetcher(project_to_analysis) + source_path = self.prepare_to_get_project_code(project_to_analysis) if not source_path: return None analyzers = fetcher.analyzers() - source_to_analysis['results'] = {} - for analyzer in analyzers: - firehose_report = self.analyze(analyzer, source_path) - if firehose_report: - source_to_analysis['results'][analyzer] = firehose_report + analysis_result = self.run_analysis(analyzers, project_to_analysis) + self.enqueue_analysis_to_monitor(analysis_result) + self.clean_temporary_directory(source_path) - if source_to_analysis['results']: - kiskadee.logger.debug( - "RUNNER: Sending {}-{} to Monitor" - .format(source_to_analysis["name"], - source_to_analysis["version"]) - ) - self.queues.enqueue_result(source_to_analysis) - shutil.rmtree(source_path) + def enqueue_analysis_to_monitor(self, analysis_result): + if analysis_result['results']: + self.queues.enqueue_result(analysis_result) def analyze(self, analyzer, source_path): """Run each analyzer on some sorce code. @@ -101,52 +121,18 @@ class Runner: kiskadee.logger.debug(err) return None - def _path_to_uncompressed_source(self, package, fetcher): + def prepare_to_get_project_code(self, project): - if not (fetcher and package): + if not (self.fetcher and project): return None - kiskadee.logger.debug( - 'ANALYSIS: Downloading {} ' - 'source...'.format(package['name']) - ) - - compressed_source = fetcher.get_sources(package) - - if compressed_source: - kiskadee.logger.debug( - 'ANALYSIS: Downloaded {} source in {} path' - .format( - package['name'], - os.path.dirname(compressed_source) - ) - ) - tmp_path = tempfile.gettempdir() - uncompressed_source_path = tempfile.mkdtemp(dir=tmp_path) - try: - shutil.unpack_archive( - compressed_source, - uncompressed_source_path + compressed_source_path = self.fetcher.get_sources(project) + if compressed_source_path : + uncompressed_source_path = uncompress_project_code( + compressed_source_path ) - kiskadee.logger.debug( - 'ANALYSIS: Unpacking {} source in {} path' - .format(package['name'], uncompressed_source_path) - ) - # not delete the source code used on tests. - if not compressed_source.find("kiskadee/tests") > -1: - shutil.rmtree(os.path.dirname(compressed_source)) - kiskadee.logger.debug( - 'ANALYSIS: Remove {} temp directory' - .format(os.path.dirname(compressed_source)) - ) - kiskadee.logger.debug( - 'ANALYSIS: Unpacked {} source'.format(package['name']) - ) - return uncompressed_source_path - except Exception as err: - kiskadee.logger.debug('Something went wrong') - kiskadee.logger.debug(err) - return None + self.clean_temporary_directory(os.path.dirname(compressed_source_path)) + return uncompressed_source_path else: kiskadee.logger.debug('RUNNER: invalid compressed source') return None diff --git a/kiskadee/tests/integration/test_monitor.py b/kiskadee/tests/integration/test_monitor.py index eae9829..3018563 100644 --- a/kiskadee/tests/integration/test_monitor.py +++ b/kiskadee/tests/integration/test_monitor.py @@ -40,46 +40,6 @@ class MonitorTestCase(unittest.TestCase): 'flawfinder': '><'}, 'fetcher_id': 1} - self.data2 = {'name': 'urlscan', - 'version': '0.8.2', - 'fetcher': kiskadee.fetchers.debian.__name__, - 'meta': {'directory': 'pool/main/u/urlscan'}, - 'results': { - 'cppcheck': '<>', - 'flawfinder': '><'}, - 'fetcher_id': 1} - - self.data3 = {'name': 'curl', - 'version': '7.52.2-5', - 'fetcher': kiskadee.fetchers.debian.__name__, - 'meta': {'directory': 'pool/main/c/curl'}, - 'results': { - 'cppcheck': '<>', - 'flawfinder': '><'}, - 'fetcher_id': 1} - self.data4 = {'name': 'urlanitya', - 'version': '0.11', - 'fetcher': kiskadee.fetchers.anitya.__name__, - 'meta': { - 'backend': 'gitHub', - 'homepage': 'https://github.com/GesielFreitas/Cros' - }, - 'results': { - 'cppcheck': '<>', - 'flawfinder': '><'}, - 'fetcher_id': 1} - self.analysis = { - 'analyzer_id': 1, - 'id': 1, - 'raw': { - 'results': [ - {'severity': 'warning'}, - {'severity': 'style'}, - {'severity': 'error'} - ] - } - } - def test_dequeue_project_from_fetcher(self): self.example_fetcher.watch() monitored_project = self.monitor.dequeue_project_from_fetchers() @@ -107,19 +67,12 @@ class MonitorTestCase(unittest.TestCase): self.monitor.get_fetcher_and_project = MagicMock( return_value=[fetcher, {}] ) - self.monitor.not_analyzed_project_version = MagicMock( + self.monitor.is_a_new_project_version = MagicMock( return_value=True ) self.monitor.send_project_to_runner(self.data1) - self.assertEqual(self.monitor.kiskadee_queue.dequeue_analysis(), self.data1) - - @unittest.skip("will implement") - def test_receive_analysis_from_runner(self): - pass - - @unittest.skip("will implement") - def test_save_analysed_runner_project(self): - pass + self.assertEqual(self.monitor.queues.dequeue_analysis(), + self.data1) if __name__ == '__main__': unittest.main() From 1174704222256d02e5d1fbc663f29f283b844bc8 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Apr 08 2018 20:51:00 +0000 Subject: [PATCH 11/18] reimplement runner integration tests --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index e39e133..f484bc4 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -46,14 +46,10 @@ def package_analysis_overview(pkg_name, version): db_session = kiskadee_db_session() #TODO: This can be a simple inner join between package, version and analysis - package_id = ( - db_session.query(Package).filter_by(name = pkg_name).id - ) - version_id = ( - db_session.query(Version) - .filter(Version.number == version) + package_id = db_session.query(Package).filter_by(name = pkg_name).id + version_id = db_session.query(Version)\ + .filter(Version.number == version)\ .filter(Version.package_id == package_id).first().id - ) analysis = ( db_session.query(Analysis) .options( @@ -62,8 +58,6 @@ def package_analysis_overview(pkg_name, version): .filter(Analysis.version_id == version_id) .all() ) - kiskadee.logger.debug("haaaaaaa") - kiskadee.logger.debug(analysis) analysis_schema = AnalysisSchema(many=True, exclude=['raw', 'report']) data, errors = analysis_schema.dump(analysis) return jsonify(data) @@ -76,10 +70,7 @@ def package_analysis_overview(pkg_name, version): def analysis_results(pkg_name, version, analysis_id): """Get the analysis results from a specific analyzer.""" db_session = kiskadee_db_session() - analysis = ( - db_session.query(Analysis) - .get(analysis_id) - ) + analysis = db_session.query(Analysis).get(analysis_id) analysis_schema = AnalysisSchema(only=['raw']) data, errors = analysis_schema.dump(analysis) response = data['raw']['results'] @@ -93,26 +84,20 @@ def analysis_results(pkg_name, version, analysis_id): def analysis_reports(pkg_name, version, analysis_id): """Get the analysis reports from a specific analyzer.""" db_session = kiskadee_db_session() - analysis = ( - db_session.query(Analysis) - .get(analysis_id) - ) - analysis_schema = AnalysisSchema(only=['report']) - data, errors = analysis_schema.dump(analysis) + analysis = db_session.query(Analysis).get(analysis_id) + data, errors = AnalysisSchema(only=['report']).dump(analysis) report = data['report'] - if (report is not None) and\ - ('results' in report.keys()) and\ - report['results'] is not None: - report['results'] = json\ - .loads(report['results']) - return jsonify({'analysis_report': report}) + try: + report['results'] = json.loads(report['results']) + return jsonify({'analysis_report': report}) + except Exception as err: + return jsonify({'analysis_report': {}}) def kiskadee_db_session(): """Return a kiskadee database session.""" return Database().session - def main(): """Initialize the kiskadee API.""" kiskadee.run('0.0.0.0') diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py index 9490b2a..0fc6755 100644 --- a/kiskadee/monitor.py +++ b/kiskadee/monitor.py @@ -23,6 +23,7 @@ class Monitor: """Return a non initialized Monitor.""" self.session = _session self.queues = queues + kiskadee.model.Analyzer.create_analyzers(self.session) def start(self): kiskadee.logger.debug('Monitor PID: {}'.format(os.getpid())) @@ -96,7 +97,7 @@ def daemon(): queues = kiskadee.queue.Queues() session = kiskadee.database.Database().session monitor = Monitor(session, queues) - runner = Runner(session, queues) + runner = Runner(queues) monitor_process = Process( target=monitor.start, ) diff --git a/kiskadee/runner.py b/kiskadee/runner.py index 607fdef..dfe02a0 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -16,12 +16,11 @@ RUNNING = True class Runner: """Provide kiskadee runner objects.""" - def __init__(self, session, queues): + def __init__(self, queues): """Return a non initialized Runner.""" self.queues = queues - self.session = session - kiskadee.model.Analyzer.create_analyzers(self.session) self.fetcher = None + self.project = None def runner(self, queues): """Run static analyzers. @@ -34,59 +33,50 @@ class Runner: while RUNNING: kiskadee.logger.debug('RUNNER: Waiting to dequeue'\ ' project to analysis...') - project_to_analysis = self.queues.dequeue_analysis() - self.call_analyzers(project_to_analysis) + self.project = self.queues.dequeue_analysis() + self.call_analyzers() - def load_project_fetcher(self, project_to_analysis): + def import_project_fetcher(self): try: - return importlib.import_module( - project_to_analysis['fetcher'] - ).Fetcher() + return importlib.import_module( + self.project['fetcher'] + ).Fetcher() except ModuleNotFoundError: kiskadee.logger.debug("Fetcher {} could not be loaded"\ - .format(project_to_analysis['fetcher']) + .format(self.project['fetcher']) ) return {} - def uncompress_project_code(self, compressed_source): - dir_to_unpack_source = tempfile.mkdtemp() - try: - shutil.unpack_archive(compressed_source, - dir_to_unpack_source) - return dir_to_unpack_source - except Exception as err: - kiskadee.logger.debug('Could not unpack project source') - kiskadee.logger.debug(err) - return {} - - def run_analysis(self, analyzers, project_to_analysis): - project_to_analysis['results'] = {} + def run_analysis(self, analyzers, source_path): + self.project['results'] = {} for analyzer in analyzers: firehose_report = self.analyze(analyzer, source_path) if firehose_report: - project_to_analysis['results'][analyzer] = firehose_report - return project_to_analysis + self.project['results'][analyzer] = firehose_report + return self.project - def clean_temporary_directory(self, dir): + def clean_temporary_directory(self, temp_dir): # not delete the source code used on tests. - if not compressed_source_path .find("kiskadee/tests") > -1: - shutil.rmtree(os.path.dirname(dir)) + if not temp_dir.find("kiskadee/tests") > -1: + shutil.rmtree(temp_dir) - def call_analyzers(self, project_to_analysis): + def call_analyzers(self): """Iterate over the package analyzers. For each analyzer defined to analysis the source, call the function :func:`analyze`, passing the source dict, the analyzer to run the analysis, and the path to a compressed source. """ - self.fetcher = self.load_project_fetcher(project_to_analysis) - source_path = self.prepare_to_get_project_code(project_to_analysis) + self.fetcher = self.import_project_fetcher() + source_path = self.get_project_code_path() if not source_path: return None - analyzers = fetcher.analyzers() - analysis_result = self.run_analysis(analyzers, project_to_analysis) + analyzers = self.fetcher.analyzers() + analysis_result = self.run_analysis( + analyzers, source_path + ) self.enqueue_analysis_to_monitor(analysis_result) self.clean_temporary_directory(source_path) @@ -121,14 +111,14 @@ class Runner: kiskadee.logger.debug(err) return None - def prepare_to_get_project_code(self, project): + def get_project_code_path(self): - if not (self.fetcher and project): + if not (self.fetcher and self.project): return None - compressed_source_path = self.fetcher.get_sources(project) + compressed_source_path = self.fetcher.get_sources(self.project) if compressed_source_path : - uncompressed_source_path = uncompress_project_code( + uncompressed_source_path = self.uncompress_project_code( compressed_source_path ) self.clean_temporary_directory(os.path.dirname(compressed_source_path)) @@ -136,3 +126,15 @@ class Runner: else: kiskadee.logger.debug('RUNNER: invalid compressed source') return None + + def uncompress_project_code(self, compressed_source): + dir_to_unpack_source = tempfile.mkdtemp() + try: + shutil.unpack_archive(compressed_source, + dir_to_unpack_source) + return dir_to_unpack_source + except Exception as err: + kiskadee.logger.debug('Could not unpack project source') + kiskadee.logger.debug(err) + return {} + diff --git a/kiskadee/tests/api/test_api.py b/kiskadee/tests/api/test_api.py index 7c8d021..1ef5b8a 100644 --- a/kiskadee/tests/api/test_api.py +++ b/kiskadee/tests/api/test_api.py @@ -17,6 +17,7 @@ class ApiTestCase(unittest.TestCase): def setUp(self): self.fetcher = kiskadee.model.Fetcher(name='kiskadee-fetcher') + self.queues = kiskadee.queue.Queues() def test_get_fetchers(self): kiskadee.api.app.kiskadee_db_session = MagicMock() @@ -39,15 +40,8 @@ class ApiTestCase(unittest.TestCase): raw='analysis result: foo' ) - def side_effect(klass): - mocked_objects = { - 'Package': package, - 'Version': version, - } - return mocked_objects[klass.__name__] - kiskadee.api.app.kiskadee_db_session = MagicMock() - Monitor(kiskadee.api.app.kiskadee_db_session()) + Monitor(kiskadee.api.app.kiskadee_db_session(), self.queues) db_session = kiskadee.api.app.kiskadee_db_session() db_session.query(kiskadee.model.Package)\ .filter_by().id = MagicMock() diff --git a/kiskadee/tests/integration/test_monitor.py b/kiskadee/tests/integration/test_monitor.py index 3018563..2369499 100644 --- a/kiskadee/tests/integration/test_monitor.py +++ b/kiskadee/tests/integration/test_monitor.py @@ -27,7 +27,6 @@ class MonitorTestCase(unittest.TestCase): queues = Queues() self.monitor = Monitor(self.session, queues) - self.session.query = MagicMock(side_effect=mocked_models) self.data1 = { diff --git a/kiskadee/tests/integration/test_runner.py b/kiskadee/tests/integration/test_runner.py index d2f08df..3cd646f 100644 --- a/kiskadee/tests/integration/test_runner.py +++ b/kiskadee/tests/integration/test_runner.py @@ -4,97 +4,54 @@ by the user that is executing the tests. It also needs selinux running in a permissive mode (setenforce 0) """ import unittest -import tempfile +import re +from unittest.mock import MagicMock -from kiskadee.runner import Runner +import kiskadee.queue +import kiskadee.runner +import kiskadee.monitor import kiskadee.fetchers.example -import kiskadee.fetchers.debian -from sqlalchemy.orm import sessionmaker -from kiskadee import model -from kiskadee.queue import Queues -from kiskadee.database import Database +import kiskadee.analyzers +class RunnerTestCase(unittest.TestCase): -class AnalyzersTestCase(unittest.TestCase): - def setUp(self): - self.engine = Database('db_test').engine - Session = sessionmaker(bind=self.engine) - self.session = Session() - model.Base.metadata.create_all(self.engine) - model.Analyzer.create_analyzers(self.session) - self.fetcher = kiskadee.fetchers.debian.Fetcher() - self.deb_pkg = {'name': 'test', - 'version': '1.0.0', - 'fetcher': kiskadee.fetchers.debian.__name__ - } - self.fetcher = model.Fetcher( - name='kiskadee-fetcher2', target='university' - ) - self.session.add(self.fetcher) - self.session.commit() - self.runner = Runner() - self.runner.queues = Queues() - - def tearDown(self): - self.session.close() - model.Base.metadata.drop_all() - - def test_run_analyzer(self): - - source_to_analysis = { - 'name': 'test', - 'version': '1.0.0', - 'fetcher': kiskadee.fetchers.example.Fetcher() - } + """Docstring for AnalyzersTestCase. """ - source_path = self.runner._path_to_uncompressed_source( - source_to_analysis, kiskadee.fetchers.example.Fetcher() - ) - firehose_report = self.runner.analyze("cppcheck", source_path) - self.assertIsNotNone(firehose_report) + def setUp(self): + self.queues = kiskadee.queue.Queues() + self.runner = kiskadee.runner.Runner(self.queues) - def test_generate_a_firehose_report(self): - source_to_analysis = { + self.project = { 'name': 'test', 'version': '1.0.0', 'fetcher': kiskadee.fetchers.example.__name__ } - self.runner.call_analyzers(source_to_analysis) - analyzed_pkg = self.runner.queues.dequeue_result() - self.assertEqual(analyzed_pkg['name'], source_to_analysis['name']) - self.assertIn('cppcheck', analyzed_pkg['results']) - self.assertIn('flawfinder', analyzed_pkg['results']) - - def test_path_to_uncompressed_source(self): - - source_to_analysis = { - 'name': 'test', - 'version': '1.0.0', - 'fetcher': kiskadee.fetchers.example - } - - source_path = self.runner._path_to_uncompressed_source( - source_to_analysis, kiskadee.fetchers.example.Fetcher() - ) - tmp_path = tempfile.gettempdir() - self.assertTrue(source_path.find(tmp_path) >= 0) - self.assertIsNotNone(source_path) - - def test_invalid_path_to_uncompressed_source(self): - - source_to_analysis = { - 'name': 'test', - 'version': '1.0.0', - 'fetcher': kiskadee.fetchers.example - } - - source_path = self.runner._path_to_uncompressed_source( - source_to_analysis, None - ) - - self.assertIsNone(source_path) - + def tearDown(self): + """TODO: to be defined1. """ + + def test_run_analyzers(self): + self.runner.project = self.project + self.runner.call_analyzers() + result = self.queues.dequeue_result() + self.assertEqual(result['name'], 'test') + self.assertIn('results', result) + + def test_run_a_single_analyzer(self): + self.runner.project = self.project + self.runner.fetcher = self.runner.import_project_fetcher() + source_path = self.runner.get_project_code_path() + analyzer = self.runner.fetcher.analyzers()[0] + self.assertIsNotNone(re.search('.*cppcheck.*', analyzer)) + self.assertIsNone(re.search('.*pylint.*', analyzer)) + + def test_analysis_an_incoming_monitor_project(self): + self.session = MagicMock() + self.session.query().filter_by().first = MagicMock(return_value={}) + self.monitor = kiskadee.monitor.Monitor(self.session, self.queues) + self.monitor.send_project_to_runner(self.project) + incoming_project = self.runner.queues.dequeue_analysis() + self.assertEqual(self.project, incoming_project) if __name__ == '__main__': unittest.main() From 5b21d26cf6f3c8c9e121320b3c37fd22e391111e Mon Sep 17 00:00:00 2001 From: David Carlos Date: Apr 08 2018 20:51:00 +0000 Subject: [PATCH 12/18] Fix remaining tests. --- diff --git a/Makefile b/Makefile index cffabee..fcc2240 100644 --- a/Makefile +++ b/Makefile @@ -2,22 +2,22 @@ SHELL:=/bin/bash help: - @printf "Available targets: check, analyzers, clean\n\n" + @printf "Available targets: test, analyzers, clean\n\n" -check: +test: coverage run --omit="lib/*","setup.py","kiskadee/tests/*",".eggs/*",".venv/*","/usr/*" -m unittest kiskadee/tests/*/*.py coverage html -check_units: +test_units: python3 -m unittest kiskadee/tests/units/*.py -check_integration: +test_integration: python3 -m unittest kiskadee/tests/integration/*.py -check_api: +test_api: python3 -m unittest kiskadee/tests/api/*.py -check_plugins: +test_plugins: python3 -m unittest kiskadee/tests/plugins/*.py analyzers: diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index f484bc4..9eb7b3a 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -14,13 +14,13 @@ kiskadee = Flask(__name__) CORS(kiskadee) +db = Database() @kiskadee.route('/fetchers') def index(): """Get the list of available fetchers.""" if request.method == 'GET': - db_session = kiskadee_db_session() - fetchers = db_session.query(Fetcher).all() + fetchers = db.session.query(Fetcher).all() fetcher_schema = FetcherSchema(many=True) result = fetcher_schema.dump(fetchers) return jsonify({'fetchers': result.data}) @@ -30,8 +30,7 @@ def index(): def packages(): """Get the list of analyzed packages.""" if request.method == 'GET': - db_session = kiskadee_db_session() - packages = db_session.query(Package).all() + packages = db.session.query(Package).all() package_schema = PackageSchema( many=True, exclude=['versions.analysis', 'versions.package_id'] @@ -42,16 +41,14 @@ def packages(): @kiskadee.route('/analysis//', methods=['GET']) def package_analysis_overview(pkg_name, version): - """Get the a analysis list of some package version.""" - db_session = kiskadee_db_session() + """Get a analysis list of some package version.""" #TODO: This can be a simple inner join between package, version and analysis - package_id = db_session.query(Package).filter_by(name = pkg_name).id - version_id = db_session.query(Version)\ - .filter(Version.number == version)\ - .filter(Version.package_id == package_id).first().id + _package_id = db.filter_by_name(Package, pkg_name).id + version_id = db.session.query(Version)\ + .filter_by(number = version, package_id = _package_id ).first().id analysis = ( - db_session.query(Analysis) + db.session.query(Analysis) .options( eagerload(Analysis.analyzers, innerjoin=True) ) @@ -69,8 +66,7 @@ def package_analysis_overview(pkg_name, version): ) def analysis_results(pkg_name, version, analysis_id): """Get the analysis results from a specific analyzer.""" - db_session = kiskadee_db_session() - analysis = db_session.query(Analysis).get(analysis_id) + analysis = db.get(Analysis, analysis_id) analysis_schema = AnalysisSchema(only=['raw']) data, errors = analysis_schema.dump(analysis) response = data['raw']['results'] @@ -83,8 +79,7 @@ def analysis_results(pkg_name, version, analysis_id): ) def analysis_reports(pkg_name, version, analysis_id): """Get the analysis reports from a specific analyzer.""" - db_session = kiskadee_db_session() - analysis = db_session.query(Analysis).get(analysis_id) + analysis = db.get(Analysis, analysis_id) data, errors = AnalysisSchema(only=['report']).dump(analysis) report = data['report'] try: @@ -93,11 +88,6 @@ def analysis_reports(pkg_name, version, analysis_id): except Exception as err: return jsonify({'analysis_report': {}}) - -def kiskadee_db_session(): - """Return a kiskadee database session.""" - return Database().session - def main(): """Initialize the kiskadee API.""" kiskadee.run('0.0.0.0') diff --git a/kiskadee/database.py b/kiskadee/database.py index 945a601..96cefe4 100644 --- a/kiskadee/database.py +++ b/kiskadee/database.py @@ -1,9 +1,10 @@ """Provide kiskadee Database operations.""" -import kiskadee -from kiskadee.model import Base from sqlalchemy import create_engine, orm +import kiskadee +from kiskadee.model import Base +from kiskadee.model import Package, Fetcher, Version, Report, Analysis class Database: """kiskadee Database class.""" @@ -22,6 +23,11 @@ class Database: DBSession = orm.sessionmaker(bind=engine) return DBSession() + def filter_by_name(self, model, _name): + return self.session.query(model).filter_by(name = _name).first() + + def get(self, model, id): + return self.session.query(model).get(id) def get_database_uri(db): """Return the Database URI of the current session.""" diff --git a/kiskadee/model.py b/kiskadee/model.py index 4e4dff2..acbd5a0 100644 --- a/kiskadee/model.py +++ b/kiskadee/model.py @@ -31,7 +31,7 @@ class Package(Base): ) @staticmethod - def save(session, data): + def save(db, data): homepage = None if ('meta' in data) and ('homepage' in data['meta']): homepage = data['meta']['homepage'] @@ -41,16 +41,16 @@ class Package(Base): homepage=homepage, fetcher_id=data['fetcher_id'] ) - session.add(_package) - session.commit() + db.session.add(_package) + db.session.commit() _version = Version(number=data['version'], package_id=_package.id) - session.add(_version) - session.commit() + db.session.add(_version) + db.session.commit() return _package @staticmethod - def update(session, project, data): + def update(db, project, data): if(project.versions[-1].number == data['version']): return project @@ -60,8 +60,8 @@ class Package(Base): project_id=project.id ) project.versions.append(_new_version) - session.add(project) - session.commit() + db.session.add(project) + db.session.commit() kiskadee.logger.debug( "MONITOR: Sending project {}_{}" "for analysis".format(data['name'], data['version']) @@ -109,7 +109,7 @@ class Analyzer(Base): analysis = orm.relationship('Analysis', backref='analyzers') @staticmethod - def create_analyzers(_session): + def create_analyzers(db): """Create the analyzers on database. The kiskadee analyzers are defined on the section `analyzers` of the @@ -118,13 +118,13 @@ class Analyzer(Base): """ list_of_analyzers = dict(kiskadee.config._sections["analyzers"]) for _name, _version in list_of_analyzers.items(): - if not _session.query(Analyzer)\ + if not db.session.query(Analyzer)\ .filter_by(name = _name, version = _version )\ .first(): new_analyzer = kiskadee.model.Analyzer(name = _name, version = _version) - _session.add(new_analyzer) - _session.commit() + db.session.add(new_analyzer) + db.session.commit() class Analysis(Base): """Abstraction of a package analysis.""" @@ -139,21 +139,21 @@ class Analysis(Base): uselist=False, back_populates='analysis') @staticmethod - def save(session, data, analyzer, result, version): + def save(db, data, analyzer, result, version): _analysis = kiskadee.model.Analysis() try: - _analyzer = session.query(kiskadee.model.Analyzer).\ + _analyzer = db.session.query(kiskadee.model.Analyzer).\ filter_by(name = analyzer).first() _analysis.analyzer_id = _analyzer.id _analysis.version_id = version.id _analysis.raw = json.loads(result) - session.add(_analysis) - session.commit() + db.session.add(_analysis) + db.session.commit() dict_analysis = { 'results': _analysis.raw['results'], 'id': _analysis.id } - Report.save(session, dict_analysis, data, _analyzer.name) + Report.save(db, dict_analysis, data, _analyzer.name) kiskadee.logger.debug( "MONITOR: Saved analysis done by {} for package: {}-{}" .format(analyzer, data["name"], data["version"]) @@ -184,7 +184,7 @@ class Report(Base): analysis = orm.relationship('Analysis', back_populates='report') @staticmethod - def save(session, analysis, data, analyzer_name): + def save(db, analysis, data, analyzer_name): try: results = analysis['results'] analyzer_report = Report.REPORTERS[analyzer_name](results) @@ -194,8 +194,8 @@ class Report(Base): ._compute_reports(analyzer_name) ) _reports.analysis_id = analysis['id'] - session.add(_reports) - session.commit() + db.session.add(_reports) + db.session.commit() kiskadee.logger.debug( "MONITOR: Saved analysis reports for {} package" .format(data["name"]) diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py index 0fc6755..7f485c2 100644 --- a/kiskadee/monitor.py +++ b/kiskadee/monitor.py @@ -19,11 +19,11 @@ RUNNING = True class Monitor: """Provide kiskadee monitoring objects.""" - def __init__(self, _session, queues): + def __init__(self, db, queues): """Return a non initialized Monitor.""" - self.session = _session + self.db = db self.queues = queues - kiskadee.model.Analyzer.create_analyzers(self.session) + kiskadee.model.Analyzer.create_analyzers(self.db) def start(self): kiskadee.logger.debug('Monitor PID: {}'.format(os.getpid())) @@ -56,8 +56,8 @@ class Monitor: def get_fetcher_and_project(self, data): fetcher_name = data['fetcher'].split('.')[-1] project_name = data['name'] - fetcher = self._query(Fetcher).filter_by(name = fetcher_name).first() - project = self._query(Package).filter_by(name = project_name).first() + fetcher = self.db.filter_by_name(Fetcher, fetcher_name) + project = self.db.filter_by_name(Package, project_name) return fetcher, project def is_a_new_project_version(self, project, data): @@ -69,17 +69,14 @@ class Monitor: def save_analyzed_project(self, data): if not data: return {} - project = self._query(Package).filter_by(name = data['name']).first() + project = self.db.filter_by_name(Package, data['name']) if not project: - project = Package.save(self.session, data) + project = Package.save(self.db, data) if project: - project = Package.update(self.session, project, data) + project = Package.update(self.db, project, data) for analyzer, result in data['results'].items(): - Analysis.save(self.session, data, analyzer, result, project.versions[-1]) - - def _query(self, arg): - return self.session.query(arg) + Analysis.save(self.db, data, analyzer, result, project.versions[-1]) @staticmethod @@ -95,8 +92,8 @@ def daemon(): """Entry point to the monitor module.""" # TODO: improve with start/stop system queues = kiskadee.queue.Queues() - session = kiskadee.database.Database().session - monitor = Monitor(session, queues) + db = kiskadee.database.Database() + monitor = Monitor(db, queues) runner = Runner(queues) monitor_process = Process( target=monitor.start, diff --git a/kiskadee/runner.py b/kiskadee/runner.py index dfe02a0..0dee08a 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -22,7 +22,7 @@ class Runner: self.fetcher = None self.project = None - def runner(self, queues): + def runner(self): """Run static analyzers. Continuously dequeue packages from `analyses_queue` and call the diff --git a/kiskadee/tests/__init__.py b/kiskadee/tests/__init__.py index 2165cb9..296e655 100644 --- a/kiskadee/tests/__init__.py +++ b/kiskadee/tests/__init__.py @@ -1 +1,13 @@ """kiskadee test suite.""" +from kiskadee import model + + +def clean_test_db(db, metadata): + """Clean test database to run tests independently.""" + # The number of openend sessions is limited by the operational system. + # If we open too much connections, the database will lock the tests + # execution. + db.session.close_all() + metadata.drop_all(db.engine) + metadata.create_all(db.engine) + model.Analyzer.create_analyzers(db) diff --git a/kiskadee/tests/api/test_api.py b/kiskadee/tests/api/test_api.py index 1ef5b8a..cff7f1b 100644 --- a/kiskadee/tests/api/test_api.py +++ b/kiskadee/tests/api/test_api.py @@ -3,9 +3,9 @@ import unittest from unittest.mock import MagicMock import kiskadee -from kiskadee.monitor import Monitor import kiskadee.api.app import kiskadee.fetchers.example +import kiskadee.database class ApiTestCase(unittest.TestCase): @@ -14,16 +14,14 @@ class ApiTestCase(unittest.TestCase): def setUpClass(cls): kiskadee.api.app.kiskadee.testing = True cls.app = kiskadee.api.app.kiskadee.test_client() + cls.db = kiskadee.api.app.db = MagicMock() def setUp(self): self.fetcher = kiskadee.model.Fetcher(name='kiskadee-fetcher') self.queues = kiskadee.queue.Queues() def test_get_fetchers(self): - kiskadee.api.app.kiskadee_db_session = MagicMock() - kiskadee.api.app.kiskadee_db_session().query().all = MagicMock( - return_value=[self.fetcher] - ) + self.db.session.query().all = MagicMock(return_value=[self.fetcher]) response = self.app.get("/fetchers") response_data = json.loads(response.data.decode("utf-8")) self.assertIn("fetchers", response_data) @@ -33,23 +31,18 @@ class ApiTestCase(unittest.TestCase): ) def test_get_analysis_as_json(self): - package = kiskadee.model.Package(name='mocked-package', id=1) - version = kiskadee.model.Version(number='1.0.0', id=1, package_id=1) analysis = kiskadee.model.Analysis( id=1, analyzer_id=1, version_id=1, raw='analysis result: foo' ) - kiskadee.api.app.kiskadee_db_session = MagicMock() - Monitor(kiskadee.api.app.kiskadee_db_session(), self.queues) - db_session = kiskadee.api.app.kiskadee_db_session() - db_session.query(kiskadee.model.Package)\ - .filter_by().id = MagicMock() - - db_session.query(kiskadee.model.Version)\ - .filter_by().id = MagicMock() - db_session.query().options()\ - .filter().all = MagicMock(return_value=[analysis]) + self.db.filter_by_name = MagicMock() + self.db.session.query(kiskadee.model.Package)\ + .filter_by().id = MagicMock() + self.db.session.query(kiskadee.model.Version)\ + .filter_by().id = MagicMock() + self.db.session.query().options()\ + .filter().all = MagicMock(return_value=[analysis]) response = self.app.get("/analysis/mocked-package/1.0.0") response_data = json.loads(response.data.decode("utf-8")) @@ -61,10 +54,7 @@ class ApiTestCase(unittest.TestCase): id=1, analyzer_id=1, version_id=1, raw={'results': 'first analysis results'} ) - - kiskadee.api.app.kiskadee_db_session().query().get = MagicMock( - return_value=analysis - ) + self.db.get = MagicMock(return_value=analysis) response = self.app.get("/analysis/kiskadee-package/7.23/1/results") response_data = json.loads(response.data.decode("utf-8")) self.assertIn("analysis_results", response_data) diff --git a/kiskadee/tests/integration/test_monitor.py b/kiskadee/tests/integration/test_monitor.py index 2369499..9691ca2 100644 --- a/kiskadee/tests/integration/test_monitor.py +++ b/kiskadee/tests/integration/test_monitor.py @@ -1,16 +1,13 @@ import unittest from unittest.mock import MagicMock -from sqlalchemy.orm import sessionmaker -from kiskadee import model from kiskadee.monitor import Monitor from kiskadee.queue import Queues -from kiskadee.model import Package, Analyzer, Report, Analysis, Fetcher +from kiskadee.model import Package, Fetcher import kiskadee.queue import kiskadee.fetchers.debian import kiskadee.fetchers.anitya import kiskadee.fetchers.example -from kiskadee.database import Database class MonitorTestCase(unittest.TestCase): @@ -23,11 +20,10 @@ class MonitorTestCase(unittest.TestCase): klass() self.example_fetcher = kiskadee.fetchers.example.Fetcher() - self.session = MagicMock() + self.db = MagicMock() queues = Queues() - self.monitor = Monitor(self.session, queues) - self.session.query = MagicMock(side_effect=mocked_models) + self.monitor = Monitor(self.db, queues) self.data1 = { 'name': 'curl', @@ -60,9 +56,8 @@ class MonitorTestCase(unittest.TestCase): self.assertIn("fetcher", second_monitored_project) def test_send_project_to_runner(self): - fetcher = kiskadee.model.Fetcher(name='example') - project = kiskadee.model.Package(name='project1', - fetcher_id=fetcher.id) + fetcher = Fetcher(name='example') + Package(name='project1', fetcher_id=fetcher.id) self.monitor.get_fetcher_and_project = MagicMock( return_value=[fetcher, {}] ) @@ -70,8 +65,7 @@ class MonitorTestCase(unittest.TestCase): return_value=True ) self.monitor.send_project_to_runner(self.data1) - self.assertEqual(self.monitor.queues.dequeue_analysis(), - self.data1) + self.assertEqual(self.monitor.queues.dequeue_analysis(), self.data1) if __name__ == '__main__': unittest.main() diff --git a/kiskadee/tests/integration/test_runner.py b/kiskadee/tests/integration/test_runner.py index 3cd646f..f8ed25b 100644 --- a/kiskadee/tests/integration/test_runner.py +++ b/kiskadee/tests/integration/test_runner.py @@ -3,6 +3,7 @@ This integration tests needs docker engine be running and accessible by the user that is executing the tests. It also needs selinux running in a permissive mode (setenforce 0) """ + import unittest import re from unittest.mock import MagicMock @@ -13,6 +14,7 @@ import kiskadee.monitor import kiskadee.fetchers.example import kiskadee.analyzers + class RunnerTestCase(unittest.TestCase): """Docstring for AnalyzersTestCase. """ @@ -40,18 +42,19 @@ class RunnerTestCase(unittest.TestCase): def test_run_a_single_analyzer(self): self.runner.project = self.project self.runner.fetcher = self.runner.import_project_fetcher() - source_path = self.runner.get_project_code_path() + self.runner.get_project_code_path() analyzer = self.runner.fetcher.analyzers()[0] self.assertIsNotNone(re.search('.*cppcheck.*', analyzer)) self.assertIsNone(re.search('.*pylint.*', analyzer)) def test_analysis_an_incoming_monitor_project(self): - self.session = MagicMock() - self.session.query().filter_by().first = MagicMock(return_value={}) - self.monitor = kiskadee.monitor.Monitor(self.session, self.queues) + self.db = MagicMock() + self.db.filter_by_name = MagicMock(return_value={}) + self.monitor = kiskadee.monitor.Monitor(self.db, self.queues) self.monitor.send_project_to_runner(self.project) incoming_project = self.runner.queues.dequeue_analysis() self.assertEqual(self.project, incoming_project) + if __name__ == '__main__': unittest.main() diff --git a/kiskadee/tests/units/test_model.py b/kiskadee/tests/units/test_model.py index aba56e7..f542aaa 100644 --- a/kiskadee/tests/units/test_model.py +++ b/kiskadee/tests/units/test_model.py @@ -1,87 +1,85 @@ import unittest from sqlalchemy import exc -from sqlalchemy.orm import sessionmaker +import kiskadee from kiskadee import model from kiskadee.database import Database class ModelTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.db = Database('db_test') + def setUp(self): - self.engine = Database('db_test').engine - Session = sessionmaker(bind=self.engine) - self.session = Session() - model.Base.metadata.create_all(self.engine) - model.Analyzer.create_analyzers(self.session) + kiskadee.tests.clean_test_db(self.db, model.Base.metadata) + self.package = model.Package(name='python-kiskadee') + self.version = model.Version(number='1.0-rc1') self.fetcher = model.Fetcher( name='kiskadee-fetcher', target='university' ) - self.package = model.Package(name='python-kiskadee') - self.version = model.Version(number='1.0-rc1') + self.db.session.add(self.fetcher) + self.db.session.commit() self.fetcher.packages.append(self.package) self.package.versions.append(self.version) - self.session.add(self.package) - self.session.add(self.fetcher) - self.session.add(self.version) + self.db.session.add(self.package) + self.db.session.add(self.fetcher) + self.db.session.add(self.version) self.analysis = model.Analysis( analyzer_id=1, version_id=1, raw="" ) - self.session.add(self.analysis) + self.db.session.add(self.analysis) self.report = model.Report( analysis_id=1, ) - self.session.add(self.report) - self.session.commit() - - def tearDown(self): - self.session.close() - model.Base.metadata.drop_all() + self.db.session.add(self.report) + self.db.session.commit() def test_query_fetcher(self): - fetchers = self.session.query(model.Fetcher).all() + fetchers = self.db.session.query(model.Fetcher).all() self.assertEqual(fetchers, [self.fetcher]) def test_query_package(self): - packages = self.session.query(model.Package).all() + packages = self.db.session.query(model.Package).all() self.assertEqual(packages, [self.package]) def test_query_version(self): - versions = self.session.query(model.Version).all() + versions = self.db.session.query(model.Version).all() self.assertEqual(versions, [self.version]) def test_query_report(self): - reports = self.session.query(model.Report).all() + reports = self.db.session.query(model.Report).all() self.assertEqual(reports, [self.report]) def test_add_fetcher(self): - fetchers = self.session.query(model.Fetcher).all() + fetchers = self.db.session.query(model.Fetcher).all() self.assertEqual(len(fetchers), 1) - self.session.add(model.Fetcher(name='foo', target='bar')) - fetchers = self.session.query(model.Fetcher).all() + self.db.session.add(model.Fetcher(name='foo', target='bar')) + fetchers = self.db.session.query(model.Fetcher).all() self.assertEqual(len(fetchers), 2) def test_add_version_without_package(self): version = model.Version(number='3.1') - self.session.add(version) + self.db.session.add(version) with self.assertRaises(exc.IntegrityError): - self.session.commit() + self.db.session.commit() def test_add_report_without_analysis(self): report = model.Report( ) - self.session.add(report) + self.db.session.add(report) with self.assertRaises(exc.IntegrityError): - self.session.commit() + self.db.session.commit() def test_add_package_without_fetcher(self): package = model.Package(name='foo-bar') - self.session.add(package) + self.db.session.add(package) with self.assertRaises(exc.IntegrityError): - self.session.commit() + self.db.session.commit() def test_unique_package_in_fetcher(self): package_1 = model.Package(name='foo-bar') @@ -89,7 +87,7 @@ class ModelTestCase(unittest.TestCase): self.fetcher.packages.append(package_1) self.fetcher.packages.append(package_2) with self.assertRaises(exc.IntegrityError): - self.session.commit() + self.db.session.commit() def test_unique_version_for_package(self): package_version_1 = model.Version(number='1.0') @@ -97,10 +95,10 @@ class ModelTestCase(unittest.TestCase): self.package.versions.append(package_version_1) self.package.versions.append(package_version_2) with self.assertRaises(exc.IntegrityError): - self.session.commit() + self.db.session.commit() def test_compose_kiskadee_source(self): - _analyzer = self.session.query(model.Analyzer)\ + _analyzer = self.db.session.query(model.Analyzer)\ .filter(model.Analyzer.name == "cppcheck").first() package = model.Package( name='bla', @@ -126,11 +124,11 @@ class ModelTestCase(unittest.TestCase): def test_save_several_analysis(self): _analyzer1 = ( - self.session.query(model.Analyzer) + self.db.session.query(model.Analyzer) .filter(model.Analyzer.name == "cppcheck").first() ) _analyzer2 = ( - self.session.query(model.Analyzer) + self.db.session.query(model.Analyzer) .filter(model.Analyzer.name == "flawfinder").first() ) @@ -146,9 +144,9 @@ class ModelTestCase(unittest.TestCase): self.fetcher.packages.append(package) package.versions.append(package_version) - self.session.add(package) - self.session.add(package_version) - self.session.commit() + self.db.session.add(package) + self.db.session.add(package_version) + self.db.session.commit() package_analysis1 = model.Analysis( raw="<>", @@ -161,12 +159,12 @@ class ModelTestCase(unittest.TestCase): version_id=package_version.id ) - self.session.add(package_analysis1) - self.session.add(package_analysis2) - self.session.commit() + self.db.session.add(package_analysis1) + self.db.session.add(package_analysis2) + self.db.session.commit() saved_package = ( - self.session.query(model.Package) + self.db.session.query(model.Package) .filter(model.Package.name == 'bla').first() ) analysis = saved_package.versions[-1].analysis diff --git a/kiskadee/tests/units/test_runner.py b/kiskadee/tests/units/test_runner.py new file mode 100644 index 0000000..502c969 --- /dev/null +++ b/kiskadee/tests/units/test_runner.py @@ -0,0 +1,18 @@ +import unittest + +import kiskadee.fetchers.example +import kiskadee.runner + + +class RunnerTestCase(unittest.TestCase): + + def setUp(self): + self.queues = kiskadee.queue.Queues() + self.runner = kiskadee.runner.Runner(self.queues) + + def test_path_to_uncompressed_source(self): + + self.runner.fetcher = kiskadee.fetchers.example.Fetcher() + self.runner.project = {'name': 'test'} + uncompressed_source_path = self.runner.get_project_code_path() + self.assertIsNotNone(uncompressed_source_path) From 7a1176932d1f5417f12f5374736152170d8100c7 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Apr 08 2018 20:51:00 +0000 Subject: [PATCH 13/18] Fix Jenkinsfile --- diff --git a/Jenkinsfile b/Jenkinsfile index 9d1ce46..341728d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,7 +6,9 @@ pipeline { stage('Create Virtualenv') { steps { sh 'virtualenv -p /usr/bin/python3 .' - sh 'source bin/activate && pip install -e .' + sh 'source bin/activate && pip install -e .' + sh "source bin/activate && pip install fedmsg[consumers]" + sh "source bin/activate && pip install -r test_requirements.txt" } } @@ -18,7 +20,7 @@ pipeline { stage('Run Tests') { steps { - sh "source bin/activate && pip install coverage && make check" + sh "source bin/activate && make test" } post { success { From 22254edd28b6ba3d2cfb93e7023d1936844a4370 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Apr 08 2018 20:51:00 +0000 Subject: [PATCH 14/18] Add initializer module. - Removes from monitor the responsibility of manage kiskadee process. Now this will be done by the initializer module. --- diff --git a/kiskadee/initializer.py b/kiskadee/initializer.py new file mode 100644 index 0000000..c39304d --- /dev/null +++ b/kiskadee/initializer.py @@ -0,0 +1,22 @@ +"""Initializes kiskadee components. + +This module will run monitor and runner as processes. +""" + +from multiprocessing import Process + +import kiskadee.model +import kiskadee.queue +import kiskadee.database +from kiskadee.runner import Runner +from kiskadee.monitor import Monitor + +def init(): + queues = kiskadee.queue.Queues() + db = kiskadee.database.Database() + monitor = Monitor(db, queues) + runner = Runner(queues) + Process(target=monitor.run,).start() + runner_process = Process(target=runner.run,) + runner_process.start() + runner_process.join() diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py index 7f485c2..1996a0b 100644 --- a/kiskadee/monitor.py +++ b/kiskadee/monitor.py @@ -4,15 +4,12 @@ kiskadee monitors repositories checking for new project versions to be analyzed. This module provides such capabilities. """ import threading -from multiprocessing import Process import time import os import importlib -import kiskadee.database -from kiskadee.runner import Runner -import kiskadee.queue -from kiskadee.model import Package, Fetcher, Version, Report, Analysis +import kiskadee +from kiskadee.model import Package, Fetcher, Version, Report, Analysis, Analyzer RUNNING = True @@ -23,9 +20,9 @@ class Monitor: """Return a non initialized Monitor.""" self.db = db self.queues = queues - kiskadee.model.Analyzer.create_analyzers(self.db) + Analyzer.create_analyzers(self.db) - def start(self): + def run(self): kiskadee.logger.debug('Monitor PID: {}'.format(os.getpid())) for fetcher in kiskadee.load_fetchers(): @@ -86,21 +83,3 @@ class Monitor: module_as_a_thread.start() if joinable or timeout: module_as_a_thread.join(timeout) - - -def daemon(): - """Entry point to the monitor module.""" - # TODO: improve with start/stop system - queues = kiskadee.queue.Queues() - db = kiskadee.database.Database() - monitor = Monitor(db, queues) - runner = Runner(queues) - monitor_process = Process( - target=monitor.start, - ) - runner_process = Process( - target=runner.runner, - ) - monitor_process.start() - runner_process.start() - runner_process.join() diff --git a/kiskadee/runner.py b/kiskadee/runner.py index 0dee08a..ea9ef60 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -22,7 +22,7 @@ class Runner: self.fetcher = None self.project = None - def runner(self): + def run(self): """Run static analyzers. Continuously dequeue packages from `analyses_queue` and call the @@ -56,7 +56,7 @@ class Runner: return self.project - def clean_temporary_directory(self, temp_dir): + def rmdtemp(self, temp_dir): # not delete the source code used on tests. if not temp_dir.find("kiskadee/tests") > -1: shutil.rmtree(temp_dir) @@ -78,7 +78,7 @@ class Runner: analyzers, source_path ) self.enqueue_analysis_to_monitor(analysis_result) - self.clean_temporary_directory(source_path) + self.rmdtemp(source_path) def enqueue_analysis_to_monitor(self, analysis_result): if analysis_result['results']: @@ -121,7 +121,7 @@ class Runner: uncompressed_source_path = self.uncompress_project_code( compressed_source_path ) - self.clean_temporary_directory(os.path.dirname(compressed_source_path)) + self.rmdtemp(os.path.dirname(compressed_source_path)) return uncompressed_source_path else: kiskadee.logger.debug('RUNNER: invalid compressed source') diff --git a/setup.py b/setup.py index 03a13f3..9a74169 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ setup( packages=find_packages(), include_package_data=False, entry_points={'console_scripts': [ - 'kiskadee = kiskadee.monitor:daemon', + 'kiskadee = kiskadee.initializer:init', 'kiskadee_api = kiskadee.api.app:main',], 'moksha.consumer': ( 'anityaconsumer = kiskadee.fetchers.anitya:AnityaConsumer')}, From 1b4894276dd17a7e24980786b7424a50e1fbfa75 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Apr 08 2018 20:51:00 +0000 Subject: [PATCH 15/18] Fix logging messages. --- diff --git a/kiskadee/fetchers/__init__.py b/kiskadee/fetchers/__init__.py index b61511b..21a6fd6 100644 --- a/kiskadee/fetchers/__init__.py +++ b/kiskadee/fetchers/__init__.py @@ -49,12 +49,8 @@ class Fetcher(abc.ABC): if not project: raise NotImplementedError('fetcher must call parent watch method ') - kiskadee.queue.Queues().enqueue_project(project) - fetcher = project['fetcher'] - kiskadee.logger.debug( - "{} fetcher: sending package {}_{} for monitor" - .format(fetcher, project['name'], project['version']) - ) + fetcher = project['fetcher'].split('.')[-1] + kiskadee.queue.Queues().enqueue_project(project, fetcher) @abc.abstractmethod def compare_versions(self, new, old): diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py index 1996a0b..58f1493 100644 --- a/kiskadee/monitor.py +++ b/kiskadee/monitor.py @@ -23,20 +23,20 @@ class Monitor: Analyzer.create_analyzers(self.db) def run(self): - kiskadee.logger.debug('Monitor PID: {}'.format(os.getpid())) + kiskadee.logger.debug('MONITOR PID: {}'.format(os.getpid())) for fetcher in kiskadee.load_fetchers(): thread = Monitor._start_fetcher(fetcher.Fetcher().watch) while RUNNING: + kiskadee.logger.debug('MONITOR STATE: Idle'.format(os.getpid())) new_project = self.dequeue_project_from_fetchers() self.send_project_to_runner(new_project) analyzed_project = self.dequeue_analysis_from_runner() self.save_analyzed_project(analyzed_project) def dequeue_project_from_fetchers(self): - new_project = self.queues.dequeue_project() - return new_project if new_project else {} + return self.queues.dequeue_project() def dequeue_analysis_from_runner(self): return self.queues.dequeue_result() @@ -45,10 +45,12 @@ class Monitor: if data: fetcher, project = self.get_fetcher_and_project(data) data["fetcher_id"] = fetcher.id if fetcher else '' - if not project: - self.queues.enqueue_analysis(data) - elif self.is_a_new_project_version(project, data): - self.queues.enqueue_analysis(data) + if not self.is_a_new_project_version(project, data): + kiskadee.logger.debug('MONITOR STATE: Project {} already'\ + 'analyzed'.format(data['name'])) + return + self.queues.enqueue_analysis(data) + def get_fetcher_and_project(self, data): fetcher_name = data['fetcher'].split('.')[-1] @@ -58,10 +60,13 @@ class Monitor: return fetcher, project def is_a_new_project_version(self, project, data): - project_version = data['version'] - analysed_version = project.versions[-1].number - fetcher = importlib.import_module(data['fetcher']).Fetcher() - fetcher.compare_versions(project_version, analysed_version) + if project: + project_version = data['version'] + analysed_version = project.versions[-1].number + fetcher = importlib.import_module(data['fetcher']).Fetcher() + return fetcher.compare_versions(project_version, analysed_version) + else: + return True def save_analyzed_project(self, data): if not data: diff --git a/kiskadee/queue.py b/kiskadee/queue.py index 672958a..bbb7128 100644 --- a/kiskadee/queue.py +++ b/kiskadee/queue.py @@ -4,34 +4,6 @@ from multiprocessing import Queue import kiskadee -def empty_queue(func): - """Decorator that checks if retrieve data from a - queue will trigger a empty exception. - """ - def wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except Exception: - return {} - return wrapper - -def enqueue_project(fetcher_watch): - """Decorator to be used with watch fetchers method. Every project returned - by the watch method, will be enqueued on the queue projects. - :arg1: watch method - :returns: a wrapper variable - """ - def wrapper(*args, **kwargs): - project = fetcher_watch(*args, **kwargs) - kiskadee.queue.queues.enqueue_project(project) - fetcher = project['fetcher'].name - kiskadee.logger.debug( - "{} fetcher: sending package {}_{} for monitor" - .format(fetcher, package['name'], package['version']) - ) - time.sleep(2) - return wrapper - analysis = Queue() results = Queue() projects = Queue() @@ -42,7 +14,7 @@ class Queues(): @staticmethod def enqueue_analysis(project_to_analysis): """Put a analysis on the analysis queue.""" - log_msg = "Sending project {}_{} for analysis"\ + log_msg = "MONITOR STATE: Sending project {}-{} for analysis"\ .format(project_to_analysis['name'], project_to_analysis['version']) kiskadee.logger.debug(log_msg) @@ -52,11 +24,12 @@ class Queues(): def dequeue_analysis(): """Get a analysis from the analysis queue.""" project_to_analysis = analysis.get() + fetcher = project_to_analysis ['fetcher'].split('.')[-1] kiskadee.logger.debug( - 'RUNNER: deqeued {}-{} from {}' + 'RUNNER STATE: dequeued {}-{} from {}' .format(project_to_analysis['name'], project_to_analysis['version'], - project_to_analysis['fetcher']) + fetcher) ) return project_to_analysis @@ -64,25 +37,39 @@ class Queues(): def enqueue_result(project): """Put a result on the results queue.""" kiskadee.logger.debug( - "RUNNER: Sending {}-{} to Monitor" + "RUNNER STATE: Sending {}-{} to Monitor" .format(project["name"], project["version"]) ) results.put(project) @staticmethod - @empty_queue def dequeue_result(): """Get a result from the results queue.""" - return results.get(timeout=1) + result = results.get() + kiskadee.logger.debug( + "MONITOR STATE: Pick Up analyzed project" + .format(result["name"], + result["version"]) + ) + return result @staticmethod - def enqueue_project(project): + def enqueue_project(project, fetcher): """Put a result on the results queue.""" + kiskadee.logger.debug( + "FETCHER {}: sending package {}-{} for monitor" + .format(fetcher, project['name'], project['version']) + ) projects.put(project) @staticmethod - @empty_queue def dequeue_project(): """Get a result from the results queue.""" - return projects.get(timeout=1) + project = projects.get() + kiskadee.logger.debug( + "MONITOR STATE: Pick Up monitored project." + .format(project["name"], + project["version"]) + ) + return project diff --git a/kiskadee/report.py b/kiskadee/report.py index 10aa7cc..1afc770 100644 --- a/kiskadee/report.py +++ b/kiskadee/report.py @@ -36,7 +36,6 @@ class CppcheckReport(Report): def _compute_reports(self, analyzer): """Compute every report type for cppcheck analyzer.""" - print(analyzer) result_dict = { 'warning': 0, 'error': 0, diff --git a/kiskadee/runner.py b/kiskadee/runner.py index ea9ef60..d1fa040 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -1,4 +1,4 @@ -"""Run each static analyzer in each package marked for analysis.""" +"""Run each static analyzer in each project marked for analysis.""" import shutil import tempfile @@ -25,14 +25,13 @@ class Runner: def run(self): """Run static analyzers. - Continuously dequeue packages from `analyses_queue` and call the - :func:`analyze` method, passing the dequeued package. - After the analysis, updates the status of this package on the database. + Continuously dequeue projects from `analyses_queue` and call the + :func:`analyze` method, passing the dequeued project as argument. + After the analysis, send the result back to monitor. """ - kiskadee.logger.debug('Runner PID: {}'.format(os.getpid())) + kiskadee.logger.debug('RUNNER PID: {}'.format(os.getpid())) while RUNNING: - kiskadee.logger.debug('RUNNER: Waiting to dequeue'\ - ' project to analysis...') + kiskadee.logger.debug('RUNNER STATE: Idle.') self.project = self.queues.dequeue_analysis() self.call_analyzers() @@ -42,8 +41,8 @@ class Runner: self.project['fetcher'] ).Fetcher() except ModuleNotFoundError: - kiskadee.logger.debug("Fetcher {} could not be loaded"\ - .format(self.project['fetcher']) + kiskadee.logger.debug("RUNNER STATE: Fetcher {}\ + could not be loaded".format(self.project['fetcher']) ) return {} @@ -95,19 +94,20 @@ class Runner: if source_path is None: return None - kiskadee.logger.debug('ANALYSIS: running {} ...'.format(analyzer)) + kiskadee.logger.debug('RUNNER STATE: analysing with {} ...'\ + .format(analyzer)) try: analysis = kiskadee.analyzers.run(analyzer, source_path) firehose_report = kiskadee.converter.to_firehose( analysis, analyzer ) kiskadee.logger.debug( - 'ANALYSIS: DONE {} analysis' + 'RUNNER STATE: DONE {} analysis' .format(analyzer) ) return firehose_report except Exception as err: - kiskadee.logger.debug('RUNNER: could not generate analysis') + kiskadee.logger.debug('RUNNER STATE: could not generate analysis') kiskadee.logger.debug(err) return None @@ -134,7 +134,8 @@ class Runner: dir_to_unpack_source) return dir_to_unpack_source except Exception as err: - kiskadee.logger.debug('Could not unpack project source') + kiskadee.logger.debug('RUNNER STATE: Could not unpack'\ + 'project source') kiskadee.logger.debug(err) return {} From 8c8a0c28c210f7a6bca80398854e4338e319080f Mon Sep 17 00:00:00 2001 From: David Carlos Date: Apr 08 2018 20:51:00 +0000 Subject: [PATCH 16/18] Rename Package model to Project. - Kiskadee monitors repositories like github, gitlab, bitbucket, but also can monitors linux distributions as Debian. It's better to call the source code as Project, instead of Package, because not always we will be dealing with packages, but we will always dealing with projects. --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index 9eb7b3a..eae5017 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -4,8 +4,8 @@ from flask import request from flask_cors import CORS from kiskadee.database import Database -from kiskadee.model import Package, Fetcher, Version, Analysis -from kiskadee.api.serializers import PackageSchema, FetcherSchema,\ +from kiskadee.model import Project, Fetcher, Version, Analysis +from kiskadee.api.serializers import ProjectSchema, FetcherSchema,\ AnalysisSchema import json from sqlalchemy.orm import eagerload @@ -30,8 +30,8 @@ def index(): def packages(): """Get the list of analyzed packages.""" if request.method == 'GET': - packages = db.session.query(Package).all() - package_schema = PackageSchema( + packages = db.session.query(Project).all() + package_schema = ProjectSchema( many=True, exclude=['versions.analysis', 'versions.package_id'] ) @@ -44,7 +44,7 @@ def package_analysis_overview(pkg_name, version): """Get a analysis list of some package version.""" #TODO: This can be a simple inner join between package, version and analysis - _package_id = db.filter_by_name(Package, pkg_name).id + _package_id = db.filter_by_name(Project, pkg_name).id version_id = db.session.query(Version)\ .filter_by(number = version, package_id = _package_id ).first().id analysis = ( diff --git a/kiskadee/api/serializers.py b/kiskadee/api/serializers.py index e7168e9..b6d027c 100644 --- a/kiskadee/api/serializers.py +++ b/kiskadee/api/serializers.py @@ -1,7 +1,7 @@ """Provide objects to serialize the kiskadee models.""" from marshmallow import Schema, fields -from kiskadee.model import Package, Fetcher, Analysis, Version,\ +from kiskadee.model import Project, Fetcher, Analysis, Version,\ Report, Analyzer @@ -49,7 +49,7 @@ class AnalysisSchema(Schema): class VersionSchema(Schema): - """Provide a serializer to the Package model.""" + """Provide a serializer to the Project model.""" id = fields.Int() number = fields.Str() @@ -57,7 +57,7 @@ class VersionSchema(Schema): analysis = fields.Nested(AnalysisSchema, many=True) def make_object(self, data): - """Serialize a Package object.""" + """Serialize a Project object.""" print('MAKING OBJECT FROM', data) return Version(**data) @@ -76,8 +76,8 @@ class FetcherSchema(Schema): return Fetcher(**data) -class PackageSchema(Schema): - """Provide a serializer to the Package model.""" +class ProjectSchema(Schema): + """Provide a serializer to the Project model.""" id = fields.Int() name = fields.Str() @@ -85,6 +85,6 @@ class PackageSchema(Schema): versions = fields.Nested(VersionSchema, many=True) def make_object(self, data): - """Serialize a Package object.""" + """Serialize a Project object.""" print('MAKING OBJECT FROM', data) - return Package(**data) + return Project(**data) diff --git a/kiskadee/database.py b/kiskadee/database.py index 96cefe4..2c7edf5 100644 --- a/kiskadee/database.py +++ b/kiskadee/database.py @@ -4,7 +4,7 @@ from sqlalchemy import create_engine, orm import kiskadee from kiskadee.model import Base -from kiskadee.model import Package, Fetcher, Version, Report, Analysis +from kiskadee.model import Project, Fetcher, Version, Report, Analysis class Database: """kiskadee Database class.""" diff --git a/kiskadee/fetchers/debian.py b/kiskadee/fetchers/debian.py index 1cd1604..9f66aa2 100644 --- a/kiskadee/fetchers/debian.py +++ b/kiskadee/fetchers/debian.py @@ -1,4 +1,4 @@ -"""Fetcher to monitor Debian Project Packages.""" +"""Fetcher to monitor Debian Packages.""" import os import tempfile @@ -16,14 +16,14 @@ RUNNING = True class Fetcher(kiskadee.fetchers.Fetcher): - """Fetcher to monitor Debian Project Packages.""" + """Fetcher to monitor Debian Packages.""" def watch(self): """Start the monitoring process for Debian Repositories. - Each project monitored by the fetcher will be + Each package monitored by the fetcher will be queued by calling the watch parent method, - passing the project data as argument. + passing it as argument. """ kiskadee.logger.debug("Starting Debian fetcher") while RUNNING: @@ -74,10 +74,10 @@ class Fetcher(kiskadee.fetchers.Fetcher): sources = os.path.join(path, 'Sources') with open(sources) as sources_file: for src in Sources.iter_paragraphs(sources_file): - project = self.project_to_enqueue(src) - super().watch(**project) + package = self.package_to_enqueue(src) + super().watch(**package) - def project_to_enqueue(self, src): + def package_to_enqueue(self, src): return {'name': src["Package"], 'version': self._parse_version(src["Version"]), 'fetcher': __name__, diff --git a/kiskadee/model.py b/kiskadee/model.py index acbd5a0..83302b9 100644 --- a/kiskadee/model.py +++ b/kiskadee/model.py @@ -11,21 +11,21 @@ from kiskadee.report import CppcheckReport, FlawfinderReport Base = declarative_base() -class Package(Base): - """Software packages abstraction. +class Project(Base): + """Software projects abstraction. - A software package is the source code for a software project. It may be + A project is the source code stored on some repository. It may be upstream's distribution or the sources provided by some other source, like a linux distribution. """ - __tablename__ = 'packages' + __tablename__ = 'projects' id = Column(Integer, - Sequence('packages_id_seq', optional=True), primary_key=True) + Sequence('projects_id_seq', optional=True), primary_key=True) name = Column(Unicode(255), nullable=False) homepage = Column(Unicode(255), nullable=True) fetcher_id = Column(Integer, ForeignKey('fetchers.id'), nullable=False) - versions = orm.relationship('Version', backref='packages') + versions = orm.relationship('Version', backref='projects') __table_args__ = ( UniqueConstraint('name', 'fetcher_id'), ) @@ -36,18 +36,18 @@ class Package(Base): if ('meta' in data) and ('homepage' in data['meta']): homepage = data['meta']['homepage'] - _package = Package( + _project = Project( name=data['name'], homepage=homepage, fetcher_id=data['fetcher_id'] ) - db.session.add(_package) + db.session.add(_project) db.session.commit() _version = Version(number=data['version'], - package_id=_package.id) + project_id=_project.id) db.session.add(_version) db.session.commit() - return _package + return _project @staticmethod def update(db, project, data): @@ -81,20 +81,20 @@ class Fetcher(Base): name = Column(Unicode(255), nullable=False, unique=True) target = Column(Unicode(255), nullable=True) description = Column(UnicodeText) - packages = orm.relationship('Package', backref='fetchers') + projects = orm.relationship('Project', backref='fetchers') class Version(Base): - """Abstraction of a package version.""" + """Abstraction of a project version.""" __tablename__ = 'versions' id = Column(Integer, Sequence('versions_id_seq', optional=True), primary_key=True) number = Column(Unicode(100), nullable=False) - package_id = Column(Integer, ForeignKey('packages.id'), nullable=False) + project_id = Column(Integer, ForeignKey('projects.id'), nullable=False) analysis = orm.relationship('Analysis', backref='versions') __table_args__ = ( - UniqueConstraint('number', 'package_id'), + UniqueConstraint('number', 'project_id'), ) @@ -127,7 +127,7 @@ class Analyzer(Base): db.session.commit() class Analysis(Base): - """Abstraction of a package analysis.""" + """Abstraction of a project analysis.""" __tablename__ = 'analysis' id = Column(Integer, @@ -155,7 +155,7 @@ class Analysis(Base): } Report.save(db, dict_analysis, data, _analyzer.name) kiskadee.logger.debug( - "MONITOR: Saved analysis done by {} for package: {}-{}" + "MONITOR: Saved analysis done by {} for project: {}-{}" .format(analyzer, data["name"], data["version"]) ) return @@ -197,7 +197,7 @@ class Report(Base): db.session.add(_reports) db.session.commit() kiskadee.logger.debug( - "MONITOR: Saved analysis reports for {} package" + "MONITOR: Saved analysis reports for {} project" .format(data["name"]) ) except KeyError as key: @@ -208,7 +208,7 @@ class Report(Base): ) except Exception as err: kiskadee.logger.debug( - "MONITOR: Failed to get analysis reports to {} package" + "MONITOR: Failed to get analysis reports to {} project" .format(data["name"]) ) kiskadee.logger.debug(err) diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py index 58f1493..d88a085 100644 --- a/kiskadee/monitor.py +++ b/kiskadee/monitor.py @@ -9,7 +9,7 @@ import os import importlib import kiskadee -from kiskadee.model import Package, Fetcher, Version, Report, Analysis, Analyzer +from kiskadee.model import Project, Fetcher, Version, Report, Analysis, Analyzer RUNNING = True @@ -56,7 +56,7 @@ class Monitor: fetcher_name = data['fetcher'].split('.')[-1] project_name = data['name'] fetcher = self.db.filter_by_name(Fetcher, fetcher_name) - project = self.db.filter_by_name(Package, project_name) + project = self.db.filter_by_name(Project, project_name) return fetcher, project def is_a_new_project_version(self, project, data): @@ -71,11 +71,11 @@ class Monitor: def save_analyzed_project(self, data): if not data: return {} - project = self.db.filter_by_name(Package, data['name']) + project = self.db.filter_by_name(Project, data['name']) if not project: - project = Package.save(self.db, data) + project = Project.save(self.db, data) if project: - project = Package.update(self.db, project, data) + project = Project.update(self.db, project, data) for analyzer, result in data['results'].items(): Analysis.save(self.db, data, analyzer, result, project.versions[-1]) diff --git a/kiskadee/tests/api/test_api.py b/kiskadee/tests/api/test_api.py index cff7f1b..919ae5f 100644 --- a/kiskadee/tests/api/test_api.py +++ b/kiskadee/tests/api/test_api.py @@ -37,7 +37,7 @@ class ApiTestCase(unittest.TestCase): ) self.db.filter_by_name = MagicMock() - self.db.session.query(kiskadee.model.Package)\ + self.db.session.query(kiskadee.model.Project)\ .filter_by().id = MagicMock() self.db.session.query(kiskadee.model.Version)\ .filter_by().id = MagicMock() diff --git a/kiskadee/tests/integration/test_monitor.py b/kiskadee/tests/integration/test_monitor.py index 9691ca2..f4acf72 100644 --- a/kiskadee/tests/integration/test_monitor.py +++ b/kiskadee/tests/integration/test_monitor.py @@ -3,7 +3,7 @@ from unittest.mock import MagicMock from kiskadee.monitor import Monitor from kiskadee.queue import Queues -from kiskadee.model import Package, Fetcher +from kiskadee.model import Project, Fetcher import kiskadee.queue import kiskadee.fetchers.debian import kiskadee.fetchers.anitya @@ -14,7 +14,7 @@ class MonitorTestCase(unittest.TestCase): def setUp(self): def mocked_models(klass): - class Package: + class Project: def all(self): [self.pkg1, self.pkg2, self.pkg3, self.pkg4] klass() @@ -57,7 +57,7 @@ class MonitorTestCase(unittest.TestCase): def test_send_project_to_runner(self): fetcher = Fetcher(name='example') - Package(name='project1', fetcher_id=fetcher.id) + Project(name='project1', fetcher_id=fetcher.id) self.monitor.get_fetcher_and_project = MagicMock( return_value=[fetcher, {}] ) diff --git a/kiskadee/tests/units/test_model.py b/kiskadee/tests/units/test_model.py index f542aaa..b0aafa7 100644 --- a/kiskadee/tests/units/test_model.py +++ b/kiskadee/tests/units/test_model.py @@ -14,7 +14,7 @@ class ModelTestCase(unittest.TestCase): def setUp(self): kiskadee.tests.clean_test_db(self.db, model.Base.metadata) - self.package = model.Package(name='python-kiskadee') + self.package = model.Project(name='python-kiskadee') self.version = model.Version(number='1.0-rc1') self.fetcher = model.Fetcher( name='kiskadee-fetcher', target='university' @@ -44,7 +44,7 @@ class ModelTestCase(unittest.TestCase): self.assertEqual(fetchers, [self.fetcher]) def test_query_package(self): - packages = self.db.session.query(model.Package).all() + packages = self.db.session.query(model.Project).all() self.assertEqual(packages, [self.package]) def test_query_version(self): @@ -76,14 +76,14 @@ class ModelTestCase(unittest.TestCase): self.db.session.commit() def test_add_package_without_fetcher(self): - package = model.Package(name='foo-bar') + package = model.Project(name='foo-bar') self.db.session.add(package) with self.assertRaises(exc.IntegrityError): self.db.session.commit() def test_unique_package_in_fetcher(self): - package_1 = model.Package(name='foo-bar') - package_2 = model.Package(name='foo-bar') + package_1 = model.Project(name='foo-bar') + package_2 = model.Project(name='foo-bar') self.fetcher.packages.append(package_1) self.fetcher.packages.append(package_2) with self.assertRaises(exc.IntegrityError): @@ -100,7 +100,7 @@ class ModelTestCase(unittest.TestCase): def test_compose_kiskadee_source(self): _analyzer = self.db.session.query(model.Analyzer)\ .filter(model.Analyzer.name == "cppcheck").first() - package = model.Package( + package = model.Project( name='bla', fetcher_id=self.fetcher.id ) @@ -132,7 +132,7 @@ class ModelTestCase(unittest.TestCase): .filter(model.Analyzer.name == "flawfinder").first() ) - package = model.Package( + package = model.Project( name='bla', fetcher_id=self.fetcher.id ) @@ -164,8 +164,8 @@ class ModelTestCase(unittest.TestCase): self.db.session.commit() saved_package = ( - self.db.session.query(model.Package) - .filter(model.Package.name == 'bla').first() + self.db.session.query(model.Project) + .filter(model.Project.name == 'bla').first() ) analysis = saved_package.versions[-1].analysis self.assertEqual(len(analysis), 2) From 8f0ba6913952bebff9922da7cc93ba7d5b4c0b57 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Apr 08 2018 20:51:00 +0000 Subject: [PATCH 17/18] Update documentation. - fix typos. - minor fix on anitya tests. - create method to save project analysis on Monitor. --- diff --git a/README.md b/README.md index b1fcc51..724c795 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,14 @@ For more information on the project, installation or development environment setup, refer to the full [kiskadee documentation](https://docs.pagure.org/kiskadee). +To build the documentation locally: + + pip install -U sphinx sphinx\_rtd\_theme + cd doc + make html + +Open the index.html file, inside the _build/html directory. + ## Repositories kiskadee core and API development are hosted at [pagure](https://pagure.io/kiskadee). diff --git a/alembic/versions/0920061a363d_rename_package_table_to_project.py b/alembic/versions/0920061a363d_rename_package_table_to_project.py new file mode 100644 index 0000000..335101d --- /dev/null +++ b/alembic/versions/0920061a363d_rename_package_table_to_project.py @@ -0,0 +1,25 @@ +"""rename package table to project. + +Revision ID: 0920061a363d +Revises: cc4887b7c3da +Create Date: 2018-03-13 17:48:05.506903 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '0920061a363d' +down_revision = 'cc4887b7c3da' +branch_labels = None +depends_on = None + + +def upgrade(): + op.rename_table('packages', 'projects') + op.alter_column('versions', 'package_id', new_column_name='project_id') + + +def downgrade(): + op.rename_table('projects', 'packages') diff --git a/doc/_static/kiskadee_arch.png b/doc/_static/kiskadee_arch.png index 767a459..6d8fc35 100644 Binary files a/doc/_static/kiskadee_arch.png and b/doc/_static/kiskadee_arch.png differ diff --git a/doc/architecture.rst b/doc/architecture.rst index 0a8ed5b..7d56cbb 100644 --- a/doc/architecture.rst +++ b/doc/architecture.rst @@ -37,7 +37,7 @@ The class defines the following public behaviors: .. autoclass:: kiskadee.monitor.Monitor() - :members: monitor + :members: Runner - Module to run static analysis -------------------------------------- diff --git a/doc/fetchers.rst b/doc/fetchers.rst index 98aaaeb..08d6089 100644 --- a/doc/fetchers.rst +++ b/doc/fetchers.rst @@ -21,30 +21,34 @@ A simple example of a kiskadee fetcher .. code-block:: python - import kiskadee - import sys - import kiskadee.queue - class Fetcher(kiskadee.fetchers.Fetcher): - def get_sources(self, source_data): - return 'kiskadee/tests/test_source/test_source.tar.gz' - - @kiskadee.queue.package_enqueuer - def watch(self): - """There is no proper API to inspect new example versions. - It should not matter, since example will not receive updates. - """ - example = {} - example['fetcher'] = sys.modules[__name__] - example['version'] = '0.1' - example['name'] = 'example' - return example - - def compare_versions(self, new, old): - """Example has only one version - - This method does not matter here, let's just pass - """ - return 0 + """Example kiskadee fetcher implementation.""" + + import kiskadee + + class Fetcher(kiskadee.fetchers.Fetcher): + """Example kiskadee fetcher implementation.""" + + def get_sources(self, source_data): + """Use sources from test suite.""" + return 'kiskadee/tests/test_source/test_source.tar.gz' + + def watch(self, **kwargs): + """There is no proper API to inspect new example versions. + It should not matter, since example will not receive updates. + """ + example = {} + example['fetcher'] = __name__ + example['version'] = '0.1' + example['name'] = 'example' + super().watch(**example) + + def compare_versions(self, new, old): + """Compare package versions. + + Example has only one version. This method does not matter here, let's + just pass. + """ + return 0 List of kiskadee fetchers ---------------------------- diff --git a/doc/installing.rst b/doc/installing.rst index 946e2c3..252cc24 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -3,11 +3,9 @@ Installing kiskadee Development ----------- -To install locally run: - First, make shure that ansible will be able to login on the vm. ansible will use the root user to do this, so you will have to add your public ssh -key inside the root ~/.ssh/authorized_keys file. You can also create the host +key inside the root **~/.ssh/authorized_keys** file. You can also create the host user inside the vm, in order to be able to test the vm access with the ping command. @@ -22,3 +20,9 @@ To deploy kiskadee locally: .. code-block:: bash ansible-playbook -c paramiko -i playbook/hosts.local playbook/local.yml + +Production +---------- + +We are still working on a rpm package to install kiskadee on production mode. +You can check the latest build `here `_. diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index eae5017..3714c17 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -26,27 +26,27 @@ def index(): return jsonify({'fetchers': result.data}) -@kiskadee.route('/packages') -def packages(): - """Get the list of analyzed packages.""" +@kiskadee.route('/projects') +def projects(): + """Get the list of analyzed projects.""" if request.method == 'GET': - packages = db.session.query(Project).all() - package_schema = ProjectSchema( + projects = db.session.query(Project).all() + project_schema = ProjectSchema( many=True, - exclude=['versions.analysis', 'versions.package_id'] + exclude=['versions.analysis', 'versions.project_id'] ) - data, errors = package_schema.dump(packages) - return jsonify({'packages': data}) + data, errors = project_schema.dump(projects) + return jsonify({'projects': data}) -@kiskadee.route('/analysis//', methods=['GET']) -def package_analysis_overview(pkg_name, version): - """Get a analysis list of some package version.""" +@kiskadee.route('/analysis//', methods=['GET']) +def project_analysis_overview(project_name, version): + """Get a analysis list of some project version.""" - #TODO: This can be a simple inner join between package, version and analysis - _package_id = db.filter_by_name(Project, pkg_name).id + #TODO: This can be a simple inner join between project, version and analysis + _project_id = db.filter_by_name(Project, project_name).id version_id = db.session.query(Version)\ - .filter_by(number = version, package_id = _package_id ).first().id + .filter_by(number = version, project_id = _project_id ).first().id analysis = ( db.session.query(Analysis) .options( diff --git a/kiskadee/model.py b/kiskadee/model.py index 83302b9..2e861df 100644 --- a/kiskadee/model.py +++ b/kiskadee/model.py @@ -139,7 +139,9 @@ class Analysis(Base): uselist=False, back_populates='analysis') @staticmethod - def save(db, data, analyzer, result, version): + def save(db, analyzer, result, project): + version = project.versions[-1] + name = project.name _analysis = kiskadee.model.Analysis() try: _analyzer = db.session.query(kiskadee.model.Analyzer).\ @@ -153,10 +155,10 @@ class Analysis(Base): 'results': _analysis.raw['results'], 'id': _analysis.id } - Report.save(db, dict_analysis, data, _analyzer.name) + Report.save(db, dict_analysis, _analyzer.name, name) kiskadee.logger.debug( "MONITOR: Saved analysis done by {} for project: {}-{}" - .format(analyzer, data["name"], data["version"]) + .format(analyzer, name, version.number) ) return except Exception as err: @@ -184,7 +186,7 @@ class Report(Base): analysis = orm.relationship('Analysis', back_populates='report') @staticmethod - def save(db, analysis, data, analyzer_name): + def save(db, analysis, analyzer_name, project_name): try: results = analysis['results'] analyzer_report = Report.REPORTERS[analyzer_name](results) @@ -198,7 +200,7 @@ class Report(Base): db.session.commit() kiskadee.logger.debug( "MONITOR: Saved analysis reports for {} project" - .format(data["name"]) + .format(project_name) ) except KeyError as key: kiskadee.logger.debug( @@ -209,7 +211,7 @@ class Report(Base): except Exception as err: kiskadee.logger.debug( "MONITOR: Failed to get analysis reports to {} project" - .format(data["name"]) + .format(project_name) ) kiskadee.logger.debug(err) return diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py index d88a085..710127a 100644 --- a/kiskadee/monitor.py +++ b/kiskadee/monitor.py @@ -23,10 +23,14 @@ class Monitor: Analyzer.create_analyzers(self.db) def run(self): + """Run Monitor process. This method resumes Monitor behavior. It dequeue + projects from fetchers, check if the project is valid and if so send it + to Runner. When Runner ends a analysis, the Monitor dequeue the analysis + and save it on database. """ kiskadee.logger.debug('MONITOR PID: {}'.format(os.getpid())) for fetcher in kiskadee.load_fetchers(): - thread = Monitor._start_fetcher(fetcher.Fetcher().watch) + thread = Monitor.start_fetcher(fetcher.Fetcher().watch) while RUNNING: kiskadee.logger.debug('MONITOR STATE: Idle'.format(os.getpid())) @@ -47,7 +51,7 @@ class Monitor: data["fetcher_id"] = fetcher.id if fetcher else '' if not self.is_a_new_project_version(project, data): kiskadee.logger.debug('MONITOR STATE: Project {} already'\ - 'analyzed'.format(data['name'])) + ' analyzed'.format(data['name'])) return self.queues.enqueue_analysis(data) @@ -74,15 +78,18 @@ class Monitor: project = self.db.filter_by_name(Project, data['name']) if not project: project = Project.save(self.db, data) - if project: + else: project = Project.update(self.db, project, data) + self.save_project_analysis(data, project) + def save_project_analysis(self, data, project): for analyzer, result in data['results'].items(): - Analysis.save(self.db, data, analyzer, result, project.versions[-1]) + Analysis.save(self.db, analyzer, result, project) @staticmethod - def _start_fetcher(module, joinable=False, timeout=None): + def start_fetcher(module, joinable=False, timeout=None): + """Start a fetcher as a thread of the Monitor process.""" module_as_a_thread = threading.Thread(target=module) module_as_a_thread.daemon = True module_as_a_thread.start() diff --git a/kiskadee/queue.py b/kiskadee/queue.py index bbb7128..0d4956f 100644 --- a/kiskadee/queue.py +++ b/kiskadee/queue.py @@ -55,12 +55,13 @@ class Queues(): return result @staticmethod - def enqueue_project(project, fetcher): + def enqueue_project(project, fetcher=None): """Put a result on the results queue.""" - kiskadee.logger.debug( - "FETCHER {}: sending package {}-{} for monitor" - .format(fetcher, project['name'], project['version']) - ) + if fetcher: + kiskadee.logger.debug( + "FETCHER {}: sending package {}-{} for monitor" + .format(fetcher, project['name'], project['version']) + ) projects.put(project) @staticmethod diff --git a/kiskadee/runner.py b/kiskadee/runner.py index d1fa040..e245552 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -27,7 +27,7 @@ class Runner: Continuously dequeue projects from `analyses_queue` and call the :func:`analyze` method, passing the dequeued project as argument. - After the analysis, send the result back to monitor. + After the analysis, send the result back to Monitor. """ kiskadee.logger.debug('RUNNER PID: {}'.format(os.getpid())) while RUNNING: @@ -89,7 +89,7 @@ class Runner: The `analyzer` is the name of a static analyzer already created on the database. The `source_path` is the directory to a uncompressed source, returned - by the :func:`_path_to_uncompressed_source`. + by the :func:`get_project_code_path`. """ if source_path is None: return None @@ -112,7 +112,8 @@ class Runner: return None def get_project_code_path(self): - + """ Returns a string, representing the path of the uncompressed + project source.""" if not (self.fetcher and self.project): return None diff --git a/kiskadee/tests/integration/test_monitor.py b/kiskadee/tests/integration/test_monitor.py index f4acf72..1017889 100644 --- a/kiskadee/tests/integration/test_monitor.py +++ b/kiskadee/tests/integration/test_monitor.py @@ -44,8 +44,8 @@ class MonitorTestCase(unittest.TestCase): self.assertEqual(monitored_project['version'], '0.1') def test_run_fetchers_as_threads(self): - Monitor._start_fetcher(self.example_fetcher.watch) - Monitor._start_fetcher(self.example_fetcher.watch) + Monitor.start_fetcher(self.example_fetcher.watch) + Monitor.start_fetcher(self.example_fetcher.watch) first_monitored_project = self.monitor.dequeue_project_from_fetchers() second_monitored_project = self.monitor.dequeue_project_from_fetchers() self.assertIn("version", first_monitored_project) diff --git a/kiskadee/tests/plugins/test_plugins.py b/kiskadee/tests/plugins/test_plugins.py index 9fe8262..d9f532d 100644 --- a/kiskadee/tests/plugins/test_plugins.py +++ b/kiskadee/tests/plugins/test_plugins.py @@ -180,13 +180,15 @@ class TestAnityaFetcher(unittest.TestCase): def test_get_sources(self): - def mock_github(self, source_data, path): + def mock_github(self, fetcher, source_data, path): return 'kiskadee/tests/test_source/Sources.gz' kiskadee.fetchers.anitya.Backends.github = mock_github source_data = {'meta': {'backend': 'GitHub'}} source_path = self.anitya_fetcher.get_sources(source_data) - self.assertEqual(source_path, mock_github("self", "foo", "bla")) + self.assertEqual(source_path, mock_github( + "self", "example", "foo", "bla") + ) def test_create_package_dict(self): diff --git a/kiskadee/tests/test_style.py b/kiskadee/tests/test_style.py deleted file mode 100644 index acf5c15..0000000 --- a/kiskadee/tests/test_style.py +++ /dev/null @@ -1,20 +0,0 @@ -# Thanks bodhi -# https://github.com/fedora-infra/bodhi/blob/develop/bodhi/tests/test_style.py - -import os -import subprocess -import unittest - -KISKADEE_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) - - -class TestStyle(unittest.TestCase): - def test_with_flake8(self): - """Enforce PEP-8""" - flake8_command = ['flake8', KISKADEE_PATH] - self.assertEqual(subprocess.call(flake8_command), 0) - - def test_with_pydocstyle(self): - """Enforce PEP-257""" - pydocstyle_command = ['pydocstyle', KISKADEE_PATH] - self.assertEqual(subprocess.call(pydocstyle_command), 0) diff --git a/kiskadee/tests/units/test_model.py b/kiskadee/tests/units/test_model.py index b0aafa7..6fda0be 100644 --- a/kiskadee/tests/units/test_model.py +++ b/kiskadee/tests/units/test_model.py @@ -14,16 +14,16 @@ class ModelTestCase(unittest.TestCase): def setUp(self): kiskadee.tests.clean_test_db(self.db, model.Base.metadata) - self.package = model.Project(name='python-kiskadee') + self.project = model.Project(name='python-kiskadee') self.version = model.Version(number='1.0-rc1') self.fetcher = model.Fetcher( name='kiskadee-fetcher', target='university' ) self.db.session.add(self.fetcher) self.db.session.commit() - self.fetcher.packages.append(self.package) - self.package.versions.append(self.version) - self.db.session.add(self.package) + self.fetcher.projects.append(self.project) + self.project.versions.append(self.version) + self.db.session.add(self.project) self.db.session.add(self.fetcher) self.db.session.add(self.version) @@ -43,9 +43,9 @@ class ModelTestCase(unittest.TestCase): fetchers = self.db.session.query(model.Fetcher).all() self.assertEqual(fetchers, [self.fetcher]) - def test_query_package(self): - packages = self.db.session.query(model.Project).all() - self.assertEqual(packages, [self.package]) + def test_query_project(self): + projects = self.db.session.query(model.Project).all() + self.assertEqual(projects, [self.project]) def test_query_version(self): versions = self.db.session.query(model.Version).all() @@ -62,7 +62,7 @@ class ModelTestCase(unittest.TestCase): fetchers = self.db.session.query(model.Fetcher).all() self.assertEqual(len(fetchers), 2) - def test_add_version_without_package(self): + def test_add_version_without_project(self): version = model.Version(number='3.1') self.db.session.add(version) with self.assertRaises(exc.IntegrityError): @@ -75,51 +75,51 @@ class ModelTestCase(unittest.TestCase): with self.assertRaises(exc.IntegrityError): self.db.session.commit() - def test_add_package_without_fetcher(self): - package = model.Project(name='foo-bar') - self.db.session.add(package) + def test_add_project_without_fetcher(self): + project = model.Project(name='foo-bar') + self.db.session.add(project) with self.assertRaises(exc.IntegrityError): self.db.session.commit() - def test_unique_package_in_fetcher(self): - package_1 = model.Project(name='foo-bar') - package_2 = model.Project(name='foo-bar') - self.fetcher.packages.append(package_1) - self.fetcher.packages.append(package_2) + def test_unique_project_in_fetcher(self): + project_1 = model.Project(name='foo-bar') + project_2 = model.Project(name='foo-bar') + self.fetcher.projects.append(project_1) + self.fetcher.projects.append(project_2) with self.assertRaises(exc.IntegrityError): self.db.session.commit() - def test_unique_version_for_package(self): - package_version_1 = model.Version(number='1.0') - package_version_2 = model.Version(number='1.0') - self.package.versions.append(package_version_1) - self.package.versions.append(package_version_2) + def test_unique_version_for_project(self): + project_version_1 = model.Version(number='1.0') + project_version_2 = model.Version(number='1.0') + self.project.versions.append(project_version_1) + self.project.versions.append(project_version_2) with self.assertRaises(exc.IntegrityError): self.db.session.commit() def test_compose_kiskadee_source(self): _analyzer = self.db.session.query(model.Analyzer)\ .filter(model.Analyzer.name == "cppcheck").first() - package = model.Project( + project = model.Project( name='bla', fetcher_id=self.fetcher.id ) - package_version = model.Version( + project_version = model.Version( number='1.0.1', - package_id=package.id + project_id=project.id ) - package_analysis = model.Analysis( + project_analysis = model.Analysis( raw="<>", analyzer_id=_analyzer.id, - version_id=package_version.id + version_id=project_version.id ) - self.fetcher.packages.append(package) - package.versions.append(package_version) - package_version.analysis.append(package_analysis) + self.fetcher.projects.append(project) + project.versions.append(project_version) + project_version.analysis.append(project_analysis) - self.assertEqual(package.versions[0].analysis[0].raw, "<>") + self.assertEqual(project.versions[0].analysis[0].raw, "<>") def test_save_several_analysis(self): @@ -132,42 +132,42 @@ class ModelTestCase(unittest.TestCase): .filter(model.Analyzer.name == "flawfinder").first() ) - package = model.Project( + project = model.Project( name='bla', fetcher_id=self.fetcher.id ) - package_version = model.Version( + project_version = model.Version( number='1.0.1', - package_id=package.id + project_id=project.id ) - self.fetcher.packages.append(package) - package.versions.append(package_version) + self.fetcher.projects.append(project) + project.versions.append(project_version) - self.db.session.add(package) - self.db.session.add(package_version) + self.db.session.add(project) + self.db.session.add(project_version) self.db.session.commit() - package_analysis1 = model.Analysis( + project_analysis1 = model.Analysis( raw="<>", analyzer_id=_analyzer1.id, - version_id=package_version.id + version_id=project_version.id ) - package_analysis2 = model.Analysis( + project_analysis2 = model.Analysis( raw="><", analyzer_id=_analyzer2.id, - version_id=package_version.id + version_id=project_version.id ) - self.db.session.add(package_analysis1) - self.db.session.add(package_analysis2) + self.db.session.add(project_analysis1) + self.db.session.add(project_analysis2) self.db.session.commit() - saved_package = ( + saved_project = ( self.db.session.query(model.Project) .filter(model.Project.name == 'bla').first() ) - analysis = saved_package.versions[-1].analysis + analysis = saved_project.versions[-1].analysis self.assertEqual(len(analysis), 2) self.assertEqual(analysis[0].raw, "<>") self.assertEqual(analysis[1].raw, "><") From 62303011e067488534c407301a65a32db94e9ddf Mon Sep 17 00:00:00 2001 From: David Carlos Date: Apr 08 2018 20:51:00 +0000 Subject: [PATCH 18/18] Revert "Rename Package model to Project." This reverts commit e405dc01d44923d3f9e693f90ff4f1cd3b1698b6. Talking with Athos, we decided to keep the model name as Package. --- diff --git a/alembic/versions/0920061a363d_rename_package_table_to_project.py b/alembic/versions/0920061a363d_rename_package_table_to_project.py deleted file mode 100644 index 335101d..0000000 --- a/alembic/versions/0920061a363d_rename_package_table_to_project.py +++ /dev/null @@ -1,25 +0,0 @@ -"""rename package table to project. - -Revision ID: 0920061a363d -Revises: cc4887b7c3da -Create Date: 2018-03-13 17:48:05.506903 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '0920061a363d' -down_revision = 'cc4887b7c3da' -branch_labels = None -depends_on = None - - -def upgrade(): - op.rename_table('packages', 'projects') - op.alter_column('versions', 'package_id', new_column_name='project_id') - - -def downgrade(): - op.rename_table('projects', 'packages') diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index 3714c17..9eb7b3a 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -4,8 +4,8 @@ from flask import request from flask_cors import CORS from kiskadee.database import Database -from kiskadee.model import Project, Fetcher, Version, Analysis -from kiskadee.api.serializers import ProjectSchema, FetcherSchema,\ +from kiskadee.model import Package, Fetcher, Version, Analysis +from kiskadee.api.serializers import PackageSchema, FetcherSchema,\ AnalysisSchema import json from sqlalchemy.orm import eagerload @@ -26,27 +26,27 @@ def index(): return jsonify({'fetchers': result.data}) -@kiskadee.route('/projects') -def projects(): - """Get the list of analyzed projects.""" +@kiskadee.route('/packages') +def packages(): + """Get the list of analyzed packages.""" if request.method == 'GET': - projects = db.session.query(Project).all() - project_schema = ProjectSchema( + packages = db.session.query(Package).all() + package_schema = PackageSchema( many=True, - exclude=['versions.analysis', 'versions.project_id'] + exclude=['versions.analysis', 'versions.package_id'] ) - data, errors = project_schema.dump(projects) - return jsonify({'projects': data}) + data, errors = package_schema.dump(packages) + return jsonify({'packages': data}) -@kiskadee.route('/analysis//', methods=['GET']) -def project_analysis_overview(project_name, version): - """Get a analysis list of some project version.""" +@kiskadee.route('/analysis//', methods=['GET']) +def package_analysis_overview(pkg_name, version): + """Get a analysis list of some package version.""" - #TODO: This can be a simple inner join between project, version and analysis - _project_id = db.filter_by_name(Project, project_name).id + #TODO: This can be a simple inner join between package, version and analysis + _package_id = db.filter_by_name(Package, pkg_name).id version_id = db.session.query(Version)\ - .filter_by(number = version, project_id = _project_id ).first().id + .filter_by(number = version, package_id = _package_id ).first().id analysis = ( db.session.query(Analysis) .options( diff --git a/kiskadee/api/serializers.py b/kiskadee/api/serializers.py index b6d027c..e7168e9 100644 --- a/kiskadee/api/serializers.py +++ b/kiskadee/api/serializers.py @@ -1,7 +1,7 @@ """Provide objects to serialize the kiskadee models.""" from marshmallow import Schema, fields -from kiskadee.model import Project, Fetcher, Analysis, Version,\ +from kiskadee.model import Package, Fetcher, Analysis, Version,\ Report, Analyzer @@ -49,7 +49,7 @@ class AnalysisSchema(Schema): class VersionSchema(Schema): - """Provide a serializer to the Project model.""" + """Provide a serializer to the Package model.""" id = fields.Int() number = fields.Str() @@ -57,7 +57,7 @@ class VersionSchema(Schema): analysis = fields.Nested(AnalysisSchema, many=True) def make_object(self, data): - """Serialize a Project object.""" + """Serialize a Package object.""" print('MAKING OBJECT FROM', data) return Version(**data) @@ -76,8 +76,8 @@ class FetcherSchema(Schema): return Fetcher(**data) -class ProjectSchema(Schema): - """Provide a serializer to the Project model.""" +class PackageSchema(Schema): + """Provide a serializer to the Package model.""" id = fields.Int() name = fields.Str() @@ -85,6 +85,6 @@ class ProjectSchema(Schema): versions = fields.Nested(VersionSchema, many=True) def make_object(self, data): - """Serialize a Project object.""" + """Serialize a Package object.""" print('MAKING OBJECT FROM', data) - return Project(**data) + return Package(**data) diff --git a/kiskadee/database.py b/kiskadee/database.py index 2c7edf5..96cefe4 100644 --- a/kiskadee/database.py +++ b/kiskadee/database.py @@ -4,7 +4,7 @@ from sqlalchemy import create_engine, orm import kiskadee from kiskadee.model import Base -from kiskadee.model import Project, Fetcher, Version, Report, Analysis +from kiskadee.model import Package, Fetcher, Version, Report, Analysis class Database: """kiskadee Database class.""" diff --git a/kiskadee/fetchers/__init__.py b/kiskadee/fetchers/__init__.py index 21a6fd6..1b89193 100644 --- a/kiskadee/fetchers/__init__.py +++ b/kiskadee/fetchers/__init__.py @@ -43,14 +43,14 @@ class Fetcher(abc.ABC): This method will be called as a thread, and will run concurrently with the main kiskadee thread. The monitoring process must be done by classes that heritage from this one. This method is only responsible - to enqueue new monitored projects, so must be called by it childrens. + to enqueue new monitored packages, so must be called by it childrens. """ - project = kwargs - if not project: + package = kwargs + if not package: raise NotImplementedError('fetcher must call parent watch method ') - fetcher = project['fetcher'].split('.')[-1] - kiskadee.queue.Queues().enqueue_project(project, fetcher) + fetcher = package['fetcher'].split('.')[-1] + kiskadee.queue.Queues().enqueue_package(package, fetcher) @abc.abstractmethod def compare_versions(self, new, old): diff --git a/kiskadee/fetchers/anitya.py b/kiskadee/fetchers/anitya.py index 589d59c..57859c3 100644 --- a/kiskadee/fetchers/anitya.py +++ b/kiskadee/fetchers/anitya.py @@ -17,9 +17,9 @@ class Fetcher(kiskadee.fetchers.Fetcher): def watch(self): """Start the monitoring process for Anitya reports. - Each project monitored by the fetcher will be + Each package monitored by the fetcher will be queued by calling the watch parent method, - passing the project data as argument. + passing the package data as argument. The fetcher will use zmq as messaging protocol to receive the fedmsg-hub events. kiskadee and fedmsg-hub runs in different @@ -36,7 +36,7 @@ class Fetcher(kiskadee.fetchers.Fetcher): if socket: while True: msg = socket.recv_string() - project = self.project_to_enqueue(msg) + package = self.package_to_enqueue(msg) def get_sources(self, source_data): """Download packages from some Anitya Backend.""" @@ -77,22 +77,22 @@ class Fetcher(kiskadee.fetchers.Fetcher): kiskadee.logger.debug(err) return False - def project_to_enqueue(self, fedmsg_event): + def package_to_enqueue(self, fedmsg_event): event = self._event_to_dict(fedmsg_event) if event: - project = event.get('body').get('msg').get('project') + package = event.get('body').get('msg').get('package') source_dict = {} - if project: - kiskadee_project = { - 'name': project.get('name'), - 'version': project.get('version'), + if package: + kiskadee_package = { + 'name': package.get('name'), + 'version': package.get('version'), 'fetcher': __name__, 'meta': { - 'backend': project.get('backend'), - 'homepage': project.get('homepage') + 'backend': package.get('backend'), + 'homepage': package.get('homepage') } } - super().watch(**kiskadee_project) + super().watch(**kiskadee_package) def _event_to_dict(self, msg): msg = msg[msg.find(" ")+1::] @@ -125,7 +125,7 @@ class Backends(): class AnityaConsumer(fedmsg.consumers.FedmsgConsumer): """Consumer used by fedmsg-hub to subscribe to fedmsg bus.""" - topic = 'org.release-monitoring.prod.anitya.project.version.update' + topic = 'org.release-monitoring.prod.anitya.package.version.update' config_key = 'anityaconsumer' validate_signatures = False diff --git a/kiskadee/fetchers/debian.py b/kiskadee/fetchers/debian.py index 9f66aa2..e3a30b9 100644 --- a/kiskadee/fetchers/debian.py +++ b/kiskadee/fetchers/debian.py @@ -1,4 +1,4 @@ -"""Fetcher to monitor Debian Packages.""" +"""Fetcher to monitor Debian package Packages.""" import os import tempfile @@ -23,7 +23,7 @@ class Fetcher(kiskadee.fetchers.Fetcher): Each package monitored by the fetcher will be queued by calling the watch parent method, - passing it as argument. + passing the package data as argument. """ kiskadee.logger.debug("Starting Debian fetcher") while RUNNING: diff --git a/kiskadee/model.py b/kiskadee/model.py index 2e861df..3d10669 100644 --- a/kiskadee/model.py +++ b/kiskadee/model.py @@ -11,21 +11,21 @@ from kiskadee.report import CppcheckReport, FlawfinderReport Base = declarative_base() -class Project(Base): - """Software projects abstraction. +class Package(Base): + """Software packages abstraction. - A project is the source code stored on some repository. It may be + A software package is the source code for a software package. It may be upstream's distribution or the sources provided by some other source, like a linux distribution. """ - __tablename__ = 'projects' + __tablename__ = 'packages' id = Column(Integer, - Sequence('projects_id_seq', optional=True), primary_key=True) + Sequence('packages_id_seq', optional=True), primary_key=True) name = Column(Unicode(255), nullable=False) homepage = Column(Unicode(255), nullable=True) fetcher_id = Column(Integer, ForeignKey('fetchers.id'), nullable=False) - versions = orm.relationship('Version', backref='projects') + versions = orm.relationship('Version', backref='packages') __table_args__ = ( UniqueConstraint('name', 'fetcher_id'), ) @@ -36,37 +36,37 @@ class Project(Base): if ('meta' in data) and ('homepage' in data['meta']): homepage = data['meta']['homepage'] - _project = Project( + _package = Package( name=data['name'], homepage=homepage, fetcher_id=data['fetcher_id'] ) - db.session.add(_project) + db.session.add(_package) db.session.commit() _version = Version(number=data['version'], - project_id=_project.id) + package_id=_package.id) db.session.add(_version) db.session.commit() - return _project + return _package @staticmethod - def update(db, project, data): + def update(db, package, data): - if(project.versions[-1].number == data['version']): - return project + if(package.versions[-1].number == data['version']): + return package try: _new_version = Version( number=data['version'], - project_id=project.id + package_id=package.id ) - project.versions.append(_new_version) - db.session.add(project) + package.versions.append(_new_version) + db.session.add(package) db.session.commit() kiskadee.logger.debug( - "MONITOR: Sending project {}_{}" + "MONITOR: Sending package {}_{}" "for analysis".format(data['name'], data['version']) ) - return project + return package except ValueError: kiskadee.logger.debug("MONITOR: Could not compare versions") return None @@ -81,20 +81,20 @@ class Fetcher(Base): name = Column(Unicode(255), nullable=False, unique=True) target = Column(Unicode(255), nullable=True) description = Column(UnicodeText) - projects = orm.relationship('Project', backref='fetchers') + packages = orm.relationship('Package', backref='fetchers') class Version(Base): - """Abstraction of a project version.""" + """Abstraction of a package version.""" __tablename__ = 'versions' id = Column(Integer, Sequence('versions_id_seq', optional=True), primary_key=True) number = Column(Unicode(100), nullable=False) - project_id = Column(Integer, ForeignKey('projects.id'), nullable=False) + package_id = Column(Integer, ForeignKey('packages.id'), nullable=False) analysis = orm.relationship('Analysis', backref='versions') __table_args__ = ( - UniqueConstraint('number', 'project_id'), + UniqueConstraint('number', 'package_id'), ) @@ -127,7 +127,7 @@ class Analyzer(Base): db.session.commit() class Analysis(Base): - """Abstraction of a project analysis.""" + """Abstraction of a package analysis.""" __tablename__ = 'analysis' id = Column(Integer, @@ -139,9 +139,9 @@ class Analysis(Base): uselist=False, back_populates='analysis') @staticmethod - def save(db, analyzer, result, project): - version = project.versions[-1] - name = project.name + def save(db, analyzer, result, package): + version = package.versions[-1] + name = package.name _analysis = kiskadee.model.Analysis() try: _analyzer = db.session.query(kiskadee.model.Analyzer).\ @@ -157,8 +157,8 @@ class Analysis(Base): } Report.save(db, dict_analysis, _analyzer.name, name) kiskadee.logger.debug( - "MONITOR: Saved analysis done by {} for project: {}-{}" - .format(analyzer, name, version.number) + "MONITOR: Saved analysis done by {} for package: {}-{}" + .format(analyzer, data["name"], data["version"]) ) return except Exception as err: @@ -186,7 +186,7 @@ class Report(Base): analysis = orm.relationship('Analysis', back_populates='report') @staticmethod - def save(db, analysis, analyzer_name, project_name): + def save(db, analysis, analyzer_name, package_name): try: results = analysis['results'] analyzer_report = Report.REPORTERS[analyzer_name](results) @@ -199,8 +199,8 @@ class Report(Base): db.session.add(_reports) db.session.commit() kiskadee.logger.debug( - "MONITOR: Saved analysis reports for {} project" - .format(project_name) + "MONITOR: Saved analysis reports for {} package" + .format(data["name"]) ) except KeyError as key: kiskadee.logger.debug( @@ -210,8 +210,8 @@ class Report(Base): ) except Exception as err: kiskadee.logger.debug( - "MONITOR: Failed to get analysis reports to {} project" - .format(project_name) + "MONITOR: Failed to get analysis reports to {} package" + .format(data["name"]) ) kiskadee.logger.debug(err) return diff --git a/kiskadee/monitor.py b/kiskadee/monitor.py index 710127a..5ff724b 100644 --- a/kiskadee/monitor.py +++ b/kiskadee/monitor.py @@ -1,6 +1,6 @@ """Provide kiskadee monitoring capabilities. -kiskadee monitors repositories checking for new project versions to be +kiskadee monitors repositories checking for new package versions to be analyzed. This module provides such capabilities. """ import threading @@ -9,7 +9,7 @@ import os import importlib import kiskadee -from kiskadee.model import Project, Fetcher, Version, Report, Analysis, Analyzer +from kiskadee.model import Package, Fetcher, Version, Report, Analysis, Analyzer RUNNING = True @@ -24,7 +24,7 @@ class Monitor: def run(self): """Run Monitor process. This method resumes Monitor behavior. It dequeue - projects from fetchers, check if the project is valid and if so send it + packages from fetchers, check if the package is valid and if so send it to Runner. When Runner ends a analysis, the Monitor dequeue the analysis and save it on database. """ kiskadee.logger.debug('MONITOR PID: {}'.format(os.getpid())) @@ -34,57 +34,56 @@ class Monitor: while RUNNING: kiskadee.logger.debug('MONITOR STATE: Idle'.format(os.getpid())) - new_project = self.dequeue_project_from_fetchers() - self.send_project_to_runner(new_project) - analyzed_project = self.dequeue_analysis_from_runner() - self.save_analyzed_project(analyzed_project) + new_package = self.dequeue_package_from_fetchers() + self.send_package_to_runner(new_package) + analyzed_package = self.dequeue_analysis_from_runner() + self.save_analyzed_package(analyzed_package) - def dequeue_project_from_fetchers(self): - return self.queues.dequeue_project() + def dequeue_package_from_fetchers(self): + return self.queues.dequeue_package() def dequeue_analysis_from_runner(self): return self.queues.dequeue_result() - def send_project_to_runner(self, data): + def send_package_to_runner(self, data): if data: - fetcher, project = self.get_fetcher_and_project(data) + fetcher, package = self.get_fetcher_and_package(data) data["fetcher_id"] = fetcher.id if fetcher else '' - if not self.is_a_new_project_version(project, data): - kiskadee.logger.debug('MONITOR STATE: Project {} already'\ + if not self.is_a_new_package_version(package, data): + kiskadee.logger.debug('MONITOR STATE: Package {} already'\ ' analyzed'.format(data['name'])) return self.queues.enqueue_analysis(data) - def get_fetcher_and_project(self, data): + def get_fetcher_and_package(self, data): fetcher_name = data['fetcher'].split('.')[-1] - project_name = data['name'] + package_name = data['name'] fetcher = self.db.filter_by_name(Fetcher, fetcher_name) - project = self.db.filter_by_name(Project, project_name) - return fetcher, project + package = self.db.filter_by_name(Package, package_name) + return fetcher, package - def is_a_new_project_version(self, project, data): - if project: - project_version = data['version'] - analysed_version = project.versions[-1].number + def is_a_new_package_version(self, package, data): + if package: + package_version = data['version'] + analysed_version = package.versions[-1].number fetcher = importlib.import_module(data['fetcher']).Fetcher() - return fetcher.compare_versions(project_version, analysed_version) + return fetcher.compare_versions(package_version, analysed_version) else: return True - def save_analyzed_project(self, data): + def save_analyzed_package(self, data): if not data: return {} - project = self.db.filter_by_name(Project, data['name']) - if not project: - project = Project.save(self.db, data) + package = self.db.filter_by_name(Package, data['name']) + if not package: + Package.save(self.db, data) else: - project = Project.update(self.db, project, data) - self.save_project_analysis(data, project) + Package.update(self.db, package, data) - def save_project_analysis(self, data, project): + def save_package_analysis(self, data, package): for analyzer, result in data['results'].items(): - Analysis.save(self.db, analyzer, result, project) + Analysis.save(self.db, analyzer, result, package) @staticmethod diff --git a/kiskadee/queue.py b/kiskadee/queue.py index 0d4956f..8eae00f 100644 --- a/kiskadee/queue.py +++ b/kiskadee/queue.py @@ -6,71 +6,71 @@ import kiskadee analysis = Queue() results = Queue() -projects = Queue() +packages = Queue() class Queues(): """Provide kiskadee queues objects.""" @staticmethod - def enqueue_analysis(project_to_analysis): + def enqueue_analysis(package_to_analysis): """Put a analysis on the analysis queue.""" - log_msg = "MONITOR STATE: Sending project {}-{} for analysis"\ - .format(project_to_analysis['name'], - project_to_analysis['version']) + log_msg = "MONITOR STATE: Sending package {}-{} for analysis"\ + .format(package_to_analysis['name'], + package_to_analysis['version']) kiskadee.logger.debug(log_msg) - analysis.put(project_to_analysis) + analysis.put(package_to_analysis) @staticmethod def dequeue_analysis(): """Get a analysis from the analysis queue.""" - project_to_analysis = analysis.get() - fetcher = project_to_analysis ['fetcher'].split('.')[-1] + package_to_analysis = analysis.get() + fetcher = package_to_analysis ['fetcher'].split('.')[-1] kiskadee.logger.debug( 'RUNNER STATE: dequeued {}-{} from {}' - .format(project_to_analysis['name'], - project_to_analysis['version'], + .format(package_to_analysis['name'], + package_to_analysis['version'], fetcher) ) - return project_to_analysis + return package_to_analysis @staticmethod - def enqueue_result(project): + def enqueue_result(package): """Put a result on the results queue.""" kiskadee.logger.debug( "RUNNER STATE: Sending {}-{} to Monitor" - .format(project["name"], - project["version"]) + .format(package["name"], + package["version"]) ) - results.put(project) + results.put(package) @staticmethod def dequeue_result(): """Get a result from the results queue.""" result = results.get() kiskadee.logger.debug( - "MONITOR STATE: Pick Up analyzed project" + "MONITOR STATE: Pick Up analyzed package" .format(result["name"], result["version"]) ) return result @staticmethod - def enqueue_project(project, fetcher=None): + def enqueue_package(package, fetcher=None): """Put a result on the results queue.""" if fetcher: kiskadee.logger.debug( "FETCHER {}: sending package {}-{} for monitor" - .format(fetcher, project['name'], project['version']) + .format(fetcher, package['name'], package['version']) ) - projects.put(project) + packages.put(package) @staticmethod - def dequeue_project(): + def dequeue_package(): """Get a result from the results queue.""" - project = projects.get() + package = packages.get() kiskadee.logger.debug( - "MONITOR STATE: Pick Up monitored project." - .format(project["name"], - project["version"]) + "MONITOR STATE: Pick Up monitored package." + .format(package["name"], + package["version"]) ) - return project + return package diff --git a/kiskadee/runner.py b/kiskadee/runner.py index e245552..14c874a 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -1,4 +1,4 @@ -"""Run each static analyzer in each project marked for analysis.""" +"""Run each static analyzer in each package marked for analysis.""" import shutil import tempfile @@ -20,39 +20,39 @@ class Runner: """Return a non initialized Runner.""" self.queues = queues self.fetcher = None - self.project = None + self.package = None def run(self): """Run static analyzers. - Continuously dequeue projects from `analyses_queue` and call the - :func:`analyze` method, passing the dequeued project as argument. + Continuously dequeue packages from `analyses_queue` and call the + :func:`analyze` method, passing the dequeued package as argument. After the analysis, send the result back to Monitor. """ kiskadee.logger.debug('RUNNER PID: {}'.format(os.getpid())) while RUNNING: kiskadee.logger.debug('RUNNER STATE: Idle.') - self.project = self.queues.dequeue_analysis() + self.package = self.queues.dequeue_analysis() self.call_analyzers() - def import_project_fetcher(self): + def import_package_fetcher(self): try: return importlib.import_module( - self.project['fetcher'] + self.package['fetcher'] ).Fetcher() except ModuleNotFoundError: kiskadee.logger.debug("RUNNER STATE: Fetcher {}\ - could not be loaded".format(self.project['fetcher']) + could not be loaded".format(self.package['fetcher']) ) return {} def run_analysis(self, analyzers, source_path): - self.project['results'] = {} + self.package['results'] = {} for analyzer in analyzers: firehose_report = self.analyze(analyzer, source_path) if firehose_report: - self.project['results'][analyzer] = firehose_report - return self.project + self.package['results'][analyzer] = firehose_report + return self.package def rmdtemp(self, temp_dir): @@ -67,8 +67,8 @@ class Runner: the function :func:`analyze`, passing the source dict, the analyzer to run the analysis, and the path to a compressed source. """ - self.fetcher = self.import_project_fetcher() - source_path = self.get_project_code_path() + self.fetcher = self.import_package_fetcher() + source_path = self.get_package_code_path() if not source_path: return None @@ -89,7 +89,7 @@ class Runner: The `analyzer` is the name of a static analyzer already created on the database. The `source_path` is the directory to a uncompressed source, returned - by the :func:`get_project_code_path`. + by the :func:`get_package_code_path`. """ if source_path is None: return None @@ -111,15 +111,15 @@ class Runner: kiskadee.logger.debug(err) return None - def get_project_code_path(self): + def get_package_code_path(self): """ Returns a string, representing the path of the uncompressed - project source.""" - if not (self.fetcher and self.project): + package source.""" + if not (self.fetcher and self.package): return None - compressed_source_path = self.fetcher.get_sources(self.project) + compressed_source_path = self.fetcher.get_sources(self.package) if compressed_source_path : - uncompressed_source_path = self.uncompress_project_code( + uncompressed_source_path = self.uncompress_package_code( compressed_source_path ) self.rmdtemp(os.path.dirname(compressed_source_path)) @@ -128,7 +128,7 @@ class Runner: kiskadee.logger.debug('RUNNER: invalid compressed source') return None - def uncompress_project_code(self, compressed_source): + def uncompress_package_code(self, compressed_source): dir_to_unpack_source = tempfile.mkdtemp() try: shutil.unpack_archive(compressed_source, @@ -136,7 +136,7 @@ class Runner: return dir_to_unpack_source except Exception as err: kiskadee.logger.debug('RUNNER STATE: Could not unpack'\ - 'project source') + 'package source') kiskadee.logger.debug(err) return {} diff --git a/kiskadee/tests/api/test_api.py b/kiskadee/tests/api/test_api.py index 919ae5f..cff7f1b 100644 --- a/kiskadee/tests/api/test_api.py +++ b/kiskadee/tests/api/test_api.py @@ -37,7 +37,7 @@ class ApiTestCase(unittest.TestCase): ) self.db.filter_by_name = MagicMock() - self.db.session.query(kiskadee.model.Project)\ + self.db.session.query(kiskadee.model.Package)\ .filter_by().id = MagicMock() self.db.session.query(kiskadee.model.Version)\ .filter_by().id = MagicMock() diff --git a/kiskadee/tests/integration/test_monitor.py b/kiskadee/tests/integration/test_monitor.py index 1017889..e7f6550 100644 --- a/kiskadee/tests/integration/test_monitor.py +++ b/kiskadee/tests/integration/test_monitor.py @@ -3,7 +3,7 @@ from unittest.mock import MagicMock from kiskadee.monitor import Monitor from kiskadee.queue import Queues -from kiskadee.model import Project, Fetcher +from kiskadee.model import Package, Fetcher import kiskadee.queue import kiskadee.fetchers.debian import kiskadee.fetchers.anitya @@ -14,7 +14,7 @@ class MonitorTestCase(unittest.TestCase): def setUp(self): def mocked_models(klass): - class Project: + class Package: def all(self): [self.pkg1, self.pkg2, self.pkg3, self.pkg4] klass() @@ -35,36 +35,36 @@ class MonitorTestCase(unittest.TestCase): 'flawfinder': '><'}, 'fetcher_id': 1} - def test_dequeue_project_from_fetcher(self): + def test_dequeue_package_from_fetcher(self): self.example_fetcher.watch() - monitored_project = self.monitor.dequeue_project_from_fetchers() - self.assertIn("version", monitored_project) - self.assertIn("name", monitored_project) - self.assertIn("fetcher", monitored_project) - self.assertEqual(monitored_project['version'], '0.1') + monitored_package = self.monitor.dequeue_package_from_fetchers() + self.assertIn("version", monitored_package) + self.assertIn("name", monitored_package) + self.assertIn("fetcher", monitored_package) + self.assertEqual(monitored_package['version'], '0.1') def test_run_fetchers_as_threads(self): Monitor.start_fetcher(self.example_fetcher.watch) Monitor.start_fetcher(self.example_fetcher.watch) - first_monitored_project = self.monitor.dequeue_project_from_fetchers() - second_monitored_project = self.monitor.dequeue_project_from_fetchers() - self.assertIn("version", first_monitored_project) - self.assertIn("version", second_monitored_project) - self.assertIn("name", first_monitored_project) - self.assertIn("name", second_monitored_project) - self.assertIn("fetcher", first_monitored_project) - self.assertIn("fetcher", second_monitored_project) + first_monitored_package = self.monitor.dequeue_package_from_fetchers() + second_monitored_package = self.monitor.dequeue_package_from_fetchers() + self.assertIn("version", first_monitored_package) + self.assertIn("version", second_monitored_package) + self.assertIn("name", first_monitored_package) + self.assertIn("name", second_monitored_package) + self.assertIn("fetcher", first_monitored_package) + self.assertIn("fetcher", second_monitored_package) - def test_send_project_to_runner(self): + def test_send_package_to_runner(self): fetcher = Fetcher(name='example') - Project(name='project1', fetcher_id=fetcher.id) - self.monitor.get_fetcher_and_project = MagicMock( + Package(name='package1', fetcher_id=fetcher.id) + self.monitor.get_fetcher_and_package = MagicMock( return_value=[fetcher, {}] ) - self.monitor.is_a_new_project_version = MagicMock( + self.monitor.is_a_new_package_version = MagicMock( return_value=True ) - self.monitor.send_project_to_runner(self.data1) + self.monitor.send_package_to_runner(self.data1) self.assertEqual(self.monitor.queues.dequeue_analysis(), self.data1) if __name__ == '__main__': diff --git a/kiskadee/tests/integration/test_runner.py b/kiskadee/tests/integration/test_runner.py index f8ed25b..c486bc4 100644 --- a/kiskadee/tests/integration/test_runner.py +++ b/kiskadee/tests/integration/test_runner.py @@ -23,7 +23,7 @@ class RunnerTestCase(unittest.TestCase): self.queues = kiskadee.queue.Queues() self.runner = kiskadee.runner.Runner(self.queues) - self.project = { + self.package = { 'name': 'test', 'version': '1.0.0', 'fetcher': kiskadee.fetchers.example.__name__ @@ -33,27 +33,27 @@ class RunnerTestCase(unittest.TestCase): """TODO: to be defined1. """ def test_run_analyzers(self): - self.runner.project = self.project + self.runner.package = self.package self.runner.call_analyzers() result = self.queues.dequeue_result() self.assertEqual(result['name'], 'test') self.assertIn('results', result) def test_run_a_single_analyzer(self): - self.runner.project = self.project - self.runner.fetcher = self.runner.import_project_fetcher() - self.runner.get_project_code_path() + self.runner.package = self.package + self.runner.fetcher = self.runner.import_package_fetcher() + self.runner.get_package_code_path() analyzer = self.runner.fetcher.analyzers()[0] self.assertIsNotNone(re.search('.*cppcheck.*', analyzer)) self.assertIsNone(re.search('.*pylint.*', analyzer)) - def test_analysis_an_incoming_monitor_project(self): + def test_analysis_an_incoming_monitor_package(self): self.db = MagicMock() self.db.filter_by_name = MagicMock(return_value={}) self.monitor = kiskadee.monitor.Monitor(self.db, self.queues) - self.monitor.send_project_to_runner(self.project) - incoming_project = self.runner.queues.dequeue_analysis() - self.assertEqual(self.project, incoming_project) + self.monitor.send_package_to_runner(self.package) + incoming_package = self.runner.queues.dequeue_analysis() + self.assertEqual(self.package, incoming_package) if __name__ == '__main__': diff --git a/kiskadee/tests/plugins/test_plugins.py b/kiskadee/tests/plugins/test_plugins.py index d9f532d..67ce20d 100644 --- a/kiskadee/tests/plugins/test_plugins.py +++ b/kiskadee/tests/plugins/test_plugins.py @@ -70,7 +70,7 @@ class DebianFetcherTestCase(unittest.TestCase): self.debian_fetcher._queue_sources_gz_pkgs(temp_dir) shutil.rmtree(temp_dir) - some_pkg = kiskadee.queue.Queues.dequeue_project() + some_pkg = kiskadee.queue.Queues.dequeue_package() self.assertTrue(isinstance(some_pkg, dict)) self.assertIn('name', some_pkg) self.assertIn('version', some_pkg) @@ -112,11 +112,11 @@ class TestAnityaFetcher(unittest.TestCase): import kiskadee.fetchers.anitya self.anitya_fetcher = kiskadee.fetchers.anitya.Fetcher() - self.msg = "anitya {'body':{'msg':{'project':{name: 'urlscan',"\ + self.msg = "anitya {'body':{'msg':{'package':{name: 'urlscan',"\ "'version':'0.8.5','backend':'GitHub',"\ "'homepage':'https://github.com/firecat53/urlscan'}}}}" - self.msg1 = "{'body':{'msg':{'project':{name: 'urlscan',"\ + self.msg1 = "{'body':{'msg':{'package':{name: 'urlscan',"\ "'version':'0.8.5','backend':'GitHub',"\ "'homepage':'https://github.com/firecat53/urlscan'}}}}" @@ -192,8 +192,8 @@ class TestAnityaFetcher(unittest.TestCase): def test_create_package_dict(self): - self.anitya_fetcher.project_to_enqueue(self.msg) - _dict = kiskadee.queue.Queues().dequeue_project() + self.anitya_fetcher.package_to_enqueue(self.msg) + _dict = kiskadee.queue.Queues().dequeue_package() self.assertEqual(_dict['name'], 'urlscan') self.assertEqual(_dict['version'], '0.8.5') self.assertEqual(_dict['meta']['backend'], 'GitHub') diff --git a/kiskadee/tests/units/test_model.py b/kiskadee/tests/units/test_model.py index 6fda0be..f542aaa 100644 --- a/kiskadee/tests/units/test_model.py +++ b/kiskadee/tests/units/test_model.py @@ -14,16 +14,16 @@ class ModelTestCase(unittest.TestCase): def setUp(self): kiskadee.tests.clean_test_db(self.db, model.Base.metadata) - self.project = model.Project(name='python-kiskadee') + self.package = model.Package(name='python-kiskadee') self.version = model.Version(number='1.0-rc1') self.fetcher = model.Fetcher( name='kiskadee-fetcher', target='university' ) self.db.session.add(self.fetcher) self.db.session.commit() - self.fetcher.projects.append(self.project) - self.project.versions.append(self.version) - self.db.session.add(self.project) + self.fetcher.packages.append(self.package) + self.package.versions.append(self.version) + self.db.session.add(self.package) self.db.session.add(self.fetcher) self.db.session.add(self.version) @@ -43,9 +43,9 @@ class ModelTestCase(unittest.TestCase): fetchers = self.db.session.query(model.Fetcher).all() self.assertEqual(fetchers, [self.fetcher]) - def test_query_project(self): - projects = self.db.session.query(model.Project).all() - self.assertEqual(projects, [self.project]) + def test_query_package(self): + packages = self.db.session.query(model.Package).all() + self.assertEqual(packages, [self.package]) def test_query_version(self): versions = self.db.session.query(model.Version).all() @@ -62,7 +62,7 @@ class ModelTestCase(unittest.TestCase): fetchers = self.db.session.query(model.Fetcher).all() self.assertEqual(len(fetchers), 2) - def test_add_version_without_project(self): + def test_add_version_without_package(self): version = model.Version(number='3.1') self.db.session.add(version) with self.assertRaises(exc.IntegrityError): @@ -75,51 +75,51 @@ class ModelTestCase(unittest.TestCase): with self.assertRaises(exc.IntegrityError): self.db.session.commit() - def test_add_project_without_fetcher(self): - project = model.Project(name='foo-bar') - self.db.session.add(project) + def test_add_package_without_fetcher(self): + package = model.Package(name='foo-bar') + self.db.session.add(package) with self.assertRaises(exc.IntegrityError): self.db.session.commit() - def test_unique_project_in_fetcher(self): - project_1 = model.Project(name='foo-bar') - project_2 = model.Project(name='foo-bar') - self.fetcher.projects.append(project_1) - self.fetcher.projects.append(project_2) + def test_unique_package_in_fetcher(self): + package_1 = model.Package(name='foo-bar') + package_2 = model.Package(name='foo-bar') + self.fetcher.packages.append(package_1) + self.fetcher.packages.append(package_2) with self.assertRaises(exc.IntegrityError): self.db.session.commit() - def test_unique_version_for_project(self): - project_version_1 = model.Version(number='1.0') - project_version_2 = model.Version(number='1.0') - self.project.versions.append(project_version_1) - self.project.versions.append(project_version_2) + def test_unique_version_for_package(self): + package_version_1 = model.Version(number='1.0') + package_version_2 = model.Version(number='1.0') + self.package.versions.append(package_version_1) + self.package.versions.append(package_version_2) with self.assertRaises(exc.IntegrityError): self.db.session.commit() def test_compose_kiskadee_source(self): _analyzer = self.db.session.query(model.Analyzer)\ .filter(model.Analyzer.name == "cppcheck").first() - project = model.Project( + package = model.Package( name='bla', fetcher_id=self.fetcher.id ) - project_version = model.Version( + package_version = model.Version( number='1.0.1', - project_id=project.id + package_id=package.id ) - project_analysis = model.Analysis( + package_analysis = model.Analysis( raw="<>", analyzer_id=_analyzer.id, - version_id=project_version.id + version_id=package_version.id ) - self.fetcher.projects.append(project) - project.versions.append(project_version) - project_version.analysis.append(project_analysis) + self.fetcher.packages.append(package) + package.versions.append(package_version) + package_version.analysis.append(package_analysis) - self.assertEqual(project.versions[0].analysis[0].raw, "<>") + self.assertEqual(package.versions[0].analysis[0].raw, "<>") def test_save_several_analysis(self): @@ -132,42 +132,42 @@ class ModelTestCase(unittest.TestCase): .filter(model.Analyzer.name == "flawfinder").first() ) - project = model.Project( + package = model.Package( name='bla', fetcher_id=self.fetcher.id ) - project_version = model.Version( + package_version = model.Version( number='1.0.1', - project_id=project.id + package_id=package.id ) - self.fetcher.projects.append(project) - project.versions.append(project_version) + self.fetcher.packages.append(package) + package.versions.append(package_version) - self.db.session.add(project) - self.db.session.add(project_version) + self.db.session.add(package) + self.db.session.add(package_version) self.db.session.commit() - project_analysis1 = model.Analysis( + package_analysis1 = model.Analysis( raw="<>", analyzer_id=_analyzer1.id, - version_id=project_version.id + version_id=package_version.id ) - project_analysis2 = model.Analysis( + package_analysis2 = model.Analysis( raw="><", analyzer_id=_analyzer2.id, - version_id=project_version.id + version_id=package_version.id ) - self.db.session.add(project_analysis1) - self.db.session.add(project_analysis2) + self.db.session.add(package_analysis1) + self.db.session.add(package_analysis2) self.db.session.commit() - saved_project = ( - self.db.session.query(model.Project) - .filter(model.Project.name == 'bla').first() + saved_package = ( + self.db.session.query(model.Package) + .filter(model.Package.name == 'bla').first() ) - analysis = saved_project.versions[-1].analysis + analysis = saved_package.versions[-1].analysis self.assertEqual(len(analysis), 2) self.assertEqual(analysis[0].raw, "<>") self.assertEqual(analysis[1].raw, "><") diff --git a/kiskadee/tests/units/test_queue.py b/kiskadee/tests/units/test_queue.py index 74b6f7e..3ef29f2 100644 --- a/kiskadee/tests/units/test_queue.py +++ b/kiskadee/tests/units/test_queue.py @@ -7,17 +7,17 @@ class QueueTestCase(unittest.TestCase): def setUp(self): self.queues = kiskadee.queue.Queues() - self.project = { + self.package = { 'name': 'bar', 'fetcher': kiskadee.fetchers.example.Fetcher(), 'version': '1.0.0' } - def test_enqueue_dequeue_project(self): - self.queues.enqueue_project(self.project) - _project = self.queues.dequeue_project() - self.assertTrue(isinstance(_project, dict)) - self.assertEqual(_project['name'], 'bar') + def test_enqueue_dequeue_package(self): + self.queues.enqueue_package(self.package) + _package = self.queues.dequeue_package() + self.assertTrue(isinstance(_package, dict)) + self.assertEqual(_package['name'], 'bar') if __name__ == '__main__': diff --git a/kiskadee/tests/units/test_runner.py b/kiskadee/tests/units/test_runner.py index 502c969..3b6d404 100644 --- a/kiskadee/tests/units/test_runner.py +++ b/kiskadee/tests/units/test_runner.py @@ -13,6 +13,6 @@ class RunnerTestCase(unittest.TestCase): def test_path_to_uncompressed_source(self): self.runner.fetcher = kiskadee.fetchers.example.Fetcher() - self.runner.project = {'name': 'test'} - uncompressed_source_path = self.runner.get_project_code_path() + self.runner.package = {'name': 'test'} + uncompressed_source_path = self.runner.get_package_code_path() self.assertIsNotNone(uncompressed_source_path)