diff --git a/Makefile b/Makefile index 516d8df..87d9e91 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ PYVERSIONS=3.4 2.7 +PYMAIN=$(firstword $(PYVERSIONS)) NAME=senpy REPO=gsiupm VERSION=$(shell cat $(NAME)/VERSION) @@ -11,28 +12,38 @@ dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS)) Dockerfile-%: Dockerfile.template sed "s/{{PYVERSION}}/$*/" Dockerfile.template > Dockerfile-$* -build: $(addprefix build-, $(PYVERSIONS)) +build: $(addprefix build-, $(PYMAIN)) + +buildall: $(addprefix build-, $(PYVERSIONS)) build-%: Dockerfile-% docker build -t '$(REPO)/$(NAME):$(VERSION)-python$*' -f Dockerfile-$* .; -test: $(addprefix test-,$(PYVERSIONS)) +test: $(addprefix test-,$(PYMAIN)) + +testall: $(addprefix test-,$(PYVERSIONS)) test-%: build-% - docker run --rm -w /usr/src/app/ --entrypoint=/usr/local/bin/python -ti '$(REPO)/$(NAME):$(VERSION)-python$*' setup.py test ; + docker run --rm -w /usr/src/app/ --entrypoint=/usr/local/bin/python -ti '$(REPO)/$(NAME):$(VERSION)-python$*' setup.py test --addopts "-vvv -s --pdb" ; + +pip_test-%: + docker run --rm -ti python:$* pip install senpy ; + +upload-%: test-% + docker push '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)' -test_pip-%: - docker run --rm -ti python:$* pip -q install senpy ; +upload: testall $(addprefix upload-,$(PYVERSIONS)) + docker tag '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)' '$(REPO)/$(NAME):$(VERSION)' + docker tag '$(REPO)/$(NAME):$(VERSION)-python$(PYVERSIONS)' '$(REPO)/$(NAME)' + docker push '$(REPO)/$(NAME):$(VERSION)' docker push '$(REPO)/$(NAME)' + python setup.py sdist upload -upload-%: - docker push '$(REPO)/$(NAME):$(VERSION)-python$(firstword $(PYVERSIONS))' +pip_upload: + python setup.py sdist upload ; -upload: test $(addprefix upload-,$(PYVERSIONS)) - docker tag '$(REPO)/$(NAME):$(VERSION)-python$(firstword $(PYVERSIONS))' '$(REPO)/$(NAME):$(VERSION)' - docker tag '$(REPO)/$(NAME):$(VERSION)-python$(firstword $(PYVERSIONS))' '$(REPO)/$(NAME)' - docker push '$(REPO)/$(NAME):$(VERSION)' - docker push '$(REPO)/$(NAME)' +pip_test: $(addprefix pip_test-,$(PYVERSIONS)) -test_pip: $(addprefix test_pip-,$(PYVERSIONS)) +run: build + docker run --rm -p 5000:5000 -ti '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)' -.PHONY: test test-% build-% build test test_pip +.PHONY: test test-% build-% build test test_pip run diff --git a/requirements.txt b/requirements.txt index e8d2f7b..8a94cf5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ requests>=2.4.1 GitPython>=0.3.2.RC1 gevent>=1.1rc4 PyLD>=0.6.5 -Flask-Testing>=0.4.2 six future jsonschema diff --git a/senpy/VERSION b/senpy/VERSION index dc2b74e..09a3acf 100644 --- a/senpy/VERSION +++ b/senpy/VERSION @@ -1 +1 @@ -0.5.7 \ No newline at end of file +0.6.0 \ No newline at end of file diff --git a/senpy/extensions.py b/senpy/extensions.py index 987928f..6e297bc 100644 --- a/senpy/extensions.py +++ b/senpy/extensions.py @@ -23,6 +23,7 @@ import logging import traceback import gevent import yaml +import pip logger = logging.getLogger(__name__) @@ -198,18 +199,29 @@ class Senpy(object): logger.error('Error reloading {}: {}'.format(name, ex)) self.plugins[name] = plugin - @staticmethod - def _load_plugin(root, filename): - logger.debug("Loading plugin: {}".format(filename)) - fpath = os.path.join(root, filename) - with open(fpath, 'r') as f: - info = yaml.load(f) - logger.debug("Info: {}".format(info)) - sys.path.append(root) + + @classmethod + def validate_info(cls, info): + return all(x in info for x in ('name', 'module', 'version')) + + @classmethod + def _load_plugin_from_info(cls, info, root): + if not cls.validate_info(info): + logger.warn('The module info is not valid.\n\t{}'.format(info)) + return None, None module = info["module"] name = info["name"] + requirements = info.get("requirements", []) + sys.path.append(root) (fp, pathname, desc) = imp.find_module(module, [root, ]) try: + if requirements: + pip_args = [] + pip_args.append('install') + for req in requirements: + pip_args.append( req ) + logger.info('Installing requirements: ' + str(requirements)) + pip.main(pip_args) tmp = imp.load_module(module, fp, pathname, desc) sys.path.remove(root) candidate = None @@ -221,20 +233,30 @@ class Senpy(object): candidate = obj break if not candidate: - logger.debug("No valid plugin for: {}".format(filename)) + logger.debug("No valid plugin for: {}".format(module)) return module = candidate(info=info) - try: - repo_path = root - module._repo = Repo(repo_path) - except InvalidGitRepositoryError: - module._repo = None + repo_path = root + module._repo = Repo(repo_path) + except InvalidGitRepositoryError: + logger.debug("The plugin {} is not in a Git repository".format(module)) + module._repo = None except Exception as ex: - logger.error("Exception importing {}: {}".format(filename, ex)) + logger.error("Exception importing {}: {}".format(module, ex)) logger.error("Trace: {}".format(traceback.format_exc())) return None, None return name, module + @classmethod + def _load_plugin(cls, root, filename): + fpath = os.path.join(root, filename) + logger.debug("Loading plugin: {}".format(fpath)) + with open(fpath, 'r') as f: + info = yaml.load(f) + logger.debug("Info: {}".format(info)) + return cls._load_plugin_from_info(info, root) + + def _load_plugins(self): plugins = {} for search_folder in self._search_folders: diff --git a/senpy/plugins.py b/senpy/plugins.py index 65bb5f1..1934843 100644 --- a/senpy/plugins.py +++ b/senpy/plugins.py @@ -91,4 +91,4 @@ class ShelfMixin(object): if hasattr(self, '_sh') and self._sh is not None: with open(self.shelf_file, 'wb') as f: pickle.dump(self._sh, f) - del(self.__dict__['_sh']) + del(self.__dict__['_sh']) diff --git a/tests/test_blueprints.py b/tests/test_blueprints.py index 7415004..454324d 100644 --- a/tests/test_blueprints.py +++ b/tests/test_blueprints.py @@ -1,9 +1,10 @@ import os import logging +import json from senpy.extensions import Senpy from flask import Flask -from flask.ext.testing import TestCase +from unittest import TestCase from gevent import sleep from itertools import product @@ -11,31 +12,38 @@ from itertools import product def check_dict(indic, template): return all(item in indic.items() for item in template.items()) +def parse_resp(resp): + return json.loads(resp.data.decode('utf-8')) + class BlueprintsTest(TestCase): - def create_app(self): + def setUp(self): self.app = Flask("test_extensions") + self.client = self.app.test_client() self.senpy = Senpy() self.senpy.init_app(self.app) self.dir = os.path.join(os.path.dirname(__file__), "..") self.senpy.add_folder(self.dir) self.senpy.activate_plugin("Dummy", sync=True) - return self.app + def assertCode(self, resp, code): + self.assertEqual(resp.status_code, code) + def test_home(self): """ Calling with no arguments should ask the user for more arguments """ resp = self.client.get("/api/") - self.assert404(resp) - logging.debug(resp.json) - assert resp.json["status"] == 404 + self.assertCode(resp, 404) + js = parse_resp(resp) + logging.debug(js) + assert js["status"] == 404 atleast = { "status": 404, "message": "Missing or invalid parameters", } - assert check_dict(resp.json, atleast) + assert check_dict(js, atleast) def test_analysis(self): """ @@ -43,81 +51,93 @@ class BlueprintsTest(TestCase): it should contain the context """ resp = self.client.get("/api/?i=My aloha mohame") - self.assert200(resp) - logging.debug("Got response: %s", resp.json) - assert "@context" in resp.json - assert "entries" in resp.json + self.assertCode(resp, 200) + js = parse_resp(resp) + logging.debug("Got response: %s", js) + assert "@context" in js + assert "entries" in js def test_list(self): """ List the plugins """ resp = self.client.get("/api/plugins/") - self.assert200(resp) - logging.debug(resp.json) - assert 'plugins' in resp.json - plugins = resp.json['plugins'] + self.assertCode(resp, 200) + js = parse_resp(resp) + logging.debug(js) + assert 'plugins' in js + plugins = js['plugins'] assert len(plugins) > 1 assert list(p for p in plugins if p['name'] == "Dummy") - assert "@context" in resp.json + assert "@context" in js def test_headers(self): for i, j in product(["/api/plugins/?nothing=", "/api/?i=test&"], ["inHeaders"]): resp = self.client.get("%s" % (i)) - assert "@context" in resp.json + js = parse_resp(resp) + assert "@context" in js resp = self.client.get("%s&%s=0" % (i, j)) - assert "@context" in resp.json + js = parse_resp(resp) + assert "@context" in js resp = self.client.get("%s&%s=1" % (i, j)) - assert "@context" not in resp.json + js = parse_resp(resp) + assert "@context" not in js resp = self.client.get("%s&%s=true" % (i, j)) - assert "@context" not in resp.json + js = parse_resp(resp) + assert "@context" not in js def test_detail(self): """ Show only one plugin""" resp = self.client.get("/api/plugins/Dummy/") - self.assert200(resp) - logging.debug(resp.json) - assert "@id" in resp.json - assert resp.json["@id"] == "Dummy_0.1" + self.assertCode(resp, 200) + js = parse_resp(resp) + logging.debug(js) + assert "@id" in js + assert js["@id"] == "Dummy_0.1" def test_activate(self): """ Activate and deactivate one plugin """ resp = self.client.get("/api/plugins/Dummy/deactivate") - self.assert200(resp) + self.assertCode(resp, 200) sleep(0.5) resp = self.client.get("/api/plugins/Dummy/") - self.assert200(resp) - assert "is_activated" in resp.json - assert resp.json["is_activated"] == False + self.assertCode(resp, 200) + js = parse_resp(resp) + assert "is_activated" in js + assert js["is_activated"] == False resp = self.client.get("/api/plugins/Dummy/activate") - self.assert200(resp) + self.assertCode(resp, 200) sleep(0.5) resp = self.client.get("/api/plugins/Dummy/") - self.assert200(resp) - assert "is_activated" in resp.json - assert resp.json["is_activated"] == True + self.assertCode(resp, 200) + js = parse_resp(resp) + assert "is_activated" in js + assert js["is_activated"] == True def test_default(self): """ Show only one plugin""" resp = self.client.get("/api/plugins/default/") - self.assert200(resp) - logging.debug(resp.json) - assert "@id" in resp.json - assert resp.json["@id"] == "Dummy_0.1" + self.assertCode(resp, 200) + js = parse_resp(resp) + logging.debug(js) + assert "@id" in js + assert js["@id"] == "Dummy_0.1" resp = self.client.get("/api/plugins/Dummy/deactivate") - self.assert200(resp) + self.assertCode(resp, 200) sleep(0.5) resp = self.client.get("/api/plugins/default/") - self.assert404(resp) + self.assertCode(resp, 404) def test_context(self): resp = self.client.get("/api/contexts/context.jsonld") - self.assert200(resp) - assert "@context" in resp.json + self.assertCode(resp, 200) + js = parse_resp(resp) + assert "@context" in js assert check_dict( - resp.json["@context"], + js["@context"], {"marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#"}) def test_schema(self): resp = self.client.get("/api/schemas/definitions.json") - self.assert200(resp) - assert "$schema" in resp.json + self.assertCode(resp, 200) + js = parse_resp(resp) + assert "$schema" in js diff --git a/tests/test_extensions.py b/tests/test_extensions.py index d95094e..9198b65 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -6,18 +6,17 @@ from functools import partial from senpy.extensions import Senpy from senpy.models import Error from flask import Flask -from flask.ext.testing import TestCase +from unittest import TestCase class ExtensionsTest(TestCase): - def create_app(self): + def setUp(self): self.app = Flask("test_extensions") self.dir = os.path.join(os.path.dirname(__file__)) self.senpy = Senpy(plugin_folder=self.dir, default_plugins=False) self.senpy.init_app(self.app) self.senpy.activate_plugin("Dummy", sync=True) - return self.app def test_init(self): """ Initialising the app with the extension. """ @@ -34,6 +33,20 @@ class ExtensionsTest(TestCase): assert "Dummy" in self.senpy.plugins def test_enabling(self): + """ Enabling a plugin """ + info = { + 'name': 'TestPip', + 'module': 'dummy', + 'requirements': ['noop'], + 'version': 0 + } + root = os.path.join(self.dir, 'dummy_plugin') + name, module = self.senpy._load_plugin_from_info(info, root=root) + assert name == 'TestPip' + assert module + import noop + + def test_installing(self): """ Enabling a plugin """ self.senpy.activate_all(sync=True) assert len(self.senpy.plugins) == 2