Bumped to 0.6.0

* Downloads pip requirements
* Modified Makefile

Makefile

@@ -1,4 +1,5 @@
 PYVERSIONS=3.4 2.7
+PYMAIN=$(firstword $(PYVERSIONS))
 NAME=senpy
 REPO=gsiupm
 VERSION=$(shell cat $(NAME)/VERSION)
@@ -11,28 +12,38 @@ dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS))
 Dockerfile-%: Dockerfile.template
 	sed "s/{{PYVERSION}}/$*/" Dockerfile.template > Dockerfile-$*
 
-build: $(addprefix build-, $(PYVERSIONS))
+build: $(addprefix build-, $(PYMAIN))
+
+buildall: $(addprefix build-, $(PYVERSIONS))
 
 build-%: Dockerfile-%
 	docker build -t '$(REPO)/$(NAME):$(VERSION)-python$*' -f Dockerfile-$* .;
 
-test: $(addprefix test-,$(PYVERSIONS))
+test: $(addprefix test-,$(PYMAIN))
+
+testall: $(addprefix test-,$(PYVERSIONS))
 
 test-%: build-%
-	docker run --rm -w /usr/src/app/ --entrypoint=/usr/local/bin/python -ti '$(REPO)/$(NAME):$(VERSION)-python$*' setup.py test ;
+	docker run --rm -w /usr/src/app/ --entrypoint=/usr/local/bin/python -ti '$(REPO)/$(NAME):$(VERSION)-python$*' setup.py test --addopts "-vvv -s --pdb" ;
 
-test_pip-%:
+pip_test-%:
-	docker run --rm -ti python:$* pip -q install senpy ;
+	docker run --rm -ti python:$* pip install senpy ;
 
-upload-%:
+upload-%: test-%
-	docker push '$(REPO)/$(NAME):$(VERSION)-python$(firstword $(PYVERSIONS))'
+	docker push '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)'
 
-upload: test $(addprefix upload-,$(PYVERSIONS))
+upload: testall $(addprefix upload-,$(PYVERSIONS))
-	docker tag '$(REPO)/$(NAME):$(VERSION)-python$(firstword $(PYVERSIONS))' '$(REPO)/$(NAME):$(VERSION)'
+	docker tag '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)' '$(REPO)/$(NAME):$(VERSION)'
-	docker tag '$(REPO)/$(NAME):$(VERSION)-python$(firstword $(PYVERSIONS))' '$(REPO)/$(NAME)'
+	docker tag '$(REPO)/$(NAME):$(VERSION)-python$(PYVERSIONS)' '$(REPO)/$(NAME)'
-	docker push '$(REPO)/$(NAME):$(VERSION)'
+	docker push '$(REPO)/$(NAME):$(VERSION)' docker push '$(REPO)/$(NAME)'
-	docker push '$(REPO)/$(NAME)'
+	python setup.py sdist upload
 
-test_pip: $(addprefix test_pip-,$(PYVERSIONS))
+pip_upload:
+	python setup.py sdist upload ;
 
-.PHONY: test test-% build-% build test test_pip
+pip_test: $(addprefix pip_test-,$(PYVERSIONS))
+
+run: build
+	docker run --rm -p 5000:5000 -ti '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)'
+
+.PHONY: test test-% build-% build test test_pip run

requirements.txt

@@ -4,7 +4,6 @@ requests>=2.4.1
 GitPython>=0.3.2.RC1
 gevent>=1.1rc4
 PyLD>=0.6.5
-Flask-Testing>=0.4.2
 six
 future
 jsonschema

senpy/VERSION

@@ -1 +1 @@
-0.5.7
+0.6.0

senpy/extensions.py

@@ -23,6 +23,7 @@ import logging
 import traceback
 import gevent
 import yaml
+import pip
 
 logger = logging.getLogger(__name__)
 
@@ -198,18 +199,29 @@ class Senpy(object):
             logger.error('Error reloading {}: {}'.format(name, ex))
             self.plugins[name] = plugin
 
-    @staticmethod
+
-    def _load_plugin(root, filename):
+    @classmethod
-        logger.debug("Loading plugin: {}".format(filename))
+    def validate_info(cls, info):
-        fpath = os.path.join(root, filename)
+        return all(x in info for x in ('name', 'module', 'version'))
-        with open(fpath, 'r') as f:
+
-            info = yaml.load(f)
+    @classmethod
-        logger.debug("Info: {}".format(info))
+    def _load_plugin_from_info(cls, info, root):
-        sys.path.append(root)
+        if not cls.validate_info(info):
+            logger.warn('The module info is not valid.\n\t{}'.format(info))
+            return None, None 
         module = info["module"]
         name = info["name"]
+        requirements = info.get("requirements", [])
+        sys.path.append(root)
         (fp, pathname, desc) = imp.find_module(module, [root, ])
         try:
+            if requirements:
+                pip_args = []
+                pip_args.append('install')
+                for req in requirements:
+                    pip_args.append( req )
+                logger.info('Installing requirements: ' + str(requirements))
+                pip.main(pip_args) 
             tmp = imp.load_module(module, fp, pathname, desc)
             sys.path.remove(root)
             candidate = None
@@ -221,20 +233,30 @@ class Senpy(object):
                     candidate = obj
                     break
             if not candidate:
-                logger.debug("No valid plugin for: {}".format(filename))
+                logger.debug("No valid plugin for: {}".format(module))
                 return
             module = candidate(info=info)
-            try:
+            repo_path = root
-                repo_path = root
+            module._repo = Repo(repo_path)
-                module._repo = Repo(repo_path)
+        except InvalidGitRepositoryError:
-            except InvalidGitRepositoryError:
+            logger.debug("The plugin {} is not in a Git repository".format(module))
-                module._repo = None
+            module._repo = None
         except Exception as ex:
-            logger.error("Exception importing {}: {}".format(filename, ex))
+            logger.error("Exception importing {}: {}".format(module, ex))
             logger.error("Trace: {}".format(traceback.format_exc()))
             return None, None
         return name, module
 
+    @classmethod
+    def _load_plugin(cls, root, filename):
+        fpath = os.path.join(root, filename)
+        logger.debug("Loading plugin: {}".format(fpath))
+        with open(fpath, 'r') as f:
+            info = yaml.load(f)
+        logger.debug("Info: {}".format(info))
+        return cls._load_plugin_from_info(info, root)
+
+
     def _load_plugins(self):
         plugins = {}
         for search_folder in self._search_folders:

senpy/plugins.py

@@ -91,4 +91,4 @@ class ShelfMixin(object):
         if hasattr(self, '_sh') and self._sh is not None:
             with open(self.shelf_file, 'wb') as f:
                 pickle.dump(self._sh, f)
-        del(self.__dict__['_sh'])
+            del(self.__dict__['_sh'])

tests/test_blueprints.py

@@ -1,9 +1,10 @@
 import os
 import logging
+import json
 
 from senpy.extensions import Senpy
 from flask import Flask
-from flask.ext.testing import TestCase
+from unittest import TestCase
 from gevent import sleep
 from itertools import product
 
@@ -11,31 +12,38 @@ from itertools import product
 def check_dict(indic, template):
     return all(item in indic.items() for item in template.items())
 
+def parse_resp(resp):
+    return json.loads(resp.data.decode('utf-8'))
+
 
 class BlueprintsTest(TestCase):
 
-    def create_app(self):
+    def setUp(self):
         self.app = Flask("test_extensions")
+        self.client = self.app.test_client()
         self.senpy = Senpy()
         self.senpy.init_app(self.app)
         self.dir = os.path.join(os.path.dirname(__file__), "..")
         self.senpy.add_folder(self.dir)
         self.senpy.activate_plugin("Dummy", sync=True)
-        return self.app
 
+    def assertCode(self, resp, code):
+        self.assertEqual(resp.status_code, code)
+        
     def test_home(self):
         """
         Calling with no arguments should ask the user for more arguments
         """
         resp = self.client.get("/api/")
-        self.assert404(resp)
+        self.assertCode(resp, 404)
-        logging.debug(resp.json)
+        js = parse_resp(resp)
-        assert resp.json["status"] == 404
+        logging.debug(js)
+        assert js["status"] == 404
         atleast = {
             "status": 404,
             "message": "Missing or invalid parameters",
         }
-        assert check_dict(resp.json, atleast)
+        assert check_dict(js, atleast)
 
     def test_analysis(self):
         """
@@ -43,81 +51,93 @@ class BlueprintsTest(TestCase):
         it should contain the context
         """
         resp = self.client.get("/api/?i=My aloha mohame")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
-        logging.debug("Got response: %s", resp.json)
+        js = parse_resp(resp)
-        assert "@context" in resp.json
+        logging.debug("Got response: %s", js)
-        assert "entries" in resp.json
+        assert "@context" in js
+        assert "entries" in js 
 
     def test_list(self):
         """ List the plugins """
         resp = self.client.get("/api/plugins/")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
-        logging.debug(resp.json)
+        js = parse_resp(resp)
-        assert 'plugins' in resp.json
+        logging.debug(js)
-        plugins = resp.json['plugins']
+        assert 'plugins' in js
+        plugins = js['plugins']
         assert len(plugins) > 1
         assert list(p for p in plugins if p['name'] == "Dummy")
-        assert "@context" in resp.json
+        assert "@context" in js
 
     def test_headers(self):
         for i, j in product(["/api/plugins/?nothing=", "/api/?i=test&"],
                             ["inHeaders"]):
             resp = self.client.get("%s" % (i))
-            assert "@context" in resp.json
+            js = parse_resp(resp)
+            assert "@context" in js
             resp = self.client.get("%s&%s=0" % (i, j))
-            assert "@context" in resp.json
+            js = parse_resp(resp)
+            assert "@context" in js 
             resp = self.client.get("%s&%s=1" % (i, j))
-            assert "@context" not in resp.json
+            js = parse_resp(resp)
+            assert "@context" not in js 
             resp = self.client.get("%s&%s=true" % (i, j))
-            assert "@context" not in resp.json
+            js = parse_resp(resp)
+            assert "@context" not in js 
 
     def test_detail(self):
         """ Show only one plugin"""
         resp = self.client.get("/api/plugins/Dummy/")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
-        logging.debug(resp.json)
+        js = parse_resp(resp)
-        assert "@id" in resp.json
+        logging.debug(js)
-        assert resp.json["@id"] == "Dummy_0.1"
+        assert "@id" in js
+        assert js["@id"] == "Dummy_0.1"
 
     def test_activate(self):
         """ Activate and deactivate one plugin """
         resp = self.client.get("/api/plugins/Dummy/deactivate")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
         sleep(0.5)
         resp = self.client.get("/api/plugins/Dummy/")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
-        assert "is_activated" in resp.json
+        js = parse_resp(resp)
-        assert resp.json["is_activated"] == False
+        assert "is_activated" in js
+        assert js["is_activated"] == False
         resp = self.client.get("/api/plugins/Dummy/activate")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
         sleep(0.5)
         resp = self.client.get("/api/plugins/Dummy/")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
-        assert "is_activated" in resp.json
+        js = parse_resp(resp)
-        assert resp.json["is_activated"] == True
+        assert "is_activated" in js 
+        assert js["is_activated"] == True
 
     def test_default(self):
         """ Show only one plugin"""
         resp = self.client.get("/api/plugins/default/")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
-        logging.debug(resp.json)
+        js = parse_resp(resp)
-        assert "@id" in resp.json
+        logging.debug(js)
-        assert resp.json["@id"] == "Dummy_0.1"
+        assert "@id" in js 
+        assert js["@id"] == "Dummy_0.1"
         resp = self.client.get("/api/plugins/Dummy/deactivate")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
         sleep(0.5)
         resp = self.client.get("/api/plugins/default/")
-        self.assert404(resp)
+        self.assertCode(resp, 404)
 
     def test_context(self):
         resp = self.client.get("/api/contexts/context.jsonld")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
-        assert "@context" in resp.json
+        js = parse_resp(resp)
+        assert "@context" in js
         assert check_dict(
-            resp.json["@context"],
+            js["@context"],
             {"marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#"})
 
     def test_schema(self):
         resp = self.client.get("/api/schemas/definitions.json")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
-        assert "$schema" in resp.json
+        js = parse_resp(resp)
+        assert "$schema" in js 

tests/test_extensions.py

@@ -6,18 +6,17 @@ from functools import partial
 from senpy.extensions import Senpy
 from senpy.models import Error
 from flask import Flask
-from flask.ext.testing import TestCase
+from unittest import TestCase
 
 
 class ExtensionsTest(TestCase):
 
-    def create_app(self):
+    def setUp(self):
         self.app = Flask("test_extensions")
         self.dir = os.path.join(os.path.dirname(__file__))
         self.senpy = Senpy(plugin_folder=self.dir, default_plugins=False)
         self.senpy.init_app(self.app)
         self.senpy.activate_plugin("Dummy", sync=True)
-        return self.app
 
     def test_init(self):
         """ Initialising the app with the extension.  """
@@ -34,6 +33,20 @@ class ExtensionsTest(TestCase):
         assert "Dummy" in self.senpy.plugins
 
     def test_enabling(self):
+        """ Enabling a plugin """
+        info = {
+            'name': 'TestPip',
+            'module': 'dummy',
+            'requirements': ['noop'],
+            'version': 0
+            }
+        root = os.path.join(self.dir, 'dummy_plugin')
+        name, module = self.senpy._load_plugin_from_info(info, root=root)
+        assert name == 'TestPip'
+        assert module
+        import noop
+
+    def test_installing(self):
         """ Enabling a plugin """
         self.senpy.activate_all(sync=True)
         assert len(self.senpy.plugins) == 2