Fix bug in py3.5

This also includes a couple of changes URIs to pass the tests with python 3.7

Closes #50

docker-compose.dev.yml

@@ -0,0 +1,10 @@
+version: '3'
+services:
+    senpy:
+        image: "${IMAGENAME-gsiupm/senpy}:${VERSION-latest}"
+        entrypoint: ["/bin/bash"]
+        working_dir: "/senpy-plugins"
+        ports:
+            - 5000:5000
+        volumes:
+            - ".:/usr/src/app/"

docker-compose.test.yml

@@ -0,0 +1,9 @@
+version: '3'
+services:
+    test:
+        image: "${IMAGENAME-gsiupm/senpy}:${VERSION-dev}"
+        entrypoint: ["py.test"]
+        volumes:
+            - ".:/usr/src/app/"
+        command:
+          []

docker-compose.yml

@@ -0,0 +1,11 @@
+version: '3'
+services:
+    senpy:
+        image: "${IMAGENAME-gsiupm/senpy}:${VERSION-dev}"
+        build:
+          context: .
+          dockerfile: Dockerfile${PYVERSION--2.7}
+        ports:
+            - 5001:5000
+        volumes:
+            - "./data:/data"

example-plugins/async_plugin.py

@@ -11,7 +11,7 @@ class Async(AnalysisPlugin):
     '''An example of an asynchronous module'''
     author = '@balkian'
     version = '0.2'
-    async = True
+    sync = False
 
     def _do_async(self, num_processes):
         pool = multiprocessing.Pool(processes=num_processes)

requirements.txt

@@ -12,3 +12,4 @@ rdflib-jsonld
 numpy
 scipy
 scikit-learn
+responses

senpy/__main__.py

@@ -78,10 +78,15 @@ def main():
         help='Do not run a server, only install plugin dependencies')
     parser.add_argument(
         '--only-test',
-        '-t',
         action='store_true',
         default=False,
         help='Do not run a server, just test all plugins')
+    parser.add_argument(
+        '--test',
+        '-t',
+        action='store_true',
+        default=False,
+        help='Test all plugins before launching the server')
     parser.add_argument(
         '--only-list',
         '--list',
@@ -99,12 +104,24 @@ def main():
         action='store_false',
         default=True,
         help='Run a threaded server')
+    parser.add_argument(
+        '--no-deps',
+        '-n',
+        action='store_true',
+        default=False,
+        help='Skip installing dependencies')
     parser.add_argument(
         '--version',
         '-v',
         action='store_true',
         default=False,
         help='Output the senpy version and exit')
+    parser.add_argument(
+        '--allow-fail',
+        '--fail',
+        action='store_true',
+        default=False,
+        help='Do not exit if some plugins fail to activate')
     args = parser.parse_args()
     if args.version:
         print('Senpy version {}'.format(senpy.__version__))
@@ -119,19 +136,27 @@ def main():
                data_folder=args.data_folder)
     if args.only_list:
         plugins = sp.plugins()
-        maxwidth = max(len(x.id) for x in plugins)
+        maxname = max(len(x.name) for x in plugins)
+        maxversion = max(len(x.version) for x in plugins)
+        print('Found {} plugins:'.format(len(plugins)))
         for plugin in plugins:
             import inspect
             fpath = inspect.getfile(plugin.__class__)
-            print('{: <{width}} @ {}'.format(plugin.id, fpath, width=maxwidth))
+            print('\t{: <{maxname}} @ {: <{maxversion}} -> {}'.format(plugin.name,
+                                                                      plugin.version,
+                                                                      fpath,
+                                                                      maxname=maxname,
+                                                                      maxversion=maxversion))
         return
-    sp.install_deps()
+    if not args.no_deps:
+        sp.install_deps()
     if args.only_install:
         return
-    sp.activate_all()
-    if args.only_test:
+    sp.activate_all(allow_fail=args.allow_fail)
+    if args.test or args.only_test:
         easy_test(sp.plugins(), debug=args.debug)
-        return
+        if args.only_test:
+            return
     print('Senpy version {}'.format(senpy.__version__))
     print('Server running on port %s:%d. Ctrl+C to quit' % (args.host,
                                                             args.port))

senpy/api.py

@@ -3,6 +3,10 @@ from .models import Error, Results, Entry, from_string
 import logging
 logger = logging.getLogger(__name__)
 
+
+boolean = [True, False]
+
+
 API_PARAMS = {
     "algorithm": {
         "aliases": ["algorithms", "a", "algo"],
@@ -13,14 +17,14 @@ API_PARAMS = {
     "expanded-jsonld": {
         "@id": "expanded-jsonld",
         "aliases": ["expanded"],
-        "options": "boolean",
+        "options": boolean,
         "required": True,
         "default": False
     },
     "with_parameters": {
         "aliases": ['withparameters',
                     'with-parameters'],
-        "options": "boolean",
+        "options": boolean,
         "default": False,
         "required": True
     },
@@ -29,14 +33,14 @@ API_PARAMS = {
         "aliases": ["o"],
         "default": "json-ld",
         "required": True,
-        "options": ["json-ld", "turtle"],
+        "options": ["json-ld", "turtle", "ntriples"],
     },
     "help": {
         "@id": "help",
         "description": "Show additional help to know more about the possible parameters",
         "aliases": ["h"],
         "required": True,
-        "options": "boolean",
+        "options": boolean,
         "default": False
     },
     "emotionModel": {
@@ -83,7 +87,7 @@ WEB_PARAMS = {
         "aliases": ["headers"],
         "required": True,
         "default": False,
-        "options": "boolean"
+        "options": boolean
     },
 }
 
@@ -132,7 +136,7 @@ NIF_PARAMS = {
         "aliases": ["u"],
         "required": False,
         "default": "RFC5147String",
-        "options": "RFC5147String"
+        "options": ["RFC5147String", ]
     }
 }
 
@@ -159,7 +163,7 @@ def parse_params(indict, *specs):
                     wrong_params[param] = spec[param]
                 continue
             if "options" in options:
-                if options["options"] == "boolean":
+                if options["options"] == boolean:
                     outdict[param] = outdict[param] in [None, True, 'true', '1']
                 elif outdict[param] not in options["options"]:
                     wrong_params[param] = spec[param]
@@ -172,7 +176,7 @@ def parse_params(indict, *specs):
             errors=wrong_params)
         raise message
     if 'algorithm' in outdict and not isinstance(outdict['algorithm'], list):
-        outdict['algorithm'] = outdict['algorithm'].split(',')
+        outdict['algorithm'] = list(outdict['algorithm'].split(','))
     return outdict
 
 
@@ -190,7 +194,8 @@ def parse_call(params):
     params = parse_params(params, NIF_PARAMS)
     if params['informat'] == 'text':
         results = Results()
-        entry = Entry(nif__isString=params['input'])
+        entry = Entry(nif__isString=params['input'],
+                      id='#')  # Use @base
         results.entries.append(entry)
     elif params['informat'] == 'json-ld':
         results = from_string(params['input'], cls=Results)

senpy/blueprints.py

@@ -18,15 +18,15 @@
 Blueprints for Senpy
 """
 from flask import (Blueprint, request, current_app, render_template, url_for,
-                   jsonify)
+                   jsonify, redirect)
 from .models import Error, Response, Help, Plugins, read_schema, dump_schema, Datasets
 from . import api
 from .version import __version__
 from functools import wraps
 
 import logging
-import traceback
 import json
+import base64
 
 logger = logging.getLogger(__name__)
 
@@ -34,6 +34,24 @@ api_blueprint = Blueprint("api", __name__)
 demo_blueprint = Blueprint("demo", __name__, template_folder='templates')
 ns_blueprint = Blueprint("ns", __name__)
 
+_mimetypes_r = {'json-ld': ['application/ld+json'],
+                'turtle': ['text/turtle'],
+                'ntriples': ['application/n-triples'],
+                'text': ['text/plain']}
+
+MIMETYPES = {}
+
+for k, vs in _mimetypes_r.items():
+    for v in vs:
+        if v in MIMETYPES:
+            raise Exception('MIMETYPE {} specified for two formats: {} and {}'.format(v,
+                                                                                      v,
+                                                                                      MIMETYPES[v]))
+        MIMETYPES[v] = k
+
+DEFAULT_MIMETYPE = 'application/ld+json'
+DEFAULT_FORMAT = 'json-ld'
+
 
 def get_params(req):
     if req.method == 'POST':
@@ -45,6 +63,30 @@ def get_params(req):
     return indict
 
 
+def encoded_url(url=None, base=None):
+    code = ''
+    if not url:
+        if request.method == 'GET':
+            url = request.full_path[1:]  # Remove the first slash
+        else:
+            hash(frozenset(request.parameters.items()))
+            code = 'hash:{}'.format(hash)
+
+    code = code or base64.urlsafe_b64encode(url.encode()).decode()
+
+    if base:
+        return base + code
+    return url_for('api.decode', code=code, _external=True)
+
+
+def decoded_url(code, base=None):
+    if code.startswith('hash:'):
+        raise Exception('Can not decode a URL for a POST request')
+    base = base or request.url_root
+    path = base64.urlsafe_b64decode(code.encode()).decode()
+    return base + path
+
+
 @demo_blueprint.route('/')
 def index():
     ev = str(get_params(request).get('evaluation', False))
@@ -59,13 +101,22 @@ def index():
 def context(entity="context"):
     context = Response._context
     context['@vocab'] = url_for('ns.index', _external=True)
+    context['endpoint'] = url_for('api.api_root', _external=True)
     return jsonify({"@context": context})
 
 
+@api_blueprint.route('/d/<code>')
+def decode(code):
+    try:
+        return redirect(decoded_url(code))
+    except Exception:
+        return Error('invalid URL').flask()
+
+
 @ns_blueprint.route('/')  # noqa: F811
 def index():
-    context = Response._context
-    context['@vocab'] = url_for('.ns', _external=True)
+    context = Response._context.copy()
+    context['endpoint'] = url_for('api.api_root', _external=True)
     return jsonify({"@context": context})
 
 
@@ -81,7 +132,7 @@ def basic_api(f):
     default_params = {
         'inHeaders': False,
         'expanded-jsonld': False,
-        'outformat': 'json-ld',
+        'outformat': None,
         'with_parameters': True,
     }
 
@@ -100,42 +151,52 @@ def basic_api(f):
                 request.parameters = params
             response = f(*args, **kwargs)
         except (Exception) as ex:
-            if current_app.debug:
+            if current_app.debug or current_app.config['TESTING']:
                 raise
             if not isinstance(ex, Error):
-                msg = "{}:\n\t{}".format(ex,
-                                         traceback.format_exc())
+                msg = "{}".format(ex)
                 ex = Error(message=msg, status=500)
-            logger.exception('Error returning analysis result')
             response = ex
             response.parameters = raw_params
-            logger.error(ex)
+            logger.exception(ex)
 
         if 'parameters' in response and not params['with_parameters']:
             del response.parameters
 
         logger.info('Response: {}'.format(response))
+        mime = request.accept_mimetypes\
+                      .best_match(MIMETYPES.keys(),
+                                  DEFAULT_MIMETYPE)
+
+        mimeformat = MIMETYPES.get(mime, DEFAULT_FORMAT)
+        outformat = params['outformat'] or mimeformat
+
         return response.flask(
             in_headers=params['inHeaders'],
             headers=headers,
-            prefix=url_for('.api_root', _external=True),
+            prefix=params.get('prefix', encoded_url()),
             context_uri=url_for('api.context',
                                 entity=type(response).__name__,
                                 _external=True),
-            outformat=params['outformat'],
+            outformat=outformat,
             expanded=params['expanded-jsonld'])
 
     return decorated_function
 
 
-@api_blueprint.route('/', methods=['POST', 'GET'])
+@api_blueprint.route('/', defaults={'plugin': None}, methods=['POST', 'GET'])
+@api_blueprint.route('/<path:plugin>', methods=['POST', 'GET'])
 @basic_api
-def api_root():
+def api_root(plugin):
     if request.parameters['help']:
         dic = dict(api.API_PARAMS, **api.NIF_PARAMS)
         response = Help(valid_parameters=dic)
         return response
     req = api.parse_call(request.parameters)
+    if plugin:
+        plugin = plugin.replace('+', '/')
+        plugin = plugin.split('/')
+        req.parameters['algorithm'] = plugin
     return current_app.senpy.analyse(req)
 
 
@@ -165,7 +226,7 @@ def plugins():
 
 @api_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
 @basic_api
-def plugin(plugin=None):
+def plugin(plugin):
     sp = current_app.senpy
     return sp.get_plugin(plugin)
 

senpy/client.py

@@ -13,7 +13,7 @@ class Client(object):
         return self.request('/', method=method, input=input, **kwargs)
 
     def evaluate(self, input, method='GET', **kwargs):
-        return self.request('/evaluate', method = method, input=input, **kwargs)
+        return self.request('/evaluate', method=method, input=input, **kwargs)
 
     def plugins(self, *args, **kwargs):
         resp = self.request(path='/plugins').plugins
@@ -24,7 +24,7 @@ class Client(object):
         return {d.name: d for d in resp}
 
     def request(self, path=None, method='GET', **params):
-        url = '{}{}'.format(self.endpoint, path)
+        url = '{}{}'.format(self.endpoint.rstrip('/'), path)
         response = requests.request(method=method, url=url, params=params)
         try:
             resp = models.from_dict(response.json())

senpy/extensions.py

@@ -18,14 +18,9 @@ import errno
 import logging
 
 
-logger = logging.getLogger(__name__)
+from . import gsitk_compat
 
-try:
-    from gsitk.datasets.datasets import DatasetManager
-    GSITK_AVAILABLE = True
-except ImportError:
-    logger.warn('GSITK is not installed. Some functions will be unavailable.')
-    GSITK_AVAILABLE = False
+logger = logging.getLogger(__name__)
 
 
 class Senpy(object):
@@ -95,7 +90,7 @@ class Senpy(object):
         if plugin in self._plugins:
             return self._plugins[plugin]
 
-        results = self.plugins(id='plugins/{}'.format(name))
+        results = self.plugins(id='endpoint:plugins/{}'.format(name))
 
         if not results:
             return Error(message="Plugin not found", status=404)
@@ -158,7 +153,6 @@ class Senpy(object):
                 yield i
             return
         plugin = plugins[0]
-        self._activate(plugin)  # Make sure the plugin is activated
         specific_params = api.parse_extra_params(req, plugin)
         req.analysis.append({'plugin': plugin,
                              'parameters': specific_params})
@@ -167,8 +161,7 @@ class Senpy(object):
             yield i
 
     def install_deps(self):
-        for plugin in self.plugins(is_activated=True):
-            plugins.install_deps(plugin)
+        plugins.install_deps(*self.plugins())
 
     def analyse(self, request):
         """
@@ -203,16 +196,14 @@ class Senpy(object):
                 raise Error(
                     status=404,
                     message="The dataset '{}' is not valid".format(dataset))
-        dm = DatasetManager()
+        dm = gsitk_compat.DatasetManager()
         datasets = dm.prepare_datasets(datasets_name)
         return datasets
 
     @property
     def datasets(self):
-        if not GSITK_AVAILABLE:
-            raise Exception('GSITK is not available. Install it to use this function.')
         self._dataset_list = {}
-        dm = DatasetManager()
+        dm = gsitk_compat.DatasetManager()
         for item in dm.get_datasets():
             for key in item:
                 if key in self._dataset_list:
@@ -223,8 +214,6 @@ class Senpy(object):
         return self._dataset_list
 
     def evaluate(self, params):
-        if not GSITK_AVAILABLE:
-            raise Exception('GSITK is not available. Install it to use this function.')
         logger.debug("evaluating request: {}".format(params))
         results = AggregatedEvaluation()
         results.parameters = params
@@ -318,10 +307,15 @@ class Senpy(object):
         else:
             self._default = self._plugins[value.lower()]
 
-    def activate_all(self, sync=True):
+    def activate_all(self, sync=True, allow_fail=False):
         ps = []
         for plug in self._plugins.keys():
-            ps.append(self.activate_plugin(plug, sync=sync))
+            try:
+                self.activate_plugin(plug, sync=sync)
+            except Exception as ex:
+                if not allow_fail:
+                    raise
+                logger.error('Could not activate {}: {}'.format(plug, ex))
         return ps
 
     def deactivate_all(self, sync=True):
@@ -346,6 +340,7 @@ class Senpy(object):
             logger.info(msg)
             success = True
             self._set_active(plugin, success)
+        return success
 
     def activate_plugin(self, plugin_name, sync=True):
         plugin_name = plugin_name.lower()
@@ -356,8 +351,8 @@ class Senpy(object):
 
         logger.info("Activating plugin: {}".format(plugin.name))
 
-        if sync or 'async' in plugin and not plugin.async:
-            self._activate(plugin)
+        if sync or not getattr(plugin, 'async', True) or getattr(plugin, 'sync', False):
+            return self._activate(plugin)
         else:
             th = Thread(target=partial(self._activate, plugin))
             th.start()
@@ -379,7 +374,7 @@ class Senpy(object):
 
         self._set_active(plugin, False)
 
-        if sync or 'async' in plugin and not plugin.async:
+        if sync or not getattr(plugin, 'async', True) or not getattr(plugin, 'sync', False):
             self._deactivate(plugin)
         else:
             th = Thread(target=partial(self._deactivate, plugin))

senpy/gsitk_compat.py

@@ -0,0 +1,31 @@
+import logging
+
+from pkg_resources import parse_version, get_distribution, DistributionNotFound
+
+logger = logging.getLogger(__name__)
+
+MSG = 'GSITK is not (properly) installed.'
+IMPORTMSG = '{} Some functions will be unavailable.'.format(MSG)
+RUNMSG = '{} Install it to use this function.'.format(MSG)
+
+
+def raise_exception(*args, **kwargs):
+    raise Exception(RUNMSG)
+
+
+try:
+    gsitk_distro = get_distribution("gsitk")
+    GSITK_VERSION = parse_version(gsitk_distro.version)
+    GSITK_AVAILABLE = GSITK_VERSION > parse_version("0.1.9.1")  # Earlier versions have a bug
+except DistributionNotFound:
+    GSITK_AVAILABLE = False
+    GSITK_VERSION = ()
+
+if GSITK_AVAILABLE:
+    from gsitk.datasets.datasets import DatasetManager
+    from gsitk.evaluation.evaluation import Evaluation as Eval
+    from sklearn.pipeline import Pipeline
+    modules = locals()
+else:
+    logger.warning(IMPORTMSG)
+    DatasetManager = Eval = Pipeline = raise_exception

senpy/models.py

@@ -138,7 +138,7 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)):
     @property
     def id(self):
         if '@id' not in self:
-            self['@id'] = ':{}_{}'.format(type(self).__name__, time.time())
+            self['@id'] = '_:{}_{}'.format(type(self).__name__, time.time())
         return self['@id']
 
     @id.setter
@@ -146,7 +146,7 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)):
         self['@id'] = value
 
     def flask(self,
-              in_headers=True,
+              in_headers=False,
               headers=None,
               outformat='json-ld',
               **kwargs):
@@ -176,20 +176,22 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)):
 
     def serialize(self, format='json-ld', with_mime=False, **kwargs):
         js = self.jsonld(**kwargs)
+        content = json.dumps(js, indent=2, sort_keys=True)
         if format == 'json-ld':
-            content = json.dumps(js, indent=2, sort_keys=True)
             mimetype = "application/json"
-        elif format in ['turtle', ]:
+        elif format in ['turtle', 'ntriples']:
             logger.debug(js)
-            content = json.dumps(js, indent=2, sort_keys=True)
+            base = kwargs.get('prefix')
             g = Graph().parse(
                 data=content,
                 format='json-ld',
-                base=kwargs.get('prefix'),
-                context=self._context)
+                base=base,
+                context=[self._context,
+                         {'@base': base}])
             logger.debug(
                 'Parsing with prefix: {}'.format(kwargs.get('prefix')))
-            content = g.serialize(format='turtle').decode('utf-8')
+            content = g.serialize(format=format,
+                                  base=base).decode('utf-8')
             mimetype = 'text/{}'.format(format)
         else:
             raise Error('Unknown outformat: {}'.format(format))
@@ -205,25 +207,21 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)):
                expanded=False):
 
         result = self.serializable()
-        if context_uri or with_context:
-            result['@context'] = context_uri or self._context
 
-        # result = jsonld.compact(result,
-        #                         self._context,
-        #                         options={
-        #                             'base': prefix,
-        #                             'expandContext': self._context,
-        #                             'senpy': prefix
-        #                         })
         if expanded:
             result = jsonld.expand(
                 result, options={'base': prefix,
-                                 'expandContext': self._context})
+                                 'expandContext': self._context})[0]
         if not with_context:
             try:
                 del result['@context']
             except KeyError:
                 pass
+        elif context_uri:
+            result['@context'] = context_uri
+        else:
+            result['@context'] = self._context
+
         return result
 
     def validate(self, obj=None):

senpy/plugins/__init__.py

@@ -3,6 +3,7 @@ standard_library.install_aliases()
 
 
 from future.utils import with_metaclass
+from functools import partial
 
 import os.path
 import os
@@ -22,18 +23,12 @@ import nltk
 
 from .. import models, utils
 from .. import api
+from .. import gsitk_compat
+from .. import testing
 
 
 logger = logging.getLogger(__name__)
 
-try:
-    from gsitk.evaluation.evaluation import Evaluation as Eval
-    from sklearn.pipeline import Pipeline
-    GSITK_AVAILABLE = True
-except ImportError:
-    logger.warn('GSITK is not installed. Some functions will be unavailable.')
-    GSITK_AVAILABLE = False
-
 
 class PluginMeta(models.BaseMeta):
     _classes = {}
@@ -92,7 +87,7 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)):
         if info:
             self.update(info)
         self.validate()
-        self.id = 'plugins/{}_{}'.format(self['name'], self['version'])
+        self.id = 'endpoint:plugins/{}_{}'.format(self['name'], self['version'])
         self.is_activated = False
         self._lock = threading.Lock()
         self._directory = os.path.abspath(os.path.dirname(inspect.getfile(self.__class__)))
@@ -149,14 +144,23 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)):
                 self.log.warn('Test case failed:\n{}'.format(pprint.pformat(case)))
                 raise
 
-    def test_case(self, case):
+    def test_case(self, case, mock=testing.MOCK_REQUESTS):
         entry = models.Entry(case['entry'])
         given_parameters = case.get('params', case.get('parameters', {}))
         expected = case.get('expected', None)
         should_fail = case.get('should_fail', False)
+        responses = case.get('responses', [])
+
         try:
             params = api.parse_params(given_parameters, self.extra_params)
-            res = list(self.analyse_entries([entry, ], params))
+
+            method = partial(self.analyse_entries, [entry, ], params)
+
+            if mock:
+                res = list(method())
+            else:
+                with testing.patch_all_requests(responses):
+                    res = list(method())
 
             if not isinstance(expected, list):
                 expected = [expected]
@@ -332,7 +336,7 @@ class Box(AnalysisPlugin):
         return self.transform(X)
 
     def as_pipe(self):
-        pipe = Pipeline([('plugin', self)])
+        pipe = gsitk_compat.Pipeline([('plugin', self)])
         pipe.name = self.name
         return pipe
 
@@ -530,7 +534,7 @@ def find_plugins(folders):
                 yield fpath
 
 
-def from_path(fpath, **kwargs):
+def from_path(fpath, install_on_fail=False, **kwargs):
     logger.debug("Loading plugin from {}".format(fpath))
     if fpath.endswith('.py'):
         # We asume root is the dir of the file, and module is the name of the file
@@ -540,7 +544,7 @@ def from_path(fpath, **kwargs):
             yield instance
     else:
         info = parse_plugin_info(fpath)
-        yield from_info(info, **kwargs)
+        yield from_info(info, install_on_fail=install_on_fail, **kwargs)
 
 
 def from_folder(folders, loader=from_path, **kwargs):
@@ -551,7 +555,7 @@ def from_folder(folders, loader=from_path, **kwargs):
     return plugins
 
 
-def from_info(info, root=None, **kwargs):
+def from_info(info, root=None, install_on_fail=True, **kwargs):
     if any(x not in info for x in ('module',)):
         raise ValueError('Plugin info is not valid: {}'.format(info))
     module = info["module"]
@@ -559,7 +563,12 @@ def from_info(info, root=None, **kwargs):
     if not root and '_path' in info:
         root = os.path.dirname(info['_path'])
 
-    return one_from_module(module, root=root, info=info, **kwargs)
+    fun = partial(one_from_module, module, root=root, info=info, **kwargs)
+    try:
+        return fun()
+    except (ImportError, LookupError):
+        install_deps(info)
+        return fun()
 
 
 def parse_plugin_info(fpath):
@@ -606,17 +615,9 @@ def _instances_in_module(module):
             yield obj
 
 
-def _from_module_name(module, root, info=None, install=True, **kwargs):
-    try:
-        module = load_module(module, root)
-    except (ImportError, LookupError):
-        if not install or not info:
-            raise
-        install_deps(info)
-        module = load_module(module, root)
+def _from_module_name(module, root, info=None, **kwargs):
+    module = load_module(module, root)
     for plugin in _from_loaded_module(module=module, root=root, info=info, **kwargs):
-        if install:
-            install_deps(plugin)
         yield plugin
 
 
@@ -628,12 +629,9 @@ def _from_loaded_module(module, info=None, **kwargs):
 
 
 def evaluate(plugins, datasets, **kwargs):
-    if not GSITK_AVAILABLE:
-        raise Exception('GSITK is not available. Install it to use this function.')
-
-    ev = Eval(tuples=None,
-              datasets=datasets,
-              pipelines=[plugin.as_pipe() for plugin in plugins])
+    ev = gsitk_compat.Eval(tuples=None,
+                           datasets=datasets,
+                           pipelines=[plugin.as_pipe() for plugin in plugins])
     ev.evaluate()
     results = ev.results
     evaluations = evaluations_to_JSONLD(results, **kwargs)

senpy/plugins/sentiment/sentiment140/sentiment140_plugin.py

@@ -4,6 +4,8 @@ import json
 from senpy.plugins import SentimentPlugin
 from senpy.models import Sentiment
 
+ENDPOINT = 'http://www.sentiment140.com/api/bulkClassifyJson'
+
 
 class Sentiment140(SentimentPlugin):
     '''Connects to the sentiment140 free API: http://sentiment140.com'''
@@ -26,7 +28,7 @@ class Sentiment140(SentimentPlugin):
 
     def analyse_entry(self, entry, params):
         lang = params["language"]
-        res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
+        res = requests.post(ENDPOINT,
                             json.dumps({
                                 "language": lang,
                                 "data": [{
@@ -52,18 +54,6 @@ class Sentiment140(SentimentPlugin):
         entry.language = lang
         yield entry
 
-    def test(self, *args, **kwargs):
-        '''
-        To avoid calling the sentiment140 API, we will mock the results
-        from requests.
-        '''
-        from senpy.testing import patch_requests
-        expected = {"data": [{"polarity": 4}]}
-        with patch_requests(expected) as (request, response):
-            super(Sentiment140, self).test(*args, **kwargs)
-            assert request.called
-            assert response.json.called
-
     test_cases = [
         {
             'entry': {
@@ -77,6 +67,9 @@ class Sentiment140(SentimentPlugin):
                         'marl:hasPolarity': 'marl:Positive',
                     }
                 ]
-            }
+            },
+            'responses': [{'url': ENDPOINT,
+                           'method': 'POST',
+                           'json': {'data': [{'polarity': 4}]}}]
         }
     ]

senpy/static/js/main.js

@@ -413,7 +413,7 @@ function evaluate_JSON(){
     url += "?algo="+plugin+"&dataset="+datasets
 
     $('#doevaluate').attr("disabled", true);
-    $.ajax({type: "GET", url: url, dataType: 'json'}).done(function(resp) {
+    $.ajax({type: "GET", url: url, dataType: 'json'}).always(function(resp) {
         $('#doevaluate').attr("disabled", false);
         response = resp.responseText;  
 

senpy/testing.py

@@ -1,36 +1,31 @@
-try:
-    from unittest.mock import patch, MagicMock
-except ImportError:
-    from mock import patch, MagicMock
-
 from past.builtins import basestring
 
-
-import json
-from contextlib import contextmanager
+import os
+import responses as requestmock
 
 from .models import BaseModel
 
 
-@contextmanager
-def patch_requests(value, code=200):
-    success = MagicMock()
-    if isinstance(value, BaseModel):
-        value = value.jsonld()
-    if not isinstance(value, basestring):
-        data = json.dumps(value)
+MOCK_REQUESTS = os.environ.get('MOCK_REQUESTS', '').lower() in ['no', 'false']
+
+
+def patch_all_requests(responses):
+
+    patched = requestmock.RequestsMock()
+
+    for response in responses or []:
+        args = response.copy()
+        if 'json' in args and isinstance(args['json'], BaseModel):
+            args['json'] = args['json'].jsonld()
+        args['method'] = getattr(requestmock, args.get('method', 'GET'))
+        patched.add(**args)
+    return patched
+
+
+def patch_requests(url, response, method='GET', status=200):
+    args = {'url': url, 'method': method, 'status': status}
+    if isinstance(response, basestring):
+        args['body'] = response
     else:
-        data = value
-
-    success.json.return_value = value
-
-    success.status_code = code
-    success.content = data
-    success.text = data
-
-    method_mocker = MagicMock()
-    method_mocker.return_value = success
-    with patch.multiple('requests', request=method_mocker,
-                        get=method_mocker, post=method_mocker):
-        yield method_mocker, success
-        assert method_mocker.called
+        args['json'] = response
+    return patch_all_requests([args])

senpy/utils.py

@@ -80,7 +80,7 @@ def easy_test(plugin_list=None, debug=True):
         for plug in plugin_list:
             plug.test()
             plug.log.info('My tests passed!')
-            logger.info('All tests passed!')
+        logger.info('All tests passed for {} plugins!'.format(len(plugin_list)))
     except Exception:
         if not debug:
             raise

tests/test_blueprints.py

@@ -21,7 +21,6 @@ class BlueprintsTest(TestCase):
     def setUpClass(cls):
         """Set up only once, and re-use in every individual test"""
         cls.app = Flask("test_extensions")
-        cls.app.debug = False
         cls.client = cls.app.test_client()
         cls.senpy = Senpy(default_plugins=True)
         cls.senpy.init_app(cls.app)
@@ -31,6 +30,9 @@ class BlueprintsTest(TestCase):
         cls.senpy.activate_plugin("DummyRequired", sync=True)
         cls.senpy.default_plugin = 'Dummy'
 
+    def setUp(self):
+        self.app.config['TESTING'] = True  # Tell Flask not to catch Exceptions
+
     def assertCode(self, resp, code):
         self.assertEqual(resp.status_code, code)
 
@@ -42,6 +44,7 @@ class BlueprintsTest(TestCase):
         """
         Calling with no arguments should ask the user for more arguments
         """
+        self.app.config['TESTING'] = False  # Errors are expected in this case
         resp = self.client.get("/api/")
         self.assertCode(resp, 400)
         js = parse_resp(resp)
@@ -64,10 +67,24 @@ class BlueprintsTest(TestCase):
         logging.debug("Got response: %s", js)
         assert "@context" in js
         assert "entries" in js
+        assert len(js['analysis']) == 1
+
+    def test_analysis_post(self):
+        """
+        The results for a POST request should be the same as for a GET request.
+        """
+        resp = self.client.post("/api/", data={'i': 'My aloha mohame',
+                                               'with_parameters': True})
+        self.assertCode(resp, 200)
+        js = parse_resp(resp)
+        logging.debug("Got response: %s", js)
+        assert "@context" in js
+        assert "entries" in js
+        assert len(js['analysis']) == 1
 
     def test_analysis_extra(self):
         """
-        Extra params that have a default should
+        Extra params that have a default should use it
         """
         resp = self.client.get("/api/?i=My aloha mohame&algo=Dummy&with_parameters=true")
         self.assertCode(resp, 200)
@@ -81,7 +98,7 @@ class BlueprintsTest(TestCase):
         Extra params that have a required argument that does not
         have a default should raise an error.
         """
-        self.app.debug = False
+        self.app.config['TESTING'] = False  # Errors are expected in this case
         resp = self.client.get("/api/?i=My aloha mohame&algo=DummyRequired")
         self.assertCode(resp, 400)
         js = parse_resp(resp)
@@ -92,12 +109,50 @@ class BlueprintsTest(TestCase):
         resp = self.client.get("/api/?i=My aloha mohame&algo=DummyRequired&example=a")
         self.assertCode(resp, 200)
 
+    def test_analysis_url(self):
+        """
+        The algorithm can also be specified as part of the URL
+        """
+        self.app.config['TESTING'] = False  # Errors are expected in this case
+        resp = self.client.get("/api/DummyRequired?i=My aloha mohame")
+        self.assertCode(resp, 400)
+        js = parse_resp(resp)
+        logging.debug("Got response: %s", js)
+        assert isinstance(js, models.Error)
+        resp = self.client.get("/api/DummyRequired?i=My aloha mohame&example=notvalid")
+        self.assertCode(resp, 400)
+        resp = self.client.get("/api/DummyRequired?i=My aloha mohame&example=a")
+        self.assertCode(resp, 200)
+
+    def test_analysis_chain(self):
+        """
+        More than one algorithm can be specified. Plugins will then be chained
+        """
+        resp = self.client.get("/api/Dummy?i=My aloha mohame")
+        js = parse_resp(resp)
+        assert len(js['analysis']) == 1
+        assert js['entries'][0]['nif:isString'] == 'My aloha mohame'[::-1]
+
+        resp = self.client.get("/api/Dummy/Dummy?i=My aloha mohame")
+        # Calling dummy twice, should return the same string
+        self.assertCode(resp, 200)
+        js = parse_resp(resp)
+        assert len(js['analysis']) == 2
+        assert js['entries'][0]['nif:isString'] == 'My aloha mohame'
+
+        resp = self.client.get("/api/Dummy+Dummy?i=My aloha mohame")
+        # Same with pluses instead of slashes
+        self.assertCode(resp, 200)
+        js = parse_resp(resp)
+        assert len(js['analysis']) == 2
+        assert js['entries'][0]['nif:isString'] == 'My aloha mohame'
+
     def test_error(self):
         """
         The dummy plugin returns an empty response,\
         it should contain the context
         """
-        self.app.debug = False
+        self.app.config['TESTING'] = False  # Errors are expected in this case
         resp = self.client.get("/api/?i=My aloha mohame&algo=DOESNOTEXIST")
         self.assertCode(resp, 404)
         js = parse_resp(resp)
@@ -139,7 +194,7 @@ class BlueprintsTest(TestCase):
         js = parse_resp(resp)
         logging.debug(js)
         assert "@id" in js
-        assert js["@id"] == "plugins/Dummy_0.1"
+        assert js["@id"] == "endpoint:plugins/Dummy_0.1"
 
     def test_default(self):
         """ Show only one plugin"""
@@ -148,7 +203,7 @@ class BlueprintsTest(TestCase):
         js = parse_resp(resp)
         logging.debug(js)
         assert "@id" in js
-        assert js["@id"] == "plugins/Dummy_0.1"
+        assert js["@id"] == "endpoint:plugins/Dummy_0.1"
 
     def test_context(self):
         resp = self.client.get("/api/contexts/context.jsonld")
@@ -172,5 +227,6 @@ class BlueprintsTest(TestCase):
         assert "help" in js["valid_parameters"]
 
     def test_conversion(self):
+        self.app.config['TESTING'] = False  # Errors are expected in this case
         resp = self.client.get("/api/?input=hello&algo=emoRand&emotionModel=DOES NOT EXIST")
         self.assertCode(resp, 404)

tests/test_client.py

@@ -14,22 +14,28 @@ class ModelsTest(TestCase):
     def test_client(self):
         endpoint = 'http://dummy/'
         client = Client(endpoint)
-        with patch_requests(Results()) as (request, response):
+        with patch_requests('http://dummy/', Results()):
             resp = client.analyse('hello')
             assert isinstance(resp, Results)
-        request.assert_called_with(
-            url=endpoint + '/', method='GET', params={'input': 'hello'})
-        with patch_requests(Error('Nothing')) as (request, response):
+        with patch_requests('http://dummy/', Error('Nothing')):
             try:
                 client.analyse(input='hello', algorithm='NONEXISTENT')
                 raise Exception('Exceptions should be raised. This is not golang')
             except Error:
                 pass
-        request.assert_called_with(
-            url=endpoint + '/',
-            method='GET',
-            params={'input': 'hello',
-                    'algorithm': 'NONEXISTENT'})
+
+    def test_client_post(self):
+        endpoint = 'http://dummy/'
+        client = Client(endpoint)
+        with patch_requests('http://dummy/', Results()):
+            resp = client.analyse('hello')
+            assert isinstance(resp, Results)
+        with patch_requests('http://dummy/', Error('Nothing'), method='POST'):
+            try:
+                client.analyse(input='hello', method='POST', algorithm='NONEXISTENT')
+                raise Exception('Exceptions should be raised. This is not golang')
+            except Error:
+                pass
 
     def test_plugins(self):
         endpoint = 'http://dummy/'
@@ -37,11 +43,8 @@ class ModelsTest(TestCase):
         plugins = Plugins()
         p1 = AnalysisPlugin({'name': 'AnalysisP1', 'version': 0, 'description': 'No'})
         plugins.plugins = [p1, ]
-        with patch_requests(plugins) as (request, response):
+        with patch_requests('http://dummy/plugins', plugins):
             response = client.plugins()
             assert isinstance(response, dict)
             assert len(response) == 1
             assert "AnalysisP1" in response
-        request.assert_called_with(
-            url=endpoint + '/plugins', method='GET',
-            params={})

tests/test_extensions.py

@@ -121,8 +121,8 @@ class ExtensionsTest(TestCase):
         # Leaf (defaultdict with  __setattr__ and __getattr__.
         r1 = analyse(self.senpy, algorithm="Dummy", input="tupni", output="tuptuo")
         r2 = analyse(self.senpy, input="tupni", output="tuptuo")
-        assert r1.analysis[0] == "plugins/Dummy_0.1"
-        assert r2.analysis[0] == "plugins/Dummy_0.1"
+        assert r1.analysis[0] == "endpoint:plugins/Dummy_0.1"
+        assert r2.analysis[0] == "endpoint:plugins/Dummy_0.1"
         assert r1.entries[0]['nif:isString'] == 'input'
 
     def test_analyse_empty(self):
@@ -156,8 +156,8 @@ class ExtensionsTest(TestCase):
         r2 = analyse(self.senpy,
                      input="tupni",
                      output="tuptuo")
-        assert r1.analysis[0] == "plugins/Dummy_0.1"
-        assert r2.analysis[0] == "plugins/Dummy_0.1"
+        assert r1.analysis[0] == "endpoint:plugins/Dummy_0.1"
+        assert r2.analysis[0] == "endpoint:plugins/Dummy_0.1"
         assert r1.entries[0]['nif:isString'] == 'input'
 
     def test_analyse_error(self):

tests/test_plugins.py

@@ -1,7 +1,6 @@
 #!/bin/env python
 
 import os
-import sys
 import pickle
 import shutil
 import tempfile
@@ -10,6 +9,7 @@ from unittest import TestCase, skipIf
 from senpy.models import Results, Entry, EmotionSet, Emotion, Plugins
 from senpy import plugins
 from senpy.plugins.conversion.emotion.centroids import CentroidConversion
+from senpy.gsitk_compat import GSITK_AVAILABLE
 
 import pandas as pd
 
@@ -312,9 +312,7 @@ class PluginsTest(TestCase):
         res = c._backwards_conversion(e)
         assert res["onyx:hasEmotionCategory"] == "c2"
 
-    @skipIf(sys.version_info < (3, 0),
-            reason="requires Python3")
-    def test_evaluation(self):
+    def _test_evaluation(self):
         testdata = []
         for i in range(50):
             testdata.append(["good", 1])
@@ -348,6 +346,16 @@ class PluginsTest(TestCase):
         smart_metrics = results[0].metrics[0]
         assert abs(smart_metrics['accuracy'] - 1) < 0.01
 
+    @skipIf(not GSITK_AVAILABLE, "GSITK is not available")
+    def test_evaluation(self):
+        self._test_evaluation()
+
+    @skipIf(GSITK_AVAILABLE, "GSITK is available")
+    def test_evaluation_unavailable(self):
+        with self.assertRaises(Exception) as context:
+            self._test_evaluation()
+        self.assertTrue('GSITK ' in str(context.exception))
+
 
 def make_mini_test(fpath):
     def mini_test(self):

tests/test_test.py

@@ -5,28 +5,29 @@ import json
 from senpy.testing import patch_requests
 from senpy.models import Results
 
+ENDPOINT = 'http://example.com'
+
 
 class TestTest(TestCase):
     def test_patch_text(self):
-        with patch_requests('hello'):
-            r = requests.get('http://example.com')
+        with patch_requests(ENDPOINT, 'hello'):
+            r = requests.get(ENDPOINT)
             assert r.text == 'hello'
-            assert r.content == 'hello'
 
     def test_patch_json(self):
         r = Results()
-        with patch_requests(r):
-            res = requests.get('http://example.com')
-            assert res.content == json.dumps(r.jsonld())
+        with patch_requests(ENDPOINT, r):
+            res = requests.get(ENDPOINT)
+            assert res.text == json.dumps(r.jsonld())
             js = res.json()
             assert js
             assert js['@type'] == r['@type']
 
     def test_patch_dict(self):
         r = {'nothing': 'new'}
-        with patch_requests(r):
-            res = requests.get('http://example.com')
-            assert res.content == json.dumps(r)
+        with patch_requests(ENDPOINT, r):
+            res = requests.get(ENDPOINT)
+            assert res.text == json.dumps(r)
             js = res.json()
             assert js
             assert js['nothing'] == 'new'