Several changes

* Add flag to run tests
* Add ntriples outformat

senpy/__main__.py

@@ -78,10 +78,15 @@ def main():
         help='Do not run a server, only install plugin dependencies')
     parser.add_argument(
         '--only-test',
-        '-t',
         action='store_true',
         default=False,
         help='Do not run a server, just test all plugins')
+    parser.add_argument(
+        '--test',
+        '-t',
+        action='store_true',
+        default=False,
+        help='Test all plugins before launching the server')
     parser.add_argument(
         '--only-list',
         '--list',
@@ -99,6 +104,12 @@ def main():
         action='store_false',
         default=True,
         help='Run a threaded server')
+    parser.add_argument(
+        '--no-deps',
+        '-n',
+        action='store_true',
+        default=False,
+        help='Skip installing dependencies')
     parser.add_argument(
         '--version',
         '-v',
@@ -125,18 +136,26 @@ def main():
                data_folder=args.data_folder)
     if args.only_list:
         plugins = sp.plugins()
-        maxwidth = max(len(x.id) for x in plugins)
+        maxname = max(len(x.name) for x in plugins)
+        maxversion = max(len(x.version) for x in plugins)
+        print('Found {} plugins:'.format(len(plugins)))
         for plugin in plugins:
             import inspect
             fpath = inspect.getfile(plugin.__class__)
-            print('{: <{width}} @ {}'.format(plugin.id, fpath, width=maxwidth))
+            print('\t{: <{maxname}} @ {: <{maxversion}} -> {}'.format(plugin.name,
+                                                                      plugin.version,
+                                                                      fpath,
+                                                                      maxname=maxname,
+                                                                      maxversion=maxversion))
         return
+    if not args.no_deps:
         sp.install_deps()
     if args.only_install:
         return
     sp.activate_all(allow_fail=args.allow_fail)
-    if args.only_test:
+    if args.test or args.only_test:
         easy_test(sp.plugins(), debug=args.debug)
+        if args.only_test:
             return
     print('Senpy version {}'.format(senpy.__version__))
     print('Server running on port %s:%d. Ctrl+C to quit' % (args.host,

senpy/api.py

@@ -4,7 +4,7 @@ import logging
 logger = logging.getLogger(__name__)
 
 
-boolean = (True, False)
+boolean = [True, False]
 
 
 API_PARAMS = {
@@ -33,7 +33,7 @@ API_PARAMS = {
         "aliases": ["o"],
         "default": "json-ld",
         "required": True,
-        "options": ["json-ld", "turtle"],
+        "options": ["json-ld", "turtle", "ntriples"],
     },
     "help": {
         "@id": "help",
@@ -175,8 +175,8 @@ def parse_params(indict, *specs):
             parameters=outdict,
             errors=wrong_params)
         raise message
-    if 'algorithm' in outdict and not isinstance(outdict['algorithm'], tuple):
+    if 'algorithm' in outdict and not isinstance(outdict['algorithm'], list):
-        outdict['algorithm'] = tuple(outdict['algorithm'].split(','))
+        outdict['algorithm'] = list(outdict['algorithm'].split(','))
     return outdict
 
 
@@ -194,7 +194,8 @@ def parse_call(params):
     params = parse_params(params, NIF_PARAMS)
     if params['informat'] == 'text':
         results = Results()
-        entry = Entry(nif__isString=params['input'])
+        entry = Entry(nif__isString=params['input'],
+                      id='#')  # Use @base
         results.entries.append(entry)
     elif params['informat'] == 'json-ld':
         results = from_string(params['input'], cls=Results)

senpy/blueprints.py

@@ -25,7 +25,6 @@ from .version import __version__
 from functools import wraps
 
 import logging
-import traceback
 import json
 import base64
 
@@ -37,12 +36,17 @@ ns_blueprint = Blueprint("ns", __name__)
 
 _mimetypes_r = {'json-ld': ['application/ld+json'],
                 'turtle': ['text/turtle'],
+                'ntriples': ['application/n-triples'],
                 'text': ['text/plain']}
 
 MIMETYPES = {}
 
 for k, vs in _mimetypes_r.items():
     for v in vs:
+        if v in MIMETYPES:
+            raise Exception('MIMETYPE {} specified for two formats: {} and {}'.format(v,
+                                                                                      v,
+                                                                                      MIMETYPES[v]))
         MIMETYPES[v] = k
 
 DEFAULT_MIMETYPE = 'application/ld+json'
@@ -147,16 +151,14 @@ def basic_api(f):
                 request.parameters = params
             response = f(*args, **kwargs)
         except (Exception) as ex:
-            if current_app.debug:
+            if current_app.debug or current_app.config['TESTING']:
                 raise
             if not isinstance(ex, Error):
-                msg = "{}:\n\t{}".format(ex,
+                msg = "{}".format(ex)
-                                         traceback.format_exc())
                 ex = Error(message=msg, status=500)
-            logger.exception('Error returning analysis result')
             response = ex
             response.parameters = raw_params
-            logger.error(ex)
+            logger.exception(ex)
 
         if 'parameters' in response and not params['with_parameters']:
             del response.parameters

senpy/extensions.py

@@ -18,14 +18,9 @@ import errno
 import logging
 
 
-logger = logging.getLogger(__name__)
+from . import gsitk_compat
 
-try:
+logger = logging.getLogger(__name__)
-    from gsitk.datasets.datasets import DatasetManager
-    GSITK_AVAILABLE = True
-except ImportError:
-    logger.warn('GSITK is not installed. Some functions will be unavailable.')
-    GSITK_AVAILABLE = False
 
 
 class Senpy(object):
@@ -167,8 +162,7 @@ class Senpy(object):
             yield i
 
     def install_deps(self):
-        for plugin in self.plugins(is_activated=True):
+        plugins.install_deps(*self.plugins())
-            plugins.install_deps(plugin)
 
     def analyse(self, request):
         """
@@ -203,16 +197,14 @@ class Senpy(object):
                 raise Error(
                     status=404,
                     message="The dataset '{}' is not valid".format(dataset))
-        dm = DatasetManager()
+        dm = gsitk_compat.DatasetManager()
         datasets = dm.prepare_datasets(datasets_name)
         return datasets
 
     @property
     def datasets(self):
-        if not GSITK_AVAILABLE:
-            raise Exception('GSITK is not available. Install it to use this function.')
         self._dataset_list = {}
-        dm = DatasetManager()
+        dm = gsitk_compat.DatasetManager()
         for item in dm.get_datasets():
             for key in item:
                 if key in self._dataset_list:
@@ -223,8 +215,6 @@ class Senpy(object):
         return self._dataset_list
 
     def evaluate(self, params):
-        if not GSITK_AVAILABLE:
-            raise Exception('GSITK is not available. Install it to use this function.')
         logger.debug("evaluating request: {}".format(params))
         results = AggregatedEvaluation()
         results.parameters = params
@@ -351,6 +341,7 @@ class Senpy(object):
             logger.info(msg)
             success = True
             self._set_active(plugin, success)
+        return success
 
     def activate_plugin(self, plugin_name, sync=True):
         plugin_name = plugin_name.lower()
@@ -362,7 +353,7 @@ class Senpy(object):
         logger.info("Activating plugin: {}".format(plugin.name))
 
         if sync or 'async' in plugin and not plugin.async:
-            self._activate(plugin)
+            return self._activate(plugin)
         else:
             th = Thread(target=partial(self._activate, plugin))
             th.start()

senpy/gsitk_compat.py

@@ -0,0 +1,23 @@
+import logging
+
+logger = logging.getLogger(__name__)
+
+MSG = 'GSITK is not (properly) installed.'
+IMPORTMSG = '{} Some functions will be unavailable.'.format(MSG)
+RUNMSG = '{} Install it to use this function.'.format(MSG)
+
+
+def raise_exception(*args, **kwargs):
+    raise Exception(RUNMSG)
+
+
+try:
+    from gsitk.datasets.datasets import DatasetManager
+    from gsitk.evaluation.evaluation import Evaluation as Eval
+    from sklearn.pipeline import Pipeline
+    GSITK_AVAILABLE = True
+    modules = locals()
+except ImportError:
+    logger.warn(IMPORTMSG)
+    GSITK_AVAILABLE = False
+    DatasetManager = Eval = Pipeline = raise_exception

senpy/models.py

@@ -179,17 +179,18 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)):
         content = json.dumps(js, indent=2, sort_keys=True)
         if format == 'json-ld':
             mimetype = "application/json"
-        elif format in ['turtle', ]:
+        elif format in ['turtle', 'ntriples']:
             logger.debug(js)
             base = kwargs.get('prefix')
             g = Graph().parse(
                 data=content,
                 format='json-ld',
                 base=base,
-                context=self._context)
+                context=[self._context,
+                         {'@base': base}])
             logger.debug(
                 'Parsing with prefix: {}'.format(kwargs.get('prefix')))
-            content = g.serialize(format='turtle',
+            content = g.serialize(format=format,
                                   base=base).decode('utf-8')
             mimetype = 'text/{}'.format(format)
         else:
@@ -207,26 +208,20 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)):
 
         result = self.serializable()
 
-        ctx = context_uri or self._context
-
-        result['@context'] = ctx
-        # result = jsonld.compact(result,
-        #                         ctx,
-        #                         options={
-        #                             'base': prefix,
-        #                             'expandContext': self._context,
-        #                             'senpy': prefix
-        #                         })
-
         if expanded:
             result = jsonld.expand(
                 result, options={'base': prefix,
-                                 'expandContext': ctx})
+                                 'expandContext': self._context})[0]
         if not with_context:
             try:
                 del result['@context']
             except KeyError:
                 pass
+        elif context_uri:
+            result['@context'] = context_uri
+        else:
+            result['@context'] = self._context
+
         return result
 
     def validate(self, obj=None):

senpy/plugins/__init__.py

@@ -23,18 +23,11 @@ import nltk
 
 from .. import models, utils
 from .. import api
+from .. import gsitk_compat
 
 
 logger = logging.getLogger(__name__)
 
-try:
-    from gsitk.evaluation.evaluation import Evaluation as Eval
-    from sklearn.pipeline import Pipeline
-    GSITK_AVAILABLE = True
-except ImportError:
-    logger.warn('GSITK is not installed. Some functions will be unavailable.')
-    GSITK_AVAILABLE = False
-
 
 class PluginMeta(models.BaseMeta):
     _classes = {}
@@ -333,7 +326,7 @@ class Box(AnalysisPlugin):
         return self.transform(X)
 
     def as_pipe(self):
-        pipe = Pipeline([('plugin', self)])
+        pipe = gsitk_compat.Pipeline([('plugin', self)])
         pipe.name = self.name
         return pipe
 
@@ -626,10 +619,7 @@ def _from_loaded_module(module, info=None, **kwargs):
 
 
 def evaluate(plugins, datasets, **kwargs):
-    if not GSITK_AVAILABLE:
+    ev = gsitk_compat.Eval(tuples=None,
-        raise Exception('GSITK is not available. Install it to use this function.')
-
-    ev = Eval(tuples=None,
                            datasets=datasets,
                            pipelines=[plugin.as_pipe() for plugin in plugins])
     ev.evaluate()

senpy/static/js/main.js

@@ -413,7 +413,7 @@ function evaluate_JSON(){
     url += "?algo="+plugin+"&dataset="+datasets
 
     $('#doevaluate').attr("disabled", true);
-    $.ajax({type: "GET", url: url, dataType: 'json'}).done(function(resp) {
+    $.ajax({type: "GET", url: url, dataType: 'json'}).always(function(resp) {
         $('#doevaluate').attr("disabled", false);
         response = resp.responseText;  
 

senpy/utils.py

@@ -80,7 +80,7 @@ def easy_test(plugin_list=None, debug=True):
         for plug in plugin_list:
             plug.test()
             plug.log.info('My tests passed!')
-            logger.info('All tests passed!')
+        logger.info('All tests passed for {} plugins!'.format(len(plugin_list)))
     except Exception:
         if not debug:
             raise

tests/test_blueprints.py

@@ -21,7 +21,6 @@ class BlueprintsTest(TestCase):
     def setUpClass(cls):
         """Set up only once, and re-use in every individual test"""
         cls.app = Flask("test_extensions")
-        cls.app.debug = False
         cls.client = cls.app.test_client()
         cls.senpy = Senpy(default_plugins=True)
         cls.senpy.init_app(cls.app)
@@ -31,6 +30,9 @@ class BlueprintsTest(TestCase):
         cls.senpy.activate_plugin("DummyRequired", sync=True)
         cls.senpy.default_plugin = 'Dummy'
 
+    def setUp(self):
+        self.app.config['TESTING'] = True  # Tell Flask not to catch Exceptions
+
     def assertCode(self, resp, code):
         self.assertEqual(resp.status_code, code)
 
@@ -42,6 +44,7 @@ class BlueprintsTest(TestCase):
         """
         Calling with no arguments should ask the user for more arguments
         """
+        self.app.config['TESTING'] = False  # Errors are expected in this case
         resp = self.client.get("/api/")
         self.assertCode(resp, 400)
         js = parse_resp(resp)
@@ -81,7 +84,7 @@ class BlueprintsTest(TestCase):
         Extra params that have a required argument that does not
         have a default should raise an error.
         """
-        self.app.debug = False
+        self.app.config['TESTING'] = False  # Errors are expected in this case
         resp = self.client.get("/api/?i=My aloha mohame&algo=DummyRequired")
         self.assertCode(resp, 400)
         js = parse_resp(resp)
@@ -97,7 +100,7 @@ class BlueprintsTest(TestCase):
         The dummy plugin returns an empty response,\
         it should contain the context
         """
-        self.app.debug = False
+        self.app.config['TESTING'] = False  # Errors are expected in this case
         resp = self.client.get("/api/?i=My aloha mohame&algo=DOESNOTEXIST")
         self.assertCode(resp, 404)
         js = parse_resp(resp)
@@ -172,5 +175,6 @@ class BlueprintsTest(TestCase):
         assert "help" in js["valid_parameters"]
 
     def test_conversion(self):
+        self.app.config['TESTING'] = False  # Errors are expected in this case
         resp = self.client.get("/api/?input=hello&algo=emoRand&emotionModel=DOES NOT EXIST")
         self.assertCode(resp, 404)

tests/test_plugins.py

@@ -6,7 +6,7 @@ import pickle
 import shutil
 import tempfile
 
-from unittest import TestCase, skipIf
+from unittest import TestCase
 from senpy.models import Results, Entry, EmotionSet, Emotion, Plugins
 from senpy import plugins
 from senpy.plugins.conversion.emotion.centroids import CentroidConversion
@@ -312,9 +312,7 @@ class PluginsTest(TestCase):
         res = c._backwards_conversion(e)
         assert res["onyx:hasEmotionCategory"] == "c2"
 
-    @skipIf(sys.version_info < (3, 0),
+    def _test_evaluation(self):
-            reason="requires Python3")
-    def test_evaluation(self):
         testdata = []
         for i in range(50):
             testdata.append(["good", 1])
@@ -348,6 +346,14 @@ class PluginsTest(TestCase):
         smart_metrics = results[0].metrics[0]
         assert abs(smart_metrics['accuracy'] - 1) < 0.01
 
+    def test_evaluation(self):
+        if sys.version_info < (3, 0):
+            with self.assertRaises(Exception) as context:
+                self._test_evaluation()
+            self.assertTrue('GSITK ' in str(context.exception))
+        else:
+            self._test_evaluation()
+
 
 def make_mini_test(fpath):
     def mini_test(self):