Added CLI and refactored argument parsing

senpy/__init__.py

@@ -18,4 +18,4 @@
 Sentiment analysis server in Python
 """
 
-__version__  = "0.5.4"
+__version__  = "0.5.5rc1"

senpy/api.py

@@ -0,0 +1,105 @@
+from future.utils import iteritems
+import logging
+logger = logging.getLogger(__name__)
+
+from .models import Error
+
+API_PARAMS = {
+    "algorithm": {
+        "aliases": ["algorithm", "a", "algo"],
+        "required": False,
+    }
+}
+
+WEB_PARAMS = {
+    "inHeaders": {
+        "aliases": ["inHeaders", "headers"],
+        "required": True,
+        "default": "0"
+    },
+}
+
+CLI_PARAMS = {
+    "plugin_folder": {
+        "aliases": ["plugin_folder", "folder"],
+        "required": True,
+        "default": "."
+    },
+    }
+
+NIF_PARAMS = {
+    "input": {
+        "@id": "input",
+        "aliases": ["i", "input"],
+        "required": True,
+        "help": "Input text"
+    },
+    "informat": {
+        "@id": "informat",
+        "aliases": ["f", "informat"],
+        "required": False,
+        "default": "text",
+        "options": ["turtle", "text"],
+    },
+    "intype": {
+        "@id": "intype",
+        "aliases": ["intype", "t"],
+        "required": False,
+        "default": "direct",
+        "options": ["direct", "url", "file"],
+    },
+    "outformat": {
+        "@id": "outformat",
+        "aliases": ["outformat", "o"],
+        "default": "json-ld",
+        "required": False,
+        "options": ["json-ld"],
+    },
+    "language": {
+        "@id": "language",
+        "aliases": ["language", "l"],
+        "required": False,
+    },
+    "prefix": {
+        "@id": "prefix",
+        "aliases": ["prefix", "p"],
+        "required": True,
+        "default": "",
+    },
+    "urischeme": {
+        "@id": "urischeme",
+        "aliases": ["urischeme", "u"],
+        "required": False,
+        "default": "RFC5147String",
+        "options": "RFC5147String"
+    },
+}
+
+
+def parse_params(indict, spec=NIF_PARAMS):
+    outdict = {}
+    wrong_params = {}
+    for param, options in iteritems(spec):
+        if param[0] != "@":  # Exclude json-ld properties
+            logger.debug("Param: %s - Options: %s", param, options)
+            for alias in options["aliases"]:
+                if alias in indict:
+                    outdict[param] = indict[alias]
+            if param not in outdict:
+                if options.get("required", False) and "default" not in options:
+                    wrong_params[param] = spec[param]
+                else:
+                    if "default" in options:
+                        outdict[param] = options["default"]
+            else:
+                if "options" in spec[param] and \
+                   outdict[param] not in spec[param]["options"]:
+                    wrong_params[param] = spec[param]
+    if wrong_params:
+        message = Error(status=404,
+                        message="Missing or invalid parameters",
+                        parameters=outdict,
+                        errors={param: error for param, error in
+                                iteritems(wrong_params)})
+        raise message
+    return outdict

senpy/blueprints.py

@@ -19,7 +19,7 @@ Blueprints for Senpy
 """
 from flask import Blueprint, request, current_app, render_template, url_for, jsonify
 from .models import Error, Response, Plugins, read_schema
-from future.utils import iteritems
+from .api import NIF_PARAMS, WEB_PARAMS, parse_params
 from functools import wraps
 
 import json
@@ -30,119 +30,14 @@ logger = logging.getLogger(__name__)
 api_blueprint = Blueprint("api", __name__)
 demo_blueprint = Blueprint("demo", __name__)
 
-API_PARAMS = {
+def get_params(req):
-    "algorithm": {
-        "aliases": ["algorithm", "a", "algo"],
-        "required": False,
-    },
-    "inHeaders": {
-        "aliases": ["inHeaders", "headers"],
-        "required": True,
-        "default": "0"
-    },
-    "prefix": {
-        "@id": "prefix",
-        "aliases": ["prefix", "p"],
-        "required": True,
-        "default": "",
-    },
-}
-
-NIF_PARAMS = {
-    "algorithm": {
-        "aliases": ["algorithm", "a", "algo"],
-        "required": False,
-    },
-    "inHeaders": {
-        "aliases": ["inHeaders", "headers"],
-        "required": True,
-        "default": "0"
-    },
-    "input": {
-        "@id": "input",
-        "aliases": ["i", "input"],
-        "required": True,
-        "help": "Input text"
-    },
-    "informat": {
-        "@id": "informat",
-        "aliases": ["f", "informat"],
-        "required": False,
-        "default": "text",
-        "options": ["turtle", "text"],
-    },
-    "intype": {
-        "@id": "intype",
-        "aliases": ["intype", "t"],
-        "required": False,
-        "default": "direct",
-        "options": ["direct", "url", "file"],
-    },
-    "outformat": {
-        "@id": "outformat",
-        "aliases": ["outformat", "o"],
-        "default": "json-ld",
-        "required": False,
-        "options": ["json-ld"],
-    },
-    "language": {
-        "@id": "language",
-        "aliases": ["language", "l"],
-        "required": False,
-    },
-    "prefix": {
-        "@id": "prefix",
-        "aliases": ["prefix", "p"],
-        "required": True,
-        "default": "",
-    },
-    "urischeme": {
-        "@id": "urischeme",
-        "aliases": ["urischeme", "u"],
-        "required": False,
-        "default": "RFC5147String",
-        "options": "RFC5147String"
-    },
-}
-
-def update_params(req, params=NIF_PARAMS):
     if req.method == 'POST':
-        indict = req.form
+        indict = req.form.to_dict(flat=True)
     elif req.method == 'GET':
-        indict = req.args
+        indict = req.args.to_dict(flat=True)
     else:
         raise Error(message="Invalid data")
-
+    return indict
-    outdict = {}
-    wrong_params = {}
-    for param, options in iteritems(params):
-        if param[0] != "@":  # Exclude json-ld properties
-            logger.debug("Param: %s - Options: %s", param, options)
-            for alias in options["aliases"]:
-                if alias in indict:
-                    outdict[param] = indict[alias]
-            if param not in outdict:
-                if options.get("required", False) and "default" not in options:
-                    wrong_params[param] = params[param]
-                else:
-                    if "default" in options:
-                        outdict[param] = options["default"]
-            else:
-                if "options" in params[param] and \
-                   outdict[param] not in params[param]["options"]:
-                    wrong_params[param] = params[param]
-    if wrong_params:
-        message = Error(status=404,
-                        message="Missing or invalid parameters",
-                        parameters=outdict,
-                        errors={param: error for param, error in
-                                iteritems(wrong_params)})
-        raise message
-    if hasattr(request, 'params'):
-        request.params.update(outdict)
-    else:
-        request.params = outdict
-    return outdict
 
 
 @demo_blueprint.route('/')
@@ -165,17 +60,20 @@ def basic_api(f):
     def decorated_function(*args, **kwargs):
         print('Getting request:')
         print(request)
-        update_params(request, params=API_PARAMS)
+        raw_params = get_params(request)
-        print('Params: %s' % request.params)
+        web_params = parse_params(raw_params, spec=WEB_PARAMS)
+
+        if hasattr(request, 'params'):
+            request.params.update(raw_params)
+        else:
+            request.params = raw_params
         try:
             response = f(*args, **kwargs)
         except Error as ex:
             response = ex
-        in_headers = request.params["inHeaders"] != "0"
+        in_headers = web_params["inHeaders"] != "0"
-        prefix = request.params["prefix"]
+        headers = {'X-ORIGINAL-PARAMS': raw_params}
-        headers = {'X-ORIGINAL-PARAMS': request.params}
         return response.flask(in_headers=in_headers,
-                              prefix=prefix,
                               headers=headers,
                               context_uri=url_for('api.context', entity=type(response).__name__,
                                                   _external=True))
@@ -184,11 +82,6 @@ def basic_api(f):
 @api_blueprint.route('/', methods=['POST', 'GET'])
 @basic_api
 def api():
-    algo = request.params.get("algorithm", None)
-    specific_params = current_app.senpy.parameters(algo)
-    update_params(request, params=NIF_PARAMS)
-    logger.debug("Specific params: %s", json.dumps(specific_params, indent=4))
-    update_params(request, specific_params)
     response = current_app.senpy.analyse(**request.params)
     return response
 

senpy/cli.py

@@ -0,0 +1,50 @@
+import sys
+from .models import Error
+from .api import parse_params, CLI_PARAMS
+from .extensions import Senpy
+
+def argv_to_dict(argv):
+    '''Turns parameters in the form of '--key value' into a dict {'key': 'value'}
+    '''
+    cli_dict = {}
+
+    for i in range(len(argv)):
+        if argv[i][0] == '-':
+            key = argv[i].strip('-')
+            value = argv[i+1] if len(argv)>i+1 else None
+            if value and value[0] == '-':
+                cli_dict[key] = ""
+            else:
+                cli_dict[key] = value
+    return cli_dict
+
+def parse_cli(argv):
+    cli_dict = argv_to_dict(argv)
+    cli_params = parse_params(cli_dict, spec=CLI_PARAMS)
+    return cli_params, cli_dict
+
+
+def main_function(argv):
+    '''This is the method for unit testing
+    '''
+    cli_params, cli_dict = parse_cli(argv)
+    plugin_folder = cli_params['plugin_folder']
+    sp = Senpy(default_plugins=False, plugin_folder=plugin_folder)
+    sp.activate_all(sync=True)
+    res = sp.analyse(**cli_dict)
+    return res
+
+def main():
+    '''This method is the entrypoint for the CLI (as configured un setup.py)
+    '''
+    try:
+        res = main_function(sys.argv[1:])
+        print(res.to_JSON())
+    except Error as err:
+        print(err.to_JSON())
+        sys.exit(2)
+    
+
+if __name__ == '__main__':
+    main()
+

senpy/extensions.py

@@ -9,6 +9,7 @@ monkey.patch_all()
 from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
 from .models import Error
 from .blueprints import api_blueprint, demo_blueprint
+from .api import API_PARAMS, NIF_PARAMS, parse_params
 
 from git import Repo, InvalidGitRepositoryError
 from functools import partial
@@ -72,8 +73,9 @@ class Senpy(object):
     def analyse(self, **params):
         algo = None
         logger.debug("analysing with params: {}".format(params))
-        if "algorithm" in params:
+        api_params = parse_params(params, spec=API_PARAMS)
-            algo = params["algorithm"]
+        if "algorithm" in api_params and api_params["algorithm"]:
+            algo = api_params["algorithm"]
         elif self.plugins:
             algo = self.default_plugin and self.default_plugin.name
         if not algo:
@@ -94,8 +96,12 @@ class Senpy(object):
                         message=("The algorithm '{}'"
                                     " is not activated yet").format(algo))
         plug = self.plugins[algo]
+        nif_params = parse_params(params, spec=NIF_PARAMS)
+        extra_params = plug.get('extra_params', {})
+        specific_params = parse_params(params, spec=extra_params)
+        nif_params.update(specific_params)
         try:
-            resp = plug.analyse(**params)
+            resp = plug.analyse(**nif_params)
             resp.analysis.append(plug)
             logger.debug("Returning analysis result: {}".format(resp))
         except Exception as ex:

senpy/models.py

@@ -69,7 +69,6 @@ class SenpyMixin(object):
     def flask(self,
               in_headers=False,
               headers=None,
-              prefix=None,
               **kwargs):
         """
         Return the values and error to be used in flask.
@@ -110,7 +109,7 @@ class SenpyMixin(object):
         return ser_or_down(self._plain_dict())
 
 
-    def jsonld(self, prefix=None, with_context=True, context_uri=None):
+    def jsonld(self, with_context=True, context_uri=None):
         ser = self.serializable()
 
         if  with_context:
@@ -119,13 +118,13 @@ class SenpyMixin(object):
                 context = context_uri
             else:
                 context = self.context.copy()
-            if prefix:
+            if hasattr(self, 'prefix'):
                 # This sets @base for the document, which will be used in
                 # all relative URIs will. For example, if a uri is "Example" and
                 # prefix =s "http://example.com", the absolute URI after expanding
                 # with JSON-LD will be "http://example.com/Example"
 
-                prefix_context = {"@base": prefix}
+                prefix_context = {"@base": self.prefix}
                 if isinstance(context, list):
                     context.append(prefix_context)
                 else:

setup.py

@@ -37,7 +37,8 @@ extendable, so new algorithms and sources can be used.
     include_package_data=True,
     entry_points={
         'console_scripts': [
-            'senpy = senpy.__main__:main'
+            'senpy = senpy.__main__:main',
+            'senpy-cli = senpy.cli:main'
             ]
         }
 )

tests/sleep_plugin/sleep.py

@@ -5,10 +5,6 @@ from time import sleep
 
 class SleepPlugin(SenpyPlugin):
 
-    def __init__(self, info, *args, **kwargs):
-        super(SleepPlugin, self).__init__(info, *args, **kwargs)
-        self.timeout = int(info["timeout"])
-
     def activate(self, *args, **kwargs):
         sleep(self.timeout)
 

tests/sleep_plugin/sleep.senpy

@@ -4,7 +4,7 @@
     "description": "I am dummy",
     "author": "@balkian",
     "version": "0.1",
-    "timeout": "2",
+    "timeout": 0.5,
     "extra_params": {
         "timeout": {
             "@id": "timeout_sleep",

tests/test_blueprints.py

@@ -61,7 +61,7 @@ class BlueprintsTest(TestCase):
 
     def test_headers(self):
         for i, j in product(["/api/plugins/?nothing=", "/api/?i=test&"],
-                            ["headers", "inHeaders"]):
+                            ["inHeaders"]):
             resp = self.client.get("%s" % (i))
             assert "@context" in resp.json
             resp = self.client.get("%s&%s=0" % (i, j))

tests/test_cli.py

@@ -0,0 +1,21 @@
+import os
+import logging
+from functools import partial
+
+logger = logging.getLogger(__name__)
+
+from unittest import TestCase
+from senpy.cli import main_function
+from senpy.models import Error
+
+
+class CLITest(TestCase):
+
+    def test_basic(self):
+        self.assertRaises(Error, partial(main_function, []))
+        res = main_function(['--input', 'test'])
+        assert 'entries' in res
+        res = main_function(['--input', 'test', '--algo', 'rand'])
+        assert 'entries' in res
+        assert 'analysis' in res
+        assert res['analysis'][0]['name'] == 'rand'