balkian/senpy
1
0
Fork 0
mirror of https://github.com/gsi-upm/senpy synced 2024-12-22 13:08:13 +00:00
Code Issues Releases Wiki Activity

Added CLI and refactored argument parsing

Browse Source
This commit is contained in:
J. Fernando Sánchez 2016-03-02 05:07:48 +01:00
parent a50f026701
commit 03eb38c12d
11 changed files with 207 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
senpy/__init__.py
View File

@ -18,4 +18,4 @@
Sentiment analysis server in Python
"""
__version__ = "0.5.4"
__version__ = "0.5.5rc1"

105
senpy/api.py Normal file
View File

@ -0,0 +1,105 @@
from future.utils import iteritems
import logging
logger = logging.getLogger(__name__)
from .models import Error
API_PARAMS = {
"algorithm": {
"aliases": ["algorithm", "a", "algo"],
"required": False,
}
}
WEB_PARAMS = {
"inHeaders": {
"aliases": ["inHeaders", "headers"],
"required": True,
"default": "0"
},
}
CLI_PARAMS = {
"plugin_folder": {
"aliases": ["plugin_folder", "folder"],
"required": True,
"default": "."
},
}
NIF_PARAMS = {
"input": {
"@id": "input",
"aliases": ["i", "input"],
"required": True,
"help": "Input text"
},
"informat": {
"@id": "informat",
"aliases": ["f", "informat"],
"required": False,
"default": "text",
"options": ["turtle", "text"],
},
"intype": {
"@id": "intype",
"aliases": ["intype", "t"],
"required": False,
"default": "direct",
"options": ["direct", "url", "file"],
},
"outformat": {
"@id": "outformat",
"aliases": ["outformat", "o"],
"default": "json-ld",
"required": False,
"options": ["json-ld"],
},
"language": {
"@id": "language",
"aliases": ["language", "l"],
"required": False,
},
"prefix": {
"@id": "prefix",
"aliases": ["prefix", "p"],
"required": True,
"default": "",
},
"urischeme": {
"@id": "urischeme",
"aliases": ["urischeme", "u"],
"required": False,
"default": "RFC5147String",
"options": "RFC5147String"
},
}
def parse_params(indict, spec=NIF_PARAMS):
outdict = {}
wrong_params = {}
for param, options in iteritems(spec):
if param[0] != "@": # Exclude json-ld properties
logger.debug("Param: %s - Options: %s", param, options)
for alias in options["aliases"]:
if alias in indict:
outdict[param] = indict[alias]
if param not in outdict:
if options.get("required", False) and "default" not in options:
wrong_params[param] = spec[param]
else:
if "default" in options:
outdict[param] = options["default"]
else:
if "options" in spec[param] and \
outdict[param] not in spec[param]["options"]:
wrong_params[param] = spec[param]
if wrong_params:
message = Error(status=404,
message="Missing or invalid parameters",
parameters=outdict,
errors={param: error for param, error in
iteritems(wrong_params)})
raise message
return outdict

135
senpy/blueprints.py
View File

@ -19,7 +19,7 @@ Blueprints for Senpy
"""
from flask import Blueprint, request, current_app, render_template, url_for, jsonify
from .models import Error, Response, Plugins, read_schema
from future.utils import iteritems
from .api import NIF_PARAMS, WEB_PARAMS, parse_params
from functools import wraps
import json
@ -30,119 +30,14 @@ logger = logging.getLogger(__name__)
api_blueprint = Blueprint("api", __name__)
demo_blueprint = Blueprint("demo", __name__)
API_PARAMS = {
"algorithm": {
"aliases": ["algorithm", "a", "algo"],
"required": False,
},
"inHeaders": {
"aliases": ["inHeaders", "headers"],
"required": True,
"default": "0"
},
"prefix": {
"@id": "prefix",
"aliases": ["prefix", "p"],
"required": True,
"default": "",
},
}
NIF_PARAMS = {
"algorithm": {
"aliases": ["algorithm", "a", "algo"],
"required": False,
},
"inHeaders": {
"aliases": ["inHeaders", "headers"],
"required": True,
"default": "0"
},
"input": {
"@id": "input",
"aliases": ["i", "input"],
"required": True,
"help": "Input text"
},
"informat": {
"@id": "informat",
"aliases": ["f", "informat"],
"required": False,
"default": "text",
"options": ["turtle", "text"],
},
"intype": {
"@id": "intype",
"aliases": ["intype", "t"],
"required": False,
"default": "direct",
"options": ["direct", "url", "file"],
},
"outformat": {
"@id": "outformat",
"aliases": ["outformat", "o"],
"default": "json-ld",
"required": False,
"options": ["json-ld"],
},
"language": {
"@id": "language",
"aliases": ["language", "l"],
"required": False,
},
"prefix": {
"@id": "prefix",
"aliases": ["prefix", "p"],
"required": True,
"default": "",
},
"urischeme": {
"@id": "urischeme",
"aliases": ["urischeme", "u"],
"required": False,
"default": "RFC5147String",
"options": "RFC5147String"
},
}
def update_params(req, params=NIF_PARAMS):
def get_params(req):
if req.method == 'POST':
indict = req.form
indict = req.form.to_dict(flat=True)
elif req.method == 'GET':
indict = req.args
indict = req.args.to_dict(flat=True)
else:
raise Error(message="Invalid data")
outdict = {}
wrong_params = {}
for param, options in iteritems(params):
if param[0] != "@": # Exclude json-ld properties
logger.debug("Param: %s - Options: %s", param, options)
for alias in options["aliases"]:
if alias in indict:
outdict[param] = indict[alias]
if param not in outdict:
if options.get("required", False) and "default" not in options:
wrong_params[param] = params[param]
else:
if "default" in options:
outdict[param] = options["default"]
else:
if "options" in params[param] and \
outdict[param] not in params[param]["options"]:
wrong_params[param] = params[param]
if wrong_params:
message = Error(status=404,
message="Missing or invalid parameters",
parameters=outdict,
errors={param: error for param, error in
iteritems(wrong_params)})
raise message
if hasattr(request, 'params'):
request.params.update(outdict)
else:
request.params = outdict
return outdict
return indict
@demo_blueprint.route('/')
@ -165,17 +60,20 @@ def basic_api(f):
def decorated_function(*args, **kwargs):
print('Getting request:')
print(request)
update_params(request, params=API_PARAMS)
print('Params: %s' % request.params)
raw_params = get_params(request)
web_params = parse_params(raw_params, spec=WEB_PARAMS)
if hasattr(request, 'params'):
request.params.update(raw_params)
else:
request.params = raw_params
try:
response = f(*args, **kwargs)
except Error as ex:
response = ex
in_headers = request.params["inHeaders"] != "0"
prefix = request.params["prefix"]
headers = {'X-ORIGINAL-PARAMS': request.params}
in_headers = web_params["inHeaders"] != "0"
headers = {'X-ORIGINAL-PARAMS': raw_params}
return response.flask(in_headers=in_headers,
prefix=prefix,
headers=headers,
context_uri=url_for('api.context', entity=type(response).__name__,
_external=True))
@ -184,11 +82,6 @@ def basic_api(f):
@api_blueprint.route('/', methods=['POST', 'GET'])
@basic_api
def api():
algo = request.params.get("algorithm", None)
specific_params = current_app.senpy.parameters(algo)
update_params(request, params=NIF_PARAMS)
logger.debug("Specific params: %s", json.dumps(specific_params, indent=4))
update_params(request, specific_params)
response = current_app.senpy.analyse(**request.params)
return response

50
senpy/cli.py Normal file
View File

@ -0,0 +1,50 @@
import sys
from .models import Error
from .api import parse_params, CLI_PARAMS
from .extensions import Senpy
def argv_to_dict(argv):
'''Turns parameters in the form of '--key value' into a dict {'key': 'value'}
'''
cli_dict = {}
for i in range(len(argv)):
if argv[i][0] == '-':
key = argv[i].strip('-')
value = argv[i+1] if len(argv)>i+1 else None
if value and value[0] == '-':
cli_dict[key] = ""
else:
cli_dict[key] = value
return cli_dict
def parse_cli(argv):
cli_dict = argv_to_dict(argv)
cli_params = parse_params(cli_dict, spec=CLI_PARAMS)
return cli_params, cli_dict
def main_function(argv):
'''This is the method for unit testing
'''
cli_params, cli_dict = parse_cli(argv)
plugin_folder = cli_params['plugin_folder']
sp = Senpy(default_plugins=False, plugin_folder=plugin_folder)
sp.activate_all(sync=True)
res = sp.analyse(**cli_dict)
return res
def main():
'''This method is the entrypoint for the CLI (as configured un setup.py)
'''
try:
res = main_function(sys.argv[1:])
print(res.to_JSON())
except Error as err:
print(err.to_JSON())
sys.exit(2)
if __name__ == '__main__':
main()

12
senpy/extensions.py
View File

@ -9,6 +9,7 @@ monkey.patch_all()
from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
from .models import Error
from .blueprints import api_blueprint, demo_blueprint
from .api import API_PARAMS, NIF_PARAMS, parse_params
from git import Repo, InvalidGitRepositoryError
from functools import partial
@ -72,8 +73,9 @@ class Senpy(object):
def analyse(self, **params):
algo = None
logger.debug("analysing with params: {}".format(params))
if "algorithm" in params:
algo = params["algorithm"]
api_params = parse_params(params, spec=API_PARAMS)
if "algorithm" in api_params and api_params["algorithm"]:
algo = api_params["algorithm"]
elif self.plugins:
algo = self.default_plugin and self.default_plugin.name
if not algo:
@ -94,8 +96,12 @@ class Senpy(object):
message=("The algorithm '{}'"
" is not activated yet").format(algo))
plug = self.plugins[algo]
nif_params = parse_params(params, spec=NIF_PARAMS)
extra_params = plug.get('extra_params', {})
specific_params = parse_params(params, spec=extra_params)
nif_params.update(specific_params)
try:
resp = plug.analyse(**params)
resp = plug.analyse(**nif_params)
resp.analysis.append(plug)
logger.debug("Returning analysis result: {}".format(resp))
except Exception as ex:

7
senpy/models.py
View File

@ -69,7 +69,6 @@ class SenpyMixin(object):
def flask(self,
in_headers=False,
headers=None,
prefix=None,
**kwargs):
"""
Return the values and error to be used in flask.
@ -110,7 +109,7 @@ class SenpyMixin(object):
return ser_or_down(self._plain_dict())
def jsonld(self, prefix=None, with_context=True, context_uri=None):
def jsonld(self, with_context=True, context_uri=None):
ser = self.serializable()
if with_context:
@ -119,13 +118,13 @@ class SenpyMixin(object):
context = context_uri
else:
context = self.context.copy()
if prefix:
if hasattr(self, 'prefix'):
# This sets @base for the document, which will be used in
# all relative URIs will. For example, if a uri is "Example" and
# prefix =s "http://example.com", the absolute URI after expanding
# with JSON-LD will be "http://example.com/Example"
prefix_context = {"@base": prefix}
prefix_context = {"@base": self.prefix}
if isinstance(context, list):
context.append(prefix_context)
else:

3
setup.py
View File

@ -37,7 +37,8 @@ extendable, so new algorithms and sources can be used.
include_package_data=True,
entry_points={
'console_scripts': [
'senpy = senpy.__main__:main'
'senpy = senpy.__main__:main',
'senpy-cli = senpy.cli:main'
]
}
)

4
tests/sleep_plugin/sleep.py
View File

@ -5,10 +5,6 @@ from time import sleep
class SleepPlugin(SenpyPlugin):
def __init__(self, info, *args, **kwargs):
super(SleepPlugin, self).__init__(info, *args, **kwargs)
self.timeout = int(info["timeout"])
def activate(self, *args, **kwargs):
sleep(self.timeout)

2
tests/sleep_plugin/sleep.senpy
View File

@ -4,7 +4,7 @@
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"timeout": "2",
"timeout": 0.5,
"extra_params": {
"timeout": {
"@id": "timeout_sleep",

2
tests/test_blueprints.py
View File

@ -61,7 +61,7 @@ class BlueprintsTest(TestCase):
def test_headers(self):
for i, j in product(["/api/plugins/?nothing=", "/api/?i=test&"],
["headers", "inHeaders"]):
["inHeaders"]):
resp = self.client.get("%s" % (i))
assert "@context" in resp.json
resp = self.client.get("%s&%s=0" % (i, j))

21
tests/test_cli.py Normal file
View File

@ -0,0 +1,21 @@
import os
import logging
from functools import partial
logger = logging.getLogger(__name__)
from unittest import TestCase
from senpy.cli import main_function
from senpy.models import Error
class CLITest(TestCase):
def test_basic(self):
self.assertRaises(Error, partial(main_function, []))
res = main_function(['--input', 'test'])
assert 'entries' in res
res = main_function(['--input', 'test', '--algo', 'rand'])
assert 'entries' in res
assert 'analysis' in res
assert res['analysis'][0]['name'] == 'rand'