mirror of
https://github.com/gsi-upm/senpy
synced 2024-12-22 13:08:13 +00:00
WIP
* Modify dependency installation logic (avoid installing several times) * Add encoded URLs for as base/prefix
This commit is contained in:
parent
e5662d482e
commit
13cf0c71c5
20
senpy/api.py
20
senpy/api.py
@ -3,6 +3,10 @@ from .models import Error, Results, Entry, from_string
|
|||||||
import logging
|
import logging
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
boolean = (True, False)
|
||||||
|
|
||||||
|
|
||||||
API_PARAMS = {
|
API_PARAMS = {
|
||||||
"algorithm": {
|
"algorithm": {
|
||||||
"aliases": ["algorithms", "a", "algo"],
|
"aliases": ["algorithms", "a", "algo"],
|
||||||
@ -13,14 +17,14 @@ API_PARAMS = {
|
|||||||
"expanded-jsonld": {
|
"expanded-jsonld": {
|
||||||
"@id": "expanded-jsonld",
|
"@id": "expanded-jsonld",
|
||||||
"aliases": ["expanded"],
|
"aliases": ["expanded"],
|
||||||
"options": "boolean",
|
"options": boolean,
|
||||||
"required": True,
|
"required": True,
|
||||||
"default": False
|
"default": False
|
||||||
},
|
},
|
||||||
"with_parameters": {
|
"with_parameters": {
|
||||||
"aliases": ['withparameters',
|
"aliases": ['withparameters',
|
||||||
'with-parameters'],
|
'with-parameters'],
|
||||||
"options": "boolean",
|
"options": boolean,
|
||||||
"default": False,
|
"default": False,
|
||||||
"required": True
|
"required": True
|
||||||
},
|
},
|
||||||
@ -36,7 +40,7 @@ API_PARAMS = {
|
|||||||
"description": "Show additional help to know more about the possible parameters",
|
"description": "Show additional help to know more about the possible parameters",
|
||||||
"aliases": ["h"],
|
"aliases": ["h"],
|
||||||
"required": True,
|
"required": True,
|
||||||
"options": "boolean",
|
"options": boolean,
|
||||||
"default": False
|
"default": False
|
||||||
},
|
},
|
||||||
"emotionModel": {
|
"emotionModel": {
|
||||||
@ -83,7 +87,7 @@ WEB_PARAMS = {
|
|||||||
"aliases": ["headers"],
|
"aliases": ["headers"],
|
||||||
"required": True,
|
"required": True,
|
||||||
"default": False,
|
"default": False,
|
||||||
"options": "boolean"
|
"options": boolean
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -132,7 +136,7 @@ NIF_PARAMS = {
|
|||||||
"aliases": ["u"],
|
"aliases": ["u"],
|
||||||
"required": False,
|
"required": False,
|
||||||
"default": "RFC5147String",
|
"default": "RFC5147String",
|
||||||
"options": "RFC5147String"
|
"options": ["RFC5147String", ]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -159,7 +163,7 @@ def parse_params(indict, *specs):
|
|||||||
wrong_params[param] = spec[param]
|
wrong_params[param] = spec[param]
|
||||||
continue
|
continue
|
||||||
if "options" in options:
|
if "options" in options:
|
||||||
if options["options"] == "boolean":
|
if options["options"] == boolean:
|
||||||
outdict[param] = outdict[param] in [None, True, 'true', '1']
|
outdict[param] = outdict[param] in [None, True, 'true', '1']
|
||||||
elif outdict[param] not in options["options"]:
|
elif outdict[param] not in options["options"]:
|
||||||
wrong_params[param] = spec[param]
|
wrong_params[param] = spec[param]
|
||||||
@ -171,8 +175,8 @@ def parse_params(indict, *specs):
|
|||||||
parameters=outdict,
|
parameters=outdict,
|
||||||
errors=wrong_params)
|
errors=wrong_params)
|
||||||
raise message
|
raise message
|
||||||
if 'algorithm' in outdict and not isinstance(outdict['algorithm'], list):
|
if 'algorithm' in outdict and not isinstance(outdict['algorithm'], tuple):
|
||||||
outdict['algorithm'] = outdict['algorithm'].split(',')
|
outdict['algorithm'] = tuple(outdict['algorithm'].split(','))
|
||||||
return outdict
|
return outdict
|
||||||
|
|
||||||
|
|
||||||
|
@ -18,7 +18,7 @@
|
|||||||
Blueprints for Senpy
|
Blueprints for Senpy
|
||||||
"""
|
"""
|
||||||
from flask import (Blueprint, request, current_app, render_template, url_for,
|
from flask import (Blueprint, request, current_app, render_template, url_for,
|
||||||
jsonify)
|
jsonify, redirect)
|
||||||
from .models import Error, Response, Help, Plugins, read_schema, dump_schema, Datasets
|
from .models import Error, Response, Help, Plugins, read_schema, dump_schema, Datasets
|
||||||
from . import api
|
from . import api
|
||||||
from .version import __version__
|
from .version import __version__
|
||||||
@ -27,6 +27,7 @@ from functools import wraps
|
|||||||
import logging
|
import logging
|
||||||
import traceback
|
import traceback
|
||||||
import json
|
import json
|
||||||
|
import base64
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@ -34,6 +35,19 @@ api_blueprint = Blueprint("api", __name__)
|
|||||||
demo_blueprint = Blueprint("demo", __name__, template_folder='templates')
|
demo_blueprint = Blueprint("demo", __name__, template_folder='templates')
|
||||||
ns_blueprint = Blueprint("ns", __name__)
|
ns_blueprint = Blueprint("ns", __name__)
|
||||||
|
|
||||||
|
_mimetypes_r = {'json-ld': ['application/ld+json'],
|
||||||
|
'turtle': ['text/turtle'],
|
||||||
|
'text': ['text/plain']}
|
||||||
|
|
||||||
|
MIMETYPES = {}
|
||||||
|
|
||||||
|
for k, vs in _mimetypes_r.items():
|
||||||
|
for v in vs:
|
||||||
|
MIMETYPES[v] = k
|
||||||
|
|
||||||
|
DEFAULT_MIMETYPE = 'application/ld+json'
|
||||||
|
DEFAULT_FORMAT = 'json-ld'
|
||||||
|
|
||||||
|
|
||||||
def get_params(req):
|
def get_params(req):
|
||||||
if req.method == 'POST':
|
if req.method == 'POST':
|
||||||
@ -45,6 +59,30 @@ def get_params(req):
|
|||||||
return indict
|
return indict
|
||||||
|
|
||||||
|
|
||||||
|
def encoded_url(url=None, base=None):
|
||||||
|
code = ''
|
||||||
|
if not url:
|
||||||
|
if request.method == 'GET':
|
||||||
|
url = request.full_path[1:] # Remove the first slash
|
||||||
|
else:
|
||||||
|
hash(frozenset(request.form.params().items()))
|
||||||
|
code = 'hash:{}'.format(hash)
|
||||||
|
|
||||||
|
code = code or base64.urlsafe_b64encode(url.encode()).decode()
|
||||||
|
|
||||||
|
if base:
|
||||||
|
return base + code
|
||||||
|
return url_for('api.decode', code=code, _external=True)
|
||||||
|
|
||||||
|
|
||||||
|
def decoded_url(code, base=None):
|
||||||
|
if code.startswith('hash:'):
|
||||||
|
raise Exception('Can not decode a URL for a POST request')
|
||||||
|
base = base or request.url_root
|
||||||
|
path = base64.urlsafe_b64decode(code.encode()).decode()
|
||||||
|
return base + path
|
||||||
|
|
||||||
|
|
||||||
@demo_blueprint.route('/')
|
@demo_blueprint.route('/')
|
||||||
def index():
|
def index():
|
||||||
ev = str(get_params(request).get('evaluation', False))
|
ev = str(get_params(request).get('evaluation', False))
|
||||||
@ -59,13 +97,22 @@ def index():
|
|||||||
def context(entity="context"):
|
def context(entity="context"):
|
||||||
context = Response._context
|
context = Response._context
|
||||||
context['@vocab'] = url_for('ns.index', _external=True)
|
context['@vocab'] = url_for('ns.index', _external=True)
|
||||||
|
context['endpoint'] = url_for('api.api_root', _external=True)
|
||||||
return jsonify({"@context": context})
|
return jsonify({"@context": context})
|
||||||
|
|
||||||
|
|
||||||
|
@api_blueprint.route('/d/<code>')
|
||||||
|
def decode(code):
|
||||||
|
try:
|
||||||
|
return redirect(decoded_url(code))
|
||||||
|
except Exception:
|
||||||
|
return Error('invalid URL').flask()
|
||||||
|
|
||||||
|
|
||||||
@ns_blueprint.route('/') # noqa: F811
|
@ns_blueprint.route('/') # noqa: F811
|
||||||
def index():
|
def index():
|
||||||
context = Response._context
|
context = Response._context.copy()
|
||||||
context['@vocab'] = url_for('.ns', _external=True)
|
context['endpoint'] = url_for('api.api_root', _external=True)
|
||||||
return jsonify({"@context": context})
|
return jsonify({"@context": context})
|
||||||
|
|
||||||
|
|
||||||
@ -81,7 +128,7 @@ def basic_api(f):
|
|||||||
default_params = {
|
default_params = {
|
||||||
'inHeaders': False,
|
'inHeaders': False,
|
||||||
'expanded-jsonld': False,
|
'expanded-jsonld': False,
|
||||||
'outformat': 'json-ld',
|
'outformat': None,
|
||||||
'with_parameters': True,
|
'with_parameters': True,
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -115,14 +162,21 @@ def basic_api(f):
|
|||||||
del response.parameters
|
del response.parameters
|
||||||
|
|
||||||
logger.info('Response: {}'.format(response))
|
logger.info('Response: {}'.format(response))
|
||||||
|
mime = request.accept_mimetypes\
|
||||||
|
.best_match(MIMETYPES.keys(),
|
||||||
|
DEFAULT_MIMETYPE)
|
||||||
|
|
||||||
|
mimeformat = MIMETYPES.get(mime, DEFAULT_FORMAT)
|
||||||
|
outformat = params['outformat'] or mimeformat
|
||||||
|
|
||||||
return response.flask(
|
return response.flask(
|
||||||
in_headers=params['inHeaders'],
|
in_headers=params['inHeaders'],
|
||||||
headers=headers,
|
headers=headers,
|
||||||
prefix=url_for('.api_root', _external=True),
|
prefix=params.get('prefix', encoded_url()),
|
||||||
context_uri=url_for('api.context',
|
context_uri=url_for('api.context',
|
||||||
entity=type(response).__name__,
|
entity=type(response).__name__,
|
||||||
_external=True),
|
_external=True),
|
||||||
outformat=params['outformat'],
|
outformat=outformat,
|
||||||
expanded=params['expanded-jsonld'])
|
expanded=params['expanded-jsonld'])
|
||||||
|
|
||||||
return decorated_function
|
return decorated_function
|
||||||
|
@ -95,7 +95,7 @@ class Senpy(object):
|
|||||||
if plugin in self._plugins:
|
if plugin in self._plugins:
|
||||||
return self._plugins[plugin]
|
return self._plugins[plugin]
|
||||||
|
|
||||||
results = self.plugins(id='plugins/{}'.format(name))
|
results = self.plugins(id='endpoint:plugins/{}'.format(name))
|
||||||
|
|
||||||
if not results:
|
if not results:
|
||||||
return Error(message="Plugin not found", status=404)
|
return Error(message="Plugin not found", status=404)
|
||||||
|
@ -138,7 +138,7 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)):
|
|||||||
@property
|
@property
|
||||||
def id(self):
|
def id(self):
|
||||||
if '@id' not in self:
|
if '@id' not in self:
|
||||||
self['@id'] = ':{}_{}'.format(type(self).__name__, time.time())
|
self['@id'] = '_:{}_{}'.format(type(self).__name__, time.time())
|
||||||
return self['@id']
|
return self['@id']
|
||||||
|
|
||||||
@id.setter
|
@id.setter
|
||||||
@ -146,7 +146,7 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)):
|
|||||||
self['@id'] = value
|
self['@id'] = value
|
||||||
|
|
||||||
def flask(self,
|
def flask(self,
|
||||||
in_headers=True,
|
in_headers=False,
|
||||||
headers=None,
|
headers=None,
|
||||||
outformat='json-ld',
|
outformat='json-ld',
|
||||||
**kwargs):
|
**kwargs):
|
||||||
@ -176,20 +176,21 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)):
|
|||||||
|
|
||||||
def serialize(self, format='json-ld', with_mime=False, **kwargs):
|
def serialize(self, format='json-ld', with_mime=False, **kwargs):
|
||||||
js = self.jsonld(**kwargs)
|
js = self.jsonld(**kwargs)
|
||||||
if format == 'json-ld':
|
|
||||||
content = json.dumps(js, indent=2, sort_keys=True)
|
content = json.dumps(js, indent=2, sort_keys=True)
|
||||||
|
if format == 'json-ld':
|
||||||
mimetype = "application/json"
|
mimetype = "application/json"
|
||||||
elif format in ['turtle', ]:
|
elif format in ['turtle', ]:
|
||||||
logger.debug(js)
|
logger.debug(js)
|
||||||
content = json.dumps(js, indent=2, sort_keys=True)
|
base = kwargs.get('prefix')
|
||||||
g = Graph().parse(
|
g = Graph().parse(
|
||||||
data=content,
|
data=content,
|
||||||
format='json-ld',
|
format='json-ld',
|
||||||
base=kwargs.get('prefix'),
|
base=base,
|
||||||
context=self._context)
|
context=self._context)
|
||||||
logger.debug(
|
logger.debug(
|
||||||
'Parsing with prefix: {}'.format(kwargs.get('prefix')))
|
'Parsing with prefix: {}'.format(kwargs.get('prefix')))
|
||||||
content = g.serialize(format='turtle').decode('utf-8')
|
content = g.serialize(format='turtle',
|
||||||
|
base=base).decode('utf-8')
|
||||||
mimetype = 'text/{}'.format(format)
|
mimetype = 'text/{}'.format(format)
|
||||||
else:
|
else:
|
||||||
raise Error('Unknown outformat: {}'.format(format))
|
raise Error('Unknown outformat: {}'.format(format))
|
||||||
@ -205,20 +206,22 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)):
|
|||||||
expanded=False):
|
expanded=False):
|
||||||
|
|
||||||
result = self.serializable()
|
result = self.serializable()
|
||||||
if context_uri or with_context:
|
|
||||||
result['@context'] = context_uri or self._context
|
|
||||||
|
|
||||||
|
ctx = context_uri or self._context
|
||||||
|
|
||||||
|
result['@context'] = ctx
|
||||||
# result = jsonld.compact(result,
|
# result = jsonld.compact(result,
|
||||||
# self._context,
|
# ctx,
|
||||||
# options={
|
# options={
|
||||||
# 'base': prefix,
|
# 'base': prefix,
|
||||||
# 'expandContext': self._context,
|
# 'expandContext': self._context,
|
||||||
# 'senpy': prefix
|
# 'senpy': prefix
|
||||||
# })
|
# })
|
||||||
|
|
||||||
if expanded:
|
if expanded:
|
||||||
result = jsonld.expand(
|
result = jsonld.expand(
|
||||||
result, options={'base': prefix,
|
result, options={'base': prefix,
|
||||||
'expandContext': self._context})
|
'expandContext': ctx})
|
||||||
if not with_context:
|
if not with_context:
|
||||||
try:
|
try:
|
||||||
del result['@context']
|
del result['@context']
|
||||||
|
@ -3,6 +3,7 @@ standard_library.install_aliases()
|
|||||||
|
|
||||||
|
|
||||||
from future.utils import with_metaclass
|
from future.utils import with_metaclass
|
||||||
|
from functools import partial
|
||||||
|
|
||||||
import os.path
|
import os.path
|
||||||
import os
|
import os
|
||||||
@ -92,7 +93,7 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)):
|
|||||||
if info:
|
if info:
|
||||||
self.update(info)
|
self.update(info)
|
||||||
self.validate()
|
self.validate()
|
||||||
self.id = 'plugins/{}_{}'.format(self['name'], self['version'])
|
self.id = 'endpoint:plugins/{}_{}'.format(self['name'], self['version'])
|
||||||
self.is_activated = False
|
self.is_activated = False
|
||||||
self._lock = threading.Lock()
|
self._lock = threading.Lock()
|
||||||
self._directory = os.path.abspath(os.path.dirname(inspect.getfile(self.__class__)))
|
self._directory = os.path.abspath(os.path.dirname(inspect.getfile(self.__class__)))
|
||||||
@ -530,7 +531,7 @@ def find_plugins(folders):
|
|||||||
yield fpath
|
yield fpath
|
||||||
|
|
||||||
|
|
||||||
def from_path(fpath, **kwargs):
|
def from_path(fpath, install_on_fail=False, **kwargs):
|
||||||
logger.debug("Loading plugin from {}".format(fpath))
|
logger.debug("Loading plugin from {}".format(fpath))
|
||||||
if fpath.endswith('.py'):
|
if fpath.endswith('.py'):
|
||||||
# We asume root is the dir of the file, and module is the name of the file
|
# We asume root is the dir of the file, and module is the name of the file
|
||||||
@ -540,7 +541,7 @@ def from_path(fpath, **kwargs):
|
|||||||
yield instance
|
yield instance
|
||||||
else:
|
else:
|
||||||
info = parse_plugin_info(fpath)
|
info = parse_plugin_info(fpath)
|
||||||
yield from_info(info, **kwargs)
|
yield from_info(info, install_on_fail=install_on_fail, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
def from_folder(folders, loader=from_path, **kwargs):
|
def from_folder(folders, loader=from_path, **kwargs):
|
||||||
@ -551,7 +552,7 @@ def from_folder(folders, loader=from_path, **kwargs):
|
|||||||
return plugins
|
return plugins
|
||||||
|
|
||||||
|
|
||||||
def from_info(info, root=None, **kwargs):
|
def from_info(info, root=None, install_on_fail=True, **kwargs):
|
||||||
if any(x not in info for x in ('module',)):
|
if any(x not in info for x in ('module',)):
|
||||||
raise ValueError('Plugin info is not valid: {}'.format(info))
|
raise ValueError('Plugin info is not valid: {}'.format(info))
|
||||||
module = info["module"]
|
module = info["module"]
|
||||||
@ -559,7 +560,12 @@ def from_info(info, root=None, **kwargs):
|
|||||||
if not root and '_path' in info:
|
if not root and '_path' in info:
|
||||||
root = os.path.dirname(info['_path'])
|
root = os.path.dirname(info['_path'])
|
||||||
|
|
||||||
return one_from_module(module, root=root, info=info, **kwargs)
|
fun = partial(one_from_module, module, root=root, info=info, **kwargs)
|
||||||
|
try:
|
||||||
|
return fun()
|
||||||
|
except (ImportError, LookupError):
|
||||||
|
install_deps(info)
|
||||||
|
return fun()
|
||||||
|
|
||||||
|
|
||||||
def parse_plugin_info(fpath):
|
def parse_plugin_info(fpath):
|
||||||
@ -606,17 +612,9 @@ def _instances_in_module(module):
|
|||||||
yield obj
|
yield obj
|
||||||
|
|
||||||
|
|
||||||
def _from_module_name(module, root, info=None, install=True, **kwargs):
|
def _from_module_name(module, root, info=None, **kwargs):
|
||||||
try:
|
|
||||||
module = load_module(module, root)
|
|
||||||
except (ImportError, LookupError):
|
|
||||||
if not install or not info:
|
|
||||||
raise
|
|
||||||
install_deps(info)
|
|
||||||
module = load_module(module, root)
|
module = load_module(module, root)
|
||||||
for plugin in _from_loaded_module(module=module, root=root, info=info, **kwargs):
|
for plugin in _from_loaded_module(module=module, root=root, info=info, **kwargs):
|
||||||
if install:
|
|
||||||
install_deps(plugin)
|
|
||||||
yield plugin
|
yield plugin
|
||||||
|
|
||||||
|
|
||||||
|
@ -139,7 +139,7 @@ class BlueprintsTest(TestCase):
|
|||||||
js = parse_resp(resp)
|
js = parse_resp(resp)
|
||||||
logging.debug(js)
|
logging.debug(js)
|
||||||
assert "@id" in js
|
assert "@id" in js
|
||||||
assert js["@id"] == "plugins/Dummy_0.1"
|
assert js["@id"] == "endpoint:plugins/Dummy_0.1"
|
||||||
|
|
||||||
def test_default(self):
|
def test_default(self):
|
||||||
""" Show only one plugin"""
|
""" Show only one plugin"""
|
||||||
@ -148,7 +148,7 @@ class BlueprintsTest(TestCase):
|
|||||||
js = parse_resp(resp)
|
js = parse_resp(resp)
|
||||||
logging.debug(js)
|
logging.debug(js)
|
||||||
assert "@id" in js
|
assert "@id" in js
|
||||||
assert js["@id"] == "plugins/Dummy_0.1"
|
assert js["@id"] == "endpoint:plugins/Dummy_0.1"
|
||||||
|
|
||||||
def test_context(self):
|
def test_context(self):
|
||||||
resp = self.client.get("/api/contexts/context.jsonld")
|
resp = self.client.get("/api/contexts/context.jsonld")
|
||||||
|
@ -121,8 +121,8 @@ class ExtensionsTest(TestCase):
|
|||||||
# Leaf (defaultdict with __setattr__ and __getattr__.
|
# Leaf (defaultdict with __setattr__ and __getattr__.
|
||||||
r1 = analyse(self.senpy, algorithm="Dummy", input="tupni", output="tuptuo")
|
r1 = analyse(self.senpy, algorithm="Dummy", input="tupni", output="tuptuo")
|
||||||
r2 = analyse(self.senpy, input="tupni", output="tuptuo")
|
r2 = analyse(self.senpy, input="tupni", output="tuptuo")
|
||||||
assert r1.analysis[0] == "plugins/Dummy_0.1"
|
assert r1.analysis[0] == "endpoint:plugins/Dummy_0.1"
|
||||||
assert r2.analysis[0] == "plugins/Dummy_0.1"
|
assert r2.analysis[0] == "endpoint:plugins/Dummy_0.1"
|
||||||
assert r1.entries[0]['nif:isString'] == 'input'
|
assert r1.entries[0]['nif:isString'] == 'input'
|
||||||
|
|
||||||
def test_analyse_empty(self):
|
def test_analyse_empty(self):
|
||||||
@ -156,8 +156,8 @@ class ExtensionsTest(TestCase):
|
|||||||
r2 = analyse(self.senpy,
|
r2 = analyse(self.senpy,
|
||||||
input="tupni",
|
input="tupni",
|
||||||
output="tuptuo")
|
output="tuptuo")
|
||||||
assert r1.analysis[0] == "plugins/Dummy_0.1"
|
assert r1.analysis[0] == "endpoint:plugins/Dummy_0.1"
|
||||||
assert r2.analysis[0] == "plugins/Dummy_0.1"
|
assert r2.analysis[0] == "endpoint:plugins/Dummy_0.1"
|
||||||
assert r1.entries[0]['nif:isString'] == 'input'
|
assert r1.entries[0]['nif:isString'] == 'input'
|
||||||
|
|
||||||
def test_analyse_error(self):
|
def test_analyse_error(self):
|
||||||
|
Loading…
Reference in New Issue
Block a user