balkian/senpy
1
0
Fork 0
mirror of https://github.com/gsi-upm/senpy synced 2024-11-21 15:52:28 +00:00
Code Issues Releases Wiki Activity

Improved request handling

Browse Source 
Also:
 * Shelve -> Pickle to avoid weird db problems
 * Serving schemas and contexts
This commit is contained in:
J. Fernando Sánchez 2016-02-21 19:36:24 +01:00
parent 407d17b2b9
commit b8339e397b
12 changed files with 191 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -2,5 +2,6 @@
.*
*egg-info
dist
build
README.html
__pycache__

116
senpy/blueprints.py
View File

@ -17,17 +17,18 @@
"""
Blueprints for Senpy
"""
from flask import Blueprint, request, current_app, render_template
from .models import Error, Response, Plugins
from flask import Blueprint, request, current_app, render_template, url_for, jsonify
from .models import Error, Response, Plugins, read_schema
from future.utils import iteritems
from functools import wraps
import json
import logging
logger = logging.getLogger(__name__)
nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__)
demo_blueprint = Blueprint("Demo of the service. It includes an HTML+Javascript playground to test senpy", __name__)
api_blueprint = Blueprint("api", __name__)
demo_blueprint = Blueprint("demo", __name__)
API_PARAMS = {
"algorithm": {
@ -47,7 +48,7 @@ API_PARAMS = {
},
}
BASIC_PARAMS = {
NIF_PARAMS = {
"algorithm": {
"aliases": ["algorithm", "a", "algo"],
"required": False,
@ -104,7 +105,7 @@ BASIC_PARAMS = {
},
}
def get_params(req, params=BASIC_PARAMS):
def update_params(req, params=NIF_PARAMS):
if req.method == 'POST':
indict = req.form
elif req.method == 'GET':
@ -136,64 +137,73 @@ def get_params(req, params=BASIC_PARAMS):
parameters=outdict,
errors={param: error for param, error in
iteritems(wrong_params)})
raise Error(message=message)
raise message
if hasattr(request, 'params'):
request.params.update(outdict)
else:
request.params = outdict
return outdict
def basic_analysis(params):
response = {"@context":
[("http://demos.gsi.dit.upm.es/"
"eurosentiment/static/context.jsonld"),
{
"@base": "{}#".format(request.url.encode('utf-8'))
}
],
"analysis": [{"@type": "marl:SentimentAnalysis"}],
"entries": []
}
if "language" in params:
response["language"] = params["language"]
for idx, sentence in enumerate(params["input"].split(".")):
response["entries"].append({
"@id": "Sentence{}".format(idx),
"nif:isString": sentence
})
return response
@demo_blueprint.route('/')
def index():
return render_template("index.html")
@api_blueprint.route('/contexts/<entity>.jsonld')
def context(entity="context"):
return jsonify({"@context": Response.context})
@nif_blueprint.route('/', methods=['POST', 'GET'])
def api():
@api_blueprint.route('/schemas/<schema>')
def schema(schema="definitions"):
try:
params = get_params(request)
algo = params.get("algorithm", None)
specific_params = current_app.senpy.parameters(algo)
logger.debug(
"Specific params: %s", json.dumps(specific_params, indent=4))
params.update(get_params(request, specific_params))
response = current_app.senpy.analyse(**params)
in_headers = params["inHeaders"] != "0"
prefix = params["prefix"]
return response.flask(in_headers=in_headers, prefix=prefix)
return jsonify(read_schema(schema))
except Exception: # Should be FileNotFoundError, but it's missing from py2
return Error(message="Schema not found", status=404).flask()
def basic_api(f):
@wraps(f)
def decorated_function(*args, **kwargs):
print('Getting request:')
print(request)
update_params(request, params=API_PARAMS)
print('Params: %s' % request.params)
try:
response = f(*args, **kwargs)
except Error as ex:
return ex.message.flask()
response = ex
in_headers = request.params["inHeaders"] != "0"
prefix = request.params["prefix"]
headers = {'X-ORIGINAL-PARAMS': request.params}
return response.flask(in_headers=in_headers,
prefix=prefix,
headers=headers,
context_uri=url_for('api.context', entity=type(response).__name__,
_external=True))
return decorated_function
@api_blueprint.route('/', methods=['POST', 'GET'])
@basic_api
def api():
algo = request.params.get("algorithm", None)
specific_params = current_app.senpy.parameters(algo)
update_params(request, params=NIF_PARAMS)
logger.debug("Specific params: %s", json.dumps(specific_params, indent=4))
update_params(request, specific_params)
response = current_app.senpy.analyse(**request.params)
return response
@nif_blueprint.route('/plugins/', methods=['POST', 'GET'])
@api_blueprint.route('/plugins/', methods=['POST', 'GET'])
@basic_api
def plugins():
in_headers = get_params(request, API_PARAMS)["inHeaders"] != "0"
sp = current_app.senpy
dic = Plugins(plugins=list(sp.plugins.values()))
return dic.flask(in_headers=in_headers)
return dic
@nif_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
@nif_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
@api_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
@api_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
@basic_api
def plugin(plugin=None, action="list"):
params = get_params(request, API_PARAMS)
filt = {}
sp = current_app.senpy
plugs = sp.filter_plugins(name=plugin)
@ -203,21 +213,19 @@ def plugin(plugin=None, action="list"):
elif plugin in sp.plugins:
response = sp.plugins[plugin]
else:
return Error(message="Plugin not found", status=404).flask()
return Error(message="Plugin not found", status=404)
if action == "list":
in_headers = params["inHeaders"] != "0"
prefix = params['prefix']
return response.flask(in_headers=in_headers, prefix=prefix)
return response
method = "{}_plugin".format(action)
if(hasattr(sp, method)):
getattr(sp, method)(plugin)
return Response(message="Ok").flask()
return Response(message="Ok")
else:
return Error(message="action '{}' not allowed".format(action)).flask()
return Error(message="action '{}' not allowed".format(action))
if __name__ == '__main__':
import config
app.register_blueprint(nif_blueprint)
app.register_blueprint(api_blueprint)
app.debug = config.DEBUG
app.run(host='0.0.0.0', port=5000)

36
senpy/extensions.py
View File

@ -8,7 +8,7 @@ monkey.patch_all()
from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
from .models import Error
from .blueprints import nif_blueprint, demo_blueprint
from .blueprints import api_blueprint, demo_blueprint
from git import Repo, InvalidGitRepositoryError
from functools import partial
@ -58,7 +58,7 @@ class Senpy(object):
app.teardown_appcontext(self.teardown)
else:
app.teardown_request(self.teardown)
app.register_blueprint(nif_blueprint, url_prefix="/api")
app.register_blueprint(api_blueprint, url_prefix="/api")
app.register_blueprint(demo_blueprint, url_prefix="/")
def add_folder(self, folder):
@ -77,29 +77,31 @@ class Senpy(object):
elif self.plugins:
algo = self.default_plugin and self.default_plugin.name
if not algo:
return Error(status=404,
raise Error(status=404,
message=("No plugins found."
" Please install one.").format(algo))
if algo in self.plugins:
if self.plugins[algo].is_activated:
plug = self.plugins[algo]
resp = plug.analyse(**params)
resp.analysis.append(plug)
logger.debug("Returning analysis result: {}".format(resp))
return resp
else:
logger.debug("Plugin not activated: {}".format(algo))
return Error(status=400,
message=("The algorithm '{}'"
" is not activated yet").format(algo))
else:
if algo not in self.plugins:
logger.debug(("The algorithm '{}' is not valid\n"
"Valid algorithms: {}").format(algo,
self.plugins.keys()))
return Error(status=404,
raise Error(status=404,
message="The algorithm '{}' is not valid"
.format(algo))
if not self.plugins[algo].is_activated:
logger.debug("Plugin not activated: {}".format(algo))
raise Error(status=400,
message=("The algorithm '{}'"
" is not activated yet").format(algo))
plug = self.plugins[algo]
try:
resp = plug.analyse(**params)
resp.analysis.append(plug)
logger.debug("Returning analysis result: {}".format(resp))
except Exception as ex:
resp = Error(message=str(ex), status=500)
return resp
@property
def default_plugin(self):
candidates = self.filter_plugins(is_activated=True)

46
senpy/models.py
View File

@ -63,28 +63,31 @@ class Context(dict):
base_context = Context.load(CONTEXT_PATH)
class SenpyMixin(object):
context = base_context
context = base_context["@context"]
def flask(self,
in_headers=False,
url="http://demos.gsi.dit.upm.es/senpy/senpy.jsonld",
prefix=None):
headers=None,
prefix=None,
**kwargs):
"""
Return the values and error to be used in flask.
So far, it returns a fixed context. We should store/generate different
contexts if the plugin adds more aliases.
"""
headers = None
headers = headers or {}
kwargs["with_context"] = True
js = self.jsonld(**kwargs)
if in_headers:
headers = {
url = js["@context"]
del js["@context"]
headers.update({
"Link": ('<%s>;'
'rel="http://www.w3.org/ns/json-ld#context";'
' type="application/ld+json"' % url)
}
return FlaskResponse(self.to_JSON(with_context=not in_headers,
prefix=prefix),
})
return FlaskResponse(json.dumps(js, indent=2, sort_keys=True),
status=getattr(self, "status", 200),
headers=headers,
mimetype="application/json")
@ -107,15 +110,27 @@ class SenpyMixin(object):
return ser_or_down(self._plain_dict())
def jsonld(self, context=None, prefix=None, with_context=False):
def jsonld(self, prefix=None, with_context=True, context_uri=None):
ser = self.serializable()
if with_context:
ser["@context"] = self.context.copy()
context = []
if context_uri:
context = context_uri
else:
context = self.context.copy()
if prefix:
ser["@context"]["@base"] = prefix
# This sets @base for the document, which will be used in
# all relative URIs will. For example, if a uri is "Example" and
# prefix =s "http://example.com", the absolute URI after expanding
# with JSON-LD will be "http://example.com/Example"
prefix_context = {"@base": prefix}
if isinstance(context, list):
context.append(prefix_context)
else:
context = [context, prefix_context]
ser["@context"] = context
return ser
@ -184,11 +199,6 @@ class SenpyModel(SenpyMixin, dict):
self.__delitem__(self._get_key(key))
@classmethod
def from_base(cls, name):
subschema = base_schema[name]
return warlock.model_factory(subschema, base_class=cls)
def _plain_dict(self):
d = { k: v for (k,v) in self.items() if k[0] != "_"}
d["@id"] = d.pop('id')

29
senpy/plugins.py
View File

@ -3,7 +3,7 @@ standard_library.install_aliases()
import inspect
import os.path
import shelve
import pickle
import logging
from .models import Response, PluginModel, Error
@ -18,10 +18,7 @@ class SenpyPlugin(PluginModel):
logger.debug("Initialising {}".format(info))
super(SenpyPlugin, self).__init__(info)
self.id = '{}_{}'.format(self.name, self.version)
self.params = info.get("extra_params", {})
self._info = info
if "@id" not in self.params:
self.params["@id"] = "params_%s" % self.id
self.is_activated = False
def get_folder(self):
@ -64,28 +61,34 @@ class ShelfMixin(object):
@property
def sh(self):
if not hasattr(self, '_sh') or self._sh is None:
self._sh = shelve.open(self.shelf_file, writeback=True)
self.__dict__['_sh'] = {}
if os.path.isfile(self.shelf_file):
self.__dict__['_sh'] = pickle.load(open(self.shelf_file, 'rb'))
return self._sh
@sh.deleter
def sh(self):
if os.path.isfile(self.shelf_file):
os.remove(self.shelf_file)
self.close()
del self.__dict__['_sh']
self.save()
def __del__(self):
self.close()
self.deactivate()
self.save()
super(ShelfMixin, self).__del__()
@property
def shelf_file(self):
if not hasattr(self, '_shelf_file') or not self._shelf_file:
if hasattr(self, '_info') and 'shelf_file' in self._info:
self._shelf_file = self._info['shelf_file']
self.__dict__['_shelf_file'] = self._info['shelf_file']
else:
self._shelf_file = os.path.join(self.get_folder(), self.name + '.db')
self._shelf_file = os.path.join(self.get_folder(), self.name + '.p')
return self._shelf_file
def close(self):
self.sh.close()
del(self._sh)
def save(self):
logger.debug('closing pickle')
if hasattr(self, '_sh') and self._sh is not None:
with open(self.shelf_file, 'wb') as f:
pickle.dump(self._sh, f)
del(self.__dict__['_sh'])

3
senpy/plugins/rand/rand.py
View File

@ -9,8 +9,7 @@ class Sentiment140Plugin(SentimentPlugin):
def analyse(self, **params):
lang = params.get("language", "auto")
p = params.get("prefix", None)
response = Results(prefix=p)
response = Results()
polarity_value = max(-1, min(1, random.gauss(0.2, 0.2)))
polarity = "marl:Neutral"
if polarity_value > 0:

2
senpy/schemas/context.jsonld
View File

@ -1,4 +1,5 @@
{
"@context": {
"@vocab": "http://www.gsi.dit.upm.es/ontologies/senpy#",
"dc": "http://dublincore.org/2012/06/14/dcelements#",
"me": "http://www.mixedemotions-project.eu/ns/model#",
@ -31,3 +32,4 @@
}
}
}

6
senpy/schemas/definitions.json
View File

@ -152,10 +152,14 @@
},
"Plugin": {
"type": "object",
"required": ["@id"],
"required": ["@id", "extra_params"],
"properties": {
"@id": {
"type": "string"
},
"extra_params": {
"type": "object",
"default": {}
}
}
},

16
tests/test_blueprints.py
View File

@ -46,9 +46,6 @@ class BlueprintsTest(TestCase):
self.assert200(resp)
logging.debug("Got response: %s", resp.json)
assert "@context" in resp.json
assert check_dict(
resp.json["@context"],
{"marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#"})
assert "entries" in resp.json
def test_list(self):
@ -111,3 +108,16 @@ class BlueprintsTest(TestCase):
sleep(0.5)
resp = self.client.get("/api/plugins/default/")
self.assert404(resp)
def test_context(self):
resp = self.client.get("/api/contexts/context.jsonld")
self.assert200(resp)
assert "@context" in resp.json
assert check_dict(
resp.json["@context"],
{"marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#"})
def test_schema(self):
resp = self.client.get("/api/schemas/definitions.json")
self.assert200(resp)
assert "$schema" in resp.json

12
tests/test_extensions.py
View File

@ -2,7 +2,9 @@ from __future__ import print_function
import os
import logging
from functools import partial
from senpy.extensions import Senpy
from senpy.models import Error
from flask import Flask
from flask.ext.testing import TestCase
@ -54,9 +56,7 @@ class ExtensionsTest(TestCase):
def test_noplugin(self):
""" Don't analyse if there isn't any plugin installed """
self.senpy.deactivate_all(sync=True)
resp = self.senpy.analyse(input="tupni")
logging.debug("Response: {}".format(resp))
assert resp.status == 404
self.assertRaises(Error, partial(self.senpy.analyse, input="tupni"))
def test_analyse(self):
""" Using a plugin """
@ -67,12 +67,6 @@ class ExtensionsTest(TestCase):
r2 = self.senpy.analyse(input="tupni", output="tuptuo")
assert r1.analysis[0].id[:5] == "Dummy"
assert r2.analysis[0].id[:5] == "Dummy"
for plug in self.senpy.plugins:
self.senpy.deactivate_plugin(plug, sync=True)
resp = self.senpy.analyse(input="tupni")
logging.debug("Response: {}".format(resp))
assert resp.status == 404
def test_filtering(self):
""" Filtering plugins """

2
tests/test_models.py
View File

@ -98,7 +98,7 @@ class ModelsTest(TestCase):
c = p.jsonld()
assert "info" not in c
assert "repo" not in c
assert "params" in c
assert "extra_params" in c
logging.debug("Framed:")
logging.debug(c)
p.validate()

9
tests/test_plugins.py
View File

@ -2,7 +2,7 @@
import os
import logging
import shelve
import pickle
import shutil
import tempfile
@ -16,7 +16,6 @@ from senpy.plugins import SenpyPlugin, ShelfMixin
class ShelfTest(ShelfMixin, SenpyPlugin):
def test(self, key=None, value=None):
assert isinstance(self.sh, shelve.Shelf)
assert key in self.sh
print('Checking: sh[{}] == {}'.format(key, value))
print('SH[{}]: {}'.format(key, self.sh[key]))
@ -49,7 +48,9 @@ class ModelsTest(TestCase):
a.sh['a'] = 'fromA'
a.test(key='a', value='fromA')
sh = shelve.open(self.shelf_file)
a.save()
sh = pickle.load(open(self.shelf_file, 'rb'))
assert sh['a'] == 'fromA'
@ -61,7 +62,7 @@ class ModelsTest(TestCase):
'shelf_file': self.shelf_file})
print('Shelf file: %s' % a.shelf_file)
a.sh['a'] = 'fromA'
a.close()
a.save()
b = ShelfTest(info={'name': 'shelve',
'version': 'test',