balkian/senpy
1
0
Fork 0
mirror of https://github.com/gsi-upm/senpy synced 2024-12-21 20:48:14 +00:00
Code Issues Releases Wiki Activity

Python 3 compatible

Browse Source 
There are also some slight changes to the JSON schemas and the use of
JSON-LD.
This commit is contained in:
J. Fernando Sánchez 2016-02-19 19:24:09 +01:00
parent a79df7a3da
commit 14c9f61864
32 changed files with 621 additions and 349 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -3,3 +3,4 @@
*egg-info
dist
README.html
__pycache__

6
requirements.txt
View File

@ -2,6 +2,10 @@ Flask>=0.10.1
gunicorn>=19.0.0
requests>=2.4.1
GitPython>=0.3.2.RC1
gevent>=1.0.1
gevent>=1.1rc4
PyLD>=0.6.5
Flask-Testing>=0.4.2
six
future
jsonschema
jsonref

45
senpy/blueprints.py
View File

@ -17,8 +17,8 @@
"""
Blueprints for Senpy
"""
from flask import Blueprint, request, current_app, Flask, redirect, url_for, render_template
from .models import Error, Response, Leaf
from flask import Blueprint, request, current_app, render_template
from .models import Error, Response
from future.utils import iteritems
import json
@ -27,6 +27,7 @@ import logging
logger = logging.getLogger(__name__)
nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__)
demo_blueprint = Blueprint("Demo of the service. It includes an HTML+Javascript playground to test senpy", __name__)
BASIC_PARAMS = {
"algorithm": {
@ -40,15 +41,6 @@ BASIC_PARAMS = {
}
}
LIST_PARAMS = {
"params": {
"aliases": ["params", "with_params"],
"required": False,
"default": "0"
},
}
def get_params(req, params=BASIC_PARAMS):
if req.method == 'POST':
indict = req.form
@ -76,12 +68,11 @@ def get_params(req, params=BASIC_PARAMS):
outdict[param] not in params[param]["options"]:
wrong_params[param] = params[param]
if wrong_params:
message = Error({"status": 404,
"message": "Missing or invalid parameters",
"parameters": outdict,
"errors": {param: error for param, error in
iteritems(wrong_params)}
})
message = Error(status=404,
message="Missing or invalid parameters",
parameters=outdict,
errors={param: error for param, error in
iteritems(wrong_params)})
raise Error(message=message)
return outdict
@ -107,12 +98,12 @@ def basic_analysis(params):
return response
@nif_blueprint.route('/')
@demo_blueprint.route('/')
def index():
return render_template("index.html")
@nif_blueprint.route('/api', methods=['POST', 'GET'])
@nif_blueprint.route('/', methods=['POST', 'GET'])
def api():
try:
params = get_params(request)
@ -128,7 +119,7 @@ def api():
return ex.message.flask()
@nif_blueprint.route("/api/default")
@nif_blueprint.route("/default")
def default():
# return current_app.senpy.default_plugin
plug = current_app.senpy.default_plugin
@ -139,9 +130,9 @@ def default():
return error.flask()
@nif_blueprint.route('/api/plugins/', methods=['POST', 'GET'])
@nif_blueprint.route('/api/plugins/<plugin>', methods=['POST', 'GET'])
@nif_blueprint.route('/api/plugins/<plugin>/<action>', methods=['POST', 'GET'])
@nif_blueprint.route('/plugins/', methods=['POST', 'GET'])
@nif_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
@nif_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
def plugins(plugin=None, action="list"):
filt = {}
sp = current_app.senpy
@ -151,21 +142,19 @@ def plugins(plugin=None, action="list"):
if plugin and not plugs:
return "Plugin not found", 400
if action == "list":
with_params = get_params(request, LIST_PARAMS)["params"] == "1"
in_headers = get_params(request, BASIC_PARAMS)["inHeaders"] != "0"
if plugin:
dic = plugs[plugin]
else:
dic = Response(
{plug: plugs[plug].jsonld(with_params) for plug in plugs},
frame={})
{plug: plugs[plug].serializable() for plug in plugs})
return dic.flask(in_headers=in_headers)
method = "{}_plugin".format(action)
if(hasattr(sp, method)):
getattr(sp, method)(plugin)
return Leaf(message="Ok").flask()
return Response(message="Ok").flask()
else:
return Error("action '{}' not allowed".format(action)).flask()
return Error(message="action '{}' not allowed".format(action)).flask()
if __name__ == '__main__':

42
senpy/context.jsonld
View File

@ -1,42 +0,0 @@
{
"dc": "http://purl.org/dc/terms/",
"dc:subject": {
"@type": "@id"
},
"xsd": "http://www.w3.org/2001/XMLSchema#",
"marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#",
"nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#",
"onyx": "http://www.gsi.dit.upm.es/ontologies/onyx/ns#",
"emotions": {
"@container": "@set",
"@id": "onyx:hasEmotionSet"
},
"opinions": {
"@container": "@set",
"@id": "marl:hasOpinion"
},
"prov": "http://www.w3.org/ns/prov#",
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
"analysis": {
"@container": "@set",
"@id": "prov:wasInformedBy"
},
"entries": {
"@container": "@set",
"@id": "prov:generated"
},
"strings": {
"@container": "@set",
"@reverse": "nif:hasContext"
},
"date":
{
"@id": "dc:date",
"@type": "xsd:dateTime"
},
"text": { "@id": "nif:isString" },
"wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#",
"xsd": "http://www.w3.org/2001/XMLSchema#",
"senpy": "http://www.gsi.dit.upm.es/ontologies/senpy/ns#",
"@vocab": "http://www.gsi.dit.upm.es/ontologies/senpy/ns#"
}

5
senpy/extensions.py
View File

@ -8,7 +8,7 @@ monkey.patch_all()
from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
from .models import Error
from .blueprints import nif_blueprint
from .blueprints import nif_blueprint, demo_blueprint
from git import Repo, InvalidGitRepositoryError
from functools import partial
@ -57,7 +57,8 @@ class Senpy(object):
app.teardown_appcontext(self.teardown)
else:
app.teardown_request(self.teardown)
app.register_blueprint(nif_blueprint)
app.register_blueprint(nif_blueprint, url_prefix="/api")
app.register_blueprint(demo_blueprint, url_prefix="/")
def add_folder(self, folder):
logger.debug("Adding folder: %s", folder)

264
senpy/models.py
View File

@ -1,71 +1,71 @@
'''
Senpy Models.
This implementation should mirror the JSON schema definition.
For compatibility with Py3 and for easier debugging, this new version drops introspection
and adds all arguments to the models.
'''
from __future__ import print_function
from six import string_types
import time
import copy
import json
import os
import logging
import jsonref
import jsonschema
from collections import defaultdict
from pyld import jsonld
from flask import Response as FlaskResponse
class Response(object):
DEFINITIONS_FILE = 'definitions.json'
CONTEXT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'schemas', 'context.jsonld')
@property
def context(self):
if not hasattr(self, '_context'):
self._context = None
return self._context
def get_schema_path(schema_file, absolute=False):
if absolute:
return os.path.realpath(schema_file)
else:
return os.path.join(os.path.dirname(os.path.realpath(__file__)), 'schemas', schema_file)
def read_schema(schema_file, absolute=False):
schema_path = get_schema_path(schema_file, absolute)
schema_uri = 'file://{}'.format(schema_path)
return jsonref.load(open(schema_path), base_uri=schema_uri)
base_schema = read_schema(DEFINITIONS_FILE)
logging.debug(base_schema)
class Context(dict):
@staticmethod
def get_context(context):
if isinstance(context, list):
def load(context):
logging.debug('Loading context: {}'.format(context))
if not context:
return context
elif isinstance(context, list):
contexts = []
for c in context:
contexts.append(Response.get_context(c))
contexts.append(Context.load(c))
return contexts
elif isinstance(context, dict):
return context
return Context(context)
elif isinstance(context, string_types):
try:
with open(context) as f:
return json.loads(f.read())
return Context(json.loads(f.read()))
except IOError:
return context
else:
raise AttributeError('Please, provide a valid context')
def jsonld(self, frame=None, options=None,
context=None, removeContext=None):
if removeContext is None:
removeContext = Response._context # Loop?
if frame is None:
frame = self._frame
if context is None:
context = self.context
else:
context = self.get_context(context)
# For some reason, this causes errors with pyld
# if options is None:
# options = {"expandContext": context.copy() }
js = self
if frame:
logging.debug("Framing: %s", json.dumps(self, indent=4))
logging.debug("Framing with %s", json.dumps(frame, indent=4))
js = jsonld.frame(js, frame, options)
logging.debug("Result: %s", json.dumps(js, indent=4))
logging.debug("Compacting with %s", json.dumps(context, indent=4))
js = jsonld.compact(js, context, options)
logging.debug("Result: %s", json.dumps(js, indent=4))
if removeContext == context:
del js["@context"]
return js
base_context = Context.load(CONTEXT_PATH)
def to_JSON(self, removeContext=None):
return json.dumps(self.jsonld(removeContext=removeContext),
default=lambda o: o.__dict__,
sort_keys=True, indent=4)
class SenpyMixin(object):
context = base_context
def flask(self,
in_headers=False,
@ -73,46 +73,166 @@ class Response(object):
"""
Return the values and error to be used in flask
"""
js = self.jsonld()
headers = None
if in_headers:
ctx = js["@context"]
headers = {
"Link": ('<%s>;'
'rel="http://www.w3.org/ns/json-ld#context";'
' type="application/ld+json"' % url)
}
del js["@context"]
return FlaskResponse(json.dumps(js, indent=4),
status=self.get("status", 200),
return FlaskResponse(self.to_JSON(with_context=not in_headers),
status=getattr(self, "status", 200),
headers=headers,
mimetype="application/json")
def serializable(self):
def ser_or_down(item):
if hasattr(item, 'serializable'):
return item.serializable()
elif isinstance(item, dict):
temp = dict()
for kp in item:
vp = item[kp]
temp[kp] = ser_or_down(vp)
return temp
elif isinstance(item, list):
return list(ser_or_down(i) for i in item)
else:
return item
return ser_or_down(self._plain_dict())
def jsonld(self, context=None, with_context=False):
ser = self.serializable()
if with_context:
ser["@context"] = self.context
return ser
def to_JSON(self, *args, **kwargs):
js = json.dumps(self.jsonld(*args, **kwargs), indent=4,
sort_keys=True)
return js
class Entry(JSONLD):
pass
class SenpyModel(SenpyMixin, dict):
class Sentiment(JSONLD):
pass
class EmotionSet(JSONLD):
pass
class Emotion(JSONLD):
pass
class Suggestion(JSONLD):
pass
class Error(BaseException, JSONLD):
# A better pattern would be this:
# htp://flask.pocoo.org/docs/0.10/patterns/apierrors/
_frame = {}
_context = {}
schema = base_schema
prefix = None
def __init__(self, *args, **kwargs):
self.message = kwargs.get('message', None)
super(Error, self).__init__(*args)
temp = dict(*args, **kwargs)
reqs = self.schema.get('required', [])
for i in reqs:
if i not in temp:
prop = self.schema['properties'][i]
if 'default' in prop:
temp[i] = copy.deepcopy(prop['default'])
if 'context' in temp:
context = temp['context']
del temp['context']
self.__dict__['context'] = Context.load(context)
super(SenpyModel, self).__init__(temp)
def _get_key(self, key):
key = key.replace("__", ":", 1)
return key
def __setitem__(self, key, value):
dict.__setitem__(self, key, value)
def __delitem__(self, key):
dict.__delitem__(self, key)
def __getattr__(self, key):
try:
return self.__getitem__(self._get_key(key))
except KeyError:
raise AttributeError(key)
def __setattr__(self, key, value):
self.__setitem__(self._get_key(key), value)
def __delattr__(self, key):
self.__delitem__(self._get_key(key))
def validate(self, obj=None):
if not obj:
obj = self
if hasattr(obj, "jsonld"):
obj = obj.jsonld()
jsonschema.validate(obj, self.schema)
@classmethod
def from_base(cls, name):
subschema = base_schema[name]
return warlock.model_factory(subschema, base_class=cls)
def _plain_dict(self):
d = { k: v for (k,v) in self.items() if k[0] != "_"}
if hasattr(self, "id"):
d["@id"] = self.id
return d
@property
def id(self):
if not hasattr(self, '_id'):
self.__dict__["_id"] = '_:{}_{}'.format(type(self).__name__, time.time())
return self._id
@id.setter
def id(self, value):
self._id = value
class Response(SenpyModel):
schema = read_schema('response.json')
class Results(SenpyModel):
schema = read_schema('results.json')
def jsonld(self, context=None, with_context=True):
return super(Results, self).jsonld(context, with_context)
class Entry(SenpyModel):
schema = read_schema('entry.json')
class Sentiment(SenpyModel):
schema = read_schema('sentiment.json')
class Analysis(SenpyModel):
schema = read_schema('analysis.json')
class EmotionSet(SenpyModel):
schema = read_schema('emotionSet.json')
class Suggestion(SenpyModel):
schema = read_schema('suggestion.json')
class PluginModel(SenpyModel):
schema = read_schema('plugin.json')
class Plugins(SenpyModel):
schema = read_schema('plugins.json')
class Error(SenpyMixin, BaseException ):
def __init__(self, message, status=500, params=None, errors=None, *args, **kwargs):
self.message = message
self.status = status
self.params = params or {}
self.errors = errors or ""
def _plain_dict(self):
return self.__dict__
def __str__(self):
return str(self.jsonld())

35
senpy/plugins.py
View File

@ -5,7 +5,7 @@ import inspect
import os.path
import shelve
import logging
from .models import Response, Leaf
from .models import Response, PluginModel, Error
logger = logging.getLogger(__name__)
@ -58,36 +58,21 @@ PARAMS = {
}
class SenpyPlugin(Leaf):
_context = Leaf.get_context(Response._context)
_frame = {"@context": _context,
"name": {},
"extra_params": {"@container": "@index"},
"@explicit": True,
"version": {},
"repo": None,
"is_activated": {},
"params": None,
}
class SenpyPlugin(PluginModel):
def __init__(self, info=None):
if not info:
raise Error(message=("You need to provide configuration"
"information for the plugin."))
logger.debug("Initialising {}".format(info))
super(SenpyPlugin, self).__init__()
self.name = info["name"]
self.version = info["version"]
self.id = "{}_{}".format(self.name, self.version)
self.params = info.get("params", PARAMS.copy())
if "@id" not in self.params:
self.params["@id"] = "params_%s" % self.id
self.extra_params = info.get("extra_params", {})
self.params.update(self.extra_params.copy())
if "@id" not in self.extra_params:
self.extra_params["@id"] = "extra_params_%s" % self.id
self.is_activated = False
self._info = info
super(SenpyPlugin, self).__init__()
def get_folder(self):
return os.path.dirname(inspect.getfile(self.__class__))
@ -102,13 +87,6 @@ class SenpyPlugin(Leaf):
def deactivate(self):
pass
def jsonld(self, parameters=False, *args, **kwargs):
nframe = kwargs.pop("frame", self._frame)
if parameters:
nframe = nframe.copy()
nframe["params"] = {}
return super(SenpyPlugin, self).jsonld(frame=nframe, *args, **kwargs)
@property
def id(self):
return "{}_{}".format(self.name, self.version)
@ -123,6 +101,7 @@ class SentimentPlugin(SenpyPlugin):
super(SentimentPlugin, self).__init__(info, *args, **kwargs)
self.minPolarityValue = float(info.get("minPolarityValue", 0))
self.maxPolarityValue = float(info.get("maxPolarityValue", 1))
self["@type"] = "marl:SentimentAnalysis"
class EmotionPlugin(SenpyPlugin):
@ -131,6 +110,7 @@ class EmotionPlugin(SenpyPlugin):
resp = super(EmotionPlugin, self).__init__(info, *args, **kwargs)
self.minEmotionValue = float(info.get("minEmotionValue", 0))
self.maxEmotionValue = float(info.get("maxEmotionValue", 0))
self["@type"] = "onyx:EmotionAnalysis"
class ShelfMixin(object):
@ -145,6 +125,11 @@ class ShelfMixin(object):
def sh(self):
if os.path.isfile(self.shelf_file):
os.remove(self.shelf_file)
self.close()
def __del__(self):
self.close()
self.deactivate()
@property
def shelf_file(self):

33
senpy/plugins/rand/rand.py
View File

@ -2,7 +2,7 @@ import json
import random
from senpy.plugins import SentimentPlugin
from senpy.models import Response, Opinion, Entry
from senpy.models import Results, Sentiment, Entry
class Sentiment140Plugin(SentimentPlugin):
@ -10,22 +10,33 @@ class Sentiment140Plugin(SentimentPlugin):
lang = params.get("language", "auto")
p = params.get("prefix", None)
response = Response(prefix=p)
response = Results(prefix=p)
polarity_value = max(-1, min(1, random.gauss(0.2, 0.2)))
polarity = "marl:Neutral"
if polarity_value > 0:
polarity = "marl:Positive"
elif polarity_value < 0:
polarity = "marl:Negative"
entry = Entry(id="Entry0",
text=params["input"],
prefix=p)
opinion = Opinion(id="Opinion0",
prefix=p,
hasPolarity=polarity,
polarityValue=polarity_value)
opinion["prov:wasGeneratedBy"] = self.id
entry.opinions.append(opinion)
entry = Entry({"id":":Entry0",
"nif:isString": params["input"]})
sentiment = Sentiment({"id": ":Sentiment0",
"marl:hasPolarity": polarity,
"marl:polarityValue": polarity_value})
sentiment["prov:wasGeneratedBy"] = self.id
entry.sentiments = []
entry.sentiments.append(sentiment)
entry.language = lang
response.entries.append(entry)
return response

21
senpy/plugins/sentiment140/sentiment140.py
View File

@ -2,7 +2,7 @@ import requests
import json
from senpy.plugins import SentimentPlugin
from senpy.models import Response, Opinion, Entry
from senpy.models import Results, Sentiment, Entry
class Sentiment140Plugin(SentimentPlugin):
@ -16,7 +16,7 @@ class Sentiment140Plugin(SentimentPlugin):
)
p = params.get("prefix", None)
response = Response(prefix=p)
response = Results(prefix=p)
polarity_value = self.maxPolarityValue*int(res.json()["data"][0]
["polarity"]) * 0.25
polarity = "marl:Neutral"
@ -25,15 +25,16 @@ class Sentiment140Plugin(SentimentPlugin):
polarity = "marl:Positive"
elif polarity_value < neutral_value:
polarity = "marl:Negative"
entry = Entry(id="Entry0",
text=params["input"],
prefix=p)
opinion = Opinion(id="Opinion0",
prefix=p,
hasPolarity=polarity,
polarityValue=polarity_value)
opinion["prov:wasGeneratedBy"] = self.id
entry.opinions.append(opinion)
nif__isString=params["input"])
sentiment = Sentiment(id="Sentiment0",
prefix=p,
marl__hasPolarity=polarity,
marl__polarityValue=polarity_value)
sentiment.prov__wasGeneratedBy = self.id
entry.sentiments = []
entry.sentiments.append(sentiment)
entry.language = lang
response.entries.append(entry)
return response

4
senpy/schemas/analysis.json Normal file
View File

@ -0,0 +1,4 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "definitions.json#/Analysis"
}

33
senpy/schemas/context.jsonld Normal file
View File

@ -0,0 +1,33 @@
{
"@vocab": "http://www.gsi.dit.upm.es/ontologies/senpy#",
"dc": "http://dublincore.org/2012/06/14/dcelements#",
"me": "http://www.mixedemotions-project.eu/ns/model#",
"prov": "http://www.w3.org/ns/prov#",
"nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#",
"marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#",
"onyx": "http://www.gsi.dit.upm.es/ontologies/onyx#",
"wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#",
"xsd": "http://www.w3.org/2001/XMLSchema#",
"topics": {
"@id": "dc:subject"
},
"entities": {
"@id": "me:hasEntities"
},
"suggestions": {
"@id": "me:hasSuggestions"
},
"emotions": {
"@id": "onyx:hasEmotionSet"
},
"sentiments": {
"@id": "marl:hasOpinion"
},
"entries": {
"@id": "prov:used"
},
"analysis": {
"@id": "prov:wasGeneratedBy"
}
}

161
senpy/schemas/definitions.json Normal file
View File

@ -0,0 +1,161 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"Results": {
"title": "Results",
"description": "The results of an analysis",
"type": "object",
"properties": {
"@context": {
"$ref": "#/Context"
},
"@id": {
"description": "ID of the analysis",
"type": "string"
},
"analysis": {
"type": "array",
"default": [],
"items": {
"$ref": "#/Analysis"
}
},
"entries": {
"type": "array",
"default": [],
"items": {
"$ref": "#/Entry"
}
}
},
"required": ["@id", "analysis", "entries"]
},
"Context": {
"description": "JSON-LD Context",
"type": ["array", "string", "object"]
},
"Analysis": {
"description": "Senpy analysis",
"type": "object",
"properties": {
"@id": {
"type": "string"
},
"@type": {
"type": "string",
"description": "Type of the analysis. e.g. marl:SentimentAnalysis"
}
},
"required": ["@id", "@type"]
},
"Entry": {
"properties": {
"@id": {
"type": "string"
},
"@type": {
"enum": [["nif:RFC5147String", "nif:Context"]]
},
"nif:isString": {
"description": "String contained in this Context",
"type": "string"
},
"sentiments": {
"type": "array",
"items": {"$ref": "#/Sentiment" }
},
"emotions": {
"type": "array",
"items": {"$ref": "#/EmotionSet" }
},
"entities": {
"type": "array",
"items": {"$ref": "#/Entity" }
},
"topics": {
"type": "array",
"items": {"$ref": "#/Topic" }
},
"suggestions": {
"type": "array",
"items": {"$ref": "#/Suggestion" }
}
},
"required": ["@id", "nif:isString"]
},
"Sentiment": {
"properties": {
"@id": {"type": "string"},
"nif:beginIndex": {"type": "integer"},
"nif:endIndex": {"type": "integer"},
"nif:anchorOf": {
"description": "Piece of context that contains the Sentiment",
"type": "string"
},
"marl:hasPolarity": {
"enum": ["marl:Positive", "marl:Negative", "marl:Neutral"]
},
"marl:polarityValue": {
"type": "number"
},
"prov:wasGeneratedBy": {
"type": "string",
"description": "The ID of the analysis that generated this Sentiment. The full object should be included in the \"analysis\" property of the root object"
}
},
"required": ["@id", "prov:wasGeneratedBy"]
},
"EmotionSet": {
"properties": {
"@id": {"type": "string"},
"nif:beginIndex": {"type": "integer"},
"nif:endIndex": {"type": "integer"},
"nif:anchorOf": {
"description": "Piece of context that contains the Sentiment",
"type": "string"
},
"onyx:hasEmotion": {
"$ref": "#/Emotion"
},
"prov:wasGeneratedBy": {
"type": "string",
"description": "The ID of the analysis that generated this Emotion. The full object should be included in the \"analysis\" property of the root object"
}
},
"required": ["@id", "prov:wasGeneratedBy", "onyx:hasEmotion"]
},
"Emotion": {
"type": "object"
},
"Entity": {
"type": "object"
},
"Topic": {
"type": "object"
},
"Suggestion": {
"type": "object"
},
"Plugins": {
"properties": {
"plugins": {
"type": "array",
"items": {
"$ref": "#/Plugin"
}
}
}
},
"Plugin": {
"type": "object",
"required": ["@id"],
"properties": {
"@id": {
"type": "string"
}
}
},
"Response": {
"type": "object"
}
}

4
senpy/schemas/emotion.json Normal file
View File

@ -0,0 +1,4 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "definitions.json#/Emotion"
}

4
senpy/schemas/emotionSet.json Normal file
View File

@ -0,0 +1,4 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "definitions.json#/EmotionSet"
}

4
senpy/schemas/entry.json Normal file
View File

@ -0,0 +1,4 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "definitions.json#/Entry"
}

3
senpy/schemas/plugin.json Normal file
View File

@ -0,0 +1,3 @@
{
"$ref": "definitions.json#/Plugin"
}

3
senpy/schemas/plugins.json Normal file
View File

@ -0,0 +1,3 @@
{
"$ref": "definitions.json#/Plugins"
}

4
senpy/schemas/response.json Normal file
View File

@ -0,0 +1,4 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "definitions.json#/Response"
}

4
senpy/schemas/results.json Normal file
View File

@ -0,0 +1,4 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "definitions.json#/Results"
}

4
senpy/schemas/sentiment.json Normal file
View File

@ -0,0 +1,4 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "definitions.json#/Sentiment"
}

4
senpy/schemas/suggestion.json Normal file
View File

@ -0,0 +1,4 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "definitions.json#/Suggestion"
}

2
setup.cfg
View File

@ -1,2 +1,4 @@
[metadata]
description-file = README.rst
[aliases]
test=pytest

4
setup.py
View File

@ -15,7 +15,7 @@ except AttributeError:
install_reqs = [str(ir.req) for ir in install_reqs]
test_reqs = [str(ir.req) for ir in test_reqs]
VERSION = "0.4.11"
VERSION = "0.5"
setup(
name='senpy',
@ -34,7 +34,7 @@ extendable, so new algorithms and sources can be used.
classifiers=[],
install_requires=install_reqs,
tests_require=test_reqs,
test_suite="nose.collector",
setup_requires=['pytest-runner',],
include_package_data=True,
entry_points={
'console_scripts': [

3
test-requirements.txt
View File

@ -1,3 +1,2 @@
nose
pytest
mock
pbr

40
tests/context.jsonld
View File

@ -1,40 +0,0 @@
{
"dc": "http://purl.org/dc/terms/",
"dc:subject": {
"@type": "@id"
},
"xsd": "http://www.w3.org/2001/XMLSchema#",
"marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#",
"nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#",
"onyx": "http://www.gsi.dit.upm.es/ontologies/onyx/ns#",
"emotions": {
"@container": "@set",
"@id": "onyx:hasEmotionSet"
},
"opinions": {
"@container": "@set",
"@id": "marl:hasOpinion"
},
"prov": "http://www.w3.org/ns/prov#",
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
"analysis": {
"@container": "@set",
"@id": "prov:wasInformedBy"
},
"entries": {
"@container": "@set",
"@id": "prov:generated"
},
"strings": {
"@container": "@set",
"@reverse": "nif:hasContext"
},
"date":
{
"@id": "dc:date",
"@type": "xsd:dateTime"
},
"text": { "@id": "nif:isString" },
"wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#",
"xsd": "http://www.w3.org/2001/XMLSchema#"
}

4
tests/dummy_plugin/dummy.py
View File

@ -1,8 +1,8 @@
from senpy.plugins import SentimentPlugin
from senpy.models import Response
from senpy.models import Results
class DummyPlugin(SentimentPlugin):
def analyse(self, *args, **kwargs):
return Response()
return Results()

81
tests/models_test/__init__.py
View File

@ -1,81 +0,0 @@
import os
import logging
try:
import unittest.mock as mock
except ImportError:
import mock
import json
import os
from unittest import TestCase
from senpy.models import Response, Entry
from senpy.plugins import SenpyPlugin
class ModelsTest(TestCase):
def test_response(self):
r = Response(context=os.path.normpath(
os.path.join(__file__, "..", "..", "context.jsonld")))
assert("@context" in r)
assert(r._frame)
logging.debug("Default frame: %s", r._frame)
assert("marl" in r.context)
assert("entries" in r.context)
r2 = Response(context=json.loads('{"test": "roger"}'))
assert("test" in r2.context)
r3 = Response(context=None)
del r3.context
assert("@context" not in r3)
assert("entries" in r3)
assert("analysis" in r3)
r4 = Response()
assert("@context" in r4)
assert("entries" in r4)
assert("analysis" in r4)
dummy = SenpyPlugin({"name": "dummy", "version": 0})
r5 = Response({"dummy": dummy}, context=None, frame=None)
logging.debug("Response 5: %s", r5)
assert("dummy" in r5)
assert(r5["dummy"].name == "dummy")
js = r5.jsonld(context={}, frame={})
logging.debug("jsonld 5: %s", js)
assert("dummy" in js)
assert(js["dummy"].name == "dummy")
r6 = Response()
r6.entries.append(Entry(text="Just testing"))
logging.debug("Reponse 6: %s", r6)
assert("@context" in r6)
assert("marl" in r6.context)
assert("entries" in r6.context)
js = r6.jsonld()
logging.debug("jsonld: %s", js)
assert("entries" in js)
assert("entries" in js)
assert("analysis" in js)
resp = r6.flask()
received = json.loads(resp.data.decode())
logging.debug("Response: %s", js)
assert(received["entries"])
assert(received["entries"][0]["text"] == "Just testing")
assert(received["entries"][0]["text"] != "Not testing")
def test_opinions(self):
pass
def test_plugins(self):
p = SenpyPlugin({"name": "dummy", "version": 0})
c = p.jsonld()
assert "info" not in c
assert "repo" not in c
assert "params" not in c
logging.debug("Framed: %s", c)
assert "extra_params" in c
def test_frame_response(self):
pass

4
tests/sleep_plugin/sleep.py
View File

@ -1,5 +1,5 @@
from senpy.plugins import SenpyPlugin
from senpy.models import Response
from senpy.models import Results
from time import sleep
@ -14,4 +14,4 @@ class SleepPlugin(SenpyPlugin):
def analyse(self, *args, **kwargs):
sleep(float(kwargs.get("timeout", self.timeout)))
return Response()
return Results()

16
tests/blueprints_test/__init__.py → tests/test_blueprints.py
View File

@ -1,10 +1,6 @@
import os
import logging
try:
import unittest.mock as mock
except ImportError:
import mock
from senpy.extensions import Senpy
from flask import Flask
from flask.ext.testing import TestCase
@ -31,7 +27,7 @@ class BlueprintsTest(TestCase):
"""
Calling with no arguments should ask the user for more arguments
"""
resp = self.client.get("/api")
resp = self.client.get("/api/")
self.assert404(resp)
logging.debug(resp.json)
assert resp.json["status"] == 404
@ -46,7 +42,7 @@ class BlueprintsTest(TestCase):
The dummy plugin returns an empty response,\
it should contain the context
"""
resp = self.client.get("/api?i=My aloha mohame")
resp = self.client.get("/api/?i=My aloha mohame")
self.assert200(resp)
logging.debug("Got response: %s", resp.json)
assert "@context" in resp.json
@ -64,7 +60,7 @@ class BlueprintsTest(TestCase):
assert "@context" in resp.json
def test_headers(self):
for i, j in product(["/api/plugins/?nothing=", "/api?i=test&"],
for i, j in product(["/api/plugins/?nothing=", "/api/?i=test&"],
["headers", "inHeaders"]):
resp = self.client.get("%s" % (i))
assert "@context" in resp.json
@ -77,7 +73,7 @@ class BlueprintsTest(TestCase):
def test_detail(self):
""" Show only one plugin"""
resp = self.client.get("/api/plugins/Dummy")
resp = self.client.get("/api/plugins/Dummy/")
self.assert200(resp)
logging.debug(resp.json)
assert "@id" in resp.json
@ -88,14 +84,14 @@ class BlueprintsTest(TestCase):
resp = self.client.get("/api/plugins/Dummy/deactivate")
self.assert200(resp)
sleep(0.5)
resp = self.client.get("/api/plugins/Dummy")
resp = self.client.get("/api/plugins/Dummy/")
self.assert200(resp)
assert "is_activated" in resp.json
assert resp.json["is_activated"] == False
resp = self.client.get("/api/plugins/Dummy/activate")
self.assert200(resp)
sleep(0.5)
resp = self.client.get("/api/plugins/Dummy")
resp = self.client.get("/api/plugins/Dummy/")
self.assert200(resp)
assert "is_activated" in resp.json
assert resp.json["is_activated"] == True

10
tests/extensions_test/__init__.py → tests/test_extensions.py
View File

@ -2,10 +2,6 @@ from __future__ import print_function
import os
import logging
try:
import unittest.mock as mock
except ImportError:
import mock
from senpy.extensions import Senpy
from flask import Flask
from flask.ext.testing import TestCase
@ -15,7 +11,7 @@ class ExtensionsTest(TestCase):
def create_app(self):
self.app = Flask("test_extensions")
self.dir = os.path.join(os.path.dirname(__file__), "..")
self.dir = os.path.join(os.path.dirname(__file__))
self.senpy = Senpy(plugin_folder=self.dir, default_plugins=False)
self.senpy.init_app(self.app)
self.senpy.activate_plugin("Dummy", sync=True)
@ -60,7 +56,7 @@ class ExtensionsTest(TestCase):
self.senpy.deactivate_all(sync=True)
resp = self.senpy.analyse(input="tupni")
logging.debug("Response: {}".format(resp))
assert resp["status"] == 404
assert resp.status == 404
def test_analyse(self):
""" Using a plugin """
@ -75,7 +71,7 @@ class ExtensionsTest(TestCase):
self.senpy.deactivate_plugin(plug, sync=True)
resp = self.senpy.analyse(input="tupni")
logging.debug("Response: {}".format(resp))
assert resp["status"] == 404
assert resp.status == 404
def test_filtering(self):

97
tests/test_models.py Normal file
View File

@ -0,0 +1,97 @@
import os
import logging
import jsonschema
import json
import os
from unittest import TestCase
from senpy.models import Response, Entry, Results, Sentiment, EmotionSet, Error
from senpy.plugins import SenpyPlugin
from pprint import pprint
class ModelsTest(TestCase):
def test_jsonld(self):
ctx = os.path.normpath(os.path.join(__file__, "..", "..", "..", "senpy", "schemas", "context.jsonld"))
prueba = {"@id": "test",
"analysis": [],
"entries": []}
r = Results(**prueba)
print("Response's context: ")
pprint(r.context)
j = r.jsonld(with_context=True)
print("As JSON:")
pprint(j)
assert("@context" in j)
assert("marl" in j["@context"])
assert("entries" in j["@context"])
r6 = Results(**prueba)
r6.entries.append(Entry({"@id":"ohno", "nif:isString":"Just testing"}))
logging.debug("Reponse 6: %s", r6)
assert("marl" in r6.context)
assert("entries" in r6.context)
j6 = r6.jsonld(with_context=True)
logging.debug("jsonld: %s", j6)
assert("@context" in j6)
assert("entries" in j6)
assert("analysis" in j6)
resp = r6.flask()
received = json.loads(resp.data.decode())
logging.debug("Response: %s", j6)
assert(received["entries"])
assert(received["entries"][0]["nif:isString"] == "Just testing")
assert(received["entries"][0]["nif:isString"] != "Not testing")
def test_entries(self):
e = Entry()
self.assertRaises(jsonschema.ValidationError, e.validate)
e.nif__isString = "this is a test"
e.nif__beginIndex = 0
e.nif__endIndex = 10
e.validate()
def test_sentiment(self):
s = Sentiment()
self.assertRaises(jsonschema.ValidationError, s.validate)
s.nif__anchorOf = "so much testing"
s.prov__wasGeneratedBy = ""
s.validate()
def test_emotion_set(self):
e = EmotionSet()
self.assertRaises(jsonschema.ValidationError, e.validate)
e.nif__anchorOf = "so much testing"
e.prov__wasGeneratedBy = ""
self.assertRaises(jsonschema.ValidationError, e.validate)
e.onyx__hasEmotion = {}
e.validate()
def test_results(self):
r = Results()
e = Entry()
e.nif__isString = "Results test"
r.entries.append(e)
r.id = ":test_results"
r.validate()
def test_sentiments(self):
pass
def test_plugins(self):
self.assertRaises(Error, SenpyPlugin)
p = SenpyPlugin({"name": "dummy", "version": 0})
c = p.jsonld()
assert "info" not in c
assert "repo" not in c
assert "params" in c
logging.debug("Framed:")
logging.debug(c)
p.validate()
def test_frame_response(self):
pass

25
tests/plugins_test/__init__.py → tests/test_plugins.py
View File

@ -1,17 +1,15 @@
#!/bin/env python2
# -*- py-which-shell: "python2"; -*-
#!/bin/env python
import os
import logging
import shelve
import shutil
import tempfile
try:
import unittest.mock as mock
except ImportError:
import mock
import json
import os
from unittest import TestCase
from senpy.models import Response, Entry
from senpy.models import Results, Entry
from senpy.plugins import SenpyPlugin, ShelfMixin
@ -27,14 +25,18 @@ class ShelfTest(ShelfMixin, SenpyPlugin):
class ModelsTest(TestCase):
shelf_file = 'shelf_test.db'
def tearDown(self):
if os.path.exists(self.shelf_dir):
shutil.rmtree(self.shelf_dir)
if os.path.isfile(self.shelf_file):
os.remove(self.shelf_file)
setUp = tearDown
def setUp(self):
self.shelf_dir = tempfile.mkdtemp()
self.shelf_file = os.path.join(self.shelf_dir, "shelf")
def test_shelf(self):
''' A shelf is created and the value is stored '''
@ -45,11 +47,10 @@ class ModelsTest(TestCase):
assert a.shelf_file == self.shelf_file
a.sh['a'] = 'fromA'
a.test(key='a', value='fromA')
del(a)
assert os.path.isfile(self.shelf_file)
sh = shelve.open(self.shelf_file)
assert sh['a'] == 'fromA'