balkian/senpy
1
0
Fork 0
mirror of https://github.com/gsi-upm/senpy synced 2024-11-24 00:52:28 +00:00
Code Issues Releases Wiki Activity

Loads of changes!

Browse Source 
* Added conversion plugins (API might change!)
* Added conversion to the analysis pipeline
* Changed behaviour of --default-plugins (it adds conversion plugins regardless)
* Added emotionModel [sic] and emotionConversion models

//TODO add conversion tests
//TODO add conversion to docs
This commit is contained in:
J. Fernando Sánchez 2017-02-27 11:37:43 +01:00
parent 3cea7534ef
commit 9f6a6f5ecd
55 changed files with 986 additions and 461 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitlab-ci.yml
View File

@ -7,6 +7,9 @@ variables:
DOCKER_DRIVER: overlay
DOCKERFILE: Dockerfile
before_script:
- sh version.sh > senpy/VERSION
stages:
- test
- images

9
Dockerfile-3.4
View File

@ -1,10 +1,13 @@
from python:3.4-slim
from python:3.4
RUN mkdir /cache/
ENV PIP_CACHE_DIR=/cache/
WORKDIR /usr/src/app
ADD requirements.txt /usr/src/app/
RUN pip install --use-wheel -r requirements.txt
ADD . /usr/src/app/
RUN pip install --use-wheel .
RUN pip install .
VOLUME /data/
@ -13,6 +16,6 @@ RUN mkdir /senpy-plugins/
WORKDIR /senpy-plugins/
ONBUILD ADD . /senpy-plugins/
ONBUILD RUN python -m senpy -f /senpy-plugins
ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]

2
Dockerfile-3.5
View File

@ -1,4 +1,4 @@
from python:3.5
FROM python:3.5
RUN mkdir /cache/
ENV PIP_CACHE_DIR=/cache/

24
Makefile
View File

@ -2,13 +2,19 @@ PYVERSIONS=3.5 3.4 2.7
PYMAIN=$(firstword $(PYVERSIONS))
NAME=senpy
REPO=gsiupm
VERSION=$(shell git describe --tags)
TARNAME=$(NAME)-$(subst -,.,$(VERSION)).tar.gz
VERSION=$(shell ./version.sh)
TARNAME=$(NAME)-$(VERSION).tar.gz
IMAGENAME=$(REPO)/$(NAME):$(VERSION)
TEST_COMMAND=gitlab-runner exec docker --cache-dir=/tmp/gitlabrunner --docker-volumes /tmp/gitlabrunner:/tmp/gitlabrunner --env CI_PROJECT_NAME=$(NAME)
all: build run
FORCE:
version: FORCE
@echo $(VERSION) > $(NAME)/VERSION
@echo $(NAME) $(VERSION)
yapf:
yapf -i -r senpy
yapf -i -r tests
@ -36,7 +42,13 @@ quick_test: $(addprefix test-,$(PYMAIN))
test: $(addprefix test-,$(PYVERSIONS))
debug-%:
(docker start $(NAME)-debug && docker attach $(NAME)-debug) || docker run -w /usr/src/app/ -v $$PWD:/usr/src/app --entrypoint=/bin/bash -ti --name $(NAME)-debug '$(IMAGENAME)-python$* pip install -r test-requirements.txt'
@docker start $(NAME)-debug || (\
$(MAKE) build-$*; \
docker run -d -w /usr/src/app/ -v $$PWD:/usr/src/app --entrypoint=/bin/bash -p 5000:5000 -ti --name $(NAME)-debug '$(IMAGENAME)-python$*'; \
docker exec -ti $(NAME)-debug pip install -r test-requirements.txt; \
)\
docker attach $(NAME)-debug
debug: debug-$(PYMAIN)
@ -77,7 +89,9 @@ pip_upload:
pip_test: $(addprefix pip_test-,$(PYVERSIONS))
run: build
docker run --rm -p 5000:5000 -ti '$(IMAGENAME)-python$(PYMAIN)'
run-%: build-%
docker run --rm -p 5000:5000 -ti '$(IMAGENAME)-python$(PYMAIN)' --default-plugins
run: run-$(PYMAIN)
.PHONY: test test-% build-% build test pip_test run yapf dev

3
docs/examples/results/example-complete.json
View File

@ -53,7 +53,8 @@
"@id": "http://micro.blog/status1#char=16,77",
"nif:beginIndex": 16,
"nif:endIndex": 77,
"nif:anchorOf": "put your Windows Phone on your newest #open technology program"
"nif:anchorOf": "put your Windows Phone on your newest #open technology program",
"prov:wasGeneratedBy": "me:SgAnalysis1"
}
],
"sentiments": [

3
docs/examples/results/example-suggestion.json
View File

@ -24,7 +24,8 @@
"@id": "http://micro.blog/status1#char=16,77",
"nif:beginIndex": 16,
"nif:endIndex": 77,
"nif:anchorOf": "put your Windows Phone on your newest #open technology program"
"nif:anchorOf": "put your Windows Phone on your newest #open technology program",
"prov:wasGeneratedBy": "me:SgAnalysis1"
}
],
"sentiments": [

138
docs/plugins.rst
View File

@ -2,46 +2,127 @@ Developing new plugins
----------------------
Each plugin represents a different analysis process.There are two types of files that are needed by senpy for loading a plugin:
Plugins Interface
=======
- Definition file, has the ".senpy" extension.
- Code file, is a python file.
This separation will allow us to deploy plugins that use the same code but employ different parameters.
For instance, one could use the same classifier and processing in several plugins, but train with different datasets.
This scenario is particularly useful for evaluation purposes.
The only limitation is that the name of each plugin needs to be unique.
Plugins Definitions
===================
The definition file can be written in JSON or YAML, where the data representation consists on attribute-value pairs.
The principal attributes are:
The definition file contains all the attributes of the plugin, and can be written in YAML or JSON.
The most important attributes are:
* name: plugin name used in senpy to call the plugin.
* module: indicates the module that will be loaded
* **name**: unique name that senpy will use internally to identify the plugin.
* **module**: indicates the module that contains the plugin code, which will be automatically loaded by senpy.
* **version**
* extra_params: used to specify parameters that the plugin accepts that are not already part of the senpy API. Those parameters may be required, and have aliased names. For instance:
.. code:: python
.. code:: yaml
extra_params:
hello_param:
aliases: # required
- hello_param
- hello
required: true
default: Hi you
values:
- Hi you
- Hello y'all
- Howdy
Parameter validation will fail if a required parameter without a default has not been provided, or if the definition includes a set of values and the provided one does not match one of them.
A complete example:
.. code:: yaml
name: <Name of the plugin>
module: <Python file>
version: 0.1
And the json equivalent:
.. code:: json
{
"name" : "senpyPlugin",
"module" : "{python code file}"
"name": "<Name of the plugin>",
"module": "<Python file>",
"version": "0.1"
}
.. code:: python
name: senpyPlugin
module: {python code file}
Plugins Code
=================
============
The basic methods in a plugin are:
* __init__
* activate: used to load memory-hungry resources
* deactivate: used to free up resources
* analyse: called in every user requests. It takes in the parameters supplied by a user and should return a senpy Response.
* analyse_entry: called in every user requests. It takes in the parameters supplied by a user and should yield one or more ``Entry`` objects.
Plugins are loaded asynchronously, so don't worry if the activate method takes too long. The plugin will be marked as activated once it is finished executing the method.
Example plugin
==============
In this section, we will implement a basic sentiment analysis plugin.
To determine the polarity of each entry, the plugin will compare the length of the string to a threshold.
This threshold will be included in the definition file.
The definition file would look like this:
.. code:: yaml
name: helloworld
module: helloworld
version: 0.0
threshold: 10
Now, in a file named ``helloworld.py``:
.. code:: python
#!/bin/env python
#helloworld.py
from senpy.plugins import SenpyPlugin
from senpy.models import Sentiment
class HelloWorld(SenpyPlugin):
def analyse_entry(entry, params):
'''Basically do nothing with each entry'''
sentiment = Sentiment()
if len(entry.text) < self.threshold:
sentiment['marl:hasPolarity'] = 'marl:Positive'
else:
sentiment['marl:hasPolarity'] = 'marl:Negative'
entry.sentiments.append(sentiment)
yield entry
F.A.Q.
======
Why does the analyse function yield instead of return?
??????????????????????????????????????????????????????
This is so that plugins may add new entries to the response or filter some of them.
For instance, a `context detection` plugin may add a new entry for each context in the original entry.
On the other hand, a conveersion plugin may leave out those entries that do not contain relevant information.
If I'm using a classifier, where should I train it?
???????????????????????????????????????????????????
@ -78,17 +159,17 @@ This example ilustrate how to implement the Sentiment140 service as a plugin in
.. code:: python
class Sentiment140Plugin(SentimentPlugin):
def analyse(self, **params):
def analyse_entry(self, entry, params):
text = entry.text
lang = params.get("language", "auto")
res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
json.dumps({"language": lang,
"data": [{"text": params["input"]}]
"data": [{"text": text}]
}
)
)
p = params.get("prefix", None)
response = Results(prefix=p)
polarity_value = self.maxPolarityValue*int(res.json()["data"][0]
["polarity"]) * 0.25
polarity = "marl:Neutral"
@ -98,18 +179,13 @@ This example ilustrate how to implement the Sentiment140 service as a plugin in
elif polarity_value < neutral_value:
polarity = "marl:Negative"
entry = Entry(id="Entry0",
nif__isString=params["input"])
sentiment = Sentiment(id="Sentiment0",
prefix=p,
marl__hasPolarity=polarity,
marl__polarityValue=polarity_value)
sentiment.prov__wasGeneratedBy = self.id
entry.sentiments = []
entry.sentiments.append(sentiment)
entry.language = lang
response.entries.append(entry)
return response
yield entry
Where can I define extra parameters to be introduced in the request to my plugin?
@ -143,9 +219,9 @@ The extraction of this paremeter is used in the analyse method of the Plugin int
Where can I set up variables for using them in my plugin?
?????????????????????????????????????????????????????????
You can add these variables in the definition file with the extracture of attribute-value pair.
You can add these variables in the definition file with the structure of attribute-value pairs.
Once you have added your variables, the next step is to extract them into the plugin. The plugin's __init__ method has a parameter called `info` where you can extract the values of the variables. This info parameter has the structure of a python dictionary.
Every field added to the definition file is available to the plugin instance.
Can I activate a DEBUG mode for my plugin?
???????????????????????????????????????????
@ -154,7 +230,15 @@ You can activate the DEBUG mode by the command-line tool using the option -d.
.. code:: bash
python -m senpy -d
senpy -d
Additionally, with the ``--pdb`` option you will be dropped into a pdb post mortem shell if an exception is raised.
.. code:: bash
senpy --pdb
Where can I find more code examples?
????????????????????????????????????

5
requirements.txt
View File

@ -1,7 +1,5 @@
Flask>=0.10.1
gunicorn>=19.0.0
requests>=2.4.1
GitPython>=0.3.2.RC1
gevent>=1.1rc4
PyLD>=0.6.5
six
@ -9,4 +7,5 @@ future
jsonschema
jsonref
PyYAML
semver
rdflib
rdflib-jsonld

19
senpy/__init__.py
View File

@ -20,21 +20,10 @@ Sentiment analysis server in Python
from __future__ import print_function
from .version import __version__
try:
import semver
__version_info__ = semver.parse_version_info(__version__)
import logging
if __version_info__.prerelease:
import logging
logger = logging.getLogger(__name__)
msg = 'WARNING: You are using a pre-release version of {} ({})'.format(
__name__, __version__)
if len(logging.root.handlers) > 0:
logger.info(msg)
else:
import sys
print(msg, file=sys.stderr)
except ImportError:
print('semver not installed. Not doing version checking')
logger = logging.getLogger(__name__)
logger.info('Using senpy version: {}'.format(__version__))
__all__ = ['api', 'blueprints', 'cli', 'extensions', 'models', 'plugins']

5
senpy/__main__.py
View File

@ -74,7 +74,7 @@ def main():
parser.add_argument(
'--host',
type=str,
default="127.0.0.1",
default="0.0.0.0",
help='Use 0.0.0.0 to accept requests from any host.')
parser.add_argument(
'--port',
@ -93,8 +93,7 @@ def main():
'-i',
action='store_true',
default=False,
help='Do not run a server, only install plugin dependencies'
)
help='Do not run a server, only install plugin dependencies')
args = parser.parse_args()
logging.basicConfig()
rl = logging.getLogger()

41
senpy/api.py
View File

@ -7,6 +7,31 @@ API_PARAMS = {
"algorithm": {
"aliases": ["algorithm", "a", "algo"],
"required": False,
},
"outformat": {
"@id": "outformat",
"aliases": ["outformat", "o"],
"default": "json-ld",
"required": True,
"options": ["json-ld", "turtle"],
},
"expanded-jsonld": {
"@id": "expanded-jsonld",
"aliases": ["expanded", "expanded-jsonld"],
"required": True,
"default": 0
},
"emotionModel": {
"@id": "emotionModel",
"aliases": ["emotionModel", "emoModel"],
"required": False
},
"conversion": {
"@id": "conversion",
"description": "How to show the elements that have (not) been converted",
"required": True,
"options": ["filtered", "nested", "full"],
"default": "full"
}
}
@ -47,13 +72,6 @@ NIF_PARAMS = {
"default": "direct",
"options": ["direct", "url", "file"],
},
"outformat": {
"@id": "outformat",
"aliases": ["outformat", "o"],
"default": "json-ld",
"required": False,
"options": ["json-ld"],
},
"language": {
"@id": "language",
"aliases": ["language", "l"],
@ -76,12 +94,12 @@ NIF_PARAMS = {
def parse_params(indict, spec=NIF_PARAMS):
outdict = {}
logger.debug("Parsing: {}\n{}".format(indict, spec))
outdict = indict.copy()
wrong_params = {}
for param, options in iteritems(spec):
if param[0] != "@": # Exclude json-ld properties
logger.debug("Param: %s - Options: %s", param, options)
for alias in options["aliases"]:
for alias in options.get("aliases", []):
if alias in indict:
outdict[param] = indict[alias]
if param not in outdict:
@ -95,8 +113,9 @@ def parse_params(indict, spec=NIF_PARAMS):
outdict[param] not in spec[param]["options"]:
wrong_params[param] = spec[param]
if wrong_params:
logger.debug("Error parsing: %s", wrong_params)
message = Error(
status=404,
status=400,
message="Missing or invalid parameters",
parameters=outdict,
errors={param: error

61
senpy/blueprints.py
View File

@ -17,10 +17,11 @@
"""
Blueprints for Senpy
"""
from flask import (Blueprint, request, current_app,
render_template, url_for, jsonify)
from flask import (Blueprint, request, current_app, render_template, url_for,
jsonify)
from .models import Error, Response, Plugins, read_schema
from .api import WEB_PARAMS, parse_params
from .api import WEB_PARAMS, API_PARAMS, parse_params
from .version import __version__
from functools import wraps
import logging
@ -29,6 +30,7 @@ logger = logging.getLogger(__name__)
api_blueprint = Blueprint("api", __name__)
demo_blueprint = Blueprint("demo", __name__)
ns_blueprint = Blueprint("ns", __name__)
def get_params(req):
@ -43,12 +45,21 @@ def get_params(req):
@demo_blueprint.route('/')
def index():
return render_template("index.html")
return render_template("index.html", version=__version__)
@api_blueprint.route('/contexts/<entity>.jsonld')
def context(entity="context"):
return jsonify({"@context": Response.context})
context = Response._context
context['@vocab'] = url_for('ns.index', _external=True)
return jsonify({"@context": context})
@ns_blueprint.route('/') # noqa: F811
def index():
context = Response._context
context['@vocab'] = url_for('.ns', _external=True)
return jsonify({"@context": context})
@api_blueprint.route('/schemas/<schema>')
@ -62,26 +73,39 @@ def schema(schema="definitions"):
def basic_api(f):
@wraps(f)
def decorated_function(*args, **kwargs):
raw_params = get_params(request)
headers = {'X-ORIGINAL-PARAMS': raw_params}
# Get defaults
web_params = parse_params({}, spec=WEB_PARAMS)
api_params = parse_params({}, spec=API_PARAMS)
outformat = 'json-ld'
try:
print('Getting request:')
print(request)
raw_params = get_params(request)
web_params = parse_params(raw_params, spec=WEB_PARAMS)
api_params = parse_params(raw_params, spec=API_PARAMS)
if hasattr(request, 'params'):
request.params.update(raw_params)
request.params.update(api_params)
else:
request.params = raw_params
try:
request.params = api_params
response = f(*args, **kwargs)
except Error as ex:
response = ex
in_headers = web_params["inHeaders"] != "0"
headers = {'X-ORIGINAL-PARAMS': raw_params}
in_headers = web_params['inHeaders'] != "0"
expanded = api_params['expanded-jsonld']
outformat = api_params['outformat']
return response.flask(
in_headers=in_headers,
headers=headers,
context_uri=url_for(
'api.context', entity=type(response).__name__, _external=True))
prefix=url_for('.api', _external=True),
context_uri=url_for('api.context',
entity=type(response).__name__,
_external=True),
outformat=outformat,
expanded=expanded)
return decorated_function
@ -106,10 +130,11 @@ def plugins():
def plugin(plugin=None):
sp = current_app.senpy
if plugin == 'default' and sp.default_plugin:
response = sp.default_plugin
plugin = response.name
elif plugin in sp.plugins:
response = sp.plugins[plugin]
return sp.default_plugin
plugins = sp.filter_plugins(
id='plugins/{}'.format(plugin)) or sp.filter_plugins(name=plugin)
if plugins:
response = list(plugins.values())[0]
else:
return Error(message="Plugin not found", status=404)
return response

8
senpy/client.py
View File

@ -6,7 +6,6 @@ logger = logging.getLogger(__name__)
class Client(object):
def __init__(self, endpoint):
self.endpoint = endpoint
@ -15,9 +14,7 @@ class Client(object):
def request(self, path=None, method='GET', **params):
url = '{}{}'.format(self.endpoint, path)
response = requests.request(method=method,
url=url,
params=params)
response = requests.request(method=method, url=url, params=params)
try:
resp = models.from_dict(response.json())
resp.validate(resp)
@ -30,7 +27,8 @@ class Client(object):
'#### Response:\n'
'\tCode: {code}'
'\tContent: {content}'
'\n').format(error=ex,
'\n').format(
error=ex,
url=url,
code=response.status_code,
content=response.content))

145
senpy/extensions.py
View File

@ -1,15 +1,15 @@
"""
Main class for Senpy.
It orchestrates plugin (de)activation and analysis.
"""
from future import standard_library
standard_library.install_aliases()
from .plugins import SentimentPlugin
from .models import Error
from .blueprints import api_blueprint, demo_blueprint
from .plugins import SentimentPlugin, SenpyPlugin
from .models import Error, Entry, Results
from .blueprints import api_blueprint, demo_blueprint, ns_blueprint
from .api import API_PARAMS, NIF_PARAMS, parse_params
from git import Repo, InvalidGitRepositoryError
from threading import Thread
import os
@ -30,18 +30,21 @@ class Senpy(object):
def __init__(self,
app=None,
plugin_folder="plugins",
plugin_folder=".",
default_plugins=False):
self.app = app
self._search_folders = set()
self._plugin_list = []
self._outdated = True
self._default = None
self.add_folder(plugin_folder)
if default_plugins:
base_folder = os.path.join(os.path.dirname(__file__), "plugins")
self.add_folder(base_folder)
self.add_folder('plugins', from_root=True)
else:
# Add only conversion plugins
self.add_folder(os.path.join('plugins', 'conversion'),
from_root=True)
if app is not None:
self.init_app(app)
@ -60,9 +63,12 @@ class Senpy(object):
else:
app.teardown_request(self.teardown)
app.register_blueprint(api_blueprint, url_prefix="/api")
app.register_blueprint(ns_blueprint, url_prefix="/ns")
app.register_blueprint(demo_blueprint, url_prefix="/")
def add_folder(self, folder):
def add_folder(self, folder, from_root=False):
if from_root:
folder = os.path.join(os.path.dirname(__file__), folder)
logger.debug("Adding folder: %s", folder)
if os.path.isdir(folder):
self._search_folders.add(folder)
@ -70,10 +76,9 @@ class Senpy(object):
else:
logger.debug("Not a folder: %s", folder)
def analyse(self, **params):
algo = None
logger.debug("analysing with params: {}".format(params))
def _find_plugin(self, params):
api_params = parse_params(params, spec=API_PARAMS)
algo = None
if "algorithm" in api_params and api_params["algorithm"]:
algo = api_params["algorithm"]
elif self.plugins:
@ -97,32 +102,114 @@ class Senpy(object):
status=400,
message=("The algorithm '{}'"
" is not activated yet").format(algo))
plug = self.plugins[algo]
return self.plugins[algo]
def _get_params(self, params, plugin):
nif_params = parse_params(params, spec=NIF_PARAMS)
extra_params = plug.get('extra_params', {})
extra_params = plugin.get('extra_params', {})
specific_params = parse_params(params, spec=extra_params)
nif_params.update(specific_params)
return nif_params
def _get_entries(self, params):
entry = None
if params['informat'] == 'text':
entry = Entry(text=params['input'])
else:
raise NotImplemented('Only text input format implemented')
yield entry
def analyse(self, **api_params):
logger.debug("analysing with params: {}".format(api_params))
plugin = self._find_plugin(api_params)
nif_params = self._get_params(api_params, plugin)
resp = Results()
if 'with_parameters' in api_params:
resp.parameters = nif_params
try:
resp = plug.analyse(**nif_params)
resp.analysis.append(plug)
entries = []
for i in self._get_entries(nif_params):
entries += list(plugin.analyse_entry(i, nif_params))
resp.entries = entries
self.convert_emotions(resp, plugin, nif_params)
resp.analysis.append(plugin.id)
logger.debug("Returning analysis result: {}".format(resp))
except Error as ex:
logger.exception('Error returning analysis result')
resp = ex
except Exception as ex:
resp = Error(message=str(ex), status=500)
logger.exception('Error returning analysis result')
resp = Error(message=str(ex), status=500)
return resp
def _conversion_candidates(self, fromModel, toModel):
candidates = self.filter_plugins(**{'@type': 'emotionConversionPlugin'})
for name, candidate in candidates.items():
for pair in candidate.onyx__doesConversion:
logging.debug(pair)
if pair['onyx:conversionFrom'] == fromModel \
and pair['onyx:conversionTo'] == toModel:
# logging.debug('Found candidate: {}'.format(candidate))
yield candidate
def convert_emotions(self, resp, plugin, params):
"""
Conversion of all emotions in a response.
In addition to converting from one model to another, it has
to include the conversion plugin to the analysis list.
Needless to say, this is far from an elegant solution, but it works.
@todo refactor and clean up
"""
fromModel = plugin.get('onyx:usesEmotionModel', None)
toModel = params.get('emotionModel', None)
output = params.get('conversion', None)
logger.debug('Asked for model: {}'.format(toModel))
logger.debug('Analysis plugin uses model: {}'.format(fromModel))
if not toModel:
return
try:
candidate = next(self._conversion_candidates(fromModel, toModel))
except StopIteration:
e = Error(('No conversion plugin found for: '
'{} -> {}'.format(fromModel, toModel)))
e.original_response = resp
e.parameters = params
raise e
newentries = []
for i in resp.entries:
if output == "full":
newemotions = i.emotions.copy()
else:
newemotions = []
for j in i.emotions:
for k in candidate.convert(j, fromModel, toModel, params):
k.prov__wasGeneratedBy = candidate.id
if output == 'nested':
k.prov__wasDerivedFrom = j
newemotions.append(k)
i.emotions = newemotions
newentries.append(i)
resp.entries = newentries
resp.analysis.append(candidate.id)
@property
def default_plugin(self):
candidate = self._default
if not candidate:
candidates = self.filter_plugins(is_activated=True)
if len(candidates) > 0:
candidate = list(candidates.values())[0]
logger.debug("Default: {}".format(candidate.name))
logger.debug("Default: {}".format(candidate))
return candidate
@default_plugin.setter
def default_plugin(self, value):
if isinstance(value, SenpyPlugin):
self._default = value
else:
return None
self._default = self.plugins[value]
def activate_all(self, sync=False):
ps = []
@ -164,6 +251,7 @@ class Senpy(object):
plugin.name, ex, traceback.format_exc())
logger.error(msg)
raise Error(msg)
if sync:
act()
else:
@ -184,8 +272,8 @@ class Senpy(object):
plugin.deactivate()
logger.info("Plugin deactivated: {}".format(plugin.name))
except Exception as ex:
logger.error("Error deactivating plugin {}: {}".format(
plugin.name, ex))
logger.error(
"Error deactivating plugin {}: {}".format(plugin.name, ex))
logger.error("Trace: {}".format(traceback.format_exc()))
if sync:
@ -237,13 +325,6 @@ class Senpy(object):
logger.debug("No valid plugin for: {}".format(module))
return
module = candidate(info=info)
repo_path = root
try:
module._repo = Repo(repo_path)
except InvalidGitRepositoryError:
logger.debug("The plugin {} is not in a Git repository".format(
module))
module._repo = None
return name, module
@classmethod
@ -261,7 +342,7 @@ class Senpy(object):
for root, dirnames, filenames in os.walk(search_folder):
for filename in fnmatch.filter(filenames, '*.senpy'):
name, plugin = self._load_plugin(root, filename)
if plugin and name not in self._plugin_list:
if plugin and name:
plugins[name] = plugin
self._outdated = False
@ -282,8 +363,8 @@ class Senpy(object):
def matches(plug):
res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
logger.debug("matching {} with {}: {}".format(plug.name, kwargs,
res))
logger.debug(
"matching {} with {}: {}".format(plug.name, kwargs, res))
return res
if not kwargs:

138
senpy/models.py
View File

@ -16,6 +16,9 @@ import jsonref
import jsonschema
from flask import Response as FlaskResponse
from pyld import jsonld
from rdflib import Graph
import logging
@ -72,31 +75,60 @@ base_context = Context.load(CONTEXT_PATH)
class SenpyMixin(object):
context = base_context["@context"]
_context = base_context["@context"]
def flask(self, in_headers=True, headers=None, **kwargs):
def flask(self,
in_headers=True,
headers=None,
outformat='json-ld',
**kwargs):
"""
Return the values and error to be used in flask.
So far, it returns a fixed context. We should store/generate different
contexts if the plugin adds more aliases.
"""
headers = headers or {}
kwargs["with_context"] = True
js = self.jsonld(**kwargs)
if in_headers:
url = js["@context"]
del js["@context"]
kwargs["with_context"] = not in_headers
content, mimetype = self.serialize(format=outformat,
with_mime=True,
**kwargs)
if outformat == 'json-ld' and in_headers:
headers.update({
"Link": ('<%s>;'
"Link":
('<%s>;'
'rel="http://www.w3.org/ns/json-ld#context";'
' type="application/ld+json"' % url)
' type="application/ld+json"' % kwargs.get('context_uri'))
})
return FlaskResponse(
json.dumps(
js, indent=2, sort_keys=True),
response=content,
status=getattr(self, "status", 200),
headers=headers,
mimetype="application/json")
mimetype=mimetype)
def serialize(self, format='json-ld', with_mime=False, **kwargs):
js = self.jsonld(**kwargs)
if format == 'json-ld':
content = json.dumps(js, indent=2, sort_keys=True)
mimetype = "application/json"
elif format in ['turtle', ]:
logger.debug(js)
content = json.dumps(js, indent=2, sort_keys=True)
g = Graph().parse(
data=content,
format='json-ld',
base=kwargs.get('prefix'),
context=self._context)
logger.debug(
'Parsing with prefix: {}'.format(kwargs.get('prefix')))
content = g.serialize(format='turtle').decode('utf-8')
mimetype = 'text/{}'.format(format)
else:
raise Error('Unknown outformat: {}'.format(format))
if with_mime:
return content, mimetype
else:
return content
def serializable(self):
def ser_or_down(item):
@ -115,28 +147,30 @@ class SenpyMixin(object):
return ser_or_down(self._plain_dict())
def jsonld(self, with_context=False, context_uri=None):
def jsonld(self,
with_context=True,
context_uri=None,
prefix=None,
expanded=False):
ser = self.serializable()
if with_context:
context = []
result = jsonld.compact(
ser,
self._context,
options={
'base': prefix,
'expandContext': self._context,
'senpy': prefix
})
if context_uri:
context = context_uri
else:
context = self.context.copy()
if hasattr(self, 'prefix'):
# This sets @base for the document, which will be used in
# all relative URIs. For example, if a uri is "Example" and
# prefix =s "http://example.com", the absolute URI after
# expanding with JSON-LD will be "http://example.com/Example"
prefix_context = {"@base": self.prefix}
if isinstance(context, list):
context.append(prefix_context)
else:
context = [context, prefix_context]
ser["@context"] = context
return ser
result['@context'] = context_uri
if expanded:
result = jsonld.expand(
result, options={'base': prefix,
'expandContext': self._context})
if not with_context:
del result['@context']
return result
def to_JSON(self, *args, **kwargs):
js = json.dumps(self.jsonld(*args, **kwargs), indent=4, sort_keys=True)
@ -161,13 +195,14 @@ class BaseModel(SenpyMixin, dict):
if 'id' in kwargs:
self.id = kwargs.pop('id')
elif kwargs.pop('_auto_id', True):
self.id = '_:{}_{}'.format(
type(self).__name__, time.time())
self.id = '_:{}_{}'.format(type(self).__name__, time.time())
temp = dict(*args, **kwargs)
for obj in [self.schema, ] + self.schema.get('allOf', []):
for obj in [
self.schema,
] + self.schema.get('allOf', []):
for k, v in obj.get('properties', {}).items():
if 'default' in v:
if 'default' in v and k not in temp:
temp[k] = copy.deepcopy(v['default'])
for i in temp:
@ -175,10 +210,6 @@ class BaseModel(SenpyMixin, dict):
if nk != i:
temp[nk] = temp[i]
del temp[i]
if 'context' in temp:
context = temp['context']
del temp['context']
self.__dict__['context'] = Context.load(context)
try:
temp['@type'] = getattr(self, '@type')
except AttributeError:
@ -239,10 +270,11 @@ def from_schema(name, schema_file=None, base_classes=None):
base_classes = base_classes or []
base_classes.append(BaseModel)
schema_file = schema_file or '{}.json'.format(name)
class_name = '{}{}'.format(i[0].upper(), i[1:])
class_name = '{}{}'.format(name[0].upper(), name[1:])
newclass = type(class_name, tuple(base_classes), {})
setattr(newclass, '@type', name)
setattr(newclass, 'schema', read_schema(schema_file))
setattr(newclass, 'class_name', class_name)
register(newclass, name)
return newclass
@ -253,29 +285,31 @@ def _add_from_schema(*args, **kwargs):
del generatedClass
for i in ['response',
'results',
'entry',
'sentiment',
for i in [
'analysis',
'emotionSet',
'emotion',
'emotionConversion',
'emotionConversionPlugin',
'emotionAnalysis',
'emotionModel',
'suggestion',
'plugin',
'emotionPlugin',
'emotionSet',
'entry',
'plugin',
'plugins',
'response',
'results',
'sentiment',
'sentimentPlugin',
'plugins']:
'suggestion',
]:
_add_from_schema(i)
_ErrorModel = from_schema('error')
class Error(SenpyMixin, BaseException):
def __init__(self,
message,
*args,
**kwargs):
def __init__(self, message, *args, **kwargs):
super(Error, self).__init__(self, message, message)
self._error = _ErrorModel(message=message, *args, **kwargs)
self.message = message

56
senpy/plugins.py
View File

@ -6,6 +6,7 @@ import os.path
import pickle
import logging
import tempfile
import copy
from . import models
logger = logging.getLogger(__name__)
@ -13,21 +14,38 @@ logger = logging.getLogger(__name__)
class SenpyPlugin(models.Plugin):
def __init__(self, info=None):
"""
Provides a canonical name for plugins and serves as base for other
kinds of plugins.
"""
if not info:
raise models.Error(message=("You need to provide configuration"
"information for the plugin."))
logger.debug("Initialising {}".format(info))
super(SenpyPlugin, self).__init__(info)
self.id = '{}_{}'.format(self.name, self.version)
self._info = info
id = 'plugins/{}_{}'.format(info['name'], info['version'])
super(SenpyPlugin, self).__init__(id=id, **info)
self.is_activated = False
def get_folder(self):
return os.path.dirname(inspect.getfile(self.__class__))
def analyse(self, *args, **kwargs):
logger.debug("Analysing with: {} {}".format(self.name, self.version))
pass
raise NotImplemented(
'Your method should implement either analyse or analyse_entry')
def analyse_entry(self, entry, parameters):
""" An implemented plugin should override this method.
This base method is here to adapt old style plugins which only
implement the *analyse* function.
Note that this method may yield an annotated entry or a list of
entries (e.g. in a tokenizer)
"""
text = entry['text']
params = copy.copy(parameters)
params['input'] = text
results = self.analyse(**params)
for i in results.entries:
yield i
def activate(self):
pass
@ -35,25 +53,24 @@ class SenpyPlugin(models.Plugin):
def deactivate(self):
pass
def __del__(self):
''' Destructor, to make sure all the resources are freed '''
self.deactivate()
class SentimentPlugin(SenpyPlugin, models.SentimentPlugin):
class SentimentPlugin(models.SentimentPlugin, SenpyPlugin):
def __init__(self, info, *args, **kwargs):
super(SentimentPlugin, self).__init__(info, *args, **kwargs)
self.minPolarityValue = float(info.get("minPolarityValue", 0))
self.maxPolarityValue = float(info.get("maxPolarityValue", 1))
self["@type"] = "marl:SentimentAnalysis"
class EmotionPlugin(SentimentPlugin, models.EmotionPlugin):
class EmotionPlugin(models.EmotionPlugin, SenpyPlugin):
def __init__(self, info, *args, **kwargs):
super(EmotionPlugin, self).__init__(info, *args, **kwargs)
self.minEmotionValue = float(info.get("minEmotionValue", 0))
self.maxEmotionValue = float(info.get("maxEmotionValue", 0))
self["@type"] = "onyx:EmotionAnalysis"
self.minEmotionValue = float(info.get("minEmotionValue", -1))
self.maxEmotionValue = float(info.get("maxEmotionValue", 1))
class EmotionConversionPlugin(models.EmotionConversionPlugin, SenpyPlugin):
def __init__(self, info, *args, **kwargs):
super(EmotionConversionPlugin, self).__init__(info, *args, **kwargs)
class ShelfMixin(object):
@ -74,13 +91,10 @@ class ShelfMixin(object):
@property
def shelf_file(self):
if not hasattr(self, '_shelf_file') or not self._shelf_file:
if hasattr(self, '_info') and 'shelf_file' in self._info:
self.__dict__['_shelf_file'] = self._info['shelf_file']
else:
self._shelf_file = os.path.join(tempfile.gettempdir(),
if 'shelf_file' not in self or not self['shelf_file']:
self.shelf_file = os.path.join(tempfile.gettempdir(),
self.name + '.p')
return self._shelf_file
return self['shelf_file']
def save(self):
logger.debug('saving pickle')

56
senpy/plugins/conversion/emotion/ekman2vad.py Normal file
View File

@ -0,0 +1,56 @@
from senpy.plugins import EmotionConversionPlugin
from senpy.models import EmotionSet, Emotion, Error
import logging
logger = logging.getLogger(__name__)
import math
class WNA2VAD(EmotionConversionPlugin):
def _ekman_to_vad(self, ekmanSet):
potency = 0
arousal = 0
dominance = 0
for e in ekmanSet.onyx__hasEmotion:
category = e.onyx__hasEmotionCategory
centroid = self.centroids[category]
potency += centroid['V']
arousal += centroid['A']
dominance += centroid['D']
e = Emotion({'emoml:potency': potency,
'emoml:arousal': arousal,
'emoml:dominance': dominance})
return e
def _vad_to_ekman(self, VADEmotion):
V = VADEmotion['emoml:valence']
A = VADEmotion['emoml:potency']
D = VADEmotion['emoml:dominance']
emotion = ''
value = 10000000000000000000000.0
for state in self.centroids:
valence = V - self.centroids[state]['V']
arousal = A - self.centroids[state]['A']
dominance = D - self.centroids[state]['D']
new_value = math.sqrt((valence**2) +
(arousal**2) +
(dominance**2))
if new_value < value:
value = new_value
emotion = state
result = Emotion(onyx__hasEmotionCategory=emotion)
return result
def convert(self, emotionSet, fromModel, toModel, params):
logger.debug('{}\n{}\n{}\n{}'.format(emotionSet, fromModel, toModel, params))
e = EmotionSet()
if fromModel == 'emoml:big6':
e.onyx__hasEmotion.append(self._ekman_to_vad(emotionSet))
elif fromModel == 'emoml:fsre-dimensions':
for i in emotionSet.onyx__hasEmotion:
e.onyx__hasEmotion.append(self._vad_to_ekman(e))
else:
raise Error('EMOTION MODEL NOT KNOWN')
yield e

35
senpy/plugins/conversion/emotion/ekman2vad.senpy Normal file
View File

@ -0,0 +1,35 @@
---
name: Ekman2VAD
module: ekman2vad
description: Plugin to convert from Ekman to VAD
version: 0.1
onyx:doesConversion:
- onyx:conversionFrom: emoml:big6
onyx:conversionTo: emoml:fsre-dimensions
- onyx:conversionFrom: emoml:fsre-dimensions
onyx:conversionTo: wna:WNAModel
centroids:
emoml:big6anger:
A: 6.95
D: 5.1
V: 2.7
emoml:big6disgust:
A: 5.3
D: 8.05
V: 2.7
emoml:big6fear:
A: 6.5
D: 3.6
V: 3.2
emoml:big6happiness:
A: 7.22
D: 6.28
V: 8.6
emoml:big6sadness:
A: 5.21
D: 2.82
V: 2.21
aliases:
A: emoml:arousal
V: emoml:potency
D: emoml:dominance

18
senpy/plugins/example/emoRand/emoRand.py Normal file
View File

@ -0,0 +1,18 @@
import random
from senpy.plugins import EmotionPlugin
from senpy.models import EmotionSet, Emotion
class RmoRandPlugin(EmotionPlugin):
def analyse_entry(self, entry, params):
category = "emoml:big6happiness"
number = max(-1, min(1, random.gauss(0, 0.5)))
if number > 0:
category = "emoml:big6anger"
emotionSet = EmotionSet()
emotion = Emotion({"onyx:hasEmotionCategory": category})
emotionSet.onyx__hasEmotion.append(emotion)
emotionSet.prov__wasGeneratedBy = self.id
entry.emotions.append(emotionSet)
yield entry

9
senpy/plugins/example/emoRand/emoRand.senpy Normal file
View File

@ -0,0 +1,9 @@
---
name: emoRand
module: emoRand
description: A sample plugin that returns a random emotion annotation
author: "@balkian"
version: '0.1'
url: "https://github.com/gsi-upm/senpy-plugins-community"
requirements: {}
onyx:usesEmotionModel: "emoml:big6"

13
senpy/plugins/rand/rand.py → senpy/plugins/example/rand/rand.py
View File

@ -1,29 +1,24 @@
import random
from senpy.plugins import SentimentPlugin
from senpy.models import Results, Sentiment, Entry
from senpy.models import Sentiment
class Sentiment140Plugin(SentimentPlugin):
def analyse(self, **params):
class RandPlugin(SentimentPlugin):
def analyse_entry(self, entry, params):
lang = params.get("language", "auto")
response = Results()
polarity_value = max(-1, min(1, random.gauss(0.2, 0.2)))
polarity = "marl:Neutral"
if polarity_value > 0:
polarity = "marl:Positive"
elif polarity_value < 0:
polarity = "marl:Negative"
entry = Entry({"id": ":Entry0", "nif:isString": params["input"]})
sentiment = Sentiment({
"id": ":Sentiment0",
"marl:hasPolarity": polarity,
"marl:polarityValue": polarity_value
})
sentiment["prov:wasGeneratedBy"] = self.id
entry.sentiments = []
entry.sentiments.append(sentiment)
entry.language = lang
response.entries.append(entry)
return response
yield entry

10
senpy/plugins/example/rand/rand.senpy Normal file
View File

@ -0,0 +1,10 @@
---
name: rand
module: rand
description: A sample plugin that returns a random sentiment annotation
author: "@balkian"
version: '0.1'
url: "https://github.com/gsi-upm/senpy-plugins-community"
requirements: {}
marl:maxPolarityValue: '1'
marl:minPolarityValue: "-1"

18
senpy/plugins/rand/rand.senpy
View File

@ -1,18 +0,0 @@
{
"name": "rand",
"module": "rand",
"description": "What my plugin broadly does",
"author": "@balkian",
"version": "0.1",
"extra_params": {
"language": {
"@id": "lang_rand",
"aliases": ["language", "l"],
"required": false,
"options": ["es", "en", "auto"]
}
},
"requirements": {},
"marl:maxPolarityValue": "1",
"marl:minPolarityValue": "-1"
}

17
senpy/plugins/sentiment140/sentiment140.py → senpy/plugins/sentiment/sentiment140/sentiment140.py
View File

@ -2,24 +2,22 @@ import requests
import json
from senpy.plugins import SentimentPlugin
from senpy.models import Results, Sentiment, Entry
from senpy.models import Sentiment
class Sentiment140Plugin(SentimentPlugin):
def analyse(self, **params):
def analyse_entry(self, entry, params):
lang = params.get("language", "auto")
res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
json.dumps({
"language": lang,
"data": [{
"text": params["input"]
"text": entry.text
}]
}))
p = params.get("prefix", None)
response = Results(prefix=p)
polarity_value = self.maxPolarityValue * int(res.json()["data"][0][
"polarity"]) * 0.25
polarity_value = self.maxPolarityValue * int(
res.json()["data"][0]["polarity"]) * 0.25
polarity = "marl:Neutral"
neutral_value = self.maxPolarityValue / 2.0
if polarity_value > neutral_value:
@ -27,9 +25,7 @@ class Sentiment140Plugin(SentimentPlugin):
elif polarity_value < neutral_value:
polarity = "marl:Negative"
entry = Entry(id="Entry0", nif__isString=params["input"])
sentiment = Sentiment(
id="Sentiment0",
prefix=p,
marl__hasPolarity=polarity,
marl__polarityValue=polarity_value)
@ -37,5 +33,4 @@ class Sentiment140Plugin(SentimentPlugin):
entry.sentiments = []
entry.sentiments.append(sentiment)
entry.language = lang
response.entries.append(entry)
return response
yield entry

21
senpy/plugins/sentiment/sentiment140/sentiment140.senpy Normal file
View File

@ -0,0 +1,21 @@
---
name: sentiment140
module: sentiment140
description: "Connects to the sentiment140 free API: http://sentiment140.com"
author: "@balkian"
version: '0.2'
url: "https://github.com/gsi-upm/senpy-plugins-community"
extra_params:
language:
"@id": lang_sentiment140
aliases:
- language
- l
required: false
options:
- es
- en
- auto
requirements: {}
maxPolarityValue: 1
minPolarityValue: 0

18
senpy/plugins/sentiment140/sentiment140.senpy
View File

@ -1,18 +0,0 @@
{
"name": "sentiment140",
"module": "sentiment140",
"description": "What my plugin broadly does",
"author": "@balkian",
"version": "0.1",
"extra_params": {
"language": {
"@id": "lang_sentiment140",
"aliases": ["language", "l"],
"required": false,
"options": ["es", "en", "auto"]
}
},
"requirements": {},
"maxPolarityValue": "1",
"minPolarityValue": "0"
}

7
senpy/schemas/\
View File

@ -1,7 +0,0 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Senpy analysis",
"allOf": [{
"$ref": "atom.json"
}]
}

25
senpy/schemas/context.jsonld
View File

@ -6,8 +6,9 @@
"prov": "http://www.w3.org/ns/prov#",
"nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#",
"marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#",
"onyx": "http://www.gsi.dit.upm.es/ontologies/onyx#",
"wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#",
"onyx": "http://www.gsi.dit.upm.es/ontologies/onyx/ns#",
"wna": "http://www.gsi.dit.upm.es/ontologies/wnaffect/ns#",
"emoml": "http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/emotionml/ns#",
"xsd": "http://www.w3.org/2001/XMLSchema#",
"topics": {
"@id": "dc:subject"
@ -32,8 +33,24 @@
"@container": "@set"
},
"analysis": {
"@id": "prov:wasGeneratedBy"
"@id": "AnalysisInvolved",
"@type": "@id",
"@container": "@set"
},
"prov:wasGeneratedBy": {
"@type": "@id"
},
"onyx:usesEmotionModel": {
"@type": "@id"
},
"onyx:hasEmotionCategory": {
"@type": "@id"
},
"onyx:conversionFrom": {
"@type": "@id"
},
"onyx:conversionTo": {
"@type": "@id"
}
}
}

5
senpy/schemas/emotionAnalysis.json
View File

@ -6,13 +6,14 @@
{"$ref": "analysis.json"},
{"properties":
{
"onyx:hasEmotionModel": {
"onyx:usesEmotionModel": {
"anyOf": [
{"type": "string"},
{"$ref": "emotionModel.json"}
]
}
},
"required": ["onyx:hasEmotionModel"]
"required": ["onyx:hasEmotionModel",
"@type"]
}]
}

12
senpy/schemas/emotionConversion.json Normal file
View File

@ -0,0 +1,12 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"properties": {
"onyx:conversionFrom": {
"$ref": "emotionModel.json"
},
"onyx:conversionTo": {
"$ref": "emotionModel.json"
}
},
"required": ["onyx:conversionFrom", "onyx:conversionTo"]
}

19
senpy/schemas/emotionConversionPlugin.json Normal file
View File

@ -0,0 +1,19 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"allOf": [
{
"$ref": "plugin.json"
},
{
"properties": {
"onyx:doesConversion": {
"type": "array",
"items": {
"$ref": "emotionConversion.json"
}
}
}
}
]
}

2
senpy/schemas/emotionPlugin.json
View File

@ -1,7 +1,7 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"$allOf": [
"allOf": [
{
"$ref": "plugin.json"
},

15
senpy/schemas/entry.json
View File

@ -11,23 +11,28 @@
},
"sentiments": {
"type": "array",
"items": {"$ref": "sentiment.json" }
"items": {"$ref": "sentiment.json" },
"default": []
},
"emotions": {
"type": "array",
"items": {"$ref": "emotionSet.json" }
"items": {"$ref": "emotionSet.json" },
"default": []
},
"entities": {
"type": "array",
"items": {"$ref": "entity.json" }
"items": {"$ref": "entity.json" },
"default": []
},
"topics": {
"type": "array",
"items": {"$ref": "topic.json" }
"items": {"$ref": "topic.json" },
"default": []
},
"suggestions": {
"type": "array",
"items": {"$ref": "suggestion.json" }
"items": {"$ref": "suggestion.json" },
"default": []
}
},
"required": ["@id", "nif:isString"]

8
senpy/schemas/error.json
View File

@ -2,7 +2,7 @@
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Base schema for all Senpy objects",
"type": "object",
"$allOf": [
"allOf": [
{"$ref": "atom.json"},
{
"properties": {
@ -10,14 +10,14 @@
"type": "string"
},
"errors": {
"type": "list",
"type": "array",
"items": {"type": "object"}
},
"status": {
"type": "int"
"type": "number"
}
},
"required": ["message"]
}
}
]
}

2
senpy/schemas/sentimentPlugin.json
View File

@ -1,7 +1,7 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"$allOf": [
"allOf": [
{
"$ref": "plugin.json"
},

3
senpy/schemas/suggestion.json
View File

@ -1,4 +1,5 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object"
"type": "object",
"required": ["@id", "prov:wasGeneratedBy"]
}

30
senpy/static/css/main.css
View File

@ -8,7 +8,7 @@ body {
}
#inputswrapper {
min-height:100%;
background: white;
/* background: white; */
position:relative;
min-width: 800px;
height: 100%;
@ -50,25 +50,16 @@ body {
#form {
width: 100%;
}
#results {
.results {
overflow: auto;
padding: 20px;
background: lightgray;
-moz-border-radius: 20px;
-webkit-border-radius: 20px;
-khtml-border-radius: 20px;
border-radius: 20px;
/* padding: 20px; */
background: white;
/* -moz-border-radius: 20px; */
/* -webkit-border-radius: 20px; */
/* -khtml-border-radius: 20px; */
/* border-radius: 20px; */
}
#jsonraw {
overflow: auto;
padding: 20px;
background: lightgray;
-moz-border-radius: 20px;
-webkit-border-radius: 20px;
-khtml-border-radius: 20px;
border-radius: 20px;
}
#input_request {
margin-top: 5px;
display:block;
@ -156,3 +147,8 @@ textarea{
#header {
font-family: 'Architects Daughter', cursive;
}
#results-div {
/* background: white; */
display: none;
}

57
senpy/static/js/main.js
View File

@ -32,39 +32,48 @@ $(document).ready(function() {
var availablePlugins = document.getElementById('availablePlugins');
plugins = response.plugins;
for (r in plugins){
if (plugins[r]["name"]){
if (plugins[r]["name"] == defaultPlugin["name"]){
if (plugins[r]["is_activated"]){
html+= "<option value=\""+plugins[r]["name"]+"\" selected=\"selected\">"+plugins[r]["name"]+"</option>"
plugin = plugins[r]
if (plugin["name"]){
if (plugin["name"] == defaultPlugin["name"]){
if (plugin["is_activated"]){
html+= "<option value=\""+plugin["name"]+"\" selected=\"selected\">"+plugin["name"]+"</option>"
}else{
html+= "<option value=\""+plugins[r]["name"]+"\" selected=\"selected\" disabled=\"disabled\">"+plugins[r]["name"]+"</option>"
html+= "<option value=\""+plugin["name"]+"\" selected=\"selected\" disabled=\"disabled\">"+plugin["name"]+"</option>"
}
}
else{
if (plugins[r]["is_activated"]){
html+= "<option value=\""+plugins[r]["name"]+"\">"+plugins[r]["name"]+"</option>"
if (plugin["is_activated"]){
html+= "<option value=\""+plugin["name"]+"\">"+plugin["name"]+"</option>"
}
else{
html+= "<option value=\""+plugins[r]["name"]+"\" disabled=\"disabled\">"+plugins[r]["name"]+"</option>"
html+= "<option value=\""+plugin["name"]+"\" disabled=\"disabled\">"+plugin["name"]+"</option>"
}
}
}
if (plugins[r]["extra_params"]){
plugins_params[plugins[r]["name"]]={};
for (param in plugins[r]["extra_params"]){
if (typeof plugins[r]["extra_params"][param] !="string"){
if (plugin["extra_params"]){
plugins_params[plugin["name"]]={};
for (param in plugin["extra_params"]){
if (typeof plugin["extra_params"][param] !="string"){
var params = new Array();
var alias = plugins[r]["extra_params"][param]["aliases"][0];
var alias = plugin["extra_params"][param]["aliases"][0];
params[alias]=new Array();
for (option in plugins[r]["extra_params"][param]["options"]){
params[alias].push(plugins[r]["extra_params"][param]["options"][option])
for (option in plugin["extra_params"][param]["options"]){
params[alias].push(plugin["extra_params"][param]["options"][option])
}
plugins_params[plugins[r]["name"]][alias] = (params[alias])
plugins_params[plugin["name"]][alias] = (params[alias])
}
}
}
var pluginList = document.createElement('li');
pluginList.innerHTML = "<a href=https://github.com/gsi-upm/senpy-plugins-community>" + plugins[r]["name"] + "</a>" + ": " + plugins[r]["description"]
newHtml = ""
if(plugin.url) {
newHtml= "<a href="+plugin.url+">" + plugin.name + "</a>";
}else {
newHtml= plugin["name"];
}
newHtml += ": " + replaceURLWithHTMLLinks(plugin.description);
pluginList.innerHTML = newHtml;
availablePlugins.appendChild(pluginList)
}
document.getElementById('plugins').innerHTML = html;
@ -96,6 +105,10 @@ function change_params(){
function load_JSON(){
url = "/api";
var container = document.getElementById('results');
var rawcontainer = document.getElementById("jsonraw");
rawcontainer.innerHTML = '';
container.innerHTML = '';
var plugin = document.getElementById("plugins").options[document.getElementById("plugins").selectedIndex].value;
var input = encodeURIComponent(document.getElementById("input").value);
url += "?algo="+plugin+"&i="+input
@ -108,18 +121,14 @@ function load_JSON(){
}
}
var response = JSON.parse($.ajax({type: "GET", url: url , async: false}).responseText);
var container = document.getElementById('results');
var options = {
mode: 'view'
};
try {
container.removeChild(container.firstChild);
}
catch(err) {
}
var editor = new JSONEditor(container, options, response);
document.getElementById("jsonraw").innerHTML = replaceURLWithHTMLLinks(JSON.stringify(response, undefined, 2))
editor.expandAll();
rawcontainer.innerHTML = replaceURLWithHTMLLinks(JSON.stringify(response, undefined, 2))
document.getElementById("input_request").innerHTML = "<a href='"+url+"'>"+url+"</a>"
document.getElementById("results-div").style.display = 'block';
}

79
senpy/templates/index.html
View File

@ -2,7 +2,7 @@
<html>
<head>
<meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>Playground</title>
<title>Playground {{version}}</title>
</head>
<script src="static/js/jquery-2.1.1.min.js" ></script>
@ -25,49 +25,68 @@
<h3 id="header-title">
<a href="https://github.com/gsi-upm/senpy" target="_blank">
<img id="header-logo" class="imsg-responsive" src="static/img/header.png"/></a> Playground
</h3>
<h4>v{{ version}}</h4>
</div>
<ul class="nav nav-tabs" role="tablist">
<li role="presentation" class="active"><a class="active" href="#about">About</a></li>
<li role="presentation"><a class="active" href="#test">Test it</a></li>
<li role="presentation" ><a class="active" href="#about">About</a></li>
<li role="presentation"class="active"><a class="active" href="#test">Test it</a></li>
</ul>
<div class="tab-content">
<div class="tab-pane active" id="about">
<div class="tab-pane" id="about">
<div class="row">
<div class="col-lg-6 ">
<div class="well">
<h2>Test Senpy</h2>
<div>
<p class="text-center">
<a class="btn btn-lg btn-primary" href="#test" role="button">Test it »</a>
<div class="col-lg-6">
<h2>About Senpy</h2>
<p>Senpy is a framework to build semantic sentiment and emotion analysis services. It does so by using a mix of web and semantic technologies, such as JSON-LD, RDFlib and Flask.</p>
<p>Senpy makes it easy to develop and publish your own analysis algorithms (plugins in senpy terms).
</p>
<p>
This website is the senpy Playground, which allows you to test the instance of senpy in this server. It provides a user-friendly interface to the functions exposed by the senpy API.
</p>
<p>
Once you get comfortable with the parameters and results, you are encouraged to issue your own requests to the API endpoint, which should be <a href="/api">here</a>.
</p>
<p>
These are some of the things you can do with the API:
<ul>
<li>List all available plugins: <a href="/api/plugins">/api/plugins</a></li>
<li>Get information about the default plugin: <a href="/api/plugins/default">/api/plugins/default</a></li>
<li>Download the JSON-LD context used: <a href="/api/contexts/Results.jsonld">/api/contexts/Results.jsonld</a></li>
</ul>
</p>
</div>
</div>
<div class="col-lg-6 ">
<div class="panel panel-default">
<div class="panel-heading"><i class="fa fa-sign-in"></i> Follow us on <a href="http://www.github.com/gsi-upm/senpy">GitHub</a></div>
<div class="panel-heading">
Available Plugins
</div>
<div class="panel-body"><ul id=availablePlugins></ul></div>
</div>
</div>
<div class="col-lg-6 ">
<a href="http://senpy.readthedocs.io">
<div class="panel panel-default">
<div class="panel-heading"><i class="fa fa-book"></i> If you are new to senpy, you might want to read senpy's documentation</div>
</div>
</a>
<a href="http://www.github.com/gsi-upm/senpy">
<div class="panel panel-default">
<div class="panel-heading"><i class="fa fa-sign-in"></i> Feel free to follow us on GitHub</div>
</div>
</a>
<div class="panel panel-default">
<div class="panel-heading"><i class="fa fa-child"></i> Enjoy.</div>
</div>
</div>
<div class="col-lg-6 ">
<div class="well">
<h2>Available Plugins</h2>
<div>
<span><ul id=availablePlugins></ul></span>
</div>
</div>
</div>
</div>
</div>
<div class="tab-pane" id="test">
<div class="tab-pane active" id="test">
<div class="well">
<form id="form" onsubmit="return getPlugins();" accept-charset="utf-8">
<div id="inputswrapper">
@ -81,26 +100,28 @@ I cannot believe it!</textarea></div>
<div id ="params">
</div>
</br>
<a id="preview" class="btn btn-lg btn-primary" href="#" onclick="load_JSON()">Analyse!</a>
<a id="preview" class="btn btn-lg btn-primary" onclick="load_JSON()">Analyse!</a>
<!--<button id="visualise" name="type" type="button">Visualise!</button>-->
</div>
</form>
</div>
<span id="input_request"></span>
<div id="results-div">
<ul class="nav nav-tabs" role="tablist">
<li role="presentation" class="active"><a class="active" href="#viewer">Viewer</a></li>
<li role="presentation"><a class="active" href="#raw">Raw</a></li>
</ul>
<div class="tab-content">
<div class="tab-content" id="results-container">
<div class="tab-pane active" id="viewer">
<div id="content">
<pre id="results"></pre>
<pre id="results" class="results"></pre>
</div>
</div>
<div class="tab-pane" id="raw">
<div id="content">
<pre id="jsonraw"></pre>
</div>
<pre id="jsonraw" class="results"></pre>
</div>
</div>
</div>

21
senpy/version.py
View File

@ -1,5 +1,4 @@
import os
import subprocess
import logging
logger = logging.getLogger(__name__)
@ -8,27 +7,9 @@ ROOT = os.path.dirname(__file__)
DEFAULT_FILE = os.path.join(ROOT, 'VERSION')
def git_version():
try:
res = subprocess.check_output(['git', 'describe',
'--tags', '--dirty']).decode('utf-8')
return res.strip()
except subprocess.CalledProcessError:
return None
def read_version(versionfile=DEFAULT_FILE):
with open(versionfile) as f:
return f.read().strip()
def write_version(version, versionfile=DEFAULT_FILE):
version = version or git_version()
if not version:
raise ValueError('You need to provide a valid version')
with open(versionfile, 'w') as f:
f.write(version)
__version__ = git_version() or read_version()
write_version(__version__)
__version__ = read_version()

3
setup.cfg
View File

@ -7,3 +7,6 @@ test=pytest
# finishing the imports. flake8 thinks that we're doing the imports too late,
# but it's actually ok
ignore = E402
max-line-length = 100
[bdist_wheel]
universal=1

4
setup.py
View File

@ -1,7 +1,8 @@
import pip
from setuptools import setup
from pip.req import parse_requirements
# parse_requirements() returns generator of pip.req.InstallRequirement objects
from pip.req import parse_requirements
from senpy import __version__
try:
install_reqs = parse_requirements(
@ -15,7 +16,6 @@ except AttributeError:
install_reqs = [str(ir.req) for ir in install_reqs]
test_reqs = [str(ir.req) for ir in test_reqs]
from senpy import __version__
setup(
name='senpy',

7
tests/dummy_plugin/dummy.py
View File

@ -1,7 +0,0 @@
from senpy.plugins import SentimentPlugin
from senpy.models import Results
class DummyPlugin(SentimentPlugin):
def analyse(self, *args, **kwargs):
return Results()

7
tests/dummy_plugin/dummy.senpy
View File

@ -1,7 +0,0 @@
{
"name": "Dummy",
"module": "dummy",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1"
}

7
tests/plugins/dummy_plugin/dummy.py Normal file
View File

@ -0,0 +1,7 @@
from senpy.plugins import SentimentPlugin
class DummyPlugin(SentimentPlugin):
def analyse_entry(self, entry, params):
entry.text = entry.text[::-1]
yield entry

15
tests/plugins/dummy_plugin/dummy.senpy Normal file
View File

@ -0,0 +1,15 @@
{
"name": "Dummy",
"module": "dummy",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"extra_params": {
"example": {
"@id": "example_parameter",
"aliases": ["example", "ex"],
"required": false,
"default": 0
}
}
}

14
tests/plugins/dummy_plugin/dummy_required.senpy Normal file
View File

@ -0,0 +1,14 @@
{
"name": "DummyRequired",
"module": "dummy",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"extra_params": {
"example": {
"@id": "example_parameter",
"aliases": ["example", "ex"],
"required": true
}
}
}

7
tests/sleep_plugin/sleep.py → tests/plugins/sleep_plugin/sleep.py
View File

@ -1,5 +1,4 @@
from senpy.plugins import SenpyPlugin
from senpy.models import Results
from time import sleep
@ -7,6 +6,6 @@ class SleepPlugin(SenpyPlugin):
def activate(self, *args, **kwargs):
sleep(self.timeout)
def analyse(self, *args, **kwargs):
sleep(float(kwargs.get("timeout", self.timeout)))
return Results()
def analyse_entry(self, entry, params):
sleep(float(params.get("timeout", self.timeout)))
yield entry

0
tests/sleep_plugin/sleep.senpy → tests/plugins/sleep_plugin/sleep.senpy
View File

34
tests/test_blueprints.py
View File

@ -25,6 +25,8 @@ class BlueprintsTest(TestCase):
self.dir = os.path.join(os.path.dirname(__file__), "..")
self.senpy.add_folder(self.dir)
self.senpy.activate_plugin("Dummy", sync=True)
self.senpy.activate_plugin("DummyRequired", sync=True)
self.senpy.default_plugin = 'Dummy'
def assertCode(self, resp, code):
self.assertEqual(resp.status_code, code)
@ -34,12 +36,12 @@ class BlueprintsTest(TestCase):
Calling with no arguments should ask the user for more arguments
"""
resp = self.client.get("/api/")
self.assertCode(resp, 404)
self.assertCode(resp, 400)
js = parse_resp(resp)
logging.debug(js)
assert js["status"] == 404
assert js["status"] == 400
atleast = {
"status": 404,
"status": 400,
"message": "Missing or invalid parameters",
}
assert check_dict(js, atleast)
@ -56,6 +58,28 @@ class BlueprintsTest(TestCase):
assert "@context" in js
assert "entries" in js
def test_analysis_extra(self):
"""
Extra params that have a default should
"""
resp = self.client.get("/api/?i=My aloha mohame&algo=Dummy")
self.assertCode(resp, 200)
js = parse_resp(resp)
logging.debug("Got response: %s", js)
assert "@context" in js
assert "entries" in js
def test_analysis_extra_required(self):
"""
Extra params that have a required argument that does not
have a default should raise an error.
"""
resp = self.client.get("/api/?i=My aloha mohame&algo=DummyRequired")
self.assertCode(resp, 400)
js = parse_resp(resp)
logging.debug("Got response: %s", js)
assert isinstance(js, models.Error)
def test_error(self):
"""
The dummy plugin returns an empty response,\
@ -102,7 +126,7 @@ class BlueprintsTest(TestCase):
js = parse_resp(resp)
logging.debug(js)
assert "@id" in js
assert js["@id"] == "Dummy_0.1"
assert js["@id"] == "plugins/Dummy_0.1"
def test_default(self):
""" Show only one plugin"""
@ -111,7 +135,7 @@ class BlueprintsTest(TestCase):
js = parse_resp(resp)
logging.debug(js)
assert "@id" in js
assert js["@id"] == "Dummy_0.1"
assert js["@id"] == "plugins/Dummy_0.1"
def test_context(self):
resp = self.client.get("/api/contexts/context.jsonld")

11
tests/test_client.py
View File

@ -9,9 +9,10 @@ from senpy.models import Results, Error
class Call(dict):
def __init__(self, obj):
self.obj = obj.jsonld()
self.status_code = 200
self.content = self.json()
def json(self):
return self.obj
@ -29,14 +30,14 @@ class ModelsTest(TestCase):
with patch('requests.request', return_value=success) as patched:
resp = client.analyse('hello')
assert isinstance(resp, Results)
patched.assert_called_with(url=endpoint + '/',
method='GET',
params={'input': 'hello'})
patched.assert_called_with(
url=endpoint + '/', method='GET', params={'input': 'hello'})
error = Call(Error('Nothing'))
with patch('requests.request', return_value=error) as patched:
resp = client.analyse(input='hello', algorithm='NONEXISTENT')
assert isinstance(resp, Error)
patched.assert_called_with(url=endpoint + '/',
patched.assert_called_with(
url=endpoint + '/',
method='GET',
params={'input': 'hello',
'algorithm': 'NONEXISTENT'})

25
tests/test_extensions.py
View File

@ -16,7 +16,7 @@ from unittest import TestCase
class ExtensionsTest(TestCase):
def setUp(self):
self.app = Flask("test_extensions")
self.app = Flask('test_extensions')
self.dir = os.path.join(os.path.dirname(__file__))
self.senpy = Senpy(plugin_folder=self.dir,
app=self.app,
@ -45,7 +45,7 @@ class ExtensionsTest(TestCase):
'requirements': ['noop'],
'version': 0
}
root = os.path.join(self.dir, 'dummy_plugin')
root = os.path.join(self.dir, 'plugins', 'dummy_plugin')
name, module = self.senpy._load_plugin_from_info(info, root=root)
assert name == 'TestPip'
assert module
@ -55,7 +55,7 @@ class ExtensionsTest(TestCase):
def test_installing(self):
""" Enabling a plugin """
self.senpy.activate_all(sync=True)
assert len(self.senpy.plugins) == 2
assert len(self.senpy.plugins) >= 3
assert self.senpy.plugins["Sleep"].is_activated
def test_disabling(self):
@ -75,9 +75,10 @@ class ExtensionsTest(TestCase):
def test_noplugin(self):
""" Don't analyse if there isn't any plugin installed """
self.senpy.deactivate_all(sync=True)
self.assertRaises(Error, partial(self.senpy.analyse,
input="tupni"))
self.assertRaises(Error, partial(self.senpy.analyse,
self.assertRaises(Error, partial(self.senpy.analyse, input="tupni"))
self.assertRaises(Error,
partial(
self.senpy.analyse,
input="tupni",
algorithm='Dummy'))
@ -88,17 +89,20 @@ class ExtensionsTest(TestCase):
r1 = self.senpy.analyse(
algorithm="Dummy", input="tupni", output="tuptuo")
r2 = self.senpy.analyse(input="tupni", output="tuptuo")
assert r1.analysis[0].id[:5] == "Dummy"
assert r2.analysis[0].id[:5] == "Dummy"
assert r1.analysis[0] == "plugins/Dummy_0.1"
assert r2.analysis[0] == "plugins/Dummy_0.1"
assert r1.entries[0].text == 'input'
def test_analyse_error(self):
mm = mock.MagicMock()
mm.analyse.side_effect = Error('error on analysis', status=900)
mm.analyse_entry.side_effect = Error('error on analysis', status=900)
self.senpy.plugins['MOCK'] = mm
resp = self.senpy.analyse(input='nothing', algorithm='MOCK')
assert resp['message'] == 'error on analysis'
assert resp['status'] == 900
mm.analyse.side_effect = Exception('generic exception on analysis')
mm.analyse_entry.side_effect = Exception(
'generic exception on analysis')
resp = self.senpy.analyse(input='nothing', algorithm='MOCK')
assert resp['message'] == 'generic exception on analysis'
assert resp['status'] == 500
@ -110,8 +114,7 @@ class ExtensionsTest(TestCase):
assert self.senpy.filter_plugins(name="Dummy", is_activated=True)
self.senpy.deactivate_plugin("Dummy", sync=True)
assert not len(
self.senpy.filter_plugins(
name="Dummy", is_activated=True))
self.senpy.filter_plugins(name="Dummy", is_activated=True))
def test_load_default_plugins(self):
senpy = Senpy(plugin_folder=self.dir, default_plugins=True)

56
tests/test_models.py
View File

@ -3,20 +3,25 @@ import logging
import jsonschema
import json
import rdflib
from unittest import TestCase
from senpy.models import Entry, Results, Sentiment, EmotionSet, Error
from senpy.models import (Emotion,
EmotionAnalysis,
EmotionSet,
Entry,
Error,
Results,
Sentiment)
from senpy.plugins import SenpyPlugin
from pprint import pprint
class ModelsTest(TestCase):
def test_jsonld(self):
prueba = {"id": "test",
"analysis": [],
"entries": []}
prueba = {"id": "test", "analysis": [], "entries": []}
r = Results(**prueba)
print("Response's context: ")
pprint(r.context)
pprint(r._context)
assert r.id == "test"
@ -30,14 +35,11 @@ class ModelsTest(TestCase):
assert "id" not in j
r6 = Results(**prueba)
e = Entry({
"@id": "ohno",
"nif:isString": "Just testing"
})
e = Entry({"@id": "ohno", "nif:isString": "Just testing"})
r6.entries.append(e)
logging.debug("Reponse 6: %s", r6)
assert ("marl" in r6.context)
assert ("entries" in r6.context)
assert ("marl" in r6._context)
assert ("entries" in r6._context)
j6 = r6.jsonld(with_context=True)
logging.debug("jsonld: %s", j6)
assert ("@context" in j6)
@ -113,5 +115,35 @@ class ModelsTest(TestCase):
s = str(r)
assert "_testing" not in s
def test_frame_response(self):
def test_turtle(self):
"""Any model should be serializable as a turtle file"""
ana = EmotionAnalysis()
res = Results()
res.analysis.append(ana)
entry = Entry(text='Just testing')
eSet = EmotionSet()
emotion = Emotion()
entry.emotions.append(eSet)
res.entries.append(entry)
eSet.onyx__hasEmotion.append(emotion)
eSet.prov__wasGeneratedBy = ana.id
triples = ('ana a :Analysis',
'entry a :entry',
' nif:isString "Just testing"',
' onyx:hasEmotionSet eSet',
'eSet a onyx:EmotionSet',
' prov:wasGeneratedBy ana',
' onyx:hasEmotion emotion',
'emotion a onyx:Emotion',
'res a :results',
' me:AnalysisInvoloved ana',
' prov:used entry')
t = res.serialize(format='turtle')
print(t)
g = rdflib.Graph().parse(data=t, format='turtle')
assert len(g) == len(triples)
def test_convert_emotions(self):
"""It should be possible to convert between different emotion models"""
pass

17
tests/test_plugins.py
View File

@ -77,6 +77,7 @@ class PluginsTest(TestCase):
})
a.activate()
assert a.shelf_file == self.shelf_file
res1 = a.analyse(input=1)
assert res1.entries[0].nif__isString == 1
res2 = a.analyse(input=1)
@ -103,3 +104,19 @@ class PluginsTest(TestCase):
assert b.sh['a'] == 'fromA'
b.sh['a'] = 'fromB'
assert b.sh['a'] == 'fromB'
def test_extra_params(self):
''' Should be able to set extra parameters'''
a = ShelfDummyPlugin(info={
'name': 'shelve',
'version': 'test',
'shelf_file': self.shelf_file,
'extra_params': {
'example': {
'aliases': ['example', 'ex'],
'required': True,
'default': 'nonsense'
}
}
})
assert 'example' in a.extra_params

2
version.sh Executable file
View File

@ -0,0 +1,2 @@
#!/bin/sh
VERSION=$(git describe --long --tags --dirty)