Browse Source

Multiple changes in the API, schemas and UI

Check out the CHANGELOG.md file for more information
tags/0.20
J. Fernando Sánchez 7 months ago
parent
commit
8a516d927e
100 changed files with 9432 additions and 46051 deletions
  1. 1
    0
      .gitlab-ci.yml
  2. 1
    1
      .makefiles/python.mk
  3. 53
    0
      CHANGELOG.md
  4. 1
    1
      Makefile
  5. 1
    1
      Procfile
  6. 23
    4
      README.rst
  7. 0
    4
      config.py
  8. 4
    0
      docs/Makefile
  9. 4659
    0
      docs/Quickstart.ipynb
  10. 2599
    0
      docs/Quickstart.rst
  11. 0
    317
      docs/SenpyClientUse.ipynb
  12. 0
    106
      docs/SenpyClientUse.rst
  13. 0
    11
      docs/about.rst
  14. 9
    0
      docs/advanced.rst
  15. 4
    13
      docs/api.rst
  16. 3
    2
      docs/apischema.rst
  17. 1
    1
      docs/bad-examples/results/example-analysis-as-id-FAIL.json
  18. 1
    1
      docs/bad-examples/results/example-basic-FAIL.json
  19. 2
    1
      docs/commandline.rst
  20. 6
    6
      docs/conversion.rst
  21. 2
    4
      docs/demo.rst
  22. 27
    0
      docs/development.rst
  23. BIN
      docs/evaluation-results.png
  24. 3
    1
      docs/examples.rst
  25. 1
    1
      docs/examples/results/example-analysis-as-id.json
  26. 1
    1
      docs/examples/results/example-basic.json
  27. 1
    1
      docs/examples/results/example-complete.json
  28. 1
    1
      docs/examples/results/example-emotion.json
  29. 1
    1
      docs/examples/results/example-ner.json
  30. 1
    1
      docs/examples/results/example-pad.json
  31. 1
    1
      docs/examples/results/example-sentiment.json
  32. 1
    1
      docs/examples/results/example-suggestion.json
  33. 14
    15
      docs/index.rst
  34. 7
    9
      docs/installation.rst
  35. 1
    1
      docs/plugins-definition.rst
  36. 22
    79
      docs/plugins-faq.rst
  37. 86
    0
      docs/plugins-quickstart.rst
  38. 46
    0
      docs/publications.rst
  39. 15
    42
      docs/senpy.rst
  40. 18
    12
      docs/server-cli.rst
  41. 0
    15
      docs/usage.rst
  42. 2
    2
      example-plugins/README.md
  43. 5
    3
      example-plugins/basic.py
  44. 3
    3
      example-plugins/basic_analyse_entry_plugin.py
  45. 8
    10
      example-plugins/basic_box_plugin.py
  46. 14
    15
      example-plugins/basic_plugin.py
  47. 2
    2
      example-plugins/configurable_plugin.py
  48. 4
    3
      example-plugins/emorand_plugin.py
  49. 4
    3
      example-plugins/parameterized_plugin.py
  50. 6
    4
      example-plugins/rand_plugin.py
  51. 8
    13
      example-plugins/sklearn/pipeline_plugin.py
  52. 1
    1
      extra-requirements.txt
  53. 0
    2
      k8s/senpy-deployment.yaml
  54. 7
    0
      k8s/senpy-ingress.yaml
  55. 2
    1
      requirements.txt
  56. 49
    28
      senpy/__main__.py
  57. 115
    22
      senpy/api.py
  58. 83
    56
      senpy/blueprints.py
  59. 5
    3
      senpy/cli.py
  60. 83
    66
      senpy/extensions.py
  61. 7
    7
      senpy/gsitk_compat.py
  62. 54
    25
      senpy/meta.py
  63. 188
    73
      senpy/models.py
  64. 283
    115
      senpy/plugins/__init__.py
  65. 0
    34
      senpy/plugins/example/emoRand/emoRand.py
  66. 13
    9
      senpy/plugins/misc/split_plugin.py
  67. 3
    1
      senpy/plugins/postprocessing/emotion/centroids.py
  68. 1
    1
      senpy/plugins/postprocessing/emotion/ekman2vad.senpy
  69. 9
    14
      senpy/plugins/postprocessing/emotion/maxEmotion_plugin.py
  70. 25
    29
      senpy/plugins/sentiment/sentiment140/sentiment140_plugin.py
  71. 0
    2
      senpy/schemas/aggregatedEvaluation.json
  72. 0
    1
      senpy/schemas/analysis.json
  73. 13
    6
      senpy/schemas/context.jsonld
  74. 0
    1
      senpy/schemas/datasets.json
  75. 1
    2
      senpy/schemas/emotionModel.json
  76. 1
    2
      senpy/schemas/emotionSet.json
  77. 4
    21
      senpy/schemas/entry.json
  78. 1
    2
      senpy/schemas/evaluation.json
  79. 1
    2
      senpy/schemas/plugin.json
  80. 8
    2
      senpy/schemas/plugins.json
  81. 3
    7
      senpy/schemas/results.json
  82. 0
    347
      senpy/static/css/bootstrap-theme.css
  83. 0
    1
      senpy/static/css/bootstrap-theme.css.map
  84. 0
    7
      senpy/static/css/bootstrap-theme.min.css
  85. 0
    5785
      senpy/static/css/bootstrap.css
  86. 0
    1
      senpy/static/css/bootstrap.css.map
  87. 5
    5
      senpy/static/css/bootstrap.min.css
  88. 1
    0
      senpy/static/css/bootstrap.min.css.map
  89. 61
    2
      senpy/static/css/main.css
  90. BIN
      senpy/static/img/ribbon.png
  91. 0
    1951
      senpy/static/js/bootstrap.js
  92. 6
    5
      senpy/static/js/bootstrap.min.js
  93. 1
    0
      senpy/static/js/bootstrap.min.js.map
  94. 5
    0
      senpy/static/js/d3.min.js
  95. 0
    36377
      senpy/static/js/jsoneditor.js
  96. 211
    94
      senpy/static/js/main.js
  97. 223
    0
      senpy/static/js/nodes.js
  98. 3
    0
      senpy/static/js/showdown.min.js
  99. 283
    228
      senpy/templates/index.html
  100. 0
    0
      senpy/utils.py

+ 1
- 0
.gitlab-ci.yml View File

@@ -28,6 +28,7 @@ test-3.5:

test-2.7:
<<: *test_definition
allow_failure: true
variables:
PYTHON_VERSION: "2.7"


+ 1
- 1
.makefiles/python.mk View File

@@ -29,7 +29,7 @@ build: $(addprefix build-, $(PYVERSIONS)) ## Build all images / python versions
docker tag $(IMAGEWTAG)-python$(PYMAIN) $(IMAGEWTAG)

build-%: version Dockerfile-% ## Build a specific version (e.g. build-2.7)
docker build -t '$(IMAGEWTAG)-python$*' -f Dockerfile-$* .;
docker build --pull -t '$(IMAGEWTAG)-python$*' -f Dockerfile-$* .;

dev-%: ## Launch a specific development environment using docker (e.g. dev-2.7)
@docker start $(NAME)-dev$* || (\

+ 53
- 0
CHANGELOG.md View File

@@ -0,0 +1,53 @@
# Changelog
All notable changes to this project will be documented in this file.

The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased]

### Added
* Objects can control the keys that will be used in `serialize`/`jsonld`/`as_dict` by specifying a list of keys in `terse_keys`.
e.g.
```python
>>> class MyModel(senpy.models.BaseModel):
... _terse_keys = ['visible']
... invisible = 5
... visible = 1
...
>>> m = MyModel(id='testing')
>>> m.jsonld()
{'invisible': 5, 'visible': 1, '@id': 'testing'}
>>> m.jsonld(verbose=False)
{'visible': 1}
```
* Configurable logging format.
* Added default terse keys for the most common classes (entry, sentiment, emotion...).
* Flag parameters (boolean) are set to true even when no value is added (e.g. `&verbose` is the same as `&verbose=true`).
* Plugin and parameter descriptions are now formatted with (showdown)[https://github.com/showdownjs/showdown].
* The web UI requests extra_parameters from the server. This is useful for pipelines. See #52
* First batch of semantic tests (using SPARQL)

### Changed
* `install_deps` now checks what requirements are already met before installing with pip.
* Help is now provided verbosely by default
* Other outputs are terse by default. This means some properties are now hidden unless verbose is set.
* `sentiments` and `emotions` are now `marl:hasOpinion` and `onyx:hasEmotionSet`, respectively.
* Nicer logging format
* Context aliases (e.g. `sentiments` and `emotions` properties) have been replaced with the original properties (e.g. `marl:hasOpinion` and `onyx:hasEmotionSet**), to use aliases, pass the `aliases** parameter.
* Several UI improvements
* Dedicated tab to show the list of plugins
* URLs in plugin descriptions are shown as links
* The format of the response is selected by clicking on a tab instead of selecting from a drop-down
* list of examples
* Bootstrap v4
* RandEmotion and RandSentiment are no longer included in the base set of plugins
* The `--plugin-folder` option can be used more than once, and every folder will be added to the app.

### Deprecated
### Removed
* Python 2.7 is no longer test or officially supported
### Fixed
* Plugin descriptions are now dedented when they are extracted from the docstring.
### Security


+ 1
- 1
Makefile View File

@@ -5,7 +5,7 @@ IMAGENAME=gsiupm/senpy

# The first version is the main one (used for quick builds)
# See .makefiles/python.mk for more info
PYVERSIONS=3.5 2.7
PYVERSIONS=3.6 3.7

DEVPORT=5000


+ 1
- 1
Procfile View File

@@ -1 +1 @@
web: python -m senpy --host 0.0.0.0 --port $PORT --default-plugins
web: python -m senpy --host 0.0.0.0 --port $PORT

+ 23
- 4
README.rst View File

@@ -1,10 +1,19 @@
.. image:: img/header.png
:width: 100%
:target: http://demos.gsi.dit.upm.es/senpy
:target: http://senpy.gsi.upm.es

.. image:: https://travis-ci.org/gsi-upm/senpy.svg?branch=master
:target: https://travis-ci.org/gsi-upm/senpy

.. image:: https://lab.gsi.upm.es/senpy/senpy/badges/master/pipeline.svg
:target: https://lab.gsi.upm.es/senpy/senpy/commits/master

.. image:: https://lab.gsi.upm.es/senpy/senpy/badges/master/coverage.svg
:target: https://lab.gsi.upm.es/senpy/senpy/commits/master

.. image:: https://img.shields.io/pypi/l/requests.svg
:target: https://lab.gsi.upm.es/senpy/senpy/

Senpy lets you create sentiment analysis web services easily, fast and using a well known API.
As a bonus, senpy services use semantic vocabularies (e.g. `NIF <http://persistence.uni-leipzig.org/nlp2rdf/>`_, `Marl <http://www.gsi.dit.upm.es/ontologies/marl>`_, `Onyx <http://www.gsi.dit.upm.es/ontologies/onyx>`_) and formats (turtle, JSON-LD, xml-rdf).

@@ -12,7 +21,7 @@ Have you ever wanted to turn your sentiment analysis algorithms into a service?
With senpy, now you can.
It provides all the tools so you just have to worry about improving your algorithms:

`See it in action. <http://senpy.cluster.gsi.dit.upm.es/>`_
`See it in action. <http://senpy.gsi.upm.es/>`_

Installation
------------
@@ -38,9 +47,9 @@ If you want to install senpy globally, use sudo instead of the ``--user`` flag.

Docker Image
************
Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/senpy --default-plugins``.
Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/senpy``.

To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --default-plugins -f /plugins``
To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy -f /plugins``


Developing
@@ -125,6 +134,16 @@ For more information, check out the `documentation <http://senpy.readthedocs.org
------------------------------------------------------------------------------------


Python 2.x compatibility
------------------------

Keeping compatibility between python 2.7 and 3.x is not always easy, especially for a framework that deals both with text and web requests.
Hence, starting February 2019, this project will no longer make efforts to support python 2.7, which will reach its end of life in 2020.
Most of the functionality should still work, and the compatibility shims will remain for now, but we cannot make any guarantees at this point.
Instead, the maintainers will focus their efforts on keeping the codebase compatible across different Python 3.3+ versions, including upcoming ones.
We apologize for the inconvenience.


Acknowledgement
---------------
This development has been partially funded by the European Union through the MixedEmotions Project (project number H2020 655632), as part of the `RIA ICT 15 Big data and Open Data Innovation and take-up` programme.

+ 0
- 4
config.py View File

@@ -1,4 +0,0 @@
import os

SERVER_PORT = os.environ.get("SERVER_PORT", 5000)
DEBUG = os.environ.get("DEBUG", True)

+ 4
- 0
docs/Makefile View File

@@ -24,6 +24,7 @@ I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
help:
@echo "Please use \`make <target>' where <target> is one of"
@echo " html to make standalone HTML files"
@echo " entr to watch for changes and continuously make HTML files"
@echo " dirhtml to make HTML files named index.html in directories"
@echo " singlehtml to make a single large HTML file"
@echo " pickle to make pickle files"
@@ -49,6 +50,9 @@ help:
clean:
rm -rf $(BUILDDIR)/*

entr:
while true; do ag -g rst | entr -d make html; done

html:
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
@echo

+ 4659
- 0
docs/Quickstart.ipynb
File diff suppressed because it is too large
View File


+ 2599
- 0
docs/Quickstart.rst
File diff suppressed because it is too large
View File


+ 0
- 317
docs/SenpyClientUse.ipynb View File

@@ -1,317 +0,0 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"ExecuteTime": {
"end_time": "2017-04-10T17:05:31.465571Z",
"start_time": "2017-04-10T19:05:31.458282+02:00"
},
"deletable": true,
"editable": true
},
"source": [
"# Client"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true,
"deletable": true,
"editable": true
},
"source": [
"The built-in senpy client allows you to query any Senpy endpoint. We will illustrate how to use it with the public demo endpoint, and then show you how to spin up your own endpoint using docker."
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"Demo Endpoint\n",
"-------------"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"To start using senpy, simply create a new Client and point it to your endpoint. In this case, the latest version of Senpy at GSI."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2017-04-10T17:29:12.827640Z",
"start_time": "2017-04-10T19:29:12.818617+02:00"
},
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"from senpy.client import Client\n",
"\n",
"c = Client('http://latest.senpy.cluster.gsi.dit.upm.es/api')\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"Now, let's use that client analyse some queries:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2017-04-10T17:29:14.011657Z",
"start_time": "2017-04-10T19:29:13.701808+02:00"
},
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"r = c.analyse('I like sugar!!', algorithm='sentiment140')\n",
"r"
]
},
{
"cell_type": "markdown",
"metadata": {
"ExecuteTime": {
"end_time": "2017-04-10T17:08:19.616754Z",
"start_time": "2017-04-10T19:08:19.610767+02:00"
},
"deletable": true,
"editable": true
},
"source": [
"As you can see, that gave us the full JSON result. A more concise way to print it would be:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2017-04-10T17:29:14.854213Z",
"start_time": "2017-04-10T19:29:14.842068+02:00"
},
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"for entry in r.entries:\n",
" print('{} -> {}'.format(entry['text'], entry['sentiments'][0]['marl:hasPolarity']))"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"We can also obtain a list of available plugins with the client:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2017-04-10T17:29:16.245198Z",
"start_time": "2017-04-10T19:29:16.056545+02:00"
},
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"c.plugins()"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"Or, more concisely:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2017-04-10T17:29:17.663275Z",
"start_time": "2017-04-10T19:29:17.484623+02:00"
},
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"c.plugins().keys()"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"Local Endpoint\n",
"--------------"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"To run your own instance of senpy, just create a docker container with the latest Senpy image. Using `--default-plugins` you will get some extra plugins to start playing with the API."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2017-04-10T17:29:20.637539Z",
"start_time": "2017-04-10T19:29:19.938322+02:00"
},
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"!docker run -ti --name 'SenpyEndpoint' -d -p 6000:5000 gsiupm/senpy:0.8.6 --host 0.0.0.0 --default-plugins"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"To use this endpoint:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2017-04-10T17:29:21.263976Z",
"start_time": "2017-04-10T19:29:21.260595+02:00"
},
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"c_local = Client('http://127.0.0.1:6000/api')"
]
},
{
"cell_type": "markdown",
"metadata": {
"deletable": true,
"editable": true
},
"source": [
"That's all! After you are done with your analysis, stop the docker container:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2017-04-10T17:29:33.226686Z",
"start_time": "2017-04-10T19:29:22.392121+02:00"
},
"collapsed": false,
"deletable": true,
"editable": true
},
"outputs": [],
"source": [
"!docker stop SenpyEndpoint\n",
"!docker rm SenpyEndpoint"
]
}
],
"metadata": {
"anaconda-cloud": {},
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.0"
},
"toc": {
"colors": {
"hover_highlight": "#DAA520",
"running_highlight": "#FF0000",
"selected_highlight": "#FFD700"
},
"moveMenuLeft": true,
"nav_menu": {
"height": "68px",
"width": "252px"
},
"navigate_menu": true,
"number_sections": true,
"sideBar": true,
"threshold": 4,
"toc_cell": false,
"toc_section_display": "block",
"toc_window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 1
}

+ 0
- 106
docs/SenpyClientUse.rst View File

@@ -1,106 +0,0 @@

Client
======

Demo Endpoint
-------------

Import Client and send a request

.. code:: python

from senpy.client import Client
c = Client('http://latest.senpy.cluster.gsi.dit.upm.es/api')
r = c.analyse('I like Pizza', algorithm='sentiment140')

Print response

.. code:: python

for entry in r.entries:
print('{} -> {}'.format(entry['text'], entry['sentiments'][0]['marl:hasPolarity']))


.. parsed-literal::

I like Pizza -> marl:Positive


Obtain a list of available plugins

.. code:: python

for plugin in c.request('/plugins')['plugins']:
print(plugin['name'])


.. parsed-literal::

emoRand
rand
sentiment140


Local Endpoint
--------------

Run a docker container with Senpy image and default plugins

.. code::

docker run -ti --name 'SenpyEndpoint' -d -p 5000:5000 gsiupm/senpy:0.8.6 --host 0.0.0.0 --default-plugins


.. parsed-literal::

a0157cd98057072388bfebeed78a830da7cf0a796f4f1a3fd9188f9f2e5fe562


Import client and send a request to localhost

.. code:: python

c_local = Client('http://127.0.0.1:5000/api')
r = c_local.analyse('Hello world', algorithm='sentiment140')

Print response

.. code:: python

for entry in r.entries:
print('{} -> {}'.format(entry['text'], entry['sentiments'][0]['marl:hasPolarity']))


.. parsed-literal::

Hello world -> marl:Neutral


Obtain a list of available plugins deployed locally

.. code:: python

c_local.plugins().keys()


.. parsed-literal::

rand
sentiment140
emoRand


Stop the docker container

.. code:: python

!docker stop SenpyEndpoint
!docker rm SenpyEndpoint


.. parsed-literal::

SenpyEndpoint
SenpyEndpoint


+ 0
- 11
docs/about.rst View File

@@ -1,11 +0,0 @@
About
--------

If you use Senpy in your research, please cite `Senpy: A Pragmatic Linked Sentiment Analysis Framework <http://gsi.dit.upm.es/index.php/es/investigacion/publicaciones?view=publication&task=show&id=417>`__ (`BibTex <http://gsi.dit.upm.es/index.php/es/investigacion/publicaciones?controller=publications&task=export&format=bibtex&id=417>`__):

.. code-block:: text

Sánchez-Rada, J. F., Iglesias, C. A., Corcuera, I., & Araque, Ó. (2016, October).
Senpy: A Pragmatic Linked Sentiment Analysis Framework.
In Data Science and Advanced Analytics (DSAA),
2016 IEEE International Conference on (pp. 735-742). IEEE.

+ 9
- 0
docs/advanced.rst View File

@@ -0,0 +1,9 @@
Advanced usage
--------------

.. toctree::
:maxdepth: 1

server-cli
conversion
commandline

+ 4
- 13
docs/api.rst View File

@@ -25,7 +25,7 @@ NIF API
"@context":"http://127.0.0.1/api/contexts/Results.jsonld",
"@id":"_:Results_11241245.22",
"@type":"results"
"analysis": [
"activities": [
"plugins/sentiment-140_0.1"
],
"entries": [
@@ -73,7 +73,7 @@ NIF API
.. http:get:: /api/plugins

Returns a list of installed plugins.
**Example request**:
**Example request and response**:

.. sourcecode:: http

@@ -82,10 +82,6 @@ NIF API
Accept: application/json, text/javascript


**Example response**:

.. sourcecode:: http

{
"@id": "plugins/sentiment-140_0.1",
"@type": "sentimentPlugin",
@@ -143,19 +139,14 @@ NIF API
.. http:get:: /api/plugins/<pluginname>

Returns the information of a specific plugin.
**Example request**:
**Example request and response**:

.. sourcecode:: http

GET /api/plugins/rand/ HTTP/1.1
GET /api/plugins/sentiment-random/ HTTP/1.1
Host: localhost
Accept: application/json, text/javascript


**Example response**:

.. sourcecode:: http

{
"@context": "http://127.0.0.1/api/contexts/ExamplePlugin.jsonld",
"@id": "plugins/ExamplePlugin_0.1",

+ 3
- 2
docs/apischema.rst View File

@@ -1,5 +1,6 @@
API and Examples
################
API and vocabularies
####################

.. toctree::

vocabularies.rst

+ 1
- 1
docs/bad-examples/results/example-analysis-as-id-FAIL.json View File

@@ -2,7 +2,7 @@
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "me:Result1",
"@type": "results",
"analysis": [
"activities": [
"me:SAnalysis1",
"me:SgAnalysis1",
"me:EmotionAnalysis1",

+ 1
- 1
docs/bad-examples/results/example-basic-FAIL.json View File

@@ -2,7 +2,7 @@
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "http://example.com#NIFExample",
"@type": "results",
"analysis": [
"activities": [
],
"entries": [
{

+ 2
- 1
docs/commandline.rst View File

@@ -1,7 +1,8 @@
Command line
============

This video shows how to analyse text directly on the command line using the senpy tool.
Although the main use of senpy is to publish services, the tool can also be used locally to analyze text in the command line.
This is a short video demonstration:

.. image:: https://asciinema.org/a/9uwef1ghkjk062cw2t4mhzpyk.png
:width: 100%

+ 6
- 6
docs/conversion.rst View File

@@ -7,9 +7,9 @@ Senpy includes experimental support for emotion/sentiment conversion plugins.
Use
===

Consider the original query: http://127.0.0.1:5000/api/?i=hello&algo=emoRand
Consider the original query: http://127.0.0.1:5000/api/?i=hello&algo=emotion-random

The requested plugin (emoRand) returns emotions using Ekman's model (or big6 in EmotionML):
The requested plugin (emotion-random) returns emotions using Ekman's model (or big6 in EmotionML):

.. code:: json

@@ -21,14 +21,14 @@ The requested plugin (emoRand) returns emotions using Ekman's model (or big6 in
"@type": "emotion",
"onyx:hasEmotionCategory": "emoml:big6anger"
},
"prov:wasGeneratedBy": "plugins/emoRand_0.1"
"prov:wasGeneratedBy": "plugins/emotion-random_0.1"
}


To get these emotions in VAD space (FSRE dimensions in EmotionML), we'd do this:

http://127.0.0.1:5000/api/?i=hello&algo=emoRand&emotionModel=emoml:fsre-dimensions
http://127.0.0.1:5000/api/?i=hello&algo=emotion-random&emotionModel=emoml:fsre-dimensions

This call, provided there is a valid conversion plugin from Ekman's to VAD, would return something like this:

@@ -42,7 +42,7 @@ This call, provided there is a valid conversion plugin from Ekman's to VAD, woul
"@type": "emotion",
"onyx:hasEmotionCategory": "emoml:big6anger"
},
"prov:wasGeneratedBy": "plugins/emoRand_0.1"
"prov:wasGeneratedBy": "plugins/emotion-random.1"
}, {
"@type": "emotionSet",
"onyx:hasEmotion": {
@@ -69,7 +69,7 @@ It is also possible to get the original emotion nested within the new converted
"@type": "emotion",
"onyx:hasEmotionCategory": "emoml:big6anger"
},
"prov:wasGeneratedBy": "plugins/emoRand_0.1"
"prov:wasGeneratedBy": "plugins/emotion-random.1"
"onyx:wasDerivedFrom": {
"@type": "emotionSet",
"onyx:hasEmotion": {

+ 2
- 4
docs/demo.rst View File

@@ -1,7 +1,7 @@
Demo
----

There is a demo available on http://senpy.cluster.gsi.dit.upm.es/, where you can test a serie of different plugins.
There is a demo available on http://senpy.gsi.upm.es/, where you can test a live instance of Senpy, with several open source plugins.
You can use the playground (a web interface) or make HTTP requests to the service API.

.. image:: senpy-playground.png
@@ -10,7 +10,5 @@ You can use the playground (a web interface) or make HTTP requests to the servic
:scale: 100 %
:align: center

Plugins Demo
============

The source code and description of the plugins used in the demo is available here: https://lab.cluster.gsi.dit.upm.es/senpy/senpy-plugins-community/.
The source code and description of the plugins used in the demo are available here: https://lab.gsi.upm.es/senpy/senpy-plugins-community/.

+ 27
- 0
docs/development.rst View File

@@ -0,0 +1,27 @@
Developing new services
-----------------------

Developing web services can be hard.
To illustrate it, the figure below summarizes the typical features in a text analysis service.

.. image:: senpy-framework.png
:width: 60%
:align: center

Senpy implements all the common blocks, so developers can focus on what really matters: great analysis algorithms that solve real problems.
Among other things, Senpy takes care of these tasks:

* Interfacing with the user: parameter validation, error handling.
* Formatting: JSON-LD, Turtle/n-triples input and output, or simple text input
* Linked Data: senpy results are semantically annotated, using a series of well established vocabularies, and sane default URIs.
* User interface: a web UI where users can explore your service and test different settings
* A client to interact with the service. Currently only available in Python.

You only need to provide the algorithm to turn a piece of text into an annotation
Sharing your sentiment analysis with the world has never been easier!

.. toctree::
:maxdepth: 1

plugins-quickstart
plugins-faq

BIN
docs/evaluation-results.png View File


+ 3
- 1
docs/examples.rst View File

@@ -1,5 +1,6 @@
Examples
------
--------

All the examples in this page use the :download:`the main schema <_static/schemas/definitions.json>`.

Simple NIF annotation
@@ -17,6 +18,7 @@ Sentiment Analysis
.....................
Description
,,,,,,,,,,,

This annotation corresponds to the sentiment analysis of an input. The example shows the sentiment represented according to Marl format.
The sentiments detected are contained in the Sentiments array with their related part of the text.


+ 1
- 1
docs/examples/results/example-analysis-as-id.json View File

@@ -2,7 +2,7 @@
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "me:Result1",
"@type": "results",
"analysis": [
"activities": [
{
"@id": "_:SAnalysis1_Activity",
"@type": "marl:SentimentAnalysis",

+ 1
- 1
docs/examples/results/example-basic.json View File

@@ -2,7 +2,7 @@
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "me:Result1",
"@type": "results",
"analysis": [ ],
"activities": [ ],
"entries": [
{
"@id": "http://example.org#char=0,40",

+ 1
- 1
docs/examples/results/example-complete.json View File

@@ -2,7 +2,7 @@
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "me:Result1",
"@type": "results",
"analysis": [
"activities": [
{
"@id": "_:SAnalysis1_Activity",
"@type": "marl:SentimentAnalysis",

+ 1
- 1
docs/examples/results/example-emotion.json View File

@@ -2,7 +2,7 @@
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "me:Result1",
"@type": "results",
"analysis": [
"activities": [
{
"@id": "me:EmotionAnalysis1_Activity",
"@type": "me:EmotionAnalysis1",

+ 1
- 1
docs/examples/results/example-ner.json View File

@@ -2,7 +2,7 @@
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "me:Result1",
"@type": "results",
"analysis": [
"activities": [
{
"@id": "_:NER1_Activity",
"@type": "me:NERAnalysis",

+ 1
- 1
docs/examples/results/example-pad.json View File

@@ -7,7 +7,7 @@
],
"@id": "me:Result1",
"@type": "results",
"analysis": [
"activities": [
{
"@id": "me:HesamsAnalysis_Activity",
"@type": "onyx:EmotionAnalysis",

+ 1
- 1
docs/examples/results/example-sentiment.json View File

@@ -2,7 +2,7 @@
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "me:Result1",
"@type": "results",
"analysis": [
"activities": [
{
"@id": "_:SAnalysis1_Activity",
"@type": "marl:SentimentAnalysis",

+ 1
- 1
docs/examples/results/example-suggestion.json View File

@@ -2,7 +2,7 @@
"@context": "http://mixedemotions-project.eu/ns/context.jsonld",
"@id": "me:Result1",
"@type": "results",
"analysis": [
"activities": [
{
"@id": "_:SgAnalysis1_Activity",
"@type": "me:SuggestionAnalysis",

+ 14
- 15
docs/index.rst View File

@@ -4,32 +4,31 @@ Welcome to Senpy's documentation!
:target: http://senpy.readthedocs.io/en/latest/
.. image:: https://badge.fury.io/py/senpy.svg
:target: https://badge.fury.io/py/senpy
.. image:: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/badges/master/build.svg
:target: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/commits/master
.. image:: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/badges/master/coverage.svg
:target: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/commits/master
.. image:: https://lab.gsi.upm.es/senpy/senpy/badges/master/build.svg
:target: https://lab.gsi.upm.es/senpy/senpy/commits/master
.. image:: https://lab.gsi.upm.es/senpy/senpy/badges/master/coverage.svg
:target: https://lab.gsi.upm.es/senpy/senpy/commits/master
.. image:: https://img.shields.io/pypi/l/requests.svg
:target: https://lab.cluster.gsi.dit.upm.es/senpy/senpy/
:target: https://lab.gsi.upm.es/senpy/senpy/



Senpy is a framework for sentiment and emotion analysis services.
Services built with senpy are interchangeable and easy to use because they share a common :doc:`apischema`.
It also simplifies service development.
Senpy services are interchangeable and easy to use because they share a common semantic :doc:`apischema`.

.. image:: senpy-architecture.png
:width: 100%
:align: center
If you interested in consuming Senpy services, read :doc:`Quickstart`.
To get familiar with the concepts behind Senpy, and what it can offer for service developers, check out :doc:`development`.
:doc:`apischema` contains information about the semantic models and vocabularies used by Senpy.

.. toctree::
:caption: Learn more about senpy:
:maxdepth: 2

senpy
Quickstart
installation
demo
usage
development
apischema
plugins
conversion
about
advanced
demo
publications

+ 7
- 9
docs/installation.rst View File

@@ -32,27 +32,25 @@ If you want to install senpy globally, use sudo instead of the ``--user`` flag.

Docker Image
************
Build the image or use the pre-built one:
The base image of senpy comes with some builtin plugins that you can use:

.. code:: bash

docker run -ti -p 5000:5000 gsiupm/senpy --host 0.0.0.0 --default-plugins
docker run -ti -p 5000:5000 gsiupm/senpy --host 0.0.0.0

To add custom plugins, use a docker volume:
To add your custom plugins, you can use a docker volume:
.. code:: bash

docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --host 0.0.0.0 --default-plugins -f /plugins
docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --host 0.0.0.0 --plugins -f /plugins

Python 2
........

There is a Senpy version for python2 too:
There is a Senpy image for **python 2**, too:
.. code:: bash

docker run -ti -p 5000:5000 gsiupm/senpy:python2.7 --host 0.0.0.0 --default-plugins
docker run -ti -p 5000:5000 gsiupm/senpy:python2.7 --host 0.0.0.0


Alias
@@ -62,7 +60,7 @@ If you are using the docker approach regularly, it is advisable to use a script

.. code:: bash

alias senpy='docker run --rm -ti -p 5000:5000 -v $PWD:/senpy-plugins gsiupm/senpy --default-plugins'
alias senpy='docker run --rm -ti -p 5000:5000 -v $PWD:/senpy-plugins gsiupm/senpy'


Now, you may run senpy from any folder in your computer like so:

+ 1
- 1
docs/plugins-definition.rst View File

@@ -110,4 +110,4 @@ Now, in a file named ``helloworld.py``:
entry.sentiments.append(sentiment)
yield entry

The complete code of the example plugin is available `here <https://lab.cluster.gsi.dit.upm.es/senpy/plugin-prueba>`__.
The complete code of the example plugin is available `here <https://lab.gsi.upm.es/senpy/plugin-prueba>`__.

docs/plugins.rst → docs/plugins-faq.rst View File

@@ -1,61 +1,18 @@
Developing new plugins
----------------------
This document contains the minimum to get you started with developing new analysis plugin.
For an example of conversion plugins, see :doc:`conversion`.
For a description of definition files, see :doc:`plugins-definition`.

A more step-by-step tutorial with slides is available `here <https://lab.cluster.gsi.dit.upm.es/senpy/senpy-tutorial>`__
F.A.Q.
======

.. contents:: :local:

What is a plugin?
=================

A plugin is a python object that can process entries. Given an entry, it will modify it, add annotations to it, or generate new entries.


What is an entry?
=================

Entries are objects that can be annotated.
In general, they will be a piece of text.
By default, entries are `NIF contexts <http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core/nif-core.html>`_ represented in JSON-LD format.
It is a dictionary/JSON object that looks like this:

.. code:: python

{
"@id": "<unique identifier or blank node name>",
"nif:isString": "input text",
"sentiments": [ {
...
}
],
...
}

Annotations are added to the object like this:

.. code:: python

entry = Entry()
entry.vocabulary__annotationName = 'myvalue'
entry['vocabulary:annotationName'] = 'myvalue'
entry['annotationNameURI'] = 'myvalue'

Where vocabulary is one of the prefixes defined in the default senpy context, and annotationURI is a full URI.
The value may be any valid JSON-LD dictionary.
For simplicity, senpy includes a series of models by default in the ``senpy.models`` module.


What are annotations?
=====================
#####################
They are objects just like entries.
Senpy ships with several default annotations, including: ``Sentiment``, ``Emotion``, ``EmotionSet``...jk bb


What's a plugin made of?
========================
########################

When receiving a query, senpy selects what plugin or plugins should process each entry, and in what order.
It also makes sure the every entry and the parameters provided by the user meet the plugin requirements.
@@ -73,37 +30,25 @@ In practice, this is what a plugin looks like, tests included:
The lines highlighted contain some information about the plugin.
In particular, the following information is mandatory:

* A unique name for the class. In our example, Rand.
* A unique name for the class. In our example, sentiment-random.
* The subclass/type of plugin. This is typically either `SentimentPlugin` or `EmotionPlugin`. However, new types of plugin can be created for different annotations. The only requirement is that these new types inherit from `senpy.Analysis`
* A description of the plugin. This can be done simply by adding a doc to the class.
* A version, which should get updated.
* An author name.


Plugins Code
============

The basic methods in a plugin are:

* analyse_entry: called in every user requests. It takes two parameters: ``Entry``, the entry object, and ``params``, the parameters supplied by the user. It should yield one or more ``Entry`` objects.
* activate: used to load memory-hungry resources. For instance, to train a classifier.
* deactivate: used to free up resources when the plugin is no longer needed.

Plugins are loaded asynchronously, so don't worry if the activate method takes too long. The plugin will be marked as activated once it is finished executing the method.


How does senpy find modules?
============================
############################

Senpy looks for files of two types:

* Python files of the form `senpy_<NAME>.py` or `<NAME>_plugin.py`. In these files, it will look for: 1) Instances that inherit from `senpy.Plugin`, or subclasses of `senpy.Plugin` that can be initialized without a configuration file. i.e. classes that contain all the required attributes for a plugin.
* Plugin definition files (see :doc:`advanced-plugins`)
* Plugin definition files (see :doc:`plugins-definition`)

Defining additional parameters
==============================
How can I define additional parameters for my plugin?
#####################################################

Your plugin may ask for additional parameters from the users of the service by using the attribute ``extra_params`` in your plugin definition.
Your plugin may ask for additional parameters from users by using the attribute ``extra_params`` in your plugin definition.
It takes a dictionary, where the keys are the name of the argument/parameter, and the value has the following fields:

* aliases: the different names which can be used in the request to use the parameter.
@@ -124,8 +69,8 @@ It takes a dictionary, where the keys are the name of the argument/parameter, an



Loading data and files
======================
How should I load external data and files
#########################################

Most plugins will need access to files (dictionaries, lexicons, etc.).
These files are usually heavy or under a license that does not allow redistribution.
@@ -144,7 +89,7 @@ Plugins have a convenience function `self.open` which will automatically prepend
file_in_data = <FILE PATH>
file_in_sources = <FILE PATH>

def activate(self):
def on activate(self):
with self.open(self.file_in_data) as f:
self._classifier = train_from_file(f)
file_in_source = os.path.join(self.get_folder(), self.file_in_sources)
@@ -155,8 +100,8 @@ Plugins have a convenience function `self.open` which will automatically prepend
It is good practice to specify the paths of these files in the plugin configuration, so the same code can be reused with different resources.


Docker image
============
Can I build a docker image for my plugin?
#########################################

Add the following dockerfile to your project to generate a docker image with your plugin:

@@ -204,17 +149,15 @@ Adding data to the image:
FROM gsiupm/senpy:1.0.1
COPY data /

F.A.Q.
======
What annotations can I use?
???????????????????????????
###########################

You can add almost any annotation to an entry.
The most common use cases are covered in the :doc:`apischema`.


Why does the analyse function yield instead of return?
??????????????????????????????????????????????????????
######################################################

This is so that plugins may add new entries to the response or filter some of them.
For instance, a chunker may split one entry into several.
@@ -222,7 +165,7 @@ On the other hand, a conversion plugin may leave out those entries that do not c


If I'm using a classifier, where should I train it?
???????????????????????????????????????????????????
###################################################

Training a classifier can be time time consuming. To avoid running the training unnecessarily, you can use ShelfMixin to store the classifier. For instance:

@@ -256,7 +199,7 @@ A corrupt shelf prevents the plugin from loading.
If you do not care about the data in the shelf, you can force your plugin to remove the corrupted file and load anyway, set the 'force_shelf' to True in your plugin and start it again.

How can I turn an external service into a plugin?
?????????????????????????????????????????????????
#################################################

This example ilustrate how to implement a plugin that accesses the Sentiment140 service.

@@ -292,8 +235,8 @@ This example ilustrate how to implement a plugin that accesses the Sentiment140
yield entry


Can I activate a DEBUG mode for my plugin?
???????????????????????????????????????????
How can I activate a DEBUG mode for my plugin?
###############################################

You can activate the DEBUG mode by the command-line tool using the option -d.

@@ -309,6 +252,6 @@ Additionally, with the ``--pdb`` option you will be dropped into a pdb post mort
python -m pdb yourplugin.py

Where can I find more code examples?
????????????????????????????????????
####################################

See: `<http://github.com/gsi-upm/senpy-plugins-community>`_.

+ 86
- 0
docs/plugins-quickstart.rst View File

@@ -0,0 +1,86 @@
Quickstart for service developers
=================================
This document contains the minimum to get you started with developing new services using Senpy.

For an example of conversion plugins, see :doc:`conversion`.
For a description of definition files, see :doc:`plugins-definition`.

A more step-by-step tutorial with slides is available `here <https://lab.gsi.upm.es/senpy/senpy-tutorial>`__

.. contents:: :local:

Installation
############

First of all, you need to install the package.
See :doc:`installation` for instructions.
Once installed, the `senpy` command should be available.

Architecture
############

The main component of a sentiment analysis service is the algorithm itself. However, for the algorithm to work, it needs to get the appropriate parameters from the user, format the results according to the defined API, interact with the user whn errors occur or more information is needed, etc.

Senpy proposes a modular and dynamic architecture that allows:

* Implementing different algorithms in a extensible way, yet offering a common interface.
* Offering common services that facilitate development, so developers can focus on implementing new and better algorithms.

The framework consists of two main modules: Senpy core, which is the building block of the service, and Senpy plugins, which consist of the analysis algorithm. The next figure depicts a simplified version of the processes involved in an analysis with the Senpy framework.

.. image:: senpy-architecture.png
:width: 100%
:align: center


What is a plugin?
#################

A plugin is a python object that can process entries. Given an entry, it will modify it, add annotations to it, or generate new entries.


What is an entry?
#################

Entries are objects that can be annotated.
In general, they will be a piece of text.
By default, entries are `NIF contexts <http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core/nif-core.html>`_ represented in JSON-LD format.
It is a dictionary/JSON object that looks like this:

.. code:: python

{
"@id": "<unique identifier or blank node name>",
"nif:isString": "input text",
"sentiments": [ {
...
}
],
...
}

Annotations are added to the object like this:

.. code:: python

entry = Entry()
entry.vocabulary__annotationName = 'myvalue'
entry['vocabulary:annotationName'] = 'myvalue'
entry['annotationNameURI'] = 'myvalue'

Where vocabulary is one of the prefixes defined in the default senpy context, and annotationURI is a full URI.
The value may be any valid JSON-LD dictionary.
For simplicity, senpy includes a series of models by default in the ``senpy.models`` module.

Plugins Code
############

The basic methods in a plugin are:

* analyse_entry: called in every user requests. It takes two parameters: ``Entry``, the entry object, and ``params``, the parameters supplied by the user. It should yield one or more ``Entry`` objects.
* activate: used to load memory-hungry resources. For instance, to train a classifier.
* deactivate: used to free up resources when the plugin is no longer needed.

Plugins are loaded asynchronously, so don't worry if the activate method takes too long. The plugin will be marked as activated once it is finished executing the method.


+ 46
- 0
docs/publications.rst View File

@@ -0,0 +1,46 @@
Publications
============


If you use Senpy in your research, please cite `Senpy: A Pragmatic Linked Sentiment Analysis Framework <http://gsi.dit.upm.es/index.php/es/investigacion/publicaciones?view=publication&task=show&id=417>`__ (`BibTex <http://gsi.dit.upm.es/index.php/es/investigacion/publicaciones?controller=publications&task=export&format=bibtex&id=417>`__):

.. code-block:: text

Sánchez-Rada, J. F., Iglesias, C. A., Corcuera, I., & Araque, Ó. (2016, October).
Senpy: A Pragmatic Linked Sentiment Analysis Framework.
In Data Science and Advanced Analytics (DSAA),
2016 IEEE International Conference on (pp. 735-742). IEEE.



Senpy uses Onyx for emotion representation, first introduced in:

.. code-block:: text

Sánchez-Rada, J. F., & Iglesias, C. A. (2016).
Onyx: A linked data approach to emotion representation.
Information Processing & Management, 52(1), 99-114.

Senpy uses Marl for sentiment representation, which was presented in:

.. code-block:: text

Westerski, A., Iglesias Fernandez, C. A., & Tapia Rico, F. (2011).
Linked opinions: Describing sentiments on the structured web of data.


Senpy has been used extensively in the toolbox of the MixedEmotions project:

.. code-block:: text

Buitelaar, P., Wood, I. D., Arcan, M., McCrae, J. P., Abele, A., Robin, C., … Tummarello, G. (2018).
MixedEmotions: An Open-Source Toolbox for Multi-Modal Emotion Analysis.
IEEE Transactions on Multimedia.

The representation models, formats and challenges are partially covered in a chapter of the book Sentiment Analysis in Social Networks:

.. code-block:: text

Iglesias, C. A., Sánchez-Rada, J. F., Vulcu, G., & Buitelaar, P. (2017).
Linked Data Models for Sentiment and Emotion Analysis in Social Networks.
In Sentiment Analysis in Social Networks (pp. 49-69).

+ 15
- 42
docs/senpy.rst View File

@@ -1,54 +1,27 @@
What is Senpy?
--------------

Senpy is a framework for text analysis using Linked Data. There are three main applications of Senpy so far: sentiment and emotion analysis, user profiling and entity recoginition. Annotations and Services are compliant with NIF (NLP Interchange Format).
Senpy is a framework for sentiment and emotion analysis services.
Its goal is to produce analysis services that are interchangeable and fully interoperable.

Senpy aims at providing a framework where analysis modules can be integrated easily as plugins, and providing a core functionality for managing tasks such as data validation, user interaction, formatting, logging, translation to linked data, etc.

The figure below summarizes the typical features in a text analysis service.
Senpy implements all the common blocks, so developers can focus on what really matters: great analysis algorithms that solve real problems.

.. image:: senpy-framework.png
:width: 60%
.. image:: senpy-architecture.png
:width: 100%
:align: center


Senpy for end users
===================

All services built using senpy share a common interface.
This allows users to use them (almost) interchangeably.
Senpy comes with a :ref:`built-in client`.


Senpy for service developers
============================
This allows users to use them (almost) interchangeably, with the same API and tools, simply by pointing to a different URL or changing a parameter.
The common schema also makes it easier to evaluate the performance of different algorithms and services.
In fact, Senpy has a built-in evaluation API you can use to compare results with different algorithms.

Senpy is a framework that turns your sentiment or emotion analysis algorithm into a full blown semantic service.
Senpy takes care of:
Services can also use the common interface to communicate with each other.
And higher level features can be built on top of these services, such as automatic fusion of results, emotion model conversion, and service discovery.

* Interfacing with the user: parameter validation, error handling.
* Formatting: JSON-LD, Turtle/n-triples input and output, or simple text input
* Linked Data: senpy results are semantically annotated, using a series of well established vocabularies, and sane default URIs.
* User interface: a web UI where users can explore your service and test different settings
* A client to interact with the service. Currently only available in Python.

Sharing your sentiment analysis with the world has never been easier!

Check out the :doc:`plugins` if you have developed an analysis algorithm (e.g. sentiment analysis) and you want to publish it as a service.

Architecture
============

The main component of a sentiment analysis service is the algorithm itself. However, for the algorithm to work, it needs to get the appropriate parameters from the user, format the results according to the defined API, interact with the user whn errors occur or more information is needed, etc.

Senpy proposes a modular and dynamic architecture that allows:
These benefits are not limited to new services.
The community has developed wrappers for some proprietary and commercial services (such as sentiment140 and Meaning Cloud), so you can consult them as.
Senpy comes with a :ref:`built-in client`.

* Implementing different algorithms in a extensible way, yet offering a common interface.
* Offering common services that facilitate development, so developers can focus on implementing new and better algorithms.

The framework consists of two main modules: Senpy core, which is the building block of the service, and Senpy plugins, which consist of the analysis algorithm. The next figure depicts a simplified version of the processes involved in an analysis with the Senpy framework.
To achieve this goal, Senpy uses a Linked Data principled approach, based on the NIF (NLP Interchange Format) specification, and open vocabularies such as Marl and Onyx.
You can learn more about this in :doc:`vocabularies`.

.. image:: senpy-architecture.png
:width: 100%
:align: center
Check out :doc:`plugins` if you have developed an analysis algorithm (e.g. sentiment analysis) and you want to publish it as a service.

docs/server.rst → docs/server-cli.rst View File

@@ -5,10 +5,11 @@ The senpy server is launched via the `senpy` command:

.. code:: text

usage: senpy [-h] [--level logging_level] [--debug] [--default-plugins]
[--host HOST] [--port PORT] [--plugins-folder PLUGINS_FOLDER]
[--only-install] [--only-list] [--data-folder DATA_FOLDER]
[--threaded] [--version]
usage: senpy [-h] [--level logging_level] [--log-format log_format] [--debug]
[--no-default-plugins] [--host HOST] [--port PORT]
[--plugins-folder PLUGINS_FOLDER] [--only-install] [--only-test]
[--test] [--only-list] [--data-folder DATA_FOLDER]
[--no-threaded] [--no-deps] [--version] [--allow-fail]

Run a Senpy server

@@ -16,20 +17,25 @@ The senpy server is launched via the `senpy` command:
-h, --help show this help message and exit
--level logging_level, -l logging_level
Logging level
--log-format log_format
Logging format
--debug, -d Run the application in debug mode
--default-plugins Load the default plugins
--no-default-plugins Do not load the default plugins
--host HOST Use 0.0.0.0 to accept requests from any host.
--port PORT, -p PORT Port to listen on.
--plugins-folder PLUGINS_FOLDER, -f PLUGINS_FOLDER
Where to look for plugins.
--only-install, -i Do not run a server, only install plugin dependencies
--only-test Do not run a server, just test all plugins
--test, -t Test all plugins before launching the server
--only-list, --list Do not run a server, only list plugins found
--data-folder DATA_FOLDER, --data DATA_FOLDER
Where to look for data. It be set with the SENPY_DATA
environment variable as well.
--threaded Run a threaded server
--no-threaded Run the server without threading
--no-deps, -n Skip installing dependencies
--version, -v Output the senpy version and exit
--allow-fail, --fail Do not exit if some plugins fail to activate


When launched, the server will recursively look for plugins in the specified plugins folder (the current working directory by default).
@@ -40,9 +46,9 @@ Let's run senpy with the default plugins:

.. code:: bash

senpy -f . --default-plugins
senpy -f .

Now go to `http://localhost:5000 <http://localhost:5000>`_, you should be greeted by the senpy playground:
Now open your browser and go to `http://localhost:5000 <http://localhost:5000>`_, where you should be greeted by the senpy playground:

.. image:: senpy-playground.png
:width: 100%
@@ -51,9 +57,9 @@ Now go to `http://localhost:5000 <http://localhost:5000>`_, you should be greete
The playground is a user-friendly way to test your plugins, but you can always use the service directly: `http://localhost:5000/api?input=hello <http://localhost:5000/api?input=hello>`_.


By default, senpy will listen only on the `127.0.0.1` address.
That means you can only access the API from your (or localhost).
You can listen on a different address using the `--host` flag (e.g., 0.0.0.0).
By default, senpy will listen only on `127.0.0.1`.
That means you can only access the API from your PC (i.e. localhost).
You can listen on a different address using the `--host` flag (e.g., 0.0.0.0, to allow any computer to access it).
The default port is 5000.
You can change it with the `--port` flag.


+ 0
- 15
docs/usage.rst View File

@@ -1,15 +0,0 @@
Usage
-----

First of all, you need to install the package.
See :doc:`installation` for instructions.
Once installed, the `senpy` command should be available.

.. toctree::
:maxdepth: 1

server
SenpyClientUse
commandline



+ 2
- 2
example-plugins/README.md View File

@@ -1,6 +1,6 @@
This is a collection of plugins that exemplify certain aspects of plugin development with senpy.

The first series of plugins the `basic` ones.
The first series of plugins are the `basic` ones.
Their starting point is a classification function defined in `basic.py`.
They all include testing and running them as a script will run all tests.
In ascending order of customization, the plugins are:
@@ -19,5 +19,5 @@ In rest of the plugins show advanced topics:

All of the plugins in this folder include a set of test cases and they are periodically tested with the latest version of senpy.

Additioanlly, for an example of stand-alone plugin that can be tested and deployed with docker, take a look at: lab.cluster.gsi.dit.upm.es/senpy/plugin-example
Additioanlly, for an example of stand-alone plugin that can be tested and deployed with docker, take a look at: lab.gsi.upm.es/senpy/plugin-example
bbm

+ 5
- 3
example-plugins/basic.py View File

@@ -1,5 +1,5 @@
#!/usr/local/bin/python
# coding: utf-8
# -*- coding: utf-8 -*-

emoticons = {
'pos': [':)', ':]', '=)', ':D'],
@@ -7,17 +7,19 @@ emoticons = {
}

emojis = {
'pos': ['😁', '😂', '😃', '😄', '😆', '😅', '😄' '😍'],
'neg': ['😢', '😡', '😠', '😞', '😖', '😔', '😓', '😒']
'pos': [u'😁', u'😂', u'😃', u'😄', u'😆', u'😅', u'😄', u'😍'],
'neg': [u'😢', u'😡', u'😠', u'😞', u'😖', u'😔', u'😓', u'😒']
}


def get_polarity(text, dictionaries=[emoticons, emojis]):
polarity = 'marl:Neutral'
print('Input for get_polarity', text)
for dictionary in dictionaries:
for label, values in dictionary.items():
for emoticon in values:
if emoticon and emoticon in text:
polarity = label
break
print('Polarity', polarity)
return polarity

+ 3
- 3
example-plugins/basic_analyse_entry_plugin.py View File

@@ -1,5 +1,5 @@
#!/usr/local/bin/python
# coding: utf-8
# -*- coding: utf-8 -*-

from senpy import easy_test, models, plugins

@@ -18,13 +18,13 @@ class BasicAnalyseEntry(plugins.SentimentPlugin):
'default': 'marl:Neutral'
}

def analyse_entry(self, entry, params):
def analyse_entry(self, entry, activity):
polarity = basic.get_polarity(entry.text)

polarity = self.mappings.get(polarity, self.mappings['default'])

s = models.Sentiment(marl__hasPolarity=polarity)
s.prov(self)
s.prov(activity)
entry.sentiments.append(s)
yield entry


+ 8
- 10
example-plugins/basic_box_plugin.py View File

@@ -1,5 +1,5 @@
#!/usr/local/bin/python
# coding: utf-8
# -*- coding: utf-8 -*-

from senpy import easy_test, SentimentBox

@@ -12,15 +12,13 @@ class BasicBox(SentimentBox):
author = '@balkian'
version = '0.1'

mappings = {
'pos': 'marl:Positive',
'neg': 'marl:Negative',
'default': 'marl:Neutral'
}

def predict_one(self, input):
output = basic.get_polarity(input)
return self.mappings.get(output, self.mappings['default'])
def predict_one(self, features, **kwargs):
output = basic.get_polarity(features[0])
if output == 'pos':
return [1, 0, 0]
if output == 'neg':
return [0, 0, 1]
return [0, 1, 0]

test_cases = [{
'input': 'Hello :)',

+ 14
- 15
example-plugins/basic_plugin.py View File

@@ -1,37 +1,36 @@
#!/usr/local/bin/python
# coding: utf-8
# -*- coding: utf-8 -*-

from senpy import easy_test, SentimentBox, MappingMixin
from senpy import easy_test, SentimentBox

import basic


class Basic(MappingMixin, SentimentBox):
class Basic(SentimentBox):
'''Provides sentiment annotation using a lexicon'''

author = '@balkian'
version = '0.1'

mappings = {
'pos': 'marl:Positive',
'neg': 'marl:Negative',
'default': 'marl:Neutral'
}

def predict_one(self, input):
return basic.get_polarity(input)
def predict_one(self, features, **kwargs):
output = basic.get_polarity(features[0])
if output == 'pos':
return [1, 0, 0]
if output == 'neu':
return [0, 1, 0]
return [0, 0, 1]

test_cases = [{
'input': 'Hello :)',
'input': u'Hello :)',
'polarity': 'marl:Positive'
}, {
'input': 'So sad :(',
'input': u'So sad :(',
'polarity': 'marl:Negative'
}, {
'input': 'Yay! Emojis 😁',
'input': u'Yay! Emojis 😁',
'polarity': 'marl:Positive'
}, {
'input': 'But no emoticons 😢',
'input': u'But no emoticons 😢',
'polarity': 'marl:Negative'
}]


+ 2
- 2
example-plugins/configurable_plugin.py View File

@@ -1,5 +1,5 @@
#!/usr/local/bin/python
# coding: utf-8
# -*- coding: utf-8 -*-

from senpy import easy_test, models, plugins

@@ -16,7 +16,7 @@ class Dictionary(plugins.SentimentPlugin):

mappings = {'pos': 'marl:Positive', 'neg': 'marl:Negative'}

def analyse_entry(self, entry, params):
def analyse_entry(self, entry, *args, **kwargs):
polarity = basic.get_polarity(entry.text, self.dictionaries)
if polarity in self.mappings:
polarity = self.mappings[polarity]

senpy/plugins/example/emorand_plugin.py → example-plugins/emorand_plugin.py View File

@@ -6,12 +6,13 @@ from senpy.models import EmotionSet, Emotion, Entry

class EmoRand(EmotionPlugin):
'''A sample plugin that returns a random emotion annotation'''
name = 'emotion-random'
author = '@balkian'
version = '0.1'
url = "https://github.com/gsi-upm/senpy-plugins-community"
onyx__usesEmotionModel = "emoml:big6"

def analyse_entry(self, entry, params):
def analyse_entry(self, entry, activity):
category = "emoml:big6happiness"
number = max(-1, min(1, random.gauss(0, 0.5)))
if number > 0:
@@ -19,7 +20,7 @@ class EmoRand(EmotionPlugin):
emotionSet = EmotionSet()
emotion = Emotion({"onyx:hasEmotionCategory": category})
emotionSet.onyx__hasEmotion.append(emotion)
emotionSet.prov__wasGeneratedBy = self.id
emotionSet.prov(activity)
entry.emotions.append(emotionSet)
yield entry

@@ -27,6 +28,6 @@ class EmoRand(EmotionPlugin):
params = dict()
results = list()
for i in range(100):
res = next(self.analyse_entry(Entry(nif__isString="Hello"), params))
res = next(self.analyse_entry(Entry(nif__isString="Hello"), self.activity(params)))
res.validate()
results.append(res.emotions[0]['onyx:hasEmotion'][0]['onyx:hasEmotionCategory'])

+ 4
- 3
example-plugins/parameterized_plugin.py View File

@@ -1,5 +1,5 @@
#!/usr/local/bin/python
# coding: utf-8
# -*- coding: utf-8 -*-

from senpy import easy_test, models, plugins

@@ -25,7 +25,8 @@ class ParameterizedDictionary(plugins.SentimentPlugin):
}
}

def analyse_entry(self, entry, params):
def analyse_entry(self, entry, activity):
params = activity.params
positive_words = params['positive-words'].split(',')
negative_words = params['negative-words'].split(',')
dictionary = {
@@ -35,7 +36,7 @@ class ParameterizedDictionary(plugins.SentimentPlugin):
polarity = basic.get_polarity(entry.text, [dictionary])

s = models.Sentiment(marl__hasPolarity=polarity)
s.prov(self)
s.prov(activity)
entry.sentiments.append(s)
yield entry


senpy/plugins/example/rand_plugin.py → example-plugins/rand_plugin.py View File

@@ -2,15 +2,16 @@ import random
from senpy import SentimentPlugin, Sentiment, Entry


class Rand(SentimentPlugin):
class RandSent(SentimentPlugin):
'''A sample plugin that returns a random sentiment annotation'''
name = 'sentiment-random'
author = "@balkian"
version = '0.1'
url = "https://github.com/gsi-upm/senpy-plugins-community"
marl__maxPolarityValue = '1'
marl__minPolarityValue = "-1"

def analyse_entry(self, entry, params):
def analyse_entry(self, entry, activity):
polarity_value = max(-1, min(1, random.gauss(0.2, 0.2)))
polarity = "marl:Neutral"
if polarity_value > 0:
@@ -19,7 +20,7 @@ class Rand(SentimentPlugin):
polarity = "marl:Negative"
sentiment = Sentiment(marl__hasPolarity=polarity,
marl__polarityValue=polarity_value)
sentiment.prov(self)
sentiment.prov(activity)
entry.sentiments.append(sentiment)
yield entry

@@ -28,8 +29,9 @@ class Rand(SentimentPlugin):
params = dict()
results = list()
for i in range(50):
activity = self.activity(params)
res = next(self.analyse_entry(Entry(nif__isString="Hello"),
params))
activity))
res.validate()
results.append(res.sentiments[0]['marl:hasPolarity'])
assert 'marl:Positive' in results

+ 8
- 13
example-plugins/sklearn/pipeline_plugin.py View File

@@ -1,25 +1,20 @@
from senpy import SentimentBox, MappingMixin, easy_test
from senpy import SentimentBox, easy_test

from mypipeline import pipeline


class PipelineSentiment(MappingMixin, SentimentBox):
'''
This is a pipeline plugin that wraps a classifier defined in another module
(mypipeline).
'''
class PipelineSentiment(SentimentBox):
'''This is a pipeline plugin that wraps a classifier defined in another module
(mypipeline).'''
author = '@balkian'
version = 0.1
maxPolarityValue = 1
minPolarityValue = -1

mappings = {
1: 'marl:Positive',
-1: 'marl:Negative'
}

def predict_one(self, input):
return pipeline.predict([input, ])[0]
def predict_one(self, features, **kwargs):
if pipeline.predict(features) > 0:
return [1, 0, 0]
return [0, 0, 1]

test_cases = [
{

+ 1
- 1
extra-requirements.txt View File

@@ -1 +1 @@
gsitk
gsitk>0.1.9.1

+ 0
- 2
k8s/senpy-deployment.yaml View File

@@ -15,8 +15,6 @@ spec:
- name: senpy-latest
image: $IMAGEWTAG
imagePullPolicy: Always
args:
- "--default-plugins"
resources:
limits:
memory: "512Mi"

+ 7
- 0
k8s/senpy-ingress.yaml View File

@@ -12,3 +12,10 @@ spec:
backend:
serviceName: senpy-latest
servicePort: 5000
- host: latest.senpy.gsi.upm.es
http:
paths:
- path: /
backend:
serviceName: senpy-latest
servicePort: 5000

+ 2
- 1
requirements.txt View File

@@ -11,5 +11,6 @@ rdflib
rdflib-jsonld
numpy
scipy
scikit-learn
scikit-learn>=0.20
responses
jmespath

+ 49
- 28
senpy/__main__.py View File

@@ -40,8 +40,14 @@ def main():
'-l',
metavar='logging_level',
type=str,
default="WARN",
default="INFO",
help='Logging level')
parser.add_argument(
'--log-format',
metavar='log_format',
type=str,
default='%(asctime)s %(levelname)-10s %(name)-30s \t %(message)s',
help='Logging format')
parser.add_argument(
'--debug',
'-d',
@@ -49,10 +55,10 @@ def main():
default=False,
help='Run the application in debug mode')
parser.add_argument(
'--default-plugins',
'--no-default-plugins',
action='store_true',
default=False,
help='Load the default plugins')
help='Do not load the default plugins')
parser.add_argument(
'--host',
type=str,
@@ -68,7 +74,7 @@ def main():
'--plugins-folder',
'-f',
type=str,
default='.',
action='append',
help='Where to look for plugins.')
parser.add_argument(
'--only-install',
@@ -100,10 +106,10 @@ def main():
default=None,
help='Where to look for data. It be set with the SENPY_DATA environment variable as well.')
parser.add_argument(
'--threaded',
action='store_false',
default=True,
help='Run a threaded server')
'--no-threaded',
action='store_true',
default=False,
help='Run a single-threaded server')
parser.add_argument(
'--no-deps',
'-n',
@@ -123,30 +129,42 @@ def main():
default=False,
help='Do not exit if some plugins fail to activate')
args = parser.parse_args()
print('Senpy version {}'.format(senpy.__version__))
print(sys.version)
if args.version:
print('Senpy version {}'.format(senpy.__version__))
print(sys.version)
exit(1)
rl = logging.getLogger()
rl.setLevel(getattr(logging, args.level))
logger_handler = rl.handlers[0]

# First, generic formatter:
logger_handler.setFormatter(logging.Formatter(args.log_format))

app = Flask(__name__)
app.debug = args.debug
sp = Senpy(app, args.plugins_folder,
default_plugins=args.default_plugins,
sp = Senpy(app,
plugin_folder=None,
default_plugins=not args.no_default_plugins,
data_folder=args.data_folder)
folders = list(args.plugins_folder) if args.plugins_folder else []
if not folders:
folders.append(".")
for p in folders:
sp.add_folder(p)

plugins = sp.plugins(plugin_type=None, is_activated=False)
maxname = max(len(x.name) for x in plugins)
maxversion = max(len(str(x.version)) for x in plugins)
print('Found {} plugins:'.format(len(plugins)))
for plugin in plugins:
import inspect
fpath = inspect.getfile(plugin.__class__)
print('\t{: <{maxname}} @ {: <{maxversion}} -> {}'.format(plugin.name,
plugin.version,
fpath,
maxname=maxname,
maxversion=maxversion))
if args.only_list:
plugins = sp.plugins()
maxname = max(len(x.name) for x in plugins)
maxversion = max(len(x.version) for x in plugins)
print('Found {} plugins:'.format(len(plugins)))
for plugin in plugins:
import inspect
fpath = inspect.getfile(plugin.__class__)
print('\t{: <{maxname}} @ {: <{maxversion}} -> {}'.format(plugin.name,
plugin.version,
fpath,
maxname=maxname,
maxversion=maxversion))
return
if not args.no_deps:
sp.install_deps()
@@ -160,10 +178,13 @@ def main():
print('Senpy version {}'.format(senpy.__version__))
print('Server running on port %s:%d. Ctrl+C to quit' % (args.host,
args.port))
app.run(args.host,