Bumped to 0.6.1

* Downloads pip requirements
* Modified Makefile

.gitignore

@@ -4,4 +4,5 @@
 dist
 build
 README.html
-__pycache__
+__pycache__
+Dockerfile-*

.travis.yml

@@ -1,6 +1,8 @@
 language: python
 python:
   - "2.7"
+  - "3.4"
+  - "3.5"
 install: "pip install -r requirements.txt"
 # run nosetests - Tests
 script: nosetests

Dockerfile.template

@@ -0,0 +1,5 @@
+from python:{{PYVERSION}}-onbuild
+
+RUN pip install .
+
+ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"]

MANIFEST.in

@@ -2,6 +2,7 @@ include requirements.txt
 include test-requirements.txt
 include README.md
 include senpy/context.jsonld
+include senpy/VERSION
 graft senpy/plugins
 graft senpy/schemas
 graft senpy/templates

Makefile

@@ -0,0 +1,57 @@
+PYVERSIONS=3.4 2.7
+PYMAIN=$(firstword $(PYVERSIONS))
+NAME=senpy
+REPO=gsiupm
+VERSION=$(shell cat $(NAME)/VERSION)
+
+
+all: build run
+
+dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS))
+
+Dockerfile-%: Dockerfile.template
+	sed "s/{{PYVERSION}}/$*/" Dockerfile.template > Dockerfile-$*
+
+build: $(addprefix build-, $(PYMAIN))
+
+buildall: $(addprefix build-, $(PYVERSIONS))
+
+build-%: Dockerfile-%
+	docker build -t '$(REPO)/$(NAME):$(VERSION)-python$*' -f Dockerfile-$* .;
+
+test: $(addprefix test-,$(PYMAIN))
+
+testall: $(addprefix test-,$(PYVERSIONS))
+
+test-%: build-%
+	docker run --rm -w /usr/src/app/ --entrypoint=/usr/local/bin/python -ti '$(REPO)/$(NAME):$(VERSION)-python$*' setup.py test --addopts "-vvv -s --pdb" ;
+
+pip_test-%:
+	docker run --rm -ti python:$* pip install senpy ;
+
+upload-%: test-%
+	docker push '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)'
+
+upload: testall $(addprefix upload-,$(PYVERSIONS))
+	docker tag '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)' '$(REPO)/$(NAME):$(VERSION)'
+	docker tag '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)' '$(REPO)/$(NAME)'
+	docker push '$(REPO)/$(NAME):$(VERSION)'
+
+clean:
+	@docker ps -a | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[1] != "${VERSION}"){ print $$1;}}' | xargs docker rm 2>/dev/null|| true
+	@docker images | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[1] != "${VERSION}"){ print $$1":"$$2;}}' | xargs docker rmi 2>/dev/null|| true
+
+upload_git:
+	git commit -a
+	git tag ${VERSION}
+	git push --tags origin master
+
+pip_upload:
+	python setup.py sdist upload ;
+
+pip_test: $(addprefix pip_test-,$(PYVERSIONS))
+
+run: build
+	docker run --rm -p 5000:5000 -ti '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)'
+
+.PHONY: test test-% build-% build test test_pip run

README.rst

@@ -30,7 +30,7 @@ Alternatively, you can use the development version:
  
 .. code:: bash
 
-   git clone git@github.com:gsi-upm/senpy
+   git clone http://github.com/gsi-upm/senpy
    cd senpy
    pip install --user .
 
@@ -38,9 +38,9 @@ If you want to install senpy globally, use sudo instead of the ``--user`` flag.
 
 Docker Image
 ************
-Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 balkian/senpy --host 0.0.0.0 --default-plugins``.
+Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/senpy --host 0.0.0.0 --default-plugins``.
 
-To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins balkian/senpy --host 0.0.0.0 --default-plugins -f /plugins``
+To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --host 0.0.0.0 --default-plugins -f /plugins``
 
 Usage
 -----

docs/_static/schemas

@@ -0,0 +1 @@
+../../senpy/schemas/

docs/conf.py

@@ -52,16 +52,17 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Senpy'
-copyright = u'2015, J. Fernando Sánchez'
+copyright = u'2016, J. Fernando Sánchez'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '0.4'
+with open('../senpy/VERSION') as f:
+    version = f.read().strip()
 # The full version, including alpha/beta/rc tags.
-release = '0.4'
+release = version
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/demo.rst

@@ -0,0 +1,75 @@
+Demo
+----
+
+There is a demo available on http://senpy.demos.gsi.dit.upm.es/, where you can a serie of different plugins. You can use them in the playground or make a directly requests to the service.
+
+.. image:: senpy-playground.png
+  :height: 400px
+  :width: 800px
+  :scale: 100 %
+  :align: center
+
+Plugins Demo
+============
+
+The next plugins are available at the demo:
+
+* emoTextAnew extracts the VAD (valence-arousal-dominance) of a sentence by matching words from the ANEW dictionary.
+* emoTextWordnetAffect based on the hierarchy of WordnetAffect to calculate the emotion of the sentence.
+* vaderSentiment utilizes the software from vaderSentiment to calculate the sentiment of a sentence.
+* sentiText is a software developed during the TASS 2015 competition, it has been adapted for English and Spanish.
+
+emoTextANEW plugin
+******************
+
+This plugin is going to used the ANEW lexicon dictionary to calculate de VAD (valence-arousal-dominance) of the sentence and the determinate which emotion is closer to this value.
+
+Each emotion has a centroid, which it has been approximated using the formula described in this article:
+
+http://www.aclweb.org/anthology/W10-0208
+
+The plugin is going to look for the words in the sentence that appear in the ANEW dictionary and calculate the average VAD score for the sentence. Once this score is calculated, it is going to seek the emotion that is closest to this value.
+
+emoTextWAF plugin
+*****************
+
+This plugin uses WordNet-Affect (http://wndomains.fbk.eu/wnaffect.html) to calculate the percentage of each emotion. The emotions that are going to be used are: anger, fear, disgust, joy and sadness. It is has been used a emotion mapping enlarge the emotions:
+
+* anger : general-dislike
+* fear : negative-fear
+* disgust : shame
+* joy : gratitude, affective, enthusiasm, love, joy, liking
+* sadness : ingrattitude, daze, humlity, compassion, despair, anxiety, sadness
+
+sentiText plugin
+****************
+
+This plugin is based in the classifier developed for the TASS 2015 competition. It has been developed for Spanish and English. The different phases that has this plugin when it is activated:
+
+* Train both classifiers (English and Spanish).
+* Initialize resources (dictionaries,stopwords,etc.).
+* Extract bag of words,lemmas and chars.
+
+Once the plugin is activated, the features that are going to be extracted for the classifiers are:
+
+* Matches with the bag of words extracted from the train corpus.
+* Sentiment score of the sentences extracted from the dictionaries (lexicons and emoticons).
+* Identify negations and intensifiers in the sentences.
+* Complementary features such as exclamation and interrogation marks, eloganted and caps words, hashtags, etc.
+
+The plugin has a preprocessor, which is focues on Twitter corpora, that is going to be used for cleaning the text to simplify the feature extraction.
+
+There is more information avaliable in the next article.
+
+Aspect based Sentiment Analysis of Spanish Tweets, Oscar Araque and Ignacio Corcuera-Platas and Constantino Román-Gómez and Carlos A. Iglesias and J. Fernando Sánchez-Rada. http://gsi.dit.upm.es/es/investigacion/publicaciones?view=publication&task=show&id=37
+
+vaderSentiment plugin
+*********************
+
+For developing this plugin, it has been used the module vaderSentiment, which is described in the paper: VADER: A Parsimonious Rule-based Model for Sentiment Analysis of Social Media Text C.J. Hutto and Eric Gilbert Eighth International Conference on Weblogs and Social Media (ICWSM-14). Ann Arbor, MI, June 2014.
+
+If you use this plugin in your research, please cite the above paper
+
+For more information about the functionality, check the official repository
+
+https://github.com/cjhutto/vaderSentiment

docs/examples/example-basic.json

@@ -0,0 +1,18 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "http://example.com#NIFExample",
+  "analysis": [
+  ],
+  "entries": [
+    {
+      "@id": "http://example.org#char=0,40",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:beginIndex": 0,
+      "nif:endIndex": 40,
+      "nif:isString": "My favourite actress is Natalie Portman"
+    }
+  ]
+}

docs/examples/example-complete.json

@@ -0,0 +1,86 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "me:Result1",
+  "analysis": [
+    {
+      "@id": "me:SAnalysis1",
+      "@type": "marl:SentimentAnalysis",
+      "marl:maxPolarityValue": 1,
+      "marl:minPolarityValue": 0
+    },
+    {
+      "@id": "me:SgAnalysis1",
+      "@type": "me:SuggestionAnalysis"
+    },
+    {
+      "@id": "me:EmotionAnalysis1",
+      "@type": "me:EmotionAnalysis"
+    },
+    {
+      "@id": "me:NER1",
+      "@type": "me:NER"
+    }
+  ],
+  "entries": [
+    {
+      "@id": "http://micro.blog/status1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+        {
+          "@id": "http://micro.blog/status1#char=5,13",
+          "nif:beginIndex": 5,
+          "nif:endIndex": 13,
+          "nif:anchorOf": "Microsoft",
+          "me:references": "http://dbpedia.org/page/Microsoft",
+          "prov:wasGeneratedBy": "me:NER1"
+        },
+        {
+          "@id": "http://micro.blog/status1#char=25,37",
+          "nif:beginIndex": 25,
+          "nif:endIndex": 37,
+          "nif:anchorOf": "Windows Phone",
+          "me:references": "http://dbpedia.org/page/Windows_Phone",
+          "prov:wasGeneratedBy": "me:NER1"
+        }
+      ],
+      "suggestions": [
+        {
+          "@id": "http://micro.blog/status1#char=16,77",
+          "nif:beginIndex": 16,
+          "nif:endIndex": 77,
+          "nif:anchorOf": "put your Windows Phone on your newest #open technology program"
+        }
+      ],
+      "sentiments": [
+        {
+          "@id": "http://micro.blog/status1#char=80,97",
+          "nif:beginIndex": 80,
+          "nif:endIndex": 97,
+          "nif:anchorOf": "You'll be awesome.",
+          "marl:hasPolarity": "marl:Positive",
+          "marl:polarityValue": 0.9,
+          "prov:wasGeneratedBy": "me:SAnalysis1"
+        }
+      ],
+      "emotions": [
+        {
+          "@id": "http://micro.blog/status1#char=0,109",
+          "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+          "prov:wasGeneratedBy": "me:EAnalysis1",
+          "onyx:hasEmotion": [
+            {
+              "onyx:hasEmotionCategory": "wna:liking"
+            },
+            {
+              "onyx:hasEmotionCategory": "wna:excitement"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

docs/examples/example-emotion.json

@@ -0,0 +1,41 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "me:Result1",
+  "analysis": [
+    {
+      "@id": "me:EmotionAnalysis1",
+      "@type": "onyx:EmotionAnalysis"
+    }
+  ],
+  "entries": [
+    {
+      "@id": "http://micro.blog/status1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+      ],
+      "suggestions": [
+      ],
+      "sentiments": [
+      ],
+      "emotions": [
+        {
+          "@id": "http://micro.blog/status1#char=0,109",
+          "nif:anchorOf": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+          "prov:wasGeneratedBy": "me:EmotionAnalysis1",
+          "onyx:hasEmotion": [
+            {
+              "onyx:hasEmotionCategory": "wna:liking"
+            },
+            {
+              "onyx:hasEmotionCategory": "wna:excitement"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

docs/examples/example-ner.json

@@ -0,0 +1,44 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "me:Result1",
+  "analysis": [
+    {
+      "@id": "me:NER1",
+      "@type": "me:NERAnalysis"
+    }
+  ],
+  "entries": [
+    {
+      "@id": "http://micro.blog/status1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+        {
+          "@id": "http://micro.blog/status1#char=5,13",
+          "nif:beginIndex": 5,
+          "nif:endIndex": 13,
+          "nif:anchorOf": "Microsoft",
+          "me:references": "http://dbpedia.org/page/Microsoft",
+          "prov:wasGeneratedBy": "me:NER1"
+        },
+        {
+          "@id": "http://micro.blog/status1#char=25,37",
+          "nif:beginIndex": 25,
+          "nif:endIndex": 37,
+          "nif:anchorOf": "Windows Phone",
+          "me:references": "http://dbpedia.org/page/Windows_Phone",
+          "prov:wasGeneratedBy": "me:NER1"
+        }
+      ],
+      "suggestions": [
+      ],
+      "sentiments": [
+      ],
+      "emotionSets": [
+      ]
+    }
+  ]
+}

docs/examples/example-sentiment.json

@@ -0,0 +1,39 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "me:Result1",
+  "analysis": [
+    {
+      "@id": "me:SAnalysis1",
+      "@type": "marl:SentimentAnalysis",
+      "marl:maxPolarityValue": 1,
+      "marl:minPolarityValue": 0
+    }
+  ],
+  "entries": [
+    {
+      "@id": "http://micro.blog/status1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+      ],
+      "suggestions": [
+      ],
+      "sentiments": [
+        {
+          "@id": "http://micro.blog/status1#char=80,97",
+          "nif:beginIndex": 80,
+          "nif:endIndex": 97,
+          "nif:anchorOf": "You'll be awesome.",
+          "marl:hasPolarity": "marl:Positive",
+          "marl:polarityValue": 0.9,
+          "prov:wasGeneratedBy": "me:SAnalysis1"
+        }
+      ],
+      "emotionSets": [
+      ]
+    }
+  ]
+}

docs/examples/example-suggestion.json

@@ -0,0 +1,35 @@
+{
+  "@context": "http://mixedemotions-project.eu/ns/context.jsonld",
+  "@id": "me:Result1",
+  "analysis": [
+    {
+      "@id": "me:SgAnalysis1",
+      "@type": "me:SuggestionAnalysis"
+    }
+  ],
+  "entries": [
+    {
+      "@id": "http://micro.blog/status1",
+      "@type": [
+        "nif:RFC5147String",
+        "nif:Context"
+      ],
+      "prov:wasGeneratedBy": "me:SAnalysis1",
+      "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
+      "entities": [
+      ],
+      "suggestions": [
+        {
+          "@id": "http://micro.blog/status1#char=16,77",
+          "nif:beginIndex": 16,
+          "nif:endIndex": 77,
+          "nif:anchorOf": "put your Windows Phone on your newest #open technology program"
+        }
+      ],
+      "sentiments": [
+      ],
+      "emotionSets": [
+      ]
+    }
+  ]
+}

docs/index.rst

@@ -9,8 +9,11 @@ Welcome to Senpy's documentation!
 Contents:
 
 .. toctree::
+   senpy
    installation
    usage
    api
+   schema
    plugins
+   demo
    :maxdepth: 2

docs/installation.rst

@@ -22,6 +22,6 @@ If you want to install senpy globally, use sudo instead of the ``--user`` flag.
 
 Docker Image
 ************
-Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 balkian/senpy --host 0.0.0.0 --default-plugins``.
+Build the image or use the pre-built one: ``docker run -ti -p 5000:5000 gsiupm/senpy --host 0.0.0.0 --default-plugins``.
 
-To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins balkian/senpy --host 0.0.0.0 --default-plugins -f /plugins``
+To add custom plugins, add a volume and tell senpy where to find the plugins: ``docker run -ti -p 5000:5000 -v <PATH OF PLUGINS>:/plugins gsiupm/senpy --host 0.0.0.0 --default-plugins -f /plugins``

docs/plugins.rst

@@ -1,7 +1,34 @@
 Developing new plugins
 ----------------------
+Each plugin represents a different analysis process.There are two types of files that are needed by senpy for loading a plugin:
 
 Plugins Interface
+=======
+- Definition file, has the ".senpy" extension.
+- Code file, is a python file.
+
+Plugins Definitions
+===================
+
+The definition file can be written in JSON or YAML, where the data representation consists on attribute-value pairs.
+The principal attributes are:
+
+* name: plugin name used in senpy to call the plugin.
+* module: indicates the module that will be loaded
+
+.. code:: python
+
+          {
+            "name" : "senpyPlugin",
+            "module" : "{python code file}"
+          }
+
+.. code:: python
+          
+          name: senpyPlugin
+          module: {python code file}
+
+Plugins Code
 =================
 
 The basic methods in a plugin are:
@@ -43,6 +70,92 @@ Training a classifier can be time time consuming. To avoid running the training
 
 You can speficy a 'shelf_file' in your .senpy file. By default the ShelfMixin creates a file based on the plugin name and stores it in that plugin's folder.
 
+I want to implement my service as a plugin, How i can do it?
+????????????????????????????????????????????????????????????
+
+This example ilustrate how to implement the Sentiment140 service as a plugin in senpy
+
+.. code:: python
+
+          class Sentiment140Plugin(SentimentPlugin):
+              def analyse(self, **params):
+                  lang = params.get("language", "auto")
+                  res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
+                                      json.dumps({"language": lang,
+                                                  "data": [{"text": params["input"]}]
+                                                  }
+                                                 )
+                                      )
+
+                  p = params.get("prefix", None)
+                  response = Results(prefix=p)
+                  polarity_value = self.maxPolarityValue*int(res.json()["data"][0]
+                                                             ["polarity"]) * 0.25
+                  polarity = "marl:Neutral"
+                  neutral_value = self.maxPolarityValue / 2.0
+                  if polarity_value > neutral_value:
+                      polarity = "marl:Positive"
+                  elif polarity_value < neutral_value:
+                      polarity = "marl:Negative"
+
+                  entry = Entry(id="Entry0",
+                                nif__isString=params["input"])
+                  sentiment = Sentiment(id="Sentiment0",
+                                      prefix=p,
+                                      marl__hasPolarity=polarity,
+                                      marl__polarityValue=polarity_value)
+                  sentiment.prov__wasGeneratedBy = self.id
+                  entry.sentiments = []
+                  entry.sentiments.append(sentiment)
+                  entry.language = lang
+                  response.entries.append(entry)
+                  return response
+
+
+Where can I define extra parameters to be introduced in the request to my plugin?
+?????????????????????????????????????????????????????????????????????????????????
+
+You can add these parameters in the definition file under the attribute "extra_params" : "{param_name}". The name of the parameter has new attributes-value pairs. The basic attributes are:
+
+* aliases: the different names which can be used in the request to use the parameter.
+* required: this option is a boolean and indicates if the parameters is binding in operation plugin.
+* options: the different values of the paremeter.
+* default: the default value of the parameter, this is useful in case the paremeter is required and you want to have a default value.
+
+.. code:: python
+
+          "extra_params": {
+             "language": {
+                "aliases": ["language", "l"],
+                "required": true,
+                "options": ["es","en"],
+                "default": "es"
+             }
+          }
+
+This example shows how to introduce a parameter associated with language.
+The extraction of this paremeter is used in the analyse method of the Plugin interface.
+
+.. code:: python
+
+          lang = params.get("language")
+
+Where can I set up variables for using them in my plugin?
+?????????????????????????????????????????????????????????
+
+You can add these variables in the definition file with the extracture of attribute-value pair.
+
+Once you have added your variables, the next step is to extract them into the plugin. The plugin's __init__ method has a parameter called `info` where you can extract the values of the variables. This info parameter has the structure of a python dictionary.
+
+Can I activate a DEBUG mode for my plugin?
+???????????????????????????????????????????
+
+You can activate the DEBUG mode by the command-line tool using the option -d.
+
+.. code:: bash
+
+   python -m senpy -d
+
 Where can I find more code examples?
 ????????????????????????????????????
 

docs/schema.rst

@@ -0,0 +1,74 @@
+Schema Examples
+===============
+All the examples in this page use the :download:`the main schema <_static/schemas/definitions.json>`.
+
+Simple NIF annotation
+---------------------
+Description
+...........
+This example covers the basic example in the NIF documentation: `<http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core/nif-core.html>`_.
+
+Representation
+..............
+.. literalinclude:: examples/example-basic.json
+   :language: json-ld
+
+Sentiment Analysis
+---------------------
+Description
+...........
+
+Representation
+..............
+
+.. literalinclude:: examples/example-sentiment.json
+   :emphasize-lines: 5-10,25-33
+   :language: json-ld
+
+Suggestion Mining
+-----------------
+Description
+...........
+
+Representation
+..............
+
+.. literalinclude:: examples/example-suggestion.json
+   :emphasize-lines: 5-8,22-27
+   :language: json-ld
+
+Emotion Analysis
+----------------
+Description
+...........
+
+Representation
+..............
+
+.. literalinclude:: examples/example-emotion.json
+   :language: json-ld
+   :emphasize-lines: 5-8,25-37
+
+Named Entity Recognition
+------------------------
+Description
+...........
+
+Representation
+..............
+
+.. literalinclude:: examples/example-ner.json
+   :emphasize-lines: 5-8,19-34
+   :language: json-ld
+
+Complete example
+----------------
+Description
+...........
+This example covers all of the above cases, integrating all the annotations in the same document.
+
+Representation
+..............
+
+.. literalinclude:: examples/example-complete.json
+   :language: json-ld

docs/senpy.rst

@@ -0,0 +1,35 @@
+What is Senpy?
+--------------
+
+Senpy is an open source reference implementation of a linked data model for sentiment and emotion analysis services based on the vocabularies NIF, Marl and Onyx. 
+
+The overall goal of the reference implementation Senpy is easing the adoption of the proposed linked data model for sentiment and emotion analysis services, so that services from different providers become interoperable. With this aim, the design of the reference implementation has focused on its extensibility and reusability. 
+
+A modular approach allows organizations to replace individual components with custom ones developed in-house. Furthermore, organizations can benefit from reusing prepackages modules that provide advanced functionalities, such as algorithms for sentiment and emotion analysis, linked data publication or emotion and sentiment mapping between different providers.
+
+Specifications
+==============
+
+The model used in Senpy is based on the following specifications:
+
+* Marl, a vocabulary designed to annotate and describe subjetive opinions expressed on the web or in information systems.
+* Onyx, which is built one the same principles as Marl to annotate and describe emotions, and provides interoperability with Emotion Markup Language.
+* NIF 2.0, which defines a semantic format and APO for improving interoperability among natural language processing services
+
+Architecture
+============
+
+The main component of a sentiment analysis service is the algorithm itself. However, for the algorithm to work, it needs to get the appropriate parameters from the user, format the results according to the defined API, interact with the user whn errors occur or more information is needed, etc.
+
+Senpy proposes a modular and dynamic architecture that allows:
+
+* Implementing different algorithms in a extensible way, yet offering a common interface.
+* Offering common services that facilitate development, so developers can focus on implementing new and better algorithms.
+
+The framework consists of two main modules: Senpy core, which is the building block of the service, and Senpy plugins, which consist of the analysis algorithm. The next figure depicts a simplified version of the processes involved in an analysis with the Senpy framework.
+
+.. image:: senpy-architecture.png
+  :height: 400px
+  :width: 800px
+  :scale: 100 %
+  :align: center

docs/usage.rst

@@ -15,6 +15,61 @@ Or, alternatively:
 
 
 This will create a server with any modules found in the current path.
+
+Useful command-line options
+===========================
+
+In case you want to load modules, which are located in different folders under the root folder, use the next option.
+
+.. code:: bash
+
+    python -m senpy -f .
+
+The default port used by senpy is 5000, but you can change it using the option `--port`.
+
+.. code:: bash
+
+    python -m senpy --port 8080
+
+Also, the host can be changed where senpy is deployed. The default value is `127.0.0.1`.
+
+.. code:: bash
+
+    python -m senpy --host 0.0.0.0
+
 For more options, see the `--help` page.
 
 Alternatively, you can use the modules included in senpy to build your own application.
+
+Senpy server
+============
+
+Once the server is launched, there is a basic endpoint in the server, which provides a playground to use the plugins that have been loaded.
+
+In case you want to know the different endpoints of the server, there is more information available in the NIF API section_.
+
+Video example
+=============
+
+This video shows how to use senpy through command-line tool.
+
+https://asciinema.org/a/9uwef1ghkjk062cw2t4mhzpyk
+
+Request example in python
+=========================
+
+This example shows how to make a request to a plugin.
+
+.. code:: python
+
+    import requests
+    import json
+
+    r = requests.get('http://127.0.0.1:5000/api/?algo=rand&i=Testing')
+    response = r.content.decode('utf-8')
+    response_json = json.loads(response)
+
+
+
+.. _section: http://senpy.readthedocs.org/en/latest/api.html
+

requirements.txt

@@ -4,8 +4,8 @@ requests>=2.4.1
 GitPython>=0.3.2.RC1
 gevent>=1.1rc4
 PyLD>=0.6.5
-Flask-Testing>=0.4.2
 six
 future
 jsonschema
 jsonref
+PyYAML

senpy/VERSION

@@ -0,0 +1 @@
+0.6.1

senpy/__init__.py

@@ -18,4 +18,8 @@
 Sentiment analysis server in Python
 """
 
-__version__  = "0.5.5"
+import os
+VFILE = os.path.join(os.path.dirname(__file__), "VERSION")
+
+with open(VFILE, 'r') as f:
+    __version__  = f.read().strip()

senpy/__main__.py

@@ -65,6 +65,11 @@ def main():
                         type=str,
                         default='plugins',
                         help='Where to look for plugins.')
+    parser.add_argument('--only-install',
+                        '-i',
+                        action='store_true',
+                        default=False,
+                        help='Do not run a server, only install the dependencies of the plugins.')
     args = parser.parse_args()
     logging.basicConfig()
     rl = logging.getLogger()
@@ -72,6 +77,9 @@ def main():
     app = Flask(__name__)
     app.debug = args.debug
     sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins)
+    if args.only_install:
+        sp.install_deps()
+        return
     sp.activate_all()
     http_server = WSGIServer((args.host, args.port), app)
     try:

senpy/extensions.py

@@ -22,7 +22,8 @@ import imp
 import logging
 import traceback
 import gevent
-import json
+import yaml
+import pip
 
 logger = logging.getLogger(__name__)
 
@@ -198,18 +199,38 @@ class Senpy(object):
             logger.error('Error reloading {}: {}'.format(name, ex))
             self.plugins[name] = plugin
 
-    @staticmethod
+
-    def _load_plugin(root, filename):
+    @classmethod
-        logger.debug("Loading plugin: {}".format(filename))
+    def validate_info(cls, info):
-        fpath = os.path.join(root, filename)
+        return all(x in info for x in ('name', 'module', 'version'))
-        with open(fpath, 'r') as f:
+
-            info = json.load(f)
+    def install_deps(self):
-        logger.debug("Info: {}".format(info))
+        for i in self.plugins.values():
-        sys.path.append(root)
+            self._install_deps(i._info)
+
+    @classmethod
+    def _install_deps(cls, info=None):
+        requirements = info.get('requirements', [])
+        if requirements:
+            pip_args = []
+            pip_args.append('install')
+            for req in requirements:
+                pip_args.append( req )
+            logger.info('Installing requirements: ' + str(requirements))
+            pip.main(pip_args) 
+
+    @classmethod
+    def _load_plugin_from_info(cls, info, root):
+        if not cls.validate_info(info):
+            logger.warn('The module info is not valid.\n\t{}'.format(info))
+            return None, None 
         module = info["module"]
         name = info["name"]
+        requirements = info.get("requirements", [])
+        sys.path.append(root)
         (fp, pathname, desc) = imp.find_module(module, [root, ])
         try:
+            cls._install_deps(info)
             tmp = imp.load_module(module, fp, pathname, desc)
             sys.path.remove(root)
             candidate = None
@@ -221,20 +242,30 @@ class Senpy(object):
                     candidate = obj
                     break
             if not candidate:
-                logger.debug("No valid plugin for: {}".format(filename))
+                logger.debug("No valid plugin for: {}".format(module))
                 return
             module = candidate(info=info)
-            try:
+            repo_path = root
-                repo_path = root
+            module._repo = Repo(repo_path)
-                module._repo = Repo(repo_path)
+        except InvalidGitRepositoryError:
-            except InvalidGitRepositoryError:
+            logger.debug("The plugin {} is not in a Git repository".format(module))
-                module._repo = None
+            module._repo = None
         except Exception as ex:
-            logger.error("Exception importing {}: {}".format(filename, ex))
+            logger.error("Exception importing {}: {}".format(module, ex))
             logger.error("Trace: {}".format(traceback.format_exc()))
             return None, None
         return name, module
 
+    @classmethod
+    def _load_plugin(cls, root, filename):
+        fpath = os.path.join(root, filename)
+        logger.debug("Loading plugin: {}".format(fpath))
+        with open(fpath, 'r') as f:
+            info = yaml.load(f)
+        logger.debug("Info: {}".format(info))
+        return cls._load_plugin_from_info(info, root)
+
+
     def _load_plugins(self):
         plugins = {}
         for search_folder in self._search_folders:

senpy/plugins.py

@@ -91,4 +91,4 @@ class ShelfMixin(object):
         if hasattr(self, '_sh') and self._sh is not None:
             with open(self.shelf_file, 'wb') as f:
                 pickle.dump(self._sh, f)
-        del(self.__dict__['_sh'])
+            del(self.__dict__['_sh'])

setup.py

@@ -15,7 +15,8 @@ except AttributeError:
 install_reqs = [str(ir.req) for ir in install_reqs]
 test_reqs = [str(ir.req) for ir in test_reqs]
 
-exec(open('senpy/__init__.py').read())
+with open('senpy/VERSION') as f:
+    __version__ = f.read().strip()
 
 setup(
     name='senpy',

tests/test_blueprints.py

@@ -1,9 +1,10 @@
 import os
 import logging
+import json
 
 from senpy.extensions import Senpy
 from flask import Flask
-from flask.ext.testing import TestCase
+from unittest import TestCase
 from gevent import sleep
 from itertools import product
 
@@ -11,31 +12,38 @@ from itertools import product
 def check_dict(indic, template):
     return all(item in indic.items() for item in template.items())
 
+def parse_resp(resp):
+    return json.loads(resp.data.decode('utf-8'))
+
 
 class BlueprintsTest(TestCase):
 
-    def create_app(self):
+    def setUp(self):
         self.app = Flask("test_extensions")
+        self.client = self.app.test_client()
         self.senpy = Senpy()
         self.senpy.init_app(self.app)
         self.dir = os.path.join(os.path.dirname(__file__), "..")
         self.senpy.add_folder(self.dir)
         self.senpy.activate_plugin("Dummy", sync=True)
-        return self.app
 
+    def assertCode(self, resp, code):
+        self.assertEqual(resp.status_code, code)
+        
     def test_home(self):
         """
         Calling with no arguments should ask the user for more arguments
         """
         resp = self.client.get("/api/")
-        self.assert404(resp)
+        self.assertCode(resp, 404)
-        logging.debug(resp.json)
+        js = parse_resp(resp)
-        assert resp.json["status"] == 404
+        logging.debug(js)
+        assert js["status"] == 404
         atleast = {
             "status": 404,
             "message": "Missing or invalid parameters",
         }
-        assert check_dict(resp.json, atleast)
+        assert check_dict(js, atleast)
 
     def test_analysis(self):
         """
@@ -43,81 +51,93 @@ class BlueprintsTest(TestCase):
         it should contain the context
         """
         resp = self.client.get("/api/?i=My aloha mohame")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
-        logging.debug("Got response: %s", resp.json)
+        js = parse_resp(resp)
-        assert "@context" in resp.json
+        logging.debug("Got response: %s", js)
-        assert "entries" in resp.json
+        assert "@context" in js
+        assert "entries" in js 
 
     def test_list(self):
         """ List the plugins """
         resp = self.client.get("/api/plugins/")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
-        logging.debug(resp.json)
+        js = parse_resp(resp)
-        assert 'plugins' in resp.json
+        logging.debug(js)
-        plugins = resp.json['plugins']
+        assert 'plugins' in js
+        plugins = js['plugins']
         assert len(plugins) > 1
         assert list(p for p in plugins if p['name'] == "Dummy")
-        assert "@context" in resp.json
+        assert "@context" in js
 
     def test_headers(self):
         for i, j in product(["/api/plugins/?nothing=", "/api/?i=test&"],
                             ["inHeaders"]):
             resp = self.client.get("%s" % (i))
-            assert "@context" in resp.json
+            js = parse_resp(resp)
+            assert "@context" in js
             resp = self.client.get("%s&%s=0" % (i, j))
-            assert "@context" in resp.json
+            js = parse_resp(resp)
+            assert "@context" in js 
             resp = self.client.get("%s&%s=1" % (i, j))
-            assert "@context" not in resp.json
+            js = parse_resp(resp)
+            assert "@context" not in js 
             resp = self.client.get("%s&%s=true" % (i, j))
-            assert "@context" not in resp.json
+            js = parse_resp(resp)
+            assert "@context" not in js 
 
     def test_detail(self):
         """ Show only one plugin"""
         resp = self.client.get("/api/plugins/Dummy/")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
-        logging.debug(resp.json)
+        js = parse_resp(resp)
-        assert "@id" in resp.json
+        logging.debug(js)
-        assert resp.json["@id"] == "Dummy_0.1"
+        assert "@id" in js
+        assert js["@id"] == "Dummy_0.1"
 
     def test_activate(self):
         """ Activate and deactivate one plugin """
         resp = self.client.get("/api/plugins/Dummy/deactivate")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
         sleep(0.5)
         resp = self.client.get("/api/plugins/Dummy/")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
-        assert "is_activated" in resp.json
+        js = parse_resp(resp)
-        assert resp.json["is_activated"] == False
+        assert "is_activated" in js
+        assert js["is_activated"] == False
         resp = self.client.get("/api/plugins/Dummy/activate")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
         sleep(0.5)
         resp = self.client.get("/api/plugins/Dummy/")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
-        assert "is_activated" in resp.json
+        js = parse_resp(resp)
-        assert resp.json["is_activated"] == True
+        assert "is_activated" in js 
+        assert js["is_activated"] == True
 
     def test_default(self):
         """ Show only one plugin"""
         resp = self.client.get("/api/plugins/default/")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
-        logging.debug(resp.json)
+        js = parse_resp(resp)
-        assert "@id" in resp.json
+        logging.debug(js)
-        assert resp.json["@id"] == "Dummy_0.1"
+        assert "@id" in js 
+        assert js["@id"] == "Dummy_0.1"
         resp = self.client.get("/api/plugins/Dummy/deactivate")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
         sleep(0.5)
         resp = self.client.get("/api/plugins/default/")
-        self.assert404(resp)
+        self.assertCode(resp, 404)
 
     def test_context(self):
         resp = self.client.get("/api/contexts/context.jsonld")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
-        assert "@context" in resp.json
+        js = parse_resp(resp)
+        assert "@context" in js
         assert check_dict(
-            resp.json["@context"],
+            js["@context"],
             {"marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#"})
 
     def test_schema(self):
         resp = self.client.get("/api/schemas/definitions.json")
-        self.assert200(resp)
+        self.assertCode(resp, 200)
-        assert "$schema" in resp.json
+        js = parse_resp(resp)
+        assert "$schema" in js 

tests/test_extensions.py

@@ -6,18 +6,17 @@ from functools import partial
 from senpy.extensions import Senpy
 from senpy.models import Error
 from flask import Flask
-from flask.ext.testing import TestCase
+from unittest import TestCase
 
 
 class ExtensionsTest(TestCase):
 
-    def create_app(self):
+    def setUp(self):
         self.app = Flask("test_extensions")
         self.dir = os.path.join(os.path.dirname(__file__))
         self.senpy = Senpy(plugin_folder=self.dir, default_plugins=False)
         self.senpy.init_app(self.app)
         self.senpy.activate_plugin("Dummy", sync=True)
-        return self.app
 
     def test_init(self):
         """ Initialising the app with the extension.  """
@@ -34,6 +33,20 @@ class ExtensionsTest(TestCase):
         assert "Dummy" in self.senpy.plugins
 
     def test_enabling(self):
+        """ Enabling a plugin """
+        info = {
+            'name': 'TestPip',
+            'module': 'dummy',
+            'requirements': ['noop'],
+            'version': 0
+            }
+        root = os.path.join(self.dir, 'dummy_plugin')
+        name, module = self.senpy._load_plugin_from_info(info, root=root)
+        assert name == 'TestPip'
+        assert module
+        import noop
+
+    def test_installing(self):
         """ Enabling a plugin """
         self.senpy.activate_all(sync=True)
         assert len(self.senpy.plugins) == 2