Update schema examples

.gitlab-ci.yml

@@ -21,16 +21,16 @@ before_script:
   except:
     - tags # Avoid unnecessary double testing
     
-test-3.5:
+test-3.6:
   <<: *test_definition
   variables:
-    PYTHON_VERSION: "3.5"
+    PYTHON_VERSION: "3.6"
 
-test-2.7:
+test-3.7:
   <<: *test_definition
   allow_failure: true
   variables:
-    PYTHON_VERSION: "2.7"
+    PYTHON_VERSION: "3.7"
 
 push:
   stage: push
@@ -101,4 +101,3 @@ cleanup_py:
    when: always   # this is important; run even if preceding stages failed.
    script:
     - rm -vf ~/.pypirc  # we don't want to leave these around, but GitLab may clean up anyway.
-    - docker logout

.travis.yml

@@ -6,7 +6,10 @@ services:
 language: python
 
 env:
-  - PYV=2.7
+  - PYV=3.4
   - PYV=3.5
+  - PYV=3.6
+  - PYV=3.7
+#  - PYV=3.3 # Apt fails in this docker image
 # run nosetests - Tests
 script: make test-$PYV

CHANGELOG.md

@@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+* Restored hash changing function in `main.js`
+
+## 0.20
+
 ### Added
 * Objects can control the keys that will be used in `serialize`/`jsonld`/`as_dict` by specifying a list of keys in `terse_keys`.
 e.g.
@@ -27,6 +32,7 @@ e.g.
 * Plugin and parameter descriptions are now formatted with (showdown)[https://github.com/showdownjs/showdown].
 * The web UI requests extra_parameters from the server. This is useful for pipelines. See #52
 * First batch of semantic tests (using SPARQL)
+* `Plugin.path()` method to get a file path from a relative path (using the senpy data folder)
 
 ### Changed
 * `install_deps` now checks what requirements are already met before installing with pip.

MANIFEST.in

@@ -2,9 +2,10 @@ include requirements.txt
 include test-requirements.txt
 include extra-requirements.txt
 include README.rst
+include LICENSE.txt
 include senpy/VERSION
 graft senpy/plugins
 graft senpy/schemas
 graft senpy/templates
 graft senpy/static
-graft img
+graft img

docs/Quickstart-1minute.ipynb

@@ -0,0 +1,152 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Senpy in 1 minute\n",
+    "\n",
+    "This mini-tutorial only shows how to annotate with a service.\n",
+    "We will use the [demo server](http://senpy.gsi.upm.es), which runs some open source plugins for sentiment and emotion analysis."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Annotating with senpy is as simple as issuing an HTTP request to the API using your favourite tool.\n",
+    "This is just an example using curl:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\r\n",
+      "  \"@context\": \"http://senpy.gsi.upm.es/api/contexts/YXBpL3NlbnRpbWVudDE0MD8j\",\r\n",
+      "  \"@type\": \"Results\",\r\n",
+      "  \"entries\": [\r\n",
+      "    {\r\n",
+      "      \"@id\": \"prefix:\",\r\n",
+      "      \"@type\": \"Entry\",\r\n",
+      "      \"marl:hasOpinion\": [\r\n",
+      "        {\r\n",
+      "          \"@type\": \"Sentiment\",\r\n",
+      "          \"marl:hasPolarity\": \"marl:Positive\",\r\n",
+      "          \"prov:wasGeneratedBy\": \"prefix:Analysis_1554389334.6431913\"\r\n",
+      "        }\r\n",
+      "      ],\r\n",
+      "      \"nif:isString\": \"Senpy is awesome\",\r\n",
+      "      \"onyx:hasEmotionSet\": []\r\n",
+      "    }\r\n",
+      "  ]\r\n",
+      "}"
+     ]
+    }
+   ],
+   "source": [
+    "!curl \"http://senpy.gsi.upm.es/api/sentiment140\" --data-urlencode \"input=Senpy is awesome\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**Congratulations**, you've used your first senpy service!"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Here is the equivalent using the `requests` library:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\n",
+      "  \"@context\": \"http://senpy.gsi.upm.es/api/contexts/YXBpL3NlbnRpbWVudDE0MD9pbnB1dD1TZW5weStpcythd2Vzb21lIw%3D%3D\",\n",
+      "  \"@type\": \"Results\",\n",
+      "  \"entries\": [\n",
+      "    {\n",
+      "      \"@id\": \"prefix:\",\n",
+      "      \"@type\": \"Entry\",\n",
+      "      \"marl:hasOpinion\": [\n",
+      "        {\n",
+      "          \"@type\": \"Sentiment\",\n",
+      "          \"marl:hasPolarity\": \"marl:Positive\",\n",
+      "          \"prov:wasGeneratedBy\": \"prefix:Analysis_1554389335.9803226\"\n",
+      "        }\n",
+      "      ],\n",
+      "      \"nif:isString\": \"Senpy is awesome\",\n",
+      "      \"onyx:hasEmotionSet\": []\n",
+      "    }\n",
+      "  ]\n",
+      "}\n"
+     ]
+    }
+   ],
+   "source": [
+    "import requests\n",
+    "res = requests.get('http://senpy.gsi.upm.es/api/sentiment140',\n",
+    "                   params={\"input\": \"Senpy is awesome\",})\n",
+    "print(res.text)"
+   ]
+  }
+ ],
+ "metadata": {
+  "anaconda-cloud": {},
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.3"
+  },
+  "toc": {
+   "colors": {
+    "hover_highlight": "#DAA520",
+    "running_highlight": "#FF0000",
+    "selected_highlight": "#FFD700"
+   },
+   "moveMenuLeft": true,
+   "nav_menu": {
+    "height": "68px",
+    "width": "252px"
+   },
+   "navigate_menu": true,
+   "number_sections": true,
+   "sideBar": true,
+   "threshold": 4,
+   "toc_cell": false,
+   "toc_section_display": "block",
+   "toc_window_display": false
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}

docs/advanced.rst

@@ -6,4 +6,5 @@ Advanced usage
 
     server-cli
     conversion
-    commandline
+    commandline
+    development

docs/conf.py

@@ -38,6 +38,8 @@ extensions = [
     'sphinxcontrib.httpdomain',
     'sphinx.ext.coverage',
     'sphinx.ext.autosectionlabel',
+    'nbsphinx',
+    'sphinx.ext.mathjax',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -54,7 +56,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Senpy'
-copyright = u'2016, J. Fernando Sánchez'
+copyright = u'2019, J. Fernando Sánchez'
 description = u'A framework for sentiment and emotion analysis services'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -79,7 +81,9 @@ language = None
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['_build']
+exclude_patterns = ['_build', '**.ipynb_checkpoints']
+
+
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.
@@ -286,3 +290,13 @@ texinfo_documents = [
 
 # If true, do not generate a @detailmenu in the "Top" node's menu.
 #texinfo_no_detailmenu = False
+
+nbsphinx_prolog = """
+.. note:: This page has been auto-generated from a Jupyter notebook using nbsphinx_.
+
+    The original source is available at: https://github.com/gsi-upm/senpy/tree/master/docs//{{ env.doc2path(env.docname, base=None) }}
+
+    .. _nbsphinx: https://nbsphinx.readthedocs.io/
+
+----
+"""

docs/demo.rst

@@ -4,11 +4,10 @@ Demo
 There is a demo available on http://senpy.gsi.upm.es/, where you can test a live instance of Senpy, with several open source plugins.
 You can use the playground (a web interface) or make HTTP requests to the service API.
 
-.. image:: senpy-playground.png
-  :height: 400px
+.. image:: playground-0.20.png
+  :target: http://senpy.gsi.upm.es
   :width: 800px
-  :scale: 100 %
   :align: center
 
 
-The source code and description of the plugins used in the demo are available here: https://lab.gsi.upm.es/senpy/senpy-plugins-community/.
+The source code and description of the plugins used in the demo are available here: https://github.com/gsi-upm/senpy-plugins-community/.

docs/development.rst

@@ -2,11 +2,7 @@ Developing new services
 -----------------------
 
 Developing web services can be hard.
-To illustrate it, the figure below summarizes the typical features in a text analysis service.
-
-.. image:: senpy-framework.png
-  :width: 60%
-  :align: center
+A text analysis service must implement all the typical features, such as: extraction of parameters, validation, format conversion, visualization...
 
 Senpy implements all the common blocks, so developers can focus on what really matters: great analysis algorithms that solve real problems.
 Among other things, Senpy takes care of these tasks:
@@ -24,4 +20,5 @@ Sharing your sentiment analysis with the world has never been easier!
     :maxdepth: 1
 
     plugins-quickstart
-    plugins-faq
+    plugins-faq
+    plugins-definition

docs/examples.rst

@@ -26,20 +26,7 @@ Representation
 ,,,,,,,,,,,,,,
 
 .. literalinclude:: examples/results/example-sentiment.json
-   :emphasize-lines: 5-10,25-33
-   :language: json-ld
-
-Suggestion Mining
-.................
-Description
-,,,,,,,,,,,
-The suggestions schema represented below shows the suggestions detected in the text. Within it, we can find the NIF fields highlighted that corresponds to the text of the detected suggestion. 
-
-Representation
-,,,,,,,,,,,,,,
-
-.. literalinclude:: examples/results/example-suggestion.json
-   :emphasize-lines: 5-8,22-27
+   :emphasize-lines: 5-11,20-30
    :language: json-ld
 
 Emotion Analysis
@@ -53,28 +40,6 @@ Representation
 
 .. literalinclude:: examples/results/example-emotion.json
    :language: json-ld
-   :emphasize-lines: 5-8,25-37
+   :emphasize-lines: 5-11,22-36
 
-Named Entity Recognition
-........................
-Description
-,,,,,,,,,,,
-The Named Entity Recognition is represented as follows. In this particular case, it can be seen within the entities array the entities recognised. For the example input, Microsoft and Windows Phone are the ones detected. 
-Representation
-,,,,,,,,,,,,,,
 
-.. literalinclude:: examples/results/example-ner.json
-   :emphasize-lines: 5-8,19-34
-   :language: json-ld
-
-Complete example
-................
-Description
-,,,,,,,,,,,
-This example covers all of the above cases, integrating all the annotations in the same document.
-
-Representation
-,,,,,,,,,,,,,,
-
-.. literalinclude:: examples/results/example-complete.json
-   :language: json-ld

docs/examples/results/example-emotion.json

@@ -17,10 +17,6 @@
         "nif:Context"
       ],
       "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
-      "entities": [
-      ],
-      "suggestions": [
-      ],
       "sentiments": [
       ],
       "emotions": [

docs/examples/results/example-sentiment.json

@@ -17,10 +17,6 @@
         "nif:Context"
       ],
       "nif:isString": "Dear Microsoft, put your Windows Phone on your newest #open technology program. You'll be awesome. #opensource",
-      "entities": [
-      ],
-      "suggestions": [
-      ],
       "sentiments": [
         {
           "@id": "http://micro.blog/status1#char=80,97",
@@ -32,7 +28,7 @@
           "prov:wasGeneratedBy": "_:SAnalysis1_Activity"
         }
       ],
-      "emotionSets": [
+      "emotions": [
       ]
     }
   ]

docs/index.rst

@@ -1,16 +1,16 @@
 Welcome to Senpy's documentation!
 =================================
-.. image:: https://readthedocs.org/projects/senpy/badge/?version=latest
-   :target: http://senpy.readthedocs.io/en/latest/
-.. image:: https://badge.fury.io/py/senpy.svg
-   :target: https://badge.fury.io/py/senpy
-.. image:: https://lab.gsi.upm.es/senpy/senpy/badges/master/build.svg
-   :target: https://lab.gsi.upm.es/senpy/senpy/commits/master
-.. image:: https://lab.gsi.upm.es/senpy/senpy/badges/master/coverage.svg
-   :target: https://lab.gsi.upm.es/senpy/senpy/commits/master
-.. image:: https://img.shields.io/pypi/l/requests.svg
-   :target: https://lab.gsi.upm.es/senpy/senpy/
 
+.. image:: https://readthedocs.org/projects/senpy/badge/?version=latest
+  :target: http://senpy.readthedocs.io/en/latest/
+.. image:: https://badge.fury.io/py/senpy.svg
+  :target: https://badge.fury.io/py/senpy
+.. image:: https://lab.gsi.upm.es/senpy/senpy/badges/master/build.svg
+  :target: https://lab.gsi.upm.es/senpy/senpy/commits/master
+.. image:: https://lab.gsi.upm.es/senpy/senpy/badges/master/coverage.svg
+  :target: https://lab.gsi.upm.es/senpy/senpy/commits/master
+.. image:: https://img.shields.io/pypi/l/requests.svg
+  :target: https://lab.gsi.upm.es/senpy/senpy/
 
 
 Senpy is a framework for sentiment and emotion analysis services.
@@ -20,15 +20,16 @@ If you interested in consuming Senpy services, read :doc:`Quickstart`.
 To get familiar with the concepts behind Senpy, and what it can offer for service developers, check out :doc:`development`.
 :doc:`apischema` contains information about the semantic models and vocabularies used by Senpy.
 
-.. toctree::
-   :caption: Learn more about senpy:
-   :maxdepth: 2
 
-   senpy
-   Quickstart
-   installation
-   development
-   apischema
-   advanced
-   demo
-   publications
+.. toctree::
+  :caption: Learn more about senpy:
+  :maxdepth: 2
+
+  senpy
+  demo
+  Quickstart.ipynb
+  installation
+  apischema
+  advanced
+  publications
+

docs/plugins-faq.rst

@@ -8,7 +8,7 @@ F.A.Q.
 What are annotations?
 #####################
 They are objects just like entries.
-Senpy ships with several default annotations, including: ``Sentiment``, ``Emotion``, ``EmotionSet``...jk bb
+Senpy ships with several default annotations, including ``Sentiment`` and ``Emotion``.
 
 
 What's a plugin made of?
@@ -22,7 +22,7 @@ Hence, two parts are necessary: 1) the code that will process the entry, and 2)
 In practice, this is what a plugin looks like, tests included:
 
 
-.. literalinclude:: ../senpy/plugins/example/rand_plugin.py
+.. literalinclude:: ../example-plugins/rand_plugin.py
    :emphasize-lines: 5-11
    :language: python
 
@@ -76,8 +76,9 @@ Most plugins will need access to files (dictionaries, lexicons, etc.).
 These files are usually heavy or under a license that does not allow redistribution.
 For this reason, senpy has a `data_folder` that is separated from the source files.
 The location of this folder is controlled programmatically or by setting the `SENPY_DATA` environment variable.
+You can use the `self.path(filepath)` function to get the path of a given `filepath` within the data folder.
 
-Plugins have a convenience function `self.open` which will automatically prepend the data folder to relative paths:
+Plugins have a convenience function `self.open` which will automatically look for the file if it exists, or open a new one if it doesn't:
 
 
 .. code:: python
@@ -132,7 +133,7 @@ And you can run it with:
    docker run -p 5000:5000 gsiupm/exampleplugin
 
 
-If the plugin uses non-source files (:ref:`loading data and files`), the recommended way is to use `SENPY_DATA` folder.
+If the plugin uses non-source files (:ref:`How should I load external data and files`), the recommended way is to use `SENPY_DATA` folder.
 Data can then be mounted in the container or added to the image.
 The former is recommended for open source plugins with licensed resources, whereas the latter is the most convenient and can be used for private images.
 

docs/requirements.txt

@@ -1,2 +1,3 @@
 sphinxcontrib-httpdomain>=1.4
+ipykernel
 nbsphinx

docs/senpy.rst

@@ -18,10 +18,10 @@ And higher level features can be built on top of these services, such as automat
 
 These benefits are not limited to new services.
 The community has developed wrappers for some proprietary and commercial services (such as sentiment140 and Meaning Cloud), so you can consult them as.
-Senpy comes with a :ref:`built-in client`.
+Senpy comes with a built-in client in the client package.
 
 
 To achieve this goal, Senpy uses a Linked Data principled approach, based on the NIF (NLP Interchange Format) specification, and open vocabularies such as Marl and Onyx.
 You can learn more about this in :doc:`vocabularies`.
 
-Check out :doc:`plugins` if you have developed an analysis algorithm (e.g. sentiment analysis) and you want to publish it as a service.
+Check out :doc:`development` if you have developed an analysis algorithm (e.g. sentiment analysis) and you want to publish it as a service.

senpy/plugins/__init__.py

@@ -251,11 +251,15 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)):
                 return alternative
         raise IOError('File does not exist: {}'.format(fname))
 
+    def path(self, fpath):
+        if not os.path.isabs(fpath):
+            fpath = os.path.join(self.data_folder, fpath)
+        return fpath
+
     def open(self, fpath, mode='r'):
         if 'w' in mode:
             # When writing, only use absolute paths or data_folder
-            if not os.path.isabs(fpath):
-                fpath = os.path.join(self.data_folder, fpath)
+            fpath = self.path(fpath)
         else:
             fpath = self.find_file(fpath)
 
@@ -381,7 +385,9 @@ class SentimentPlugin(Analyser, Evaluable, models.SentimentPlugin):
             activity = self.activity(parameters)
         entries = []
         for feat in X:
-            entries.append(models.Entry(nif__isString=feat[0]))
+            if isinstance(feat, list):
+                feat = ' '.join(feat)
+            entries.append(models.Entry(nif__isString=feat))
         labels = []
         for e in self.process_entries(entries, activity):
             sent = e.sentiments[0].polarity

senpy/static/js/main.js

@@ -203,8 +203,8 @@ $(document).ready(function() {
         draw_datasets();
     }
 
-    // $(window).on('hashchange', hashchanged);
-    // hashchanged();
+    $(window).on('hashchange', hashchanged);
+    hashchanged();
     $('.tooltip-form').tooltip();
 
     $('.nav-pills a').on('shown.bs.tab', function (e) {

tests/test_plugins.py

@@ -320,24 +320,32 @@ class PluginsTest(TestCase):
         for i in range(50):
             testdata.append(["good", 1])
         for i in range(50):
-            testdata.append(["bad", 0])
+            testdata.append(["bad", -1])
         dataset = pd.DataFrame(testdata, columns=['text', 'polarity'])
 
         class DummyPlugin(plugins.SentimentBox):
             description = 'Plugin to test evaluation'
             version = 0
 
+            classes = ['marl:Positive', 'marl:Negative']
+
             def predict_one(self, features, **kwargs):
-                return 0
+                print(features[0])
+                return [0, 1]
 
         class SmartPlugin(plugins.SentimentBox):
             description = 'Plugin to test evaluation'
             version = 0
 
+            classes = ['marl:Positive', 'marl:Negative']
+
             def predict_one(self, features, **kwargs):
+                print(features[0])
                 if features[0] == 'good':
-                    return 1
-                return 0
+                    print('positive')
+                    return [1, 0]
+                print('negative')
+                return [0, 1]
 
         dpipe = DummyPlugin()
         results = plugins.evaluate(datasets={'testdata': dataset}, plugins=[dpipe], flatten=True)