Fix schema issues and parameter validation

.gitlab-ci.yml

@@ -18,8 +18,6 @@ before_script:
   stage: test
   script:
     - make -e test-$PYTHON_VERSION
-  except:
-    - tags # Avoid unnecessary double testing
     
 test-3.5:
   <<: *test_definition

.makefiles/docker.mk

@@ -1,14 +1,5 @@
-ifndef IMAGENAME
-	ifdef CI_REGISTRY_IMAGE
-		IMAGENAME=$(CI_REGISTRY_IMAGE)
-	else
-		IMAGENAME=$(NAME)
-	endif
-endif
-
+IMAGENAME?=$(NAME)
 IMAGEWTAG?=$(IMAGENAME):$(VERSION)
-DOCKER_FLAGS?=$(-ti)
-DOCKER_CMD?=
 
 docker-login: ## Log in to the registry. It will only be used in the server, or when running a CI task locally (if CI_BUILD_TOKEN is set).
 ifeq ($(CI_BUILD_TOKEN),)
@@ -28,19 +19,6 @@ else
 	@docker logout
 endif
 
-docker-run: ## Build a generic docker image
-	docker run $(DOCKER_FLAGS) $(IMAGEWTAG) $(DOCKER_CMD)
-
-docker-build: ## Build a generic docker image
-	docker build . -t $(IMAGEWTAG)
-
-docker-push: docker-login ## Push a generic docker image
-	docker push $(IMAGEWTAG)
-
-docker-latest-push: docker-login ## Push the latest image
-	docker tag $(IMAGEWTAG) $(IMAGENAME)
-	docker push $(IMAGENAME)
-
 login:: docker-login
 
 clean:: docker-clean

.makefiles/git.mk

@@ -14,7 +14,7 @@ push-github: ## Push the code to github. You need to set up GITHUB_DEPLOY_KEY
 ifeq ($(GITHUB_DEPLOY_KEY),)
 else
 	$(eval KEY_FILE := "$(shell mktemp)")
-	@printf '%b' '$(GITHUB_DEPLOY_KEY)' > $(KEY_FILE)
+	@echo "$(GITHUB_DEPLOY_KEY)" > $(KEY_FILE)
 	@git remote rm github-deploy || true
 	git remote add github-deploy $(GITHUB_REPO)
 	-@GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git fetch github-deploy $(CI_COMMIT_REF_NAME)

.makefiles/k8s.mk

@@ -13,7 +13,7 @@
 KUBE_CA_TEMP=false
 ifndef KUBE_CA_PEM_FILE
 KUBE_CA_PEM_FILE:=$$PWD/.ca.crt
-CREATED:=$(shell printf '%b\n' '$(KUBE_CA_BUNDLE)' > $(KUBE_CA_PEM_FILE))
+CREATED:=$(shell echo -e "$(KUBE_CA_BUNDLE)" > $(KUBE_CA_PEM_FILE))
 endif 
 KUBE_TOKEN?=""
 KUBE_NAMESPACE?=$(NAME)

.makefiles/python.mk

@@ -26,7 +26,6 @@ Dockerfile-%: Dockerfile.template  ## Generate a specific dockerfile (e.g. Docke
 quick_build: $(addprefix build-, $(PYMAIN))
 
 build: $(addprefix build-, $(PYVERSIONS)) ## Build all images / python versions
-	docker tag $(IMAGEWTAG)-python$(PYMAIN) $(IMAGEWTAG)
 
 build-%: version Dockerfile-%  ## Build a specific version (e.g. build-2.7)
 	docker build -t '$(IMAGEWTAG)-python$*' -f Dockerfile-$* .;
@@ -76,9 +75,8 @@ pip_upload: pip_test  ## Upload package to pip
 
 push-latest: $(addprefix push-latest-,$(PYVERSIONS)) ## Push the "latest" tag to dockerhub
 	docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGEWTAG)'
-	docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGENAME):latest'
+	docker tag '$(IMAGEWTAG)-python$(PYMAIN)' '$(IMAGENAME)'
 	docker push '$(IMAGENAME):latest'
-	docker push '$(IMAGEWTAG)'
 
 push-latest-%: build-%  ## Push the latest image for a specific python version
 	docker tag $(IMAGENAME):$(VERSION)-python$* $(IMAGENAME):python$*

docker-compose.dev.yml

@@ -1,10 +0,0 @@
-version: '3'
-services:
-    senpy:
-        image: "${IMAGENAME-gsiupm/senpy}:${VERSION-latest}"
-        entrypoint: ["/bin/bash"]
-        working_dir: "/senpy-plugins"
-        ports:
-            - 5000:5000
-        volumes:
-            - ".:/usr/src/app/"

docker-compose.test.yml

@@ -1,9 +0,0 @@
-version: '3'
-services:
-    test:
-        image: "${IMAGENAME-gsiupm/senpy}:${VERSION-dev}"
-        entrypoint: ["py.test"]
-        volumes:
-            - ".:/usr/src/app/"
-        command:
-          []

docker-compose.yml

@@ -1,11 +0,0 @@
-version: '3'
-services:
-    senpy:
-        image: "${IMAGENAME-gsiupm/senpy}:${VERSION-dev}"
-        build:
-          context: .
-          dockerfile: Dockerfile${PYVERSION--2.7}
-        ports:
-            - 5001:5000
-        volumes:
-            - "./data:/data"

example-plugins/configurable_plugin.py

@@ -43,6 +43,7 @@ class Dictionary(plugins.SentimentPlugin):
 
 class EmojiOnly(Dictionary):
     '''Sentiment annotation with a basic lexicon of emojis'''
+    description = 'A plugin'
     dictionaries = [basic.emojis]
 
     test_cases = [{

senpy/__main__.py

@@ -105,12 +105,6 @@ def main():
         action='store_true',
         default=False,
         help='Output the senpy version and exit')
-    parser.add_argument(
-        '--allow-fail',
-        '--fail',
-        action='store_true',
-        default=False,
-        help='Do not exit if some plugins fail to activate')
     args = parser.parse_args()
     if args.version:
         print('Senpy version {}'.format(senpy.__version__))
@@ -134,9 +128,9 @@ def main():
     sp.install_deps()
     if args.only_install:
         return
-    sp.activate_all(allow_fail=args.allow_fail)
+    sp.activate_all()
     if args.only_test:
-        easy_test(sp.plugins(), debug=args.debug)
+        easy_test(sp.plugins())
         return
     print('Senpy version {}'.format(senpy.__version__))
     print('Server running on port %s:%d. Ctrl+C to quit' % (args.host,

senpy/blueprints.py

@@ -47,12 +47,7 @@ def get_params(req):
 
 @demo_blueprint.route('/')
 def index():
-    ev = str(get_params(request).get('evaluation', False))
-    evaluation_enabled = ev.lower() not in ['false', 'no', 'none']
-
-    return render_template("index.html",
-                           evaluation=evaluation_enabled,
-                           version=__version__)
+    return render_template("index.html", version=__version__)
 
 
 @api_blueprint.route('/contexts/<entity>.jsonld')

senpy/extensions.py

@@ -318,15 +318,10 @@ class Senpy(object):
         else:
             self._default = self._plugins[value.lower()]
 
-    def activate_all(self, sync=True, allow_fail=False):
+    def activate_all(self, sync=True):
         ps = []
         for plug in self._plugins.keys():
-            try:
-                self.activate_plugin(plug, sync=sync)
-            except Exception as ex:
-                if not allow_fail:
-                    raise
-                logger.error('Could not activate {}: {}'.format(plug, ex))
+            ps.append(self.activate_plugin(plug, sync=sync))
         return ps
 
     def deactivate_all(self, sync=True):

senpy/models.py

@@ -203,27 +203,24 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)):
                context_uri=None,
                prefix=None,
                expanded=False):
+        ser = self.serializable()
 
-        result = self.serializable()
-        if context_uri or with_context:
-            result['@context'] = context_uri or self._context
-
-        # result = jsonld.compact(result,
-        #                         self._context,
-        #                         options={
-        #                             'base': prefix,
-        #                             'expandContext': self._context,
-        #                             'senpy': prefix
-        #                         })
+        result = jsonld.compact(
+            ser,
+            self._context,
+            options={
+                'base': prefix,
+                'expandContext': self._context,
+                'senpy': prefix
+            })
+        if context_uri:
+            result['@context'] = context_uri
         if expanded:
             result = jsonld.expand(
                 result, options={'base': prefix,
                                  'expandContext': self._context})
         if not with_context:
-            try:
-                del result['@context']
-            except KeyError:
-                pass
+            del result['@context']
         return result
 
     def validate(self, obj=None):
@@ -326,10 +323,7 @@ def _add_class_from_schema(*args, **kwargs):
 
 
 for i in [
-        'aggregatedEvaluation',
         'analysis',
-        'dataset',
-        'datasets',
         'emotion',
         'emotionConversion',
         'emotionConversionPlugin',
@@ -337,17 +331,19 @@ for i in [
         'emotionModel',
         'emotionPlugin',
         'emotionSet',
-        'evaluation',
         'entity',
         'help',
-        'metric',
         'plugin',
         'plugins',
         'response',
         'results',
         'sentimentPlugin',
         'suggestion',
-        'topic',
+        'aggregatedEvaluation',
+        'evaluation',
+        'metric',
+        'dataset',
+        'datasets',
 
 ]:
     _add_class_from_schema(i)

senpy/plugins/__init__.py

@@ -18,7 +18,6 @@ import subprocess
 import importlib
 import yaml
 import threading
-import nltk
 
 from .. import models, utils
 from .. import api
@@ -95,27 +94,7 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)):
         self.id = 'plugins/{}_{}'.format(self['name'], self['version'])
         self.is_activated = False
         self._lock = threading.Lock()
-        self._directory = os.path.abspath(os.path.dirname(inspect.getfile(self.__class__)))
-
-        data_folder = data_folder or os.getcwd()
-        subdir = os.path.join(data_folder, self.name)
-
-        self._data_paths = [
-            data_folder,
-            subdir,
-            self._directory,
-            os.path.join(self._directory, 'data'),
-        ]
-
-        if os.path.exists(subdir):
-            data_folder = subdir
-        self.data_folder = data_folder
-
-        self._log = logging.getLogger('{}.{}'.format(__name__, self.name))
-
-    @property
-    def log(self):
-        return self._log
+        self.data_folder = data_folder or os.getcwd()
 
     def validate(self):
         missing = []
@@ -144,9 +123,9 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)):
         for case in test_cases:
             try:
                 self.test_case(case)
-                self.log.debug('Test case passed:\n{}'.format(pprint.pformat(case)))
+                logger.debug('Test case passed:\n{}'.format(pprint.pformat(case)))
             except Exception as ex:
-                self.log.warn('Test case failed:\n{}'.format(pprint.pformat(case)))
+                logger.warn('Test case failed:\n{}'.format(pprint.pformat(case)))
                 raise
 
     def test_case(self, case):
@@ -169,22 +148,10 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)):
             raise
         assert not should_fail
 
-    def find_file(self, fname):
-        for p in self._data_paths:
-            alternative = os.path.join(p, fname)
-            if os.path.exists(alternative):
-                return alternative
-        raise IOError('File does not exist: {}'.format(fname))
-
-    def open(self, fpath, mode='r'):
-        if 'w' in mode:
-            # When writing, only use absolute paths or data_folder
-            if not os.path.isabs(fpath):
-                fpath = os.path.join(self.data_folder, fpath)
-        else:
-            fpath = self.find_file(fpath)
-
-        return open(fpath, mode=mode)
+    def open(self, fpath, *args, **kwargs):
+        if not os.path.isabs(fpath):
+            fpath = os.path.join(self.data_folder, fpath)
+        return open(fpath, *args, **kwargs)
 
     def serve(self, debug=True, **kwargs):
         utils.easy(plugin_list=[self, ], plugin_folder=None, debug=debug, **kwargs)
@@ -219,7 +186,7 @@ class Analysis(Plugin):
 
     def analyse_entries(self, entries, parameters):
         for entry in entries:
-            self.log.debug('Analysing entry with plugin {}: {}'.format(self, entry))
+            logger.debug('Analysing entry with plugin {}: {}'.format(self, entry))
             results = self.analyse_entry(entry, parameters)
             if inspect.isgenerator(results):
                 for result in results:
@@ -408,7 +375,7 @@ class ShelfMixin(object):
                     with self.open(self.shelf_file, 'rb') as p:
                         self._sh = pickle.load(p)
                 except (IndexError, EOFError, pickle.UnpicklingError):
-                    self.log.warning('Corrupted shelf file: {}'.format(self.shelf_file))
+                    logger.warning('{} has a corrupted shelf file!'.format(self.id))
                     if not self.get('force_shelf', False):
                         raise
         return self._sh
@@ -435,31 +402,32 @@ class ShelfMixin(object):
         self._shelf_file = value
 
     def save(self):
-        self.log.debug('Saving pickle')
+        logger.debug('saving pickle')
         if hasattr(self, '_sh') and self._sh is not None:
             with self.open(self.shelf_file, 'wb') as f:
                 pickle.dump(self._sh, f)
 
 
-def pfilter(plugins, plugin_type=Analysis, **kwargs):
+def pfilter(plugins, **kwargs):
     """ Filter plugins by different criteria """
     if isinstance(plugins, models.Plugins):
         plugins = plugins.plugins
     elif isinstance(plugins, dict):
         plugins = plugins.values()
+    ptype = kwargs.pop('plugin_type', Plugin)
     logger.debug('#' * 100)
-    logger.debug('plugin_type {}'.format(plugin_type))
-    if plugin_type:
-        if isinstance(plugin_type, PluginMeta):
-            plugin_type = plugin_type.__name__
+    logger.debug('ptype {}'.format(ptype))
+    if ptype:
+        if isinstance(ptype, PluginMeta):
+            ptype = ptype.__name__
         try:
-            plugin_type = plugin_type[0].upper() + plugin_type[1:]
-            pclass = globals()[plugin_type]
+            ptype = ptype[0].upper() + ptype[1:]
+            pclass = globals()[ptype]
             logger.debug('Class: {}'.format(pclass))
             candidates = filter(lambda x: isinstance(x, pclass),
                                 plugins)
         except KeyError:
-            raise models.Error('{} is not a valid type'.format(plugin_type))
+            raise models.Error('{} is not a valid type'.format(ptype))
     else:
         candidates = plugins
 
@@ -494,7 +462,6 @@ def _log_subprocess_output(process):
 
 def install_deps(*plugins):
     installed = False
-    nltk_resources = set()
     for info in plugins:
         requirements = info.get('requirements', [])
         if requirements:
@@ -509,10 +476,7 @@ def install_deps(*plugins):
             exitcode = process.wait()
             installed = True
             if exitcode != 0:
-                raise models.Error("Dependencies not properly installed: {}".format(pip_args))
-        nltk_resources |= set(info.get('nltk_resources', []))
-
-    installed |= nltk.download(list(nltk_resources))
+                raise models.Error("Dependencies not properly installed")
     return installed
 
 
@@ -609,14 +573,12 @@ def _instances_in_module(module):
 def _from_module_name(module, root, info=None, install=True, **kwargs):
     try:
         module = load_module(module, root)
-    except (ImportError, LookupError):
+    except ImportError:
         if not install or not info:
             raise
         install_deps(info)
         module = load_module(module, root)
     for plugin in _from_loaded_module(module=module, root=root, info=info, **kwargs):
-        if install:
-            install_deps(plugin)
         yield plugin
 
 

senpy/plugins/misc/split.py

@@ -5,27 +5,13 @@ from nltk.tokenize.simple import LineTokenizer
 import nltk
 
 
-class Split(AnalysisPlugin):
+class SplitPlugin(AnalysisPlugin):
     '''description: A sample plugin that chunks input text'''
 
-    author = ["@militarpancho", '@balkian']
-    version = '0.2'
-    url = "https://github.com/gsi-upm/senpy"
-
-    extra_params = {
-        'delimiter': {
-            'aliases': ['type', 't'],
-            'required': False,
-            'default': 'sentence',
-            'options': ['sentence', 'paragraph']
-        },
-    }
-
     def activate(self):
         nltk.download('punkt')
 
     def analyse_entry(self, entry, params):
-        yield entry
         chunker_type = params["delimiter"]
         original_text = entry['nif:isString']
         if chunker_type == "sentence":

senpy/plugins/misc/split.senpy

@@ -0,0 +1,19 @@
+---
+name: split
+module: senpy.plugins.misc.split
+description: A sample plugin that chunks input text
+author: "@militarpancho"
+version: '0.2'
+url: "https://github.com/gsi-upm/senpy"
+requirements:
+    - nltk
+extra_params:
+  delimiter:
+    aliases:
+    - type
+    - t
+    required: false
+    default: sentence
+    options:
+    - sentence
+    - paragraph

senpy/plugins/sentiment/sentiment140/sentiment140.py

@@ -5,25 +5,8 @@ from senpy.plugins import SentimentPlugin
 from senpy.models import Sentiment
 
 
-class Sentiment140(SentimentPlugin):
+class Sentiment140Plugin(SentimentPlugin):
     '''Connects to the sentiment140 free API: http://sentiment140.com'''
-
-    author = "@balkian"
-    version = '0.2'
-    url = "https://github.com/gsi-upm/senpy-plugins-community"
-    extra_params = {
-        'language': {
-            "@id": 'lang_sentiment140',
-            'aliases': ['language', 'l'],
-            'required': False,
-            'default': 'auto',
-            'options': ['es', 'en', 'auto']
-        }
-    }
-
-    maxPolarityValue = 1
-    minPolarityValue = 0
-
     def analyse_entry(self, entry, params):
         lang = params["language"]
         res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
@@ -48,6 +31,7 @@ class Sentiment140(SentimentPlugin):
             marl__hasPolarity=polarity,
             marl__polarityValue=polarity_value)
         sentiment.prov__wasGeneratedBy = self.id
+        entry.sentiments = []
         entry.sentiments.append(sentiment)
         entry.language = lang
         yield entry
@@ -57,10 +41,10 @@ class Sentiment140(SentimentPlugin):
         To avoid calling the sentiment140 API, we will mock the results
         from requests.
         '''
-        from senpy.testing import patch_requests
+        from senpy.test import patch_requests
         expected = {"data": [{"polarity": 4}]}
         with patch_requests(expected) as (request, response):
-            super(Sentiment140, self).test(*args, **kwargs)
+            super(Sentiment140Plugin, self).test(*args, **kwargs)
             assert request.called
             assert response.json.called
 

senpy/plugins/sentiment/sentiment140/sentiment140.senpy

@@ -0,0 +1,22 @@
+---
+name: sentiment140
+module: sentiment140
+description: "Connects to the sentiment140 free API: http://sentiment140.com"
+author: "@balkian"
+version: '0.2'
+url: "https://github.com/gsi-upm/senpy-plugins-community"
+extra_params:
+  language:
+    "@id": lang_sentiment140
+    aliases:
+    - language
+    - l
+    required: false
+    options:
+    - es
+    - en
+    - auto
+    default: auto
+requirements: {}
+maxPolarityValue: 1
+minPolarityValue: 0

senpy/schemas/context.jsonld

@@ -10,10 +10,8 @@
     "wna": "http://www.gsi.dit.upm.es/ontologies/wnaffect/ns#",
     "emoml": "http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/emotionml/ns#",
     "xsd": "http://www.w3.org/2001/XMLSchema#",
-    "fam": "http://vocab.fusepool.info/fam#",
     "topics": {
-      "@id": "nif:topic",
-      "@container": "@set"
+      "@id": "dc:subject"
     },
     "entities": {
       "@id": "me:hasEntities"

senpy/static/css/main.css

@@ -167,36 +167,3 @@ textarea{
     color: inherit;
     text-decoration: inherit;
 }
-
-.collapsed .collapseicon {
-    display: none !important;
-}
-
-.collapsed .expandicon {
-    display: inline-block !important;
-}
-
-.expandicon {
-    display: none !important;
-}
-
-.collapseicon {
-    display: inline-block !important;
-}
-
-.loader {
-    border: 6px solid #f3f3f3; /* Light grey */
-    border-top: 6px solid blue;
-    border-bottom: 6px solid blue;
-
-    border-radius: 50%;
-    width: 3em;
-    height: 3em;
-    animation: spin 2s linear infinite;
-
-}
-
-@keyframes spin {
-    0% { transform: rotate(0deg); }
-    100% { transform: rotate(360deg); }
-}

senpy/static/js/main.js

@@ -4,7 +4,6 @@ var plugins_params = default_params = {};
 var plugins = [];
 var defaultPlugin = {};
 var gplugins = {};
-var pipeline = [];
 
 function replaceURLWithHTMLLinks(text) {
     console.log('Text: ' + text);
@@ -31,10 +30,7 @@ function hashchanged(){
 
 
 function get_plugins(response){
-    for(ix in response.plugins){
-        plug = response.plugins[ix];
-        plugins[plug.name] = plug;
-    }
+    plugins = response.plugins;
 }
 
 function get_datasets(response){
@@ -87,32 +83,10 @@ function draw_plugins_selection(){
     html += "</optgroup>"
     // Two elements with plugin class
     // One from the evaluate tab and another one from the analyse tab
-    plugin_lists  = document.getElementsByClassName('plugin')
-    for (element in plugin_lists){
-        plugin_lists[element].innerHTML = html;
-    }
-    draw_plugin_pipeline();
+    document.getElementsByClassName('plugin')[0].innerHTML = html;
+    document.getElementsByClassName('plugin')[1].innerHTML = html;
 }
 
-function draw_plugin_pipeline(){
-    var pipeHTML = "";
-    console.log("Drawing pipeline: ", pipeline);
-    for (ix in pipeline){
-        plug = pipeline[ix];
-        pipeHTML += '<span onclick="remove_plugin_pipeline(\'' + plug + '\')" class="btn btn-primary"><span ><i class="fa fa-minus"></i></span> ' + plug + '</span> <i class="fa fa-arrow-right"></i> ';
-    }
-    console.log(pipeHTML);
-    $("#pipeline").html(pipeHTML);
-}
-
-
-function remove_plugin_pipeline(name){
-    console.log("Removing plugin: ", name);
-    var index = pipeline.indexOf(name);
-    pipeline.splice(index, 1);
-    draw_plugin_pipeline();
-
-}
 function draw_plugins_list(){
     var availablePlugins = document.getElementById('availablePlugins');
 
@@ -131,13 +105,6 @@ function draw_plugins_list(){
     }
 }
 
-function add_plugin_pipeline(){
-    var selected = get_selected_plugin();
-    pipeline.push(selected);
-    console.log("Adding ", selected);
-    draw_plugin_pipeline();
-}
-
 function draw_datasets(){
     html = "";
     repeated_html = "<input class=\"checks-datasets\" type=\"checkbox\" value=\"";
@@ -151,20 +118,16 @@ function draw_datasets(){
 $(document).ready(function() {
     var response = JSON.parse($.ajax({type: "GET", url: "/api/plugins/" , async: false}).responseText);
     defaultPlugin= JSON.parse($.ajax({type: "GET", url: "/api/plugins/default" , async: false}).responseText);
+    var response2 = JSON.parse($.ajax({type: "GET", url: "/api/datasets/" , async: false}).responseText);
 
     get_plugins(response);
     get_default_parameters();
+    get_datasets(response2);
 
     draw_plugins_list();
     draw_plugins_selection();
     draw_parameters();
-    draw_plugin_description();
-
-    if (evaluation_enabled) {
-        var response2 = JSON.parse($.ajax({type: "GET", url: "/api/datasets/" , async: false}).responseText);
-        get_datasets(response2);
-        draw_datasets();
-    }
+    draw_datasets();
 
     $(window).on('hashchange', hashchanged);
     hashchanged();
@@ -181,34 +144,17 @@ function get_default_parameters(){
 
 }
 
-function get_selected_plugin(){
-    return document.getElementsByClassName('plugin')[0].options[document.getElementsByClassName('plugin')[0].selectedIndex].value;
-}
-
 function draw_default_parameters(){
     var basic_params = document.getElementById("basic_params");
     basic_params.innerHTML = params_div(default_params);
 }
 
-function update_params(params, plug){
-    ep = plugins_params[plug];
-    for(k in ep){
-        params[k] = ep[k];
-    }
-    return params
-}
-
 function draw_extra_parameters(){
-    var plugin = get_selected_plugin();
+    var plugin = document.getElementsByClassName('plugin')[0].options[document.getElementsByClassName('plugin')[0].selectedIndex].value;
     get_parameters();
 
     var extra_params = document.getElementById("extra_params");
-    var params = {};
-    for (sel in pipeline){
-        update_params(params, pipeline[sel]);
-    }
-    update_params(params, plugin);
-    extra_params.innerHTML = params_div(params);
+    extra_params.innerHTML = params_div(plugins_params[plugin]);
 }
 
 function draw_parameters(){
@@ -315,15 +261,6 @@ function add_param(key, value){
     return "&"+key+"="+value;
 }
 
-function get_pipeline_arg(){
-    arg = "";
-    for (ix in pipeline){
-        arg = arg + pipeline[ix] + ",";
-    }
-    arg = arg + get_selected_plugin();
-    return arg;
-}
-
 
 function load_JSON(){
     url = "/api";
@@ -332,9 +269,7 @@ function load_JSON(){
     rawcontainer.innerHTML = '';
     container.innerHTML = '';
 
-    var plugin = get_pipeline_arg();
-    $(".loading").addClass("loader");
-    $("#preview").hide();
+    var plugin = document.getElementsByClassName("plugin")[0].options[document.getElementsByClassName("plugin")[0].selectedIndex].value;
 
     var input = encodeURIComponent(document.getElementById("input").value);
     url += "?algo="+plugin+"&i="+input
@@ -345,27 +280,27 @@ function load_JSON(){
         url += add_param(key, params[key]);
     }
 
-    $.ajax({type: "GET", url: url}).always(function(response){
-        document.getElementById("results-div").style.display = 'block';
-        if(typeof response=="object") {
-            var options = {
-                mode: 'view'
-            };
-            var editor = new JSONEditor(container, options, response);
-            editor.expandAll();
-            $('#results-div a[href="#viewer"]').click();
-            response = JSON.stringify(response, null, 4);
-        } else {
-            console.log("Got turtle?");
-            $('#results-div a[href="#raw"]').click();
-        }
+    var response =  $.ajax({type: "GET", url: url , async: false}).responseText;
+    rawcontainer.innerHTML = replaceURLWithHTMLLinks(response);
 
-        rawcontainer.innerHTML = replaceURLWithHTMLLinks(response);
-        document.getElementById("input_request").innerHTML = "<a href='"+url+"'>"+url+"</a>"
-
-        $(".loading").removeClass("loader");
-        $("#preview").show();
-    });
+    document.getElementById("input_request").innerHTML = "<a href='"+url+"'>"+url+"</a>"
+    document.getElementById("results-div").style.display = 'block';
+    try {
+        response = JSON.parse(response);
+        var options = {
+            mode: 'view'
+        };
+        var editor = new JSONEditor(container, options, response);
+        editor.expandAll();
+        // $('#results-div a[href="#viewer"]').tab('show');
+        $('#results-div a[href="#viewer"]').click();
+        // location.hash = 'raw';
+    }
+    catch(err){
+        console.log("Error decoding JSON (got turtle?)");
+        $('#results-div a[href="#raw"]').click();
+        // location.hash = 'raw';
+    }
 }
 
 function get_datasets_from_checkbox(){
@@ -412,53 +347,40 @@ function evaluate_JSON(){
 
     url += "?algo="+plugin+"&dataset="+datasets
 
-    $('#doevaluate').attr("disabled", true);
-    $.ajax({type: "GET", url: url, dataType: 'json'}).done(function(resp) {
-        $('#doevaluate').attr("disabled", false);
-        response = resp.responseText;  
-
-        rawcontainer.innerHTML = replaceURLWithHTMLLinks(response);
-
-        document.getElementById("input_request_eval").innerHTML = "<a href='"+url+"'>"+url+"</a>"
-        document.getElementById("evaluate-div").style.display = 'block';
-
-        try {
-            response = JSON.parse(response);
-            var options = {
-                mode: 'view'
-            };
-
-            //Control the single response results
-            if (!(Array.isArray(response.evaluations))){
-                response.evaluations = [response.evaluations]
-            }
-
-            new_tbody = create_body_metrics(response.evaluations)
-            table.replaceChild(new_tbody, table.lastElementChild)
-
-            var editor = new JSONEditor(container, options, response);
-            editor.expandAll();
-            // $('#results-div a[href="#viewer"]').tab('show');
-            $('#evaluate-div a[href="#evaluate-table"]').click();
-            // location.hash = 'raw';
+    var response =  $.ajax({type: "GET", url: url , async: false, dataType: 'json'}).responseText;
+    rawcontainer.innerHTML = replaceURLWithHTMLLinks(response);
 
+    document.getElementById("input_request_eval").innerHTML = "<a href='"+url+"'>"+url+"</a>"
+    document.getElementById("evaluate-div").style.display = 'block';
+    
+    try {
+        response = JSON.parse(response);
+        var options = {
+            mode: 'view'
+        };
 
+        //Control the single response results
+        if (!(Array.isArray(response.evaluations))){
+            response.evaluations = [response.evaluations]
         }
-        catch(err){
-            console.log("Error decoding JSON (got turtle?)");
-            $('#evaluate-div a[href="#evaluate-raw"]').click();
-            // location.hash = 'raw';
-        }
-    })
-}
 
-function draw_plugin_description(){
-    var plugin = plugins[get_selected_plugin()];
-    $("#plugdescription").text(plugin.description);
-    console.log(plugin);
-}
+        new_tbody = create_body_metrics(response.evaluations)
+        table.replaceChild(new_tbody, table.lastElementChild)
 
-function plugin_selected(){
-    draw_extra_parameters();
-    draw_plugin_description();
-}
+        var editor = new JSONEditor(container, options, response);
+        editor.expandAll();
+        // $('#results-div a[href="#viewer"]').tab('show');
+        $('#evaluate-div a[href="#evaluate-table"]').click();
+        // location.hash = 'raw';
+        
+        
+    }
+    catch(err){
+        console.log("Error decoding JSON (got turtle?)");
+        $('#evaluate-div a[href="#evaluate-raw"]').click();
+        // location.hash = 'raw';
+    }
+
+    
+
+}

senpy/templates/index.html

@@ -5,9 +5,6 @@
     <title>Playground {{version}}</title>
 
   </head>
-  <script>
-   this.evaluation_enabled = {% if evaluation %}true{%else %}false{%endif%};
-  </script>
   <script src="static/js/jquery-2.1.1.min.js" ></script>
   <!--<script src="jquery.autosize.min.js"></script>-->
   <link rel="stylesheet" href="static/css/bootstrap.min.css">
@@ -35,9 +32,7 @@
       <ul class="nav nav-tabs" role="tablist">
         <li role="presentation" ><a class="active" href="#about">About</a></li>
         <li role="presentation"class="active"><a class="active" href="#test">Test it</a></li>
-        {% if evaluation %}
         <li role="presentation"><a class="active" href="#evaluate">Evaluate Plugins</a></li>
-        {% endif %}
 
       </ul>
 
@@ -66,14 +61,6 @@
                     </ul>
 
                   </p>
-                  <p>Senpy is a research project. If you use it in your research, please cite:
-                  <pre>
-Senpy: A Pragmatic Linked Sentiment Analysis Framework.
-Sánchez-Rada, J. F., Iglesias, C. A., Corcuera, I., & Araque, Ó.
-In Data Science and Advanced Analytics (DSAA),
-2016 IEEE International Conference on (pp. 735-742). IEEE.
-                  </pre>
-                  </p>
                   
                 </div>
             <div class="col-lg-6 ">
@@ -83,6 +70,8 @@ In Data Science and Advanced Analytics (DSAA),
                 </div>
                 <div class="panel-body"><ul id=availablePlugins></ul></div>
               </div>
+            </div>
+            <div class="col-lg-6 ">
               <a href="http://senpy.readthedocs.io">
               <div class="panel panel-default">
                 <div class="panel-heading"><i class="fa fa-book"></i> If you are new to senpy, you might want to read senpy's documentation</div>
@@ -93,6 +82,9 @@ In Data Science and Advanced Analytics (DSAA),
                 <div class="panel-heading"><i class="fa fa-sign-in"></i> Feel free to follow us on GitHub</div>
               </div>
               </a>
+              <div class="panel panel-default">
+                <div class="panel-heading"><i class="fa fa-child"></i> Enjoy.</div>
+              </div>
             </div>     
           </div>
         </div>
@@ -104,28 +96,17 @@ In Data Science and Advanced Analytics (DSAA),
 whilst this text makes me happy and surprised at the same time.
 I cannot believe it!</textarea>
                 </div>
+                <div>
+                    <label>Select the plugin:</label>
+                    <select id="plugins" name="plugins" class=plugin onchange="draw_extra_parameters()">
+                    </select>
+                </div>
                 <!-- PARAMETERS -->
                 <div class="panel-group" id="parameters">
                     <div class="panel panel-default">
-                        <div class="panel-heading">
-                            <h4 class="panel-title">
-                                Select the plugin.
-                            </h4>
-                        </div>
-                        <div id="plugin_selection" class="panel-collapse panel-body">
-                            <span id="pipeline"></span>
-                            <select name="plugins" class="plugin" onchange="plugin_selected()">
-                            </select>
-                            <span onclick="add_plugin_pipeline()"><span class="btn"><i class="fa fa-plus" title="Add more plugins to the pipeline. Processing order is left to right. i.e. the results of the leftmost plugin will be used as input for the second leftmost, and so on."></i></span></span>
-                            <label class="help-block " id="plugdescription"></label>
-                        </div>
-                    </div>
-                    <div class="panel panel-default">
-                        <a data-toggle="collapse" class="deco-none collapsed" href="#basic_params">
+                        <a data-toggle="collapse" class="deco-none" href="#basic_params">
                             <div class="panel-heading">
                                 <h4 class="panel-title">
-                                    <i class="fa fa-chevron-right pull-left expandicon"></i>
-                                    <i class="fa fa-chevron-down pull-left collapseicon"></i>
                                     Basic API parameters
                                 </h4>
                             </div>
@@ -137,8 +118,6 @@ I cannot believe it!</textarea>
                         <a data-toggle="collapse" class="deco-none" href="#extra_params">
                             <div class="panel-heading">
                                 <h4 class="panel-title">
-                                    <i class="fa fa-chevron-right pull-left expandicon"></i>
-                                    <i class="fa fa-chevron-down pull-left collapseicon"></i>
                                     Plugin extra parameters
                                 </h4>
                             </div>
@@ -150,7 +129,6 @@ I cannot believe it!</textarea>
                 <!-- END PARAMETERS -->
  
                 <a id="preview" class="btn btn-lg btn-primary" onclick="load_JSON()">Analyse!</a>
-                <div id="loading-results" class="loading"></div>
                 <!--<button id="visualise" name="type" type="button">Visualise!</button>-->
             </form>
           </div>
@@ -177,8 +155,6 @@ I cannot believe it!</textarea>
             </div>
         </div>
 
-        {% if evaluation %}
-
         <div class="tab-pane" id="evaluate">
           <div class="well">
             <form id="form" class="container" onsubmit="return getPlugins();" accept-charset="utf-8">
@@ -193,7 +169,7 @@ I cannot believe it!</textarea>
                     </select>
                 </div>
  
-                <a id="doevaluate" class="btn btn-lg btn-primary" onclick="evaluate_JSON()">Evaluate Plugin!</a>
+                <a id="preview" class="btn btn-lg btn-primary" onclick="evaluate_JSON()">Evaluate Plugin!</a>
                 <!--<button id="visualise" name="type" type="button">Visualise!</button>-->
             </form>
           </div>
@@ -240,7 +216,6 @@ I cannot believe it!</textarea>
                 </div>
               </div>
             </div>
-            {% endif %}
         </div>
 
         <a href="http://www.gsi.dit.upm.es" target="_blank"><img class="center-block" src="static/img/gsi.png"/>  </a> 

senpy/utils.py

@@ -1,7 +1,6 @@
 from . import models, __version__
 from collections import MutableMapping
 import pprint
-import pdb
 
 import logging
 logger = logging.getLogger(__name__)
@@ -33,8 +32,8 @@ def check_template(indict, template):
         if indict != template:
             raise models.Error(('Differences found.\n'
                                 '\tExpected: {}\n'
-                                '\tFound: {}').format(pprint.pformat(template),
-                                                      pprint.pformat(indict)))
+                                '\tFound: {}').format(pprint.pformat(indict),
+                                                      pprint.pformat(template)))
 
 
 def convert_dictionary(original, mappings):
@@ -68,23 +67,18 @@ def easy_load(app=None, plugin_list=None, plugin_folder=None, **kwargs):
     return sp, app
 
 
-def easy_test(plugin_list=None, debug=True):
+def easy_test(plugin_list=None):
     logger.setLevel(logging.DEBUG)
     logging.getLogger().setLevel(logging.INFO)
-    try:
-        if not plugin_list:
-            import __main__
-            logger.info('Loading classes from {}'.format(__main__))
-            from . import plugins
-            plugin_list = plugins.from_module(__main__)
-        for plug in plugin_list:
-            plug.test()
-            plug.log.info('My tests passed!')
-            logger.info('All tests passed!')
-    except Exception:
-        if not debug:
-            raise
-        pdb.post_mortem()
+    if not plugin_list:
+        import __main__
+        logger.info('Loading classes from {}'.format(__main__))
+        from . import plugins
+        plugin_list = plugins.from_module(__main__)
+    for plug in plugin_list:
+        plug.test()
+        logger.info('The tests for {} passed!'.format(plug.name))
+    logger.info('All tests passed!')
 
 
 def easy(host='0.0.0.0', port=5000, debug=True, **kwargs):

tests/test_blueprints.py

@@ -36,7 +36,7 @@ class BlueprintsTest(TestCase):
 
     def test_playground(self):
         resp = self.client.get("/")
-        assert "main.js" in resp.get_data(as_text=True)
+        assert "main.js" in resp.data.decode()
 
     def test_home(self):
         """

tests/test_client.py

@@ -1,6 +1,6 @@
 from unittest import TestCase
 
-from senpy.testing import patch_requests
+from senpy.test import patch_requests
 from senpy.client import Client
 from senpy.models import Results, Plugins, Error
 from senpy.plugins import AnalysisPlugin

tests/test_extensions.py

@@ -47,7 +47,7 @@ class ExtensionsTest(TestCase):
 
     def test_add_delete(self):
         '''Should be able to add and delete new plugins. '''
-        new = plugins.Analysis(name='new', description='new', version=0)
+        new = plugins.Plugin(name='new', description='new', version=0)
         self.senpy.add_plugin(new)
         assert new in self.senpy.plugins()
         self.senpy.delete_plugin(new)

tests/test_test.py

@@ -2,7 +2,7 @@ from unittest import TestCase
 
 import requests
 import json
-from senpy.testing import patch_requests
+from senpy.test import patch_requests
 from senpy.models import Results