UI improvements

* Add option to add multiple plugins
* Improve UI hints for collapsed parameters
* Refactored plugins without requirements
* Hide evaluation tab for the moment. You can see it by adding "?evaluation" to
  the URL.

example-plugins/configurable_plugin.py

@@ -43,7 +43,6 @@ class Dictionary(plugins.SentimentPlugin):
 
 class EmojiOnly(Dictionary):
     '''Sentiment annotation with a basic lexicon of emojis'''
-    description = 'A plugin'
     dictionaries = [basic.emojis]
 
     test_cases = [{

senpy/blueprints.py

@@ -47,7 +47,12 @@ def get_params(req):
 
 @demo_blueprint.route('/')
 def index():
-    return render_template("index.html", version=__version__)
+    ev = str(get_params(request).get('evaluation', False))
+    evaluation_enabled = ev.lower() not in ['false', 'no', 'none']
+
+    return render_template("index.html",
+                           evaluation=evaluation_enabled,
+                           version=__version__)
 
 
 @api_blueprint.route('/contexts/<entity>.jsonld')

senpy/plugins/__init__.py

@@ -509,7 +509,7 @@ def install_deps(*plugins):
             exitcode = process.wait()
             installed = True
             if exitcode != 0:
-                raise models.Error("Dependencies not properly installed")
+                raise models.Error("Dependencies not properly installed: {}".format(pip_args))
         nltk_resources |= set(info.get('nltk_resources', []))
 
     installed |= nltk.download(list(nltk_resources))

senpy/plugins/misc/split.senpy

@@ -1,19 +0,0 @@
----
-name: split
-module: senpy.plugins.misc.split
-description: A sample plugin that chunks input text
-author: "@militarpancho"
-version: '0.2'
-url: "https://github.com/gsi-upm/senpy"
-requirements:
-    - nltk
-extra_params:
-  delimiter:
-    aliases:
-    - type
-    - t
-    required: false
-    default: sentence
-    options:
-    - sentence
-    - paragraph

senpy/plugins/misc/split_plugin.py

@@ -5,13 +5,27 @@ from nltk.tokenize.simple import LineTokenizer
 import nltk
 
 
-class SplitPlugin(AnalysisPlugin):
+class Split(AnalysisPlugin):
     '''description: A sample plugin that chunks input text'''
 
+    author = ["@militarpancho", '@balkian']
+    version = '0.2'
+    url = "https://github.com/gsi-upm/senpy"
+
+    extra_params = {
+        'delimiter': {
+            'aliases': ['type', 't'],
+            'required': False,
+            'default': 'sentence',
+            'options': ['sentence', 'paragraph']
+        },
+    }
+
     def activate(self):
         nltk.download('punkt')
 
     def analyse_entry(self, entry, params):
+        yield entry
         chunker_type = params["delimiter"]
         original_text = entry['nif:isString']
         if chunker_type == "sentence":

senpy/plugins/sentiment/sentiment140/sentiment140.senpy

@@ -1,22 +0,0 @@
----
-name: sentiment140
-module: sentiment140
-description: "Connects to the sentiment140 free API: http://sentiment140.com"
-author: "@balkian"
-version: '0.2'
-url: "https://github.com/gsi-upm/senpy-plugins-community"
-extra_params:
-  language:
-    "@id": lang_sentiment140
-    aliases:
-    - language
-    - l
-    required: false
-    options:
-    - es
-    - en
-    - auto
-    default: auto
-requirements: {}
-maxPolarityValue: 1
-minPolarityValue: 0

senpy/plugins/sentiment/sentiment140/sentiment140_plugin.py

@@ -5,8 +5,25 @@ from senpy.plugins import SentimentPlugin
 from senpy.models import Sentiment
 
 
-class Sentiment140Plugin(SentimentPlugin):
+class Sentiment140(SentimentPlugin):
     '''Connects to the sentiment140 free API: http://sentiment140.com'''
+
+    author = "@balkian"
+    version = '0.2'
+    url = "https://github.com/gsi-upm/senpy-plugins-community"
+    extra_params = {
+        'language': {
+            "@id": 'lang_sentiment140',
+            'aliases': ['language', 'l'],
+            'required': False,
+            'default': 'auto',
+            'options': ['es', 'en', 'auto']
+        }
+    }
+
+    maxPolarityValue = 1
+    minPolarityValue = 0
+
     def analyse_entry(self, entry, params):
         lang = params["language"]
         res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
@@ -44,7 +61,7 @@ class Sentiment140Plugin(SentimentPlugin):
         from senpy.testing import patch_requests
         expected = {"data": [{"polarity": 4}]}
         with patch_requests(expected) as (request, response):
-            super(Sentiment140Plugin, self).test(*args, **kwargs)
+            super(Sentiment140, self).test(*args, **kwargs)
             assert request.called
             assert response.json.called
 

senpy/static/css/main.css

@@ -167,3 +167,36 @@ textarea{
     color: inherit;
     text-decoration: inherit;
 }
+
+.collapsed .collapseicon {
+    display: none !important;
+}
+
+.collapsed .expandicon {
+    display: inline-block !important;
+}
+
+.expandicon {
+    display: none !important;
+}
+
+.collapseicon {
+    display: inline-block !important;
+}
+
+.loader {
+    border: 6px solid #f3f3f3; /* Light grey */
+    border-top: 6px solid blue;
+    border-bottom: 6px solid blue;
+
+    border-radius: 50%;
+    width: 3em;
+    height: 3em;
+    animation: spin 2s linear infinite;
+
+}
+
+@keyframes spin {
+    0% { transform: rotate(0deg); }
+    100% { transform: rotate(360deg); }
+}

senpy/static/js/main.js

@@ -4,6 +4,7 @@ var plugins_params = default_params = {};
 var plugins = [];
 var defaultPlugin = {};
 var gplugins = {};
+var pipeline = [];
 
 function replaceURLWithHTMLLinks(text) {
     console.log('Text: ' + text);
@@ -30,7 +31,10 @@ function hashchanged(){
 
 
 function get_plugins(response){
-    plugins = response.plugins;
+    for(ix in response.plugins){
+        plug = response.plugins[ix];
+        plugins[plug.name] = plug;
+    }
 }
 
 function get_datasets(response){
@@ -83,10 +87,32 @@ function draw_plugins_selection(){
     html += "</optgroup>"
     // Two elements with plugin class
     // One from the evaluate tab and another one from the analyse tab
-    document.getElementsByClassName('plugin')[0].innerHTML = html;
-    document.getElementsByClassName('plugin')[1].innerHTML = html;
+    plugin_lists  = document.getElementsByClassName('plugin')
+    for (element in plugin_lists){
+        plugin_lists[element].innerHTML = html;
+    }
+    draw_plugin_pipeline();
+}
+
+function draw_plugin_pipeline(){
+    var pipeHTML = "";
+    console.log("Drawing pipeline: ", pipeline);
+    for (ix in pipeline){
+        plug = pipeline[ix];
+        pipeHTML += '<span onclick="remove_plugin_pipeline(\'' + plug + '\')" class="btn btn-primary"><span ><i class="fa fa-minus"></i></span> ' + plug + '</span> <i class="fa fa-arrow-right"></i> ';
+    }
+    console.log(pipeHTML);
+    $("#pipeline").html(pipeHTML);
 }
 
+
+function remove_plugin_pipeline(name){
+    console.log("Removing plugin: ", name);
+    var index = pipeline.indexOf(name);
+    pipeline.splice(index, 1);
+    draw_plugin_pipeline();
+
+}
 function draw_plugins_list(){
     var availablePlugins = document.getElementById('availablePlugins');
 
@@ -105,6 +131,13 @@ function draw_plugins_list(){
     }
 }
 
+function add_plugin_pipeline(){
+    var selected = get_selected_plugin();
+    pipeline.push(selected);
+    console.log("Adding ", selected);
+    draw_plugin_pipeline();
+}
+
 function draw_datasets(){
     html = "";
     repeated_html = "<input class=\"checks-datasets\" type=\"checkbox\" value=\"";
@@ -118,16 +151,20 @@ function draw_datasets(){
 $(document).ready(function() {
     var response = JSON.parse($.ajax({type: "GET", url: "/api/plugins/" , async: false}).responseText);
     defaultPlugin= JSON.parse($.ajax({type: "GET", url: "/api/plugins/default" , async: false}).responseText);
-    var response2 = JSON.parse($.ajax({type: "GET", url: "/api/datasets/" , async: false}).responseText);
 
     get_plugins(response);
     get_default_parameters();
-    get_datasets(response2);
 
     draw_plugins_list();
     draw_plugins_selection();
     draw_parameters();
-    draw_datasets();
+    draw_plugin_description();
+
+    if (evaluation_enabled) {
+        var response2 = JSON.parse($.ajax({type: "GET", url: "/api/datasets/" , async: false}).responseText);
+        get_datasets(response2);
+        draw_datasets();
+    }
 
     $(window).on('hashchange', hashchanged);
     hashchanged();
@@ -144,17 +181,34 @@ function get_default_parameters(){
 
 }
 
+function get_selected_plugin(){
+    return document.getElementsByClassName('plugin')[0].options[document.getElementsByClassName('plugin')[0].selectedIndex].value;
+}
+
 function draw_default_parameters(){
     var basic_params = document.getElementById("basic_params");
     basic_params.innerHTML = params_div(default_params);
 }
 
+function update_params(params, plug){
+    ep = plugins_params[plug];
+    for(k in ep){
+        params[k] = ep[k];
+    }
+    return params
+}
+
 function draw_extra_parameters(){
-    var plugin = document.getElementsByClassName('plugin')[0].options[document.getElementsByClassName('plugin')[0].selectedIndex].value;
+    var plugin = get_selected_plugin();
     get_parameters();
 
     var extra_params = document.getElementById("extra_params");
-    extra_params.innerHTML = params_div(plugins_params[plugin]);
+    var params = {};
+    for (sel in pipeline){
+        update_params(params, pipeline[sel]);
+    }
+    update_params(params, plugin);
+    extra_params.innerHTML = params_div(params);
 }
 
 function draw_parameters(){
@@ -261,6 +315,15 @@ function add_param(key, value){
     return "&"+key+"="+value;
 }
 
+function get_pipeline_arg(){
+    arg = "";
+    for (ix in pipeline){
+        arg = arg + pipeline[ix] + ",";
+    }
+    arg = arg + get_selected_plugin();
+    return arg;
+}
+
 
 function load_JSON(){
     url = "/api";
@@ -269,7 +332,9 @@ function load_JSON(){
     rawcontainer.innerHTML = '';
     container.innerHTML = '';
 
-    var plugin = document.getElementsByClassName("plugin")[0].options[document.getElementsByClassName("plugin")[0].selectedIndex].value;
+    var plugin = get_pipeline_arg();
+    $(".loading").addClass("loader");
+    $("#preview").hide();
 
     var input = encodeURIComponent(document.getElementById("input").value);
     url += "?algo="+plugin+"&i="+input
@@ -280,27 +345,27 @@ function load_JSON(){
         url += add_param(key, params[key]);
     }
 
-    var response =  $.ajax({type: "GET", url: url , async: false}).responseText;
-    rawcontainer.innerHTML = replaceURLWithHTMLLinks(response);
-
-    document.getElementById("input_request").innerHTML = "<a href='"+url+"'>"+url+"</a>"
-    document.getElementById("results-div").style.display = 'block';
-    try {
-        response = JSON.parse(response);
-        var options = {
-            mode: 'view'
-        };
-        var editor = new JSONEditor(container, options, response);
-        editor.expandAll();
-        // $('#results-div a[href="#viewer"]').tab('show');
-        $('#results-div a[href="#viewer"]').click();
-        // location.hash = 'raw';
-    }
-    catch(err){
-        console.log("Error decoding JSON (got turtle?)");
-        $('#results-div a[href="#raw"]').click();
-        // location.hash = 'raw';
-    }
+    $.ajax({type: "GET", url: url}).always(function(response){
+        document.getElementById("results-div").style.display = 'block';
+        if(typeof response=="object") {
+            var options = {
+                mode: 'view'
+            };
+            var editor = new JSONEditor(container, options, response);
+            editor.expandAll();
+            $('#results-div a[href="#viewer"]').click();
+            response = JSON.stringify(response, null, 4);
+        } else {
+            console.log("Got turtle?");
+            $('#results-div a[href="#raw"]').click();
+        }
+
+        rawcontainer.innerHTML = replaceURLWithHTMLLinks(response);
+        document.getElementById("input_request").innerHTML = "<a href='"+url+"'>"+url+"</a>"
+
+        $(".loading").removeClass("loader");
+        $("#preview").show();
+    });
 }
 
 function get_datasets_from_checkbox(){
@@ -347,40 +412,53 @@ function evaluate_JSON(){
 
     url += "?algo="+plugin+"&dataset="+datasets
 
-    var response =  $.ajax({type: "GET", url: url , async: false, dataType: 'json'}).responseText;
-    rawcontainer.innerHTML = replaceURLWithHTMLLinks(response);
-
-    document.getElementById("input_request_eval").innerHTML = "<a href='"+url+"'>"+url+"</a>"
-    document.getElementById("evaluate-div").style.display = 'block';
-    
-    try {
-        response = JSON.parse(response);
-        var options = {
-            mode: 'view'
-        };
-
-        //Control the single response results
-        if (!(Array.isArray(response.evaluations))){
-            response.evaluations = [response.evaluations]
-        }
+    $('#doevaluate').attr("disabled", true);
+    $.ajax({type: "GET", url: url, dataType: 'json'}).done(function(resp) {
+        $('#doevaluate').attr("disabled", false);
+        response = resp.responseText;  
 
-        new_tbody = create_body_metrics(response.evaluations)
-        table.replaceChild(new_tbody, table.lastElementChild)
+        rawcontainer.innerHTML = replaceURLWithHTMLLinks(response);
 
-        var editor = new JSONEditor(container, options, response);
-        editor.expandAll();
-        // $('#results-div a[href="#viewer"]').tab('show');
-        $('#evaluate-div a[href="#evaluate-table"]').click();
-        // location.hash = 'raw';
-        
-        
-    }
-    catch(err){
-        console.log("Error decoding JSON (got turtle?)");
-        $('#evaluate-div a[href="#evaluate-raw"]').click();
-        // location.hash = 'raw';
-    }
+        document.getElementById("input_request_eval").innerHTML = "<a href='"+url+"'>"+url+"</a>"
+        document.getElementById("evaluate-div").style.display = 'block';
 
-    
+        try {
+            response = JSON.parse(response);
+            var options = {
+                mode: 'view'
+            };
+
+            //Control the single response results
+            if (!(Array.isArray(response.evaluations))){
+                response.evaluations = [response.evaluations]
+            }
 
-}
+            new_tbody = create_body_metrics(response.evaluations)
+            table.replaceChild(new_tbody, table.lastElementChild)
+
+            var editor = new JSONEditor(container, options, response);
+            editor.expandAll();
+            // $('#results-div a[href="#viewer"]').tab('show');
+            $('#evaluate-div a[href="#evaluate-table"]').click();
+            // location.hash = 'raw';
+
+
+        }
+        catch(err){
+            console.log("Error decoding JSON (got turtle?)");
+            $('#evaluate-div a[href="#evaluate-raw"]').click();
+            // location.hash = 'raw';
+        }
+    })
+}
+
+function draw_plugin_description(){
+    var plugin = plugins[get_selected_plugin()];
+    $("#plugdescription").text(plugin.description);
+    console.log(plugin);
+}
+
+function plugin_selected(){
+    draw_extra_parameters();
+    draw_plugin_description();
+}

senpy/templates/index.html

@@ -5,6 +5,9 @@
     <title>Playground {{version}}</title>
 
   </head>
+  <script>
+   this.evaluation_enabled = {% if evaluation %}true{%else %}false{%endif%};
+  </script>
   <script src="static/js/jquery-2.1.1.min.js" ></script>
   <!--<script src="jquery.autosize.min.js"></script>-->
   <link rel="stylesheet" href="static/css/bootstrap.min.css">
@@ -32,7 +35,9 @@
       <ul class="nav nav-tabs" role="tablist">
         <li role="presentation" ><a class="active" href="#about">About</a></li>
         <li role="presentation"class="active"><a class="active" href="#test">Test it</a></li>
+        {% if evaluation %}
         <li role="presentation"><a class="active" href="#evaluate">Evaluate Plugins</a></li>
+        {% endif %}
 
       </ul>
 
@@ -61,6 +66,14 @@
                     </ul>
 
                   </p>
+                  <p>Senpy is a research project. If you use it in your research, please cite:
+                  <pre>
+Senpy: A Pragmatic Linked Sentiment Analysis Framework.
+Sánchez-Rada, J. F., Iglesias, C. A., Corcuera, I., & Araque, Ó.
+In Data Science and Advanced Analytics (DSAA),
+2016 IEEE International Conference on (pp. 735-742). IEEE.
+                  </pre>
+                  </p>
                   
                 </div>
             <div class="col-lg-6 ">
@@ -70,8 +83,6 @@
                 </div>
                 <div class="panel-body"><ul id=availablePlugins></ul></div>
               </div>
-            </div>
-            <div class="col-lg-6 ">
               <a href="http://senpy.readthedocs.io">
               <div class="panel panel-default">
                 <div class="panel-heading"><i class="fa fa-book"></i> If you are new to senpy, you might want to read senpy's documentation</div>
@@ -82,9 +93,6 @@
                 <div class="panel-heading"><i class="fa fa-sign-in"></i> Feel free to follow us on GitHub</div>
               </div>
               </a>
-              <div class="panel panel-default">
-                <div class="panel-heading"><i class="fa fa-child"></i> Enjoy.</div>
-              </div>
             </div>     
           </div>
         </div>
@@ -96,17 +104,28 @@
 whilst this text makes me happy and surprised at the same time.
 I cannot believe it!</textarea>
                 </div>
-                <div>
-                    <label>Select the plugin:</label>
-                    <select id="plugins" name="plugins" class=plugin onchange="draw_extra_parameters()">
-                    </select>
-                </div>
                 <!-- PARAMETERS -->
                 <div class="panel-group" id="parameters">
                     <div class="panel panel-default">
-                        <a data-toggle="collapse" class="deco-none" href="#basic_params">
+                        <div class="panel-heading">
+                            <h4 class="panel-title">
+                                Select the plugin.
+                            </h4>
+                        </div>
+                        <div id="plugin_selection" class="panel-collapse panel-body">
+                            <span id="pipeline"></span>
+                            <select name="plugins" class="plugin" onchange="plugin_selected()">
+                            </select>
+                            <span onclick="add_plugin_pipeline()"><span class="btn"><i class="fa fa-plus" title="Add more plugins to the pipeline. Processing order is left to right. i.e. the results of the leftmost plugin will be used as input for the second leftmost, and so on."></i></span></span>
+                            <label class="help-block " id="plugdescription"></label>
+                        </div>
+                    </div>
+                    <div class="panel panel-default">
+                        <a data-toggle="collapse" class="deco-none collapsed" href="#basic_params">
                             <div class="panel-heading">
                                 <h4 class="panel-title">
+                                    <i class="fa fa-chevron-right pull-left expandicon"></i>
+                                    <i class="fa fa-chevron-down pull-left collapseicon"></i>
                                     Basic API parameters
                                 </h4>
                             </div>
@@ -118,6 +137,8 @@ I cannot believe it!</textarea>
                         <a data-toggle="collapse" class="deco-none" href="#extra_params">
                             <div class="panel-heading">
                                 <h4 class="panel-title">
+                                    <i class="fa fa-chevron-right pull-left expandicon"></i>
+                                    <i class="fa fa-chevron-down pull-left collapseicon"></i>
                                     Plugin extra parameters
                                 </h4>
                             </div>
@@ -129,6 +150,7 @@ I cannot believe it!</textarea>
                 <!-- END PARAMETERS -->
  
                 <a id="preview" class="btn btn-lg btn-primary" onclick="load_JSON()">Analyse!</a>
+                <div id="loading-results" class="loading"></div>
                 <!--<button id="visualise" name="type" type="button">Visualise!</button>-->
             </form>
           </div>
@@ -155,6 +177,8 @@ I cannot believe it!</textarea>
             </div>
         </div>
 
+        {% if evaluation %}
+
         <div class="tab-pane" id="evaluate">
           <div class="well">
             <form id="form" class="container" onsubmit="return getPlugins();" accept-charset="utf-8">
@@ -169,7 +193,7 @@ I cannot believe it!</textarea>
                     </select>
                 </div>
  
-                <a id="preview" class="btn btn-lg btn-primary" onclick="evaluate_JSON()">Evaluate Plugin!</a>
+                <a id="doevaluate" class="btn btn-lg btn-primary" onclick="evaluate_JSON()">Evaluate Plugin!</a>
                 <!--<button id="visualise" name="type" type="button">Visualise!</button>-->
             </form>
           </div>
@@ -216,6 +240,7 @@ I cannot believe it!</textarea>
                 </div>
               </div>
             </div>
+            {% endif %}
         </div>
 
         <a href="http://www.gsi.dit.upm.es" target="_blank"><img class="center-block" src="static/img/gsi.png"/>  </a> 

tests/test_blueprints.py

@@ -36,7 +36,7 @@ class BlueprintsTest(TestCase):
 
     def test_playground(self):
         resp = self.client.get("/")
-        assert "main.js" in resp.data.decode()
+        assert "main.js" in resp.get_data(as_text=True)
 
     def test_home(self):
         """