Merge branch 'split-fix'

Fix #48

.gitlab-ci.yml

@@ -63,11 +63,13 @@ push-github:
   only:
     - master
     - triggers
+    - fix-makefiles
 
 deploy_pypi:
   stage: deploy
   script:   # Configure the PyPI credentials, then push the package, and cleanup the creds.
     - echo "[server-login]" >> ~/.pypirc
+    - echo "repository=https://upload.pypi.org/legacy/" >> ~/.pypirc
     - echo "username=" ${PYPI_USER} >> ~/.pypirc
     - echo "password=" ${PYPI_PASSWORD} >> ~/.pypirc
     - make pip_upload

.makefiles/git.mk

@@ -13,12 +13,12 @@ git-pull:
 push-github: ## Push the code to github. You need to set up GITHUB_DEPLOY_KEY
 ifeq ($(GITHUB_DEPLOY_KEY),)
 else
-	$(eval KEY_FILE := $(shell mktemp))
+	$(eval KEY_FILE := "$(shell mktemp)")
 	@echo "$(GITHUB_DEPLOY_KEY)" > $(KEY_FILE)
 	@git remote rm github-deploy || true
 	git remote add github-deploy $(GITHUB_REPO)
-	@GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git fetch github-deploy $(CI_COMMIT_REF_NAME) || true
-	@GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git push github-deploy $(CI_COMMIT_REF_NAME)
+	-@GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git fetch github-deploy $(CI_COMMIT_REF_NAME)
+	@GIT_SSH_COMMAND="ssh -i $(KEY_FILE)" git push github-deploy HEAD:$(CI_COMMIT_REF_NAME)
 	rm $(KEY_FILE)
 endif
 

README.rst

@@ -1,5 +1,5 @@
 .. image:: img/header.png
-   :height: 6em
+   :width: 100%
    :target: http://demos.gsi.dit.upm.es/senpy
 
 .. image:: https://travis-ci.org/gsi-upm/senpy.svg?branch=master

senpy/__main__.py

@@ -25,6 +25,7 @@ from senpy.extensions import Senpy
 
 import logging
 import os
+import sys
 import argparse
 import senpy
 
@@ -74,6 +75,12 @@ def main():
         action='store_true',
         default=False,
         help='Do not run a server, only install plugin dependencies')
+    parser.add_argument(
+        '--data-folder',
+        '--data',
+        type=str,
+        default=None,
+        help='Where to look for data. It be set with the SENPY_DATA environment variable as well.')
     parser.add_argument(
         '--threaded',
         action='store_false',
@@ -88,13 +95,16 @@ def main():
     args = parser.parse_args()
     if args.version:
         print('Senpy version {}'.format(senpy.__version__))
+        print(sys.version)
         exit(1)
     logging.basicConfig()
     rl = logging.getLogger()
     rl.setLevel(getattr(logging, args.level))
     app = Flask(__name__)
     app.debug = args.debug
-    sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins)
+    sp = Senpy(app, args.plugins_folder,
+               default_plugins=args.default_plugins,
+               data_folder=args.data_folder)
     sp.install_deps()
     if args.only_install:
         return

senpy/extensions.py

@@ -15,6 +15,7 @@ from functools import partial
 
 import os
 import copy
+import errno
 import logging
 import traceback
 
@@ -27,6 +28,7 @@ class Senpy(object):
     def __init__(self,
                  app=None,
                  plugin_folder=".",
+                 data_folder=None,
                  default_plugins=False):
         self.app = app
         self._search_folders = set()
@@ -42,6 +44,17 @@ class Senpy(object):
             self.add_folder(os.path.join('plugins', 'conversion'),
                             from_root=True)
 
+        self.data_folder = data_folder or os.environ.get('SENPY_DATA',
+                                                         os.path.join(os.getcwd(),
+                                                                      'senpy_data'))
+        try:
+            os.makedirs(self.data_folder)
+        except OSError as e:
+            if e.errno == errno.EEXIST:
+                print('Directory not created.')
+            else:
+                raise
+
         if app is not None:
             self.init_app(app)
 
@@ -312,7 +325,8 @@ class Senpy(object):
     def plugins(self):
         """ Return the plugins registered for a given application.  """
         if self._outdated:
-            self._plugin_list = plugins.load_plugins(self._search_folders)
+            self._plugin_list = plugins.load_plugins(self._search_folders,
+                                                     data_folder=self.data_folder)
             self._outdated = False
         return self._plugin_list
 

senpy/plugins/__init__.py

@@ -5,7 +5,6 @@ import os.path
 import os
 import pickle
 import logging
-import tempfile
 import copy
 
 import fnmatch
@@ -16,6 +15,8 @@ import importlib
 import yaml
 import threading
 
+from contextlib import contextmanager
+
 from .. import models, utils
 from ..api import API_PARAMS
 
@@ -23,7 +24,7 @@ logger = logging.getLogger(__name__)
 
 
 class Plugin(models.Plugin):
-    def __init__(self, info=None):
+    def __init__(self, info=None, data_folder=None):
         """
         Provides a canonical name for plugins and serves as base for other
         kinds of plugins.
@@ -36,6 +37,7 @@ class Plugin(models.Plugin):
         super(Plugin, self).__init__(id=id, **info)
         self.is_activated = False
         self._lock = threading.Lock()
+        self.data_folder = data_folder or os.getcwd()
 
     def get_folder(self):
         return os.path.dirname(inspect.getfile(self.__class__))
@@ -61,6 +63,13 @@ class Plugin(models.Plugin):
             for r in res:
                 r.validate()
 
+    @contextmanager
+    def open(self, fpath, *args, **kwargs):
+        if not os.path.isabs(fpath):
+            fpath = os.path.join(self.data_folder, fpath)
+        with open(fpath, *args, **kwargs) as f:
+            yield f
+
 
 SenpyPlugin = Plugin
 
@@ -121,7 +130,8 @@ class ShelfMixin(object):
             self.__dict__['_sh'] = {}
             if os.path.isfile(self.shelf_file):
                 try:
-                    self.__dict__['_sh'] = pickle.load(open(self.shelf_file, 'rb'))
+                    with self.open(self.shelf_file, 'rb') as p:
+                        self.__dict__['_sh'] = pickle.load(p)
                 except (IndexError, EOFError, pickle.UnpicklingError):
                     logger.warning('{} has a corrupted shelf file!'.format(self.id))
                     if not self.get('force_shelf', False):
@@ -138,14 +148,13 @@ class ShelfMixin(object):
     @property
     def shelf_file(self):
         if 'shelf_file' not in self or not self['shelf_file']:
-            sd = os.environ.get('SENPY_DATA', tempfile.gettempdir())
-            self.shelf_file = os.path.join(sd, self.name + '.p')
+            self.shelf_file = os.path.join(self.data_folder, self.name + '.p')
         return self['shelf_file']
 
     def save(self):
         logger.debug('saving pickle')
         if hasattr(self, '_sh') and self._sh is not None:
-            with open(self.shelf_file, 'wb') as f:
+            with self.open(self.shelf_file, 'wb') as f:
                 pickle.dump(self._sh, f)
 
 
@@ -207,12 +216,11 @@ def log_subprocess_output(process):
 
 
 def install_deps(*plugins):
+    installed = False
     for info in plugins:
         requirements = info.get('requirements', [])
         if requirements:
-            pip_args = ['pip']
-            pip_args.append('install')
-            pip_args.append('--use-wheel')
+            pip_args = [sys.executable, '-m', 'pip', 'install', '--use-wheel']
             for req in requirements:
                 pip_args.append(req)
             logger.info('Installing requirements: ' + str(requirements))
@@ -221,11 +229,13 @@ def install_deps(*plugins):
                                        stderr=subprocess.PIPE)
             log_subprocess_output(process)
             exitcode = process.wait()
+            installed = True
             if exitcode != 0:
                 raise models.Error("Dependencies not properly installed")
+    return installed
 
 
-def load_plugin_from_info(info, root=None, validator=validate_info, install=True):
+def load_plugin_from_info(info, root=None, validator=validate_info, install=True, *args, **kwargs):
     if not root and '_path' in info:
         root = os.path.dirname(info['_path'])
     if not validator(info):
@@ -249,7 +259,7 @@ def load_plugin_from_info(info, root=None, validator=validate_info, install=True
     if not candidate:
         logger.debug("No valid plugin for: {}".format(module))
         return
-    module = candidate(info=info)
+    module = candidate(info=info, *args, **kwargs)
     return module
 
 
@@ -262,14 +272,14 @@ def parse_plugin_info(fpath):
     return name, info
 
 
-def load_plugin(fpath):
+def load_plugin(fpath, *args, **kwargs):
     name, info = parse_plugin_info(fpath)
     logger.debug("Info: {}".format(info))
-    plugin = load_plugin_from_info(info)
+    plugin = load_plugin_from_info(info, *args, **kwargs)
     return name, plugin
 
 
-def load_plugins(folders, loader=load_plugin):
+def load_plugins(folders, loader=load_plugin, *args, **kwargs):
     plugins = {}
     for search_folder in folders:
         for root, dirnames, filenames in os.walk(search_folder):
@@ -277,7 +287,7 @@ def load_plugins(folders, loader=load_plugin):
             dirnames[:] = [d for d in dirnames if d[0] not in ['.', '_']]
             for filename in fnmatch.filter(filenames, '*.senpy'):
                 fpath = os.path.join(root, filename)
-                name, plugin = loader(fpath)
+                name, plugin = loader(fpath, *args, **kwargs)
                 if plugin and name:
                     plugins[name] = plugin
     return plugins

senpy/plugins/conversion/emotion/centroids.py

@@ -6,7 +6,7 @@ logger = logging.getLogger(__name__)
 
 
 class CentroidConversion(EmotionConversionPlugin):
-    def __init__(self, info):
+    def __init__(self, info, *args, **kwargs):
         if 'centroids' not in info:
             raise Error('Centroid conversion plugins should provide '
                         'the centroids in their senpy file')
@@ -33,7 +33,7 @@ class CentroidConversion(EmotionConversionPlugin):
                 ncentroids[aliases.get(k1, k1)] = nv1
             info['centroids'] = ncentroids
 
-        super(CentroidConversion, self).__init__(info)
+        super(CentroidConversion, self).__init__(info, *args, **kwargs)
 
         self.dimensions = set()
         for c in self.centroids.values():

senpy/plugins/misc/split.py

@@ -12,13 +12,14 @@ class SplitPlugin(AnalysisPlugin):
 
     def analyse_entry(self, entry, params):
         chunker_type = params.get("delimiter", "sentence")
-        original_text = entry.get('nif:isString', None)
+        original_text = entry['nif:isString']
         if chunker_type == "sentence":
             tokenizer = PunktSentenceTokenizer()
         if chunker_type == "paragraph":
             tokenizer = LineTokenizer()
-        chars = tokenizer.span_tokenize(original_text)
+        chars = list(tokenizer.span_tokenize(original_text))
         for i, chunk in enumerate(tokenizer.tokenize(original_text)):
+            print(chunk)
             e = Entry()
             e['nif:isString'] = chunk
             if entry.id:
@@ -45,19 +46,19 @@ class SplitPlugin(AnalysisPlugin):
         {
             'entry': {
                 "id": ":test",
-                'nif:isString': 'Hello. World.'
+                'nif:isString': 'Hello\nWorld'
             },
             'params': {
-                'delimiter': 'sentence',
+                'delimiter': 'paragraph',
             },
             'expected': [
                 {
-                    "@id": ":test#char=0,6",
-                    'nif:isString': 'Hello.'
+                    "@id": ":test#char=0,5",
+                    'nif:isString': 'Hello'
                 },
                 {
-                    "@id": ":test#char=7,13",
-                    'nif:isString': 'World.'
+                    "@id": ":test#char=6,11",
+                    'nif:isString': 'World'
                 }
             ]
         }

senpy/static/js/main.js

@@ -1,7 +1,7 @@
 var ONYX = "http://www.gsi.dit.upm.es/ontologies/onyx/ns#";
 var RDF_TYPE =  "http://www.w3.org/1999/02/22-rdf-syntax-ns#type";
 var plugins_params={};
-var default_params = JSON.parse($.ajax({type: "GET", url: "/api?help=True" , async: false}).responseText);
+var default_params = JSON.parse($.ajax({type: "GET", url: "/api?help=true" , async: false}).responseText);
 function replaceURLWithHTMLLinks(text) {
     console.log('Text: ' + text);
     var exp = /(\b(https?|ftp|file):\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])/ig;
@@ -105,29 +105,40 @@ function change_params(){
         html=""
 		for (param in default_params){
 		  if ((default_params[param]['options']) && (['help','conversion'].indexOf(param) < 0)){  
-			html+= "<label> "+param+"</label>"
-            html+= "<select id=\""+param+"\" name=\""+param+"\">"
-			for (option in default_params[param]['options']){
-				if (default_params[param]['options'][option] == default_params[param]['default']){  
-					html+="<option value \""+default_params[param]['options'][option]+"\" selected >"+default_params[param]['options'][option]+"</option>"
-				}
-				else{
-					html+="<option value \""+default_params[param]['options'][option]+"\">"+default_params[param]['options'][option]+"</option>"
-							
-				}
-			}
-			html+="</select><br>"
-		  }
+          html+= "<label> "+param+"</label>"
+          if (default_params[param]['options'].length < 1) {
+              html +="<input></input>";
+          }
+          else {
+              html+= "<select id=\""+param+"\" name=\""+param+"\">"
+              for (option in default_params[param]['options']){
+                  if (default_params[param]['options'][option] == default_params[param]['default']){  
+                      html+="<option value \""+default_params[param]['options'][option]+"\" selected >"+default_params[param]['options'][option]+"</option>"
+                  }
+                  else{
+                      html+="<option value \""+default_params[param]['options'][option]+"\">"+default_params[param]['options'][option]+"</option>"
+                      
+                  }
+              }
+          }
+      html+="</select><br>"
+      }
 		}
         for (param in plugins_params[plugin]){
           if (param || plugins_params[plugin][param].length > 1){
-            html+= "<label> Parameter "+param+"</label>"
-            html+= "<select id=\""+param+"\" name=\""+param+"\">"
-            for (option in plugins_params[plugin][param]){
-                  html+="<option value \""+plugins_params[plugin][param][option]+"\">"+plugins_params[plugin][param][option]+"</option>"
-            }
+              html+= "<label> Parameter "+param+"</label>"
+              param_opts = plugins_params[plugin][param]
+              if (param_opts.length > 0) {
+                  html+= "<select id=\""+param+"\" name=\""+param+"\">"
+                  for (option in param_opts){
+                      html+="<option value \""+param_opts[option]+"\">"+param_opts[option]+"</option>"
+                  }
+                  html+="</select>"
+              }
+              else {
+                  html +="<input id=\""+param+"\" name=\""+param+"\"></input>";
+              }
           }
-          html+="</select>"
         }
         document.getElementById("params").innerHTML = html
 };
@@ -143,9 +154,13 @@ function load_JSON(){
       url += "?algo="+plugin+"&i="+input
       for (param in plugins_params[plugin]){
         if (param != null){
-            var param_value = encodeURIComponent(document.getElementById(param).options[document.getElementById(param).selectedIndex].text);
-            if (param_value){
-              url+="&"+param+"="+param_value
+            if (plugins_params[plugin].length > 0){
+                var param_value = encodeURIComponent(document.getElementById(param).options[document.getElementById(param).selectedIndex].text);
+            } else {
+                var param_value = encodeURIComponent(document.getElementById(param).value);
+            }
+            if (param_value !== "undefined" && param_value.length > 0){
+                url+="&"+param+"="+param_value
             }
         }
       }

tests/plugins/noop/noop_plugin.py

@@ -0,0 +1,5 @@
+from senpy.plugins import SentimentPlugin
+
+
+class DummyPlugin(SentimentPlugin):
+    import noop

tests/test_extensions.py

@@ -49,14 +49,13 @@ class ExtensionsTest(TestCase):
         """ Installing a plugin """
         info = {
             'name': 'TestPip',
-            'module': 'dummy',
+            'module': 'noop_plugin',
             'description': None,
             'requirements': ['noop'],
             'version': 0
         }
-        root = os.path.join(self.dir, 'plugins', 'dummy_plugin')
-        module = plugins.load_plugin_from_info(info, root=root)
-        plugins.install_deps(info)
+        root = os.path.join(self.dir, 'plugins', 'noop')
+        module = plugins.load_plugin_from_info(info, root=root, install=True)
         assert module.name == 'TestPip'
         assert module
         import noop

tests/test_plugins.py

@@ -222,8 +222,9 @@ def make_mini_test(plugin_info):
 
 
 def _add_tests():
-    root = os.path.dirname(__file__)
-    plugs = plugins.load_plugins(os.path.join(root, ".."), loader=plugins.parse_plugin_info)
+    root = os.path.join(os.path.dirname(__file__), '..')
+    print(root)
+    plugs = plugins.load_plugins([root, ], loader=plugins.parse_plugin_info)
     for k, v in plugs.items():
         pass
         t_method = make_mini_test(v)