Merge branch '46-make-data-folder-configurable' into 'master'

Resolve "Make data folder configurable"

Closes #46 and #47

See merge request senpy/senpy!19

README.rst

@@ -1,5 +1,5 @@
 .. image:: img/header.png
-   :height: 6em
+   :width: 100%
    :target: http://demos.gsi.dit.upm.es/senpy
 
 .. image:: https://travis-ci.org/gsi-upm/senpy.svg?branch=master

senpy/__main__.py

@@ -25,6 +25,7 @@ from senpy.extensions import Senpy
 
 import logging
 import os
+import sys
 import argparse
 import senpy
 
@@ -74,6 +75,12 @@ def main():
         action='store_true',
         default=False,
         help='Do not run a server, only install plugin dependencies')
+    parser.add_argument(
+        '--data-folder',
+        '--data',
+        type=str,
+        default=None,
+        help='Where to look for data. It be set with the SENPY_DATA environment variable as well.')
     parser.add_argument(
         '--threaded',
         action='store_false',
@@ -88,13 +95,16 @@ def main():
     args = parser.parse_args()
     if args.version:
         print('Senpy version {}'.format(senpy.__version__))
+        print(sys.version)
         exit(1)
     logging.basicConfig()
     rl = logging.getLogger()
     rl.setLevel(getattr(logging, args.level))
     app = Flask(__name__)
     app.debug = args.debug
-    sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins)
+    sp = Senpy(app, args.plugins_folder,
+               default_plugins=args.default_plugins,
+               data_folder=args.data_folder)
     sp.install_deps()
     if args.only_install:
         return

senpy/extensions.py

@@ -15,6 +15,7 @@ from functools import partial
 
 import os
 import copy
+import errno
 import logging
 import traceback
 
@@ -27,6 +28,7 @@ class Senpy(object):
     def __init__(self,
                  app=None,
                  plugin_folder=".",
+                 data_folder=None,
                  default_plugins=False):
         self.app = app
         self._search_folders = set()
@@ -42,6 +44,17 @@ class Senpy(object):
             self.add_folder(os.path.join('plugins', 'conversion'),
                             from_root=True)
 
+        self.data_folder = data_folder or os.environ.get('SENPY_DATA',
+                                                         os.path.join(os.getcwd(),
+                                                                      'senpy_data'))
+        try:
+            os.makedirs(self.data_folder)
+        except OSError as e:
+            if e.errno == errno.EEXIST:
+                print('Directory not created.')
+            else:
+                raise
+
         if app is not None:
             self.init_app(app)
 
@@ -312,7 +325,8 @@ class Senpy(object):
     def plugins(self):
         """ Return the plugins registered for a given application.  """
         if self._outdated:
-            self._plugin_list = plugins.load_plugins(self._search_folders)
+            self._plugin_list = plugins.load_plugins(self._search_folders,
+                                                     data_folder=self.data_folder)
             self._outdated = False
         return self._plugin_list
 

senpy/plugins/__init__.py

@@ -5,7 +5,6 @@ import os.path
 import os
 import pickle
 import logging
-import tempfile
 import copy
 
 import fnmatch
@@ -16,6 +15,8 @@ import importlib
 import yaml
 import threading
 
+from contextlib import contextmanager
+
 from .. import models, utils
 from ..api import API_PARAMS
 
@@ -23,7 +24,7 @@ logger = logging.getLogger(__name__)
 
 
 class Plugin(models.Plugin):
-    def __init__(self, info=None):
+    def __init__(self, info=None, data_folder=None):
         """
         Provides a canonical name for plugins and serves as base for other
         kinds of plugins.
@@ -36,6 +37,7 @@ class Plugin(models.Plugin):
         super(Plugin, self).__init__(id=id, **info)
         self.is_activated = False
         self._lock = threading.Lock()
+        self.data_folder = data_folder or os.getcwd()
 
     def get_folder(self):
         return os.path.dirname(inspect.getfile(self.__class__))
@@ -61,6 +63,13 @@ class Plugin(models.Plugin):
             for r in res:
                 r.validate()
 
+    @contextmanager
+    def open(self, fpath, *args, **kwargs):
+        if not os.path.isabs(fpath):
+            fpath = os.path.join(self.data_folder, fpath)
+        with open(fpath, *args, **kwargs) as f:
+            yield f
+
 
 SenpyPlugin = Plugin
 
@@ -121,7 +130,8 @@ class ShelfMixin(object):
             self.__dict__['_sh'] = {}
             if os.path.isfile(self.shelf_file):
                 try:
-                    self.__dict__['_sh'] = pickle.load(open(self.shelf_file, 'rb'))
+                    with self.open(self.shelf_file, 'rb') as p:
+                        self.__dict__['_sh'] = pickle.load(p)
                 except (IndexError, EOFError, pickle.UnpicklingError):
                     logger.warning('{} has a corrupted shelf file!'.format(self.id))
                     if not self.get('force_shelf', False):
@@ -138,14 +148,13 @@ class ShelfMixin(object):
     @property
     def shelf_file(self):
         if 'shelf_file' not in self or not self['shelf_file']:
-            sd = os.environ.get('SENPY_DATA', tempfile.gettempdir())
+            self.shelf_file = os.path.join(self.data_folder, self.name + '.p')
-            self.shelf_file = os.path.join(sd, self.name + '.p')
         return self['shelf_file']
 
     def save(self):
         logger.debug('saving pickle')
         if hasattr(self, '_sh') and self._sh is not None:
-            with open(self.shelf_file, 'wb') as f:
+            with self.open(self.shelf_file, 'wb') as f:
                 pickle.dump(self._sh, f)
 
 
@@ -207,12 +216,11 @@ def log_subprocess_output(process):
 
 
 def install_deps(*plugins):
+    installed = False
     for info in plugins:
         requirements = info.get('requirements', [])
         if requirements:
-            pip_args = ['pip']
+            pip_args = [sys.executable, '-m', 'pip', 'install', '--use-wheel']
-            pip_args.append('install')
-            pip_args.append('--use-wheel')
             for req in requirements:
                 pip_args.append(req)
             logger.info('Installing requirements: ' + str(requirements))
@@ -221,11 +229,13 @@ def install_deps(*plugins):
                                        stderr=subprocess.PIPE)
             log_subprocess_output(process)
             exitcode = process.wait()
+            installed = True
             if exitcode != 0:
                 raise models.Error("Dependencies not properly installed")
+    return installed
 
 
-def load_plugin_from_info(info, root=None, validator=validate_info, install=True):
+def load_plugin_from_info(info, root=None, validator=validate_info, install=True, *args, **kwargs):
     if not root and '_path' in info:
         root = os.path.dirname(info['_path'])
     if not validator(info):
@@ -249,7 +259,7 @@ def load_plugin_from_info(info, root=None, validator=validate_info, install=True
     if not candidate:
         logger.debug("No valid plugin for: {}".format(module))
         return
-    module = candidate(info=info)
+    module = candidate(info=info, *args, **kwargs)
     return module
 
 
@@ -262,14 +272,14 @@ def parse_plugin_info(fpath):
     return name, info
 
 
-def load_plugin(fpath):
+def load_plugin(fpath, *args, **kwargs):
     name, info = parse_plugin_info(fpath)
     logger.debug("Info: {}".format(info))
-    plugin = load_plugin_from_info(info)
+    plugin = load_plugin_from_info(info, *args, **kwargs)
     return name, plugin
 
 
-def load_plugins(folders, loader=load_plugin):
+def load_plugins(folders, loader=load_plugin, *args, **kwargs):
     plugins = {}
     for search_folder in folders:
         for root, dirnames, filenames in os.walk(search_folder):
@@ -277,7 +287,7 @@ def load_plugins(folders, loader=load_plugin):
             dirnames[:] = [d for d in dirnames if d[0] not in ['.', '_']]
             for filename in fnmatch.filter(filenames, '*.senpy'):
                 fpath = os.path.join(root, filename)
-                name, plugin = loader(fpath)
+                name, plugin = loader(fpath, *args, **kwargs)
                 if plugin and name:
                     plugins[name] = plugin
     return plugins

senpy/plugins/conversion/emotion/centroids.py

@@ -6,7 +6,7 @@ logger = logging.getLogger(__name__)
 
 
 class CentroidConversion(EmotionConversionPlugin):
-    def __init__(self, info):
+    def __init__(self, info, *args, **kwargs):
         if 'centroids' not in info:
             raise Error('Centroid conversion plugins should provide '
                         'the centroids in their senpy file')
@@ -33,7 +33,7 @@ class CentroidConversion(EmotionConversionPlugin):
                 ncentroids[aliases.get(k1, k1)] = nv1
             info['centroids'] = ncentroids
 
-        super(CentroidConversion, self).__init__(info)
+        super(CentroidConversion, self).__init__(info, *args, **kwargs)
 
         self.dimensions = set()
         for c in self.centroids.values():

tests/plugins/noop/noop_plugin.py

@@ -0,0 +1,5 @@
+from senpy.plugins import SentimentPlugin
+
+
+class DummyPlugin(SentimentPlugin):
+    import noop

tests/test_extensions.py

@@ -49,14 +49,13 @@ class ExtensionsTest(TestCase):
         """ Installing a plugin """
         info = {
             'name': 'TestPip',
-            'module': 'dummy',
+            'module': 'noop_plugin',
             'description': None,
             'requirements': ['noop'],
             'version': 0
         }
-        root = os.path.join(self.dir, 'plugins', 'dummy_plugin')
+        root = os.path.join(self.dir, 'plugins', 'noop')
-        module = plugins.load_plugin_from_info(info, root=root)
+        module = plugins.load_plugin_from_info(info, root=root, install=True)
-        plugins.install_deps(info)
         assert module.name == 'TestPip'
         assert module
         import noop

tests/test_plugins.py

@@ -222,8 +222,9 @@ def make_mini_test(plugin_info):
 
 
 def _add_tests():
-    root = os.path.dirname(__file__)
+    root = os.path.join(os.path.dirname(__file__), '..')
-    plugs = plugins.load_plugins(os.path.join(root, ".."), loader=plugins.parse_plugin_info)
+    print(root)
+    plugs = plugins.load_plugins([root, ], loader=plugins.parse_plugin_info)
     for k, v in plugs.items():
         pass
         t_method = make_mini_test(v)