From 778746c5e835ed2e120c5dfd80e497cb3aaa3671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Wed, 22 Nov 2017 17:46:52 +0100 Subject: [PATCH] Added data folder configuration Closes #46 --- README.rst | 2 +- senpy/__main__.py | 10 ++++- senpy/extensions.py | 16 +++++++- senpy/plugins/__init__.py | 40 ++++++++++++------- senpy/plugins/conversion/emotion/centroids.py | 4 +- tests/plugins/noop/noop_plugin.py | 5 +++ tests/test_extensions.py | 7 ++-- tests/test_plugins.py | 5 ++- 8 files changed, 63 insertions(+), 26 deletions(-) create mode 100644 tests/plugins/noop/noop_plugin.py diff --git a/README.rst b/README.rst index 33904b9..631671e 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ .. image:: img/header.png - :height: 6em + :width: 100% :target: http://demos.gsi.dit.upm.es/senpy .. image:: https://travis-ci.org/gsi-upm/senpy.svg?branch=master diff --git a/senpy/__main__.py b/senpy/__main__.py index 851e637..51998c9 100644 --- a/senpy/__main__.py +++ b/senpy/__main__.py @@ -75,6 +75,12 @@ def main(): action='store_true', default=False, help='Do not run a server, only install plugin dependencies') + parser.add_argument( + '--data-folder', + '--data', + type=str, + default=None, + help='Where to look for data. It be set with the SENPY_DATA environment variable as well.') parser.add_argument( '--threaded', action='store_false', @@ -96,7 +102,9 @@ def main(): rl.setLevel(getattr(logging, args.level)) app = Flask(__name__) app.debug = args.debug - sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins) + sp = Senpy(app, args.plugins_folder, + default_plugins=args.default_plugins, + data_folder=args.data_folder) sp.install_deps() if args.only_install: return diff --git a/senpy/extensions.py b/senpy/extensions.py index 57422ab..a11dd5a 100644 --- a/senpy/extensions.py +++ b/senpy/extensions.py @@ -15,6 +15,7 @@ from functools import partial import os import copy +import errno import logging import traceback @@ -27,6 +28,7 @@ class Senpy(object): def __init__(self, app=None, plugin_folder=".", + data_folder=None, default_plugins=False): self.app = app self._search_folders = set() @@ -42,6 +44,17 @@ class Senpy(object): self.add_folder(os.path.join('plugins', 'conversion'), from_root=True) + self.data_folder = data_folder or os.environ.get('SENPY_DATA', + os.path.join(os.getcwd(), + 'senpy_data')) + try: + os.makedirs(self.data_folder) + except OSError as e: + if e.errno == errno.EEXIST: + print('Directory not created.') + else: + raise + if app is not None: self.init_app(app) @@ -312,7 +325,8 @@ class Senpy(object): def plugins(self): """ Return the plugins registered for a given application. """ if self._outdated: - self._plugin_list = plugins.load_plugins(self._search_folders) + self._plugin_list = plugins.load_plugins(self._search_folders, + data_folder=self.data_folder) self._outdated = False return self._plugin_list diff --git a/senpy/plugins/__init__.py b/senpy/plugins/__init__.py index 567dd1b..c3abb0e 100644 --- a/senpy/plugins/__init__.py +++ b/senpy/plugins/__init__.py @@ -5,7 +5,6 @@ import os.path import os import pickle import logging -import tempfile import copy import fnmatch @@ -16,6 +15,8 @@ import importlib import yaml import threading +from contextlib import contextmanager + from .. import models, utils from ..api import API_PARAMS @@ -23,7 +24,7 @@ logger = logging.getLogger(__name__) class Plugin(models.Plugin): - def __init__(self, info=None): + def __init__(self, info=None, data_folder=None): """ Provides a canonical name for plugins and serves as base for other kinds of plugins. @@ -36,6 +37,7 @@ class Plugin(models.Plugin): super(Plugin, self).__init__(id=id, **info) self.is_activated = False self._lock = threading.Lock() + self.data_folder = data_folder or os.getcwd() def get_folder(self): return os.path.dirname(inspect.getfile(self.__class__)) @@ -61,6 +63,13 @@ class Plugin(models.Plugin): for r in res: r.validate() + @contextmanager + def open(self, fpath, *args, **kwargs): + if not os.path.isabs(fpath): + fpath = os.path.join(self.data_folder, fpath) + with open(fpath, *args, **kwargs) as f: + yield f + SenpyPlugin = Plugin @@ -121,7 +130,8 @@ class ShelfMixin(object): self.__dict__['_sh'] = {} if os.path.isfile(self.shelf_file): try: - self.__dict__['_sh'] = pickle.load(open(self.shelf_file, 'rb')) + with self.open(self.shelf_file, 'rb') as p: + self.__dict__['_sh'] = pickle.load(p) except (IndexError, EOFError, pickle.UnpicklingError): logger.warning('{} has a corrupted shelf file!'.format(self.id)) if not self.get('force_shelf', False): @@ -138,14 +148,13 @@ class ShelfMixin(object): @property def shelf_file(self): if 'shelf_file' not in self or not self['shelf_file']: - sd = os.environ.get('SENPY_DATA', tempfile.gettempdir()) - self.shelf_file = os.path.join(sd, self.name + '.p') + self.shelf_file = os.path.join(self.data_folder, self.name + '.p') return self['shelf_file'] def save(self): logger.debug('saving pickle') if hasattr(self, '_sh') and self._sh is not None: - with open(self.shelf_file, 'wb') as f: + with self.open(self.shelf_file, 'wb') as f: pickle.dump(self._sh, f) @@ -207,12 +216,11 @@ def log_subprocess_output(process): def install_deps(*plugins): + installed = False for info in plugins: requirements = info.get('requirements', []) if requirements: - pip_args = [sys.executable, '-m', 'pip'] - pip_args.append('install') - pip_args.append('--use-wheel') + pip_args = [sys.executable, '-m', 'pip', 'install', '--use-wheel'] for req in requirements: pip_args.append(req) logger.info('Installing requirements: ' + str(requirements)) @@ -221,11 +229,13 @@ def install_deps(*plugins): stderr=subprocess.PIPE) log_subprocess_output(process) exitcode = process.wait() + installed = True if exitcode != 0: raise models.Error("Dependencies not properly installed") + return installed -def load_plugin_from_info(info, root=None, validator=validate_info, install=True): +def load_plugin_from_info(info, root=None, validator=validate_info, install=True, *args, **kwargs): if not root and '_path' in info: root = os.path.dirname(info['_path']) if not validator(info): @@ -249,7 +259,7 @@ def load_plugin_from_info(info, root=None, validator=validate_info, install=True if not candidate: logger.debug("No valid plugin for: {}".format(module)) return - module = candidate(info=info) + module = candidate(info=info, *args, **kwargs) return module @@ -262,14 +272,14 @@ def parse_plugin_info(fpath): return name, info -def load_plugin(fpath): +def load_plugin(fpath, *args, **kwargs): name, info = parse_plugin_info(fpath) logger.debug("Info: {}".format(info)) - plugin = load_plugin_from_info(info) + plugin = load_plugin_from_info(info, *args, **kwargs) return name, plugin -def load_plugins(folders, loader=load_plugin): +def load_plugins(folders, loader=load_plugin, *args, **kwargs): plugins = {} for search_folder in folders: for root, dirnames, filenames in os.walk(search_folder): @@ -277,7 +287,7 @@ def load_plugins(folders, loader=load_plugin): dirnames[:] = [d for d in dirnames if d[0] not in ['.', '_']] for filename in fnmatch.filter(filenames, '*.senpy'): fpath = os.path.join(root, filename) - name, plugin = loader(fpath) + name, plugin = loader(fpath, *args, **kwargs) if plugin and name: plugins[name] = plugin return plugins diff --git a/senpy/plugins/conversion/emotion/centroids.py b/senpy/plugins/conversion/emotion/centroids.py index d8b5b67..6222152 100644 --- a/senpy/plugins/conversion/emotion/centroids.py +++ b/senpy/plugins/conversion/emotion/centroids.py @@ -6,7 +6,7 @@ logger = logging.getLogger(__name__) class CentroidConversion(EmotionConversionPlugin): - def __init__(self, info): + def __init__(self, info, *args, **kwargs): if 'centroids' not in info: raise Error('Centroid conversion plugins should provide ' 'the centroids in their senpy file') @@ -33,7 +33,7 @@ class CentroidConversion(EmotionConversionPlugin): ncentroids[aliases.get(k1, k1)] = nv1 info['centroids'] = ncentroids - super(CentroidConversion, self).__init__(info) + super(CentroidConversion, self).__init__(info, *args, **kwargs) self.dimensions = set() for c in self.centroids.values(): diff --git a/tests/plugins/noop/noop_plugin.py b/tests/plugins/noop/noop_plugin.py new file mode 100644 index 0000000..ba851b5 --- /dev/null +++ b/tests/plugins/noop/noop_plugin.py @@ -0,0 +1,5 @@ +from senpy.plugins import SentimentPlugin + + +class DummyPlugin(SentimentPlugin): + import noop diff --git a/tests/test_extensions.py b/tests/test_extensions.py index 9ff9ffa..a9bf767 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -49,14 +49,13 @@ class ExtensionsTest(TestCase): """ Installing a plugin """ info = { 'name': 'TestPip', - 'module': 'dummy', + 'module': 'noop_plugin', 'description': None, 'requirements': ['noop'], 'version': 0 } - root = os.path.join(self.dir, 'plugins', 'dummy_plugin') - module = plugins.load_plugin_from_info(info, root=root) - plugins.install_deps(info) + root = os.path.join(self.dir, 'plugins', 'noop') + module = plugins.load_plugin_from_info(info, root=root, install=True) assert module.name == 'TestPip' assert module import noop diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 1fee8c4..e3cdb54 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -222,8 +222,9 @@ def make_mini_test(plugin_info): def _add_tests(): - root = os.path.dirname(__file__) - plugs = plugins.load_plugins(os.path.join(root, ".."), loader=plugins.parse_plugin_info) + root = os.path.join(os.path.dirname(__file__), '..') + print(root) + plugs = plugins.load_plugins([root, ], loader=plugins.parse_plugin_info) for k, v in plugs.items(): pass t_method = make_mini_test(v)