balkian/senpy
1
0
Fork 0
mirror of https://github.com/gsi-upm/senpy synced 2024-12-25 22:48:11 +00:00
Code Issues Releases Wiki Activity

Added data folder configuration

Browse Source 
Closes #46
This commit is contained in:
J. Fernando Sánchez 2017-11-22 17:46:52 +01:00
parent 19278d0acd
commit 778746c5e8
8 changed files with 63 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.rst
View File

@ -1,5 +1,5 @@
.. image:: img/header.png
:height: 6em
:width: 100%
:target: http://demos.gsi.dit.upm.es/senpy
.. image:: https://travis-ci.org/gsi-upm/senpy.svg?branch=master

10
senpy/__main__.py
View File

@ -75,6 +75,12 @@ def main():
action='store_true',
default=False,
help='Do not run a server, only install plugin dependencies')
parser.add_argument(
'--data-folder',
'--data',
type=str,
default=None,
help='Where to look for data. It be set with the SENPY_DATA environment variable as well.')
parser.add_argument(
'--threaded',
action='store_false',
@ -96,7 +102,9 @@ def main():
rl.setLevel(getattr(logging, args.level))
app = Flask(__name__)
app.debug = args.debug
sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins)
sp = Senpy(app, args.plugins_folder,
default_plugins=args.default_plugins,
data_folder=args.data_folder)
sp.install_deps()
if args.only_install:
return

16
senpy/extensions.py
View File

@ -15,6 +15,7 @@ from functools import partial
import os
import copy
import errno
import logging
import traceback
@ -27,6 +28,7 @@ class Senpy(object):
def __init__(self,
app=None,
plugin_folder=".",
data_folder=None,
default_plugins=False):
self.app = app
self._search_folders = set()
@ -42,6 +44,17 @@ class Senpy(object):
self.add_folder(os.path.join('plugins', 'conversion'),
from_root=True)
self.data_folder = data_folder or os.environ.get('SENPY_DATA',
os.path.join(os.getcwd(),
'senpy_data'))
try:
os.makedirs(self.data_folder)
except OSError as e:
if e.errno == errno.EEXIST:
print('Directory not created.')
else:
raise
if app is not None:
self.init_app(app)
@ -312,7 +325,8 @@ class Senpy(object):
def plugins(self):
""" Return the plugins registered for a given application. """
if self._outdated:
self._plugin_list = plugins.load_plugins(self._search_folders)
self._plugin_list = plugins.load_plugins(self._search_folders,
data_folder=self.data_folder)
self._outdated = False
return self._plugin_list

40
senpy/plugins/__init__.py
View File

@ -5,7 +5,6 @@ import os.path
import os
import pickle
import logging
import tempfile
import copy
import fnmatch
@ -16,6 +15,8 @@ import importlib
import yaml
import threading
from contextlib import contextmanager
from .. import models, utils
from ..api import API_PARAMS
@ -23,7 +24,7 @@ logger = logging.getLogger(__name__)
class Plugin(models.Plugin):
def __init__(self, info=None):
def __init__(self, info=None, data_folder=None):
"""
Provides a canonical name for plugins and serves as base for other
kinds of plugins.
@ -36,6 +37,7 @@ class Plugin(models.Plugin):
super(Plugin, self).__init__(id=id, **info)
self.is_activated = False
self._lock = threading.Lock()
self.data_folder = data_folder or os.getcwd()
def get_folder(self):
return os.path.dirname(inspect.getfile(self.__class__))
@ -61,6 +63,13 @@ class Plugin(models.Plugin):
for r in res:
r.validate()
@contextmanager
def open(self, fpath, *args, **kwargs):
if not os.path.isabs(fpath):
fpath = os.path.join(self.data_folder, fpath)
with open(fpath, *args, **kwargs) as f:
yield f
SenpyPlugin = Plugin
@ -121,7 +130,8 @@ class ShelfMixin(object):
self.__dict__['_sh'] = {}
if os.path.isfile(self.shelf_file):
try:
self.__dict__['_sh'] = pickle.load(open(self.shelf_file, 'rb'))
with self.open(self.shelf_file, 'rb') as p:
self.__dict__['_sh'] = pickle.load(p)
except (IndexError, EOFError, pickle.UnpicklingError):
logger.warning('{} has a corrupted shelf file!'.format(self.id))
if not self.get('force_shelf', False):
@ -138,14 +148,13 @@ class ShelfMixin(object):
@property
def shelf_file(self):
if 'shelf_file' not in self or not self['shelf_file']:
sd = os.environ.get('SENPY_DATA', tempfile.gettempdir())
self.shelf_file = os.path.join(sd, self.name + '.p')
self.shelf_file = os.path.join(self.data_folder, self.name + '.p')
return self['shelf_file']
def save(self):
logger.debug('saving pickle')
if hasattr(self, '_sh') and self._sh is not None:
with open(self.shelf_file, 'wb') as f:
with self.open(self.shelf_file, 'wb') as f:
pickle.dump(self._sh, f)
@ -207,12 +216,11 @@ def log_subprocess_output(process):
def install_deps(*plugins):
installed = False
for info in plugins:
requirements = info.get('requirements', [])
if requirements:
pip_args = [sys.executable, '-m', 'pip']
pip_args.append('install')
pip_args.append('--use-wheel')
pip_args = [sys.executable, '-m', 'pip', 'install', '--use-wheel']
for req in requirements:
pip_args.append(req)
logger.info('Installing requirements: ' + str(requirements))
@ -221,11 +229,13 @@ def install_deps(*plugins):
stderr=subprocess.PIPE)
log_subprocess_output(process)
exitcode = process.wait()
installed = True
if exitcode != 0:
raise models.Error("Dependencies not properly installed")
return installed
def load_plugin_from_info(info, root=None, validator=validate_info, install=True):
def load_plugin_from_info(info, root=None, validator=validate_info, install=True, *args, **kwargs):
if not root and '_path' in info:
root = os.path.dirname(info['_path'])
if not validator(info):
@ -249,7 +259,7 @@ def load_plugin_from_info(info, root=None, validator=validate_info, install=True
if not candidate:
logger.debug("No valid plugin for: {}".format(module))
return
module = candidate(info=info)
module = candidate(info=info, *args, **kwargs)
return module
@ -262,14 +272,14 @@ def parse_plugin_info(fpath):
return name, info
def load_plugin(fpath):
def load_plugin(fpath, *args, **kwargs):
name, info = parse_plugin_info(fpath)
logger.debug("Info: {}".format(info))
plugin = load_plugin_from_info(info)
plugin = load_plugin_from_info(info, *args, **kwargs)
return name, plugin
def load_plugins(folders, loader=load_plugin):
def load_plugins(folders, loader=load_plugin, *args, **kwargs):
plugins = {}
for search_folder in folders:
for root, dirnames, filenames in os.walk(search_folder):
@ -277,7 +287,7 @@ def load_plugins(folders, loader=load_plugin):
dirnames[:] = [d for d in dirnames if d[0] not in ['.', '_']]
for filename in fnmatch.filter(filenames, '*.senpy'):
fpath = os.path.join(root, filename)
name, plugin = loader(fpath)
name, plugin = loader(fpath, *args, **kwargs)
if plugin and name:
plugins[name] = plugin
return plugins

4
senpy/plugins/conversion/emotion/centroids.py
View File

@ -6,7 +6,7 @@ logger = logging.getLogger(__name__)
class CentroidConversion(EmotionConversionPlugin):
def __init__(self, info):
def __init__(self, info, *args, **kwargs):
if 'centroids' not in info:
raise Error('Centroid conversion plugins should provide '
'the centroids in their senpy file')
@ -33,7 +33,7 @@ class CentroidConversion(EmotionConversionPlugin):
ncentroids[aliases.get(k1, k1)] = nv1
info['centroids'] = ncentroids
super(CentroidConversion, self).__init__(info)
super(CentroidConversion, self).__init__(info, *args, **kwargs)
self.dimensions = set()
for c in self.centroids.values():

5
tests/plugins/noop/noop_plugin.py Normal file
View File

@ -0,0 +1,5 @@
from senpy.plugins import SentimentPlugin
class DummyPlugin(SentimentPlugin):
import noop

7
tests/test_extensions.py
View File

@ -49,14 +49,13 @@ class ExtensionsTest(TestCase):
""" Installing a plugin """
info = {
'name': 'TestPip',
'module': 'dummy',
'module': 'noop_plugin',
'description': None,
'requirements': ['noop'],
'version': 0
}
root = os.path.join(self.dir, 'plugins', 'dummy_plugin')
module = plugins.load_plugin_from_info(info, root=root)
plugins.install_deps(info)
root = os.path.join(self.dir, 'plugins', 'noop')
module = plugins.load_plugin_from_info(info, root=root, install=True)
assert module.name == 'TestPip'
assert module
import noop

5
tests/test_plugins.py
View File

@ -222,8 +222,9 @@ def make_mini_test(plugin_info):
def _add_tests():
root = os.path.dirname(__file__)
plugs = plugins.load_plugins(os.path.join(root, ".."), loader=plugins.parse_plugin_info)
root = os.path.join(os.path.dirname(__file__), '..')
print(root)
plugs = plugins.load_plugins([root, ], loader=plugins.parse_plugin_info)
for k, v in plugs.items():
pass
t_method = make_mini_test(v)