From 1313853788c0cbdde0a9d7aca5d496be746ca3bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Thu, 14 Jun 2018 15:10:16 +0200 Subject: [PATCH] Several fixes and improvements * Add Topic model * Add PDB post-mortem debugging * Add logger to plugins (`self.log`) * Add NLTK resource auto-download * Force installation of requirements even if adding doesn't work * Add a method to find files in several possible locations. Now the plugin.open method will try these locations IF the file is to be opened in read mode. Otherwise only the SENPY_DATA folder will be used (to avoid writing to the package folder). --- senpy/__main__.py | 2 +- senpy/models.py | 40 ++++++++++++---------- senpy/plugins/__init__.py | 66 +++++++++++++++++++++++++----------- senpy/schemas/context.jsonld | 4 ++- senpy/utils.py | 30 +++++++++------- tests/test_extensions.py | 2 +- 6 files changed, 92 insertions(+), 52 deletions(-) diff --git a/senpy/__main__.py b/senpy/__main__.py index 886675c..a35f774 100644 --- a/senpy/__main__.py +++ b/senpy/__main__.py @@ -130,7 +130,7 @@ def main(): return sp.activate_all() if args.only_test: - easy_test(sp.plugins()) + easy_test(sp.plugins(), debug=args.debug) return print('Senpy version {}'.format(senpy.__version__)) print('Server running on port %s:%d. Ctrl+C to quit' % (args.host, diff --git a/senpy/models.py b/senpy/models.py index 2d74087..7bb8bbe 100644 --- a/senpy/models.py +++ b/senpy/models.py @@ -203,24 +203,27 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)): context_uri=None, prefix=None, expanded=False): - ser = self.serializable() - - result = jsonld.compact( - ser, - self._context, - options={ - 'base': prefix, - 'expandContext': self._context, - 'senpy': prefix - }) - if context_uri: - result['@context'] = context_uri + + result = self.serializable() + if context_uri or with_context: + result['@context'] = context_uri or self._context + + # result = jsonld.compact(result, + # self._context, + # options={ + # 'base': prefix, + # 'expandContext': self._context, + # 'senpy': prefix + # }) if expanded: result = jsonld.expand( result, options={'base': prefix, 'expandContext': self._context}) if not with_context: - del result['@context'] + try: + del result['@context'] + except KeyError: + pass return result def validate(self, obj=None): @@ -323,7 +326,10 @@ def _add_class_from_schema(*args, **kwargs): for i in [ + 'aggregatedEvaluation', 'analysis', + 'dataset', + 'datasets', 'emotion', 'emotionConversion', 'emotionConversionPlugin', @@ -331,19 +337,17 @@ for i in [ 'emotionModel', 'emotionPlugin', 'emotionSet', + 'evaluation', 'entity', 'help', + 'metric', 'plugin', 'plugins', 'response', 'results', 'sentimentPlugin', 'suggestion', - 'aggregatedEvaluation', - 'evaluation', - 'metric', - 'dataset', - 'datasets', + 'topic', ]: _add_class_from_schema(i) diff --git a/senpy/plugins/__init__.py b/senpy/plugins/__init__.py index 80fea08..b3d5b0e 100644 --- a/senpy/plugins/__init__.py +++ b/senpy/plugins/__init__.py @@ -18,6 +18,7 @@ import subprocess import importlib import yaml import threading +import nltk from .. import models, utils from .. import api @@ -95,6 +96,16 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)): self.is_activated = False self._lock = threading.Lock() self.data_folder = data_folder or os.getcwd() + self._directory = os.path.abspath(os.path.dirname(inspect.getfile(self.__class__))) + self._data_paths = ['', + self._directory, + os.path.join(self._directory, 'data'), + self.data_folder] + self._log = logging.getLogger('{}.{}'.format(__name__, self.name)) + + @property + def log(self): + return self._log def validate(self): missing = [] @@ -123,9 +134,9 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)): for case in test_cases: try: self.test_case(case) - logger.debug('Test case passed:\n{}'.format(pprint.pformat(case))) + self.log.debug('Test case passed:\n{}'.format(pprint.pformat(case))) except Exception as ex: - logger.warn('Test case failed:\n{}'.format(pprint.pformat(case))) + self.log.warn('Test case failed:\n{}'.format(pprint.pformat(case))) raise def test_case(self, case): @@ -148,10 +159,22 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)): raise assert not should_fail - def open(self, fpath, *args, **kwargs): - if not os.path.isabs(fpath): - fpath = os.path.join(self.data_folder, fpath) - return open(fpath, *args, **kwargs) + def find_file(self, fname): + for p in self._data_paths: + alternative = os.path.join(p, fname) + if os.path.exists(alternative): + return alternative + raise IOError('File does not exist: {}'.format(fname)) + + def open(self, fpath, mode='r'): + if 'w' in mode: + # When writing, only use absolute paths or data_folder + if not os.path.isabs(fpath): + fpath = os.path.join(self.data_folder, fpath) + else: + fpath = self.find_file(fpath) + + return open(fpath, mode=mode) def serve(self, debug=True, **kwargs): utils.easy(plugin_list=[self, ], plugin_folder=None, debug=debug, **kwargs) @@ -186,7 +209,7 @@ class Analysis(Plugin): def analyse_entries(self, entries, parameters): for entry in entries: - logger.debug('Analysing entry with plugin {}: {}'.format(self, entry)) + self.log.debug('Analysing entry with plugin {}: {}'.format(self, entry)) results = self.analyse_entry(entry, parameters) if inspect.isgenerator(results): for result in results: @@ -375,7 +398,7 @@ class ShelfMixin(object): with self.open(self.shelf_file, 'rb') as p: self._sh = pickle.load(p) except (IndexError, EOFError, pickle.UnpicklingError): - logger.warning('{} has a corrupted shelf file!'.format(self.id)) + self.log.warning('Corrupted shelf file: {}'.format(self.shelf_file)) if not self.get('force_shelf', False): raise return self._sh @@ -402,32 +425,31 @@ class ShelfMixin(object): self._shelf_file = value def save(self): - logger.debug('saving pickle') + self.log.debug('Saving pickle') if hasattr(self, '_sh') and self._sh is not None: with self.open(self.shelf_file, 'wb') as f: pickle.dump(self._sh, f) -def pfilter(plugins, **kwargs): +def pfilter(plugins, plugin_type=Analysis, **kwargs): """ Filter plugins by different criteria """ if isinstance(plugins, models.Plugins): plugins = plugins.plugins elif isinstance(plugins, dict): plugins = plugins.values() - ptype = kwargs.pop('plugin_type', Plugin) logger.debug('#' * 100) - logger.debug('ptype {}'.format(ptype)) - if ptype: - if isinstance(ptype, PluginMeta): - ptype = ptype.__name__ + logger.debug('plugin_type {}'.format(plugin_type)) + if plugin_type: + if isinstance(plugin_type, PluginMeta): + plugin_type = plugin_type.__name__ try: - ptype = ptype[0].upper() + ptype[1:] - pclass = globals()[ptype] + plugin_type = plugin_type[0].upper() + plugin_type[1:] + pclass = globals()[plugin_type] logger.debug('Class: {}'.format(pclass)) candidates = filter(lambda x: isinstance(x, pclass), plugins) except KeyError: - raise models.Error('{} is not a valid type'.format(ptype)) + raise models.Error('{} is not a valid type'.format(plugin_type)) else: candidates = plugins @@ -462,6 +484,7 @@ def _log_subprocess_output(process): def install_deps(*plugins): installed = False + nltk_resources = set() for info in plugins: requirements = info.get('requirements', []) if requirements: @@ -477,6 +500,9 @@ def install_deps(*plugins): installed = True if exitcode != 0: raise models.Error("Dependencies not properly installed") + nltk_resources |= set(info.get('nltk_resources', [])) + + installed |= nltk.download(list(nltk_resources)) return installed @@ -573,12 +599,14 @@ def _instances_in_module(module): def _from_module_name(module, root, info=None, install=True, **kwargs): try: module = load_module(module, root) - except ImportError: + except (ImportError, LookupError): if not install or not info: raise install_deps(info) module = load_module(module, root) for plugin in _from_loaded_module(module=module, root=root, info=info, **kwargs): + if install: + install_deps(plugin) yield plugin diff --git a/senpy/schemas/context.jsonld b/senpy/schemas/context.jsonld index 2e21811..6879e4a 100644 --- a/senpy/schemas/context.jsonld +++ b/senpy/schemas/context.jsonld @@ -10,8 +10,10 @@ "wna": "http://www.gsi.dit.upm.es/ontologies/wnaffect/ns#", "emoml": "http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/emotionml/ns#", "xsd": "http://www.w3.org/2001/XMLSchema#", + "fam": "http://vocab.fusepool.info/fam#", "topics": { - "@id": "dc:subject" + "@id": "nif:topic", + "@container": "@set" }, "entities": { "@id": "me:hasEntities" diff --git a/senpy/utils.py b/senpy/utils.py index 1f14de3..370bb8c 100644 --- a/senpy/utils.py +++ b/senpy/utils.py @@ -1,6 +1,7 @@ from . import models, __version__ from collections import MutableMapping import pprint +import pdb import logging logger = logging.getLogger(__name__) @@ -32,8 +33,8 @@ def check_template(indict, template): if indict != template: raise models.Error(('Differences found.\n' '\tExpected: {}\n' - '\tFound: {}').format(pprint.pformat(indict), - pprint.pformat(template))) + '\tFound: {}').format(pprint.pformat(template), + pprint.pformat(indict))) def convert_dictionary(original, mappings): @@ -67,18 +68,23 @@ def easy_load(app=None, plugin_list=None, plugin_folder=None, **kwargs): return sp, app -def easy_test(plugin_list=None): +def easy_test(plugin_list=None, debug=True): logger.setLevel(logging.DEBUG) logging.getLogger().setLevel(logging.INFO) - if not plugin_list: - import __main__ - logger.info('Loading classes from {}'.format(__main__)) - from . import plugins - plugin_list = plugins.from_module(__main__) - for plug in plugin_list: - plug.test() - logger.info('The tests for {} passed!'.format(plug.name)) - logger.info('All tests passed!') + try: + if not plugin_list: + import __main__ + logger.info('Loading classes from {}'.format(__main__)) + from . import plugins + plugin_list = plugins.from_module(__main__) + for plug in plugin_list: + plug.test() + plug.log.info('My tests passed!') + logger.info('All tests passed!') + except Exception: + if not debug: + raise + pdb.post_mortem() def easy(host='0.0.0.0', port=5000, debug=True, **kwargs): diff --git a/tests/test_extensions.py b/tests/test_extensions.py index af8b11b..b2e8e3b 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -47,7 +47,7 @@ class ExtensionsTest(TestCase): def test_add_delete(self): '''Should be able to add and delete new plugins. ''' - new = plugins.Plugin(name='new', description='new', version=0) + new = plugins.Analysis(name='new', description='new', version=0) self.senpy.add_plugin(new) assert new in self.senpy.plugins() self.senpy.delete_plugin(new)