From 9414b0e3e6589126c052329724ee820b2b555580 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Thu, 28 Nov 2024 13:51:21 +0100 Subject: [PATCH] Minor updates --- CHANGELOG.md | 4 +++- extra-requirements.txt | 2 +- senpy/extensions.py | 2 ++ senpy/plugins/__init__.py | 9 ++++---- senpy/plugins/emotion/anew/emotion-anew.senpy | 6 +++--- ...emotion-anew.py => emotion_anew_plugin.py} | 0 senpy/plugins/emotion/depechemood_plugin.py | 2 ++ .../emotion/wnaffect/emotion-wnaffect.senpy | 7 ------- ...wnaffect.py => emotion_wnaffect_plugin.py} | 7 ++++++- .../sentiment/basic/sentiment-basic.senpy | 8 ------- ...ent-basic.py => sentiment_basic_plugin.py} | 5 +++-- test-requirements.txt | 1 + tests/test_plugins.py | 21 +++++++++++++++---- 13 files changed, 43 insertions(+), 31 deletions(-) rename senpy/plugins/emotion/anew/{emotion-anew.py => emotion_anew_plugin.py} (100%) delete mode 100644 senpy/plugins/emotion/wnaffect/emotion-wnaffect.senpy rename senpy/plugins/emotion/wnaffect/{emotion-wnaffect.py => emotion_wnaffect_plugin.py} (98%) delete mode 100644 senpy/plugins/sentiment/basic/sentiment-basic.senpy rename senpy/plugins/sentiment/basic/{sentiment-basic.py => sentiment_basic_plugin.py} (97%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79995a2..29e2e09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Fixed +* Tests now use strict mode to detect errors on optional plugins ### Added * The code of many senpy community plugins have been included by default. However, additional files (e.g., licensed data) and/or installing additional dependencies may be necessary for some plugins. Read each plugin's documentation for more information. -* `--strict` flag, to fail and not start when a * `optional` attribute in plugins. Optional plugins may fail to load or activate but the server will be started regardless, unless running in strict mode +* `--strict` flag to set strict mode. In this mode, the server will not start when optional plugins fail to load * Option in shelf plugins to ignore pickling errors ### Removed * `--only-install`, `--only-test` and `--only-list` flags were removed in favor of `--no-run` + `--install`/`--test`/`--dependencies` diff --git a/extra-requirements.txt b/extra-requirements.txt index 79ac7ea..aa907f7 100644 --- a/extra-requirements.txt +++ b/extra-requirements.txt @@ -1,6 +1,6 @@ gsitk>0.1.9.1 flask_cors==3.0.10 -Pattern==3.6 lxml==4.9.3 pandas==2.1.1 textblob==0.17.1 +git+https://github.com/uob-vil/pattern.git#pattern diff --git a/senpy/extensions.py b/senpy/extensions.py index cbbb041..3f68fcc 100644 --- a/senpy/extensions.py +++ b/senpy/extensions.py @@ -77,6 +77,8 @@ class Senpy(object): if app is not None: self.init_app(app) self._conversion_candidates = {} + if self.install: + self.install_deps() def init_app(self, app): """ Initialise a flask app to add plugins to its context """ diff --git a/senpy/plugins/__init__.py b/senpy/plugins/__init__.py index 42307b4..de3c290 100644 --- a/senpy/plugins/__init__.py +++ b/senpy/plugins/__init__.py @@ -777,7 +777,8 @@ def install_deps(*plugins): if exitcode != 0: raise models.Error( "Dependencies not properly installed: {}".format(pip_args)) - return installed or download(list(nltk_resources)) + installed_nltk = download(list(nltk_resources)) + return installed or installed_nltk is_plugin_file = re.compile(r'.*\.senpy$|senpy_[a-zA-Z0-9_]+\.py$|' @@ -826,10 +827,10 @@ def from_info(info, root=None, strict=False, **kwargs): fun = partial(one_from_module, module, root=root, info=info, **kwargs) try: return fun() - except (ImportError, LookupError): - if strict or not str(info.get("optional", "false")).lower() in ["True", "true", "t"]: + except (ImportError, LookupError) as ex: + if strict or str(info.get("optional", "false")).lower() not in ["true", "t"]: raise - print(f"Could not import plugin: { info }") + print(f"Could not import plugin: { info }: {ex}", file=sys.stderr) return FailedPlugin(info, fun) diff --git a/senpy/plugins/emotion/anew/emotion-anew.senpy b/senpy/plugins/emotion/anew/emotion-anew.senpy index b9d72fa..9052047 100644 --- a/senpy/plugins/emotion/anew/emotion-anew.senpy +++ b/senpy/plugins/emotion/anew/emotion-anew.senpy @@ -1,5 +1,5 @@ --- -module: emotion-anew +module: emotion_anew_plugin optional: true requirements: - numpy @@ -8,5 +8,5 @@ requirements: - scipy - scikit-learn - textblob -- pattern -- lxml \ No newline at end of file +- 'git+https://github.com/uob-vil/pattern.git' +- lxml diff --git a/senpy/plugins/emotion/anew/emotion-anew.py b/senpy/plugins/emotion/anew/emotion_anew_plugin.py similarity index 100% rename from senpy/plugins/emotion/anew/emotion-anew.py rename to senpy/plugins/emotion/anew/emotion_anew_plugin.py diff --git a/senpy/plugins/emotion/depechemood_plugin.py b/senpy/plugins/emotion/depechemood_plugin.py index 4595c56..3f26ab1 100644 --- a/senpy/plugins/emotion/depechemood_plugin.py +++ b/senpy/plugins/emotion/depechemood_plugin.py @@ -10,6 +10,8 @@ import sys import string import numpy as np from six.moves import urllib +import nltk + from nltk.corpus import stopwords from senpy import EmotionBox, models diff --git a/senpy/plugins/emotion/wnaffect/emotion-wnaffect.senpy b/senpy/plugins/emotion/wnaffect/emotion-wnaffect.senpy deleted file mode 100644 index 2ecdf63..0000000 --- a/senpy/plugins/emotion/wnaffect/emotion-wnaffect.senpy +++ /dev/null @@ -1,7 +0,0 @@ ---- -module: emotion-wnaffect -optional: true -requirements: -- nltk>=3.0.5 -- lxml>=3.4.2 -async: false diff --git a/senpy/plugins/emotion/wnaffect/emotion-wnaffect.py b/senpy/plugins/emotion/wnaffect/emotion_wnaffect_plugin.py similarity index 98% rename from senpy/plugins/emotion/wnaffect/emotion-wnaffect.py rename to senpy/plugins/emotion/wnaffect/emotion_wnaffect_plugin.py index 6e91ddf..7c152e2 100644 --- a/senpy/plugins/emotion/wnaffect/emotion-wnaffect.py +++ b/senpy/plugins/emotion/wnaffect/emotion_wnaffect_plugin.py @@ -31,11 +31,16 @@ class WNAffect(EmotionPlugin, ShelfMixin): 'options': ['en',] } } + optional = True + requirements = [ + "nltk>=3.0.5", + "lxml>=3.4.2" + ] synsets_path = "a-synsets.xml" hierarchy_path = "a-hierarchy.xml" wn16_path = "wordnet1.6/dict" onyx__usesEmotionModel = "emoml:big6" - nltk_resources = ['stopwords', 'averaged_perceptron_tagger', 'wordnet'] + nltk_resources = ['stopwords', 'averaged_perceptron_tagger_eng', 'wordnet'] def _load_synsets(self, synsets_path): """Returns a dictionary POS tag -> synset offset -> emotion (str -> int -> str).""" diff --git a/senpy/plugins/sentiment/basic/sentiment-basic.senpy b/senpy/plugins/sentiment/basic/sentiment-basic.senpy deleted file mode 100644 index c038a8f..0000000 --- a/senpy/plugins/sentiment/basic/sentiment-basic.senpy +++ /dev/null @@ -1,8 +0,0 @@ ---- -module: sentiment-basic -optional: true -requirements: -- nltk>=3.0.5 -- scipy>=0.14.0 -- textblob - diff --git a/senpy/plugins/sentiment/basic/sentiment-basic.py b/senpy/plugins/sentiment/basic/sentiment_basic_plugin.py similarity index 97% rename from senpy/plugins/sentiment/basic/sentiment-basic.py rename to senpy/plugins/sentiment/basic/sentiment_basic_plugin.py index b639e2d..d76d70b 100644 --- a/senpy/plugins/sentiment/basic/sentiment-basic.py +++ b/senpy/plugins/sentiment/basic/sentiment_basic_plugin.py @@ -39,8 +39,9 @@ class SentimentBasic(SentimentBox): pos_path = "unigram_spanish.pickle" maxPolarityValue = 1 minPolarityValue = -1 - nltk_resources = ['punkt','wordnet', 'omw', 'omw-1.4'] - + nltk_resources = ['punkt_tab','wordnet', 'omw', 'omw-1.4'] + optional = True + requirements = ['nltk>=3.0.5', 'scipy>=0.14.0', 'textblob==0.17'] with_polarity = False def _load_swn(self): diff --git a/test-requirements.txt b/test-requirements.txt index 3a4ac9a..a15b24e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,3 +3,4 @@ pytest-cov pytest pandas noop +gsitk diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 2760e0f..44f7a5e 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -31,6 +31,7 @@ from senpy import config import pandas as pd +ROOT = os.path.join(os.path.dirname(__file__), '..') class ShelfDummyPlugin(plugins.SentimentPlugin, plugins.ShelfMixin): '''Dummy plugin for tests.''' @@ -332,6 +333,18 @@ class PluginsTest(TestCase): res = c._backwards_conversion(e) assert res["onyx:hasEmotionCategory"] == "c2" + def test_installation(self): + sentiment = next(plugins.from_path('senpy/plugins/sentiment/basic/sentiment_basic_plugin.py')) + assert sentiment + inst, missing, nltk_deps = plugins.list_dependencies(sentiment) + assert 'punkt_tab' in nltk_deps + + emotion = next(plugins.from_path('senpy/plugins/emotion/wnaffect/emotion_wnaffect_plugin.py')) + assert emotion + inst, missing, nltk_deps = plugins.list_dependencies(emotion) + assert 'averaged_perceptron_tagger_eng' in nltk_deps + + def _test_evaluation(self): testdata = [] for i in range(50): @@ -387,15 +400,15 @@ class PluginsTest(TestCase): def make_mini_test(fpath): def mini_test(self): - for plugin in plugins.from_path(fpath, install=True, strict=config.strict): + for plugin in plugins.from_path(fpath, strict=True): + plugins.install_deps(plugin) plugin.test() return mini_test def _add_tests(): - root = os.path.join(os.path.dirname(__file__), '..') - print(root) - for fpath in plugins.find_plugins([root, ]): + print(ROOT) + for fpath in plugins.find_plugins([ROOT, ]): pass t_method = make_mini_test(fpath) t_method.__name__ = 'test_plugin_{}'.format(fpath)