balkian/senpy
1
0
Fork 0
mirror of https://github.com/gsi-upm/senpy synced 2024-11-28 10:52:28 +00:00
Code Issues Releases Wiki Activity

Added tests for all "discoverable" plugins

Browse Source 
Closes #39
This commit is contained in:
J. Fernando Sánchez 2017-07-12 17:41:14 +02:00
parent 8e4578dc25
commit 4d7e8e7589
9 changed files with 130 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
senpy/extensions.py
View File

@ -15,25 +15,12 @@ from threading import Thread
import os
import copy
import fnmatch
import inspect
import sys
import importlib
import logging
import traceback
import yaml
import subprocess
logger = logging.getLogger(__name__)
def log_subprocess_output(process):
for line in iter(process.stdout.readline, b''):
logger.info('%r', line)
for line in iter(process.stderr.readline, b''):
logger.error('%r', line)
class Senpy(object):
""" Default Senpy extension for Flask """
@ -330,84 +317,6 @@ class Senpy(object):
th.start()
return th
@classmethod
def validate_info(cls, info):
return all(x in info for x in ('name', 'module', 'description', 'version'))
def install_deps(self):
for i in self.plugins.values():
self._install_deps(i)
@classmethod
def _install_deps(cls, info=None):
requirements = info.get('requirements', [])
if requirements:
pip_args = ['pip']
pip_args.append('install')
pip_args.append('--use-wheel')
for req in requirements:
pip_args.append(req)
logger.info('Installing requirements: ' + str(requirements))
process = subprocess.Popen(pip_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
log_subprocess_output(process)
exitcode = process.wait()
if exitcode != 0:
raise Error("Dependencies not properly installed")
@classmethod
def _load_module(cls, name, root):
sys.path.append(root)
tmp = importlib.import_module(name)
sys.path.remove(root)
return tmp
@classmethod
def _load_plugin_from_info(cls, info, root):
if not cls.validate_info(info):
logger.warn('The module info is not valid.\n\t{}'.format(info))
return None, None
module = info["module"]
name = info["name"]
cls._install_deps(info)
tmp = cls._load_module(module, root)
candidate = None
for _, obj in inspect.getmembers(tmp):
if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
logger.debug(("Found plugin class:"
" {}@{}").format(obj, inspect.getmodule(obj)))
candidate = obj
break
if not candidate:
logger.debug("No valid plugin for: {}".format(module))
return
module = candidate(info=info)
return name, module
@classmethod
def _load_plugin(cls, root, filename):
fpath = os.path.join(root, filename)
logger.debug("Loading plugin: {}".format(fpath))
with open(fpath, 'r') as f:
info = yaml.load(f)
logger.debug("Info: {}".format(info))
return cls._load_plugin_from_info(info, root)
def _load_plugins(self):
plugins = {}
for search_folder in self._search_folders:
for root, dirnames, filenames in os.walk(search_folder):
for filename in fnmatch.filter(filenames, '*.senpy'):
name, plugin = self._load_plugin(root, filename)
if plugin and name:
plugins[name] = plugin
self._outdated = False
return plugins
def teardown(self, exception):
pass
@ -415,7 +324,8 @@ class Senpy(object):
def plugins(self):
""" Return the plugins registered for a given application. """
if self._outdated:
self._plugin_list = self._load_plugins()
self._plugin_list = plugins.load_plugins(self._search_folders)
self._outdated = False
return self._plugin_list
def filter_plugins(self, **kwargs):

6
senpy/models.py
View File

@ -223,12 +223,6 @@ class BaseModel(SenpyMixin, dict):
key = key.replace("__", ":", 1)
return key
def __getitem__(self, key):
return dict.__getitem__(self, self._get_key(key))
def __setitem__(self, key, value):
dict.__setitem__(self, self._get_key(key), value)
def __delitem__(self, key):
dict.__delitem__(self, key)

92
senpy/plugins/__init__.py
View File

@ -1,13 +1,20 @@
from future import standard_library
standard_library.install_aliases()
import inspect
import os.path
import os
import pickle
import logging
import tempfile
import copy
import fnmatch
import inspect
import sys
import subprocess
import importlib
import yaml
from .. import models
from ..api import API_PARAMS
@ -69,6 +76,8 @@ class Plugin(models.Plugin):
if not isinstance(exp, list):
exp = [exp]
check_template(res, exp)
for r in res:
r.validate()
SenpyPlugin = Plugin
@ -193,3 +202,84 @@ def pfilter(plugins, **kwargs):
if kwargs:
candidates = filter(matches, candidates)
return {p.name: p for p in candidates}
def validate_info(info):
return all(x in info for x in ('name', 'module', 'description', 'version'))
def load_module(name, root):
sys.path.append(root)
tmp = importlib.import_module(name)
sys.path.remove(root)
return tmp
def log_subprocess_output(process):
for line in iter(process.stdout.readline, b''):
logger.info('%r', line)
for line in iter(process.stderr.readline, b''):
logger.error('%r', line)
def install_deps(*plugins):
for info in plugins:
requirements = info.get('requirements', [])
if requirements:
pip_args = ['pip']
pip_args.append('install')
pip_args.append('--use-wheel')
for req in requirements:
pip_args.append(req)
logger.info('Installing requirements: ' + str(requirements))
process = subprocess.Popen(pip_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
log_subprocess_output(process)
exitcode = process.wait()
if exitcode != 0:
raise models.Error("Dependencies not properly installed")
def load_plugin_from_info(info, root, validator=validate_info):
if not validator(info):
logger.warn('The module info is not valid.\n\t{}'.format(info))
return None, None
module = info["module"]
name = info["name"]
install_deps(info)
tmp = load_module(module, root)
candidate = None
for _, obj in inspect.getmembers(tmp):
if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
logger.debug(("Found plugin class:"
" {}@{}").format(obj, inspect.getmodule(obj)))
candidate = obj
break
if not candidate:
logger.debug("No valid plugin for: {}".format(module))
return
module = candidate(info=info)
return name, module
def load_plugin(root, filename):
fpath = os.path.join(root, filename)
logger.debug("Loading plugin: {}".format(fpath))
with open(fpath, 'r') as f:
info = yaml.load(f)
logger.debug("Info: {}".format(info))
return load_plugin_from_info(info, root)
def load_plugins(folders, loader=load_plugin):
plugins = {}
for search_folder in folders:
for root, dirnames, filenames in os.walk(search_folder):
for filename in fnmatch.filter(filenames, '*.senpy'):
name, plugin = loader(root, filename)
if plugin and name:
plugins[name] = plugin
return plugins

2
senpy/plugins/example/emoRand/emoRand.py
View File

@ -21,6 +21,6 @@ class RmoRandPlugin(EmotionPlugin):
params = dict()
results = list()
for i in range(100):
res = next(self.analyse_entry(Entry(text="Hello"), params))
res = next(self.analyse_entry(Entry(nif__isString="Hello"), params))
res.validate()
results.append(res.emotions[0]['onyx:hasEmotion'][0]['onyx:hasEmotionCategory'])

2
senpy/plugins/example/rand/rand.py
View File

@ -27,7 +27,7 @@ class RandPlugin(SentimentPlugin):
params = dict()
results = list()
for i in range(100):
res = next(self.analyse_entry(Entry(text="Hello"), params))
res = next(self.analyse_entry(Entry(nif__isString="Hello"), params))
res.validate()
results.append(res.sentiments[0]['marl:hasPolarity'])
assert 'marl:Positive' in results

6
senpy/plugins/sentiment/sentiment140/sentiment140.py
View File

@ -12,7 +12,7 @@ class Sentiment140Plugin(SentimentPlugin):
json.dumps({
"language": lang,
"data": [{
"text": entry.text
"text": entry.nif__isString
}]
}))
p = params.get("prefix", None)
@ -38,11 +38,11 @@ class Sentiment140Plugin(SentimentPlugin):
test_cases = [
{
'entry': {
'text': 'I love Titanic'
'nif:isString': 'I love Titanic'
},
'params': {},
'expected': {
"text": "I love Titanic",
"nif:isString": "I love Titanic",
'sentiments': [
{
'marl:hasPolarity': 'marl:Positive',

3
senpy/schemas/context.jsonld
View File

@ -20,6 +20,9 @@
"@id": "me:hasSuggestions",
"@container": "@set"
},
"onyx:hasEmotion": {
"@container": "@set"
},
"emotions": {
"@id": "onyx:hasEmotionSet",
"@container": "@set"

16
tests/test_extensions.py
View File

@ -10,6 +10,7 @@ except ImportError:
from functools import partial
from senpy.extensions import Senpy
from senpy import plugins
from senpy.models import Error, Results, Entry, EmotionSet, Emotion, Plugin
from flask import Flask
from unittest import TestCase
@ -18,7 +19,7 @@ from unittest import TestCase
class ExtensionsTest(TestCase):
def setUp(self):
self.app = Flask('test_extensions')
self.dir = os.path.join(os.path.dirname(__file__))
self.dir = os.path.dirname(__file__)
self.senpy = Senpy(plugin_folder=self.dir,
app=self.app,
default_plugins=False)
@ -38,8 +39,8 @@ class ExtensionsTest(TestCase):
print(self.senpy.plugins)
assert "Dummy" in self.senpy.plugins
def test_enabling(self):
""" Enabling a plugin """
def test_installing(self):
""" Installing a plugin """
info = {
'name': 'TestPip',
'module': 'dummy',
@ -48,14 +49,13 @@ class ExtensionsTest(TestCase):
'version': 0
}
root = os.path.join(self.dir, 'plugins', 'dummy_plugin')
name, module = self.senpy._load_plugin_from_info(info, root=root)
name, module = plugins.load_plugin_from_info(info, root=root)
assert name == 'TestPip'
assert module
import noop
dir(noop)
self.senpy.install_deps()
def test_installing(self):
def test_enabling(self):
""" Enabling a plugin """
self.senpy.activate_all(sync=True)
assert len(self.senpy.plugins) >= 3
@ -72,7 +72,7 @@ class ExtensionsTest(TestCase):
}
root = os.path.join(self.dir, 'plugins', 'dummy_plugin')
with self.assertRaises(Error):
name, module = self.senpy._load_plugin_from_info(info, root=root)
name, module = plugins.load_plugin_from_info(info, root=root)
def test_disabling(self):
""" Disabling a plugin """
@ -173,7 +173,7 @@ class ExtensionsTest(TestCase):
'onyx:usesEmotionModel': 'emoml:fsre-dimensions'
})
eSet1 = EmotionSet()
eSet1.prov__wasGeneratedBy = plugin['id']
eSet1.prov__wasGeneratedBy = plugin['@id']
eSet1['onyx:hasEmotion'].append(Emotion({
'emoml:arousal': 1,
'emoml:potency': 0,

23
tests/test_plugins.py
View File

@ -7,11 +7,11 @@ import tempfile
from unittest import TestCase
from senpy.models import Results, Entry, EmotionSet, Emotion
from senpy.plugins import SentimentPlugin, ShelfMixin
from senpy import plugins
from senpy.plugins.conversion.emotion.centroids import CentroidConversion
class ShelfDummyPlugin(SentimentPlugin, ShelfMixin):
class ShelfDummyPlugin(plugins.SentimentPlugin, plugins.ShelfMixin):
def activate(self, *args, **kwargs):
if 'counter' not in self.sh:
self.sh['counter'] = 0
@ -202,3 +202,22 @@ class PluginsTest(TestCase):
e["Y-dimension"] = 0.3
res = c._backwards_conversion(e)
assert res["onyx:hasEmotionCategory"] == "c2"
def make_mini_test(plugin):
def mini_test(self):
plugin.test()
return mini_test
def add_tests():
root = os.path.dirname(__file__)
plugs = plugins.load_plugins(os.path.join(root, ".."))
for k, v in plugs.items():
t_method = make_mini_test(v)
t_method.__name__ = 'test_plugin_{}'.format(k)
setattr(PluginsTest, t_method.__name__, t_method)
del t_method
add_tests()