balkian/senpy
1
0
Fork 0
mirror of https://github.com/gsi-upm/senpy synced 2024-11-22 00:02:28 +00:00
Code Issues Releases Wiki Activity

Improved plugins, better tests, gevent

Browse Source 
Moved from Yapsy again (it is not flexible enough), now we use a
custom solution.
The activation and deactivation of plugins is asynchronous, so
that plugins that take a long time don't interfere with the rest.
This commit is contained in:
J. Fernando Sánchez 2014-12-01 18:27:20 +01:00
parent 10f4782ad7
commit ff14925056
17 changed files with 241 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
app.py
View File

@ -19,18 +19,25 @@ Simple Sentiment Analysis server for EUROSENTIMENT
This class shows how to use the nif_server module to create custom services.
"""
from gevent.monkey import patch_all; patch_all()
import gevent
import config
from flask import Flask
from senpy.extensions import Senpy
import logging
import os
from gevent.wsgi import WSGIServer
logging.basicConfig(level=logging.DEBUG)
app = Flask(__name__)
mypath = os.path.dirname(os.path.realpath(__file__))
sp = Senpy(app, os.path.join(mypath, "plugins"))
sp.activate_all()
if __name__ == '__main__':
import logging
logging.basicConfig(level=config.DEBUG)
app.debug = config.DEBUG
app.run(host="0.0.0.0", use_reloader=False)
http_server = WSGIServer(('', 5000), app)
http_server.serve_forever()

15
plugins/sentiment140/__init__.py → plugins/sentiment140/sentiment140.py
View File

@ -6,19 +6,6 @@ from senpy.models import Response, Opinion, Entry
class Sentiment140Plugin(SentimentPlugin):
EXTRA_PARAMS = {
"language": {"aliases": ["language", "l"],
"required": False,
"options": ["es", "en", "auto"],
}
}
def __init__(self, **kwargs):
super(Sentiment140Plugin, self).__init__(name="sentiment140",
version="2.0",
extraparams=self.EXTRA_PARAMS,
**kwargs)
def analyse(self, **params):
lang = params.get("language", "auto")
res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
@ -45,5 +32,3 @@ class Sentiment140Plugin(SentimentPlugin):
entry.language = lang
response.entries.append(entry)
return response
plugin = Sentiment140Plugin()

15
plugins/sentiment140/sentiment140.senpy Normal file
View File

@ -0,0 +1,15 @@
{
"name": "sentiment140",
"module": "sentiment140",
"description": "What my plugin broadly does",
"author": "@balkian",
"version": "0.1",
"extra_params": {
"language": {
"aliases": ["language", "l"],
"required": false,
"options": ["es", "en", "auto"]
}
},
"requirements": {}
}

4
requirements.txt
View File

@ -1,4 +1,6 @@
Flask==0.10.1
gunicorn==19.0.0
requests==2.4.1
GitPython==0.3.2.RC1
GitPython==0.3.2.RC1
Yapsy>=1.10.423
gevent>=1.0.1

19
senpy/blueprints.py
View File

@ -110,23 +110,22 @@ def default():
@nif_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
def plugins(plugin=None, action="list"):
filt = {}
sp = current_app.senpy
if plugin:
filt["name"] = plugin
plugs = current_app.senpy.filter_plugins(**filt)
plugs = sp.filter_plugins(**filt)
if plugin and not plugs:
return "Plugin not found", 400
if action == "list":
with_params = request.args.get("params", "") == "1"
dic = {plug: plugs[plug].jsonable(with_params) for plug in plugs}
if plugin:
dic = plugs[plugin].jsonable(with_params)
else:
dic = {plug: plugs[plug].jsonable(with_params) for plug in plugs}
return jsonify(dic)
if action == "disable":
current_app.senpy.disable_plugin(plugin)
return "Ok"
elif action == "enable":
current_app.senpy.enable_plugin(plugin)
return "Ok"
elif action == "reload":
current_app.senpy.reload_plugin(plugin)
method = "{}_plugin".format(action)
if(hasattr(sp, method)):
getattr(sp, method)(plugin)
return "Ok"
else:
return "action '{}' not allowed".format(action), 400

118
senpy/extensions.py
View File

@ -1,16 +1,21 @@
"""
"""
import os
import fnmatch
import inspect
import sys
import imp
import logging
import gevent
import json
logger = logging.getLogger(__name__)
from .plugins import SentimentPlugin, EmotionPlugin
from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
from .blueprints import nif_blueprint
from git import Repo, InvalidGitRepositoryError
from functools import partial
class Senpy(object):
@ -45,11 +50,13 @@ class Senpy(object):
app.register_blueprint(nif_blueprint)
def add_folder(self, folder):
logger.debug("Adding folder: %s", folder)
if os.path.isdir(folder):
self._search_folders.add(folder)
self._outdated = True
return True
else:
logger.debug("Not a folder: %s", folder)
return False
def analyse(self, **params):
@ -59,17 +66,20 @@ class Senpy(object):
algo = params["algorithm"]
elif self.plugins:
algo = self.default_plugin
if algo in self.plugins and self.plugins[algo].enabled:
plug = self.plugins[algo]
resp = plug.analyse(**params)
resp.analysis.append(plug.jsonable())
return resp
if algo in self.plugins:
if self.plugins[algo].is_activated:
plug = self.plugins[algo]
resp = plug.analyse(**params)
resp.analysis.append(plug.jsonable())
return resp
logger.debug("Plugin not activated: {}".format(algo))
else:
logger.debug("The algorithm '{}' is not valid\nValid algorithms: {}".format(algo, self.plugins.keys()))
return {"status": 400, "message": "The algorithm '{}' is not valid".format(algo)}
@property
def default_plugin(self):
candidates = self.filter_plugins(enabled=True)
candidates = self.filter_plugins(is_activated=True)
if len(candidates) > 0:
candidate = candidates.keys()[0]
logger.debug("Default: {}".format(candidate))
@ -80,11 +90,38 @@ class Senpy(object):
def parameters(self, algo):
return getattr(self.plugins.get(algo or self.default_plugin), "params", {})
def enable_plugin(self, plugin):
self.plugins[plugin].enable()
def activate_all(self, sync=False):
ps = []
for plug in self.plugins.keys():
ps.append(self.activate_plugin(plug, sync=sync))
return ps
def disable_plugin(self, plugin):
self.plugins[plugin].disable()
def deactivate_all(self, sync=False):
ps = []
for plug in self.plugins.keys():
ps.append(self.deactivate_plugin(plug, sync=sync))
return ps
def _set_active_plugin(self, plugin_name, active=True, *args, **kwargs):
self.plugins[plugin_name].is_activated = active
def activate_plugin(self, plugin_name, sync=False):
plugin = self.plugins[plugin_name]
th = gevent.spawn(plugin.activate)
th.link_value(partial(self._set_active_plugin, plugin_name, True))
if sync:
th.join()
else:
return th
def deactivate_plugin(self, plugin_name, sync=False):
plugin = self.plugins[plugin_name]
th = gevent.spawn(plugin.deactivate)
th.link_value(partial(self._set_active_plugin, plugin_name, False))
if sync:
th.join()
else:
return th
def reload_plugin(self, plugin):
logger.debug("Reloading {}".format(plugin))
@ -94,38 +131,47 @@ class Senpy(object):
self.plugins[nplug.name] = nplug
@staticmethod
def _load_plugin(plugin, search_folder, enabled=True):
logger.debug("Loading plugins")
sys.path.append(search_folder)
(fp, pathname, desc) = imp.find_module(plugin)
def _load_plugin(root, filename):
logger.debug("Loading plugin: {}".format(filename))
fpath = os.path.join(root, filename)
with open(fpath,'r') as f:
info = json.load(f)
logger.debug("Info: {}".format(info))
sys.path.append(root)
module = info["module"]
name = info["name"]
(fp, pathname, desc) = imp.find_module(module, [root,])
try:
tmp = imp.load_module(plugin, fp, pathname, desc).plugin
sys.path.remove(search_folder)
tmp.path = search_folder
tmp = imp.load_module(module, fp, pathname, desc)
sys.path.remove(root)
candidate = None
for _, obj in inspect.getmembers(tmp):
if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
logger.debug("Found plugin class: {}@{}".format(obj, inspect.getmodule(obj)))
candidate = obj
break
if not candidate:
logger.debug("No valid plugin for: {}".format(filename))
return
module = candidate(info=info)
try:
repo_path = os.path.join(search_folder, plugin)
tmp.repo = Repo(repo_path)
repo_path = root
module.repo = Repo(repo_path)
except InvalidGitRepositoryError:
tmp.repo = None
if not hasattr(tmp, "enabled"):
tmp.enabled = enabled
tmp.module = plugin
module.repo = None
except Exception as ex:
tmp = None
logger.debug("Exception importing {}: {}".format(plugin, ex))
return tmp
logger.debug("Exception importing {}: {}".format(filename, ex))
return None, None
return name, module
def _load_plugins(self):
plugins = {}
for search_folder in self._search_folders:
for item in os.listdir(search_folder):
if os.path.isdir(os.path.join(search_folder, item)) \
and os.path.exists(os.path.join(search_folder,
item,
"__init__.py")):
plugin = self._load_plugin(item, search_folder)
for root, dirnames, filenames in os.walk(search_folder):
for filename in fnmatch.filter(filenames, '*.senpy'):
name, plugin = self._load_plugin(root, filename)
if plugin:
plugins[plugin.name] = plugin
plugins[name] = plugin
self._outdated = False
return plugins
@ -133,10 +179,6 @@ class Senpy(object):
def teardown(self, exception):
pass
def enable_all(self):
for plugin in self.plugins:
self.enable_plugin(plugin)
@property
def plugins(self):
""" Return the plugins registered for a given application. """

11
senpy/models.py
View File

@ -77,7 +77,7 @@ class Entry(Leaf):
class Opinion(Leaf):
opinionContext = {}
opinionContext = {"@vocab": "http://www.gsi.dit.upm.es/ontologies/marl/ns#"}
def __init__(self, polarityValue=None, hasPolarity=None, *args, **kwargs):
super(Opinion, self).__init__(context=self.opinionContext,
*args,
@ -93,7 +93,14 @@ class EmotionSet(Leaf):
def __init__(self, emotions=None, *args, **kwargs):
if not emotions:
emotions = []
super(EmotionSet, self).__init__(context=self.emotionContext,
super(EmotionSet, self).__init__(context=EmotionSet.emotionContext,
*args,
**kwargs)
self.emotions = emotions or []
class Emotion(Leaf):
emotionContext = {}
def __init__(self, emotions=None, *args, **kwargs):
super(EmotionSet, self).__init__(context=Emotion.emotionContext,
*args,
**kwargs)

42
senpy/plugins.py
View File

@ -1,4 +1,5 @@
import logging
import ConfigParser
logger = logging.getLogger(__name__)
@ -38,27 +39,28 @@ PARAMS = {"input": {"aliases": ["i", "input"],
class SenpyPlugin(object):
def __init__(self, name=None, version=None, extraparams=None, params=None):
logger.debug("Initialising {}".format(name))
self.name = name
self.version = version
if params:
self.params = params
else:
self.params = PARAMS.copy()
if extraparams:
self.params.update(extraparams)
self.extraparams = extraparams or {}
self.enabled = True
def __init__(self, info=None):
if not info:
raise ValueError("You need to provide configuration information for the plugin.")
logger.debug("Initialising {}".format(info))
self.name = info["name"]
self.version = info["version"]
self.params = info.get("params", PARAMS.copy())
self.extra_params = info.get("extra_params", {})
self.params.update(self.extra_params)
self.is_activated = False
self.info = info
def analyse(self, *args, **kwargs):
logger.debug("Analysing with: {} {}".format(self.name, self.version))
pass
def enable(self):
self.enabled = True
def activate(self):
pass
def deactivate(self):
pass
def disable(self):
self.enabled = False
@property
def id(self):
@ -66,15 +68,15 @@ class SenpyPlugin(object):
def jsonable(self, parameters=False):
resp = {
"@id": self.id,
"enabled": self.enabled,
"@id": "{}_{}".format(self.name, self.version),
"is_activated": self.is_activated,
}
if hasattr(self, "repo") and self.repo:
resp["repo"] = self.repo.remotes[0].url
if parameters:
resp["parameters"] = self.params
elif self.extraparams:
resp["extra_parameters"] = self.extraparams
elif self.extra_params:
resp["extra_parameters"] = self.extra_params
return resp

2
setup.py
View File

@ -8,7 +8,7 @@ install_reqs = parse_requirements("requirements.txt")
# e.g. ['django==1.5.1', 'mezzanine==1.4.6']
reqs = [str(ir.req) for ir in install_reqs]
VERSION = "0.2.9"
VERSION = "0.3.0"
print(reqs)

1
tests/__init__.py
View File

@ -1 +0,0 @@

42
tests/blueprints_test/__init__.py
View File

@ -9,6 +9,7 @@ except ImportError:
from senpy.extensions import Senpy
from flask import Flask
from flask.ext.testing import TestCase
from gevent import sleep
def check_dict(indic, template):
@ -22,6 +23,7 @@ class BlueprintsTest(TestCase):
self.senpy.init_app(self.app)
self.dir = os.path.join(os.path.dirname(__file__), "..")
self.senpy.add_folder(self.dir)
self.senpy.activate_plugin("Dummy", sync=True)
return self.app
def test_home(self):
@ -36,3 +38,43 @@ class BlueprintsTest(TestCase):
}
assert check_dict(resp.json, atleast)
def test_analysis(self):
""" The dummy plugin returns an empty response, it should contain the context """
resp = self.client.get("/?i=My aloha mohame")
self.assert200(resp)
logging.debug(resp.json)
assert "@context" in resp.json
assert check_dict(resp.json["@context"], {"marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#"})
assert "entries" in resp.json
def test_list(self):
""" List the plugins """
resp = self.client.get("/plugins/")
self.assert200(resp)
logging.debug(resp.json)
assert "Dummy" in resp.json
def test_detail(self):
""" Show only one plugin"""
resp = self.client.get("/plugins/Dummy")
self.assert200(resp)
logging.debug(resp.json)
assert "@id" in resp.json
assert resp.json["@id"] == "Dummy_0.1"
def test_activate(self):
""" Activate and deactivate one plugin """
resp = self.client.get("/plugins/Dummy/deactivate")
self.assert200(resp)
sleep(0.5)
resp = self.client.get("/plugins/Dummy")
self.assert200(resp)
assert "is_activated" in resp.json
assert resp.json["is_activated"] == False
resp = self.client.get("/plugins/Dummy/activate")
self.assert200(resp)
sleep(0.5)
resp = self.client.get("/plugins/Dummy")
self.assert200(resp)
assert "is_activated" in resp.json
assert resp.json["is_activated"] == True

3
tests/dummy_plugin/__init__.py
View File

@ -1,3 +0,0 @@
from senpy.plugins import SenpyPlugin
plugin = SenpyPlugin("dummy")

8
tests/dummy_plugin/dummy.py Normal file
View File

@ -0,0 +1,8 @@
from senpy.plugins import SentimentPlugin
from senpy.models import Response
class DummyPlugin(SentimentPlugin):
pass
def analyse(self, *args, **kwargs):
return Response()

7
tests/dummy_plugin/dummy.senpy Normal file
View File

@ -0,0 +1,7 @@
{
"name": "Dummy",
"module": "dummy",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1"
}

37
tests/extensions_test/__init__.py
View File

@ -13,10 +13,10 @@ from flask.ext.testing import TestCase
class ExtensionsTest(TestCase):
def create_app(self):
self.app = Flask("test_extensions")
self.senpy = Senpy()
self.senpy.init_app(self.app)
self.dir = os.path.join(os.path.dirname(__file__), "..")
self.senpy.add_folder(self.dir)
self.senpy = Senpy(plugin_folder=self.dir)
self.senpy.init_app(self.app)
self.senpy.activate_plugin("Dummy", sync=True)
return self.app
def test_init(self):
@ -31,41 +31,42 @@ class ExtensionsTest(TestCase):
# noinspection PyProtectedMember
assert self.dir in self.senpy._search_folders
print self.senpy.plugins
assert "dummy" in self.senpy.plugins
assert "Dummy" in self.senpy.plugins
def test_enabling(self):
""" Enabling a plugin """
self.senpy.enable_plugin("dummy")
assert self.senpy.plugins["dummy"].enabled
self.senpy.activate_all(sync=True)
assert len(self.senpy.plugins) == 2
assert self.senpy.plugins["Sleep"].is_activated
def test_disabling(self):
""" Disabling a plugin """
self.senpy.enable_plugin("dummy")
self.senpy.disable_plugin("dummy")
assert not self.senpy.plugins["dummy"].enabled
self.senpy.deactivate_all(sync=True)
assert self.senpy.plugins["Dummy"].is_activated == False
assert self.senpy.plugins["Sleep"].is_activated == False
def test_default(self):
""" Default plugin should be set """
assert self.senpy.default_plugin
assert self.senpy.default_plugin == "dummy"
assert self.senpy.default_plugin == "Dummy"
def test_analyse(self):
""" Using a plugin """
with mock.patch.object(self.senpy.plugins["dummy"], "analyse") as mocked:
self.senpy.analyse(algorithm="dummy", input="tupni", output="tuptuo")
with mock.patch.object(self.senpy.plugins["Dummy"], "analyse") as mocked:
self.senpy.analyse(algorithm="Dummy", input="tupni", output="tuptuo")
self.senpy.analyse(input="tupni", output="tuptuo")
mocked.assert_any_call(input="tupni", output="tuptuo", algorithm="dummy")
mocked.assert_any_call(input="tupni", output="tuptuo", algorithm="Dummy")
mocked.assert_any_call(input="tupni", output="tuptuo")
for plug in self.senpy.plugins:
self.senpy.disable_plugin(plug)
self.senpy.deactivate_plugin(plug, sync=True)
resp = self.senpy.analyse(input="tupni")
logging.debug("Response: {}".format(resp))
assert resp["status"] == 400
def test_filtering(self):
""" Filtering plugins """
assert len(self.senpy.filter_plugins(name="dummy")) > 0
assert len(self.senpy.filter_plugins(name="Dummy")) > 0
assert not len(self.senpy.filter_plugins(name="notdummy"))
assert self.senpy.filter_plugins(name="dummy", enabled=True)
self.senpy.disable_plugin("dummy")
assert not len(self.senpy.filter_plugins(name="dummy", enabled=True))
assert self.senpy.filter_plugins(name="Dummy", is_activated=True)
self.senpy.deactivate_plugin("Dummy", sync=True)
assert not len(self.senpy.filter_plugins(name="Dummy", is_activated=True))

10
tests/sleep_plugin/sleep.py Normal file
View File

@ -0,0 +1,10 @@
from senpy.plugins import SenpyPlugin
from time import sleep
class SleepPlugin(SenpyPlugin):
def __init__(self, info, *args, **kwargs):
super(SleepPlugin, self).__init__(info, *args, **kwargs)
self.timeout = int(info["timeout"])
def activate(self, *args, **kwargs):
sleep(self.timeout)

8
tests/sleep_plugin/sleep.senpy Normal file
View File

@ -0,0 +1,8 @@
{
"name": "Sleep",
"module": "sleep",
"description": "I am dummy",
"author": "@balkian",
"version": "0.1",
"timeout": "2"
}