balkian/senpy
1
0
Fork 0
mirror of https://github.com/gsi-upm/senpy synced 2024-11-22 08:12:27 +00:00
Code Issues Releases Wiki Activity

YAPFed

Browse Source
This commit is contained in:
J. Fernando Sánchez 2017-01-10 10:16:45 +01:00
parent b543a4614e
commit 7fd69cc690
19 changed files with 283 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Makefile
View File

@ -7,6 +7,10 @@ VERSION=$(shell cat $(NAME)/VERSION)
all: build run all: build run
yapf:
yapf -i -r senpy
yapf -i -r tests
dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS)) dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS))
ln -s Dockerfile-$(PYMAIN) Dockerfile ln -s Dockerfile-$(PYMAIN) Dockerfile
@ -71,4 +75,4 @@ pip_test: $(addprefix pip_test-,$(PYVERSIONS))
run: build run: build
docker run --rm -p 5000:5000 -ti '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)' docker run --rm -p 5000:5000 -ti '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)'
.PHONY: test test-% build-% build test test_pip run .PHONY: test test-% build-% build test test_pip run yapf

1
senpy/__init__.py
View File

@ -22,5 +22,4 @@ import os
from .version import __version__ from .version import __version__
__all__ = ['api', 'blueprints', 'cli', 'extensions', 'models', 'plugins'] __all__ = ['api', 'blueprints', 'cli', 'extensions', 'models', 'plugins']

78
senpy/__main__.py
View File

@ -34,42 +34,51 @@ patch_all(thread=False)
SERVER_PORT = os.environ.get("PORT", 5000) SERVER_PORT = os.environ.get("PORT", 5000)
def main(): def main():
parser = argparse.ArgumentParser(description='Run a Senpy server') parser = argparse.ArgumentParser(description='Run a Senpy server')
parser.add_argument('--level', parser.add_argument(
'-l', '--level',
metavar='logging_level', '-l',
type=str, metavar='logging_level',
default="INFO", type=str,
help='Logging level') default="INFO",
parser.add_argument('--debug', help='Logging level')
'-d', parser.add_argument(
action='store_true', '--debug',
default=False, '-d',
help='Run the application in debug mode') action='store_true',
parser.add_argument('--default-plugins', default=False,
action='store_true', help='Run the application in debug mode')
default=False, parser.add_argument(
help='Load the default plugins') '--default-plugins',
parser.add_argument('--host', action='store_true',
type=str, default=False,
default="127.0.0.1", help='Load the default plugins')
help='Use 0.0.0.0 to accept requests from any host.') parser.add_argument(
parser.add_argument('--port', '--host',
'-p', type=str,
type=int, default="127.0.0.1",
default=SERVER_PORT, help='Use 0.0.0.0 to accept requests from any host.')
help='Port to listen on.') parser.add_argument(
parser.add_argument('--plugins-folder', '--port',
'-f', '-p',
type=str, type=int,
default='plugins', default=SERVER_PORT,
help='Where to look for plugins.') help='Port to listen on.')
parser.add_argument('--only-install', parser.add_argument(
'-i', '--plugins-folder',
action='store_true', '-f',
default=False, type=str,
help='Do not run a server, only install the dependencies of the plugins.') default='plugins',
help='Where to look for plugins.')
parser.add_argument(
'--only-install',
'-i',
action='store_true',
default=False,
help='Do not run a server, only install the dependencies of the plugins.'
)
args = parser.parse_args() args = parser.parse_args()
logging.basicConfig() logging.basicConfig()
rl = logging.getLogger() rl = logging.getLogger()
@ -92,5 +101,6 @@ def main():
http_server.stop() http_server.stop()
sp.deactivate_all() sp.deactivate_all()
if __name__ == '__main__': if __name__ == '__main__':
main() main()

13
senpy/api.py
View File

@ -25,7 +25,7 @@ CLI_PARAMS = {
"required": True, "required": True,
"default": "." "default": "."
}, },
} }
NIF_PARAMS = { NIF_PARAMS = {
"input": { "input": {
@ -96,10 +96,11 @@ def parse_params(indict, spec=NIF_PARAMS):
outdict[param] not in spec[param]["options"]: outdict[param] not in spec[param]["options"]:
wrong_params[param] = spec[param] wrong_params[param] = spec[param]
if wrong_params: if wrong_params:
message = Error(status=404, message = Error(
message="Missing or invalid parameters", status=404,
parameters=outdict, message="Missing or invalid parameters",
errors={param: error for param, error in parameters=outdict,
iteritems(wrong_params)}) errors={param: error
for param, error in iteritems(wrong_params)})
raise message raise message
return outdict return outdict

21
senpy/blueprints.py
View File

@ -30,6 +30,7 @@ logger = logging.getLogger(__name__)
api_blueprint = Blueprint("api", __name__) api_blueprint = Blueprint("api", __name__)
demo_blueprint = Blueprint("demo", __name__) demo_blueprint = Blueprint("demo", __name__)
def get_params(req): def get_params(req):
if req.method == 'POST': if req.method == 'POST':
indict = req.form.to_dict(flat=True) indict = req.form.to_dict(flat=True)
@ -44,17 +45,20 @@ def get_params(req):
def index(): def index():
return render_template("index.html") return render_template("index.html")
@api_blueprint.route('/contexts/<entity>.jsonld') @api_blueprint.route('/contexts/<entity>.jsonld')
def context(entity="context"): def context(entity="context"):
return jsonify({"@context": Response.context}) return jsonify({"@context": Response.context})
@api_blueprint.route('/schemas/<schema>') @api_blueprint.route('/schemas/<schema>')
def schema(schema="definitions"): def schema(schema="definitions"):
try: try:
return jsonify(read_schema(schema)) return jsonify(read_schema(schema))
except Exception: # Should be FileNotFoundError, but it's missing from py2 except Exception: # Should be FileNotFoundError, but it's missing from py2
return Error(message="Schema not found", status=404).flask() return Error(message="Schema not found", status=404).flask()
def basic_api(f): def basic_api(f):
@wraps(f) @wraps(f)
def decorated_function(*args, **kwargs): def decorated_function(*args, **kwargs):
@ -73,12 +77,15 @@ def basic_api(f):
response = ex response = ex
in_headers = web_params["inHeaders"] != "0" in_headers = web_params["inHeaders"] != "0"
headers = {'X-ORIGINAL-PARAMS': raw_params} headers = {'X-ORIGINAL-PARAMS': raw_params}
return response.flask(in_headers=in_headers, return response.flask(
headers=headers, in_headers=in_headers,
context_uri=url_for('api.context', entity=type(response).__name__, headers=headers,
_external=True)) context_uri=url_for(
'api.context', entity=type(response).__name__, _external=True))
return decorated_function return decorated_function
@api_blueprint.route('/', methods=['POST', 'GET']) @api_blueprint.route('/', methods=['POST', 'GET'])
@basic_api @basic_api
def api(): def api():
@ -93,6 +100,7 @@ def plugins():
dic = Plugins(plugins=list(sp.plugins.values())) dic = Plugins(plugins=list(sp.plugins.values()))
return dic return dic
@api_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET']) @api_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
@api_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET']) @api_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
@basic_api @basic_api
@ -110,12 +118,13 @@ def plugin(plugin=None, action="list"):
if action == "list": if action == "list":
return response return response
method = "{}_plugin".format(action) method = "{}_plugin".format(action)
if(hasattr(sp, method)): if (hasattr(sp, method)):
getattr(sp, method)(plugin) getattr(sp, method)(plugin)
return Response(message="Ok") return Response(message="Ok")
else: else:
return Error(message="action '{}' not allowed".format(action)) return Error(message="action '{}' not allowed".format(action))
if __name__ == '__main__': if __name__ == '__main__':
import config import config

5
senpy/cli.py
View File

@ -3,6 +3,7 @@ from .models import Error
from .api import parse_params, CLI_PARAMS from .api import parse_params, CLI_PARAMS
from .extensions import Senpy from .extensions import Senpy
def argv_to_dict(argv): def argv_to_dict(argv):
'''Turns parameters in the form of '--key value' into a dict {'key': 'value'} '''Turns parameters in the form of '--key value' into a dict {'key': 'value'}
''' '''
@ -11,13 +12,14 @@ def argv_to_dict(argv):
for i in range(len(argv)): for i in range(len(argv)):
if argv[i][0] == '-': if argv[i][0] == '-':
key = argv[i].strip('-') key = argv[i].strip('-')
value = argv[i+1] if len(argv)>i+1 else None value = argv[i + 1] if len(argv) > i + 1 else None
if value and value[0] == '-': if value and value[0] == '-':
cli_dict[key] = "" cli_dict[key] = ""
else: else:
cli_dict[key] = value cli_dict[key] = value
return cli_dict return cli_dict
def parse_cli(argv): def parse_cli(argv):
cli_dict = argv_to_dict(argv) cli_dict = argv_to_dict(argv)
cli_params = parse_params(cli_dict, spec=CLI_PARAMS) cli_params = parse_params(cli_dict, spec=CLI_PARAMS)
@ -34,6 +36,7 @@ def main_function(argv):
res = sp.analyse(**cli_dict) res = sp.analyse(**cli_dict)
return res return res
def main(): def main():
'''This method is the entrypoint for the CLI (as configured un setup.py) '''This method is the entrypoint for the CLI (as configured un setup.py)
''' '''

70
senpy/extensions.py
View File

@ -29,10 +29,12 @@ logger = logging.getLogger(__name__)
class Senpy(object): class Senpy(object):
""" Default Senpy extension for Flask """ """ Default Senpy extension for Flask """
def __init__(self, app=None, plugin_folder="plugins", default_plugins=False): def __init__(self,
app=None,
plugin_folder="plugins",
default_plugins=False):
self.app = app self.app = app
self._search_folders = set() self._search_folders = set()
@ -80,22 +82,24 @@ class Senpy(object):
elif self.plugins: elif self.plugins:
algo = self.default_plugin and self.default_plugin.name algo = self.default_plugin and self.default_plugin.name
if not algo: if not algo:
raise Error(status=404, raise Error(
message=("No plugins found." status=404,
" Please install one.").format(algo)) message=("No plugins found."
" Please install one.").format(algo))
if algo not in self.plugins: if algo not in self.plugins:
logger.debug(("The algorithm '{}' is not valid\n" logger.debug(("The algorithm '{}' is not valid\n"
"Valid algorithms: {}").format(algo, "Valid algorithms: {}").format(algo,
self.plugins.keys())) self.plugins.keys()))
raise Error(status=404, raise Error(
message="The algorithm '{}' is not valid" status=404,
.format(algo)) message="The algorithm '{}' is not valid".format(algo))
if not self.plugins[algo].is_activated: if not self.plugins[algo].is_activated:
logger.debug("Plugin not activated: {}".format(algo)) logger.debug("Plugin not activated: {}".format(algo))
raise Error(status=400, raise Error(
message=("The algorithm '{}'" status=400,
" is not activated yet").format(algo)) message=("The algorithm '{}'"
" is not activated yet").format(algo))
plug = self.plugins[algo] plug = self.plugins[algo]
nif_params = parse_params(params, spec=NIF_PARAMS) nif_params = parse_params(params, spec=NIF_PARAMS)
extra_params = plug.get('extra_params', {}) extra_params = plug.get('extra_params', {})
@ -120,9 +124,8 @@ class Senpy(object):
return None return None
def parameters(self, algo): def parameters(self, algo):
return getattr(self.plugins.get(algo) or self.default_plugin, return getattr(
"extra_params", self.plugins.get(algo) or self.default_plugin, "extra_params", {})
{})
def activate_all(self, sync=False): def activate_all(self, sync=False):
ps = [] ps = []
@ -146,18 +149,20 @@ class Senpy(object):
try: try:
plugin = self.plugins[plugin_name] plugin = self.plugins[plugin_name]
except KeyError: except KeyError:
raise Error(message="Plugin not found: {}".format(plugin_name), raise Error(
status=404) message="Plugin not found: {}".format(plugin_name), status=404)
logger.info("Activating plugin: {}".format(plugin.name)) logger.info("Activating plugin: {}".format(plugin.name))
def act(): def act():
try: try:
plugin.activate() plugin.activate()
logger.info("Plugin activated: {}".format(plugin.name)) logger.info("Plugin activated: {}".format(plugin.name))
except Exception as ex: except Exception as ex:
logger.error("Error activating plugin {}: {}".format(plugin.name, logger.error("Error activating plugin {}: {}".format(
ex)) plugin.name, ex))
logger.error("Trace: {}".format(traceback.format_exc())) logger.error("Trace: {}".format(traceback.format_exc()))
th = gevent.spawn(act) th = gevent.spawn(act)
th.link_value(partial(self._set_active_plugin, plugin_name, True)) th.link_value(partial(self._set_active_plugin, plugin_name, True))
if sync: if sync:
@ -169,16 +174,16 @@ class Senpy(object):
try: try:
plugin = self.plugins[plugin_name] plugin = self.plugins[plugin_name]
except KeyError: except KeyError:
raise Error(message="Plugin not found: {}".format(plugin_name), raise Error(
status=404) message="Plugin not found: {}".format(plugin_name), status=404)
def deact(): def deact():
try: try:
plugin.deactivate() plugin.deactivate()
logger.info("Plugin deactivated: {}".format(plugin.name)) logger.info("Plugin deactivated: {}".format(plugin.name))
except Exception as ex: except Exception as ex:
logger.error("Error deactivating plugin {}: {}".format(plugin.name, logger.error("Error deactivating plugin {}: {}".format(
ex)) plugin.name, ex))
logger.error("Trace: {}".format(traceback.format_exc())) logger.error("Trace: {}".format(traceback.format_exc()))
th = gevent.spawn(deact) th = gevent.spawn(deact)
@ -199,7 +204,6 @@ class Senpy(object):
logger.error('Error reloading {}: {}'.format(name, ex)) logger.error('Error reloading {}: {}'.format(name, ex))
self.plugins[name] = plugin self.plugins[name] = plugin
@classmethod @classmethod
def validate_info(cls, info): def validate_info(cls, info):
return all(x in info for x in ('name', 'module', 'version')) return all(x in info for x in ('name', 'module', 'version'))
@ -215,7 +219,7 @@ class Senpy(object):
pip_args = [] pip_args = []
pip_args.append('install') pip_args.append('install')
for req in requirements: for req in requirements:
pip_args.append( req ) pip_args.append(req)
logger.info('Installing requirements: ' + str(requirements)) logger.info('Installing requirements: ' + str(requirements))
pip.main(pip_args) pip.main(pip_args)
@ -237,8 +241,8 @@ class Senpy(object):
for _, obj in inspect.getmembers(tmp): for _, obj in inspect.getmembers(tmp):
if inspect.isclass(obj) and inspect.getmodule(obj) == tmp: if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
logger.debug(("Found plugin class:" logger.debug(("Found plugin class:"
" {}@{}").format(obj, inspect.getmodule(obj)) " {}@{}").format(obj, inspect.getmodule(
) obj)))
candidate = obj candidate = obj
break break
if not candidate: if not candidate:
@ -248,7 +252,8 @@ class Senpy(object):
repo_path = root repo_path = root
module._repo = Repo(repo_path) module._repo = Repo(repo_path)
except InvalidGitRepositoryError: except InvalidGitRepositoryError:
logger.debug("The plugin {} is not in a Git repository".format(module)) logger.debug("The plugin {} is not in a Git repository".format(
module))
module._repo = None module._repo = None
except Exception as ex: except Exception as ex:
logger.error("Exception importing {}: {}".format(module, ex)) logger.error("Exception importing {}: {}".format(module, ex))
@ -265,7 +270,6 @@ class Senpy(object):
logger.debug("Info: {}".format(info)) logger.debug("Info: {}".format(info))
return cls._load_plugin_from_info(info, root) return cls._load_plugin_from_info(info, root)
def _load_plugins(self): def _load_plugins(self):
plugins = {} plugins = {}
for search_folder in self._search_folders: for search_folder in self._search_folders:
@ -293,8 +297,7 @@ class Senpy(object):
def matches(plug): def matches(plug):
res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items()) res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
logger.debug("matching {} with {}: {}".format(plug.name, logger.debug("matching {} with {}: {}".format(plug.name, kwargs,
kwargs,
res)) res))
return res return res
@ -305,5 +308,8 @@ class Senpy(object):
def sentiment_plugins(self): def sentiment_plugins(self):
""" Return only the sentiment plugins """ """ Return only the sentiment plugins """
return {p: plugin for p, plugin in self.plugins.items() if return {
isinstance(plugin, SentimentPlugin)} p: plugin
for p, plugin in self.plugins.items()
if isinstance(plugin, SentimentPlugin)
}

100
senpy/models.py
View File

@ -18,15 +18,18 @@ import jsonschema
from flask import Response as FlaskResponse from flask import Response as FlaskResponse
DEFINITIONS_FILE = 'definitions.json' DEFINITIONS_FILE = 'definitions.json'
CONTEXT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'schemas', 'context.jsonld') CONTEXT_PATH = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'schemas', 'context.jsonld')
def get_schema_path(schema_file, absolute=False): def get_schema_path(schema_file, absolute=False):
if absolute: if absolute:
return os.path.realpath(schema_file) return os.path.realpath(schema_file)
else: else:
return os.path.join(os.path.dirname(os.path.realpath(__file__)), 'schemas', schema_file) return os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'schemas',
schema_file)
def read_schema(schema_file, absolute=False): def read_schema(schema_file, absolute=False):
@ -39,8 +42,8 @@ def read_schema(schema_file, absolute=False):
base_schema = read_schema(DEFINITIONS_FILE) base_schema = read_schema(DEFINITIONS_FILE)
logging.debug(base_schema) logging.debug(base_schema)
class Context(dict):
class Context(dict):
@staticmethod @staticmethod
def load(context): def load(context):
logging.debug('Loading context: {}'.format(context)) logging.debug('Loading context: {}'.format(context))
@ -62,15 +65,14 @@ class Context(dict):
else: else:
raise AttributeError('Please, provide a valid context') raise AttributeError('Please, provide a valid context')
base_context = Context.load(CONTEXT_PATH) base_context = Context.load(CONTEXT_PATH)
class SenpyMixin(object): class SenpyMixin(object):
context = base_context["@context"] context = base_context["@context"]
def flask(self, def flask(self, in_headers=False, headers=None, **kwargs):
in_headers=False,
headers=None,
**kwargs):
""" """
Return the values and error to be used in flask. Return the values and error to be used in flask.
So far, it returns a fixed context. We should store/generate different So far, it returns a fixed context. We should store/generate different
@ -87,33 +89,34 @@ class SenpyMixin(object):
'rel="http://www.w3.org/ns/json-ld#context";' 'rel="http://www.w3.org/ns/json-ld#context";'
' type="application/ld+json"' % url) ' type="application/ld+json"' % url)
}) })
return FlaskResponse(json.dumps(js, indent=2, sort_keys=True), return FlaskResponse(
status=getattr(self, "status", 200), json.dumps(
headers=headers, js, indent=2, sort_keys=True),
mimetype="application/json") status=getattr(self, "status", 200),
headers=headers,
mimetype="application/json")
def serializable(self): def serializable(self):
def ser_or_down(item): def ser_or_down(item):
if hasattr(item, 'serializable'): if hasattr(item, 'serializable'):
return item.serializable() return item.serializable()
elif isinstance(item, dict): elif isinstance(item, dict):
temp = dict() temp = dict()
for kp in item: for kp in item:
vp = item[kp] vp = item[kp]
temp[kp] = ser_or_down(vp) temp[kp] = ser_or_down(vp)
return temp return temp
elif isinstance(item, list): elif isinstance(item, list):
return list(ser_or_down(i) for i in item) return list(ser_or_down(i) for i in item)
else: else:
return item return item
return ser_or_down(self._plain_dict())
return ser_or_down(self._plain_dict())
def jsonld(self, with_context=True, context_uri=None): def jsonld(self, with_context=True, context_uri=None):
ser = self.serializable() ser = self.serializable()
if with_context: if with_context:
context = [] context = []
if context_uri: if context_uri:
context = context_uri context = context_uri
@ -133,10 +136,8 @@ class SenpyMixin(object):
ser["@context"] = context ser["@context"] = context
return ser return ser
def to_JSON(self, *args, **kwargs): def to_JSON(self, *args, **kwargs):
js = json.dumps(self.jsonld(*args, **kwargs), indent=4, js = json.dumps(self.jsonld(*args, **kwargs), indent=4, sort_keys=True)
sort_keys=True)
return js return js
def validate(self, obj=None): def validate(self, obj=None):
@ -146,17 +147,18 @@ class SenpyMixin(object):
obj = obj.jsonld() obj = obj.jsonld()
jsonschema.validate(obj, self.schema) jsonschema.validate(obj, self.schema)
class SenpyModel(SenpyMixin, dict): class SenpyModel(SenpyMixin, dict):
schema = base_schema schema = base_schema
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.id = kwargs.pop('id', '{}_{}'.format(type(self).__name__, self.id = kwargs.pop('id', '{}_{}'.format(
time.time())) type(self).__name__, time.time()))
temp = dict(*args, **kwargs) temp = dict(*args, **kwargs)
for obj in [self.schema,]+self.schema.get('allOf', []): for obj in [self.schema, ] + self.schema.get('allOf', []):
for k, v in obj.get('properties', {}).items(): for k, v in obj.get('properties', {}).items():
if 'default' in v: if 'default' in v:
temp[k] = copy.deepcopy(v['default']) temp[k] = copy.deepcopy(v['default'])
@ -172,7 +174,6 @@ class SenpyModel(SenpyMixin, dict):
self.__dict__['context'] = Context.load(context) self.__dict__['context'] = Context.load(context)
super(SenpyModel, self).__init__(temp) super(SenpyModel, self).__init__(temp)
def _get_key(self, key): def _get_key(self, key):
key = key.replace("__", ":", 1) key = key.replace("__", ":", 1)
return key return key
@ -180,7 +181,6 @@ class SenpyModel(SenpyMixin, dict):
def __setitem__(self, key, value): def __setitem__(self, key, value):
dict.__setitem__(self, key, value) dict.__setitem__(self, key, value)
def __delitem__(self, key): def __delitem__(self, key):
dict.__delitem__(self, key) dict.__delitem__(self, key)
@ -196,61 +196,79 @@ class SenpyModel(SenpyMixin, dict):
def __delattr__(self, key): def __delattr__(self, key):
self.__delitem__(self._get_key(key)) self.__delitem__(self._get_key(key))
def _plain_dict(self): def _plain_dict(self):
d = { k: v for (k,v) in self.items() if k[0] != "_"} d = {k: v for (k, v) in self.items() if k[0] != "_"}
d["@id"] = d.pop('id') d["@id"] = d.pop('id')
return d return d
class Response(SenpyModel): class Response(SenpyModel):
schema = read_schema('response.json') schema = read_schema('response.json')
class Results(SenpyModel): class Results(SenpyModel):
schema = read_schema('results.json') schema = read_schema('results.json')
class Entry(SenpyModel): class Entry(SenpyModel):
schema = read_schema('entry.json') schema = read_schema('entry.json')
class Sentiment(SenpyModel): class Sentiment(SenpyModel):
schema = read_schema('sentiment.json') schema = read_schema('sentiment.json')
class Analysis(SenpyModel): class Analysis(SenpyModel):
schema = read_schema('analysis.json') schema = read_schema('analysis.json')
class EmotionSet(SenpyModel): class EmotionSet(SenpyModel):
schema = read_schema('emotionSet.json') schema = read_schema('emotionSet.json')
class Emotion(SenpyModel): class Emotion(SenpyModel):
schema = read_schema('emotion.json') schema = read_schema('emotion.json')
class EmotionModel(SenpyModel): class EmotionModel(SenpyModel):
schema = read_schema('emotionModel.json') schema = read_schema('emotionModel.json')
class Suggestion(SenpyModel): class Suggestion(SenpyModel):
schema = read_schema('suggestion.json') schema = read_schema('suggestion.json')
class PluginModel(SenpyModel): class PluginModel(SenpyModel):
schema = read_schema('plugin.json') schema = read_schema('plugin.json')
class EmotionPluginModel(SenpyModel): class EmotionPluginModel(SenpyModel):
schema = read_schema('plugin.json') schema = read_schema('plugin.json')
class SentimentPluginModel(SenpyModel): class SentimentPluginModel(SenpyModel):
schema = read_schema('plugin.json') schema = read_schema('plugin.json')
class Plugins(SenpyModel): class Plugins(SenpyModel):
schema = read_schema('plugins.json') schema = read_schema('plugins.json')
class Error(SenpyMixin, BaseException ):
def __init__(self, message, status=500, params=None, errors=None, *args, **kwargs): class Error(SenpyMixin, BaseException):
def __init__(self,
message,
status=500,
params=None,
errors=None,
*args,
**kwargs):
self.message = message self.message = message
self.status = status self.status = status
self.params = params or {} self.params = params or {}
self.errors = errors or "" self.errors = errors or ""
def _plain_dict(self): def _plain_dict(self):
return self.__dict__ return self.__dict__
def __str__(self): def __str__(self):
return str(self.jsonld()) return str(self.jsonld())

11
senpy/plugins.py
View File

@ -10,8 +10,8 @@ from .models import Response, PluginModel, Error
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class SenpyPlugin(PluginModel):
class SenpyPlugin(PluginModel):
def __init__(self, info=None): def __init__(self, info=None):
if not info: if not info:
raise Error(message=("You need to provide configuration" raise Error(message=("You need to provide configuration"
@ -39,8 +39,8 @@ class SenpyPlugin(PluginModel):
''' Destructor, to make sure all the resources are freed ''' ''' Destructor, to make sure all the resources are freed '''
self.deactivate() self.deactivate()
class SentimentPlugin(SenpyPlugin):
class SentimentPlugin(SenpyPlugin):
def __init__(self, info, *args, **kwargs): def __init__(self, info, *args, **kwargs):
super(SentimentPlugin, self).__init__(info, *args, **kwargs) super(SentimentPlugin, self).__init__(info, *args, **kwargs)
self.minPolarityValue = float(info.get("minPolarityValue", 0)) self.minPolarityValue = float(info.get("minPolarityValue", 0))
@ -49,7 +49,6 @@ class SentimentPlugin(SenpyPlugin):
class EmotionPlugin(SenpyPlugin): class EmotionPlugin(SenpyPlugin):
def __init__(self, info, *args, **kwargs): def __init__(self, info, *args, **kwargs):
resp = super(EmotionPlugin, self).__init__(info, *args, **kwargs) resp = super(EmotionPlugin, self).__init__(info, *args, **kwargs)
self.minEmotionValue = float(info.get("minEmotionValue", 0)) self.minEmotionValue = float(info.get("minEmotionValue", 0))
@ -58,7 +57,6 @@ class EmotionPlugin(SenpyPlugin):
class ShelfMixin(object): class ShelfMixin(object):
@property @property
def sh(self): def sh(self):
if not hasattr(self, '_sh') or self._sh is None: if not hasattr(self, '_sh') or self._sh is None:
@ -84,7 +82,8 @@ class ShelfMixin(object):
if hasattr(self, '_info') and 'shelf_file' in self._info: if hasattr(self, '_info') and 'shelf_file' in self._info:
self.__dict__['_shelf_file'] = self._info['shelf_file'] self.__dict__['_shelf_file'] = self._info['shelf_file']
else: else:
self._shelf_file = os.path.join(tempfile.gettempdir(), self.name + '.p') self._shelf_file = os.path.join(tempfile.gettempdir(),
self.name + '.p')
return self._shelf_file return self._shelf_file
def save(self): def save(self):
@ -92,4 +91,4 @@ class ShelfMixin(object):
if hasattr(self, '_sh') and self._sh is not None: if hasattr(self, '_sh') and self._sh is not None:
with open(self.shelf_file, 'wb') as f: with open(self.shelf_file, 'wb') as f:
pickle.dump(self._sh, f) pickle.dump(self._sh, f)
del(self.__dict__['_sh']) del (self.__dict__['_sh'])

23
senpy/plugins/rand/rand.py
View File

@ -16,26 +16,15 @@ class Sentiment140Plugin(SentimentPlugin):
polarity = "marl:Positive" polarity = "marl:Positive"
elif polarity_value < 0: elif polarity_value < 0:
polarity = "marl:Negative" polarity = "marl:Negative"
entry = Entry({"id":":Entry0", entry = Entry({"id": ":Entry0", "nif:isString": params["input"]})
"nif:isString": params["input"]}) sentiment = Sentiment({
sentiment = Sentiment({"id": ":Sentiment0", "id": ":Sentiment0",
"marl:hasPolarity": polarity, "marl:hasPolarity": polarity,
"marl:polarityValue": polarity_value}) "marl:polarityValue": polarity_value
})
sentiment["prov:wasGeneratedBy"] = self.id sentiment["prov:wasGeneratedBy"] = self.id
entry.sentiments = [] entry.sentiments = []
entry.sentiments.append(sentiment) entry.sentiments.append(sentiment)
entry.language = lang entry.language = lang
response.entries.append(entry) response.entries.append(entry)
return response return response

27
senpy/plugins/sentiment140/sentiment140.py
View File

@ -9,16 +9,17 @@ class Sentiment140Plugin(SentimentPlugin):
def analyse(self, **params): def analyse(self, **params):
lang = params.get("language", "auto") lang = params.get("language", "auto")
res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson", res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
json.dumps({"language": lang, json.dumps({
"data": [{"text": params["input"]}] "language": lang,
} "data": [{
) "text": params["input"]
) }]
}))
p = params.get("prefix", None) p = params.get("prefix", None)
response = Results(prefix=p) response = Results(prefix=p)
polarity_value = self.maxPolarityValue*int(res.json()["data"][0] polarity_value = self.maxPolarityValue * int(res.json()["data"][0][
["polarity"]) * 0.25 "polarity"]) * 0.25
polarity = "marl:Neutral" polarity = "marl:Neutral"
neutral_value = self.maxPolarityValue / 2.0 neutral_value = self.maxPolarityValue / 2.0
if polarity_value > neutral_value: if polarity_value > neutral_value:
@ -26,12 +27,12 @@ class Sentiment140Plugin(SentimentPlugin):
elif polarity_value < neutral_value: elif polarity_value < neutral_value:
polarity = "marl:Negative" polarity = "marl:Negative"
entry = Entry(id="Entry0", entry = Entry(id="Entry0", nif__isString=params["input"])
nif__isString=params["input"]) sentiment = Sentiment(
sentiment = Sentiment(id="Sentiment0", id="Sentiment0",
prefix=p, prefix=p,
marl__hasPolarity=polarity, marl__hasPolarity=polarity,
marl__polarityValue=polarity_value) marl__polarityValue=polarity_value)
sentiment.prov__wasGeneratedBy = self.id sentiment.prov__wasGeneratedBy = self.id
entry.sentiments = [] entry.sentiments = []
entry.sentiments.append(sentiment) entry.sentiments.append(sentiment)

1
tests/dummy_plugin/dummy.py
View File

@ -3,6 +3,5 @@ from senpy.models import Results
class DummyPlugin(SentimentPlugin): class DummyPlugin(SentimentPlugin):
def analyse(self, *args, **kwargs): def analyse(self, *args, **kwargs):
return Results() return Results()

1
tests/sleep_plugin/sleep.py
View File

@ -4,7 +4,6 @@ from time import sleep
class SleepPlugin(SenpyPlugin): class SleepPlugin(SenpyPlugin):
def activate(self, *args, **kwargs): def activate(self, *args, **kwargs):
sleep(self.timeout) sleep(self.timeout)

2
tests/test_blueprints.py
View File

@ -12,12 +12,12 @@ from itertools import product
def check_dict(indic, template): def check_dict(indic, template):
return all(item in indic.items() for item in template.items()) return all(item in indic.items() for item in template.items())
def parse_resp(resp): def parse_resp(resp):
return json.loads(resp.data.decode('utf-8')) return json.loads(resp.data.decode('utf-8'))
class BlueprintsTest(TestCase): class BlueprintsTest(TestCase):
def setUp(self): def setUp(self):
self.app = Flask("test_extensions") self.app = Flask("test_extensions")
self.client = self.app.test_client() self.client = self.app.test_client()

1
tests/test_cli.py
View File

@ -10,7 +10,6 @@ from senpy.models import Error
class CLITest(TestCase): class CLITest(TestCase):
def test_basic(self): def test_basic(self):
self.assertRaises(Error, partial(main_function, [])) self.assertRaises(Error, partial(main_function, []))
res = main_function(['--input', 'test']) res = main_function(['--input', 'test'])

6
tests/test_extensions.py
View File

@ -10,7 +10,6 @@ from unittest import TestCase
class ExtensionsTest(TestCase): class ExtensionsTest(TestCase):
def setUp(self): def setUp(self):
self.app = Flask("test_extensions") self.app = Flask("test_extensions")
self.dir = os.path.join(os.path.dirname(__file__)) self.dir = os.path.join(os.path.dirname(__file__))
@ -39,7 +38,7 @@ class ExtensionsTest(TestCase):
'module': 'dummy', 'module': 'dummy',
'requirements': ['noop'], 'requirements': ['noop'],
'version': 0 'version': 0
} }
root = os.path.join(self.dir, 'dummy_plugin') root = os.path.join(self.dir, 'dummy_plugin')
name, module = self.senpy._load_plugin_from_info(info, root=root) name, module = self.senpy._load_plugin_from_info(info, root=root)
assert name == 'TestPip' assert name == 'TestPip'
@ -88,4 +87,5 @@ class ExtensionsTest(TestCase):
assert self.senpy.filter_plugins(name="Dummy", is_activated=True) assert self.senpy.filter_plugins(name="Dummy", is_activated=True)
self.senpy.deactivate_plugin("Dummy", sync=True) self.senpy.deactivate_plugin("Dummy", sync=True)
assert not len( assert not len(
self.senpy.filter_plugins(name="Dummy", is_activated=True)) self.senpy.filter_plugins(
name="Dummy", is_activated=True))

40
tests/test_models.py
View File

@ -12,12 +12,11 @@ from pprint import pprint
class ModelsTest(TestCase): class ModelsTest(TestCase):
def test_jsonld(self): def test_jsonld(self):
ctx = os.path.normpath(os.path.join(__file__, "..", "..", "..", "senpy", "schemas", "context.jsonld")) ctx = os.path.normpath(
prueba = {"id": "test", os.path.join(__file__, "..", "..", "..", "senpy", "schemas",
"analysis": [], "context.jsonld"))
"entries": []} prueba = {"id": "test", "analysis": [], "entries": []}
r = Results(**prueba) r = Results(**prueba)
print("Response's context: ") print("Response's context: ")
pprint(r.context) pprint(r.context)
@ -27,28 +26,32 @@ class ModelsTest(TestCase):
j = r.jsonld(with_context=True) j = r.jsonld(with_context=True)
print("As JSON:") print("As JSON:")
pprint(j) pprint(j)
assert("@context" in j) assert ("@context" in j)
assert("marl" in j["@context"]) assert ("marl" in j["@context"])
assert("entries" in j["@context"]) assert ("entries" in j["@context"])
assert(j["@id"] == "test") assert (j["@id"] == "test")
assert "id" not in j assert "id" not in j
r6 = Results(**prueba) r6 = Results(**prueba)
r6.entries.append(Entry({"@id":"ohno", "nif:isString":"Just testing"})) r6.entries.append(
Entry({
"@id": "ohno",
"nif:isString": "Just testing"
}))
logging.debug("Reponse 6: %s", r6) logging.debug("Reponse 6: %s", r6)
assert("marl" in r6.context) assert ("marl" in r6.context)
assert("entries" in r6.context) assert ("entries" in r6.context)
j6 = r6.jsonld(with_context=True) j6 = r6.jsonld(with_context=True)
logging.debug("jsonld: %s", j6) logging.debug("jsonld: %s", j6)
assert("@context" in j6) assert ("@context" in j6)
assert("entries" in j6) assert ("entries" in j6)
assert("analysis" in j6) assert ("analysis" in j6)
resp = r6.flask() resp = r6.flask()
received = json.loads(resp.data.decode()) received = json.loads(resp.data.decode())
logging.debug("Response: %s", j6) logging.debug("Response: %s", j6)
assert(received["entries"]) assert (received["entries"])
assert(received["entries"][0]["nif:isString"] == "Just testing") assert (received["entries"][0]["nif:isString"] == "Just testing")
assert(received["entries"][0]["nif:isString"] != "Not testing") assert (received["entries"][0]["nif:isString"] != "Not testing")
def test_id(self): def test_id(self):
''' Adding the id after creation should overwrite the automatic ID ''' Adding the id after creation should overwrite the automatic ID
@ -61,7 +64,6 @@ class ModelsTest(TestCase):
assert j2['@id'] == 'test' assert j2['@id'] == 'test'
assert 'id' not in j2 assert 'id' not in j2
def test_entries(self): def test_entries(self):
e = Entry() e = Entry()
self.assertRaises(jsonschema.ValidationError, e.validate) self.assertRaises(jsonschema.ValidationError, e.validate)

43
tests/test_plugins.py
View File

@ -13,8 +13,8 @@ from flask import Flask
from senpy.models import Results, Entry from senpy.models import Results, Entry
from senpy.plugins import SentimentPlugin, ShelfMixin from senpy.plugins import SentimentPlugin, ShelfMixin
class ShelfDummyPlugin(SentimentPlugin, ShelfMixin):
class ShelfDummyPlugin(SentimentPlugin, ShelfMixin):
def activate(self, *args, **kwargs): def activate(self, *args, **kwargs):
if 'counter' not in self.sh: if 'counter' not in self.sh:
self.sh['counter'] = 0 self.sh['counter'] = 0
@ -24,15 +24,15 @@ class ShelfDummyPlugin(SentimentPlugin, ShelfMixin):
self.save() self.save()
def analyse(self, *args, **kwargs): def analyse(self, *args, **kwargs):
self.sh['counter'] = self.sh['counter']+1 self.sh['counter'] = self.sh['counter'] + 1
e = Entry() e = Entry()
e.nif__isString = self.sh['counter'] e.nif__isString = self.sh['counter']
r = Results() r = Results()
r.entries.append(e) r.entries.append(e)
return r return r
class PluginsTest(TestCase):
class PluginsTest(TestCase):
def tearDown(self): def tearDown(self):
if os.path.exists(self.shelf_dir): if os.path.exists(self.shelf_dir):
shutil.rmtree(self.shelf_dir) shutil.rmtree(self.shelf_dir)
@ -45,8 +45,9 @@ class PluginsTest(TestCase):
self.shelf_file = os.path.join(self.shelf_dir, "shelf") self.shelf_file = os.path.join(self.shelf_dir, "shelf")
def test_shelf_file(self): def test_shelf_file(self):
a = ShelfDummyPlugin(info={'name': 'default_shelve_file', a = ShelfDummyPlugin(
'version': 'test'}) info={'name': 'default_shelve_file',
'version': 'test'})
assert os.path.dirname(a.shelf_file) == tempfile.gettempdir() assert os.path.dirname(a.shelf_file) == tempfile.gettempdir()
a.activate() a.activate()
assert os.path.isfile(a.shelf_file) assert os.path.isfile(a.shelf_file)
@ -54,9 +55,11 @@ class PluginsTest(TestCase):
def test_shelf(self): def test_shelf(self):
''' A shelf is created and the value is stored ''' ''' A shelf is created and the value is stored '''
a = ShelfDummyPlugin(info={'name': 'shelve', a = ShelfDummyPlugin(info={
'version': 'test', 'name': 'shelve',
'shelf_file': self.shelf_file}) 'version': 'test',
'shelf_file': self.shelf_file
})
assert a.sh == {} assert a.sh == {}
a.activate() a.activate()
assert a.sh == {'counter': 0} assert a.sh == {'counter': 0}
@ -72,9 +75,11 @@ class PluginsTest(TestCase):
assert sh['a'] == 'fromA' assert sh['a'] == 'fromA'
def test_dummy_shelf(self): def test_dummy_shelf(self):
a = ShelfDummyPlugin(info={'name': 'DummyShelf', a = ShelfDummyPlugin(info={
'shelf_file': self.shelf_file, 'name': 'DummyShelf',
'version': 'test'}) 'shelf_file': self.shelf_file,
'version': 'test'
})
a.activate() a.activate()
res1 = a.analyse(input=1) res1 = a.analyse(input=1)
@ -84,17 +89,21 @@ class PluginsTest(TestCase):
def test_two(self): def test_two(self):
''' Reusing the values of a previous shelf ''' ''' Reusing the values of a previous shelf '''
a = ShelfDummyPlugin(info={'name': 'shelve', a = ShelfDummyPlugin(info={
'version': 'test', 'name': 'shelve',
'shelf_file': self.shelf_file}) 'version': 'test',
'shelf_file': self.shelf_file
})
a.activate() a.activate()
print('Shelf file: %s' % a.shelf_file) print('Shelf file: %s' % a.shelf_file)
a.sh['a'] = 'fromA' a.sh['a'] = 'fromA'
a.save() a.save()
b = ShelfDummyPlugin(info={'name': 'shelve', b = ShelfDummyPlugin(info={
'version': 'test', 'name': 'shelve',
'shelf_file': self.shelf_file}) 'version': 'test',
'shelf_file': self.shelf_file
})
b.activate() b.activate()
assert b.sh['a'] == 'fromA' assert b.sh['a'] == 'fromA'
b.sh['a'] = 'fromB' b.sh['a'] = 'fromB'

15
tests/test_schemas.py
View File

@ -14,9 +14,11 @@ schema_folder = path.join(root_path, 'senpy', 'schemas')
examples_path = path.join(root_path, 'docs', 'examples') examples_path = path.join(root_path, 'docs', 'examples')
bad_examples_path = path.join(root_path, 'docs', 'bad-examples') bad_examples_path = path.join(root_path, 'docs', 'bad-examples')
class JSONSchemaTests(unittest.TestCase): class JSONSchemaTests(unittest.TestCase):
pass pass
def do_create_(jsfile, success): def do_create_(jsfile, success):
def do_expected(self): def do_expected(self):
with open(jsfile) as f: with open(jsfile) as f:
@ -24,7 +26,8 @@ def do_create_(jsfile, success):
try: try:
assert '@type' in js assert '@type' in js
schema_name = js['@type'] schema_name = js['@type']
with open(os.path.join(schema_folder, schema_name+".json")) as file_object: with open(os.path.join(schema_folder, schema_name +
".json")) as file_object:
schema = json.load(file_object) schema = json.load(file_object)
resolver = RefResolver('file://' + schema_folder + '/', schema) resolver = RefResolver('file://' + schema_folder + '/', schema)
validator = Draft4Validator(schema, resolver=resolver) validator = Draft4Validator(schema, resolver=resolver)
@ -32,19 +35,25 @@ def do_create_(jsfile, success):
except (AssertionError, ValidationError, KeyError) as ex: except (AssertionError, ValidationError, KeyError) as ex:
if success: if success:
raise raise
return do_expected return do_expected
def add_examples(dirname, success): def add_examples(dirname, success):
for dirpath, dirnames, filenames in os.walk(dirname): for dirpath, dirnames, filenames in os.walk(dirname):
for i in filenames: for i in filenames:
if fnmatch(i, '*.json'): if fnmatch(i, '*.json'):
filename = path.join(dirpath, i) filename = path.join(dirpath, i)
test_method = do_create_(filename, success) test_method = do_create_(filename, success)
test_method.__name__ = 'test_file_%s_success_%s' % (filename, success) test_method.__name__ = 'test_file_%s_success_%s' % (filename,
test_method.__doc__ = '%s should %svalidate' % (filename, '' if success else 'not' ) success)
test_method.__doc__ = '%s should %svalidate' % (filename, ''
if success else
'not')
setattr(JSONSchemaTests, test_method.__name__, test_method) setattr(JSONSchemaTests, test_method.__name__, test_method)
del test_method del test_method
add_examples(examples_path, True) add_examples(examples_path, True)
add_examples(bad_examples_path, False) add_examples(bad_examples_path, False)