balkian/senpy
1
0
Fork 0
mirror of https://github.com/gsi-upm/senpy synced 2025-09-18 04:22:21 +00:00
Code Issues Releases Wiki Activity

YAPFed

Browse Source
This commit is contained in:
J. Fernando Sánchez
2017-01-10 10:16:45 +01:00
parent b543a4614e
commit 7fd69cc690
19 changed files with 283 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
senpy/__init__.py
View File

@@ -22,5 +22,4 @@ import os
from .version import __version__
__all__ = ['api', 'blueprints', 'cli', 'extensions', 'models', 'plugins']

78
senpy/__main__.py
View File

@@ -34,42 +34,51 @@ patch_all(thread=False)
SERVER_PORT = os.environ.get("PORT", 5000)
def main():
parser = argparse.ArgumentParser(description='Run a Senpy server')
parser.add_argument('--level',
'-l',
metavar='logging_level',
type=str,
default="INFO",
help='Logging level')
parser.add_argument('--debug',
'-d',
action='store_true',
default=False,
help='Run the application in debug mode')
parser.add_argument('--default-plugins',
action='store_true',
default=False,
help='Load the default plugins')
parser.add_argument('--host',
type=str,
default="127.0.0.1",
help='Use 0.0.0.0 to accept requests from any host.')
parser.add_argument('--port',
'-p',
type=int,
default=SERVER_PORT,
help='Port to listen on.')
parser.add_argument('--plugins-folder',
'-f',
type=str,
default='plugins',
help='Where to look for plugins.')
parser.add_argument('--only-install',
'-i',
action='store_true',
default=False,
help='Do not run a server, only install the dependencies of the plugins.')
parser.add_argument(
'--level',
'-l',
metavar='logging_level',
type=str,
default="INFO",
help='Logging level')
parser.add_argument(
'--debug',
'-d',
action='store_true',
default=False,
help='Run the application in debug mode')
parser.add_argument(
'--default-plugins',
action='store_true',
default=False,
help='Load the default plugins')
parser.add_argument(
'--host',
type=str,
default="127.0.0.1",
help='Use 0.0.0.0 to accept requests from any host.')
parser.add_argument(
'--port',
'-p',
type=int,
default=SERVER_PORT,
help='Port to listen on.')
parser.add_argument(
'--plugins-folder',
'-f',
type=str,
default='plugins',
help='Where to look for plugins.')
parser.add_argument(
'--only-install',
'-i',
action='store_true',
default=False,
help='Do not run a server, only install the dependencies of the plugins.'
)
args = parser.parse_args()
logging.basicConfig()
rl = logging.getLogger()
@@ -92,5 +101,6 @@ def main():
http_server.stop()
sp.deactivate_all()
if __name__ == '__main__':
main()

13
senpy/api.py
View File

@@ -25,7 +25,7 @@ CLI_PARAMS = {
"required": True,
"default": "."
},
}
}
NIF_PARAMS = {
"input": {
@@ -96,10 +96,11 @@ def parse_params(indict, spec=NIF_PARAMS):
outdict[param] not in spec[param]["options"]:
wrong_params[param] = spec[param]
if wrong_params:
message = Error(status=404,
message="Missing or invalid parameters",
parameters=outdict,
errors={param: error for param, error in
iteritems(wrong_params)})
message = Error(
status=404,
message="Missing or invalid parameters",
parameters=outdict,
errors={param: error
for param, error in iteritems(wrong_params)})
raise message
return outdict

25
senpy/blueprints.py
View File

@@ -30,6 +30,7 @@ logger = logging.getLogger(__name__)
api_blueprint = Blueprint("api", __name__)
demo_blueprint = Blueprint("demo", __name__)
def get_params(req):
if req.method == 'POST':
indict = req.form.to_dict(flat=True)
@@ -44,17 +45,20 @@ def get_params(req):
def index():
return render_template("index.html")
@api_blueprint.route('/contexts/<entity>.jsonld')
def context(entity="context"):
return jsonify({"@context": Response.context})
@api_blueprint.route('/schemas/<schema>')
def schema(schema="definitions"):
try:
return jsonify(read_schema(schema))
except Exception: # Should be FileNotFoundError, but it's missing from py2
except Exception: # Should be FileNotFoundError, but it's missing from py2
return Error(message="Schema not found", status=404).flask()
def basic_api(f):
@wraps(f)
def decorated_function(*args, **kwargs):
@@ -73,12 +77,15 @@ def basic_api(f):
response = ex
in_headers = web_params["inHeaders"] != "0"
headers = {'X-ORIGINAL-PARAMS': raw_params}
return response.flask(in_headers=in_headers,
headers=headers,
context_uri=url_for('api.context', entity=type(response).__name__,
_external=True))
return response.flask(
in_headers=in_headers,
headers=headers,
context_uri=url_for(
'api.context', entity=type(response).__name__, _external=True))
return decorated_function
@api_blueprint.route('/', methods=['POST', 'GET'])
@basic_api
def api():
@@ -92,7 +99,8 @@ def plugins():
sp = current_app.senpy
dic = Plugins(plugins=list(sp.plugins.values()))
return dic
@api_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
@api_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
@basic_api
@@ -110,12 +118,13 @@ def plugin(plugin=None, action="list"):
if action == "list":
return response
method = "{}_plugin".format(action)
if(hasattr(sp, method)):
if (hasattr(sp, method)):
getattr(sp, method)(plugin)
return Response(message="Ok")
else:
return Error(message="action '{}' not allowed".format(action))
if __name__ == '__main__':
import config

7
senpy/cli.py
View File

@@ -3,6 +3,7 @@ from .models import Error
from .api import parse_params, CLI_PARAMS
from .extensions import Senpy
def argv_to_dict(argv):
'''Turns parameters in the form of '--key value' into a dict {'key': 'value'}
'''
@@ -11,13 +12,14 @@ def argv_to_dict(argv):
for i in range(len(argv)):
if argv[i][0] == '-':
key = argv[i].strip('-')
value = argv[i+1] if len(argv)>i+1 else None
value = argv[i + 1] if len(argv) > i + 1 else None
if value and value[0] == '-':
cli_dict[key] = ""
else:
cli_dict[key] = value
return cli_dict
def parse_cli(argv):
cli_dict = argv_to_dict(argv)
cli_params = parse_params(cli_dict, spec=CLI_PARAMS)
@@ -34,6 +36,7 @@ def main_function(argv):
res = sp.analyse(**cli_dict)
return res
def main():
'''This method is the entrypoint for the CLI (as configured un setup.py)
'''
@@ -43,7 +46,7 @@ def main():
except Error as err:
print(err.to_JSON())
sys.exit(2)
if __name__ == '__main__':
main()

76
senpy/extensions.py
View File

@@ -29,10 +29,12 @@ logger = logging.getLogger(__name__)
class Senpy(object):
""" Default Senpy extension for Flask """
def __init__(self, app=None, plugin_folder="plugins", default_plugins=False):
def __init__(self,
app=None,
plugin_folder="plugins",
default_plugins=False):
self.app = app
self._search_folders = set()
@@ -80,22 +82,24 @@ class Senpy(object):
elif self.plugins:
algo = self.default_plugin and self.default_plugin.name
if not algo:
raise Error(status=404,
message=("No plugins found."
" Please install one.").format(algo))
raise Error(
status=404,
message=("No plugins found."
" Please install one.").format(algo))
if algo not in self.plugins:
logger.debug(("The algorithm '{}' is not valid\n"
"Valid algorithms: {}").format(algo,
self.plugins.keys()))
raise Error(status=404,
message="The algorithm '{}' is not valid"
.format(algo))
raise Error(
status=404,
message="The algorithm '{}' is not valid".format(algo))
if not self.plugins[algo].is_activated:
logger.debug("Plugin not activated: {}".format(algo))
raise Error(status=400,
message=("The algorithm '{}'"
" is not activated yet").format(algo))
raise Error(
status=400,
message=("The algorithm '{}'"
" is not activated yet").format(algo))
plug = self.plugins[algo]
nif_params = parse_params(params, spec=NIF_PARAMS)
extra_params = plug.get('extra_params', {})
@@ -120,9 +124,8 @@ class Senpy(object):
return None
def parameters(self, algo):
return getattr(self.plugins.get(algo) or self.default_plugin,
"extra_params",
{})
return getattr(
self.plugins.get(algo) or self.default_plugin, "extra_params", {})
def activate_all(self, sync=False):
ps = []
@@ -146,18 +149,20 @@ class Senpy(object):
try:
plugin = self.plugins[plugin_name]
except KeyError:
raise Error(message="Plugin not found: {}".format(plugin_name),
status=404)
raise Error(
message="Plugin not found: {}".format(plugin_name), status=404)
logger.info("Activating plugin: {}".format(plugin.name))
def act():
try:
plugin.activate()
logger.info("Plugin activated: {}".format(plugin.name))
except Exception as ex:
logger.error("Error activating plugin {}: {}".format(plugin.name,
ex))
logger.error("Error activating plugin {}: {}".format(
plugin.name, ex))
logger.error("Trace: {}".format(traceback.format_exc()))
th = gevent.spawn(act)
th.link_value(partial(self._set_active_plugin, plugin_name, True))
if sync:
@@ -169,16 +174,16 @@ class Senpy(object):
try:
plugin = self.plugins[plugin_name]
except KeyError:
raise Error(message="Plugin not found: {}".format(plugin_name),
status=404)
raise Error(
message="Plugin not found: {}".format(plugin_name), status=404)
def deact():
try:
plugin.deactivate()
logger.info("Plugin deactivated: {}".format(plugin.name))
except Exception as ex:
logger.error("Error deactivating plugin {}: {}".format(plugin.name,
ex))
logger.error("Error deactivating plugin {}: {}".format(
plugin.name, ex))
logger.error("Trace: {}".format(traceback.format_exc()))
th = gevent.spawn(deact)
@@ -199,7 +204,6 @@ class Senpy(object):
logger.error('Error reloading {}: {}'.format(name, ex))
self.plugins[name] = plugin
@classmethod
def validate_info(cls, info):
return all(x in info for x in ('name', 'module', 'version'))
@@ -215,15 +219,15 @@ class Senpy(object):
pip_args = []
pip_args.append('install')
for req in requirements:
pip_args.append( req )
pip_args.append(req)
logger.info('Installing requirements: ' + str(requirements))
pip.main(pip_args)
pip.main(pip_args)
@classmethod
def _load_plugin_from_info(cls, info, root):
if not cls.validate_info(info):
logger.warn('The module info is not valid.\n\t{}'.format(info))
return None, None
return None, None
module = info["module"]
name = info["name"]
requirements = info.get("requirements", [])
@@ -237,8 +241,8 @@ class Senpy(object):
for _, obj in inspect.getmembers(tmp):
if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
logger.debug(("Found plugin class:"
" {}@{}").format(obj, inspect.getmodule(obj))
)
" {}@{}").format(obj, inspect.getmodule(
obj)))
candidate = obj
break
if not candidate:
@@ -248,7 +252,8 @@ class Senpy(object):
repo_path = root
module._repo = Repo(repo_path)
except InvalidGitRepositoryError:
logger.debug("The plugin {} is not in a Git repository".format(module))
logger.debug("The plugin {} is not in a Git repository".format(
module))
module._repo = None
except Exception as ex:
logger.error("Exception importing {}: {}".format(module, ex))
@@ -265,7 +270,6 @@ class Senpy(object):
logger.debug("Info: {}".format(info))
return cls._load_plugin_from_info(info, root)
def _load_plugins(self):
plugins = {}
for search_folder in self._search_folders:
@@ -293,8 +297,7 @@ class Senpy(object):
def matches(plug):
res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
logger.debug("matching {} with {}: {}".format(plug.name,
kwargs,
logger.debug("matching {} with {}: {}".format(plug.name, kwargs,
res))
return res
@@ -305,5 +308,8 @@ class Senpy(object):
def sentiment_plugins(self):
""" Return only the sentiment plugins """
return {p: plugin for p, plugin in self.plugins.items() if
isinstance(plugin, SentimentPlugin)}
return {
p: plugin
for p, plugin in self.plugins.items()
if isinstance(plugin, SentimentPlugin)
}

110
senpy/models.py
View File

@@ -18,15 +18,18 @@ import jsonschema
from flask import Response as FlaskResponse
DEFINITIONS_FILE = 'definitions.json'
CONTEXT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'schemas', 'context.jsonld')
CONTEXT_PATH = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'schemas', 'context.jsonld')
def get_schema_path(schema_file, absolute=False):
if absolute:
return os.path.realpath(schema_file)
else:
return os.path.join(os.path.dirname(os.path.realpath(__file__)), 'schemas', schema_file)
return os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'schemas',
schema_file)
def read_schema(schema_file, absolute=False):
@@ -34,13 +37,13 @@ def read_schema(schema_file, absolute=False):
schema_uri = 'file://{}'.format(schema_path)
with open(schema_path) as f:
return jsonref.load(f, base_uri=schema_uri)
base_schema = read_schema(DEFINITIONS_FILE)
logging.debug(base_schema)
class Context(dict):
class Context(dict):
@staticmethod
def load(context):
logging.debug('Loading context: {}'.format(context))
@@ -60,17 +63,16 @@ class Context(dict):
except IOError:
return context
else:
raise AttributeError('Please, provide a valid context')
raise AttributeError('Please, provide a valid context')
base_context = Context.load(CONTEXT_PATH)
class SenpyMixin(object):
context = base_context["@context"]
def flask(self,
in_headers=False,
headers=None,
**kwargs):
def flask(self, in_headers=False, headers=None, **kwargs):
"""
Return the values and error to be used in flask.
So far, it returns a fixed context. We should store/generate different
@@ -87,33 +89,34 @@ class SenpyMixin(object):
'rel="http://www.w3.org/ns/json-ld#context";'
' type="application/ld+json"' % url)
})
return FlaskResponse(json.dumps(js, indent=2, sort_keys=True),
status=getattr(self, "status", 200),
headers=headers,
mimetype="application/json")
return FlaskResponse(
json.dumps(
js, indent=2, sort_keys=True),
status=getattr(self, "status", 200),
headers=headers,
mimetype="application/json")
def serializable(self):
def ser_or_down(item):
if hasattr(item, 'serializable'):
return item.serializable()
elif isinstance(item, dict):
temp = dict()
for kp in item:
vp = item[kp]
temp[kp] = ser_or_down(vp)
return temp
elif isinstance(item, list):
return list(ser_or_down(i) for i in item)
else:
return item
return ser_or_down(self._plain_dict())
if hasattr(item, 'serializable'):
return item.serializable()
elif isinstance(item, dict):
temp = dict()
for kp in item:
vp = item[kp]
temp[kp] = ser_or_down(vp)
return temp
elif isinstance(item, list):
return list(ser_or_down(i) for i in item)
else:
return item
return ser_or_down(self._plain_dict())
def jsonld(self, with_context=True, context_uri=None):
ser = self.serializable()
if with_context:
if with_context:
context = []
if context_uri:
context = context_uri
@@ -133,10 +136,8 @@ class SenpyMixin(object):
ser["@context"] = context
return ser
def to_JSON(self, *args, **kwargs):
js = json.dumps(self.jsonld(*args, **kwargs), indent=4,
sort_keys=True)
js = json.dumps(self.jsonld(*args, **kwargs), indent=4, sort_keys=True)
return js
def validate(self, obj=None):
@@ -145,18 +146,19 @@ class SenpyMixin(object):
if hasattr(obj, "jsonld"):
obj = obj.jsonld()
jsonschema.validate(obj, self.schema)
class SenpyModel(SenpyMixin, dict):
schema = base_schema
def __init__(self, *args, **kwargs):
self.id = kwargs.pop('id', '{}_{}'.format(type(self).__name__,
time.time()))
self.id = kwargs.pop('id', '{}_{}'.format(
type(self).__name__, time.time()))
temp = dict(*args, **kwargs)
for obj in [self.schema,]+self.schema.get('allOf', []):
for obj in [self.schema, ] + self.schema.get('allOf', []):
for k, v in obj.get('properties', {}).items():
if 'default' in v:
temp[k] = copy.deepcopy(v['default'])
@@ -172,7 +174,6 @@ class SenpyModel(SenpyMixin, dict):
self.__dict__['context'] = Context.load(context)
super(SenpyModel, self).__init__(temp)
def _get_key(self, key):
key = key.replace("__", ":", 1)
return key
@@ -180,7 +181,6 @@ class SenpyModel(SenpyMixin, dict):
def __setitem__(self, key, value):
dict.__setitem__(self, key, value)
def __delitem__(self, key):
dict.__delitem__(self, key)
@@ -195,62 +195,80 @@ class SenpyModel(SenpyMixin, dict):
def __delattr__(self, key):
self.__delitem__(self._get_key(key))
def _plain_dict(self):
d = { k: v for (k,v) in self.items() if k[0] != "_"}
d = {k: v for (k, v) in self.items() if k[0] != "_"}
d["@id"] = d.pop('id')
return d
class Response(SenpyModel):
schema = read_schema('response.json')
class Results(SenpyModel):
schema = read_schema('results.json')
class Entry(SenpyModel):
schema = read_schema('entry.json')
class Sentiment(SenpyModel):
schema = read_schema('sentiment.json')
class Analysis(SenpyModel):
schema = read_schema('analysis.json')
class EmotionSet(SenpyModel):
schema = read_schema('emotionSet.json')
class Emotion(SenpyModel):
schema = read_schema('emotion.json')
class EmotionModel(SenpyModel):
schema = read_schema('emotionModel.json')
class Suggestion(SenpyModel):
schema = read_schema('suggestion.json')
class PluginModel(SenpyModel):
schema = read_schema('plugin.json')
class EmotionPluginModel(SenpyModel):
schema = read_schema('plugin.json')
class SentimentPluginModel(SenpyModel):
schema = read_schema('plugin.json')
class Plugins(SenpyModel):
schema = read_schema('plugins.json')
class Error(SenpyMixin, BaseException ):
def __init__(self, message, status=500, params=None, errors=None, *args, **kwargs):
class Error(SenpyMixin, BaseException):
def __init__(self,
message,
status=500,
params=None,
errors=None,
*args,
**kwargs):
self.message = message
self.status = status
self.params = params or {}
self.errors = errors or ""
def _plain_dict(self):
return self.__dict__
return self.__dict__
def __str__(self):
return str(self.jsonld())
return str(self.jsonld())

15
senpy/plugins.py
View File

@@ -10,8 +10,8 @@ from .models import Response, PluginModel, Error
logger = logging.getLogger(__name__)
class SenpyPlugin(PluginModel):
class SenpyPlugin(PluginModel):
def __init__(self, info=None):
if not info:
raise Error(message=("You need to provide configuration"
@@ -39,8 +39,8 @@ class SenpyPlugin(PluginModel):
''' Destructor, to make sure all the resources are freed '''
self.deactivate()
class SentimentPlugin(SenpyPlugin):
class SentimentPlugin(SenpyPlugin):
def __init__(self, info, *args, **kwargs):
super(SentimentPlugin, self).__init__(info, *args, **kwargs)
self.minPolarityValue = float(info.get("minPolarityValue", 0))
@@ -49,7 +49,6 @@ class SentimentPlugin(SenpyPlugin):
class EmotionPlugin(SenpyPlugin):
def __init__(self, info, *args, **kwargs):
resp = super(EmotionPlugin, self).__init__(info, *args, **kwargs)
self.minEmotionValue = float(info.get("minEmotionValue", 0))
@@ -58,7 +57,6 @@ class EmotionPlugin(SenpyPlugin):
class ShelfMixin(object):
@property
def sh(self):
if not hasattr(self, '_sh') or self._sh is None:
@@ -75,7 +73,7 @@ class ShelfMixin(object):
self.save()
def __del__(self):
self.save()
self.save()
super(ShelfMixin, self).__del__()
@property
@@ -84,12 +82,13 @@ class ShelfMixin(object):
if hasattr(self, '_info') and 'shelf_file' in self._info:
self.__dict__['_shelf_file'] = self._info['shelf_file']
else:
self._shelf_file = os.path.join(tempfile.gettempdir(), self.name + '.p')
return self._shelf_file
self._shelf_file = os.path.join(tempfile.gettempdir(),
self.name + '.p')
return self._shelf_file
def save(self):
logger.debug('closing pickle')
if hasattr(self, '_sh') and self._sh is not None:
with open(self.shelf_file, 'wb') as f:
pickle.dump(self._sh, f)
del(self.__dict__['_sh'])
del (self.__dict__['_sh'])

23
senpy/plugins/rand/rand.py
View File

@@ -16,26 +16,15 @@ class Sentiment140Plugin(SentimentPlugin):
polarity = "marl:Positive"
elif polarity_value < 0:
polarity = "marl:Negative"
entry = Entry({"id":":Entry0",
"nif:isString": params["input"]})
sentiment = Sentiment({"id": ":Sentiment0",
"marl:hasPolarity": polarity,
"marl:polarityValue": polarity_value})
entry = Entry({"id": ":Entry0", "nif:isString": params["input"]})
sentiment = Sentiment({
"id": ":Sentiment0",
"marl:hasPolarity": polarity,
"marl:polarityValue": polarity_value
})
sentiment["prov:wasGeneratedBy"] = self.id
entry.sentiments = []
entry.sentiments.append(sentiment)
entry.language = lang
response.entries.append(entry)
return response

27
senpy/plugins/sentiment140/sentiment140.py
View File

@@ -9,16 +9,17 @@ class Sentiment140Plugin(SentimentPlugin):
def analyse(self, **params):
lang = params.get("language", "auto")
res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
json.dumps({"language": lang,
"data": [{"text": params["input"]}]
}
)
)
json.dumps({
"language": lang,
"data": [{
"text": params["input"]
}]
}))
p = params.get("prefix", None)
response = Results(prefix=p)
polarity_value = self.maxPolarityValue*int(res.json()["data"][0]
["polarity"]) * 0.25
polarity_value = self.maxPolarityValue * int(res.json()["data"][0][
"polarity"]) * 0.25
polarity = "marl:Neutral"
neutral_value = self.maxPolarityValue / 2.0
if polarity_value > neutral_value:
@@ -26,12 +27,12 @@ class Sentiment140Plugin(SentimentPlugin):
elif polarity_value < neutral_value:
polarity = "marl:Negative"
entry = Entry(id="Entry0",
nif__isString=params["input"])
sentiment = Sentiment(id="Sentiment0",
prefix=p,
marl__hasPolarity=polarity,
marl__polarityValue=polarity_value)
entry = Entry(id="Entry0", nif__isString=params["input"])
sentiment = Sentiment(
id="Sentiment0",
prefix=p,
marl__hasPolarity=polarity,
marl__polarityValue=polarity_value)
sentiment.prov__wasGeneratedBy = self.id
entry.sentiments = []
entry.sentiments.append(sentiment)