balkian/senpy
1
0
Fork 0
mirror of https://github.com/gsi-upm/senpy synced 2024-11-14 04:32:29 +00:00
Code Issues Releases Wiki Activity

Update to senpy 0.20

Browse Source
This commit is contained in:
J. Fernando Sánchez 2019-04-04 12:56:46 +02:00
parent fa993c6e2a
commit 4f286057c9
13 changed files with 239 additions and 247 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -1 +1 @@
from gsiupm/senpy:0.11.4-python2.7 from gsiupm/senpy:0.20-python3.6

4
Makefile
View File

@ -1,4 +1,4 @@
PYVERSION=2.7 PYVERSION=3.7
NAME=senpycommunity NAME=senpycommunity
REPO=gsiupm REPO=gsiupm
PLUGINS= $(filter %/, $(wildcard */)) PLUGINS= $(filter %/, $(wildcard */))
@ -19,7 +19,7 @@ test-fast: test-fast-/
test: docker-build test-fast test: docker-build test-fast
dev: dev: docker-build
docker run -p $(DEV_PORT):5000 $(DOCKER_FLAGS) -ti $(DOCKER_FLAGS) -v $$PWD/$*:/senpy-plugins/ --entrypoint /bin/bash -v $$PWD/data:/data/ --rm $(IMAGEWTAG) docker run -p $(DEV_PORT):5000 $(DOCKER_FLAGS) -ti $(DOCKER_FLAGS) -v $$PWD/$*:/senpy-plugins/ --entrypoint /bin/bash -v $$PWD/data:/data/ --rm $(IMAGEWTAG)
.PHONY:: test test-fast dev .PHONY:: test test-fast dev

78
emotion-anew/emotion-anew.py
View File

@ -19,11 +19,11 @@ from nltk.corpus import stopwords
from pattern.en import parse as parse_en from pattern.en import parse as parse_en
from pattern.es import parse as parse_es from pattern.es import parse as parse_es
from senpy.plugins import SentimentPlugin, SenpyPlugin from senpy.plugins import EmotionPlugin, SenpyPlugin
from senpy.models import Results, EmotionSet, Entry, Emotion from senpy.models import Results, EmotionSet, Entry, Emotion
class ANEW(SentimentPlugin): class ANEW(EmotionPlugin):
description = "This plugin consists on an emotion classifier using ANEW lexicon dictionary to calculate VAD (valence-arousal-dominance) of the sentence and determinate which emotion is closer to this value. Each emotion has a centroid, calculated according to this article: http://www.aclweb.org/anthology/W10-0208. The plugin is going to look for the words in the sentence that appear in the ANEW dictionary and calculate the average VAD score for the sentence. Once this score is calculated, it is going to seek the emotion that is closest to this value." description = "This plugin consists on an emotion classifier using ANEW lexicon dictionary to calculate VAD (valence-arousal-dominance) of the sentence and determinate which emotion is closer to this value. Each emotion has a centroid, calculated according to this article: http://www.aclweb.org/anthology/W10-0208. The plugin is going to look for the words in the sentence that appear in the ANEW dictionary and calculate the average VAD score for the sentence. Once this score is calculated, it is going to seek the emotion that is closest to this value."
author = "@icorcuera" author = "@icorcuera"
version = "0.5.1" version = "0.5.1"
@ -31,6 +31,7 @@ class ANEW(SentimentPlugin):
extra_params = { extra_params = {
"language": { "language": {
"description": "language of the input",
"aliases": ["language", "l"], "aliases": ["language", "l"],
"required": True, "required": True,
"options": ["es","en"], "options": ["es","en"],
@ -75,7 +76,7 @@ class ANEW(SentimentPlugin):
"neutral": "http://gsi.dit.upm.es/ontologies/wnaffect/ns#neutral-emotion", "neutral": "http://gsi.dit.upm.es/ontologies/wnaffect/ns#neutral-emotion",
"sadness": "http://gsi.dit.upm.es/ontologies/wnaffect/ns#sadness" "sadness": "http://gsi.dit.upm.es/ontologies/wnaffect/ns#sadness"
} }
onyx__usesEmotionModel = "emoml:big6" onyx__usesEmotionModel = "emoml:pad"
nltk_resources = ['stopwords'] nltk_resources = ['stopwords']
def activate(self, *args, **kwargs): def activate(self, *args, **kwargs):
@ -119,14 +120,13 @@ class ANEW(SentimentPlugin):
return s return s
def _extract_ngrams(self, text, lang): def _extract_ngrams(self, text, lang):
unigrams_lemmas = [] unigrams_lemmas = []
unigrams_words = [] unigrams_words = []
pos_tagged = [] pos_tagged = []
if lang == 'es': if lang == 'es':
sentences = parse_es(text,lemmata=True).split() sentences = list(parse_es(text, lemmata=True).split())
else: else:
sentences = parse_en(text,lemmata=True).split() sentences = list(parse_en(text, lemmata=True).split())
for sentence in sentences: for sentence in sentences:
for token in sentence: for token in sentence:
@ -140,19 +140,6 @@ class ANEW(SentimentPlugin):
def _find_ngrams(self, input_list, n): def _find_ngrams(self, input_list, n):
return zip(*[input_list[i:] for i in range(n)]) return zip(*[input_list[i:] for i in range(n)])
def _emotion_calculate(self, VAD):
emotion=''
value=10000000000000000000000.0
for state in self.centroids:
valence=VAD[0]-self.centroids[state]['V']
arousal=VAD[1]-self.centroids[state]['A']
dominance=VAD[2]-self.centroids[state]['D']
new_value=math.sqrt((valence*valence)+(arousal*arousal)+(dominance*dominance))
if new_value < value:
value=new_value
emotion=state
return emotion
def _extract_features(self, tweet,dictionary,lang): def _extract_features(self, tweet,dictionary,lang):
feature_set={} feature_set={}
ngrams_lemmas,ngrams_words,pos_tagged = self._extract_ngrams(tweet,lang) ngrams_lemmas,ngrams_words,pos_tagged = self._extract_ngrams(tweet,lang)
@ -176,14 +163,13 @@ class ANEW(SentimentPlugin):
emotion='neutral' emotion='neutral'
else: else:
totalVAD=[totalVAD[0]/matches,totalVAD[1]/matches,totalVAD[2]/matches] totalVAD=[totalVAD[0]/matches,totalVAD[1]/matches,totalVAD[2]/matches]
emotion=self._emotion_calculate(totalVAD) feature_set['V'] = totalVAD[0]
feature_set['emotion']=emotion feature_set['A'] = totalVAD[1]
feature_set['V']=totalVAD[0] feature_set['D'] = totalVAD[2]
feature_set['A']=totalVAD[1]
feature_set['D']=totalVAD[2]
return feature_set return feature_set
def analyse_entry(self, entry, params): def analyse_entry(self, entry, activity):
params = activity.params
text_input = entry.text text_input = entry.text
@ -196,13 +182,12 @@ class ANEW(SentimentPlugin):
emotions.id = "Emotions0" emotions.id = "Emotions0"
emotion1 = Emotion(id="Emotion0") emotion1 = Emotion(id="Emotion0")
emotion1["onyx:hasEmotionCategory"] = self.emotions_ontology[feature_set['emotion']]
emotion1["http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/anew/ns#valence"] = feature_set['V'] emotion1["http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/anew/ns#valence"] = feature_set['V']
emotion1["http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/anew/ns#arousal"] = feature_set['A'] emotion1["http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/anew/ns#arousal"] = feature_set['A']
emotion1["http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/anew/ns#dominance"] = feature_set['D'] emotion1["http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/anew/ns#dominance"] = feature_set['D']
emotion1.prov(self) emotion1.prov(activity)
emotions.prov(self) emotions.prov(activity)
emotions.onyx__hasEmotion.append(emotion1) emotions.onyx__hasEmotion.append(emotion1)
entry.emotions = [emotions, ] entry.emotions = [emotions, ]
@ -212,47 +197,64 @@ class ANEW(SentimentPlugin):
ontology = "http://gsi.dit.upm.es/ontologies/wnaffect/ns#" ontology = "http://gsi.dit.upm.es/ontologies/wnaffect/ns#"
test_cases = [ test_cases = [
{ {
'name': 'anger with VAD=(2.12, 6.95, 5.05)',
'input': 'I hate you', 'input': 'I hate you',
'expected': { 'expected': {
'emotions': [{ 'onyx:hasEmotionSet': [{
'onyx:hasEmotion': [{ 'onyx:hasEmotion': [{
'onyx:hasEmotionCategory': ontology + 'anger', "http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/anew/ns#arousal": 6.95,
"http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/anew/ns#dominance": 5.05,
"http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/anew/ns#valence": 2.12,
}] }]
}] }]
} }
}, { }, {
'input': 'i am sad', 'input': 'i am sad',
'expected': { 'expected': {
'emotions': [{ 'onyx:hasEmotionSet': [{
'onyx:hasEmotion': [{ 'onyx:hasEmotion': [{
'onyx:hasEmotionCategory': ontology + 'sadness', "http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/anew/ns#arousal": 4.13,
"http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/anew/ns#dominance": 3.45,
"http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/anew/ns#valence": 1.61,
}] }]
}] }]
} }
}, { }, {
'name': 'joy',
'input': 'i am happy with my marks', 'input': 'i am happy with my marks',
'expected': { 'expected': {
'emotions': [{ 'onyx:hasEmotionSet': [{
'onyx:hasEmotion': [{ 'onyx:hasEmotion': [{
'onyx:hasEmotionCategory': ontology + 'joy', "http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/anew/ns#arousal": 6.49,
"http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/anew/ns#dominance": 6.63,
"http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/anew/ns#valence": 8.21,
}] }]
}] }]
} }
}, { }, {
'name': 'negative-feat',
'input': 'This movie is scary', 'input': 'This movie is scary',
'expected': { 'expected': {
'emotions': [{ 'onyx:hasEmotionSet': [{
'onyx:hasEmotion': [{ 'onyx:hasEmotion': [{
'onyx:hasEmotionCategory': ontology + 'negative-fear', "http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/anew/ns#arousal": 5.8100000000000005,
"http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/anew/ns#dominance": 4.33,
"http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/anew/ns#valence": 5.050000000000001,
}] }]
}] }]
} }
}, { }, {
'name': 'negative-fear',
'input': 'this cake is disgusting' , 'input': 'this cake is disgusting' ,
'expected': { 'expected': {
'emotions': [{ 'onyx:hasEmotionSet': [{
'onyx:hasEmotion': [{ 'onyx:hasEmotion': [{
'onyx:hasEmotionCategory': ontology + 'negative-fear', "http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/anew/ns#arousal": 5.09,
"http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/anew/ns#dominance": 4.4,
"http://www.gsi.dit.upm.es/ontologies/onyx/vocabularies/anew/ns#valence": 5.109999999999999,
}] }]
}] }]
} }

3
emotion-anew/emotion-anew.senpy
View File

@ -8,5 +8,4 @@ requirements:
- scikit-learn - scikit-learn
- textblob - textblob
- pattern - pattern
- lxml - lxml
onyx:usesEmotionModel: "emoml:big6"

126
emotion-depechemood/depechemood_plugin.py
View File

@ -1,49 +1,56 @@
#!/usr/local/bin/python #!/usr/local/bin/python
# coding: utf-8 # coding: utf-8
from future import standard_library
standard_library.install_aliases()
import os import os
import re import re
import sys import sys
import string import string
import numpy as np import numpy as np
import pandas as pd
from six.moves import urllib from six.moves import urllib
from nltk.corpus import stopwords from nltk.corpus import stopwords
from senpy import EmotionPlugin, TextBox, models from senpy import EmotionBox, models
def ignore(dchars): def ignore(dchars):
deletechars = "".join(dchars) deletechars = "".join(dchars)
if sys.version_info[0] >= 3: tbl = str.maketrans("", "", deletechars)
tbl = str.maketrans("", "", deletechars) ignore = lambda s: s.translate(tbl)
ignore = lambda s: s.translate(tbl)
else:
def ignore(s):
return string.translate(s, None, deletechars)
return ignore return ignore
class DepecheMood(TextBox, EmotionPlugin): class DepecheMood(EmotionBox):
'''Plugin that uses the DepecheMood++ emotion lexicon.''' '''
Plugin that uses the DepecheMood emotion lexicon.
DepecheMood is an emotion lexicon automatically generated from news articles where users expressed their associated emotions. It contains two languages (English and Italian), as well as three types of word representations (token, lemma and lemma#PoS). For English, the lexicon contains 165k tokens, while the Italian version contains 116k. Unsupervised techniques can be applied to generate simple but effective baselines. To learn more, please visit https://github.com/marcoguerini/DepecheMood and http://www.depechemood.eu/
'''
author = 'Oscar Araque' author = 'Oscar Araque'
name = 'emotion-depechemood'
version = '0.1' version = '0.1'
requirements = ['pandas']
nltk_resources = ["stopwords"]
onyx__usesEmotionModel = 'wna:WNAModel'
EMOTIONS = ['wna:negative-fear',
'wna:amusement',
'wna:anger',
'wna:annoyance',
'wna:indifference',
'wna:joy',
'wna:awe',
'wna:sadness']
DM_EMOTIONS = ['AFRAID', 'AMUSED', 'ANGRY', 'ANNOYED', 'DONT_CARE', 'HAPPY', 'INSPIRED', 'SAD',]
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super(DepecheMood, self).__init__(*args, **kwargs) super(DepecheMood, self).__init__(*args, **kwargs)
self.LEXICON_URL = "https://github.com/marcoguerini/DepecheMood/raw/master/DepecheMood%2B%2B/DepecheMood_english_token_full.tsv" self.LEXICON_URL = "https://github.com/marcoguerini/DepecheMood/raw/master/DepecheMood%2B%2B/DepecheMood_english_token_full.tsv"
self.EMOTIONS = ['AFRAID', 'AMUSED', 'ANGRY', 'ANNOYED', 'DONT_CARE', 'HAPPY', 'INSPIRED', 'SAD',]
self._mapping = {
'AFRAID': 'wna:negative-fear',
'AMUSED': 'wna:amusement',
'ANGRY': 'wna:anger',
'ANNOYED': 'wna:annoyance',
'DONT_CARE': 'wna:indifference',
'HAPPY': 'wna:joy',
'INSPIRED': 'wna:awe',
'SAD': 'wna:sadness',
}
self._denoise = ignore(set(string.punctuation)|set('«»')) self._denoise = ignore(set(string.punctuation)|set('«»'))
self._stop_words = [] self._stop_words = []
self._lex_vocab = None self._lex_vocab = None
@ -89,19 +96,21 @@ class DepecheMood(TextBox, EmotionPlugin):
return S return S
def estimate_all_emotions(self, tokens): def estimate_all_emotions(self, tokens):
S = {} S = []
intersection = set(tokens) & self._lex_vocab intersection = set(tokens) & self._lex_vocab
for emotion in self.EMOTIONS: for emotion in self.DM_EMOTIONS:
s = self.estimate_emotion(intersection, emotion) s = self.estimate_emotion(intersection, emotion)
emotion_mapped = self._mapping[emotion] S.append(s)
S[emotion_mapped] = s
return S return S
def download_lex(self, file_path='DepecheMood_english_token_full.tsv', freq_threshold=10): def download_lex(self, file_path='DepecheMood_english_token_full.tsv', freq_threshold=10):
import pandas as pd
try: try:
file_path = self.find_file(file_path) file_path = self.find_file(file_path)
except IOError: except IOError:
file_path = self.path(file_path)
filename, _ = urllib.request.urlretrieve(self.LEXICON_URL, file_path) filename, _ = urllib.request.urlretrieve(self.LEXICON_URL, file_path)
lexicon = pd.read_csv(file_path, sep='\t', index_col=0) lexicon = pd.read_csv(file_path, sep='\t', index_col=0)
@ -110,18 +119,8 @@ class DepecheMood(TextBox, EmotionPlugin):
lexicon = lexicon.T.to_dict() lexicon = lexicon.T.to_dict()
return lexicon return lexicon
def output(self, output, entry, **kwargs): def predict_one(self, features, **kwargs):
s = models.EmotionSet() tokens = self.preprocess(features[0])
s.prov__wasGeneratedBy = self.id
entry.emotions.append(s)
for label, value in output.items():
e = models.Emotion(onyx__hasEmotionCategory=label,
onyx__hasEmotionIntensity=value)
s.onyx__hasEmotion.append(e)
return entry
def predict_one(self, input, **kwargs):
tokens = self.preprocess(input)
estimation = self.estimate_all_emotions(tokens) estimation = self.estimate_all_emotions(tokens)
return estimation return estimation
@ -131,26 +130,41 @@ class DepecheMood(TextBox, EmotionPlugin):
'nif:isString': 'My cat is very happy', 'nif:isString': 'My cat is very happy',
}, },
'expected': { 'expected': {
'emotions': [ 'onyx:hasEmotionSet': [
{ {
'@type': 'emotionSet',
'onyx:hasEmotion': [ 'onyx:hasEmotion': [
{'@type': 'emotion', 'onyx:hasEmotionCategory': 'wna:negative-fear', {
'onyx:hasEmotionIntensity': 0.05278117640010922, }, 'onyx:hasEmotionCategory': 'wna:negative-fear',
{'@type': 'emotion', 'onyx:hasEmotionCategory': 'wna:amusement', 'onyx:hasEmotionIntensity': 0.05278117640010922
'onyx:hasEmotionIntensity': 0.2114806151413433, }, },
{'@type': 'emotion', 'onyx:hasEmotionCategory': 'wna:anger', {
'onyx:hasEmotionIntensity': 0.05726119426520887, }, 'onyx:hasEmotionCategory': 'wna:amusement',
{'@type': 'emotion', 'onyx:hasEmotionCategory': 'wna:annoyance', 'onyx:hasEmotionIntensity': 0.2114806151413433,
'onyx:hasEmotionIntensity': 0.12295990731053638, }, },
{'@type': 'emotion', 'onyx:hasEmotionCategory': 'wna:indifference', {
'onyx:hasEmotionIntensity': 0.1860159893608025, }, 'onyx:hasEmotionCategory': 'wna:anger',
{'@type': 'emotion', 'onyx:hasEmotionCategory': 'wna:joy', 'onyx:hasEmotionIntensity': 0.05726119426520887
'onyx:hasEmotionIntensity': 0.12904050973724163, }, },
{'@type': 'emotion', 'onyx:hasEmotionCategory': 'wna:awe', {
'onyx:hasEmotionIntensity': 0.17973650399862967, }, 'onyx:hasEmotionCategory': 'wna:annoyance',
{'@type': 'emotion', 'onyx:hasEmotionCategory': 'wna:sadness', 'onyx:hasEmotionIntensity': 0.12295990731053638,
'onyx:hasEmotionIntensity': 0.060724103786128455, }, },
{
'onyx:hasEmotionCategory': 'wna:indifference',
'onyx:hasEmotionIntensity': 0.1860159893608025,
},
{
'onyx:hasEmotionCategory': 'wna:joy',
'onyx:hasEmotionIntensity': 0.12904050973724163,
},
{
'onyx:hasEmotionCategory': 'wna:awe',
'onyx:hasEmotionIntensity': 0.17973650399862967,
},
{
'onyx:hasEmotionCategory': 'wna:sadness',
'onyx:hasEmotionIntensity': 0.060724103786128455,
},
] ]
} }
] ]
@ -164,4 +178,4 @@ if __name__ == '__main__':
# sp, app = easy_load() # sp, app = easy_load()
# for plug in sp.analysis_plugins: # for plug in sp.analysis_plugins:
# plug.test() # plug.test()
easy() easy_test(debug=False)

8
emotion-wnaffect/emotion-wnaffect.py
View File

@ -25,6 +25,7 @@ class WNAffect(EmotionPlugin, ShelfMixin):
extra_params = { extra_params = {
'language': { 'language': {
"@id": 'lang_wnaffect', "@id": 'lang_wnaffect',
'description': 'language of the input',
'aliases': ['language', 'l'], 'aliases': ['language', 'l'],
'required': True, 'required': True,
'options': ['en',] 'options': ['en',]
@ -223,7 +224,8 @@ class WNAffect(EmotionPlugin, ShelfMixin):
return feature_set return feature_set
def analyse_entry(self, entry, params): def analyse_entry(self, entry, activity):
params = activity.params
text_input = entry['nif:isString'] text_input = entry['nif:isString']
@ -262,14 +264,14 @@ class WNAffect(EmotionPlugin, ShelfMixin):
'algorithm': 'emotion-wnaffect'} 'algorithm': 'emotion-wnaffect'}
self.activate() self.activate()
res = next(self.analyse_entry(Entry(nif__isString="This text make me sad"), params))
texts = {'I hate you': 'anger', texts = {'I hate you': 'anger',
'i am sad': 'sadness', 'i am sad': 'sadness',
'i am happy with my marks': 'joy', 'i am happy with my marks': 'joy',
'This movie is scary': 'negative-fear'} 'This movie is scary': 'negative-fear'}
for text in texts: for text in texts:
response = next(self.analyse_entry(Entry(nif__isString=text), params)) response = next(self.analyse_entry(Entry(nif__isString=text),
self.activity(params)))
expected = texts[text] expected = texts[text]
emotionSet = response.emotions[0] emotionSet = response.emotions[0]
max_emotion = max(emotionSet['onyx:hasEmotion'], key=lambda x: x['onyx:hasEmotionIntensity']) max_emotion = max(emotionSet['onyx:hasEmotion'], key=lambda x: x['onyx:hasEmotionIntensity'])

8
example-plugin/example_plugin.py
View File

@ -1,8 +1,8 @@
from senpy.plugins import Analysis from senpy.plugins import AnalysisPlugin
from senpy.models import Response, Entry from senpy.models import Response, Entry
class ExamplePlugin(Analysis): class ExamplePlugin(AnalysisPlugin):
'''A *VERY* simple plugin that exemplifies the development of Senpy Plugins''' '''A *VERY* simple plugin that exemplifies the development of Senpy Plugins'''
name = "example-plugin" name = "example-plugin"
author = "@balkian" author = "@balkian"
@ -10,6 +10,7 @@ class ExamplePlugin(Analysis):
extra_params = { extra_params = {
"parameter": { "parameter": {
"@id": "parameter", "@id": "parameter",
"description": "this parameter does nothing, it is only an example",
"aliases": ["parameter", "param"], "aliases": ["parameter", "param"],
"required": True, "required": True,
"default": 42 "default": 42
@ -17,7 +18,8 @@ class ExamplePlugin(Analysis):
} }
custom_attribute = "42" custom_attribute = "42"
def analyse_entry(self, entry, params): def analyse_entry(self, entry, activity):
params = activity.params
self.log.debug('Analysing with the example.') self.log.debug('Analysing with the example.')
self.log.debug('The answer to this response is: %s.' % params['parameter']) self.log.debug('The answer to this response is: %s.' % params['parameter'])
resp = Response() resp = Response()

4
k8s/senpy-deployment.yaml
View File

@ -15,11 +15,9 @@ spec:
- name: senpy-latest - name: senpy-latest
image: $IMAGEWTAG image: $IMAGEWTAG
imagePullPolicy: Always imagePullPolicy: Always
args:
- "--default-plugins"
resources: resources:
limits: limits:
memory: "512Mi" memory: "2048Mi"
cpu: "1000m" cpu: "1000m"
ports: ports:
- name: web - name: web

155
sentiment-basic/sentiment-basic.py
View File

@ -12,14 +12,14 @@ from textblob import TextBlob
from scipy.interpolate import interp1d from scipy.interpolate import interp1d
from os import path from os import path
from senpy.plugins import SentimentPlugin, SenpyPlugin from senpy.plugins import SentimentBox, SenpyPlugin
from senpy.models import Results, Entry, Sentiment from senpy.models import Results, Entry, Sentiment, Error
if sys.version_info[0] >= 3: if sys.version_info[0] >= 3:
unicode = str unicode = str
class SentimentBasic(SentimentPlugin): class SentimentBasic(SentimentBox):
''' '''
Sentiment classifier using rule-based classification for Spanish. Based on english to spanish translation and SentiWordNet sentiment knowledge. This is a demo plugin that uses only some features from the TASS 2015 classifier. To use the entirely functional classifier you can use the service in: http://senpy.cluster.gsi.dit.upm.es. Sentiment classifier using rule-based classification for Spanish. Based on english to spanish translation and SentiWordNet sentiment knowledge. This is a demo plugin that uses only some features from the TASS 2015 classifier. To use the entirely functional classifier you can use the service in: http://senpy.cluster.gsi.dit.upm.es.
''' '''
@ -28,10 +28,11 @@ class SentimentBasic(SentimentPlugin):
version = "0.1.1" version = "0.1.1"
extra_params = { extra_params = {
"language": { "language": {
"description": "language of the text",
"aliases": ["language", "l"], "aliases": ["language", "l"],
"required": True, "required": True,
"options": ["en","es", "it", "fr", "auto"], "options": ["en","es", "it", "fr"],
"default": "auto" "default": "en"
} }
} }
sentiword_path = "SentiWordNet_3.0.txt" sentiword_path = "SentiWordNet_3.0.txt"
@ -40,6 +41,8 @@ class SentimentBasic(SentimentPlugin):
minPolarityValue = -1 minPolarityValue = -1
nltk_resources = ['punkt','wordnet', 'omw'] nltk_resources = ['punkt','wordnet', 'omw']
with_polarity = False
def _load_swn(self): def _load_swn(self):
self.swn_path = self.find_file(self.sentiword_path) self.swn_path = self.find_file(self.sentiword_path)
swn = SentiWordNet(self.swn_path) swn = SentiWordNet(self.swn_path)
@ -59,128 +62,116 @@ class SentimentBasic(SentimentPlugin):
return [t for t in tokens if t not in string.punctuation] return [t for t in tokens if t not in string.punctuation]
def _tokenize(self, text): def _tokenize(self, text):
data = {} sentence_ = {}
sentences = nltk.sent_tokenize(text) words = nltk.word_tokenize(text)
for i, sentence in enumerate(sentences): sentence_['sentence'] = text
sentence_ = {} tokens_ = [w.lower() for w in words]
words = nltk.word_tokenize(sentence) sentence_['tokens'] = self._remove_punctuation(tokens_)
sentence_['sentence'] = sentence return sentence_
tokens_ = [w.lower() for w in words]
sentence_['tokens'] = self._remove_punctuation(tokens_)
data[i] = sentence_
return data
def _pos(self, tokens): def _pos(self, tokens):
for i in tokens: tokens['tokens'] = self._pos_tagger.tag(tokens['tokens'])
tokens[i]['tokens'] = self._pos_tagger.tag(tokens[i]['tokens'])
return tokens return tokens
def _compare_synsets(self, synsets, tokens, i): def _compare_synsets(self, synsets, tokens):
for synset in synsets: for synset in synsets:
for word in tokens[i]['lemmas']: for word, lemmas in tokens['lemmas'].items():
for lemma in tokens[i]['lemmas'][word]: for lemma in lemmas:
synset_ = lemma.synset() synset_ = lemma.synset()
if synset == synset_: if synset == synset_:
return synset return synset
return None return None
def predict_one(self, features, activity):
def analyse_entry(self, entry, params): language = activity.param("language")
language = params.get("language") text = features[0]
text = entry.text
tokens = self._tokenize(text) tokens = self._tokenize(text)
tokens = self._pos(tokens) tokens = self._pos(tokens)
sufixes = {'es':'spa','en':'eng','it':'ita','fr':'fra'} sufixes = {'es':'spa','en':'eng','it':'ita','fr':'fra'}
for i in tokens: tokens['lemmas'] = {}
tokens[i]['lemmas'] = {} for w in tokens['tokens']:
for w in tokens[i]['tokens']: lemmas = wn.lemmas(w[0], lang=sufixes[language])
lemmas = wn.lemmas(w[0], lang=sufixes[language]) if len(lemmas) == 0:
if len(lemmas) == 0: continue
continue tokens['lemmas'][w[0]] = lemmas
tokens[i]['lemmas'][w[0]] = lemmas
if language == "en": if language == "en":
trans = TextBlob(unicode(text)) trans = TextBlob(unicode(text))
else: else:
trans = TextBlob(unicode(text)).translate(from_lang=language,to='en') try:
trans = TextBlob(unicode(text)).translate(from_lang=language,to='en')
except Exception as ex:
raise Error('Could not translate the text from "{}" to "{}": {}'.format(language,
'en',
str(ex)))
useful_synsets = {} useful_synsets = {}
for s_i, t_s in enumerate(trans.sentences): for w_i, t_w in enumerate(trans.sentences[0].words):
useful_synsets[s_i] = {} synsets = wn.synsets(trans.sentences[0].words[w_i])
for w_i, t_w in enumerate(trans.sentences[s_i].words): if len(synsets) == 0:
synsets = wn.synsets(trans.sentences[s_i].words[w_i]) continue
if len(synsets) == 0: eq_synset = self._compare_synsets(synsets, tokens)
continue useful_synsets[t_w] = eq_synset
eq_synset = self._compare_synsets(synsets, tokens, s_i)
useful_synsets[s_i][t_w] = eq_synset
scores = {} scores = {}
for i in tokens: scores = {}
scores[i] = {} if useful_synsets != None:
if useful_synsets != None: for word in useful_synsets:
for word in useful_synsets[i]: if useful_synsets[word] is None:
if useful_synsets[i][word] is None: continue
continue temp_scores = self._swn.get_score(useful_synsets[word].name().split('.')[0].replace(' ',' '))
temp_scores = self._swn.get_score(useful_synsets[i][word].name().split('.')[0].replace(' ',' ')) for score in temp_scores:
for score in temp_scores: if score['synset'] == useful_synsets[word]:
if score['synset'] == useful_synsets[i][word]: t_score = score['pos'] - score['neg']
t_score = score['pos'] - score['neg'] f_score = 'neu'
f_score = 'neu' if t_score > 0:
if t_score > 0: f_score = 'pos'
f_score = 'pos' elif t_score < 0:
elif t_score < 0: f_score = 'neg'
f_score = 'neg' score['score'] = f_score
score['score'] = f_score scores[word] = score
scores[i][word] = score break
break g_score = 0.5
p = params.get("prefix", None)
for i in scores: for i in scores:
n_pos = 0.0 n_pos = 0.0
n_neg = 0.0 n_neg = 0.0
for w in scores[i]: for w in scores:
if scores[i][w]['score'] == 'pos': if scores[w]['score'] == 'pos':
n_pos += 1.0 n_pos += 1.0
elif scores[i][w]['score'] == 'neg': elif scores[w]['score'] == 'neg':
n_neg += 1.0 n_neg += 1.0
inter = interp1d([-1.0, 1.0], [0.0, 1.0]) inter = interp1d([-1.0, 1.0], [0.0, 1.0])
try: try:
g_score = (n_pos - n_neg) / (n_pos + n_neg) g_score = (n_pos - n_neg) / (n_pos + n_neg)
g_score = float(inter(g_score)) g_score = float(inter(g_score))
except: except:
if n_pos == 0 and n_neg == 0: if n_pos == 0 and n_neg == 0:
g_score = 0.5 g_score = 0.5
if g_score >= 0.5:
polarity = 'marl:Positive'
polarity_value = 1
elif g_score < 0.5:
polarity = 'marl:Negative'
polarity_value = -1
else:
polarity = 'marl:Neutral'
polarity_value = 0
opinion = Sentiment(id="Opinion0"+'_'+str(i),
marl__hasPolarity=polarity,
marl__polarityValue=polarity_value)
opinion.prov(self) if g_score > 0.5: # Positive
entry.sentiments.append(opinion) return [1, 0, 0]
elif g_score < 0.5: # Negative
return [0, 0, 1]
else:
return [0, 1, 0]
yield entry
test_cases = [ test_cases = [
{ {
'input': u'Odio ir al cine', 'input': 'Odio ir al cine',
'params': {'language': 'es'}, 'params': {'language': 'es'},
'polarity': 'marl:Negative' 'polarity': 'marl:Negative'
}, },
{ {
'input': u'El cielo está nublado', 'input': 'El cielo está nublado',
'params': {'language': 'es'}, 'params': {'language': 'es'},
'polarity': 'marl:Positive' 'polarity': 'marl:Neutral'
}, },
{ {
'input': u'Esta tarta está muy buena', 'input': 'Esta tarta está muy buena',
'params': {'language': 'es'}, 'params': {'language': 'es'},
'polarity': 'marl:Negative' 'polarity': 'marl:Negative' # SURPRISINGLY!
} }
] ]

21
sentiment-meaningCloud/meaningcloud_plugin.py
View File

@ -21,7 +21,7 @@ class MeaningCloudPlugin(SentimentPlugin):
When you had obtained the meaningCloud API Key, you have to provide it to the plugin, using param apiKey. When you had obtained the meaningCloud API Key, you have to provide it to the plugin, using param apiKey.
Example request: Example request:
http://senpy.cluster.gsi.dit.upm.es/api/?algo=meaningCloud&language=en&apiKey=<API key>&input=I%20love%20Madrid. http://senpy.cluster.gsi.dit.upm.es/api/?algo=meaningCloud&language=en&apiKey=YOUR_API_KEY&input=I%20love%20Madrid.
''' '''
name = 'sentiment-meaningcloud' name = 'sentiment-meaningcloud'
author = 'GSI UPM' author = 'GSI UPM'
@ -31,12 +31,14 @@ class MeaningCloudPlugin(SentimentPlugin):
extra_params = { extra_params = {
"language": { "language": {
"description": "language of the input",
"aliases": ["language", "l"], "aliases": ["language", "l"],
"required": True, "required": True,
"options": ["en","es","ca","it","pt","fr","auto"], "options": ["en","es","ca","it","pt","fr","auto"],
"default": "auto" "default": "auto"
}, },
"apikey":{ "apikey":{
"description": "API key for the meaningcloud service. See https://www.meaningcloud.com/developer/login",
"aliases": ["apiKey", "meaningcloud-key", "meaningcloud-apikey"], "aliases": ["apiKey", "meaningcloud-key", "meaningcloud-apikey"],
"required": True "required": True
} }
@ -55,7 +57,8 @@ class MeaningCloudPlugin(SentimentPlugin):
polarityValue = 1 polarityValue = 1
return polarity, polarityValue return polarity, polarityValue
def analyse_entry(self, entry, params): def analyse_entry(self, entry, activity):
params = activity.params
txt = entry['nif:isString'] txt = entry['nif:isString']
api = 'http://api.meaningcloud.com/' api = 'http://api.meaningcloud.com/'
@ -129,7 +132,9 @@ class MeaningCloudPlugin(SentimentPlugin):
sementity = sent_entity['sementity'].get('type', None).split(">")[-1] sementity = sent_entity['sementity'].get('type', None).split(">")[-1]
entity['@type'] = "ODENTITY_{}".format(sementity) entity['@type'] = "ODENTITY_{}".format(sementity)
entity.prov(self) entity.prov(self)
entry.entities.append(entity) if 'senpy:hasEntity' not in entry:
entry['senpy:hasEntity'] = []
entry['senpy:hasEntity'].append(entity)
for topic in api_response_topics['concept_list']: for topic in api_response_topics['concept_list']:
if 'semtheme_list' in topic: if 'semtheme_list' in topic:
@ -139,7 +144,9 @@ class MeaningCloudPlugin(SentimentPlugin):
concept['@type'] = "ODTHEME_{}".format(theme['type'].split(">")[-1]) concept['@type'] = "ODTHEME_{}".format(theme['type'].split(">")[-1])
concept['fam:topic-reference'] = "http://dbpedia.org/resource/{}".format(theme['type'].split('>')[-1]) concept['fam:topic-reference'] = "http://dbpedia.org/resource/{}".format(theme['type'].split('>')[-1])
entry.prov(self) entry.prov(self)
entry.topics.append(concept) if 'senpy:hasTopic' not in entry:
entry['senpy:hasTopic'] = []
entry['senpy:hasTopic'].append(concept)
yield entry yield entry
test_cases = [ test_cases = [
@ -160,11 +167,11 @@ class MeaningCloudPlugin(SentimentPlugin):
}, },
'input': 'Hello World Obama', 'input': 'Hello World Obama',
'expected': { 'expected': {
'sentiments': [ 'marl:hasOpinion': [
{'marl:hasPolarity': 'marl:Neutral'}], {'marl:hasPolarity': 'marl:Neutral'}],
'entities': [ 'senpy:hasEntity': [
{'itsrdf:taIdentRef': 'http://dbpedia.org/resource/Obama'}], {'itsrdf:taIdentRef': 'http://dbpedia.org/resource/Obama'}],
'topics': [ 'senpy:hasTopic': [
{'fam:topic-reference': 'http://dbpedia.org/resource/Astronomy'}] {'fam:topic-reference': 'http://dbpedia.org/resource/Astronomy'}]
}, },
'responses': [ 'responses': [

9
sentiment-taiger/taiger3c_plugin.py
View File

@ -43,7 +43,8 @@ class TaigerPlugin3cats(SentimentPlugin):
raise ValueError('unknown polarity: {}'.format(value)) raise ValueError('unknown polarity: {}'.format(value))
return polarity, value return polarity, value
def analyse_entry(self, entry, params): def analyse_entry(self, entry, activity):
params = activity.params
txt = entry['nif:isString'] txt = entry['nif:isString']
api = TAIGER_ENDPOINT api = TAIGER_ENDPOINT
@ -89,7 +90,7 @@ class TaigerPlugin3cats(SentimentPlugin):
}, },
'input': 'I hate to say this', 'input': 'I hate to say this',
'expected': { 'expected': {
'sentiments': [ 'marl:hasOpinion': [
{'marl:hasPolarity': 'marl:Negative'}], {'marl:hasPolarity': 'marl:Negative'}],
}, },
'responses': [ 'responses': [
@ -116,7 +117,7 @@ class TaigerPlugin3cats(SentimentPlugin):
}, },
'input': 'This is amazing', 'input': 'This is amazing',
'expected': { 'expected': {
'sentiments': [ 'marl:hasOpinion': [
{'marl:hasPolarity': 'marl:Positive'}], {'marl:hasPolarity': 'marl:Positive'}],
}, },
'responses': [ 'responses': [
@ -143,7 +144,7 @@ class TaigerPlugin3cats(SentimentPlugin):
}, },
'input': 'The pillow is in the wardrobe', 'input': 'The pillow is in the wardrobe',
'expected': { 'expected': {
'sentiments': [ 'marl:hasOpinion': [
{'marl:hasPolarity': 'marl:Neutral'}], {'marl:hasPolarity': 'marl:Neutral'}],
}, },
'responses': [ 'responses': [

12
sentiment-taiger/taiger_plugin.py
View File

@ -38,7 +38,8 @@ class TaigerPlugin(SentimentPlugin):
polarity = 'marl:Positive' polarity = 'marl:Positive'
return polarity return polarity
def analyse_entry(self, entry, params): def analyse_entry(self, entry, activity):
params = activity.params
txt = entry['nif:isString'] txt = entry['nif:isString']
api = TAIGER_ENDPOINT api = TAIGER_ENDPOINT
@ -77,7 +78,6 @@ class TaigerPlugin(SentimentPlugin):
'expanded-jsonld': 0, 'expanded-jsonld': 0,
'informat': 'text', 'informat': 'text',
'prefix': '', 'prefix': '',
'plugin_type': 'analysisPlugin',
'urischeme': 'RFC5147String', 'urischeme': 'RFC5147String',
'outformat': 'json-ld', 'outformat': 'json-ld',
'conversion': 'full', 'conversion': 'full',
@ -87,7 +87,7 @@ class TaigerPlugin(SentimentPlugin):
}, },
'input': 'I hate to say this', 'input': 'I hate to say this',
'expected': { 'expected': {
'sentiments': [ 'marl:hasOpinion': [
{'marl:hasPolarity': 'marl:Negative'}], {'marl:hasPolarity': 'marl:Negative'}],
}, },
'responses': [ 'responses': [
@ -109,7 +109,6 @@ class TaigerPlugin(SentimentPlugin):
'expanded-jsonld': 0, 'expanded-jsonld': 0,
'informat': 'text', 'informat': 'text',
'prefix': '', 'prefix': '',
'plugin_type': 'analysisPlugin',
'urischeme': 'RFC5147String', 'urischeme': 'RFC5147String',
'outformat': 'json-ld', 'outformat': 'json-ld',
'conversion': 'full', 'conversion': 'full',
@ -119,7 +118,7 @@ class TaigerPlugin(SentimentPlugin):
}, },
'input': 'This is amazing', 'input': 'This is amazing',
'expected': { 'expected': {
'sentiments': [ 'marl:hasOpinion': [
{'marl:hasPolarity': 'marl:Positive'}], {'marl:hasPolarity': 'marl:Positive'}],
}, },
'responses': [ 'responses': [
@ -141,7 +140,6 @@ class TaigerPlugin(SentimentPlugin):
'expanded-jsonld': 0, 'expanded-jsonld': 0,
'informat': 'text', 'informat': 'text',
'prefix': '', 'prefix': '',
'plugin_type': 'analysisPlugin',
'urischeme': 'RFC5147String', 'urischeme': 'RFC5147String',
'outformat': 'json-ld', 'outformat': 'json-ld',
'conversion': 'full', 'conversion': 'full',
@ -151,7 +149,7 @@ class TaigerPlugin(SentimentPlugin):
}, },
'input': 'The pillow is in the wardrobe', 'input': 'The pillow is in the wardrobe',
'expected': { 'expected': {
'sentiments': [ 'marl:hasOpinion': [
{'marl:hasPolarity': 'marl:Neutral'}], {'marl:hasPolarity': 'marl:Neutral'}],
}, },
'responses': [ 'responses': [

56
sentiment-vader/vader_plugin.py
View File

@ -1,12 +1,12 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from vaderSentiment import sentiment from vaderSentiment import sentiment
from senpy.plugins import SentimentPlugin, SenpyPlugin from senpy.plugins import SentimentBox, SenpyPlugin
from senpy.models import Results, Sentiment, Entry from senpy.models import Results, Sentiment, Entry
import logging import logging
class VaderSentimentPlugin(SentimentPlugin): class VaderSentimentPlugin(SentimentBox):
''' '''
Sentiment classifier using vaderSentiment module. Params accepted: Language: {en, es}. The output uses Marl ontology developed at GSI UPM for semantic web. Sentiment classifier using vaderSentiment module. Params accepted: Language: {en, es}. The output uses Marl ontology developed at GSI UPM for semantic web.
''' '''
@ -16,6 +16,7 @@ class VaderSentimentPlugin(SentimentPlugin):
version = "0.1.1" version = "0.1.1"
extra_params = { extra_params = {
"language": { "language": {
"description": "language of the input",
"@id": "lang_rand", "@id": "lang_rand",
"aliases": ["language", "l"], "aliases": ["language", "l"],
"default": "auto", "default": "auto",
@ -23,55 +24,32 @@ class VaderSentimentPlugin(SentimentPlugin):
}, },
"aggregate": { "aggregate": {
"description": "Show only the strongest sentiment (aggregate) or all sentiments",
"aliases": ["aggregate","agg"], "aliases": ["aggregate","agg"],
"options": ["true", "false"], "options": [True, False],
"default": False "default": False
} }
} }
requirements = {} requirements = {}
def analyse_entry(self, entry, params): _VADER_KEYS = ['pos', 'neu', 'neg']
binary = False
self.log.debug("Analysing with params {}".format(params))
text_input = entry.text def predict_one(self, features, activity):
aggregate = params['aggregate'] text_input = ' '.join(features)
scores = sentiment(text_input)
score = sentiment(text_input) sentiments = []
for k in self._VADER_KEYS:
sentiments.append(scores[k])
opinion0 = Sentiment(id= "Opinion_positive", if activity.param('aggregate'):
marl__hasPolarity= "marl:Positive", m = max(sentiments)
marl__algorithmConfidence= score['pos'] sentiments = [k if k==m else None for k in sentiments]
)
opinion0.prov(self)
opinion1 = Sentiment(id= "Opinion_negative",
marl__hasPolarity= "marl:Negative",
marl__algorithmConfidence= score['neg']
)
opinion1.prov(self)
opinion2 = Sentiment(id= "Opinion_neutral",
marl__hasPolarity = "marl:Neutral",
marl__algorithmConfidence = score['neu']
)
opinion2.prov(self)
if aggregate == 'true': return sentiments
res = None
confident = max(score['neg'],score['neu'],score['pos'])
if opinion0.marl__algorithmConfidence == confident:
res = opinion0
elif opinion1.marl__algorithmConfidence == confident:
res = opinion1
elif opinion2.marl__algorithmConfidence == confident:
res = opinion2
entry.sentiments.append(res)
else:
entry.sentiments.append(opinion0)
entry.sentiments.append(opinion1)
entry.sentiments.append(opinion2)
yield entry
test_cases = [] test_cases = []