@ -18,6 +18,7 @@ import subprocess
import importlib
import importlib
import yaml
import yaml
import threading
import threading
import nltk
from . . import models , utils
from . . import models , utils
from . . import api
from . . import api
@ -95,6 +96,16 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)):
self . is_activated = False
self . is_activated = False
self . _lock = threading . Lock ( )
self . _lock = threading . Lock ( )
self . data_folder = data_folder or os . getcwd ( )
self . data_folder = data_folder or os . getcwd ( )
self . _directory = os . path . abspath ( os . path . dirname ( inspect . getfile ( self . __class__ ) ) )
self . _data_paths = [ ' ' ,
self . _directory ,
os . path . join ( self . _directory , ' data ' ) ,
self . data_folder ]
self . _log = logging . getLogger ( ' {} . {} ' . format ( __name__ , self . name ) )
@property
def log ( self ) :
return self . _log
def validate ( self ) :
def validate ( self ) :
missing = [ ]
missing = [ ]
@ -123,9 +134,9 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)):
for case in test_cases :
for case in test_cases :
try :
try :
self . test_case ( case )
self . test_case ( case )
log ger . debug ( ' Test case passed: \n {} ' . format ( pprint . pformat ( case ) ) )
self . log . debug ( ' Test case passed: \n {} ' . format ( pprint . pformat ( case ) ) )
except Exception as ex :
except Exception as ex :
log ger . warn ( ' Test case failed: \n {} ' . format ( pprint . pformat ( case ) ) )
self . log . warn ( ' Test case failed: \n {} ' . format ( pprint . pformat ( case ) ) )
raise
raise
def test_case ( self , case ) :
def test_case ( self , case ) :
@ -148,10 +159,22 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)):
raise
raise
assert not should_fail
assert not should_fail
def open ( self , fpath , * args , * * kwargs ) :
def find_file ( self , fname ) :
for p in self . _data_paths :
alternative = os . path . join ( p , fname )
if os . path . exists ( alternative ) :
return alternative
raise IOError ( ' File does not exist: {} ' . format ( fname ) )
def open ( self , fpath , mode = ' r ' ) :
if ' w ' in mode :
# When writing, only use absolute paths or data_folder
if not os . path . isabs ( fpath ) :
if not os . path . isabs ( fpath ) :
fpath = os . path . join ( self . data_folder , fpath )
fpath = os . path . join ( self . data_folder , fpath )
return open ( fpath , * args , * * kwargs )
else :
fpath = self . find_file ( fpath )
return open ( fpath , mode = mode )
def serve ( self , debug = True , * * kwargs ) :
def serve ( self , debug = True , * * kwargs ) :
utils . easy ( plugin_list = [ self , ] , plugin_folder = None , debug = debug , * * kwargs )
utils . easy ( plugin_list = [ self , ] , plugin_folder = None , debug = debug , * * kwargs )
@ -186,7 +209,7 @@ class Analysis(Plugin):
def analyse_entries ( self , entries , parameters ) :
def analyse_entries ( self , entries , parameters ) :
for entry in entries :
for entry in entries :
log ger . debug ( ' Analysing entry with plugin {} : {} ' . format ( self , entry ) )
self . log . debug ( ' Analysing entry with plugin {} : {} ' . format ( self , entry ) )
results = self . analyse_entry ( entry , parameters )
results = self . analyse_entry ( entry , parameters )
if inspect . isgenerator ( results ) :
if inspect . isgenerator ( results ) :
for result in results :
for result in results :
@ -375,7 +398,7 @@ class ShelfMixin(object):
with self . open ( self . shelf_file , ' rb ' ) as p :
with self . open ( self . shelf_file , ' rb ' ) as p :
self . _sh = pickle . load ( p )
self . _sh = pickle . load ( p )
except ( IndexError , EOFError , pickle . UnpicklingError ) :
except ( IndexError , EOFError , pickle . UnpicklingError ) :
log ger . warning ( ' {} has a corrupted shelf file! ' . format ( self . id ) )
self . log . warning ( ' Corrupted shelf file: {} ' . format ( self . shelf_file ) )
if not self . get ( ' force_shelf ' , False ) :
if not self . get ( ' force_shelf ' , False ) :
raise
raise
return self . _sh
return self . _sh
@ -402,32 +425,31 @@ class ShelfMixin(object):
self . _shelf_file = value
self . _shelf_file = value
def save ( self ) :
def save ( self ) :
log ger . debug ( ' s aving pickle' )
self . log . debug ( ' S aving pickle' )
if hasattr ( self , ' _sh ' ) and self . _sh is not None :
if hasattr ( self , ' _sh ' ) and self . _sh is not None :
with self . open ( self . shelf_file , ' wb ' ) as f :
with self . open ( self . shelf_file , ' wb ' ) as f :
pickle . dump ( self . _sh , f )
pickle . dump ( self . _sh , f )
def pfilter ( plugins , * * kwargs ) :
def pfilter ( plugins , plugin_type = Analysis , * * kwargs ) :
""" Filter plugins by different criteria """
""" Filter plugins by different criteria """
if isinstance ( plugins , models . Plugins ) :
if isinstance ( plugins , models . Plugins ) :
plugins = plugins . plugins
plugins = plugins . plugins
elif isinstance ( plugins , dict ) :
elif isinstance ( plugins , dict ) :
plugins = plugins . values ( )
plugins = plugins . values ( )
ptype = kwargs . pop ( ' plugin_type ' , Plugin )
logger . debug ( ' # ' * 100 )
logger . debug ( ' # ' * 100 )
logger . debug ( ' p type {} ' . format ( p type) )
logger . debug ( ' p lugin_ type {} ' . format ( p lugin_ type) )
if p type:
if p lugin_ type:
if isinstance ( p type, PluginMeta ) :
if isinstance ( p lugin_ type, PluginMeta ) :
p type = p type. __name__
p lugin_ type = p lugin_ type. __name__
try :
try :
p type = p type[ 0 ] . upper ( ) + p type[ 1 : ]
p lugin_ type = p lugin_ type[ 0 ] . upper ( ) + p lugin_ type[ 1 : ]
pclass = globals ( ) [ p type]
pclass = globals ( ) [ p lugin_ type]
logger . debug ( ' Class: {} ' . format ( pclass ) )
logger . debug ( ' Class: {} ' . format ( pclass ) )
candidates = filter ( lambda x : isinstance ( x , pclass ) ,
candidates = filter ( lambda x : isinstance ( x , pclass ) ,
plugins )
plugins )
except KeyError :
except KeyError :
raise models . Error ( ' {} is not a valid type ' . format ( p type) )
raise models . Error ( ' {} is not a valid type ' . format ( p lugin_ type) )
else :
else :
candidates = plugins
candidates = plugins
@ -462,6 +484,7 @@ def _log_subprocess_output(process):
def install_deps ( * plugins ) :
def install_deps ( * plugins ) :
installed = False
installed = False
nltk_resources = set ( )
for info in plugins :
for info in plugins :
requirements = info . get ( ' requirements ' , [ ] )
requirements = info . get ( ' requirements ' , [ ] )
if requirements :
if requirements :
@ -477,6 +500,9 @@ def install_deps(*plugins):
installed = True
installed = True
if exitcode != 0 :
if exitcode != 0 :
raise models . Error ( " Dependencies not properly installed " )
raise models . Error ( " Dependencies not properly installed " )
nltk_resources | = set ( info . get ( ' nltk_resources ' , [ ] ) )
installed | = nltk . download ( list ( nltk_resources ) )
return installed
return installed
@ -573,12 +599,14 @@ def _instances_in_module(module):
def _from_module_name ( module , root , info = None , install = True , * * kwargs ) :
def _from_module_name ( module , root , info = None , install = True , * * kwargs ) :
try :
try :
module = load_module ( module , root )
module = load_module ( module , root )
except ImportError :
except ( ImportError , LookupError ) :
if not install or not info :
if not install or not info :
raise
raise
install_deps ( info )
install_deps ( info )
module = load_module ( module , root )
module = load_module ( module , root )
for plugin in _from_loaded_module ( module = module , root = root , info = info , * * kwargs ) :
for plugin in _from_loaded_module ( module = module , root = root , info = info , * * kwargs ) :
if install :
install_deps ( plugin )
yield plugin
yield plugin