From f76b777b9f491514cb2c38a601f23f45d89f939f Mon Sep 17 00:00:00 2001 From: Ian Wood Date: Tue, 16 May 2017 15:09:46 +0100 Subject: [PATCH 1/3] don't fail if shelf pickle file broken --- senpy/plugins/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/senpy/plugins/__init__.py b/senpy/plugins/__init__.py index b45a7f4..a02465d 100644 --- a/senpy/plugins/__init__.py +++ b/senpy/plugins/__init__.py @@ -96,7 +96,10 @@ class ShelfMixin(object): if not hasattr(self, '_sh') or self._sh is None: self.__dict__['_sh'] = {} if os.path.isfile(self.shelf_file): - self.__dict__['_sh'] = pickle.load(open(self.shelf_file, 'rb')) + try: + self.__dict__['_sh'] = pickle.load(open(self.shelf_file, 'rb')) + except EOFError: + logger.warning('corrupted shelf file!') return self._sh @sh.deleter From ea91e3e4a468ca0ec14f8a6b02bfa7c644da3842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Wed, 17 May 2017 16:25:13 +0200 Subject: [PATCH 2/3] Add an option to force the load of shelf plugins Closes gsi-upm/senpy#34 --- docs/plugins.rst | 6 +++++- senpy/models.py | 5 ++++- senpy/plugins/__init__.py | 6 ++++-- tests/test_plugins.py | 35 ++++++++++++++++++++++++++++++++++- 4 files changed, 47 insertions(+), 5 deletions(-) diff --git a/docs/plugins.rst b/docs/plugins.rst index 2954a05..decf558 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -183,7 +183,11 @@ Training a classifier can be time time consuming. To avoid running the training def deactivate(self): self.close() -You can speficy a 'shelf_file' in your .senpy file. By default the ShelfMixin creates a file based on the plugin name and stores it in that plugin's folder. +You can specify a 'shelf_file' in your .senpy file. By default the ShelfMixin creates a file based on the plugin name and stores it in that plugin's folder. + +Shelves may get corrupted if the plugin exists unexpectedly. +A corrupt shelf prevents the plugin from loading. +If you do not care about the pickle, you can force your plugin to remove the corrupted file and load anyway, set the 'force_shelf' to True in your .senpy file. I want to implement my service as a plugin, How i can do it? ???????????????????????????????????????????????????????????? diff --git a/senpy/models.py b/senpy/models.py index 2025d31..b47947f 100644 --- a/senpy/models.py +++ b/senpy/models.py @@ -237,7 +237,10 @@ class BaseModel(SenpyMixin, dict): self.__setitem__(self._get_key(key), value) def __delattr__(self, key): - self.__delitem__(self._get_key(key)) + try: + object.__delattr__(self, key) + except AttributeError: + self.__delitem__(self._get_key(key)) def _plain_dict(self): d = {k: v for (k, v) in self.items() if k[0] != "_"} diff --git a/senpy/plugins/__init__.py b/senpy/plugins/__init__.py index a02465d..517d957 100644 --- a/senpy/plugins/__init__.py +++ b/senpy/plugins/__init__.py @@ -98,8 +98,10 @@ class ShelfMixin(object): if os.path.isfile(self.shelf_file): try: self.__dict__['_sh'] = pickle.load(open(self.shelf_file, 'rb')) - except EOFError: - logger.warning('corrupted shelf file!') + except (EOFError, pickle.UnpicklingError): + logger.warning('{} has a corrupted shelf file!'.format(self.id)) + if not self.get('force_shelf', False): + raise return self._sh @sh.deleter diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 946f77e..3f874f5 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -83,7 +83,40 @@ class PluginsTest(TestCase): res2 = a.analyse(input=1) assert res2.entries[0].nif__isString == 2 - def test_two(self): + def test_corrupt_shelf(self): + ''' Reusing the values of a previous shelf ''' + + emptyfile = os.path.join(self.shelf_dir, "emptyfile") + invalidfile = os.path.join(self.shelf_dir, "invalid_file") + with open(emptyfile, 'w+b'), open(invalidfile, 'w+b') as inf: + inf.write(b'ohno') + + files = {emptyfile: ['empty file', EOFError], + invalidfile: ['invalid file', pickle.UnpicklingError]} + + for fn in files: + with open(fn, 'rb') as f: + msg, error = files[fn] + a = ShelfDummyPlugin(info={ + 'name': 'shelve', + 'version': 'test', + 'shelf_file': f.name + }) + assert os.path.isfile(a.shelf_file) + print('Shelf file: %s' % a.shelf_file) + with self.assertRaises(error): + # By default, raise an error + a.sh['a'] = 'fromA' + a.save() + del a._sh + assert os.path.isfile(a.shelf_file) + a.force_shelf = True + a.sh['a'] = 'fromA' + a.save() + b = pickle.load(f) + assert b['a'] == 'fromA' + + def test_reuse_shelf(self): ''' Reusing the values of a previous shelf ''' a = ShelfDummyPlugin(info={ 'name': 'shelve', From 0f89b92457ebfa33cb41236bdbad80f07a897e36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Wed, 17 May 2017 16:51:01 +0200 Subject: [PATCH 3/3] Fixed pickling error in py2.7 --- senpy/plugins/__init__.py | 2 +- tests/test_plugins.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/senpy/plugins/__init__.py b/senpy/plugins/__init__.py index 517d957..63bb66e 100644 --- a/senpy/plugins/__init__.py +++ b/senpy/plugins/__init__.py @@ -98,7 +98,7 @@ class ShelfMixin(object): if os.path.isfile(self.shelf_file): try: self.__dict__['_sh'] = pickle.load(open(self.shelf_file, 'rb')) - except (EOFError, pickle.UnpicklingError): + except (IndexError, EOFError, pickle.UnpicklingError): logger.warning('{} has a corrupted shelf file!'.format(self.id)) if not self.get('force_shelf', False): raise diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 3f874f5..dbfe60d 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -91,8 +91,8 @@ class PluginsTest(TestCase): with open(emptyfile, 'w+b'), open(invalidfile, 'w+b') as inf: inf.write(b'ohno') - files = {emptyfile: ['empty file', EOFError], - invalidfile: ['invalid file', pickle.UnpicklingError]} + files = {emptyfile: ['empty file', (EOFError, IndexError)], + invalidfile: ['invalid file', (pickle.UnpicklingError, IndexError)]} for fn in files: with open(fn, 'rb') as f: @@ -105,7 +105,6 @@ class PluginsTest(TestCase): assert os.path.isfile(a.shelf_file) print('Shelf file: %s' % a.shelf_file) with self.assertRaises(error): - # By default, raise an error a.sh['a'] = 'fromA' a.save() del a._sh