adapt deployment

.gitlab-ci.yml

@@ -12,6 +12,7 @@ stages:
 variables:
   KUBENS: senpy
   LATEST_IMAGE: "${HUB_REPO}:${CI_COMMIT_SHORT_SHA}"
+  SENPY_DATA: "/senpy-data/" # This is configured in the CI job
 
 docker:
   stage: publish

CHANGELOG.md

@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 ### Added
 * The code of many senpy community plugins have been included by default. However, additional files (e.g., licensed data) and/or installing additional dependencies may be necessary for some plugins. Read each plugin's documentation for more information.
+* `--strict` flag, to fail and not start when a 
+* `optional` attribute in plugins. Optional plugins may fail to load or activate but the server will be started regardless, unless running in strict mode
+* Option in shelf plugins to ignore pickling errors
+### Removed
+* `--only-install`, `--only-test` and `--only-list` flags were removed in favor of `--no-run` + `--install`/`--test`/`--dependencies`
+### Changed
+* data directory selection logic is slightly modified, and will choose one of the following (in this order): `data_folder` (argument), `$SENPY_DATA` or `$CWD`
 
 ## [1.0.6]
 ### Fixed

k8s/senpy-deployment.yaml.tmpl

@@ -26,3 +26,11 @@ spec:
         ports:
           - name: web
             containerPort: 5000
+        volumeMounts:
+          - name: senpy-data
+            mountPath: /senpy-data
+            subpath: data
+      volumes:
+        - name: senpy-data
+          persistentVolumeClaim:
+            claimName: pvc-senpy

senpy/extensions.py

@@ -365,9 +365,10 @@ class Senpy(object):
     def _activate(self, plugin):
         with plugin._lock:
             if plugin.is_activated:
-                logger.info(f"Plugin is already activated: {plugin.name}")
                 return
             try:
+                logger.info("Activating plugin: {}".format(plugin.name))
+
                 assert plugin._activate()
                 logger.info(f"Plugin activated: {plugin.name}")
             except Exception as ex:

senpy/plugins/__init__.py

@@ -122,7 +122,13 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)):
         self._directory = os.path.abspath(
             os.path.dirname(inspect.getfile(self.__class__)))
 
-        data_folder = data_folder or os.getcwd()
+        if not data_folder:
+            data_folder = os.environ['SENPY_DATA']
+        if not data_folder:
+            data_folder = os.getcwd()
+
+
+        data_folder = os.path.abspath(data_folder)
         subdir = os.path.join(data_folder, self.name)
 
         self._data_paths = [
@@ -652,11 +658,17 @@ class ShelfMixin(object):
     def shelf_file(self, value):
         self._shelf_file = value
 
-    def save(self):
+    def save(self, ignore_errors=False):
-        self.log.debug('Saving pickle')
+        try:
-        if hasattr(self, '_sh') and self._sh is not None:
+            self.log.debug('Saving pickle')
-            with self.open(self.shelf_file, 'wb') as f:
+            if hasattr(self, '_sh') and self._sh is not None:
-                pickle.dump(self._sh, f)
+                with self.open(self.shelf_file, 'wb') as f:
+                    pickle.dump(self._sh, f)
+        except Exception as ex:
+            self.log.warning("Could not save shelf state. Check folder permissions for: "
+                             f" {self.shelf_file}. Error: { ex }")
+            if not ignore_errors:
+                raise
 
 
 def pfilter(plugins, plugin_type=Analyser, **kwargs):

senpy/plugins/emotion/depechemood_plugin.py

@@ -175,8 +175,5 @@ class DepecheMood(EmotionBox):
 
 
 if __name__ == '__main__':
-    from senpy.utils import easy, easy_load, easy_test
+    from senpy.utils import easy_test
-    # sp, app = easy_load()
-    # for plug in sp.analysis_plugins:
-    #     plug.test()
     easy_test(debug=False)

senpy/plugins/emotion/wnaffect/emotion-wnaffect.py

@@ -117,7 +117,7 @@ class WNAffect(EmotionPlugin, ShelfMixin):
 
 
     def deactivate(self, *args, **kwargs):
-        self.save()
+        self.save(ignore_errors=True)
 
     def _my_preprocessor(self, text):