Change name to split, according to issue #37

.gitlab-ci.yml

@@ -98,7 +98,7 @@ clean :
     - make -e clean
   when: manual
 
-cleanup_pypirc:
+cleanup_py:
    stage: clean
    when: always   # this is important; run even if preceding stages failed.
    script:

Makefile

@@ -76,14 +76,12 @@ test-%:
 test: test-$(PYMAIN)
 
 dist/$(TARNAME): version
-	docker run --rm -v $$PWD:/usr/src/app/ -w /usr/src/app/ python:$(PYMAIN) python setup.py sdist;
+	python setup.py sdist;
-	docker run --rm -v $$PWD:/usr/src/app/ -w /usr/src/app/ python:$(PYMAIN) chmod -R a+rwx dist;
-
 
 sdist: dist/$(TARNAME)
 
 pip_test-%: sdist
-	docker run --rm -v $$PWD/dist:/dist/ -ti python:$* pip install /dist/$(TARNAME);
+	docker run --rm -v $$PWD/dist:/dist/ python:$* pip install /dist/$(TARNAME);
 
 pip_test: $(addprefix pip_test-,$(PYVERSIONS))
 

senpy/plugins/split/split.py

@@ -0,0 +1,30 @@
+from senpy.plugins import AnalysisPlugin
+from senpy.models import Entry
+from nltk.tokenize.punkt import PunktSentenceTokenizer
+from nltk.tokenize.simple import LineTokenizer
+import nltk
+class SplitPlugin(AnalysisPlugin):
+    def activate(self):
+        nltk.download('punkt')
+    
+    def analyse_entry(self, entry, params):
+        chunker_type = params.get("delimiter", "sentence")
+        original_id = entry.id
+        original_text = entry.get("text", None)
+        if chunker_type == "sentence":
+            tokenizer = PunktSentenceTokenizer()
+            chars = tokenizer.span_tokenize(original_text)
+            for i, sentence in enumerate(tokenizer.tokenize(original_text)):
+                e = Entry()
+                e.text = sentence
+                e.id = original_id + "#char={},{}".format(chars[i][0], chars[i][1])
+                yield e
+        if chunker_type == "paragraph":
+            tokenizer = LineTokenizer()
+            chars = tokenizer.span_tokenize(original_text)
+            for i, paragraph in enumerate(tokenizer.tokenize(original_text)):
+                e = Entry() 
+                e.text = paragraph
+                chars = [char for char in chars]
+                e.id = original_id + "#char={},{}".format(chars[i][0], chars[i][1])
+                yield e

senpy/plugins/split/split.senpy

@@ -0,0 +1,18 @@
+---
+name: split
+module: split
+description: A sample plugin that chunks input text
+author: "@militarpancho"
+version: '0.1'
+url: "https://github.com/gsi-upm/senpy"
+requirements: {nltk}
+extra_params:
+  delimiter:
+    aliases:
+    - type
+    - t
+    required: false
+    default: sentence
+    options:
+    - sentence
+    - paragraph