balkian/senpy
1
0
Fork 0
mirror of https://github.com/gsi-upm/senpy synced 2024-11-23 08:32:29 +00:00
Code Issues Releases Wiki Activity

Avoid duplication in split plugin

Browse Source
This commit is contained in:
J. Fernando Sánchez 2018-08-20 14:07:33 +02:00
parent 6dd4a44924
commit aa35e62a27
1 changed files with 8 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
senpy/plugins/misc/split_plugin.py
View File

@ -9,7 +9,7 @@ class Split(AnalysisPlugin):
'''description: A sample plugin that chunks input text'''
author = ["@militarpancho", '@balkian']
version = '0.2'
version = '0.3'
url = "https://github.com/gsi-upm/senpy"
extra_params = {
@ -33,12 +33,15 @@ class Split(AnalysisPlugin):
if chunker_type == "paragraph":
tokenizer = LineTokenizer()
chars = list(tokenizer.span_tokenize(original_text))
for i, chunk in enumerate(tokenizer.tokenize(original_text)):
print(chunk)
if len(chars) == 1:
# This sentence was already split
return
for i, chunk in enumerate(chars):
start, end = chunk
e = Entry()
e['nif:isString'] = chunk
e['nif:isString'] = original_text[start:end]
if entry.id:
e.id = entry.id + "#char={},{}".format(chars[i][0], chars[i][1])
e.id = entry.id + "#char={},{}".format(start, end)
yield e
test_cases = [