mirror of
https://github.com/gsi-upm/sitc
synced 2025-01-07 03:31:28 +00:00
Compare commits
2 Commits
e70689072f
...
75f08ea170
Author | SHA1 | Date | |
---|---|---|---|
|
75f08ea170 | ||
|
19ea5dff09 |
@ -326,7 +326,7 @@
|
||||
"def preprocess(words, type='doc'):\n",
|
||||
" if (type == 'tweet'):\n",
|
||||
" tknzr = TweetTokenizer(strip_handles=True, reduce_len=True)\n",
|
||||
" tokens = tknzr.tokenize(tweet)\n",
|
||||
" tokens = tknzr.tokenize(words)\n",
|
||||
" else:\n",
|
||||
" tokens = nltk.word_tokenize(words.lower())\n",
|
||||
" porter = nltk.PorterStemmer()\n",
|
||||
|
Loading…
Reference in New Issue
Block a user