balkian/sitc
1
0
Fork 0
mirror of https://github.com/gsi-upm/sitc synced 2025-08-23 10:12:20 +00:00
Code Issues Releases Wiki Activity

Merge pull request #5 from gsi-upm/dveni-patch-2

Browse Source 
Update 4_1_Lexical_Processing.ipynb
This commit is contained in:
Carlos A. Iglesias
2019-11-27 10:19:12 +01:00
committed by GitHub
parent e70689072f 19ea5dff09
commit 75f08ea170
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
nlp/4_1_Lexical_Processing.ipynb
View File

@@ -326,7 +326,7 @@
"def preprocess(words, type='doc'):\n",
" if (type == 'tweet'):\n",
" tknzr = TweetTokenizer(strip_handles=True, reduce_len=True)\n",
" tokens = tknzr.tokenize(tweet)\n",
" tokens = tknzr.tokenize(words)\n",
" else:\n",
" tokens = nltk.word_tokenize(words.lower())\n",
" porter = nltk.PorterStemmer()\n",