|
|
@ -326,7 +326,7 @@
|
|
|
|
"def preprocess(words, type='doc'):\n",
|
|
|
|
"def preprocess(words, type='doc'):\n",
|
|
|
|
" if (type == 'tweet'):\n",
|
|
|
|
" if (type == 'tweet'):\n",
|
|
|
|
" tknzr = TweetTokenizer(strip_handles=True, reduce_len=True)\n",
|
|
|
|
" tknzr = TweetTokenizer(strip_handles=True, reduce_len=True)\n",
|
|
|
|
" tokens = tknzr.tokenize(tweet)\n",
|
|
|
|
" tokens = tknzr.tokenize(words)\n",
|
|
|
|
" else:\n",
|
|
|
|
" else:\n",
|
|
|
|
" tokens = nltk.word_tokenize(words.lower())\n",
|
|
|
|
" tokens = nltk.word_tokenize(words.lower())\n",
|
|
|
|
" porter = nltk.PorterStemmer()\n",
|
|
|
|
" porter = nltk.PorterStemmer()\n",
|
|
|
|