balkian/sitc
1
0
Fork 0
mirror of https://github.com/gsi-upm/sitc synced 2025-08-24 02:22:21 +00:00
Code Issues Releases Wiki Activity

Remove outputs and metadata

Browse Source
This commit is contained in:
J. Fernando Sánchez
2019-02-28 15:30:33 +01:00
parent a1be167cc0
commit c1d3ca38ea
25 changed files with 989 additions and 14268 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
nlp/4_1_Lexical_Processing.ipynb
View File

@@ -68,9 +68,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"metadata": {},
"outputs": [],
"source": [
"review = \"\"\"I purchased this monitor because of budgetary concerns. This item was the most inexpensive 17 inch monitor \n",
@@ -111,9 +109,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"metadata": {},
"outputs": [],
"source": [
"import nltk\n",
@@ -171,9 +167,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"metadata": {},
"outputs": [],
"source": [
"from nltk.tokenize import sent_tokenize, word_tokenize\n",
@@ -199,10 +193,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"scrolled": true
},
"metadata": {},
"outputs": [],
"source": [
"words = [word_tokenize(t) for t in sent_tokenize(review)]\n",
@@ -219,9 +210,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"metadata": {},
"outputs": [],
"source": [
"words = word_tokenize(review)\n",
@@ -239,9 +228,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"metadata": {},
"outputs": [],
"source": [
"from nltk.tokenize import TweetTokenizer\n",
@@ -268,9 +255,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"metadata": {},
"outputs": [],
"source": [
"from nltk.stem import PorterStemmer, LancasterStemmer, WordNetLemmatizer\n",
@@ -304,9 +289,7 @@
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"metadata": {},
"source": [
"As we can see, we get the forms *are* and *is* instead of *be*. This is because we have not introduce the Part-Of-Speech (POS), and the default POS is 'n' (name).\n",
"\n",
@@ -316,9 +299,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"metadata": {},
"outputs": [],
"source": [
"verbs = \"are crying is have has\"\n",
@@ -327,9 +308,7 @@
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"metadata": {},
"source": [
"Depending of the application, we can select stemmers or lemmatizers. \n",
"\n",
@@ -341,9 +320,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"metadata": {},
"outputs": [],
"source": [
"def preprocess(words, type='doc'):\n",
@@ -376,9 +353,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"metadata": {},
"outputs": [],
"source": [
"from nltk.corpus import stopwords\n",
@@ -390,9 +365,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"metadata": {},
"outputs": [],
"source": [
"def preprocess(words, type='doc'):\n",
@@ -428,9 +401,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"metadata": {},
"outputs": [],
"source": [
"import string\n",
@@ -474,9 +445,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"metadata": {},
"outputs": [],
"source": [
"frec = nltk.FreqDist(nltk.word_tokenize(review))\n",