diff --git a/your-code/Q1.ipynb b/your-code/.ipynb_checkpoints/Q1 - Miguel-checkpoint.ipynb similarity index 99% rename from your-code/Q1.ipynb rename to your-code/.ipynb_checkpoints/Q1 - Miguel-checkpoint.ipynb index 8b07d3d..7c2c26c 100644 --- a/your-code/Q1.ipynb +++ b/your-code/.ipynb_checkpoints/Q1 - Miguel-checkpoint.ipynb @@ -19,7 +19,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -170,7 +170,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.6" + "version": "3.7.3" } }, "nbformat": 4, diff --git a/your-code/Q1 - Miguel.ipynb b/your-code/Q1 - Miguel.ipynb new file mode 100644 index 0000000..58ed918 --- /dev/null +++ b/your-code/Q1 - Miguel.ipynb @@ -0,0 +1,226 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the cell below, create a Python function that wraps your previous solution for the Bag of Words lab.\n", + "\n", + "Requirements:\n", + "\n", + "1. Your function should accept the following parameters:\n", + " * `docs` [REQUIRED] - array of document paths.\n", + " * `stop_words` [OPTIONAL] - array of stop words. The default value is an empty array.\n", + "\n", + "1. Your function should return a Python object that contains the following:\n", + " * `bag_of_words` - array of strings of normalized unique words in the corpus.\n", + " * `term_freq` - array of the term-frequency vectors." + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'bag_of_words': ['ironhack', 'is', 'cool', 'i', 'love', 'am', 'a', 'student', 'at'], 'term_freq': [[1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1], [1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1], [1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1]]}\n" + ] + } + ], + "source": [ + "# Import required libraries\n", + "import os\n", + "import re\n", + "\n", + "docs = ['doc1.txt', 'doc2.txt', 'doc3.txt']\n", + "stop_words = ['all', 'six', 'less', 'being', 'indeed', 'over', 'move', 'anyway', 'fifty', 'four', 'not', 'own', 'through', 'yourselves', 'go', 'where', 'mill', 'only', 'find', 'before', 'one', 'whose', 'system', 'how', 'somewhere', 'with', 'thick', 'show', 'had', 'enough', 'should', 'to', 'must', 'whom', 'seeming', 'under', 'ours', 'has', 'might', 'thereafter', 'latterly', 'do', 'them', 'his', 'around', 'than', 'get', 'very', 'de', 'none', 'cannot', 'every', 'whether', 'they', 'front', 'during', 'thus', 'now', 'him', 'nor', 'name', 'several', 'hereafter', 'always', 'who', 'cry', 'whither', 'this', 'someone', 'either', 'each', 'become', 'thereupon', 'sometime', 'side', 'two', 'therein', 'twelve', 'because', 'often', 'ten', 'our', 'eg', 'some', 'back', 'up', 'namely', 'towards', 'are', 'further', 'beyond', 'ourselves', 'yet', 'out', 'even', 'will', 'what', 'still', 'for', 'bottom', 'mine', 'since', 'please', 'forty', 'per', 'its', 'everything', 'behind', 'un', 'above', 'between', 'it', 'neither', 'seemed', 'ever', 'across', 'she', 'somehow', 'be', 'we', 'full', 'never', 'sixty', 'however', 'here', 'otherwise', 'were', 'whereupon', 'nowhere', 'although', 'found', 'alone', 're', 'along', 'fifteen', 'by', 'both', 'about', 'last', 'would', 'anything', 'via', 'many', 'could', 'thence', 'put', 'against', 'keep', 'etc', 'amount', 'became', 'ltd', 'hence', 'onto', 'or', 'con', 'among', 'already', 'co', 'afterwards', 'formerly', 'within', 'seems', 'into', 'others', 'while', 'whatever', 'except', 'down', 'hers', 'everyone', 'done', 'least', 'another', 'whoever', 'moreover', 'couldnt', 'throughout', 'anyhow', 'yourself', 'three', 'from', 'her', 'few', 'together', 'top', 'there', 'due', 'been', 'next', 'anyone', 'eleven', 'much', 'call', 'therefore', 'interest', 'then', 'thru', 'themselves', 'hundred', 'was', 'sincere', 'empty', 'more', 'himself', 'elsewhere', 'mostly', 'on', 'fire', 'am', 'becoming', 'hereby', 'amongst', 'else', 'part', 'everywhere', 'too', 'herself', 'former', 'those', 'he', 'me', 'myself', 'made', 'twenty', 'these', 'bill', 'cant', 'us', 'until', 'besides', 'nevertheless', 'below', 'anywhere', 'nine', 'can', 'of', 'your', 'toward', 'my', 'something', 'and', 'whereafter', 'whenever', 'give', 'almost', 'wherever', 'is', 'describe', 'beforehand', 'herein', 'an', 'as', 'itself', 'at', 'have', 'in', 'seem', 'whence', 'ie', 'any', 'fill', 'again', 'hasnt', 'inc', 'thereby', 'thin', 'no', 'perhaps', 'latter', 'meanwhile', 'when', 'detail', 'same', 'wherein', 'beside', 'also', 'that', 'other', 'take', 'which', 'becomes', 'you', 'if', 'nobody', 'see', 'though', 'may', 'after', 'upon', 'most', 'hereupon', 'eight', 'but', 'serious', 'nothing', 'such', 'why', 'a', 'off', 'whereby', 'third', 'i', 'whole', 'noone', 'sometimes', 'well', 'amoungst', 'yours', 'their', 'rather', 'without', 'so', 'five', 'the', 'first', 'whereas', 'once']\n", + "\n", + "# Define function\n", + "def get_bow_from_docs(docs, stop_words=[]):\n", + " corpus = []\n", + " bag_of_words = []\n", + " term_freq = []\n", + " for i in docs:\n", + " corpus.append(open(i,\"r\").read().lower().strip('.'))\n", + " corpus2 = ' '.join(corpus).split()\n", + " for i in corpus2:\n", + " if i not in bag_of_words and i not in stop_words:\n", + " bag_of_words.append(i)\n", + "\n", + " for i in corpus:\n", + " corpus3 = []\n", + " if stop_words == []:\n", + " for word in corpus2:\n", + " #term_freq.append(bag_of_words.count(i))\n", + " corpus3.append(i.split(' ').count(word))\n", + " else:\n", + " for word in bag_of_words:\n", + " corpus3.append(i.count(word))\n", + " term_freq.append(corpus3)\n", + "\n", + "#print(corpus) # Todas las oraciones\n", + "#print(corpus2) #Todas las palabras de las oraciones\n", + "#print(corpus3) #\n", + "#print(bag_of_words) #Todas las palabras sin stop_words\n", + "#print(term_freq) #Conteo de palabras en \n", + "\n", + "# In the function, first define the variables you will use such as `corpus`, `bag_of_words`, and `term_freq`.\n", + " #\"\"\"\n", + " #Loop `corpus` again. For each doc string, count the number of occurrences of each term in `bag_of_words`. \n", + " #Create an array for each doc's term frequency and append it to `term_freq`.\n", + " #\"\"\"\n", + " \n", + " # Now return your output as an object\n", + " return {\n", + " \"bag_of_words\": bag_of_words,\n", + " \"term_freq\": term_freq\n", + " }\n", + "print(get_bow_from_docs(docs,))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Test your function without stop words. You should see the output like below:\n", + "\n", + "```{'bag_of_words': ['ironhack', 'is', 'cool', 'i', 'love', 'am', 'a', 'student', 'at'], 'term_freq': [[1, 1, 1, 0, 0, 0, 0, 0, 0], [1, 0, 0, 1, 1, 0, 0, 0, 0], [1, 0, 0, 1, 0, 1, 1, 1, 1]]}```" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'bag_of_words': ['ironhack', 'is', 'cool', 'i', 'love', 'am', 'a', 'student', 'at'], 'term_freq': [[1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1], [1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1], [1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1]]}\n" + ] + } + ], + "source": [ + "# Define doc paths array\n", + "docs = ['doc1.txt', 'doc2.txt', 'doc3.txt']\n", + "\n", + "# Obtain BoW from your function\n", + "bow = get_bow_from_docs(docs)\n", + "\n", + "# Print BoW\n", + "print(bow)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If your attempt above is successful, nice work done!\n", + "\n", + "Now test your function again with the stop words. In the previous lab we defined the stop words in a large array. In this lab, we'll import the stop words from Scikit-Learn." + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "frozenset({'by', 'than', 'herein', 'whereupon', 'to', 'whole', 'serious', 'before', 'less', 'some', 'everywhere', 'nor', 'very', 'already', 'co', 'over', 'side', 'thereafter', 'am', 'became', 'anyhow', 'may', 'name', 'each', 'eight', 'every', 'find', 'would', 'otherwise', 'seems', 'empty', 'while', 'none', 'here', 'please', 'the', 'again', 'becoming', 'thereupon', 'beside', 'into', 'me', 'through', 'yourselves', 'be', 'besides', 'someone', 'amongst', 'fire', 'full', 'therefore', 'ours', 'either', 'thin', 'this', 'put', 'how', 'nevertheless', 'fill', 'bill', 'there', 'latterly', 'from', 'with', 'and', 'mill', 'done', 'inc', 'though', 'whose', 'moreover', 'former', 'ourselves', 'enough', 'during', 'seem', 'few', 'his', 'themselves', 'us', 'next', 'several', 'are', 'always', 'elsewhere', 'hasnt', 'is', 'an', 'well', 'sincere', 'together', 'whom', 'above', 'whither', 'amoungst', 'ten', 'myself', 'something', 'ie', 'below', 'de', 'across', 'back', 'go', 'made', 'onto', 'not', 'ltd', 'only', 'as', 'part', 'whereas', 'nothing', 'eg', 'we', 'much', 'seeming', 'whatever', 'until', 'sixty', 'him', 'even', 'thick', 'top', 'noone', 'two', 'also', 'never', 'himself', 'i', 'detail', 'at', 'six', 'thence', 'had', 'if', 'same', 'toward', 'hers', 'will', 'others', 'four', 'least', 'which', 'of', 'wherein', 'rather', 'fifty', 'due', 'perhaps', 'whether', 'after', 'however', 'hereupon', 'nowhere', 'wherever', 'show', 'off', 'meanwhile', 'eleven', 'around', 'yours', 'she', 'he', 'five', 'three', 'throughout', 'via', 'hereby', 'anywhere', 'except', 'were', 'because', 'nine', 'all', 'now', 'itself', 'under', 'could', 'mine', 'or', 'without', 'but', 'thereby', 'further', 'another', 'un', 'its', 'other', 'in', 'yet', 'anyone', 'beforehand', 'sometimes', 'con', 'almost', 'where', 'why', 'indeed', 'it', 'too', 'many', 'afterwards', 'down', 'do', 'move', 'become', 'herself', 'seemed', 'bottom', 'once', 'somewhere', 'fifteen', 'else', 'up', 'mostly', 'twelve', 'should', 'sometime', 'twenty', 'call', 'anything', 'about', 'thru', 'yourself', 'namely', 'upon', 'although', 'third', 'among', 'hence', 'so', 'that', 'these', 'still', 'couldnt', 'everyone', 'hundred', 'a', 'ever', 'since', 'has', 'must', 'whence', 'get', 'they', 'within', 'was', 'them', 'give', 'etc', 'therein', 'being', 'between', 'forty', 'formerly', 'cry', 'then', 'their', 'our', 'whereby', 'along', 'such', 'thus', 'system', 'for', 'can', 'interest', 'beyond', 'who', 'whenever', 'behind', 'cant', 'hereafter', 'whoever', 'describe', 'front', 'latter', 'own', 'neither', 'no', 'last', 'most', 'might', 'more', 'anyway', 'becomes', 'nobody', 'cannot', 'have', 'you', 'keep', 'her', 'any', 'first', 'what', 'towards', 'when', 'out', 'those', 'both', 're', 'somehow', 'your', 'alone', 'take', 'amount', 'found', 'one', 'my', 'per', 'everything', 'whereafter', 'see', 'often', 'on', 'been', 'against'})\n" + ] + } + ], + "source": [ + "from sklearn.feature_extraction import stop_words\n", + "print(stop_words.ENGLISH_STOP_WORDS)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You should have seen a large list of words that looks like:\n", + "\n", + "```frozenset({'across', 'mine', 'cannot', ...})```\n", + "\n", + "`frozenset` is a type of Python object that is immutable. In this lab you can use it just like an array without conversion." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, test your function with supplying `stop_words.ENGLISH_STOP_WORDS` as the second parameter." + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'bag_of_words': ['ironhack', 'cool', 'love', 'student'], 'term_freq': [[1, 1, 0, 0], [1, 0, 1, 0], [1, 0, 0, 1]]}\n" + ] + } + ], + "source": [ + "bow = get_bow_from_docs(docs, stop_words.ENGLISH_STOP_WORDS)\n", + "\n", + "print(bow)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You should have seen:\n", + "\n", + "```{'bag_of_words': ['ironhack', 'cool', 'love', 'student'], 'term_freq': [[1, 1, 0, 0], [1, 0, 1, 0], [1, 0, 0, 1]]}```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/your-code/doc1.txt b/your-code/doc1.txt new file mode 100644 index 0000000..e66288d --- /dev/null +++ b/your-code/doc1.txt @@ -0,0 +1 @@ +Ironhack is cool. \ No newline at end of file diff --git a/your-code/doc2.txt b/your-code/doc2.txt new file mode 100644 index 0000000..b21feac --- /dev/null +++ b/your-code/doc2.txt @@ -0,0 +1 @@ +I love Ironhack. \ No newline at end of file diff --git a/your-code/doc3.txt b/your-code/doc3.txt new file mode 100644 index 0000000..653c5b7 --- /dev/null +++ b/your-code/doc3.txt @@ -0,0 +1 @@ +I am a student at Ironhack. \ No newline at end of file