From 53484454749a4890ce1ab0f274640038a69fe869 Mon Sep 17 00:00:00 2001 From: Alejandro Arroyo Yamin Date: Thu, 4 Jul 2019 15:30:36 -0500 Subject: [PATCH] =?UTF-8?q?Completamos=20Q1,=20falt=C3=B3=20Q2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.ipynb_checkpoints/Q1-checkpoint.ipynb | 207 ++++++++++++++++++ .../.ipynb_checkpoints/Q2-checkpoint.ipynb | 119 ++++++++++ your-code/Q1.ipynb | 111 ++++++---- your-code/Q2.ipynb | 2 +- your-code/doc1.txt | 1 + your-code/doc2.txt | 1 + your-code/doc3.txt | 1 + 7 files changed, 400 insertions(+), 42 deletions(-) create mode 100644 your-code/.ipynb_checkpoints/Q1-checkpoint.ipynb create mode 100644 your-code/.ipynb_checkpoints/Q2-checkpoint.ipynb create mode 100644 your-code/doc1.txt create mode 100644 your-code/doc2.txt create mode 100644 your-code/doc3.txt diff --git a/your-code/.ipynb_checkpoints/Q1-checkpoint.ipynb b/your-code/.ipynb_checkpoints/Q1-checkpoint.ipynb new file mode 100644 index 0000000..7f0c6e5 --- /dev/null +++ b/your-code/.ipynb_checkpoints/Q1-checkpoint.ipynb @@ -0,0 +1,207 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the cell below, create a Python function that wraps your previous solution for the Bag of Words lab.\n", + "\n", + "Requirements:\n", + "\n", + "1. Your function should accept the following parameters:\n", + " * `docs` [REQUIRED] - array of document paths.\n", + " * `stop_words` [OPTIONAL] - array of stop words. The default value is an empty array.\n", + "\n", + "1. Your function should return a Python object that contains the following:\n", + " * `bag_of_words` - array of strings of normalized unique words in the corpus.\n", + " * `term_freq` - array of the term-frequency vectors." + ] + }, + { + "cell_type": "code", + "execution_count": 97, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'bag_of_words': ['ironhack', 'is', 'cool', 'i', 'love', 'am', 'a', 'student', 'at'], 'term_freq': [[1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1], [1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1], [1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1]]}\n" + ] + } + ], + "source": [ + "# Import required libraries\n", + "\n", + "import os\n", + "import re\n", + "\n", + "stop_words = ['all', 'six', 'less', 'being', 'indeed', 'over', 'move', 'anyway', 'fifty', 'four', 'not', 'own', 'through', 'yourselves', 'go', 'where', 'mill', 'only', 'find', 'before', 'one', 'whose', 'system', 'how', 'somewhere', 'with', 'thick', 'show', 'had', 'enough', 'should', 'to', 'must', 'whom', 'seeming', 'under', 'ours', 'has', 'might', 'thereafter', 'latterly', 'do', 'them', 'his', 'around', 'than', 'get', 'very', 'de', 'none', 'cannot', 'every', 'whether', 'they', 'front', 'during', 'thus', 'now', 'him', 'nor', 'name', 'several', 'hereafter', 'always', 'who', 'cry', 'whither', 'this', 'someone', 'either', 'each', 'become', 'thereupon', 'sometime', 'side', 'two', 'therein', 'twelve', 'because', 'often', 'ten', 'our', 'eg', 'some', 'back', 'up', 'namely', 'towards', 'are', 'further', 'beyond', 'ourselves', 'yet', 'out', 'even', 'will', 'what', 'still', 'for', 'bottom', 'mine', 'since', 'please', 'forty', 'per', 'its', 'everything', 'behind', 'un', 'above', 'between', 'it', 'neither', 'seemed', 'ever', 'across', 'she', 'somehow', 'be', 'we', 'full', 'never', 'sixty', 'however', 'here', 'otherwise', 'were', 'whereupon', 'nowhere', 'although', 'found', 'alone', 're', 'along', 'fifteen', 'by', 'both', 'about', 'last', 'would', 'anything', 'via', 'many', 'could', 'thence', 'put', 'against', 'keep', 'etc', 'amount', 'became', 'ltd', 'hence', 'onto', 'or', 'con', 'among', 'already', 'co', 'afterwards', 'formerly', 'within', 'seems', 'into', 'others', 'while', 'whatever', 'except', 'down', 'hers', 'everyone', 'done', 'least', 'another', 'whoever', 'moreover', 'couldnt', 'throughout', 'anyhow', 'yourself', 'three', 'from', 'her', 'few', 'together', 'top', 'there', 'due', 'been', 'next', 'anyone', 'eleven', 'much', 'call', 'therefore', 'interest', 'then', 'thru', 'themselves', 'hundred', 'was', 'sincere', 'empty', 'more', 'himself', 'elsewhere', 'mostly', 'on', 'fire', 'am', 'becoming', 'hereby', 'amongst', 'else', 'part', 'everywhere', 'too', 'herself', 'former', 'those', 'he', 'me', 'myself', 'made', 'twenty', 'these', 'bill', 'cant', 'us', 'until', 'besides', 'nevertheless', 'below', 'anywhere', 'nine', 'can', 'of', 'your', 'toward', 'my', 'something', 'and', 'whereafter', 'whenever', 'give', 'almost', 'wherever', 'is', 'describe', 'beforehand', 'herein', 'an', 'as', 'itself', 'at', 'have', 'in', 'seem', 'whence', 'ie', 'any', 'fill', 'again', 'hasnt', 'inc', 'thereby', 'thin', 'no', 'perhaps', 'latter', 'meanwhile', 'when', 'detail', 'same', 'wherein', 'beside', 'also', 'that', 'other', 'take', 'which', 'becomes', 'you', 'if', 'nobody', 'see', 'though', 'may', 'after', 'upon', 'most', 'hereupon', 'eight', 'but', 'serious', 'nothing', 'such', 'why', 'a', 'off', 'whereby', 'third', 'i', 'whole', 'noone', 'sometimes', 'well', 'amoungst', 'yours', 'their', 'rather', 'without', 'so', 'five', 'the', 'first', 'whereas', 'once']\n", + "docs = ['./doc1.txt', './doc2.txt', './doc3.txt']\n", + "\n", + "# Define function\n", + "\n", + "def get_bow_from_docs(docs, stop_words=[]):\n", + " corpus = []\n", + " bag_of_words = []\n", + " term_freq = []\n", + " for i in docs:\n", + " corpus.append(open(i,'r').read().lower().strip('.'))\n", + " corpus2 = ' '.join(corpus).split()\n", + " for i in corpus2:\n", + " if i not in bag_of_words and i not in stop_words:\n", + " bag_of_words.append(i)\n", + " for i in corpus:\n", + " corpus3 = []\n", + " if stop_words == []:\n", + " for word in corpus2:\n", + " corpus3.append(i.split(' ').count(word)) \n", + " else:\n", + " for word in bag_of_words:\n", + " corpus3.append(i.count(word))\n", + " term_freq.append(corpus3)\n", + " return {\n", + " \"bag_of_words\": bag_of_words,\n", + " \"term_freq\": term_freq\n", + " }\n", + "\n", + "print(get_bow_from_docs(docs))\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Test your function without stop words. You should see the output like below:\n", + "\n", + "```{'bag_of_words': ['ironhack', 'is', 'cool', 'i', 'love', 'am', 'a', 'student', 'at'], 'term_freq': [[1, 1, 1, 0, 0, 0, 0, 0, 0], [1, 0, 0, 1, 1, 0, 0, 0, 0], [1, 0, 0, 1, 0, 1, 1, 1, 1]]}```" + ] + }, + { + "cell_type": "code", + "execution_count": 98, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'bag_of_words': ['ironhack', 'is', 'cool', 'i', 'love', 'am', 'a', 'student', 'at'], 'term_freq': [[1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1], [1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1], [1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1]]}\n" + ] + } + ], + "source": [ + "# Define doc paths array\n", + "docs = ['./doc1.txt', './doc2.txt', './doc3.txt']\n", + "\n", + "# Obtain BoW from your function\n", + "bow = get_bow_from_docs(docs)\n", + "\n", + "# Print BoW\n", + "print(bow)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If your attempt above is successful, nice work done!\n", + "\n", + "Now test your function again with the stop words. In the previous lab we defined the stop words in a large array. In this lab, we'll import the stop words from Scikit-Learn." + ] + }, + { + "cell_type": "code", + "execution_count": 101, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "frozenset({'against', 'off', 'twelve', 'whither', 'give', 'you', 'eight', 'eg', 'all', 'will', 'four', 'hence', 'well', 'seem', 'out', 'behind', 'in', 'much', 'sometime', 'after', 'these', 'along', 'never', 'also', 'hundred', 'me', 'mostly', 'but', 'or', 'her', 'may', 'an', 'thereupon', 'amount', 'would', 'whereby', 'its', 'get', 'how', 'top', 'whom', 'full', 'either', 'third', 'do', 'being', 'my', 'due', 'hers', 'with', 'be', 'con', 'detail', 'now', 'nowhere', 'former', 'whole', 'first', 'that', 'herself', 'become', 'whose', 'latter', 'at', 'what', 'anyone', 'than', 'cant', 'nor', 'beside', 'as', 'co', 'two', 'therefore', 'made', 'namely', 'otherwise', 'since', 'which', 'back', 'those', 'latterly', 'bill', 'should', 'he', 'un', 'another', 'both', 'if', 'most', 'formerly', 'one', 'because', 'other', 'serious', 'next', 'whenever', 'there', 'been', 'hereafter', 'empty', 'until', 'everywhere', 'per', 'six', 'had', 'only', 'of', 'where', 'on', 'three', 'whence', 'sincere', 'moreover', 'meanwhile', 'therein', 'thru', 'everyone', 'is', 'eleven', 'yourselves', 'fifty', 'afterwards', 'own', 'etc', 'above', 'however', 'least', 'themselves', 'nobody', 'while', 'whatever', 'indeed', 'move', 'his', 'de', 'call', 'for', 'always', 'between', 'him', 'becoming', 'a', 'by', 'yet', 'whoever', 'thereby', 'see', 'already', 'them', 'toward', 'several', 'seems', 'besides', 'cry', 'although', 'over', 'thence', 'bottom', 'ten', 'are', 'it', 'why', 'amoungst', 'we', 'almost', 'myself', 'yours', 'under', 'their', 'further', 'they', 'whereupon', 'others', 'something', 'sometimes', 'sixty', 'ie', 'none', 'beyond', 'our', 'else', 'elsewhere', 'wherein', 'enough', 'beforehand', 'so', 'mine', 'across', 'many', 'and', 'some', 'every', 'again', 'found', 'somehow', 'i', 'once', 'might', 'fill', 'though', 'anything', 'cannot', 'ltd', 'thus', 'five', 'even', 'among', 'were', 'herein', 'during', 'ourselves', 'whereafter', 'below', 'less', 'perhaps', 'ours', 'must', 'fire', 'amongst', 'has', 'last', 'whereas', 'himself', 'without', 'yourself', 'name', 'wherever', 'becomes', 'up', 'forty', 'here', 'couldnt', 'somewhere', 'go', 'hasnt', 'to', 'thereafter', 'noone', 'thin', 'front', 'thick', 'could', 'from', 'then', 'few', 'no', 'rather', 'someone', 'such', 'inc', 'was', 'this', 'neither', 'any', 'your', 'put', 'anyway', 'still', 'interest', 'about', 'hereby', 'more', 'into', 'who', 'upon', 'via', 'anyhow', 'seemed', 'describe', 'mill', 'twenty', 'am', 'not', 'often', 'same', 'show', 'system', 'done', 'find', 'became', 'fifteen', 'seeming', 'nevertheless', 'take', 'before', 'she', 'itself', 'the', 'part', 'nothing', 'very', 'through', 'towards', 'except', 'alone', 'throughout', 'us', 'nine', 'please', 'keep', 'too', 'around', 're', 'onto', 'have', 'anywhere', 'each', 'within', 'hereupon', 'together', 'side', 'ever', 'can', 'down', 'whether', 'everything', 'when'})\n" + ] + } + ], + "source": [ + "from sklearn.feature_extraction import stop_words\n", + "print(stop_words.ENGLISH_STOP_WORDS)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You should have seen a large list of words that looks like:\n", + "\n", + "```frozenset({'across', 'mine', 'cannot', ...})```\n", + "\n", + "`frozenset` is a type of Python object that is immutable. In this lab you can use it just like an array without conversion." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, test your function with supplying `stop_words.ENGLISH_STOP_WORDS` as the second parameter." + ] + }, + { + "cell_type": "code", + "execution_count": 103, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'bag_of_words': ['ironhack', 'cool', 'love', 'student'], 'term_freq': [[1, 1, 0, 0], [1, 0, 1, 0], [1, 0, 0, 1]]}\n" + ] + } + ], + "source": [ + "bow = get_bow_from_docs(docs, stop_words.ENGLISH_STOP_WORDS)\n", + "\n", + "print(bow)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You should have seen:\n", + "\n", + "```{'bag_of_words': ['ironhack', 'cool', 'love', 'student'], 'term_freq': [[1, 1, 0, 0], [1, 0, 1, 0], [1, 0, 0, 1]]}```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/your-code/.ipynb_checkpoints/Q2-checkpoint.ipynb b/your-code/.ipynb_checkpoints/Q2-checkpoint.ipynb new file mode 100644 index 0000000..f6d3967 --- /dev/null +++ b/your-code/.ipynb_checkpoints/Q2-checkpoint.ipynb @@ -0,0 +1,119 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we want to enhance the `get_bow_from_docs` function so that it will work with HTML webpages. In HTML, there are a lot of messy codes such as HTML tags, Javascripts, [unicodes](https://www.w3schools.com/charsets/ref_utf_misc_symbols.asp) that will mess up your bag of words. We need to clean up those junk before generating BoW.\n", + "\n", + "Next, what you will do is to define several new functions each of which is specialized to clean up the HTML codes in one aspect. For instance, you can have a `strip_html_tags` function to remove all HTML tags, a `remove_punctuation` function to remove all punctuation, a `to_lower_case` function to convert string to lowercase, and a `remove_unicode` function to remove all unicodes.\n", + "\n", + "Then in your `get_bow_from_doc` function, you will call each of those functions you created to clean up the HTML before you generate the corpus.\n", + "\n", + "Note: Please use Python string operations and regular expression only in this lab. Do not use extra libraries such as `beautifulsoup` because otherwise you loose the purpose of practicing." + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": {}, + "outputs": [], + "source": [ + "# Define your string handling functions below\n", + "# Minimal 3 functions\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, paste your previously written `get_bow_from_docs` function below. Call your functions above at the appropriate place." + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "metadata": {}, + "outputs": [], + "source": [ + "def get_bow_from_docs(docs, stop_words=[]):\n", + " # In the function, first define the variables you will use such as `corpus`, `bag_of_words`, and `term_freq`.\n", + " corpus = []\n", + " bag_of_words = []\n", + " term_freq = []\n", + " \n", + " # write your codes here\n", + " \n", + " return {\n", + " \"bag_of_words\": bag_of_words,\n", + " \"term_freq\": term_freq\n", + " }\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, read the content from the three HTML webpages in the `your-codes` directory to test your function." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.feature_extraction import stop_words\n", + "bow = get_bow_from_docs([\n", + " 'www.coursereport.com_ironhack.html',\n", + " 'en.wikipedia.org_Data_analysis.html',\n", + " 'www.lipsum.com.html'\n", + " ],\n", + " stop_words.ENGLISH_STOP_WORDS\n", + ")\n", + "\n", + "print(bow)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Do you see any problem in the output? How do you improve the output?\n", + "\n", + "A good way to improve your codes is to look into the HTML data sources and try to understand where the messy output came from. A good data analyst always learns about the data in depth in order to perform the job well.\n", + "\n", + "Spend 20-30 minutes to improve your functions or until you feel you are good at string operations. This lab is just a practice so you don't need to stress yourself out. If you feel you've practiced enough you can stop and move on the next challenge question." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/your-code/Q1.ipynb b/your-code/Q1.ipynb index 8b07d3d..7f0c6e5 100644 --- a/your-code/Q1.ipynb +++ b/your-code/Q1.ipynb @@ -19,48 +19,53 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 97, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'bag_of_words': ['ironhack', 'is', 'cool', 'i', 'love', 'am', 'a', 'student', 'at'], 'term_freq': [[1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1], [1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1], [1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1]]}\n" + ] + } + ], "source": [ "# Import required libraries\n", "\n", + "import os\n", + "import re\n", + "\n", + "stop_words = ['all', 'six', 'less', 'being', 'indeed', 'over', 'move', 'anyway', 'fifty', 'four', 'not', 'own', 'through', 'yourselves', 'go', 'where', 'mill', 'only', 'find', 'before', 'one', 'whose', 'system', 'how', 'somewhere', 'with', 'thick', 'show', 'had', 'enough', 'should', 'to', 'must', 'whom', 'seeming', 'under', 'ours', 'has', 'might', 'thereafter', 'latterly', 'do', 'them', 'his', 'around', 'than', 'get', 'very', 'de', 'none', 'cannot', 'every', 'whether', 'they', 'front', 'during', 'thus', 'now', 'him', 'nor', 'name', 'several', 'hereafter', 'always', 'who', 'cry', 'whither', 'this', 'someone', 'either', 'each', 'become', 'thereupon', 'sometime', 'side', 'two', 'therein', 'twelve', 'because', 'often', 'ten', 'our', 'eg', 'some', 'back', 'up', 'namely', 'towards', 'are', 'further', 'beyond', 'ourselves', 'yet', 'out', 'even', 'will', 'what', 'still', 'for', 'bottom', 'mine', 'since', 'please', 'forty', 'per', 'its', 'everything', 'behind', 'un', 'above', 'between', 'it', 'neither', 'seemed', 'ever', 'across', 'she', 'somehow', 'be', 'we', 'full', 'never', 'sixty', 'however', 'here', 'otherwise', 'were', 'whereupon', 'nowhere', 'although', 'found', 'alone', 're', 'along', 'fifteen', 'by', 'both', 'about', 'last', 'would', 'anything', 'via', 'many', 'could', 'thence', 'put', 'against', 'keep', 'etc', 'amount', 'became', 'ltd', 'hence', 'onto', 'or', 'con', 'among', 'already', 'co', 'afterwards', 'formerly', 'within', 'seems', 'into', 'others', 'while', 'whatever', 'except', 'down', 'hers', 'everyone', 'done', 'least', 'another', 'whoever', 'moreover', 'couldnt', 'throughout', 'anyhow', 'yourself', 'three', 'from', 'her', 'few', 'together', 'top', 'there', 'due', 'been', 'next', 'anyone', 'eleven', 'much', 'call', 'therefore', 'interest', 'then', 'thru', 'themselves', 'hundred', 'was', 'sincere', 'empty', 'more', 'himself', 'elsewhere', 'mostly', 'on', 'fire', 'am', 'becoming', 'hereby', 'amongst', 'else', 'part', 'everywhere', 'too', 'herself', 'former', 'those', 'he', 'me', 'myself', 'made', 'twenty', 'these', 'bill', 'cant', 'us', 'until', 'besides', 'nevertheless', 'below', 'anywhere', 'nine', 'can', 'of', 'your', 'toward', 'my', 'something', 'and', 'whereafter', 'whenever', 'give', 'almost', 'wherever', 'is', 'describe', 'beforehand', 'herein', 'an', 'as', 'itself', 'at', 'have', 'in', 'seem', 'whence', 'ie', 'any', 'fill', 'again', 'hasnt', 'inc', 'thereby', 'thin', 'no', 'perhaps', 'latter', 'meanwhile', 'when', 'detail', 'same', 'wherein', 'beside', 'also', 'that', 'other', 'take', 'which', 'becomes', 'you', 'if', 'nobody', 'see', 'though', 'may', 'after', 'upon', 'most', 'hereupon', 'eight', 'but', 'serious', 'nothing', 'such', 'why', 'a', 'off', 'whereby', 'third', 'i', 'whole', 'noone', 'sometimes', 'well', 'amoungst', 'yours', 'their', 'rather', 'without', 'so', 'five', 'the', 'first', 'whereas', 'once']\n", + "docs = ['./doc1.txt', './doc2.txt', './doc3.txt']\n", + "\n", "# Define function\n", + "\n", "def get_bow_from_docs(docs, stop_words=[]):\n", - " \n", - " # In the function, first define the variables you will use such as `corpus`, `bag_of_words`, and `term_freq`.\n", - " \n", - " \n", - " \n", - " \"\"\"\n", - " Loop `docs` and read the content of each doc into a string in `corpus`.\n", - " Remember to convert the doc content to lowercases and remove punctuation.\n", - " \"\"\"\n", - "\n", - " \n", - " \n", - " \"\"\"\n", - " Loop `corpus`. Append the terms in each doc into the `bag_of_words` array. The terms in `bag_of_words` \n", - " should be unique which means before adding each term you need to check if it's already added to the array.\n", - " In addition, check if each term is in the `stop_words` array. Only append the term to `bag_of_words`\n", - " if it is not a stop word.\n", - " \"\"\"\n", - "\n", - " \n", - " \n", - " \n", - " \"\"\"\n", - " Loop `corpus` again. For each doc string, count the number of occurrences of each term in `bag_of_words`. \n", - " Create an array for each doc's term frequency and append it to `term_freq`.\n", - " \"\"\"\n", - "\n", - " \n", - " \n", - " # Now return your output as an object\n", + " corpus = []\n", + " bag_of_words = []\n", + " term_freq = []\n", + " for i in docs:\n", + " corpus.append(open(i,'r').read().lower().strip('.'))\n", + " corpus2 = ' '.join(corpus).split()\n", + " for i in corpus2:\n", + " if i not in bag_of_words and i not in stop_words:\n", + " bag_of_words.append(i)\n", + " for i in corpus:\n", + " corpus3 = []\n", + " if stop_words == []:\n", + " for word in corpus2:\n", + " corpus3.append(i.split(' ').count(word)) \n", + " else:\n", + " for word in bag_of_words:\n", + " corpus3.append(i.count(word))\n", + " term_freq.append(corpus3)\n", " return {\n", " \"bag_of_words\": bag_of_words,\n", " \"term_freq\": term_freq\n", " }\n", + "\n", + "print(get_bow_from_docs(docs))\n", " " ] }, @@ -75,12 +80,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 98, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'bag_of_words': ['ironhack', 'is', 'cool', 'i', 'love', 'am', 'a', 'student', 'at'], 'term_freq': [[1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1], [1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1], [1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1]]}\n" + ] + } + ], "source": [ "# Define doc paths array\n", - "docs = []\n", + "docs = ['./doc1.txt', './doc2.txt', './doc3.txt']\n", "\n", "# Obtain BoW from your function\n", "bow = get_bow_from_docs(docs)\n", @@ -100,9 +113,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 101, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "frozenset({'against', 'off', 'twelve', 'whither', 'give', 'you', 'eight', 'eg', 'all', 'will', 'four', 'hence', 'well', 'seem', 'out', 'behind', 'in', 'much', 'sometime', 'after', 'these', 'along', 'never', 'also', 'hundred', 'me', 'mostly', 'but', 'or', 'her', 'may', 'an', 'thereupon', 'amount', 'would', 'whereby', 'its', 'get', 'how', 'top', 'whom', 'full', 'either', 'third', 'do', 'being', 'my', 'due', 'hers', 'with', 'be', 'con', 'detail', 'now', 'nowhere', 'former', 'whole', 'first', 'that', 'herself', 'become', 'whose', 'latter', 'at', 'what', 'anyone', 'than', 'cant', 'nor', 'beside', 'as', 'co', 'two', 'therefore', 'made', 'namely', 'otherwise', 'since', 'which', 'back', 'those', 'latterly', 'bill', 'should', 'he', 'un', 'another', 'both', 'if', 'most', 'formerly', 'one', 'because', 'other', 'serious', 'next', 'whenever', 'there', 'been', 'hereafter', 'empty', 'until', 'everywhere', 'per', 'six', 'had', 'only', 'of', 'where', 'on', 'three', 'whence', 'sincere', 'moreover', 'meanwhile', 'therein', 'thru', 'everyone', 'is', 'eleven', 'yourselves', 'fifty', 'afterwards', 'own', 'etc', 'above', 'however', 'least', 'themselves', 'nobody', 'while', 'whatever', 'indeed', 'move', 'his', 'de', 'call', 'for', 'always', 'between', 'him', 'becoming', 'a', 'by', 'yet', 'whoever', 'thereby', 'see', 'already', 'them', 'toward', 'several', 'seems', 'besides', 'cry', 'although', 'over', 'thence', 'bottom', 'ten', 'are', 'it', 'why', 'amoungst', 'we', 'almost', 'myself', 'yours', 'under', 'their', 'further', 'they', 'whereupon', 'others', 'something', 'sometimes', 'sixty', 'ie', 'none', 'beyond', 'our', 'else', 'elsewhere', 'wherein', 'enough', 'beforehand', 'so', 'mine', 'across', 'many', 'and', 'some', 'every', 'again', 'found', 'somehow', 'i', 'once', 'might', 'fill', 'though', 'anything', 'cannot', 'ltd', 'thus', 'five', 'even', 'among', 'were', 'herein', 'during', 'ourselves', 'whereafter', 'below', 'less', 'perhaps', 'ours', 'must', 'fire', 'amongst', 'has', 'last', 'whereas', 'himself', 'without', 'yourself', 'name', 'wherever', 'becomes', 'up', 'forty', 'here', 'couldnt', 'somewhere', 'go', 'hasnt', 'to', 'thereafter', 'noone', 'thin', 'front', 'thick', 'could', 'from', 'then', 'few', 'no', 'rather', 'someone', 'such', 'inc', 'was', 'this', 'neither', 'any', 'your', 'put', 'anyway', 'still', 'interest', 'about', 'hereby', 'more', 'into', 'who', 'upon', 'via', 'anyhow', 'seemed', 'describe', 'mill', 'twenty', 'am', 'not', 'often', 'same', 'show', 'system', 'done', 'find', 'became', 'fifteen', 'seeming', 'nevertheless', 'take', 'before', 'she', 'itself', 'the', 'part', 'nothing', 'very', 'through', 'towards', 'except', 'alone', 'throughout', 'us', 'nine', 'please', 'keep', 'too', 'around', 're', 'onto', 'have', 'anywhere', 'each', 'within', 'hereupon', 'together', 'side', 'ever', 'can', 'down', 'whether', 'everything', 'when'})\n" + ] + } + ], "source": [ "from sklearn.feature_extraction import stop_words\n", "print(stop_words.ENGLISH_STOP_WORDS)" @@ -128,11 +149,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 103, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'bag_of_words': ['ironhack', 'cool', 'love', 'student'], 'term_freq': [[1, 1, 0, 0], [1, 0, 1, 0], [1, 0, 0, 1]]}\n" + ] + } + ], "source": [ - "bow = get_bow_from_docs(bow, stop_words.ENGLISH_STOP_WORDS)\n", + "bow = get_bow_from_docs(docs, stop_words.ENGLISH_STOP_WORDS)\n", "\n", "print(bow)" ] @@ -170,7 +199,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.6" + "version": "3.7.3" } }, "nbformat": 4, diff --git a/your-code/Q2.ipynb b/your-code/Q2.ipynb index f50f442..f6d3967 100644 --- a/your-code/Q2.ipynb +++ b/your-code/Q2.ipynb @@ -111,7 +111,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.6" + "version": "3.7.3" } }, "nbformat": 4, diff --git a/your-code/doc1.txt b/your-code/doc1.txt new file mode 100644 index 0000000..e66288d --- /dev/null +++ b/your-code/doc1.txt @@ -0,0 +1 @@ +Ironhack is cool. \ No newline at end of file diff --git a/your-code/doc2.txt b/your-code/doc2.txt new file mode 100644 index 0000000..b21feac --- /dev/null +++ b/your-code/doc2.txt @@ -0,0 +1 @@ +I love Ironhack. \ No newline at end of file diff --git a/your-code/doc3.txt b/your-code/doc3.txt new file mode 100644 index 0000000..653c5b7 --- /dev/null +++ b/your-code/doc3.txt @@ -0,0 +1 @@ +I am a student at Ironhack. \ No newline at end of file