diff --git a/your-code/.ipynb_checkpoints/Q1-checkpoint.ipynb b/your-code/.ipynb_checkpoints/Q1-checkpoint.ipynb new file mode 100644 index 0000000..dfee27f --- /dev/null +++ b/your-code/.ipynb_checkpoints/Q1-checkpoint.ipynb @@ -0,0 +1,254 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the cell below, create a Python function that wraps your previous solution for the Bag of Words lab.\n", + "\n", + "Requirements:\n", + "\n", + "1. Your function should accept the following parameters:\n", + " * `docs` [REQUIRED] - array of document paths.\n", + " * `stop_words` [OPTIONAL] - array of stop words. The default value is an empty array.\n", + "\n", + "1. Your function should return a Python object that contains the following:\n", + " * `bag_of_words` - array of strings of normalized unique words in the corpus.\n", + " * `term_freq` - array of the term-frequency vectors." + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [], + "source": [ + "# Import required libraries\n", + "import re\n", + "import pandas as pd\n", + "import math\n", + "import os\n", + "# Define function\n", + "docs = ['doc1.txt', 'doc2.txt', 'doc3.txt']\n", + "def get_bow_from_docs(docs, stop_words=[]):\n", + " corpus = []\n", + " bag_of_words = []\n", + " term_freq = []\n", + " corpus2 = []\n", + " # In the function, first define the variables you will use such as `corpus`, `bag_of_words`, and `term_freq`.\n", + " for x in docs:\n", + " abrir = open(x).read()\n", + " corpus.append(abrir)\n", + " for x in corpus:\n", + " y = x.lower().replace(\".\", \"\")\n", + " corpus2.append(y)\n", + " \n", + " \"\"\"\n", + " Loop `docs` and read the content of each doc into a string in `corpus`.\n", + " Remember to convert the doc content to lowercases and remove punctuation.\n", + " \"\"\"\n", + " for x in corpus2:\n", + " y = re.split(\" \", x)\n", + " for z in y:\n", + " if z not in bag_of_words and z not in stop_words:\n", + " bag_of_words.append(z)\n", + " else:\n", + " pass\n", + " \n", + " \n", + " \n", + " pieces = []\n", + " for x in corpus2:\n", + " pieces.append(x.split(\" \"))\n", + " \n", + " \n", + " \n", + " \n", + " \"\"\"\n", + " Loop `corpus`. Append the terms in each doc into the `bag_of_words` array. The terms in `bag_of_words` \n", + " should be unique which means before adding each term you need to check if it's already added to the array.\n", + " In addition, check if each term is in the `stop_words` array. Only append the term to `bag_of_words`\n", + " if it is not a stop word.\n", + " \"\"\"\n", + " \n", + " \n", + " \n", + " \n", + " for s in pieces:\n", + " temp = []\n", + " for b in bag_of_words:\n", + " y = s.count(b)\n", + " temp.append(y)\n", + " term_freq.append(temp)\n", + "\n", + " \n", + " \n", + " \n", + " \"\"\"\n", + " Loop `corpus` again. For each doc string, count the number of occurrences of each term in `bag_of_words`. \n", + " Create an array for each doc's term frequency and append it to `term_freq`.\n", + " \"\"\"\n", + "\n", + " \n", + " \n", + " # Now return your output as an object\n", + " return {\n", + " \"bag_of_words\": bag_of_words,\n", + " \"term_freq\": term_freq\n", + " }\n" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'bag_of_words': ['ironhack', 'is', 'cool', 'i', 'love', 'am', 'a', 'student', 'at'], 'term_freq': [[1, 1, 1, 0, 0, 0, 0, 0, 0], [1, 0, 0, 1, 1, 0, 0, 0, 0], [1, 0, 0, 1, 0, 1, 1, 1, 1]]}\n" + ] + } + ], + "source": [ + "print(get_bow_from_docs(docs, stop_words=[]))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Test your function without stop words. You should see the output like below:\n", + "\n", + "```{'bag_of_words': ['ironhack', 'is', 'cool', 'i', 'love', 'am', 'a', 'student', 'at'], 'term_freq': [[1, 1, 1, 0, 0, 0, 0, 0, 0], [1, 0, 0, 1, 1, 0, 0, 0, 0], [1, 0, 0, 1, 0, 1, 1, 1, 1]]}```" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'bag_of_words': ['ironhack', 'is', 'cool', 'i', 'love', 'am', 'a', 'student', 'at'], 'term_freq': [[1, 1, 1, 0, 0, 0, 0, 0, 0], [1, 0, 0, 1, 1, 0, 0, 0, 0], [1, 0, 0, 1, 0, 1, 1, 1, 1]]}\n" + ] + } + ], + "source": [ + "# Define doc paths array\n", + "docs = ['doc1.txt', 'doc2.txt', 'doc3.txt']\n", + "\n", + "# Obtain BoW from your function\n", + "bow = get_bow_from_docs(docs)\n", + "\n", + "# Print BoW\n", + "print(bow)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If your attempt above is successful, nice work done!\n", + "\n", + "Now test your function again with the stop words. In the previous lab we defined the stop words in a large array. In this lab, we'll import the stop words from Scikit-Learn." + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "frozenset({'for', 'up', 'please', 'were', 'so', 'besides', 'beyond', 'against', 'these', 'any', 'has', 'ours', 'although', 'that', 'why', 'then', 'none', 'detail', 'latter', 'herein', 'twelve', 'find', 'myself', 'further', 'which', 'though', 'should', 'he', 'thence', 'yourself', 'side', 'itself', 'are', 'very', 'ourselves', 'your', 'empty', 'hence', 'put', 'across', 'elsewhere', 'this', 'un', 'former', 'out', 'found', 'moreover', 'still', 'rather', 'most', 'afterwards', 'have', 'at', 'now', 'either', 'themselves', 'therein', 'until', 'from', 'both', 'less', 'four', 'i', 'whereas', 'keep', 'even', 'becoming', 're', 'nine', 'been', 'ie', 'around', 'everywhere', 'anyhow', 'con', 'sometimes', 'take', 'where', 'can', 'front', 'several', 'hereafter', 'others', 'over', 'else', 'such', 'before', 'during', 'because', 'they', 'do', 'latterly', 'hers', 'almost', 'under', 'part', 'nobody', 'whither', 'bottom', 'about', 'move', 'done', 'became', 'sincere', 'whereby', 'anything', 'name', 'serious', 'seemed', 'much', 'her', 'through', 'below', 'whatever', 'and', 'being', 'among', 'them', 'therefore', 'show', 'our', 'sometime', 'herself', 'nor', 'after', 'there', 'top', 'least', 'nowhere', 'might', 'whose', 'must', 'every', 'to', 'no', 'also', 'somewhere', 'two', 'enough', 'him', 'on', 'seeming', 'us', 'we', 'without', 'per', 'thereby', 'eleven', 'when', 'ltd', 'those', 'would', 'mostly', 'mine', 'thru', 'amongst', 'perhaps', 'hereupon', 'within', 'never', 'forty', 'along', 'hundred', 'cant', 'becomes', 'due', 'more', 'anyone', 'inc', 'five', 'same', 'down', 'back', 'formerly', 'yours', 'describe', 'etc', 'full', 'neither', 'each', 'could', 'last', 'or', 'whereupon', 'behind', 'it', 'in', 'few', 'noone', 'via', 'thus', 'thereafter', 'alone', 'thick', 'towards', 'somehow', 'twenty', 'anyway', 'three', 'beforehand', 'me', 'call', 'ever', 'amount', 'will', 'whole', 'onto', 'namely', 'she', 'everyone', 'yourselves', 'wherein', 'whether', 'with', 'made', 'cry', 'whenever', 'something', 'his', 'mill', 'first', 'give', 'get', 'become', 'interest', 'than', 'eight', 'had', 'by', 'bill', 'am', 'as', 'go', 'but', 'hereby', 'all', 'nothing', 'thin', 'here', 'beside', 'meanwhile', 'ten', 'was', 'an', 'always', 'who', 'only', 'third', 'fifty', 'toward', 'the', 'see', 'eg', 'since', 'of', 'many', 'some', 'their', 'amoungst', 'system', 'couldnt', 'whence', 'everything', 'one', 'what', 'fire', 'next', 'whom', 'my', 'off', 'six', 'between', 'upon', 'wherever', 'cannot', 'throughout', 'once', 'whoever', 'whereafter', 'indeed', 'seem', 'seems', 'fill', 'himself', 'too', 'its', 'may', 'fifteen', 'you', 'nevertheless', 'except', 'while', 'de', 'sixty', 'thereupon', 'again', 'a', 'co', 'be', 'above', 'often', 'already', 'someone', 'yet', 'together', 'hasnt', 'how', 'anywhere', 'another', 'well', 'if', 'not', 'own', 'otherwise', 'into', 'is', 'however', 'other'})\n" + ] + } + ], + "source": [ + "from sklearn.feature_extraction import stop_words\n", + "\n", + "print(stop_words.ENGLISH_STOP_WORDS)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You should have seen a large list of words that looks like:\n", + "\n", + "```frozenset({'across', 'mine', 'cannot', ...})```\n", + "\n", + "`frozenset` is a type of Python object that is immutable. In this lab you can use it just like an array without conversion." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, test your function with supplying `stop_words.ENGLISH_STOP_WORDS` as the second parameter." + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'bag_of_words': ['ironhack', 'cool', 'love', 'student'], 'term_freq': [[1, 1, 0, 0], [1, 0, 1, 0], [1, 0, 0, 1]]}\n" + ] + } + ], + "source": [ + "bow = get_bow_from_docs(docs, stop_words.ENGLISH_STOP_WORDS)\n", + "\n", + "print(bow)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You should have seen:\n", + "\n", + "```{'bag_of_words': ['ironhack', 'cool', 'love', 'student'], 'term_freq': [[1, 1, 0, 0], [1, 0, 1, 0], [1, 0, 0, 1]]}```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/your-code/.ipynb_checkpoints/Q2-checkpoint.ipynb b/your-code/.ipynb_checkpoints/Q2-checkpoint.ipynb new file mode 100644 index 0000000..6ea0a0b --- /dev/null +++ b/your-code/.ipynb_checkpoints/Q2-checkpoint.ipynb @@ -0,0 +1,188 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we want to enhance the `get_bow_from_docs` function so that it will work with HTML webpages. In HTML, there are a lot of messy codes such as HTML tags, Javascripts, [unicodes](https://www.w3schools.com/charsets/ref_utf_misc_symbols.asp) that will mess up your bag of words. We need to clean up those junk before generating BoW.\n", + "\n", + "Next, what you will do is to define several new functions each of which is specialized to clean up the HTML codes in one aspect. For instance, you can have a `strip_html_tags` function to remove all HTML tags, a `remove_punctuation` function to remove all punctuation, a `to_lower_case` function to convert string to lowercase, and a `remove_unicode` function to remove all unicodes.\n", + "\n", + "Then in your `get_bow_from_doc` function, you will call each of those functions you created to clean up the HTML before you generate the corpus.\n", + "\n", + "Note: Please use Python string operations and regular expression only in this lab. Do not use extra libraries such as `beautifulsoup` because otherwise you loose the purpose of practicing." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "# Define your string handling functions below\n", + "# Minimal 3 functions\n", + "import os\n", + "import pandas as pd\n", + "import re\n", + "import math\n", + "\n", + "def sin_puntuacion(corpus):\n", + " corpus2 = []\n", + " for x in corpus:\n", + " y = x.lower().replace(\".\", \"\")\n", + " corpus2.append(y)\n", + " return corpus2\n", + "\n", + "def sin_tags(corpus):\n", + " \n", + " \n", + " quotes = re.sub(r\"\\<.*?\\>\", \"\", corpus)\n", + " \n", + " return quotes\n", + "\n", + "def sin_unicodes(corpus):\n", + " \n", + " sincodes = re.sub(r\"\\&.*?\\;\", \"\", corpus)\n", + " \n", + " return sincodes\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, paste your previously written `get_bow_from_docs` function below. Call your functions above at the appropriate place." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "def get_bow_from_docs(docs, stop_words=[]):\n", + " # In the function, first define the variables you will use such as `corpus`, `bag_of_words`, and `term_freq`.\n", + " corpus = []\n", + " corpus2 = []\n", + " bag_of_words = []\n", + " term_freq = []\n", + " \n", + " # write your codes here\n", + " \n", + " for x in docs:\n", + " abrir = open(x).read()\n", + " corpus.append(abrir)\n", + " \n", + " sin_puntuacion(corpus)\n", + " \n", + " sin_tags(corpus2)\n", + " \n", + " sin_unicodes(quotes)\n", + " \n", + " for x in sincodes:\n", + " y = re.split(\" \", x)\n", + " for z in y:\n", + " if z not in bag_of_words and z not in stop_words:\n", + " bag_of_words.append(z)\n", + " else:\n", + " pass\n", + " \n", + " pieces = []\n", + " for x in sincodes:\n", + " pieces.append(x.split(\" \"))\n", + " \n", + " for s in pieces:\n", + " temp = []\n", + " for b in bag_of_words:\n", + " y = s.count(b)\n", + " temp.append(y)\n", + " term_freq.append(temp)\n", + " return {\n", + " \"bag_of_words\": bag_of_words,\n", + " \"term_freq\": term_freq\n", + " }\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, read the content from the three HTML webpages in the `your-codes` directory to test your function." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "ename": "TypeError", + "evalue": "expected string or bytes-like object", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;34m'www.lipsum.com.html'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m ],\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mstop_words\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mENGLISH_STOP_WORDS\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 8\u001b[0m )\n\u001b[1;32m 9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mget_bow_from_docs\u001b[0;34m(docs, stop_words)\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0msin_puntuacion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcorpus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m \u001b[0msin_tags\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcorpus2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[0msin_unicodes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquotes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36msin_tags\u001b[0;34m(corpus)\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0mquotes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mre\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msub\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mr\"\\<.*?\\>\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcorpus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mquotes\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/anaconda3/lib/python3.7/re.py\u001b[0m in \u001b[0;36msub\u001b[0;34m(pattern, repl, string, count, flags)\u001b[0m\n\u001b[1;32m 190\u001b[0m \u001b[0ma\u001b[0m \u001b[0mcallable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mit\u001b[0m\u001b[0;31m'\u001b[0m\u001b[0ms\u001b[0m \u001b[0mpassed\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mMatch\u001b[0m \u001b[0mobject\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mmust\u001b[0m \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 191\u001b[0m a replacement string to be used.\"\"\"\n\u001b[0;32m--> 192\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_compile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpattern\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msub\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrepl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstring\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcount\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 193\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 194\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0msubn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpattern\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrepl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstring\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcount\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mTypeError\u001b[0m: expected string or bytes-like object" + ] + } + ], + "source": [ + "from sklearn.feature_extraction import stop_words\n", + "bow = get_bow_from_docs([\n", + " 'www.coursereport.com_ironhack.html',\n", + " 'en.wikipedia.org_Data_analysis.html',\n", + " 'www.lipsum.com.html'\n", + " ],\n", + " stop_words.ENGLISH_STOP_WORDS\n", + ")\n", + "\n", + "print(bow)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Do you see any problem in the output? How do you improve the output?\n", + "\n", + "A good way to improve your codes is to look into the HTML data sources and try to understand where the messy output came from. A good data analyst always learns about the data in depth in order to perform the job well.\n", + "\n", + "Spend 20-30 minutes to improve your functions or until you feel you are good at string operations. This lab is just a practice so you don't need to stress yourself out. If you feel you've practiced enough you can stop and move on the next challenge question." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/your-code/Q1.ipynb b/your-code/Q1.ipynb index 8b07d3d..c5a0d62 100644 --- a/your-code/Q1.ipynb +++ b/your-code/Q1.ipynb @@ -19,49 +19,70 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ "# Import required libraries\n", - "\n", + "import re\n", + "import pandas as pd\n", + "import math\n", + "import os\n", "# Define function\n", + "docs = ['doc1.txt', 'doc2.txt', 'doc3.txt']\n", "def get_bow_from_docs(docs, stop_words=[]):\n", - " \n", + " corpus = []\n", + " bag_of_words = []\n", + " term_freq = []\n", + " corpus2 = []\n", " # In the function, first define the variables you will use such as `corpus`, `bag_of_words`, and `term_freq`.\n", + " for x in docs:\n", + " abrir = open(x).read()\n", + " corpus.append(abrir)\n", + " for x in corpus:\n", + " y = x.lower().replace(\".\", \"\")\n", + " corpus2.append(y)\n", " \n", - " \n", - " \n", - " \"\"\"\n", - " Loop `docs` and read the content of each doc into a string in `corpus`.\n", - " Remember to convert the doc content to lowercases and remove punctuation.\n", - " \"\"\"\n", - "\n", - " \n", - " \n", - " \"\"\"\n", - " Loop `corpus`. Append the terms in each doc into the `bag_of_words` array. The terms in `bag_of_words` \n", - " should be unique which means before adding each term you need to check if it's already added to the array.\n", - " In addition, check if each term is in the `stop_words` array. Only append the term to `bag_of_words`\n", - " if it is not a stop word.\n", - " \"\"\"\n", + " for x in corpus2:\n", + " y = re.split(\" \", x)\n", + " for z in y:\n", + " if z not in bag_of_words and z not in stop_words:\n", + " bag_of_words.append(z)\n", + " else:\n", + " pass\n", + " \n", + " pieces = []\n", + " for x in corpus2:\n", + " pieces.append(x.split(\" \"))\n", + " \n", + " for s in pieces:\n", + " temp = []\n", + " for b in bag_of_words:\n", + " y = s.count(b)\n", + " temp.append(y)\n", + " term_freq.append(temp)\n", "\n", - " \n", - " \n", - " \n", - " \"\"\"\n", - " Loop `corpus` again. For each doc string, count the number of occurrences of each term in `bag_of_words`. \n", - " Create an array for each doc's term frequency and append it to `term_freq`.\n", - " \"\"\"\n", - "\n", - " \n", - " \n", - " # Now return your output as an object\n", " return {\n", " \"bag_of_words\": bag_of_words,\n", " \"term_freq\": term_freq\n", - " }\n", - " " + " }\n" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'bag_of_words': ['ironhack', 'is', 'cool', 'i', 'love', 'am', 'a', 'student', 'at'], 'term_freq': [[1, 1, 1, 0, 0, 0, 0, 0, 0], [1, 0, 0, 1, 1, 0, 0, 0, 0], [1, 0, 0, 1, 0, 1, 1, 1, 1]]}\n" + ] + } + ], + "source": [ + "print(get_bow_from_docs(docs, stop_words=[]))\n" ] }, { @@ -75,12 +96,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'bag_of_words': ['ironhack', 'is', 'cool', 'i', 'love', 'am', 'a', 'student', 'at'], 'term_freq': [[1, 1, 1, 0, 0, 0, 0, 0, 0], [1, 0, 0, 1, 1, 0, 0, 0, 0], [1, 0, 0, 1, 0, 1, 1, 1, 1]]}\n" + ] + } + ], "source": [ "# Define doc paths array\n", - "docs = []\n", + "docs = ['doc1.txt', 'doc2.txt', 'doc3.txt']\n", "\n", "# Obtain BoW from your function\n", "bow = get_bow_from_docs(docs)\n", @@ -100,12 +129,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "frozenset({'for', 'up', 'please', 'were', 'so', 'besides', 'beyond', 'against', 'these', 'any', 'has', 'ours', 'although', 'that', 'why', 'then', 'none', 'detail', 'latter', 'herein', 'twelve', 'find', 'myself', 'further', 'which', 'though', 'should', 'he', 'thence', 'yourself', 'side', 'itself', 'are', 'very', 'ourselves', 'your', 'empty', 'hence', 'put', 'across', 'elsewhere', 'this', 'un', 'former', 'out', 'found', 'moreover', 'still', 'rather', 'most', 'afterwards', 'have', 'at', 'now', 'either', 'themselves', 'therein', 'until', 'from', 'both', 'less', 'four', 'i', 'whereas', 'keep', 'even', 'becoming', 're', 'nine', 'been', 'ie', 'around', 'everywhere', 'anyhow', 'con', 'sometimes', 'take', 'where', 'can', 'front', 'several', 'hereafter', 'others', 'over', 'else', 'such', 'before', 'during', 'because', 'they', 'do', 'latterly', 'hers', 'almost', 'under', 'part', 'nobody', 'whither', 'bottom', 'about', 'move', 'done', 'became', 'sincere', 'whereby', 'anything', 'name', 'serious', 'seemed', 'much', 'her', 'through', 'below', 'whatever', 'and', 'being', 'among', 'them', 'therefore', 'show', 'our', 'sometime', 'herself', 'nor', 'after', 'there', 'top', 'least', 'nowhere', 'might', 'whose', 'must', 'every', 'to', 'no', 'also', 'somewhere', 'two', 'enough', 'him', 'on', 'seeming', 'us', 'we', 'without', 'per', 'thereby', 'eleven', 'when', 'ltd', 'those', 'would', 'mostly', 'mine', 'thru', 'amongst', 'perhaps', 'hereupon', 'within', 'never', 'forty', 'along', 'hundred', 'cant', 'becomes', 'due', 'more', 'anyone', 'inc', 'five', 'same', 'down', 'back', 'formerly', 'yours', 'describe', 'etc', 'full', 'neither', 'each', 'could', 'last', 'or', 'whereupon', 'behind', 'it', 'in', 'few', 'noone', 'via', 'thus', 'thereafter', 'alone', 'thick', 'towards', 'somehow', 'twenty', 'anyway', 'three', 'beforehand', 'me', 'call', 'ever', 'amount', 'will', 'whole', 'onto', 'namely', 'she', 'everyone', 'yourselves', 'wherein', 'whether', 'with', 'made', 'cry', 'whenever', 'something', 'his', 'mill', 'first', 'give', 'get', 'become', 'interest', 'than', 'eight', 'had', 'by', 'bill', 'am', 'as', 'go', 'but', 'hereby', 'all', 'nothing', 'thin', 'here', 'beside', 'meanwhile', 'ten', 'was', 'an', 'always', 'who', 'only', 'third', 'fifty', 'toward', 'the', 'see', 'eg', 'since', 'of', 'many', 'some', 'their', 'amoungst', 'system', 'couldnt', 'whence', 'everything', 'one', 'what', 'fire', 'next', 'whom', 'my', 'off', 'six', 'between', 'upon', 'wherever', 'cannot', 'throughout', 'once', 'whoever', 'whereafter', 'indeed', 'seem', 'seems', 'fill', 'himself', 'too', 'its', 'may', 'fifteen', 'you', 'nevertheless', 'except', 'while', 'de', 'sixty', 'thereupon', 'again', 'a', 'co', 'be', 'above', 'often', 'already', 'someone', 'yet', 'together', 'hasnt', 'how', 'anywhere', 'another', 'well', 'if', 'not', 'own', 'otherwise', 'into', 'is', 'however', 'other'})\n" + ] + } + ], "source": [ "from sklearn.feature_extraction import stop_words\n", - "print(stop_words.ENGLISH_STOP_WORDS)" + "\n", + "print(stop_words.ENGLISH_STOP_WORDS)\n" ] }, { @@ -128,11 +166,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'bag_of_words': ['ironhack', 'cool', 'love', 'student'], 'term_freq': [[1, 1, 0, 0], [1, 0, 1, 0], [1, 0, 0, 1]]}\n" + ] + } + ], "source": [ - "bow = get_bow_from_docs(bow, stop_words.ENGLISH_STOP_WORDS)\n", + "bow = get_bow_from_docs(docs, stop_words.ENGLISH_STOP_WORDS)\n", "\n", "print(bow)" ] @@ -170,7 +216,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.6" + "version": "3.7.3" } }, "nbformat": 4, diff --git a/your-code/Q2.ipynb b/your-code/Q2.ipynb index f50f442..6ea0a0b 100644 --- a/your-code/Q2.ipynb +++ b/your-code/Q2.ipynb @@ -15,12 +15,37 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "# Define your string handling functions below\n", - "# Minimal 3 functions\n" + "# Minimal 3 functions\n", + "import os\n", + "import pandas as pd\n", + "import re\n", + "import math\n", + "\n", + "def sin_puntuacion(corpus):\n", + " corpus2 = []\n", + " for x in corpus:\n", + " y = x.lower().replace(\".\", \"\")\n", + " corpus2.append(y)\n", + " return corpus2\n", + "\n", + "def sin_tags(corpus):\n", + " \n", + " \n", + " quotes = re.sub(r\"\\<.*?\\>\", \"\", corpus)\n", + " \n", + " return quotes\n", + "\n", + "def sin_unicodes(corpus):\n", + " \n", + " sincodes = re.sub(r\"\\&.*?\\;\", \"\", corpus)\n", + " \n", + " return sincodes\n", + " " ] }, { @@ -32,18 +57,47 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ "def get_bow_from_docs(docs, stop_words=[]):\n", " # In the function, first define the variables you will use such as `corpus`, `bag_of_words`, and `term_freq`.\n", " corpus = []\n", + " corpus2 = []\n", " bag_of_words = []\n", " term_freq = []\n", " \n", " # write your codes here\n", " \n", + " for x in docs:\n", + " abrir = open(x).read()\n", + " corpus.append(abrir)\n", + " \n", + " sin_puntuacion(corpus)\n", + " \n", + " sin_tags(corpus2)\n", + " \n", + " sin_unicodes(quotes)\n", + " \n", + " for x in sincodes:\n", + " y = re.split(\" \", x)\n", + " for z in y:\n", + " if z not in bag_of_words and z not in stop_words:\n", + " bag_of_words.append(z)\n", + " else:\n", + " pass\n", + " \n", + " pieces = []\n", + " for x in sincodes:\n", + " pieces.append(x.split(\" \"))\n", + " \n", + " for s in pieces:\n", + " temp = []\n", + " for b in bag_of_words:\n", + " y = s.count(b)\n", + " temp.append(y)\n", + " term_freq.append(temp)\n", " return {\n", " \"bag_of_words\": bag_of_words,\n", " \"term_freq\": term_freq\n", @@ -60,9 +114,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "TypeError", + "evalue": "expected string or bytes-like object", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;34m'www.lipsum.com.html'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m ],\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mstop_words\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mENGLISH_STOP_WORDS\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 8\u001b[0m )\n\u001b[1;32m 9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mget_bow_from_docs\u001b[0;34m(docs, stop_words)\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0msin_puntuacion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcorpus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m \u001b[0msin_tags\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcorpus2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[0msin_unicodes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquotes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36msin_tags\u001b[0;34m(corpus)\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0mquotes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mre\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msub\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mr\"\\<.*?\\>\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcorpus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mquotes\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/anaconda3/lib/python3.7/re.py\u001b[0m in \u001b[0;36msub\u001b[0;34m(pattern, repl, string, count, flags)\u001b[0m\n\u001b[1;32m 190\u001b[0m \u001b[0ma\u001b[0m \u001b[0mcallable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mit\u001b[0m\u001b[0;31m'\u001b[0m\u001b[0ms\u001b[0m \u001b[0mpassed\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mMatch\u001b[0m \u001b[0mobject\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mmust\u001b[0m \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 191\u001b[0m a replacement string to be used.\"\"\"\n\u001b[0;32m--> 192\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_compile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpattern\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msub\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrepl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstring\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcount\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 193\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 194\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0msubn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpattern\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrepl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstring\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcount\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mTypeError\u001b[0m: expected string or bytes-like object" + ] + } + ], "source": [ "from sklearn.feature_extraction import stop_words\n", "bow = get_bow_from_docs([\n", @@ -111,7 +180,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.6" + "version": "3.7.3" } }, "nbformat": 4, diff --git a/your-code/doc1.txt b/your-code/doc1.txt new file mode 100644 index 0000000..e66288d --- /dev/null +++ b/your-code/doc1.txt @@ -0,0 +1 @@ +Ironhack is cool. \ No newline at end of file diff --git a/your-code/doc2.txt b/your-code/doc2.txt new file mode 100644 index 0000000..b21feac --- /dev/null +++ b/your-code/doc2.txt @@ -0,0 +1 @@ +I love Ironhack. \ No newline at end of file diff --git a/your-code/doc3.txt b/your-code/doc3.txt new file mode 100644 index 0000000..653c5b7 --- /dev/null +++ b/your-code/doc3.txt @@ -0,0 +1 @@ +I am a student at Ironhack. \ No newline at end of file