diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..07425ce Binary files /dev/null and b/.DS_Store differ diff --git a/your-code/.DS_Store b/your-code/.DS_Store new file mode 100644 index 0000000..c46ab82 Binary files /dev/null and b/your-code/.DS_Store differ diff --git a/your-code/.ipynb_checkpoints/INTERNATIONAL NEWS PARSER-checkpoint.ipynb b/your-code/.ipynb_checkpoints/INTERNATIONAL NEWS PARSER-checkpoint.ipynb new file mode 100644 index 0000000..979bf2b --- /dev/null +++ b/your-code/.ipynb_checkpoints/INTERNATIONAL NEWS PARSER-checkpoint.ipynb @@ -0,0 +1,441 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

INTERNATIONAL NEWS

\n", + "\n", + "This is a web parser to identify title, summary and links of the international breaking news as published by different media of the Ibero-american region:\n", + "\n", + "Right now parses:\n", + "\n", + "\n", + "\n", + "
  • El País (Spain)
    • \n", + "
  • La Vanguardia (Spain)
    • \n", + "
  • El Milenio (México)
    • \n", + "
  • Clarín (Argentina)
    • \n" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from bs4 import BeautifulSoup\n", + "import html5lib\n", + "import lxml\n", + "import requests\n", + "import re" + ] + }, + { + "cell_type": "code", + "execution_count": 154, + "metadata": {}, + "outputs": [], + "source": [ + "import re" + ] + }, + { + "cell_type": "code", + "execution_count": 205, + "metadata": {}, + "outputs": [], + "source": [ + "class InternationalNews:\n", + " \n", + " def __init__(self):\n", + " \n", + " self.el_pais_internacional = []\n", + " self.el_clarin_internacional = []\n", + " self.output = pd.DataFrame(columns=['Titular', 'Summary', 'Link', 'Source', 'Country'])\n", + " self.lv_internacional = []\n", + " self.milenio_int = []\n", + " \n", + " def get_el_pais(self):\n", + " \n", + " el_pais = requests.get('https://elpais.com/tag/c/15148420ba519668342b7a63149cad97').content\n", + " el_pais_soup = BeautifulSoup(el_pais, 'html5lib')\n", + " el_pais_soup = el_pais_soup.select('body')\n", + " el_pais_soup = el_pais_soup[0]\n", + " el_pais_soup1 = el_pais_soup.find_all('div', {'id':'contenedor'})\n", + " el_pais_soup1 = el_pais_soup1[0]\n", + " el_pais_soup1 = el_pais_soup1.find_all('main', {'class':'principal'})\n", + " el_pais_soup1 = el_pais_soup1[0]\n", + " el_pais_soup1 = el_pais_soup1.find_all('section', {'class':'bloque_automatico'})\n", + " el_pais_soup1 = el_pais_soup1[0]\n", + " el_pais_soup1 = el_pais_soup1.find_all('div', {'class':'bloque__interior'})\n", + " el_pais_soup1 = el_pais_soup1[0]\n", + " el_pais_soup1 = el_pais_soup1.find_all('div', {'class':'articulos__interior'})\n", + " \n", + " for i in range(len(el_pais_soup1)):\n", + " l = []\n", + " titular = el_pais_soup1[i].select('h2')\n", + " titular = titular[0].text\n", + " l.append(titular)\n", + " subtitulo = el_pais_soup1[i].select('p', {'class':'articulo-entradilla'})\n", + " subtitulo = subtitulo[0].text\n", + " l.append(subtitulo)\n", + " link = el_pais_soup1[i].a['href']\n", + " l.append('https:'+link)\n", + " l.append('El Pais')\n", + " l.append('España')\n", + "\n", + " self.el_pais_internacional.append(l)\n", + "\n", + " return self.el_pais_internacional\n", + " \n", + " def clarin(self):\n", + " \n", + " requests.get('https://www.clarin.com/mundo/').content\n", + " clarin_soup = BeautifulSoup(clarin, 'html5lib')\n", + " clarin_soup = clarin_soup.find_all('div', {'class':'content home'})\n", + " clarin_soup = clarin_soup[0]\n", + " clarin_soup = clarin_soup.find_all('div', {'class':'box-notas'})\n", + " clarin_soup = clarin_soup[0]\n", + " l = []\n", + " nota1 = clarin_soup.find_all('div', {'class':'mt pull-left'})\n", + " nota1 = nota1[0]\n", + " nota1 = nota1.a\n", + " link = nota1['href']\n", + " l.append('https://www.clarin.com'+link)\n", + " titular = nota1.select('h2')\n", + " titular = titular[0]\n", + " l.append(titular.text)\n", + " summary = nota1.select('p')\n", + " summary = summary[1].text\n", + " l.append(summary)\n", + " l.append('Clarin')\n", + " l.append('Argentina')\n", + " self.el_clarin_internacional.append(l)\n", + " l = []\n", + " nota2 = clarin_soup.find_all('div', {'class':'box col-lg-3 col-md-4 col-sm-6 col-xs-12 no-p border'})\n", + " nota2 = nota2[0]\n", + " nota2 = nota2.a\n", + " link = nota2['href']\n", + " title = nota2.select('figure')\n", + " title = title[0]\n", + " title = title.select('img')\n", + " title = title[0]\n", + " title = title['alt']\n", + " l.append(title)\n", + " l.append('-')\n", + " l.append('https://www.clarin.com'+link)\n", + " l.append('Clarín')\n", + " l.append('Argentina')\n", + " self.el_clarin_internacional.append(l)\n", + "\n", + " return self.el_clarin_internacional\n", + " \n", + " \n", + " def lv(self):\n", + " lv = requests.get('https://www.lavanguardia.com/internacional').content\n", + " lv_soup = BeautifulSoup(lv,'html5lib')\n", + " lv_soup = BeautifulSoup(lv,'html5lib')\n", + " lv_body = lv_soup.select('body')\n", + " lv_div1 = lv_body[0].select('section', {'class':\"section-block breakingnews small\"})\n", + " lv_div1 = lv_div1[0].select('header', {'class':\"story-header-title-link\"})\n", + " lv_div1 = [self.lv_internacional.append([lv_div1[i].a['title'], '-', lv_div1[i].a['href'], 'La Vanguardia', 'España']) for i in range(5)]\n", + "\n", + " return self.lv_internacional\n", + " \n", + " def milenio(self):\n", + " \n", + " milenio = requests.get('https://www.milenio.com/internacional').content\n", + " milenio_soup = BeautifulSoup(milenio,'html5lib')\n", + " body = milenio_soup.select('body')\n", + " notas = milenio_soup.find_all('div',{'class':'contenedor-notas-block'})\n", + " notas = notas[0].find_all('div', {'class':'title'})\n", + " for i in range(5):\n", + " l = []\n", + " link = notas[i].a['href']\n", + " titular = notas[i].text\n", + " titular = re.sub('\\s\\s+', '', titular)\n", + " l.append([titular, '-', link, 'Milenio', 'México'])\n", + " self.milenio_int.append(l)\n", + "\n", + " self.milenio_int = [y for i in self.milenio_int for y in i] \n", + " return self.milenio_int\n", + " \n", + " def main(self):\n", + " \n", + " self.clarin()\n", + " self.get_el_pais()\n", + " self.lv()\n", + " self.milenio()\n", + "\n", + " \n", + " dfc = [i for i in self.el_pais_internacional]+[i for i in self.el_clarin_internacional] + [i for i in self.lv_internacional] + [i for i in self.milenio_int]\n", + " self.output = pd.DataFrame(dfc, columns=['Titular', 'Summary', 'Link', 'Source', 'Country'])\n", + " \n", + " return self.output" + ] + }, + { + "cell_type": "code", + "execution_count": 207, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    TitularSummaryLinkSourceCountry
    0Mueren al menos 18 civiles, entre ellos un rep...Los ataques aéreos han sido en la región de Id...https://elpais.com/tag/guerra_civil_siria/aEl PaisEspaña
    1La ‘guerra herbicida’ de Israel arrasa cultivo...Un investigación constata que las fumigaciones...https://elpais.com/internacional/2019/07/21/ac...El PaisEspaña
    2Miles de manifestantes piden elecciones libres...Los asistentes exigen a la comisión electoral ...https://elpais.com/tag/moscu/aEl PaisEspaña
    3https://www.clarin.com/mundo/nueva-york-derrit...Nueva York se derrite y ubican a los “sin tech...La temperatura llegó a los 38° con una sensaci...ClarinArgentina
    4Theresa May cuenta las horas para su retiro fo...-https://www.clarin.com/mundo/theresa-may-cuent...ClarínArgentina
    5El día a día de los simpapeles bajo Trump-https://www.lavanguardia.com/internacional/201...La VanguardiaEspaña
    6“Si obedecéis, estaréis a salvo”, así abordó l...-https://www.lavanguardia.com/internacional/201...La VanguardiaEspaña
    7Las tropas de EE.UU. vuelven a Arabia al calor...-https://www.lavanguardia.com/internacional/201...La VanguardiaEspaña
    8British Airways cancela todos sus vuelos a El ...-https://www.lavanguardia.com/internacional/201...La VanguardiaEspaña
    9Un gran incendio sigue activo en Portugal, don...-https://www.lavanguardia.com/internacional/201...La VanguardiaEspaña
    10Trump instruye a NASA enviar a primera mujer a...-/internacional/donald-trump-instruye-nasa-envi...MilenioMéxico
    11Panamá, primer país en prohibir bolsas de plás...-/internacional/panama-el-primer-pais-en-prohib...MilenioMéxico
    12Michelle Obama es la mujer más admirada del mundo-/internacional/michelle-obama-es-la-mujer-mas-...MilenioMéxico
    13No nos arrodillaremos ante Washington: Maduro-/internacional/latinoamerica/no-nos-arrodillar...MilenioMéxico
    14Maduro recuerda muerte de Pancho Villa; envía ...-/internacional/maduro-recuerda-muerte-pancho-v...MilenioMéxico
    \n", + "
    " + ], + "text/plain": [ + " Titular \\\n", + "0 Mueren al menos 18 civiles, entre ellos un rep... \n", + "1 La ‘guerra herbicida’ de Israel arrasa cultivo... \n", + "2 Miles de manifestantes piden elecciones libres... \n", + "3 https://www.clarin.com/mundo/nueva-york-derrit... \n", + "4 Theresa May cuenta las horas para su retiro fo... \n", + "5 El día a día de los simpapeles bajo Trump \n", + "6 “Si obedecéis, estaréis a salvo”, así abordó l... \n", + "7 Las tropas de EE.UU. vuelven a Arabia al calor... \n", + "8 British Airways cancela todos sus vuelos a El ... \n", + "9 Un gran incendio sigue activo en Portugal, don... \n", + "10 Trump instruye a NASA enviar a primera mujer a... \n", + "11 Panamá, primer país en prohibir bolsas de plás... \n", + "12 Michelle Obama es la mujer más admirada del mundo \n", + "13 No nos arrodillaremos ante Washington: Maduro \n", + "14 Maduro recuerda muerte de Pancho Villa; envía ... \n", + "\n", + " Summary \\\n", + "0 Los ataques aéreos han sido en la región de Id... \n", + "1 Un investigación constata que las fumigaciones... \n", + "2 Los asistentes exigen a la comisión electoral ... \n", + "3 Nueva York se derrite y ubican a los “sin tech... \n", + "4 - \n", + "5 - \n", + "6 - \n", + "7 - \n", + "8 - \n", + "9 - \n", + "10 - \n", + "11 - \n", + "12 - \n", + "13 - \n", + "14 - \n", + "\n", + " Link Source \\\n", + "0 https://elpais.com/tag/guerra_civil_siria/a El Pais \n", + "1 https://elpais.com/internacional/2019/07/21/ac... El Pais \n", + "2 https://elpais.com/tag/moscu/a El Pais \n", + "3 La temperatura llegó a los 38° con una sensaci... Clarin \n", + "4 https://www.clarin.com/mundo/theresa-may-cuent... Clarín \n", + "5 https://www.lavanguardia.com/internacional/201... La Vanguardia \n", + "6 https://www.lavanguardia.com/internacional/201... La Vanguardia \n", + "7 https://www.lavanguardia.com/internacional/201... La Vanguardia \n", + "8 https://www.lavanguardia.com/internacional/201... La Vanguardia \n", + "9 https://www.lavanguardia.com/internacional/201... La Vanguardia \n", + "10 /internacional/donald-trump-instruye-nasa-envi... Milenio \n", + "11 /internacional/panama-el-primer-pais-en-prohib... Milenio \n", + "12 /internacional/michelle-obama-es-la-mujer-mas-... Milenio \n", + "13 /internacional/latinoamerica/no-nos-arrodillar... Milenio \n", + "14 /internacional/maduro-recuerda-muerte-pancho-v... Milenio \n", + "\n", + " Country \n", + "0 España \n", + "1 España \n", + "2 España \n", + "3 Argentina \n", + "4 Argentina \n", + "5 España \n", + "6 España \n", + "7 España \n", + "8 España \n", + "9 España \n", + "10 México \n", + "11 México \n", + "12 México \n", + "13 México \n", + "14 México " + ] + }, + "execution_count": 207, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "news = InternationalNews()\n", + "news.main()\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/your-code/.ipynb_checkpoints/YOUR APARTMENT-checkpoint.ipynb b/your-code/.ipynb_checkpoints/YOUR APARTMENT-checkpoint.ipynb new file mode 100644 index 0000000..7a0d210 --- /dev/null +++ b/your-code/.ipynb_checkpoints/YOUR APARTMENT-checkpoint.ipynb @@ -0,0 +1,545 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

    YOUR APARTMENT

    \n", + "\n", + "When moving, lots of things have to be considered, here we want to simplify a little of the task of looking for an apartment.\n", + "\n", + "The code, working with Google Geocode's API, Google Distance Matrix API and Foursquare API gives you an insight of:\n", + "
  • Your time to work from the apartment
    • \n", + "
  • Nearby filtered things and " + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": {}, + "outputs": [], + "source": [ + "from bs4 import BeautifulSoup\n", + "import re\n", + "import pandas as pd\n", + "import requests\n", + "import html5lib\n", + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": 119, + "metadata": {}, + "outputs": [], + "source": [ + "class YourApartment:\n", + " \n", + " def __init__(self):\n", + " \n", + " self.workplace = []\n", + " self.usr_home = []\n", + " self.categories = []\n", + " self.getwork = []\n", + " self.output_df = pd.DataFrame()\n", + " self.g_key = 'AIzaSyA19iq_u3Mqt8VQCgB_bflpHYxrIgBOzMM'\n", + " self.CLIENT_ID = 'SIWT0GVUWOBFB50H0TS5ZZ0LGKHMZHQUPKLLVKMYFXOYIQ2B' # your Foursquare ID\n", + " self.CLIENT_SECRET = 'F3FI2LD0U4KB0TQOPQPUM5QOE33KQLTTWPPP42IYBNSST4D4' # your Foursquare Secret\n", + " self.VERSION = '20180605'\n", + " self.radius = 500\n", + " self.LIMIT = 5 \n", + " \n", + " \n", + " def get_preferences(self):\n", + " SUB_CATEGORIES = {\"Ocio\":{\n", + " \"Cine\":'4bf58dd8d48988d17f941735', \"Galería de arte\":'4bf58dd8d48988d1e2931735', \n", + " \"Bowling\":\"4bf58dd8d48988d1e4931735\", \"Museo\":\"4bf58dd8d48988d181941735\",\n", + " \"Lugar de música\":\"4bf58dd8d48988d1e5931735\"}, \n", + " \"Comida\":{\"Restaurante asiático\":\"4bf58dd8d48988d142941735\",\n", + " \"Panadería\":\"4bf58dd8d48988d16a941735\",\n", + " \"Café\":\"4bf58dd8d48988d143941735\",\n", + " \"Buffet\":\"52e81612bcbc57f1066b79f4\",\n", + " \"Cafetería\":\"4bf58dd8d48988d1e0931735\",\n", + " \"Confitería\":\"4bf58dd8d48988d1d0941735\",\n", + " \"Cafetería\":\"4bf58dd8d48988d147941735\",\n", + " \"Tienda de donuts\":\"4bf58dd8d48988d148941735\",\n", + " \"Restaurante inglés\":\"52e81612bcbc57f1066b7a05\",\n", + " \"Restaurante de comida rápida\":\"4bf58dd8d48988d16e941735\",\n", + " \"Puesto de comida\":\"56aa371be4b08b9a8d57350b\",\n", + " \"Restaurante francés\":\"4bf58dd8d48988d10c941735\",\n", + " \"Restaurante italiano\":\"4bf58dd8d48988d110941735\",\n", + " \"Restaurante latinoamericano\":\"4bf58dd8d48988d1be941735\",\n", + " \"Restaurante mexicano\":\"4bf58dd8d48988d1c1941735\",\n", + " \"Pizzería\":\"4bf58dd8d48988d1ca941735\",\n", + " \"Restaurante español\":\"4bf58dd8d48988d150941735\",\n", + " \"Restaurante vegetariano/vegano\":\"4bf58dd8d48988d1d3941735\"},\n", + " \"Bares\":{\"Bar de cerveza\":\"56aa371ce4b08b9a8d57356c\",\n", + " \"Bar al aire libre\":\"4bf58dd8d48988d117941735\",\n", + " \"Antro\":\"4bf58dd8d48988d118941735\",\n", + " \"Cocteles\":\"4bf58dd8d48988d11e941735\",\n", + " \"Bar gay\":\"4bf58dd8d48988d1d8941735\",\n", + " \"Karaoke Bar\":\"4bf58dd8d48988d120941735\",\n", + " \"Pub\":\"4bf58dd8d48988d11b941735\",\n", + " \"Bar clandestino\":\"4bf58dd8d48988d1d4941735\",\n", + " \"Putero\":\"4bf58dd8d48988d1d6941735\"},\n", + " \"Deportes\":{\"Boxeo\":\"52f2ab2ebcbc57f1066b8b47\",\n", + " \"Escalada\":\"503289d391d4c4b30a586d6a\",\n", + " \"Spinning\":\"52f2ab2ebcbc57f1066b8b49\",\n", + " \"Gym Fitness\":\"52f2ab2ebcbc57f1066b8b48,4bf58dd8d48988d176941735\",\n", + " \"Dojo\":\"4bf58dd8d48988d101941735\",\n", + " \"Pilates\":\"\",\n", + " \"Yoga\":\"\",\n", + " \"Cancha de Fútbol\":\"4cce455aebf7b749d5e191f5\",\n", + " \"Cancha de Rugby\":\"52e81612bcbc57f1066b7a2c\",\n", + " \"Cancha de Tenis\":\"4e39a956bd410d7aed40cbc3\"},\n", + " \"Educación\":{\"Colegio de primaria\":\"4f4533804b9074f6e4fb0105\",\n", + " \"Colegio Secundaria\":\"4bf58dd8d48988d13d941735\",\n", + " \"Escuela de idiomas\":\"52e81612bcbc57f1066b7a48\",\n", + " \"Escuela de música\":\"4f04b10d2fb6e1c99f3db0be\",\n", + " \"Kinder\": \"4f4533814b9074f6e4fb0107\"}}\n", + "\n", + " MAJOR_CATEGORIES = {\"Ocio\":\"4d4b7104d754a06370d81259\", \"Comida\":\"4d4b7105d754a06374d81259\", \"Bares\":\"4bf58dd8d48988d116941735\",\n", + " \"Atletismo y Deportes\":\"4f4528bc4b90abdf24c9de85\", \"Educación\":\"4bf58dd8d48988d13b941735\"}\n", + "\n", + " m_keys = list(MAJOR_CATEGORIES.keys())\n", + "\n", + " for category in m_keys:\n", + " print( str(m_keys.index(category)) +'.'+category)\n", + "\n", + " l = [] \n", + " while len(l)<=2:\n", + " category = input('Wich places are you interested in?: (Number of the category i.e. 2)\\n')\n", + " l.append(category)\n", + "\n", + " l = [int(i) for i in l]\n", + "\n", + " for category in l:\n", + " continue_ = input(f'Do you want to filter subcategories of {m_keys[category]}[y/n]? ')\n", + "\n", + " if continue_ == 'y':\n", + "\n", + " sub_cat_keys = list(SUB_CATEGORIES[m_keys[category]].keys())\n", + "\n", + " for x in range(len(sub_cat_keys)):\n", + " print( str(x) + \".\" + sub_cat_keys[x] +\"\\n\")\n", + "\n", + "\n", + " sub_cats = input('Choose your sub-categories in a list with no spaces:(i.e. 1,2,3):\\n')\n", + " sub_cats = sub_cats.split(',')\n", + " sub_cats_l = [SUB_CATEGORIES[m_keys[category]][sub_cat_keys[int(i)]] for i in sub_cats]\n", + " sub_cat_vals = \",\".join(sub_cats_l)\n", + " self.categories.append(sub_cat_vals)\n", + " else:\n", + " self.categories.append(MAJOR_CATEGORIES[m_keys[category]])\n", + "\n", + " return self.categories\n", + " \n", + " \n", + " def get_work(self):\n", + " Trafic = (requests.get('https://maps.googleapis.com/maps/api/distancematrix/json?origins=%s,%s&destinations=%s,%s&arrival_time=1563180306&key=%s'%(self.usr_home[0], self.usr_home[1], self.workplace[0], self.workplace[1], self.g_key))).json()\n", + " self.getwork = [Trafic['rows'][0]['elements'][0]['distance']['text'], Trafic['rows'][0]['elements'][0]['duration']['text']]\n", + "\n", + " return self.getwork\n", + " \n", + " def user_home(self):\n", + " #INPUTS PARA LA UBICACIÓN DEL APARTAMENTO\n", + " print('Input your apartment address:')\n", + " num = input('Number adress: ')\n", + " street = input('Street name: ').split()\n", + " city = input('City name: ').split()\n", + " state = input('State name: ').split()\n", + " country = input('Country name: ').split()\n", + " query = num + \"+\" + \"+\".join(street) +','+ '+' + '+'.join(state) +','+ '+' + '+'.join(country)\n", + " lang_lat = (requests.get('https://maps.googleapis.com/maps/api/geocode/json?address=%s&key=%s' % (query, self.g_key) )).json()\n", + " self.usr_home = [lang_lat['results'][0]['geometry']['location']['lat'], lang_lat['results'][0]['geometry']['location']['lng']]\n", + " return self.usr_home\n", + "\n", + " def user_workplace(self):\n", + " print('Input your workplace address to calculate the distance to work:')\n", + " num = input('Number adress: ')\n", + " street = input('Street name: ').split()\n", + " city = input('City name: ').split()\n", + " state = input('State name: ').split()\n", + " country = input('Country name: ').split()\n", + " query = num + \"+\" + \"+\".join(street) +','+ '+' + '+'.join(state) +','+ '+' + '+'.join(country)\n", + " w_lang_lat = (requests.get('https://maps.googleapis.com/maps/api/geocode/json?address=%s&key=%s' % (query, self.g_key) )).json()\n", + " self.workplace = [ w_lang_lat['results'][0]['geometry']['location']['lat'], w_lang_lat['results'][0]['geometry']['location']['lng']]\n", + " return self.workplace\n", + " \n", + " def get_venue_data_frame_by_category(self):\n", + " \n", + " for category in self.categories:\n", + " details = (requests.get('https://api.foursquare.com/v2/venues/search?client_id=%s&client_secret=%s&ll=%s,%s&v=%s&radius=%s&limit=%s&categoryId=%s'% (self.CLIENT_ID, self.CLIENT_SECRET, self.usr_home[0], self.usr_home[1], self.VERSION, self.radius, self.LIMIT, category))).json()\n", + " venue_ids = [details['response']['venues'][i]['id'] for i in range(self.LIMIT)]\n", + " info = [[] for i in range(len(venue_ids))]\n", + " for ID in venue_ids:\n", + " for i in range(len(venue_ids)):\n", + " venue_detail = (requests.get('https://api.foursquare.com/v2/venues/%s?client_id=%s&client_secret=%s&v=%s'%(ID, self.CLIENT_ID, self.CLIENT_SECRET, self.VERSION))).json()\n", + " try:\n", + " info[i].append(venue_detail['response']['venue']['categories'][1]['name'])\n", + " except:\n", + " info[i].append('nan')\n", + " try:\n", + " info[i].append(venue_detail['response']['venue']['price']['message'])\n", + " except:\n", + " info[i].append('nan')\n", + " try:\n", + " info[i].append(venue_detail['response']['venue']['rating'])\n", + " except:\n", + " info[i].append('nan')\n", + " venues = [] \n", + " for i in range(len(venue_ids)):\n", + " venues.append([\n", + " details['response']['venues'][i]['name'],\n", + " details['response']['venues'][i]['categories'][0]['name'],\n", + " info[i][0],\n", + " info[i][1],\n", + " info[i][2],\n", + " (details['response']['venues'][i]['location']['lat'],\n", + " details['response']['venues'][i]['location']['lng'])\n", + " ])\n", + " self.output_df = pd.concat([self.output_df, pd.DataFrame(venues)])\n", + " \n", + " return self.output_df \n", + " \n", + "\n", + " def main(self):\n", + " \n", + " self.user_home()\n", + " \n", + " self.user_workplace()\n", + " \n", + " self.get_work()\n", + " \n", + " self.get_preferences()\n", + " \n", + " self.get_venue_data_frame_by_category()\n", + " \n", + " print('\\n\\n\\n It will take you %s to get to your workplace, it is %s away.'%(self.getwork[0],self.getwork[1]) ) \n", + " print('\\n\\n These are the best places around the apartment:')\n", + " self.output_df.columns = ['Venue Name', 'Category', 'Sub-category', 'Price', 'Rating', 'Location']\n", + " return self.output_df\n", + " \n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 121, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Input your apartment address:\n", + "Number adress: 1031\n", + "Street name: ejercito nacional\n", + "City name: cdmx\n", + "State name: cdmx\n", + "Country name: mexico\n", + "Input your workplace address to calculate the distance to work:\n", + "Number adress: 10\n", + "Street name: tonala\n", + "City name: cdmx\n", + "State name: cdmx\n", + "Country name: mexico\n", + "0.Ocio\n", + "1.Comida\n", + "2.Bares\n", + "3.Atletismo y Deportes\n", + "4.Educación\n", + "Wich places are you interested in?: (Number of the category i.e. 2)\n", + "1\n", + "Wich places are you interested in?: (Number of the category i.e. 2)\n", + "2\n", + "Wich places are you interested in?: (Number of the category i.e. 2)\n", + "3\n", + "Do you want to filter subcategories of Comida[y/n]? y\n", + "0.Restaurante asiático\n", + "\n", + "1.Panadería\n", + "\n", + "2.Café\n", + "\n", + "3.Buffet\n", + "\n", + "4.Cafetería\n", + "\n", + "5.Confitería\n", + "\n", + "6.Tienda de donuts\n", + "\n", + "7.Restaurante inglés\n", + "\n", + "8.Restaurante de comida rápida\n", + "\n", + "9.Puesto de comida\n", + "\n", + "10.Restaurante francés\n", + "\n", + "11.Restaurante italiano\n", + "\n", + "12.Restaurante latinoamericano\n", + "\n", + "13.Restaurante mexicano\n", + "\n", + "14.Pizzería\n", + "\n", + "15.Restaurante español\n", + "\n", + "16.Restaurante vegetariano/vegano\n", + "\n", + "Choose your sub-categories in a list with no spaces:(i.e. 1,2,3):\n", + "0,15,9\n", + "Do you want to filter subcategories of Bares[y/n]? n\n", + "Do you want to filter subcategories of Atletismo y Deportes[y/n]? n\n", + "It will take you 8.2 km to get to your workplace, it is 23 mins away.\n", + "These are the best places around the apartment:\n" + ] + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    Venue NameCategorySub-categoryPriceRatingLocation
    0Panda ExpressChinese Restaurantnannannan(19.44073457176492, -99.20676725053112)
    1Sushi RollSushi Restaurantnannannan(19.437018923638274, -99.20711674331213)
    2Sushi RollSushi Restaurantnannannan(19.441363343847247, -99.2074353401175)
    3Tianguis De Los LunesFood Standnannannan(19.44016, -99.20423)
    4P.F. Chang'sChinese Restaurantnannannan(19.440989706016158, -99.20456329250248)
    0Peace & Love New YorkDeli / Bodeganannannan(19.43786620111166, -99.2100337783336)
    1Chili's PolancoTex-Mex Restaurantnannannan(19.436844054803938, -99.20694667338729)
    2La Palapa @clubmundetBarnannannan(19.437516983898295, -99.21268414601322)
    3Guarida del Mononúcleo (ter)Whisky Barnannannan(19.436774013837002, -99.21207287357824)
    4James & Collins Boutique RestaurantWine Barnannannan(19.436205719162544, -99.20497541086422)
    0Club MundetParknannannan(19.439580795674093, -99.21140210208443)
    1Sport City Fitness ClubGymnannannan(19.441073919865204, -99.20640538528268)
    2AcuaticGym / Fitness Centernannannan(19.43612368046696, -99.20755108007091)
    3Sport City Fitness ClubGym / Fitness Centernannannan(19.441243633660456, -99.20442987145387)
    4CommandoGym / Fitness Centernannannan(19.43611546973738, -99.2051436792559)
    \n", + "
    " + ], + "text/plain": [ + " Venue Name Category Sub-category \\\n", + "0 Panda Express Chinese Restaurant nan \n", + "1 Sushi Roll Sushi Restaurant nan \n", + "2 Sushi Roll Sushi Restaurant nan \n", + "3 Tianguis De Los Lunes Food Stand nan \n", + "4 P.F. Chang's Chinese Restaurant nan \n", + "0 Peace & Love New York Deli / Bodega nan \n", + "1 Chili's Polanco Tex-Mex Restaurant nan \n", + "2 La Palapa @clubmundet Bar nan \n", + "3 Guarida del Mononúcleo (ter) Whisky Bar nan \n", + "4 James & Collins Boutique Restaurant Wine Bar nan \n", + "0 Club Mundet Park nan \n", + "1 Sport City Fitness Club Gym nan \n", + "2 Acuatic Gym / Fitness Center nan \n", + "3 Sport City Fitness Club Gym / Fitness Center nan \n", + "4 Commando Gym / Fitness Center nan \n", + "\n", + " Price Rating Location \n", + "0 nan nan (19.44073457176492, -99.20676725053112) \n", + "1 nan nan (19.437018923638274, -99.20711674331213) \n", + "2 nan nan (19.441363343847247, -99.2074353401175) \n", + "3 nan nan (19.44016, -99.20423) \n", + "4 nan nan (19.440989706016158, -99.20456329250248) \n", + "0 nan nan (19.43786620111166, -99.2100337783336) \n", + "1 nan nan (19.436844054803938, -99.20694667338729) \n", + "2 nan nan (19.437516983898295, -99.21268414601322) \n", + "3 nan nan (19.436774013837002, -99.21207287357824) \n", + "4 nan nan (19.436205719162544, -99.20497541086422) \n", + "0 nan nan (19.439580795674093, -99.21140210208443) \n", + "1 nan nan (19.441073919865204, -99.20640538528268) \n", + "2 nan nan (19.43612368046696, -99.20755108007091) \n", + "3 nan nan (19.441243633660456, -99.20442987145387) \n", + "4 nan nan (19.43611546973738, -99.2051436792559) " + ] + }, + "execution_count": 121, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "departamento = YourApartment()\n", + "departamento.main()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/your-code/INTERNATIONAL NEWS PARSER.ipynb b/your-code/INTERNATIONAL NEWS PARSER.ipynb new file mode 100644 index 0000000..979bf2b --- /dev/null +++ b/your-code/INTERNATIONAL NEWS PARSER.ipynb @@ -0,0 +1,441 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

    INTERNATIONAL NEWS

    \n", + "\n", + "This is a web parser to identify title, summary and links of the international breaking news as published by different media of the Ibero-american region:\n", + "\n", + "Right now parses:\n", + "\n", + "\n", + "\n", + "
  • El País (Spain)
    • \n", + "
  • La Vanguardia (Spain)
    • \n", + "
  • El Milenio (México)
    • \n", + "
  • Clarín (Argentina)
    • \n" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from bs4 import BeautifulSoup\n", + "import html5lib\n", + "import lxml\n", + "import requests\n", + "import re" + ] + }, + { + "cell_type": "code", + "execution_count": 154, + "metadata": {}, + "outputs": [], + "source": [ + "import re" + ] + }, + { + "cell_type": "code", + "execution_count": 205, + "metadata": {}, + "outputs": [], + "source": [ + "class InternationalNews:\n", + " \n", + " def __init__(self):\n", + " \n", + " self.el_pais_internacional = []\n", + " self.el_clarin_internacional = []\n", + " self.output = pd.DataFrame(columns=['Titular', 'Summary', 'Link', 'Source', 'Country'])\n", + " self.lv_internacional = []\n", + " self.milenio_int = []\n", + " \n", + " def get_el_pais(self):\n", + " \n", + " el_pais = requests.get('https://elpais.com/tag/c/15148420ba519668342b7a63149cad97').content\n", + " el_pais_soup = BeautifulSoup(el_pais, 'html5lib')\n", + " el_pais_soup = el_pais_soup.select('body')\n", + " el_pais_soup = el_pais_soup[0]\n", + " el_pais_soup1 = el_pais_soup.find_all('div', {'id':'contenedor'})\n", + " el_pais_soup1 = el_pais_soup1[0]\n", + " el_pais_soup1 = el_pais_soup1.find_all('main', {'class':'principal'})\n", + " el_pais_soup1 = el_pais_soup1[0]\n", + " el_pais_soup1 = el_pais_soup1.find_all('section', {'class':'bloque_automatico'})\n", + " el_pais_soup1 = el_pais_soup1[0]\n", + " el_pais_soup1 = el_pais_soup1.find_all('div', {'class':'bloque__interior'})\n", + " el_pais_soup1 = el_pais_soup1[0]\n", + " el_pais_soup1 = el_pais_soup1.find_all('div', {'class':'articulos__interior'})\n", + " \n", + " for i in range(len(el_pais_soup1)):\n", + " l = []\n", + " titular = el_pais_soup1[i].select('h2')\n", + " titular = titular[0].text\n", + " l.append(titular)\n", + " subtitulo = el_pais_soup1[i].select('p', {'class':'articulo-entradilla'})\n", + " subtitulo = subtitulo[0].text\n", + " l.append(subtitulo)\n", + " link = el_pais_soup1[i].a['href']\n", + " l.append('https:'+link)\n", + " l.append('El Pais')\n", + " l.append('España')\n", + "\n", + " self.el_pais_internacional.append(l)\n", + "\n", + " return self.el_pais_internacional\n", + " \n", + " def clarin(self):\n", + " \n", + " requests.get('https://www.clarin.com/mundo/').content\n", + " clarin_soup = BeautifulSoup(clarin, 'html5lib')\n", + " clarin_soup = clarin_soup.find_all('div', {'class':'content home'})\n", + " clarin_soup = clarin_soup[0]\n", + " clarin_soup = clarin_soup.find_all('div', {'class':'box-notas'})\n", + " clarin_soup = clarin_soup[0]\n", + " l = []\n", + " nota1 = clarin_soup.find_all('div', {'class':'mt pull-left'})\n", + " nota1 = nota1[0]\n", + " nota1 = nota1.a\n", + " link = nota1['href']\n", + " l.append('https://www.clarin.com'+link)\n", + " titular = nota1.select('h2')\n", + " titular = titular[0]\n", + " l.append(titular.text)\n", + " summary = nota1.select('p')\n", + " summary = summary[1].text\n", + " l.append(summary)\n", + " l.append('Clarin')\n", + " l.append('Argentina')\n", + " self.el_clarin_internacional.append(l)\n", + " l = []\n", + " nota2 = clarin_soup.find_all('div', {'class':'box col-lg-3 col-md-4 col-sm-6 col-xs-12 no-p border'})\n", + " nota2 = nota2[0]\n", + " nota2 = nota2.a\n", + " link = nota2['href']\n", + " title = nota2.select('figure')\n", + " title = title[0]\n", + " title = title.select('img')\n", + " title = title[0]\n", + " title = title['alt']\n", + " l.append(title)\n", + " l.append('-')\n", + " l.append('https://www.clarin.com'+link)\n", + " l.append('Clarín')\n", + " l.append('Argentina')\n", + " self.el_clarin_internacional.append(l)\n", + "\n", + " return self.el_clarin_internacional\n", + " \n", + " \n", + " def lv(self):\n", + " lv = requests.get('https://www.lavanguardia.com/internacional').content\n", + " lv_soup = BeautifulSoup(lv,'html5lib')\n", + " lv_soup = BeautifulSoup(lv,'html5lib')\n", + " lv_body = lv_soup.select('body')\n", + " lv_div1 = lv_body[0].select('section', {'class':\"section-block breakingnews small\"})\n", + " lv_div1 = lv_div1[0].select('header', {'class':\"story-header-title-link\"})\n", + " lv_div1 = [self.lv_internacional.append([lv_div1[i].a['title'], '-', lv_div1[i].a['href'], 'La Vanguardia', 'España']) for i in range(5)]\n", + "\n", + " return self.lv_internacional\n", + " \n", + " def milenio(self):\n", + " \n", + " milenio = requests.get('https://www.milenio.com/internacional').content\n", + " milenio_soup = BeautifulSoup(milenio,'html5lib')\n", + " body = milenio_soup.select('body')\n", + " notas = milenio_soup.find_all('div',{'class':'contenedor-notas-block'})\n", + " notas = notas[0].find_all('div', {'class':'title'})\n", + " for i in range(5):\n", + " l = []\n", + " link = notas[i].a['href']\n", + " titular = notas[i].text\n", + " titular = re.sub('\\s\\s+', '', titular)\n", + " l.append([titular, '-', link, 'Milenio', 'México'])\n", + " self.milenio_int.append(l)\n", + "\n", + " self.milenio_int = [y for i in self.milenio_int for y in i] \n", + " return self.milenio_int\n", + " \n", + " def main(self):\n", + " \n", + " self.clarin()\n", + " self.get_el_pais()\n", + " self.lv()\n", + " self.milenio()\n", + "\n", + " \n", + " dfc = [i for i in self.el_pais_internacional]+[i for i in self.el_clarin_internacional] + [i for i in self.lv_internacional] + [i for i in self.milenio_int]\n", + " self.output = pd.DataFrame(dfc, columns=['Titular', 'Summary', 'Link', 'Source', 'Country'])\n", + " \n", + " return self.output" + ] + }, + { + "cell_type": "code", + "execution_count": 207, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    TitularSummaryLinkSourceCountry
    0Mueren al menos 18 civiles, entre ellos un rep...Los ataques aéreos han sido en la región de Id...https://elpais.com/tag/guerra_civil_siria/aEl PaisEspaña
    1La ‘guerra herbicida’ de Israel arrasa cultivo...Un investigación constata que las fumigaciones...https://elpais.com/internacional/2019/07/21/ac...El PaisEspaña
    2Miles de manifestantes piden elecciones libres...Los asistentes exigen a la comisión electoral ...https://elpais.com/tag/moscu/aEl PaisEspaña
    3https://www.clarin.com/mundo/nueva-york-derrit...Nueva York se derrite y ubican a los “sin tech...La temperatura llegó a los 38° con una sensaci...ClarinArgentina
    4Theresa May cuenta las horas para su retiro fo...-https://www.clarin.com/mundo/theresa-may-cuent...ClarínArgentina
    5El día a día de los simpapeles bajo Trump-https://www.lavanguardia.com/internacional/201...La VanguardiaEspaña
    6“Si obedecéis, estaréis a salvo”, así abordó l...-https://www.lavanguardia.com/internacional/201...La VanguardiaEspaña
    7Las tropas de EE.UU. vuelven a Arabia al calor...-https://www.lavanguardia.com/internacional/201...La VanguardiaEspaña
    8British Airways cancela todos sus vuelos a El ...-https://www.lavanguardia.com/internacional/201...La VanguardiaEspaña
    9Un gran incendio sigue activo en Portugal, don...-https://www.lavanguardia.com/internacional/201...La VanguardiaEspaña
    10Trump instruye a NASA enviar a primera mujer a...-/internacional/donald-trump-instruye-nasa-envi...MilenioMéxico
    11Panamá, primer país en prohibir bolsas de plás...-/internacional/panama-el-primer-pais-en-prohib...MilenioMéxico
    12Michelle Obama es la mujer más admirada del mundo-/internacional/michelle-obama-es-la-mujer-mas-...MilenioMéxico
    13No nos arrodillaremos ante Washington: Maduro-/internacional/latinoamerica/no-nos-arrodillar...MilenioMéxico
    14Maduro recuerda muerte de Pancho Villa; envía ...-/internacional/maduro-recuerda-muerte-pancho-v...MilenioMéxico
    \n", + "
    " + ], + "text/plain": [ + " Titular \\\n", + "0 Mueren al menos 18 civiles, entre ellos un rep... \n", + "1 La ‘guerra herbicida’ de Israel arrasa cultivo... \n", + "2 Miles de manifestantes piden elecciones libres... \n", + "3 https://www.clarin.com/mundo/nueva-york-derrit... \n", + "4 Theresa May cuenta las horas para su retiro fo... \n", + "5 El día a día de los simpapeles bajo Trump \n", + "6 “Si obedecéis, estaréis a salvo”, así abordó l... \n", + "7 Las tropas de EE.UU. vuelven a Arabia al calor... \n", + "8 British Airways cancela todos sus vuelos a El ... \n", + "9 Un gran incendio sigue activo en Portugal, don... \n", + "10 Trump instruye a NASA enviar a primera mujer a... \n", + "11 Panamá, primer país en prohibir bolsas de plás... \n", + "12 Michelle Obama es la mujer más admirada del mundo \n", + "13 No nos arrodillaremos ante Washington: Maduro \n", + "14 Maduro recuerda muerte de Pancho Villa; envía ... \n", + "\n", + " Summary \\\n", + "0 Los ataques aéreos han sido en la región de Id... \n", + "1 Un investigación constata que las fumigaciones... \n", + "2 Los asistentes exigen a la comisión electoral ... \n", + "3 Nueva York se derrite y ubican a los “sin tech... \n", + "4 - \n", + "5 - \n", + "6 - \n", + "7 - \n", + "8 - \n", + "9 - \n", + "10 - \n", + "11 - \n", + "12 - \n", + "13 - \n", + "14 - \n", + "\n", + " Link Source \\\n", + "0 https://elpais.com/tag/guerra_civil_siria/a El Pais \n", + "1 https://elpais.com/internacional/2019/07/21/ac... El Pais \n", + "2 https://elpais.com/tag/moscu/a El Pais \n", + "3 La temperatura llegó a los 38° con una sensaci... Clarin \n", + "4 https://www.clarin.com/mundo/theresa-may-cuent... Clarín \n", + "5 https://www.lavanguardia.com/internacional/201... La Vanguardia \n", + "6 https://www.lavanguardia.com/internacional/201... La Vanguardia \n", + "7 https://www.lavanguardia.com/internacional/201... La Vanguardia \n", + "8 https://www.lavanguardia.com/internacional/201... La Vanguardia \n", + "9 https://www.lavanguardia.com/internacional/201... La Vanguardia \n", + "10 /internacional/donald-trump-instruye-nasa-envi... Milenio \n", + "11 /internacional/panama-el-primer-pais-en-prohib... Milenio \n", + "12 /internacional/michelle-obama-es-la-mujer-mas-... Milenio \n", + "13 /internacional/latinoamerica/no-nos-arrodillar... Milenio \n", + "14 /internacional/maduro-recuerda-muerte-pancho-v... Milenio \n", + "\n", + " Country \n", + "0 España \n", + "1 España \n", + "2 España \n", + "3 Argentina \n", + "4 Argentina \n", + "5 España \n", + "6 España \n", + "7 España \n", + "8 España \n", + "9 España \n", + "10 México \n", + "11 México \n", + "12 México \n", + "13 México \n", + "14 México " + ] + }, + "execution_count": 207, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "news = InternationalNews()\n", + "news.main()\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/your-code/YOUR APARTMENT.ipynb b/your-code/YOUR APARTMENT.ipynb new file mode 100644 index 0000000..8d66d0a --- /dev/null +++ b/your-code/YOUR APARTMENT.ipynb @@ -0,0 +1,545 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

    YOUR APARTMENT

    \n", + "\n", + "When moving, lots of things have to be considered, here we want to simplify a little of the task of looking for an apartment.\n", + "\n", + "The code, working with Google Geocode's API, Google Distance Matrix API and Foursquare API gives you an insight of:\n", + "
  • Your time to work from the apartment
    • \n", + "
  • Nearby filtered places of interestest
    • \n" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": {}, + "outputs": [], + "source": [ + "from bs4 import BeautifulSoup\n", + "import re\n", + "import pandas as pd\n", + "import requests\n", + "import html5lib\n", + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": 119, + "metadata": {}, + "outputs": [], + "source": [ + "class YourApartment:\n", + " \n", + " def __init__(self):\n", + " \n", + " self.workplace = []\n", + " self.usr_home = []\n", + " self.categories = []\n", + " self.getwork = []\n", + " self.output_df = pd.DataFrame()\n", + " self.g_key = 'AIzaSyA19iq_u3Mqt8VQCgB_bflpHYxrIgBOzMM'\n", + " self.CLIENT_ID = 'SIWT0GVUWOBFB50H0TS5ZZ0LGKHMZHQUPKLLVKMYFXOYIQ2B' # your Foursquare ID\n", + " self.CLIENT_SECRET = 'F3FI2LD0U4KB0TQOPQPUM5QOE33KQLTTWPPP42IYBNSST4D4' # your Foursquare Secret\n", + " self.VERSION = '20180605'\n", + " self.radius = 500\n", + " self.LIMIT = 5 \n", + " \n", + " \n", + " def get_preferences(self):\n", + " SUB_CATEGORIES = {\"Ocio\":{\n", + " \"Cine\":'4bf58dd8d48988d17f941735', \"Galería de arte\":'4bf58dd8d48988d1e2931735', \n", + " \"Bowling\":\"4bf58dd8d48988d1e4931735\", \"Museo\":\"4bf58dd8d48988d181941735\",\n", + " \"Lugar de música\":\"4bf58dd8d48988d1e5931735\"}, \n", + " \"Comida\":{\"Restaurante asiático\":\"4bf58dd8d48988d142941735\",\n", + " \"Panadería\":\"4bf58dd8d48988d16a941735\",\n", + " \"Café\":\"4bf58dd8d48988d143941735\",\n", + " \"Buffet\":\"52e81612bcbc57f1066b79f4\",\n", + " \"Cafetería\":\"4bf58dd8d48988d1e0931735\",\n", + " \"Confitería\":\"4bf58dd8d48988d1d0941735\",\n", + " \"Cafetería\":\"4bf58dd8d48988d147941735\",\n", + " \"Tienda de donuts\":\"4bf58dd8d48988d148941735\",\n", + " \"Restaurante inglés\":\"52e81612bcbc57f1066b7a05\",\n", + " \"Restaurante de comida rápida\":\"4bf58dd8d48988d16e941735\",\n", + " \"Puesto de comida\":\"56aa371be4b08b9a8d57350b\",\n", + " \"Restaurante francés\":\"4bf58dd8d48988d10c941735\",\n", + " \"Restaurante italiano\":\"4bf58dd8d48988d110941735\",\n", + " \"Restaurante latinoamericano\":\"4bf58dd8d48988d1be941735\",\n", + " \"Restaurante mexicano\":\"4bf58dd8d48988d1c1941735\",\n", + " \"Pizzería\":\"4bf58dd8d48988d1ca941735\",\n", + " \"Restaurante español\":\"4bf58dd8d48988d150941735\",\n", + " \"Restaurante vegetariano/vegano\":\"4bf58dd8d48988d1d3941735\"},\n", + " \"Bares\":{\"Bar de cerveza\":\"56aa371ce4b08b9a8d57356c\",\n", + " \"Bar al aire libre\":\"4bf58dd8d48988d117941735\",\n", + " \"Antro\":\"4bf58dd8d48988d118941735\",\n", + " \"Cocteles\":\"4bf58dd8d48988d11e941735\",\n", + " \"Bar gay\":\"4bf58dd8d48988d1d8941735\",\n", + " \"Karaoke Bar\":\"4bf58dd8d48988d120941735\",\n", + " \"Pub\":\"4bf58dd8d48988d11b941735\",\n", + " \"Bar clandestino\":\"4bf58dd8d48988d1d4941735\",\n", + " \"Putero\":\"4bf58dd8d48988d1d6941735\"},\n", + " \"Deportes\":{\"Boxeo\":\"52f2ab2ebcbc57f1066b8b47\",\n", + " \"Escalada\":\"503289d391d4c4b30a586d6a\",\n", + " \"Spinning\":\"52f2ab2ebcbc57f1066b8b49\",\n", + " \"Gym Fitness\":\"52f2ab2ebcbc57f1066b8b48,4bf58dd8d48988d176941735\",\n", + " \"Dojo\":\"4bf58dd8d48988d101941735\",\n", + " \"Pilates\":\"\",\n", + " \"Yoga\":\"\",\n", + " \"Cancha de Fútbol\":\"4cce455aebf7b749d5e191f5\",\n", + " \"Cancha de Rugby\":\"52e81612bcbc57f1066b7a2c\",\n", + " \"Cancha de Tenis\":\"4e39a956bd410d7aed40cbc3\"},\n", + " \"Educación\":{\"Colegio de primaria\":\"4f4533804b9074f6e4fb0105\",\n", + " \"Colegio Secundaria\":\"4bf58dd8d48988d13d941735\",\n", + " \"Escuela de idiomas\":\"52e81612bcbc57f1066b7a48\",\n", + " \"Escuela de música\":\"4f04b10d2fb6e1c99f3db0be\",\n", + " \"Kinder\": \"4f4533814b9074f6e4fb0107\"}}\n", + "\n", + " MAJOR_CATEGORIES = {\"Ocio\":\"4d4b7104d754a06370d81259\", \"Comida\":\"4d4b7105d754a06374d81259\", \"Bares\":\"4bf58dd8d48988d116941735\",\n", + " \"Atletismo y Deportes\":\"4f4528bc4b90abdf24c9de85\", \"Educación\":\"4bf58dd8d48988d13b941735\"}\n", + "\n", + " m_keys = list(MAJOR_CATEGORIES.keys())\n", + "\n", + " for category in m_keys:\n", + " print( str(m_keys.index(category)) +'.'+category)\n", + "\n", + " l = [] \n", + " while len(l)<=2:\n", + " category = input('Wich places are you interested in?: (Number of the category i.e. 2)\\n')\n", + " l.append(category)\n", + "\n", + " l = [int(i) for i in l]\n", + "\n", + " for category in l:\n", + " continue_ = input(f'Do you want to filter subcategories of {m_keys[category]}[y/n]? ')\n", + "\n", + " if continue_ == 'y':\n", + "\n", + " sub_cat_keys = list(SUB_CATEGORIES[m_keys[category]].keys())\n", + "\n", + " for x in range(len(sub_cat_keys)):\n", + " print( str(x) + \".\" + sub_cat_keys[x] +\"\\n\")\n", + "\n", + "\n", + " sub_cats = input('Choose your sub-categories in a list with no spaces:(i.e. 1,2,3):\\n')\n", + " sub_cats = sub_cats.split(',')\n", + " sub_cats_l = [SUB_CATEGORIES[m_keys[category]][sub_cat_keys[int(i)]] for i in sub_cats]\n", + " sub_cat_vals = \",\".join(sub_cats_l)\n", + " self.categories.append(sub_cat_vals)\n", + " else:\n", + " self.categories.append(MAJOR_CATEGORIES[m_keys[category]])\n", + "\n", + " return self.categories\n", + " \n", + " \n", + " def get_work(self):\n", + " Trafic = (requests.get('https://maps.googleapis.com/maps/api/distancematrix/json?origins=%s,%s&destinations=%s,%s&arrival_time=1563180306&key=%s'%(self.usr_home[0], self.usr_home[1], self.workplace[0], self.workplace[1], self.g_key))).json()\n", + " self.getwork = [Trafic['rows'][0]['elements'][0]['distance']['text'], Trafic['rows'][0]['elements'][0]['duration']['text']]\n", + "\n", + " return self.getwork\n", + " \n", + " def user_home(self):\n", + " #INPUTS PARA LA UBICACIÓN DEL APARTAMENTO\n", + " print('Input your apartment address:')\n", + " num = input('Number adress: ')\n", + " street = input('Street name: ').split()\n", + " city = input('City name: ').split()\n", + " state = input('State name: ').split()\n", + " country = input('Country name: ').split()\n", + " query = num + \"+\" + \"+\".join(street) +','+ '+' + '+'.join(state) +','+ '+' + '+'.join(country)\n", + " lang_lat = (requests.get('https://maps.googleapis.com/maps/api/geocode/json?address=%s&key=%s' % (query, self.g_key) )).json()\n", + " self.usr_home = [lang_lat['results'][0]['geometry']['location']['lat'], lang_lat['results'][0]['geometry']['location']['lng']]\n", + " return self.usr_home\n", + "\n", + " def user_workplace(self):\n", + " print('Input your workplace address to calculate the distance to work:')\n", + " num = input('Number adress: ')\n", + " street = input('Street name: ').split()\n", + " city = input('City name: ').split()\n", + " state = input('State name: ').split()\n", + " country = input('Country name: ').split()\n", + " query = num + \"+\" + \"+\".join(street) +','+ '+' + '+'.join(state) +','+ '+' + '+'.join(country)\n", + " w_lang_lat = (requests.get('https://maps.googleapis.com/maps/api/geocode/json?address=%s&key=%s' % (query, self.g_key) )).json()\n", + " self.workplace = [ w_lang_lat['results'][0]['geometry']['location']['lat'], w_lang_lat['results'][0]['geometry']['location']['lng']]\n", + " return self.workplace\n", + " \n", + " def get_venue_data_frame_by_category(self):\n", + " \n", + " for category in self.categories:\n", + " details = (requests.get('https://api.foursquare.com/v2/venues/search?client_id=%s&client_secret=%s&ll=%s,%s&v=%s&radius=%s&limit=%s&categoryId=%s'% (self.CLIENT_ID, self.CLIENT_SECRET, self.usr_home[0], self.usr_home[1], self.VERSION, self.radius, self.LIMIT, category))).json()\n", + " venue_ids = [details['response']['venues'][i]['id'] for i in range(self.LIMIT)]\n", + " info = [[] for i in range(len(venue_ids))]\n", + " for ID in venue_ids:\n", + " for i in range(len(venue_ids)):\n", + " venue_detail = (requests.get('https://api.foursquare.com/v2/venues/%s?client_id=%s&client_secret=%s&v=%s'%(ID, self.CLIENT_ID, self.CLIENT_SECRET, self.VERSION))).json()\n", + " try:\n", + " info[i].append(venue_detail['response']['venue']['categories'][1]['name'])\n", + " except:\n", + " info[i].append('nan')\n", + " try:\n", + " info[i].append(venue_detail['response']['venue']['price']['message'])\n", + " except:\n", + " info[i].append('nan')\n", + " try:\n", + " info[i].append(venue_detail['response']['venue']['rating'])\n", + " except:\n", + " info[i].append('nan')\n", + " venues = [] \n", + " for i in range(len(venue_ids)):\n", + " venues.append([\n", + " details['response']['venues'][i]['name'],\n", + " details['response']['venues'][i]['categories'][0]['name'],\n", + " info[i][0],\n", + " info[i][1],\n", + " info[i][2],\n", + " (details['response']['venues'][i]['location']['lat'],\n", + " details['response']['venues'][i]['location']['lng'])\n", + " ])\n", + " self.output_df = pd.concat([self.output_df, pd.DataFrame(venues)])\n", + " \n", + " return self.output_df \n", + " \n", + "\n", + " def main(self):\n", + " \n", + " self.user_home()\n", + " \n", + " self.user_workplace()\n", + " \n", + " self.get_work()\n", + " \n", + " self.get_preferences()\n", + " \n", + " self.get_venue_data_frame_by_category()\n", + " \n", + " print('\\n\\n\\n It will take you %s to get to your workplace, it is %s away.'%(self.getwork[0],self.getwork[1]) ) \n", + " print('\\n\\n These are the best places around the apartment:')\n", + " self.output_df.columns = ['Venue Name', 'Category', 'Sub-category', 'Price', 'Rating', 'Location']\n", + " return self.output_df\n", + " \n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 121, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Input your apartment address:\n", + "Number adress: 1031\n", + "Street name: ejercito nacional\n", + "City name: cdmx\n", + "State name: cdmx\n", + "Country name: mexico\n", + "Input your workplace address to calculate the distance to work:\n", + "Number adress: 10\n", + "Street name: tonala\n", + "City name: cdmx\n", + "State name: cdmx\n", + "Country name: mexico\n", + "0.Ocio\n", + "1.Comida\n", + "2.Bares\n", + "3.Atletismo y Deportes\n", + "4.Educación\n", + "Wich places are you interested in?: (Number of the category i.e. 2)\n", + "1\n", + "Wich places are you interested in?: (Number of the category i.e. 2)\n", + "2\n", + "Wich places are you interested in?: (Number of the category i.e. 2)\n", + "3\n", + "Do you want to filter subcategories of Comida[y/n]? y\n", + "0.Restaurante asiático\n", + "\n", + "1.Panadería\n", + "\n", + "2.Café\n", + "\n", + "3.Buffet\n", + "\n", + "4.Cafetería\n", + "\n", + "5.Confitería\n", + "\n", + "6.Tienda de donuts\n", + "\n", + "7.Restaurante inglés\n", + "\n", + "8.Restaurante de comida rápida\n", + "\n", + "9.Puesto de comida\n", + "\n", + "10.Restaurante francés\n", + "\n", + "11.Restaurante italiano\n", + "\n", + "12.Restaurante latinoamericano\n", + "\n", + "13.Restaurante mexicano\n", + "\n", + "14.Pizzería\n", + "\n", + "15.Restaurante español\n", + "\n", + "16.Restaurante vegetariano/vegano\n", + "\n", + "Choose your sub-categories in a list with no spaces:(i.e. 1,2,3):\n", + "0,15,9\n", + "Do you want to filter subcategories of Bares[y/n]? n\n", + "Do you want to filter subcategories of Atletismo y Deportes[y/n]? n\n", + "It will take you 8.2 km to get to your workplace, it is 23 mins away.\n", + "These are the best places around the apartment:\n" + ] + }, + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    Venue NameCategorySub-categoryPriceRatingLocation
    0Panda ExpressChinese Restaurantnannannan(19.44073457176492, -99.20676725053112)
    1Sushi RollSushi Restaurantnannannan(19.437018923638274, -99.20711674331213)
    2Sushi RollSushi Restaurantnannannan(19.441363343847247, -99.2074353401175)
    3Tianguis De Los LunesFood Standnannannan(19.44016, -99.20423)
    4P.F. Chang'sChinese Restaurantnannannan(19.440989706016158, -99.20456329250248)
    0Peace & Love New YorkDeli / Bodeganannannan(19.43786620111166, -99.2100337783336)
    1Chili's PolancoTex-Mex Restaurantnannannan(19.436844054803938, -99.20694667338729)
    2La Palapa @clubmundetBarnannannan(19.437516983898295, -99.21268414601322)
    3Guarida del Mononúcleo (ter)Whisky Barnannannan(19.436774013837002, -99.21207287357824)
    4James & Collins Boutique RestaurantWine Barnannannan(19.436205719162544, -99.20497541086422)
    0Club MundetParknannannan(19.439580795674093, -99.21140210208443)
    1Sport City Fitness ClubGymnannannan(19.441073919865204, -99.20640538528268)
    2AcuaticGym / Fitness Centernannannan(19.43612368046696, -99.20755108007091)
    3Sport City Fitness ClubGym / Fitness Centernannannan(19.441243633660456, -99.20442987145387)
    4CommandoGym / Fitness Centernannannan(19.43611546973738, -99.2051436792559)
    \n", + "
    " + ], + "text/plain": [ + " Venue Name Category Sub-category \\\n", + "0 Panda Express Chinese Restaurant nan \n", + "1 Sushi Roll Sushi Restaurant nan \n", + "2 Sushi Roll Sushi Restaurant nan \n", + "3 Tianguis De Los Lunes Food Stand nan \n", + "4 P.F. Chang's Chinese Restaurant nan \n", + "0 Peace & Love New York Deli / Bodega nan \n", + "1 Chili's Polanco Tex-Mex Restaurant nan \n", + "2 La Palapa @clubmundet Bar nan \n", + "3 Guarida del Mononúcleo (ter) Whisky Bar nan \n", + "4 James & Collins Boutique Restaurant Wine Bar nan \n", + "0 Club Mundet Park nan \n", + "1 Sport City Fitness Club Gym nan \n", + "2 Acuatic Gym / Fitness Center nan \n", + "3 Sport City Fitness Club Gym / Fitness Center nan \n", + "4 Commando Gym / Fitness Center nan \n", + "\n", + " Price Rating Location \n", + "0 nan nan (19.44073457176492, -99.20676725053112) \n", + "1 nan nan (19.437018923638274, -99.20711674331213) \n", + "2 nan nan (19.441363343847247, -99.2074353401175) \n", + "3 nan nan (19.44016, -99.20423) \n", + "4 nan nan (19.440989706016158, -99.20456329250248) \n", + "0 nan nan (19.43786620111166, -99.2100337783336) \n", + "1 nan nan (19.436844054803938, -99.20694667338729) \n", + "2 nan nan (19.437516983898295, -99.21268414601322) \n", + "3 nan nan (19.436774013837002, -99.21207287357824) \n", + "4 nan nan (19.436205719162544, -99.20497541086422) \n", + "0 nan nan (19.439580795674093, -99.21140210208443) \n", + "1 nan nan (19.441073919865204, -99.20640538528268) \n", + "2 nan nan (19.43612368046696, -99.20755108007091) \n", + "3 nan nan (19.441243633660456, -99.20442987145387) \n", + "4 nan nan (19.43611546973738, -99.2051436792559) " + ] + }, + "execution_count": 121, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "departamento = YourApartment()\n", + "departamento.main()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}