web_scraping.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python374jvsc74a57bd01aa32c3e39cce6b6a2a7f2e9227e06d8322f53ac0b3d2a2ea2c1a4485a851f90",
   "display_name": "Python 3.7.4 64-bit ('base': conda)"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "source": [
    "## Practicing web-scraping"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import bs4\n",
    "import lxml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_url = 'http://quotes.toscrape.com/page/{}/' # URL to quotes to scrape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'http://quotes.toscrape.com/page/1/'"
      ]
     },
     "metadata": {},
     "execution_count": 4
    }
   ],
   "source": [
    "base_url.format(1) # access the first page url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = requests.get(base_url.format(1)) # access the first page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = bs4.BeautifulSoup(res.text, 'lxml') # reorders it nicely"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "bs4.BeautifulSoup"
      ]
     },
     "metadata": {},
     "execution_count": 7
    }
   ],
   "source": [
    "type(soup)\n"
   ]
  },
  {
   "source": [
    "## I'm accessing the authors name in the first page of the url"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "{'Albert Einstein',\n",
       " 'André Gide',\n",
       " 'Eleanor Roosevelt',\n",
       " 'J.K. Rowling',\n",
       " 'Jane Austen',\n",
       " 'Marilyn Monroe',\n",
       " 'Steve Martin',\n",
       " 'Thomas A. Edison'}"
      ]
     },
     "metadata": {},
     "execution_count": 10
    }
   ],
   "source": [
    "tag = soup.select('.author')\n",
    "authors = set()\n",
    "for item in tag:\n",
    "    authors.add(item.text)\n",
    "authors"
   ]
  },
  {
   "source": [
    "## Below I obtain a list of all quotes in the first page of the above url"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "quotes = soup.select('.text')\n",
    "quotes_list = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”\n“It is our choices, Harry, that show what we truly are, far more than our abilities.”\n“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”\n“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”\n“Imperfection is beauty, madness is genius and it's better to be absolutely ridiculous than absolutely boring.”\n“Try not to become a man of success. Rather become a man of value.”\n“It is better to be hated for what you are than to be loved for what you are not.”\n“I have not failed. I've just found 10,000 ways that won't work.”\n“A woman is like a tea bag; you never know how strong it is until it's in hot water.”\n“A day without sunshine is like, you know, night.”\n"
     ]
    }
   ],
   "source": [
    "for itms in quotes:\n",
    "    quotes_list.append(itms.text)\n",
    "print(*quotes_list,sep='\\n')"
   ]
  },
  {
   "source": [
    "## Accessing the TOP TEN TAGS in the first page"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags = []\n",
    "tag_list = soup.select('.tag-item .tag')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['love',\n",
       " 'inspirational',\n",
       " 'life',\n",
       " 'humor',\n",
       " 'books',\n",
       " 'reading',\n",
       " 'friendship',\n",
       " 'friends',\n",
       " 'truth',\n",
       " 'simile']"
      ]
     },
     "metadata": {},
     "execution_count": 23
    }
   ],
   "source": [
    "for itms in tag_list:\n",
    "    tags.append(itms.text)\n",
    "tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = requests.get(base_url.format(15)) # access the first page\n",
    "soup = bs4.BeautifulSoup(res.text, 'lxml') # reorders it nicely"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "info = soup.select('.col-md-8')[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "metadata": {},
     "execution_count": 17
    }
   ],
   "source": [
    "'No quotes found' not in info.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "page_indx = 0\n",
    "authors = set()\n",
    "\n",
    "while True:\n",
    "    page_indx += 1\n",
    "    res = requests.get(base_url.format(page_indx)) # access the first page\n",
    "    soup = bs4.BeautifulSoup(res.text, 'lxml') # reorders it nicely\n",
    "    info = soup.select('.col-md-8')[1]\n",
    "\n",
    "    condition = 'No quotes found' not in info.text\n",
    "    if condition:\n",
    "        tag = soup.select('.author')\n",
    "        for item in tag:\n",
    "            authors.add(item.text)\n",
    "    else:\n",
    "        page_indx -= 1\n",
    "        break\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "George Eliot\nHelen Keller\nGeorge Carlin\nCharles Bukowski\nWilliam Nicholson\nJorge Luis Borges\nRalph Waldo Emerson\nJane Austen\nTerry Pratchett\nDr. Seuss\nJ.R.R. Tolkien\nJames Baldwin\nCharles M. Schulz\nStephenie Meyer\nJ.D. Salinger\nJimi Hendrix\nErnest Hemingway\nDouglas Adams\nMother Teresa\nSteve Martin\nAlbert Einstein\nPablo Neruda\nSuzanne Collins\nHarper Lee\nAllen Saunders\nElie Wiesel\nE.E. Cummings\nAlexandre Dumas fils\nJohn Lennon\nEleanor Roosevelt\nKhaled Hosseini\nMadeleine L'Engle\nMartin Luther King Jr.\nAyn Rand\nC.S. Lewis\nHaruki Murakami\nJ.M. Barrie\nW.C. Fields\nGeorge Bernard Shaw\nThomas A. Edison\nJ.K. Rowling\nAndré Gide\nAlfred Tennyson\nBob Marley\nFriedrich Nietzsche\nMarilyn Monroe\nMark Twain\nJim Henson\nGeorge R.R. Martin\nGarrison Keillor\n"
     ]
    }
   ],
   "source": [
    "print(*authors, sep='\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}