{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "## Web scraping with BeautifulSoup" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Import required libraries" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "from bs4 import BeautifulSoup as soup\n", "from urllib.request import urlopen as uReq\n", "import pandas as pd" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Create an object and turn an HTML page into a soup object" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Fetching the raw HTML page" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "query = \"playstation+5\" # Put your search term here\n", "my_url = 'YOUR_EBAY_URL'.format(query) # Paste the url for an Ebay search from your browser and replace the query part with a {}\n", "my_url" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "uClient = uReq(my_url)\n", "page_html = uClient.read()\n", "uClient.close()\n", "\n", "page_soup = soup(page_html, 'html.parser')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Find all the relevant classes and extract data" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "divs = page_soup.findAll('div',{'class':'s-item__details clearfix'}) # Find the HTML tag with the data we are looking for and fetch all instances of it\n", "prices = []\n", "\n", "for PS5 in divs: \n", " price = PS5.find('span',{'class':\"s-item__price\"})\n", " price = price.text[1:]\n", " price = price.replace(',','')\n", " prices.append(price)\n", "\n", "\n", "prices = pd.to_numeric(prices,errors='coerce')\n", "prices" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Use the scraped data to get insights into the market for PS5 consoles" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import seaborn as sns\n", "sns.set()\n", "sns.boxplot(data=prices)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "interpreter": { "hash": "8fe041c8bf3f6c634cfbe99daa1d0835869ac67f33d4cde2619e21380e56c003" }, "kernelspec": { "display_name": "Python 3.9.7 64-bit", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.0" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }