sri_webscraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3 number of pages were found\n"
     ]
    }
   ],
   "source": [
    "#Get the first page to extract page numbers\n",
    "import requests, re\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "r=requests.get(\"http://www.pythonhow.com/real-estate/rock-springs-wy/LCWYROCKSPRINGS/\")\n",
    "c=r.content\n",
    "\n",
    "soup=BeautifulSoup(c,\"html.parser\")\n",
    "\n",
    "all=soup.find_all(\"div\",{\"class\":\"propertyRow\"})\n",
    "\n",
    "all[0].find(\"h4\",{\"class\":\"propPrice\"}).text.replace(\"\\n\",\"\").replace(\" \",\"\")\n",
    "\n",
    "page_nr=soup.find_all(\"a\",{\"class\":\"Page\"})[-1].text\n",
    "print(page_nr,\"number of pages were found\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "l=[]\n",
    "base_url=\"http://www.pythonhow.com/real-estate/rock-springs-wy/LCWYROCKSPRINGS/t=0&s=\"\n",
    "for page in range(0,int(page_nr)*10,10):\n",
    "    print( )\n",
    "    r=requests.get(base_url+str(page)+\".html\")\n",
    "    c=r.content\n",
    "    #c=r.json()[\"list\"]\n",
    "    soup=BeautifulSoup(c,\"html.parser\")\n",
    "    all=soup.find_all(\"div\",{\"class\":\"propertyRow\"})\n",
    "    for item in all:\n",
    "        d={}\n",
    "        d[\"Address\"]=item.find_all(\"span\",{\"class\",\"propAddressCollapse\"})[0].text\n",
    "        try:\n",
    "            d[\"Locality\"]=item.find_all(\"span\",{\"class\",\"propAddressCollapse\"})[1].text\n",
    "        except:\n",
    "            d[\"Locality\"]=None\n",
    "        d[\"Price\"]=item.find(\"h4\",{\"class\",\"propPrice\"}).text.replace(\"\\n\",\"\").replace(\" \",\"\")\n",
    "        try:\n",
    "            d[\"Beds\"]=item.find(\"span\",{\"class\",\"infoBed\"}).find(\"b\").text\n",
    "        except:\n",
    "            d[\"Beds\"]=None\n",
    "    \n",
    "        try:\n",
    "            d[\"Area\"]=item.find(\"span\",{\"class\",\"infoSqFt\"}).find(\"b\").text\n",
    "        except:\n",
    "            d[\"Area\"]=None\n",
    "    \n",
    "        try:\n",
    "            d[\"Full Baths\"]=item.find(\"span\",{\"class\",\"infoValueFullBath\"}).find(\"b\").text\n",
    "        except:\n",
    "            d[\"Full Baths\"]=None\n",
    "\n",
    "        try:\n",
    "            d[\"Half Baths\"]=item.find(\"span\",{\"class\",\"infoValueHalfBath\"}).find(\"b\").text\n",
    "        except:\n",
    "            d[\"Half Baths\"]=None\n",
    "        for column_group in item.find_all(\"div\",{\"class\":\"columnGroup\"}):\n",
    "            for feature_group, feature_name in zip(column_group.find_all(\"span\",{\"class\":\"featureGroup\"}),column_group.find_all(\"span\",{\"class\":\"featureName\"})):\n",
    "                if \"Lot Size\" in feature_group.text:\n",
    "                    d[\"Lot Size\"]=feature_name.text\n",
    "        l.append(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'Address': '0 Gateway',\n",
       "  'Area': None,\n",
       "  'Beds': None,\n",
       "  'Full Baths': None,\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Rock Springs, WY 82901',\n",
       "  'Price': '$725,000'},\n",
       " {'Address': '1003 Winchester Blvd.',\n",
       "  'Area': None,\n",
       "  'Beds': '4',\n",
       "  'Full Baths': '4',\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Rock Springs, WY 82901',\n",
       "  'Lot Size': '0.21 Acres',\n",
       "  'Price': '$452,900'},\n",
       " {'Address': '600 Talladega',\n",
       "  'Area': '3,154',\n",
       "  'Beds': '5',\n",
       "  'Full Baths': '3',\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Rock Springs, WY 82901',\n",
       "  'Price': '$396,900'},\n",
       " {'Address': '3239 Spearhead Way',\n",
       "  'Area': '3,076',\n",
       "  'Beds': '4',\n",
       "  'Full Baths': '3',\n",
       "  'Half Baths': '1',\n",
       "  'Locality': 'Rock Springs, WY 82901',\n",
       "  'Lot Size': 'Under 1/2 Acre, ',\n",
       "  'Price': '$389,900'},\n",
       " {'Address': '522 Emerald Street',\n",
       "  'Area': '1,172',\n",
       "  'Beds': '3',\n",
       "  'Full Baths': '3',\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Rock Springs, WY 82901',\n",
       "  'Lot Size': 'Under 1/2 Acre, ',\n",
       "  'Price': '$254,000'},\n",
       " {'Address': \"1302 Veteran's Drive\",\n",
       "  'Area': '1,932',\n",
       "  'Beds': '4',\n",
       "  'Full Baths': '2',\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Rock Springs, WY 82901',\n",
       "  'Lot Size': '0.27 Acres',\n",
       "  'Price': '$252,900'},\n",
       " {'Address': '1021 Cypress Cir',\n",
       "  'Area': '1,676',\n",
       "  'Beds': '4',\n",
       "  'Full Baths': '3',\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Rock Springs, WY 82901',\n",
       "  'Lot Size': 'Under 1/2 Acre, ',\n",
       "  'Price': '$210,000'},\n",
       " {'Address': '913 Madison Dr',\n",
       "  'Area': '1,344',\n",
       "  'Beds': '3',\n",
       "  'Full Baths': '2',\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Rock Springs, WY 82901',\n",
       "  'Lot Size': 'Under 1/2 Acre, ',\n",
       "  'Price': '$209,000'},\n",
       " {'Address': '1344 Teton Street',\n",
       "  'Area': '1,920',\n",
       "  'Beds': '3',\n",
       "  'Full Baths': '2',\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Rock Springs, WY 82901',\n",
       "  'Lot Size': 'Under 1/2 Acre, ',\n",
       "  'Price': '$199,900'},\n",
       " {'Address': '4 Minnies Lane',\n",
       "  'Area': '1,664',\n",
       "  'Beds': '3',\n",
       "  'Full Baths': '2',\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Rock Springs, WY 82901',\n",
       "  'Lot Size': '2.02 Acres',\n",
       "  'Price': '$196,900'},\n",
       " {'Address': '9339 Sd 26900',\n",
       "  'Area': '2,560',\n",
       "  'Beds': None,\n",
       "  'Full Baths': None,\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Rocksprings, TX 78880',\n",
       "  'Price': '$1,700,000'},\n",
       " {'Address': 'RR674P13 Hwy 377',\n",
       "  'Area': '2,000',\n",
       "  'Beds': None,\n",
       "  'Full Baths': None,\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Rocksprings, TX 78880',\n",
       "  'Price': '$1,100,000'},\n",
       " {'Address': '0 Hwy 41',\n",
       "  'Area': None,\n",
       "  'Beds': None,\n",
       "  'Full Baths': None,\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Rocksprings, TX 78880',\n",
       "  'Price': '$1,080,000'},\n",
       " {'Address': '9339 Sd 26900',\n",
       "  'Area': '2,560',\n",
       "  'Beds': None,\n",
       "  'Full Baths': None,\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Rocksprings, TX 78880',\n",
       "  'Price': '$908,350'},\n",
       " {'Address': 'CR450 Hwy 377',\n",
       "  'Area': None,\n",
       "  'Beds': None,\n",
       "  'Full Baths': None,\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Rocksprings, TX 78880',\n",
       "  'Price': '$905,000'},\n",
       " {'Address': 'Cr 240 Cr 240',\n",
       "  'Area': '1,398',\n",
       "  'Beds': None,\n",
       "  'Full Baths': None,\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Rocksprings, TX 78880',\n",
       "  'Price': '$695,000'},\n",
       " {'Address': 'RR674 Hwy 377',\n",
       "  'Area': '1,738',\n",
       "  'Beds': None,\n",
       "  'Full Baths': None,\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Rocksprings, TX 78880',\n",
       "  'Price': '$605,000'},\n",
       " {'Address': '9770a Sd 26900',\n",
       "  'Area': '1,080',\n",
       "  'Beds': None,\n",
       "  'Full Baths': None,\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Rocksprings, TX 78880',\n",
       "  'Price': '$559,805'},\n",
       " {'Address': 'Lot17 CR 2630',\n",
       "  'Area': None,\n",
       "  'Beds': None,\n",
       "  'Full Baths': None,\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Rocksprings, TX 78880',\n",
       "  'Price': '$504,000'},\n",
       " {'Address': 'Tr12,16 CR 520',\n",
       "  'Area': None,\n",
       "  'Beds': None,\n",
       "  'Full Baths': None,\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Rocksprings, TX 78880',\n",
       "  'Price': '$410,000'},\n",
       " {'Address': '32575 S Shadow Mountain Road',\n",
       "  'Area': '2,318',\n",
       "  'Beds': '3',\n",
       "  'Full Baths': '2',\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Black Canyon City, AZ 85324',\n",
       "  'Price': '$299,900'},\n",
       " {'Address': '32750 S Shangrila Drive',\n",
       "  'Area': '2,120',\n",
       "  'Beds': '3',\n",
       "  'Full Baths': '2',\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Black Canyon City, AZ 85324',\n",
       "  'Price': '$167,500'},\n",
       " {'Address': '0000 Black Canyon Highway',\n",
       "  'Area': None,\n",
       "  'Beds': None,\n",
       "  'Full Baths': None,\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Black Canyon City, AZ 85324',\n",
       "  'Lot Size': '5 Acres',\n",
       "  'Price': '$150,000'},\n",
       " {'Address': '34775 S CHOLLA Drive',\n",
       "  'Area': '1,220',\n",
       "  'Beds': '3',\n",
       "  'Full Baths': '2',\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Black Canyon City, AZ 85324',\n",
       "  'Price': '$129,500'},\n",
       " {'Address': '33403 S. HA-WA-SI TERRACE',\n",
       "  'Area': '2,000',\n",
       "  'Beds': '4',\n",
       "  'Full Baths': '2',\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'BLACK CANYON CITY, AZ 85324',\n",
       "  'Price': '$129,000'},\n",
       " {'Address': '34263 S Bertha Street',\n",
       "  'Area': '2,260',\n",
       "  'Beds': '5',\n",
       "  'Full Baths': '2',\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Black Canyon City, AZ 85324',\n",
       "  'Price': '$80,000'},\n",
       " {'Address': '33160 S Canyon Road',\n",
       "  'Area': '1,248',\n",
       "  'Beds': '3',\n",
       "  'Full Baths': '2',\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Black Canyon City, AZ 85324',\n",
       "  'Price': '$77,900'},\n",
       " {'Address': '19421 E Todd Evans Road',\n",
       "  'Area': '1,404',\n",
       "  'Beds': '3',\n",
       "  'Full Baths': '2',\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Black Canyon City, AZ 85324',\n",
       "  'Price': '$70,500'},\n",
       " {'Address': '18688 E AGUA Vista',\n",
       "  'Area': None,\n",
       "  'Beds': None,\n",
       "  'Full Baths': None,\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Black Canyon City, AZ 85324',\n",
       "  'Lot Size': '0.7 Acres',\n",
       "  'Price': '$70,000'},\n",
       " {'Address': '50600 N Old Black Canyon Road',\n",
       "  'Area': None,\n",
       "  'Beds': None,\n",
       "  'Full Baths': None,\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Black Canyon City, AZ 85324',\n",
       "  'Lot Size': '3 Acres',\n",
       "  'Price': '$67,500'},\n",
       " {'Address': '20101 E SQUAW VALLEY Road',\n",
       "  'Area': None,\n",
       "  'Beds': None,\n",
       "  'Full Baths': None,\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Black Canyon City, AZ 85324',\n",
       "  'Price': '$54,900'},\n",
       " {'Address': '33259 S Canyon Road',\n",
       "  'Area': '1,056',\n",
       "  'Beds': '3',\n",
       "  'Full Baths': '1',\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Black Canyon City, AZ 85324',\n",
       "  'Price': '$45,600'},\n",
       " {'Address': '34558 S ROADRUNNER RD',\n",
       "  'Area': '784',\n",
       "  'Beds': '2',\n",
       "  'Full Baths': '1',\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Black Canyon City, AZ 85324',\n",
       "  'Lot Size': 'Under 1/2 Acre',\n",
       "  'Price': '$40,000'},\n",
       " {'Address': '19260 E Scenic Loop Road',\n",
       "  'Area': None,\n",
       "  'Beds': None,\n",
       "  'Full Baths': None,\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Black Canyon City, AZ 85324',\n",
       "  'Lot Size': '2.35 Acres',\n",
       "  'Price': '$30,000'},\n",
       " {'Address': '19000 E MAREN Avenue',\n",
       "  'Area': None,\n",
       "  'Beds': None,\n",
       "  'Full Baths': None,\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Black Canyon City, AZ 85324',\n",
       "  'Lot Size': '2.05 Acres',\n",
       "  'Price': '$29,000'},\n",
       " {'Address': '19350 E SAGUARO Drive',\n",
       "  'Area': None,\n",
       "  'Beds': None,\n",
       "  'Full Baths': None,\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Black Canyon City, AZ 85324',\n",
       "  'Lot Size': '0.73 Acres',\n",
       "  'Price': '$28,995'},\n",
       " {'Address': '20650 E Amethyst Place',\n",
       "  'Area': None,\n",
       "  'Beds': None,\n",
       "  'Full Baths': None,\n",
       "  'Half Baths': None,\n",
       "  'Locality': 'Black Canyon City, AZ 85324',\n",
       "  'Lot Size': '0.31 Acres',\n",
       "  'Price': '$15,000'}]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas\n",
    "df=pandas.DataFrame(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Address</th>\n",
       "      <th>Area</th>\n",
       "      <th>Beds</th>\n",
       "      <th>Full Baths</th>\n",
       "      <th>Half Baths</th>\n",
       "      <th>Locality</th>\n",
       "      <th>Lot Size</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0 Gateway</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Rock Springs, WY 82901</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$725,000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1003 Winchester Blvd.</td>\n",
       "      <td>None</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>Rock Springs, WY 82901</td>\n",
       "      <td>0.21 Acres</td>\n",
       "      <td>$452,900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>600 Talladega</td>\n",
       "      <td>3,154</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>Rock Springs, WY 82901</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$396,900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3239 Spearhead Way</td>\n",
       "      <td>3,076</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>Rock Springs, WY 82901</td>\n",
       "      <td>Under 1/2 Acre,</td>\n",
       "      <td>$389,900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>522 Emerald Street</td>\n",
       "      <td>1,172</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>Rock Springs, WY 82901</td>\n",
       "      <td>Under 1/2 Acre,</td>\n",
       "      <td>$254,000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1302 Veteran's Drive</td>\n",
       "      <td>1,932</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>Rock Springs, WY 82901</td>\n",
       "      <td>0.27 Acres</td>\n",
       "      <td>$252,900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1021 Cypress Cir</td>\n",
       "      <td>1,676</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>Rock Springs, WY 82901</td>\n",
       "      <td>Under 1/2 Acre,</td>\n",
       "      <td>$210,000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>913 Madison Dr</td>\n",
       "      <td>1,344</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>Rock Springs, WY 82901</td>\n",
       "      <td>Under 1/2 Acre,</td>\n",
       "      <td>$209,000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1344 Teton Street</td>\n",
       "      <td>1,920</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>Rock Springs, WY 82901</td>\n",
       "      <td>Under 1/2 Acre,</td>\n",
       "      <td>$199,900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>4 Minnies Lane</td>\n",
       "      <td>1,664</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>Rock Springs, WY 82901</td>\n",
       "      <td>2.02 Acres</td>\n",
       "      <td>$196,900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>9339 Sd 26900</td>\n",
       "      <td>2,560</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Rocksprings, TX 78880</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$1,700,000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>RR674P13 Hwy 377</td>\n",
       "      <td>2,000</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Rocksprings, TX 78880</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$1,100,000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0 Hwy 41</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Rocksprings, TX 78880</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$1,080,000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>9339 Sd 26900</td>\n",
       "      <td>2,560</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Rocksprings, TX 78880</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$908,350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>CR450 Hwy 377</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Rocksprings, TX 78880</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$905,000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Cr 240 Cr 240</td>\n",
       "      <td>1,398</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Rocksprings, TX 78880</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$695,000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>RR674 Hwy 377</td>\n",
       "      <td>1,738</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Rocksprings, TX 78880</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$605,000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>9770a Sd 26900</td>\n",
       "      <td>1,080</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Rocksprings, TX 78880</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$559,805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Lot17 CR 2630</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Rocksprings, TX 78880</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$504,000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Tr12,16 CR 520</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Rocksprings, TX 78880</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$410,000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>32575 S Shadow Mountain Road</td>\n",
       "      <td>2,318</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>Black Canyon City, AZ 85324</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$299,900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>32750 S Shangrila Drive</td>\n",
       "      <td>2,120</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>Black Canyon City, AZ 85324</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$167,500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0000 Black Canyon Highway</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Black Canyon City, AZ 85324</td>\n",
       "      <td>5 Acres</td>\n",
       "      <td>$150,000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>34775 S CHOLLA Drive</td>\n",
       "      <td>1,220</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>Black Canyon City, AZ 85324</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$129,500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>33403 S. HA-WA-SI TERRACE</td>\n",
       "      <td>2,000</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>BLACK CANYON CITY, AZ 85324</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$129,000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>34263 S Bertha Street</td>\n",
       "      <td>2,260</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>Black Canyon City, AZ 85324</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$80,000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>33160 S Canyon Road</td>\n",
       "      <td>1,248</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>Black Canyon City, AZ 85324</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$77,900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>19421 E Todd Evans Road</td>\n",
       "      <td>1,404</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>Black Canyon City, AZ 85324</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$70,500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>18688 E AGUA Vista</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Black Canyon City, AZ 85324</td>\n",
       "      <td>0.7 Acres</td>\n",
       "      <td>$70,000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>50600 N Old Black Canyon Road</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Black Canyon City, AZ 85324</td>\n",
       "      <td>3 Acres</td>\n",
       "      <td>$67,500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>20101 E SQUAW VALLEY Road</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Black Canyon City, AZ 85324</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$54,900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>33259 S Canyon Road</td>\n",
       "      <td>1,056</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>Black Canyon City, AZ 85324</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$45,600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>34558 S ROADRUNNER RD</td>\n",
       "      <td>784</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>Black Canyon City, AZ 85324</td>\n",
       "      <td>Under 1/2 Acre</td>\n",
       "      <td>$40,000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>19260 E Scenic Loop Road</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Black Canyon City, AZ 85324</td>\n",
       "      <td>2.35 Acres</td>\n",
       "      <td>$30,000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>19000 E MAREN Avenue</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Black Canyon City, AZ 85324</td>\n",
       "      <td>2.05 Acres</td>\n",
       "      <td>$29,000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>19350 E SAGUARO Drive</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Black Canyon City, AZ 85324</td>\n",
       "      <td>0.73 Acres</td>\n",
       "      <td>$28,995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>20650 E Amethyst Place</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Black Canyon City, AZ 85324</td>\n",
       "      <td>0.31 Acres</td>\n",
       "      <td>$15,000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          Address   Area  Beds Full Baths Half Baths  \\\n",
       "0                       0 Gateway   None  None       None       None   \n",
       "1           1003 Winchester Blvd.   None     4          4       None   \n",
       "2                   600 Talladega  3,154     5          3       None   \n",
       "3              3239 Spearhead Way  3,076     4          3          1   \n",
       "4              522 Emerald Street  1,172     3          3       None   \n",
       "5            1302 Veteran's Drive  1,932     4          2       None   \n",
       "6                1021 Cypress Cir  1,676     4          3       None   \n",
       "7                  913 Madison Dr  1,344     3          2       None   \n",
       "8               1344 Teton Street  1,920     3          2       None   \n",
       "9                  4 Minnies Lane  1,664     3          2       None   \n",
       "10                  9339 Sd 26900  2,560  None       None       None   \n",
       "11               RR674P13 Hwy 377  2,000  None       None       None   \n",
       "12                       0 Hwy 41   None  None       None       None   \n",
       "13                  9339 Sd 26900  2,560  None       None       None   \n",
       "14                  CR450 Hwy 377   None  None       None       None   \n",
       "15                  Cr 240 Cr 240  1,398  None       None       None   \n",
       "16                  RR674 Hwy 377  1,738  None       None       None   \n",
       "17                 9770a Sd 26900  1,080  None       None       None   \n",
       "18                  Lot17 CR 2630   None  None       None       None   \n",
       "19                 Tr12,16 CR 520   None  None       None       None   \n",
       "20   32575 S Shadow Mountain Road  2,318     3          2       None   \n",
       "21        32750 S Shangrila Drive  2,120     3          2       None   \n",
       "22      0000 Black Canyon Highway   None  None       None       None   \n",
       "23           34775 S CHOLLA Drive  1,220     3          2       None   \n",
       "24      33403 S. HA-WA-SI TERRACE  2,000     4          2       None   \n",
       "25          34263 S Bertha Street  2,260     5          2       None   \n",
       "26            33160 S Canyon Road  1,248     3          2       None   \n",
       "27        19421 E Todd Evans Road  1,404     3          2       None   \n",
       "28             18688 E AGUA Vista   None  None       None       None   \n",
       "29  50600 N Old Black Canyon Road   None  None       None       None   \n",
       "30      20101 E SQUAW VALLEY Road   None  None       None       None   \n",
       "31            33259 S Canyon Road  1,056     3          1       None   \n",
       "32          34558 S ROADRUNNER RD    784     2          1       None   \n",
       "33       19260 E Scenic Loop Road   None  None       None       None   \n",
       "34           19000 E MAREN Avenue   None  None       None       None   \n",
       "35          19350 E SAGUARO Drive   None  None       None       None   \n",
       "36         20650 E Amethyst Place   None  None       None       None   \n",
       "\n",
       "                       Locality          Lot Size       Price  \n",
       "0        Rock Springs, WY 82901               NaN    $725,000  \n",
       "1        Rock Springs, WY 82901        0.21 Acres    $452,900  \n",
       "2        Rock Springs, WY 82901               NaN    $396,900  \n",
       "3        Rock Springs, WY 82901  Under 1/2 Acre,     $389,900  \n",
       "4        Rock Springs, WY 82901  Under 1/2 Acre,     $254,000  \n",
       "5        Rock Springs, WY 82901        0.27 Acres    $252,900  \n",
       "6        Rock Springs, WY 82901  Under 1/2 Acre,     $210,000  \n",
       "7        Rock Springs, WY 82901  Under 1/2 Acre,     $209,000  \n",
       "8        Rock Springs, WY 82901  Under 1/2 Acre,     $199,900  \n",
       "9        Rock Springs, WY 82901        2.02 Acres    $196,900  \n",
       "10        Rocksprings, TX 78880               NaN  $1,700,000  \n",
       "11        Rocksprings, TX 78880               NaN  $1,100,000  \n",
       "12        Rocksprings, TX 78880               NaN  $1,080,000  \n",
       "13        Rocksprings, TX 78880               NaN    $908,350  \n",
       "14        Rocksprings, TX 78880               NaN    $905,000  \n",
       "15        Rocksprings, TX 78880               NaN    $695,000  \n",
       "16        Rocksprings, TX 78880               NaN    $605,000  \n",
       "17        Rocksprings, TX 78880               NaN    $559,805  \n",
       "18        Rocksprings, TX 78880               NaN    $504,000  \n",
       "19        Rocksprings, TX 78880               NaN    $410,000  \n",
       "20  Black Canyon City, AZ 85324               NaN    $299,900  \n",
       "21  Black Canyon City, AZ 85324               NaN    $167,500  \n",
       "22  Black Canyon City, AZ 85324           5 Acres    $150,000  \n",
       "23  Black Canyon City, AZ 85324               NaN    $129,500  \n",
       "24  BLACK CANYON CITY, AZ 85324               NaN    $129,000  \n",
       "25  Black Canyon City, AZ 85324               NaN     $80,000  \n",
       "26  Black Canyon City, AZ 85324               NaN     $77,900  \n",
       "27  Black Canyon City, AZ 85324               NaN     $70,500  \n",
       "28  Black Canyon City, AZ 85324         0.7 Acres     $70,000  \n",
       "29  Black Canyon City, AZ 85324           3 Acres     $67,500  \n",
       "30  Black Canyon City, AZ 85324               NaN     $54,900  \n",
       "31  Black Canyon City, AZ 85324               NaN     $45,600  \n",
       "32  Black Canyon City, AZ 85324    Under 1/2 Acre     $40,000  \n",
       "33  Black Canyon City, AZ 85324        2.35 Acres     $30,000  \n",
       "34  Black Canyon City, AZ 85324        2.05 Acres     $29,000  \n",
       "35  Black Canyon City, AZ 85324        0.73 Acres     $28,995  \n",
       "36  Black Canyon City, AZ 85324        0.31 Acres     $15,000  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3 number of pages were found\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Get the first page to extract page numbers\n",
    "import requests, re\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "r=requests.get(\"http://www.pythonhow.com/real-estate/rock-springs-wy/LCWYROCKSPRINGS/\")\n",
    "c=r.content\n",
    "\n",
    "soup=BeautifulSoup(c,\"html.parser\")\n",
    "\n",
    "all=soup.find_all(\"div\",{\"class\":\"propertyRow\"})\n",
    "\n",
    "all[0].find(\"h4\",{\"class\":\"propPrice\"}).text.replace(\"\\n\",\"\").replace(\" \",\"\")\n",
    "\n",
    "page_nr=soup.find_all(\"a\",{\"class\":\"Page\"})[-1].text\n",
    "print(page_nr,\"number of pages were found\")\n",
    "\n",
    "l=[]\n",
    "base_url=\"http://www.pythonhow.com/real-estate/rock-springs-wy/LCWYROCKSPRINGS/t=0&s=\"\n",
    "for page in range(0,int(page_nr)*10,10):\n",
    "    print( )\n",
    "    r=requests.get(base_url+str(page)+\".html\")\n",
    "    c=r.content\n",
    "    #c=r.json()[\"list\"]\n",
    "    soup=BeautifulSoup(c,\"html.parser\")\n",
    "    all=soup.find_all(\"div\",{\"class\":\"propertyRow\"})\n",
    "    for item in all:\n",
    "        d={}\n",
    "        d[\"Address\"]=item.find_all(\"span\",{\"class\",\"propAddressCollapse\"})[0].text\n",
    "        try:\n",
    "            d[\"Locality\"]=item.find_all(\"span\",{\"class\",\"propAddressCollapse\"})[1].text\n",
    "        except:\n",
    "            d[\"Locality\"]=None\n",
    "        d[\"Price\"]=item.find(\"h4\",{\"class\",\"propPrice\"}).text.replace(\"\\n\",\"\").replace(\" \",\"\")\n",
    "        try:\n",
    "            d[\"Beds\"]=item.find(\"span\",{\"class\",\"infoBed\"}).find(\"b\").text\n",
    "        except:\n",
    "            d[\"Beds\"]=None\n",
    "    \n",
    "        try:\n",
    "            d[\"Area\"]=item.find(\"span\",{\"class\",\"infoSqFt\"}).find(\"b\").text\n",
    "        except:\n",
    "            d[\"Area\"]=None\n",
    "    \n",
    "        try:\n",
    "            d[\"Full Baths\"]=item.find(\"span\",{\"class\",\"infoValueFullBath\"}).find(\"b\").text\n",
    "        except:\n",
    "            d[\"Full Baths\"]=None\n",
    "\n",
    "        try:\n",
    "            d[\"Half Baths\"]=item.find(\"span\",{\"class\",\"infoValueHalfBath\"}).find(\"b\").text\n",
    "        except:\n",
    "            d[\"Half Baths\"]=None\n",
    "        for column_group in item.find_all(\"div\",{\"class\":\"columnGroup\"}):\n",
    "            for feature_group, feature_name in zip(column_group.find_all(\"span\",{\"class\":\"featureGroup\"}),column_group.find_all(\"span\",{\"class\":\"featureName\"})):\n",
    "                if \"Lot Size\" in feature_group.text:\n",
    "                    d[\"Lot Size\"]=feature_name.text\n",
    "        l.append(d)\n",
    "import pandas\n",
    "df=pandas.DataFrame(l)\n",
    "df\n",
    "df.to_csv(\"Output.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}