Python 实战：房价数据采集与分析

requests

BeautifulSoup

lxml

Selenium

import requests
from bs4 import BeautifulSoup
import time
import random

def fetch_page_content(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
        'Referer': 'https://www.example.com' # 替换为实际的Referer
    }
    try:
        response = requests.get(url, headers=headers, timeout=10)
        response.raise_for_status() # 检查HTTP请求是否成功
        return response.text
    except requests.exceptions.RequestException as e:
        print(f"请求失败: {e}")
        return None

def parse_house_info(html_content):
    if not html_content:
        return []
    soup = BeautifulSoup(html_content, 'html.parser')
    house_list = []
    # 假设房源信息在一个class为'house-item'的div中
    items = soup.find_all('div', class_='house-item')
    for item in items:
        title = item.find('a', class_='title').text.strip() if item.find('a', class_='title') else 'N/A'
        price = item.find('span', class_='total-price').text.strip() if item.find('span', class_='total-price') else 'N/A'
        area = item.find('span', class_='area').text.strip() if item.find('span', class_='area') else 'N/A'
        location = item.find('span', class_='location').text.strip() if item.find('span', class_='location') else 'N/A'

        house_list.append({
            'title': title,
            'price': price,
            'area': area,
            'location': location
        })
    return house_list

# 示例用法
# target_url = "https://www.some-real-estate-website.com/zufang/pg1/"
# html = fetch_page_content(target_url)
# if html:
#     houses = parse_house_info(html)
#     for house in houses:
#         print(house)
#     time.sleep(random.uniform(2, 5)) # 模拟人类行为，增加随机延迟

pandas

import pandas as pd
import numpy as np

# 假设我们已经有了一个包含房价数据的DataFrame
# df = pd.DataFrame(houses) # 从上面爬取的数据创建DataFrame

# 示例数据清洗过程
data = {
    'title': ['三居室好房', '两居室精装', '一居室学区房', '三居室', '两居室'],
    'price': ['250万', '180万', '120万', '260万', '190万'],
    'area': ['90平米', '70㎡', '50平', '95平米', '72㎡'],
    'location': ['朝阳区', '海淀区', '朝阳区', '丰台区', '海淀区'],
    'description': ['近地铁', '采光好', np.nan, '学区房', '新装修']
}
df = pd.DataFrame(data)

# 1. 清洗价格：转换为数值型，单位统一为万元
def clean_price(price_str):
    if isinstance(price_str, str):
        price_str = price_str.replace('万', '').replace('元', '').strip()
        try:
            return float(price_str)
        except ValueError:
            return np.nan
    return np.nan

df['price_cleaned'] = df['price'].apply(clean_price)

# 2. 清洗面积：转换为数值型，单位统一为平方米
def clean_area(area_str):
    if isinstance(area_str, str):
        area_str = area_str.replace('平米', '').replace('㎡', '').replace('平', '').strip()
        try:
            return float(area_str)
        except ValueError:
            return np.nan
    return np.nan

df['area_cleaned'] = df['area'].apply(clean_area)

# 3. 计算单价
df['price_per_sqm'] = (df['price_cleaned'] * 10000) / df['area_cleaned'] # 价格单位是万，面积是平米

# 4. 处理缺失值（例如，填充description的缺失值）
df['description'].fillna('无描述', inplace=True)

print(df.head())

matplotlib

seaborn

scikit-learn

import matplotlib.pyplot as plt
import seaborn as sns

# 设置matplotlib中文显示
plt.rcParams['font.sans-serif'] = ['SimHei'] # 指定默认字体
plt.rcParams['axes.unicode_minus'] = False # 解决保存图像是负号'-'显示为方块的问题

# 假设df已经经过清洗，并且有'price_cleaned', 'area_cleaned', 'price_per_sqm', 'location'等列

# 1. 房价分布直方图
plt.figure(figsize=(10, 6))
sns.histplot(df['price_cleaned'], bins=20, kde=True)
plt.title('房价分布')
plt.xlabel('总价 (万元)')
plt.ylabel('房源数量')
plt.show()

# 2. 不同区域的房价箱线图
plt.figure(figsize=(12, 7))
sns.boxplot(x='location', y='price_cleaned', data=df)
plt.title('不同区域房价分布')
plt.xlabel('区域')
plt.ylabel('总价 (万元)')
plt.show()

# 3. 面积与房价的散点图
plt.figure(figsize=(10, 6))
sns.scatterplot(x='area_cleaned', y='price_cleaned', data=df)
plt.title('面积与房价关系')
plt.xlabel('面积 (平方米)')
plt.ylabel('总价 (万元)')
plt.show()

# 4. 区域平均单价
avg_price_per_sqm_by_location = df.groupby('location')['price_per_sqm'].mean().sort_values(ascending=False)
print("\n各区域平均单价:\n", avg_price_per_sqm_by_location)

plt.figure(figsize=(12, 7))
sns.barplot(x=avg_price_per_sqm_by_location.index, y=avg_price_per_sqm_by_location.values)
plt.title('各区域平均单价')
plt.xlabel('区域')
plt.ylabel('平均单价 (元/平方米)')
plt.show()

requests

BeautifulSoup

lxml

Scrapy

Selenium

Playwright

pandas

numpy

matplotlib

seaborn

scikit-learn

Selenium

Playwright