Python网络爬虫：高效处理分页数据与Pandas Excel存储实践

# 示例：构建分页URL
base_url = "https://www.jumia.ma/catalog/?q=pc&page="
page_number = 1
url = f"{base_url}{page_number}#catalog-listing"

import requests
import time

def fetch_page_content(page_num):
    url = f"https://www.jumia.ma/catalog/?q=pc&page={page_num}#catalog-listing"
    try:
        html_text = requests.get(url).text
        return html_text
    except requests.exceptions.RequestException as e:
        print(f"请求页面 {page_num} 失败: {e}")
        return None

# 示例循环结构
max_pages = 5 # 假设最多抓取5页
current_page = 1
while current_page <= max_pages:
    page_content = fetch_page_content(current_page)
    if page_content:
        # 在这里处理页面内容
        pass
    current_page += 1
    time.sleep(2) # 添加延迟，避免对服务器造成过大压力

from bs4 import BeautifulSoup
from bs4 import Tag # 导入Tag类型用于类型检查

def parse_page_data(html_content):
    soup = BeautifulSoup(html_content, "lxml")
    computers = soup.find_all("a", class_="core")
    page_data = []
    for computer in computers:
        name_element = computer.find("h3", class_="name")
        price_element = computer.find("div", class_="prc")
        original_price_element = computer.find("div", class_="old")
        promo_element = computer.find("div", class_="bdg _dsct _sm")

        # 健壮的数据提取：检查元素是否存在
        name = name_element.text.strip() if name_element else "N/A"
        price = price_element.text.strip() if price_element else "N/A"
        original_price = original_price_element.text.strip() if original_price_element else "N/A"
        promo = promo_element.text.strip() if promo_element else "N/A"

        page_data.append({
            'name': name,
            'price': price,
            'original_price': original_price,
            'promo': promo
        })
    return page_data

# 整合到分页循环中
all_data = []
max_pages = 5
current_page = 1
while current_page <= max_pages:
    html_content = fetch_page_content(current_page)
    if html_content:
        page_records = parse_page_data(html_content)
        all_data.extend(page_records) # 将当前页的数据添加到总列表中
    current_page += 1
    time.sleep(2)

import pandas as pd

# 假设 all_data 已经包含了所有抓取到的数据
df = pd.DataFrame(all_data)

# 假设 df 已经包含了所有需要保存的数据
output_filename = "output.xlsx"
with pd.ExcelWriter(output_filename) as writer:
    df.to_excel(writer, sheet_name="sheet1", index=False) # index=False 避免将DataFrame索引写入Excel

import pandas as pd
from bs4 import BeautifulSoup
import requests
import time

def find_computers():
    """
    抓取Jumia网站PC分类下多页的电脑信息。
    """
    all_collected_data = [] # 用于存储所有页面抓取到的数据
    max_pages = 5 # 设置要抓取的最大页数，可以根据实际情况调整

    current_page = 1
    while current_page <= max_pages:
        print(f"正在抓取第 {current_page} 页...")
        url = f"https://www.jumia.ma/catalog/?q=pc&page={current_page}#catalog-listing"

        try:
            html_text = requests.get(url, timeout=10).text # 增加请求超时
            soup = BeautifulSoup(html_text, "lxml")
            computers = soup.find_all("a", class_="core")

            if not computers:
                print(f"第 {current_page} 页未找到商品，可能已达最后一页或页面结构改变。")
                break # 如果当前页没有找到商品，则认为已到达最后一页，停止循环

            for computer in computers:
                name_element = computer.find("h3", class_="name")
                price_element = computer.find("div", class_="prc")
                original_price_element = computer.find("div", class_="old")
                promo_element = computer.find("div", class_="bdg _dsct _sm")

                # 健壮性检查：确保元素存在才提取文本
                name = name_element.text.strip() if name_element else "N/A"
                price = price_element.text.strip() if price_element else "N/A"
                original_price = original_price_element.text.strip() if original_price_element else "N/A"
                promo = promo_element.text.strip() if promo_element else "N/A"

                all_collected_data.append({
                    'name': name,
                    'price': price,
                    'original_price': original_price,
                    'promo': promo
                })
        except requests.exceptions.RequestException as e:
            print(f"请求第 {current_page} 页失败: {e}")
        except Exception as e:
            print(f"处理第 {current_page} 页时发生错误: {e}")

        current_page += 1
        time.sleep(6) # 每次请求后暂停6秒，避免请求过于频繁

    return all_collected_data

if __name__ == "__main__":
    print("开始抓取数据...")
    data_to_save = find_computers()

    if data_to_save:
        df = pd.DataFrame(data_to_save)
        output_filename = "output.xlsx"
        try:
            with pd.ExcelWriter(output_filename, engine="openpyxl") as writer:
                df.to_excel(writer, sheet_name="sheet1", index=False)
            print(f"所有数据已成功保存到 {output_filename} 的 'sheet1' 中。")
        except Exception as e:
            print(f"保存数据到Excel时发生错误: {e}")
    else:
        print("未抓取到任何数据，未生成Excel文件。")