如何用Python源码整合影视剧评论数据 利用API与爬虫协作的源码结构

# project_root/
# ├── config.py             # 存放API密钥、User-Agent、延迟时间等配置
# ├── api_client.py         # 负责所有API交互逻辑
# ├── web_scraper.py        # 负责所有网页抓取与解析逻辑
# ├── data_processor.py     # 负责数据清洗、标准化和去重
# ├── database_manager.py   # 负责数据存储（例如SQLite或MongoDB）
# ├── main.py               # 程序的入口点，协调各模块工作
# └── requirements.txt      # 项目依赖

# config.py 示例
API_KEYS = {
    "douban": "YOUR_DOUBAN_API_KEY",
    "tmdb": "YOUR_TMDB_API_KEY"
}
USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
]
REQUEST_DELAY_SECONDS = (1, 3) # 随机延迟范围

# api_client.py 示例
import requests
import random
from config import API_KEYS, USER_AGENTS

def _get_headers():
    return {'User-Agent': random.choice(USER_AGENTS)}

def fetch_movie_details_from_api(movie_id, api_source='douban'):
    """从指定API获取电影详情，包含基础信息和部分评论ID"""
    api_key = API_KEYS.get(api_source)
    if not api_key:
        print(f"错误: 未找到 {api_source} 的API密钥。")
        return None

    url = f"https://api.{api_source}.com/v2/movie/{movie_id}?apikey={api_key}" # 示例URL
    try:
        response = requests.get(url, headers=_get_headers(), timeout=10)
        response.raise_for_status() # 检查HTTP错误
        return response.json()
    except requests.exceptions.RequestException as e:
        print(f"API请求失败 ({api_source}, ID: {movie_id}): {e}")
        return None

# web_scraper.py 示例
from bs4 import BeautifulSoup
import time
import random
from config import USER_AGENTS, REQUEST_DELAY_SECONDS

def scrape_movie_reviews(review_page_url):
    """从指定评论页面抓取用户评论"""
    time.sleep(random.uniform(*REQUEST_DELAY_SECONDS)) # 模拟人类行为，避免被封
    headers = {'User-Agent': random.choice(USER_AGENTS)}

    try:
        response = requests.get(review_page_url, headers=headers, timeout=15)
        response.raise_for_status()
        soup = BeautifulSoup(response.text, 'html.parser')

        reviews = []
        # 假设评论都在特定的div或li中，例如 class="comment-item"
        for comment_div in soup.find_all('div', class_='comment-item'):
            user_name = comment_div.find('a', class_='user-name').text.strip() if comment_div.find('a', class_='user-name') else '匿名'
            rating_span = comment_div.find('span', class_='rating')
            rating = rating_span['title'] if rating_span and 'title' in rating_span.attrs else 'N/A'
            comment_text = comment_div.find('p', class_='comment-content').text.strip() if comment_div.find('p', class_='comment-content') else ''

            reviews.append({
                'user': user_name,
                'rating': rating,
                'text': comment_text,
                'source_url': review_page_url # 记录来源，方便溯源
            })
        return reviews
    except requests.exceptions.RequestException as e:
        print(f"网页抓取失败 ({review_page_url}): {e}")
        return []
    except Exception as e:
        print(f"解析页面失败 ({review_page_url}): {e}")
        return []

# data_processor.py 示例
def clean_and_standardize_reviews(raw_reviews):
    """清洗并标准化评论数据，例如去除HTML标签、统一评分格式"""
    processed_reviews = []
    for review in raw_reviews:
        cleaned_text = review['text'].replace('\n', ' ').strip() # 简单的文本清理
        # 评分转换示例：将“力荐”转换为数字
        rating_map = {'很差': 1, '较差': 2, '还行': 3, '推荐': 4, '力荐': 5}
        standardized_rating = rating_map.get(review['rating'], review['rating'])

        processed_reviews.append({
            'user': review['user'],
            'rating': standardized_rating,
            'text': cleaned_text,
            'source_url': review['source_url']
        })
    return processed_reviews

def deduplicate_reviews(reviews):
    """基于用户ID和评论内容进行去重"""
    unique_reviews = []
    seen = set()
    for review in reviews:
        # 创建一个简单的哈希键，实际可能需要更复杂的逻辑
        key = (review.get('user'), review.get('text')) 
        if key not in seen:
            unique_reviews.append(review)
            seen.add(key)
    return unique_reviews

# main.py 示例
import api_client
import web_scraper
import data_processor
import time

def run_integration_pipeline(movie_ids_to_process):
    all_raw_reviews = []

    for movie_id in movie_ids_to_process:
        print(f"处理电影ID: {movie_id}")
        # 1. 从API获取基础信息和潜在的评论入口
        movie_data = api_client.fetch_movie_details_from_api(movie_id, api_source='douban')
        if movie_data:
            print(f"  - 从API获取到电影 '{movie_data.get('title', '未知')}' 的数据。")
            # 假设API返回了评论页面的URL或ID
            review_url = movie_data.get('reviews_url') # 这是一个假设的字段
            if review_url:
                # 2. 从网页抓取更详细的评论
                print(f"  - 尝试抓取评论页面: {review_url}")
                scraped_reviews = web_scraper.scrape_movie_reviews(review_url)
                if scraped_reviews:
                    print(f"    - 抓取到 {len(scraped_reviews)} 条评论。")
                    all_raw_reviews.extend(scraped_reviews)
                else:
                    print("    - 未抓取到评论或抓取失败。")
            else:
                print("  - API未提供评论页面URL，跳过爬虫部分。")
        else:
            print(f"  - 未能从API获取到电影ID {movie_id} 的数据。")

        time.sleep(1) # 每次处理完一部电影，稍微休息一下

    # 3. 数据清洗与标准化
    print("\n开始清洗和标准化数据...")
    cleaned_reviews = data_processor.clean_and_standardize_reviews(all_raw_reviews)
    print(f"清洗后剩余 {len(cleaned_reviews)} 条评论。")

    # 4. 数据去重
    print("开始去重...")
    final_reviews = data_processor.deduplicate_reviews(cleaned_reviews)
    print(f"去重后最终得到 {len(final_reviews)} 条独立评论。")

    # 5. 存储数据 (这里省略了database_manager的调用，但实际会在这里写入数据库)
    print("\n整合完成，准备存储数据。")
    # database_manager.save_reviews(final_reviews) 

    return final_reviews

if __name__ == "__main__":
    # 示例：处理几部电影的评论
    target_movie_ids = ['26709849', '35295982'] # 假设的电影ID
    integrated_data = run_integration_pipeline(target_movie_ids)

    # 打印一些结果
    # import json
    # print("\n部分整合后的评论数据：")
    # for i, review in enumerate(integrated_data[:5]): # 只打印前5条
    #     print(json.dumps(review, ensure_ascii=False, indent=2))
    #     if i == 4: break