使用Selenium高效抓取Google地图完整评论：处理“更多”按钮与动态加载

pip install selenium webdriver-manager

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import TimeoutException, NoSuchElementException, StaleElementReferenceException
import time

# 目标Google地图商家评论页面的URL
# 请替换为你要抓取的实际URL
TARGET_URL = 'https://www.google.com/maps/place/Henn+na+Hotel+Tokyo+Asakusa+Tawaramachi/@35.7081692,139.7888494,17z/data=!4m22!1m12!3m11!1s0x60188f36ab21f05b:0x9241dab287ff62c9!2sHenn+na+Hotel+Tokyo+Asakusa+Tawaramachi!5m2!4m1!1i2!8m2!3d35.7081692!4d139.7914243!9m1!1b1!16s%2Fg%2F11h0gzlhht!3m8!1s0x60188f36ab21f05b:0x9241dab287ff62c9!5m2!4m1!1i2!8m2!3d35.7081692!4d139.7914243!16s%2Fg%2F11h0gzlhht?entry=ttu'

# 配置Chrome选项
chrome_options = Options()
# chrome_options.add_argument('--headless') # 无头模式运行，不显示浏览器界面
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_argument('--start-maximized') # 启动时最大化窗口，确保元素可见

# 初始化WebDriver
driver = webdriver.Chrome(options=chrome_options)
driver.get(TARGET_URL)

def accept_cookie_policy(driver):
    """尝试点击Cookie同意按钮。"""
    try:
        # 等待页面加载，并尝试找到“Accept all”按钮
        # 寻找所有按钮，然后通过文本内容筛选
        WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.TAG_NAME, 'button'))
        )
        buttons = driver.find_elements(By.TAG_NAME, 'button')
        for button in buttons:
            if "Accept all" in button.text: # 使用in操作符更健壮
                print("点击 'Accept all' Cookie按钮。")
                button.click()
                time.sleep(2) # 等待弹窗消失
                return True
    except TimeoutException:
        print("未找到或无法点击Cookie政策按钮。")
    except Exception as e:
        print(f"处理Cookie时发生错误: {e}")
    return False

accept_cookie_policy(driver)

def navigate_to_reviews(driver):
    """导航到评论区。"""
    try:
        # 等待页面加载，并找到“Reviews”按钮
        WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.TAG_NAME, 'button'))
        )
        all_buttons = driver.find_elements(By.TAG_NAME, 'button')
        for button in all_buttons:
            if "Reviews" in button.text:
                print("点击 'Reviews' 按钮。")
                button.click()
                time.sleep(3) # 等待评论区加载
                return True
    except TimeoutException:
        print("未找到 'Reviews' 按钮。")
    except Exception as e:
        print(f"导航到评论区时发生错误: {e}")
    return False

navigate_to_reviews(driver)

def scroll_to_load_all_reviews(driver):
    """滚动评论区以加载所有评论。"""
    print("开始滚动加载所有评论...")
    # Google Maps评论的滚动容器通常具有特定的class或aria-label
    # 经过观察，评论列表本身是一个可滚动区域，其父级可能是一个具有特定样式的div
    # 尝试找到评论列表的父级滚动容器，这里使用一个常见的类名组合
    # 如果这个选择器失效，需要根据实际页面结构调整
    try:
        # 查找包含评论的滚动容器。这里假定评论列表的直接父元素是可滚动的。
        # 实际的Google Maps页面结构可能会有所不同，可能需要更精确的定位。
        # 示例中使用的类名 'm6QErb' 'DxyBCb' 'kA9KIf' 'dS8AEf' 是一个常见的组合，但可能变化。
        # 更稳健的方法可能是寻找一个具有 `role="feed"` 或 `aria-label="Reviews"` 的元素。
        review_scroll_container = WebDriverWait(driver, 15).until(
            EC.presence_of_element_located((By.XPATH, "//div[contains(@class, 'm6QErb') and contains(@class, 'DxyBCb') and contains(@class, 'kA9KIf') and contains(@class, 'dS8AEf')]"))
        )
    except TimeoutException:
        print("未找到评论滚动容器，请检查XPath或页面结构。")
        return

    last_height = driver.execute_script("return arguments[0].scrollHeight", review_scroll_container)

    while True:
        # 模拟滚动到底部
        driver.execute_script("arguments[0].scrollTop = arguments[0].scrollHeight", review_scroll_container)
        time.sleep(3) # 等待新内容加载

        new_height = driver.execute_script("return arguments[0].scrollHeight", review_scroll_container)
        if new_height == last_height:
            # 如果滚动高度没有变化，说明已经到达底部
            print("所有评论已加载。")
            break
        last_height = new_height

    # 滚动到顶部，确保所有元素在视口内以便后续操作（可选，但有时有用）
    driver.execute_script("arguments[0].scrollTop = 0", review_scroll_container)
    time.sleep(2)

scroll_to_load_all_reviews(driver)

def extract_full_reviews(driver):
    """展开所有“更多”按钮并提取完整评论。"""
    print("开始展开评论并提取内容...")
    all_reviews_data = []

    # 查找所有评论容器。Google Maps中每个评论通常在一个具有特定类名的div中。
    # 'jftiEf' 是一个常见的评论容器类名。
    review_elements = WebDriverWait(driver, 10).until(
        EC.presence_of_all_elements_located((By.CLASS_NAME, 'jftiEf'))
    )

    print(f"找到 {len(review_elements)} 条评论容器。")

    for i, review_element in enumerate(review_elements):
        try:
            # 尝试在当前评论容器内查找“More”按钮
            # 'w8nwRe' 是“More”按钮的常见类名
            more_button = review_element.find_element(By.CLASS_NAME, "w8nwRe")
            if "More" in more_button.text: # 再次确认文本是“More”
                more_button.click()
                time.sleep(1) # 点击后等待内容展开
                print(f"点击了第 {i+1} 条评论的 'More' 按钮。")
        except NoSuchElementException:
            # 没有“More”按钮，评论已是完整内容
            pass
        except StaleElementReferenceException:
            # 元素过时，通常是因为页面内容发生变化，尝试重新定位
            print(f"第 {i+1} 条评论的元素过时，尝试重新定位...")
            review_elements = WebDriverWait(driver, 5).until(
                EC.presence_of_all_elements_located((By.CLASS_NAME, 'jftiEf'))
            )
            # 重新获取当前评论元素并重试
            try:
                review_element = review_elements[i]
                more_button = review_element.find_element(By.CLASS_NAME, "w8nwRe")
                if "More" in more_button.text:
                    more_button.click()
                    time.sleep(1)
                    print(f"重新点击了第 {i+1} 条评论的 'More' 按钮。")
            except Exception as e:
                print(f"重新处理第 {i+1} 条评论时仍然出错: {e}")
                pass # 忽略错误，继续下一条

        # 提取评论的完整文本
        # 'wiI7pd' 可能是包含评论文本的元素
        try:
            review_text_element = review_element.find_element(By.CLASS_NAME, 'wiI7pd')
            full_review_text = review_text_element.text
            all_reviews_data.append(full_review_text)
        except NoSuchElementException:
            print(f"未找到第 {i+1} 条评论的文本内容元素。")
            all_reviews_data.append("评论文本未找到") # 记录缺失
        except Exception as e:
            print(f"提取第 {i+1} 条评论文本时发生错误: {e}")
            all_reviews_data.append(f"提取失败: {e}")

    return all_reviews_data

# 执行提取
reviews = extract_full_reviews(driver)

# 打印结果
print("\n--- 抓取到的完整评论 ---")
for idx, review in enumerate(reviews):
    print(f"评论 {idx+1}:\n{review}\n---")

# 关闭浏览器
driver.quit()

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import TimeoutException, NoSuchElementException, StaleElementReferenceException
import time
import csv

# 目标Google地图商家评论页面的URL
TARGET_URL = 'https://www.google.com/maps/place/Henn+na+Hotel+Tokyo+Asakusa+Tawaramachi/@35.7081692,139.7888494,17z/data=!4m22!1m12!3m11!1s0x60188f36ab21f05b:0x9241dab287ff62c9!2sHenn+na+Hotel+Tokyo+Asakusa+Tawaramachi!5m2!4m1!1i2!8m2!3d35.7081692!4d139.7914243!9m1!1b1!16s%2Fg%2F11h0gzlhht!3m8!1s0x60188f36ab21f05b:0x9241dab287ff62c9!5m2!4m1!1i2!8m2!3d35.7081692!4d139.7914243!16s%2Fg%2F11h0gzlhht?entry=ttu'

def initialize_driver():
    """初始化Chrome WebDriver并返回实例。"""
    chrome_options = Options()
    # chrome_options.add_argument('--headless') # 无头模式运行，不显示浏览器界面
    chrome_options.add_argument('--no-sandbox')
    chrome_options.add_argument('--disable-dev-shm-usage')
    chrome_options.add_argument('--start-maximized') # 启动时最大化窗口，确保元素可见
    chrome_options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")

    driver = webdriver.Chrome(options=chrome_options)
    driver.get(TARGET_URL)
    return driver

def accept_cookie_policy(driver):
    """尝试点击Cookie同意按钮。"""
    try:
        WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.TAG_NAME, 'button'))
        )
        buttons = driver.find_elements(By.TAG_NAME, 'button')
        for button in buttons:
            if "Accept all" in button.text:
                print("点击 'Accept all' Cookie按钮。")
                button.click()
                time.sleep(2)
                return True
    except TimeoutException:
        print("未找到或无法点击Cookie政策按钮。")
    except Exception as e:
        print(f"处理Cookie时发生错误: {e}")
    return False

def navigate_to_reviews(driver):
    """导航到评论区。"""
    try:
        WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.TAG_NAME, 'button'))
        )
        all_buttons = driver.find_elements(By.TAG_NAME, 'button')
        for button in all_buttons:
            if "Reviews" in button.text:
                print("点击 'Reviews' 按钮。")
                button.click()
                time.sleep(3)
                return True
    except TimeoutException:
        print("未找到 'Reviews' 按钮。")
    except Exception as e:
        print(f"导航到评论区时发生错误: {e}")
    return False

def scroll_to_load_all_reviews(driver):
    """滚动评论区以加载所有评论。"""
    print("开始滚动加载所有评论...")
    try:
        review_scroll_container = WebDriverWait(driver, 15).until(
            EC.presence_of_element_located((By.XPATH, "//div[contains(@class, 'm6QErb') and contains(@class, 'DxyBCb') and contains(@class, 'kA9KIf') and contains(@class, 'dS8AEf')]"))
        )
    except TimeoutException:
        print("未找到评论滚动容器，请检查XPath或页面结构。")
        return

    last_height = driver.execute_script("return arguments[0].scrollHeight", review_scroll_container)

    while True: