使用 Selenium 和 CSS 选择器高效抓取 Patagonia 产品数据

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, NoSuchElementException
import csv

# 配置 Chrome WebDriver
service = Service(executable_path='/path/to/chromedriver') # 替换为你的 chromedriver 路径
driver = webdriver.Chrome(service=service)

# 目标 URL
url = "https://www.patagonia.com/shop/womens"
driver.get(url)

# 用于存储产品信息的列表
product_data = []

def extract_product_data():
    """提取当前页面上的产品数据"""
    products = driver.find_elements(By.CSS_SELECTOR, '#product-search-results > div.row.product-grid.load-more-present > div[class^="col"]')
    for product in products:
        try:
            title = product.find_element(By.CSS_SELECTOR, 'div.product-tile__title > a > span').text
            product_url = product.find_element(By.CSS_SELECTOR, 'div.product-tile__title > a').get_attribute('href')
            image_url = product.find_element(By.CSS_SELECTOR, 'div.product-tile__image > a > img').get_attribute('src')
            price = product.find_element(By.CSS_SELECTOR, 'div.product-tile__price').text
            # 尝试获取评分和评论数量，如果不存在则设置为 None
            try:
                rating = product.find_element(By.CSS_SELECTOR, 'span.sr-only').get_attribute('aria-label')
                review_count = product.find_element(By.CSS_SELECTOR, 'span.oc-rating-count').text.strip('()')
            except NoSuchElementException:
                rating = None
                review_count = None

            product_data.append([title, product_url, image_url, price, rating, review_count])

        except NoSuchElementException as e:
            print(f"Error extracting data from a product: {e}")
            continue

def load_all_products():
    """滚动加载所有产品"""
    while True:
        try:
            # 找到并点击 "加载更多" 按钮
            load_more_button = WebDriverWait(driver, 10).until(
                EC.presence_of_element_located((By.CSS_SELECTOR, '#product-search-results > div.row.product-grid.load-more-present > div.col-12.grid-footer > div > div > button'))
            )
            driver.execute_script("arguments[0].scrollIntoView({ behavior: 'auto', block: 'center' });", load_more_button)
            load_more_button.click()
            # 等待新内容加载 (可以根据实际情况调整等待时间)
            WebDriverWait(driver, 10).until(
                EC.presence_of_element_located((By.CSS_SELECTOR, '#product-search-results > div.row.product-grid.load-more-present > div[class^="col"]:nth-child(24)'))
            ) # 假设每次加载 24 个产品
        except (TimeoutException, NoSuchElementException):
            # 如果没有 "加载更多" 按钮或加载超时，则停止加载
            break

# 执行抓取流程
load_all_products()
extract_product_data()

# 保存数据到 CSV 文件
csv_file = "patagonia_womens_jackets.csv"
with open(csv_file, "w", newline="", encoding="utf-8") as file:
    writer = csv.writer(file)
    writer.writerow(["product title", "product URL", "product image URL", "product price", "rating", "review count"]) # 写入表头
    writer.writerows(product_data)

print(f"Data saved to {csv_file}")

# 关闭浏览器
driver.quit()