使用Selenium与CSS选择器：动态网页数据提取实战指南

from selenium.webdriver.common.by import By

# 定位ID为'product-search-results'的元素
element = driver.find_element(By.CSS_SELECTOR, '#product-search-results')

# 定位所有class为'product-grid-item'的产品卡片
product_cards = driver.find_elements(By.CSS_SELECTOR, '.product-grid-item')

pip install selenium

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, NoSuchElementException
import time
import csv

# 配置ChromeDriver路径 (根据实际情况修改)
# service = Service(executable_path='/path/to/chromedriver')
# driver = webdriver.Chrome(service=service)

# 推荐使用webdriver_manager自动管理ChromeDriver
from webdriver_manager.chrome import ChromeDriverManager
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))

# 可选：配置为无头模式，不显示浏览器界面，提高效率
# options = webdriver.ChromeOptions()
# options.add_argument('--headless')
# options.add_argument('--disable-gpu') # 某些Linux系统需要
# driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)

driver.maximize_window() # 最大化窗口，确保元素可见

target_url = "https://www.patagonia.com/shop/womens-jackets" # 示例URL，请替换为实际目标
driver.get(target_url)

def load_all_products(driver):
    """
    尝试点击“加载更多”按钮，直到所有产品加载完毕。
    """
    while True:
        try:
            # 等待“加载更多”按钮出现并可点击
            # 这里的CSS选择器需要根据实际网站结构调整
            load_more_button_selector = '#product-search-results > div.row.product-grid.load-more-present > div.col-12.grid-footer > div > div > button'
            load_more_button = WebDriverWait(driver, 10).until(
                EC.element_to_be_clickable((By.CSS_SELECTOR, load_more_button_selector))
            )

            # 滚动到按钮位置，确保其在视图内
            driver.execute_script("arguments[0].scrollIntoView({ behavior: 'auto', block: 'center' });", load_more_button)
            time.sleep(0.5) # 短暂等待，模拟用户操作

            # 点击按钮
            load_more_button.click()
            time.sleep(2) # 等待新内容加载

        except TimeoutException:
            print("未找到或无法点击“加载更多”按钮，或所有内容已加载。")
            break # 按钮不再出现或不可点击，说明已加载完毕
        except NoSuchElementException:
            print("“加载更多”按钮的CSS选择器不正确或元素不存在。")
            break
        except Exception as e:
            print(f"加载更多时发生未知错误: {e}")
            break

print("开始加载所有产品...")
load_all_products(driver)
print("所有产品加载完成。")

products_data = []

# 定位所有产品容器，这里的CSS选择器需要根据实际网站结构调整
# 示例：每个产品项可能是一个带有特定类名的div
product_items_selector = '#product-search-results > div.row.product-grid.load-more-present > div.col-6.col-sm-4.col-md-3.product-grid-item'
product_elements = driver.find_elements(By.CSS_SELECTOR, product_items_selector)

print(f"找到 {len(product_elements)} 个产品。")

for i, product_element in enumerate(product_elements):
    product_info = {
        'product_title': '',
        'product_url': '',
        'product_image_url': '',
        'product_price': '',
        'rating': '',
        'review_count': ''
    }
    try:
        # 提取产品标题和URL
        # 示例：标题通常在h2或h3标签内的a链接中
        title_link = product_element.find_element(By.CSS_SELECTOR, 'h2.product-name > a')
        product_info['product_title'] = title_link.text.strip()
        product_info['product_url'] = title_link.get_attribute('href')

        # 提取产品图片URL
        # 示例：图片通常在img标签的src属性中
        image_element = product_element.find_element(By.CSS_SELECTOR, 'img.product-image')
        product_info['product_image_url'] = image_element.get_attribute('src')

        # 提取产品价格
        # 示例：价格可能在一个带有特定类名的span或div中
        price_element = product_element.find_element(By.CSS_SELECTOR, '.price-sales')
        product_info['product_price'] = price_element.text.strip()

        # 提取评分和评论数量（如果可用）
        # 这些元素可能不存在，需要用try-except处理
        try:
            rating_element = product_element.find_element(By.CSS_SELECTOR, '.rating-stars span.sr-only')
            product_info['rating'] = rating_element.get_attribute('innerHTML').strip() # 假设评分在sr-only的innerHTML中
        except NoSuchElementException:
            product_info['rating'] = 'N/A'

        try:
            review_count_element = product_element.find_element(By.CSS_SELECTOR, '.review-count')
            product_info['review_count'] = review_count_element.text.strip().replace('(', '').replace(')', '')
        except NoSuchElementException:
            product_info['review_count'] = 'N/A'

        products_data.append(product_info)

    except NoSuchElementException as e:
        print(f"第 {i+1} 个产品元素缺少部分信息: {e}")
        # 可以选择跳过此产品或记录部分信息
    except Exception as e:
        print(f"处理第 {i+1} 个产品时发生未知错误: {e}")

# 打印部分结果
for product in products_data[:5]:
    print(product)

# 定义CSV文件头
csv_headers = ['product_title', 'product_url', 'product_image_url', 'product_price', 'rating', 'review_count']
csv_file_name = 'patagonia_womens_jackets.csv'

with open(csv_file_name, 'w', newline='', encoding='utf-8') as csvfile:
    writer = csv.DictWriter(csvfile, fieldnames=csv_headers)
    writer.writeheader()
    writer.writerows(products_data)

print(f"数据已成功保存到 {csv_file_name}")

# 关闭浏览器
driver.quit()