使用Selenium高效抓取层级结构数据：关联H2标题与P标签内容

<h2><span>Title1</span></h2>
<p>text I want</p>
<p>text I want</p>
<h2><span>Title2</span></h2>
<p>text I want</p>
<p>text I want</p>
<p>text I want</p>
<h2><span>Title3</span></h2>
<p>text I want</p>
<p>text I want</p>

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
import time

# 1. 初始化Selenium WebDriver
# 推荐使用webdriver_manager自动管理驱动
service = Service(ChromeDriverManager().install())
driver = webdriver.Chrome(service=service)

# 2. 加载HTML内容
# 这里使用data URI加载一个字符串HTML，实际应用中替换为driver.get("your_url_here")
html_content = '''
<h2><span>Title1</span></h2>
<p>text I want 1.1</p>
<p>text I want 1.2</p>
<h2><span>Title2</span></h2>
<p>text I want 2.1</p>
<p>text I want 2.2</p>
<p>text I want 2.3</p>
<h2><span>Title3</span></h2>
<p>text I want 3.1</p>
<p>text I want 3.2</p>
'''
driver.get("data:text/html;charset=utf-8," + html_content)
time.sleep(1) # 给浏览器一些时间加载内容

# 3. 初始化数据字典
# 遍历所有h2元素，以其文本为键，空字符串为值初始化字典
# 注意：h2.text 会自动处理内部的span标签，提取出正确的文本
data_mapping = dict((h2_element.text, '') for h2_element in driver.find_elements(By.CSS_SELECTOR, 'h2'))

# 4. 遍历并聚合p标签内容
# 遍历所有p元素
for p_element in driver.find_elements(By.CSS_SELECTOR, 'p'):
    # 找到当前p元素最近的前一个h2同级元素
    # XPath './preceding-sibling::h2[1]' 表示当前节点（.）之前（preceding-sibling）的h2元素中的第一个（[1]）
    try:
        preceding_h2 = p_element.find_element(By.XPATH, './preceding-sibling::h2[1]')
        h2_text = preceding_h2.text
        # 将p标签的文本追加到对应h2键的值中，用空格分隔
        data_mapping[h2_text] += ' ' + p_element.text
    except Exception as e:
        print(f"Error processing p tag: {p_element.text}. No preceding h2 found or other error: {e}")
        # 可以选择跳过或记录此类p标签

# 5. 构建结构化输出
# 将字典转换为列表，每个元素是一个包含'title'和'content'键的字典
structured_output = [{'title': title, 'content': content.strip()} for title, content in data_mapping.items()]

# 打印结果
print(structured_output)

# 关闭浏览器
driver.quit()

[{'title': 'Title1', 'content': 'text I want 1.1 text I want 1.2'},
 {'title': 'Title2', 'content': 'text I want 2.1 text I want 2.2 text I want 2.3'},
 {'title': 'Title3', 'content': 'text I want 3.1 text I want 3.2'}]