如何高效抓取网页图表中的动态数据

from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC

# 初始化WebDriver
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service()
driver = webdriver.Chrome(options=options, service=webdriver_service)

# 访问目标网页
driver.get('https://www.cpubenchmark.net/cpu.php?cpu=Intel+Core+i9-11900K+%403.50GHz&id=3904')

# 等待图表元素加载
# 注意：EC.presence_of_element_located 返回的是单个WebElement对象
element = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//*[@id='placeholder']/div/canvas[2]")))

# 尝试遍历元素（此处会报错，因为element是单个WebElement，不可迭代）
# for el in element:
#     ActionChains(driver).move_to_element(el).perform()
#     mouseover = WebDriverWait(driver, 30).until(EC.visibility_of_element_located((By.CSS_SELECTOR, ".placeholder > div > div.canvasjs-chart-tooltip > div > span")))
#     print(mouseover.text)

import re
import pandas as pd
import requests

# 目标URL
url = "https://www.cpubenchmark.net/cpu.php?cpu=Intel+Core+i9-11900K+%403.50GHz&id=3904"

# 使用requests获取网页HTML内容
html_text = requests.get(url).text

# 使用正则表达式从HTML中查找并提取数据
# 模式解释：
# dataArray\.push\({x: (\d+), y: ([\d.]+)}
# - dataArray\.push\({x: : 匹配字面字符串 "dataArray.push({x: "
# - (\d+) : 捕获一个或多个数字（时间戳）
# - , y: : 匹配字面字符串 ", y: "
# - ([\d.]+) : 捕获一个或多个数字或点（价格，可能包含小数点）
# - } : 匹配字面字符串 "}"
df = pd.DataFrame(
    re.findall(r"dataArray\.push\({x: (\d+), y: ([\d.]+)}", html_text),
    columns=["time", "price"],
)

# 将时间戳（Unix时间戳，通常是毫秒）转换为可读的日期时间格式
# 网站给出的时间戳是毫秒，需要除以1000转换为秒
df["time"] = pd.to_datetime(df["time"].astype(int) // 1000, unit="s")

# 打印数据框的最后几行以验证结果
print(df.tail())

                   time   price
236 2023-05-28 06:00:00  317.86
237 2023-05-29 06:00:00  319.43
238 2023-05-30 06:00:00  429.99
239 2023-05-31 06:00:00  314.64
240 2023-06-01 06:00:00   318.9