Python网络爬虫：高效提取图表数据，告别鼠标悬停烦恼

from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC

options = Options()
options.add_argument("start-maximized")
webdriver_service = Service()
driver = webdriver.Chrome(options=options, service=webdriver_service)

driver.get('https://www.cpubenchmark.net/cpu.php?cpu=Intel+Core+i9-11900K+%403.50GHz&id=3904')
# 尝试定位图表元素
element = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//*[@id='placeholder']/div/canvas[2]")))

# 尝试遍历元素并模拟鼠标悬停
# for el in element: # 错误：element是一个WebElement对象，不可迭代
#     ActionChains(driver).move_to_element(el).perform()
#     mouseover = WebDriverWait(driver, 30).until(EC.visibility_of_element_located((By.CSS_SELECTOR, ".placeholder > div > div.canvasjs-chart-tooltip > div > span")))
#     print(mouseover.text)

import requests
import re
import pandas as pd

url = "https://www.cpubenchmark.net/cpu.php?cpu=Intel+Core+i9-11900K+%403.50GHz&id=3904"
html_text = requests.get(url).text

df = pd.DataFrame(
    re.findall(r"dataArray\.push\({x: (\d+), y: ([\d.]+)}", html_text),
    columns=["time", "price"],
)

# 将时间戳从字符串转换为整数，并除以1000（如果原始是毫秒级时间戳），然后转换为datetime对象
df["time"] = pd.to_datetime(df["time"].astype(int) // 1000, unit="s")
# 将价格从字符串转换为浮点数
df["price"] = df["price"].astype(float)

import requests
import re
import pandas as pd

# 目标网页URL
url = "https://www.cpubenchmark.net/cpu.php?cpu=Intel+Core+i9-11900K+%403.50GHz&id=3904"

# 步骤一：获取网页内容
html_text = requests.get(url).text

# 步骤二：识别并提取JavaScript数据
# 使用正则表达式匹配 dataArray.push({x: timestamp, y: price}) 格式的数据
# (\d+) 捕获时间戳， ([\d.]+) 捕获价格（可能包含小数）
df = pd.DataFrame(
    re.findall(r"dataArray\.push\({x: (\d+), y: ([\d.]+)}", html_text),
    columns=["time", "price"],
)

# 步骤三：数据清洗与转换
# 将时间戳从字符串转换为整数，并将其除以1000（如果原始是毫秒级时间戳），然后转换为datetime对象
# unit='s' 表示时间戳的单位是秒
df["time"] = pd.to_datetime(df["time"].astype(int) // 1000, unit="s")
# 将价格从字符串转换为浮点数
df["price"] = df["price"].astype(float)

# 打印DataFrame的最后几行，验证数据
print(df.tail())

                   time   price
236 2023-05-28 06:00:00  317.86
237 2023-05-29 06:00:00  319.43
238 2023-05-30 06:00:00  429.99
239 2023-05-31 06:00:00  314.64
240 2023-06-01 06:00:00  318.90