使用Selenium和Python执行异步Fetch请求并获取响应

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
import json

# 示例代理设置 (注意：对于execute_async_script发起的fetch，此设置可能不总是生效)
# 如果fetch请求需要通过代理，通常需要更复杂的网络拦截或直接在fetch选项中指定代理
proxy_server_url = "127.0.0.1:8888"
chrome_options = webdriver.ChromeOptions()
# chrome_options.add_argument('--proxy-server=%s' % proxy_server_url)
chrome_options.add_argument("ignore-certificate-errors")
chrome_options.add_argument("--headless") # 可选：无头模式运行

driver = webdriver.Chrome(options=chrome_options)

# 设置异步脚本的超时时间，默认为0，需要手动设置
driver.set_script_timeout(10) # 设置为10秒

# 导航到一个空白页或任意页面，以确保浏览器环境已准备好执行JS
driver.get("about:blank")
print("WebDriver initialized and navigated to about:blank.")

# 示例：GET请求，获取JSON数据
get_url = "https://jsonplaceholder.typicode.com/posts/1" # 示例URL
get_headers = {
    "Accept": "application/json",
    "Custom-Header-GET": "get_value_123"
}

# 构建JavaScript fetch请求脚本
js_script_get = """
let callback = arguments[0]; // Selenium注入的回调函数
let url = arguments[1];
let headers = arguments[2];

fetch(url, {
    method: 'GET',
    headers: headers
})
.then(response => {
    if (!response.ok) {
        // 如果响应状态码不是2xx，抛出错误
        return response.text().then(text => { throw new Error(`Network response was not ok: ${response.status} - ${text}`); });
    }
    return response.json(); // 尝试解析为JSON
})
.then(data => callback(data)) // 将解析后的数据通过回调返回给Python
.catch(error => callback({ error: error.message })); // 捕获错误并返回错误信息
"""

print(f"\nExecuting GET request to: {get_url}")
try:
    response_data_get = driver.execute_async_script(js_script_get, get_url, get_headers)
    print("GET Request Response (Python):")
    print(json.dumps(response_data_get, indent=2, ensure_ascii=False))
except TimeoutException:
    print("GET request timed out.")
except Exception as e:
    print(f"An error occurred during GET request: {e}")

# 示例：POST请求，发送JSON数据
post_url = "https://jsonplaceholder.typicode.com/posts" # 示例URL
post_data = {"title": "foo", "body": "bar", "userId": 1}
post_headers = {
    "Content-Type": "application/json",
    "Accept": "application/json",
    "Custom-Header-POST": "post_value_456"
}

# 构建JavaScript fetch请求脚本
js_script_post = """
let callback = arguments[0];
let url = arguments[1];
let headers = arguments[2];
let body = arguments[3];

fetch(url, {
    method: 'POST',
    headers: headers,
    body: JSON.stringify(body) // 将POST数据转换为JSON字符串
})
.then(response => {
    if (!response.ok) {
        return response.text().then(text => { throw new Error(`Network response was not ok: ${response.status} - ${text}`); });
    }
    return response.json();
})
.then(data => callback(data))
.catch(error => callback({ error: error.message }));
"""

print(f"\nExecuting POST request to: {post_url}")
try:
    response_data_post = driver.execute_async_script(js_script_post, post_url, post_headers, post_data)
    print("POST Request Response (Python):")
    print(json.dumps(response_data_post, indent=2, ensure_ascii=False))
except TimeoutException:
    print("POST request timed out.")
except Exception as e:
    print(f"An error occurred during POST request: {e}")

# 完成后关闭WebDriver
driver.quit()
print("\nWebDriver closed.")