解决Python requests循环请求中遇到的401未授权错误

from requests import get
from time import sleep
from multiprocessing.pool import ThreadPool
import logging

# 配置日志，以便更好地追踪请求状态和错误
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

# 配置常量
MAX_RETRIES = 5  # 最大重试次数
DELAY = 10       # 每次重试批次之间的延迟时间（秒）
THREAD_POOL_SIZE = 5 # 线程池大小，可根据实际情况调整

# 示例游戏ID列表，实际应用中可能远大于此
gameIds = [
    "2022020022", "2022020028", "2022020044", "2022020072", "2022020088",
    "2022020090", "2022020091", "2022020092", "2022020093", "2022020094",
    "2022020095", "2022020096", "2022020097", "2022020098", "2022020099"
]

all_data = []  # 用于存储所有成功获取的数据

# 定义一个处理单个游戏ID的函数
def process_game_data(gameId: str) -> tuple[dict, str]:
    """
    尝试获取单个游戏ID的数据。
    返回一个元组：(数据字典, 成功获取数据的gameId)
    如果失败，返回({}, "")
    """
    rv = {}, ""
    url = f"https://api-web.nhle.com/v1/gamecenter/{gameId}/play-by-play"

    # 可以在这里添加自定义的headers和proxies
    # headers = {'User-Agent': 'YourCustomUserAgent'}
    # proxies = {'http': 'http://your_proxy:port', 'https': 'https://your_proxy:port'}

    try:
        # 使用with语句确保response资源被正确关闭
        with get(url) as response:
            match response.status_code:
                case 200:
                    data = response.json()
                    rv = data, gameId
                    logging.info(f"成功获取数据: {gameId}")
                case 401:
                    logging.warning(f"获取数据失败，Game ID: {gameId}，状态码: 401 Unauthorized. 将在下一轮重试。")
                    # 对于401，我们不抛出异常，而是让其进入下一轮重试
                case _:
                    # 对于其他非200/401的状态码，抛出HTTPError
                    response.raise_for_status()
                    logging.error(f"获取数据失败，Game ID: {gameId}，状态码: {response.status_code}")
    except Exception as e:
        logging.error(f"无法获取Game ID: {gameId} 的数据，原因: {e}")
    return rv

# 主循环：执行多次重试批次
remaining_game_ids = gameIds.copy() # 复制原始列表，以便在重试时修改
for attempt in range(MAX_RETRIES):
    if not remaining_game_ids:
        logging.info("所有数据已成功获取。")
        break

    logging.info(f"开始第 {attempt + 1} 次尝试，剩余 {len(remaining_game_ids)} 个游戏ID。")

    # 使用线程池并发处理当前批次的请求
    # imap_unordered可以无序地返回结果，提高效率
    with ThreadPool(THREAD_POOL_SIZE) as pool:
        # 使用list()来强制迭代并收集所有结果，避免在remove时修改迭代器
        results = list(pool.imap_unordered(process_game_data, remaining_game_ids))

        # 清空当前剩余ID列表，重新构建
        current_successful_ids = []
        for data, gameId in results:
            if data:
                all_data.append(data)
                current_successful_ids.append(gameId)

        # 从remaining_game_ids中移除已成功获取的ID
        remaining_game_ids = [gid for gid in remaining_game_ids if gid not in current_successful_ids]

    if remaining_game_ids:
        logging.info(f"本轮尝试后，仍有 {len(remaining_game_ids)} 个游戏ID未获取。将在 {DELAY} 秒后进行下一次尝试。")
        sleep(DELAY) # 在每次重试批次之间加入延迟
    else:
        logging.info("所有数据已成功获取。")
        break

# 最终检查未获取的数据
if remaining_game_ids:
    logging.warning(f"经过 {MAX_RETRIES} 次尝试后，仍无法获取以下游戏ID的数据: {remaining_game_ids}")
else:
    logging.info("所有指定的游戏数据均已成功获取。")

# all_data 列表现在包含了所有成功获取的游戏数据
# print(f"总共获取了 {len(all_data)} 条数据。")