Python批量构建URL与GET请求：多文件迭代的正确实践

import requests

# 假设 hosts.txt 内容:
# google.com
# target.com
# bing.com

# 假设 strings.txt 内容:
# x
# y
# z

# 错误示范：文件迭代器耗尽
with open('hosts.txt', 'r') as file:
    with open('strings.txt', 'r') as strings:
        for line in file:
            host = line.strip()
            # 内部循环迭代 strings 文件
            for string in strings:
                param_value = string.strip()
                url = f"https://{host}/?test={param_value}"
                # resp = requests.get(url) # 实际请求
                print(f'构建URL: {url}')

import requests
import os # 用于检查文件是否存在

# 假设 hosts.txt 和 strings.txt 文件存在于脚本同级目录
hosts_file_path = 'hosts.txt'
strings_file_path = 'strings.txt'

# 1. 读取主机列表
hosts = []
if os.path.exists(hosts_file_path):
    try:
        with open(hosts_file_path, 'r') as f_hosts:
            # 使用列表推导式读取并清理数据，同时过滤掉完全为空的行
            hosts = [line.strip() for line in f_hosts if line.strip()]
    except IOError as e:
        print(f"读取文件 {hosts_file_path} 失败: {e}")
else:
    print(f"错误：文件 {hosts_file_path} 未找到。")

# 2. 读取参数字符串列表
strings = []
if os.path.exists(strings_file_path):
    try:
        with open(strings_file_path, 'r') as f_strings:
            # 使用列表推导式读取并清理数据，同时过滤掉完全为空的行
            strings = [line.strip() for line in f_strings if line.strip()]
    except IOError as e:
        print(f"读取文件 {strings_file_path} 失败: {e}")
else:
    print(f"错误：文件 {strings_file_path} 未找到。")

print(f"加载的主机: {hosts}")
print(f"加载的参数字符串: {strings}")

# 3. 嵌套循环构建URL并发送请求
if not hosts or not strings:
    print("没有足够的数据来构建URL和发送请求，请检查输入文件。")
else:
    print("\n开始构建URL并发送请求:")
    for host in hosts:
        for param_value in strings:
            # 构建URL，推荐使用requests库的params参数来处理查询字符串
            # 这样可以自动处理URL编码，避免手动拼接可能出现的错误
            base_url = f"https://{host}/"
            params = {'test': param_value}

            try:
                # 发送GET请求，并设置超时时间以避免长时间阻塞
                response = requests.get(base_url, params=params, timeout=10)
                # requests库会自动构建完整的URL，我们可以从response对象中获取
                full_url = response.url
                print(f"请求URL: {full_url}, 状态码: {response.status_code}")

                # 根据需要处理响应内容，例如检查状态码或解析JSON/HTML
                # if response.status_code == 200:
                #     print(f"响应内容片段: {response.text[:100]}...")
                # else:
                #     print(f"请求失败，响应状态码: {response.status_code}")

            except requests.exceptions.Timeout:
                print(f"请求 {base_url} (参数: {params}) 超时。")
            except requests.exceptions.ConnectionError as e:
                print(f"请求 {base_url} (参数: {params}) 连接错误: {e}")
            except requests.exceptions.RequestException as e:
                print(f"请求 {base_url} (参数: {params}) 发生其他请求错误: {e}")
            except Exception as e:
                print(f"发生未知错误: {e}")

加载的主机: ['google.com', 'target.com', 'bing.com']
加载的参数字符串: ['x', 'y', 'z']

开始构建URL并发送请求:
请求URL: https://google.com/?test=x, 状态码: 302
请求URL: https://google.com/?test=y, 状态码: 302
请求URL: https://google.com/?test=z, 状态码: 302
请求URL: https://target.com/?test=x, 状态码: 200
请求URL: https://target.com/?test=y, 状态码: 200
请求URL: https://target.com/?test=z, 状态码: 200
请求URL: https://bing.com/?test=x, 状态码: 200
请求URL: https://bing.com/?test=y, 状态码: 200
请求URL: https://bing.com/?test=z, 状态码: 200