Python urllib 遇到 403 Forbidden 错误的排查与解决

import urllib.request
import urllib.error

# 替换为你的目标URL
url = "http://example.com/some/protected/path" 

# 模拟更全面的浏览器请求头
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,application/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
    "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.3",
    "Accept-Encoding": "gzip, deflate, br", # 注意：如果设置此项，可能需要处理压缩响应
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-US;q=0.7",
    "Connection": "keep-alive",
    "Referer": "http://example.com/" # 替换为合适的Referer，通常是目标域名的根目录或上一页
}

try:
    req = urllib.request.Request(url=url, headers=headers)
    with urllib.request.urlopen(req) as response:
        html_content = response.read().decode("utf-8")
        print("请求成功，内容长度:", len(html_content))
except urllib.error.HTTPError as e:
    print(f"HTTP Error: {e.code} - {e.reason}")
    print(f"URL: {e.url}")
    # 打印服务器返回的详细错误信息（如果有）
    try:
        error_page = e.read().decode("utf-8")
        print("Error Page Content:", error_page[:500]) # 打印前500字符
    except Exception as read_e:
        print(f"Failed to read error page: {read_e}")
except urllib.error.URLError as e:
    print(f"URL Error: {e.reason}")

import urllib.request
import urllib.error
import http.cookiejar

# 替换为你的目标URL
url = "http://example.com/some/protected/path" 

# 1. 创建一个CookieJar实例来存储和管理Cookies
cookie_jar = http.cookiejar.CookieJar()

# 2. 创建一个opener，它将自动处理Cookies
# HTTPCookieProcessor 会自动从服务器响应中提取Cookies并将其添加到后续请求中
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookie_jar))

# 3. 将这个opener设置为全局默认，这样所有urllib.request.urlopen都会使用它
urllib.request.install_opener(opener)

# 模拟更全面的浏览器请求头
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,application/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-US;q=0.7",
    "Connection": "keep-alive",
    "Referer": "http://example.com/"
}

try:
    # 发起请求。如果目标URL需要登录，可能需要先访问登录页并POST表单
    # 这里简化为直接对目标URL发起请求，让cookiejar自动处理可能返回的cookies
    req = urllib.request.Request(url=url, headers=headers)
    with urllib.request.urlopen(req) as response:
        print("请求成功，状态码:", response.getcode())
        html_content = response.read().decode("utf-8")
        print("内容长度:", len(html_content))

        # 此时cookie_jar中可能已经包含了服务器返回的cookies
        print("\n当前Cookie Jar中的Cookies:")
        for cookie in cookie_jar:
            print(f"  Name: {cookie.name}, Value: {cookie.value}, Domain: {cookie.domain}")

except urllib.error.HTTPError as e:
    print(f"HTTP Error: {e.code} - {e.reason}")
    try:
        error_page = e.read().decode("utf-8")
        print("Error Page Content:", error_page[:500])
    except Exception as read_e:
        print(f"Failed to read error page: {read_e}")
except urllib.error.URLError as e:
    print(f"URL Error: {e.reason}")