Indiegogo网站产品URL爬取失败：如何正确处理clickthrough_url列数据及应对反爬机制？

def extract_project_url(df_input):
    list_url = []
    for ele in df_input["clickthrough_url"]:
        list_url.append("https://www.indiegogo.com" + ele)
    return list_url

def extract_project_url(df_input):
    list_url = []
    for ele in df_input["clickthrough_url"]:
        try:
            list_url.append("https://www.indiegogo.com" + str(ele))
        except TypeError as e:
            print(f"Warning: Skipping invalid element: {ele}, Error: {e}")
            continue #跳过无效元素
    return list_url

def extract_project_url(df_input):
    return "https://www.indiegogo.com" + df_input["clickthrough_url"].astype(str)

import pandas as pd
import requests
import time
import random

def extract_project_url(df_input):
    return "https://www.indiegogo.com" + df_input["clickthrough_url"].astype(str)

def fetch_page(url, headers):
    time.sleep(random.uniform(1, 3)) #添加随机延迟
    try:
        response = requests.get(url, headers=headers)
        response.raise_for_status() #检查HTTP状态码
        return response.text
    except requests.exceptions.RequestException as e:
        print(f"Error fetching {url}: {e}")
        return None

# 读取CSV文件
df = pd.read_csv("1.csv")

# 提取URL
urls = extract_project_url(df)

# 设置headers (包括用户代理)
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36'
}

# 抓取页面内容
for url in urls:
    page_content = fetch_page(url, headers)
    if page_content:
        # 处理页面内容 (根据需要修改)
        print(f"Successfully fetched: {url}")
        # ... your code to process page_content ...