高效处理URL文件下载：解决压缩包内文件损坏问题

import requests
import pandas as pd
from tqdm import tqdm

def download_and_process_csv(url, save_path):
    try:
        response = requests.get(url, stream=True)
        response.raise_for_status() # 检查HTTP请求是否成功

        with open(save_path, "wb") as file:
            for chunk in tqdm(response.iter_content(chunk_size=8192)):
                if chunk:
                    file.write(chunk)
        print(f"文件已下载到: {save_path}")

        # 尝试直接读取CSV
        csv_data = pd.read_csv(save_path)
        print("CSV文件内容前5行:")
        print(csv_data.head())

    except requests.exceptions.RequestException as e:
        print(f"请求错误: {e}")
    except pd.errors.EmptyDataError:
        print(f"文件 {save_path} 为空或不包含数据。")
    except Exception as e:
        print(f"发生错误: {e}")

# 示例URL (假设它直接是CSV，但实际可能是ZIP)
# download_and_process_csv("your_csv_url_here", "downloaded_file.csv")

import requests
import zipfile
import tempfile
import os
from tqdm import tqdm

def download_and_extract_zip(url, extract_path="."):
    """
    从URL下载ZIP文件并解压其内容。

    Args:
        url (str): ZIP文件的下载URL。
        extract_path (str): 解压文件的目标路径。默认为当前工作目录。
    """
    CHUNK_SIZE = 32 * 1024  # 定义分块大小，例如32KB

    try:
        print(f"开始从 {url} 下载...")
        with requests.get(url, stream=True) as response:
            response.raise_for_status()  # 检查HTTP请求是否成功

            # 使用临时文件存储下载的ZIP内容
            with tempfile.TemporaryFile() as temp_zip_file:
                # 获取文件总大小以显示进度
                total_size = int(response.headers.get('content-length', 0))
                with tqdm(total=total_size, unit='B', unit_scale=True, desc="下载ZIP") as pbar:
                    for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
                        if chunk:  # 过滤掉保持连接的新块
                            temp_zip_file.write(chunk)
                            pbar.update(len(chunk))

                temp_zip_file.flush() # 确保所有数据都写入临时文件
                temp_zip_file.seek(0) # 将文件指针移回文件开头，以便zipfile读取

                print("下载完成，开始解压...")
                with zipfile.ZipFile(temp_zip_file) as zf:
                    print(f"ZIP文件包含以下内容: {zf.namelist()}")
                    # 确保解压路径存在
                    os.makedirs(extract_path, exist_ok=True)
                    zf.extractall(path=extract_path)
                    print(f"文件已成功解压到: {os.path.abspath(extract_path)}")
                    return zf.namelist() # 返回解压的文件名列表

    except requests.exceptions.RequestException as e:
        print(f"请求错误: {e}")
    except zipfile.BadZipFile:
        print(f"下载的文件不是一个有效的ZIP文件。请检查URL或文件内容。")
    except Exception as e:
        print(f"发生未知错误: {e}")
    return []

# 示例用法：
# 假设此URL指向一个包含CSV文件的ZIP文件
ZIP_URL = "https://prod-dcd-datasets-cache-zipfiles.s3.eu-west-1.amazonaws.com/mpjzbtfgfr-1.zip"
extracted_files = download_and_extract_zip(ZIP_URL, extract_path="./extracted_data")

if extracted_files:
    print("\n解压后的文件列表:")
    for f_name in extracted_files:
        print(f"- {f_name}")
        # 如果需要，可以进一步处理解压后的文件，例如读取CSV
        if f_name.endswith(".csv"):
            try:
                # 假设CSV文件在解压路径下
                csv_path = os.path.join("./extracted_data", f_name)
                df = pd.read_csv(csv_path)
                print(f"成功读取 {f_name} 的前5行数据:")
                print(df.head())
            except Exception as e:
                print(f"读取CSV文件 {f_name} 时发生错误: {e}")