Python从API获取并解析Parquet数据实战指南

import requests

def get_orders_data_incorrect(date):
    url = "YOUR_API_BASE_URL/orders" # 替换为你的API地址
    headers = {}
    params = {"date": date}

    response = requests.get(url, headers=headers, params=params)

    if response.status_code == 200:
        # 错误：Parquet是二进制数据，不应使用response.text
        data = response.text.strip()
        return data
    else:
        print(f"Failed to fetch orders data: {response.status_code}")
        return None

# 假设orders_info是使用上述错误方法获取的乱码字符串
# parquet_data_str = orders_info
# buffer = io.BytesIO(parquet_data_str.encode()) # 此时encode会再次出错或生成无效字节流

import requests
import io
import pandas as pd
import pyarrow.parquet as pq # 虽然这里直接用pandas，但pyarrow是其底层依赖

def get_orders_data_solution1(date: str) -> pd.DataFrame | None:
    """
    从API获取订单数据并直接解析为Pandas DataFrame。
    """
    url = "YOUR_API_BASE_URL/orders" # 替换为你的API地址
    headers = {} # 根据需要添加认证或其他头部
    params = {"date": date}

    try:
        response = requests.get(url, headers=headers, params=params)
        response.raise_for_status() # 如果状态码不是200，则抛出HTTPError

        # 核心：使用response.content获取原始二进制数据
        # 并通过io.BytesIO封装，然后由pd.read_parquet直接读取
        df = pd.read_parquet(io.BytesIO(response.content))
        return df
    except requests.exceptions.RequestException as e:
        print(f"请求失败: {e}")
        return None
    except Exception as e:
        print(f"数据解析失败: {e}")
        return None

# 示例调用
date_to_fetch = "2023-12-08"
orders_df = get_orders_data_solution1(date_to_fetch)

if orders_df is not None:
    print("成功获取并解析订单数据，前5行：")
    print(orders_df.head())
    print(f"DataFrame形状: {orders_df.shape}")
else:
    print("未能获取或解析订单数据。")

import requests
import io
import pandas as pd
import pyarrow.parquet as pq

def get_orders_data_solution2(date: str) -> pd.DataFrame | None:
    """
    从API获取订单数据，通过pyarrow解析，然后转换为Pandas DataFrame。
    """
    url = "YOUR_API_BASE_URL/orders" # 替换为你的API地址
    headers = {}
    params = {"date": date}

    try:
        response = requests.get(url, headers=headers, params=params)
        response.raise_for_status() # 如果状态码不是200，则抛出HTTPError

        # 核心：使用response.content获取原始二进制数据
        buffer = io.BytesIO(response.content)

        # 通过pyarrow.parquet读取数据到Arrow Table
        table = pq.read_table(buffer)

        # 将Arrow Table转换为Pandas DataFrame
        df = table.to_pandas()
        return df
    except requests.exceptions.RequestException as e:
        print(f"请求失败: {e}")
        return None
    except Exception as e:
        print(f"数据解析失败: {e}")
        return None

# 示例调用
date_to_fetch = "2023-12-08"
orders_df_alt = get_orders_data_solution2(date_to_fetch)

if orders_df_alt is not None:
    print("\n成功获取并解析订单数据（方案二），前5行：")
    print(orders_df_alt.head())
    print(f"DataFrame形状: {orders_df_alt.shape}")
else:
    print("未能获取或解析订单数据（方案二）。")

# 假设 orders_df 是从API获取并解析后的DataFrame
if orders_df is not None:
    output_filename = f"orders_{date_to_fetch}.parquet"
    try:
        orders_df.to_parquet(output_filename, index=False) # index=False表示不将DataFrame的索引写入Parquet文件
        print(f"\nDataFrame已成功保存到 {output_filename}")
    except Exception as e:
        print(f"保存Parquet文件失败: {e}")