Numpy数组与Python列表：意外的存储大小差异及其优化策略

import numpy as np
import random

# 假设 all_games 是一个包含多个7元素浮点数列表的列表
# 例如：all_games = [[float(i), float(i+1), ..., float(i+6)] for i in range(100)]

def sample_games_numpy(all_games_list, file_name):
    # 将Python列表转换为Numpy数组
    all_games_np = np.array(all_games_list, dtype=np.float16)
    DRAW = 10000
    SAMPLE = 10000

    # 从 all_games_np 中随机采样
    # sampled_indices 会生成一个 (SAMPLE, DRAW) 的索引数组
    # sampled_data 会根据这些索引从 all_games_np 中提取数据
    # 此时 sampled_data 是一个全新的、独立的Numpy数组，其元素是原始数据的副本
    rng = np.random.default_rng() # 推荐使用新的随机数生成器
    sampled_indices = rng.choice(all_games_np.shape[0], size=(SAMPLE, DRAW), replace=True)
    sampled_data = all_games_np[sampled_indices]

    # 保存为Numpy文件，默认不压缩
    np.save(file_name, sampled_data)
    print(f"Numpy array saved to {file_name}.npy with shape {sampled_data.shape}")

# 示例调用 (all_games_list 需要实际数据)
# all_games_list = [[random.random() for _ in range(7)] for _ in range(1000)]
# sample_games_numpy(all_games_list, 'sampled_numpy_data')

import random
import pickle

# 假设 all_games 是一个包含多个7元素浮点数列表的列表
# 例如：all_games = [[float(i), float(i+1), ..., float(i+6)] for i in range(100)]

def sample_games_pickle(all_games_list, file_name):
    DRAW = 10000
    SAMPLE = 10000

    # 随机采样，注意这里直接从 all_games_list 中选择对象
    # 如果 all_games_list 中包含重复的7元素列表对象，
    # random.choice 可能会多次返回同一个内存地址的列表对象
    sampled_data = [[random.choice(all_games_list) for _ in range(DRAW)] for _ in range(SAMPLE)]

    # 使用pickle保存Python列表
    with open(file_name, 'wb') as file:
        pickle.dump(sampled_data, file)
    print(f"Python list saved to {file_name}.pkl")

# 示例调用 (all_games_list 需要实际数据)
# all_games_list = [[random.random() for _ in range(7)] for _ in range(1000)]
# sample_games_pickle(all_games_list, 'sampled_pickle_data')

def sample_games_pickle_with_copy(all_games_list, file_name):
    DRAW = 10000
    SAMPLE = 10000

    # 强制创建副本，破坏pickle的引用优化
    sampled_data = [[random.choice(all_games_list).copy() for _ in range(DRAW)] for _ in range(SAMPLE)]

    with open(file_name, 'wb') as file:
        pickle.dump(sampled_data, file)
    print(f"Python list (with copies) saved to {file_name}.pkl")

# 示例调用
# all_games_list = [[random.random() for _ in range(7)] for _ in range(1000)]
# sample_games_pickle_with_copy(all_games_list, 'sampled_pickle_data_with_copy')

import numpy as np

def save_compressed_numpy_array(data_array, file_name):
    """
    使用 numpy.savez_compressed 保存Numpy数组。
    """
    np.savez_compressed(file_name, data=data_array)
    print(f"Compressed Numpy array saved to {file_name}.npz with shape {data_array.shape}")

# 假设 sampled_data 是之前生成的10000x10000x7的Numpy数组
# sampled_data = np.random.rand(10000, 10000, 7).astype(np.float16)
# save_compressed_numpy_array(sampled_data, 'sampled_numpy_data_compressed')