Golang如何加速文件IO操作 配置缓冲区大小与mmap技巧

package main

import (
    "bufio"
    "fmt"
    "os"
    "time"
)

func main() {
    filePath := "large_file.txt" // 替换成你的大文件路径
    bufferSizes := []int{4096, 8192, 16384, 32768, 65536} // 尝试不同的缓冲区大小

    for _, bufferSize := range bufferSizes {
        startTime := time.Now()
        file, err := os.Open(filePath)
        if err != nil {
            fmt.Println("Error opening file:", err)
            return
        }
        defer file.Close()

        reader := bufio.NewReaderSize(file, bufferSize)
        count := 0
        buf := make([]byte, 1024) // 每次读取1KB
        for {
            n, err := reader.Read(buf)
            if n > 0 {
                count += n
            }
            if err != nil {
                break // 假设io.EOF是唯一需要停止的错误
            }
        }

        elapsed := time.Since(startTime)
        fmt.Printf("BufferSize: %d, Read %d bytes, Time: %s\n", bufferSize, count, elapsed)
    }
}

package main

import (
    "fmt"
    "os"
    "syscall"
    "unsafe"
)

func main() {
    filePath := "large_file.txt" // 替换成你的大文件路径

    file, err := os.Open(filePath)
    if err != nil {
        fmt.Println("Error opening file:", err)
        return
    }
    defer file.Close()

    fileInfo, err := file.Stat()
    if err != nil {
        fmt.Println("Error getting file info:", err)
        return
    }
    fileSize := fileInfo.Size()

    // 将文件映射到内存
    data, err := syscall.Mmap(int(file.Fd()), 0, int(fileSize), syscall.PROT_READ, syscall.MAP_SHARED)
    if err != nil {
        fmt.Println("Error mmapping file:", err)
        return
    }
    defer syscall.Munmap(data)

    // 访问文件内容，例如读取前10个字节
    if fileSize > 10 {
        fmt.Println("First 10 bytes:", string(data[:10]))
    }

    // 注意：修改 mmap 区域需要额外的处理，例如使用 PROT_READ|PROT_WRITE 标志
}