Go语言文本去重：如何优化75万行文本的去重效率？

package distinct

import (
    "bufio"
    "fmt"
    "io"
    "os"
    "strings"
)

//distinctfile 为指定文件去重
func distinctfile(file string, output string) {
    f, err := os.Open(file)
    if err != nil {
        fmt.Println(err)
        return
    }
    defer f.Close()

    reader := bufio.NewReader(f)
    set := make(map[string]struct{}, 1000000) // 预分配map容量
    builder := strings.Builder{}              // 使用strings.Builder

    for {
        line, isPrefix, err := reader.ReadLine()
        if err == io.EOF {
            break
        }
        if err != nil {
            fmt.Println(err)
            return
        }

        if !isPrefix {
            lineStr := string(line)
            if _, ok := set[lineStr]; !ok {
                builder.WriteString(lineStr)
                builder.WriteString("\n")
                set[lineStr] = struct{}{}
            }
        }
    }

    nf, err := os.Create(output)
    if err != nil {
        fmt.Println(err)
        return
    }
    defer nf.Close()

    _, err = io.Copy(nf, strings.NewReader(builder.String()))
    if err != nil {
        fmt.Println(err)
    }
}

package distinct

import "testing"

func TestDistinctFile(t *testing.T) {
    distinctfile("result.txt", "out.txt")
}