如何在Golang中实现并发任务分片执行_Golang并发任务分片处理方法汇总

func processTasks(tasks []int, chunkSize int) {
    var wg sync.WaitGroup
    resultChan := make(chan []int, len(tasks)/chunkSize+1)
<pre class="brush:php;toolbar:false;"><pre class="brush:php;toolbar:false;">for i := 0; i < len(tasks); i += chunkSize {
    end := i + chunkSize
    if end > len(tasks) {
        end = len(tasks)
    }
    chunk := tasks[i:end]

    wg.Add(1)
    go func(c []int) {
        defer wg.Done()
        // 模拟处理逻辑
        processed := make([]int, len(c))
        for j, v := range c {
            processed[j] = v * 2
        }
        resultChan <- processed
    }(chunk)
}

go func() {
    wg.Wait()
    close(resultChan)
}()

// 收集结果
for result := range resultChan {
    fmt.Println("Processed:", result)
}

func processWithWorkerPool(tasks []string, chunkSize, maxWorkers int) {
    var wg sync.WaitGroup
    taskChan := make(chan []string, maxWorkers)
<pre class="brush:php;toolbar:false;"><pre class="brush:php;toolbar:false;">// 启动 worker
for w := 0; w < maxWorkers; w++ {
    wg.Add(1)
    go func() {
        defer wg.Done()
        for chunk := range taskChan {
            // 处理分片
            for _, task := range chunk {
                fmt.Printf("Worker processing: %s\n", task)
                time.Sleep(100 * time.Millisecond) // 模拟耗时
            }
        }
    }()
}

// 发送分片任务
for i := 0; i < len(tasks); i += chunkSize {
    end := i + chunkSize
    if end > len(tasks) {
        end = len(tasks)
    }
    taskChan <- tasks[i:end]
}
close(taskChan)

wg.Wait()

func processWithErrorGroup(tasks []int, chunkSize int) error {
    ctx := context.Background()
    g, ctx := errgroup.WithContext(ctx)
<pre class="brush:php;toolbar:false;"><pre class="brush:php;toolbar:false;">for i := 0; i < len(tasks); i += chunkSize {
    end := i + chunkSize
    if end > len(tasks) {
        end = len(tasks)
    }
    chunk := tasks[i:end]

    g.Go(func() error {
        // 模拟处理，可能出错
        for _, v := range chunk {
            if v == 999 {
                return fmt.Errorf("invalid task: %d", v)
            }
            fmt.Printf("Processed %d\n", v*2)
        }
        return nil
    })
}

return g.Wait()

func processWithTimeout(tasks []int, chunkSize int, timeout time.Duration) {
    var wg sync.WaitGroup
<pre class="brush:php;toolbar:false;"><pre class="brush:php;toolbar:false;">for i := 0; i < len(tasks); i += chunkSize {
    end := i + chunkSize
    if end > len(tasks) {
        end = len(tasks)
    }
    chunk := tasks[i:end]

    wg.Add(1)
    go func(c []int) {
        defer wg.Done()

        ctx, cancel := context.WithTimeout(context.Background(), timeout)
        defer cancel()

        timer := time.NewTimer(2 * timeout)
        done := make(chan bool, 1)

        go func() {
            // 模拟处理
            time.Sleep(500 * time.Millisecond)
            fmt.Printf("Finished chunk: %v\n", c)
            done <- true
        }()

        select {
        case <-done:
            return
        case <-ctx.Done():
            fmt.Println("Task timed out")
        case <-timer.C:
            fmt.Println("Hard timeout triggered")
        }
    }(chunk)
}

wg.Wait()