Golang如何实现多协程爬虫_Golang 多协程爬虫实践

urls := []string{
    "https://httpbin.org/delay/1",
    "https://httpbin.org/delay/2",
    "https://httpbin.org/get",
}

for _, url := range urls {
    go fetch(url)
}

semaphore := make(chan struct{}, 10) // 最大 10 个并发
var wg sync.WaitGroup
<p>for _, url := range urls {
wg.Add(1)
go func(u string) {
defer wg.Done()
semaphore <- struct{}{} // 获取令牌
defer func() { <-semaphore }() // 释放令牌</p><pre class='brush:php;toolbar:false;'>    fetch(u)
}(url)

go get github.com/PuerkitoBio/goquery

func parseTitle(body io.Reader) string {
    doc, _ := goquery.NewDocumentFromReader(body)
    return doc.Find("title").Text()
}

results := make(chan string, len(urls))
// 在协程中：
title := parseTitle(resp.Body)
results <- fmt.Sprintf("%s: %s", url, title)

func fetchWithRetry(url string, maxRetries int) error {
    var err error
    for i := 0; i < maxRetries; i++ {
        resp, err := http.Get(url)
        if err == nil && resp.StatusCode == 200 {
            // 处理成功
            return nil
        }
        time.Sleep(time.Second << uint(i)) // 指数退避
    }
    return err
}

ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
req, _ := http.NewRequestWithContext(ctx, "GET", url, nil)
client.Do(req)