Golang多协程爬虫实现教程

urls := []string{
    "https://httpbin.org/delay/1",
    "https://httpbin.org/delay/2",
    "https://httpbin.org/get",
}

for _, url := range urls {
    go fetch(url)
}

semaphore := make(chan struct{}, 10) // 最大 10 个并发
var wg sync.WaitGroup
<p>for _, url := range urls {
wg.Add(1)
go func(u string) {
defer wg.Done()
semaphore <- struct{}{} // 获取令牌
defer func() { <-semaphore }() // 释放令牌</p><pre class="brush:php;toolbar:false;">    fetch(u)
}(url)

} wg.Wait() // 等待所有任务完成

go get github.com/PuerkitoBio/goquery

func parseTitle(body io.Reader) string {
    doc, _ := goquery.NewDocumentFromReader(body)
    return doc.Find("title").Text()
}

results := make(chan string, len(urls))
// 在协程中：
title := parseTitle(resp.Body)
results 

func fetchWithRetry(url string, maxRetries int) error {
    var err error
    for i := 0; i 

ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
req, _ := http.NewRequestWithContext(ctx, "GET", url, nil)
client.Do(req)