Golang高效并发爬虫实现教程

func fetch(url string, ch chan string) {
    resp, err := http.Get(url)
    if err != nil {
        ch // 主函数中并发调用
urls := []string{"https://example.com", "https://httpbin.org/get", ...}
ch := make(chan string, len(urls))
for _, url := range urls {
go fetch(url, ch)
}
for range urls {
fmt.Println(<-ch)
}

semaphore := make(chan struct{}, 10) // 最多10个并发
ch := make(chan string, len(urls))
for _, url := range urls {
go func(u string) {
semaphore <- struct{}{} // 获取令牌
resp, err := http.Get(u)
result := ""
if err != nil {
result = "Failed"
} else {
result = fmt.Sprintf("Status: %d", resp.StatusCode)
resp.Body.Close()
}
ch <- result
<-semaphore // 释放令牌
}(url)
}

client := &http.Client{
    Timeout: 10 * time.Second,
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
req, _ := http.NewRequestWithContext(ctx, "GET", url, nil)
resp, err := client.Do(req)

doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
    log.Fatal(err)
}
title := doc.Find("title").Text()
fmt.Println("Title:", title)