Golang高效并发爬虫实现教程

func fetch(url string, ch chan string) {
    resp, err := http.Get(url)
    if err != nil {
        ch // 主函数中并发调用
urls := []string{"<a target='_blank'  href='https://www.17golang.com/gourl/?redirect=MDAwMDAwMDAwML57hpSHp6VpkrqbYLx2eayza4KafaOkbLS3zqSBrJvPsa5_0Ia6sWuR4Juaq6t9nq5roGCUgXuytMyero6Kn83GjHPXkraZo5qYYJqrq32ermuKnH59oK-zqrttgnmI3rGIlc2R3a1uh6qbZLyGfWe-s4Wqio2PorPQs20' rel='nofollow'>https://example.com</a>", "<a target='_blank'  href='https://www.17golang.com/gourl/?redirect=MDAwMDAwMDAwML57hpSHp6VpkrqbYLx2eayza4KafaOkbLS3zqSBrJvPsa5_0Ia6sWuR4Juaq6t9nq5roGCUgXuytMyerpV6iZXHe3vUmsyZr5vTk6a8ZIZkrrKFmnmyh6O_t7dsgXaJ0bOIg8-FzalskdN9qbGGl2m0gI2qfmuGsrKVu2mNrJHPs4VuoQ' rel='nofollow'>https://httpbin.org/get</a>", ...}
ch := make(chan string, len(urls))
for _, url := range urls {
go fetch(url, ch)
}
for range urls {
fmt.Println(<-ch)
}

semaphore := make(chan struct{}, 10) // 最多10个并发
ch := make(chan string, len(urls))
<p>for _, url := range urls {
go func(u string) {
semaphore <- struct{}{} // 获取令牌
resp, err := http.Get(u)
result := ""
if err != nil {
result = "Failed"
} else {
result = fmt.Sprintf("Status: %d", resp.StatusCode)
resp.Body.Close()
}
ch <- result
<-semaphore // 释放令牌
}(url)
}</p>

client := &http.Client{
    Timeout: 10 * time.Second,
}
<p>ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()</p><p>req, _ := http.NewRequestWithContext(ctx, "GET", url, nil)
resp, err := client.Do(req)</p>

doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
    log.Fatal(err)
}
title := doc.Find("title").Text()
fmt.Println("Title:", title)