Go中限制并发的简单方法

Question

我有一个包含大约 10k 个 URL 的 CSV 文件，我需要通过 HTTP 获取。将 Go 例程的并发限制为一次不超过 16 个的最简单方法是什么？

func getUrl(url string) {

    request := gorequest.New()

    resp, body, errs := request.Get(each[1]).End()

    _ = resp

    _ = body

    _ = errs

}

func main() {

    csvfile, err := os.Open("urls.csv")

    defer csvfile.Close()

    reader := csv.NewReader(csvfile)

    reader.FieldsPerRecord = -1 

    rawCSVdata, err := reader.ReadAll()

    completed := 0

    for _, each := range rawCSVdata {

        go getUrl(each[1])

        completed++

    }

}

Answer 1

生产者-消费者模式：package mainimport (&nbsp; &nbsp; "encoding/csv"&nbsp; &nbsp; "os"&nbsp; &nbsp; "sync"&nbsp; &nbsp; "github.com/parnurzeal/gorequest")const workersCount = 16func getUrlWorker(urlChan chan string) {&nbsp; &nbsp; for url := range urlChan {&nbsp; &nbsp; &nbsp; &nbsp; request := gorequest.New()&nbsp; &nbsp; &nbsp; &nbsp; resp, body, errs := request.Get(url).End()&nbsp; &nbsp; &nbsp; &nbsp; _ = resp&nbsp; &nbsp; &nbsp; &nbsp; _ = body&nbsp; &nbsp; &nbsp; &nbsp; _ = errs&nbsp; &nbsp; }}func main() {&nbsp; &nbsp; csvfile, err := os.Open("urls.csv")&nbsp; &nbsp; if err != nil {&nbsp; &nbsp; &nbsp; &nbsp; panic(err)&nbsp; &nbsp; }&nbsp; &nbsp; defer csvfile.Close()&nbsp; &nbsp; reader := csv.NewReader(csvfile)&nbsp; &nbsp; reader.FieldsPerRecord = -1&nbsp; &nbsp; rawCSVdata, err := reader.ReadAll()&nbsp; &nbsp; var wg sync.WaitGroup&nbsp; &nbsp; urlChan := make(chan string)&nbsp; &nbsp; wg.Add(workersCount)&nbsp; &nbsp; for i := 0; i < workersCount; i++ {&nbsp; &nbsp; &nbsp; &nbsp; go func() {&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; getUrlWorker(urlChan)&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; wg.Done()&nbsp; &nbsp; &nbsp; &nbsp; }()&nbsp; &nbsp; }&nbsp; &nbsp; completed := 0&nbsp; &nbsp; for _, each := range rawCSVdata {&nbsp; &nbsp; &nbsp; &nbsp; urlChan <- each[1]&nbsp; &nbsp; &nbsp; &nbsp; completed++&nbsp; &nbsp; }&nbsp; &nbsp; close(urlChan)&nbsp; &nbsp; wg.Wait()}