Golang上传整个目录并发返回许多打开的文件

Question

我正在尝试将整个目录上传到服务器。它适用于小目录，但有 100 多张图片，它返回“许多打开的文件”错误。我在读取文件后立即关闭该文件。知道如何解决这个问题吗？

这是我的代码

    func uploadDir(path string) error {

    dir, err := os.Open(path)

    if err != nil {

        return err

    }

    files, err := dir.Readdirnames(-1)

    if err != nil {

        return err

    }

    dir.Close()

    errChan := make(chan error)

    resChan := make(chan *client.PutResult)

    remaining := len(files)

    for _, file := range files {

        file := file

        go func() {

            file, err := os.Open(path + "/" + file)

            if err != nil {

                errChan <- err

            }

            c := client.NewClient(os.Getenv("DROPS_SERVER"))

            res, err := c.Upload(client.NewUploadHandleFromReader(file))

            file.Close()

            if err != nil {

                errChan <- err

            }

            resChan <- res

        }()

    }

    for {

        select {

        case res := <-resChan:

            log.Println(res)

            remaining--

        case err := <-errChan:

            if err != nil {

                return err

            }

        }

        if remaining == 0 {

            break

        }

    }

    return nil

}

Answer 1

原始代码不限制活动 go 例程的数量，因此不限制打开的文件描述符的数量。一些操作系统对打开的文件描述符的数量有限制。解决方法是创建固定数量的工作程序 go 例程。func uploadDir(path string) error {&nbsp; &nbsp; // Read directory and close.&nbsp; &nbsp; dir, err := os.Open(path)&nbsp; &nbsp; if err != nil {&nbsp; &nbsp; &nbsp; &nbsp; return err&nbsp; &nbsp; }&nbsp; &nbsp; names, err := dir.Readdirnames(-1)&nbsp; &nbsp; if err != nil {&nbsp; &nbsp; &nbsp; &nbsp; return err&nbsp; &nbsp; }&nbsp; &nbsp; dir.Close()&nbsp; &nbsp; // Copy names to a channel for workers to consume. Close the&nbsp; &nbsp; // channel so that workers stop when all work is complete.&nbsp; &nbsp; namesChan := make(chan string, len(names))&nbsp; &nbsp; for _, name := range names {&nbsp; &nbsp; &nbsp; &nbsp; namesChan <- name&nbsp; &nbsp; }&nbsp; &nbsp; close(namesChan)&nbsp; &nbsp; // Create a maximum of 8 workers&nbsp; &nbsp; workers := 8&nbsp; &nbsp; if len(names) < workers {&nbsp; &nbsp; &nbsp; &nbsp; workers = len(names)&nbsp; &nbsp; }&nbsp; &nbsp; errChan := make(chan error, 1)&nbsp; &nbsp; resChan := make(chan *client.PutResult, len(names))&nbsp; &nbsp; // Run workers&nbsp; &nbsp; for i := 0; i < workers; i++ {&nbsp; &nbsp; &nbsp; &nbsp; go func() {&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; // Consume work from namesChan. Loop will end when no more work.&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; for name := range namesChan {&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; file, err := os.Open(filepath.Join(path, name))&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; if err != nil {&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; select {&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; case errChan <- err:&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; // will break parent goroutine out of loop&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; default:&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;// don't care, first error wins&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; }&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; return&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; }&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; c := client.NewClient(os.Getenv("DROPS_SERVER"))&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; res, err := c.Upload(client.NewUploadHandleFromReader(file))&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; file.Close()&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; if err != nil {&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; select {&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; case errChan <- err:&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; // will break parent goroutine out of loop&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; default:&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;// don't care, first error wins&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; }&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; return&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; }&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; resChan <- res&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; }&nbsp; &nbsp; &nbsp; &nbsp; }()&nbsp; &nbsp; }&nbsp; &nbsp; // Collect results from workers&nbsp;&nbsp; &nbsp; for i := 0; i < len(names); i++ {&nbsp; &nbsp; &nbsp; &nbsp; select {&nbsp; &nbsp; &nbsp; &nbsp; case res := <-resChan:&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; log.Println(res)&nbsp; &nbsp; &nbsp; &nbsp; case err := <-errChan:&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; return err&nbsp; &nbsp; &nbsp; &nbsp; }&nbsp; &nbsp; }&nbsp; &nbsp; return nil}作为奖励，我修改了通道大小并发送操作，以便在出现错误时不会卡住 goroutine。