Golang批量文件处理工具实现教程

filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
    if err != nil {
        return err
    }
    if !info.IsDir() && strings.HasSuffix(info.Name(), ".txt") {
        // 加入待处理队列
        processFile(path)
    }
    return nil
})

var wg sync.WaitGroup
fileChan := make(chan string, 100)
// 启动多个worker
for i := 0; i < 5; i++ {
go func() {
for file := range fileChan {
processSingleFile(file) // 实际处理逻辑
wg.Done()
}
}()
}
// 发现文件后发送到通道
filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
if !info.IsDir() && matchCondition(info) {
wg.Add(1)
fileChan <- path
}
return nil
})
close(fileChan)
wg.Wait() // 等待全部完成

func convertEncoding(filePath string) error {
    content, _ := os.ReadFile(filePath)
    decoded := iconv.ConvertString(string(content), "gbk", "utf-8")
    return os.WriteFile(filePath+".utf8", []byte(decoded), 0644)
}

go func() {
    for file := range fileChan {
        defer wg.Done()
        if err := processSingleFile(file); err != nil {
            log.Printf("failed to process %s: %v", file, err)
        } else {
            log.Printf("processed %s", file)
        }
    }
}()