luzifer/s3sync
1
0
Fork 0
mirror of https://github.com/Luzifer/s3sync.git synced 2024-12-20 19:41:15 +00:00
Code Issues Releases Wiki Activity

Do file sync with more than 1 thread in parallel

Browse source
This commit is contained in:
Knut Ahlers 2015-07-26 17:21:41 +02:00
parent f0a5a40c51
commit 5dfb0e4b45
1 changed files with 54 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
main.go
View file

@ -17,6 +17,7 @@ var (
Delete bool
Public bool
PrintVersion bool
MaxThreads int
}{}
version = "dev"
)
@ -51,6 +52,7 @@ func main() {
app.Flags().BoolVarP(&cfg.Public, "public", "P", false, "Make files public when syncing to S3")
app.Flags().BoolVarP(&cfg.Delete, "delete", "d", false, "Delete files on remote not existing on local")
app.Flags().BoolVarP(&cfg.PrintVersion, "version", "v", false, "Print version and quit")
app.Flags().IntVar(&cfg.MaxThreads, "max-threads", 10, "Use max N parallel threads for file sync")
app.Execute()
}
@ -76,8 +78,12 @@ func execSync(cmd *cobra.Command, args []string) {
remoteFiles, err := remote.ListFiles(remotePath)
errExit(err)
syncChannel := make(chan bool, cfg.MaxThreads)
for i, localFile := range localFiles {
fmt.Printf("(%d / %d) %s ", i+1, len(localFiles), localFile.Filename)
syncChannel <- true
go func(i int, localFile file) {
defer func() { <-syncChannel }()
needsCopy := true
for _, remoteFile := range remoteFiles {
if remoteFile.Filename == localFile.Filename && remoteFile.MD5 == localFile.MD5 {
@ -88,32 +94,37 @@ func execSync(cmd *cobra.Command, args []string) {
if needsCopy {
l, err := local.ReadFile(path.Join(localPath, localFile.Filename))
if err != nil {
fmt.Printf("ERR: %s\n", err)
continue
fmt.Printf("(%d / %d) %s ERR: %s\n", i+1, len(localFiles), localFile.Filename, err)
return
}
buffer, err := ioutil.ReadAll(l)
if err != nil {
fmt.Printf("ERR: %s\n", err)
continue
fmt.Printf("(%d / %d) %s ERR: %s\n", i+1, len(localFiles), localFile.Filename, err)
return
}
l.Close()
err = remote.WriteFile(path.Join(remotePath, localFile.Filename), bytes.NewReader(buffer), cfg.Public)
if err != nil {
fmt.Printf("ERR: %s\n", err)
continue
fmt.Printf("(%d / %d) %s ERR: %s\n", i+1, len(localFiles), localFile.Filename, err)
return
}
fmt.Printf("OK\n")
continue
fmt.Printf("(%d / %d) %s OK\n", i+1, len(localFiles), localFile.Filename)
return
}
fmt.Printf("Skip\n")
fmt.Printf("(%d / %d) %s Skip\n", i+1, len(localFiles), localFile.Filename)
}(i, localFile)
}
if cfg.Delete {
for _, remoteFile := range remoteFiles {
syncChannel <- true
go func(remoteFile file) {
defer func() { <-syncChannel }()
needsDeletion := true
for _, localFile := range localFiles {
if localFile.Filename == remoteFile.Filename {
@ -122,13 +133,13 @@ func execSync(cmd *cobra.Command, args []string) {
}
if needsDeletion {
fmt.Printf("delete: %s ", remoteFile.Filename)
if err := remote.DeleteFile(path.Join(remotePath, remoteFile.Filename)); err != nil {
fmt.Printf("ERR: %s\n", err)
continue
fmt.Printf("delete: %s ERR: %s\n", remoteFile.Filename, err)
return
}
fmt.Printf("OK\n")
fmt.Printf("delete: %s OK\n", remoteFile.Filename)
}
}(remoteFile)
}
}
}