luzifer/s3sync
1
0
Fork 0
mirror of https://github.com/Luzifer/s3sync.git synced 2024-12-20 19:41:15 +00:00
Code Issues Releases Wiki Activity

Do file sync with more than 1 thread in parallel

Browse source
This commit is contained in:
Knut Ahlers 2015-07-26 17:21:41 +02:00
parent f0a5a40c51
commit 5dfb0e4b45
1 changed files with 54 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
main.go
View file

@ -17,6 +17,7 @@ var (
Delete bool Delete bool
Public bool Public bool
PrintVersion bool PrintVersion bool
MaxThreads int
}{} }{}
version = "dev" version = "dev"
) )
@ -51,6 +52,7 @@ func main() {
app.Flags().BoolVarP(&cfg.Public, "public", "P", false, "Make files public when syncing to S3") app.Flags().BoolVarP(&cfg.Public, "public", "P", false, "Make files public when syncing to S3")
app.Flags().BoolVarP(&cfg.Delete, "delete", "d", false, "Delete files on remote not existing on local") app.Flags().BoolVarP(&cfg.Delete, "delete", "d", false, "Delete files on remote not existing on local")
app.Flags().BoolVarP(&cfg.PrintVersion, "version", "v", false, "Print version and quit") app.Flags().BoolVarP(&cfg.PrintVersion, "version", "v", false, "Print version and quit")
app.Flags().IntVar(&cfg.MaxThreads, "max-threads", 10, "Use max N parallel threads for file sync")
app.Execute() app.Execute()
} }
@ -76,8 +78,12 @@ func execSync(cmd *cobra.Command, args []string) {
remoteFiles, err := remote.ListFiles(remotePath) remoteFiles, err := remote.ListFiles(remotePath)
errExit(err) errExit(err)
syncChannel := make(chan bool, cfg.MaxThreads)
for i, localFile := range localFiles { for i, localFile := range localFiles {
fmt.Printf("(%d / %d) %s ", i+1, len(localFiles), localFile.Filename) syncChannel <- true
go func(i int, localFile file) {
defer func() { <-syncChannel }()
needsCopy := true needsCopy := true
for _, remoteFile := range remoteFiles { for _, remoteFile := range remoteFiles {
if remoteFile.Filename == localFile.Filename && remoteFile.MD5 == localFile.MD5 { if remoteFile.Filename == localFile.Filename && remoteFile.MD5 == localFile.MD5 {
@ -88,32 +94,37 @@ func execSync(cmd *cobra.Command, args []string) {
if needsCopy { if needsCopy {
l, err := local.ReadFile(path.Join(localPath, localFile.Filename)) l, err := local.ReadFile(path.Join(localPath, localFile.Filename))
if err != nil { if err != nil {
fmt.Printf("ERR: %s\n", err) fmt.Printf("(%d / %d) %s ERR: %s\n", i+1, len(localFiles), localFile.Filename, err)
continue return
} }
buffer, err := ioutil.ReadAll(l) buffer, err := ioutil.ReadAll(l)
if err != nil { if err != nil {
fmt.Printf("ERR: %s\n", err) fmt.Printf("(%d / %d) %s ERR: %s\n", i+1, len(localFiles), localFile.Filename, err)
continue return
} }
l.Close() l.Close()
err = remote.WriteFile(path.Join(remotePath, localFile.Filename), bytes.NewReader(buffer), cfg.Public) err = remote.WriteFile(path.Join(remotePath, localFile.Filename), bytes.NewReader(buffer), cfg.Public)
if err != nil { if err != nil {
fmt.Printf("ERR: %s\n", err) fmt.Printf("(%d / %d) %s ERR: %s\n", i+1, len(localFiles), localFile.Filename, err)
continue return
} }
fmt.Printf("OK\n") fmt.Printf("(%d / %d) %s OK\n", i+1, len(localFiles), localFile.Filename)
continue return
} }
fmt.Printf("Skip\n") fmt.Printf("(%d / %d) %s Skip\n", i+1, len(localFiles), localFile.Filename)
}(i, localFile)
} }
if cfg.Delete { if cfg.Delete {
for _, remoteFile := range remoteFiles { for _, remoteFile := range remoteFiles {
syncChannel <- true
go func(remoteFile file) {
defer func() { <-syncChannel }()
needsDeletion := true needsDeletion := true
for _, localFile := range localFiles { for _, localFile := range localFiles {
if localFile.Filename == remoteFile.Filename { if localFile.Filename == remoteFile.Filename {
@ -122,13 +133,13 @@ func execSync(cmd *cobra.Command, args []string) {
} }
if needsDeletion { if needsDeletion {
fmt.Printf("delete: %s ", remoteFile.Filename)
if err := remote.DeleteFile(path.Join(remotePath, remoteFile.Filename)); err != nil { if err := remote.DeleteFile(path.Join(remotePath, remoteFile.Filename)); err != nil {
fmt.Printf("ERR: %s\n", err) fmt.Printf("delete: %s ERR: %s\n", remoteFile.Filename, err)
continue return
} }
fmt.Printf("OK\n") fmt.Printf("delete: %s OK\n", remoteFile.Filename)
} }
}(remoteFile)
} }
} }
} }