luzifer/cloudbox
1
0
Fork 0
mirror of https://github.com/Luzifer/cloudbox.git synced 2024-12-22 10:41:19 +00:00
Code Issues Releases Wiki Activity

Fix broken behaviour when using checksums

Browse source 
Signed-off-by: Knut Ahlers <knut@ahlers.me>
This commit is contained in:
Knut Ahlers 2019-06-17 01:09:16 +02:00
parent 122eebbd25
commit e98a3dc32d
Signed by: luzifer
GPG key ID: DC2729FDD34BE99E
3 changed files with 61 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
providers/s3/provider.go
View file

@ -36,7 +36,12 @@ func New(uri string) (providers.CloudProvider, error) {
return nil, errors.Wrap(err, "Invalid URI specified")
}
cfg := aws.NewConfig()
region, err := s3manager.GetBucketRegion(context.Background(), session.New(), u.Host, "us-east-1")
if err != nil {
return nil, errors.Wrap(err, "Unable to find bucket region")
}
cfg := aws.NewConfig().WithRegion(region)
if u.User != nil {
user := u.User.Username()
pass, _ := u.User.Password()
@ -46,11 +51,6 @@ func New(uri string) (providers.CloudProvider, error) {
sess := session.Must(session.NewSession(cfg))
svc := s3.New(sess)
region, err := s3manager.GetBucketRegion(context.Background(), sess, u.Host, "us-east-1")
if err != nil {
return nil, errors.Wrap(err, "Unable to find bucket region")
}
return &Provider{
bucket: u.Host,
bucketRegion: region,
@ -165,6 +165,9 @@ func (p *Provider) getFileACL(relativeName string) string {
}
for _, g := range objACL.Grants {
if g.Grantee == nil || g.Grantee.URI == nil {
continue
}
if *g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && *g.Permission == "READ" {
return s3.ObjectCannedACLPublicRead
}

28
sync/execute.go
View file

@ -36,11 +36,21 @@ func (s *Sync) addBothCreated(fileName string) error {
return errors.New("Checksums differ")
}
if err := s.setDBFileInfo(sideLocal, local.Info()); err != nil {
localInfo, err := s.getFileInfo(local)
if err != nil {
return errors.Wrap(err, "Unable to get file info for local file")
}
if err := s.setDBFileInfo(sideLocal, localInfo); err != nil {
return errors.Wrap(err, "Unable to update DB info for local file")
}
if err := s.setDBFileInfo(sideRemote, remote.Info()); err != nil {
remoteInfo, err := s.getFileInfo(remote)
if err != nil {
return errors.Wrap(err, "Unable to get file info for remote file")
}
if err := s.setDBFileInfo(sideRemote, remoteInfo); err != nil {
return errors.Wrap(err, "Unable to update DB info for remote file")
}
@ -74,11 +84,21 @@ func (s *Sync) transferFile(from, to providers.CloudProvider, sideFrom, sideTo,
return errors.Wrap(err, "Unable to put file")
}
if err := s.setDBFileInfo(sideTo, newFile.Info()); err != nil {
newFileInfo, err := s.getFileInfo(newFile)
if err != nil {
return errors.Wrap(err, "Unable to get file info for target file")
}
if err := s.setDBFileInfo(sideTo, newFileInfo); err != nil {
return errors.Wrap(err, "Unable to update DB info for target file")
}
if err := s.setDBFileInfo(sideFrom, file.Info()); err != nil {
fileInfo, err := s.getFileInfo(file)
if err != nil {
return errors.Wrap(err, "Unable to get file info for source file")
}
if err := s.setDBFileInfo(sideFrom, fileInfo); err != nil {
return errors.Wrap(err, "Unable to update DB info for source file")
}

41
sync/sync.go
View file

@ -23,6 +23,9 @@ type Sync struct {
log *log.Entry
useChecksum bool
hashMethod hash.Hash
stop chan struct{}
}
@ -62,20 +65,32 @@ func (s *Sync) Run() error {
func (s *Sync) Stop() { s.stop <- struct{}{} }
func (s *Sync) fillStateFromProvider(syncState *state, provider providers.CloudProvider, side string, useChecksum bool, hashMethod hash.Hash) error {
func (s *Sync) getFileInfo(f providers.File) (providers.FileInfo, error) {
var info = f.Info()
if !s.useChecksum || info.Checksum != "" {
return info, nil
}
cs, err := f.Checksum(s.hashMethod)
if err != nil {
return info, errors.Wrap(err, "Unable to fetch checksum")
}
info.Checksum = cs
return info, nil
}
func (s *Sync) fillStateFromProvider(syncState *state, provider providers.CloudProvider, side string) error {
files, err := provider.ListFiles()
if err != nil {
return errors.Wrap(err, "Unable to list files")
}
for _, f := range files {
info := f.Info()
if useChecksum && info.Checksum == "" {
cs, err := f.Checksum(hashMethod)
info, err := s.getFileInfo(f)
if err != nil {
return errors.Wrap(err, "Unable to fetch checksum")
}
info.Checksum = cs
return errors.Wrap(err, "Unable to get file info")
}
syncState.Set(side, sourceScan, info)
@ -85,21 +100,19 @@ func (s *Sync) fillStateFromProvider(syncState *state, provider providers.CloudP
}
func (s *Sync) runSync() error {
var (
hashMethod = s.remote.GetChecksumMethod()
syncState = newState()
useChecksum = s.remote.Capabilities().Has(providers.CapAutoChecksum) || s.conf.ForceUseChecksum
)
var syncState = newState()
s.hashMethod = s.remote.GetChecksumMethod()
s.useChecksum = s.remote.Capabilities().Has(providers.CapAutoChecksum) || s.conf.ForceUseChecksum
if err := s.updateStateFromDatabase(syncState); err != nil {
return errors.Wrap(err, "Unable to load database state")
}
if err := s.fillStateFromProvider(syncState, s.local, sideLocal, useChecksum, hashMethod); err != nil {
if err := s.fillStateFromProvider(syncState, s.local, sideLocal); err != nil {
return errors.Wrap(err, "Unable to load local files")
}
if err := s.fillStateFromProvider(syncState, s.remote, sideRemote, useChecksum, hashMethod); err != nil {
if err := s.fillStateFromProvider(syncState, s.remote, sideRemote); err != nil {
return errors.Wrap(err, "Unable to load remote files")
}