luzifer/cloudbox
1
0
Fork 0
mirror of https://github.com/Luzifer/cloudbox.git synced 2024-12-22 10:41:19 +00:00
Code Issues Releases Wiki Activity

Add prefix sync for s3 buckets

Browse source 
Signed-off-by: Knut Ahlers <knut@ahlers.me>
This commit is contained in:
Knut Ahlers 2019-06-30 05:33:34 +02:00
parent 5cf68673a3
commit 74a496e44a
Signed by: luzifer
GPG key ID: DC2729FDD34BE99E
2 changed files with 17 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
providers/s3/file.go
View file

@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"hash" "hash"
"io" "io"
"strings"
"time" "time"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws"
@ -22,11 +23,12 @@ type File struct {
s3Conn *s3.S3 s3Conn *s3.S3
bucket string bucket string
prefix string
} }
func (f File) Info() providers.FileInfo { func (f File) Info() providers.FileInfo {
return providers.FileInfo{ return providers.FileInfo{
RelativeName: f.key, RelativeName: strings.Trim(strings.TrimPrefix(f.key, f.prefix), "/"),
LastModified: f.lastModified, LastModified: f.lastModified,
Checksum: f.checksum, Checksum: f.checksum,
Size: f.size, Size: f.size,

19
providers/s3/provider.go
View file

@ -24,6 +24,7 @@ type Provider struct {
bucket string bucket string
bucketRegion string bucketRegion string
defaultACL string defaultACL string
prefix string
s3 *s3.S3 s3 *s3.S3
} }
@ -56,6 +57,7 @@ func New(uri string) (providers.CloudProvider, error) {
bucket: u.Host, bucket: u.Host,
bucketRegion: region, bucketRegion: region,
defaultACL: s3.ObjectCannedACLPrivate, defaultACL: s3.ObjectCannedACLPrivate,
prefix: strings.Trim(u.Path, "/"),
s3: svc, s3: svc,
} }
@ -75,7 +77,7 @@ func (p *Provider) GetChecksumMethod() hash.Hash { return md5.New() }
func (p *Provider) DeleteFile(relativeName string) error { func (p *Provider) DeleteFile(relativeName string) error {
_, err := p.s3.DeleteObject(&s3.DeleteObjectInput{ _, err := p.s3.DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(p.bucket), Bucket: aws.String(p.bucket),
Key: aws.String(relativeName), Key: p.relativeNameToKey(relativeName),
}) })
return errors.Wrap(err, "Unable to delete object") return errors.Wrap(err, "Unable to delete object")
@ -84,7 +86,7 @@ func (p *Provider) DeleteFile(relativeName string) error {
func (p *Provider) GetFile(relativeName string) (providers.File, error) { func (p *Provider) GetFile(relativeName string) (providers.File, error) {
resp, err := p.s3.HeadObject(&s3.HeadObjectInput{ resp, err := p.s3.HeadObject(&s3.HeadObjectInput{
Bucket: aws.String(p.bucket), Bucket: aws.String(p.bucket),
Key: aws.String(relativeName), Key: p.relativeNameToKey(relativeName),
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "Unable to fetch head information") return nil, errors.Wrap(err, "Unable to fetch head information")
@ -106,6 +108,7 @@ func (p *Provider) ListFiles() ([]providers.File, error) {
err := p.s3.ListObjectsPages(&s3.ListObjectsInput{ err := p.s3.ListObjectsPages(&s3.ListObjectsInput{
Bucket: aws.String(p.bucket), Bucket: aws.String(p.bucket),
Prefix: aws.String(p.prefix),
}, func(out *s3.ListObjectsOutput, lastPage bool) bool { }, func(out *s3.ListObjectsOutput, lastPage bool) bool {
for _, obj := range out.Contents { for _, obj := range out.Contents {
files = append(files, File{ files = append(files, File{
@ -116,6 +119,7 @@ func (p *Provider) ListFiles() ([]providers.File, error) {
s3Conn: p.s3, s3Conn: p.s3,
bucket: p.bucket, bucket: p.bucket,
prefix: p.prefix,
}) })
} }
@ -141,7 +145,7 @@ func (p *Provider) PutFile(f providers.File) (providers.File, error) {
ACL: aws.String(p.getFileACL(f.Info().RelativeName)), ACL: aws.String(p.getFileACL(f.Info().RelativeName)),
Body: bytes.NewReader(buf.Bytes()), Body: bytes.NewReader(buf.Bytes()),
Bucket: aws.String(p.bucket), Bucket: aws.String(p.bucket),
Key: aws.String(f.Info().RelativeName), Key: p.relativeNameToKey(f.Info().RelativeName),
}); err != nil { }); err != nil {
return nil, errors.Wrap(err, "Unable to write file") return nil, errors.Wrap(err, "Unable to write file")
} }
@ -153,7 +157,7 @@ func (p *Provider) Share(relativeName string) (string, error) {
_, err := p.s3.PutObjectAcl(&s3.PutObjectAclInput{ _, err := p.s3.PutObjectAcl(&s3.PutObjectAclInput{
ACL: aws.String(s3.ObjectCannedACLPublicRead), ACL: aws.String(s3.ObjectCannedACLPublicRead),
Bucket: aws.String(p.bucket), Bucket: aws.String(p.bucket),
Key: aws.String(relativeName), Key: p.relativeNameToKey(relativeName),
}) })
if err != nil { if err != nil {
return "", errors.Wrap(err, "Unable to publish file") return "", errors.Wrap(err, "Unable to publish file")
@ -165,7 +169,7 @@ func (p *Provider) Share(relativeName string) (string, error) {
func (p *Provider) getFileACL(relativeName string) string { func (p *Provider) getFileACL(relativeName string) string {
objACL, err := p.s3.GetObjectAcl(&s3.GetObjectAclInput{ objACL, err := p.s3.GetObjectAcl(&s3.GetObjectAclInput{
Bucket: aws.String(p.bucket), Bucket: aws.String(p.bucket),
Key: aws.String(relativeName), Key: p.relativeNameToKey(relativeName),
}) })
if err != nil { if err != nil {
@ -183,3 +187,8 @@ func (p *Provider) getFileACL(relativeName string) string {
return p.defaultACL return p.defaultACL
} }
func (p Provider) relativeNameToKey(relativeName string) *string {
key := strings.Join([]string{p.prefix, relativeName}, "/")
return &key
}