From 627fe09bacb5a805e3398a08abade8192fefff56 Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Fri, 9 Jun 2023 12:52:04 +0200 Subject: [PATCH] Breaking: Refactor, update deps, add MinIO support - Drop MD5 checksumming - Replace logging to stdout - Add Endpoint flag for MinIO support - Switch to Go modules support Signed-off-by: Knut Ahlers --- .github/workflows/test-and-build.yml | 74 +++++++++ .gobuilder.yml | 5 - .golangci.yml | 174 +++++++++++++++++++++ .repo-runner.yaml | 10 -- Makefile | 5 + README.md | 6 +- go.mod | 9 +- go.sum | 29 +++- local.go | 68 -------- logger/logger.go | 55 ------- main.go | 225 +++++++++++++++------------ pkg/fsprovider/interface.go | 26 ++++ pkg/fsprovider/local.go | 92 +++++++++++ pkg/fsprovider/s3.go | 217 ++++++++++++++++++++++++++ s3.go | 191 ----------------------- 15 files changed, 745 insertions(+), 441 deletions(-) create mode 100644 .github/workflows/test-and-build.yml delete mode 100644 .gobuilder.yml create mode 100644 .golangci.yml delete mode 100644 .repo-runner.yaml delete mode 100644 local.go delete mode 100644 logger/logger.go create mode 100644 pkg/fsprovider/interface.go create mode 100644 pkg/fsprovider/local.go create mode 100644 pkg/fsprovider/s3.go delete mode 100644 s3.go diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml new file mode 100644 index 0000000..d63d6f6 --- /dev/null +++ b/.github/workflows/test-and-build.yml @@ -0,0 +1,74 @@ +--- + +name: test-and-build +on: [push] + +permissions: + contents: write + +jobs: + test-and-build: + defaults: + run: + shell: bash + + container: + image: luzifer/archlinux + env: + CGO_ENABLED: 0 + GOPATH: /go + + runs-on: ubuntu-latest + + steps: + - name: Enable custom AUR package repo + run: echo -e "[luzifer]\nSigLevel = Never\nServer = https://s3-eu-west-1.amazonaws.com/arch-luzifer-io/repo/\$arch" >>/etc/pacman.conf + + - name: Install required packages + run: | + pacman -Syy --noconfirm \ + awk \ + curl \ + diffutils \ + git \ + go \ + golangci-lint-bin \ + make \ + nodejs-lts-fermium \ + npm \ + tar \ + unzip \ + which \ + zip + + - uses: actions/checkout@v3 + + - name: Marking workdir safe + run: git config --global --add safe.directory /__w/s3sync/s3sync + + - name: Lint code + run: make lint + + - name: Build release + run: make publish + env: + FORCE_SKIP_UPLOAD: 'true' + MOD_MODE: readonly + NODE_ENV: production + NO_TESTS: 'true' + PACKAGES: '.' + + - name: Extract changelog + run: 'awk "/^#/ && ++c==2{exit}; /^#/f" "History.md" | tail -n +2 >release_changelog.md' + + - name: Release + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + body_path: release_changelog.md + draft: false + fail_on_unmatched_files: true + files: '.build/*' + generate_release_notes: false + +... diff --git a/.gobuilder.yml b/.gobuilder.yml deleted file mode 100644 index 18e86a8..0000000 --- a/.gobuilder.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -build_matrix: - general: - ldflags: - - "-X main.version $(git describe --tags)" diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..e4d0cf1 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,174 @@ +# Derived from https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml + +--- + +run: + # timeout for analysis, e.g. 30s, 5m, default is 1m + timeout: 5m + # Force readonly modules usage for checking + modules-download-mode: readonly + +output: + format: tab + +issues: + # This disables the included exclude-list in golangci-lint as that + # list for example fully hides G304 gosec rule, errcheck, exported + # rule of revive and other errors one really wants to see. + # Smme detail: https://github.com/golangci/golangci-lint/issues/456 + exclude-use-default: false + # Don't limit the number of shown issues: Report ALL of them + max-issues-per-linter: 0 + max-same-issues: 0 + +linters: + disable-all: true + enable: + - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers [fast: true, auto-fix: false] + - bidichk # Checks for dangerous unicode character sequences [fast: true, auto-fix: false] + - bodyclose # checks whether HTTP response body is closed successfully [fast: true, auto-fix: false] + - containedctx # containedctx is a linter that detects struct contained context.Context field [fast: true, auto-fix: false] + - contextcheck # check the function whether use a non-inherited context [fast: false, auto-fix: false] + - dogsled # Checks assignments with too many blank identifiers (e.g. x, _, _, _, := f()) [fast: true, auto-fix: false] + - durationcheck # check for two durations multiplied together [fast: false, auto-fix: false] + - errcheck # Errcheck is a program for checking for unchecked errors in go programs. These unchecked errors can be critical bugs in some cases [fast: false, auto-fix: false] + - errchkjson # Checks types passed to the json encoding functions. Reports unsupported types and optionally reports occations, where the check for the returned error can be omitted. [fast: false, auto-fix: false] + - exportloopref # checks for pointers to enclosing loop variables [fast: true, auto-fix: false] + - forbidigo # Forbids identifiers [fast: true, auto-fix: false] + - funlen # Tool for detection of long functions [fast: true, auto-fix: false] + - gocognit # Computes and checks the cognitive complexity of functions [fast: true, auto-fix: false] + - goconst # Finds repeated strings that could be replaced by a constant [fast: true, auto-fix: false] + - gocritic # The most opinionated Go source code linter [fast: true, auto-fix: false] + - gocyclo # Computes and checks the cyclomatic complexity of functions [fast: true, auto-fix: false] + - godox # Tool for detection of FIXME, TODO and other comment keywords [fast: true, auto-fix: false] + - gofmt # Gofmt checks whether code was gofmt-ed. By default this tool runs with -s option to check for code simplification [fast: true, auto-fix: true] + - gofumpt # Gofumpt checks whether code was gofumpt-ed. [fast: true, auto-fix: true] + - goimports # Goimports does everything that gofmt does. Additionally it checks unused imports [fast: true, auto-fix: true] + - gomnd # An analyzer to detect magic numbers. [fast: true, auto-fix: false] + - gosec # Inspects source code for security problems [fast: true, auto-fix: false] + - gosimple # Linter for Go source code that specializes in simplifying a code [fast: true, auto-fix: false] + - govet # Vet examines Go source code and reports suspicious constructs, such as Printf calls whose arguments do not align with the format string [fast: true, auto-fix: false] + - ineffassign # Detects when assignments to existing variables are not used [fast: true, auto-fix: false] + - misspell # Finds commonly misspelled English words in comments [fast: true, auto-fix: true] + - nakedret # Finds naked returns in functions greater than a specified function length [fast: true, auto-fix: false] + - nilerr # Finds the code that returns nil even if it checks that the error is not nil. [fast: false, auto-fix: false] + - nilnil # Checks that there is no simultaneous return of `nil` error and an invalid value. [fast: false, auto-fix: false] + - noctx # noctx finds sending http request without context.Context [fast: true, auto-fix: false] + - nolintlint # Reports ill-formed or insufficient nolint directives [fast: true, auto-fix: false] + - revive # Fast, configurable, extensible, flexible, and beautiful linter for Go. Drop-in replacement of golint. [fast: false, auto-fix: false] + - staticcheck # Staticcheck is a go vet on steroids, applying a ton of static analysis checks [fast: true, auto-fix: false] + - stylecheck # Stylecheck is a replacement for golint [fast: true, auto-fix: false] + - tenv # tenv is analyzer that detects using os.Setenv instead of t.Setenv since Go1.17 [fast: false, auto-fix: false] + - typecheck # Like the front-end of a Go compiler, parses and type-checks Go code [fast: true, auto-fix: false] + - unconvert # Remove unnecessary type conversions [fast: true, auto-fix: false] + - unused # Checks Go code for unused constants, variables, functions and types [fast: false, auto-fix: false] + - wastedassign # wastedassign finds wasted assignment statements. [fast: false, auto-fix: false] + - wrapcheck # Checks that errors returned from external packages are wrapped [fast: false, auto-fix: false] + +linters-settings: + funlen: + lines: 100 + statements: 60 + + gocyclo: + # minimal code complexity to report, 30 by default (but we recommend 10-20) + min-complexity: 15 + + gomnd: + settings: + mnd: + ignored-functions: 'strconv.(?:Format|Parse)\B+' + + revive: + rules: + #- name: add-constant # Suggests using constant for magic numbers and string literals + # Opinion: Makes sense for strings, not for numbers but checks numbers + #- name: argument-limit # Specifies the maximum number of arguments a function can receive | Opinion: Don't need this + - name: atomic # Check for common mistaken usages of the `sync/atomic` package + - name: banned-characters # Checks banned characters in identifiers + arguments: + - 'Íž' # Greek question mark + - name: bare-return # Warns on bare returns + - name: blank-imports # Disallows blank imports + - name: bool-literal-in-expr # Suggests removing Boolean literals from logic expressions + - name: call-to-gc # Warns on explicit call to the garbage collector + #- name: cognitive-complexity # Sets restriction for maximum Cognitive complexity. + # There is a dedicated linter for this + - name: confusing-naming # Warns on methods with names that differ only by capitalization + - name: confusing-results # Suggests to name potentially confusing function results + - name: constant-logical-expr # Warns on constant logical expressions + - name: context-as-argument # `context.Context` should be the first argument of a function. + - name: context-keys-type # Disallows the usage of basic types in `context.WithValue`. + #- name: cyclomatic # Sets restriction for maximum Cyclomatic complexity. + # There is a dedicated linter for this + #- name: datarace # Spots potential dataraces + # Is not (yet) available? + - name: deep-exit # Looks for program exits in funcs other than `main()` or `init()` + - name: defer # Warns on some [defer gotchas](https://blog.learngoprogramming.com/5-gotchas-of-defer-in-go-golang-part-iii-36a1ab3d6ef1) + - name: dot-imports # Forbids `.` imports. + - name: duplicated-imports # Looks for packages that are imported two or more times + - name: early-return # Spots if-then-else statements that can be refactored to simplify code reading + - name: empty-block # Warns on empty code blocks + - name: empty-lines # Warns when there are heading or trailing newlines in a block + - name: errorf # Should replace `errors.New(fmt.Sprintf())` with `fmt.Errorf()` + - name: error-naming # Naming of error variables. + - name: error-return # The error return parameter should be last. + - name: error-strings # Conventions around error strings. + - name: exported # Naming and commenting conventions on exported symbols. + arguments: ['sayRepetitiveInsteadOfStutters'] + #- name: file-header # Header which each file should have. + # Useless without config, have no config for it + - name: flag-parameter # Warns on boolean parameters that create a control coupling + #- name: function-length # Warns on functions exceeding the statements or lines max + # There is a dedicated linter for this + #- name: function-result-limit # Specifies the maximum number of results a function can return + # Opinion: Don't need this + - name: get-return # Warns on getters that do not yield any result + - name: identical-branches # Spots if-then-else statements with identical `then` and `else` branches + - name: if-return # Redundant if when returning an error. + #- name: imports-blacklist # Disallows importing the specified packages + # Useless without config, have no config for it + - name: import-shadowing # Spots identifiers that shadow an import + - name: increment-decrement # Use `i++` and `i--` instead of `i += 1` and `i -= 1`. + - name: indent-error-flow # Prevents redundant else statements. + #- name: line-length-limit # Specifies the maximum number of characters in a lined + # There is a dedicated linter for this + #- name: max-public-structs # The maximum number of public structs in a file. + # Opinion: Don't need this + - name: modifies-parameter # Warns on assignments to function parameters + - name: modifies-value-receiver # Warns on assignments to value-passed method receivers + #- name: nested-structs # Warns on structs within structs + # Opinion: Don't need this + - name: optimize-operands-order # Checks inefficient conditional expressions + #- name: package-comments # Package commenting conventions. + # Opinion: Don't need this + - name: range # Prevents redundant variables when iterating over a collection. + - name: range-val-address # Warns if address of range value is used dangerously + - name: range-val-in-closure # Warns if range value is used in a closure dispatched as goroutine + - name: receiver-naming # Conventions around the naming of receivers. + - name: redefines-builtin-id # Warns on redefinitions of builtin identifiers + #- name: string-format # Warns on specific string literals that fail one or more user-configured regular expressions + # Useless without config, have no config for it + - name: string-of-int # Warns on suspicious casts from int to string + - name: struct-tag # Checks common struct tags like `json`,`xml`,`yaml` + - name: superfluous-else # Prevents redundant else statements (extends indent-error-flow) + - name: time-equal # Suggests to use `time.Time.Equal` instead of `==` and `!=` for equality check time. + - name: time-naming # Conventions around the naming of time variables. + - name: unconditional-recursion # Warns on function calls that will lead to (direct) infinite recursion + - name: unexported-naming # Warns on wrongly named un-exported symbols + - name: unexported-return # Warns when a public return is from unexported type. + - name: unhandled-error # Warns on unhandled errors returned by funcion calls + arguments: + - "fmt.(Fp|P)rint(f|ln|)" + - name: unnecessary-stmt # Suggests removing or simplifying unnecessary statements + - name: unreachable-code # Warns on unreachable code + - name: unused-parameter # Suggests to rename or remove unused function parameters + - name: unused-receiver # Suggests to rename or remove unused method receivers + #- name: use-any # Proposes to replace `interface{}` with its alias `any` + # Is not (yet) available? + - name: useless-break # Warns on useless `break` statements in case clauses + - name: var-declaration # Reduces redundancies around variable declaration. + - name: var-naming # Naming rules. + - name: waitgroup-by-value # Warns on functions taking sync.WaitGroup as a by-value parameter + +... diff --git a/.repo-runner.yaml b/.repo-runner.yaml deleted file mode 100644 index 93c8a23..0000000 --- a/.repo-runner.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- - -image: "quay.io/luzifer/repo-runner-image" -checkout_dir: /go/src/github.com/Luzifer/s3sync - -commands: - - make publish - -environment: - CGO_ENABLED: 0 diff --git a/Makefile b/Makefile index 9338123..628935d 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,8 @@ +default: + +lint: + golangci-lint run ./... + publish: curl -sSLo golang.sh https://raw.githubusercontent.com/Luzifer/github-publish/master/golang.sh bash golang.sh diff --git a/README.md b/README.md index abf8d0a..62d0a3c 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ ## Features - Static binary, no dependencies required -- Sync files only if required (judged by MD5 checksum) +- Sync files only if required (judged by file size & modify-date) - Using multiple threads to upload the transfer is quite fast - Optionally delete files at target - Optionally make files public on sync (only if file needs sync) @@ -49,7 +49,3 @@ Flags: (2 / 3) 07/26/bkm.png Skip (3 / 3) 07/26/luzifer_io.png Skip ``` - ---- - -![](https://d2o84fseuhwkxk.cloudfront.net/s3sync.svg) diff --git a/go.mod b/go.mod index 482b72f..19e4166 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,17 @@ module github.com/Luzifer/s3sync go 1.20 require ( + github.com/Luzifer/rconfig/v2 v2.4.0 github.com/aws/aws-sdk-go v1.44.279 - github.com/spf13/cobra v1.7.0 + github.com/pkg/errors v0.9.1 + github.com/sirupsen/logrus v1.9.3 ) require ( - github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/sys v0.8.0 // indirect + gopkg.in/validator.v2 v2.0.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 7175eba..e854b56 100644 --- a/go.sum +++ b/go.sum @@ -1,23 +1,27 @@ +github.com/Luzifer/rconfig/v2 v2.4.0 h1:MAdymTlExAZ8mx5VG8xOFAtFQSpWBipKYQHPOmYTn9o= +github.com/Luzifer/rconfig/v2 v2.4.0/go.mod h1:hWF3ZVSusbYlg5bEvCwalEyUSY+0JPJWUiIu7rBmav8= github.com/aws/aws-sdk-go v1.44.279 h1:g23dxnYjIiPlQo0gIKNR0zVPsSvo1bj5frWln+5sfhk= github.com/aws/aws-sdk-go v1.44.279/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -33,8 +37,11 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -48,6 +55,12 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/validator.v2 v2.0.1 h1:xF0KWyGWXm/LM2G1TrEjqOu4pa6coO9AlWSf3msVfDY= +gopkg.in/validator.v2 v2.0.1/go.mod h1:lIUZBlB3Im4s/eYp39Ry/wkR02yOPhZ9IwIRBjuPuG8= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/local.go b/local.go deleted file mode 100644 index deac910..0000000 --- a/local.go +++ /dev/null @@ -1,68 +0,0 @@ -package main - -import ( - "crypto/md5" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "strings" -) - -type localProvider struct{} - -func newLocalProvider() *localProvider { - return &localProvider{} -} - -func (l *localProvider) WriteFile(path string, content io.ReadSeeker, public bool) error { - os.MkdirAll(filepath.Dir(path), 0755) - - f, err := os.Create(path) - if err != nil { - return err - } - if _, err := io.Copy(f, content); err != nil { - return err - } - return f.Close() -} - -func (l *localProvider) ReadFile(path string) (io.ReadCloser, error) { - return os.Open(path) -} - -func (l *localProvider) ListFiles(prefix string) ([]file, error) { - out := []file{} - - err := filepath.Walk(prefix, func(path string, f os.FileInfo, err error) error { - if err != nil { - return err - } - - if !f.IsDir() { - content, err := ioutil.ReadFile(path) - if err != nil { - return err - } - - out = append(out, file{ - Filename: strings.TrimLeft(strings.Replace(path, prefix, "", 1), string(os.PathSeparator)), - Size: f.Size(), - MD5: fmt.Sprintf("%x", md5.Sum(content)), - }) - } - return nil - }) - - return out, err -} - -func (l *localProvider) DeleteFile(path string) error { - return os.Remove(path) -} - -func (l *localProvider) GetAbsolutePath(path string) (string, error) { - return filepath.Abs(path) -} diff --git a/logger/logger.go b/logger/logger.go deleted file mode 100644 index 4972c8e..0000000 --- a/logger/logger.go +++ /dev/null @@ -1,55 +0,0 @@ -package logger - -import "fmt" - -// LogLevel defines a type for named log levels -type LogLevel uint - -// Pre-Defined log levels to be used with this logging module -const ( - Error LogLevel = iota - Warning - Info - Debug -) - -// Logger is a wrapper around output to filter according to levels -type Logger struct { - Level LogLevel -} - -// New instanciates a new Logger and sets the preferred log level -func New(logLevel LogLevel) *Logger { - return &Logger{ - Level: logLevel, - } -} - -// Log is the filtered equivalent to fmt.Println -func (l *Logger) Log(level LogLevel, line string) { - if l.Level >= level { - fmt.Println(line) - } -} - -// LogF is the filtered equivalent to fmt.Printf -func (l *Logger) LogF(level LogLevel, line string, args ...interface{}) { - if l.Level >= level { - fmt.Printf(line, args...) - } -} - -// ErrorF executes LogF with Error level -func (l *Logger) ErrorF(line string, args ...interface{}) { - l.LogF(Error, line, args...) -} - -// InfoF executes LogF with Info level -func (l *Logger) InfoF(line string, args ...interface{}) { - l.LogF(Info, line, args...) -} - -// DebugF executes LogF with Debug level -func (l *Logger) DebugF(line string, args ...interface{}) { - l.LogF(Debug, line, args...) -} diff --git a/main.go b/main.go index 23db52b..34ea46b 100644 --- a/main.go +++ b/main.go @@ -1,137 +1,182 @@ package main import ( - "bytes" - "fmt" - "io" - "io/ioutil" "os" "path" "strings" "time" - "github.com/Luzifer/s3sync/logger" - "github.com/spf13/cobra" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + "github.com/Luzifer/rconfig/v2" + "github.com/Luzifer/s3sync/pkg/fsprovider" ) var ( cfg = struct { - Delete bool - Public bool - PrintVersion bool - MaxThreads int - logLevel uint + Delete bool `flag:"delete,d" default:"false" description:"Delete files on remote not existing on local"` + Endpoint string `flag:"endpoint" default:"" description:"Switch S3 endpoint (i.e. for MinIO compatibility)"` + LogLevel string `flag:"log-level" default:"info" description:"Log level (debug, info, warn, error, fatal)"` + MaxThreads int `flag:"max-threads" default:"10" description:"Use max N parallel threads for file sync"` + Public bool `flag:"public,P" default:"false" description:"Make files public when syncing to S3"` + VersionAndExit bool `flag:"version" default:"false" description:"Prints current version and exits"` }{} - stdout *logger.Logger + version = "dev" ) -type file struct { - Filename string - Size int64 - MD5 string +func getFSProvider(prefix string) (fsprovider.Provider, error) { + if strings.HasPrefix(prefix, "s3://") { + p, err := fsprovider.NewS3(cfg.Endpoint) + return p, errors.Wrap(err, "getting s3 provider") + } + return fsprovider.NewLocal(), nil } -type filesystemProvider interface { - WriteFile(path string, content io.ReadSeeker, public bool) error - ReadFile(path string) (io.ReadCloser, error) - ListFiles(prefix string) ([]file, error) - DeleteFile(path string) error - GetAbsolutePath(path string) (string, error) +func initApp() error { + rconfig.AutoEnv(true) + if err := rconfig.ParseAndValidate(&cfg); err != nil { + return errors.Wrap(err, "parsing cli options") + } + + l, err := logrus.ParseLevel(cfg.LogLevel) + if err != nil { + return errors.Wrap(err, "parsing log-level") + } + logrus.SetLevel(l) + + return nil } func main() { - app := cobra.Command{ - Use: "s3sync ", - Short: "Sync files from to ", - Run: execSync, - PreRun: func(cmd *cobra.Command, args []string) { - if cfg.PrintVersion { - fmt.Printf("s3sync %s\n", version) - os.Exit(0) - } - }, + var err error + if err = initApp(); err != nil { + logrus.WithError(err).Fatal("initializing app") } - app.Flags().BoolVarP(&cfg.Public, "public", "P", false, "Make files public when syncing to S3") - app.Flags().BoolVarP(&cfg.Delete, "delete", "d", false, "Delete files on remote not existing on local") - app.Flags().BoolVar(&cfg.PrintVersion, "version", false, "Print version and quit") - app.Flags().IntVar(&cfg.MaxThreads, "max-threads", 10, "Use max N parallel threads for file sync") - app.Flags().UintVar(&cfg.logLevel, "loglevel", 2, "Amount of log output (0 = Error only, 3 = Debug)") + if cfg.VersionAndExit { + logrus.WithField("version", version).Info("s3sync") + os.Exit(0) + } - app.ParseFlags(os.Args[1:]) - - stdout = logger.New(logger.LogLevel(cfg.logLevel)) - - app.Execute() + if err = runSync(rconfig.Args()[1:]); err != nil { + logrus.WithError(err).Fatal("running sync") + } } -func execSync(cmd *cobra.Command, args []string) { +//nolint:funlen,gocognit,gocyclo // Should be kept as single unit +func runSync(args []string) error { + //nolint:gomnd // Simple count of parameters, makes no sense to export if len(args) != 2 { - cmd.Usage() - os.Exit(1) + return errors.New("missing required arguments: s3sync ") } local, err := getFSProvider(args[0]) - errExit(err) + if err != nil { + return errors.Wrap(err, "getting local provider") + } remote, err := getFSProvider(args[1]) - errExit(err) + if err != nil { + return errors.Wrap(err, "getting remote provider") + } localPath, err := local.GetAbsolutePath(args[0]) - errExit(err) + if err != nil { + return errors.Wrap(err, "getting local path") + } remotePath, err := remote.GetAbsolutePath(args[1]) - errExit(err) + if err != nil { + return errors.Wrap(err, "getting remote path") + } localFiles, err := local.ListFiles(localPath) - errExit(err) + if err != nil { + return errors.Wrap(err, "listing local files") + } remoteFiles, err := remote.ListFiles(remotePath) - errExit(err) + if err != nil { + return errors.Wrap(err, "listing remote files") + } + + var ( + nErr int + syncChannel = make(chan bool, cfg.MaxThreads) + ) - syncChannel := make(chan bool, cfg.MaxThreads) for i, localFile := range localFiles { syncChannel <- true - go func(i int, localFile file) { + go func(i int, localFile fsprovider.File) { defer func() { <-syncChannel }() - needsCopy := true + var ( + logger = logrus.WithField("filename", localFile.Filename) + debugLogger = logger.WithField("tx_reason", "missing") + + needsCopy bool + remoteFound bool + ) + for _, remoteFile := range remoteFiles { - if remoteFile.Filename == localFile.Filename && remoteFile.MD5 == localFile.MD5 { + if remoteFile.Filename != localFile.Filename { + // Different file, do not compare + continue + } + + // We found a match, lets check whether tx is required + remoteFound = true + + switch { + case remoteFile.Size != localFile.Size: + debugLogger = debugLogger.WithField("tx_reason", "size-mismatch").WithField("ls", localFile.Size).WithField("rs", remoteFile.Size) + needsCopy = true + + case localFile.LastModified.After(remoteFile.LastModified): + debugLogger = debugLogger.WithField("tx_reason", "local-newer") + needsCopy = true + + default: + // No reason to update needsCopy = false - break } + + break } - if needsCopy { - l, err := local.ReadFile(path.Join(localPath, localFile.Filename)) - if err != nil { - stdout.ErrorF("(%d / %d) %s ERR: %s\n", i+1, len(localFiles), localFile.Filename, err) - return - } - buffer, err := ioutil.ReadAll(l) - if err != nil { - stdout.ErrorF("(%d / %d) %s ERR: %s\n", i+1, len(localFiles), localFile.Filename, err) - return - } - l.Close() - - err = remote.WriteFile(path.Join(remotePath, localFile.Filename), bytes.NewReader(buffer), cfg.Public) - if err != nil { - stdout.ErrorF("(%d / %d) %s ERR: %s\n", i+1, len(localFiles), localFile.Filename, err) - return - } - - stdout.InfoF("(%d / %d) %s OK\n", i+1, len(localFiles), localFile.Filename) + if remoteFound && !needsCopy { + logger.Debug("skipped transfer") return } - stdout.DebugF("(%d / %d) %s Skip\n", i+1, len(localFiles), localFile.Filename) + debugLogger.Debug("starting transfer") + + l, err := local.ReadFile(path.Join(localPath, localFile.Filename)) + if err != nil { + logger.WithError(err).Error("reading local file") + nErr++ + return + } + defer func() { + if err := l.Close(); err != nil { + logger.WithError(err).Error("closing local file") + } + }() + + err = remote.WriteFile(path.Join(remotePath, localFile.Filename), l, cfg.Public) + if err != nil { + logger.WithError(err).Error("writing remote file") + nErr++ + return + } + + logger.Info("transferred file") }(i, localFile) } if cfg.Delete { for _, remoteFile := range remoteFiles { syncChannel <- true - go func(remoteFile file) { + go func(remoteFile fsprovider.File) { defer func() { <-syncChannel }() needsDeletion := true @@ -143,33 +188,19 @@ func execSync(cmd *cobra.Command, args []string) { if needsDeletion { if err := remote.DeleteFile(path.Join(remotePath, remoteFile.Filename)); err != nil { - stdout.ErrorF("delete: %s ERR: %s\n", remoteFile.Filename, err) + logrus.WithField("filename", remoteFile.Filename).WithError(err).Error("deleting remote file") + nErr++ return } - stdout.InfoF("delete: %s OK\n", remoteFile.Filename) + logrus.WithField("filename", remoteFile.Filename).Info("deleted remote file") } }(remoteFile) } } - for { - if len(syncChannel) == 0 { - break - } + for len(syncChannel) > 0 { <-time.After(time.Second) } -} -func errExit(err error) { - if err != nil { - stdout.ErrorF("ERR: %s\n", err) - os.Exit(1) - } -} - -func getFSProvider(prefix string) (filesystemProvider, error) { - if strings.HasPrefix(prefix, "s3://") { - return newS3Provider() - } - return newLocalProvider(), nil + return nil } diff --git a/pkg/fsprovider/interface.go b/pkg/fsprovider/interface.go new file mode 100644 index 0000000..3a1e33d --- /dev/null +++ b/pkg/fsprovider/interface.go @@ -0,0 +1,26 @@ +// Package fsprovider contains implementations for filesystem access +// to read files from / write files to +package fsprovider + +import ( + "io" + "time" +) + +type ( + // File contains metadata about the file to be copied + File struct { + Filename string + LastModified time.Time + Size int64 + } + + // Provider describes the implementation of a fsprovider + Provider interface { + WriteFile(path string, content io.Reader, public bool) error + ReadFile(path string) (io.ReadCloser, error) + ListFiles(prefix string) ([]File, error) + DeleteFile(path string) error + GetAbsolutePath(path string) (string, error) + } +) diff --git a/pkg/fsprovider/local.go b/pkg/fsprovider/local.go new file mode 100644 index 0000000..8c99b42 --- /dev/null +++ b/pkg/fsprovider/local.go @@ -0,0 +1,92 @@ +package fsprovider + +import ( + "io" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" +) + +const directoryCreatePerms = 0o750 + +type ( + // Local implements the Provider for filesystem access + Local struct{} +) + +// NewLocal creates a new Local file provider +func NewLocal() *Local { + return &Local{} +} + +// DeleteFile deletes a local file +func (*Local) DeleteFile(path string) error { + return errors.Wrap(os.Remove(path), "removing file") +} + +// GetAbsolutePath converts the given path into an absolute path +func (*Local) GetAbsolutePath(path string) (string, error) { + fp, err := filepath.Abs(path) + return fp, errors.Wrap(err, "getting absolute filepath") +} + +// ListFiles retrieves the metadata of all files with given prefix +func (*Local) ListFiles(prefix string) ([]File, error) { + out := []File{} + + err := filepath.Walk(prefix, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + if f.IsDir() { + return nil + } + + linkTarget, err := filepath.EvalSymlinks(path) + if err != nil { + return errors.Wrap(err, "evaluating symlinks") + } + + if path != linkTarget { + if f, err = os.Stat(linkTarget); err != nil { + return errors.Wrap(err, "getting file-stat of link target") + } + } + + out = append(out, File{ + Filename: strings.TrimLeft(strings.Replace(path, prefix, "", 1), string(os.PathSeparator)), + LastModified: f.ModTime(), + Size: f.Size(), + }) + + return nil + }) + + return out, errors.Wrap(err, "walking prefix") +} + +// ReadFile opens the local file for reading +func (*Local) ReadFile(path string) (io.ReadCloser, error) { + f, err := os.Open(path) //#nosec:G304 // The purpose is to read a dynamic file + return f, errors.Wrap(err, "opening file") +} + +// WriteFile takes the content of the file and writes it to the local +// filesystem +func (*Local) WriteFile(path string, content io.Reader, _ bool) error { + if err := os.MkdirAll(filepath.Dir(path), directoryCreatePerms); err != nil { + return errors.Wrap(err, "creating file path") + } + + f, err := os.Create(path) //#nosec:G304 // The purpose is to create a dynamic file + if err != nil { + return errors.Wrap(err, "creating file") + } + if _, err := io.Copy(f, content); err != nil { + return errors.Wrap(err, "copying content") + } + return errors.Wrap(f.Close(), "closing file") +} diff --git a/pkg/fsprovider/s3.go b/pkg/fsprovider/s3.go new file mode 100644 index 0000000..92eed34 --- /dev/null +++ b/pkg/fsprovider/s3.go @@ -0,0 +1,217 @@ +package fsprovider + +import ( + "io" + "mime" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +const ( + channelBufferSizeHuge = 10000 + channelBufferSizeSmall = 10 + doneTickerInterval = 500 * time.Millisecond + maxKeysPerPage = 1000 +) + +type ( + // S3 implements the Provider for S3 / MinIO access + S3 struct { + conn *s3.S3 + requestedPrefix string + } +) + +// NewS3 creates a new S3 / MinIO file provider +func NewS3(endpoint string) (*S3, error) { + var cfgs []*aws.Config + + if endpoint != "" { + cfgs = append(cfgs, &aws.Config{ + Endpoint: &endpoint, + S3ForcePathStyle: aws.Bool(true), + }) + } + + sess := session.Must(session.NewSession(cfgs...)) + return &S3{ + conn: s3.New(sess), + }, nil +} + +// DeleteFile deletes an object from the bucket +func (s *S3) DeleteFile(path string) error { + bucket, path, err := s.getBucketPath(path) + if err != nil { + return errors.Wrap(err, "getting bucket path") + } + + _, err = s.conn.DeleteObject(&s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(path), + }) + + return errors.Wrap(err, "deleting object") +} + +// GetAbsolutePath converts the given path into an absolute path +func (*S3) GetAbsolutePath(path string) (string, error) { + return path, nil +} + +// ListFiles retrieves the metadata of all objects with given prefix +func (s *S3) ListFiles(prefix string) ([]File, error) { + out := []File{} + + bucket, path, err := s.getBucketPath(prefix) + if err != nil { + return out, errors.Wrap(err, "getting bucket path") + } + + processedPrefixes := []string{} + + prefixChan := make(chan *string, channelBufferSizeHuge) + outputChan := make(chan File, channelBufferSizeHuge) + errChan := make(chan error, channelBufferSizeSmall) + syncChan := make(chan bool, channelBufferSizeSmall) + doneTimer := time.NewTicker(doneTickerInterval) + + prefixChan <- aws.String(path) + + for { + select { + case prefix := <-prefixChan: + if len(syncChan) == channelBufferSizeSmall { + prefixChan <- prefix + } else { + found := false + for _, v := range processedPrefixes { + if v == *prefix { + found = true + } + } + if !found { + syncChan <- true + go s.readS3FileList(bucket, prefix, outputChan, prefixChan, errChan, syncChan) + processedPrefixes = append(processedPrefixes, *prefix) + } + } + case o := <-outputChan: + out = append(out, o) + case err := <-errChan: + return out, err + case <-doneTimer.C: + logrus.Debugf("scanning prefixes (%d working, %d left)...", len(syncChan), len(prefixChan)) + if len(prefixChan) == 0 && len(syncChan) == 0 { + return out, nil + } + } + } +} + +// ReadFile retrieves the object body for reading +func (s *S3) ReadFile(path string) (io.ReadCloser, error) { + bucket, path, err := s.getBucketPath(path) + if err != nil { + return nil, errors.Wrap(err, "getting bucket path") + } + + o, err := s.conn.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(path), + }) + if err != nil { + return nil, errors.Wrap(err, "getting object") + } + + return o.Body, nil +} + +// WriteFile copies the content into an S3 object +// +//revive:disable-next-line:flag-parameter // That's not a control coupling but a config flag +func (s *S3) WriteFile(path string, content io.Reader, public bool) error { + bucket, path, err := s.getBucketPath(path) + if err != nil { + return errors.Wrap(err, "getting bucket path") + } + + ext := filepath.Ext(path) + mimeType := mime.TypeByExtension(ext) + if mimeType == "" { + mimeType = "application/octet-stream" + } + + params := &s3manager.UploadInput{ + Bucket: aws.String(bucket), + Key: aws.String(path), + Body: content, + ContentType: aws.String(mimeType), + } + if public { + params.ACL = aws.String("public-read") + } + + _, err = s3manager.NewUploaderWithClient(s.conn).Upload(params) + return errors.Wrap(err, "uploading file") +} + +func (s *S3) getBucketPath(prefix string) (bucket string, path string, err error) { + rex := regexp.MustCompile(`^s3://?([^/]+)/(.*)$`) + matches := rex.FindStringSubmatch(prefix) + if matches == nil { + return "", "", errors.New("prefix did not match requirements") + } + + bucket = matches[1] + path = strings.ReplaceAll(matches[2], string(os.PathSeparator), "/") + s.requestedPrefix = path + + return bucket, path, nil +} + +func (s *S3) readS3FileList(bucket string, path *string, outputChan chan File, prefixChan chan *string, errorChan chan error, syncChan chan bool) { + defer func() { <-syncChan }() + in := &s3.ListObjectsInput{ + Bucket: aws.String(bucket), + Prefix: path, + MaxKeys: aws.Int64(maxKeysPerPage), + Delimiter: aws.String("/"), + } + for { + o, err := s.conn.ListObjects(in) + if err != nil { + errorChan <- errors.Wrap(err, "listing objects") + return + } + + for _, v := range o.Contents { + outputChan <- File{ + Filename: strings.Replace(*v.Key, s.requestedPrefix, "", 1), + LastModified: *v.LastModified, + Size: *v.Size, + } + } + + if len(o.CommonPrefixes) > 0 { + for _, cp := range o.CommonPrefixes { + prefixChan <- cp.Prefix + } + } + + if !*o.IsTruncated { + break + } + in.Marker = o.NextMarker + } +} diff --git a/s3.go b/s3.go deleted file mode 100644 index eb1d9e1..0000000 --- a/s3.go +++ /dev/null @@ -1,191 +0,0 @@ -package main - -import ( - "fmt" - "io" - "mime" - "os" - "path/filepath" - "regexp" - "strings" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" -) - -type s3Provider struct { - conn *s3.S3 - requestedPrefix string -} - -func newS3Provider() (*s3Provider, error) { - sess := session.Must(session.NewSession()) - return &s3Provider{ - conn: s3.New(sess), - }, nil -} - -func (s *s3Provider) getBucketPath(prefix string) (bucket string, path string, err error) { - rex := regexp.MustCompile(`^s3://?([^/]+)/(.*)$`) - matches := rex.FindStringSubmatch(prefix) - if len(matches) != 3 { - err = fmt.Errorf("prefix did not match requirements") - return - } - - bucket = matches[1] - path = strings.Replace(matches[2], string(os.PathSeparator), "/", -1) - s.requestedPrefix = path - - return -} - -func (s *s3Provider) ListFiles(prefix string) ([]file, error) { - out := []file{} - - bucket, path, err := s.getBucketPath(prefix) - if err != nil { - return out, err - } - - processedPrefixes := []string{} - - prefixChan := make(chan *string, 10000) - outputChan := make(chan file, 10000) - errChan := make(chan error, 10) - syncChan := make(chan bool, 10) - doneTimer := time.NewTicker(500 * time.Millisecond) - - prefixChan <- aws.String(path) - - for { - select { - case prefix := <-prefixChan: - if len(syncChan) == 10 { - prefixChan <- prefix - } else { - found := false - for _, v := range processedPrefixes { - if v == *prefix { - found = true - } - } - if !found { - syncChan <- true - go s.readS3FileList(bucket, prefix, outputChan, prefixChan, errChan, syncChan) - processedPrefixes = append(processedPrefixes, *prefix) - } - } - case o := <-outputChan: - out = append(out, o) - case err := <-errChan: - return out, err - case <-doneTimer.C: - stdout.DebugF("Scanning prefixes (%d working, %d left)...\r", len(syncChan), len(prefixChan)) - if len(prefixChan) == 0 && len(syncChan) == 0 { - fmt.Printf("\n") - return out, nil - } - } - } -} - -func (s *s3Provider) readS3FileList(bucket string, path *string, outputChan chan file, prefixChan chan *string, errorChan chan error, syncChan chan bool) { - defer func() { <-syncChan }() - in := &s3.ListObjectsInput{ - Bucket: aws.String(bucket), - Prefix: path, - MaxKeys: aws.Int64(1000), - Delimiter: aws.String("/"), - } - for { - o, err := s.conn.ListObjects(in) - if err != nil { - errorChan <- err - return - } - - for _, v := range o.Contents { - outputChan <- file{ - Filename: strings.Replace(*v.Key, s.requestedPrefix, "", 1), - Size: *v.Size, - MD5: strings.Trim(*v.ETag, "\""), // Wat? - } - } - - if len(o.CommonPrefixes) > 0 { - for _, cp := range o.CommonPrefixes { - prefixChan <- cp.Prefix - } - } - - if !*o.IsTruncated { - break - } - in.Marker = o.NextMarker - } -} - -func (s *s3Provider) WriteFile(path string, content io.ReadSeeker, public bool) error { - bucket, path, err := s.getBucketPath(path) - if err != nil { - return err - } - - ext := filepath.Ext(path) - mimeType := mime.TypeByExtension(ext) - if mimeType == "" { - mimeType = "application/octet-stream" - } - - params := &s3.PutObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(path), - Body: content, - ContentType: aws.String(mimeType), - } - if public { - params.ACL = aws.String("public-read") - } - _, err = s.conn.PutObject(params) - - return err -} - -func (s *s3Provider) ReadFile(path string) (io.ReadCloser, error) { - bucket, path, err := s.getBucketPath(path) - if err != nil { - return nil, err - } - - o, err := s.conn.GetObject(&s3.GetObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(path), - }) - - if err != nil { - return nil, err - } - - return o.Body, nil -} - -func (s *s3Provider) DeleteFile(path string) error { - bucket, path, err := s.getBucketPath(path) - if err != nil { - return err - } - - _, err = s.conn.DeleteObject(&s3.DeleteObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(path), - }) - - return err -} - -func (s *s3Provider) GetAbsolutePath(path string) (string, error) { - return path, nil -}