From 3dfd5694d9bb7512c774de198fdd7bf7226f79d1 Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Sun, 16 Jun 2019 02:05:06 +0200 Subject: [PATCH] Add basic structure --- .gitignore | 2 + cmd/cloudbox/config.go | 94 +++++++++++++++++++++++++++++++++++++ cmd/cloudbox/help.go | 21 +++++++++ cmd/cloudbox/main.go | 77 ++++++++++++++++++++++++++++++ cmd/cloudbox/providers.go | 36 ++++++++++++++ cmd/cloudbox/sync.go | 44 +++++++++++++++++ providers/file.go | 23 +++++++++ providers/interface.go | 30 ++++++++++++ providers/local/file.go | 46 ++++++++++++++++++ providers/local/provider.go | 87 ++++++++++++++++++++++++++++++++++ sync/sync.go | 28 +++++++++++ 11 files changed, 488 insertions(+) create mode 100644 .gitignore create mode 100644 cmd/cloudbox/config.go create mode 100644 cmd/cloudbox/help.go create mode 100644 cmd/cloudbox/main.go create mode 100644 cmd/cloudbox/providers.go create mode 100644 cmd/cloudbox/sync.go create mode 100644 providers/file.go create mode 100644 providers/interface.go create mode 100644 providers/local/file.go create mode 100644 providers/local/provider.go create mode 100644 sync/sync.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..63bf876 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +config.yaml +dev_test diff --git a/cmd/cloudbox/config.go b/cmd/cloudbox/config.go new file mode 100644 index 0000000..48f6cc1 --- /dev/null +++ b/cmd/cloudbox/config.go @@ -0,0 +1,94 @@ +package main + +import ( + "os" + + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" +) + +type shareConfig struct { + OverrideURI bool `yaml:"override_uri"` + URITemplate string `yaml:"uri_template"` +} + +type syncConfig struct { + LocalDir string `yaml:"local_dir"` + RemoteURI string `yaml:"remote_uri"` +} + +type configFile struct { + ControlDir string `yaml:"control_dir"` + Sync syncConfig `yaml:"sync"` + Share shareConfig `yaml:"share"` +} + +func (c configFile) validate() error { + if c.Sync.LocalDir == "" { + return errors.New("Local directory not specified") + } + + if c.Sync.RemoteURI == "" { + return errors.New("Remote sync URI not specified") + } + + if c.Share.OverrideURI && c.Share.URITemplate == "" { + return errors.New("Share URI override enabled but no template specified") + } + + return nil +} + +func defaultConfig() *configFile { + return &configFile{ + ControlDir: "~/.cache/cloudbox", + } +} + +func execWriteSampleConfig() error { + conf := defaultConfig() + + if _, err := os.Stat(cfg.Config); err == nil { + if conf, err = loadConfig(true); err != nil { + return errors.Wrap(err, "Unable to load existing config") + } + } + + f, err := os.Create(cfg.Config) + if err != nil { + return errors.Wrap(err, "Unable to create config file") + } + defer f.Close() + + f.WriteString("---\n\n") + + if err := yaml.NewEncoder(f).Encode(conf); err != nil { + return errors.Wrap(err, "Unable to write config file") + } + + f.WriteString("\n...\n") + + log.WithField("dest", cfg.Config).Info("Config written") + return nil +} + +func loadConfig(noValidate bool) (*configFile, error) { + config := defaultConfig() + + f, err := os.Open(cfg.Config) + if err != nil { + return nil, errors.Wrap(err, "Unable to open config file") + } + defer f.Close() + + if err = yaml.NewDecoder(f).Decode(config); err != nil { + return nil, errors.Wrap(err, "Unable to decode config") + } + + if noValidate { + return config, nil + } + + return config, config.validate() +} diff --git a/cmd/cloudbox/help.go b/cmd/cloudbox/help.go new file mode 100644 index 0000000..c6abf7e --- /dev/null +++ b/cmd/cloudbox/help.go @@ -0,0 +1,21 @@ +package main + +import ( + "fmt" + + "github.com/Luzifer/rconfig" +) + +const helpText = ` +Available commands: + help Display this message + sync Executes the bi-directional sync + write-config Write a sample configuration to specified location +` + +func execHelp() error { + rconfig.Usage() + + fmt.Print(helpText) + return nil +} diff --git a/cmd/cloudbox/main.go b/cmd/cloudbox/main.go new file mode 100644 index 0000000..4ed7529 --- /dev/null +++ b/cmd/cloudbox/main.go @@ -0,0 +1,77 @@ +package main + +import ( + "fmt" + "os" + + "github.com/mitchellh/go-homedir" + log "github.com/sirupsen/logrus" + + "github.com/Luzifer/rconfig" +) + +type command string +type commandFunc func() error + +const ( + cmdHelp command = "help" + cmdSync command = "sync" + cmdWriteConfig command = "write-config" +) + +var cmdFuncs = map[command]commandFunc{ + cmdSync: execSync, + cmdWriteConfig: execWriteSampleConfig, +} + +var ( + cfg = struct { + Config string `flag:"config,c" default:"config.yaml" description:"Configuration file location"` + Force bool `flag:"force,f" default:"false" description:"Force operation"` + LogLevel string `flag:"log-level" default:"info" description:"Log level (debug, info, warn, error, fatal)"` + VersionAndExit bool `flag:"version" default:"false" description:"Prints current version and exits"` + }{} + + version = "dev" +) + +func init() { + if err := rconfig.ParseAndValidate(&cfg); err != nil { + log.Fatalf("Unable to parse commandline options: %s", err) + } + + if cfg.VersionAndExit { + fmt.Printf("cloudbox %s\n", version) + os.Exit(0) + } + + if l, err := log.ParseLevel(cfg.LogLevel); err != nil { + log.WithError(err).Fatal("Unable to parse log level") + } else { + log.SetLevel(l) + } + + if dir, err := homedir.Expand(cfg.Config); err != nil { + log.WithError(err).Fatal("Unable to expand config path") + } else { + cfg.Config = dir + } +} + +func main() { + cmd := cmdHelp + if len(rconfig.Args()) > 1 { + cmd = command(rconfig.Args()[1]) + } + + var cmdFunc commandFunc = execHelp + if f, ok := cmdFuncs[cmd]; ok { + cmdFunc = f + } + + log.WithField("version", version).Info("cloudbox started") + + if err := cmdFunc(); err != nil { + log.WithError(err).Fatal("Command execution failed") + } +} diff --git a/cmd/cloudbox/providers.go b/cmd/cloudbox/providers.go new file mode 100644 index 0000000..bed67be --- /dev/null +++ b/cmd/cloudbox/providers.go @@ -0,0 +1,36 @@ +package main + +import ( + "github.com/pkg/errors" + + "github.com/Luzifer/cloudbox/providers" + "github.com/Luzifer/cloudbox/providers/local" +) + +var providerInitFuncs = []providers.CloudProviderInitFunc{ + local.New, +} + +func providerFromURI(uri string) (providers.CloudProvider, error) { + if uri == "" { + return nil, errors.New("Empty provider URI") + } + + for _, f := range providerInitFuncs { + cp, err := f(uri) + switch err { + case nil: + if cp.Capabilities()&providers.CapBasic == 0 { + return nil, errors.Errorf("Provider %s does not support basic capabilities", cp.Name()) + } + + return cp, nil + case providers.ErrInvalidURI: + // Fine for now, try next one + default: + return nil, errors.Wrap(err, "Unable to initialize provider") + } + } + + return nil, errors.Errorf("No provider found for URI %q", uri) +} diff --git a/cmd/cloudbox/sync.go b/cmd/cloudbox/sync.go new file mode 100644 index 0000000..66f98fc --- /dev/null +++ b/cmd/cloudbox/sync.go @@ -0,0 +1,44 @@ +package main + +import ( + "database/sql" + "os" + "path" + + _ "github.com/mattn/go-sqlite3" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + + "github.com/Luzifer/cloudbox/sync" +) + +func execSync() error { + conf, err := loadConfig(false) + if err != nil { + return errors.Wrap(err, "Unable to load config") + } + + local, err := providerFromURI("file://" + conf.Sync.LocalDir) + if err != nil { + return errors.Wrap(err, "Unable to initialize local provider") + } + + remote, err := providerFromURI(conf.Sync.RemoteURI) + if err != nil { + return errors.Wrap(err, "Unable to initialize remote provider") + } + + if err := os.MkdirAll(conf.ControlDir, 0700); err != nil { + return errors.Wrap(err, "Unable to create control dir") + } + + db, err := sql.Open("sqlite3", path.Join(conf.ControlDir, "sync.db")) + if err != nil { + return errors.Wrap(err, "Unable to establish database connection") + } + + s := sync.New(local, remote, db) + + log.Info("Starting sync run...") + return errors.Wrap(s.Run(), "Unable to sync") +} diff --git a/providers/file.go b/providers/file.go new file mode 100644 index 0000000..5990da4 --- /dev/null +++ b/providers/file.go @@ -0,0 +1,23 @@ +package providers + +import ( + "io" + "time" + + "github.com/pkg/errors" +) + +var ErrFileNotFound = errors.New("File not found") + +type File interface { + Info() FileInfo + Checksum() (string, error) + Content() (io.ReadCloser, error) +} + +type FileInfo struct { + RelativeName string + LastModified time.Time + Checksum string // Expected to be present on CapAutoChecksum + Size uint64 +} diff --git a/providers/interface.go b/providers/interface.go new file mode 100644 index 0000000..b252a13 --- /dev/null +++ b/providers/interface.go @@ -0,0 +1,30 @@ +package providers + +import ( + "github.com/pkg/errors" +) + +type Capability uint8 + +const ( + CapBasic Capability = 1 << iota + CapShare + CapAutoChecksum +) + +var ( + ErrInvalidURI = errors.New("Spefified URI is invalid for this provider") + ErrFeatureNotSupported = errors.New("Feature not supported") +) + +type CloudProviderInitFunc func(string) (CloudProvider, error) + +type CloudProvider interface { + Capabilities() Capability + Name() string + DeleteFile(relativeName string) error + GetFile(relativeName string) (File, error) + ListFiles() ([]File, error) + PutFile(File) error + Share(relativeName string) (string, error) +} diff --git a/providers/local/file.go b/providers/local/file.go new file mode 100644 index 0000000..675b805 --- /dev/null +++ b/providers/local/file.go @@ -0,0 +1,46 @@ +package local + +import ( + "bytes" + "crypto/sha256" + "fmt" + "io" + "os" + + "github.com/pkg/errors" + + "github.com/Luzifer/cloudbox/providers" +) + +type File struct { + info os.FileInfo + relativeName string + fullPath string +} + +func (f File) Info() providers.FileInfo { + return providers.FileInfo{ + RelativeName: f.relativeName, + LastModified: f.info.ModTime(), + Size: uint64(f.info.Size()), + } +} + +func (f File) Checksum() (string, error) { + fc, err := f.Content() + if err != nil { + return "", errors.Wrap(err, "Unable to get file contents") + } + + buf := new(bytes.Buffer) + if _, err := io.Copy(buf, fc); err != nil { + return "", errors.Wrap(err, "Unable to read file contents") + } + + return fmt.Sprintf("%x", sha256.Sum256(buf.Bytes())), nil +} + +func (f File) Content() (io.ReadCloser, error) { + fp, err := os.Open(f.fullPath) + return fp, errors.Wrap(err, "Unable to open file") +} diff --git a/providers/local/provider.go b/providers/local/provider.go new file mode 100644 index 0000000..4180f05 --- /dev/null +++ b/providers/local/provider.go @@ -0,0 +1,87 @@ +package local + +import ( + "io" + "os" + "path" + "strings" + "time" + + "github.com/pkg/errors" + + "github.com/Luzifer/cloudbox/providers" +) + +func New(uri string) (providers.CloudProvider, error) { + if !strings.HasPrefix(uri, "file://") { + return nil, providers.ErrInvalidURI + } + + return &Provider{directory: strings.TrimPrefix(uri, "file://")}, nil +} + +type Provider struct { + directory string +} + +func (p Provider) Capabilities() providers.Capability { return providers.CapBasic } +func (p Provider) Name() string { return "local" } + +func (p Provider) ListFiles() ([]providers.File, error) { + return nil, errors.New("Not implemented") +} + +func (p Provider) DeleteFile(relativeName string) error { + return os.Remove(path.Join(p.directory, relativeName)) +} + +func (p Provider) GetFile(relativeName string) (providers.File, error) { + fullPath := path.Join(p.directory, relativeName) + + stat, err := os.Stat(fullPath) + if err != nil { + if os.IsNotExist(err) { + return nil, providers.ErrFileNotFound + } + return nil, errors.Wrap(err, "Unable to get file stat") + } + + return File{ + info: stat, + relativeName: relativeName, + fullPath: fullPath, + }, nil +} + +func (p Provider) PutFile(f providers.File) error { + fullPath := path.Join(p.directory, f.Info().RelativeName) + + fp, err := os.Create(fullPath) + if err != nil { + return errors.Wrap(err, "Unable to create file") + } + + rfp, err := f.Content() + if err != nil { + return errors.Wrap(err, "Unable to get remote file content") + } + defer rfp.Close() + + if _, err := io.Copy(fp, rfp); err != nil { + return errors.Wrap(err, "Unable to copy file contents") + } + + if err := fp.Close(); err != nil { + return errors.Wrap(err, "Unable to close local file") + } + + if err := os.Chtimes(fullPath, time.Now(), f.Info().LastModified); err != nil { + return errors.Wrap(err, "Unable to set last file mod time") + } + + return nil +} + +func (p Provider) Share(relativeName string) (string, error) { + return "", providers.ErrFeatureNotSupported +} diff --git a/sync/sync.go b/sync/sync.go new file mode 100644 index 0000000..8a9ac51 --- /dev/null +++ b/sync/sync.go @@ -0,0 +1,28 @@ +package sync + +import ( + "database/sql" + + "github.com/Luzifer/cloudbox/providers" +) + +type Sync struct { + db *sql.DB + local, remote providers.CloudProvider +} + +func New(local, remote providers.CloudProvider, db *sql.DB) *Sync { + return &Sync{ + db: db, + local: local, + remote: remote, + } +} + +func (s *Sync) Run() error { + for { + select {} + } + + return nil +}