From 632ab6001893ca8890e58ea8ab44d40db48c820f Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Mon, 6 Jul 2015 21:41:21 +0200 Subject: [PATCH] Moved towards modular storage system --- config/config.go | 30 +++++++++++++++++ main.go | 26 +++++++-------- storage/adapter.go | 36 ++++++++++++++++++++ storage/s3.go | 82 ++++++++++++++++++++++++++++++++++++++++++++++ structs.go | 15 ++++++--- web_handlers.go | 19 ++++++----- welcome_runner.go | 9 ++--- 7 files changed, 185 insertions(+), 32 deletions(-) create mode 100644 config/config.go create mode 100644 storage/adapter.go create mode 100644 storage/s3.go diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..680e2cc --- /dev/null +++ b/config/config.go @@ -0,0 +1,30 @@ +package config // import "github.com/Luzifer/mondash/config" + +import ( + "os" + + "github.com/spf13/pflag" +) + +type Config struct { + Storage string + BaseURL string + APIToken string + + S3 struct { + Bucket string + } +} + +func Load() *Config { + cfg := &Config{} + pflag.StringVar(&cfg.Storage, "storage", "s3", "Storage engine to use") + pflag.StringVar(&cfg.BaseURL, "baseurl", os.Getenv("BASE_URL"), "The Base-URL the application is running on for example https://mondash.org") + pflag.StringVar(&cfg.APIToken, "api-token", os.Getenv("API_TOKEN"), "API Token used for the /welcome dashboard (you can choose your own)") + + // S3 + pflag.StringVar(&cfg.S3.Bucket, "s3Bucket", os.Getenv("S3Bucket"), "Bucket to use for S3 storage") + + pflag.Parse() + return cfg +} diff --git a/main.go b/main.go index b8a4a87..4a18008 100644 --- a/main.go +++ b/main.go @@ -5,33 +5,31 @@ import ( "fmt" "io/ioutil" "net/http" - "os" "time" - "log" - - "launchpad.net/goamz/aws" - "launchpad.net/goamz/s3" - + "github.com/Luzifer/mondash/config" + "github.com/Luzifer/mondash/storage" "github.com/flosch/pongo2" "github.com/go-martini/martini" _ "github.com/flosch/pongo2-addons" ) -var templates = make(map[string]*pongo2.Template) -var s3Storage *s3.Bucket +var ( + templates = make(map[string]*pongo2.Template) + store storage.Storage + cfg *config.Config +) func main() { preloadTemplates() - // Initialize S3 storage - awsAuth, err := aws.EnvAuth() + var err error + cfg = config.Load() + store, err = storage.GetStorage(cfg) if err != nil { - log.Fatal(err) + fmt.Printf("An error occurred while loading the storage handler: %s", err) } - s3Conn := s3.New(awsAuth, aws.EUWest) - s3Storage = s3Conn.Bucket(os.Getenv("S3Bucket")) m := martini.Classic() @@ -48,7 +46,7 @@ func main() { m.Delete("/:dashid", handleDeleteDashboard) m.Delete("/:dashid/:metricid", handleDeleteMetric) - go runWelcomePage() + go runWelcomePage(cfg) // GO! m.Run() diff --git a/storage/adapter.go b/storage/adapter.go new file mode 100644 index 0000000..e55c0e5 --- /dev/null +++ b/storage/adapter.go @@ -0,0 +1,36 @@ +package storage // import "github.com/Luzifer/mondash/storage" + +import ( + "fmt" + + "github.com/Luzifer/mondash/config" +) + +// Storage is an interface to have all storage systems compatible to each other +type Storage interface { + Put(dashboardID string, data []byte) error + Get(dashboardID string) ([]byte, error) + Delete(dashboardID string) error + Exists(dashboardID string) (bool, error) +} + +// NotFoundError is a named error for more simple determination which +// type of error is thrown +type NotFoundError struct { + Name string +} + +func (e NotFoundError) Error() string { + return fmt.Sprintf("Storage '%s' not found.", e.Name) +} + +// GetStorage acts as a storage factory providing the storage named by input +// name parameter +func GetStorage(cfg *config.Config) (Storage, error) { + switch cfg.Storage { + case "s3": + return NewS3Storage(cfg), nil + } + + return nil, NotFoundError{cfg.Storage} +} diff --git a/storage/s3.go b/storage/s3.go new file mode 100644 index 0000000..1e8e591 --- /dev/null +++ b/storage/s3.go @@ -0,0 +1,82 @@ +package storage // import "github.com/Luzifer/mondash/storage" + +import ( + "io/ioutil" + "strings" + + "github.com/Luzifer/mondash/config" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/s3" +) + +type S3Storage struct { + s3connection *s3.S3 + cfg *config.Config +} + +func NewS3Storage(cfg *config.Config) *S3Storage { + s3connection := s3.New(&aws.Config{}) + return &S3Storage{ + s3connection: s3connection, + } +} + +func (s *S3Storage) Put(dashboardID string, data []byte) error { + _, err := s.s3connection.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(s.cfg.S3.Bucket), + ContentType: aws.String("application/json"), + Key: aws.String(dashboardID), + // TODO: Private ACL + }) + + return err +} + +func (s *S3Storage) Get(dashboardID string) ([]byte, error) { + res, err := s.s3connection.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(s.cfg.S3.Bucket), + Key: aws.String(dashboardID), + }) + if err != nil { + return nil, err + } + + defer res.Body.Close() + + data, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + + return data, nil +} + +func (s *S3Storage) Delete(dashboardID string) error { + _, err := s.s3connection.DeleteObject(&s3.DeleteObjectInput{ + Bucket: aws.String(s.cfg.S3.Bucket), + Key: aws.String(dashboardID), + }) + + return err +} + +func (s *S3Storage) Exists(dashboardID string) (bool, error) { + _, err := s.s3connection.HeadObject(&s3.HeadObjectInput{ + Bucket: aws.String(s.cfg.S3.Bucket), + Key: aws.String(dashboardID), + }) + + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + if strings.Contains(awsErr.Error(), "status code: 404") { + return false, nil + } + return false, err + } else { + return false, err + } + } + + return true, nil +} diff --git a/structs.go b/structs.go index f8c9c2d..d47814a 100644 --- a/structs.go +++ b/structs.go @@ -3,26 +3,31 @@ package main import ( "encoding/json" "errors" - "launchpad.net/goamz/s3" "log" "sort" "strconv" "time" + + "github.com/Luzifer/mondash/storage" ) type dashboard struct { DashboardID string `json:"-"` APIKey string `json:"api_key"` Metrics dashboardMetrics `json:"metrics"` + storage storage.Storage } -func loadDashboard(dashid string) (*dashboard, error) { - data, err := s3Storage.Get(dashid) +func loadDashboard(dashid string, store storage.Storage) (*dashboard, error) { + data, err := store.Get(dashid) if err != nil { return &dashboard{}, errors.New("Dashboard not found") } - tmp := &dashboard{DashboardID: dashid} + tmp := &dashboard{ + DashboardID: dashid, + storage: store, + } _ = json.Unmarshal(data, tmp) return tmp, nil @@ -34,7 +39,7 @@ func (d *dashboard) Save() { log.Printf("Error while marshalling dashboard: %s", err) return } - err = s3Storage.Put(d.DashboardID, data, "application/json", s3.Private) + err = d.storage.Put(d.DashboardID, data) if err != nil { log.Printf("Error while storing dashboard: %s", err) } diff --git a/web_handlers.go b/web_handlers.go index f7f2ad4..8b20c68 100644 --- a/web_handlers.go +++ b/web_handlers.go @@ -18,17 +18,18 @@ func handleRedirectWelcome(res http.ResponseWriter, req *http.Request) { } func handleCreateRandomDashboard(res http.ResponseWriter, req *http.Request) { - urlProposal := generateAPIKey()[0:20] - _, err := s3Storage.Get(urlProposal) - for err == nil { + var urlProposal string + for { urlProposal = generateAPIKey()[0:20] - _, err = s3Storage.Get(urlProposal) + if exists, err := store.Exists(urlProposal); err == nil && !exists { + break + } } http.Redirect(res, req, fmt.Sprintf("/%s", urlProposal), http.StatusTemporaryRedirect) } func handleDisplayDashboard(params martini.Params, res http.ResponseWriter) { - dash, err := loadDashboard(params["dashid"]) + dash, err := loadDashboard(params["dashid"], store) if err != nil { dash = &dashboard{APIKey: generateAPIKey(), Metrics: dashboardMetrics{}} } @@ -51,7 +52,7 @@ func handleDisplayDashboard(params martini.Params, res http.ResponseWriter) { } func handleDeleteDashboard(params martini.Params, req *http.Request, res http.ResponseWriter) { - dash, err := loadDashboard(params["dashid"]) + dash, err := loadDashboard(params["dashid"], store) if err != nil { http.Error(res, "This dashboard does not exist.", http.StatusInternalServerError) return @@ -62,7 +63,7 @@ func handleDeleteDashboard(params martini.Params, req *http.Request, res http.Re return } - _ = s3Storage.Del(params["dashid"]) + store.Delete(params["dashid"]) http.Error(res, "OK", http.StatusOK) } @@ -80,7 +81,7 @@ func handlePutMetric(params martini.Params, req *http.Request, res http.Response return } - dash, err := loadDashboard(params["dashid"]) + dash, err := loadDashboard(params["dashid"], store) if err != nil { if len(req.Header.Get("Authorization")) < 10 { http.Error(res, "APIKey is too insecure", http.StatusUnauthorized) @@ -122,7 +123,7 @@ func handlePutMetric(params martini.Params, req *http.Request, res http.Response } func handleDeleteMetric(params martini.Params, req *http.Request, res http.ResponseWriter) { - dash, err := loadDashboard(params["dashid"]) + dash, err := loadDashboard(params["dashid"], store) if err != nil { dash = &dashboard{APIKey: req.Header.Get("Authorization"), Metrics: dashboardMetrics{}, DashboardID: params["dashid"]} } diff --git a/welcome_runner.go b/welcome_runner.go index c342f49..1c0dcc3 100644 --- a/welcome_runner.go +++ b/welcome_runner.go @@ -7,13 +7,14 @@ import ( "log" "math/rand" "net/http" - "os" "time" + + "github.com/Luzifer/mondash/config" ) -func runWelcomePage() { - baseURL := os.Getenv("BASE_URL") - welcomeAPIToken := os.Getenv("API_TOKEN") +func runWelcomePage(cfg *config.Config) { + baseURL := cfg.BaseURL + welcomeAPIToken := cfg.APIToken generateTicker := time.NewTicker(time.Minute) for {