From 83030ad9d46075524f9dd63c0483ee48593ea90e Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Wed, 24 Apr 2019 20:53:34 +0200 Subject: [PATCH] Add more logging, cleanup code, add docs Signed-off-by: Knut Ahlers --- http.go | 112 +++++++++++++++++++++++++---------------------------- main.go | 11 +++++- storage.go | 33 ++++++++++++---- 3 files changed, 87 insertions(+), 69 deletions(-) diff --git a/http.go b/http.go index f7b5546..3a6cf3b 100644 --- a/http.go +++ b/http.go @@ -12,61 +12,15 @@ import ( ) func init() { - http.HandleFunc("/api/page", handlePage) - http.HandleFunc("/api/since", handleNewest) - http.HandleFunc("/api/favourite", handleFavourite) - http.HandleFunc("/api/refresh", handleTweetRefresh) + http.HandleFunc("/api/favourite", handleFavorite) http.HandleFunc("/api/force-reload", handleForceReload) + http.HandleFunc("/api/page", handlePage) + http.HandleFunc("/api/refresh", handleTweetRefresh) + http.HandleFunc("/api/since", handleNewest) } -func handlePage(w http.ResponseWriter, r *http.Request) { - var page int = 1 - - if p, err := strconv.Atoi(r.URL.Query().Get("n")); err == nil && p > 0 { - page = p - } - - tweets, err := tweetStore.GetTweetPage(page) - if err != nil { - log.WithError(err).Error("Unable to fetch tweets for page request") - http.Error(w, "Something went wrong", http.StatusInternalServerError) - return - } - - tweetResponse(w, tweets) -} - -func handleNewest(w http.ResponseWriter, r *http.Request) { - var since uint64 = 0 - - if s, err := strconv.ParseUint(r.URL.Query().Get("id"), 10, 64); err == nil && s > 0 { - since = s - } - - if since == 0 { - http.Error(w, "Must specify last id", http.StatusBadRequest) - return - } - - tweets, err := tweetStore.GetTweetsSince(since) - if err != nil { - log.WithError(err).Error("Unable to fetch tweets for newest request") - http.Error(w, "Something went wrong", http.StatusInternalServerError) - return - } - - tweetResponse(w, tweets) -} - -func tweetResponse(w http.ResponseWriter, tweets []tweet) { - w.Header().Set("Content-Type", "application/json") - w.Header().Set("Cache-Control", "no-cache") - w.WriteHeader(http.StatusOK) - - json.NewEncoder(w).Encode(tweets) -} - -func handleFavourite(w http.ResponseWriter, r *http.Request) { +// handleFavorite takes the ID of a tweet and submits a favorite to Twitter +func handleFavorite(w http.ResponseWriter, r *http.Request) { req := struct { ID int64 `json:"id,string"` }{} @@ -99,6 +53,46 @@ func handleFavourite(w http.ResponseWriter, r *http.Request) { tweetResponse(w, tweets) } +// handleForceReload issues a full load of the latest tweets to update their state +func handleForceReload(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "This needs to be POST", http.StatusBadRequest) + return + } + + go loadAndStoreTweets(true) + + w.WriteHeader(http.StatusNoContent) +} + +// handleNewest returns all tweets newly stored since the given tweet ID +func handleNewest(w http.ResponseWriter, r *http.Request) { + var since uint64 + + if s, err := strconv.ParseUint(r.URL.Query().Get("id"), 10, 64); err == nil && s > 0 { + since = s + } + + if since == 0 { + http.Error(w, "Must specify last id", http.StatusBadRequest) + return + } + + tweetResponse(w, tweetStore.GetTweetsSince(since)) +} + +// handlePage loads older tweets with pagination +func handlePage(w http.ResponseWriter, r *http.Request) { + var page = 1 + + if p, err := strconv.Atoi(r.URL.Query().Get("n")); err == nil && p > 0 { + page = p + } + + tweetResponse(w, tweetStore.GetTweetPage(page)) +} + +// handleTweetRefresh refreshes the state of the tweet with the given ID against the Twitter API func handleTweetRefresh(w http.ResponseWriter, r *http.Request) { req := struct { ID int64 `json:"id,string"` @@ -141,13 +135,11 @@ func handleTweetRefresh(w http.ResponseWriter, r *http.Request) { tweetResponse(w, tweets) } -func handleForceReload(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - http.Error(w, "This needs to be POST", http.StatusBadRequest) - return - } +// tweetResponse is a generic wrapper to return a list of tweets through JSON +func tweetResponse(w http.ResponseWriter, tweets []tweet) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-cache") + w.WriteHeader(http.StatusOK) - go loadAndStoreTweets(true) - - w.WriteHeader(http.StatusNoContent) + json.NewEncoder(w).Encode(tweets) } diff --git a/main.go b/main.go index a5674ff..2d75c18 100644 --- a/main.go +++ b/main.go @@ -11,6 +11,7 @@ import ( "github.com/ChimeraCoder/anaconda" log "github.com/sirupsen/logrus" + hhelp "github.com/Luzifer/go_helpers/v2/http" "github.com/Luzifer/rconfig/v2" ) @@ -69,16 +70,20 @@ func main() { } }() + log.WithField("version", version).Info("MediaTimeline Viewer started") + http.Handle("/", http.FileServer(http.Dir(cfg.Frontend))) - http.ListenAndServe(cfg.Listen, nil) + http.ListenAndServe(cfg.Listen, hhelp.NewHTTPLogHandler(http.DefaultServeMux)) } func loadAndStoreTweets(forceRefresh bool) { + log.WithField("force", forceRefresh).Debug("Starting tweets fetch") params := url.Values{ - "count": []string{"100"}, + "count": []string{"1000"}, } lastTweet := tweetStore.GetLastTweetID() + log.WithField("last", lastTweet).Debug("Found last known tweet ID") if lastTweet > 0 && !forceRefresh { params.Set("since_id", strconv.FormatUint(lastTweet, 10)) @@ -100,4 +105,6 @@ func loadAndStoreTweets(forceRefresh bool) { log.WithError(err).Error("Unable to store tweets") return } + + log.Debug("Finished tweets fetch") } diff --git a/storage.go b/storage.go index 3ed1bb5..8956c72 100644 --- a/storage.go +++ b/storage.go @@ -1,6 +1,7 @@ package main import ( + "compress/gzip" "encoding/gob" "os" "sort" @@ -31,6 +32,7 @@ func newStore(location string) (*store, error) { return s, s.load() } +// DeleteTweetByID removes the tweet with mentioned ID from the store and issues a save when required func (s *store) DeleteTweetByID(id uint64) error { s.lock.Lock() defer s.lock.Unlock() @@ -57,6 +59,7 @@ func (s *store) DeleteTweetByID(id uint64) error { return s.save() } +// GetLastTweetID returns the newest known tweet ID (or 0 if none) func (s *store) GetLastTweetID() uint64 { s.lock.RLock() defer s.lock.RUnlock() @@ -68,7 +71,8 @@ func (s *store) GetLastTweetID() uint64 { return s.s[0].ID } -func (s *store) GetTweetPage(page int) ([]tweet, error) { +// GetTweetPage returns a paginated version of the store based on the page number (1..N) +func (s *store) GetTweetPage(page int) []tweet { s.lock.RLock() defer s.lock.RUnlock() @@ -78,17 +82,18 @@ func (s *store) GetTweetPage(page int) ([]tweet, error) { ) if start > len(s.s) { - return []tweet{}, nil + return []tweet{} } if start+num >= len(s.s) { num = len(s.s) - start } - return s.s[start:num], nil + return s.s[start:num] } -func (s *store) GetTweetsSince(since uint64) ([]tweet, error) { +// GetTweetsSince returns all tweets newer than the given tweet ID +func (s *store) GetTweetsSince(since uint64) []tweet { s.lock.RLock() defer s.lock.RUnlock() @@ -100,9 +105,10 @@ func (s *store) GetTweetsSince(since uint64) ([]tweet, error) { } } - return s.s[:i], nil + return s.s[:i] } +// StoreTweets performs an "upsert" for the given tweets (update known, add new) func (s *store) StoreTweets(tweets []tweet) error { s.lock.Lock() defer s.lock.Unlock() @@ -132,6 +138,7 @@ func (s *store) StoreTweets(tweets []tweet) error { return s.save() } +// load reads the file storage with the tweet database func (s *store) load() error { s.lock.Lock() defer s.lock.Unlock() @@ -151,9 +158,14 @@ func (s *store) load() error { } defer f.Close() + zf, err := gzip.NewReader(f) + if err != nil { + return errors.Wrap(err, "Unable to open gzip reader") + } + tmp := []tweet{} - if err := gob.NewDecoder(f).Decode(&tmp); err != nil { + if err := gob.NewDecoder(zf).Decode(&tmp); err != nil { return errors.Wrap(err, "Unable to decode storage file") } @@ -162,6 +174,7 @@ func (s *store) load() error { return nil } +// save writes the file storage with the tweet database func (s *store) save() error { // No need to lock here, has write-lock from s.StoreTweets @@ -171,5 +184,11 @@ func (s *store) save() error { } defer f.Close() - return errors.Wrap(gob.NewEncoder(f).Encode(s.s), "Unable to encode store") + zf, _ := gzip.NewWriterLevel(f, gzip.BestCompression) // #nosec G104: Ignore error as using a compression constant + defer func() { + zf.Flush() + zf.Close() + }() + + return errors.Wrap(gob.NewEncoder(zf).Encode(s.s), "Unable to encode store") }