Add more logging, cleanup code, add docs

Signed-off-by: Knut Ahlers <knut@ahlers.me>

http.go

@@ -12,61 +12,15 @@ import (
 )
 
 func init() {
-	http.HandleFunc("/api/page", handlePage)
-	http.HandleFunc("/api/since", handleNewest)
-	http.HandleFunc("/api/favourite", handleFavourite)
-	http.HandleFunc("/api/refresh", handleTweetRefresh)
+	http.HandleFunc("/api/favourite", handleFavorite)
 	http.HandleFunc("/api/force-reload", handleForceReload)
+	http.HandleFunc("/api/page", handlePage)
+	http.HandleFunc("/api/refresh", handleTweetRefresh)
+	http.HandleFunc("/api/since", handleNewest)
 }
 
-func handlePage(w http.ResponseWriter, r *http.Request) {
-	var page int = 1
-
-	if p, err := strconv.Atoi(r.URL.Query().Get("n")); err == nil && p > 0 {
-		page = p
-	}
-
-	tweets, err := tweetStore.GetTweetPage(page)
-	if err != nil {
-		log.WithError(err).Error("Unable to fetch tweets for page request")
-		http.Error(w, "Something went wrong", http.StatusInternalServerError)
-		return
-	}
-
-	tweetResponse(w, tweets)
-}
-
-func handleNewest(w http.ResponseWriter, r *http.Request) {
-	var since uint64 = 0
-
-	if s, err := strconv.ParseUint(r.URL.Query().Get("id"), 10, 64); err == nil && s > 0 {
-		since = s
-	}
-
-	if since == 0 {
-		http.Error(w, "Must specify last id", http.StatusBadRequest)
-		return
-	}
-
-	tweets, err := tweetStore.GetTweetsSince(since)
-	if err != nil {
-		log.WithError(err).Error("Unable to fetch tweets for newest request")
-		http.Error(w, "Something went wrong", http.StatusInternalServerError)
-		return
-	}
-
-	tweetResponse(w, tweets)
-}
-
-func tweetResponse(w http.ResponseWriter, tweets []tweet) {
-	w.Header().Set("Content-Type", "application/json")
-	w.Header().Set("Cache-Control", "no-cache")
-	w.WriteHeader(http.StatusOK)
-
-	json.NewEncoder(w).Encode(tweets)
-}
-
-func handleFavourite(w http.ResponseWriter, r *http.Request) {
+// handleFavorite takes the ID of a tweet and submits a favorite to Twitter
+func handleFavorite(w http.ResponseWriter, r *http.Request) {
 	req := struct {
 		ID int64 `json:"id,string"`
 	}{}
@@ -99,6 +53,46 @@ func handleFavourite(w http.ResponseWriter, r *http.Request) {
 	tweetResponse(w, tweets)
 }
 
+// handleForceReload issues a full load of the latest tweets to update their state
+func handleForceReload(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "This needs to be POST", http.StatusBadRequest)
+		return
+	}
+
+	go loadAndStoreTweets(true)
+
+	w.WriteHeader(http.StatusNoContent)
+}
+
+// handleNewest returns all tweets newly stored since the given tweet ID
+func handleNewest(w http.ResponseWriter, r *http.Request) {
+	var since uint64
+
+	if s, err := strconv.ParseUint(r.URL.Query().Get("id"), 10, 64); err == nil && s > 0 {
+		since = s
+	}
+
+	if since == 0 {
+		http.Error(w, "Must specify last id", http.StatusBadRequest)
+		return
+	}
+
+	tweetResponse(w, tweetStore.GetTweetsSince(since))
+}
+
+// handlePage loads older tweets with pagination
+func handlePage(w http.ResponseWriter, r *http.Request) {
+	var page = 1
+
+	if p, err := strconv.Atoi(r.URL.Query().Get("n")); err == nil && p > 0 {
+		page = p
+	}
+
+	tweetResponse(w, tweetStore.GetTweetPage(page))
+}
+
+// handleTweetRefresh refreshes the state of the tweet with the given ID against the Twitter API
 func handleTweetRefresh(w http.ResponseWriter, r *http.Request) {
 	req := struct {
 		ID int64 `json:"id,string"`
@@ -141,13 +135,11 @@ func handleTweetRefresh(w http.ResponseWriter, r *http.Request) {
 	tweetResponse(w, tweets)
 }
 
-func handleForceReload(w http.ResponseWriter, r *http.Request) {
-	if r.Method != http.MethodPost {
-		http.Error(w, "This needs to be POST", http.StatusBadRequest)
-		return
-	}
+// tweetResponse is a generic wrapper to return a list of tweets through JSON
+func tweetResponse(w http.ResponseWriter, tweets []tweet) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Cache-Control", "no-cache")
+	w.WriteHeader(http.StatusOK)
 
-	go loadAndStoreTweets(true)
-
-	w.WriteHeader(http.StatusNoContent)
+	json.NewEncoder(w).Encode(tweets)
 }

main.go

@@ -11,6 +11,7 @@ import (
 	"github.com/ChimeraCoder/anaconda"
 	log "github.com/sirupsen/logrus"
 
+	hhelp "github.com/Luzifer/go_helpers/v2/http"
 	"github.com/Luzifer/rconfig/v2"
 )
 
@@ -69,16 +70,20 @@ func main() {
 		}
 	}()
 
+	log.WithField("version", version).Info("MediaTimeline Viewer started")
+
 	http.Handle("/", http.FileServer(http.Dir(cfg.Frontend)))
-	http.ListenAndServe(cfg.Listen, nil)
+	http.ListenAndServe(cfg.Listen, hhelp.NewHTTPLogHandler(http.DefaultServeMux))
 }
 
 func loadAndStoreTweets(forceRefresh bool) {
+	log.WithField("force", forceRefresh).Debug("Starting tweets fetch")
 	params := url.Values{
-		"count": []string{"100"},
+		"count": []string{"1000"},
 	}
 
 	lastTweet := tweetStore.GetLastTweetID()
+	log.WithField("last", lastTweet).Debug("Found last known tweet ID")
 
 	if lastTweet > 0 && !forceRefresh {
 		params.Set("since_id", strconv.FormatUint(lastTweet, 10))
@@ -100,4 +105,6 @@ func loadAndStoreTweets(forceRefresh bool) {
 		log.WithError(err).Error("Unable to store tweets")
 		return
 	}
+
+	log.Debug("Finished tweets fetch")
 }

storage.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"compress/gzip"
 	"encoding/gob"
 	"os"
 	"sort"
@@ -31,6 +32,7 @@ func newStore(location string) (*store, error) {
 	return s, s.load()
 }
 
+// DeleteTweetByID removes the tweet with mentioned ID from the store and issues a save when required
 func (s *store) DeleteTweetByID(id uint64) error {
 	s.lock.Lock()
 	defer s.lock.Unlock()
@@ -57,6 +59,7 @@ func (s *store) DeleteTweetByID(id uint64) error {
 	return s.save()
 }
 
+// GetLastTweetID returns the newest known tweet ID (or 0 if none)
 func (s *store) GetLastTweetID() uint64 {
 	s.lock.RLock()
 	defer s.lock.RUnlock()
@@ -68,7 +71,8 @@ func (s *store) GetLastTweetID() uint64 {
 	return s.s[0].ID
 }
 
-func (s *store) GetTweetPage(page int) ([]tweet, error) {
+// GetTweetPage returns a paginated version of the store based on the page number (1..N)
+func (s *store) GetTweetPage(page int) []tweet {
 	s.lock.RLock()
 	defer s.lock.RUnlock()
 
@@ -78,17 +82,18 @@ func (s *store) GetTweetPage(page int) ([]tweet, error) {
 	)
 
 	if start > len(s.s) {
-		return []tweet{}, nil
+		return []tweet{}
 	}
 
 	if start+num >= len(s.s) {
 		num = len(s.s) - start
 	}
 
-	return s.s[start:num], nil
+	return s.s[start:num]
 }
 
-func (s *store) GetTweetsSince(since uint64) ([]tweet, error) {
+// GetTweetsSince returns all tweets newer than the given tweet ID
+func (s *store) GetTweetsSince(since uint64) []tweet {
 	s.lock.RLock()
 	defer s.lock.RUnlock()
 
@@ -100,9 +105,10 @@ func (s *store) GetTweetsSince(since uint64) ([]tweet, error) {
 		}
 	}
 
-	return s.s[:i], nil
+	return s.s[:i]
 }
 
+// StoreTweets performs an "upsert" for the given tweets (update known, add new)
 func (s *store) StoreTweets(tweets []tweet) error {
 	s.lock.Lock()
 	defer s.lock.Unlock()
@@ -132,6 +138,7 @@ func (s *store) StoreTweets(tweets []tweet) error {
 	return s.save()
 }
 
+// load reads the file storage with the tweet database
 func (s *store) load() error {
 	s.lock.Lock()
 	defer s.lock.Unlock()
@@ -151,9 +158,14 @@ func (s *store) load() error {
 	}
 	defer f.Close()
 
+	zf, err := gzip.NewReader(f)
+	if err != nil {
+		return errors.Wrap(err, "Unable to open gzip reader")
+	}
+
 	tmp := []tweet{}
 
-	if err := gob.NewDecoder(f).Decode(&tmp); err != nil {
+	if err := gob.NewDecoder(zf).Decode(&tmp); err != nil {
 		return errors.Wrap(err, "Unable to decode storage file")
 	}
 
@@ -162,6 +174,7 @@ func (s *store) load() error {
 	return nil
 }
 
+// save writes the file storage with the tweet database
 func (s *store) save() error {
 	// No need to lock here, has write-lock from s.StoreTweets
 
@@ -171,5 +184,11 @@ func (s *store) save() error {
 	}
 	defer f.Close()
 
-	return errors.Wrap(gob.NewEncoder(f).Encode(s.s), "Unable to encode store")
+	zf, _ := gzip.NewWriterLevel(f, gzip.BestCompression) // #nosec G104: Ignore error as using a compression constant
+	defer func() {
+		zf.Flush()
+		zf.Close()
+	}()
+
+	return errors.Wrap(gob.NewEncoder(zf).Encode(s.s), "Unable to encode store")
 }