luzifer/mediatimeline
1
0
Fork 0
mirror of https://github.com/Luzifer/mediatimeline.git synced 2024-09-19 16:02:57 +00:00
Code Issues Releases Wiki Activity

Add more logging, cleanup code, add docs

Browse Source 
Signed-off-by: Knut Ahlers <knut@ahlers.me>
This commit is contained in:
Knut Ahlers 2019-04-24 20:53:34 +02:00
parent 6f5eb97b63
commit 83030ad9d4
Signed by: luzifer
GPG Key ID: DC2729FDD34BE99E
3 changed files with 87 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
http.go
View File

@ -12,61 +12,15 @@ import (
)
func init() {
http.HandleFunc("/api/page", handlePage)
http.HandleFunc("/api/since", handleNewest)
http.HandleFunc("/api/favourite", handleFavourite)
http.HandleFunc("/api/refresh", handleTweetRefresh)
http.HandleFunc("/api/favourite", handleFavorite)
http.HandleFunc("/api/force-reload", handleForceReload)
http.HandleFunc("/api/page", handlePage)
http.HandleFunc("/api/refresh", handleTweetRefresh)
http.HandleFunc("/api/since", handleNewest)
}
func handlePage(w http.ResponseWriter, r *http.Request) {
var page int = 1
if p, err := strconv.Atoi(r.URL.Query().Get("n")); err == nil && p > 0 {
page = p
}
tweets, err := tweetStore.GetTweetPage(page)
if err != nil {
log.WithError(err).Error("Unable to fetch tweets for page request")
http.Error(w, "Something went wrong", http.StatusInternalServerError)
return
}
tweetResponse(w, tweets)
}
func handleNewest(w http.ResponseWriter, r *http.Request) {
var since uint64 = 0
if s, err := strconv.ParseUint(r.URL.Query().Get("id"), 10, 64); err == nil && s > 0 {
since = s
}
if since == 0 {
http.Error(w, "Must specify last id", http.StatusBadRequest)
return
}
tweets, err := tweetStore.GetTweetsSince(since)
if err != nil {
log.WithError(err).Error("Unable to fetch tweets for newest request")
http.Error(w, "Something went wrong", http.StatusInternalServerError)
return
}
tweetResponse(w, tweets)
}
func tweetResponse(w http.ResponseWriter, tweets []tweet) {
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "no-cache")
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(tweets)
}
func handleFavourite(w http.ResponseWriter, r *http.Request) {
// handleFavorite takes the ID of a tweet and submits a favorite to Twitter
func handleFavorite(w http.ResponseWriter, r *http.Request) {
req := struct {
ID int64 `json:"id,string"`
}{}
@ -99,6 +53,46 @@ func handleFavourite(w http.ResponseWriter, r *http.Request) {
tweetResponse(w, tweets)
}
// handleForceReload issues a full load of the latest tweets to update their state
func handleForceReload(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "This needs to be POST", http.StatusBadRequest)
return
}
go loadAndStoreTweets(true)
w.WriteHeader(http.StatusNoContent)
}
// handleNewest returns all tweets newly stored since the given tweet ID
func handleNewest(w http.ResponseWriter, r *http.Request) {
var since uint64
if s, err := strconv.ParseUint(r.URL.Query().Get("id"), 10, 64); err == nil && s > 0 {
since = s
}
if since == 0 {
http.Error(w, "Must specify last id", http.StatusBadRequest)
return
}
tweetResponse(w, tweetStore.GetTweetsSince(since))
}
// handlePage loads older tweets with pagination
func handlePage(w http.ResponseWriter, r *http.Request) {
var page = 1
if p, err := strconv.Atoi(r.URL.Query().Get("n")); err == nil && p > 0 {
page = p
}
tweetResponse(w, tweetStore.GetTweetPage(page))
}
// handleTweetRefresh refreshes the state of the tweet with the given ID against the Twitter API
func handleTweetRefresh(w http.ResponseWriter, r *http.Request) {
req := struct {
ID int64 `json:"id,string"`
@ -141,13 +135,11 @@ func handleTweetRefresh(w http.ResponseWriter, r *http.Request) {
tweetResponse(w, tweets)
}
func handleForceReload(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "This needs to be POST", http.StatusBadRequest)
return
}
// tweetResponse is a generic wrapper to return a list of tweets through JSON
func tweetResponse(w http.ResponseWriter, tweets []tweet) {
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "no-cache")
w.WriteHeader(http.StatusOK)
go loadAndStoreTweets(true)
w.WriteHeader(http.StatusNoContent)
json.NewEncoder(w).Encode(tweets)
}

11
main.go
View File

@ -11,6 +11,7 @@ import (
"github.com/ChimeraCoder/anaconda"
log "github.com/sirupsen/logrus"
hhelp "github.com/Luzifer/go_helpers/v2/http"
"github.com/Luzifer/rconfig/v2"
)
@ -69,16 +70,20 @@ func main() {
}
}()
log.WithField("version", version).Info("MediaTimeline Viewer started")
http.Handle("/", http.FileServer(http.Dir(cfg.Frontend)))
http.ListenAndServe(cfg.Listen, nil)
http.ListenAndServe(cfg.Listen, hhelp.NewHTTPLogHandler(http.DefaultServeMux))
}
func loadAndStoreTweets(forceRefresh bool) {
log.WithField("force", forceRefresh).Debug("Starting tweets fetch")
params := url.Values{
"count": []string{"100"},
"count": []string{"1000"},
}
lastTweet := tweetStore.GetLastTweetID()
log.WithField("last", lastTweet).Debug("Found last known tweet ID")
if lastTweet > 0 && !forceRefresh {
params.Set("since_id", strconv.FormatUint(lastTweet, 10))
@ -100,4 +105,6 @@ func loadAndStoreTweets(forceRefresh bool) {
log.WithError(err).Error("Unable to store tweets")
return
}
log.Debug("Finished tweets fetch")
}

33
storage.go
View File

@ -1,6 +1,7 @@
package main
import (
"compress/gzip"
"encoding/gob"
"os"
"sort"
@ -31,6 +32,7 @@ func newStore(location string) (*store, error) {
return s, s.load()
}
// DeleteTweetByID removes the tweet with mentioned ID from the store and issues a save when required
func (s *store) DeleteTweetByID(id uint64) error {
s.lock.Lock()
defer s.lock.Unlock()
@ -57,6 +59,7 @@ func (s *store) DeleteTweetByID(id uint64) error {
return s.save()
}
// GetLastTweetID returns the newest known tweet ID (or 0 if none)
func (s *store) GetLastTweetID() uint64 {
s.lock.RLock()
defer s.lock.RUnlock()
@ -68,7 +71,8 @@ func (s *store) GetLastTweetID() uint64 {
return s.s[0].ID
}
func (s *store) GetTweetPage(page int) ([]tweet, error) {
// GetTweetPage returns a paginated version of the store based on the page number (1..N)
func (s *store) GetTweetPage(page int) []tweet {
s.lock.RLock()
defer s.lock.RUnlock()
@ -78,17 +82,18 @@ func (s *store) GetTweetPage(page int) ([]tweet, error) {
)
if start > len(s.s) {
return []tweet{}, nil
return []tweet{}
}
if start+num >= len(s.s) {
num = len(s.s) - start
}
return s.s[start:num], nil
return s.s[start:num]
}
func (s *store) GetTweetsSince(since uint64) ([]tweet, error) {
// GetTweetsSince returns all tweets newer than the given tweet ID
func (s *store) GetTweetsSince(since uint64) []tweet {
s.lock.RLock()
defer s.lock.RUnlock()
@ -100,9 +105,10 @@ func (s *store) GetTweetsSince(since uint64) ([]tweet, error) {
}
}
return s.s[:i], nil
return s.s[:i]
}
// StoreTweets performs an "upsert" for the given tweets (update known, add new)
func (s *store) StoreTweets(tweets []tweet) error {
s.lock.Lock()
defer s.lock.Unlock()
@ -132,6 +138,7 @@ func (s *store) StoreTweets(tweets []tweet) error {
return s.save()
}
// load reads the file storage with the tweet database
func (s *store) load() error {
s.lock.Lock()
defer s.lock.Unlock()
@ -151,9 +158,14 @@ func (s *store) load() error {
}
defer f.Close()
zf, err := gzip.NewReader(f)
if err != nil {
return errors.Wrap(err, "Unable to open gzip reader")
}
tmp := []tweet{}
if err := gob.NewDecoder(f).Decode(&tmp); err != nil {
if err := gob.NewDecoder(zf).Decode(&tmp); err != nil {
return errors.Wrap(err, "Unable to decode storage file")
}
@ -162,6 +174,7 @@ func (s *store) load() error {
return nil
}
// save writes the file storage with the tweet database
func (s *store) save() error {
// No need to lock here, has write-lock from s.StoreTweets
@ -171,5 +184,11 @@ func (s *store) save() error {
}
defer f.Close()
return errors.Wrap(gob.NewEncoder(f).Encode(s.s), "Unable to encode store")
zf, _ := gzip.NewWriterLevel(f, gzip.BestCompression) // #nosec G104: Ignore error as using a compression constant
defer func() {
zf.Flush()
zf.Close()
}()
return errors.Wrap(gob.NewEncoder(zf).Encode(s.s), "Unable to encode store")
}