From bf68f32c0e91bf47a98016e343654eb292ed9d5d Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Tue, 23 Apr 2019 15:54:42 +0200 Subject: [PATCH] Initial implementation --- .gitignore | 4 + frontend/index.html | 260 ++++++++++++++++++++++++++++++++++++++++++++ http.go | 131 ++++++++++++++++++++++ main.go | 101 +++++++++++++++++ storage.go | 149 +++++++++++++++++++++++++ tweet.go | 88 +++++++++++++++ 6 files changed, 733 insertions(+) create mode 100644 .gitignore create mode 100644 frontend/index.html create mode 100644 http.go create mode 100644 main.go create mode 100644 storage.go create mode 100644 tweet.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..549d4d7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.env +mediatimeline +runwrap.sh +tweets.db diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 0000000..727ef6a --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,260 @@ + + + + + + + + MediaTimeline + + + + + + + + + + + +
+ +
+ +
+
+
+
+ +
+ +
+
{{ tweet.user.screen_name }}
+ {{ moment(tweet.posted).format('lll') }} +
+
+ +
+
+ + + + + + +
+
+
+ +
+ + + +
+ + + + + + + + + + + + + + + + + diff --git a/http.go b/http.go new file mode 100644 index 0000000..abffd5f --- /dev/null +++ b/http.go @@ -0,0 +1,131 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/url" + "strconv" + + "github.com/ChimeraCoder/anaconda" + log "github.com/sirupsen/logrus" +) + +func init() { + http.HandleFunc("/api/page", handlePage) + http.HandleFunc("/api/since", handleNewest) + http.HandleFunc("/api/favourite", handleFavourite) + http.HandleFunc("/api/refresh", handleTweetRefresh) +} + +func handlePage(w http.ResponseWriter, r *http.Request) { + var page int = 1 + + if p, err := strconv.Atoi(r.URL.Query().Get("n")); err == nil && p > 0 { + page = p + } + + tweets, err := tweetStore.GetTweetPage(page) + if err != nil { + log.WithError(err).Error("Unable to fetch tweets for page request") + http.Error(w, "Something went wrong", http.StatusInternalServerError) + return + } + + tweetResponse(w, tweets) +} + +func handleNewest(w http.ResponseWriter, r *http.Request) { + var since uint64 = 0 + + if s, err := strconv.ParseUint(r.URL.Query().Get("id"), 10, 64); err == nil && s > 0 { + since = s + } + + if since == 0 { + http.Error(w, "Must specify last id", http.StatusBadRequest) + return + } + + tweets, err := tweetStore.GetTweetsSince(since) + if err != nil { + log.WithError(err).Error("Unable to fetch tweets for newest request") + http.Error(w, "Something went wrong", http.StatusInternalServerError) + return + } + + tweetResponse(w, tweets) +} + +func tweetResponse(w http.ResponseWriter, tweets []tweet) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-cache") + w.WriteHeader(http.StatusOK) + + json.NewEncoder(w).Encode(tweets) +} + +func handleFavourite(w http.ResponseWriter, r *http.Request) { + req := struct { + ID int64 `json:"id,string"` + }{} + + if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.ID == 0 { + http.Error(w, "Need to specify id", http.StatusBadRequest) + return + } + + tweet, err := twitter.Favorite(req.ID) + if err != nil { + log.WithError(err).Error("Unable to favourite tweet") + http.Error(w, "Something went wrong", http.StatusInternalServerError) + return + } + + tweets, err := convertTweetList([]anaconda.Tweet{tweet}, true) + if err != nil { + log.WithError(err).Error("Unable to convert tweet for storing") + http.Error(w, "Something went wrong", http.StatusInternalServerError) + return + } + + if err = tweetStore.StoreTweets(tweets); err != nil { + log.WithError(err).Error("Unable to update tweet") + http.Error(w, "Something went wrong", http.StatusInternalServerError) + return + } + + tweetResponse(w, tweets) +} + +func handleTweetRefresh(w http.ResponseWriter, r *http.Request) { + req := struct { + ID int64 `json:"id,string"` + }{} + + if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.ID == 0 { + http.Error(w, "Need to specify id", http.StatusBadRequest) + return + } + + tweet, err := twitter.GetTweet(req.ID, url.Values{}) + if err != nil { + log.WithError(err).Error("Unable to fetch tweet") + http.Error(w, "Something went wrong", http.StatusInternalServerError) + return + } + + tweets, err := convertTweetList([]anaconda.Tweet{tweet}, true) + if err != nil { + log.WithError(err).Error("Unable to convert tweet for storing") + http.Error(w, "Something went wrong", http.StatusInternalServerError) + return + } + + if err = tweetStore.StoreTweets(tweets); err != nil { + log.WithError(err).Error("Unable to update tweet") + http.Error(w, "Something went wrong", http.StatusInternalServerError) + return + } + + tweetResponse(w, tweets) +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..c9dd674 --- /dev/null +++ b/main.go @@ -0,0 +1,101 @@ +package main + +import ( + "fmt" + "net/http" + "net/url" + "os" + "strconv" + "time" + + "github.com/ChimeraCoder/anaconda" + log "github.com/sirupsen/logrus" + + "github.com/Luzifer/rconfig" +) + +var ( + cfg = struct { + AppToken string `flag:"app-token" description:"Token for the application" validate:"nonzero"` + AppSecret string `flag:"app-secret" description:"Secret for the provided token" validate:"nonzero"` + Database string `flag:"database" default:"tweets.db" description:"Database storage location"` + Frontend string `flag:"frontend" default:"frontend" description:"Directory containing frontend files"` + Listen string `flag:"listen" default:":3000" description:"Port/IP to listen on"` + ListOwner string `flag:"list-owner" description:"Owner of the specified list" validate:"nonzero"` + ListSlug string `flag:"list-slug" description:"Slug of the list" validate:"nonzero"` + LogLevel string `flag:"log-level" default:"info" description:"Log level (debug, info, warn, error, fatal)"` + UserToken string `flag:"user-token" description:"Token for the user" validate:"nonzero"` + UserSecret string `flag:"user-secret" description:"Secret for the provided token" validate:"nonzero"` + VersionAndExit bool `flag:"version" default:"false" description:"Prints current version and exits"` + }{} + + tweetStore *store + twitter *anaconda.TwitterApi + + version = "dev" +) + +func init() { + rconfig.AutoEnv(true) + if err := rconfig.ParseAndValidate(&cfg); err != nil { + log.Fatalf("Unable to parse commandline options: %s", err) + } + + if cfg.VersionAndExit { + fmt.Printf("mediatimeline %s\n", version) + os.Exit(0) + } + + if l, err := log.ParseLevel(cfg.LogLevel); err != nil { + log.WithError(err).Fatal("Unable to parse log level") + } else { + log.SetLevel(l) + } +} + +func main() { + var err error + if tweetStore, err = newStore(cfg.Database); err != nil { + log.WithError(err).Fatal("Unable to create store") + } + + twitter = anaconda.NewTwitterApiWithCredentials( + cfg.UserToken, cfg.UserSecret, + cfg.AppToken, cfg.AppSecret, + ) + go loadAndStoreTweets() + + http.Handle("/", http.FileServer(http.Dir(cfg.Frontend))) + http.ListenAndServe(cfg.Listen, nil) +} + +func loadAndStoreTweets() { + for t := time.NewTicker(time.Minute); true; <-t.C { + params := url.Values{ + "count": []string{"100"}, + } + + lastTweet := tweetStore.GetLastTweetID() + + if lastTweet > 0 { + params.Set("since_id", strconv.FormatUint(lastTweet, 10)) + } + + anacondaTweets, err := twitter.GetListTweetsBySlug(cfg.ListSlug, cfg.ListOwner, false, params) + if err != nil { + log.WithError(err).Error("Unable to fetch tweets") + continue + } + + tweets, err := convertTweetList(anacondaTweets, true) + if err != nil { + log.WithError(err).Error("Unable to parse tweets") + continue + } + + if err := tweetStore.StoreTweets(tweets); err != nil { + log.WithError(err).Error("Unable to store tweets") + continue + } + } +} diff --git a/storage.go b/storage.go new file mode 100644 index 0000000..c573f14 --- /dev/null +++ b/storage.go @@ -0,0 +1,149 @@ +package main + +import ( + "encoding/gob" + "os" + "sort" + "sync" + + "github.com/pkg/errors" +) + +const tweetPageSize = 50 + +type store struct { + s []tweet + location string + + lock sync.RWMutex +} + +func init() { + gob.Register(tweet{}) +} + +func newStore(location string) (*store, error) { + s := &store{ + s: []tweet{}, + location: location, + } + + return s, s.load() +} + +func (s *store) StoreTweets(tweets []tweet) error { + s.lock.Lock() + defer s.lock.Unlock() + + tmp := s.s + + for _, t := range tweets { + var stored bool + + for i := 0; i < len(tmp); i++ { + if tmp[i].ID == t.ID { + tmp[i] = t + stored = true + break + } + } + + if !stored { + tmp = append(tmp, t) + } + } + + sort.Slice(tmp, func(j, i int) bool { return tmp[i].ID < tmp[j].ID }) + + s.s = tmp + + return s.save() +} + +func (s *store) GetLastTweetID() uint64 { + s.lock.RLock() + defer s.lock.RUnlock() + + if len(s.s) == 0 { + return 0 + } + + return s.s[0].ID +} + +func (s *store) GetTweetPage(page int) ([]tweet, error) { + s.lock.RLock() + defer s.lock.RUnlock() + + var ( + start = (page - 1) * tweetPageSize + num = tweetPageSize + ) + + if start > len(s.s) { + return []tweet{}, nil + } + + if start+num >= len(s.s) { + num = len(s.s) - start + } + + return s.s[start:num], nil +} + +func (s *store) GetTweetsSince(since uint64) ([]tweet, error) { + s.lock.RLock() + defer s.lock.RUnlock() + + var i int + + for i = 0; i < len(s.s); i++ { + if s.s[i].ID <= since { + break + } + } + + return s.s[:i], nil +} + +func (s *store) load() error { + s.lock.Lock() + defer s.lock.Unlock() + + if _, err := os.Stat(s.location); err != nil { + if os.IsNotExist(err) { + // Leave a fresh store + return nil + } + + return errors.Wrap(err, "Unable to stat storage file") + } + + f, err := os.Open(s.location) + if err != nil { + return errors.Wrap(err, "Unable to open storage file") + } + defer f.Close() + + tmp := []tweet{} + + if err := gob.NewDecoder(f).Decode(&tmp); err != nil { + return errors.Wrap(err, "Unable to decode storage file") + } + + s.s = tmp + + return nil +} + +func (s *store) save() error { + // No need to lock here, has write-lock from s.StoreTweets + + f, err := os.Create(s.location) + if err != nil { + return errors.Wrap(err, "Unable to open store for writing") + } + defer f.Close() + + return errors.Wrap(gob.NewEncoder(f).Encode(s.s), "Unable to encode store") +} diff --git a/tweet.go b/tweet.go new file mode 100644 index 0000000..fc2a5e0 --- /dev/null +++ b/tweet.go @@ -0,0 +1,88 @@ +package main + +import ( + "strconv" + "time" + + "github.com/ChimeraCoder/anaconda" + "github.com/pkg/errors" +) + +func convertTweetList(in []anaconda.Tweet, filterForMedia bool) ([]tweet, error) { + out := []tweet{} + + for _, t := range in { + tw, err := tweetFromAnaconda(t) + if err != nil { + return nil, errors.Wrap(err, "Unable to parse tweet") + } + + if len(tw.Images) == 0 && filterForMedia { + continue + } + out = append(out, tw) + } + + return out, nil +} + +type tweet struct { + Favorited bool `json:"favorited"` + ID uint64 `json:"id,string"` + Images []media `json:"images"` + Posted time.Time `json:"posted"` + User user `json:"user"` + Text string `json:"text"` +} + +func tweetFromAnaconda(in anaconda.Tweet) (tweet, error) { + medias := []media{} + for _, m := range in.ExtendedEntities.Media { + if m.Type != "photo" { + continue + } + + medias = append(medias, mediaFromAnaconda(m)) + } + + created, _ := in.CreatedAtTime() + id, err := strconv.ParseUint(in.IdStr, 10, 64) + if err != nil { + return tweet{}, errors.Wrap(err, "Unable to parse tweet ID") + } + + return tweet{ + Favorited: in.Favorited, + ID: id, + Images: medias, + Posted: created, + User: userFromAnaconda(in.User), + Text: in.Text, + }, nil +} + +type user struct { + ID int64 `json:"id"` + ScreenName string `json:"screen_name"` + Image string `json:"image"` +} + +func userFromAnaconda(in anaconda.User) user { + return user{ + ID: in.Id, + ScreenName: in.ScreenName, + Image: in.ProfileImageUrlHttps, + } +} + +type media struct { + ID int64 `json:"id"` + Image string `json:"image"` +} + +func mediaFromAnaconda(in anaconda.EntityMedia) media { + return media{ + ID: in.Id, + Image: in.Media_url_https, + } +}