Move from individual check intervals to fixed check distribution

Signed-off-by: Knut Ahlers <knut@ahlers.me>

.gitignore

@@ -1,3 +1,4 @@
 config.yaml
 .env
+go-latestver
 latestver.db

internal/database/store.go

@@ -20,8 +20,6 @@ type (
 		Fetcher       string                          `json:"-" yaml:"fetcher"`
 		FetcherConfig fieldcollection.FieldCollection `json:"-" yaml:"fetcher_config"`
 
-		CheckInterval time.Duration `json:"-" yaml:"check_interval"`
-
 		Links []CatalogLink `json:"links" yaml:"links"`
 	}
 

main.go

@@ -18,20 +18,22 @@ import (
 
 var (
 	cfg = struct {
-		BaseURL        string        `flag:"base-url" default:"https://example.com/" description:"Base-URL the application is reachable at"`
+		BaseURL           string        `flag:"base-url" default:"https://example.com/" description:"Base-URL the application is reachable at"`
-		Config         string        `flag:"config,c" default:"config.yaml" description:"Configuration file with catalog entries"`
+		Config            string        `flag:"config,c" default:"config.yaml" description:"Configuration file with catalog entries"`
-		Listen         string        `flag:"listen" default:":3000" description:"Port/IP to listen on"`
+		Listen            string        `flag:"listen" default:":3000" description:"Port/IP to listen on"`
-		LogLevel       string        `flag:"log-level" default:"info" description:"Log level (debug, info, warn, error, fatal)"`
+		LogLevel          string        `flag:"log-level" default:"info" description:"Log level (debug, info, warn, error, fatal)"`
-		MaxJitter      time.Duration `flag:"max-jitter" default:"30m" description:"Maximum jitter to add to the check interval for load balancing"`
+		CheckDistribution time.Duration `flag:"check-distribution" default:"1h" description:"Checks are executed at static times every [value]"`
-		Storage        string        `flag:"storage" default:"sqlite" description:"Storage adapter to use (mysql, sqlite)"`
+		Storage           string        `flag:"storage" default:"sqlite" description:"Storage adapter to use (mysql, sqlite)"`
-		StorageDSN     string        `flag:"storage-dsn" default:"file::memory:?cache=shared" description:"DSN to connect to the database"`
+		StorageDSN        string        `flag:"storage-dsn" default:"file::memory:?cache=shared" description:"DSN to connect to the database"`
-		VersionAndExit bool          `flag:"version" default:"false" description:"Prints current version and exits"`
+		VersionAndExit    bool          `flag:"version" default:"false" description:"Prints current version and exits"`
 	}{}
 
 	configFile = config.New()
 	router     *mux.Router
 	storage    *database.Client
 
+	processStart = time.Now()
+
 	version = "dev"
 )
 

scheduler.go

@@ -2,6 +2,9 @@ package main
 
 import (
 	"context"
+	"crypto/md5"
+	"fmt"
+	"math"
 	"strings"
 	"time"
 
@@ -36,23 +39,13 @@ func checkForUpdates(ce *database.CatalogEntry) error {
 		return errors.Wrap(err, "getting catalog meta")
 	}
 
-	checkTime := time.Now()
+	nct := nextCheckTime(ce, cm.LastChecked)
-	if cm.LastChecked != nil {
+	logger = logger.WithFields(log.Fields{
-		checkTime = *cm.LastChecked
+		"last": cm.LastChecked,
-
+		"next": nct,
-		switch {
+	})
-		case ce.CheckInterval > 0:
+	logger.Trace("Next check time found")
-			checkTime = checkTime.Add(ce.CheckInterval)
+	if nct.After(time.Now()) {
-
-		case configFile.CheckInterval > 0:
-			checkTime = checkTime.Add(configFile.CheckInterval)
-
-		default:
-			checkTime = checkTime.Add(time.Hour)
-		}
-	}
-
-	if checkTime.After(time.Now()) {
 		// Not yet ready to check
 		return nil
 	}
@@ -85,7 +78,7 @@ func checkForUpdates(ce *database.CatalogEntry) error {
 			return errors.Wrap(err, "adding log entry")
 		}
 
-		cm.VersionTime = func(v time.Time) *time.Time { return &v }(vertime)
+		cm.VersionTime = ptrTime(vertime)
 		cm.CurrentVersion = ver
 		fallthrough
 
@@ -94,6 +87,32 @@ func checkForUpdates(ce *database.CatalogEntry) error {
 
 	}
 
-	cm.LastChecked = func(v time.Time) *time.Time { return &v }(time.Now())
+	cm.LastChecked = ptrTime(time.Now())
 	return errors.Wrap(storage.Catalog.PutMeta(cm), "updating meta entry")
 }
+
+func nextCheckTime(ce *database.CatalogEntry, lastCheck *time.Time) time.Time {
+	hash := md5.New()
+	fmt.Fprint(hash, ce.Key())
+
+	var jitter int64
+	for i, c := range hash.Sum(nil) {
+		jitter += int64(c) * int64(math.Pow(10, float64(i)))
+	}
+
+	if lastCheck == nil {
+		lastCheck = ptrTime(processStart)
+	}
+
+	next := lastCheck.
+		Truncate(cfg.CheckDistribution).
+		Add(time.Duration(jitter) % cfg.CheckDistribution)
+
+	if next.Before(*lastCheck) {
+		next = next.Add(cfg.CheckDistribution)
+	}
+
+	return next.Truncate(time.Second)
+}
+
+func ptrTime(t time.Time) *time.Time { return &t }