Implement file.Watcher

Signed-off-by: Knut Ahlers <knut@ahlers.me>

file/watcher.go

@@ -0,0 +1,141 @@
+package file
+
+import (
+	"crypto/sha256"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+type (
+	// Watcher creates a background routine and emits events when the
+	// watched file changes on its C channel. If an error occurs the
+	// loop is stopped and the error is exposed on the Err property.
+	Watcher struct {
+		C             <-chan WatcherEvent
+		CheckInterval time.Duration
+		Err           error
+		FilePath      string
+
+		c          chan WatcherEvent
+		checks     []WatcherCheck
+		lock       sync.RWMutex
+		stateCache map[string]any
+	}
+
+	// WatcherCheck is an interface to implement own checks
+	WatcherCheck func(*Watcher) (WatcherEvent, error)
+
+	// WatcherEvent is the detected change to be signeld through the
+	// channel within the Watcher
+	WatcherEvent uint
+)
+
+const (
+	WatcherEventInvalid WatcherEvent = iota
+	WatcherEventNoChange
+	WatcherEventFileAppeared
+	WatcherEventFileModified
+	WatcherEventFileVanished
+)
+
+// NewCryptographicWatcher is a wrapper around NewWatcher to configure
+// the Watcher with presence and sha256 hash checks.
+func NewCryptographicWatcher(filePath string, interval time.Duration) (*Watcher, error) {
+	return NewWatcher(filePath, interval, WatcherCheckPresence, WatcherCheckHash(sha256.New))
+}
+
+// NewSimpleWatcher is a wrapper around NewWatcher to configure the
+// Watcher with presence, size and mtime checks.
+func NewSimpleWatcher(filePath string, interval time.Duration) (*Watcher, error) {
+	return NewWatcher(filePath, interval, WatcherCheckPresence, WatcherCheckSize, WatcherCheckMtime)
+}
+
+// NewWatcher creates a new Watcher configured with the given filePath,
+// interval and checks given. The checks are executed once during
+// initialization and will not cause an event to be sent. The created
+// Watcher will automatically start its periodic check and the C
+// channel should immediately be watched for changes. If the channel
+// is not listened on the check loop will be paused until events are
+// retrieved. If during the initial checks an error is detected the
+// loop is NOT started and the watcher needs to be initialized again.
+func NewWatcher(filePath string, interval time.Duration, checks ...WatcherCheck) (*Watcher, error) {
+	w, err := newWatcher(filePath, interval, checks...)
+
+	if err == nil {
+		go w.loop()
+	}
+
+	return w, err
+}
+
+func newWatcher(filePath string, interval time.Duration, checks ...WatcherCheck) (*Watcher, error) {
+	notify := make(chan WatcherEvent, 1)
+
+	w := &Watcher{
+		C:             notify,
+		CheckInterval: interval,
+		FilePath:      filePath,
+
+		c:          notify,
+		checks:     checks,
+		stateCache: make(map[string]any),
+	}
+	// Initially run checks once
+	_, err := w.runStateChecks(true)
+
+	return w, errors.Wrap(err, "executing initial checks")
+}
+
+// GetState is a helper to retrieve state from the internal store for
+// usage in checks to have their state retained.
+func (w *Watcher) GetState(key string) any {
+	w.lock.RLock()
+	defer w.lock.RUnlock()
+
+	return w.stateCache[key]
+}
+
+// SetState is a helper to set state into the internal store for
+// usage in checks to have their state retained.
+func (w *Watcher) SetState(key string, value any) {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+
+	w.stateCache[key] = value
+}
+
+func (w *Watcher) loop() {
+	for {
+		evt, err := w.runStateChecks(false)
+		if err != nil {
+			w.Err = err
+			break
+		}
+
+		if evt != WatcherEventNoChange && evt != WatcherEventInvalid {
+			// On "no change" and "invalid" events sending the new event is skipped
+			w.c <- evt
+		}
+
+		time.Sleep(w.CheckInterval)
+	}
+}
+
+func (w *Watcher) runStateChecks(runAll bool) (WatcherEvent, error) {
+	for _, c := range w.checks {
+		evt, err := c(w)
+		if err != nil {
+			return WatcherEventInvalid, errors.Wrap(err, "checking file state")
+		}
+
+		if evt == WatcherEventNoChange && !runAll {
+			continue
+		}
+
+		return evt, nil
+	}
+
+	return WatcherEventNoChange, nil
+}

file/watcher_checks.go

@@ -0,0 +1,145 @@
+package file
+
+import (
+	"crypto/sha512"
+	"fmt"
+	"hash"
+	"io"
+	"io/fs"
+	"os"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	keyWatcherCheckHash     = "WatcherCheckHash"
+	keyWatcherCheckMtime    = "WatcherCheckMtime"
+	keyWatcherCheckPresence = "WatcherCheckPresence"
+	keyWatcherCheckSize     = "WatcherCheckSize"
+)
+
+// WatcherCheckHash returns a WatcherCheck configured with the given
+// hash method (i.e. provide md5.New, sha1.New, ...). If the file is
+// not present at the time of the check the check is skipped and will
+// NOT cause an error.
+func WatcherCheckHash(hcf func() hash.Hash) WatcherCheck {
+	return func(w *Watcher) (WatcherEvent, error) {
+		var lastHash string
+		if v, ok := w.GetState(keyWatcherCheckHash).(string); ok {
+			lastHash = v
+		}
+
+		if _, err := os.Stat(w.FilePath); errors.Is(err, fs.ErrNotExist) {
+			return WatcherEventInvalid, nil
+		}
+
+		f, err := os.Open(w.FilePath)
+		if err != nil {
+			return WatcherEventInvalid, errors.Wrap(err, "opening file")
+		}
+		defer f.Close()
+
+		h := hcf()
+		if _, err = io.Copy(h, f); err != nil {
+			return WatcherEventInvalid, errors.Wrap(err, "reading file")
+		}
+
+		currentHash := fmt.Sprintf("%x", h.Sum(nil))
+		if lastHash == currentHash {
+			return WatcherEventNoChange, nil
+		}
+
+		w.SetState(keyWatcherCheckHash, currentHash)
+		return WatcherEventFileModified, nil
+	}
+}
+
+var _ WatcherCheck = WatcherCheckHash(sha512.New)
+
+// WatcherCheckMtime checks whether the mtime attribute of the file
+// has changed. If the file is not present at the time of the check
+// the check is skipped and will NOT cause an error.
+func WatcherCheckMtime(w *Watcher) (WatcherEvent, error) {
+	var lastChange int64
+	if v, ok := w.GetState(keyWatcherCheckMtime).(int64); ok {
+		lastChange = v
+	}
+
+	s, err := os.Stat(w.FilePath)
+	switch {
+	case err == nil:
+		// handle size change
+	case errors.Is(err, fs.ErrNotExist):
+		return WatcherEventInvalid, nil
+	default:
+		return WatcherEventInvalid, errors.Wrap(err, "getting file stat")
+	}
+
+	if s.ModTime().UnixNano() == lastChange {
+		return WatcherEventNoChange, nil
+	}
+
+	w.SetState(keyWatcherCheckMtime, s.ModTime().UnixNano())
+	return WatcherEventFileModified, nil
+}
+
+var _ WatcherCheck = WatcherCheckMtime
+
+// WatcherCheckPresence simply checks whether the file is present and
+// allows to emit WatcherEventFileAppeared / WatcherEventFileVanished
+// events when the file existence state changes.
+func WatcherCheckPresence(w *Watcher) (WatcherEvent, error) {
+	var wasPresent bool
+	if v, ok := w.GetState(keyWatcherCheckPresence).(bool); ok {
+		wasPresent = v
+	}
+
+	_, err := os.Stat(w.FilePath)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		// Some weird error occurred
+		return WatcherEventInvalid, errors.Wrap(err, "getting file stat")
+	}
+
+	isPresent := err == nil
+	w.SetState(keyWatcherCheckPresence, isPresent)
+
+	switch {
+	case !wasPresent && isPresent:
+		return WatcherEventFileAppeared, nil
+	case wasPresent && !isPresent:
+		return WatcherEventFileVanished, nil
+	default:
+		return WatcherEventNoChange, nil
+	}
+}
+
+var _ WatcherCheck = WatcherCheckPresence
+
+// WatcherCheckSize checks whether the size of the file has changed.
+// If the file is not present at the time of the check the check is
+// skipped and will NOT cause an error.
+func WatcherCheckSize(w *Watcher) (WatcherEvent, error) {
+	var knownSize int64
+	if v, ok := w.GetState(keyWatcherCheckSize).(int64); ok {
+		knownSize = v
+	}
+
+	s, err := os.Stat(w.FilePath)
+	switch {
+	case err == nil:
+		// handle size change
+	case errors.Is(err, fs.ErrNotExist):
+		return WatcherEventInvalid, nil
+	default:
+		return WatcherEventInvalid, errors.Wrap(err, "getting file stat")
+	}
+
+	if s.Size() == knownSize {
+		return WatcherEventNoChange, nil
+	}
+
+	w.SetState(keyWatcherCheckSize, s.Size())
+	return WatcherEventFileModified, nil
+}
+
+var _ WatcherCheck = WatcherCheckSize

file/watcher_test.go

@@ -0,0 +1,209 @@
+package file
+
+import (
+	"crypto/sha256"
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestWatcherCheckHash(t *testing.T) {
+	testDir, err := os.MkdirTemp("", "")
+	require.NoError(t, err, "creating test-tempdir")
+	t.Cleanup(func() {
+		if err := os.RemoveAll(testDir); err != nil {
+			t.Logf("failed to clean tempdir %q: %s", testDir, err)
+		}
+	})
+
+	testFile := path.Join(testDir, "test.txt")
+
+	w, err := newWatcher(testFile, time.Second, WatcherCheckHash(sha256.New))
+	require.NoError(t, err, "initial check should not error on non existing file")
+
+	evt, err := w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on non existing file")
+	assert.Equal(t, WatcherEventInvalid, evt, "expect invalid as file is still missing")
+
+	err = os.WriteFile(testFile, []byte("test"), 0o644)
+	require.NoError(t, err, "creating test file")
+
+	evt, err = w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on existing file")
+	assert.Equal(t, WatcherEventFileModified, evt, "expect change as file now exists and has hash change")
+
+	evt, err = w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on existing file")
+	assert.Equal(t, WatcherEventNoChange, evt, "expect no change as the file has the same hash")
+
+	err = os.WriteFile(testFile, []byte("hello world"), 0o644)
+	require.NoError(t, err, "updating test file")
+
+	evt, err = w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on existing file")
+	assert.Equal(t, WatcherEventFileModified, evt, "expect change as file was modified")
+
+	err = os.Remove(testFile)
+	require.NoError(t, err, "deleting test file")
+
+	evt, err = w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on non existing file")
+	assert.Equal(t, WatcherEventInvalid, evt, "expect check to be invalid as file is no longer there")
+
+	err = os.WriteFile(testFile, []byte("hello world"), 0o644)
+	require.NoError(t, err, "updating test file")
+
+	evt, err = w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on existing file")
+	assert.Equal(t, WatcherEventNoChange, evt, "expect change as file has same hash")
+}
+
+func TestWatcherCheckMtime(t *testing.T) {
+	testDir, err := os.MkdirTemp("", "")
+	require.NoError(t, err, "creating test-tempdir")
+	t.Cleanup(func() {
+		if err := os.RemoveAll(testDir); err != nil {
+			t.Logf("failed to clean tempdir %q: %s", testDir, err)
+		}
+	})
+
+	testFile := path.Join(testDir, "test.txt")
+
+	w, err := newWatcher(testFile, time.Second, WatcherCheckMtime)
+	require.NoError(t, err, "initial check should not error on non existing file")
+
+	evt, err := w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on non existing file")
+	assert.Equal(t, WatcherEventInvalid, evt, "expect invalid as file is still missing")
+
+	err = os.WriteFile(testFile, []byte("test"), 0o644)
+	require.NoError(t, err, "creating test file")
+
+	evt, err = w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on existing file")
+	assert.Equal(t, WatcherEventFileModified, evt, "expect change as file now exists and has mtime change")
+
+	evt, err = w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on existing file")
+	assert.Equal(t, WatcherEventNoChange, evt, "expect no change as the file has the same mtime")
+
+	time.Sleep(time.Second) // Unix mtime is second-based, wait a moment
+
+	err = os.WriteFile(testFile, []byte("hello world"), 0o644)
+	require.NoError(t, err, "updating test file")
+
+	evt, err = w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on existing file")
+	assert.Equal(t, WatcherEventFileModified, evt, "expect change as file was modified")
+
+	err = os.Remove(testFile)
+	require.NoError(t, err, "deleting test file")
+
+	evt, err = w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on non existing file")
+	assert.Equal(t, WatcherEventInvalid, evt, "expect check to be invalid as file is no longer there")
+
+	time.Sleep(time.Second) // Unix mtime is second-based, wait a moment
+
+	err = os.WriteFile(testFile, []byte("hello world"), 0o644)
+	require.NoError(t, err, "updating test file")
+
+	evt, err = w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on existing file")
+	assert.Equal(t, WatcherEventFileModified, evt, "expect change as file is newer")
+}
+
+func TestWatcherCheckPresence(t *testing.T) {
+	testDir, err := os.MkdirTemp("", "")
+	require.NoError(t, err, "creating test-tempdir")
+	t.Cleanup(func() {
+		if err := os.RemoveAll(testDir); err != nil {
+			t.Logf("failed to clean tempdir %q: %s", testDir, err)
+		}
+	})
+
+	testFile := path.Join(testDir, "test.txt")
+
+	w, err := newWatcher(testFile, time.Second, WatcherCheckPresence)
+	require.NoError(t, err, "initial check should not error on non existing file")
+
+	evt, err := w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on non existing file")
+	assert.Equal(t, WatcherEventNoChange, evt, "expect no change as file is still missing")
+
+	err = os.WriteFile(testFile, []byte("test"), 0o644)
+	require.NoError(t, err, "creating test file")
+
+	evt, err = w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on existing file")
+	assert.Equal(t, WatcherEventFileAppeared, evt, "expect check to state file is now there")
+
+	evt, err = w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on existing file")
+	assert.Equal(t, WatcherEventNoChange, evt, "expect check to state nothing changed")
+
+	err = os.Remove(testFile)
+	require.NoError(t, err, "deleting test file")
+
+	evt, err = w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on non existing file")
+	assert.Equal(t, WatcherEventFileVanished, evt, "expect check to state file vanished again")
+}
+
+func TestWatcherCheckSize(t *testing.T) {
+	testDir, err := os.MkdirTemp("", "")
+	require.NoError(t, err, "creating test-tempdir")
+	t.Cleanup(func() {
+		if err := os.RemoveAll(testDir); err != nil {
+			t.Logf("failed to clean tempdir %q: %s", testDir, err)
+		}
+	})
+
+	testFile := path.Join(testDir, "test.txt")
+
+	w, err := newWatcher(testFile, time.Second, WatcherCheckSize)
+	require.NoError(t, err, "initial check should not error on non existing file")
+
+	evt, err := w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on non existing file")
+	assert.Equal(t, WatcherEventInvalid, evt, "expect invalid as file is still missing")
+
+	err = os.WriteFile(testFile, []byte("test"), 0o644)
+	require.NoError(t, err, "creating test file")
+
+	evt, err = w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on existing file")
+	assert.Equal(t, WatcherEventFileModified, evt, "expect change as file has now 4 instead of 0 bytes")
+
+	err = os.WriteFile(testFile, []byte("tset"), 0o644)
+	require.NoError(t, err, "updating test file")
+
+	evt, err = w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on existing file")
+	assert.Equal(t, WatcherEventNoChange, evt, "expect no change as the file has the same size")
+
+	err = os.WriteFile(testFile, []byte("hello world"), 0o644)
+	require.NoError(t, err, "updating test file")
+
+	evt, err = w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on existing file")
+	assert.Equal(t, WatcherEventFileModified, evt, "expect change as we went from 4 to 11 chars")
+
+	err = os.Remove(testFile)
+	require.NoError(t, err, "deleting test file")
+
+	evt, err = w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on non existing file")
+	assert.Equal(t, WatcherEventInvalid, evt, "expect check to be invalid as file is no longer there")
+
+	err = os.WriteFile(testFile, []byte("hello world"), 0o644)
+	require.NoError(t, err, "updating test file")
+
+	evt, err = w.runStateChecks(false)
+	require.NoError(t, err, "check should not error on existing file")
+	assert.Equal(t, WatcherEventNoChange, evt, "expect no change as we restored the file with same content")
+}

go.mod

@@ -1,24 +1,30 @@
 module github.com/Luzifer/go_helpers/v2
 
-go 1.15
+go 1.19
 
 require (
-	github.com/golang/protobuf v1.4.3 // indirect
-	github.com/google/go-cmp v0.5.4 // indirect
 	github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
-	github.com/kr/text v0.2.0 // indirect
 	github.com/leekchan/gtf v0.0.0-20190214083521-5fba33c5b00b
-	github.com/nxadm/tail v1.4.6 // indirect
 	github.com/onsi/ginkgo v1.15.0
 	github.com/onsi/gomega v1.10.5
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.7.0
-	github.com/stretchr/testify v1.7.0 // indirect
+	github.com/stretchr/testify v1.7.0
+	gopkg.in/yaml.v2 v2.4.0
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/golang/protobuf v1.4.3 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/nxadm/tail v1.4.6 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect
 	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect
 	golang.org/x/text v0.3.5 // indirect
 	google.golang.org/protobuf v1.25.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
-	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )

go.sum

@@ -29,8 +29,6 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
-github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8=
 github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
@@ -119,7 +117,6 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=