From 79116db2a4591aea7a18042d9ef7e72070181ebe Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Sat, 25 Jan 2020 17:00:43 +0100 Subject: [PATCH] Initial version --- .gitignore | 1 + go.mod | 10 +++++ go.sum | 67 +++++++++++++++++++++++++++++++++ main.go | 106 +++++++++++++++++++++++++++++++++++++++++++++++++++++ metrics.go | 94 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 278 insertions(+) create mode 100644 .gitignore create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 metrics.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4f9df20 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +bind-log-metrics diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..ac92552 --- /dev/null +++ b/go.mod @@ -0,0 +1,10 @@ +module github.com/Luzifer/bind-log-metrics + +go 1.13 + +require ( + github.com/Luzifer/rconfig/v2 v2.2.1 + github.com/influxdata/influxdb v1.7.9 + github.com/pkg/errors v0.9.1 + github.com/sirupsen/logrus v1.4.2 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..14198e3 --- /dev/null +++ b/go.sum @@ -0,0 +1,67 @@ +github.com/Luzifer/rconfig v2.2.0+incompatible h1:Kle3+rshPM7LxciOheaR4EfHUzibkDDGws04sefQ5m8= +github.com/Luzifer/rconfig v2.2.0+incompatible/go.mod h1:9pet6z2+mm/UAB0jF/rf0s62USfHNolzgR6Q4KpsJI0= +github.com/Luzifer/rconfig/v2 v2.2.1 h1:zcDdLQlnlzwcBJ8E0WFzOkQE1pCMn3EbX0dFYkeTczg= +github.com/Luzifer/rconfig/v2 v2.2.1/go.mod h1:OKIX0/JRZrPJ/ZXXWklQEFXA6tBfWaljZbW37w+sqBw= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/influxdata/influxdb v1.7.9 h1:uSeBTNO4rBkbp1Be5FKRsAmglM9nlx25TzVQRQt1An4= +github.com/influxdata/influxdb v1.7.9/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34= +github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db h1:6/JqlYfC1CCaLnGceQTI+sDGhC9UBSPAsBqI0Gun6kU= +golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/validator.v2 v2.0.0-20180514200540-135c24b11c19 h1:WB265cn5OpO+hK3pikC9hpP1zI/KTwmyMFKloW9eOVc= +gopkg.in/validator.v2 v2.0.0-20180514200540-135c24b11c19/go.mod h1:o4V0GXN9/CAmCsvJ0oXYZvrZOe7syiDZSN1GWGZTGzc= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go new file mode 100644 index 0000000..7314480 --- /dev/null +++ b/main.go @@ -0,0 +1,106 @@ +package main + +import ( + "bufio" + "fmt" + "os" + "regexp" + + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + + "github.com/Luzifer/rconfig/v2" +) + +type metric string + +const ( + metricQuery metric = "dns_query" + metricBlock metric = "dns_block" +) + +var ( + cfg = struct { + InfluxDBName string `description:"Database name of the InfluxDB" validate:"nonzero"` + InfluxHost string `description:"Hostname of the InfluxDB" validate:"nonzero"` + InfluxPass string `description:"Password of the InfluxDB" validate:"nonzero"` + InfluxUser string `description:"Username of the InfluxDB" validate:"nonzero"` + LogLevel string `flag:"log-level" default:"info" description:"Log level (debug, info, warn, error, fatal)"` + VersionAndExit bool `flag:"version" default:"false" description:"Prints current version and exits"` + }{} + + metrics *metricsSender + + regBlock = regexp.MustCompile(`([0-9:.]+)#.*: rpz QNAME NXDOMAIN rewrite ([^/]+)/([^/]+)/IN`) + regQuery = regexp.MustCompile(`([0-9:.]+)#.*: query: ([^ ]+) IN ([^ ]+) \+`) + + version = "dev" +) + +func init() { + rconfig.AutoEnv(true) + if err := rconfig.ParseAndValidate(&cfg); err != nil { + log.Fatalf("Unable to parse commandline options: %s", err) + } + + if cfg.VersionAndExit { + fmt.Printf("bind-log-metrics %s\n", version) + os.Exit(0) + } + + if l, err := log.ParseLevel(cfg.LogLevel); err != nil { + log.WithError(err).Fatal("Unable to parse log level") + } else { + log.SetLevel(l) + } +} + +func main() { + var err error + + if metrics, err = newMetricsSender(cfg.InfluxHost, cfg.InfluxUser, cfg.InfluxPass, cfg.InfluxDBName); err != nil { + log.WithError(err).Fatal("Unable to create metrics client") + } + + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + var line = scanner.Text() + + // Re-yield the line scanned from stdin + fmt.Fprintln(os.Stdout, line) + + switch { + + case regBlock.MatchString(line): + err = handleRecord(metricBlock, regBlock.FindStringSubmatch(line)) + + case regQuery.MatchString(line): + err = handleRecord(metricQuery, regBlock.FindStringSubmatch(line)) + + default: + continue + + } + + if err != nil { + log.WithError(err).Error("Unable to process line") + } + } + + if err := scanner.Err(); err != nil { + log.WithError(err).Fatal("Scanner caused an error") + } +} + +func handleRecord(m metric, groups []string) error { + return errors.Wrap( + metrics.RecordPoint(string(m), map[string]string{ + "client": groups[1], + "domain": groups[2], + "type": groups[3], + }, map[string]interface{}{ + "count": 1, + }), + "Unable to add metrics point", + ) +} diff --git a/metrics.go b/metrics.go new file mode 100644 index 0000000..7af115f --- /dev/null +++ b/metrics.go @@ -0,0 +1,94 @@ +package main + +import ( + "sync" + "time" + + influx "github.com/influxdata/influxdb/client/v2" +) + +const ( + influxWriteInterval = 10 * time.Second +) + +type metricsSender struct { + batch influx.BatchPoints + batchLock sync.Mutex + client influx.Client + database string + errs chan error +} + +func newMetricsSender(host, user, pass, database string) (*metricsSender, error) { + out := &metricsSender{ + database: database, + errs: make(chan error, 10), + } + return out, out.initialize(host, user, pass) +} + +func (m *metricsSender) Errors() <-chan error { + return m.errs +} + +func (m *metricsSender) RecordPoint(name string, tags map[string]string, fields map[string]interface{}) error { + pt, err := influx.NewPoint(name, tags, fields, time.Now()) + if err != nil { + return err + } + + m.batchLock.Lock() + defer m.batchLock.Unlock() + m.batch.AddPoint(pt) + + return nil +} + +func (m *metricsSender) resetBatch() error { + b, err := influx.NewBatchPoints(influx.BatchPointsConfig{ + Database: m.database, + }) + + if err != nil { + return err + } + + m.batch = b + return nil +} + +func (m *metricsSender) sendLoop() { + for range time.Tick(influxWriteInterval) { + + m.batchLock.Lock() + if err := m.client.Write(m.batch); err != nil { + m.errs <- err + m.batchLock.Unlock() + continue + } + m.resetBatch() + m.batchLock.Unlock() + + } +} + +func (m *metricsSender) initialize(host, user, pass string) error { + influxClient, err := influx.NewHTTPClient(influx.HTTPConfig{ + Addr: host, + Username: user, + Password: pass, + Timeout: 2 * time.Second, + }) + + if err != nil { + return err + } + + m.client = influxClient + if err := m.resetBatch(); err != nil { + return err + } + go m.sendLoop() + + return nil +}