Implement metrics collection for API server (#143)
This commit is contained in:
GitHub
parent
1623e09225
commit
5ad6449757
12 changed files with 402 additions and 95 deletions
29
api.go
29
api.go
|
|
@ -8,13 +8,24 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Luzifer/ots/pkg/metrics"
|
||||
"github.com/Luzifer/ots/pkg/storage"
|
||||
"github.com/gofrs/uuid"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const (
|
||||
errorReasonInvalidJSON = "invalid_json"
|
||||
errorReasonSecretMissing = "secret_missing"
|
||||
errorReasonSecretSize = "secret_size"
|
||||
errorReasonStorageError = "storage_error"
|
||||
errorReasonSecretNotFound = "secret_not_found"
|
||||
)
|
||||
|
||||
type apiServer struct {
|
||||
store storage
|
||||
collector *metrics.Collector
|
||||
store storage.Storage
|
||||
}
|
||||
|
||||
type apiResponse struct {
|
||||
|
|
@ -29,8 +40,9 @@ type apiRequest struct {
|
|||
Secret string `json:"secret"`
|
||||
}
|
||||
|
||||
func newAPI(s storage) *apiServer {
|
||||
func newAPI(s storage.Storage, c *metrics.Collector) *apiServer {
|
||||
return &apiServer{
|
||||
collector: c,
|
||||
store: s,
|
||||
}
|
||||
}
|
||||
|
|
@ -57,6 +69,7 @@ func (a apiServer) handleCreate(res http.ResponseWriter, r *http.Request) {
|
|||
if strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
|
||||
tmp := apiRequest{}
|
||||
if err := json.NewDecoder(r.Body).Decode(&tmp); err != nil {
|
||||
a.collector.CountSecretCreateError(errorReasonInvalidJSON)
|
||||
a.errorResponse(res, http.StatusBadRequest, err, "decoding request body")
|
||||
return
|
||||
}
|
||||
|
|
@ -66,17 +79,20 @@ func (a apiServer) handleCreate(res http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
|
||||
if secret == "" {
|
||||
a.collector.CountSecretCreateError(errorReasonSecretMissing)
|
||||
a.errorResponse(res, http.StatusBadRequest, errors.New("secret missing"), "")
|
||||
return
|
||||
}
|
||||
|
||||
if cust.MaxSecretSize > 0 && len(secret) > int(cust.MaxSecretSize) {
|
||||
a.collector.CountSecretCreateError(errorReasonSecretSize)
|
||||
a.errorResponse(res, http.StatusBadRequest, errors.New("secret size exceeds maximum"), "")
|
||||
return
|
||||
}
|
||||
|
||||
id, err := a.store.Create(secret, time.Duration(expiry)*time.Second)
|
||||
if err != nil {
|
||||
a.collector.CountSecretCreateError(errorReasonStorageError)
|
||||
a.errorResponse(res, http.StatusInternalServerError, err, "creating secret")
|
||||
return
|
||||
}
|
||||
|
|
@ -86,6 +102,8 @@ func (a apiServer) handleCreate(res http.ResponseWriter, r *http.Request) {
|
|||
expiresAt = func(v time.Time) *time.Time { return &v }(time.Now().UTC().Add(time.Duration(expiry) * time.Second))
|
||||
}
|
||||
|
||||
a.collector.CountSecretCreated()
|
||||
go updateStoredSecretsCount(a.store, a.collector)
|
||||
a.jsonResponse(res, http.StatusCreated, apiResponse{
|
||||
ExpiresAt: expiresAt,
|
||||
Success: true,
|
||||
|
|
@ -104,13 +122,18 @@ func (a apiServer) handleRead(res http.ResponseWriter, r *http.Request) {
|
|||
secret, err := a.store.ReadAndDestroy(id)
|
||||
if err != nil {
|
||||
status := http.StatusInternalServerError
|
||||
if err == errSecretNotFound {
|
||||
if errors.Is(err, storage.ErrSecretNotFound) {
|
||||
a.collector.CountSecretReadError(errorReasonSecretNotFound)
|
||||
status = http.StatusNotFound
|
||||
} else {
|
||||
a.collector.CountSecretReadError(errorReasonStorageError)
|
||||
}
|
||||
a.errorResponse(res, status, err, "reading & destroying secret")
|
||||
return
|
||||
}
|
||||
|
||||
a.collector.CountSecretRead()
|
||||
go updateStoredSecretsCount(a.store, a.collector)
|
||||
a.jsonResponse(res, http.StatusOK, apiResponse{
|
||||
Success: true,
|
||||
Secret: secret,
|
||||
|
|
|
|||
8
go.mod
8
go.mod
|
|
@ -14,6 +14,7 @@ require (
|
|||
github.com/gofrs/uuid v4.4.0+incompatible
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/prometheus/client_golang v1.17.0
|
||||
github.com/redis/go-redis/v9 v9.2.1
|
||||
github.com/sirupsen/logrus v1.9.3
|
||||
)
|
||||
|
|
@ -21,18 +22,25 @@ require (
|
|||
require (
|
||||
github.com/Masterminds/goutils v1.1.1 // indirect
|
||||
github.com/Masterminds/semver/v3 v3.2.1 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/google/uuid v1.3.1 // indirect
|
||||
github.com/huandu/xstrings v1.4.0 // indirect
|
||||
github.com/imdario/mergo v0.3.16 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/mitchellh/copystructure v1.2.0 // indirect
|
||||
github.com/mitchellh/reflectwalk v1.0.2 // indirect
|
||||
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
|
||||
github.com/prometheus/common v0.44.0 // indirect
|
||||
github.com/prometheus/procfs v0.11.1 // indirect
|
||||
github.com/shopspring/decimal v1.3.1 // indirect
|
||||
github.com/spf13/cast v1.5.1 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
golang.org/x/crypto v0.14.0 // indirect
|
||||
golang.org/x/sys v0.13.0 // indirect
|
||||
google.golang.org/protobuf v1.31.0 // indirect
|
||||
gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
)
|
||||
|
|
|
|||
27
go.sum
27
go.sum
|
|
@ -9,6 +9,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0
|
|||
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
|
||||
github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
|
||||
github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
|
|
@ -26,6 +28,11 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4
|
|||
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
|
||||
github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
|
||||
github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
|
||||
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
|
|
@ -43,6 +50,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
|||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
|
||||
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
|
||||
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
|
||||
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
|
||||
|
|
@ -59,10 +68,18 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
|||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
|
||||
github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
|
||||
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM=
|
||||
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
|
||||
github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
|
||||
github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
|
||||
github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
|
||||
github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
|
||||
github.com/redis/go-redis/v9 v9.2.1 h1:WlYJg71ODF0dVspZZCpYmoF1+U1Jjk9Rwd7pq6QmlCg=
|
||||
github.com/redis/go-redis/v9 v9.2.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
|
||||
github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
|
|
@ -92,6 +109,7 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
|
|||
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
|
||||
golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU=
|
||||
golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
|
@ -116,6 +134,11 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
|
|||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
|
||||
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
|
|
|
|||
52
helpers.go
Normal file
52
helpers.go
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
|
||||
"github.com/Luzifer/ots/pkg/metrics"
|
||||
"github.com/Luzifer/ots/pkg/storage"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func requestInSubnetList(r *http.Request, subnets []string) bool {
|
||||
if len(subnets) == 0 {
|
||||
// No subnets specififed: None allowed (without doing the parsing)
|
||||
return false
|
||||
}
|
||||
|
||||
remote, _, err := net.SplitHostPort(r.RemoteAddr)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Error("parsing remote address")
|
||||
return false
|
||||
}
|
||||
|
||||
remoteIP := net.ParseIP(remote)
|
||||
if remoteIP == nil {
|
||||
logrus.WithError(err).Error("parsing remote address")
|
||||
return false
|
||||
}
|
||||
|
||||
for _, sn := range subnets {
|
||||
_, netw, err := net.ParseCIDR(sn)
|
||||
if err != nil {
|
||||
logrus.WithError(err).WithField("subnet", sn).Warn("invalid subnet specified")
|
||||
continue
|
||||
}
|
||||
|
||||
if netw.Contains(remoteIP) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func updateStoredSecretsCount(store storage.Storage, collector *metrics.Collector) {
|
||||
n, err := store.Count()
|
||||
if err != nil {
|
||||
logrus.WithError(err).Error("counting stored secrets")
|
||||
return
|
||||
}
|
||||
collector.UpdateSecretsCount(n)
|
||||
}
|
||||
46
main.go
46
main.go
|
|
@ -19,6 +19,7 @@ import (
|
|||
file_helpers "github.com/Luzifer/go_helpers/v2/file"
|
||||
http_helpers "github.com/Luzifer/go_helpers/v2/http"
|
||||
"github.com/Luzifer/ots/pkg/customization"
|
||||
"github.com/Luzifer/ots/pkg/metrics"
|
||||
"github.com/Luzifer/rconfig/v2"
|
||||
)
|
||||
|
||||
|
|
@ -99,6 +100,9 @@ func main() {
|
|||
os.Exit(0)
|
||||
}
|
||||
|
||||
// Initialize metrics collector
|
||||
collector := metrics.New()
|
||||
|
||||
// Initialize index template in order not to parse it multiple times
|
||||
source, err := assets.ReadFile("index.html")
|
||||
if err != nil {
|
||||
|
|
@ -111,27 +115,49 @@ func main() {
|
|||
if err != nil {
|
||||
logrus.WithError(err).Fatal("initializing storage")
|
||||
}
|
||||
api := newAPI(store)
|
||||
api := newAPI(store, collector)
|
||||
|
||||
// Initialize server
|
||||
r := mux.NewRouter()
|
||||
r.Use(http_helpers.GzipHandler)
|
||||
|
||||
api.Register(r.PathPrefix("/api").Subrouter())
|
||||
|
||||
r.HandleFunc("/", handleIndex)
|
||||
r.PathPrefix("/").HandlerFunc(assetDelivery)
|
||||
r.Handle("/metrics", metrics.Handler()).
|
||||
Methods(http.MethodGet).
|
||||
MatcherFunc(func(r *http.Request, _ *mux.RouteMatch) bool {
|
||||
return requestInSubnetList(r, cust.MetricsAllowedSubnets)
|
||||
})
|
||||
|
||||
r.HandleFunc("/", handleIndex).
|
||||
Methods(http.MethodGet)
|
||||
r.PathPrefix("/").HandlerFunc(assetDelivery).
|
||||
Methods(http.MethodGet)
|
||||
|
||||
var hdl http.Handler = r
|
||||
hdl = http_helpers.GzipHandler(hdl)
|
||||
hdl = http_helpers.NewHTTPLogHandlerWithLogger(hdl, logrus.StandardLogger())
|
||||
|
||||
server := &http.Server{
|
||||
Addr: cfg.Listen,
|
||||
Handler: hdl,
|
||||
ReadHeaderTimeout: time.Second,
|
||||
}
|
||||
|
||||
// Start periodic stored metrics update (required for multi-instance
|
||||
// OTS hosting as other instances will create / delete secrets and
|
||||
// we need to keep up with that)
|
||||
go func() {
|
||||
for t := time.NewTicker(time.Minute); ; <-t.C {
|
||||
updateStoredSecretsCount(store, collector)
|
||||
}
|
||||
}()
|
||||
|
||||
// Start server
|
||||
logrus.WithFields(logrus.Fields{
|
||||
"secret_expiry": time.Duration(cfg.SecretExpiry) * time.Second,
|
||||
"version": version,
|
||||
}).Info("ots started")
|
||||
|
||||
server := &http.Server{
|
||||
Addr: cfg.Listen,
|
||||
Handler: http_helpers.NewHTTPLogHandlerWithLogger(r, logrus.StandardLogger()),
|
||||
ReadHeaderTimeout: time.Second,
|
||||
}
|
||||
|
||||
if err = server.ListenAndServe(); err != nil {
|
||||
logrus.WithError(err).Fatal("HTTP server quit unexpectedly")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ type (
|
|||
MaxAttachmentSizeTotal int64 `json:"maxAttachmentSizeTotal" yaml:"maxAttachmentSizeTotal"`
|
||||
|
||||
MaxSecretSize int64 `json:"-" yaml:"maxSecretSize"`
|
||||
MetricsAllowedSubnets []string `json:"-" yaml:"metricsAllowedSubnets"`
|
||||
OverlayFSPath string `json:"-" yaml:"overlayFSPath"`
|
||||
UseFormalLanguage bool `json:"-" yaml:"useFormalLanguage"`
|
||||
}
|
||||
|
|
|
|||
99
pkg/metrics/metrics.go
Normal file
99
pkg/metrics/metrics.go
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
// Package metrics provides an abstraction around metrics collection
|
||||
// in order to bundle all metrics related calls in one location
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
)
|
||||
|
||||
const (
|
||||
metricSecretsCreated = "secrets_created"
|
||||
metricSecretsRead = "secrets_read"
|
||||
metricSecretsCreateErrors = "secrets_create_errors"
|
||||
meticsSecretsReadErrors = "secrets_read_errors"
|
||||
metricsSecretsStored = "secrets_stored"
|
||||
|
||||
labelReason = "reason"
|
||||
|
||||
namespace = "ots"
|
||||
)
|
||||
|
||||
type (
|
||||
// Collector contains all required methods to collect metrics
|
||||
// and to populate them into the Handler
|
||||
Collector struct {
|
||||
secretsCreated prometheus.Counter
|
||||
secretsRead prometheus.Counter
|
||||
secretsCreateErrors *prometheus.CounterVec
|
||||
secretsReadErrors *prometheus.CounterVec
|
||||
secretsStored prometheus.Gauge
|
||||
}
|
||||
)
|
||||
|
||||
// Handler returns the handler to be registered at /metrics
|
||||
func Handler() http.Handler { return promhttp.Handler() }
|
||||
|
||||
// New creates a new Collector and registers the metrics
|
||||
func New() *Collector {
|
||||
return &Collector{
|
||||
secretsCreated: promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: metricSecretsCreated,
|
||||
Help: "number of successfully created secrets",
|
||||
}),
|
||||
|
||||
secretsRead: promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: metricSecretsRead,
|
||||
Help: "number of fetched (and destroyed) secrets",
|
||||
}),
|
||||
|
||||
secretsCreateErrors: promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: metricSecretsCreateErrors,
|
||||
Help: "number of errors on secret creation for each reason",
|
||||
}, []string{labelReason}),
|
||||
|
||||
secretsReadErrors: promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: meticsSecretsReadErrors,
|
||||
Help: "number of read-errors for each reason",
|
||||
}, []string{labelReason}),
|
||||
|
||||
secretsStored: promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
Name: metricsSecretsStored,
|
||||
Help: "number of secrets currently held in the backend store",
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
// CountSecretCreated signalizes a secret has successfully been created
|
||||
func (c Collector) CountSecretCreated() { c.secretsCreated.Inc() }
|
||||
|
||||
// CountSecretRead signalizes a secret has successfully been read and destroyed
|
||||
func (c Collector) CountSecretRead() { c.secretsRead.Inc() }
|
||||
|
||||
// CountSecretCreateError signalizes an error occurred during secret
|
||||
// creation. The reason must not be the error.Error() but a simple
|
||||
// static string describing the error.
|
||||
func (c Collector) CountSecretCreateError(reason string) {
|
||||
c.secretsCreateErrors.WithLabelValues(reason).Inc()
|
||||
}
|
||||
|
||||
// CountSecretReadError signalizes an error occurred during secret
|
||||
// read. The reason must not be the error.Error() but a simple
|
||||
// static string describing the error.
|
||||
func (c Collector) CountSecretReadError(reason string) {
|
||||
c.secretsReadErrors.WithLabelValues(reason).Inc()
|
||||
}
|
||||
|
||||
// UpdateSecretsCount sets the current amount of secrets stored in the
|
||||
// backend storage
|
||||
func (c Collector) UpdateSecretsCount(count int64) {
|
||||
c.secretsStored.Set(float64(count))
|
||||
}
|
||||
76
pkg/storage/memory/memory.go
Normal file
76
pkg/storage/memory/memory.go
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
// Package memory implements a pure in-memory store for secrets which
|
||||
// is suitable for testing and should not be used for productive use
|
||||
package memory
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Luzifer/ots/pkg/storage"
|
||||
"github.com/gofrs/uuid"
|
||||
)
|
||||
|
||||
type (
|
||||
memStorageSecret struct {
|
||||
Expiry time.Time
|
||||
Secret string
|
||||
}
|
||||
|
||||
storageMem struct {
|
||||
sync.RWMutex
|
||||
store map[string]memStorageSecret
|
||||
}
|
||||
)
|
||||
|
||||
// New creates a new In-Mem storage
|
||||
func New() storage.Storage {
|
||||
return &storageMem{
|
||||
store: make(map[string]memStorageSecret),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *storageMem) Count() (int64, error) {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
return int64(len(s.store)), nil
|
||||
}
|
||||
|
||||
func (s *storageMem) Create(secret string, expireIn time.Duration) (string, error) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
var (
|
||||
expire time.Time
|
||||
id = uuid.Must(uuid.NewV4()).String()
|
||||
)
|
||||
|
||||
if expireIn > 0 {
|
||||
expire = time.Now().Add(expireIn)
|
||||
}
|
||||
|
||||
s.store[id] = memStorageSecret{
|
||||
Expiry: expire,
|
||||
Secret: secret,
|
||||
}
|
||||
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (s *storageMem) ReadAndDestroy(id string) (string, error) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
secret, ok := s.store[id]
|
||||
if !ok {
|
||||
return "", storage.ErrSecretNotFound
|
||||
}
|
||||
|
||||
defer delete(s.store, id)
|
||||
|
||||
if !secret.Expiry.IsZero() && secret.Expiry.Before(time.Now()) {
|
||||
return "", storage.ErrSecretNotFound
|
||||
}
|
||||
|
||||
return secret.Secret, nil
|
||||
}
|
||||
|
|
@ -1,24 +1,30 @@
|
|||
package main
|
||||
// Package redis implements a Redis backed storage for secrets
|
||||
package redis
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Luzifer/ots/pkg/storage"
|
||||
"github.com/gofrs/uuid"
|
||||
"github.com/pkg/errors"
|
||||
redis "github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const redisDefaultPrefix = "io.luzifer.ots"
|
||||
const (
|
||||
redisDefaultPrefix = "io.luzifer.ots"
|
||||
redisScanCount = 10
|
||||
)
|
||||
|
||||
type storageRedis struct {
|
||||
conn *redis.Client
|
||||
}
|
||||
|
||||
func newStorageRedis() (storage, error) {
|
||||
// New returns a new Redis backed storage
|
||||
func New() (storage.Storage, error) {
|
||||
if os.Getenv("REDIS_URL") == "" {
|
||||
return nil, fmt.Errorf("REDIS_URL environment variable not set")
|
||||
}
|
||||
|
|
@ -30,7 +36,7 @@ func newStorageRedis() (storage, error) {
|
|||
// in order to maintain backwards compatibility
|
||||
opt, err := redis.ParseURL(strings.Replace(os.Getenv("REDIS_URL"), "tcp://", "redis://", 1))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "parsing REDIS_URL")
|
||||
return nil, fmt.Errorf("parsing REDIS_URL: %w", err)
|
||||
}
|
||||
|
||||
s := &storageRedis{
|
||||
|
|
@ -40,24 +46,50 @@ func newStorageRedis() (storage, error) {
|
|||
return s, nil
|
||||
}
|
||||
|
||||
func (s storageRedis) Count() (n int64, err error) {
|
||||
var cursor uint64
|
||||
|
||||
for {
|
||||
var keys []string
|
||||
|
||||
keys, cursor, err = s.conn.Scan(context.Background(), cursor, s.redisKey("*"), redisScanCount).Result()
|
||||
if err != nil {
|
||||
return n, fmt.Errorf("scanning stored keys: %w", err)
|
||||
}
|
||||
|
||||
n += int64(len(keys))
|
||||
if cursor == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (s storageRedis) Create(secret string, expireIn time.Duration) (string, error) {
|
||||
id := uuid.Must(uuid.NewV4()).String()
|
||||
err := s.conn.Set(context.Background(), s.redisKey(id), secret, expireIn).Err()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("writing redis key: %w", err)
|
||||
}
|
||||
|
||||
return id, errors.Wrap(err, "writing redis key")
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (s storageRedis) ReadAndDestroy(id string) (string, error) {
|
||||
secret, err := s.conn.Get(context.Background(), s.redisKey(id)).Result()
|
||||
if err != nil {
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return "", errSecretNotFound
|
||||
return "", storage.ErrSecretNotFound
|
||||
}
|
||||
return "", errors.Wrap(err, "getting key")
|
||||
return "", fmt.Errorf("getting key: %w", err)
|
||||
}
|
||||
|
||||
err = s.conn.Del(context.Background(), s.redisKey(id)).Err()
|
||||
return secret, errors.Wrap(err, "deleting key")
|
||||
if err != nil {
|
||||
return secret, fmt.Errorf("deleting key: %w", err)
|
||||
}
|
||||
return secret, nil
|
||||
}
|
||||
|
||||
func (storageRedis) redisKey(id string) string {
|
||||
21
pkg/storage/storage.go
Normal file
21
pkg/storage/storage.go
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
// Package storage describes the requirements a storage provider
|
||||
// has to fulfill ot be usable in OTS
|
||||
package storage
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
type (
|
||||
// Storage is the interface to implement in each storage provider
|
||||
Storage interface {
|
||||
Count() (int64, error)
|
||||
Create(secret string, expireIn time.Duration) (string, error)
|
||||
ReadAndDestroy(id string) (string, error)
|
||||
}
|
||||
)
|
||||
|
||||
// ErrSecretNotFound is a generic error to be returned when a secret
|
||||
// does not exist in the backend. It will then be handled by API.
|
||||
var ErrSecretNotFound = errors.New("secret not found")
|
||||
25
storage.go
25
storage.go
|
|
@ -1,24 +1,25 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/Luzifer/ots/pkg/storage"
|
||||
"github.com/Luzifer/ots/pkg/storage/memory"
|
||||
"github.com/Luzifer/ots/pkg/storage/redis"
|
||||
)
|
||||
|
||||
var errSecretNotFound = errors.New("secret not found")
|
||||
|
||||
type storage interface {
|
||||
Create(secret string, expireIn time.Duration) (string, error)
|
||||
ReadAndDestroy(id string) (string, error)
|
||||
}
|
||||
|
||||
func getStorageByType(t string) (storage, error) {
|
||||
func getStorageByType(t string) (storage.Storage, error) {
|
||||
switch t {
|
||||
case "mem":
|
||||
return newStorageMem(), nil
|
||||
return memory.New(), nil
|
||||
|
||||
case "redis":
|
||||
return newStorageRedis()
|
||||
s, err := redis.New()
|
||||
if err != nil {
|
||||
return s, fmt.Errorf("creating redis storage: %w", err)
|
||||
}
|
||||
return s, nil
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("storage type %q not found", t)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,55 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/gofrs/uuid"
|
||||
)
|
||||
|
||||
type memStorageSecret struct {
|
||||
Expiry time.Time
|
||||
Secret string
|
||||
}
|
||||
|
||||
type storageMem struct {
|
||||
store map[string]memStorageSecret
|
||||
}
|
||||
|
||||
func newStorageMem() storage {
|
||||
return &storageMem{
|
||||
store: make(map[string]memStorageSecret),
|
||||
}
|
||||
}
|
||||
|
||||
func (s storageMem) Create(secret string, expireIn time.Duration) (string, error) {
|
||||
var (
|
||||
expire time.Time
|
||||
id = uuid.Must(uuid.NewV4()).String()
|
||||
)
|
||||
|
||||
if expireIn > 0 {
|
||||
expire = time.Now().Add(expireIn)
|
||||
}
|
||||
|
||||
s.store[id] = memStorageSecret{
|
||||
Expiry: expire,
|
||||
Secret: secret,
|
||||
}
|
||||
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (s storageMem) ReadAndDestroy(id string) (string, error) {
|
||||
secret, ok := s.store[id]
|
||||
if !ok {
|
||||
return "", errSecretNotFound
|
||||
}
|
||||
|
||||
defer delete(s.store, id)
|
||||
|
||||
if !secret.Expiry.IsZero() && secret.Expiry.Before(time.Now()) {
|
||||
return "", errSecretNotFound
|
||||
}
|
||||
|
||||
return secret.Secret, nil
|
||||
}
|
||||
Loading…
Reference in a new issue