Implement metrics collection for API server (#143)

This commit is contained in:
Knut Ahlers 2023-10-23 14:05:20 +02:00 committed by GitHub
parent 1623e09225
commit 5ad6449757
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 402 additions and 95 deletions

29
api.go
View File

@ -8,13 +8,24 @@ import (
"strings" "strings"
"time" "time"
"github.com/Luzifer/ots/pkg/metrics"
"github.com/Luzifer/ots/pkg/storage"
"github.com/gofrs/uuid" "github.com/gofrs/uuid"
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
const (
errorReasonInvalidJSON = "invalid_json"
errorReasonSecretMissing = "secret_missing"
errorReasonSecretSize = "secret_size"
errorReasonStorageError = "storage_error"
errorReasonSecretNotFound = "secret_not_found"
)
type apiServer struct { type apiServer struct {
store storage collector *metrics.Collector
store storage.Storage
} }
type apiResponse struct { type apiResponse struct {
@ -29,8 +40,9 @@ type apiRequest struct {
Secret string `json:"secret"` Secret string `json:"secret"`
} }
func newAPI(s storage) *apiServer { func newAPI(s storage.Storage, c *metrics.Collector) *apiServer {
return &apiServer{ return &apiServer{
collector: c,
store: s, store: s,
} }
} }
@ -57,6 +69,7 @@ func (a apiServer) handleCreate(res http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") { if strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
tmp := apiRequest{} tmp := apiRequest{}
if err := json.NewDecoder(r.Body).Decode(&tmp); err != nil { if err := json.NewDecoder(r.Body).Decode(&tmp); err != nil {
a.collector.CountSecretCreateError(errorReasonInvalidJSON)
a.errorResponse(res, http.StatusBadRequest, err, "decoding request body") a.errorResponse(res, http.StatusBadRequest, err, "decoding request body")
return return
} }
@ -66,17 +79,20 @@ func (a apiServer) handleCreate(res http.ResponseWriter, r *http.Request) {
} }
if secret == "" { if secret == "" {
a.collector.CountSecretCreateError(errorReasonSecretMissing)
a.errorResponse(res, http.StatusBadRequest, errors.New("secret missing"), "") a.errorResponse(res, http.StatusBadRequest, errors.New("secret missing"), "")
return return
} }
if cust.MaxSecretSize > 0 && len(secret) > int(cust.MaxSecretSize) { if cust.MaxSecretSize > 0 && len(secret) > int(cust.MaxSecretSize) {
a.collector.CountSecretCreateError(errorReasonSecretSize)
a.errorResponse(res, http.StatusBadRequest, errors.New("secret size exceeds maximum"), "") a.errorResponse(res, http.StatusBadRequest, errors.New("secret size exceeds maximum"), "")
return return
} }
id, err := a.store.Create(secret, time.Duration(expiry)*time.Second) id, err := a.store.Create(secret, time.Duration(expiry)*time.Second)
if err != nil { if err != nil {
a.collector.CountSecretCreateError(errorReasonStorageError)
a.errorResponse(res, http.StatusInternalServerError, err, "creating secret") a.errorResponse(res, http.StatusInternalServerError, err, "creating secret")
return return
} }
@ -86,6 +102,8 @@ func (a apiServer) handleCreate(res http.ResponseWriter, r *http.Request) {
expiresAt = func(v time.Time) *time.Time { return &v }(time.Now().UTC().Add(time.Duration(expiry) * time.Second)) expiresAt = func(v time.Time) *time.Time { return &v }(time.Now().UTC().Add(time.Duration(expiry) * time.Second))
} }
a.collector.CountSecretCreated()
go updateStoredSecretsCount(a.store, a.collector)
a.jsonResponse(res, http.StatusCreated, apiResponse{ a.jsonResponse(res, http.StatusCreated, apiResponse{
ExpiresAt: expiresAt, ExpiresAt: expiresAt,
Success: true, Success: true,
@ -104,13 +122,18 @@ func (a apiServer) handleRead(res http.ResponseWriter, r *http.Request) {
secret, err := a.store.ReadAndDestroy(id) secret, err := a.store.ReadAndDestroy(id)
if err != nil { if err != nil {
status := http.StatusInternalServerError status := http.StatusInternalServerError
if err == errSecretNotFound { if errors.Is(err, storage.ErrSecretNotFound) {
a.collector.CountSecretReadError(errorReasonSecretNotFound)
status = http.StatusNotFound status = http.StatusNotFound
} else {
a.collector.CountSecretReadError(errorReasonStorageError)
} }
a.errorResponse(res, status, err, "reading & destroying secret") a.errorResponse(res, status, err, "reading & destroying secret")
return return
} }
a.collector.CountSecretRead()
go updateStoredSecretsCount(a.store, a.collector)
a.jsonResponse(res, http.StatusOK, apiResponse{ a.jsonResponse(res, http.StatusOK, apiResponse{
Success: true, Success: true,
Secret: secret, Secret: secret,

8
go.mod
View File

@ -14,6 +14,7 @@ require (
github.com/gofrs/uuid v4.4.0+incompatible github.com/gofrs/uuid v4.4.0+incompatible
github.com/gorilla/mux v1.8.0 github.com/gorilla/mux v1.8.0
github.com/pkg/errors v0.9.1 github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.17.0
github.com/redis/go-redis/v9 v9.2.1 github.com/redis/go-redis/v9 v9.2.1
github.com/sirupsen/logrus v1.9.3 github.com/sirupsen/logrus v1.9.3
) )
@ -21,18 +22,25 @@ require (
require ( require (
github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Masterminds/semver/v3 v3.2.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/uuid v1.3.1 // indirect github.com/google/uuid v1.3.1 // indirect
github.com/huandu/xstrings v1.4.0 // indirect github.com/huandu/xstrings v1.4.0 // indirect
github.com/imdario/mergo v0.3.16 // indirect github.com/imdario/mergo v0.3.16 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/shopspring/decimal v1.3.1 // indirect github.com/shopspring/decimal v1.3.1 // indirect
github.com/spf13/cast v1.5.1 // indirect github.com/spf13/cast v1.5.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/crypto v0.14.0 // indirect golang.org/x/crypto v0.14.0 // indirect
golang.org/x/sys v0.13.0 // indirect golang.org/x/sys v0.13.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/validator.v2 v2.0.1 // indirect gopkg.in/validator.v2 v2.0.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect
) )

27
go.sum
View File

@ -9,6 +9,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
@ -26,6 +28,11 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@ -43,6 +50,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
@ -59,10 +68,18 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM=
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
github.com/redis/go-redis/v9 v9.2.1 h1:WlYJg71ODF0dVspZZCpYmoF1+U1Jjk9Rwd7pq6QmlCg= github.com/redis/go-redis/v9 v9.2.1 h1:WlYJg71ODF0dVspZZCpYmoF1+U1Jjk9Rwd7pq6QmlCg=
github.com/redis/go-redis/v9 v9.2.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/redis/go-redis/v9 v9.2.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
@ -92,6 +109,7 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU=
golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -116,6 +134,11 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=

52
helpers.go Normal file
View File

@ -0,0 +1,52 @@
package main
import (
"net"
"net/http"
"github.com/Luzifer/ots/pkg/metrics"
"github.com/Luzifer/ots/pkg/storage"
"github.com/sirupsen/logrus"
)
func requestInSubnetList(r *http.Request, subnets []string) bool {
if len(subnets) == 0 {
// No subnets specififed: None allowed (without doing the parsing)
return false
}
remote, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
logrus.WithError(err).Error("parsing remote address")
return false
}
remoteIP := net.ParseIP(remote)
if remoteIP == nil {
logrus.WithError(err).Error("parsing remote address")
return false
}
for _, sn := range subnets {
_, netw, err := net.ParseCIDR(sn)
if err != nil {
logrus.WithError(err).WithField("subnet", sn).Warn("invalid subnet specified")
continue
}
if netw.Contains(remoteIP) {
return true
}
}
return false
}
func updateStoredSecretsCount(store storage.Storage, collector *metrics.Collector) {
n, err := store.Count()
if err != nil {
logrus.WithError(err).Error("counting stored secrets")
return
}
collector.UpdateSecretsCount(n)
}

46
main.go
View File

@ -19,6 +19,7 @@ import (
file_helpers "github.com/Luzifer/go_helpers/v2/file" file_helpers "github.com/Luzifer/go_helpers/v2/file"
http_helpers "github.com/Luzifer/go_helpers/v2/http" http_helpers "github.com/Luzifer/go_helpers/v2/http"
"github.com/Luzifer/ots/pkg/customization" "github.com/Luzifer/ots/pkg/customization"
"github.com/Luzifer/ots/pkg/metrics"
"github.com/Luzifer/rconfig/v2" "github.com/Luzifer/rconfig/v2"
) )
@ -99,6 +100,9 @@ func main() {
os.Exit(0) os.Exit(0)
} }
// Initialize metrics collector
collector := metrics.New()
// Initialize index template in order not to parse it multiple times // Initialize index template in order not to parse it multiple times
source, err := assets.ReadFile("index.html") source, err := assets.ReadFile("index.html")
if err != nil { if err != nil {
@ -111,27 +115,49 @@ func main() {
if err != nil { if err != nil {
logrus.WithError(err).Fatal("initializing storage") logrus.WithError(err).Fatal("initializing storage")
} }
api := newAPI(store) api := newAPI(store, collector)
// Initialize server
r := mux.NewRouter() r := mux.NewRouter()
r.Use(http_helpers.GzipHandler)
api.Register(r.PathPrefix("/api").Subrouter()) api.Register(r.PathPrefix("/api").Subrouter())
r.HandleFunc("/", handleIndex) r.Handle("/metrics", metrics.Handler()).
r.PathPrefix("/").HandlerFunc(assetDelivery) Methods(http.MethodGet).
MatcherFunc(func(r *http.Request, _ *mux.RouteMatch) bool {
return requestInSubnetList(r, cust.MetricsAllowedSubnets)
})
r.HandleFunc("/", handleIndex).
Methods(http.MethodGet)
r.PathPrefix("/").HandlerFunc(assetDelivery).
Methods(http.MethodGet)
var hdl http.Handler = r
hdl = http_helpers.GzipHandler(hdl)
hdl = http_helpers.NewHTTPLogHandlerWithLogger(hdl, logrus.StandardLogger())
server := &http.Server{
Addr: cfg.Listen,
Handler: hdl,
ReadHeaderTimeout: time.Second,
}
// Start periodic stored metrics update (required for multi-instance
// OTS hosting as other instances will create / delete secrets and
// we need to keep up with that)
go func() {
for t := time.NewTicker(time.Minute); ; <-t.C {
updateStoredSecretsCount(store, collector)
}
}()
// Start server
logrus.WithFields(logrus.Fields{ logrus.WithFields(logrus.Fields{
"secret_expiry": time.Duration(cfg.SecretExpiry) * time.Second, "secret_expiry": time.Duration(cfg.SecretExpiry) * time.Second,
"version": version, "version": version,
}).Info("ots started") }).Info("ots started")
server := &http.Server{
Addr: cfg.Listen,
Handler: http_helpers.NewHTTPLogHandlerWithLogger(r, logrus.StandardLogger()),
ReadHeaderTimeout: time.Second,
}
if err = server.ListenAndServe(); err != nil { if err = server.ListenAndServe(); err != nil {
logrus.WithError(err).Fatal("HTTP server quit unexpectedly") logrus.WithError(err).Fatal("HTTP server quit unexpectedly")
} }

View File

@ -30,6 +30,7 @@ type (
MaxAttachmentSizeTotal int64 `json:"maxAttachmentSizeTotal" yaml:"maxAttachmentSizeTotal"` MaxAttachmentSizeTotal int64 `json:"maxAttachmentSizeTotal" yaml:"maxAttachmentSizeTotal"`
MaxSecretSize int64 `json:"-" yaml:"maxSecretSize"` MaxSecretSize int64 `json:"-" yaml:"maxSecretSize"`
MetricsAllowedSubnets []string `json:"-" yaml:"metricsAllowedSubnets"`
OverlayFSPath string `json:"-" yaml:"overlayFSPath"` OverlayFSPath string `json:"-" yaml:"overlayFSPath"`
UseFormalLanguage bool `json:"-" yaml:"useFormalLanguage"` UseFormalLanguage bool `json:"-" yaml:"useFormalLanguage"`
} }

99
pkg/metrics/metrics.go Normal file
View File

@ -0,0 +1,99 @@
// Package metrics provides an abstraction around metrics collection
// in order to bundle all metrics related calls in one location
package metrics
import (
"net/http"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
const (
metricSecretsCreated = "secrets_created"
metricSecretsRead = "secrets_read"
metricSecretsCreateErrors = "secrets_create_errors"
meticsSecretsReadErrors = "secrets_read_errors"
metricsSecretsStored = "secrets_stored"
labelReason = "reason"
namespace = "ots"
)
type (
// Collector contains all required methods to collect metrics
// and to populate them into the Handler
Collector struct {
secretsCreated prometheus.Counter
secretsRead prometheus.Counter
secretsCreateErrors *prometheus.CounterVec
secretsReadErrors *prometheus.CounterVec
secretsStored prometheus.Gauge
}
)
// Handler returns the handler to be registered at /metrics
func Handler() http.Handler { return promhttp.Handler() }
// New creates a new Collector and registers the metrics
func New() *Collector {
return &Collector{
secretsCreated: promauto.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Name: metricSecretsCreated,
Help: "number of successfully created secrets",
}),
secretsRead: promauto.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Name: metricSecretsRead,
Help: "number of fetched (and destroyed) secrets",
}),
secretsCreateErrors: promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: namespace,
Name: metricSecretsCreateErrors,
Help: "number of errors on secret creation for each reason",
}, []string{labelReason}),
secretsReadErrors: promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: namespace,
Name: meticsSecretsReadErrors,
Help: "number of read-errors for each reason",
}, []string{labelReason}),
secretsStored: promauto.NewGauge(prometheus.GaugeOpts{
Namespace: namespace,
Name: metricsSecretsStored,
Help: "number of secrets currently held in the backend store",
}),
}
}
// CountSecretCreated signalizes a secret has successfully been created
func (c Collector) CountSecretCreated() { c.secretsCreated.Inc() }
// CountSecretRead signalizes a secret has successfully been read and destroyed
func (c Collector) CountSecretRead() { c.secretsRead.Inc() }
// CountSecretCreateError signalizes an error occurred during secret
// creation. The reason must not be the error.Error() but a simple
// static string describing the error.
func (c Collector) CountSecretCreateError(reason string) {
c.secretsCreateErrors.WithLabelValues(reason).Inc()
}
// CountSecretReadError signalizes an error occurred during secret
// read. The reason must not be the error.Error() but a simple
// static string describing the error.
func (c Collector) CountSecretReadError(reason string) {
c.secretsReadErrors.WithLabelValues(reason).Inc()
}
// UpdateSecretsCount sets the current amount of secrets stored in the
// backend storage
func (c Collector) UpdateSecretsCount(count int64) {
c.secretsStored.Set(float64(count))
}

View File

@ -0,0 +1,76 @@
// Package memory implements a pure in-memory store for secrets which
// is suitable for testing and should not be used for productive use
package memory
import (
"sync"
"time"
"github.com/Luzifer/ots/pkg/storage"
"github.com/gofrs/uuid"
)
type (
memStorageSecret struct {
Expiry time.Time
Secret string
}
storageMem struct {
sync.RWMutex
store map[string]memStorageSecret
}
)
// New creates a new In-Mem storage
func New() storage.Storage {
return &storageMem{
store: make(map[string]memStorageSecret),
}
}
func (s *storageMem) Count() (int64, error) {
s.RLock()
defer s.RUnlock()
return int64(len(s.store)), nil
}
func (s *storageMem) Create(secret string, expireIn time.Duration) (string, error) {
s.Lock()
defer s.Unlock()
var (
expire time.Time
id = uuid.Must(uuid.NewV4()).String()
)
if expireIn > 0 {
expire = time.Now().Add(expireIn)
}
s.store[id] = memStorageSecret{
Expiry: expire,
Secret: secret,
}
return id, nil
}
func (s *storageMem) ReadAndDestroy(id string) (string, error) {
s.Lock()
defer s.Unlock()
secret, ok := s.store[id]
if !ok {
return "", storage.ErrSecretNotFound
}
defer delete(s.store, id)
if !secret.Expiry.IsZero() && secret.Expiry.Before(time.Now()) {
return "", storage.ErrSecretNotFound
}
return secret.Secret, nil
}

View File

@ -1,24 +1,30 @@
package main // Package redis implements a Redis backed storage for secrets
package redis
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"os" "os"
"strings" "strings"
"time" "time"
"github.com/Luzifer/ots/pkg/storage"
"github.com/gofrs/uuid" "github.com/gofrs/uuid"
"github.com/pkg/errors"
redis "github.com/redis/go-redis/v9" redis "github.com/redis/go-redis/v9"
) )
const redisDefaultPrefix = "io.luzifer.ots" const (
redisDefaultPrefix = "io.luzifer.ots"
redisScanCount = 10
)
type storageRedis struct { type storageRedis struct {
conn *redis.Client conn *redis.Client
} }
func newStorageRedis() (storage, error) { // New returns a new Redis backed storage
func New() (storage.Storage, error) {
if os.Getenv("REDIS_URL") == "" { if os.Getenv("REDIS_URL") == "" {
return nil, fmt.Errorf("REDIS_URL environment variable not set") return nil, fmt.Errorf("REDIS_URL environment variable not set")
} }
@ -30,7 +36,7 @@ func newStorageRedis() (storage, error) {
// in order to maintain backwards compatibility // in order to maintain backwards compatibility
opt, err := redis.ParseURL(strings.Replace(os.Getenv("REDIS_URL"), "tcp://", "redis://", 1)) opt, err := redis.ParseURL(strings.Replace(os.Getenv("REDIS_URL"), "tcp://", "redis://", 1))
if err != nil { if err != nil {
return nil, errors.Wrap(err, "parsing REDIS_URL") return nil, fmt.Errorf("parsing REDIS_URL: %w", err)
} }
s := &storageRedis{ s := &storageRedis{
@ -40,24 +46,50 @@ func newStorageRedis() (storage, error) {
return s, nil return s, nil
} }
func (s storageRedis) Count() (n int64, err error) {
var cursor uint64
for {
var keys []string
keys, cursor, err = s.conn.Scan(context.Background(), cursor, s.redisKey("*"), redisScanCount).Result()
if err != nil {
return n, fmt.Errorf("scanning stored keys: %w", err)
}
n += int64(len(keys))
if cursor == 0 {
break
}
}
return n, nil
}
func (s storageRedis) Create(secret string, expireIn time.Duration) (string, error) { func (s storageRedis) Create(secret string, expireIn time.Duration) (string, error) {
id := uuid.Must(uuid.NewV4()).String() id := uuid.Must(uuid.NewV4()).String()
err := s.conn.Set(context.Background(), s.redisKey(id), secret, expireIn).Err() err := s.conn.Set(context.Background(), s.redisKey(id), secret, expireIn).Err()
if err != nil {
return "", fmt.Errorf("writing redis key: %w", err)
}
return id, errors.Wrap(err, "writing redis key") return id, nil
} }
func (s storageRedis) ReadAndDestroy(id string) (string, error) { func (s storageRedis) ReadAndDestroy(id string) (string, error) {
secret, err := s.conn.Get(context.Background(), s.redisKey(id)).Result() secret, err := s.conn.Get(context.Background(), s.redisKey(id)).Result()
if err != nil { if err != nil {
if errors.Is(err, redis.Nil) { if errors.Is(err, redis.Nil) {
return "", errSecretNotFound return "", storage.ErrSecretNotFound
} }
return "", errors.Wrap(err, "getting key") return "", fmt.Errorf("getting key: %w", err)
} }
err = s.conn.Del(context.Background(), s.redisKey(id)).Err() err = s.conn.Del(context.Background(), s.redisKey(id)).Err()
return secret, errors.Wrap(err, "deleting key") if err != nil {
return secret, fmt.Errorf("deleting key: %w", err)
}
return secret, nil
} }
func (storageRedis) redisKey(id string) string { func (storageRedis) redisKey(id string) string {

21
pkg/storage/storage.go Normal file
View File

@ -0,0 +1,21 @@
// Package storage describes the requirements a storage provider
// has to fulfill ot be usable in OTS
package storage
import (
"errors"
"time"
)
type (
// Storage is the interface to implement in each storage provider
Storage interface {
Count() (int64, error)
Create(secret string, expireIn time.Duration) (string, error)
ReadAndDestroy(id string) (string, error)
}
)
// ErrSecretNotFound is a generic error to be returned when a secret
// does not exist in the backend. It will then be handled by API.
var ErrSecretNotFound = errors.New("secret not found")

View File

@ -1,24 +1,25 @@
package main package main
import ( import (
"errors"
"fmt" "fmt"
"time"
"github.com/Luzifer/ots/pkg/storage"
"github.com/Luzifer/ots/pkg/storage/memory"
"github.com/Luzifer/ots/pkg/storage/redis"
) )
var errSecretNotFound = errors.New("secret not found") func getStorageByType(t string) (storage.Storage, error) {
type storage interface {
Create(secret string, expireIn time.Duration) (string, error)
ReadAndDestroy(id string) (string, error)
}
func getStorageByType(t string) (storage, error) {
switch t { switch t {
case "mem": case "mem":
return newStorageMem(), nil return memory.New(), nil
case "redis": case "redis":
return newStorageRedis() s, err := redis.New()
if err != nil {
return s, fmt.Errorf("creating redis storage: %w", err)
}
return s, nil
default: default:
return nil, fmt.Errorf("storage type %q not found", t) return nil, fmt.Errorf("storage type %q not found", t)
} }

View File

@ -1,55 +0,0 @@
package main
import (
"time"
"github.com/gofrs/uuid"
)
type memStorageSecret struct {
Expiry time.Time
Secret string
}
type storageMem struct {
store map[string]memStorageSecret
}
func newStorageMem() storage {
return &storageMem{
store: make(map[string]memStorageSecret),
}
}
func (s storageMem) Create(secret string, expireIn time.Duration) (string, error) {
var (
expire time.Time
id = uuid.Must(uuid.NewV4()).String()
)
if expireIn > 0 {
expire = time.Now().Add(expireIn)
}
s.store[id] = memStorageSecret{
Expiry: expire,
Secret: secret,
}
return id, nil
}
func (s storageMem) ReadAndDestroy(id string) (string, error) {
secret, ok := s.store[id]
if !ok {
return "", errSecretNotFound
}
defer delete(s.store, id)
if !secret.Expiry.IsZero() && secret.Expiry.Before(time.Now()) {
return "", errSecretNotFound
}
return secret.Secret, nil
}