luzifer/sqlapi
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Rewrite to support multiple database types

Browse source
This commit is contained in:
Knut Ahlers 2024-06-28 22:31:45 +02:00
parent ee41ba43b8
commit 5580e98911
Signed by: luzifer
SSH key fingerprint: SHA256:/xtE5lCgiRDQr8SLxHMS92ZBlACmATUmF1crK16Ks4E
10 changed files with 287 additions and 164 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -1 +1 @@
mysqlapi
sqlapi

8
Dockerfile
View file

@ -1,7 +1,7 @@
FROM golang:alpine as builder
COPY . /go/src/github.com/Luzifer/mysqlapi
WORKDIR /go/src/github.com/Luzifer/mysqlapi
COPY . /go/src/github.com/Luzifer/sqlapi
WORKDIR /go/src/github.com/Luzifer/sqlapi
RUN set -ex \
&& apk add --update git \
@ -19,11 +19,11 @@ RUN set -ex \
&& apk --no-cache add \
ca-certificates
COPY --from=builder /go/bin/mysqlapi /usr/local/bin/mysqlapi
COPY --from=builder /go/bin/sqlapi /usr/local/bin/sqlapi
EXPOSE 3000
ENTRYPOINT ["/usr/local/bin/mysqlapi"]
ENTRYPOINT ["/usr/local/bin/sqlapi"]
CMD ["--"]
# vim: set ft=Dockerfile:

41
README.md
View file

@ -1,10 +1,10 @@
# Luzifer / mysqlapi
# Luzifer / sqlapi
This repo contains a simple-ish web-application to translate HTTP POST requests into MySQL queries.
This repo contains a simple-ish web-application to translate HTTP POST requests into SQL queries.
## Why?!?
I had the requirement to do SQL queries from fairly simple scripts without the possibility to add a MySQL client. As HTTP calls are possible in nearly every environement the idea was to have an API to execute arbitrary SQL statements over a JSON POST-API.
I had the requirement to do SQL queries from fairly simple scripts without the possibility to add a SQL client. As HTTP calls are possible in nearly every environement the idea was to have an API to execute arbitrary SQL statements over a JSON POST-API.
## Security
@ -12,7 +12,7 @@ I had the requirement to do SQL queries from fairly simple scripts without the p
Having stated that as clearly as possible: This API does not limit the type of queries being executed. The only thing saving you might be the permissions of the user you configured in the DSN given to the tool. If you gave it global admin permissions, well - you've just handed over your database server.
In general make sure you understood what is possible using this and limit access to an absolute minimum. Your data got lost / leaked? I did warn you.
In general make sure you understood what is possible using this and limit access to an absolute minimum. Your data got lost / leaked? **I did warn you.**
## How to use?
@ -24,22 +24,39 @@ Content-Type: application/json
```
```console
# mysqlapi --help
Usage of mysqlapi:
--dsn string MySQL DSN to connect to: [username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]
# sqlapi --help
Usage of sqlapi:
--db-type string Database type to connect to
--dsn string DSN to connect to (see README for formats)
--listen string Port/IP to listen on (default ":3000")
--log-level string Log level (debug, info, warn, error, fatal) (default "info")
--version Prints current version and exits
# mysqlapi \
# sqlapi \
--db-type mysql \
--dsn "limiteduser:verysecretpass@tcp(mydatabase.cluster.local:3306)/?charset=utf8mb4&parseTime=True&loc=Local" \
--listen 127.0.0.1:7895
INFO[0000] mysqlapi started addr="127.0.0.1:7895" version=dev
INFO[0000] sqlapi started addr="127.0.0.1:7895" version=dev
# curl -s --data-binary @select.json localhost:7895/mysqlapi_test | jq .
# curl -s --data-binary @select.json localhost:7895/sqlapi_test | jq .
```
**Request format**
### DSN formats
- **MySQL / MariaDB**
- `[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]`
- For parameters see [github.com/go-sql-driver/mysql](https://pkg.go.dev/github.com/go-sql-driver/mysql#readme-dsn-data-source-name)
- **Postgres / CockroachDB**
- `postgres://jack:secret@pg.example.com:5432/mydb?sslmode=verify-ca`
- `user=jack password=secret host=pg.example.com port=5432 dbname=mydb sslmode=verify-ca`
- For parameters see [github.com/jackc/pgx/v5](https://pkg.go.dev/github.com/jackc/pgx/v5@v5.6.0/pgconn#ParseConfig)
### Request format
- **MySQL / MariaDB**
- Placeholders use `?` for arguments
- **Postgres / CockroachDB**
- Placeholders use `$1, $2, ...` for arguments
```json
[
@ -50,7 +67,7 @@ INFO[0000] mysqlapi started addr="127.0.0.1:7895" v
]
```
**Response format**
### Response format
```json
[

42
connect.go Normal file
View file

@ -0,0 +1,42 @@
package main
import (
"database/sql"
"fmt"
"github.com/go-sql-driver/mysql"
"github.com/jackc/pgx/v5"
)
func connect(database string) (db *sql.DB, err error) {
switch cfg.DBType {
case "mysql", "mariadb":
connInfo, err := mysql.ParseDSN(cfg.DSN)
if err != nil {
return nil, fmt.Errorf("parsing DSN: %w", err)
}
connInfo.DBName = database
if db, err = sql.Open("mysql", connInfo.FormatDSN()); err != nil {
return nil, fmt.Errorf("opening db connection: %w", err)
}
return db, nil
case "postgres", "pg", "crdb":
connInfo, err := pgx.ParseConfig(cfg.DSN)
if err != nil {
return nil, fmt.Errorf("parsing DSN: %w", err)
}
connInfo.Database = database
if db, err = sql.Open("pgx", connInfo.ConnString()); err != nil {
return nil, fmt.Errorf("opening db connection: %w", err)
}
return db, nil
default:
return nil, fmt.Errorf("unknown database type %q", cfg.DBType)
}
}

8
go.mod
View file

@ -1,4 +1,4 @@
module github.com/Luzifer/mysqlapi
module github.com/Luzifer/sqlapi
go 1.20
@ -7,14 +7,20 @@ require (
github.com/go-sql-driver/mysql v1.8.1
github.com/gofrs/uuid v4.4.0+incompatible
github.com/gorilla/mux v1.8.1
github.com/jackc/pgx/v5 v5.6.0
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.9.3
)
require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/crypto v0.17.0 // indirect
golang.org/x/sys v0.21.0 // indirect
golang.org/x/text v0.14.0 // indirect
gopkg.in/validator.v2 v2.0.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

17
go.sum
View file

@ -11,22 +11,37 @@ github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1
github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY=
github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw=
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/validator.v2 v2.0.1 h1:xF0KWyGWXm/LM2G1TrEjqOu4pa6coO9AlWSf3msVfDY=

153
http.go
View file

@ -1,22 +1,20 @@
package main
import (
"database/sql"
"encoding/json"
"fmt"
"net/http"
"reflect"
"github.com/go-sql-driver/mysql"
"github.com/Luzifer/sqlapi/pkg/query"
"github.com/Luzifer/sqlapi/pkg/types"
"github.com/gofrs/uuid"
"github.com/gorilla/mux"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
type (
request [][]any
response [][]map[string]any
request []types.Query
response []types.QueryResult
)
/*
@ -31,58 +29,6 @@ type (
]
*/
func executeQuery(db *sql.DB, query []any, resp *response) error {
if len(query) == 0 {
return errors.New("no query given")
}
qs, ok := query[0].(string)
if !ok {
return errors.Errorf("expected query as string in first argument, got %T", query[0])
}
rows, err := db.Query(qs, query[1:]...)
if err != nil {
return errors.Wrap(err, "executing query")
}
var respForQuery []map[string]any
colTypes, err := rows.ColumnTypes()
if err != nil {
return errors.Wrap(err, "getting column types")
}
for rows.Next() {
var (
scanNames []string
scanSet []any
)
for _, col := range colTypes {
scanNames = append(scanNames, col.Name())
scanSet = append(scanSet, reflect.New(col.ScanType()).Interface())
}
if err = rows.Err(); err != nil {
return errors.Wrap(err, "iterating rows")
}
if err = rows.Scan(scanSet...); err != nil {
return errors.Wrap(err, "scanning row")
}
respForQuery = append(respForQuery, scanSetToObject(scanNames, scanSet))
}
if err = rows.Err(); err != nil {
return errors.Wrap(err, "iterating rows (final)")
}
*resp = append(*resp, respForQuery)
return nil
}
func handleRequest(w http.ResponseWriter, r *http.Request) {
var (
connID = uuid.Must(uuid.NewV4()).String()
@ -94,7 +40,7 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
connError = func(err error, reason string, code int) {
logger.WithError(err).Error(reason)
http.Error(w, fmt.Sprintf("an error occurred: %s", connID), http.StatusInternalServerError)
http.Error(w, fmt.Sprintf("an error occurred: %s", connID), code)
}
)
@ -102,16 +48,9 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("X-Conn-ID", connID)
connInfo, err := mysql.ParseDSN(cfg.DSN)
db, err := connect(database)
if err != nil {
connError(err, "parsing DSN", http.StatusInternalServerError)
return
}
connInfo.DBName = database
db, err := sql.Open("mysql", connInfo.FormatDSN())
if err != nil {
connError(err, "opening db connection", http.StatusInternalServerError)
connError(err, "connecting to server", http.StatusInternalServerError)
return
}
defer func() {
@ -122,6 +61,7 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
var (
req request
res types.QueryResult
resp response
)
@ -130,11 +70,12 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
return
}
for i, query := range req {
if err = executeQuery(db, query, &resp); err != nil {
for i, qry := range req {
if res, err = query.RunQuery(db, qry); err != nil {
connError(err, fmt.Sprintf("executing query %d", i), http.StatusInternalServerError)
return
}
resp = append(resp, res)
}
if err = json.NewEncoder(w).Encode(resp); err != nil {
@ -142,75 +83,3 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
return
}
}
//nolint:gocognit,gocyclo // contains simple type conversions
func scanSetToObject(scanNames []string, scanSet []any) map[string]any {
row := make(map[string]any)
for idx, name := range scanNames {
// Some types are not very JSON friendly, lets make them
switch tv := scanSet[idx].(type) {
case *sql.NullBool:
if tv.Valid {
scanSet[idx] = tv.Bool
} else {
scanSet[idx] = nil
}
case *sql.NullByte:
if tv.Valid {
scanSet[idx] = tv.Byte
} else {
scanSet[idx] = nil
}
case *sql.NullFloat64:
if tv.Valid {
scanSet[idx] = tv.Float64
} else {
scanSet[idx] = nil
}
case *sql.NullInt16:
if tv.Valid {
scanSet[idx] = tv.Int16
} else {
scanSet[idx] = nil
}
case *sql.NullInt32:
if tv.Valid {
scanSet[idx] = tv.Int32
} else {
scanSet[idx] = nil
}
case *sql.NullInt64:
if tv.Valid {
scanSet[idx] = tv.Int64
} else {
scanSet[idx] = nil
}
case *sql.NullString:
if tv.Valid {
scanSet[idx] = tv.String
} else {
scanSet[idx] = nil
}
case *sql.NullTime:
if tv.Valid {
scanSet[idx] = tv.Time
} else {
scanSet[idx] = nil
}
case *sql.RawBytes:
scanSet[idx] = string(*tv)
}
row[name] = scanSet[idx]
}
return row
}

7
main.go
View file

@ -15,9 +15,10 @@ import (
var (
cfg = struct {
DBType string `flag:"db-type" default:"" description:"Database type to connect to"`
DSN string `flag:"dsn" default:"" description:"DSN to connect to (see README for formats)"`
Listen string `flag:"listen" default:":3000" description:"Port/IP to listen on"`
LogLevel string `flag:"log-level" default:"info" description:"Log level (debug, info, warn, error, fatal)"`
DSN string `flag:"dsn" default:"" description:"MySQL DSN to connect to: [username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]"`
VersionAndExit bool `flag:"version" default:"false" description:"Prints current version and exits"`
}{}
@ -46,7 +47,7 @@ func main() {
}
if cfg.VersionAndExit {
fmt.Printf("mysqlapi %s\n", version)
fmt.Printf("sqlapi %s\n", version) //nolint:forbidigo
os.Exit(0)
}
@ -62,7 +63,7 @@ func main() {
logrus.WithFields(logrus.Fields{
"addr": cfg.Listen,
"version": version,
}).Info("mysqlapi started")
}).Info("sqlapi started")
if err = server.ListenAndServe(); err != nil {
logrus.WithError(err).Fatal("listening for HTTP")

134
pkg/query/query.go Normal file
View file

@ -0,0 +1,134 @@
// Package query converts data from the query into the response types
package query
import (
"database/sql"
"fmt"
"reflect"
"github.com/Luzifer/sqlapi/pkg/types"
)
// RunQuery takes a Query containing at least the query-string as
// the first parameter and optionally any argument referenced. It
// runs the query using the connection stored inside the Adapter.
// The result then is parsed into the QueryResult form using the
// field names as keys and values as typed values.
func RunQuery(db *sql.DB, q types.Query) (types.QueryResult, error) {
qs, err := q.QueryString()
if err != nil {
return nil, fmt.Errorf("getting query-string: %w", err)
}
rows, err := db.Query(qs, q.Args()...)
if err != nil {
return nil, fmt.Errorf("executing query: %w", err)
}
var respForQuery types.QueryResult
colTypes, err := rows.ColumnTypes()
if err != nil {
return nil, fmt.Errorf("getting column types: %w", err)
}
for rows.Next() {
var (
scanNames []string
scanSet []any
)
for _, col := range colTypes {
scanNames = append(scanNames, col.Name())
scanSet = append(scanSet, reflect.New(col.ScanType()).Interface())
}
if err = rows.Err(); err != nil {
return nil, fmt.Errorf("iterating rows: %w", err)
}
if err = rows.Scan(scanSet...); err != nil {
return nil, fmt.Errorf("scanning row: %w", err)
}
respForQuery = append(respForQuery, scanSetToObject(scanNames, scanSet))
}
if err = rows.Err(); err != nil {
return nil, fmt.Errorf("iterating rows (final): %w", err)
}
return respForQuery, nil
}
//nolint:gocognit,gocyclo // contains simple type conversions
func scanSetToObject(scanNames []string, scanSet []any) map[string]any {
row := make(map[string]any)
for idx, name := range scanNames {
// Some types are not very JSON friendly, lets make them
switch tv := scanSet[idx].(type) {
case *sql.NullBool:
if tv.Valid {
scanSet[idx] = tv.Bool
} else {
scanSet[idx] = nil
}
case *sql.NullByte:
if tv.Valid {
scanSet[idx] = tv.Byte
} else {
scanSet[idx] = nil
}
case *sql.NullFloat64:
if tv.Valid {
scanSet[idx] = tv.Float64
} else {
scanSet[idx] = nil
}
case *sql.NullInt16:
if tv.Valid {
scanSet[idx] = tv.Int16
} else {
scanSet[idx] = nil
}
case *sql.NullInt32:
if tv.Valid {
scanSet[idx] = tv.Int32
} else {
scanSet[idx] = nil
}
case *sql.NullInt64:
if tv.Valid {
scanSet[idx] = tv.Int64
} else {
scanSet[idx] = nil
}
case *sql.NullString:
if tv.Valid {
scanSet[idx] = tv.String
} else {
scanSet[idx] = nil
}
case *sql.NullTime:
if tv.Valid {
scanSet[idx] = tv.Time
} else {
scanSet[idx] = nil
}
case *sql.RawBytes:
scanSet[idx] = string(*tv)
}
row[name] = scanSet[idx]
}
return row
}

39
pkg/types/types.go Normal file
View file

@ -0,0 +1,39 @@
// Package types defines datatypes to work with
package types
import (
"fmt"
)
type (
// Query contains the query itself in the form the Adapter requires
// and all arguments referenced in the query-string
// (i.e. `[]any{"INSERT INTO foo VALUES (?, ?)", "bar", 1}`)
Query []any
// QueryResult contains the fields returned from the Query as a map.
// The names of the fields are used as keys.
QueryResult []map[string]any
)
// Args returns the arguments for the QueryString
func (q Query) Args() []any {
if len(q) == 0 {
return nil
}
return q[1:]
}
// QueryString returns the first argument as the query string
func (q Query) QueryString() (string, error) {
if len(q) == 0 {
return "", fmt.Errorf("no query given")
}
qs, ok := q[0].(string)
if !ok {
return "", fmt.Errorf("expected query as string in first argument, got %T", q[0])
}
return qs, nil
}