From 8351f955645287ef1a5336e0aca9343fe01b97c2 Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Wed, 6 Sep 2023 12:28:03 +0200 Subject: [PATCH] Refactor & add support for direct PDF download Signed-off-by: Knut Ahlers --- assets.go | 87 +++++++++++----- go.mod | 1 + go.sum | 2 + helpers.go | 26 +++++ jobStatus.go | 88 ++++++++++++++++ main.go | 283 +++++++++----------------------------------------- processing.go | 137 ++++++++++++++++++++++++ 7 files changed, 365 insertions(+), 259 deletions(-) create mode 100644 helpers.go create mode 100644 jobStatus.go create mode 100644 processing.go diff --git a/assets.go b/assets.go index 23635e5..f96a21c 100644 --- a/assets.go +++ b/assets.go @@ -11,15 +11,52 @@ import ( "strings" "github.com/Luzifer/go_helpers/v2/str" + "github.com/pkg/errors" "github.com/gofrs/uuid" ) -func shouldPackFile(extension string) bool { - return str.StringInSlice(extension, []string{ - ".log", - ".pdf", +func buildAssetsTAR(uid uuid.UUID) (io.Reader, error) { + buf := new(bytes.Buffer) + w := tar.NewWriter(buf) + + basePath := pathFromUUID(uid, filenameOutputDir) + err := filepath.Walk(basePath, func(p string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if !shouldPackFile(path.Ext(info.Name())) { + return nil + } + + tarInfo, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + tarInfo.Name = strings.TrimLeft(strings.Replace(p, basePath, "", 1), "/\\") + err = w.WriteHeader(tarInfo) + if err != nil { + return err + } + osFile, err := os.Open(p) // #nosec G304 + if err != nil { + return err + } + + if _, err := io.Copy(w, osFile); err != nil { + return err + } + osFile.Close() // #nosec G104 + + return nil }) + + if err != nil { + return nil, err + } + + return buf, w.Close() } func buildAssetsZIP(uid uuid.UUID) (io.Reader, error) { @@ -65,9 +102,11 @@ func buildAssetsZIP(uid uuid.UUID) (io.Reader, error) { return buf, w.Close() } -func buildAssetsTAR(uid uuid.UUID) (io.Reader, error) { - buf := new(bytes.Buffer) - w := tar.NewWriter(buf) +func getAssetsPDF(uid uuid.UUID) (io.Reader, error) { + var ( + buf = new(bytes.Buffer) + found bool + ) basePath := pathFromUUID(uid, filenameOutputDir) err := filepath.Walk(basePath, func(p string, info os.FileInfo, err error) error { @@ -75,35 +114,35 @@ func buildAssetsTAR(uid uuid.UUID) (io.Reader, error) { return err } - if !shouldPackFile(path.Ext(info.Name())) { + if path.Ext(info.Name()) != ".pdf" { return nil } - tarInfo, err := tar.FileInfoHeader(info, "") - if err != nil { - return err - } - tarInfo.Name = strings.TrimLeft(strings.Replace(p, basePath, "", 1), "/\\") - err = w.WriteHeader(tarInfo) - if err != nil { - return err - } osFile, err := os.Open(p) // #nosec G304 if err != nil { - return err + return errors.Wrap(err, "opening file") } - if _, err := io.Copy(w, osFile); err != nil { - return err + if _, err := io.Copy(buf, osFile); err != nil { + return errors.Wrap(err, "reading file") } osFile.Close() // #nosec G104 - return nil + found = true + return filepath.SkipAll }) - if err != nil { - return nil, err + if !found { + // We found no file + return nil, errors.New("no pdf found") } - return buf, w.Close() + return buf, err +} + +func shouldPackFile(extension string) bool { + return str.StringInSlice(extension, []string{ + ".log", + ".pdf", + }) } diff --git a/go.mod b/go.mod index 68166df..1545836 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/Luzifer/rconfig/v2 v2.4.0 github.com/gofrs/uuid v4.4.0+incompatible github.com/gorilla/mux v1.8.0 + github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 ) diff --git a/go.sum b/go.sum index fd9904b..6675d1a 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,8 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= diff --git a/helpers.go b/helpers.go new file mode 100644 index 0000000..215f0a0 --- /dev/null +++ b/helpers.go @@ -0,0 +1,26 @@ +package main + +import ( + "net/http" + "net/url" + "path" + + "github.com/gofrs/uuid" + "github.com/sirupsen/logrus" +) + +func pathFromUUID(uid uuid.UUID, filename string) string { + return path.Join(cfg.StorageDir, uid.String(), filename) +} + +func serverErrorf(res http.ResponseWriter, err error, tpl string, args ...interface{}) { + logrus.WithError(err).Errorf(tpl, args...) + http.Error(res, "An error occurred. See details in log.", http.StatusInternalServerError) +} + +func urlMust(u *url.URL, err error) *url.URL { + if err != nil { + logrus.WithError(err).Fatal("Unable to retrieve URL from router") + } + return u +} diff --git a/jobStatus.go b/jobStatus.go new file mode 100644 index 0000000..5fcaee2 --- /dev/null +++ b/jobStatus.go @@ -0,0 +1,88 @@ +package main + +import ( + "encoding/json" + "net/http" + "os" + "time" + + "github.com/gofrs/uuid" + "github.com/gorilla/mux" +) + +type ( + jobStatus struct { + UUID string `json:"uuid"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Status status `json:"status"` + } + + status string +) + +const ( + statusCreated = "created" + statusStarted = "started" + statusError = "error" + statusFinished = "finished" +) + +func getJobStatus(res http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + uid, err := uuid.FromString(vars["uid"]) + if err != nil { + http.Error(res, "UUID had unexpected format!", http.StatusBadRequest) + return + } + + if status, err := loadStatusByUUID(uid); err == nil { + if encErr := json.NewEncoder(res).Encode(status); encErr != nil { + serverErrorf(res, encErr, "Unable to serialize status file") + return + } + } else { + serverErrorf(res, err, "Unable to read status file") + return + } +} + +func loadStatusByUUID(uid uuid.UUID) (*jobStatus, error) { + statusFile := pathFromUUID(uid, filenameStatus) + + status := jobStatus{} + // #nosec G304 + if f, err := os.Open(statusFile); err == nil { + defer f.Close() + if err = json.NewDecoder(f).Decode(&status); err != nil { + return nil, err + } + } else { + return nil, err + } + + return &status, nil +} + +func (s *jobStatus) UpdateStatus(st status) { + s.Status = st + s.UpdatedAt = time.Now() +} + +func (s jobStatus) Save() error { + uid, _ := uuid.FromString(s.UUID) // #nosec G104 + f, err := os.Create(pathFromUUID(uid, filenameStatusTemp)) + if err != nil { + return err + } + defer f.Close() + + if err = json.NewEncoder(f).Encode(s); err != nil { + return err + } + + return os.Rename( + pathFromUUID(uid, filenameStatusTemp), + pathFromUUID(uid, filenameStatus), + ) +} diff --git a/main.go b/main.go index 3a8a85d..5ed9b99 100644 --- a/main.go +++ b/main.go @@ -1,23 +1,26 @@ package main import ( - "encoding/json" "fmt" "io" - "math" "net/http" - "net/url" "os" - "os/exec" - "path" - "strconv" "time" "github.com/Luzifer/rconfig/v2" + "github.com/pkg/errors" "github.com/gofrs/uuid" "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus" +) + +const ( + filenameInput = "input.zip" + filenameStatus = "status.json" + filenameStatusTemp = "status.tmp.json" + filenameOutputDir = "output" + sleepBase = 1.5 ) var ( @@ -32,207 +35,50 @@ var ( router = mux.NewRouter() ) -type status string - -const ( - statusCreated = "created" - statusStarted = "started" - statusError = "error" - statusFinished = "finished" -) - -const ( - filenameInput = "input.zip" - filenameStatus = "status.json" - filenameStatusTemp = "status.tmp.json" - filenameOutputDir = "output" - sleepBase = 1.5 -) - -type jobStatus struct { - UUID string `json:"uuid"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - Status status `json:"status"` -} - -func loadStatusByUUID(uid uuid.UUID) (*jobStatus, error) { - statusFile := pathFromUUID(uid, filenameStatus) - - status := jobStatus{} - // #nosec G304 - if f, err := os.Open(statusFile); err == nil { - defer f.Close() - if err = json.NewDecoder(f).Decode(&status); err != nil { - return nil, err - } - } else { - return nil, err - } - - return &status, nil -} - -func (s *jobStatus) UpdateStatus(st status) { - s.Status = st - s.UpdatedAt = time.Now() -} - -func (s jobStatus) Save() error { - uid, _ := uuid.FromString(s.UUID) // #nosec G104 - f, err := os.Create(pathFromUUID(uid, filenameStatusTemp)) - if err != nil { - return err - } - defer f.Close() - - if err = json.NewEncoder(f).Encode(s); err != nil { - return err - } - - return os.Rename( - pathFromUUID(uid, filenameStatusTemp), - pathFromUUID(uid, filenameStatus), - ) -} - -func urlMust(u *url.URL, err error) *url.URL { - if err != nil { - log.WithError(err).Fatal("Unable to retrieve URL from router") - } - return u -} - -func init() { +func initApp() error { rconfig.AutoEnv(true) if err := rconfig.Parse(&cfg); err != nil { - log.WithError(err).Fatal("Unable to parse commandline options") + return errors.Wrap(err, "parsing cli options") } - if cfg.VersionAndExit { - fmt.Printf("tex-api %s\n", version) - os.Exit(0) - } + return nil } func main() { - router.HandleFunc("/job", startNewJob).Methods("POST").Name("startNewJob") - router.HandleFunc("/job/{uid:[0-9a-z-]{36}}", getJobStatus).Methods("GET").Name("getJobStatus") - router.HandleFunc("/job/{uid:[0-9a-z-]{36}}/wait", waitForJob).Methods("GET").Name("waitForJob") - router.HandleFunc("/job/{uid:[0-9a-z-]{36}}/download", downloadAssets).Methods("GET").Name("downloadAssets") - - if err := http.ListenAndServe(cfg.Listen, router); err != nil { - log.WithError(err).Fatal("HTTP server exited with error") - } -} - -func serverErrorf(res http.ResponseWriter, err error, tpl string, args ...interface{}) { - log.WithError(err).Errorf(tpl, args...) - http.Error(res, "An error occurred. See details in log.", http.StatusInternalServerError) -} - -func pathFromUUID(uid uuid.UUID, filename string) string { - return path.Join(cfg.StorageDir, uid.String(), filename) -} - -func startNewJob(res http.ResponseWriter, r *http.Request) { - jobUUID := uuid.Must(uuid.NewV4()) - inputFile := pathFromUUID(jobUUID, filenameInput) - statusFile := pathFromUUID(jobUUID, filenameStatus) - - if err := os.Mkdir(path.Dir(inputFile), 0750); err != nil { - log.WithError(err).Errorf("Unable to create job dir %q", path.Dir(inputFile)) + var err error + if err = initApp(); err != nil { + logrus.WithError(err).Fatal("app initialization failed") } - if f, err := os.Create(inputFile); err == nil { - defer f.Close() - if _, copyErr := io.Copy(f, r.Body); copyErr != nil { - serverErrorf(res, copyErr, "Unable to copy input file %q", inputFile) - return - } - f.Sync() // #nosec G104 - } else { - serverErrorf(res, err, "Unable to write input file %q", inputFile) - return + if cfg.VersionAndExit { + logrus.WithField("version", version).Info("tex-api") + os.Exit(0) } - status := jobStatus{ - UUID: jobUUID.String(), - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - Status: statusCreated, - } - if err := status.Save(); err != nil { - serverErrorf(res, err, "Unable to create status file %q", statusFile) - return + router.HandleFunc("/job", startNewJob). + Methods("POST"). + Name("startNewJob") + + router.HandleFunc("/job/{uid:[0-9a-z-]{36}}", getJobStatus). + Methods("GET"). + Name("getJobStatus") + + router.HandleFunc("/job/{uid:[0-9a-z-]{36}}/wait", waitForJob). + Methods("GET"). + Name("waitForJob") + + router.HandleFunc("/job/{uid:[0-9a-z-]{36}}/download", downloadAssets). + Methods("GET"). + Name("downloadAssets") + + server := &http.Server{ + Addr: cfg.Listen, + Handler: router, + ReadHeaderTimeout: time.Second, } - go jobProcessor(jobUUID) - - u := urlMust(router.Get("waitForJob").URL("uid", jobUUID.String())) - http.Redirect(res, r, u.String(), http.StatusFound) -} - -func getJobStatus(res http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - uid, err := uuid.FromString(vars["uid"]) - if err != nil { - http.Error(res, "UUID had unexpected format!", http.StatusBadRequest) - return - } - - if status, err := loadStatusByUUID(uid); err == nil { - if encErr := json.NewEncoder(res).Encode(status); encErr != nil { - serverErrorf(res, encErr, "Unable to serialize status file") - return - } - } else { - serverErrorf(res, err, "Unable to read status file") - return - } -} - -func waitForJob(res http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - uid, err := uuid.FromString(vars["uid"]) - if err != nil { - http.Error(res, "UUID had unexpected format!", http.StatusBadRequest) - return - } - - var loop int - if v := r.URL.Query().Get("loop"); v != "" { - if pv, convErr := strconv.Atoi(v); convErr == nil { - loop = pv - } - } - loop++ - - status, err := loadStatusByUUID(uid) - if err != nil { - serverErrorf(res, err, "Unable to read status file") - return - } - - switch status.Status { - case statusCreated: - fallthrough - - case statusStarted: - u := urlMust(router.Get("waitForJob").URL("uid", uid.String())) - u.Query().Set("loop", strconv.Itoa(loop)) - - <-time.After(time.Duration(math.Pow(sleepBase, float64(loop))) * time.Second) - - http.Redirect(res, r, u.String(), http.StatusFound) - return - - case statusError: - http.Error(res, "Processing ran into an error.", http.StatusInternalServerError) - - case statusFinished: - u := urlMust(router.Get("downloadAssets").URL("uid", uid.String())) - http.Redirect(res, r, u.String(), http.StatusFound) + if err := server.ListenAndServe(); err != nil { + logrus.WithError(err).Fatal("HTTP server exited with error") } } @@ -255,13 +101,19 @@ func downloadAssets(res http.ResponseWriter, r *http.Request) { contentType = "application/tar" content, err = buildAssetsTAR(uid) filename = uid.String() + ".tar" + + case "application/pdf": + contentType = "application/pdf" + content, err = getAssetsPDF(uid) + filename = uid.String() + ".pdf" + default: content, err = buildAssetsZIP(uid) filename = uid.String() + ".zip" } if err != nil { - serverErrorf(res, err, "Unable to generate downloadable asset") + serverErrorf(res, err, "generating downloadable asset") return } @@ -271,42 +123,3 @@ func downloadAssets(res http.ResponseWriter, r *http.Request) { io.Copy(res, content) // #nosec G104 } - -func jobProcessor(uid uuid.UUID) { - logger := log.WithField("uuid", uid) - logger.Info("Started processing") - - processingDir := path.Dir(pathFromUUID(uid, filenameStatus)) - status, err := loadStatusByUUID(uid) - if err != nil { - logger.WithError(err).Error("Unable to load status file in processing job") - return - } - - cmd := exec.Command("/bin/bash", cfg.Script) // #nosec G204 - cmd.Dir = processingDir - cmd.Stderr = logger.WriterLevel(log.InfoLevel) // Bash uses stderr for `-x` parameter - - status.UpdateStatus(statusStarted) - if err := status.Save(); err != nil { - logger.WithError(err).Error("Unable to save status file") - return - } - - if err := cmd.Run(); err != nil { - logger.WithError(err).Error("Processing failed") - status.UpdateStatus(statusError) - if err := status.Save(); err != nil { - logger.WithError(err).Error("Unable to save status file") - return - } - return - } - - status.UpdateStatus(statusFinished) - if err := status.Save(); err != nil { - logger.WithError(err).Error("Unable to save status file") - return - } - logger.Info("Finished processing") -} diff --git a/processing.go b/processing.go new file mode 100644 index 0000000..156471c --- /dev/null +++ b/processing.go @@ -0,0 +1,137 @@ +package main + +import ( + "io" + "math" + "net/http" + "os" + "os/exec" + "path" + "strconv" + "time" + + "github.com/gofrs/uuid" + "github.com/gorilla/mux" + "github.com/sirupsen/logrus" +) + +func jobProcessor(uid uuid.UUID) { + logger := logrus.WithField("uuid", uid) + logger.Info("Started processing") + + processingDir := path.Dir(pathFromUUID(uid, filenameStatus)) + status, err := loadStatusByUUID(uid) + if err != nil { + logger.WithError(err).Error("Unable to load status file in processing job") + return + } + + cmd := exec.Command("/bin/bash", cfg.Script) // #nosec G204 + cmd.Dir = processingDir + cmd.Stderr = logger.WriterLevel(logrus.InfoLevel) // Bash uses stderr for `-x` parameter + + status.UpdateStatus(statusStarted) + if err := status.Save(); err != nil { + logger.WithError(err).Error("Unable to save status file") + return + } + + if err := cmd.Run(); err != nil { + logger.WithError(err).Error("Processing failed") + status.UpdateStatus(statusError) + if err := status.Save(); err != nil { + logger.WithError(err).Error("Unable to save status file") + return + } + return + } + + status.UpdateStatus(statusFinished) + if err := status.Save(); err != nil { + logger.WithError(err).Error("Unable to save status file") + return + } + logger.Info("Finished processing") +} + +func startNewJob(res http.ResponseWriter, r *http.Request) { + jobUUID := uuid.Must(uuid.NewV4()) + inputFile := pathFromUUID(jobUUID, filenameInput) + statusFile := pathFromUUID(jobUUID, filenameStatus) + + if err := os.Mkdir(path.Dir(inputFile), 0750); err != nil { + logrus.WithError(err).Errorf("Unable to create job dir %q", path.Dir(inputFile)) + } + + if f, err := os.Create(inputFile); err == nil { + defer f.Close() + if _, copyErr := io.Copy(f, r.Body); copyErr != nil { + serverErrorf(res, copyErr, "Unable to copy input file %q", inputFile) + return + } + f.Sync() // #nosec G104 + } else { + serverErrorf(res, err, "Unable to write input file %q", inputFile) + return + } + + status := jobStatus{ + UUID: jobUUID.String(), + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + Status: statusCreated, + } + if err := status.Save(); err != nil { + serverErrorf(res, err, "Unable to create status file %q", statusFile) + return + } + + go jobProcessor(jobUUID) + + u := urlMust(router.Get("waitForJob").URL("uid", jobUUID.String())) + http.Redirect(res, r, u.String(), http.StatusFound) +} + +func waitForJob(res http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + uid, err := uuid.FromString(vars["uid"]) + if err != nil { + http.Error(res, "UUID had unexpected format!", http.StatusBadRequest) + return + } + + var loop int + if v := r.URL.Query().Get("loop"); v != "" { + if pv, convErr := strconv.Atoi(v); convErr == nil { + loop = pv + } + } + loop++ + + status, err := loadStatusByUUID(uid) + if err != nil { + serverErrorf(res, err, "Unable to read status file") + return + } + + switch status.Status { + case statusCreated: + fallthrough + + case statusStarted: + u := urlMust(router.Get("waitForJob").URL("uid", uid.String())) + u.Query().Set("loop", strconv.Itoa(loop)) + + <-time.After(time.Duration(math.Pow(sleepBase, float64(loop))) * time.Second) + + http.Redirect(res, r, u.String(), http.StatusFound) + return + + case statusError: + http.Error(res, "Processing ran into an error.", http.StatusInternalServerError) + + case statusFinished: + u := urlMust(router.Get("downloadAssets").URL("uid", uid.String())) + http.Redirect(res, r, u.String(), http.StatusFound) + } +}