Refactor & add support for direct PDF download

Signed-off-by: Knut Ahlers <knut@ahlers.me>

assets.go

@@ -11,15 +11,52 @@ import (
 	"strings"
 
 	"github.com/Luzifer/go_helpers/v2/str"
+	"github.com/pkg/errors"
 
 	"github.com/gofrs/uuid"
 )
 
-func shouldPackFile(extension string) bool {
+func buildAssetsTAR(uid uuid.UUID) (io.Reader, error) {
-	return str.StringInSlice(extension, []string{
+	buf := new(bytes.Buffer)
-		".log",
+	w := tar.NewWriter(buf)
-		".pdf",
+
+	basePath := pathFromUUID(uid, filenameOutputDir)
+	err := filepath.Walk(basePath, func(p string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if !shouldPackFile(path.Ext(info.Name())) {
+			return nil
+		}
+
+		tarInfo, err := tar.FileInfoHeader(info, "")
+		if err != nil {
+			return err
+		}
+		tarInfo.Name = strings.TrimLeft(strings.Replace(p, basePath, "", 1), "/\\")
+		err = w.WriteHeader(tarInfo)
+		if err != nil {
+			return err
+		}
+		osFile, err := os.Open(p) // #nosec G304
+		if err != nil {
+			return err
+		}
+
+		if _, err := io.Copy(w, osFile); err != nil {
+			return err
+		}
+		osFile.Close() // #nosec G104
+
+		return nil
 	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return buf, w.Close()
 }
 
 func buildAssetsZIP(uid uuid.UUID) (io.Reader, error) {
@@ -65,9 +102,11 @@ func buildAssetsZIP(uid uuid.UUID) (io.Reader, error) {
 	return buf, w.Close()
 }
 
-func buildAssetsTAR(uid uuid.UUID) (io.Reader, error) {
+func getAssetsPDF(uid uuid.UUID) (io.Reader, error) {
-	buf := new(bytes.Buffer)
+	var (
-	w := tar.NewWriter(buf)
+		buf   = new(bytes.Buffer)
+		found bool
+	)
 
 	basePath := pathFromUUID(uid, filenameOutputDir)
 	err := filepath.Walk(basePath, func(p string, info os.FileInfo, err error) error {
@@ -75,35 +114,35 @@ func buildAssetsTAR(uid uuid.UUID) (io.Reader, error) {
 			return err
 		}
 
-		if !shouldPackFile(path.Ext(info.Name())) {
+		if path.Ext(info.Name()) != ".pdf" {
 			return nil
 		}
 
-		tarInfo, err := tar.FileInfoHeader(info, "")
-		if err != nil {
-			return err
-		}
-		tarInfo.Name = strings.TrimLeft(strings.Replace(p, basePath, "", 1), "/\\")
-		err = w.WriteHeader(tarInfo)
-		if err != nil {
-			return err
-		}
 		osFile, err := os.Open(p) // #nosec G304
 		if err != nil {
-			return err
+			return errors.Wrap(err, "opening file")
 		}
 
-		if _, err := io.Copy(w, osFile); err != nil {
+		if _, err := io.Copy(buf, osFile); err != nil {
-			return err
+			return errors.Wrap(err, "reading file")
 		}
 		osFile.Close() // #nosec G104
 
-		return nil
+		found = true
+		return filepath.SkipAll
 	})
 
-	if err != nil {
+	if !found {
-		return nil, err
+		// We found no file
+		return nil, errors.New("no pdf found")
 	}
 
-	return buf, w.Close()
+	return buf, err
+}
+
+func shouldPackFile(extension string) bool {
+	return str.StringInSlice(extension, []string{
+		".log",
+		".pdf",
+	})
 }

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/Luzifer/rconfig/v2 v2.4.0
 	github.com/gofrs/uuid v4.4.0+incompatible
 	github.com/gorilla/mux v1.8.0
+	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.9.3
 )
 

go.sum

@@ -23,6 +23,8 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc=
 github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=

helpers.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"net/http"
+	"net/url"
+	"path"
+
+	"github.com/gofrs/uuid"
+	"github.com/sirupsen/logrus"
+)
+
+func pathFromUUID(uid uuid.UUID, filename string) string {
+	return path.Join(cfg.StorageDir, uid.String(), filename)
+}
+
+func serverErrorf(res http.ResponseWriter, err error, tpl string, args ...interface{}) {
+	logrus.WithError(err).Errorf(tpl, args...)
+	http.Error(res, "An error occurred. See details in log.", http.StatusInternalServerError)
+}
+
+func urlMust(u *url.URL, err error) *url.URL {
+	if err != nil {
+		logrus.WithError(err).Fatal("Unable to retrieve URL from router")
+	}
+	return u
+}

jobStatus.go

@@ -0,0 +1,88 @@
+package main
+
+import (
+	"encoding/json"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/gofrs/uuid"
+	"github.com/gorilla/mux"
+)
+
+type (
+	jobStatus struct {
+		UUID      string    `json:"uuid"`
+		CreatedAt time.Time `json:"created_at"`
+		UpdatedAt time.Time `json:"updated_at"`
+		Status    status    `json:"status"`
+	}
+
+	status string
+)
+
+const (
+	statusCreated  = "created"
+	statusStarted  = "started"
+	statusError    = "error"
+	statusFinished = "finished"
+)
+
+func getJobStatus(res http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	uid, err := uuid.FromString(vars["uid"])
+	if err != nil {
+		http.Error(res, "UUID had unexpected format!", http.StatusBadRequest)
+		return
+	}
+
+	if status, err := loadStatusByUUID(uid); err == nil {
+		if encErr := json.NewEncoder(res).Encode(status); encErr != nil {
+			serverErrorf(res, encErr, "Unable to serialize status file")
+			return
+		}
+	} else {
+		serverErrorf(res, err, "Unable to read status file")
+		return
+	}
+}
+
+func loadStatusByUUID(uid uuid.UUID) (*jobStatus, error) {
+	statusFile := pathFromUUID(uid, filenameStatus)
+
+	status := jobStatus{}
+	// #nosec G304
+	if f, err := os.Open(statusFile); err == nil {
+		defer f.Close()
+		if err = json.NewDecoder(f).Decode(&status); err != nil {
+			return nil, err
+		}
+	} else {
+		return nil, err
+	}
+
+	return &status, nil
+}
+
+func (s *jobStatus) UpdateStatus(st status) {
+	s.Status = st
+	s.UpdatedAt = time.Now()
+}
+
+func (s jobStatus) Save() error {
+	uid, _ := uuid.FromString(s.UUID) // #nosec G104
+	f, err := os.Create(pathFromUUID(uid, filenameStatusTemp))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	if err = json.NewEncoder(f).Encode(s); err != nil {
+		return err
+	}
+
+	return os.Rename(
+		pathFromUUID(uid, filenameStatusTemp),
+		pathFromUUID(uid, filenameStatus),
+	)
+}

main.go

@@ -1,23 +1,26 @@
 package main
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
-	"math"
 	"net/http"
-	"net/url"
 	"os"
-	"os/exec"
-	"path"
-	"strconv"
 	"time"
 
 	"github.com/Luzifer/rconfig/v2"
+	"github.com/pkg/errors"
 
 	"github.com/gofrs/uuid"
 	"github.com/gorilla/mux"
-	log "github.com/sirupsen/logrus"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	filenameInput      = "input.zip"
+	filenameStatus     = "status.json"
+	filenameStatusTemp = "status.tmp.json"
+	filenameOutputDir  = "output"
+	sleepBase          = 1.5
 )
 
 var (
@@ -32,207 +35,50 @@ var (
 	router  = mux.NewRouter()
 )
 
-type status string
+func initApp() error {
-
-const (
-	statusCreated  = "created"
-	statusStarted  = "started"
-	statusError    = "error"
-	statusFinished = "finished"
-)
-
-const (
-	filenameInput      = "input.zip"
-	filenameStatus     = "status.json"
-	filenameStatusTemp = "status.tmp.json"
-	filenameOutputDir  = "output"
-	sleepBase          = 1.5
-)
-
-type jobStatus struct {
-	UUID      string    `json:"uuid"`
-	CreatedAt time.Time `json:"created_at"`
-	UpdatedAt time.Time `json:"updated_at"`
-	Status    status    `json:"status"`
-}
-
-func loadStatusByUUID(uid uuid.UUID) (*jobStatus, error) {
-	statusFile := pathFromUUID(uid, filenameStatus)
-
-	status := jobStatus{}
-	// #nosec G304
-	if f, err := os.Open(statusFile); err == nil {
-		defer f.Close()
-		if err = json.NewDecoder(f).Decode(&status); err != nil {
-			return nil, err
-		}
-	} else {
-		return nil, err
-	}
-
-	return &status, nil
-}
-
-func (s *jobStatus) UpdateStatus(st status) {
-	s.Status = st
-	s.UpdatedAt = time.Now()
-}
-
-func (s jobStatus) Save() error {
-	uid, _ := uuid.FromString(s.UUID) // #nosec G104
-	f, err := os.Create(pathFromUUID(uid, filenameStatusTemp))
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	if err = json.NewEncoder(f).Encode(s); err != nil {
-		return err
-	}
-
-	return os.Rename(
-		pathFromUUID(uid, filenameStatusTemp),
-		pathFromUUID(uid, filenameStatus),
-	)
-}
-
-func urlMust(u *url.URL, err error) *url.URL {
-	if err != nil {
-		log.WithError(err).Fatal("Unable to retrieve URL from router")
-	}
-	return u
-}
-
-func init() {
 	rconfig.AutoEnv(true)
 	if err := rconfig.Parse(&cfg); err != nil {
-		log.WithError(err).Fatal("Unable to parse commandline options")
+		return errors.Wrap(err, "parsing cli options")
 	}
 
-	if cfg.VersionAndExit {
+	return nil
-		fmt.Printf("tex-api %s\n", version)
-		os.Exit(0)
-	}
 }
 
 func main() {
-	router.HandleFunc("/job", startNewJob).Methods("POST").Name("startNewJob")
+	var err error
-	router.HandleFunc("/job/{uid:[0-9a-z-]{36}}", getJobStatus).Methods("GET").Name("getJobStatus")
+	if err = initApp(); err != nil {
-	router.HandleFunc("/job/{uid:[0-9a-z-]{36}}/wait", waitForJob).Methods("GET").Name("waitForJob")
+		logrus.WithError(err).Fatal("app initialization failed")
-	router.HandleFunc("/job/{uid:[0-9a-z-]{36}}/download", downloadAssets).Methods("GET").Name("downloadAssets")
-
-	if err := http.ListenAndServe(cfg.Listen, router); err != nil {
-		log.WithError(err).Fatal("HTTP server exited with error")
-	}
-}
-
-func serverErrorf(res http.ResponseWriter, err error, tpl string, args ...interface{}) {
-	log.WithError(err).Errorf(tpl, args...)
-	http.Error(res, "An error occurred. See details in log.", http.StatusInternalServerError)
-}
-
-func pathFromUUID(uid uuid.UUID, filename string) string {
-	return path.Join(cfg.StorageDir, uid.String(), filename)
-}
-
-func startNewJob(res http.ResponseWriter, r *http.Request) {
-	jobUUID := uuid.Must(uuid.NewV4())
-	inputFile := pathFromUUID(jobUUID, filenameInput)
-	statusFile := pathFromUUID(jobUUID, filenameStatus)
-
-	if err := os.Mkdir(path.Dir(inputFile), 0750); err != nil {
-		log.WithError(err).Errorf("Unable to create job dir %q", path.Dir(inputFile))
 	}
 
-	if f, err := os.Create(inputFile); err == nil {
+	if cfg.VersionAndExit {
-		defer f.Close()
+		logrus.WithField("version", version).Info("tex-api")
-		if _, copyErr := io.Copy(f, r.Body); copyErr != nil {
+		os.Exit(0)
-			serverErrorf(res, copyErr, "Unable to copy input file %q", inputFile)
-			return
-		}
-		f.Sync() // #nosec G104
-	} else {
-		serverErrorf(res, err, "Unable to write input file %q", inputFile)
-		return
 	}
 
-	status := jobStatus{
+	router.HandleFunc("/job", startNewJob).
-		UUID:      jobUUID.String(),
+		Methods("POST").
-		CreatedAt: time.Now(),
+		Name("startNewJob")
-		UpdatedAt: time.Now(),
+
-		Status:    statusCreated,
+	router.HandleFunc("/job/{uid:[0-9a-z-]{36}}", getJobStatus).
-	}
+		Methods("GET").
-	if err := status.Save(); err != nil {
+		Name("getJobStatus")
-		serverErrorf(res, err, "Unable to create status file %q", statusFile)
+
-		return
+	router.HandleFunc("/job/{uid:[0-9a-z-]{36}}/wait", waitForJob).
+		Methods("GET").
+		Name("waitForJob")
+
+	router.HandleFunc("/job/{uid:[0-9a-z-]{36}}/download", downloadAssets).
+		Methods("GET").
+		Name("downloadAssets")
+
+	server := &http.Server{
+		Addr:              cfg.Listen,
+		Handler:           router,
+		ReadHeaderTimeout: time.Second,
 	}
 
-	go jobProcessor(jobUUID)
+	if err := server.ListenAndServe(); err != nil {
-
+		logrus.WithError(err).Fatal("HTTP server exited with error")
-	u := urlMust(router.Get("waitForJob").URL("uid", jobUUID.String()))
-	http.Redirect(res, r, u.String(), http.StatusFound)
-}
-
-func getJobStatus(res http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	uid, err := uuid.FromString(vars["uid"])
-	if err != nil {
-		http.Error(res, "UUID had unexpected format!", http.StatusBadRequest)
-		return
-	}
-
-	if status, err := loadStatusByUUID(uid); err == nil {
-		if encErr := json.NewEncoder(res).Encode(status); encErr != nil {
-			serverErrorf(res, encErr, "Unable to serialize status file")
-			return
-		}
-	} else {
-		serverErrorf(res, err, "Unable to read status file")
-		return
-	}
-}
-
-func waitForJob(res http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	uid, err := uuid.FromString(vars["uid"])
-	if err != nil {
-		http.Error(res, "UUID had unexpected format!", http.StatusBadRequest)
-		return
-	}
-
-	var loop int
-	if v := r.URL.Query().Get("loop"); v != "" {
-		if pv, convErr := strconv.Atoi(v); convErr == nil {
-			loop = pv
-		}
-	}
-	loop++
-
-	status, err := loadStatusByUUID(uid)
-	if err != nil {
-		serverErrorf(res, err, "Unable to read status file")
-		return
-	}
-
-	switch status.Status {
-	case statusCreated:
-		fallthrough
-
-	case statusStarted:
-		u := urlMust(router.Get("waitForJob").URL("uid", uid.String()))
-		u.Query().Set("loop", strconv.Itoa(loop))
-
-		<-time.After(time.Duration(math.Pow(sleepBase, float64(loop))) * time.Second)
-
-		http.Redirect(res, r, u.String(), http.StatusFound)
-		return
-
-	case statusError:
-		http.Error(res, "Processing ran into an error.", http.StatusInternalServerError)
-
-	case statusFinished:
-		u := urlMust(router.Get("downloadAssets").URL("uid", uid.String()))
-		http.Redirect(res, r, u.String(), http.StatusFound)
 	}
 }
 
@@ -255,13 +101,19 @@ func downloadAssets(res http.ResponseWriter, r *http.Request) {
 		contentType = "application/tar"
 		content, err = buildAssetsTAR(uid)
 		filename = uid.String() + ".tar"
+
+	case "application/pdf":
+		contentType = "application/pdf"
+		content, err = getAssetsPDF(uid)
+		filename = uid.String() + ".pdf"
+
 	default:
 		content, err = buildAssetsZIP(uid)
 		filename = uid.String() + ".zip"
 	}
 
 	if err != nil {
-		serverErrorf(res, err, "Unable to generate downloadable asset")
+		serverErrorf(res, err, "generating downloadable asset")
 		return
 	}
 
@@ -271,42 +123,3 @@ func downloadAssets(res http.ResponseWriter, r *http.Request) {
 
 	io.Copy(res, content) // #nosec G104
 }
-
-func jobProcessor(uid uuid.UUID) {
-	logger := log.WithField("uuid", uid)
-	logger.Info("Started processing")
-
-	processingDir := path.Dir(pathFromUUID(uid, filenameStatus))
-	status, err := loadStatusByUUID(uid)
-	if err != nil {
-		logger.WithError(err).Error("Unable to load status file in processing job")
-		return
-	}
-
-	cmd := exec.Command("/bin/bash", cfg.Script) // #nosec G204
-	cmd.Dir = processingDir
-	cmd.Stderr = logger.WriterLevel(log.InfoLevel) // Bash uses stderr for `-x` parameter
-
-	status.UpdateStatus(statusStarted)
-	if err := status.Save(); err != nil {
-		logger.WithError(err).Error("Unable to save status file")
-		return
-	}
-
-	if err := cmd.Run(); err != nil {
-		logger.WithError(err).Error("Processing failed")
-		status.UpdateStatus(statusError)
-		if err := status.Save(); err != nil {
-			logger.WithError(err).Error("Unable to save status file")
-			return
-		}
-		return
-	}
-
-	status.UpdateStatus(statusFinished)
-	if err := status.Save(); err != nil {
-		logger.WithError(err).Error("Unable to save status file")
-		return
-	}
-	logger.Info("Finished processing")
-}

processing.go

@@ -0,0 +1,137 @@
+package main
+
+import (
+	"io"
+	"math"
+	"net/http"
+	"os"
+	"os/exec"
+	"path"
+	"strconv"
+	"time"
+
+	"github.com/gofrs/uuid"
+	"github.com/gorilla/mux"
+	"github.com/sirupsen/logrus"
+)
+
+func jobProcessor(uid uuid.UUID) {
+	logger := logrus.WithField("uuid", uid)
+	logger.Info("Started processing")
+
+	processingDir := path.Dir(pathFromUUID(uid, filenameStatus))
+	status, err := loadStatusByUUID(uid)
+	if err != nil {
+		logger.WithError(err).Error("Unable to load status file in processing job")
+		return
+	}
+
+	cmd := exec.Command("/bin/bash", cfg.Script) // #nosec G204
+	cmd.Dir = processingDir
+	cmd.Stderr = logger.WriterLevel(logrus.InfoLevel) // Bash uses stderr for `-x` parameter
+
+	status.UpdateStatus(statusStarted)
+	if err := status.Save(); err != nil {
+		logger.WithError(err).Error("Unable to save status file")
+		return
+	}
+
+	if err := cmd.Run(); err != nil {
+		logger.WithError(err).Error("Processing failed")
+		status.UpdateStatus(statusError)
+		if err := status.Save(); err != nil {
+			logger.WithError(err).Error("Unable to save status file")
+			return
+		}
+		return
+	}
+
+	status.UpdateStatus(statusFinished)
+	if err := status.Save(); err != nil {
+		logger.WithError(err).Error("Unable to save status file")
+		return
+	}
+	logger.Info("Finished processing")
+}
+
+func startNewJob(res http.ResponseWriter, r *http.Request) {
+	jobUUID := uuid.Must(uuid.NewV4())
+	inputFile := pathFromUUID(jobUUID, filenameInput)
+	statusFile := pathFromUUID(jobUUID, filenameStatus)
+
+	if err := os.Mkdir(path.Dir(inputFile), 0750); err != nil {
+		logrus.WithError(err).Errorf("Unable to create job dir %q", path.Dir(inputFile))
+	}
+
+	if f, err := os.Create(inputFile); err == nil {
+		defer f.Close()
+		if _, copyErr := io.Copy(f, r.Body); copyErr != nil {
+			serverErrorf(res, copyErr, "Unable to copy input file %q", inputFile)
+			return
+		}
+		f.Sync() // #nosec G104
+	} else {
+		serverErrorf(res, err, "Unable to write input file %q", inputFile)
+		return
+	}
+
+	status := jobStatus{
+		UUID:      jobUUID.String(),
+		CreatedAt: time.Now(),
+		UpdatedAt: time.Now(),
+		Status:    statusCreated,
+	}
+	if err := status.Save(); err != nil {
+		serverErrorf(res, err, "Unable to create status file %q", statusFile)
+		return
+	}
+
+	go jobProcessor(jobUUID)
+
+	u := urlMust(router.Get("waitForJob").URL("uid", jobUUID.String()))
+	http.Redirect(res, r, u.String(), http.StatusFound)
+}
+
+func waitForJob(res http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	uid, err := uuid.FromString(vars["uid"])
+	if err != nil {
+		http.Error(res, "UUID had unexpected format!", http.StatusBadRequest)
+		return
+	}
+
+	var loop int
+	if v := r.URL.Query().Get("loop"); v != "" {
+		if pv, convErr := strconv.Atoi(v); convErr == nil {
+			loop = pv
+		}
+	}
+	loop++
+
+	status, err := loadStatusByUUID(uid)
+	if err != nil {
+		serverErrorf(res, err, "Unable to read status file")
+		return
+	}
+
+	switch status.Status {
+	case statusCreated:
+		fallthrough
+
+	case statusStarted:
+		u := urlMust(router.Get("waitForJob").URL("uid", uid.String()))
+		u.Query().Set("loop", strconv.Itoa(loop))
+
+		<-time.After(time.Duration(math.Pow(sleepBase, float64(loop))) * time.Second)
+
+		http.Redirect(res, r, u.String(), http.StatusFound)
+		return
+
+	case statusError:
+		http.Error(res, "Processing ran into an error.", http.StatusInternalServerError)
+
+	case statusFinished:
+		u := urlMust(router.Get("downloadAssets").URL("uid", uid.String()))
+		http.Redirect(res, r, u.String(), http.StatusFound)
+	}
+}