go-latestver/internal/fetcher/json.go

123 lines
3.5 KiB
Go
Raw Normal View History

package fetcher
import (
"bytes"
"context"
"io/ioutil"
"net/http"
"regexp"
"time"
"github.com/antchfx/jsonquery"
"github.com/antchfx/xpath"
"github.com/pkg/errors"
"github.com/Luzifer/go-latestver/internal/database"
"github.com/Luzifer/go_helpers/v2/fieldcollection"
)
/*
* @module json
* @module_desc Fetches a JSON / JSONP file from remote source and traverses it using XPath expression
*/
var (
jsonFetcherDefaultRegex = `(v?(?:[0-9]+\.?){2,})`
jsonpStripRegex = regexp.MustCompile(`(?m)^[^\(]+\((.*)\)$`)
ptrBoolFalse = func(v bool) *bool { return &v }(false)
)
type (
JSONFetcher struct{}
)
func init() { registerFetcher("json", func() Fetcher { return &JSONFetcher{} }) }
func (JSONFetcher) FetchVersion(ctx context.Context, attrs *fieldcollection.FieldCollection) (string, time.Time, error) {
var (
doc *jsonquery.Node
err error
)
// @attr jsonp optional boolean "false" File contains JSONP function, strip it to get the raw JSON
if attrs.MustBool("jsonp", ptrBoolFalse) {
Add PR testing, fix linter errors Squashed commit of the following: commit 2a83adf6c54d6abcf6762760fd38f2505511f545 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:34:49 2021 +0100 Lint: Fix copylocks errors Signed-off-by: Knut Ahlers <knut@ahlers.me> commit 418f85d504203a6968329e280ecd9cf7d2365373 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:31:38 2021 +0100 Lint: Fix gosec warnings Signed-off-by: Knut Ahlers <knut@ahlers.me> commit 1a977875740be3c40884aa0985578721ceb4ae37 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:28:02 2021 +0100 Lint: Disable gomnd for certain cases Signed-off-by: Knut Ahlers <knut@ahlers.me> commit 5e81cf79ba7256b321442530715a2b53de0a18e1 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:26:01 2021 +0100 Lint: fix ineffassign errors Signed-off-by: Knut Ahlers <knut@ahlers.me> commit cb14fae2dad985368e1f05d62f8e778817d01c6f Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:23:42 2021 +0100 Lint: Fix revive linter errors Signed-off-by: Knut Ahlers <knut@ahlers.me> commit b3390b8dff9b939caa4e3821a48dd848af0bfba4 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:21:35 2021 +0100 Lint: Remove unrequired dereference Signed-off-by: Knut Ahlers <knut@ahlers.me> commit f9052e6aa530c5b5017249fc6c31bdbb94252760 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:20:43 2021 +0100 Lint: Remove deadcode Signed-off-by: Knut Ahlers <knut@ahlers.me> commit 72b88adaa25a3bb5a7af21da7ed12f08cae36573 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 02:52:27 2021 +0100 Add PR-testing Signed-off-by: Knut Ahlers <knut@ahlers.me> Signed-off-by: Knut Ahlers <knut@ahlers.me>
2021-12-01 02:38:52 +00:00
var (
body []byte
req *http.Request
resp *http.Response
)
req, err = http.NewRequestWithContext(ctx, http.MethodGet, attrs.MustString("url", nil), nil)
if err != nil {
return "", time.Time{}, errors.Wrap(err, "creating request")
}
Add PR testing, fix linter errors Squashed commit of the following: commit 2a83adf6c54d6abcf6762760fd38f2505511f545 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:34:49 2021 +0100 Lint: Fix copylocks errors Signed-off-by: Knut Ahlers <knut@ahlers.me> commit 418f85d504203a6968329e280ecd9cf7d2365373 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:31:38 2021 +0100 Lint: Fix gosec warnings Signed-off-by: Knut Ahlers <knut@ahlers.me> commit 1a977875740be3c40884aa0985578721ceb4ae37 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:28:02 2021 +0100 Lint: Disable gomnd for certain cases Signed-off-by: Knut Ahlers <knut@ahlers.me> commit 5e81cf79ba7256b321442530715a2b53de0a18e1 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:26:01 2021 +0100 Lint: fix ineffassign errors Signed-off-by: Knut Ahlers <knut@ahlers.me> commit cb14fae2dad985368e1f05d62f8e778817d01c6f Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:23:42 2021 +0100 Lint: Fix revive linter errors Signed-off-by: Knut Ahlers <knut@ahlers.me> commit b3390b8dff9b939caa4e3821a48dd848af0bfba4 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:21:35 2021 +0100 Lint: Remove unrequired dereference Signed-off-by: Knut Ahlers <knut@ahlers.me> commit f9052e6aa530c5b5017249fc6c31bdbb94252760 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:20:43 2021 +0100 Lint: Remove deadcode Signed-off-by: Knut Ahlers <knut@ahlers.me> commit 72b88adaa25a3bb5a7af21da7ed12f08cae36573 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 02:52:27 2021 +0100 Add PR-testing Signed-off-by: Knut Ahlers <knut@ahlers.me> Signed-off-by: Knut Ahlers <knut@ahlers.me>
2021-12-01 02:38:52 +00:00
resp, err = http.DefaultClient.Do(req)
if err != nil {
return "", time.Time{}, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
Add PR testing, fix linter errors Squashed commit of the following: commit 2a83adf6c54d6abcf6762760fd38f2505511f545 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:34:49 2021 +0100 Lint: Fix copylocks errors Signed-off-by: Knut Ahlers <knut@ahlers.me> commit 418f85d504203a6968329e280ecd9cf7d2365373 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:31:38 2021 +0100 Lint: Fix gosec warnings Signed-off-by: Knut Ahlers <knut@ahlers.me> commit 1a977875740be3c40884aa0985578721ceb4ae37 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:28:02 2021 +0100 Lint: Disable gomnd for certain cases Signed-off-by: Knut Ahlers <knut@ahlers.me> commit 5e81cf79ba7256b321442530715a2b53de0a18e1 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:26:01 2021 +0100 Lint: fix ineffassign errors Signed-off-by: Knut Ahlers <knut@ahlers.me> commit cb14fae2dad985368e1f05d62f8e778817d01c6f Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:23:42 2021 +0100 Lint: Fix revive linter errors Signed-off-by: Knut Ahlers <knut@ahlers.me> commit b3390b8dff9b939caa4e3821a48dd848af0bfba4 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:21:35 2021 +0100 Lint: Remove unrequired dereference Signed-off-by: Knut Ahlers <knut@ahlers.me> commit f9052e6aa530c5b5017249fc6c31bdbb94252760 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:20:43 2021 +0100 Lint: Remove deadcode Signed-off-by: Knut Ahlers <knut@ahlers.me> commit 72b88adaa25a3bb5a7af21da7ed12f08cae36573 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 02:52:27 2021 +0100 Add PR-testing Signed-off-by: Knut Ahlers <knut@ahlers.me> Signed-off-by: Knut Ahlers <knut@ahlers.me>
2021-12-01 02:38:52 +00:00
body, err = ioutil.ReadAll(resp.Body)
if err != nil {
return "", time.Time{}, errors.Wrap(err, "reading response body")
}
matches := jsonpStripRegex.FindSubmatch(body)
if matches == nil {
return "", time.Time{}, errors.New("document does not match jsonp syntax")
}
doc, err = jsonquery.Parse(bytes.NewReader(matches[1]))
} else {
doc, err = jsonquery.LoadURL(attrs.MustString("url", nil))
}
if err != nil {
return "", time.Time{}, errors.New("parsing JSON document")
}
node, err := jsonquery.Query(doc, attrs.MustString("xpath", nil))
if err != nil {
return "", time.Time{}, errors.Wrap(err, "querying xpath")
}
if node == nil {
return "", time.Time{}, errors.New("xpath expression lead to nil-node")
}
if node.Type == jsonquery.ElementNode && node.FirstChild != nil && node.FirstChild.Type == jsonquery.TextNode {
node = node.FirstChild
}
if node.Type != jsonquery.TextNode {
return "", time.Time{}, errors.Errorf("xpath expression lead to unexpected node type: %d", node.Type)
}
match := regexp.MustCompile(attrs.MustString("regex", &jsonFetcherDefaultRegex)).FindStringSubmatch(node.Data)
Add PR testing, fix linter errors Squashed commit of the following: commit 2a83adf6c54d6abcf6762760fd38f2505511f545 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:34:49 2021 +0100 Lint: Fix copylocks errors Signed-off-by: Knut Ahlers <knut@ahlers.me> commit 418f85d504203a6968329e280ecd9cf7d2365373 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:31:38 2021 +0100 Lint: Fix gosec warnings Signed-off-by: Knut Ahlers <knut@ahlers.me> commit 1a977875740be3c40884aa0985578721ceb4ae37 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:28:02 2021 +0100 Lint: Disable gomnd for certain cases Signed-off-by: Knut Ahlers <knut@ahlers.me> commit 5e81cf79ba7256b321442530715a2b53de0a18e1 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:26:01 2021 +0100 Lint: fix ineffassign errors Signed-off-by: Knut Ahlers <knut@ahlers.me> commit cb14fae2dad985368e1f05d62f8e778817d01c6f Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:23:42 2021 +0100 Lint: Fix revive linter errors Signed-off-by: Knut Ahlers <knut@ahlers.me> commit b3390b8dff9b939caa4e3821a48dd848af0bfba4 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:21:35 2021 +0100 Lint: Remove unrequired dereference Signed-off-by: Knut Ahlers <knut@ahlers.me> commit f9052e6aa530c5b5017249fc6c31bdbb94252760 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 03:20:43 2021 +0100 Lint: Remove deadcode Signed-off-by: Knut Ahlers <knut@ahlers.me> commit 72b88adaa25a3bb5a7af21da7ed12f08cae36573 Author: Knut Ahlers <knut@ahlers.me> Date: Wed Dec 1 02:52:27 2021 +0100 Add PR-testing Signed-off-by: Knut Ahlers <knut@ahlers.me> Signed-off-by: Knut Ahlers <knut@ahlers.me>
2021-12-01 02:38:52 +00:00
if len(match) < 2 { //nolint:gomnd // Simple count of fields, no need for constant
return "", time.Time{}, errors.New("regular expression did not yield version")
}
return match[1], time.Now(), nil
}
func (JSONFetcher) Links(attrs *fieldcollection.FieldCollection) []database.CatalogLink { return nil }
func (JSONFetcher) Validate(attrs *fieldcollection.FieldCollection) error {
// @attr url required string "" URL to fetch the HTML from
if v, err := attrs.String("url"); err != nil || v == "" {
return errors.New("url is expected to be non-empty string")
}
// @attr xpath required string "" XPath expression leading to the text-node containing the version
if v, err := attrs.String("xpath"); err != nil || v == "" {
return errors.New("xpath is expected to be non-empty string")
}
if _, err := xpath.Compile(attrs.MustString("xpath", nil)); err != nil {
return errors.Wrap(err, "compiling xpath expression")
}
return nil
}