Implement json and atlassian fetchers

Signed-off-by: Knut Ahlers <knut@ahlers.me>
This commit is contained in:
Knut Ahlers 2021-11-29 16:53:15 +01:00
parent 4a200e12e8
commit 02ec355192
Signed by: luzifer
GPG key ID: 0066F03ED215AD7D
6 changed files with 295 additions and 0 deletions

1
go.mod
View file

@ -21,6 +21,7 @@ require (
github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect
github.com/acomagu/bufpipe v1.0.3 // indirect
github.com/antchfx/htmlquery v1.2.4 // indirect
github.com/antchfx/jsonquery v1.1.5 // indirect
github.com/antchfx/xpath v1.2.0 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/emirpasic/gods v1.12.0 // indirect

2
go.sum
View file

@ -17,6 +17,8 @@ github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/g
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/antchfx/htmlquery v1.2.4 h1:qLteofCMe/KGovBI6SQgmou2QNyedFUW+pE+BpeZ494=
github.com/antchfx/htmlquery v1.2.4/go.mod h1:2xO6iu3EVWs7R2JYqBbp8YzG50gj/ofqs5/0VZoDZLc=
github.com/antchfx/jsonquery v1.1.5 h1:1YWrNFYCcIuJPIjFeOP5b6TXbLSUYY8qqxWbuZOB1qE=
github.com/antchfx/jsonquery v1.1.5/go.mod h1:RtMzTHohKaAerkfslTNjr3Y9MdxjKlSgIgaVjVKNiug=
github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8=
github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=

View file

@ -0,0 +1,109 @@
package fetcher
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"sort"
"strings"
"time"
"github.com/pkg/errors"
"github.com/Luzifer/go-latestver/internal/database"
"github.com/Luzifer/go_helpers/v2/fieldcollection"
)
var (
atlassianDefaultEdition = ""
atlassianDefaultSearch = "TAR.GZ"
)
type (
AtlassianFetcher struct{}
)
func init() { registerFetcher("atlassian", func() Fetcher { return &AtlassianFetcher{} }) }
func (a AtlassianFetcher) FetchVersion(ctx context.Context, attrs *fieldcollection.FieldCollection) (string, time.Time, error) {
url := fmt.Sprintf("https://my.atlassian.com/download/feeds/current/%s.json", attrs.MustString("product", nil))
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return "", time.Time{}, errors.Wrap(err, "creating request")
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", time.Time{}, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", time.Time{}, errors.Wrap(err, "reading response body")
}
matches := jsonpStripRegex.FindSubmatch(body)
if matches == nil {
return "", time.Time{}, errors.New("document does not match jsonp syntax")
}
var payload []struct {
Description string `json:"description"`
Edition string `json:"edition"`
Zipurl string `json:"zipUrl"`
Md5 string `json:"md5"`
Size string `json:"size"`
Released string `json:"released"`
Type string `json:"type"`
Platform string `json:"platform"`
Version string `json:"version"`
Releasenotes string `json:"releaseNotes"`
Upgradenotes string `json:"upgradeNotes"`
}
if err = json.Unmarshal(matches[1], &payload); err != nil {
return "", time.Time{}, errors.Wrap(err, "parsing response JSON")
}
sort.Slice(payload, func(j, i int) bool { // j, i -> Reverse sort, biggest date at the top
iRelease, _ := time.Parse("02-Jan-2006", payload[i].Released)
jRelease, _ := time.Parse("02-Jan-2006", payload[j].Released)
return iRelease.Before(jRelease)
})
var (
edition = attrs.MustString("edition", &atlassianDefaultEdition)
search = attrs.MustString("search", &atlassianDefaultSearch)
)
for _, r := range payload {
if edition != "" && !strings.Contains(r.Edition, edition) {
continue
}
if search != "" && !strings.Contains(r.Description, search) {
continue
}
rt, _ := time.Parse("02-Jan-2006", r.Released)
return r.Version, rt, nil
}
return "", time.Time{}, ErrNoVersionFound
}
func (AtlassianFetcher) Links(attrs *fieldcollection.FieldCollection) []database.CatalogLink {
return nil
}
func (AtlassianFetcher) Validate(attrs *fieldcollection.FieldCollection) error {
if v, err := attrs.String("product"); err != nil || v == "" {
return errors.New("product is expected to be non-empty string")
}
return nil
}

View file

@ -0,0 +1,40 @@
package fetcher
import (
"context"
"testing"
"github.com/Luzifer/go_helpers/v2/fieldcollection"
)
func Test_AtlassianFetcher(t *testing.T) {
attrs := fieldcollection.FieldCollectionFromData(map[string]interface{}{
"product": "confluence",
"edition": "Standard",
})
f := Get("atlassian")
if err := f.Validate(attrs); err != nil {
t.Fatalf("validating attributes: %s", err)
}
ver, _, err := f.FetchVersion(context.Background(), attrs)
if err != nil {
t.Fatalf("fetching version: %s", err)
}
// Uses tag format: 1.0.0
if len(ver) < 5 {
t.Errorf("version has unexpected format: %s != X.X.X", ver)
}
t.Logf("found version: %s", ver)
attrs.Set("edition", "ThisDoesNotExist")
_, _, err = f.FetchVersion(context.Background(), attrs)
if err == nil {
t.Errorf("fetching non existing edition did not error")
}
}

109
internal/fetcher/json.go Normal file
View file

@ -0,0 +1,109 @@
package fetcher
import (
"bytes"
"context"
"io/ioutil"
"net/http"
"regexp"
"time"
"github.com/antchfx/jsonquery"
"github.com/antchfx/xpath"
"github.com/pkg/errors"
"github.com/Luzifer/go-latestver/internal/database"
"github.com/Luzifer/go_helpers/v2/fieldcollection"
)
var (
jsonFetcherDefaultRegex = `(v?(?:[0-9]+\.?){2,})`
jsonpStripRegex = regexp.MustCompile(`(?m)^[^\(]+\((.*)\)$`)
ptrBoolFalse = func(v bool) *bool { return &v }(false)
)
type (
JSONFetcher struct{}
)
func init() { registerFetcher("json", func() Fetcher { return &JSONFetcher{} }) }
func (JSONFetcher) FetchVersion(ctx context.Context, attrs *fieldcollection.FieldCollection) (string, time.Time, error) {
var (
doc *jsonquery.Node
err error
)
if attrs.MustBool("jsonp", ptrBoolFalse) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, attrs.MustString("url", nil), nil)
if err != nil {
return "", time.Time{}, errors.Wrap(err, "creating request")
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", time.Time{}, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", time.Time{}, errors.Wrap(err, "reading response body")
}
matches := jsonpStripRegex.FindSubmatch(body)
if matches == nil {
return "", time.Time{}, errors.New("document does not match jsonp syntax")
}
doc, err = jsonquery.Parse(bytes.NewReader(matches[1]))
} else {
doc, err = jsonquery.LoadURL(attrs.MustString("url", nil))
}
if err != nil {
return "", time.Time{}, errors.New("parsing JSON document")
}
node, err := jsonquery.Query(doc, attrs.MustString("xpath", nil))
if err != nil {
return "", time.Time{}, errors.Wrap(err, "querying xpath")
}
if node == nil {
return "", time.Time{}, errors.New("xpath expression lead to nil-node")
}
if node.Type == jsonquery.ElementNode && node.FirstChild != nil && node.FirstChild.Type == jsonquery.TextNode {
node = node.FirstChild
}
if node.Type != jsonquery.TextNode {
return "", time.Time{}, errors.Errorf("xpath expression lead to unexpected node type: %d", node.Type)
}
match := regexp.MustCompile(attrs.MustString("regex", &jsonFetcherDefaultRegex)).FindStringSubmatch(node.Data)
if len(match) < 2 {
return "", time.Time{}, errors.New("regular expression did not yield version")
}
return match[1], time.Now(), nil
}
func (JSONFetcher) Links(attrs *fieldcollection.FieldCollection) []database.CatalogLink { return nil }
func (JSONFetcher) Validate(attrs *fieldcollection.FieldCollection) error {
if v, err := attrs.String("url"); err != nil || v == "" {
return errors.New("url is expected to be non-empty string")
}
if v, err := attrs.String("xpath"); err != nil || v == "" {
return errors.New("xpath is expected to be non-empty string")
}
if _, err := xpath.Compile(attrs.MustString("xpath", nil)); err != nil {
return errors.Wrap(err, "compiling xpath expression")
}
return nil
}

View file

@ -0,0 +1,34 @@
package fetcher
import (
"context"
"testing"
"github.com/Luzifer/go_helpers/v2/fieldcollection"
)
func Test_JSONFetcher(t *testing.T) {
attrs := fieldcollection.FieldCollectionFromData(map[string]interface{}{
"jsonp": true,
"url": "https://my.atlassian.com/download/feeds/current/crowd.json", // This is JSONP
"xpath": "*[1]/version",
})
f := Get("json")
if err := f.Validate(attrs); err != nil {
t.Fatalf("validating attributes: %s", err)
}
ver, _, err := f.FetchVersion(context.Background(), attrs)
if err != nil {
t.Fatalf("fetching version: %s", err)
}
// Uses tag format: 1.0.0
if len(ver) < 5 {
t.Errorf("version has unexpected format: %s != X.X.X", ver)
}
t.Logf("found version: %s", ver)
}