mirror of
https://github.com/Luzifer/go-latestver.git
synced 2024-11-08 15:10:04 +00:00
Implement json
and atlassian
fetchers
Signed-off-by: Knut Ahlers <knut@ahlers.me>
This commit is contained in:
parent
4a200e12e8
commit
02ec355192
6 changed files with 295 additions and 0 deletions
1
go.mod
1
go.mod
|
@ -21,6 +21,7 @@ require (
|
|||
github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect
|
||||
github.com/acomagu/bufpipe v1.0.3 // indirect
|
||||
github.com/antchfx/htmlquery v1.2.4 // indirect
|
||||
github.com/antchfx/jsonquery v1.1.5 // indirect
|
||||
github.com/antchfx/xpath v1.2.0 // indirect
|
||||
github.com/blang/semver/v4 v4.0.0 // indirect
|
||||
github.com/emirpasic/gods v1.12.0 // indirect
|
||||
|
|
2
go.sum
2
go.sum
|
@ -17,6 +17,8 @@ github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/g
|
|||
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
|
||||
github.com/antchfx/htmlquery v1.2.4 h1:qLteofCMe/KGovBI6SQgmou2QNyedFUW+pE+BpeZ494=
|
||||
github.com/antchfx/htmlquery v1.2.4/go.mod h1:2xO6iu3EVWs7R2JYqBbp8YzG50gj/ofqs5/0VZoDZLc=
|
||||
github.com/antchfx/jsonquery v1.1.5 h1:1YWrNFYCcIuJPIjFeOP5b6TXbLSUYY8qqxWbuZOB1qE=
|
||||
github.com/antchfx/jsonquery v1.1.5/go.mod h1:RtMzTHohKaAerkfslTNjr3Y9MdxjKlSgIgaVjVKNiug=
|
||||
github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8=
|
||||
github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
|
||||
|
|
109
internal/fetcher/atlassian.go
Normal file
109
internal/fetcher/atlassian.go
Normal file
|
@ -0,0 +1,109 @@
|
|||
package fetcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/Luzifer/go-latestver/internal/database"
|
||||
"github.com/Luzifer/go_helpers/v2/fieldcollection"
|
||||
)
|
||||
|
||||
var (
|
||||
atlassianDefaultEdition = ""
|
||||
atlassianDefaultSearch = "TAR.GZ"
|
||||
)
|
||||
|
||||
type (
|
||||
AtlassianFetcher struct{}
|
||||
)
|
||||
|
||||
func init() { registerFetcher("atlassian", func() Fetcher { return &AtlassianFetcher{} }) }
|
||||
|
||||
func (a AtlassianFetcher) FetchVersion(ctx context.Context, attrs *fieldcollection.FieldCollection) (string, time.Time, error) {
|
||||
url := fmt.Sprintf("https://my.atlassian.com/download/feeds/current/%s.json", attrs.MustString("product", nil))
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return "", time.Time{}, errors.Wrap(err, "creating request")
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return "", time.Time{}, errors.Wrap(err, "executing request")
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", time.Time{}, errors.Wrap(err, "reading response body")
|
||||
}
|
||||
|
||||
matches := jsonpStripRegex.FindSubmatch(body)
|
||||
if matches == nil {
|
||||
return "", time.Time{}, errors.New("document does not match jsonp syntax")
|
||||
}
|
||||
|
||||
var payload []struct {
|
||||
Description string `json:"description"`
|
||||
Edition string `json:"edition"`
|
||||
Zipurl string `json:"zipUrl"`
|
||||
Md5 string `json:"md5"`
|
||||
Size string `json:"size"`
|
||||
Released string `json:"released"`
|
||||
Type string `json:"type"`
|
||||
Platform string `json:"platform"`
|
||||
Version string `json:"version"`
|
||||
Releasenotes string `json:"releaseNotes"`
|
||||
Upgradenotes string `json:"upgradeNotes"`
|
||||
}
|
||||
|
||||
if err = json.Unmarshal(matches[1], &payload); err != nil {
|
||||
return "", time.Time{}, errors.Wrap(err, "parsing response JSON")
|
||||
}
|
||||
|
||||
sort.Slice(payload, func(j, i int) bool { // j, i -> Reverse sort, biggest date at the top
|
||||
iRelease, _ := time.Parse("02-Jan-2006", payload[i].Released)
|
||||
jRelease, _ := time.Parse("02-Jan-2006", payload[j].Released)
|
||||
return iRelease.Before(jRelease)
|
||||
})
|
||||
|
||||
var (
|
||||
edition = attrs.MustString("edition", &atlassianDefaultEdition)
|
||||
search = attrs.MustString("search", &atlassianDefaultSearch)
|
||||
)
|
||||
|
||||
for _, r := range payload {
|
||||
if edition != "" && !strings.Contains(r.Edition, edition) {
|
||||
continue
|
||||
}
|
||||
|
||||
if search != "" && !strings.Contains(r.Description, search) {
|
||||
continue
|
||||
}
|
||||
|
||||
rt, _ := time.Parse("02-Jan-2006", r.Released)
|
||||
return r.Version, rt, nil
|
||||
}
|
||||
|
||||
return "", time.Time{}, ErrNoVersionFound
|
||||
}
|
||||
|
||||
func (AtlassianFetcher) Links(attrs *fieldcollection.FieldCollection) []database.CatalogLink {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (AtlassianFetcher) Validate(attrs *fieldcollection.FieldCollection) error {
|
||||
if v, err := attrs.String("product"); err != nil || v == "" {
|
||||
return errors.New("product is expected to be non-empty string")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
40
internal/fetcher/atlassian_test.go
Normal file
40
internal/fetcher/atlassian_test.go
Normal file
|
@ -0,0 +1,40 @@
|
|||
package fetcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/Luzifer/go_helpers/v2/fieldcollection"
|
||||
)
|
||||
|
||||
func Test_AtlassianFetcher(t *testing.T) {
|
||||
attrs := fieldcollection.FieldCollectionFromData(map[string]interface{}{
|
||||
"product": "confluence",
|
||||
"edition": "Standard",
|
||||
})
|
||||
|
||||
f := Get("atlassian")
|
||||
|
||||
if err := f.Validate(attrs); err != nil {
|
||||
t.Fatalf("validating attributes: %s", err)
|
||||
}
|
||||
|
||||
ver, _, err := f.FetchVersion(context.Background(), attrs)
|
||||
if err != nil {
|
||||
t.Fatalf("fetching version: %s", err)
|
||||
}
|
||||
|
||||
// Uses tag format: 1.0.0
|
||||
if len(ver) < 5 {
|
||||
t.Errorf("version has unexpected format: %s != X.X.X", ver)
|
||||
}
|
||||
|
||||
t.Logf("found version: %s", ver)
|
||||
|
||||
attrs.Set("edition", "ThisDoesNotExist")
|
||||
|
||||
_, _, err = f.FetchVersion(context.Background(), attrs)
|
||||
if err == nil {
|
||||
t.Errorf("fetching non existing edition did not error")
|
||||
}
|
||||
}
|
109
internal/fetcher/json.go
Normal file
109
internal/fetcher/json.go
Normal file
|
@ -0,0 +1,109 @@
|
|||
package fetcher
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/antchfx/jsonquery"
|
||||
"github.com/antchfx/xpath"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/Luzifer/go-latestver/internal/database"
|
||||
"github.com/Luzifer/go_helpers/v2/fieldcollection"
|
||||
)
|
||||
|
||||
var (
|
||||
jsonFetcherDefaultRegex = `(v?(?:[0-9]+\.?){2,})`
|
||||
jsonpStripRegex = regexp.MustCompile(`(?m)^[^\(]+\((.*)\)$`)
|
||||
ptrBoolFalse = func(v bool) *bool { return &v }(false)
|
||||
)
|
||||
|
||||
type (
|
||||
JSONFetcher struct{}
|
||||
)
|
||||
|
||||
func init() { registerFetcher("json", func() Fetcher { return &JSONFetcher{} }) }
|
||||
|
||||
func (JSONFetcher) FetchVersion(ctx context.Context, attrs *fieldcollection.FieldCollection) (string, time.Time, error) {
|
||||
var (
|
||||
doc *jsonquery.Node
|
||||
err error
|
||||
)
|
||||
|
||||
if attrs.MustBool("jsonp", ptrBoolFalse) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, attrs.MustString("url", nil), nil)
|
||||
if err != nil {
|
||||
return "", time.Time{}, errors.Wrap(err, "creating request")
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return "", time.Time{}, errors.Wrap(err, "executing request")
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", time.Time{}, errors.Wrap(err, "reading response body")
|
||||
}
|
||||
|
||||
matches := jsonpStripRegex.FindSubmatch(body)
|
||||
if matches == nil {
|
||||
return "", time.Time{}, errors.New("document does not match jsonp syntax")
|
||||
}
|
||||
|
||||
doc, err = jsonquery.Parse(bytes.NewReader(matches[1]))
|
||||
} else {
|
||||
doc, err = jsonquery.LoadURL(attrs.MustString("url", nil))
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return "", time.Time{}, errors.New("parsing JSON document")
|
||||
}
|
||||
|
||||
node, err := jsonquery.Query(doc, attrs.MustString("xpath", nil))
|
||||
if err != nil {
|
||||
return "", time.Time{}, errors.Wrap(err, "querying xpath")
|
||||
}
|
||||
|
||||
if node == nil {
|
||||
return "", time.Time{}, errors.New("xpath expression lead to nil-node")
|
||||
}
|
||||
|
||||
if node.Type == jsonquery.ElementNode && node.FirstChild != nil && node.FirstChild.Type == jsonquery.TextNode {
|
||||
node = node.FirstChild
|
||||
}
|
||||
|
||||
if node.Type != jsonquery.TextNode {
|
||||
return "", time.Time{}, errors.Errorf("xpath expression lead to unexpected node type: %d", node.Type)
|
||||
}
|
||||
|
||||
match := regexp.MustCompile(attrs.MustString("regex", &jsonFetcherDefaultRegex)).FindStringSubmatch(node.Data)
|
||||
if len(match) < 2 {
|
||||
return "", time.Time{}, errors.New("regular expression did not yield version")
|
||||
}
|
||||
|
||||
return match[1], time.Now(), nil
|
||||
}
|
||||
|
||||
func (JSONFetcher) Links(attrs *fieldcollection.FieldCollection) []database.CatalogLink { return nil }
|
||||
|
||||
func (JSONFetcher) Validate(attrs *fieldcollection.FieldCollection) error {
|
||||
if v, err := attrs.String("url"); err != nil || v == "" {
|
||||
return errors.New("url is expected to be non-empty string")
|
||||
}
|
||||
|
||||
if v, err := attrs.String("xpath"); err != nil || v == "" {
|
||||
return errors.New("xpath is expected to be non-empty string")
|
||||
}
|
||||
|
||||
if _, err := xpath.Compile(attrs.MustString("xpath", nil)); err != nil {
|
||||
return errors.Wrap(err, "compiling xpath expression")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
34
internal/fetcher/json_test.go
Normal file
34
internal/fetcher/json_test.go
Normal file
|
@ -0,0 +1,34 @@
|
|||
package fetcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/Luzifer/go_helpers/v2/fieldcollection"
|
||||
)
|
||||
|
||||
func Test_JSONFetcher(t *testing.T) {
|
||||
attrs := fieldcollection.FieldCollectionFromData(map[string]interface{}{
|
||||
"jsonp": true,
|
||||
"url": "https://my.atlassian.com/download/feeds/current/crowd.json", // This is JSONP
|
||||
"xpath": "*[1]/version",
|
||||
})
|
||||
|
||||
f := Get("json")
|
||||
|
||||
if err := f.Validate(attrs); err != nil {
|
||||
t.Fatalf("validating attributes: %s", err)
|
||||
}
|
||||
|
||||
ver, _, err := f.FetchVersion(context.Background(), attrs)
|
||||
if err != nil {
|
||||
t.Fatalf("fetching version: %s", err)
|
||||
}
|
||||
|
||||
// Uses tag format: 1.0.0
|
||||
if len(ver) < 5 {
|
||||
t.Errorf("version has unexpected format: %s != X.X.X", ver)
|
||||
}
|
||||
|
||||
t.Logf("found version: %s", ver)
|
||||
}
|
Loading…
Reference in a new issue