Implement json and atlassian fetchers

Signed-off-by: Knut Ahlers <knut@ahlers.me>

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect
 	github.com/acomagu/bufpipe v1.0.3 // indirect
 	github.com/antchfx/htmlquery v1.2.4 // indirect
+	github.com/antchfx/jsonquery v1.1.5 // indirect
 	github.com/antchfx/xpath v1.2.0 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/emirpasic/gods v1.12.0 // indirect

go.sum

@@ -17,6 +17,8 @@ github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/g
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/antchfx/htmlquery v1.2.4 h1:qLteofCMe/KGovBI6SQgmou2QNyedFUW+pE+BpeZ494=
 github.com/antchfx/htmlquery v1.2.4/go.mod h1:2xO6iu3EVWs7R2JYqBbp8YzG50gj/ofqs5/0VZoDZLc=
+github.com/antchfx/jsonquery v1.1.5 h1:1YWrNFYCcIuJPIjFeOP5b6TXbLSUYY8qqxWbuZOB1qE=
+github.com/antchfx/jsonquery v1.1.5/go.mod h1:RtMzTHohKaAerkfslTNjr3Y9MdxjKlSgIgaVjVKNiug=
 github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8=
 github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=

internal/fetcher/atlassian.go

@@ -0,0 +1,109 @@
+package fetcher
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/Luzifer/go-latestver/internal/database"
+	"github.com/Luzifer/go_helpers/v2/fieldcollection"
+)
+
+var (
+	atlassianDefaultEdition = ""
+	atlassianDefaultSearch  = "TAR.GZ"
+)
+
+type (
+	AtlassianFetcher struct{}
+)
+
+func init() { registerFetcher("atlassian", func() Fetcher { return &AtlassianFetcher{} }) }
+
+func (a AtlassianFetcher) FetchVersion(ctx context.Context, attrs *fieldcollection.FieldCollection) (string, time.Time, error) {
+	url := fmt.Sprintf("https://my.atlassian.com/download/feeds/current/%s.json", attrs.MustString("product", nil))
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return "", time.Time{}, errors.Wrap(err, "creating request")
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", time.Time{}, errors.Wrap(err, "executing request")
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", time.Time{}, errors.Wrap(err, "reading response body")
+	}
+
+	matches := jsonpStripRegex.FindSubmatch(body)
+	if matches == nil {
+		return "", time.Time{}, errors.New("document does not match jsonp syntax")
+	}
+
+	var payload []struct {
+		Description  string `json:"description"`
+		Edition      string `json:"edition"`
+		Zipurl       string `json:"zipUrl"`
+		Md5          string `json:"md5"`
+		Size         string `json:"size"`
+		Released     string `json:"released"`
+		Type         string `json:"type"`
+		Platform     string `json:"platform"`
+		Version      string `json:"version"`
+		Releasenotes string `json:"releaseNotes"`
+		Upgradenotes string `json:"upgradeNotes"`
+	}
+
+	if err = json.Unmarshal(matches[1], &payload); err != nil {
+		return "", time.Time{}, errors.Wrap(err, "parsing response JSON")
+	}
+
+	sort.Slice(payload, func(j, i int) bool { // j, i -> Reverse sort, biggest date at the top
+		iRelease, _ := time.Parse("02-Jan-2006", payload[i].Released)
+		jRelease, _ := time.Parse("02-Jan-2006", payload[j].Released)
+		return iRelease.Before(jRelease)
+	})
+
+	var (
+		edition = attrs.MustString("edition", &atlassianDefaultEdition)
+		search  = attrs.MustString("search", &atlassianDefaultSearch)
+	)
+
+	for _, r := range payload {
+		if edition != "" && !strings.Contains(r.Edition, edition) {
+			continue
+		}
+
+		if search != "" && !strings.Contains(r.Description, search) {
+			continue
+		}
+
+		rt, _ := time.Parse("02-Jan-2006", r.Released)
+		return r.Version, rt, nil
+	}
+
+	return "", time.Time{}, ErrNoVersionFound
+}
+
+func (AtlassianFetcher) Links(attrs *fieldcollection.FieldCollection) []database.CatalogLink {
+	return nil
+}
+
+func (AtlassianFetcher) Validate(attrs *fieldcollection.FieldCollection) error {
+	if v, err := attrs.String("product"); err != nil || v == "" {
+		return errors.New("product is expected to be non-empty string")
+	}
+
+	return nil
+}

internal/fetcher/atlassian_test.go

@@ -0,0 +1,40 @@
+package fetcher
+
+import (
+	"context"
+	"testing"
+
+	"github.com/Luzifer/go_helpers/v2/fieldcollection"
+)
+
+func Test_AtlassianFetcher(t *testing.T) {
+	attrs := fieldcollection.FieldCollectionFromData(map[string]interface{}{
+		"product": "confluence",
+		"edition": "Standard",
+	})
+
+	f := Get("atlassian")
+
+	if err := f.Validate(attrs); err != nil {
+		t.Fatalf("validating attributes: %s", err)
+	}
+
+	ver, _, err := f.FetchVersion(context.Background(), attrs)
+	if err != nil {
+		t.Fatalf("fetching version: %s", err)
+	}
+
+	// Uses tag format: 1.0.0
+	if len(ver) < 5 {
+		t.Errorf("version has unexpected format: %s != X.X.X", ver)
+	}
+
+	t.Logf("found version: %s", ver)
+
+	attrs.Set("edition", "ThisDoesNotExist")
+
+	_, _, err = f.FetchVersion(context.Background(), attrs)
+	if err == nil {
+		t.Errorf("fetching non existing edition did not error")
+	}
+}

internal/fetcher/json.go

@@ -0,0 +1,109 @@
+package fetcher
+
+import (
+	"bytes"
+	"context"
+	"io/ioutil"
+	"net/http"
+	"regexp"
+	"time"
+
+	"github.com/antchfx/jsonquery"
+	"github.com/antchfx/xpath"
+	"github.com/pkg/errors"
+
+	"github.com/Luzifer/go-latestver/internal/database"
+	"github.com/Luzifer/go_helpers/v2/fieldcollection"
+)
+
+var (
+	jsonFetcherDefaultRegex = `(v?(?:[0-9]+\.?){2,})`
+	jsonpStripRegex         = regexp.MustCompile(`(?m)^[^\(]+\((.*)\)$`)
+	ptrBoolFalse            = func(v bool) *bool { return &v }(false)
+)
+
+type (
+	JSONFetcher struct{}
+)
+
+func init() { registerFetcher("json", func() Fetcher { return &JSONFetcher{} }) }
+
+func (JSONFetcher) FetchVersion(ctx context.Context, attrs *fieldcollection.FieldCollection) (string, time.Time, error) {
+	var (
+		doc *jsonquery.Node
+		err error
+	)
+
+	if attrs.MustBool("jsonp", ptrBoolFalse) {
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, attrs.MustString("url", nil), nil)
+		if err != nil {
+			return "", time.Time{}, errors.Wrap(err, "creating request")
+		}
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return "", time.Time{}, errors.Wrap(err, "executing request")
+		}
+		defer resp.Body.Close()
+
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return "", time.Time{}, errors.Wrap(err, "reading response body")
+		}
+
+		matches := jsonpStripRegex.FindSubmatch(body)
+		if matches == nil {
+			return "", time.Time{}, errors.New("document does not match jsonp syntax")
+		}
+
+		doc, err = jsonquery.Parse(bytes.NewReader(matches[1]))
+	} else {
+		doc, err = jsonquery.LoadURL(attrs.MustString("url", nil))
+	}
+
+	if err != nil {
+		return "", time.Time{}, errors.New("parsing JSON document")
+	}
+
+	node, err := jsonquery.Query(doc, attrs.MustString("xpath", nil))
+	if err != nil {
+		return "", time.Time{}, errors.Wrap(err, "querying xpath")
+	}
+
+	if node == nil {
+		return "", time.Time{}, errors.New("xpath expression lead to nil-node")
+	}
+
+	if node.Type == jsonquery.ElementNode && node.FirstChild != nil && node.FirstChild.Type == jsonquery.TextNode {
+		node = node.FirstChild
+	}
+
+	if node.Type != jsonquery.TextNode {
+		return "", time.Time{}, errors.Errorf("xpath expression lead to unexpected node type: %d", node.Type)
+	}
+
+	match := regexp.MustCompile(attrs.MustString("regex", &jsonFetcherDefaultRegex)).FindStringSubmatch(node.Data)
+	if len(match) < 2 {
+		return "", time.Time{}, errors.New("regular expression did not yield version")
+	}
+
+	return match[1], time.Now(), nil
+}
+
+func (JSONFetcher) Links(attrs *fieldcollection.FieldCollection) []database.CatalogLink { return nil }
+
+func (JSONFetcher) Validate(attrs *fieldcollection.FieldCollection) error {
+	if v, err := attrs.String("url"); err != nil || v == "" {
+		return errors.New("url is expected to be non-empty string")
+	}
+
+	if v, err := attrs.String("xpath"); err != nil || v == "" {
+		return errors.New("xpath is expected to be non-empty string")
+	}
+
+	if _, err := xpath.Compile(attrs.MustString("xpath", nil)); err != nil {
+		return errors.Wrap(err, "compiling xpath expression")
+	}
+
+	return nil
+}

internal/fetcher/json_test.go

@@ -0,0 +1,34 @@
+package fetcher
+
+import (
+	"context"
+	"testing"
+
+	"github.com/Luzifer/go_helpers/v2/fieldcollection"
+)
+
+func Test_JSONFetcher(t *testing.T) {
+	attrs := fieldcollection.FieldCollectionFromData(map[string]interface{}{
+		"jsonp": true,
+		"url":   "https://my.atlassian.com/download/feeds/current/crowd.json", // This is JSONP
+		"xpath": "*[1]/version",
+	})
+
+	f := Get("json")
+
+	if err := f.Validate(attrs); err != nil {
+		t.Fatalf("validating attributes: %s", err)
+	}
+
+	ver, _, err := f.FetchVersion(context.Background(), attrs)
+	if err != nil {
+		t.Fatalf("fetching version: %s", err)
+	}
+
+	// Uses tag format: 1.0.0
+	if len(ver) < 5 {
+		t.Errorf("version has unexpected format: %s != X.X.X", ver)
+	}
+
+	t.Logf("found version: %s", ver)
+}