Add fetcher documentation

Signed-off-by: Knut Ahlers <knut@ahlers.me>

Makefile

@@ -15,3 +15,13 @@ node_modules:
 go_test:
 	go test -cover -v ./...
 	golangci-lint run
+
+# --- Documentation
+
+gendoc: .venv
+	.venv/bin/python3 ci/gendoc.py $(shell grep -l '@module ' internal/fetcher/*.go) >docs/config.md
+	git add docs/config.md
+
+.venv:
+	python -m venv .venv
+	.venv/bin/pip install -r ci/requirements.txt

README.md

@@ -23,6 +23,8 @@ Usage of go-latestver:
       --version                       Prints current version and exits
 ```
 
+The documentation for the format of the `config` file can be found in the [`docs/config.md`](docs/config.md) file.
+
 To use the `github_release` fetcher without hitting the API limits quite fast provide `GITHUB_CLIENT_ID` and `GITHUB_CLIENT_SECRET` of an [OAuth App](https://github.com/settings/developers) in environment variables.
 
 ## Screenshots

ci/gendoc.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+import jinja2
+import sys
+import re
+
+
+def main(args):
+    modules = []
+
+    for filename in args:
+        with open(filename, 'r') as codefile:
+            mod = {
+                'attributes': [],
+            }
+
+            for line in codefile:
+                match = re.search(r'@module (.*)', line)
+                if match is not None:
+                    mod['type'] = match[1]
+
+                match = re.search(r'@module_desc (.*)', line)
+                if match is not None:
+                    mod['description'] = match[1]
+
+                match = re.search(
+                    r'@attr ([^\s]+) ([^\s]+) ([^\s]+) "([^"]*)" (.*)', line)
+                if match is not None:
+                    mod['attributes'].append({
+                        'name': match[1],
+                        'required': match[2],
+                        'type': match[3],
+                        'default': match[4],
+                        'description': match[5],
+                    })
+
+        mod['attributes'] = sorted(
+            mod['attributes'], key=lambda a: ('0' if a['required'] == 'required' else '1') + ':' + a['name'])
+
+        modules.append(mod)
+
+    modules = sorted(modules, key=lambda m: m['type'])
+
+    with open('docs/config.md.tpl', 'r') as f:
+        tpl = jinja2.Template(f.read())
+        print(tpl.render(modules=modules))
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

ci/requirements.txt

@@ -0,0 +1 @@
+jinja2

docs/config.md

@@ -0,0 +1,84 @@
+# Configuration file format
+
+```yaml
+---
+
+catalog:
+
+  - name: alpine
+    tag: stable
+
+    fetcher: html
+    fetcher_config:
+      url: https://alpinelinux.org/downloads/
+      xpath: '//div[@class="l-box"]/p/strong'
+
+check_interval: 1h
+
+...
+```
+
+Each catalog entry contains a `name` and a `tag` representing the entry. You can choose those freely but they should be URL-safe. Some examples I'm using are: `alpine:stable`, `google-chrome:dev`, `google-chrome:stable`, `factorio:latest`, …
+
+Additionally you will configure a `fetcher` with its corresponding `fetcher_config` for the catalog entry. In the example above the `html` fetcher is used with two attributes configured. The attributes for each fetcher can be found below.
+
+## Available Fetchers
+
+## Fetcher: `atlassian`
+
+Fetches latest version of an Atlassian product
+
+| Attribute | Req. | Type | Default Value | Description |
+| --------- | :--: | ---- | ------------- | ----------- |
+| `product` | ✅ | string |  | Lowercase name of the product to fetch (e.g. confluence, crowd, jira-software, ...) |
+| `edition` |  | string |  | Filter down the versions according to its edition (e.g. "Enterprise" or "Standard" for Confluence) |
+| `search` |  | string | `TAR.GZ` | What to search in the download description: default is to search for the standalone .tar.gz file |
+
+## Fetcher: `git_tag`
+
+Reads git tags (annotated and leightweight) from a remote repository and returns the newest one
+
+| Attribute | Req. | Type | Default Value | Description |
+| --------- | :--: | ---- | ------------- | ----------- |
+| `remote` | ✅ | string |  | Repository remote to fetch the tags from (should accept everything you can use in `git remote set-url` command) |
+
+## Fetcher: `github_release`
+
+Fetches the latest release from Github for a given repository not marked as pre-release
+
+| Attribute | Req. | Type | Default Value | Description |
+| --------- | :--: | ---- | ------------- | ----------- |
+| `repository` | ✅ | string |  | Repository to fetch in form `owner/repo` |
+
+## Fetcher: `html`
+
+Downloads website, selects text-node using XPath and optionally applies custom regular expression
+
+| Attribute | Req. | Type | Default Value | Description |
+| --------- | :--: | ---- | ------------- | ----------- |
+| `url` | ✅ | string |  | URL to fetch the HTML from |
+| `xpath` | ✅ | string |  | XPath expression leading to the text-node containing the version |
+| `regex` |  | string | `(v?(?:[0-9]+\.?){2,})` | Regular expression to apply to the text from the XPath expression |
+
+## Fetcher: `json`
+
+Fetches a JSON / JSONP file from remote source and traverses it using XPath expression
+
+| Attribute | Req. | Type | Default Value | Description |
+| --------- | :--: | ---- | ------------- | ----------- |
+| `url` | ✅ | string |  | URL to fetch the HTML from |
+| `xpath` | ✅ | string |  | XPath expression leading to the text-node containing the version |
+| `jsonp` |  | boolean | `false` | File contains JSONP function, strip it to get the raw JSON |
+
+## Fetcher: `regex`
+
+Fetches URL and applies a regular expression to extract a version from it
+
+| Attribute | Req. | Type | Default Value | Description |
+| --------- | :--: | ---- | ------------- | ----------- |
+| `regex` | ✅ | string |  | Regular expression (RE2) to apply to the text fetched from the URL. The regex MUST have exactly one submatch containing the version. |
+| `url` | ✅ | string |  | URL to fetch the content from |
+
+
+
+<!-- vim: set ft=markdown : -->

docs/config.md.tpl

@@ -0,0 +1,40 @@
+# Configuration file format
+
+```yaml
+---
+
+catalog:
+
+  - name: alpine
+    tag: stable
+
+    fetcher: html
+    fetcher_config:
+      url: https://alpinelinux.org/downloads/
+      xpath: '//div[@class="l-box"]/p/strong'
+
+check_interval: 1h
+
+...
+```
+
+Each catalog entry contains a `name` and a `tag` representing the entry. You can choose those freely but they should be URL-safe. Some examples I'm using are: `alpine:stable`, `google-chrome:dev`, `google-chrome:stable`, `factorio:latest`, …
+
+Additionally you will configure a `fetcher` with its corresponding `fetcher_config` for the catalog entry. In the example above the `html` fetcher is used with two attributes configured. The attributes for each fetcher can be found below.
+
+## Available Fetchers
+
+{% for module in modules -%}
+## Fetcher: `{{ module.type }}`
+
+{{ module.description }}
+
+| Attribute | Req. | Type | Default Value | Description |
+| --------- | :--: | ---- | ------------- | ----------- |
+{%- for attr in module.attributes %}
+| `{{ attr.name }}` | {% if attr.required == 'required' %}✅{% endif %} | {{ attr.type }} | {% if attr.default != "" %}`{{ attr.default }}`{% endif %} | {{ attr.description }} |
+{%- endfor %}
+
+{% endfor %}
+
+<!-- vim: set ft=markdown : -->

internal/fetcher/atlassian.go

@@ -16,6 +16,11 @@ import (
 	"github.com/Luzifer/go_helpers/v2/fieldcollection"
 )
 
+/*
+ * @module atlassian
+ * @module_desc Fetches latest version of an Atlassian product
+ */
+
 var (
 	atlassianDefaultEdition = ""
 	atlassianDefaultSearch  = "TAR.GZ"
@@ -76,7 +81,9 @@ func (a AtlassianFetcher) FetchVersion(ctx context.Context, attrs *fieldcollecti
 	})
 
 	var (
+		// @attr edition optional string "" Filter down the versions according to its edition (e.g. "Enterprise" or "Standard" for Confluence)
 		edition = attrs.MustString("edition", &atlassianDefaultEdition)
+		// @attr search optional string "TAR.GZ" What to search in the download description: default is to search for the standalone .tar.gz file
 		search = attrs.MustString("search", &atlassianDefaultSearch)
 	)
 
@@ -101,6 +108,7 @@ func (AtlassianFetcher) Links(attrs *fieldcollection.FieldCollection) []database
 }
 
 func (AtlassianFetcher) Validate(attrs *fieldcollection.FieldCollection) error {
+	// @attr product required string "" Lowercase name of the product to fetch (e.g. confluence, crowd, jira-software, ...)
 	if v, err := attrs.String("product"); err != nil || v == "" {
 		return errors.New("product is expected to be non-empty string")
 	}

internal/fetcher/git.go

@@ -14,6 +14,11 @@ import (
 	"github.com/Luzifer/go_helpers/v2/fieldcollection"
 )
 
+/*
+ * @module git_tag
+ * @module_desc Reads git tags (annotated and leightweight) from a remote repository and returns the newest one
+ */
+
 type (
 	GitTagFetcher struct{}
 )
@@ -79,6 +84,7 @@ func (g GitTagFetcher) Links(attrs *fieldcollection.FieldCollection) []database.
 }
 
 func (g GitTagFetcher) Validate(attrs *fieldcollection.FieldCollection) error {
+	// @attr remote required string "" Repository remote to fetch the tags from (should accept everything you can use in `git remote set-url` command)
 	if v, err := attrs.String("remote"); err != nil || v == "" {
 		return errors.New("remote is expected to be non-empty string")
 	}

internal/fetcher/github.go

@@ -14,6 +14,11 @@ import (
 	"github.com/Luzifer/go_helpers/v2/fieldcollection"
 )
 
+/*
+ * @module github_release
+ * @module_desc Fetches the latest release from Github for a given repository not marked as pre-release
+ */
+
 const githubHTTPTimeout = 2 * time.Second
 
 type (
@@ -91,6 +96,7 @@ func (g GithubReleaseFetcher) Links(attrs *fieldcollection.FieldCollection) []da
 }
 
 func (g GithubReleaseFetcher) Validate(attrs *fieldcollection.FieldCollection) error {
+	// @attr repository required string "" Repository to fetch in form `owner/repo`
 	if v, err := attrs.String("repository"); err != nil || v == "" {
 		return errors.New("repository is expected to be non-empty string")
 	}

internal/fetcher/html.go

@@ -14,6 +14,11 @@ import (
 	"github.com/Luzifer/go_helpers/v2/fieldcollection"
 )
 
+/*
+ * @module html
+ * @module_desc Downloads website, selects text-node using XPath and optionally applies custom regular expression
+ */
+
 var htmlFetcherDefaultRegex = `(v?(?:[0-9]+\.?){2,})`
 
 type (
@@ -64,10 +69,12 @@ func (h HTMLFetcher) Links(attrs *fieldcollection.FieldCollection) []database.Ca
 }
 
 func (h HTMLFetcher) Validate(attrs *fieldcollection.FieldCollection) error {
+	// @attr url required string "" URL to fetch the HTML from
 	if v, err := attrs.String("url"); err != nil || v == "" {
 		return errors.New("url is expected to be non-empty string")
 	}
 
+	// @attr xpath required string "" XPath expression leading to the text-node containing the version
 	if v, err := attrs.String("xpath"); err != nil || v == "" {
 		return errors.New("xpath is expected to be non-empty string")
 	}
@@ -76,6 +83,7 @@ func (h HTMLFetcher) Validate(attrs *fieldcollection.FieldCollection) error {
 		return errors.Wrap(err, "compiling xpath expression")
 	}
 
+	// @attr regex optional string "(v?(?:[0-9]+\.?){2,})" Regular expression to apply to the text from the XPath expression
 	if attrs.CanString("regex") {
 		if _, err := regexp.Compile(attrs.MustString("regex", nil)); err != nil {
 			return errors.Wrap(err, "invalid regex given")

internal/fetcher/json.go

@@ -16,6 +16,11 @@ import (
 	"github.com/Luzifer/go_helpers/v2/fieldcollection"
 )
 
+/*
+ * @module json
+ * @module_desc Fetches a JSON / JSONP file from remote source and traverses it using XPath expression
+ */
+
 var (
 	jsonFetcherDefaultRegex = `(v?(?:[0-9]+\.?){2,})`
 	jsonpStripRegex         = regexp.MustCompile(`(?m)^[^\(]+\((.*)\)$`)
@@ -34,6 +39,7 @@ func (JSONFetcher) FetchVersion(ctx context.Context, attrs *fieldcollection.Fiel
 		err error
 	)
 
+	// @attr jsonp optional boolean "false" File contains JSONP function, strip it to get the raw JSON
 	if attrs.MustBool("jsonp", ptrBoolFalse) {
 		var (
 			body []byte
@@ -98,10 +104,12 @@ func (JSONFetcher) FetchVersion(ctx context.Context, attrs *fieldcollection.Fiel
 func (JSONFetcher) Links(attrs *fieldcollection.FieldCollection) []database.CatalogLink { return nil }
 
 func (JSONFetcher) Validate(attrs *fieldcollection.FieldCollection) error {
+	// @attr url required string "" URL to fetch the HTML from
 	if v, err := attrs.String("url"); err != nil || v == "" {
 		return errors.New("url is expected to be non-empty string")
 	}
 
+	// @attr xpath required string "" XPath expression leading to the text-node containing the version
 	if v, err := attrs.String("xpath"); err != nil || v == "" {
 		return errors.New("xpath is expected to be non-empty string")
 	}

internal/fetcher/regex.go

@@ -13,6 +13,11 @@ import (
 	"github.com/Luzifer/go_helpers/v2/fieldcollection"
 )
 
+/*
+ * @module regex
+ * @module_desc Fetches URL and applies a regular expression to extract a version from it
+ */
+
 const (
 	httpStatus3xx               = 300
 	regexpFetcherExpectedLength = 2
@@ -68,10 +73,12 @@ func (h RegexFetcher) Links(attrs *fieldcollection.FieldCollection) []database.C
 }
 
 func (h RegexFetcher) Validate(attrs *fieldcollection.FieldCollection) error {
+	// @attr url required string "" URL to fetch the content from
 	if v, err := attrs.String("url"); err != nil || v == "" {
 		return errors.New("url is expected to be non-empty string")
 	}
 
+	// @attr regex required string "" Regular expression (RE2) to apply to the text fetched from the URL. The regex MUST have exactly one submatch containing the version.
 	if v, err := attrs.String("regex"); err != nil || v == "" {
 		return errors.New("regex is expected to be non-empty string")
 	}