[core] Add rule-subscription feature

- Add remote url to rules
- Add cron to update remote URLs hourly
- Add frontend display for shared rules
- Add frontend feature to subscribe rules

Signed-off-by: Knut Ahlers <knut@ahlers.me>

History.md

@@ -7,6 +7,7 @@
 **Changelog:**
 
   * New Features
+    * [core] Add rule-subscription feature
     * [templating] Add jsonAPI template function
 
   * Improvements

config.go

@@ -28,6 +28,8 @@ var (
 
 	configReloadHooks     = map[string]func(){}
 	configReloadHooksLock sync.RWMutex
+
+	errSaveNotRequired = errors.New("save not required")
 )
 
 func registerConfigReloadHook(hook func()) func() {
@@ -187,7 +189,16 @@ func patchConfig(filename, authorName, authorEmail, summary string, patcher func
 
 	cfgFile.fixMissingUUIDs()
 
-	if err = patcher(cfgFile); err != nil {
+	err = patcher(cfgFile)
+	switch {
+	case errors.Is(err, nil):
+		// This is fine
+
+	case errors.Is(err, errSaveNotRequired):
+		// This is also fine but we don't need to save
+		return nil
+
+	default:
 		return errors.Wrap(err, "patching config")
 	}
 

configEditor_rules.go

@@ -89,7 +89,14 @@ func configEditorRulesAdd(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	msg.UUID = uuid.Must(uuid.NewV4()).String()
+	if msg.SubscribeFrom != nil {
+		if _, err = msg.UpdateFromSubscription(); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	} else {
+		msg.UUID = uuid.Must(uuid.NewV4()).String()
+	}
 
 	if err := patchConfig(cfg.Config, user, "", "Add rule", func(c *configFile) error {
 		c.Rules = append(c.Rules, msg)

configRemoteUpdate.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"fmt"
+	"math/rand"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func updateConfigCron() string {
+	minute := rand.Intn(60) //nolint:gomnd,gosec // Only used to distribute load
+	return fmt.Sprintf("%d * * * *", minute)
+}
+
+func updateConfigFromRemote() {
+	err := patchConfig(
+		cfg.Config,
+		"Remote Update", "twitch-bot@luzifer.io",
+		"update rules from subscription URLs",
+		func(cfg *configFile) error {
+			var hasUpdate bool
+
+			for _, r := range cfg.Rules {
+				logger := log.WithField("rule", r.MatcherID())
+
+				rhu, err := r.UpdateFromSubscription()
+				if err != nil {
+					logger.WithError(err).Error("updating rule")
+					continue
+				}
+
+				if rhu {
+					hasUpdate = true
+					logger.Info("updated rule from remote URL")
+				}
+
+			}
+
+			if !hasUpdate {
+				return errSaveNotRequired
+			}
+			return nil
+		},
+	)
+	if err != nil {
+		log.WithError(err).Error("updating config rules from subscriptions")
+	}
+}

main.go

@@ -247,6 +247,11 @@ func main() {
 	// are retried on error each time
 	cronService.AddFunc("@every 30s", twitchWatch.Check)
 
+	// Allow config to subscribe to external rules
+	updCron := updateConfigCron()
+	cronService.AddFunc(updCron, updateConfigFromRemote)
+	log.WithField("cron", updCron).Debug("Initialized remote update cron")
+
 	router.Use(corsMiddleware)
 	router.HandleFunc("/openapi.html", handleSwaggerHTML)
 	router.HandleFunc("/openapi.json", handleSwaggerRequest)

plugins/rule.go

@@ -1,7 +1,12 @@
 package plugins
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
+	"net/http"
+	"net/url"
+	"path"
 	"regexp"
 	"strings"
 	"time"
@@ -10,15 +15,19 @@ import (
 	"github.com/mitchellh/hashstructure/v2"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
 
 	"github.com/Luzifer/go_helpers/v2/str"
 	"github.com/Luzifer/twitch-bot/pkg/twitch"
 )
 
+const remoteRuleFetchTimeout = 2 * time.Second
+
 type (
 	Rule struct {
-		UUID        string `hash:"-" json:"uuid,omitempty" yaml:"uuid,omitempty"`
-		Description string `json:"description,omitempty" yaml:"description,omitempty"`
+		UUID          string  `hash:"-" json:"uuid,omitempty" yaml:"uuid,omitempty"`
+		Description   string  `json:"description,omitempty" yaml:"description,omitempty"`
+		SubscribeFrom *string `json:"subscribe_from,omitempty" yaml:"subscribe_from,omitempty"`
 
 		Actions []*RuleAction `json:"actions,omitempty" yaml:"actions,omitempty"`
 
@@ -60,11 +69,7 @@ func (r Rule) MatcherID() string {
 		return r.UUID
 	}
 
-	h, err := hashstructure.Hash(r, hashstructure.FormatV2, nil)
-	if err != nil {
-		panic(errors.Wrap(err, "hashing automessage"))
-	}
-	return fmt.Sprintf("hashstructure:%x", h)
+	return r.hash()
 }
 
 func (r *Rule) Matches(m *irc.Message, event *string, timerStore TimerStore, msgFormatter MsgFormatter, twitchClient *twitch.Client, eventData *FieldCollection) bool {
@@ -132,6 +137,62 @@ func (r *Rule) SetCooldown(timerStore TimerStore, m *irc.Message, evtData *Field
 	}
 }
 
+func (r *Rule) UpdateFromSubscription() (bool, error) {
+	if r.SubscribeFrom == nil || len(*r.SubscribeFrom) == 0 {
+		return false, nil
+	}
+
+	prevHash := r.hash()
+
+	remoteURL, err := url.Parse(*r.SubscribeFrom)
+	if err != nil {
+		return false, errors.Wrap(err, "parsing remote subscription url")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), remoteRuleFetchTimeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, remoteURL.String(), nil)
+	if err != nil {
+		return false, errors.Wrap(err, "assembling request")
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return false, errors.Wrap(err, "executing request")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return false, errors.Errorf("unxpected HTTP status %d", resp.StatusCode)
+	}
+
+	var newRule Rule
+	switch path.Ext(remoteURL.Path) {
+	case ".json":
+		err = json.NewDecoder(resp.Body).Decode(&newRule)
+
+	case ".yaml", ".yml":
+		err = yaml.NewDecoder(resp.Body).Decode(&newRule)
+
+	default:
+		return false, errors.New("unexpected format")
+	}
+
+	if err != nil {
+		return false, errors.Wrap(err, "decoding remote rule")
+	}
+
+	if newRule.hash() == prevHash {
+		// No update, exit now
+		return false, nil
+	}
+
+	*r = newRule
+
+	return true, nil
+}
+
 func (r *Rule) allowExecuteBadgeBlacklist(logger *log.Entry, m *irc.Message, event *string, badges twitch.BadgeCollection, evtData *FieldCollection) bool {
 	for _, b := range r.DisableOn {
 		if badges.Has(b) {
@@ -397,3 +458,11 @@ func (r *Rule) allowExecuteUserWhitelist(logger *log.Entry, m *irc.Message, even
 
 	return true
 }
+
+func (r Rule) hash() string {
+	h, err := hashstructure.Hash(r, hashstructure.FormatV2, nil)
+	if err != nil {
+		panic(errors.Wrap(err, "hashing rule"))
+	}
+	return fmt.Sprintf("hashstructure:%x", h)
+}

src/rules.vue

@@ -30,7 +30,19 @@
         >
           <template #cell(_actions)="data">
             <b-button-group size="sm">
-              <b-button @click="editRule(data.item)">
+              <b-button
+                v-if="data.item.subscribe_from"
+                disabled
+              >
+                <font-awesome-icon
+                  fixed-width
+                  :icon="['fas', 'download']"
+                />
+              </b-button>
+              <b-button
+                v-else
+                @click="editRule(data.item)"
+              >
                 <font-awesome-icon
                   fixed-width
                   :icon="['fas', 'pen']"
@@ -63,6 +75,13 @@
             <template v-if="data.item.description">
               {{ data.item.description }}<br>
             </template>
+            <b-badge
+              v-if="data.item.subscribe_from"
+              class="mt-1 mr-1"
+              variant="primary"
+            >
+              Shared
+            </b-badge>
             <b-badge
               v-if="data.item.disable"
               class="mt-1 mr-1"
@@ -90,12 +109,45 @@
                   :icon="['fas', 'plus']"
                 />
               </b-button>
+              <b-button
+                variant="secondary"
+                @click="showRuleSubscribeModal=true"
+              >
+                <font-awesome-icon
+                  fixed-width
+                  :icon="['fas', 'download']"
+                />
+              </b-button>
             </b-button-group>
           </template>
         </b-table>
       </b-col>
     </b-row>
 
+    <b-modal
+      v-if="showRuleSubscribeModal"
+      hide-header-close
+      :ok-disabled="!models.subscriptionURL"
+      ok-title="Subscribe"
+      size="md"
+      :visible="showRuleSubscribeModal"
+      title="Subscribe Rule"
+      @hidden="showRuleSubscribeModal=false"
+      @ok="subscribeRule"
+    >
+      <b-form-group
+        label="Rule Subscription URL"
+        label-for="formRuleSubURL"
+      >
+        <b-form-input
+          id="formRuleSubURL"
+          v-model="models.subscriptionURL"
+          :state="Boolean(models.subscriptionURL)"
+          type="text"
+        />
+      </b-form-group>
+    </b-modal>
+
     <!-- Rule Editor -->
     <b-modal
       v-if="showRuleEditModal"
@@ -702,6 +754,7 @@ export default {
         addAction: '',
         addException: '',
         rule: {},
+        subscriptionURL: '',
       },
 
       rules: [],
@@ -727,6 +780,7 @@ export default {
       ],
 
       showRuleEditModal: false,
+      showRuleSubscribeModal: false,
       validateReason: null,
     }
   },
@@ -1027,6 +1081,17 @@ export default {
         .catch(err => this.$bus.$emit(constants.NOTIFY_FETCH_ERROR, err))
     },
 
+    subscribeRule() {
+      axios.post(`config-editor/rules`, {
+        subscribe_from: this.models.subscriptionURL,
+      }, this.$root.axiosOptions)
+        .then(() => {
+          this.models.subscriptionURL = ''
+          this.$bus.$emit(constants.NOTIFY_CHANGE_PENDING, true)
+        })
+        .catch(err => this.$bus.$emit(constants.NOTIFY_FETCH_ERROR, err))
+    },
+
     validateActionArgument(idx, key) {
       const action = this.models.rule.actions[idx]
       const def = this.getActionDefinitionByType(action.type)