From 0f544d9ac7d24ed43c39404a0a477c1a72b28bd7 Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Thu, 5 Oct 2023 15:29:06 +0200 Subject: [PATCH] Add automated Issue generation for missing translations (#119) --- .github/workflows/test-and-build.yml | 10 ++++ .gitignore | 1 + Makefile | 2 +- ci/translate/issue.tpl.md | 22 ++++++++ ci/translate/issuegen.go | 64 ++++++++++++++++++++++ ci/translate/{translate.go => main.go} | 46 +++++++--------- ci/translate/translation.go | 66 +++++++++++++++++++++++ ci/translate/verify.go | 74 +++++++++++++------------- i18n.yaml | 17 ++++++ 9 files changed, 238 insertions(+), 64 deletions(-) create mode 100644 ci/translate/issue.tpl.md create mode 100644 ci/translate/issuegen.go rename ci/translate/{translate.go => main.go} (88%) create mode 100644 ci/translate/translation.go diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml index 772b01c..cfa7881 100644 --- a/.github/workflows/test-and-build.yml +++ b/.github/workflows/test-and-build.yml @@ -8,6 +8,7 @@ on: permissions: contents: write + issues: write jobs: test-and-build: @@ -70,6 +71,15 @@ jobs: - name: Generate (and validate) translations run: make translate + - name: Update Translations Issue + uses: JasonEtco/create-an-issue@v2 + if: github.ref == 'refs/heads/master' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + filename: translate-issue.md + update_existing: true + - name: Build release run: make publish env: diff --git a/.gitignore b/.gitignore index 48dace5..11a2ef1 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ frontend/webfonts frontend/*.woff2 node_modules ots +translate-issue.md diff --git a/Makefile b/Makefile index f5d80eb..cdc0d67 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ publish: download_libs generate-inner generate-apidocs bash ./ci/build.sh translate: - cd ci/translate && go run . + cd ci/translate && go run . --write-issue-file # -- Download / refresh external libraries -- diff --git a/ci/translate/issue.tpl.md b/ci/translate/issue.tpl.md new file mode 100644 index 0000000..72bfeea --- /dev/null +++ b/ci/translate/issue.tpl.md @@ -0,0 +1,22 @@ +--- +title: Missing Translations +--- +> As a developer I want my application to have correct translations in all available languages and not to have them to experience mixed translations in their native language and English. + +**In order to achieve this we need to fix the following missing translations.** + +To do so please either **create a pull-request** updating the `i18n.yaml` in the root of the repository and add the missing translations to the corresponding language or **just leave a comment** below and ping @Luzifer in your comment. He then will integrate the new translation strings and mark your comment hidden after this issue has been automatically updated (kind of a to-do list for translations until we have something better in place). + +{{ range $lang, $translation := .Translations -}} +{{ if MissingTranslations $lang -}} +### Language: `{{ $lang }}` + +Please add the following translations: +{{ range MissingTranslations $lang }} +- `{{ . }}` + > {{ English . }} +{{ end }} +_{{ Ping $lang }}_ + +{{ end -}} +{{ end -}} diff --git a/ci/translate/issuegen.go b/ci/translate/issuegen.go new file mode 100644 index 0000000..9ac77d6 --- /dev/null +++ b/ci/translate/issuegen.go @@ -0,0 +1,64 @@ +package main + +import ( + _ "embed" + "os" + "sort" + "strings" + "text/template" + + "github.com/pkg/errors" +) + +//go:embed issue.tpl.md +var issueTemplate string + +func generateIssue(tf translationFile) error { + fm := template.FuncMap{ + "English": tplEnglish(tf), + "MissingTranslations": tplMissingTranslations(tf), + "Ping": tplPing(tf), + } + + tpl, err := template.New("issue").Funcs(fm).Parse(issueTemplate) + if err != nil { + return errors.Wrap(err, "parsing issue template") + } + + f, err := os.Create(cfg.IssueFile) + if err != nil { + return errors.Wrap(err, "opening issue file") + } + defer f.Close() //nolint:errcheck // Short-lived fd-leak + + return errors.Wrap(tpl.Execute(f, tf), "executing issue template") +} + +func tplEnglish(tf translationFile) func(string) any { + return func(key string) any { + return tf.Reference.Translations[key] + } +} + +func tplMissingTranslations(tf translationFile) func(string) []string { + return func(lang string) []string { + missing, _, _ := tf.Translations[lang].Translations.GetErrorKeys(tf.Reference.Translations) + sort.Strings(missing) + return missing + } +} + +func tplPing(tf translationFile) func(string) string { + return func(lang string) string { + if len(tf.Translations[lang].Translators) == 0 { + return "No translators to ping for this language." + } + + var pings []string + for _, t := range tf.Translations[lang].Translators { + pings = append(pings, "@"+t) + } + + return strings.Join([]string{"Ping", strings.Join(pings, ", ")}, " ") + } +} diff --git a/ci/translate/translate.go b/ci/translate/main.go similarity index 88% rename from ci/translate/translate.go rename to ci/translate/main.go index 9cfb2b8..0e3d4ff 100644 --- a/ci/translate/translate.go +++ b/ci/translate/main.go @@ -20,31 +20,19 @@ import ( const deeplRequestTimeout = 10 * time.Second -type ( - translation map[string]any - translationFile struct { - Reference translationMapping `yaml:"reference"` - Translations map[string]*translationMapping `yaml:"translations"` - } - translationMapping struct { - DeeplLanguage string `yaml:"deeplLanguage,omitempty"` - LanguageKey string `yaml:"languageKey,omitempty"` - Translations translation `yaml:"translations"` - FormalTranslations translation `yaml:"formalTranslations,omitempty"` - } -) - var ( cfg = struct { AutoTranslate bool `flag:"auto-translate" default:"false" description:"Enable auto-translation through DeepL"` DeeplAPIEndpoint string `flag:"deepl-api-endpoint" default:"https://api-free.deepl.com/v2/translate" description:"DeepL API endpoint to request translations from"` DeeplAPIKey string `flag:"deepl-api-key" default:"" description:"API key for the DeepL API"` + IssueFile string `flag:"issue-file" default:"../../translate-issue.md" description:"Where to create the translate issue"` OutputFile string `flag:"output-file,o" default:"../../src/langs/langs.js" description:"Where to put rendered translations"` Template string `flag:"template" default:"../../src/langs/langs.tpl.js" description:"Template to load for translation JS file"` TranslationFile string `flag:"translation-file,t" default:"../../i18n.yaml" description:"File to use for translations"` LogLevel string `flag:"log-level" default:"info" description:"Log level (debug, info, warn, error, fatal)"` Verify bool `flag:"verify" default:"true" description:"Run verification against translation file"` VersionAndExit bool `flag:"version" default:"false" description:"Prints current version and exits"` + WriteIssueFile bool `flag:"write-issue-file" default:"false" description:"Generates an issue body for missing translations"` }{} version = "dev" @@ -113,6 +101,13 @@ func main() { if err = renderJSFile(tf); err != nil { logrus.WithError(err).Fatal("rendering JS output") } + + if cfg.WriteIssueFile { + logrus.Info("writing issue template...") + if err = generateIssue(tf); err != nil { + logrus.WithError(err).Fatal("generating issue template") + } + } } func autoTranslate(tf *translationFile) error { @@ -212,7 +207,11 @@ func fetchTranslation(srcLang, destLang, text string) (string, error) { if err != nil { return "", errors.Wrap(err, "executing request") } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + logrus.WithError(err).Error("closing response body (leaked fd)") + } + }() var payload struct { Translations []struct { @@ -237,7 +236,7 @@ func loadTranslationFile() (translationFile, error) { if err != nil { return tf, errors.Wrap(err, "opening translation file") } - defer f.Close() + defer f.Close() //nolint:errcheck // Short-lived fd-leak decoder := yaml.NewDecoder(f) decoder.KnownFields(true) @@ -262,11 +261,11 @@ func renderJSFile(tf translationFile) error { } if err = tpl.Execute(f, tf); err != nil { - f.Close() + f.Close() //nolint:errcheck,gosec,revive // Short-lived fd-leak return errors.Wrap(err, "rendering js template") } - f.Close() + f.Close() //nolint:errcheck,gosec,revive // Short-lived fd-leak return errors.Wrap(os.Rename(cfg.OutputFile+".tmp", cfg.OutputFile), "moving file in place") } @@ -277,18 +276,13 @@ func saveTranslationFile(tf translationFile) error { } encoder := yaml.NewEncoder(f) - encoder.SetIndent(2) + encoder.SetIndent(2) //nolint:gomnd if err = encoder.Encode(tf); err != nil { - f.Close() + f.Close() //nolint:errcheck,gosec,revive // Short-lived fd-leak return errors.Wrap(err, "encoding translation file") } - f.Close() + f.Close() //nolint:errcheck,gosec,revive // Short-lived fd-leak return errors.Wrap(os.Rename(cfg.TranslationFile+".tmp", cfg.TranslationFile), "moving file in place") } - -func (t translation) ToJSON() (string, error) { - j, err := json.Marshal(t) - return strings.ReplaceAll(string(j), "'", "\\'"), errors.Wrap(err, "marshalling JSON") -} diff --git a/ci/translate/translation.go b/ci/translate/translation.go new file mode 100644 index 0000000..de53c4e --- /dev/null +++ b/ci/translate/translation.go @@ -0,0 +1,66 @@ +package main + +import ( + "encoding/json" + "reflect" + "strings" + + "github.com/Luzifer/go_helpers/v2/str" + "github.com/pkg/errors" +) + +type ( + translation map[string]any + translationFile struct { + Reference translationMapping `yaml:"reference"` + Translations map[string]*translationMapping `yaml:"translations"` + } + translationMapping struct { + DeeplLanguage string `yaml:"deeplLanguage,omitempty"` + LanguageKey string `yaml:"languageKey,omitempty"` + Translators []string `yaml:"translators"` + Translations translation `yaml:"translations"` + FormalTranslations translation `yaml:"formalTranslations,omitempty"` + } +) + +func (t translation) ToJSON() (string, error) { + j, err := json.Marshal(t) + return strings.ReplaceAll(string(j), "'", "\\'"), errors.Wrap(err, "marshalling JSON") +} + +func (t translation) GetErrorKeys(ref translation) (missing, extra, wrongType []string) { + var ( + keys []string + keyType = map[string]reflect.Type{} + seenKeys []string + ) + + for k, v := range ref { + keys = append(keys, k) + keyType[k] = reflect.TypeOf(v) + } + + for k, v := range t { + if !str.StringInSlice(k, keys) { + // Contains extra key, is error + extra = append(extra, k) + continue // No further checks for that key + } + + seenKeys = append(seenKeys, k) + if kt := reflect.TypeOf(v); keyType[k] != kt { + // Type mismatches (i.e. string vs []string) + wrongType = append(wrongType, k) + continue + } + } + + for _, k := range keys { + if !str.StringInSlice(k, seenKeys) { + missing = append(missing, k) + } + } + + return missing, extra, wrongType +} diff --git a/ci/translate/verify.go b/ci/translate/verify.go index a4fd6d7..80e2337 100644 --- a/ci/translate/verify.go +++ b/ci/translate/verify.go @@ -1,41 +1,35 @@ package main import ( - "reflect" "regexp" + "sort" "github.com/pkg/errors" "github.com/sirupsen/logrus" - - "github.com/Luzifer/go_helpers/v2/str" ) var langKeyFormat = regexp.MustCompile(`^[a-z]{2}(-[A-Z]{2})?$`) func verify(tf translationFile) error { - var ( - err error - keys []string - keyType = map[string]reflect.Type{} - ) - - for k, v := range tf.Reference.Translations { - keys = append(keys, k) - keyType[k] = reflect.TypeOf(v) - } + var err error if !langKeyFormat.MatchString(tf.Reference.LanguageKey) { return errors.New("reference contains invalid languageKey") } - if len(keys) == 0 { + if len(tf.Reference.Translations) == 0 { return errors.New("reference does not contain translations") } - logrus.Infof("found %d translation keys in reference", len(keys)) + logrus.Infof("found %d translation keys in reference", len(tf.Reference.Translations)) if tf.Reference.FormalTranslations != nil { - if verifyTranslationKeys(logrus.NewEntry(logrus.StandardLogger()), tf.Reference.FormalTranslations, keys, keyType, false); err != nil { + if verifyTranslationKeys( + logrus.NewEntry(logrus.StandardLogger()), + tf.Reference.FormalTranslations, + tf.Reference.Translations, + false, + ); err != nil { return errors.New("reference contains error in formalTranslations") } } @@ -54,8 +48,18 @@ func verify(tf translationFile) error { logger.Info("no deeplLanguage is set") } - hadErrors = hadErrors || verifyTranslationKeys(logger, tm.Translations, keys, keyType, true) - hadErrors = hadErrors || verifyTranslationKeys(logger, tm.FormalTranslations, keys, keyType, false) + hadErrors = hadErrors || verifyTranslationKeys( + logger, + tm.Translations, + tf.Reference.Translations, + true, + ) + hadErrors = hadErrors || verifyTranslationKeys( + logger, + tm.FormalTranslations, + tf.Reference.Translations, + false, + ) } if hadErrors { @@ -64,31 +68,27 @@ func verify(tf translationFile) error { return nil } -func verifyTranslationKeys(logger *logrus.Entry, t translation, keys []string, keyType map[string]reflect.Type, warnMissing bool) (hadErrors bool) { - var seenKeys []string +//revive:disable-next-line:flag-parameter +func verifyTranslationKeys(logger *logrus.Entry, t, ref translation, warnMissing bool) (hadErrors bool) { + missing, extra, wrongType := t.GetErrorKeys(ref) - for k, v := range t { - keyLogger := logger.WithField("translation_key", k) - if !str.StringInSlice(k, keys) { - // Contains extra key, is error - hadErrors = true - keyLogger.Error("extra key found") - continue // No further checks for that key - } + sort.Strings(extra) + sort.Strings(missing) + sort.Strings(wrongType) - seenKeys = append(seenKeys, k) - if kt := reflect.TypeOf(v); keyType[k] != kt { - // Type mismatches (i.e. string vs []string) - hadErrors = true - keyLogger.Errorf("key has invalid type %s != %s", kt, keyType[k]) - } + for _, k := range extra { + logger.WithField("translation_key", k).Error("extra key found") } - for _, k := range keys { - if warnMissing && !str.StringInSlice(k, seenKeys) { + for _, k := range wrongType { + logger.WithField("translation_key", k).Error("key has invalid type") + } + + if warnMissing { + for _, k := range missing { logger.WithField("translation_key", k).Warn("missing translation") } } - return hadErrors + return len(extra)+len(wrongType) > 0 } diff --git a/i18n.yaml b/i18n.yaml index 472e783..79a5a72 100644 --- a/i18n.yaml +++ b/i18n.yaml @@ -1,6 +1,8 @@ reference: deeplLanguage: en languageKey: en + translators: + - Luzifer translations: alert-secret-not-found: This is not the secret you are looking for… - If you expected the secret to be here it might be compromised as someone else might have opened the link already. alert-something-went-wrong: Something went wrong. I'm very sorry about this… @@ -43,6 +45,7 @@ reference: title-secret-created: Secret created! translations: ca: + translators: [] translations: alert-secret-not-found: Aquest no és el secret que busques… - Si esperaves que el secret estiguera ací, és possible que s'haja vist compromés, ja que una altra persona podria haver obert l'enllaç en comptes de tu. alert-something-went-wrong: Alguna cosa ha eixit malament. Ens sap molt greu… @@ -79,6 +82,8 @@ translations: title-secret-created: Secret creat! de: deeplLanguage: de + translators: + - Luzifer translations: alert-secret-not-found: Das ist nicht das Secret, was du suchst… - Falls du diesen Link noch nicht selbst geöffnet hast, könnte das Secret kompromittiert sein, da jemand anderes den Link geöffnet haben könnte. alert-something-went-wrong: Irgendwas ging schief. Entschuldigung… @@ -138,6 +143,7 @@ translations: title-new-secret: Ein neues Secret erstellen es: deeplLanguage: es + translators: [] translations: alert-secret-not-found: Este no es el secreto que buscas… - Si esperabas que el secreto estuviera aquí, es posible que se haya visto comprometido, ya que otra persona podría haber abierto el enlace en tu lugar. alert-something-went-wrong: Algo ha salido mal. Lo sentimos mucho… @@ -174,6 +180,7 @@ translations: title-secret-created: ¡Secreto creado! fr: deeplLanguage: fr + translators: [] translations: alert-secret-not-found: Ce secret n'est pas celui que vous cherchez… - Si vous comptiez trouvez ce secret ici, il a pu être compromis car quelqu'un a probablement déjà ouvert le lien. alert-something-went-wrong: Un problème est survenu. Nous en sommes désolés… @@ -207,6 +214,7 @@ translations: title-secret-created: Secret créé! lv: deeplLanguage: lv + translators: [] translations: alert-secret-not-found: Ziņa nav atrasta!… - Ja ievadītā saite ir pareiza, tad ir beidzies ziņas glabāšanas laiks, vai arī tā jau vienreiz ir atvērta. alert-something-went-wrong: Neparedzēta sistēmas kļūda. Atvainojiet par sagādātajām neērtībām… @@ -240,6 +248,7 @@ translations: title-secret-created: Ziņa nošifrēta! nl: deeplLanguage: nl + translators: [] translations: alert-secret-not-found: De gegevens die je zocht bestaan niet (meer)… - Als je hier informatie verwachtte dan is de link mogelijk al door iemand anders bekeken! alert-something-went-wrong: Er ging iets verkeerd, sorry… @@ -273,6 +282,7 @@ translations: title-secret-created: Vertrouwelijke info opgeslaan! pl: deeplLanguage: pl + translators: [] translations: alert-secret-not-found: TO nie jest sekret, którego szukasz… - Jeśli spodziewałeś się tu sekretu, to może być on zagrożony, ponieważ ktoś inny mógł już otworzyć ten link. alert-something-went-wrong: Coś poszło nie tak. Bardzo mi przykro… @@ -309,6 +319,7 @@ translations: title-secret-created: Sekret utworzony! pt-BR: deeplLanguage: pt-BR + translators: [] translations: alert-secret-not-found: Esta não é o segredo que você está procurando… - Se você esperava que o segredo estaria aqui, ele pode ter sido comprometido por alguém que já acessou o link. alert-something-went-wrong: Desculpe, algo deu errado… @@ -342,6 +353,7 @@ translations: title-secret-created: Segredo criado! ru: deeplLanguage: ru + translators: [] translations: alert-secret-not-found: Секрет недоступен… - Помните, он может быть скомпрометирован. Возможно кто-то другой уже открыл вашу ссылку. alert-something-went-wrong: Что-то пошло не так. Приносим свои извинения… @@ -375,6 +387,7 @@ translations: title-secret-created: Секрет создан! sv: deeplLanguage: sv + translators: [] translations: alert-secret-not-found: Hemlighet hittades inte… - Om du förväntade dig att hemligheten skulle finnas här kan den vara röjd då någon annan kan ha öppnat denna länk tidigare. alert-something-went-wrong: Något gick fel. Jag ber om ursäkt för detta!… @@ -408,6 +421,7 @@ translations: title-secret-created: Hemlighet skapad! tr: deeplLanguage: tr + translators: [] translations: alert-secret-not-found: Aradığınız sır bu değil… - Sırrın burada olmasını bekliyorsanız, bu link başkası tarafından açılmış ve sırrınız tehlikede olabilir. alert-something-went-wrong: Bir şeyler ters gitti. Bunun için çok üzgünüm… @@ -441,6 +455,7 @@ translations: title-secret-created: Sır oluşturuldu! uk: deeplLanguage: uk + translators: [] translations: alert-secret-not-found: Це не секрет, який ви шукаєте… - Якщо ви очікували, що секрет буде тут, він міг бути скомпрометований, оскільки хтось інший міг уже відкрити посилання. alert-something-went-wrong: Щось пішло не так. Ми дуже шкодуємо про це… @@ -476,6 +491,7 @@ translations: title-secret-create-disabled: Створення секрету вимкнено… title-secret-created: Секрет створений! zh: + translators: [] translations: alert-secret-not-found: 该秘密不存在 - 如果这与您的预期不符，那么该链接可能已经泄露，且您的秘密已经被其他人查看了。 alert-something-went-wrong: 运行异常，对此我深感抱歉… @@ -511,6 +527,7 @@ translations: title-secret-create-disabled: 创建秘密被禁止… title-secret-created: 秘密已创建！ zh-TW: + translators: [] translations: alert-secret-not-found: 這不是您正在尋找的機密… - 如果您期望機密會出現在這裡，它可能已經被泄漏了，因為可能有其他人已經打開了此連結。 alert-something-went-wrong: 看樣子出了一些問題，對此我非常抱歉…