diff --git a/.github/workflows/doc-generator.yml b/.github/workflows/doc-generator.yml deleted file mode 100644 index ca14f76..0000000 --- a/.github/workflows/doc-generator.yml +++ /dev/null @@ -1,57 +0,0 @@ ---- - -name: doc-generator -on: push - -jobs: - doc-generator: - defaults: - run: - shell: bash - - container: - image: luzifer/archlinux - env: - CGO_ENABLED: 0 - GOPATH: /go - - permissions: - contents: read - pages: write - id-token: write - - runs-on: ubuntu-latest - - steps: - - - name: Install required packages - run: | - pacman -Syy --noconfirm \ - curl \ - git \ - git-lfs \ - make \ - tar - - - uses: actions/checkout@v3 - with: - lfs: true - submodules: true - - - name: Marking workdir safe - run: git config --global --add safe.directory /__w/twitch-bot/twitch-bot - - - name: Generate documentation - run: make render_docs DOCS_BASE_URL=https://luzifer.github.io/twitch-bot/ - - - name: Upload GitHub Pages artifact - if: github.ref == 'refs/heads/master' - uses: actions/upload-pages-artifact@v1 - with: - path: .rendered-docs - - - name: Deploy artifact - if: github.ref == 'refs/heads/master' - uses: actions/deploy-pages@v1 - -... diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml deleted file mode 100644 index ae15e85..0000000 --- a/.github/workflows/docker-publish.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- - -name: docker-publish -on: - push: - branches: ['master'] - tags: ['v*'] - -permissions: - packages: write - -jobs: - docker-publish: - defaults: - run: - shell: bash - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - with: - lfs: true - show-progress: false - - - name: Log into registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Docker Build & Publish - run: bash ci/docker-publish.sh - -... diff --git a/.github/workflows/generated_workflow.yml b/.github/workflows/generated_workflow.yml new file mode 100644 index 0000000..19f7f70 --- /dev/null +++ b/.github/workflows/generated_workflow.yml @@ -0,0 +1,341 @@ +name: CI Workflow +on: push +permissions: + packages: write +jobs: + doc-generator: + if: ${{ startsWith(github.ref, 'refs/tags/v') }} + needs: + - test + defaults: + run: + shell: bash + container: + image: luzifer/archlinux + env: + CGO_ENABLED: 0 + GOPATH: /go + permissions: + contents: read + pages: write + id-token: write + runs-on: ubuntu-latest + steps: + - name: Install required packages + run: | + pacman -Syy --noconfirm \ + curl \ + git \ + git-lfs \ + make \ + tar + - uses: actions/checkout@v4 + with: + lfs: true + show-progress: false + - name: Marking workdir safe + run: git config --global --add safe.directory /__w/twitch-bot/twitch-bot + - name: Generate documentation + run: make render_docs DOCS_BASE_URL=https://luzifer.github.io/twitch-bot/ + - name: Upload GitHub Pages artifact + uses: actions/upload-pages-artifact@v1 + with: + path: .rendered-docs + - name: Deploy artifact + uses: actions/deploy-pages@v1 + docker-publish: + if: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master' }} + needs: + - test + defaults: + run: + shell: bash + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + lfs: true + show-progress: false + - name: Log into registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Docker Build & Publish + run: bash ci/docker-publish.sh + integration-crdb: + if: ${{ github.ref == 'refs/heads/master' }} + needs: + - test + defaults: + run: + shell: bash + container: + image: luzifer/archlinux + env: + CGO_ENABLED: 0 + GOPATH: /go + runs-on: ubuntu-latest + services: + crdb: + image: luzifer/crdb-gh-service + steps: + - name: Enable custom AUR package repo + run: echo -e "[luzifer]\nSigLevel = Never\nServer = https://archrepo.hub.luzifer.io/\$arch" >>/etc/pacman.conf + - name: Install required packages + run: | + pacman -Syy --noconfirm \ + cockroachdb-bin \ + git \ + go \ + make + - uses: actions/checkout@v4 + with: + show-progress: false + - name: Marking workdir safe + run: git config --global --add safe.directory /__w/twitch-bot/twitch-bot + - name: Set up CRDB service + run: | + cockroach sql --host crdb --insecure <>/etc/pacman.conf + - name: Install required packages + run: | + pacman -Syy --noconfirm \ + git \ + go \ + make \ + mariadb-clients + - uses: actions/checkout@v4 + with: + show-progress: false + - name: Marking workdir safe + run: git config --global --add safe.directory /__w/twitch-bot/twitch-bot + - name: Set up MariaDB service + run: | + mariadb -h mariadb -u root --password=root-pass <>/etc/pacman.conf + - name: Install required packages + run: | + pacman -Syy --noconfirm \ + git \ + go \ + make \ + mariadb-clients + - uses: actions/checkout@v4 + with: + show-progress: false + - name: Marking workdir safe + run: git config --global --add safe.directory /__w/twitch-bot/twitch-bot + - name: Set up MySQL service + run: | + mariadb -h mysql -u root --password=root-pass <>/etc/pacman.conf + - name: Install required packages + run: | + pacman -Syy --noconfirm \ + git \ + go \ + make + - uses: actions/checkout@v4 + with: + show-progress: false + - name: Marking workdir safe + run: git config --global --add safe.directory /__w/twitch-bot/twitch-bot + - name: Run tests against PostgreSQL + env: + GO_TEST_DB_ENGINE: postgres + GO_TEST_DB_DSN: host=postgres user=postgres password=twitch-bot-pass dbname=postgres port=5432 sslmode=disable timezone=UTC + run: make short_test + release: + if: ${{ startsWith(github.ref, 'refs/tags/v') }} + needs: + - test + defaults: + run: + shell: bash + container: + image: luzifer/archlinux + env: + CGO_ENABLED: 0 + GOPATH: /go + runs-on: ubuntu-latest + steps: + - name: Enable custom AUR package repo + run: echo -e "[luzifer]\nSigLevel = Never\nServer = https://archrepo.hub.luzifer.io/\$arch" >>/etc/pacman.conf + - name: Install required packages + run: | + pacman -Syy --noconfirm \ + awk \ + curl \ + diffutils \ + git \ + git-lfs \ + go \ + golangci-lint-bin \ + make \ + nodejs-lts-hydrogen \ + npm \ + tar \ + trivy \ + unzip \ + which \ + zip + - uses: actions/checkout@v4 + with: + lfs: true + show-progress: false + - name: Marking workdir safe + run: | + git config --global --add safe.directory /__w/twitch-bot/twitch-bot + - name: Build release + run: make publish + - name: Extract changelog + run: awk "/^#/ && ++c==2{exit}; /^#/f" "History.md" | tail -n +2 >release_changelog.md + - name: Update stable branch + if: startsWith(github.ref, 'refs/tags/') + run: | + git branch -f stable ${GITHUB_SHA} + git push -f origin stable + - name: Release + uses: ncipollo/release-action@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + artifacts: .build/* + bodyFile: release_changelog.md + draft: false + generateReleaseNotes: false + test: + if: ${{ github.ref != 'refs/heads/stable' }} + defaults: + run: + shell: bash + container: + image: luzifer/archlinux + env: + CGO_ENABLED: 0 + GOPATH: /go + runs-on: ubuntu-latest + steps: + - name: Enable custom AUR package repo + run: echo -e "[luzifer]\nSigLevel = Never\nServer = https://archrepo.hub.luzifer.io/\$arch" >>/etc/pacman.conf + - name: Install required packages + run: | + pacman -Syy --noconfirm \ + awk \ + curl \ + diffutils \ + git \ + git-lfs \ + go \ + golangci-lint-bin \ + make \ + nodejs-lts-hydrogen \ + npm \ + tar \ + trivy \ + unzip \ + which \ + zip + - uses: actions/checkout@v4 + with: + lfs: true + show-progress: false + - name: Marking workdir safe + run: | + git config --global --add safe.directory /__w/twitch-bot/twitch-bot + - name: Lint and test code + run: make lint test frontend_lint + - name: Build release (quick local for compile) + run: make build_prod + - name: Execute Trivy scan + run: make trivy diff --git a/.github/workflows/integration-crdb.yml b/.github/workflows/integration-crdb.yml deleted file mode 100644 index 2c39345..0000000 --- a/.github/workflows/integration-crdb.yml +++ /dev/null @@ -1,61 +0,0 @@ ---- - -name: integration-crdb -on: - push: - branches: - - master - -permissions: - contents: write - -jobs: - test: - defaults: - run: - shell: bash - - container: - image: luzifer/archlinux - env: - CGO_ENABLED: 0 - GOPATH: /go - - runs-on: ubuntu-latest - - services: - crdb: - image: luzifer/crdb-gh-service - - steps: - - name: Enable custom AUR package repo - run: echo -e "[luzifer]\nSigLevel = Never\nServer = https://archrepo.hub.luzifer.io/\$arch" >>/etc/pacman.conf - - - name: Install required packages - run: | - pacman -Syy --noconfirm \ - cockroachdb-bin \ - git \ - go \ - make - - - uses: actions/checkout@v3 - - - name: Marking workdir safe - run: git config --global --add safe.directory /__w/twitch-bot/twitch-bot - - - name: Set up CRDB service - run: | - cockroach sql --host crdb --insecure <>/etc/pacman.conf - - - name: Install required packages - run: | - pacman -Syy --noconfirm \ - git \ - go \ - make \ - mariadb-clients - - - uses: actions/checkout@v3 - - - name: Marking workdir safe - run: git config --global --add safe.directory /__w/twitch-bot/twitch-bot - - - name: Set up MariaDB service - run: | - mariadb -h mariadb -u root --password=root-pass <>/etc/pacman.conf - - - name: Install required packages - run: | - pacman -Syy --noconfirm \ - git \ - go \ - make \ - mariadb-clients - - - uses: actions/checkout@v3 - - - name: Marking workdir safe - run: git config --global --add safe.directory /__w/twitch-bot/twitch-bot - - - name: Set up MySQL service - run: | - mariadb -h mysql -u root --password=root-pass <>/etc/pacman.conf - - - name: Install required packages - run: | - pacman -Syy --noconfirm \ - git \ - go \ - make - - - uses: actions/checkout@v3 - - - name: Marking workdir safe - run: git config --global --add safe.directory /__w/twitch-bot/twitch-bot - - - name: Run tests against PostgreSQL - env: - GO_TEST_DB_ENGINE: postgres - GO_TEST_DB_DSN: host=postgres user=postgres password=twitch-bot-pass dbname=postgres port=5432 sslmode=disable timezone=UTC - run: make test - -... diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml deleted file mode 100644 index e9723c1..0000000 --- a/.github/workflows/test-and-build.yml +++ /dev/null @@ -1,87 +0,0 @@ ---- - -name: test-and-build -on: - push: - branches-ignore: - - stable - tags: - - 'v*' - -permissions: - contents: write - -jobs: - test-and-build: - defaults: - run: - shell: bash - - container: - image: luzifer/archlinux - env: - CGO_ENABLED: 0 - GOPATH: /go - - runs-on: ubuntu-latest - - steps: - - name: Enable custom AUR package repo - run: echo -e "[luzifer]\nSigLevel = Never\nServer = https://archrepo.hub.luzifer.io/\$arch" >>/etc/pacman.conf - - - name: Install required packages - run: | - pacman -Syy --noconfirm \ - awk \ - curl \ - diffutils \ - git \ - git-lfs \ - go \ - golangci-lint-bin \ - make \ - nodejs-lts-hydrogen \ - npm \ - tar \ - trivy \ - unzip \ - which \ - zip - - - uses: actions/checkout@v4 - with: - lfs: true - show-progress: false - - - name: Marking workdir safe - run: | - git config --global --add safe.directory /__w/twitch-bot/twitch-bot - - - name: Lint and test code - run: make lint test frontend_lint - - - name: Build release - run: make publish - - - name: Execute Trivy scan - run: make trivy - - - name: Extract changelog - run: 'awk "/^#/ && ++c==2{exit}; /^#/f" "History.md" | tail -n +2 >release_changelog.md' - - - name: Update stable branch - if: startsWith(github.ref, 'refs/tags/') - run: | - git branch -f stable ${GITHUB_SHA} - git push -f origin stable - - - name: Release - uses: ncipollo/release-action@v1 - if: startsWith(github.ref, 'refs/tags/') - with: - artifacts: '.build/*' - bodyFile: release_changelog.md - draft: false - generateReleaseNotes: false - -... diff --git a/Makefile b/Makefile index e787fcf..493968b 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,9 @@ lint: publish: frontend_prod bash ./ci/build.sh +short_test: + go test -cover -test.short -v ./... + test: go test -cover -v ./... @@ -41,6 +44,9 @@ update_ua_list: # User-Agents provided by https://www.useragents.me/ curl -sSf https://www.useragents.me/api | jq -r '.data[].ua' | grep -v 'Trident' >internal/linkcheck/user-agents.txt +gh-workflow: + bash ci/create-workflow.sh + # -- Vulnerability scanning -- trivy: diff --git a/ci/create-workflow.sh b/ci/create-workflow.sh new file mode 100644 index 0000000..38016a0 --- /dev/null +++ b/ci/create-workflow.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +target_file=.github/workflows/generated_workflow.yml +source_files=($(find ci/workflow-parts -name 'part_*')) +base=ci/workflow-parts/index.yaml + +cp ${base} ${target_file} + +for source_file in "${source_files[@]}"; do + job_name=${source_file##*/part_} + job_name=${job_name%%.*} + yq -P ".jobs.${job_name} |= load(\"${source_file}\")" ${target_file} >${target_file}.new + mv ${target_file}.new ${target_file} +done diff --git a/ci/workflow-parts/index.yaml b/ci/workflow-parts/index.yaml new file mode 100644 index 0000000..0280b4d --- /dev/null +++ b/ci/workflow-parts/index.yaml @@ -0,0 +1,7 @@ +name: CI Workflow +on: push + +permissions: + packages: write + +jobs: {} diff --git a/ci/workflow-parts/part_doc-generator.yml b/ci/workflow-parts/part_doc-generator.yml new file mode 100644 index 0000000..582575f --- /dev/null +++ b/ci/workflow-parts/part_doc-generator.yml @@ -0,0 +1,53 @@ +--- + +if: ${{ startsWith(github.ref, 'refs/tags/v') }} +needs: [test] + +defaults: + run: + shell: bash + +container: + image: luzifer/archlinux + env: + CGO_ENABLED: 0 + GOPATH: /go + +permissions: + contents: read + pages: write + id-token: write + +runs-on: ubuntu-latest + +steps: + + - name: Install required packages + run: | + pacman -Syy --noconfirm \ + curl \ + git \ + git-lfs \ + make \ + tar + + - uses: actions/checkout@v4 + with: + lfs: true + show-progress: false + + - name: Marking workdir safe + run: git config --global --add safe.directory /__w/twitch-bot/twitch-bot + + - name: Generate documentation + run: make render_docs DOCS_BASE_URL=https://luzifer.github.io/twitch-bot/ + + - name: Upload GitHub Pages artifact + uses: actions/upload-pages-artifact@v1 + with: + path: .rendered-docs + + - name: Deploy artifact + uses: actions/deploy-pages@v1 + +... diff --git a/ci/workflow-parts/part_docker-publish.yml b/ci/workflow-parts/part_docker-publish.yml new file mode 100644 index 0000000..b380224 --- /dev/null +++ b/ci/workflow-parts/part_docker-publish.yml @@ -0,0 +1,28 @@ +--- + +if: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master' }} +needs: [test] + +defaults: + run: + shell: bash + +runs-on: ubuntu-latest + +steps: + - uses: actions/checkout@v4 + with: + lfs: true + show-progress: false + + - name: Log into registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker Build & Publish + run: bash ci/docker-publish.sh + +... diff --git a/ci/workflow-parts/part_integration-crdb.yml b/ci/workflow-parts/part_integration-crdb.yml new file mode 100644 index 0000000..3d6c234 --- /dev/null +++ b/ci/workflow-parts/part_integration-crdb.yml @@ -0,0 +1,55 @@ +--- + +if: ${{ github.ref == 'refs/heads/master' }} +needs: [test] + +defaults: + run: + shell: bash + +container: + image: luzifer/archlinux + env: + CGO_ENABLED: 0 + GOPATH: /go + +runs-on: ubuntu-latest + +services: + crdb: + image: luzifer/crdb-gh-service + +steps: + - name: Enable custom AUR package repo + run: echo -e "[luzifer]\nSigLevel = Never\nServer = https://archrepo.hub.luzifer.io/\$arch" >>/etc/pacman.conf + + - name: Install required packages + run: | + pacman -Syy --noconfirm \ + cockroachdb-bin \ + git \ + go \ + make + + - uses: actions/checkout@v4 + with: + show-progress: false + + - name: Marking workdir safe + run: git config --global --add safe.directory /__w/twitch-bot/twitch-bot + + - name: Set up CRDB service + run: | + cockroach sql --host crdb --insecure <>/etc/pacman.conf + + - name: Install required packages + run: | + pacman -Syy --noconfirm \ + git \ + go \ + make \ + mariadb-clients + + - uses: actions/checkout@v4 + with: + show-progress: false + + - name: Marking workdir safe + run: git config --global --add safe.directory /__w/twitch-bot/twitch-bot + + - name: Set up MariaDB service + run: | + mariadb -h mariadb -u root --password=root-pass <>/etc/pacman.conf + + - name: Install required packages + run: | + pacman -Syy --noconfirm \ + git \ + go \ + make \ + mariadb-clients + + - uses: actions/checkout@v4 + with: + show-progress: false + + - name: Marking workdir safe + run: git config --global --add safe.directory /__w/twitch-bot/twitch-bot + + - name: Set up MySQL service + run: | + mariadb -h mysql -u root --password=root-pass <>/etc/pacman.conf + + - name: Install required packages + run: | + pacman -Syy --noconfirm \ + git \ + go \ + make + + - uses: actions/checkout@v4 + with: + show-progress: false + + - name: Marking workdir safe + run: git config --global --add safe.directory /__w/twitch-bot/twitch-bot + + - name: Run tests against PostgreSQL + env: + GO_TEST_DB_ENGINE: postgres + GO_TEST_DB_DSN: host=postgres user=postgres password=twitch-bot-pass dbname=postgres port=5432 sslmode=disable timezone=UTC + run: make short_test + +... diff --git a/ci/workflow-parts/part_release.yml b/ci/workflow-parts/part_release.yml new file mode 100644 index 0000000..f0fd4c9 --- /dev/null +++ b/ci/workflow-parts/part_release.yml @@ -0,0 +1,71 @@ +--- + +if: ${{ startsWith(github.ref, 'refs/tags/v') }} +needs: [test] + +defaults: + run: + shell: bash + +container: + image: luzifer/archlinux + env: + CGO_ENABLED: 0 + GOPATH: /go + +runs-on: ubuntu-latest + +steps: + - name: Enable custom AUR package repo + run: echo -e "[luzifer]\nSigLevel = Never\nServer = https://archrepo.hub.luzifer.io/\$arch" >>/etc/pacman.conf + + - name: Install required packages + run: | + pacman -Syy --noconfirm \ + awk \ + curl \ + diffutils \ + git \ + git-lfs \ + go \ + golangci-lint-bin \ + make \ + nodejs-lts-hydrogen \ + npm \ + tar \ + trivy \ + unzip \ + which \ + zip + + - uses: actions/checkout@v4 + with: + lfs: true + show-progress: false + + - name: Marking workdir safe + run: | + git config --global --add safe.directory /__w/twitch-bot/twitch-bot + + - name: Build release + run: make publish + + - name: Extract changelog + run: 'awk "/^#/ && ++c==2{exit}; /^#/f" "History.md" | tail -n +2 >release_changelog.md' + + - name: Update stable branch + if: startsWith(github.ref, 'refs/tags/') + run: | + git branch -f stable ${GITHUB_SHA} + git push -f origin stable + + - name: Release + uses: ncipollo/release-action@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + artifacts: '.build/*' + bodyFile: release_changelog.md + draft: false + generateReleaseNotes: false + +... diff --git a/ci/workflow-parts/part_test.yml b/ci/workflow-parts/part_test.yml new file mode 100644 index 0000000..ac27886 --- /dev/null +++ b/ci/workflow-parts/part_test.yml @@ -0,0 +1,58 @@ +--- + +if: ${{ github.ref != 'refs/heads/stable' }} + +defaults: + run: + shell: bash + +container: + image: luzifer/archlinux + env: + CGO_ENABLED: 0 + GOPATH: /go + +runs-on: ubuntu-latest + +steps: + - name: Enable custom AUR package repo + run: echo -e "[luzifer]\nSigLevel = Never\nServer = https://archrepo.hub.luzifer.io/\$arch" >>/etc/pacman.conf + + - name: Install required packages + run: | + pacman -Syy --noconfirm \ + awk \ + curl \ + diffutils \ + git \ + git-lfs \ + go \ + golangci-lint-bin \ + make \ + nodejs-lts-hydrogen \ + npm \ + tar \ + trivy \ + unzip \ + which \ + zip + + - uses: actions/checkout@v4 + with: + lfs: true + show-progress: false + + - name: Marking workdir safe + run: | + git config --global --add safe.directory /__w/twitch-bot/twitch-bot + + - name: Lint and test code + run: make lint test frontend_lint + + - name: Build release (quick local for compile) + run: make build_prod + + - name: Execute Trivy scan + run: make trivy + +...