diff --git a/.forgejo/workflows/deploy-community.yml b/.forgejo/workflows/deploy-community.yml new file mode 100644 index 0000000..16d1cf0 --- /dev/null +++ b/.forgejo/workflows/deploy-community.yml @@ -0,0 +1,141 @@ +# ============================================================================= +# Deploy SQLInsight Pro-webstedet (sqlinsight.seosoft.dk) til appserveren ved push til main. +# +# Samme mønster som deploy-hub.yml: runneren er host-mode, så .env og compose-filen skrives til INSTALL_DIR +# via en kortlivet helper-container, og 'docker compose up' køres i docker:cli. +# +# Påkrævede secrets: HOST_BIND COMMUNITY_PORT COMMUNITY_CONTACT +# Valgfri: TRUSTED_KEY_ID + TRUSTED_KEY (offentlig nøgle, som feedet er signeret med, hvis den ikke er indbygget) +# ============================================================================= +name: deploy community + +on: + push: + branches: [main] + paths: + - "src/**" + - "docs/**" + - "rules/**" + - "deploy/community/**" + - "Directory.*.props" + - "global.json" + - ".forgejo/workflows/deploy-community.yml" + workflow_dispatch: + +concurrency: + group: deploy-sqlinsight-community + cancel-in-progress: false + +env: + INSTALL_DIR: /opt/sqlinsight-community + IMAGE: sqlinsight-community + CONTAINER: sqlinsight-community + +jobs: + deploy: + runs-on: [self-hosted, appserver] + timeout-minutes: 30 + + steps: + - name: Configured? + id: check + env: + COMMUNITY_PORT: ${{ secrets.COMMUNITY_PORT }} + run: | + # Indtil secrets er sat, springes deploy over i stedet for at fejle hvert push. + if [ -n "${COMMUNITY_PORT:-}" ]; then + echo "configured=true" >> "$GITHUB_OUTPUT" + else + echo "::warning::COMMUNITY_PORT er ikke sat; webstedet deployes ikke (se docs/community.md)" + echo "configured=false" >> "$GITHUB_OUTPUT" + fi + + - name: Checkout + if: steps.check.outputs.configured == 'true' + uses: actions/checkout@v4 + + - name: Build image + if: steps.check.outputs.configured == 'true' + run: | + set -e + docker build -f deploy/community/Dockerfile -t "${IMAGE}:latest" -t "${IMAGE}:${{ github.sha }}" . + + - name: Write .env + docker-compose.yml to INSTALL_DIR + if: steps.check.outputs.configured == 'true' + env: + HOST_BIND: ${{ secrets.HOST_BIND }} + COMMUNITY_PORT: ${{ secrets.COMMUNITY_PORT }} + COMMUNITY_CONTACT: ${{ secrets.COMMUNITY_CONTACT }} + TRUSTED_KEY_ID: ${{ secrets.TRUSTED_KEY_ID }} + TRUSTED_KEY: ${{ secrets.TRUSTED_KEY }} + run: | + set -eu + required="HOST_BIND COMMUNITY_PORT COMMUNITY_CONTACT" + missing=0 + for v in $required; do + if [ -z "$(printenv "$v" || true)" ]; then + echo "ERROR: required secret $v is not set in repo settings" + missing=1 + fi + done + [ "$missing" -eq 0 ] || exit 1 + + for v in $required TRUSTED_KEY_ID TRUSTED_KEY; do + case "$(printenv "$v" || true)" in + *"'"*|*";"*) echo "ERROR: secret $v indeholder ' eller ; – ikke understøttet"; exit 1 ;; + esac + done + + tmp="$(mktemp -d)" + { + printf "HOST_BIND='%s'\n" "$HOST_BIND" + printf "COMMUNITY_PORT='%s'\n" "$COMMUNITY_PORT" + printf "SQLInsight__Community__ContactEmail='%s'\n" "$COMMUNITY_CONTACT" + if [ -n "${TRUSTED_KEY_ID:-}" ] && [ -n "${TRUSTED_KEY:-}" ]; then + printf "SQLInsight__Community__TrustedKeys__%s='%s'\n" "$TRUSTED_KEY_ID" "$TRUSTED_KEY" + fi + } > "$tmp/.env" + cp deploy/community/docker-compose.prod.yml "$tmp/docker-compose.yml" + + tar -C "$tmp" -cf - .env docker-compose.yml | docker run --rm -i \ + -v "${INSTALL_DIR}:/out" alpine:3 \ + sh -c 'tar -C /out -xf - && chown root:root /out/.env /out/docker-compose.yml && chmod 600 /out/.env && chmod 644 /out/docker-compose.yml' + rm -rf "$tmp" + + - name: Deploy (compose up) + if: steps.check.outputs.configured == 'true' + run: | + set -eu + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v "${INSTALL_DIR}:${INSTALL_DIR}" \ + -w "${INSTALL_DIR}" \ + docker:cli \ + docker compose --project-name sqlinsight-community up -d --force-recreate --remove-orphans + + - name: Wait for /health + if: steps.check.outputs.configured == 'true' + env: + HOST_BIND: ${{ secrets.HOST_BIND }} + COMMUNITY_PORT: ${{ secrets.COMMUNITY_PORT }} + run: | + set -eu + for i in $(seq 1 30); do + if docker run --rm alpine:3 wget -qO- -T 3 "http://${HOST_BIND}:${COMMUNITY_PORT}/health" 2>/dev/null | grep -q '"ok"'; then + echo "${CONTAINER} is healthy" + exit 0 + fi + echo "attempt $i: not ready" + sleep 5 + done + echo "${CONTAINER} did not answer /health within 150s — last 100 log lines:" + docker logs --tail 100 "${CONTAINER}" || true + exit 1 + + - name: Prune old images + if: always() && steps.check.outputs.configured == 'true' + run: | + docker image prune -f || true + docker images "${IMAGE}" --format '{{.Tag}} {{.CreatedAt}}' \ + | grep -v '^latest ' | sort -k2 -r | awk 'NR>3 {print $1}' \ + | xargs -r -I{} docker image rm "${IMAGE}:{}" || true diff --git a/.forgejo/workflows/deploy-forum.yml b/.forgejo/workflows/deploy-forum.yml new file mode 100644 index 0000000..edc1adb --- /dev/null +++ b/.forgejo/workflows/deploy-forum.yml @@ -0,0 +1,73 @@ +# ============================================================================= +# Installér eller genopbyg forummet (Discourse, forum.sqlinsight.seosoft.dk) på appserveren. +# +# Kun manuelt (Actions → deploy forum → Run workflow): en rebuild tager 10-15 minutter, og forummet er nede imens. +# Discourse opdateres ellers fra /admin/upgrade. +# +# Runneren er host-mode, så launcheren køres i en helper-container med /var/discourse og docker.sock fra værten. +# app.yml udfyldes fra secrets af render.py; intet hemmeligt ligger i repoet. +# +# Påkrævede secrets: HOST_BIND FORUM_PORT KNOWN_PROXY FORUM_ADMIN_EMAIL FORUM_SMTP_ADDRESS FORUM_SMTP_USER FORUM_SMTP_PASSWORD +# Valgfri: FORUM_SMTP_PORT (587) FORUM_NOTIFICATION_EMAIL (= FORUM_SMTP_USER) FORUM_WORKERS (4) FORUM_DB_SHARED_BUFFERS (512MB) +# ============================================================================= +name: deploy forum + +on: + workflow_dispatch: + +concurrency: + group: deploy-sqlinsight-forum + cancel-in-progress: false + +jobs: + deploy: + runs-on: [self-hosted, appserver] + timeout-minutes: 60 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build launcher helper + run: docker build -t sqlinsight-forum-launcher deploy/forum + + - name: Rebuild Discourse + env: + FORUM_BIND: ${{ secrets.HOST_BIND }} + FORUM_PORT: ${{ secrets.FORUM_PORT }} + KNOWN_PROXY: ${{ secrets.KNOWN_PROXY }} + FORUM_ADMIN_EMAIL: ${{ secrets.FORUM_ADMIN_EMAIL }} + FORUM_SMTP_ADDRESS: ${{ secrets.FORUM_SMTP_ADDRESS }} + FORUM_SMTP_PORT: ${{ secrets.FORUM_SMTP_PORT }} + FORUM_SMTP_USER: ${{ secrets.FORUM_SMTP_USER }} + FORUM_SMTP_PASSWORD: ${{ secrets.FORUM_SMTP_PASSWORD }} + FORUM_NOTIFICATION_EMAIL: ${{ secrets.FORUM_NOTIFICATION_EMAIL }} + FORUM_WORKERS: ${{ secrets.FORUM_WORKERS }} + FORUM_DB_SHARED_BUFFERS: ${{ secrets.FORUM_DB_SHARED_BUFFERS }} + run: | + set -eu + # Samme sti inde og ude, så launcherens bind mounts (/var/discourse/shared/...) peger på værtens mapper. + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/discourse:/var/discourse \ + -e FORUM_BIND -e FORUM_PORT -e KNOWN_PROXY -e FORUM_ADMIN_EMAIL \ + -e FORUM_SMTP_ADDRESS -e FORUM_SMTP_PORT -e FORUM_SMTP_USER -e FORUM_SMTP_PASSWORD \ + -e FORUM_NOTIFICATION_EMAIL -e FORUM_WORKERS -e FORUM_DB_SHARED_BUFFERS \ + sqlinsight-forum-launcher + + - name: Wait for the forum + env: + HOST_BIND: ${{ secrets.HOST_BIND }} + FORUM_PORT: ${{ secrets.FORUM_PORT }} + run: | + set -eu + for i in $(seq 1 60); do + if docker run --rm alpine:3 wget -qO- -T 3 "http://${HOST_BIND}:${FORUM_PORT}/srv/status" 2>/dev/null | grep -q ok; then + echo "forum is up" + exit 0 + fi + echo "attempt $i: not ready" + sleep 5 + done + docker logs --tail 100 app || true + exit 1 diff --git a/deploy/community/Caddyfile b/deploy/community/Caddyfile deleted file mode 100644 index 57773c6..0000000 --- a/deploy/community/Caddyfile +++ /dev/null @@ -1,5 +0,0 @@ -{$COMMUNITY_HOST} { - encode zstd gzip - reverse_proxy community:8080 - header Strict-Transport-Security "max-age=31536000" -} diff --git a/deploy/community/docker-compose.prod.yml b/deploy/community/docker-compose.prod.yml new file mode 100644 index 0000000..1632c34 --- /dev/null +++ b/deploy/community/docker-compose.prod.yml @@ -0,0 +1,31 @@ +# SQLInsight Pro-webstedet (sqlinsight.seosoft.dk) i SEOSofts drift (appserver). Kopieres til INSTALL_DIR af +# .forgejo/workflows/deploy-community.yml sammen med en .env, der genereres fra Forgejo repo-secrets. +# Reverse proxyen terminerer TLS, som for hubben. +# +# Feedet læses direkte fra hubbens feed-mappe (skrivebeskyttet), så webstedet viser præcis det, agenterne får. +# Download-links går til hubbens offentlige /updates/. +# +# Manuel drift på appserveren: +# cd /opt/sqlinsight-community && docker compose ps | logs -f | restart + +services: + community: + image: sqlinsight-community:latest + container_name: sqlinsight-community + restart: unless-stopped + env_file: + - .env + environment: + SQLInsight__Community__Feed: /feed + SQLInsight__Community__DownloadUrl: https://hub.sqlinsight.seosoft.dk/updates/ + SQLInsight__Community__ForumUrl: https://forum.sqlinsight.seosoft.dk/ + volumes: + - /opt/sqlinsight-hub/data/feed:/feed:ro + ports: + # Bind til DMZ-IP (HOST_BIND i .env). Reverse proxyen terminerer TLS. + - "${HOST_BIND}:${COMMUNITY_PORT}:8080" + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" diff --git a/deploy/community/docker-compose.yml b/deploy/community/docker-compose.yml deleted file mode 100644 index aca71c9..0000000 --- a/deploy/community/docker-compose.yml +++ /dev/null @@ -1,39 +0,0 @@ -# The SQLInsight Pro website behind Caddy, which gets and renews the TLS certificate itself. -# DNS for COMMUNITY_HOST must point at this server, and ports 80 and 443 must be free (Discourse runs on its own server). -# -# COMMUNITY_HOST=sqlinsight.seosoft.dk docker compose up -d -services: - community: - build: - context: ../.. - dockerfile: deploy/community/Dockerfile - image: sqlinsight-community - restart: unless-stopped - environment: - # The signed update feed: the hub's /updates/ (or a folder mounted into the container). - SQLInsight__Community__Feed: ${COMMUNITY_FEED:-https://hub.sqlinsight.seosoft.dk/updates/} - SQLInsight__Community__ForumUrl: ${COMMUNITY_FORUM:-https://forum.sqlinsight.seosoft.dk/} - SQLInsight__Community__ContactEmail: ${COMMUNITY_CONTACT:?set COMMUNITY_CONTACT (the address for support enquiries)} - # The public key releases are signed with, if it is not embedded in the build (the vendor tool's keygen prints it): - # SQLInsight__Community__TrustedKeys__k20260929: "MFkw..." - expose: - - "8080" - - caddy: - image: caddy:2-alpine - restart: unless-stopped - depends_on: - - community - environment: - COMMUNITY_HOST: ${COMMUNITY_HOST:?set COMMUNITY_HOST} - ports: - - "80:80" - - "443:443" - volumes: - - ./Caddyfile:/etc/caddy/Caddyfile:ro - - caddy_data:/data - - caddy_config:/config - -volumes: - caddy_data: - caddy_config: diff --git a/deploy/forum/Dockerfile b/deploy/forum/Dockerfile new file mode 100644 index 0000000..151ec99 --- /dev/null +++ b/deploy/forum/Dockerfile @@ -0,0 +1,8 @@ +# Helper image that runs the discourse_docker launcher from the self-hosted runner (host-mode): +# /var/discourse and docker.sock are mounted from the host, so the launcher's host paths are the real ones. +ARG BASE=docker:cli +FROM ${BASE} +RUN apk add --no-cache bash git python3 coreutils sed grep gawk procps +WORKDIR /src +COPY app.yml render.py install.sh configure.py ./ +ENTRYPOINT ["bash", "/src/install.sh"] diff --git a/deploy/forum/app.yml b/deploy/forum/app.yml index a84794b..aa91299 100644 --- a/deploy/forum/app.yml +++ b/deploy/forum/app.yml @@ -1,26 +1,27 @@ -## SQLInsight Pro community forum: Discourse, installed with the official discourse_docker launcher. -## See docs/community.md. Copy to /var/discourse/containers/app.yml, fill in the values marked CHANGE, and run -## ./launcher rebuild app -## (deploy/forum/install.sh does this for you.) +## SQLInsight Pro community forum (forum.sqlinsight.seosoft.dk): Discourse with the official discourse_docker launcher. +## See docs/community.md. +## +## This is a template: the double-braced names are filled in by render.py from environment variables (Forgejo secrets in +## .forgejo/workflows/deploy-forum.yml, or exported by hand before deploy/forum/install.sh). Nothing secret is committed. +## +## The forum runs behind SEOSoft's reverse proxy, which terminates TLS (like the hub). The proxy must send +## Host, X-Forwarded-For and X-Forwarded-Proto: https, and allow WebSocket/long polling (/message-bus/). templates: - "templates/postgres.template.yml" - "templates/redis.template.yml" - "templates/web.template.yml" - "templates/web.ratelimited.template.yml" - ## TLS with Let's Encrypt. Remove both lines if a reverse proxy in front already terminates TLS. - - "templates/web.ssl.template.yml" - - "templates/web.letsencrypt.ssl.template.yml" +## Only on the DMZ address, where the reverse proxy reaches it. expose: - - "80:80" - - "443:443" + - "{{FORUM_BIND}}:{{FORUM_PORT}}:80" params: ## Danish stemming for search. db_default_text_search_config: "pg_catalog.danish" - ## About 25 % of the server's memory. - db_shared_buffers: "512MB" + ## About 25 % of the memory Discourse may use. + db_shared_buffers: "{{FORUM_DB_SHARED_BUFFERS}}" env: LC_ALL: en_US.UTF-8 @@ -28,24 +29,22 @@ env: LANGUAGE: en_US.UTF-8 DISCOURSE_DEFAULT_LOCALE: da ## About 2 per CPU core. - UNICORN_WORKERS: 4 + UNICORN_WORKERS: {{FORUM_WORKERS}} - ## CHANGE: the forum's address (a DNS record must point to this server). - DISCOURSE_HOSTNAME: forum.sqlinsight.seosoft.dk + DISCOURSE_HOSTNAME: '{{FORUM_HOSTNAME}}' + ## TLS is terminated by the reverse proxy; all links must still be https. + DISCOURSE_FORCE_HTTPS: true - ## CHANGE: who becomes administrator when signing up with these addresses. - DISCOURSE_DEVELOPER_EMAILS: "admin@example.com" + ## Who becomes administrator when signing up with these addresses (comma separated). + DISCOURSE_DEVELOPER_EMAILS: '{{FORUM_ADMIN_EMAIL}}' - ## CHANGE: mail server. Discourse cannot work without it (sign-up, notifications, password reset). - DISCOURSE_SMTP_ADDRESS: smtp.example.com - DISCOURSE_SMTP_PORT: 587 - DISCOURSE_SMTP_USER_NAME: forum@example.com - DISCOURSE_SMTP_PASSWORD: "CHANGE" + ## Mail server. Discourse cannot work without it (sign-up, notifications, password reset). + DISCOURSE_SMTP_ADDRESS: '{{FORUM_SMTP_ADDRESS}}' + DISCOURSE_SMTP_PORT: {{FORUM_SMTP_PORT}} + DISCOURSE_SMTP_USER_NAME: '{{FORUM_SMTP_USER}}' + DISCOURSE_SMTP_PASSWORD: '{{FORUM_SMTP_PASSWORD}}' DISCOURSE_SMTP_ENABLE_START_TLS: true - DISCOURSE_NOTIFICATION_EMAIL: forum@example.com - - ## CHANGE: address Let's Encrypt writes to about expiring certificates. - LETSENCRYPT_ACCOUNT_EMAIL: admin@example.com + DISCOURSE_NOTIFICATION_EMAIL: '{{FORUM_NOTIFICATION_EMAIL}}' volumes: - volume: @@ -65,4 +64,14 @@ hooks: run: - exec: echo "Beginning of custom commands" + ## Real client addresses from the reverse proxy. Without this every visitor looks like the proxy, and + ## web.ratelimited would throttle them all together. + - replace: + filename: "/etc/nginx/conf.d/discourse.conf" + from: /server \{/ + to: | + server { + set_real_ip_from {{KNOWN_PROXY}}; + real_ip_header X-Forwarded-For; + real_ip_recursive on; - exec: echo "End of custom commands" diff --git a/deploy/forum/install.sh b/deploy/forum/install.sh index ad34ab5..f1f3b9f 100755 --- a/deploy/forum/install.sh +++ b/deploy/forum/install.sh @@ -1,46 +1,44 @@ #!/usr/bin/env bash -# Installs the SQLInsight Pro forum (Discourse) on a fresh Linux server with the official discourse_docker launcher. -# Run as root on the forum server, from a checkout of this repository, after filling in deploy/forum/app.yml: -# sudo deploy/forum/install.sh -# Needs: DNS for the forum's host name pointing here, ports 80 and 443 free, 2 GB memory (4 GB recommended) and a mail server. +# Installs or rebuilds the SQLInsight Pro forum (Discourse) with the official discourse_docker launcher. +# +# Normally run by .forgejo/workflows/deploy-forum.yml (in a helper container with /var/discourse and docker.sock +# from the host). By hand, as root on the server, with the values render.py needs exported: +# sudo -E deploy/forum/install.sh +# Needs 4 GB memory (2 GB minimum), 20 GB disk, and a mail server. set -euo pipefail HERE="$(cd "$(dirname "$0")" && pwd)" TARGET=/var/discourse if [[ $EUID -ne 0 ]]; then - echo "Kør som root (sudo)." >&2 + echo "Kør som root (sudo -E)." >&2 exit 1 fi -# Refuse to install with the example values still in place: Discourse is useless without a working mail server. -if grep -nE 'example\.com|"CHANGE"' "$HERE/app.yml" | grep -vE '^[0-9]+:\s*#'; then - echo "Udfyld værdierne markeret CHANGE i $HERE/app.yml først (linjerne ovenfor)." >&2 - exit 1 -fi - -if ! command -v docker >/dev/null; then - echo "Installerer Docker ..." - curl -fsSL https://get.docker.com | sh -fi +command -v docker >/dev/null || { echo "Docker mangler." >&2; exit 1; } if [[ ! -d $TARGET/.git ]]; then git clone --depth 1 https://github.com/discourse/discourse_docker.git "$TARGET" +else + git -C "$TARGET" pull -q --ff-only fi -install -m 600 "$HERE/app.yml" "$TARGET/containers/app.yml" -cd "$TARGET" -./launcher rebuild app +mkdir -p "$TARGET/containers" +python3 "$HERE/render.py" "$HERE/app.yml" "$TARGET/containers/app.yml" -host=$(sed -n 's/^\s*DISCOURSE_HOSTNAME:\s*//p' "$HERE/app.yml" | tr -d '"' | head -1) +cd "$TARGET" +# The launcher asks before continuing on low memory; answer yes when running unattended. +./launcher rebuild app --skip-prereqs + +host="${FORUM_HOSTNAME:-forum.sqlinsight.seosoft.dk}" cat < CONTACT_EMAIL= python3 $HERE/configure.py - 4. Slet API-nøglen igen under Admin → API, når opsætningen er færdig. +Discourse kører på ${FORUM_BIND}:${FORUM_PORT}. Første gang: + 1. Peg reverse proxyen for $host på http://${FORUM_BIND}:${FORUM_PORT} (med X-Forwarded-Proto: https og WebSockets). + 2. Åbn https://$host og opret administratoren med adressen i FORUM_ADMIN_EMAIL. + 3. Lav en API-nøgle til opsætningen: + docker exec app su discourse -c 'cd /var/www/discourse && RAILS_ENV=production bundle exec rake api_key:create_master[forum-setup]' + 4. Opsæt kategorier, sprog og velkomstemne (kan køres igen): + DISCOURSE_URL=https://$host DISCOURSE_API_KEY= CONTACT_EMAIL= python3 deploy/forum/configure.py + 5. Slet API-nøglen igen under Admin → API. NEXT diff --git a/deploy/forum/render.py b/deploy/forum/render.py new file mode 100755 index 0000000..7f1c113 --- /dev/null +++ b/deploy/forum/render.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Fills in deploy/forum/app.yml from environment variables and writes the result (mode 600). + + render.py