diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml new file mode 100644 index 0000000..94a22c3 --- /dev/null +++ b/.forgejo/workflows/ci.yml @@ -0,0 +1,43 @@ +# CI: lint + typecheck + unit tests on every branch / PR. +# +# Self-contained on purpose: python:3.14-trixie has git built in, so no +# node/actions-checkout dependency; uv comes from PyPI (both reachable from +# the runner network). Mirrors the Makefile lint/test-unit targets. + +name: ci + +on: + push: + branches: ["**"] + pull_request: + +jobs: + test: + runs-on: docker + container: + image: python:3.14-trixie + steps: + - name: Checkout (sha-pinned, no node actions needed) + run: | + git init -q . + git remote add origin ${{ gitea.server_url }}/${{ gitea.repository }}.git + git fetch --depth 1 origin ${{ gitea.sha }} + git checkout -q FETCH_HEAD + + - name: Install uv + run: pip install --quiet uv + + - name: Sync dev dependencies + run: uv sync --group dev --frozen + + - name: Lint (ruff) + run: | + uv run ruff check src tests + uv run ruff format --check src tests + uv run isort --check-only src tests + + - name: Typecheck (ty) + run: uv run ty check src + + - name: Unit tests + run: uv run pytest -m "not integration" tests/unit diff --git a/.forgejo/workflows/deploy.yml b/.forgejo/workflows/deploy.yml new file mode 100644 index 0000000..6cbacc3 --- /dev/null +++ b/.forgejo/workflows/deploy.yml @@ -0,0 +1,76 @@ +# Deploy: build → push → nomad job run, on every push to main. +# +# Runner requirements (verify on the Forgejo VPS): +# - job container gets the docker socket (runner config must allow +# valid_volumes for /var/run/docker.sock — see README §deploy notes), or +# switch `runs-on` to the host label and drop the container/socket bits; +# - Docker Hub reachable (nomad CLI is extracted from hashicorp/nomad:1.9 — +# releases.hashicorp.com is blocked from this network). +# +# Repo secrets (Settings → Secrets): +# REGISTRY_TOKEN forgejo token, write:package scope (docker login) +# NOMAD_TOKEN CI ACL token from deploy/nomad/README.md §5 +# NOMAD_CACERT contents of nomad-ca.crt from README §3 +# Repo variables (Settings → Variables) or edit the env block below: +# REGISTRY_HOST e.g. git.example.com +# REGISTRY_OWNER forgejo user/org owning the images +# NOMAD_ADDR_HOST services VPS public IP or DNS name (API :4646) + +name: deploy + +on: + push: + branches: [main] + +env: + REGISTRY_HOST: ${{ vars.REGISTRY_HOST }} + REGISTRY_OWNER: ${{ vars.REGISTRY_OWNER }} + +jobs: + build-and-deploy: + runs-on: docker + container: + image: docker:28-cli + options: --volume /var/run/docker.sock:/var/run/docker.sock + steps: + - name: Checkout + run: | + apk add --no-cache git + git init -q . + git remote add origin ${{ gitea.server_url }}/${{ gitea.repository }}.git + git fetch --depth 1 origin ${{ gitea.sha }} + git checkout -q FETCH_HEAD + + - name: Extract nomad CLI (official image; CDN is blocked here) + run: | + docker create --name nomad-extract hashicorp/nomad:1.9 >/dev/null + docker cp nomad-extract:/bin/nomad /usr/local/bin/nomad + docker rm nomad-extract >/dev/null + nomad version + + - name: Registry login + run: | + echo "${{ secrets.REGISTRY_TOKEN }}" | \ + docker login "$REGISTRY_HOST" -u "$REGISTRY_OWNER" --password-stdin + + - name: Build & push images (SHA tag) + run: | + set -e + TAG="${GITEA_SHA}" + for name in api worker-extract worker-analyze worker-prescreen worker-billing worker-notify; do + image="$REGISTRY_HOST/$REGISTRY_OWNER/contract-check-$name" + docker build -f "srv/$name/Dockerfile" -t "$image:$TAG" . + docker push "$image:$TAG" + done + + - name: Deploy to Nomad + run: | + mkdir -p /tmp/nomad-tls + echo "${{ secrets.NOMAD_CACERT }}" > /tmp/nomad-tls/ca.crt + export IMAGE_TAG="${GITEA_SHA}" + export NOMAD_ADDR="https://${{ vars.NOMAD_ADDR_HOST }}:4646" + export NOMAD_CACERT=/tmp/nomad-tls/ca.crt + export NOMAD_TOKEN="${{ secrets.NOMAD_TOKEN }}" + nomad job run deploy/nomad/contract-check.nomad.hcl + # the CLI follows the deployment and exits non-zero if health + # checks fail — auto_revert then rolls the job back server-side. diff --git a/deploy/nomad/README.md b/deploy/nomad/README.md new file mode 100644 index 0000000..3b72be3 --- /dev/null +++ b/deploy/nomad/README.md @@ -0,0 +1,256 @@ +# Nomad on the services VPS — install & operations runbook + +Single-node Nomad agent (server+client) running the app services +(`api` + 5 workers) while stateful infra (postgres, redis, rabbitmq, minio), +the nginx edge cascade, and observability stay on docker compose. + +Layout: + +``` +deploy/nomad/ +├── nomad.hcl # agent config → /etc/nomad.d/nomad.hcl +├── policies/ci.hcl # ACL policy for the Forgejo deploy token +├── contract-check.nomad.hcl # job: api + worker groups +└── README.md # this runbook +``` + +Placeholders used below — substitute before running: + +| Placeholder | Meaning | +|---|---| +| `` | public IP of the services VPS | +| `` | public IP of the Forgejo VPS (runner) | +| `` | Forgejo hostname, e.g. `git.example.com` | +| `` | Forgejo user/org owning the repo + images | + +--- + +## 1. Install Nomad + +### 1a. Manual install via the official Docker image + +The services VPS network 404s `apt.releases.hashicorp.com`, +`releases.hashicorp.com` and GitHub release assets, and Ubuntu `resolute` +is newer than the apt repo's suites — so extract the binary from the +official Docker image instead (Docker Hub is reachable there): + +```bash +docker pull hashicorp/nomad:1.9 +docker create --name nomad-extract hashicorp/nomad:1.9 +docker cp nomad-extract:/bin/nomad /tmp/nomad +docker rm nomad-extract +chmod +x /tmp/nomad && /tmp/nomad version +sudo install -m 0755 /tmp/nomad /usr/local/bin/nomad +# if the cp path is wrong: +# docker run --rm --entrypoint sh hashicorp/nomad:1.9 -c 'command -v nomad' +``` + +**Why 1.9.x and not 2.x:** Nomad 2.x extracts the docker driver into an +external plugin distributed via the blocked hosts above. 1.9.x has it +built in — everything this stack uses (nomadVar templates, native +services, static ports, canary updates) is fully supported there. + +No `.deb` ⇒ no systemd unit ships; install ours: + +```bash +sudo cp deploy/nomad/nomad.service /etc/systemd/system/nomad.service +``` + +### 1b. apt install (only on networks that reach HashiCorp's repo) + +```bash +wget -O- https://apt.releases.hashicorp.com/gpg | \ + sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg +echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] \ + https://apt.releases.hashicorp.com $(lsb_release -cs) main" | \ + sudo tee /etc/apt/sources.list.d/hashicorp.list +sudo apt-get update && sudo apt-get install -y nomad +nomad version # expect 1.7+ +``` + +Note: the `.deb`'s own unit lives in /lib/systemd/system — do NOT install +`deploy/nomad/nomad.service` in that case. + +## 2. Install config + data dir + +```bash +sudo mkdir -p /var/lib/nomad /etc/nomad.d/tls +# from the repo root on the VPS: +sudo cp deploy/nomad/nomad.hcl /etc/nomad.d/nomad.hcl +``` + +## 3. Generate TLS material + +The API is reachable from the internet (Forgejo runner → VPS), so TLS is +mandatory. One mini-CA + one server cert; the runner gets **only** the CA +cert. + +```bash +cd /tmp && mkdir nomad-tls && cd nomad-tls + +# CA (keep nomad-ca.key offline afterwards; only the .crt is ever copied) +openssl req -x509 -newkey rsa:2048 -nodes -days 3650 \ + -keyout nomad-ca.key -out nomad-ca.crt -subj "/CN=Contract-Check Nomad CA" + +# server cert — SANs must cover the RPC hostname AND how clients reach it +openssl req -newkey rsa:2048 -nodes \ + -keyout server.key -out server.csr \ + -subj "/CN=server.global.vps.nomad" + +cat > server.ext <<'EOF' +subjectAltName = DNS:server.global.vps.nomad, DNS:localhost, IP:127.0.0.1, IP: +extendedKeyUsage = serverAuth, clientAuth +EOF + +openssl x509 -req -in server.csr \ + -CA nomad-ca.crt -CAkey nomad-ca.key -CAcreateserial \ + -out server.crt -days 825 -extfile server.ext + +sudo install -m 644 nomad-ca.crt server.crt /etc/nomad.d/tls/ +sudo install -m 600 server.key /etc/nomad.d/tls/ +``` + +Copy `nomad-ca.crt` to the Forgejo VPS (or a repo secret / secure storage) +— the deploy workflow needs it as `NOMAD_CACERT`. + +## 4. Start the agent + bootstrap ACLs + +```bash +sudo systemctl enable --now nomad +systemctl status nomad --no-pager +``` + +For every CLI call on the VPS, export: + +```bash +export NOMAD_ADDR=https://127.0.0.1:4646 +export NOMAD_CACERT=/etc/nomad.d/tls/nomad-ca.crt +``` + +Bootstrap ACLs once: + +```bash +nomad acl bootstrap +# → prints an AccessorID/SecretID management token. Store the SecretID in a +# password manager; export it for the following commands: +export NOMAD_TOKEN= +``` + +Sanity checks: + +```bash +nomad server members # self as leader +nomad node status # client ready, docker driver detected +``` + +## 5. CI policy + token + registry variables + +```bash +# policy for the deploy token +nomad acl policy apply -description "Forgejo CI deploy" ci \ + deploy/nomad/policies/ci.hcl + +# long-lived token for the runner (no TTL) +nomad acl token create -name forgejo-ci -policy ci +# → AccessorID + SecretID. Forgejo repo secret: NOMAD_TOKEN = SecretID +``` + +Registry credentials and app secrets — the job file templates ALL of these +from `nomad/jobs/contract-check` (missing keys render empty; define every +key once). Connection strings point at the docker0 host gateway +`172.17.0.1` and the host-published infra ports (15432/17379/5672/9000) — +substitute real credentials where the compose `.env` deviates from +defaults: + +```bash +nomad var put -in=json nomad/jobs/contract-check - <<'EOF' +{ + "registry_host": "", + "registry_owner": "", + "registry_user": "", + "registry_token": "", + + "database_url": "postgresql+asyncpg://contract_check:@172.17.0.1:15432/contract_check", + "redis_url": "redis://172.17.0.1:17379/0", + "rabbitmq_url": "amqp://contract_check:@172.17.0.1:5672/", + "s3_endpoint_url": "http://172.17.0.1:9000", + "s3_access_key": "", + "s3_secret_key": "", + "s3_bucket": "contract-check-docs", + + "jwt_secret": "", + "telegram_bot_token": "", + "ollama_api_key": "", + "yandexgpt_api_key": "", + "smtp_host": "", + "smtp_username": "", + "smtp_password": "", + "metrics_bearer_token": "" +} +EOF + +nomad var get nomad/jobs/contract-check # sanity check +``` + +## 6. Firewall + +```bash +# 4646: only the Forgejo runner may talk to the API +sudo ufw allow from to any port 4646 proto tcp \ + comment 'nomad http (forgejo runner)' +# single-node: no external rpc/gossip peers +sudo ufw deny 4647/tcp comment 'nomad rpc (local only)' +sudo ufw deny 4648/tcp comment 'nomad gossip (local only)' +sudo ufw status verbose +``` + +Not using ufw? Apply the equivalent (443/80 stay as-is; only 4646 needs a +source restriction) in iptables/nft/cloud-secgroup. + +## 7. Web UI + +The UI has no login of its own — it rides on TLS+ACL, and neither should be +public. Reach it through an SSH tunnel: + +```bash +ssh -L 4646:127.0.0.1:4646 +# browser: https://localhost:4646/ui (self-signed warning is expected) +``` + +## 8. Everyday operations + +First submission (the job file requires IMAGE_TAG — it is rendered by the +CLI at submit time; CI exports it automatically): + +```bash +IMAGE_TAG= nomad job validate deploy/nomad/contract-check.nomad.hcl +IMAGE_TAG= nomad job plan deploy/nomad/contract-check.nomad.hcl +IMAGE_TAG= nomad job run deploy/nomad/contract-check.nomad.hcl +``` + +Replace ``/`` in the job file image paths first. + +```bash +nomad job status contract-check # groups, allocs, deployments +nomad alloc logs -f # or: nomad logs -f contract-check +nomad job scale contract-check worker-extract 3 +nomad deployment list / promote / rollback +nomad job revert contract-check +``` + +Health gates: the api group checks `/healthz`; every group sets +`update { auto_revert = true }` — a deployment that turns unhealthy rolls +back on its own. + +## 9. Failure behavior (single server) + +- Agent process dies → containers keep running untouched; deploys/scaling + impossible until `systemctl restart nomad`. Data in `/var/lib/nomad` + survives restarts. +- `systemctl status nomad`, `journalctl -u nomad -f` for diagnostics. + +## 10. Image retention + +Every merge pushes 6 SHA-tagged images to the Forgejo registry. Periodically +prune old tags (Forgejo UI → Packages, or the API), or keep `latest` + the +last few SHAs. Nomad also GCs unused images on the VPS automatically. diff --git a/deploy/nomad/contract-check.nomad.hcl b/deploy/nomad/contract-check.nomad.hcl new file mode 100644 index 0000000..953b6fd --- /dev/null +++ b/deploy/nomad/contract-check.nomad.hcl @@ -0,0 +1,527 @@ +# «Контракт-чек» app services on Nomad (single VPS, docker driver). +# +# Groups: api (with prestart migrations) + 5 workers. Stateful infra +# (postgres/redis/rabbitmq/minio) stays on compose; tasks reach it via the +# docker0 host gateway 172.17.0.1 and the host-published ports +# (15432/17379/5672/9000). The nginx edge cascade is unchanged. +# +# Secrets flow (nothing sensitive lives in this file): +# - registry creds + all app secrets → Nomad Variables at +# nomad/jobs/contract-check (see deploy/nomad/README.md §5), rendered +# agent-side into env / auth files by `template` blocks; +# - image tag is injected CLI-side at submit time: +# IMAGE_TAG= nomad job run deploy/nomad/contract-check.nomad.hcl +# +# First deploy: verify with `nomad job validate` + `nomad job plan`, then +# cutover workers one by one (README / docs/DEPLOY.md §15). + +job "contract-check" { + datacenters = ["vps"] + type = "service" + + # ── Rolling defaults for every group ──────────────────────────────────────── + update { + max_parallel = 1 + min_healthy_time = "15s" + healthy_deadline = "5m" + progress_deadline = "10m" + auto_revert = true + } + + # ══ API ════════════════════════════════════════════════════════════════════ + group "api" { + count = 1 + + # No canary here: the static host port (18000) cannot be bound twice on a + # single node. Rolling = brief seconds-level gap per deploy; switch to + # dynamic ports + Traefik if zero-downtime becomes a requirement. + update { + max_parallel = 1 + } + + restart { + attempts = 3 + interval = "10m" + delay = "15s" + mode = "delay" + } + + network { + port "http" { + static = 18000 + to = 8000 + } + } + + service { + name = "contract-check-api" + port = "http" + provider = "nomad" + + check { + name = "healthz" + type = "http" + path = "/healthz" + interval = "10s" + timeout = "3s" + } + + check_restart { + limit = 3 + grace = "30s" + } + } + + # One-shot migrations before the API starts (replaces `make migrate`). + task "migrate" { + lifecycle { + hook = "prestart" + } + + driver = "docker" + + config { + image = "//contract-check-api:{{ mustEnv \"IMAGE_TAG\" }}" + args = ["alembic", "upgrade", "head"] + + auth { + usernameFile = "secrets/registry-user" + passwordFile = "secrets/registry-pass" + } + } + + template { + destination = "secrets/registry-user" + data = "{{ with nomadVar \"nomad/jobs/contract-check\" }}{{ .registry_user }}{{ end }}" + } + template { + destination = "secrets/registry-pass" + data = "{{ with nomadVar \"nomad/jobs/contract-check\" }}{{ .registry_token }}{{ end }}" + } + + template { + destination = "secrets/env" + env = true + change_mode = "restart" + data = <<-EOT + {{ with nomadVar "nomad/jobs/contract-check" }} + DATABASE_URL="{{ .database_url }}" + {{ end }} + EOT + } + + resources { + cpu = 150 + memory = 256 + } + } + + task "api" { + driver = "docker" + + config { + image = "//contract-check-api:{{ mustEnv \"IMAGE_TAG\" }}" + ports = ["http"] + + auth { + usernameFile = "secrets/registry-user" + passwordFile = "secrets/registry-pass" + } + } + + env { + ENV = "prod" + LOG_LEVEL = "INFO" + LOG_FORMAT = "json" + APP_VERSION = "{{ env \"IMAGE_TAG\" }}" + } + + template { + destination = "secrets/registry-user" + data = "{{ with nomadVar \"nomad/jobs/contract-check\" }}{{ .registry_user }}{{ end }}" + } + template { + destination = "secrets/registry-pass" + data = "{{ with nomadVar \"nomad/jobs/contract-check\" }}{{ .registry_token }}{{ end }}" + } + + # All connection strings + app secrets, agent-rendered from Nomad + # Variables. Missing keys render empty — define them all once (README §5). + template { + destination = "secrets/env" + env = true + change_mode = "restart" + data = <<-EOT + {{ with nomadVar "nomad/jobs/contract-check" }} + DATABASE_URL="{{ .database_url }}" + REDIS_URL="{{ .redis_url }}" + RABBITMQ_URL="{{ .rabbitmq_url }}" + S3_ENDPOINT_URL="{{ .s3_endpoint_url }}" + S3_ACCESS_KEY="{{ .s3_access_key }}" + S3_SECRET_KEY="{{ .s3_secret_key }}" + S3_BUCKET="{{ .s3_bucket }}" + JWT_SECRET="{{ .jwt_secret }}" + TELEGRAM_BOT_TOKEN="{{ .telegram_bot_token }}" + OLLAMA_API_KEY="{{ .ollama_api_key }}" + YANDEXGPT_API_KEY="{{ .yandexgpt_api_key }}" + SMTP_HOST="{{ .smtp_host }}" + SMTP_USERNAME="{{ .smtp_username }}" + SMTP_PASSWORD="{{ .smtp_password }}" + METRICS_BEARER_TOKEN="{{ .metrics_bearer_token }}" + {{ end }} + EOT + } + + kill_timeout = "30s" + + resources { + cpu = 300 + memory = 512 + } + } + } + + # ══ WORKERS ═══════════════════════════════════════════════════════════════ + # Workers bind no ports, so each group gets canary + auto_promote: + # new version starts alongside the old one, must pass min_healthy_time, + # then old allocations stop. auto_revert rolls back on failure. + # Scale with: nomad job scale contract-check worker- + + group "worker-extract" { + count = 1 + + update { + canary = 1 + auto_promote = true + } + + restart { + attempts = 3 + interval = "10m" + delay = "15s" + mode = "delay" + } + + task "worker-extract" { + driver = "docker" + + config { + image = "//contract-check-worker-extract:{{ mustEnv \"IMAGE_TAG\" }}" + + auth { + usernameFile = "secrets/registry-user" + passwordFile = "secrets/registry-pass" + } + } + + env { + ENV = "prod" + LOG_LEVEL = "INFO" + LOG_FORMAT = "json" + APP_VERSION = "{{ env \"IMAGE_TAG\" }}" + } + + template { + destination = "secrets/registry-user" + data = "{{ with nomadVar \"nomad/jobs/contract-check\" }}{{ .registry_user }}{{ end }}" + } + template { + destination = "secrets/registry-pass" + data = "{{ with nomadVar \"nomad/jobs/contract-check\" }}{{ .registry_token }}{{ end }}" + } + template { + destination = "secrets/env" + env = true + change_mode = "restart" + data = <<-EOT + {{ with nomadVar "nomad/jobs/contract-check" }} + DATABASE_URL="{{ .database_url }}" + RABBITMQ_URL="{{ .rabbitmq_url }}" + S3_ENDPOINT_URL="{{ .s3_endpoint_url }}" + S3_ACCESS_KEY="{{ .s3_access_key }}" + S3_SECRET_KEY="{{ .s3_secret_key }}" + S3_BUCKET="{{ .s3_bucket }}" + OLLAMA_API_KEY="{{ .ollama_api_key }}" + YANDEXGPT_API_KEY="{{ .yandexgpt_api_key }}" + {{ end }} + EOT + } + + kill_timeout = "60s" + + resources { + # CPU-bound OCR (tesseract); the heaviest task of the pipeline. + cpu = 500 + memory = 640 + } + } + } + + group "worker-analyze" { + count = 1 + + update { + canary = 1 + auto_promote = true + } + + restart { + attempts = 3 + interval = "10m" + delay = "15s" + mode = "delay" + } + + task "worker-analyze" { + driver = "docker" + + config { + image = "//contract-check-worker-analyze:{{ mustEnv \"IMAGE_TAG\" }}" + + auth { + usernameFile = "secrets/registry-user" + passwordFile = "secrets/registry-pass" + } + } + + env { + ENV = "prod" + LOG_LEVEL = "INFO" + LOG_FORMAT = "json" + APP_VERSION = "{{ env \"IMAGE_TAG\" }}" + } + + template { + destination = "secrets/registry-user" + data = "{{ with nomadVar \"nomad/jobs/contract-check\" }}{{ .registry_user }}{{ end }}" + } + template { + destination = "secrets/registry-pass" + data = "{{ with nomadVar \"nomad/jobs/contract-check\" }}{{ .registry_token }}{{ end }}" + } + template { + destination = "secrets/env" + env = true + change_mode = "restart" + data = <<-EOT + {{ with nomadVar "nomad/jobs/contract-check" }} + DATABASE_URL="{{ .database_url }}" + RABBITMQ_URL="{{ .rabbitmq_url }}" + S3_ENDPOINT_URL="{{ .s3_endpoint_url }}" + S3_ACCESS_KEY="{{ .s3_access_key }}" + S3_SECRET_KEY="{{ .s3_secret_key }}" + S3_BUCKET="{{ .s3_bucket }}" + OLLAMA_API_KEY="{{ .ollama_api_key }}" + YANDEXGPT_API_KEY="{{ .yandexgpt_api_key }}" + {{ end }} + EOT + } + + kill_timeout = "120s" + + resources { + # LLM calls (I/O bound, long in-flight requests on shutdown). + cpu = 250 + memory = 384 + } + } + } + + group "worker-prescreen" { + count = 1 + + update { + canary = 1 + auto_promote = true + } + + restart { + attempts = 3 + interval = "10m" + delay = "15s" + mode = "delay" + } + + task "worker-prescreen" { + driver = "docker" + + config { + image = "//contract-check-worker-prescreen:{{ mustEnv \"IMAGE_TAG\" }}" + + auth { + usernameFile = "secrets/registry-user" + passwordFile = "secrets/registry-pass" + } + } + + env { + ENV = "prod" + LOG_LEVEL = "INFO" + LOG_FORMAT = "json" + APP_VERSION = "{{ env \"IMAGE_TAG\" }}" + } + + template { + destination = "secrets/registry-user" + data = "{{ with nomadVar \"nomad/jobs/contract-check\" }}{{ .registry_user }}{{ end }}" + } + template { + destination = "secrets/registry-pass" + data = "{{ with nomadVar \"nomad/jobs/contract-check\" }}{{ .registry_token }}{{ end }}" + } + template { + destination = "secrets/env" + env = true + change_mode = "restart" + data = <<-EOT + {{ with nomadVar "nomad/jobs/contract-check" }} + DATABASE_URL="{{ .database_url }}" + RABBITMQ_URL="{{ .rabbitmq_url }}" + S3_ENDPOINT_URL="{{ .s3_endpoint_url }}" + S3_ACCESS_KEY="{{ .s3_access_key }}" + S3_SECRET_KEY="{{ .s3_secret_key }}" + S3_BUCKET="{{ .s3_bucket }}" + OLLAMA_API_KEY="{{ .ollama_api_key }}" + YANDEXGPT_API_KEY="{{ .yandexgpt_api_key }}" + {{ end }} + EOT + } + + kill_timeout = "60s" + + resources { + cpu = 150 + memory = 256 + } + } + } + + group "worker-billing" { + count = 1 + + update { + canary = 1 + auto_promote = true + } + + restart { + attempts = 3 + interval = "10m" + delay = "15s" + mode = "delay" + } + + task "worker-billing" { + driver = "docker" + + config { + image = "//contract-check-worker-billing:{{ mustEnv \"IMAGE_TAG\" }}" + + auth { + usernameFile = "secrets/registry-user" + passwordFile = "secrets/registry-pass" + } + } + + env { + ENV = "prod" + LOG_LEVEL = "INFO" + LOG_FORMAT = "json" + APP_VERSION = "{{ env \"IMAGE_TAG\" }}" + } + + template { + destination = "secrets/registry-user" + data = "{{ with nomadVar \"nomad/jobs/contract-check\" }}{{ .registry_user }}{{ end }}" + } + template { + destination = "secrets/registry-pass" + data = "{{ with nomadVar \"nomad/jobs/contract-check\" }}{{ .registry_token }}{{ end }}" + } + template { + destination = "secrets/env" + env = true + change_mode = "restart" + data = <<-EOT + {{ with nomadVar "nomad/jobs/contract-check" }} + DATABASE_URL="{{ .database_url }}" + {{ end }} + EOT + } + + kill_timeout = "60s" + + resources { + # Background DB-only worker (dunning, renewals); lightest of the set. + cpu = 150 + memory = 256 + } + } + } + + group "worker-notify" { + count = 1 + + update { + canary = 1 + auto_promote = true + } + + restart { + attempts = 3 + interval = "10m" + delay = "15s" + mode = "delay" + } + + task "worker-notify" { + driver = "docker" + + config { + image = "//contract-check-worker-notify:{{ mustEnv \"IMAGE_TAG\" }}" + + auth { + usernameFile = "secrets/registry-user" + passwordFile = "secrets/registry-pass" + } + } + + env { + ENV = "prod" + LOG_LEVEL = "INFO" + LOG_FORMAT = "json" + APP_VERSION = "{{ env \"IMAGE_TAG\" }}" + } + + template { + destination = "secrets/registry-user" + data = "{{ with nomadVar \"nomad/jobs/contract-check\" }}{{ .registry_user }}{{ end }}" + } + template { + destination = "secrets/registry-pass" + data = "{{ with nomadVar \"nomad/jobs/contract-check\" }}{{ .registry_token }}{{ end }}" + } + template { + destination = "secrets/env" + env = true + change_mode = "restart" + data = <<-EOT + {{ with nomadVar "nomad/jobs/contract-check" }} + DATABASE_URL="{{ .database_url }}" + RABBITMQ_URL="{{ .rabbitmq_url }}" + SMTP_HOST="{{ .smtp_host }}" + SMTP_USERNAME="{{ .smtp_username }}" + SMTP_PASSWORD="{{ .smtp_password }}" + {{ end }} + EOT + } + + kill_timeout = "60s" + + resources { + cpu = 150 + memory = 256 + } + } + } +} diff --git a/deploy/nomad/nomad.hcl b/deploy/nomad/nomad.hcl new file mode 100644 index 0000000..8e462ce --- /dev/null +++ b/deploy/nomad/nomad.hcl @@ -0,0 +1,74 @@ +# Single-node Nomad agent for «Контракт-чек» (services VPS). +# +# One process = scheduler (server) + docker workload host (client). +# Install & bootstrap runbook: deploy/nomad/README.md. +# +# Target location on the VPS: /etc/nomad.d/nomad.hcl + +datacenter = "vps" +region = "global" +data_dir = "/var/lib/nomad" +log_level = "INFO" + +# The agent binds 4646 (http) / 4647 (rpc) / 4648 (gossip) on 0.0.0.0 by +# default. The host firewall MUST restrict them (see README §6): +# - 4646 tcp: allow ONLY the Forgejo server IP (runner deploys) +# - 4647/4648: single-node cluster, no external peers — keep blocked. + +# Point-to-point TLS: the Forgejo runner reaches the API over the public +# internet. Self-signed CA (README §3); the runner only needs nomad-ca.crt. +tls { + http = true + rpc = true + + ca_file = "/etc/nomad.d/tls/nomad-ca.crt" + cert_file = "/etc/nomad.d/tls/server.crt" + key_file = "/etc/nomad.d/tls/server.key" + + # Requires the server cert SAN to carry server.global.vps.nomad + # (region.region/datacenter convention) — the README cert command adds it. + verify_server_hostname = true +} + +# Anonymous requests are rejected; every client needs a token. +acl { + enabled = true +} + +server { + # Single-server cluster: acceptable for one VPS. If the agent is down, + # running allocations keep running (docker does not kill them) — only + # scheduling/deployments pause until it is back. + enabled = true + bootstrap_expect = 1 +} + +client { + enabled = true + # Docker driver is auto-detected. Tasks pull images from the Forgejo + # registry via per-task `auth {}` templated from Nomad Variables + # (nomad/jobs/contract-check) — no host-level `docker login` needed. + + # The compose stack (postgres, rabbitmq, minio, redis, nginx, observability) + # runs OUTSIDE Nomad but on the same 4 GB VPS. Reserve its share so the + # scheduler only bin-packs what is actually left (~2.5 GB / ~1.6 GHz). + reserved { + cpu = 400 + memory = 1536 + } +} + +# Docker driver is intentionally left at defaults: +# - host bind mounts are DISABLED (docker.volumes.enabled=false by default) +# — tasks are 12-factor (env-only config), no volumes required; +# - Nomad garbage-collects unused images periodically — good for the +# 50 GB NVMe. +# Enable only if a task ever needs a bind mount: +# +# plugin "docker" { +# config { +# volumes { +# enabled = true +# } +# } +# } diff --git a/deploy/nomad/nomad.service b/deploy/nomad/nomad.service new file mode 100644 index 0000000..74dec45 --- /dev/null +++ b/deploy/nomad/nomad.service @@ -0,0 +1,22 @@ +# systemd unit for a MANUAL Nomad install (GitHub releases route, README §1b). +# The HashiCorp .deb ships an equivalent unit at /lib/systemd/system/nomad.service — +# do NOT install this file if nomad was installed via apt. +# +# Target location: /etc/systemd/system/nomad.service + +[Unit] +Description=Nomad +Documentation=https://developer.hashicorp.com/nomad/docs +Wants=network-online.target +After=network-online.target + +[Service] +# agent reads /etc/nomad.d/*.hcl (deploy/nomad/nomad.hcl copied there) +ExecStart=/usr/local/bin/nomad agent -config /etc/nomad.d/ +ExecReload=/bin/kill -HUP $MAINPID +Restart=on-failure +LimitNOFILE=65536 +# Runs as root on purpose: the docker driver needs the docker socket. + +[Install] +WantedBy=multi-user.target diff --git a/deploy/nomad/policies/ci.hcl b/deploy/nomad/policies/ci.hcl new file mode 100644 index 0000000..322d49e --- /dev/null +++ b/deploy/nomad/policies/ci.hcl @@ -0,0 +1,21 @@ +# ACL policy for the Forgejo Actions deploy token. +# +# Scopes: read the default namespace (job list / deployment status polling), +# full control over the contract-check job only. The token cannot touch +# agents, other jobs, or Nomad Variables (set via the management token). +# +# Apply with the management token (README §4): +# nomad acl policy apply -description "Forgejo CI deploy" ci \ +# deploy/nomad/policies/ci.hcl +# +# If `nomad job run` from CI ever fails with permission errors during +# deployment evaluation, fall back to namespace-wide write: +# namespace "default" { policy = "write" } + +namespace "default" { + policy = "read" + + job "contract-check" { + policy = "write" + } +}