From 6f22b0368a00777e844637226674e3d89abc2d4c Mon Sep 17 00:00:00 2001 From: febux Date: Sun, 13 Sep 2026 23:54:06 +0300 Subject: [PATCH] Fix CI workflows. Extend make commands. Fix Nomad deploy doc. --- .forgejo/workflows/ci.yml | 98 ++++++++++++++++++++++++++++++--------- Makefile | 34 +++++++++++++- deploy/nomad/README.md | 57 ++++++++++++++--------- docs/DEPLOY.md | 77 ++++++++++++++++++++++++++++++ 4 files changed, 222 insertions(+), 44 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 94a22c3..8a00a9d 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -8,36 +8,92 @@ name: ci on: push: - branches: ["**"] + branches: [master] pull_request: + branches: [master] + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - test: - runs-on: docker - container: - image: python:3.14-trixie + lint: + name: Lint & typecheck + runs-on: ubuntu-latest steps: - - name: Checkout (sha-pinned, no node actions needed) - run: | - git init -q . - git remote add origin ${{ gitea.server_url }}/${{ gitea.repository }}.git - git fetch --depth 1 origin ${{ gitea.sha }} - git checkout -q FETCH_HEAD + - uses: actions/checkout@v4 - - name: Install uv - run: pip install --quiet uv + - name: Setup uv + uses: astral-sh/setup-uv@v8.3.2 + with: + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Install Python + run: uv python install - name: Sync dev dependencies run: uv sync --group dev --frozen - - name: Lint (ruff) - run: | - uv run ruff check src tests - uv run ruff format --check src tests - uv run isort --check-only src tests + - name: Ruff check + run: uv run ruff check src tests - - name: Typecheck (ty) + - name: Ruff format check + run: uv run ruff format --check src tests + + - name: Ty type check run: uv run ty check src - - name: Unit tests - run: uv run pytest -m "not integration" tests/unit + test-unit: + name: Unit tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup uv + uses: astral-sh/setup-uv@v8.3.2 + with: + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Install Python + run: uv python install + + - name: Sync dev dependencies + run: uv sync --group dev --frozen + + - name: Run unit tests with coverage + run: uv run pytest --cov=src/contract_check --cov-branch --cov-report=term-missing --cov-fail-under=50 -m "not integration" tests/unit + + # Integration tests. + test-integration: + name: Integration tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup uv + uses: astral-sh/setup-uv@v8.3.2 + with: + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Install Python + run: uv python install + + - name: Sync dev dependencies + run: uv sync --group dev --frozen + + # No --wait: minio-init-test is a one-shot container that exits (0) after + # creating the bucket, and --wait treats any exited container as failure. + # Health polling is done by the `infra` fixture in + # tests/integration/conftest.py; this step just pre-pulls the images. + - name: Start test infrastructure (postgres/redis/rabbitmq/minio) + run: docker compose -p contract-check-test -f docker-compose.test.yml up -d + + - name: Run integration tests + run: uv run pytest -m integration + + - name: Teardown infrastructure + if: always() + run: docker compose -p contract-check-test -f docker-compose.test.yml down -v diff --git a/Makefile b/Makefile index 04ab15d..1889412 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,9 @@ api api-logs bot bot-up bot-down bot-logs bot-ps worker-extract worker-analyze worker-notify \ bot-remote-up bot-remote-down bot-remote-logs bot-remote-ps bot-webhook-path \ seed-token jwt-secret jwt-token jwt-verify health shell-api shell-bot \ - shell-db admin-promote admin-list clean dev dev-obs stop stop-obs + shell-db admin-promote admin-list \ + nomad-validate nomad-plan nomad-deploy nomad-status nomad-logs nomad-scale nomad-revert \ + clean dev dev-obs stop stop-obs # ───────────────────────────────────────────────────────────────────────────── # Help @@ -299,6 +301,36 @@ shell-api: ## Open shell inside API container shell-bot: ## Open shell inside bot container docker compose --profile bot exec bot /bin/sh +# ───────────────────────────────────────────────────────────────────────────── +# Nomad (production app services; full runbook: deploy/nomad/README.md) +# CLI env (NOMAD_ADDR/CACERT/TOKEN) is sourced from /root/.nomadrc on the VPS. +# ───────────────────────────────────────────────────────────────────────────── +NOMAD_JOB := deploy/nomad/contract-check.nomad.hcl +NOMAD_RUN := bash -c '. /root/.nomadrc 2>/dev/null; IMAGE_TAG=$${IMAGE_TAG:-manual} nomad "$$@"' -- + +nomad-validate: ## Validate the Nomad job file + @$(NOMAD_RUN) job validate $(NOMAD_JOB) + +nomad-plan: ## Dry-run diff of the next deployment + @$(NOMAD_RUN) job plan $(NOMAD_JOB) + +nomad-deploy: ## Deploy to Nomad (usage: IMAGE_TAG= make nomad-deploy) + @$(NOMAD_RUN) job run $(NOMAD_JOB) + +nomad-status: ## Job status: groups, allocations, deployments + @$(NOMAD_RUN) job status contract-check + +nomad-logs: ## Tail a group's logs (usage: make nomad-logs G=api) + @alloc=$$($(NOMAD_RUN) job allocs -json contract-check | python3 -c \ + "import json,sys; a=[x for x in json.load(sys.stdin) if x['TaskGroup']=='$(G)' and x['ClientStatus']=='running']; print(a[0]['ID'] if a else '')"); \ + [ -n "$$alloc" ] && $(NOMAD_RUN) alloc logs -f $$alloc || echo "no running alloc in group $(G)" + +nomad-scale: ## Scale a group (usage: make nomad-scale G=worker-extract N=3) + @$(NOMAD_RUN) job scale contract-check $(G) $(N) + +nomad-revert: ## Revert job to a prior version (usage: make nomad-revert V=2) + @$(NOMAD_RUN) job revert contract-check $(V) + # ───────────────────────────────────────────────────────────────────────────── # Cleanup # ───────────────────────────────────────────────────────────────────────────── diff --git a/deploy/nomad/README.md b/deploy/nomad/README.md index 0f38d18..90e7d9d 100644 --- a/deploy/nomad/README.md +++ b/deploy/nomad/README.md @@ -148,13 +148,22 @@ nomad acl bootstrap export NOMAD_TOKEN= ``` -Lost the management token? The 400 error from bootstrap tells you the reset -index; use it to mint a fresh management token (invalidates the old one): +Lost the management token? The API reset path (`ResetIndex` in the +bootstrap call) does not work on this version. Since the failure window +here is initial setup — the cluster is EMPTY (no jobs, no variables) — do a +state reset (wipes raft: jobs, tokens, variables): ```bash -nomad acl bootstrap -reset-index +sudo systemctl stop nomad +# ensure acl { enabled = true } in /etc/nomad.d/nomad.hcl +sudo rm -rf /var/lib/nomad/* +sudo systemctl start nomad +sleep 5 && nomad acl bootstrap # fresh management token — save it ``` +On a POPULATED cluster never do this — keep the bootstrap SecretID in a +password manager from day one. + Handy: keep the CLI env in a root-only file and source it on demand (never into .bashrc — tokens shouldn't leak to every shell): @@ -195,29 +204,33 @@ substitute real credentials where the compose `.env` deviates from defaults: ```bash +# NOTE: -in=json requires the {"Items": {...}} envelope — a flat key map +# is rejected with "variable missing required Items object". nomad var put -in=json nomad/jobs/contract-check - <<'EOF' { - "registry_host": "p2gnl.mu-dungeon.xyz", - "registry_owner": "admin-git", - "registry_user": "admin-git", - "registry_token": "", + "Items": { + "registry_host": "p2gnl.mu-dungeon.xyz", + "registry_owner": "admin-git", + "registry_user": "admin-git", + "registry_token": "", - "database_url": "postgresql+asyncpg://contract_check:@172.17.0.1:15432/contract_check", - "redis_url": "redis://172.17.0.1:17379/0", - "rabbitmq_url": "amqp://contract_check:@172.17.0.1:5672/", - "s3_endpoint_url": "http://172.17.0.1:9000", - "s3_access_key": "", - "s3_secret_key": "", - "s3_bucket": "contract-check-docs", + "database_url": "postgresql+asyncpg://contract_check:@172.17.0.1:15432/contract_check", + "redis_url": "redis://172.17.0.1:17379/0", + "rabbitmq_url": "amqp://contract_check:@172.17.0.1:5672/", + "s3_endpoint_url": "http://172.17.0.1:9000", + "s3_access_key": "", + "s3_secret_key": "", + "s3_bucket": "contract-check-docs", - "jwt_secret": "", - "telegram_bot_token": "", - "ollama_api_key": "", - "yandexgpt_api_key": "", - "smtp_host": "", - "smtp_username": "", - "smtp_password": "", - "metrics_bearer_token": "" + "jwt_secret": "", + "telegram_bot_token": "", + "ollama_api_key": "", + "yandexgpt_api_key": "", + "smtp_host": "", + "smtp_username": "", + "smtp_password": "", + "metrics_bearer_token": "" + } } EOF diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index 106c67f..49e9896 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -22,6 +22,7 @@ 12. [Troubleshooting](#12-troubleshooting) 13. [Edge proxy (Nginx + certbot)](#13-edge-proxy-nginx--certbot) 14. [Telegram-бот на отдельном сервере](#14-telegram-бот-на-отдельном-сервере) +15. [Nomad: app-сервисы в продакшене](#15-nomad-app-сервисы-в-продакшене) --- @@ -888,6 +889,79 @@ make bot-remote-ps --- +## 15. Nomad: app-сервисы в продакшене + +Полный runbook (установка агента, TLS, ACL, секреты, firewall): +[deploy/nomad/README.md](../deploy/nomad/README.md). + +**Разделение ответственности на VPS:** + +| Слой | Чем управляется | +|---|---| +| `api` + 5 воркеров (`extract/analyze/prescreen/billing/notify`) | **Nomad** (job `contract-check`, одиночный агент server+client) | +| postgres, redis, rabbitmq, minio | docker compose (профиль infra) | +| edge-каскад: system nginx → compose nginx | без изменений (§13, `deploy/vps/`) | +| наблюдаемость (Vector/Promtail + …) | docker compose, скрейпит docker-сокет — контейнеры Nomad видит автоматически | + +Задачи Nomad ходят в инфраструктуру compose через хостовый шлюз docker0 +`172.17.0.1` по опубликованным портам (15432/17379/5672/9000). + +### 15.1 CI/CD + +Push в `main` (Forgejo) → `.forgejo/workflows/deploy.yml`: + +1. сборка 6 образов из `srv/*/Dockerfile`; +2. push в registry `p2gnl.mu-dungeon.xyz/admin-git/contract-check-:`; +3. `nomad job run` (образ-тег подставляется CLI-шаблоном `IMAGE_TAG`). + +Деплой неуспешен в CI, если health-checks не прошли — на стороне Nomad +сработает `auto_revert` (откат на предыдущую версию job). Воркеры +обновляются canary-стратегией (новая версия рядом со старой); api — rolling +(`max_parallel=1`, секундный разрыв на деплой). + +### 15.2 Повседневные операции + +```bash +make nomad-status # группы, аллокации, деплои +make nomad-logs G=api # хвост логов группы +make nomad-scale G=worker-extract N=3 # масштабирование воркера +make nomad-revert V= # ручной откат +``` + +Web UI (логи/exec/scale/deployments): `ssh -L 4646:127.0.0.1:4646 ` → +`https://localhost:4646/ui` (сертификат self-signed — предупреждение норма). + +### 15.3 Секреты + +| Где | Что | +|---|---| +| Nomad Variables `nomad/jobs/contract-check` | connection strings, ключи LLM/SMTP/JWT, токен registry (pull) | +| Forgejo repo secrets | `REGISTRY_TOKEN` (push, write:package), `NOMAD_TOKEN` (CI ACL), `NOMAD_CACERT` (CA-сертификат) | +| Forgejo repo variables | `NOMAD_ADDR_HOST` (публичный IP VPS с Nomad) | + +Обновление секрета: `nomad var put ...` → контейнеры перезапустятся +(`change_mode = restart`). + +### 15.4 Отказоустойчивость (одиночный сервер) + +Агент упал → контейнеры продолжают работать (docker их не убивает); +недоступны только деплои/скейлинг до `systemctl restart nomad`. Состояние — +в `/var/lib/nomad` (raft). + +### 15.5 Порядок перевода со compose на Nomad + +1. Воркеры по одному: остановить compose-сервис → убедиться, что группа + Nomad поднялась и потребители вернулись в очереди RabbitMQ + (`rabbitmqctl list_queues name consumers`); +2. `api`: поднять группу Nomad (порт 18000) → проверить `/healthz` → + переключить `deploy/nginx/templates/contract-check-http.conf.template` + (`set $api http://api:8000` → `http://:18000`) → + `docker compose --profile services down`; +3. Профиль `infra` (postgres/redis/rabbitmq/minio), edge и наблюдаемость + не трогать. + +--- + ## Чек-лист перед production - [ ] `.env` заполнен, `.env.example` не содержит реальных секретов @@ -902,6 +976,9 @@ make bot-remote-ps - [ ] B2B-ключ создаётся через `/api/v1/b2b/keys` с user JWT - [ ] Backup cron настроен - [ ] Firewall: открыты только 443 (nginx), 22 (ssh), 15672 (RabbitMQ mgmt, restrict IP) +- [ ] Nomad: агент healthy (`nomad server members` / `nomad node status`), 4646 открыт только IP Forgejo, 4647/4648 закрыты +- [ ] Nomad Variables `nomad/jobs/contract-check` заполнены (включая registry-токен) +- [ ] Forgejo secrets заданы: `REGISTRY_TOKEN`, `NOMAD_TOKEN`, `NOMAD_CACERT` + переменная `NOMAD_ADDR_HOST` - [ ] Если бот на отдельном сервере: `API_URL` указывает на центральный API, `BOT_SERVICE_TOKEN` засеян, ботовый сервер имеет outbound HTTPS - [ ] Если бот в webhook-режиме: `BOT_UPDATE_MODE=webhook`, `BOT_WEBHOOK_PUBLIC_BASE_URL`/`BOT_WEBHOOK_SECRET_TOKEN` заданы, edge маршрутизирует `BOT_WEBHOOK_PATH` на бота, webhook-порт не опубликован в интернет