admin-git/DealDocumentScreening
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Init commit

Browse source
This commit is contained in:
febux 2026-08-12 21:29:36 +03:00
commit 5073d89033
110 changed files with 15867 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
.dockerignore Normal file
View file

@ -0,0 +1,23 @@
.git
.github
.venv
venv
env
__pycache__
**/__pycache__
*.py[cod]
.pytest_cache
.mypy_cache
.ruff_cache
htmlcov
.coverage
.coverage.*
.env
data
reports
tmp
*.log
.gitignore
.dockerignore
srv
docker-compose.yml

74
.env.example Normal file
View file

@ -0,0 +1,74 @@
# ============================================================
# «Контракт-чек» — production config (see docs/ARCHITECTURE.md §11)
# Copy to `.env` and fill in. All config is 12-factor (env-driven).
# ============================================================
# --- Runtime ---
ENV=dev # dev | staging | prod
LOG_LEVEL=INFO
LOG_FORMAT=json # json (prod/staging) | console (dev)
APP_VERSION=0.1.0 # added to every log line; set at build/deploy time
# --- Postgres (async) ---
DATABASE_URL=postgresql+asyncpg://contract_check:contract_check@postgres:5432/contract_check
# --- Redis (rate-limit / sessions, future; NOT the job queue) ---
REDIS_URL=redis://redis:6379/0
# --- RabbitMQ (job queue pipeline) ---
RABBITMQ_URL=amqp://contract_check:contract_check@rabbitmq:5672//
MQ_PREFETCH_EXTRACT=1 # CPU-bound extraction; tune to CPU count
MQ_PREFETCH_ANALYZE=3 # mirrors Ollama Pro concurrency
MQ_MAX_ATTEMPTS=5 # before a message lands on the DLQ
MQ_RETRY_BASE_MS=2000 # exponential backoff base (2s, 4s, 8s, ...)
# --- MinIO (object storage) ---
S3_ENDPOINT_URL=http://minio:9000
S3_ACCESS_KEY=contract_check
S3_SECRET_KEY=contract_check
S3_BUCKET=contract-check-docs
S3_REGION=us-east-1
S3_SERVER_SIDE_ENCRYPTION=false # true in prod
DOC_RETENTION_DAYS=7 # MinIO ILM expiry for raw docs (152-ФЗ lever)
TEXT_RETENTION_DAYS=30 # expiry for extracted .txt blobs
# --- Billing ---
REFUND_POLICY=all # all | infra_only (see docs/ARCHITECTURE.md §8)
# --- Observability (leave empty to disable) ---
SENTRY_DSN=
OTEL_EXPORTER_OTLP_ENDPOINT= # e.g. http://otel-collector:4317
OTEL_SERVICE_NAME=contract-check
# --- LLM provider (abstract port; first realization = Ollama Cloud) ---
# For Ollama Cloud use https://ollama.com (not api.ollama.com). Models must be
# available on the chosen host — cloud models differ from local Ollama models.
LLM_PROVIDER=ollama_cloud
OLLAMA_HOST=https://ollama.com
OLLAMA_API_KEY=replace-me
OLLAMA_MODEL=qwen2.5:14b
OLLAMA_FALLBACK_MODEL=qwen2.5:7b
OLLAMA_TEMPERATURE=0.2
OLLAMA_NUM_PREDICT=3072
OLLAMA_TIMEOUT=120
OLLAMA_MAX_CONCURRENCY=3
CHUNK_SIZE_CHARS=10000
# --- API (FastAPI) ---
API_HOST=0.0.0.0
API_PORT=8000
API_METRICS_PORT=9100
B2B_DEFAULT_RATE_LIMIT_RPS=3 # per API key; mirrors Ollama Pro concurrency
CORS_ORIGINS= # comma-separated, future web SPA
# --- Auth (JWT + Telegram identity verification) ---
# Telegram bot token is also used by the API to verify Login Widget / Mini App signatures.
TELEGRAM_BOT_TOKEN=
JWT_SECRET= # HS256 secret for signing user JWTs; generate with `openssl rand -hex 32`
JWT_ALGORITHM=HS256
JWT_ACCESS_TTL_MINUTES=1440 # 24 hours default
# --- Telegram bot (adapter, HTTP-only to api) ---
BOT_TOKEN= # same value as TELEGRAM_BOT_TOKEN (kept for the bot image)
BOT_SERVICE_TOKEN= # bearer looked up against service_tokens.name="bot-prod"
API_URL=http://api:8000 # base URL of the api service (container DNS in compose)

35
.gitignore vendored Normal file
View file

@ -0,0 +1,35 @@
# Python
__pycache__/
*.py[cod]
*$py.class
*.so
.Python
.venv/
venv/
env/
.env
*.egg-info/
.eggs/
build/
dist/
.pytest_cache/
.mypy_cache/
.ruff_cache/
htmlcov/
.coverage
.coverage.*
# Notebooks
.ipynb_checkpoints/
# OS / editor
.DS_Store
.idea/
.vscode/
*.swp
# Project
*.log
reports/
tmp/
data/

1
.python-version Normal file
View file

@ -0,0 +1 @@
3.13

162
Makefile Normal file
View file

@ -0,0 +1,162 @@
# «Контракт-чек» — everyday commands (uv + docker)
# Usage: make <target> (see `make help`)
.PHONY: help install lint typecheck test test-unit test-integration migrate \
infra-up infra-down infra-logs services-up services-down services-logs \
api api-logs bot bot-logs worker-extract worker-analyze seed-token \
jwt-secret jwt-token jwt-verify health shell-api shell-bot shell-db clean
# ─────────────────────────────────────────────────────────────────────────────
# Help
# ─────────────────────────────────────────────────────────────────────────────
help: ## Show available commands
@echo "Usage: make <target>"
@grep -E '^[a-zA-Z0-9_-]+:.*##' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*##"}; {printf " \033[36m%-18s\033[0m %s\n", $$1, $$2}'
# ─────────────────────────────────────────────────────────────────────────────
# Local dev (uv)
# ─────────────────────────────────────────────────────────────────────────────
install: ## Sync dev dependencies (uv)
uv sync --group dev
lint: ## Run ruff linter + import sorter
uv run ruff check src tests
uv run ruff format --check src tests
typecheck: ## Run mypy type checker
uv run mypy src
test: ## Run all tests (unit + integration)
uv run pytest
test-unit: ## Run unit tests only (fast)
uv run pytest -m "not integration"
test-integration: ## Run integration tests (needs docker infra)
uv run pytest -m integration
# ─────────────────────────────────────────────────────────────────────────────
# Docker: infrastructure (postgres + redis + rabbitmq + minio)
# ─────────────────────────────────────────────────────────────────────────────
infra-up: ## Start infra containers
docker compose up -d
infra-down: ## Stop infra containers
docker compose down
infra-logs: ## Tail infra logs
docker compose logs -f
# ─────────────────────────────────────────────────────────────────────────────
# Docker: all services (api + workers + bot)
# ─────────────────────────────────────────────────────────────────────────────
services-up: ## Start all services (needs infra running)
docker compose --profile services up -d --build --remove-orphans
services-down: ## Stop all services
docker compose --profile services down
services-logs: ## Tail all service logs
docker compose --profile services logs -f
services-ps: ## Show running containers
docker compose --profile services ps
# ─────────────────────────────────────────────────────────────────────────────
# Individual services
# ─────────────────────────────────────────────────────────────────────────────
api: ## Start/restart API service
docker compose --profile services up -d --build --remove-orphans api
api-logs: ## Tail API logs
docker compose logs -f api
bot: ## Start/restart Telegram bot
docker compose --profile services up -d --build --remove-orphans bot
bot-logs: ## Tail bot logs
docker compose logs -f bot
worker-extract: ## Start/restart extract worker
docker compose --profile services up -d --build --remove-orphans worker-extract
worker-analyze: ## Start/restart analyze worker
docker compose --profile services up -d --build --remove-orphans worker-analyze
worker-analyze-logs: ## Tail analyze worker logs
docker compose logs -f worker-analyze
# ─────────────────────────────────────────────────────────────────────────────
# Database
# ─────────────────────────────────────────────────────────────────────────────
migrate: ## Run Alembic migrations (inside api container)
docker compose --profile services build api
docker compose --profile services run --rm api alembic upgrade head
shell-db: ## Open psql inside postgres container
docker compose exec postgres psql -U contract_check -d contract_check
# ─────────────────────────────────────────────────────────────────────────────
# Auth / tokens
# ─────────────────────────────────────────────────────────────────────────────
seed-token: ## Generate bot service token (prints bearer token)
docker compose --profile services exec api python -m contract_check.api seed-token bot-prod bot
jwt-secret: ## Generate a fresh JWT_SECRET for .env
@openssl rand -hex 32
jwt-token: ## Exchange a telegram_id for a user JWT (usage: make jwt-token TG_ID=123456)
@if [ -z "$(TG_ID)" ]; then \
echo "Usage: make jwt-token TG_ID=123456"; \
exit 1; \
fi
@TOKEN=$$(grep -E '^BOT_SERVICE_TOKEN=' .env | cut -d= -f2); \
if [ -z "$$TOKEN" ]; then \
echo "BOT_SERVICE_TOKEN not found in .env"; \
exit 1; \
fi; \
curl -s -X POST -H "Authorization: Bearer $$TOKEN" \
-H "Content-Type: application/json" \
-d '{"telegram_id":$(TG_ID)}' \
http://localhost:8000/api/v1/auth/telegram/bot | jq .
jwt-verify: ## Introspect a user JWT (usage: make jwt-verify JWT=eyJ...)
@if [ -z "$(JWT)" ]; then \
echo "Usage: make jwt-verify JWT=eyJ..."; \
exit 1; \
fi
@curl -s -H "Authorization: Bearer $(JWT)" \
http://localhost:8000/api/v1/auth/me | jq .
# ─────────────────────────────────────────────────────────────────────────────
# Health / diagnostics
# ─────────────────────────────────────────────────────────────────────────────
health: ## Check API health endpoint
@echo "API health:"
@curl -s http://localhost:8000/healthz | jq . 2>/dev/null || curl -s http://localhost:8000/healthz
@echo ""
@echo "RabbitMQ: http://localhost:15672 (guest/guest → contract_check/contract_check)"
@echo "MinIO: http://localhost:9001 (contract_check/contract_check)"
@echo "Postgres: localhost:15432 (contract_check/contract_check)"
shell-api: ## Open shell inside API container
docker compose --profile services exec api /bin/sh
shell-bot: ## Open shell inside bot container
docker compose --profile services exec bot /bin/sh
# ─────────────────────────────────────────────────────────────────────────────
# Cleanup
# ─────────────────────────────────────────────────────────────────────────────
clean: ## Remove containers, volumes, caches
docker compose --profile services down -v
docker compose down -v
rm -rf .mypy_cache .pytest_cache .ruff_cache
uv cache clean
# ─────────────────────────────────────────────────────────────────────────────
# Full workflow shortcuts
# ─────────────────────────────────────────────────────────────────────────────
dev: install infra-up migrate services-up ## Bootstrap full dev environment
stop: services-down infra-down ## Stop everything

152
README.md Normal file
View file

@ -0,0 +1,152 @@
# Контракт-чек
LLM-сервис скрининга рисков в договорах (PDF/DOCX) по ГК РФ / ГК РБ.
Telegram-бот MVP + B2B API сейчас; веб + подписки — позже.
Pipeline: `PDF/DOCX → текст (pymupdf/tesseract OCR) → чанки → Ollama Cloud (LLM, json-schema + repair-loop) → markdown-отчёт` с цитатами, ссылкой на пункт и дисклеймером «не заменяет юриста».
## Документы
- [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) — **единственный источник истины** для production-рефакторинга. Hexagonal-архитектура, RabbitMQ-конвейер, схемы БД, очередь/ретраи/DLQ, конфиг, deploy, observability.
- [`docs/DEPLOY.md`](docs/DEPLOY.md) — руководство по развёртыванию: локально, Docker Compose, VPS, seed-token, backup, troubleshooting.
- [`docs/BUSINESS_IDEA.md`](docs/BUSINESS_IDEA.md) — идея и бизнес-модель.
- [`docs/IMPLEMENTATION_PLAN.md`](docs/IMPLEMENTATION_PLAN.md) — исходный «ленивый» план. Этап 0 (прототип) актуален; этапы 1+ superseded в `docs/ARCHITECTURE.md`.
- [`docs/TICKETS.md`](docs/TICKETS.md) — тикеты реализации со статусами.
## Архитектура (одна строка)
Hexagonal (ports & adapters). Ядро `contract_check.core` владеет всем состоянием
(Postgres/MinIO/RabbitMQ/LLM/кредиты). Четыре сервиса собираются из него:
```
Telegram ──► bot (aiogram, HTTP-only) ──HTTP──► api (FastAPI) ──publish──► RabbitMQ
владеет: PG, MinIO, │
Redis, кредитами ▼
extract.q ──► worker-extract
(pymupdf+tesseract, CPU) │
▼ publish
analyze.q ──► worker-analyze
(LLM, I/O) → Report
```
- **api** — единственный писатель для пользовательских мутаций (upload → reserve credit → MinIO → publish `DocumentUploaded`).
- **worker-extract** — CPU: достаёт текст (PDF/DOCX), при необходимости OCR, грузит `.txt` в MinIO, публикует `DocumentExtracted`.
- **worker-analyze** — I/O: LLM-анализ по чек-листу, валидация/repair, сохраняет `Report`, `status=done`.
- **bot** — адаптер: HTTP-клиент к api, **не импортирует** core.db/s3/llm/mq (граница проверяется тестом `tests/unit/test_bot_boundary.py`).
## Структура репозитория
```
src/contract_check/
__main__.py # указывает на prototype (stage-0 CLI сохранён)
core/ # общий домен (импортируется каждым сервисом)
config.py logging.py telemetry.py sentry.py metrics.py
credits.py tokens.py api_keys.py rate_limit.py redis_client.py
db/ models.py session.py enums.py
mq/ topology.py publisher.py consumer.py messages.py
s3/ port.py minio_storage.py
llm/ port.py ollama_cloud.py factory.py
analysis/ extractor.py chunker.py checklist.py report_schema.py ocr.py analyzer.py
api/ # FastAPI-образ
app.py deps.py middleware.py services.py __main__.py
routes/ health.py documents.py reports.py me.py metrics.py b2b.py
worker_extract/ # CPU-образ (pymupdf + tesseract)
consumer.py handler.py extract_document.py __main__.py
worker_analyze/ # I/O-образ (LLM provider)
consumer.py handler.py __main__.py
bot/ # aiogram-адаптер (самый «тощий» образ: только core.logging)
client.py config.py handlers.py __main__.py
prototype/ # stage-0 standalone CLI (бенчмарк go/no-go)
docs/ # документация проекта (README остаётся в корне)
ARCHITECTURE.md # архитектура, схемы БД, конфиг
DEPLOY.md # руководство по развёртыванию
TICKETS.md # статусы тикетов
IMPLEMENTATION_PLAN.md # исходный план (superseded для этапов 1+)
BUSINESS_IDEA.md # продукт / бизнес-модель
srv/ # Dockerfile-ы (один на сервис, deps заточены)
api/Dockerfile worker-extract/Dockerfile worker-analyze/Dockerfile
bot/Dockerfile prototype/Dockerfile
migrations/ # alembic (async): 0001_initial, 0002_api_keys
tests/
conftest.py
unit/ chunker, extractor, llm_ollama_cloud, credits, messages,
rate_limit, checklist_report, bot_client, bot_boundary
integration/ upload_pipeline, extract_worker, analyze_worker, b2b_api, credits_db
docker-compose.yml # default = инфра; --profile services = стек
pyproject.toml # hatchling + PEP 735 dependency-groups (db/mq/s3/obs/api/extract/analyze/bot/prototype/dev)
.env.example # полный список env (см. docs/ARCHITECTURE.md §11)
```
## Быстрый старт
### Локально (разработка)
```bash
uv sync --group dev # все группы для локальной разработки
cp .env.example .env # впишите OLLAMA_HOST / OLLAMA_API_KEY
docker compose up -d # только инфра (postgres/redis/rabbitmq/minio)
uv run alembic upgrade head # миграции
uv run python -m contract_check.api # api на :8000
uv run python -m contract_check.worker_extract # воркер экстракции
uv run python -m contract_check.worker_analyze # воркер анализа
uv run python -m contract_check.bot # Telegram-бот
```
### Через Docker Compose
```bash
cp .env.example .env # заполнить секреты (DB, Rabbit, MinIO, Ollama, BOT_TOKEN, ...)
docker compose up -d # только инфра с healthchecks
docker compose --profile services up -d --build # + api, worker-extract, worker-analyze, bot
```
Профиль `services` собирает 4 образа из `srv/<service>/Dockerfile` и поднимает их
с `depends_on: condition: service_healthy`. Observability (Prometheus/Grafana/Tempo/OTel)
и edge (Nginx/certbot) — за будущими профилями `obs`/`edge` (`docs/ARCHITECTURE.md §20`, шаги 1–5
реализованы; шаг 6 — позже).
Порты на хосте (смещены, чтобы не конфликтовать): Postgres `15432`, Redis `17379`,
RabbitMQ AMQP `5672` / UI `15672`, MinIO `9000` / console `9001`, api `8000` / metrics `9100`,
worker metrics `9101`/`9102`.
### Stage-0 прототип (бенчмарк)
```bash
uv sync --group prototype
uv run python -m contract_check prototype contract.pdf # отчёт в stdout
uv run contract-check contract.pdf -o report.md # или консольная команда
uv run python -m contract_check prototype contract.pdf --json metrics.json # + метрики go/no-go
```
## API (кратко)
Все роуты под `/api/v1`. Auth зависит от роута:
- **Пользовательские роуты** (bot / web / Mini App) — `Authorization: Bearer <user_jwt>`. JWT выдаётся через `/api/v1/auth/telegram/*` после проверки identity от Telegram.
- **Адаптер-level** (только `/api/v1/auth/telegram/bot`) — `Authorization: Bearer <service_token>`.
- **B2B** — `X-API-Key`. Управление B2B-ключами требует пользовательский JWT.
- **Health/metrics** — без auth.
| Метод | Путь | Auth | Назначение |
|---|---|---|---|
| GET | `/healthz`, `/readyz`, `/metrics` | — | liveness / readiness / Prometheus |
| POST | `/api/v1/auth/telegram/bot` | service token | бот меняет verified `telegram_id` на JWT |
| POST | `/api/v1/auth/telegram/web` | — | Telegram Login Widget → JWT |
| POST | `/api/v1/auth/telegram/miniapp` | — | Mini App `initData` → JWT |
| GET | `/api/v1/auth/me` | user JWT | introspect JWT |
| POST | `/api/v1/documents` | user JWT | multipart upload → reserve credit → MinIO → publish → `202 {document_id, correlation_id}` |
| GET | `/api/v1/documents/{id}` | user JWT | статус + stage (для поллинга) |
| GET | `/api/v1/reports/{document_id}` | user JWT | `202 {status, stage}` или `200 {markdown, findings, ...}` |
| GET | `/api/v1/me` | user JWT | `{telegram_id, credits_left}` |
| POST | `/api/v1/analyze` | `X-API-Key` | B2B: анализ документа |
| GET | `/api/v1/b2b/reports/{id}`, `/api/v1/b2b/usage` | `X-API-Key` | B2B: отчёт / usage |
| POST/GET | `/api/v1/b2b/keys`, `/api/v1/b2b/keys/{id}/revoke`, `.../usage` | user JWT | управление B2B-ключами |
Полная спецификация — `docs/ARCHITECTURE.md §15` (актуализируется).
## Проверки (DoD)
```bash
uv run ruff check . && uv run mypy src && uv run pytest -q # unit, быстро
uv run pytest -m integration -q # интеграционные (нужны контейнеры)
```

40
alembic.ini Normal file
View file

@ -0,0 +1,40 @@
# Alembic config. The DB URL is resolved at runtime from DATABASE_URL env
# (see migrations/env.py), so the [alembic] sqlalchemy.url here is a fallback.
[alembic]
script_location = migrations
prepend_sys_path = src
sqlalchemy.url = postgresql+asyncpg://contract_check:contract_check@localhost:5432/contract_check
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARNING
handlers = console
qualname =
[logger_sqlalchemy]
level = WARNING
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

299
docker-compose.yml Normal file
View file

@ -0,0 +1,299 @@
# «Контракт-чек» — infrastructure (Step 1).
#
# Default (`docker compose up`) starts ONLY infra: postgres + redis + rabbitmq
# + minio (+ minio-init). Service containers (api, worker-extract, worker-analyze,
# bot) and the observability/edge stacks are added behind profiles in later steps
# (docs/ARCHITECTURE.md §20).
#
# Durability posture (§10): quorum-ready. Postgres is configured
# wal_level=replica + WAL archiving (replica/PITR-ready). RabbitMQ quorum queues
# are declared by the app (core/mq/topology.py) — they replicate the moment a
# 3-node cluster is added. Named volumes everywhere; restart: unless-stopped.
services:
postgres:
image: postgres:18-alpine
container_name: contract_check-postgres
restart: unless-stopped
command:
- "postgres"
- "-c"
- "wal_level=replica"
- "-c"
- "archive_mode=on"
- "-c"
- "archive_command=test ! -f /walarchive/%f && cp %p /walarchive/%f"
environment:
POSTGRES_USER: ${POSTGRES_USER:-contract_check}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-contract_check}
POSTGRES_DB: ${POSTGRES_DB:-contract_check}
volumes:
- pgdata:/var/lib/postgresql
- pgwal:/walarchive
ports:
- "15432:5432"
healthcheck:
test:
- CMD-SHELL
- "pg_isready -U ${POSTGRES_USER:-contract_check} -d ${POSTGRES_DB:-contract_check}"
interval: 5s
timeout: 3s
retries: 10
redis:
image: redis:8-alpine
container_name: contract_check-redis
restart: unless-stopped
command: ["redis-server", "--appendonly", "yes"]
volumes:
- redisdata:/data
ports:
- "17379:6379"
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 3s
retries: 10
rabbitmq:
image: rabbitmq:4-management-alpine
container_name: contract_check-rabbitmq
restart: unless-stopped
environment:
RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-contract_check}
RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASS:-contract_check}
RABBITMQ_DEFAULT_VHOST: ${RABBITMQ_VHOST:-/}
volumes:
- rabbitmq:/var/lib/rabbitmq
ports:
- "5672:5672" # AMQP
- "15672:15672" # management UI (http://localhost:15672)
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 10s
timeout: 5s
retries: 10
start_period: 15s
minio:
image: minio/minio:latest
container_name: contract_check-minio
restart: unless-stopped
command: server /data --console-address ":9001"
environment:
MINIO_ROOT_USER: ${S3_ACCESS_KEY:-contract_check}
MINIO_ROOT_PASSWORD: ${S3_SECRET_KEY:-contract_check}
volumes:
- minio:/data
ports:
- "9000:9000" # S3 API
- "9001:9001" # console (http://localhost:9001)
healthcheck:
test: ["CMD", "curl", "-sf", "http://localhost:9000/minio/health/ready"]
interval: 10s
timeout: 5s
retries: 10
start_period: 10s
# One-shot: create the bucket + an ILM expiry rule (152-ФЗ retention lever).
# Service containers (Step 2+) gate on `service_completed_successfully`.
minio-init:
image: minio/mc:latest
container_name: contract_check-minio-init
depends_on:
minio:
condition: service_healthy
entrypoint: /bin/sh
command:
- -c
- |
set -e
mc alias set local http://minio:9000 "$${MINIO_ROOT_USER:-contract_check}" "$${MINIO_ROOT_PASSWORD:-contract_check}"
mc mb --ignore-existing local/${S3_BUCKET:-contract-check-docs}
mc anonymous set none local/${S3_BUCKET:-contract-check-docs} || true
# Expire raw docs + extracted text after DOC_RETENTION_DAYS (default 7).
mc ilm rule add --expire-days ${DOC_RETENTION_DAYS:-7} local/${S3_BUCKET:-contract-check-docs} || true
echo "bucket ${S3_BUCKET:-contract-check-docs} ready (ilm expire ${DOC_RETENTION_DAYS:-7}d)"
environment:
MINIO_ROOT_USER: ${S3_ACCESS_KEY:-contract_check}
MINIO_ROOT_PASSWORD: ${S3_SECRET_KEY:-contract_check}
restart: "no"
# ── SERVICES (profile: services) ────────────────────────────────────────────
# Core FastAPI service. Runs migrations separately (see deploy docs); assumes
# the DB is migrated before accepting traffic via healthcheck delay.
api:
profiles: ["services"]
build:
context: .
dockerfile: srv/api/Dockerfile
container_name: contract_check-api
restart: unless-stopped
depends_on:
postgres:
condition: service_healthy
rabbitmq:
condition: service_healthy
minio-init:
condition: service_completed_successfully
environment:
ENV: ${ENV:-dev}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-contract_check}:${POSTGRES_PASSWORD:-contract_check}@postgres:5432/${POSTGRES_DB:-contract_check}
REDIS_URL: redis://redis:6379/0
RABBITMQ_URL: amqp://${RABBITMQ_USER:-contract_check}:${RABBITMQ_PASS:-contract_check}@rabbitmq:5672/${RABBITMQ_VHOST:-/}
S3_ENDPOINT_URL: http://minio:9000
S3_ACCESS_KEY: ${S3_ACCESS_KEY:-contract_check}
S3_SECRET_KEY: ${S3_SECRET_KEY:-contract_check}
S3_BUCKET: ${S3_BUCKET:-contract-check-docs}
S3_REGION: ${S3_REGION:-us-east-1}
S3_SERVER_SIDE_ENCRYPTION: ${S3_SERVER_SIDE_ENCRYPTION:-false}
DOC_RETENTION_DAYS: ${DOC_RETENTION_DAYS:-7}
REFUND_POLICY: ${REFUND_POLICY:-all}
SENTRY_DSN: ${SENTRY_DSN:-}
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-}
OTEL_SERVICE_NAME: api
LLM_PROVIDER: ${LLM_PROVIDER:-ollama_cloud}
OLLAMA_HOST: ${OLLAMA_HOST:-}
OLLAMA_API_KEY: ${OLLAMA_API_KEY:-}
OLLAMA_MODEL: ${OLLAMA_MODEL:-qwen2.5:14b}
OLLAMA_FALLBACK_MODEL: ${OLLAMA_FALLBACK_MODEL:-qwen2.5:7b}
OLLAMA_TEMPERATURE: ${OLLAMA_TEMPERATURE:-0.2}
OLLAMA_NUM_PREDICT: ${OLLAMA_NUM_PREDICT:-3072}
OLLAMA_TIMEOUT: ${OLLAMA_TIMEOUT:-120}
OLLAMA_MAX_CONCURRENCY: ${OLLAMA_MAX_CONCURRENCY:-3}
CHUNK_SIZE_CHARS: ${CHUNK_SIZE_CHARS:-10000}
API_HOST: ${API_HOST:-0.0.0.0}
API_PORT: ${API_PORT:-8000}
API_METRICS_PORT: ${API_METRICS_PORT:-9100}
TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:-}
JWT_SECRET: ${JWT_SECRET}
JWT_ALGORITHM: ${JWT_ALGORITHM:-HS256}
JWT_ACCESS_TTL_MINUTES: ${JWT_ACCESS_TTL_MINUTES:-1440}
ports:
- "${API_PORT:-8000}:8000"
- "${API_METRICS_PORT:-9100}:9100"
healthcheck:
test:
- CMD-SHELL
- "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/healthz')\""
interval: 10s
timeout: 3s
retries: 10
start_period: 15s
worker-extract:
profiles: ["services"]
build:
context: .
dockerfile: srv/worker-extract/Dockerfile
container_name: contract_check-worker-extract
restart: unless-stopped
depends_on:
postgres:
condition: service_healthy
rabbitmq:
condition: service_healthy
minio-init:
condition: service_completed_successfully
environment:
ENV: ${ENV:-dev}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-contract_check}:${POSTGRES_PASSWORD:-contract_check}@postgres:5432/${POSTGRES_DB:-contract_check}
RABBITMQ_URL: amqp://${RABBITMQ_USER:-contract_check}:${RABBITMQ_PASS:-contract_check}@rabbitmq:5672/${RABBITMQ_VHOST:-/}
S3_ENDPOINT_URL: http://minio:9000
S3_ACCESS_KEY: ${S3_ACCESS_KEY:-contract_check}
S3_SECRET_KEY: ${S3_SECRET_KEY:-contract_check}
S3_BUCKET: ${S3_BUCKET:-contract-check-docs}
S3_REGION: ${S3_REGION:-us-east-1}
S3_SERVER_SIDE_ENCRYPTION: ${S3_SERVER_SIDE_ENCRYPTION:-false}
DOC_RETENTION_DAYS: ${DOC_RETENTION_DAYS:-7}
REFUND_POLICY: ${REFUND_POLICY:-all}
SENTRY_DSN: ${SENTRY_DSN:-}
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-}
OTEL_SERVICE_NAME: worker-extract
MQ_PREFETCH_EXTRACT: ${MQ_PREFETCH_EXTRACT:-1}
MQ_MAX_ATTEMPTS: ${MQ_MAX_ATTEMPTS:-5}
MQ_RETRY_BASE_MS: ${MQ_RETRY_BASE_MS:-2000}
TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:-}
JWT_SECRET: ${JWT_SECRET}
JWT_ALGORITHM: ${JWT_ALGORITHM:-HS256}
JWT_ACCESS_TTL_MINUTES: ${JWT_ACCESS_TTL_MINUTES:-1440}
ports:
- "9101:9101"
worker-analyze:
profiles: ["services"]
build:
context: .
dockerfile: srv/worker-analyze/Dockerfile
container_name: contract_check-worker-analyze
restart: unless-stopped
depends_on:
postgres:
condition: service_healthy
rabbitmq:
condition: service_healthy
minio-init:
condition: service_completed_successfully
environment:
ENV: ${ENV:-dev}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-contract_check}:${POSTGRES_PASSWORD:-contract_check}@postgres:5432/${POSTGRES_DB:-contract_check}
RABBITMQ_URL: amqp://${RABBITMQ_USER:-contract_check}:${RABBITMQ_PASS:-contract_check}@rabbitmq:5672/${RABBITMQ_VHOST:-/}
S3_ENDPOINT_URL: http://minio:9000
S3_ACCESS_KEY: ${S3_ACCESS_KEY:-contract_check}
S3_SECRET_KEY: ${S3_SECRET_KEY:-contract_check}
S3_BUCKET: ${S3_BUCKET:-contract-check-docs}
S3_REGION: ${S3_REGION:-us-east-1}
REFUND_POLICY: ${REFUND_POLICY:-all}
SENTRY_DSN: ${SENTRY_DSN:-}
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-}
OTEL_SERVICE_NAME: worker-analyze
MQ_PREFETCH_ANALYZE: ${MQ_PREFETCH_ANALYZE:-3}
MQ_MAX_ATTEMPTS: ${MQ_MAX_ATTEMPTS:-5}
MQ_RETRY_BASE_MS: ${MQ_RETRY_BASE_MS:-2000}
LLM_PROVIDER: ${LLM_PROVIDER:-ollama_cloud}
OLLAMA_HOST: ${OLLAMA_HOST:-}
OLLAMA_API_KEY: ${OLLAMA_API_KEY:-}
OLLAMA_MODEL: ${OLLAMA_MODEL:-qwen2.5:14b}
OLLAMA_FALLBACK_MODEL: ${OLLAMA_FALLBACK_MODEL:-qwen2.5:7b}
OLLAMA_TEMPERATURE: ${OLLAMA_TEMPERATURE:-0.2}
OLLAMA_NUM_PREDICT: ${OLLAMA_NUM_PREDICT:-3072}
OLLAMA_TIMEOUT: ${OLLAMA_TIMEOUT:-120}
OLLAMA_MAX_CONCURRENCY: ${OLLAMA_MAX_CONCURRENCY:-3}
CHUNK_SIZE_CHARS: ${CHUNK_SIZE_CHARS:-10000}
TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:-}
JWT_SECRET: ${JWT_SECRET}
JWT_ALGORITHM: ${JWT_ALGORITHM:-HS256}
JWT_ACCESS_TTL_MINUTES: ${JWT_ACCESS_TTL_MINUTES:-1440}
ports:
- "9102:9102"
# Telegram bot adapter (aiogram 3, HTTP-only to api). Per docs/ARCHITECTURE.md §17
# the bot holds no DB/MQ/S3 credentials — it depends on `api` being healthy,
# not on the infra containers directly, enforcing the hexagonal boundary even
# in dependency ordering.
bot:
profiles: ["services"]
build:
context: .
dockerfile: srv/bot/Dockerfile
container_name: contract_check-bot
restart: unless-stopped
depends_on:
api:
condition: service_healthy
environment:
ENV: ${ENV:-dev}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
BOT_TOKEN: ${BOT_TOKEN:-}
API_URL: ${API_URL:-http://api:8000}
BOT_SERVICE_TOKEN: ${BOT_SERVICE_TOKEN:-}
volumes:
pgdata:
pgwal:
redisdata:
rabbitmq:
minio:

1288
docs/ARCHITECTURE.md Normal file
View file

File diff suppressed because it is too large Load diff

96
docs/BUSINESS_IDEA.md Normal file
View file

@ -0,0 +1,96 @@
# «Контракт-чек» — AI-анализатор договоров для СНГ
> microSaaS: загрузил договор → получил отчёт с подсветкой рисков
> по чек-листу юриста, адаптированному под ГК РФ / ГК РБ.
---
## Суть одной строкой
Загружаешь договор (PDF / DOCX / скан) → получаешь отчёт с подсветкой
рисков по чек-листу юриста, адаптированному под ГК РФ / РБ.
## Почему это microSaaS и почему мне
- **Одна фича** (найти риски в договоре), ничего лишнего.
- **Боль реальная и ежедневная**: фрилансер / ИП / малый бизнес подписывают
договоры, не будучи юристами, и потом ловят штрафы 200%, чужую
подсудность, одностороннее изменение цены.
- **Стек**: FastAPI + парсинг документов (pymupdf) +
Postgres / Redis + LLM. React для веба.
- **AI commoditized сложную часть** (юр. NLP), но глобальные игроки
(LegalGPT и т.п.) сидят на common law — СНГ-право это наша вотчина.
- **ЦА сидит в Telegram-каналах** (юристы, фрилансеры, ИП) — дистрибуция
копеечная, без рекламного бюджета.
## Архитектура (ленивая, до $10k MRR)
| Слой | Технология |
|-------------|-------------------------------------------------------------------|
| Backend | FastAPI + SQLAlchemy + Postgres + Redis |
| Workers | arq (Redis-очередь) для OCR / LLM задач |
| LLM | Ollama Cloud — hosted open-модели (напр. qwen2.5:14b). Подписка $20/мес + metered overage. Fallback: лёгкая модель при 429/квоте; план Б — GigaChat/YandexGPT (РФ) |
| OCR | pymupdf для текстовых PDF. Сканы → Tesseract (локально, rus+eng); Yandex Vision — опция улучшения качества |
| Storage | Selectel Object Storage (S3-совместимый, РФ-локация под 152-ФЗ) |
| Frontend | v1 — Telegram-бот как MVP; v2 — React-веб для B2B / API клиентов |
| Payments | ЮKassa (подписки, карты СНГ) + CloudPayments |
| Deploy | один VPS (Hetzner EU — стабильный доступ к Ollama Cloud), Docker Compose. K8s — не нужен |
## Что реализуем в v1 (режем жёстко)
- Приём одного документа за раз (бизнес-бот / веб-форма).
- Отчёт в markdown / PDF: список пунктов с уровнем риска
(высокий / средний), цитатой из оригинала и номером пункта
(чтобы проверить одним кликом).
- Чек-лист ~10 пунктов: штрафы / неустойки, подсудность, сроки оплаты,
IP-права, одностороннее изменение условий, гарантии, форс-мажор, НДС,
ответственность, расторжение.
- Pay-per-doc. Без подписок, без команд, без SSO.
**Skipped**: multi-tenant orgs, API, шаблоны договоров, e-sign,
команду юристов.
**Add when**: появится 50+ платящих и запрос на подписку / командный доступ.
## Цены
| Тариф | Цена | Лимит |
|------------------|-------------|------------------------|
| Pay-per-doc | 199 ₽ / док | разовая |
| Подписка solo | 1 490 ₽/мес | 10 док-ов |
| Подписка team | 3 990 ₽/мес | 30 док-ов |
| API / B2B | 9 900 ₽/мес | 100 док-ов + REST API |
### Юнит-экономика (прикидка, Ollama Cloud)
- Опекс фиксированный до квоты: VPS Hetzner ~4 €/мес + Ollama Cloud
Pro $20/мес + Selectel S3 (копейки). Итого ~$25–30/мес на старте.
- Per-doc cost: при росте объёма — metered overage Ollama Cloud (по
токенам модели). Реальную себестоимость на документ **измеряем на
этапе 0** (квота/токены на договор); цель — заметно ниже 199 ₽.
- Маржа отлична на малом объёме (фиксированный cost); сжимается при
overage и упирается в конкарренси Pro = 3 одновременных анализа.
- 200 платящих × активность могут не прожаться на одном Pro без очередей
→ Enterprise-тариф или свой GPU-бокс в РФ по достижении ~50–100 платящих.
- Цель: **200 платящих × ~1 490 ₽ ≈ 300k ₽/мес MRR** за 6–9 мес через
посев в Telegram-каналах юристов / фрилансеров + SEO «анализ договора онлайн».
## Риски (честно)
1. **«AI даёт юр. совет»** → позиционировать как чек-лист / подсветку,
НЕ консультацию. Disclaimer везде: «не заменяет юриста, это первичный
скрининг».
2. **LLM галлюцинирует пункты** (open-модели 14B — сильнее, чем GPT-4o) →
ВСЕГДА цитата + номер пункта из оригинала, кликабельная, + серверная
валидация JSON / repair-loop. Юзер сам сверяет за 5 секунд.
3. **152-ФЗ (ПДн)**: договоры могут содержать ПДн. Хранение (S3, БД) —
в РФ (Selectel). **НО Ollama Cloud инферисит в США** → текст договора
пересекает границу = трансграничная передача ПДн. Митигация:
обезличивание ПДн перед отправкой в LLM + disclaimer + согласие,
либо план Б — РФ-LLM (GigaChat/YandexGPT) или свой GPU-бокс в РФ.
НЕ позиционировать как «данные не покидают РФ».
## Почему не раздавят
- **СНГ-право + судебная практика РФ/РБ** как калибровка чек-листа —
глобалам неинтересно, слишком узкий рынок для них и в самый раз для соло.
- **Distribution через Telegram-сообщества** — там ЦА, там и продукт (бот).

546
docs/DEPLOY.md Normal file
View file

@ -0,0 +1,546 @@
# Руководство по развёртыванию «Контракт-чек»
> Целевая аудитория: DevOps / разработчик, поднимающий стек на одном VPS или локально.
> Предполагается: Docker + Docker Compose установлены, есть SSH-доступ к VPS.
> Полная архитектура: [`ARCHITECTURE.md`](ARCHITECTURE.md).
---
## Содержание
1. [Требования](#1-требования)
2. [Быстрый старт локально](#2-быстрый-старт-локально)
3. [Конфигурация через `.env`](#3-конфигурация-через-env)
4. [Docker Compose — полный стек](#4-docker-compose--полный-стек)
5. [Первичная инициализация](#5-первичная-инициализация)
6. [Проверка работоспособности](#6-проверка-работоспособности)
7. [Telegram-бот: первый запуск](#7-telegram-бот-первый-запуск)
8. [B2B API: первый API-ключ](#8-b2b-api-первый-api-ключ)
9. [Резервное копирование](#9-резервное-копирование)
10. [Обновление (zero-downtime-ish)](#10-обновление)
11. [Масштабирование](#11-масштабирование)
12. [Troubleshooting](#12-troubleshooting)
---
## 1. Требования
| Компонент | Минимум | Рекомендуемое |
|---|---|---|
| CPU | 2 vCPU | 4 vCPU (worker-extract CPU-bound) |
| RAM | 2 GB | 4 GB |
| Disk | 20 GB SSD | 40 GB SSD (MinIO + WAL-archive) |
| OS | Ubuntu 22.04+ / Debian 12+ | Ubuntu 24.04 LTS |
| Docker | 24.0+ | 25.0+ |
| Docker Compose | v2 | v2 |
| Интернет | нужен для Ollama Cloud | — |
> **Важно:** Ollama Cloud инферит в США. Если 152-ФЗ data-residency критичен —
> рассмотрите self-hosted Ollama (GPU) или GigaChat-адаптер (`core/llm/factory.py`).
---
## 2. Быстрый старт локально
### 2.1 Клонирование и env
```bash
git clone <repo> contract-check
cd contract-check
cp .env.example .env
# Отредактируйте .env — минимум: OLLAMA_HOST, OLLAMA_API_KEY, BOT_TOKEN,
# TELEGRAM_BOT_TOKEN, JWT_SECRET
```
### 2.2 Инфраструктура (только Postgres + Redis + RabbitMQ + MinIO)
```bash
docker compose up -d
# Ждём healthy:
docker compose ps
# Ожидаемый результат: postgres, redis, rabbitmq, minio, minio-init — Up (healthy)
```
Порты на хосте (смещены, чтобы не конфликтовать):
- Postgres: `15432`
- Redis: `17379`
- RabbitMQ AMQP: `5672`, Management UI: `http://localhost:15672`
- MinIO S3 API: `9000`, Console: `http://localhost:9001`
### 2.3 Миграции
```bash
# Локально (нужен uv + dev-группа):
uv sync --group dev
uv run alembic upgrade head
# Или через временный api-контейнер:
docker compose --profile services run --rm api alembic upgrade head
```
### 2.4 Полный стек (api + workers + bot)
```bash
docker compose --profile services up -d --build
# Ждём healthy:
docker compose --profile services ps
```
---
## 3. Конфигурация через `.env`
Все секреты — в `.env` (не коммитить!). Ключевые переменные:
```bash
# --- LLM (обязательно) ---
OLLAMA_HOST=https://api.ollama.com
OLLAMA_API_KEY=sk-xxxxxxxx
OLLAMA_MODEL=qwen2.5:14b
OLLAMA_FALLBACK_MODEL=qwen2.5:7b
# --- Telegram + auth (обязательно) ---
BOT_TOKEN=123456789:ABCDEF... # для aiogram бота
TELEGRAM_BOT_TOKEN=$BOT_TOKEN # тот же токен; API использует для проверки Login Widget / Mini App
BOT_SERVICE_TOKEN=bot-prod-secret-xxx # см. §5.3
JWT_SECRET=$(openssl rand -hex 32) # HS256 secret для подписи JWT
# --- Postgres (можно оставить defaults для dev) ---
POSTGRES_USER=contract_check
POSTGRES_PASSWORD=changeme-strong-password
POSTGRES_DB=contract_check
# --- RabbitMQ ---
RABBITMQ_USER=contract_check
RABBITMQ_PASS=changeme-strong-password
# --- MinIO ---
S3_ACCESS_KEY=contract_check
S3_SECRET_KEY=changeme-strong-password
S3_BUCKET=contract-check-docs
# --- Billing ---
REFUND_POLICY=all # или infra_only
# --- Observability (опционально) ---
SENTRY_DSN=https://...@sentry.io/...
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
```
> Полный список: см. `.env.example` и `ARCHITECTURE.md §11`.
---
## 4. Docker Compose — полный стек
### 4.1 Профили
| Профиль | Что поднимает |
|---|---|
| *(default)* | `postgres`, `redis`, `rabbitmq`, `minio`, `minio-init` |
| `services` | `api`, `worker-extract`, `worker-analyze`, `bot` |
| `obs` *(планируется)* | `prometheus`, `grafana`, `otel-collector`, `tempo` |
| `edge` *(планируется)* | `nginx`, `certbot` |
```bash
# Инфра:
docker compose up -d
# + сервисы (сборка + запуск):
docker compose --profile services up -d --build
# + observability + edge (когда будут готовы конфиги в deploy/):
# docker compose --profile services --profile obs --profile edge up -d --build
```
### 4.2 depends_on и healthchecks
`api` ждёт `postgres`, `rabbitmq`, `minio-init` (healthy / completed).
`bot` ждёт только `api` (healthy) — не инфра напрямую, соблюдая hexagonal boundary.
Workers ждут `postgres`, `rabbitmq`, `minio-init`.
---
## 5. Первичная инициализация
### 5.1 Миграции
```bash
# Способ A: локально (если uv установлен)
uv run alembic upgrade head
# Способ B: через api-контейнер (если uv недоступен на хосте)
docker compose --profile services run --rm api alembic upgrade head
```
Проверка:
```bash
docker compose exec postgres psql -U contract_check -d contract_check -c "\dt"
# Должны быть: api_key_requests, api_keys, documents, invoices, jobs, reports, service_tokens, users
```
### 5.2 Seed-token для адаптеров (bot / web / cli)
Бот аутентифицируется в api через `Authorization: Bearer <BOT_SERVICE_TOKEN>`.
Этот токен нужно положить в `service_tokens`.
```bash
# Генерируем секрет
BOT_SERVICE_TOKEN=$(openssl rand -hex 32)
echo "BOT_SERVICE_TOKEN=$BOT_SERVICE_TOKEN" >> .env
# Записываем hash в БД через временный api-контейнер
docker compose --profile services run --rm api \
python -c "
import asyncio, os, sys
os.chdir('/app')
sys.path.insert(0, 'src')
from contract_check.core.config import get_settings
from contract_check.core.tokens import hash_token
from contract_check.core.db.session import create_session_factory
from sqlalchemy import text
async def seed():
factory = create_session_factory()
async with factory() as s:
h = hash_token('$BOT_SERVICE_TOKEN')
await s.execute(text('INSERT INTO service_tokens (name, token_hash, adapter) VALUES (:n, :h, :a) ON CONFLICT (name) DO UPDATE SET token_hash = EXCLUDED.token_hash'), {'n': 'bot-prod', 'h': h, 'a': 'bot'})
await s.commit()
print('seeded bot-prod')
asyncio.run(seed())
"
```
> В production используйте `python -m contract_check.api seed-token` CLI (если реализовано).
### 5.3 Перезапуск бота с новым токеном
```bash
docker compose --profile services restart bot
# Проверка логов:
docker compose --profile services logs -f bot
```
---
## 6. Проверка работоспособности
### 6.1 Health endpoints
```bash
curl http://localhost:8000/healthz # liveness — 200
curl http://localhost:8000/readyz # readiness — 200 (PG + Rabbit + MinIO)
curl http://localhost:8000/metrics # Prometheus exposition
```
### 6.2 RabbitMQ Management UI
Откройте `http://localhost:15672` (guest/guest или credentials из `.env`).
Проверьте:
- Exchange `contracts.x` (direct)
- Queues: `extract.q`, `analyze.q` (quorum), `extract.retry.q`, `analyze.retry.q`, DLQ
- Connections/consumers от worker-ов
### 6.3 MinIO Console
`http://localhost:9001` (root user = `S3_ACCESS_KEY` / `S3_SECRET_KEY`).
Бакет `contract-check-docs` должен появиться после `minio-init`.
### 6.4 End-to-end smoke-тест (через curl)
```bash
# 1. Сначала нужен JWT пользователя. В dev можно обменять telegram_id через
# service-token endpoint /auth/telegram/bot (в production этим занимается бот):
curl -X POST -H "Authorization: Bearer $BOT_SERVICE_TOKEN" \
-H "Content-Type: application/json" \
-d '{"telegram_id": 123456}' \
http://localhost:8000/api/v1/auth/telegram/bot
# → {"access_token":"eyJ...", "user_id":"...", "telegram_id":123456, ...}
USER_JWT="eyJ..." # скопируйте access_token
# 2. Проверить /me
curl -H "Authorization: Bearer $USER_JWT" http://localhost:8000/api/v1/me
# → {"telegram_id":123456, "credits_left":0}
# 3. Загрузить документ (нужен credit — пока 0, но проверим путь)
curl -X POST -H "Authorization: Bearer $USER_JWT" \
-F "file=@contract.pdf" http://localhost:8000/api/v1/documents
# → 402 Payment Required (нормально — нет кредитов)
# 4. Добавить кредит вручную (dev):
docker compose exec postgres psql -U contract_check -d contract_check \
-c "UPDATE users SET credits_left = 10 WHERE telegram_id = 123456;"
```
---
## 7. Telegram-бот: первый запуск
### 7.1 Создание бота в BotFather
1. Напишите `@BotFather` → `/newbot`
2. Скопируйте токен (`BOT_TOKEN`) в `.env`
3. Установите webhook (опционально, polling работает по умолчанию):
```bash
curl -F "url=https://your-domain.com/webhook" \
https://api.telegram.org/bot$BOT_TOKEN/setWebhook
```
### 7.2 Запуск
```bash
docker compose --profile services up -d bot
docker compose --profile services logs -f bot
```
Ожидаемый вывод при `/start`:
```
Привет! Я «Контракт-чек» — первичный скрининг рисков в договорах...
Осталось проверок: 0.
```
### 7.3 Добавление кредитов пользователю
```bash
# Найти telegram_id пользователя (из логов бота или таблицы users)
docker compose exec postgres psql -U contract_check -d contract_check \
-c "UPDATE users SET credits_left = credits_left + 5 WHERE telegram_id = <tg_id>;"
```
---
## 8. B2B API: первый API-ключ
### 8.1 Создание ключа (через user JWT)
Управление B2B-ключами теперь требует пользовательский JWT. Получите JWT через
`/auth/telegram/bot` (как в §6.4) и выполните:
```bash
curl -X POST -H "Authorization: Bearer $USER_JWT" \
-H "Content-Type: application/json" \
-d '{"name": "integration-test", "rate_limit_rps": 3, "monthly_quota": 100}' \
http://localhost:8000/api/v1/b2b/keys
# → {"api_key": "cc_abc123...", "id": "...", ...}
# Сохраните api_key — он показывается ТОЛЬКО ОДИН РАЗ
```
### 8.2 Анализ документа через B2B
```bash
API_KEY="cc_abc123..."
curl -X POST -H "X-API-Key: $API_KEY" \
-F "file=@contract.pdf" \
http://localhost:8000/api/v1/analyze
# → 202 {"document_id": "...", "correlation_id": "..."}
```
### 8.3 Проверка usage
```bash
curl -H "X-API-Key: $API_KEY" http://localhost:8000/api/v1/b2b/usage
```
---
## 9. Резервное копирование
### 9.1 Postgres
```bash
# Ручной backup
docker compose exec postgres pg_dump -U contract_check -d contract_check \
> backup_$(date +%Y%m%d_%H%M%S).sql
# Автоматический backup (cron на хосте)
# 0 2 * * * cd /opt/contract-check && docker compose exec -T postgres pg_dump -U contract_check -d contract_check | gzip > backups/pg_$(date +\%Y\%m\%d).sql.gz
```
### 9.2 WAL-архивы (PITR-ready)
Postgres настроен с `wal_level=replica`, `archive_mode=on`.
WAL-сегменты пишутся в volume `pgwal` (mount `/walarchive`).
Для полноценного PITR настройте `pg_backrest` или репликацию (см. `ARCHITECTURE.md §10`).
### 9.3 MinIO
MinIO хранит raw-документы и extracted `.txt`. ILM-правило истекает объекты
через `DOC_RETENTION_DAYS` (по умолчанию 7 дней). **Отчёты живут в Postgres** — они
выживут после истечения raw-документов. Backup MinIO не обязателен для бизнес-логики.
---
## 10. Обновление
### 10.1 Rolling update (без остановки всего)
```bash
# 1. Pull изменений
git pull origin main
# 2. Rebuild + recreate (Compose пересоздаёт только изменённые контейнеры)
docker compose --profile services up -d --build
# 3. Миграции (если есть новые)
docker compose --profile services run --rm api alembic upgrade head
# 4. Проверка:
curl http://localhost:8000/readyz
```
### 10.2 Graceful shutdown workers
Workers получают `SIGTERM` → завершают текущее сообщение → `SIGKILL` после grace period.
Compose `stop_grace_period` по умолчанию 10s; для долгих контрактов можно увеличить.
---
## 11. Масштабирование
### 11.1 На одном VPS (вертикальное)
- Увеличьте `MQ_PREFETCH_EXTRACT` до числа CPU
- Увеличьте `MQ_PREFETCH_ANALYZE` до конкарренси Ollama (3 на Pro, 10 на Max)
- Масштабируйте RAM под размер контрактов
### 11.2 Горизонтальное (несколько worker-ов)
```yaml
# docker-compose.override.yml
services:
worker-extract:
deploy:
replicas: 2
worker-analyze:
deploy:
replicas: 2
```
```bash
docker compose --profile services up -d --scale worker-extract=2 --scale worker-analyze=2
```
> Workers stateless — горизонтальное масштабирование бесплатно. Единственное ограничение:
> конкарренси Ollama Cloud (3 на Pro). Не масштабируйте `worker-analyze` выше,
> чем позволяет квота провайдера — иначе получите 429.
### 11.3 Многонодовая HA
См. `ARCHITECTURE.md §10`. Путь к HA — только инфраструктурный (Compose → кластер
RabbitMQ 3 nodes + Patroni Postgres + distributed MinIO). **Код `core/` не меняется.**
---
## 12. Troubleshooting
### 12.1 `docker compose up` зависает — сервисы не стартуют
**Причина:** `depends_on: condition: service_healthy` — кто-то не прошёл healthcheck.
```bash
# Диагностика:
docker compose ps
docker compose logs <service>
# Частые причины:
# - Postgres ещё не готов: подождите 15-20s после первого запуска
# - minio-init не выполнился: проверьте docker compose logs minio-init
# - RabbitMQ не отвечает: docker compose logs rabbitmq
```
### 12.2 `POST /documents` → 402 Payment Required
Пользователю не хватает `credits_left`. В dev добавьте вручную:
```bash
docker compose exec postgres psql -U contract_check -d contract_check \
-c "UPDATE users SET credits_left = credits_left + 1 WHERE telegram_id = <id>;"
```
### 12.3 Worker-extract падает с `ExtractionError`
Документ — скан/PDF без текстового слоя. Worker должен автоматически перейти к OCR.
Если и OCR падает — `documents.status=failed`, credit refunded (если refundable).
Проверьте:
```bash
docker compose --profile services logs worker-extract
# Или в Postgres:
docker compose exec postgres psql -U contract_check -d contract_check \
-c "SELECT id, status, stage, last_failure_class FROM documents ORDER BY created_at DESC LIMIT 5;"
```
### 12.4 Worker-analyze: постоянные 429 / `llm_quota`
- Превышена квота Ollama Cloud. Проверьте usage в dashboard ollama.com.
- Уменьшите `MQ_PREFETCH_ANALYZE` или перейдите на Enterprise тариф.
- Проверьте fallback-model (`OLLAMA_FALLBACK_MODEL`) — он должен сработать на 429.
### 12.5 Bot: `Unauthorized` / 401
- `BOT_SERVICE_TOKEN` не совпадает с `service_tokens.token_hash` в БД.
- Бот не смог получить user JWT через `/api/v1/auth/telegram/bot` (проверьте логи бота и api).
- `TELEGRAM_BOT_TOKEN` / `JWT_SECRET` не заданы в `.env` для сервиса `api`.
- Пересоздайте токен через §5.2.
### 12.6 Бот не отвечает
```bash
# Проверка логов
docker compose --profile services logs -f bot
# Проверка polling:
# Бот использует polling по умолчанию (aiogram). Если webhook установлен —
# убедитесь, что Nginx проксирует /webhook к api:8000.
```
### 12.7 MinIO: файлы не видны / bucket не создан
```bash
# Ручной init (если minio-init не отработал)
docker compose run --rm minio-init
# Или проверьте через mc:
docker compose run --rm minio-init mc ls local/
```
### 12.8 Миграции: `alembic` не видит таблицы / revision conflict
```bash
# Проверка текущей HEAD:
docker compose --profile services run --rm api alembic current
# Принудительный upgrade (осторожно — только dev!):
docker compose --profile services run --rm api alembic stamp head
```
### 12.9 Полный сброс (dev only!)
```bash
# Удалить ВСЕ данные (тома + контейнеры):
docker compose --profile services down -v
docker compose down -v
# Затем пересоздать с нуля: §2 + §5
```
---
## Чек-лист перед production
- [ ] `.env` заполнен, `.env.example` не содержит реальных секретов
- [ ] `POSTGRES_PASSWORD`, `RABBITMQ_PASS`, `S3_SECRET_KEY` — strong random
- [ ] `.env` заполнен (включая `TELEGRAM_BOT_TOKEN`, `JWT_SECRET`)
- [ ] `BOT_SERVICE_TOKEN` засеян в `service_tokens`
- [ ] Миграции накатаны (`alembic upgrade head`)
- [ ] `docker compose --profile services ps` показывает все healthy
- [ ] `/healthz` и `/readyz` отвечают 200
- [ ] Telegram-бот отвечает на `/start`
- [ ] Тестовый PDF проходит pipeline: upload → extract → analyze → report
- [ ] B2B-ключ создаётся через `/api/v1/b2b/keys` с user JWT
- [ ] Backup cron настроен
- [ ] Firewall: открыты только 443 (nginx), 22 (ssh), 15672 (RabbitMQ mgmt, restrict IP)
---
## Ссылки
- [ARCHITECTURE.md](ARCHITECTURE.md) — полная архитектура, схемы БД, RabbitMQ topology, конфиг
- [TICKETS.md](TICKETS.md) — текущие статусы задач
- [README.md](../README.md) — структура, quick start, API overview

338
docs/IMPLEMENTATION_PLAN.md Normal file
View file

@ -0,0 +1,338 @@
# План реализации «Контракт-чек» (Ollama Cloud / hosted LLM)
> **Примечание:** этот документ описывает исходную «ленивую» архитектуру с arq+Redis
> (очередь), Selectel S3 (хранилище) и единым Docker-образом с `MODE=api|worker|bot`.
> **Этап 0 (прототип) актуален.** Этапы 1+ superseded [`ARCHITECTURE.md`](ARCHITECTURE.md),
> где реализована production-архитектура: RabbitMQ-конвейер, MinIO, два worker-а,
> aiogram-бот, B2B API, 5 Dockerfile-ов в `srv/`, dependency-groups (PEP 735).
> Принцип: ленивая архитектура. Каждый этап — минимальный работающий
> срез. Никаких абстракций «на потом». K8s, RabbitMQ, микросервисы —
> не нужны до $10k MRR.
>
> **LLM:** [Ollama Cloud](https://ollama.com/cloud) — hosted-инференс
> открытых моделей. Те же модели и API (`/api/chat`, `format: json`),
> что и у локального Ollama, но без своего железа: платим подписку +
> metered overage, инференс бежит у Ollama (NCP-партнёры, в основном США).
---
## О выборе Ollama Cloud
- **API:** стандартный Ollama HTTP API на cloud-эндпоинте. Авторизация —
API-ключ (bearer) из настроек аккаунта Ollama. Структурный вывод через
`format: json` (или JSON-schema) для парсимого отчёта.
- **Модели:** только cloud-enabled (см. `ollama.com/search?c=cloud`).
Рекомендация — **`qwen2.5:14b`** (сильный русский + reasoning).
Fallback — более лёгкая **`qwen2.5:7b`** при 429/квоте.
Теги БЕЗ суффикса `-instruct` (его нет в Ollama): `qwen2.5:14b`,
`llama3.1:8b`, `gemma2:9b` и т.п.
- **`format: json` ≠ соответствие схеме.** Ollama гарантирует синтаксис
JSON; соответствие вашей pydantic-схеме проверяем на клиенте + цикл
repair/retry. Не полагаемся на модель.
## Стоимость и лимиты Ollama Cloud (актуально на момент планирования)
| План | Цена | Конкарренси | Назначение | Статус |
|-------|-------------|-------------|-------------------------|------------------|
| Free | $0 | 1 | тесты | открыт |
| Pro | $20/мес | 3 | рабочая лошадка MVP | открыт |
| Max | $100/мес | 10 | тяжёлый поток | **приостановлен** |
| Team | $25/seat | — | команда (5 seat min) | waitlist |
| Enterprise | custom | custom | продакшн SaaS | по запросу |
- **Usage:** rolling-лимиты — сессия 5 ч + недельный 7 дн. Pro = 50× Free.
При превышении — **metered overage** (добиваем баланс, оплата по
токенам модели). Жёсткой стены нет, но конкарренси = 3 на Pro.
- **Хостинг: в основном США** (подключаются EU/Singapore). Это **ломает
152-ФЗ data-residency**, см. раздел «Риски».
## Экономика проекта (честно)
- Опекс фиксированный до квоты: VPS ~4 €/мес + Ollama Cloud Pro $20/мес +
Selectel S3 (копейки). Итого ~$25–30/мес на старте.
- Рост объёма → metered overage Ollama Cloud по токенам. Маржа на
pay-per-doc (199 ₽) остаётся, **если** один договор съедает мало
квоты. Это **обязательно измерить на этапе 0** (квота/документ).
- В отличие от локального Ollama — нет потолка по железу, но есть потолок
по конкарренси (3) и плавающий cost при overage. В отличие от GigaChat —
cost в $, не в ₽, и нет РФ-локализации данных.
---
## Архитектура — hexagonal (ports & adapters)
Ядро и адаптеры разделены. Это снимает связность «бот знает про БД/S3/LLM» и
позволяет добавлять каналы доставки (CLI, веб, B2B API) без дублирования
бизнес-логики.
- **Ядро (application core):**
- `api` (FastAPI/uvicorn) — владеет БД, S3, кредитами, оплатой; принимает
документы, **резервирует кредит на enqueue**, ставит arq-задачи в redis,
отдаёт отчёты/профиль. Аутентификация адаптеров — общий `SERVICE_TOKEN`.
- `worker` (arq) — разбирает очередь: S3 → экстракт → (OCR) → chunker →
analyzer → Ollama Cloud → Report. Idempotency + refund при ошибке.
- Общий доменный пакет `contract_check/` (`extractor`, `chunker`,
`llm_client`, `analyzer`, `checklist`, `report_schema`, `ocr`, `storage`,
`db`, `models`, `payments`, `quota`) — используется только ядром.
- **Адаптеры (delivery = HTTP-клиенты к `api`):**
- `bot` (aiogram) — Telegram: приём документа → `POST /documents` (multipart)
→ отдача отчёта. **Не трогает БД/S3/redis/LLM.**
- `cli` — пользовательский/отладочный CLI поверх api (новый, отдельный от
прототипа).
- `web` (React SPA) — этап 2, тоже HTTP-клиент к api.
- **Прототип `prototype.py` (stage 0)** — standalone, in-process LLM; артефакт
go/no-go, **не адаптер и не ядро**.
> Следствие для планирования: **api + worker строятся на этапе 1** (боту-адаптеру
> нужен api). Этап 2 = добавление адаптера `web` + подписок (api уже есть).
---
## Этап 0 — Прототип (1 выходной)
**Цель:** доказать, что модель через Ollama Cloud реально находит риски,
и измерить задержку + **квоту на один договор** (для юнит-экономики).
**Деливерэбл:** один файл `prototype.py` — end-to-end.
```
PDF/DOCX → pymupdf/python-docx → текст →
prompt с чек-листом → Ollama Cloud (format: json) → отчёт в markdown
```
**Что делаем:**
1. Завести аккаунт Ollama, взять API-ключ, положить в `.env`.
2. Поставить `pymupdf`, `httpx` в venv.
3. Один скрипт: читает PDF → достаёт текст → промпт с чек-листом из 10
пунктов → дёргает Ollama Cloud → печатает отчёт.
4. Прогнать 3–5 реальных договоров (NDA, оказание услуг, поставка).
5. Зафиксировать: качество находок, задержка (сек), % битых JSON,
**сколько квоты/токенов на договор**.
**Чего НЕ делаем:** база, веб, бот, оплату, деплой. Один скрипт локально.
**Критерий успеха:** отчёт по реальному договору содержит хотя бы 3
осмысленные находки с цитатами; задержка < 60 сек; квота/договор даёт
понятную маржинальность при 199 ₽.
---
## Этап 1 — Telegram-бот MVP (2–3 недели)
**Цель:** первые платящие пользователи. Бот = самый быстрый путь до ЦА.
### 1.1 Структура проекта
```
contract_check/
├── pyproject.toml # hatchling, deps (core + adapters)
├── docker-compose.yml # postgres, redis, api, worker, bot (+ profile app = prototype)
├── .env.example
├── src/contract_check/
│ ├── main.py # диспетчер: MODE=api|worker|bot|cli
│ ├── prototype.py # stage-0 standalone (in-process LLM) — НЕ в hexagonal
│ │ # ── ЯДРО (application core): владеет БД/S3/кредитами/LLM ──
│ ├── api.py # FastAPI: /documents, /reports/{id}, /me, /healthz
│ ├── worker.py # arq analyze_document(doc_id): S3→extract→OCR→analyze→Report
│ ├── analyzer.py extractor.py chunker.py ocr.py
│ ├── llm_client.py checklist.py report_schema.py
│ ├── storage.py db.py models.py payments.py quota.py
│ │ # ── АДАПТЕРЫ (delivery): HTTP-клиенты к api, без БД/S3/LLM ──
│ ├── bot.py # aiogram: приём документа → POST /documents → отчёт
│ └── cli.py # CLI поверх api (новый, не прототип)
├── migrations/ # alembic
└── tests/
```
### 1.2 База данных — 3 таблицы
```sql
users (id, telegram_id, created_at, credits_left)
documents (id, user_id, s3_key, status, created_at)
reports (id, document_id, content_json, created_at)
```
`credits_left` — prepaid-кредиты (pay-per-doc). Подписок в v1 нет.
`status` хранить как `TEXT` + `CHECK`, не Postgres-ENUM (миграции проще).
### 1.3 Поток (резервирование кредита на enqueue)
```
Юзер кидает PDF в бот
→ бот (adapter): multipart'ом шлёт файл в api: POST /documents (+ SERVICE_TOKEN)
→ api (core): get_or_create_user, проверяет credits_left > 0, РЕЗЕРВИРУЕТ (-= 1),
грузит файл в S3, создаёт Document(status=queued), ставит arq-таску → 202 + job_id
→ worker (core): idempotency guard по status, достаёт из S3 → текст → (OCR если скан)
→ worker: chunker при необходимости → analyzer → Ollama Cloud → отчёт
→ worker: pydantic-валидация + repair-loop; пишет Report, status=done
→ при ошибке: status=failed, ВОЗВРАТ кредита (+1)
→ бот: опрашивает GET /reports/{id} (или push), отправляет отчёт
(с разбивкой/файлом при >4096 симв.), футер-disclaimer
```
### 1.4 Оплата (минимум)
- ЮKassa: бот генерирует ссылку на оплату N ₽ → webhook пополняет
`credits_left`.
- Один тариф: 199 ₽ = 1 документ. Без подписок.
### 1.5 Деплой
- **Один VPS** (Hetzner CX22 ~4 €/мес или Selectel под РФ-локацию БД).
- Docker Compose: `postgres`, `redis`, `api`, `worker`, `bot`. **Без GPU.**
(Бот — адаптер к api; веб-адаптер `web` добавится на этапе 2.)
- S3 — Selectel Object Storage.
- Домен + HTTPS только для webhook ЮKassa; на старте ngrok (с оговоркой:
URL на free-ngrok меняется → лучше сразу дешёвый домен + Caddy).
### 1.6 Критерий успеха
10 платящих. MRR ~2 000 ₽. Отчёты не вызывают жалоб «ничего не нашёл».
Ollama Cloud Pro покрывает нагрузку без ухода в overage.
---
## Этап 2 — Веб + подписки (3–4 недели)
**Цель:** B2B-веб-интерфейс и подписки.
### 2.1 Что добавляем
- **`api` уже построен на этапе 1** (T-E1-015). На этапе 2 добавляем: роуты
подписок/счетов, Telegram Login auth (сессия/JWT) для веб-адаптера, и сам
веб-адаптер `web` (React SPA) — ещё один HTTP-клиент к api.
- React SPA (Vite): загрузка, история, профиль, подписка. Отдаётся статикой
позже (отдельный `web`-контейнер / Nginx).
- Auth: Telegram Login Widget (нужен публичный HTTPS-домен в BotFather) →
сессия/JWT.
- Подписки: solo (1 490 ₽), team (3 990 ₽). ЮKassa recurring.
### 2.2 Что меняем в базе
```sql
ALTER TABLE users ADD COLUMN plan TEXT DEFAULT 'free';
ALTER TABLE users ADD COLUMN plan_renews_at TIMESTAMP;
CREATE TABLE invoices (id, user_id, amount, status, provider, external_id, created_at);
```
`credits_left` остаётся для pay-per-doc; подписка = безлимит с monthly
reset через cron-arq-таску.
### 2.3 Архитектура — без изменений в ядре
`api` и `worker` работают с этапа 1; на этапе 2 добавляется только адаптер
`web` (React, статика) перед Nginx → HTTPS (Let's Encrypt). Hexagonal-граница
сохранена: `web` — такой же HTTP-клиент к api, как `bot`. LLM по-прежнему
Ollama Cloud; при росте — Enterprise тариф или переход на свой GPU.
### 2.4 Критерий успеха
50 платящих. MRR ~30 000 ₽. Есть хотя бы один team-клиент.
---
## Этап 3 — B2B API (2 недели, только если есть спрос)
**Цель:** сторонние сервисы дёргают анализ через API.
### 3.1 Что добавляем
- API-ключи: таблица `api_keys`, header `X-API-Key`.
- Rate-limit: Redis (token bucket). Лимит = **конкарренси/квота Ollama
Cloud** (3 на Pro), а не ₽.
- `POST /api/v1/analyze` (multipart) → 202 + job_id →
`GET /api/v1/reports/{id}`.
- Дашборд: ключи, usage, счета.
### 3.2 Чего НЕ делаем
- Нет SDK, нет вебхуков (клиент поллит), нет OAuth2.
- Один тариф API: 9 900 ₽/мес за 100 запросов.
### 3.3 Критерий успеха
3 API-клиента. MRR +30 000 ₽.
---
## Инфраструктура — сводка
| Компонент | Этап 0 | Этап 1 | Этап 2–3 |
|----------------|------------|------------------------------|--------------------------------|
| Compute | ноутбук | 1 VPS (Docker Compose) | 1 VPS (апгрейд RAM) |
| LLM | Ollama Cloud (Free/Pro) | Ollama Cloud Pro | Ollama Cloud Pro/Enterprise |
| DB | — | Postgres (в compose) | Postgres (+ backup cron) |
| Queue | — | Redis (в compose) | Redis (тот же) |
| Object storage | — | Selectel S3 | Selectel S3 |
| OCR | — | Tesseract (локально) | Tesseract (+ Yandex Vision) |
| Payments | — | ЮKassa | ЮKassa + CloudPayments |
| Monitoring | — | Docker logs | Uptime Kuma + Sentry (free) |
| CI/CD | — | git push → ssh deploy | GitHub Actions → build → deploy|
**K8s / RabbitMQ / Kafka / Elasticsearch / vLLM / TGI — НЕ НУЖНЫ.** Один
VPS + Ollama Cloud держит всё до заметного объёма. Свой GPU — только если
Ollama Cloud overage станет дороже self-host (отдельное решение позже).
---
## Чек-лист пунктов анализа (v1)
Содержимое `checklist.py` — один список, без БД:
1. Неустойки / штрафы (размер, односторонний)
2. Подсудность (чужой регион)
3. Сроки оплаты (условия, просрочка)
4. IP-права (кому отходят результаты)
5. Одностороннее изменение условий
6. Гарантии и их срок
7. Форс-мажор (формулировки)
8. НДС (включён / сверх)
9. Ответственность сторон (cap, исключения)
10. Расторжение (условия, уведомление)
---
## Сроки (реалистично, соло, вечера/выходные)
| Этап | Что | Время |
|------|------------------|-----------|
| 0 | Прототип | 1 выходной|
| 1 | Telegram-бот MVP | 2–3 недели|
| 2 | Веб + подписки | 3–4 недели|
| 3 | B2B API | 2 недели |
**До первого платящего (0+1): ~3–4 недели.** Заложить ~1–2 дня на подбор
cloud-модели и тюнинг промпта (локальные/open модели капризнее GPT-4o).
---
## Риски Ollama Cloud (честно)
1. **152-ФЗ / data-residency.** Контракт улетает в Ollama Cloud (США).
Это тот же класс риска, что и GPT-4o. Митигация: disclaimer,
обезличивание ПДн перед отправкой, либо при необходимости —
отказ от Ollama Cloud в пользу РФ-LLM (GigaChat/YandexGPT) или своего
GPU-бокса. **Не позиционировать продукт как «данные не покидают РФ».**
2. **Квота/конкарренси.** Pro = 3 одновременных модели + rolling usage.
Бурст платящих юзеров упрётся в очередь/429. Трекать usage,
алертить у лимита, при росте — overage-баланс или Enterprise.
3. **Качество open-моделей.** 14B галлюцинирует/пересказывает цитаты
сильнее GPT-4o/GigaChat. Всегда: цитата + номер пункта + ремонт-цикл
валидации JSON. Disclaimer «не заменяет юриста» — в каждый отчёт.
4. **Зависимость от одного провайдера.** Один аккаунт Ollama (нельзя
несколько). Иметь готовый план Б: GigaChat/YandexGPT-фолбэк или свой
GPU при блокировке/превышении квоты.
---
## Что сознательно отложено (YAGNI)
- Multi-tenant / организации / роли — пока все юзеры = solo.
- Шаблоны договоров (генерация) — другой продукт.
- ЭЦП / Госуслуги — чужой регуляторный ад.
- Команда юристов (human-in-the-loop) — только если попросят.
- Mobile app — веб + бот закрывают 100%.
- White-label — один продукт, один бренд.
- Свой GPU / vLLM / TGI / RAG-над-векторной-базой — пока Ollama Cloud
дешевле; пересмотрим при выходе overage в минус.

216
docs/TICKETS.md Normal file
View file

@ -0,0 +1,216 @@
# Тикеты реализации «Контракт-чек» (production refactor)
> **Актуальная архитектура:** `ARCHITECTURE.md` (supersedes `IMPLEMENTATION_PLAN.md` для этапов 1+).
> Состояние кода на момент синхронизации: реализованы `api/` (вкл. B2B-роуты), `core/`,
> `worker_extract/`, `worker_analyze/`, `bot/`, `prototype/`; все 5 Dockerfile-ов в `srv/`;
> `docker-compose.yml` полностью разводит профиль `services`; миграции `0001_initial` + `0002_api_keys`;
> unit-тесты зелёные. DoD: `ruff check .`, `mypy src`, `pytest` зелёные; `.env.example` актуален.
>
> Статусы: `todo` / `in_progress` / `done` / `blocked`.
---
## Аудит реализации (текущее состояние)
> Проверено: код собирается (`pyproject.toml` + `uv.lock`), миграции накатываются,
> `api/` стартует, unit-тесты проходят (58 тестов зелёные).
| Компонент | Статус | Доказательство / пробел |
|-----------|--------|---------------------------|
| Stage 0 prototype | done | `src/contract_check/prototype/` работает; `tests/unit/test_checklist_report.py`, `test_chunker.py`, `test_extractor.py` зелёные. |
| `core/` — общий домен | done | `db/`, `mq/`, `s3/`, `llm/`, `analysis/`, `credits.py`, `tokens.py`, `api_keys.py`, `rate_limit.py`, `redis_client.py`, `config.py`, `logging.py`, `metrics.py`, `telemetry.py`, `sentry.py`. |
| Миграции / БД | done | `0001_initial.py` (6 таблиц) + `0002_api_keys.py` (`api_keys`, `api_key_requests`). |
| `api/` — FastAPI ядро | done | `POST /api/v1/documents`, `GET /api/v1/reports/{id}`, `GET /api/v1/documents/{id}`, `GET /api/v1/me`, `/healthz`, `/readyz`, `/metrics`; user JWT auth via `/api/v1/auth/telegram/*`; reserve-on-enqueue (`services.py`). |
| `worker_extract/` | done | `consumer.py`/`handler.py`/`extract_document.py`/`__main__.py`: consume `DocumentUploaded` → MinIO dl → extract/OCR → `.txt` upload → publish `DocumentExtracted`; failure-class + refund. |
| `worker_analyze/` | done | `consumer.py`/`handler.py`/`__main__.py`: consume `DocumentExtracted` → LLM → Report → `status=done`; refund-on-DLQ по политике. |
| `bot/` — Telegram adapter | done | `client.py`/`config.py`/`handlers.py`/`__main__.py`: `/start`, upload→`POST /documents`, poll→deliver; граница импортов проверяется `tests/unit/test_bot_boundary.py`. |
| Dockerfile-ы | done | `srv/{api,worker-extract,worker-analyze,bot,prototype}/Dockerfile` — все 5 (deps-группы PEP 735 заточены на сервис). |
| Docker Compose | done | Инфра (default) + профиль `services` (api, worker-extract, worker-analyze, bot) с `depends_on: service_healthy`. Профили `obs`/`edge` — позже. |
| Observability / edge | todo | Пром/Grafana/Tempo/OTel-collector/Nginx/certbot — не развёрнуты (нет `deploy/`). |
| Stage 2 — веб + подписки | todo | React SPA, Telegram Login, recurring ЮKassa — не начаты. |
| Stage 3 — B2B API | done | `api/routes/b2b.py`, `core/api_keys.py`, `core/rate_limit.py`, `core/redis_client.py`, миграция `0002_api_keys.py`, `tests/integration/test_b2b_api.py`, `tests/unit/test_rate_limit.py`; `X-API-Key` auth + token-bucket rate-limit. |
---
## Этап 0 — Прототип (go/no-go)
### T-E0-001 — Аккаунт Ollama Cloud + каркас прототипа
**Статус:** done · **Оценка:** S
`.env.example` содержит `OLLAMA_HOST`/`OLLAMA_API_KEY`/`OLLAMA_MODEL=qwen2.5:14b`/`OLLAMA_FALLBACK_MODEL=qwen2.5:7b`.
`python -m contract_check --help` работает. Теги без `-instruct`.
### T-E0-002 — Экстрактор текста (PDF/DOCX)
**Статус:** done · **Оценка:** S
`src/contract_check/core/analysis/extractor.py`; покрыт `tests/unit/test_extractor.py`.
### T-E0-003 — Чек-лист анализа (v1)
**Статус:** done · **Оценка:** S
`src/contract_check/core/analysis/checklist.py` — 10 пунктов с `id/title/description`.
### T-E0-004 — Ollama Cloud-клиент + repair-loop
**Статус:** done · **Оценка:** M
`src/contract_check/core/llm/ollama_cloud.py`; мок-тесты 200/429-fallback/битый-JSON в `tests/unit/test_llm_ollama_cloud.py`.
### T-E0-005 — Промпт + JSON-отчёт (с чанками)
**Статус:** done · **Оценка:** M
`core/analysis/analyzer.py`, `chunker.py`, `report_schema.py`; disclaimer в markdown.
### T-E0-006 — Замеры метрик и economics go/no-go
**Статус:** todo · **Оценка:** M
Требует живых прогонов 3–5 договоров через Ollama Cloud + `docs/stage0-results.md`.
---
## Этап 1 — Production refactor (api + core + workers + bot)
> Архитектура: hexagonal, RabbitMQ pipeline (`extract.q` → `analyze.q`), MinIO, Postgres, Redis.
> Профили Docker Compose: default = инфра; `services` = api + workers + bot.
### T-E1-001 — Ядро `contract_check.core`
**Статус:** done · **Оценка:** L
Пакет `src/contract_check/core/` содержит `db/`, `mq/`, `s3/`, `llm/`, `analysis/`, `credits.py`, `tokens.py`, `config.py`, `logging.py`, `metrics.py`, `telemetry.py`, `sentry.py`.
### T-E1-002 — Миграции БД (6 таблиц)
**Статус:** done · **Оценка:** M
`migrations/versions/0001_initial.py`: `users`, `documents`, `reports`, `jobs`, `service_tokens`, `invoices` (stub).
### T-E1-003 — FastAPI ядро: документы, кредиты, enqueue, отчёты
**Статус:** done · **Оценка:** L
`src/contract_check/api/`: `POST /api/v1/documents`, `GET /api/v1/reports/{id}`, `GET /api/v1/me`, `/healthz`, `/metrics`.
Reserve-on-enqueue (`core/credits.py`) и user JWT auth (`core/auth.py`, `api/routes/auth.py`).
### T-E1-004 — Worker-extract (CPU: pymupdf + tesseract)
**Статус:** done · **Оценка:** M · **Зависимости:** T-E1-001, T-E1-002
`src/contract_check/worker_extract/` (`consumer.py`/`handler.py`/`extract_document.py`/`__main__.py`):
consume `DocumentUploaded` из `extract.q` → скачать blob из MinIO →
`extractor` + `ocr.ocr_pdf()` → загрузить `.txt` → publish `DocumentExtracted` в `analyze.q`.
Failure-classification + refund-on-DLQ. Образ `srv/worker-extract/Dockerfile`.
### T-E1-005 — Worker-analyze (I/O: LLM provider)
**Статус:** done · **Оценка:** M · **Зависимости:** T-E1-001, T-E1-002, T-E1-004
`src/contract_check/worker_analyze/` (`consumer.py`/`handler.py`/`__main__.py`):
consume `DocumentExtracted` из `analyze.q` → скачать `.txt` → chunk → LLM (через `core/llm` port)
→ validate/repair → сохранить `Report`, `status=done`; refund при terminal failure (`core/credits.py`).
Образ `srv/worker-analyze/Dockerfile`.
### T-E1-006 — Telegram-бот (aiogram 3) — HTTP-адаптер
**Статус:** done · **Оценка:** M · **Зависимости:** T-E1-003
`src/contract_check/bot/` (`client.py`/`config.py`/`handlers.py`/`__main__.py`):
`/start`, приём PDF/DOCX → `POST /api/v1/documents`, poll `GET /api/v1/reports/{id}`
→ отправка отчёта (разбивка/файл >4096 симв.), реакция на 402/400/202.
Образ `srv/bot/Dockerfile`. **Не импортирует** `core.db`/`core.s3`/`core.llm`/`core.mq`/`core.credits`
(проверяется `tests/unit/test_bot_boundary.py`).
### T-E1-007 — Docker Compose: сервисы и профили
**Статус:** done · **Оценка:** M · **Зависимости:** T-E1-004, T-E1-005, T-E1-006
`docker-compose.yml`: default-профиль = инфра (`postgres`, `redis`, `rabbitmq`, `minio`, `minio-init`);
профиль `services` = `api`, `worker-extract`, `worker-analyze`, `bot` с `depends_on: service_healthy`.
Все 5 Dockerfile-ов в `srv/`. Профили `obs`/`edge` — позже (T-E1-009/T-E1-010).
### T-E1-008 — Оплата (ЮKassa) и пополнение кредитов
**Статус:** todo · **Оценка:** M · **Зависимости:** T-E1-003, T-E1-006
Роут для создания платежа 199 ₽ = 1 документ; webhook `succeeded` → `credits_left += 1`.
### T-E1-009 — Деплой + edge (Nginx/certbot)
**Статус:** todo · **Оценка:** M · **Зависимости:** T-E1-007
`deploy/nginx/nginx.conf`, `deploy/nginx/certbot-init.sh`, `DEPLOY.md`, `deploy.sh`.
### T-E1-010 — Мониторинг (Prometheus/Grafana/Tempo/Sentry)
**Статус:** todo · **Оценка:** M · **Зависимости:** T-E1-007
`deploy/observability/` + compose профиль `obs`. Дашборды: queue depth, job latency, LLM tokens, credits.
---
## Этап 2 — Веб + подписки
### T-E2-001 — Миграция БД: планы и счета
**Статус:** todo · **Оценка:** M · **Зависимости:** T-E1-002
`users.plan` (`free|solo|team`), `users.plan_renews_at`. Таблица `invoices` уже существует (stub); наполнить логикой.
### T-E2-002 — Telegram Login Widget auth
**Статус:** done · **Оценка:** M · **Зависимости:** T-E1-003
Реализовано в `core/auth.py` + `api/routes/auth.py`: `/api/v1/auth/telegram/web` и `/api/v1/auth/telegram/miniapp` проверяют HMAC-подпись Telegram и выдают тот же user JWT, что и бот.
### T-E2-003 — React SPA (Vite)
**Статус:** todo · **Оценка:** L · **Зависимости:** T-E2-002
Загрузка, история, профиль, подписка. Сборка в статику; `api` отдаёт `index.html` на `/`.
### T-E2-004 — Подписки ЮKassa recurring
**Статус:** todo · **Оценка:** M · **Зависимости:** T-E2-001, T-E2-003
Тарифы solo (1 490 ₽), team (3 990 ₽). Ежемесячное продление/reset.
---
## Этап 3 — B2B API
> **Примечание:** в текущей архитектуре B2B API строится **не после веба**, а поверх уже готового `api` и существующей очереди.
> Зависимость от T-E2-002 снята.
### T-E3-001 — Миграция БД: `api_keys` и `api_key_requests`
**Статус:** done · **Оценка:** M · **Зависимости:** T-E1-002
`migrations/versions/0002_api_keys.py`: таблицы
- `api_keys(id, user_id FK, name, key_hash, rate_limit_rps, monthly_quota, monthly_used, resets_at, revoked, created_at, last_used_at)`
- `api_key_requests(id, api_key_id FK, document_id FK, created_at)`
Накатывается/откатывается чисто; SQLAlchemy-модели в `core/db/models.py`.
### T-E3-002 — Redis-клиент и token-bucket rate limiter
**Статус:** done · **Оценка:** M · **Зависимости:** T-E3-001
`core/redis_client.py` (async Redis из `redis_url`), `core/rate_limit.py` (token bucket per `api_key_id`,
с `MemoryRateLimiter`-фолбэком для unit-тестов без Redis). Лимит по умолчанию = `B2B_DEFAULT_RATE_LIMIT_RPS` (3),
переопределяется `api_keys.rate_limit_rps`. Покрыт `tests/unit/test_rate_limit.py`; 429 при превышении.
### T-E3-003 — `X-API-Key` auth dependency
**Статус:** done · **Оценка:** M · **Зависимости:** T-E3-001, T-E3-002
`api/deps.py`: `require_api_key` — проверяет `X-API-Key` по `key_hash` (`core/api_keys.py`),
отклоняет revoked, обновляет `last_used_at`, применяет rate-limit. `api/routes/b2b.py`:
`POST /api/v1/analyze`, `GET /api/v1/b2b/reports/{document_id}`, `GET /api/v1/b2b/usage`.
Покрыт `tests/integration/test_b2b_api.py` (вкл. 401/429).
### T-E3-004 — Управление API-ключами (user JWT auth)
**Статус:** done · **Оценка:** S · **Зависимости:** T-E3-001
`api/routes/b2b.py` под user JWT auth: `POST /api/v1/b2b/keys` (plaintext ключ возвращается
**только один раз**), `GET /api/v1/b2b/keys`, `POST /api/v1/b2b/keys/{id}/revoke`,
`GET /api/v1/b2b/keys/{id}/usage`. Revoke мгновенно отключает аутентификацию.
### T-E3-005 — Интеграционные тесты B2B API
**Статус:** done · **Оценка:** M · **Зависимости:** T-E3-003
`tests/integration/test_b2b_api.py`: upload → 202 → poll report; ветки 401/429.
Seed API key в `tests/integration/conftest.py`.
---
## Сводка порядка (обновлённая)
1. **E0** — прототип (done, кроме живых замеров T-E0-006).
2. **E1.001–007** — ядро + БД + FastAPI `api` + worker-extract + worker-analyze + bot + Docker/compose (done).
3. **E3** — B2B API (done: миграция, rate-limit, auth, роуты, тесты).
4. **E1.008** — оплата ЮKassa (todo — блокирует первых платящих).
5. **E1.009–010** — деплой/edge (Nginx/certbot) и observability (Prometheus/Grafana/Tempo/Sentry) (todo).
6. **E2** — веб + подписки (todo; после 10–50 платящих).
**Ближайшие работы:** E1.008 (оплата), затем E1.009–010 (edge + observability), затем E2 (веб + подписки).

66
migrations/env.py Normal file
View file

@ -0,0 +1,66 @@
"""Alembic env — async engine, target metadata from core.db.models.
URL comes from `DATABASE_URL` (via core.config.Settings), overriding the
placeholder in alembic.ini. Importing `contract_check.core.db.models` populates
`Base.metadata`; autogenerate is used only to draft, migrations are committed
hand-written (docs/ARCHITECTURE.md §7).
"""
from __future__ import annotations
import asyncio
import sys
from logging.config import fileConfig
from pathlib import Path
from alembic import context
from sqlalchemy import pool
from sqlalchemy.engine import Connection
from sqlalchemy.ext.asyncio import create_async_engine
# ensure src/ is importable when running alembic from the repo root
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
import contract_check.core.db.models # noqa: E402,F401 — populate Base.metadata
from contract_check.core.config import get_settings # noqa: E402
from contract_check.core.db.models import Base # noqa: E402
config = context.config
if config.config_file_name is not None:
fileConfig(config.config_file_name)
target_metadata = Base.metadata
def run_migrations_offline() -> None:
url = get_settings().database_url
context.configure(
url=url,
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
compare_type=True,
)
with context.begin_transaction():
context.run_migrations()
def do_run_migrations(connection: Connection) -> None:
context.configure(connection=connection, target_metadata=target_metadata, compare_type=True)
with context.begin_transaction():
context.run_migrations()
async def run_migrations_online() -> None:
connectable = create_async_engine(get_settings().database_url, poolclass=pool.NullPool)
try:
async with connectable.connect() as connection:
await connection.run_sync(do_run_migrations)
finally:
await connectable.dispose()
if context.is_offline_mode():
run_migrations_offline()
else:
asyncio.run(run_migrations_online())

27
migrations/script.py.mako Normal file
View file

@ -0,0 +1,27 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from __future__ import annotations
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision: str = ${repr(up_revision)}
down_revision: Union[str, None] = ${repr(down_revision)}
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}

234
migrations/versions/0001_initial.py Normal file
View file

@ -0,0 +1,234 @@
"""initial schema: users, documents, reports, jobs, service_tokens, invoices
Revision ID: 0001
Revises:
Create Date: 2026-01-01
Hand-written (docs/ARCHITECTURE.md §7). status/queue/adapter columns are TEXT+CHECK
(additive migrations). UUIDs default to gen_random_uuid() server-side
(built into Postgres 13+; we run pg16).
"""
from __future__ import annotations
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision: str = "0001"
down_revision: str | None = None
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
# ── users ────────────────────────────────────────────────────────────────
op.create_table(
"users",
sa.Column(
"id",
postgresql.UUID(as_uuid=True),
primary_key=True,
server_default=sa.text("gen_random_uuid()"),
),
sa.Column("telegram_id", sa.BigInteger, unique=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.Column("credits_left", sa.Integer, nullable=False, server_default=sa.text("0")),
sa.CheckConstraint("credits_left >= 0", name="users_credits_nonneg"),
)
# ── documents ────────────────────────────────────────────────────────────
op.create_table(
"documents",
sa.Column(
"id",
postgresql.UUID(as_uuid=True),
primary_key=True,
server_default=sa.text("gen_random_uuid()"),
),
sa.Column(
"user_id",
postgresql.UUID(as_uuid=True),
sa.ForeignKey("users.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column("s3_key", sa.Text, nullable=False),
sa.Column("extracted_s3_key", sa.Text),
sa.Column("filename", sa.Text, nullable=False),
sa.Column("mime", sa.Text, nullable=False),
sa.Column("bytes", sa.BigInteger, nullable=False, server_default=sa.text("0")),
sa.Column("status", sa.String, nullable=False, server_default=sa.text("'queued'")),
sa.Column("stage", sa.Text),
sa.Column("refunded", sa.Boolean, nullable=False, server_default=sa.text("false")),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.CheckConstraint(
"status IN ('queued','extracting','ocr','analyzing','done','failed')",
name="documents_status_check",
),
)
op.create_index("documents_user_created_idx", "documents", ["user_id", sa.text("created_at DESC")])
op.create_index("documents_status_idx", "documents", ["status"])
# ── reports ──────────────────────────────────────────────────────────────
op.create_table(
"reports",
sa.Column(
"id",
postgresql.UUID(as_uuid=True),
primary_key=True,
server_default=sa.text("gen_random_uuid()"),
),
sa.Column(
"document_id",
postgresql.UUID(as_uuid=True),
sa.ForeignKey("documents.id", ondelete="CASCADE"),
nullable=False,
unique=True,
),
sa.Column("content_json", postgresql.JSONB, nullable=False),
sa.Column("markdown", sa.Text, nullable=False),
sa.Column("model_used", sa.Text),
sa.Column("prompt_tokens", sa.Integer, nullable=False, server_default=sa.text("0")),
sa.Column("eval_tokens", sa.Integer, nullable=False, server_default=sa.text("0")),
sa.Column("latency_ms", sa.Integer, nullable=False, server_default=sa.text("0")),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
)
# ── jobs ─────────────────────────────────────────────────────────────────
op.create_table(
"jobs",
sa.Column(
"id",
postgresql.UUID(as_uuid=True),
primary_key=True,
server_default=sa.text("gen_random_uuid()"),
),
sa.Column(
"document_id",
postgresql.UUID(as_uuid=True),
sa.ForeignKey("documents.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column("correlation_id", postgresql.UUID(as_uuid=True), nullable=False),
sa.Column("queue", sa.String, nullable=False),
sa.Column("attempts", sa.Integer, nullable=False, server_default=sa.text("0")),
sa.Column("max_attempts", sa.Integer, nullable=False, server_default=sa.text("5")),
sa.Column("last_failure_class", sa.Text),
sa.Column("last_error", sa.Text),
sa.Column("dlq", sa.Boolean, nullable=False, server_default=sa.text("false")),
sa.Column("status", sa.String, nullable=False, server_default=sa.text("'pending'")),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.CheckConstraint("queue IN ('extract','analyze')", name="jobs_queue_check"),
sa.CheckConstraint(
"status IN ('pending','running','retrying','dlq','done')",
name="jobs_status_check",
),
)
op.create_index("jobs_correlation_idx", "jobs", ["correlation_id"])
op.create_index("jobs_document_idx", "jobs", ["document_id"])
# ── service_tokens ───────────────────────────────────────────────────────
op.create_table(
"service_tokens",
sa.Column(
"id",
postgresql.UUID(as_uuid=True),
primary_key=True,
server_default=sa.text("gen_random_uuid()"),
),
sa.Column("name", sa.Text, unique=True, nullable=False),
sa.Column("token_hash", sa.Text, nullable=False),
sa.Column("adapter", sa.String, nullable=False),
sa.Column("revoked", sa.Boolean, nullable=False, server_default=sa.text("false")),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.Column("last_used_at", sa.DateTime(timezone=True)),
sa.CheckConstraint(
"adapter IN ('bot','web','cli')", name="service_tokens_adapter_check"
),
)
# ── invoices (STUB — no ЮKassa logic this refactor) ─────────────────────
op.create_table(
"invoices",
sa.Column(
"id",
postgresql.UUID(as_uuid=True),
primary_key=True,
server_default=sa.text("gen_random_uuid()"),
),
sa.Column(
"user_id",
postgresql.UUID(as_uuid=True),
sa.ForeignKey("users.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column("amount", sa.Integer, nullable=False),
sa.Column("status", sa.String, nullable=False, server_default=sa.text("'draft'")),
sa.Column("provider", sa.Text),
sa.Column("external_id", sa.Text),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.Column("paid_at", sa.DateTime(timezone=True)),
sa.CheckConstraint(
"status IN ('draft','pending','succeeded','cancelled','refunded')",
name="invoices_status_check",
),
)
op.create_index("invoices_user_idx", "invoices", ["user_id", sa.text("created_at DESC")])
def downgrade() -> None:
op.drop_index("invoices_user_idx", table_name="invoices")
op.drop_table("invoices")
op.drop_table("service_tokens")
op.drop_index("jobs_document_idx", table_name="jobs")
op.drop_index("jobs_correlation_idx", table_name="jobs")
op.drop_table("jobs")
op.drop_table("reports")
op.drop_index("documents_status_idx", table_name="documents")
op.drop_index("documents_user_created_idx", table_name="documents")
op.drop_table("documents")
op.drop_table("users")

111
migrations/versions/0002_api_keys.py Normal file
View file

@ -0,0 +1,111 @@
"""api_keys + api_key_requests tables for B2B API (Stage 3).
Revision ID: 0002
Revises: 0001
Create Date: 2026-08-07
Hand-written (docs/ARCHITECTURE.md §7 + docs/TICKETS.md T-E3-001). status/rate columns are
TEXT+CHECK / INTEGER so later migrations remain additive.
"""
from __future__ import annotations
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision: str = "0002"
down_revision: str | None = "0001"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
# ── api_keys ─────────────────────────────────────────────────────────────
op.create_table(
"api_keys",
sa.Column(
"id",
postgresql.UUID(as_uuid=True),
primary_key=True,
server_default=sa.text("gen_random_uuid()"),
),
sa.Column(
"user_id",
postgresql.UUID(as_uuid=True),
sa.ForeignKey("users.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column("name", sa.Text, nullable=False),
sa.Column("key_hash", sa.Text, nullable=False),
sa.Column("rate_limit_rps", sa.Integer, nullable=False, server_default=sa.text("3")),
sa.Column("monthly_quota", sa.Integer, nullable=True),
sa.Column("monthly_used", sa.Integer, nullable=False, server_default=sa.text("0")),
sa.Column(
"resets_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.text("(now() + interval '1 month')"),
),
sa.Column("revoked", sa.Boolean, nullable=False, server_default=sa.text("false")),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.Column("last_used_at", sa.DateTime(timezone=True)),
sa.CheckConstraint("rate_limit_rps > 0", name="api_keys_rate_limit_positive"),
sa.CheckConstraint("monthly_used >= 0", name="api_keys_monthly_used_nonneg"),
sa.CheckConstraint(
"monthly_quota IS NULL OR monthly_quota >= 0",
name="api_keys_monthly_quota_nonneg",
),
sa.UniqueConstraint("user_id", "name", name="api_keys_user_name_unique"),
)
op.create_index("api_keys_user_idx", "api_keys", ["user_id"])
op.create_index("api_keys_hash_idx", "api_keys", ["key_hash"])
# ── api_key_requests ─────────────────────────────────────────────────────
op.create_table(
"api_key_requests",
sa.Column(
"id",
postgresql.UUID(as_uuid=True),
primary_key=True,
server_default=sa.text("gen_random_uuid()"),
),
sa.Column(
"api_key_id",
postgresql.UUID(as_uuid=True),
sa.ForeignKey("api_keys.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column(
"document_id",
postgresql.UUID(as_uuid=True),
sa.ForeignKey("documents.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
)
op.create_index(
"api_key_requests_key_created_idx",
"api_key_requests",
["api_key_id", sa.text("created_at DESC")],
)
def downgrade() -> None:
op.drop_index("api_key_requests_key_created_idx", table_name="api_key_requests")
op.drop_table("api_key_requests")
op.drop_index("api_keys_hash_idx", table_name="api_keys")
op.drop_index("api_keys_user_idx", table_name="api_keys")
op.drop_table("api_keys")

152
pyproject.toml Normal file
View file

@ -0,0 +1,152 @@
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project]
name = "contract-check"
version = "0.1.0"
description = "AI-скрининг рисков в договорах (PDF/DOCX) для СНГ — ГК РФ / ГК РБ. Event-driven production app."
readme = "README.md"
requires-python = ">=3.13"
license = { text = "Proprietary" }
authors = [{ name = "Контракт-чек" }]
keywords = ["legal", "contracts", "llm", "risk-screening", "fastapi", "rabbitmq"]
# Core deps imported by EVERY service image (including the lean bot adapter).
# Keep this set minimal — service-specific libs go in dependency-groups below
# so each Docker image only installs what it needs.
dependencies = [
"pydantic>=2.7",
"pydantic-settings>=2.3",
"structlog>=24.1",
"python-dotenv>=1.0",
"httpx[http2]>=0.27",
]
[project.scripts]
# Stage-0 benchmark CLI (standalone, kept). Production services have their own
# entrypoints via `python -m contract_check.<service>`.
contract-check = "contract_check.prototype:main"
[tool.hatch.build.targets.wheel]
packages = ["src/contract_check"]
# ─────────────────────────────────────────────────────────────────────────────
# Dependency groups (PEP 735). Docker images: `uv sync --frozen --no-default-groups
# --group <service>`. The project deps above (core) are always installed on top.
#
# api → core + db/mq/s3/obs + fastapi/uvicorn
# worker-extract → core + db/mq/s3/obs + pymupdf/tesseract
# worker-analyze → core + db/mq/s3/obs + otel-httpx
# bot → core + aiogram (leanest image: no db/mq/s3/llm/tesseract)
# prototype → core + pymupdf/python-docx (benchmark CLI)
# dev → everything needed to lint/typecheck/test locally
# ─────────────────────────────────────────────────────────────────────────────
[dependency-groups]
db = [
"sqlalchemy>=2.0",
"asyncpg>=0.29",
"alembic>=1.13",
]
mq = [
"aio-pika>=9.4",
]
s3 = [
"minio>=7.2",
]
obs = [
"prometheus-client>=0.20",
"sentry-sdk>=2",
"opentelemetry-sdk>=1.24",
"opentelemetry-exporter-otlp>=1.24",
]
api = [
{ include-group = "db" },
{ include-group = "mq" },
{ include-group = "s3" },
{ include-group = "obs" },
"fastapi>=0.110",
"uvicorn[standard]>=0.29",
"python-multipart>=0.0.9",
"redis>=5.0",
"pyjwt[crypto]>=2.8",
"opentelemetry-instrumentation-fastapi>=0.45b0",
"opentelemetry-instrumentation-asgi>=0.45b0",
"opentelemetry-instrumentation-httpx>=0.45b0",
]
extract = [
{ include-group = "db" },
{ include-group = "mq" },
{ include-group = "s3" },
{ include-group = "obs" },
"pymupdf>=1.24",
"pytesseract>=0.3.10",
"pillow>=10",
]
analyze = [
{ include-group = "db" },
{ include-group = "mq" },
{ include-group = "s3" },
{ include-group = "obs" },
"opentelemetry-instrumentation-httpx>=0.45b0",
]
bot = [
"aiogram>=3.4",
]
prototype = [
"pymupdf>=1.24",
"python-docx>=1.1",
]
dev = [
{ include-group = "api" },
{ include-group = "extract" },
{ include-group = "analyze" },
{ include-group = "bot" },
{ include-group = "prototype" },
"pytest>=8",
"pytest-asyncio>=0.23",
"respx>=0.21",
"ruff>=0.5",
"mypy>=1.10",
"anyio>=4",
"aiosqlite>=0.20",
"testcontainers[rabbitmq,postgres,minio]>=4",
"asgi-lifespan>=2.1.0",
]
[tool.ruff]
line-length = 100
target-version = "py313"
src = ["src", "tests"]
[tool.ruff.lint]
select = ["E4", "E7", "E9", "F", "I", "UP", "B"]
[tool.mypy]
python_version = "3.13"
strict = true
packages = ["contract_check"]
# Third-party libs without complete type stubs. Our own code stays fully typed.
disallow_untyped_calls = false
[[tool.mypy.overrides]]
module = [
"aio_pika.*",
"minio.*",
"prometheus_client.*",
"sentry_sdk.*",
"opentelemetry.*",
"pymupdf.*",
"docx.*",
"pytesseract.*",
"PIL.*",
]
ignore_missing_imports = true
[tool.pytest.ini_options]
testpaths = ["tests"]
addopts = "-q"
asyncio_mode = "auto"
markers = [
"integration: slow tests needing real Postgres/RabbitMQ/MinIO (deselect with '-m \"not integration\"')",
]

11
src/contract_check/__init__.py Normal file
View file

@ -0,0 +1,11 @@
"""«Контракт-чек» — AI-скрининг рисков в договорах (СНГ / ГК РФ / ГК РБ).
Production refactor (see docs/ARCHITECTURE.md):
- core/ shared domain (config, db, mq, s3, llm, analysis, credits, tokens)
- prototype/ stage-0 standalone benchmark CLI (preserved)
Service packages (api/, worker_extract/, worker_analyze/, bot/) land in
Steps 2–5. Importable from each service's own Docker image.
"""
__version__ = "0.1.0"

6
src/contract_check/__main__.py Normal file
View file

@ -0,0 +1,6 @@
"""Top-level entrypoint: `python -m contract_check <file>` → stage-0 prototype."""
from .prototype import main
if __name__ == "__main__":
main()

69
src/contract_check/api/__main__.py Normal file
View file

@ -0,0 +1,69 @@
"""Entrypoint for the api image.
Default: run uvicorn. Supports a seed-token command for operator bootstrapping:
python -m contract_check.api
python -m contract_check.api seed-token bot-prod bot [optional-token]
"""
from __future__ import annotations
import asyncio
import sys
import uvicorn
from ..core.config import get_settings
from ..core.db.session import create_session_factory
from ..core.tokens import assert_adapter, generate_token, hash_token
from .app import create_app
async def _seed_token(name: str, adapter: str, token: str | None = None) -> None:
assert_adapter(adapter)
raw = token or generate_token()
factory = create_session_factory()
async with factory() as session:
from sqlalchemy import text
await session.execute(
text(
"INSERT INTO service_tokens (name, token_hash, adapter) "
"VALUES (:name, :hash, :adapter) "
"ON CONFLICT (name) DO UPDATE SET "
" token_hash = EXCLUDED.token_hash, "
" revoked = FALSE, "
" adapter = EXCLUDED.adapter"
),
{"name": name, "hash": hash_token(raw), "adapter": adapter},
)
await session.commit()
print(f"Token '{name}' ({adapter}) ready. Bearer:")
print(raw)
if token:
print("(provided by operator)")
else:
print("(generated — copy it now; only the hash is stored)")
def main() -> None:
args = sys.argv[1:]
if args and args[0] == "seed-token":
if len(args) not in (3, 4):
print("Usage: python -m contract_check.api seed-token NAME ADAPTER [TOKEN]")
sys.exit(1)
_, name, adapter, *provided = args
asyncio.run(_seed_token(name, adapter, provided[0] if provided else None))
return
settings = get_settings()
uvicorn.run(
create_app(),
host=settings.api_host,
port=settings.api_port,
log_config=None, # structlog handles logging
)
if __name__ == "__main__":
main()

96
src/contract_check/api/app.py Normal file
View file

@ -0,0 +1,96 @@
"""FastAPI application factory for the «Контракт-чек» API (core service).
The api is the only service that writes to Postgres/MinIO/RabbitMQ for user-
initiated document analysis. Adapters (bot, future web/cli) call it over HTTP.
See docs/ARCHITECTURE.md §15.
"""
from contextlib import asynccontextmanager
from typing import Any
from fastapi import FastAPI
from ..core.config import get_settings
from ..core.llm import port as llm_port # noqa: F401 — package loaded
from ..core.logging import bind_context, configure_logging, get_logger
from ..core.metrics import redis_connected
from ..core.mq.publisher import Publisher
from ..core.rate_limit import MemoryRateLimiter, RateLimiter, RedisRateLimiter
from ..core.redis_client import get_redis_client
from ..core.s3.minio_storage import MinioStorage
from ..core.sentry import init_sentry
from ..core.telemetry import setup_telemetry, shutdown_telemetry
from .middleware import add_middleware
from .routes import auth, b2b, documents, health, me, metrics, reports
@asynccontextmanager
async def lifespan(app: FastAPI) -> Any:
settings = get_settings()
configure_logging(
settings.log_level,
json_output=settings.json_logs,
service="api",
env=settings.env,
)
bind_context(service="api", env=settings.env)
init_sentry("api")
setup_telemetry("api")
storage = MinioStorage.from_endpoint_url(
endpoint_url=settings.s3_endpoint_url,
access_key=settings.s3_access_key,
secret_key=settings.s3_secret_key,
bucket=settings.s3_bucket,
region=settings.s3_region,
)
await storage.ensure_bucket()
publisher = Publisher(settings.rabbitmq_url, origin="api")
await publisher.connect()
# Redis is used for rate-limiting/sessions. In dev/tests without Redis we
# transparently fall back to an in-memory bucket so unit tests stay
# dependency-free.
redis_client = get_redis_client(settings.redis_url)
rate_limiter: RateLimiter
try:
await redis_client.ping()
rate_limiter = RedisRateLimiter(redis_client)
redis_connected.set(1)
except Exception as exc:
log = get_logger(__name__)
log.warning("redis_unavailable", redis_url=settings.redis_url, error=str(exc))
rate_limiter = MemoryRateLimiter()
redis_connected.set(0)
app.state.storage = storage
app.state.publisher = publisher
app.state.rate_limiter = rate_limiter
yield
await publisher.close()
try:
await redis_client.aclose()
except Exception:
pass
shutdown_telemetry()
def create_app() -> FastAPI:
app = FastAPI(
title="Контракт-чек",
version="0.1.0",
description="AI-скрининг рисков в договорах.",
lifespan=lifespan,
)
add_middleware(app)
app.include_router(health.router)
app.include_router(metrics.router)
app.include_router(auth.router)
app.include_router(documents.router)
app.include_router(reports.router)
app.include_router(me.router)
app.include_router(b2b.router)
return app

324
src/contract_check/api/deps.py Normal file
View file

@ -0,0 +1,324 @@
"""FastAPI dependencies (db session, storage, publisher, service-token auth)."""
from __future__ import annotations
from collections.abc import AsyncIterator
from typing import Annotated
from uuid import UUID
from fastapi import Depends, Header, HTTPException, Request
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from ..core.api_keys import hash_api_key
from ..core.auth import AuthError, TokenExpiredError, TokenInvalidError, verify_access_token
from ..core.config import get_settings
from ..core.db.models import User
from ..core.db.session import create_session_factory
from ..core.logging import get_logger
from ..core.mq.publisher import Publisher
from ..core.rate_limit import RateLimiter
from ..core.s3.port import Storage
from ..core.tokens import hash_token
log = get_logger(__name__)
async def get_db_session() -> AsyncIterator[AsyncSession]:
factory = create_session_factory()
async with factory() as session:
yield session
AsyncSessionDep = Annotated[AsyncSession, Depends(get_db_session)]
def get_storage(request: Request) -> Storage:
storage: Storage = request.app.state.storage
return storage
StorageDep = Annotated[Storage, Depends(get_storage)]
def get_publisher(request: Request) -> Publisher:
publisher: Publisher = request.app.state.publisher
return publisher
PublisherDep = Annotated[Publisher, Depends(get_publisher)]
async def require_service_token(
session: AsyncSessionDep,
authorization: Annotated[str | None, Header()] = None,
) -> None:
"""Validate `Authorization: Bearer <token>` against service_tokens table.
Raises 401 on missing/invalid/revoked token. Logs last_used_at on success.
"""
if not authorization or not authorization.lower().startswith("bearer "):
raise HTTPException(status_code=401, detail="Missing or invalid Authorization header")
raw = authorization[7:].strip()
token_hash = hash_token(raw)
result = await session.execute(
text("SELECT id FROM service_tokens WHERE token_hash = :h AND revoked = FALSE"),
{"h": token_hash},
)
row = result.first()
if row is None:
raise HTTPException(status_code=401, detail="Invalid or revoked token")
await session.execute(
text("UPDATE service_tokens SET last_used_at = now() WHERE id = :id"),
{"id": row[0]},
)
await session.commit()
AuthDep = Annotated[None, Depends(require_service_token)]
async def get_or_create_user_for_telegram(session: AsyncSession, telegram_id: int) -> User:
"""Fetch or create a user identified by telegram_id."""
result = await session.execute(
text("SELECT id, telegram_id, created_at, credits_left FROM users WHERE telegram_id = :t"),
{"t": telegram_id},
)
row = result.first()
if row:
return User(id=row[0], telegram_id=row[1], created_at=row[2], credits_left=row[3])
insert = await session.execute(
text(
"INSERT INTO users (telegram_id, credits_left) VALUES (:t, 0) "
"RETURNING id, telegram_id, created_at, credits_left"
),
{"t": telegram_id},
)
new = insert.first()
assert new is not None
await session.commit()
return User(id=new[0], telegram_id=new[1], created_at=new[2], credits_left=new[3])
async def get_or_create_user_by_id(session: AsyncSession, user_id: UUID) -> User | None:
"""Fetch an existing user by UUID. Returns None if not found."""
result = await session.execute(
text("SELECT id, telegram_id, created_at, credits_left FROM users WHERE id = :u"),
{"u": user_id},
)
row = result.first()
if row is None:
return None
return User(id=row[0], telegram_id=row[1], created_at=row[2], credits_left=row[3])
def get_rate_limiter(request: Request) -> RateLimiter:
"""Return the app-state rate limiter (Redis in prod, Memory in tests)."""
limiter: RateLimiter = request.app.state.rate_limiter
return limiter
RateLimiterDep = Annotated[RateLimiter, Depends(get_rate_limiter)]
class ApiKeyAuth:
"""Validated B2B API key + its owning user id."""
def __init__(self, api_key_id: UUID, user_id: UUID, rate_limit_rps: int) -> None:
self.api_key_id = api_key_id
self.user_id = user_id
self.rate_limit_rps = rate_limit_rps
async def _maybe_reset_monthly_quota(session: AsyncSession, api_key_id: UUID) -> None:
"""Reset monthly_used/resets_at if the quota window has expired."""
await session.execute(
text(
"UPDATE api_keys "
"SET monthly_used = 0, resets_at = now() + interval '1 month' "
"WHERE id = :k AND resets_at < now()"
),
{"k": api_key_id},
)
async def require_api_key(
session: AsyncSessionDep,
rate_limiter: RateLimiterDep,
x_api_key: Annotated[str | None, Header(alias="X-API-Key")] = None,
) -> ApiKeyAuth:
"""Validate `X-API-Key` header, rate-limit, and return key metadata.
Raises 401 on missing/invalid/revoked key. Raises 429 when rate-limited
or monthly quota exhausted.
"""
if not x_api_key:
raise HTTPException(status_code=401, detail="Missing X-API-Key header")
key_hash = hash_api_key(x_api_key)
result = await session.execute(
text(
"SELECT id, user_id, rate_limit_rps, monthly_quota, monthly_used, revoked "
"FROM api_keys "
"WHERE key_hash = :h"
),
{"h": key_hash},
)
row = result.first()
if row is None:
raise HTTPException(status_code=401, detail="Invalid API key")
api_key_id, user_id, rate_limit_rps, monthly_quota, monthly_used, revoked = row
if revoked:
raise HTTPException(status_code=401, detail="Revoked API key")
await _maybe_reset_monthly_quota(session, api_key_id)
# Check monthly quota (if configured) after potential reset.
if monthly_quota is not None and int(monthly_used) >= int(monthly_quota):
raise HTTPException(status_code=429, detail="Monthly quota exceeded")
# Apply token-bucket rate limit per key id.
limit = int(rate_limit_rps or get_settings().b2b_default_rate_limit_rps)
rl_result = await rate_limiter.allow(f"rate_limit:{api_key_id}", limit)
if not rl_result.allowed:
retry_after = max(1, int(rl_result.retry_after_sec or 1))
raise HTTPException(
status_code=429,
detail="Rate limit exceeded",
headers={"Retry-After": str(retry_after)},
)
await session.execute(
text("UPDATE api_keys SET last_used_at = now() WHERE id = :id"),
{"id": api_key_id},
)
await session.commit()
return ApiKeyAuth(api_key_id=api_key_id, user_id=user_id, rate_limit_rps=limit)
ApiKeyAuthDep = Annotated[ApiKeyAuth, Depends(require_api_key)]
async def fetch_document_status_for_user(
session: AsyncSession, document_id: UUID, user_id: UUID
) -> dict[str, object] | None:
"""Fetch document + report scoped to a specific user (B2B API)."""
result = await session.execute(
text(
"SELECT d.id, d.status, d.stage, d.filename, d.created_at, "
" r.markdown, r.content_json, r.model_used, "
" r.prompt_tokens, r.eval_tokens, r.latency_ms "
"FROM documents d "
"LEFT JOIN reports r ON r.document_id = d.id "
"WHERE d.id = :d AND d.user_id = :u"
),
{"d": document_id, "u": user_id},
)
row = result.first()
if row is None:
return None
return {
"id": row[0],
"status": row[1],
"stage": row[2],
"filename": row[3],
"created_at": row[4],
"markdown": row[5],
"content_json": row[6],
"model_used": row[7],
"prompt_tokens": row[8],
"eval_tokens": row[9],
"latency_ms": row[10],
}
class CurrentUser:
"""Authenticated user extracted from a Bearer JWT."""
def __init__(self, user_id: UUID, telegram_id: int) -> None:
self.user_id = user_id
self.telegram_id = telegram_id
async def require_current_user(
session: AsyncSessionDep,
authorization: Annotated[str | None, Header()] = None,
) -> CurrentUser:
"""Validate `Authorization: Bearer <user-jwt>` and return the user.
This is the common auth gate for bot, web, and Mini App users.
"""
if not authorization or not authorization.lower().startswith("bearer "):
raise HTTPException(status_code=401, detail="Missing or invalid Authorization header")
token = authorization[7:].strip()
try:
claims = verify_access_token(token)
except TokenExpiredError as exc:
raise HTTPException(status_code=401, detail="Token expired") from exc
except TokenInvalidError as exc:
raise HTTPException(status_code=401, detail="Invalid token") from exc
except AuthError as exc:
raise HTTPException(status_code=401, detail=str(exc)) from exc
# Ensure the user still exists (defense in depth: tokens are stateless,
# but a deleted user should not be able to use them).
result = await session.execute(
text("SELECT id FROM users WHERE id = :u"),
{"u": claims.sub},
)
if result.first() is None:
raise HTTPException(status_code=401, detail="User not found")
return CurrentUser(user_id=claims.sub, telegram_id=claims.telegram_id)
CurrentUserDep = Annotated[CurrentUser, Depends(require_current_user)]
async def fetch_document_status(
session: AsyncSession, document_id: UUID, user_id: UUID
) -> dict[str, object] | None:
result = await session.execute(
text(
"SELECT d.id, d.status, d.stage, d.filename, d.created_at, "
" r.markdown, r.content_json, r.model_used, "
" r.prompt_tokens, r.eval_tokens, r.latency_ms "
"FROM documents d "
"LEFT JOIN reports r ON r.document_id = d.id "
"WHERE d.id = :d AND d.user_id = :u"
),
{"d": document_id, "u": user_id},
)
row = result.first()
if row is None:
return None
return {
"id": row[0],
"status": row[1],
"stage": row[2],
"filename": row[3],
"created_at": row[4],
"markdown": row[5],
"content_json": row[6],
"model_used": row[7],
"prompt_tokens": row[8],
"eval_tokens": row[9],
"latency_ms": row[10],
}
async def get_credits(session: AsyncSession, user_id: UUID) -> int:
result = await session.execute(
text("SELECT credits_left FROM users WHERE id = :u"),
{"u": user_id},
)
row = result.first()
if row is None:
return 0
return int(row[0])

47
src/contract_check/api/middleware.py Normal file
View file

@ -0,0 +1,47 @@
"""FastAPI middleware — correlation_id, request timing/metrics, Sentry errors."""
from __future__ import annotations
import time
from typing import Any
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse
from ..core.logging import get_logger, new_correlation_id, set_correlation_id
from ..core.metrics import http_request_duration
from ..core.sentry import init_sentry
log = get_logger(__name__)
def add_middleware(app: FastAPI) -> None:
@app.middleware("http")
async def _middleware(request: Request, call_next: Any) -> Any:
cid = request.headers.get("x-correlation-id") or new_correlation_id()
set_correlation_id(cid)
start = time.perf_counter()
status = 200
try:
response = await call_next(request)
status = response.status_code
response.headers["x-correlation-id"] = cid
return response
except Exception as exc:
status = 500
log.error("unhandled_exception", path=request.url.path, error=str(exc))
raise
finally:
duration = time.perf_counter() - start
http_request_duration.labels(
method=request.method,
path=request.url.path,
status=str(status),
).observe(duration)
@app.exception_handler(Exception)
async def _exception_handler(_request: Request, exc: Exception) -> JSONResponse:
log.error("exception_handler", error=str(exc))
return JSONResponse(status_code=500, content={"detail": "internal error"})
init_sentry("api")

397
src/contract_check/api/routes/README.md Normal file
View file

@ -0,0 +1,397 @@
# API Routers
FastAPI-роутеры сервиса `contract_check.api`. Все пути — префикс `/api/v1`
(кроме инфра-эндпоинтов `/healthz`, `/readyz`, `/metrics`).
Роутеры регистрируются в приложении через `include_router`; теги соответствуют
атрибуту `tags=` каждого `APIRouter`.
## Содержание
- [Аутентификация](#аутентификация)
- [health](#health)
- [metrics](#metrics)
- [auth](#auth)
- [me](#me)
- [documents](#documents)
- [reports](#reports)
- [b2b](#b2b)
- [Коды ошибок](#коды-ошибок)
---
## Аутентификация
Три независимые схемы, реализованные в [`api/deps.py`](../deps.py):
| Зависимость | Заголовок | Кому выдаётся | Где используется |
| ------------------- | ---------------------------------- | -------------------------------------- | ---------------------------------------- |
| `AuthDep` | `Authorization: Bearer <svc-token>`| Сервисный токен (адаптеры bot/web/cli) | `/auth/telegram/*` — вход от имени бота |
| `CurrentUserDep` | `Authorization: Bearer <user-jwt>` | JWT пользователя (Telegram-логин) | `/me`, `/documents`, `/reports`, `/b2b/keys*` |
| `ApiKeyAuthDep` | `X-API-Key: <raw-key>` | B2B-ключ внешнего клиента | `/analyze`, `/b2b/reports/*`, `/b2b/usage` |
- Сервисный токен и B2B-ключ хешируются (`hash_token`, `hash_api_key`),
в БД хранится только хеш; raw-значение возвращается один раз при создании.
- `ApiKeyAuthDep` дополнительно применяет token-bucket rate limit и
месячный квоты (`429` при превышении).
- `CurrentUserDep` проверяет, что пользователь ещё есть в таблице `users`
(stateless JWT + defence-in-depth).
---
## health
Файл: [`health.py`](health.py). Тег: `health`. Без аутентификации.
### `GET /healthz`
Liveness-проба. Возвращает `200` всегда.
```json
{ "status": "ok" }
```
### `GET /readyz`
Readiness-проба. Выполняет `SELECT 1` в БД.
```json
{ "status": "ok" }
```
Если БД недоступна:
```json
{ "status": "not_ready", "reason": "db: <error>" }
```
---
## metrics
Файл: [`metrics.py`](metrics.py). Тег: `metrics`. Без аутентификации.
### `GET /metrics`
Exposition-эндпоинт Prometheus. Тело — текст в формате
`text/plain; version=0.0.4` (`prometheus_client.generate_latest`).
---
## auth
Файл: [`auth.py`](auth.py). Тег: `auth`.
Три источника идентичности сходятся к одному JWT:
### `POST /api/v1/auth/telegram/bot`
Бот (aiogram) обменивает проверенный `telegram_id` на JWT пользователя.
**Auth:** `AuthDep` (сервисный токен бота).
Тело — `TelegramBotAuthRequest`:
| Поле | Тип | Условие | Описание |
| ------------ | ---- | ------------ | ------------------------------------- |
| `telegram_id`| int | `> 0` | Verified Telegram user id из aiogram |
Ответ `200` — `AuthResponse` (см. ниже).
### `POST /api/v1/auth/telegram/web`
Callback Telegram Login Widget. **Auth:** нет (проверяется подпись Telegram).
Тело — `TelegramWebAuthRequest`:
| Поле | Тип | Условие |
| ----------- | --------- | ------- |
| `id` | int | `> 0` |
| `first_name`| str? | — |
| `last_name` | str? | — |
| `username` | str? | — |
| `photo_url` | str? | — |
| `auth_date` | int | — |
| `hash` | str | — |
`401` при ошибке проверки подписи. Ответ `200` — `AuthResponse`.
### `POST /api/v1/auth/telegram/miniapp`
Mini App initData. **Auth:** нет (проверяется подпись ` initData`).
Тело — `TelegramMiniAppAuthRequest`:
| Поле | Тип | Описание |
| ---------- | --- | ------------------------------------------ |
| `init_data`| str | Raw initData query string из `Telegram.WebApp` |
`401` при ошибке. Ответ `200` — `AuthResponse`.
#### `AuthResponse`
```json
{
"access_token": "<jwt>",
"token_type": "bearer",
"expires_in": 3600,
"user_id": "uuid",
"telegram_id": 123456789
}
```
`expires_in` = `jwt_access_ttl_minutes * 60` из настроек.
### `GET /api/v1/auth/me`
Интроспекция Bearer-JWT. **Auth:** `Authorization: Bearer <jwt>` в заголовке.
Ответ `200` — `TokenIntrospectResponse`:
| Поле | Тип |
| ------------ | --- |
| `sub` | str (UUID) |
| `telegram_id`| int |
| `type` | str |
| `exp` | int (unix) |
`401` — отсутствует/невалиден.
### `GET /api/v1/auth/me/permissions`
Заглушка под будущее RBAC.
```json
{ "permissions": ["upload", "read_reports", "read_me"] }
```
---
## me
Файл: [`me.py`](me.py). Тег: `me`. **Auth:** `CurrentUserDep` (user JWT).
### `GET /api/v1/me`
Профиль пользователя и баланс кредитов.
```json
{ "telegram_id": 123456789, "credits_left": 7 }
```
---
## documents
Файл: [`documents.py`](documents.py). Тег: `documents`.
**Auth:** `CurrentUserDep`.
### `POST /api/v1/documents`
Загрузить документ на анализ. Резервирует кредит, кладёт файл в MinIO,
создаёт строки `documents` + `jobs`, публикует `DocumentUploaded` в RabbitMQ.
Запрос — `multipart/form-data`:
| Поле | Тип | Описание |
| ----- | ----------- | ------------------------------------- |
| `file`| UploadFile | Расширения: `.pdf`, `.docx` (см. `SUPPORTED_SUFFIXES`) |
Ответ `202` (результат [`upload_and_enqueue`](../services.py)):
```json
{
"document_id": "uuid",
"correlation_id": "uuid",
"credits_left": 6
}
```
Ошибки:
- `400` — нет имени / неподдерживаемый формат / пустой файл;
- `402` — нет кредитов (`no credits available`);
- `500` — сбой MinIO/БД/RabbitMQ (кредит возвращается).
---
## reports
Файл: [`reports.py`](reports.py). Тег: `reports`. **Auth:** `CurrentUserDep`.
### `GET /api/v1/reports/{document_id}`
Опрос статуса/результата анализа. Документ обязан принадлежать
аутентифицированному пользователю.
Параметр пути: `document_id` — UUID.
Пока анализ не готов (`status != "done"`):
```json
{
"document_id": "uuid",
"status": "queued | extracting | analyzing | failed",
"stage": "<current stage>"
}
```
Готовый отчёт (`status == "done"`):
```json
{
"document_id": "uuid",
"status": "done",
"filename": "contract.pdf",
"markdown": "<markdown-отчёт>",
"findings": { ... },
"model_used": "<llm model>",
"prompt_tokens": 1234,
"eval_tokens": 5678,
"latency_ms": 91011
}
```
`404` — отчёт не найден (или чужой).
---
## b2b
Файл: [`b2b.py`](b2b.py). Тег: `b2b`.
Анализ-эндпоинты используют **`X-API-Key`** (`ApiKeyAuthDep`).
Эндпоинты управления ключами используют **user JWT** (`CurrentUserDep`),
т.к. вызываются внутренними адаптерами от имени пользователя (`telegram_id`).
### `POST /api/v1/analyze`
Загрузить документ под B2B-ключом. **Auth:** `X-API-Key`.
Запрос — `multipart/form-data` (`file`), как в `POST /api/v1/documents`.
Резервирует кредит у владельца ключа, пишет `api_key_requests`,
инкрементит `api_keys.monthly_used`. Логирование usage — best-effort
(не валит загрузку при сбое записи).
Ответ `202` — как у `POST /api/v1/documents`.
### `GET /api/v1/b2b/reports/{document_id}`
Опрос отчёта, scoped к владельцу ключа. **Auth:** `X-API-Key`.
Параметр пути: `document_id` — UUID.
Формат ответа — идентичен `GET /api/v1/reports/{document_id}`.
`404` — чужой/несуществующий.
### `GET /api/v1/b2b/usage`
Текущее использование ключа. **Auth:** `X-API-Key`.
```json
{
"api_key_id": "uuid",
"rate_limit_rps": 3,
"monthly_quota": 1000,
"monthly_used": 17,
"requests_this_month": 17,
"resets_at": "2026-09-01T00:00:00+00:00"
}
```
### `POST /api/v1/b2b/keys`
Создать новый B2B-ключ. **Auth:** `CurrentUserDep`.
Тело — `CreateApiKeyRequest`:
| Поле | Тип | Default | Описание |
| ---------------- | ----- | ------- | ------------------------------ |
| `name` | str | — | Метка ключа |
| `rate_limit_rps` | int? | `3` | Должно быть `> 0` |
| `monthly_quota` | int? | `null` | Без лимита, если не задан |
Ответ `201` — **raw-ключ возвращается только один раз**:
```json
{
"api_key": "sk_b2b_...",
"id": "uuid",
"name": "prod-webhooks",
"rate_limit_rps": 3,
"monthly_quota": null,
"monthly_used": 0,
"revoked": false,
"created_at": "2026-08-12T..."
}
```
`400` — `rate_limit_rps` не положителен; `500` — сбой БД.
### `GET /api/v1/b2b/keys`
Список ключей пользователя. **Auth:** `CurrentUserDep`.
Ответ — массив объектов формы `ApiKeyResponse`:
| Поле | Тип |
| ---------------- | -------- |
| `id` | UUID |
| `name` | str |
| `rate_limit_rps` | int |
| `monthly_quota` | int? |
| `monthly_used` | int |
| `revoked` | bool |
| `created_at` | ISO-str |
Сортировка — `created_at DESC`. Raw-ключи **не** возвращаются.
### `POST /api/v1/b2b/keys/{key_id}/revoke`
Отозвать ключ. **Auth:** `CurrentUserDep`. Параметр пути: `key_id` — UUID.
```json
{ "id": "uuid", "revoked": true }
```
`404` — ключ не найден, чужой или уже отозван.
### `GET /api/v1/b2b/keys/{key_id}/usage`
Помесячное использование конкретного ключа. **Auth:** `CurrentUserDep`.
```json
{
"key_id": "uuid",
"name": "prod-webhooks",
"rate_limit_rps": 3,
"monthly_quota": 1000,
"monthly_used": 17,
"resets_at": "2026-09-01T...",
"monthly_requests": [
{ "month": "2026-08-01T...", "requests": 17 }
]
}
```
`404` — ключ не найден или чужой.
---
## Коды ошибок
Общий формат ошибки FastAPI:
```json
{ "detail": "<сообщение>" }
```
| Код | Когда |
| --- | ----------------------------------------------------- |
| 400 | Неверный запрос (bad format / пустой файл / bad arg) |
| 401 | Нет/невалиден токен или API-ключ |
| 402 | `no credits available` (при резерве кредита) |
| 404 | Ресурс не найден или чужой |
| 429 | Rate limit / месячный квот B2B-ключа исчерпан |
| 500 | Сбой инфраструктуры (MinIO/PG/RabbitMQ) |
При `429` от rate limiter добавляется заголовок `Retry-After`.

1
src/contract_check/api/routes/__init__.py Normal file
View file

@ -0,0 +1 @@
"""API routes package."""

148
src/contract_check/api/routes/auth.py Normal file
View file

@ -0,0 +1,148 @@
"""Telegram-based authentication endpoints.
Three identity sources converge on the same JWT:
- /auth/telegram/bot — bot adapter exchanges a verified telegram_id for JWT
- /auth/telegram/web — Telegram Login Widget callback
- /auth/telegram/miniapp — Mini App initData
Protected user endpoints receive the JWT via `Authorization: Bearer <jwt>` and use
`deps.CurrentUser`.
"""
from __future__ import annotations
from typing import Annotated, Any
from fastapi import APIRouter, Header, HTTPException, status
from pydantic import BaseModel, Field
from ...core.auth import (
AuthError,
create_access_token,
verify_access_token,
verify_bot_identity,
verify_telegram_miniapp_init_data,
verify_telegram_web_payload,
)
from ...core.config import get_settings
from ...core.db.models import User
from ..deps import AsyncSessionDep, AuthDep, get_or_create_user_for_telegram
router = APIRouter(tags=["auth"])
class TelegramBotAuthRequest(BaseModel):
telegram_id: int = Field(..., gt=0, description="Verified Telegram user id from aiogram")
class TelegramWebAuthRequest(BaseModel):
id: int = Field(..., gt=0)
first_name: str | None = None
last_name: str | None = None
username: str | None = None
photo_url: str | None = None
auth_date: int
hash: str
class TelegramMiniAppAuthRequest(BaseModel):
init_data: str = Field(..., description="Raw initData query string from Telegram.WebApp")
class AuthResponse(BaseModel):
access_token: str
token_type: str = "bearer"
expires_in: int
user_id: str
telegram_id: int
class TokenIntrospectResponse(BaseModel):
sub: str
telegram_id: int
type: str
exp: int
def _issue_token(user: User) -> AuthResponse:
token = create_access_token(user.id, user.telegram_id or 0)
ttl = get_settings().jwt_access_ttl_minutes * 60
return AuthResponse(
access_token=token,
token_type="bearer",
expires_in=ttl,
user_id=str(user.id),
telegram_id=user.telegram_id or 0,
)
@router.post("/api/v1/auth/telegram/bot", status_code=status.HTTP_200_OK)
async def auth_telegram_bot(
auth: AuthDep,
session: AsyncSessionDep,
body: TelegramBotAuthRequest,
) -> AuthResponse:
"""Exchange a verified telegram_id (from the bot) for a user JWT."""
identity = verify_bot_identity(body.telegram_id)
user = await get_or_create_user_for_telegram(session, identity.telegram_id)
return _issue_token(user)
@router.post("/api/v1/auth/telegram/web", status_code=status.HTTP_200_OK)
async def auth_telegram_web(
session: AsyncSessionDep,
body: TelegramWebAuthRequest,
) -> AuthResponse:
"""Verify Telegram Login Widget payload and issue a user JWT."""
bot_token = get_settings().telegram_bot_token
try:
identity = verify_telegram_web_payload(body.model_dump(), bot_token)
except AuthError as exc:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=str(exc)) from exc
user = await get_or_create_user_for_telegram(session, identity.telegram_id)
return _issue_token(user)
@router.post("/api/v1/auth/telegram/miniapp", status_code=status.HTTP_200_OK)
async def auth_telegram_miniapp(
session: AsyncSessionDep,
body: TelegramMiniAppAuthRequest,
) -> AuthResponse:
"""Verify Telegram Mini App initData and issue a user JWT."""
bot_token = get_settings().telegram_bot_token
try:
identity = verify_telegram_miniapp_init_data(body.init_data, bot_token)
except AuthError as exc:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=str(exc)) from exc
user = await get_or_create_user_for_telegram(session, identity.telegram_id)
return _issue_token(user)
@router.get("/api/v1/auth/me", status_code=status.HTTP_200_OK)
async def introspect_token(
authorization: Annotated[str | None, Header()] = None,
) -> TokenIntrospectResponse:
"""Debug/introspection endpoint: return claims for a Bearer JWT."""
if not authorization or not authorization.lower().startswith("bearer "):
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="missing bearer token")
token = authorization[7:].strip()
try:
claims = verify_access_token(token)
except AuthError as exc:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=str(exc)) from exc
return TokenIntrospectResponse(
sub=str(claims.sub),
telegram_id=claims.telegram_id,
type=claims.type,
exp=claims.exp or 0,
)
@router.get("/api/v1/auth/me/permissions")
async def token_permissions_dummy() -> dict[str, Any]:
"""Placeholder for future RBAC expansion."""
return {"permissions": ["upload", "read_reports", "read_me"]}

294
src/contract_check/api/routes/b2b.py Normal file
View file

@ -0,0 +1,294 @@
"""B2B API routes: external clients analyze contracts via `X-API-Key`.
Management endpoints (create/revoke/list keys) use service-token auth because they
are called by internal adapters (web/cli) on behalf of a user identified by
`telegram_id`. Analysis endpoints use `X-API-Key` auth only.
"""
from __future__ import annotations
import uuid
from typing import Annotated
from fastapi import APIRouter, File, HTTPException, UploadFile, status
from pydantic import BaseModel
from sqlalchemy import text
from ...core.api_keys import generate_api_key, hash_api_key
from ...core.logging import get_logger
from ..deps import (
ApiKeyAuthDep,
AsyncSessionDep,
CurrentUserDep,
PublisherDep,
StorageDep,
fetch_document_status_for_user,
)
from ..services import upload_and_enqueue
log = get_logger(__name__)
router = APIRouter(tags=["b2b"])
@router.post("/api/v1/analyze", status_code=status.HTTP_202_ACCEPTED)
async def analyze_document(
auth: ApiKeyAuthDep,
session: AsyncSessionDep,
storage: StorageDep,
publisher: PublisherDep,
file: Annotated[UploadFile, File()] = ..., # type: ignore[assignment]
) -> dict[str, object]:
"""Upload a document for analysis using a B2B API key.
Reserves one credit from the key owner's account, stores the file, enqueues
the extraction job, and returns 202 + document_id/correlation_id for polling.
"""
result = await upload_and_enqueue(session, storage, publisher, auth.user_id, file)
# Record usage for this key.
try:
await session.execute(
text("INSERT INTO api_key_requests (api_key_id, document_id) VALUES (:k, :d)"),
{"k": auth.api_key_id, "d": uuid.UUID(str(result["document_id"]))},
)
await session.execute(
text("UPDATE api_keys SET monthly_used = monthly_used + 1 WHERE id = :k"),
{"k": auth.api_key_id},
)
await session.commit()
except Exception as exc:
log.error(
"api_key_request_log_failed",
api_key_id=str(auth.api_key_id),
document_id=result["document_id"],
error=str(exc),
)
# Usage logging is best-effort; do not fail the upload.
return result
@router.get("/api/v1/b2b/reports/{document_id}")
async def get_b2b_report(
auth: ApiKeyAuthDep,
session: AsyncSessionDep,
document_id: uuid.UUID,
) -> dict[str, object]:
"""Poll for an analysis report scoped to the API key owner."""
row = await fetch_document_status_for_user(session, document_id, auth.user_id)
if row is None:
raise HTTPException(status_code=404, detail="report not found")
doc_status = row["status"]
if doc_status != "done":
return {
"document_id": str(document_id),
"status": doc_status,
"stage": row["stage"],
}
return {
"document_id": str(document_id),
"status": doc_status,
"filename": row["filename"],
"markdown": row["markdown"],
"findings": row["content_json"],
"model_used": row["model_used"],
"prompt_tokens": row["prompt_tokens"],
"eval_tokens": row["eval_tokens"],
"latency_ms": row["latency_ms"],
}
@router.get("/api/v1/b2b/usage")
async def get_usage(
auth: ApiKeyAuthDep,
session: AsyncSessionDep,
) -> dict[str, object]:
"""Return current-month usage for the authenticating API key."""
result = await session.execute(
text("SELECT monthly_quota, monthly_used, resets_at FROM api_keys WHERE id = :k"),
{"k": auth.api_key_id},
)
row = result.first()
assert row is not None
quota, used, resets_at = row
requests_result = await session.execute(
text(
"SELECT COUNT(*) FROM api_key_requests "
"WHERE api_key_id = :k AND created_at >= DATE_TRUNC('month', now())"
),
{"k": auth.api_key_id},
)
requests_this_month = int(requests_result.scalar_one())
return {
"api_key_id": str(auth.api_key_id),
"rate_limit_rps": auth.rate_limit_rps,
"monthly_quota": quota,
"monthly_used": used,
"requests_this_month": requests_this_month,
"resets_at": resets_at.isoformat() if resets_at else None,
}
# ── Key management (service-token auth + telegram_id) ────────────────────────
class CreateApiKeyRequest(BaseModel):
name: str
rate_limit_rps: int | None = None
monthly_quota: int | None = None
class ApiKeyResponse(BaseModel):
id: uuid.UUID
name: str
rate_limit_rps: int
monthly_quota: int | None
monthly_used: int
revoked: bool
created_at: str
@router.post("/api/v1/b2b/keys", status_code=status.HTTP_201_CREATED)
async def create_api_key(
session: AsyncSessionDep,
user: CurrentUserDep,
body: CreateApiKeyRequest,
) -> dict[str, object]:
"""Create a new B2B API key for the authenticated user.
The raw key is returned **only once**; afterwards only its hash is stored.
"""
rate_limit = body.rate_limit_rps or 3
if rate_limit <= 0:
raise HTTPException(status_code=400, detail="rate_limit_rps must be positive")
raw_key = generate_api_key()
key_hash = hash_api_key(raw_key)
try:
result = await session.execute(
text(
"INSERT INTO api_keys (user_id, name, key_hash, rate_limit_rps, monthly_quota) "
"VALUES (:u, :n, :h, :r, :q) "
"RETURNING id, name, rate_limit_rps, monthly_quota, monthly_used, revoked, created_at"
),
{
"u": user.user_id,
"n": body.name,
"h": key_hash,
"r": rate_limit,
"q": body.monthly_quota,
},
)
await session.commit()
except Exception as exc:
log.error("api_key_create_failed", user_id=str(user.user_id), error=str(exc))
raise HTTPException(status_code=500, detail="failed to create API key") from exc
row = result.first()
assert row is not None
return {
"api_key": raw_key,
"id": str(row[0]),
"name": row[1],
"rate_limit_rps": row[2],
"monthly_quota": row[3],
"monthly_used": row[4],
"revoked": row[5],
"created_at": row[6].isoformat() if row[6] else None,
}
@router.get("/api/v1/b2b/keys")
async def list_api_keys(
session: AsyncSessionDep,
user: CurrentUserDep,
) -> list[dict[str, object]]:
"""List B2B API keys for the authenticated user."""
result = await session.execute(
text(
"SELECT id, name, rate_limit_rps, monthly_quota, monthly_used, revoked, created_at "
"FROM api_keys WHERE user_id = :u ORDER BY created_at DESC"
),
{"u": user.user_id},
)
return [
{
"id": str(row[0]),
"name": row[1],
"rate_limit_rps": row[2],
"monthly_quota": row[3],
"monthly_used": row[4],
"revoked": row[5],
"created_at": row[6].isoformat() if row[6] else None,
}
for row in result.all()
]
@router.post("/api/v1/b2b/keys/{key_id}/revoke")
async def revoke_api_key(
session: AsyncSessionDep,
user: CurrentUserDep,
key_id: uuid.UUID,
) -> dict[str, object]:
"""Revoke a B2B API key. Only keys owned by the user may be revoked."""
result = await session.execute(
text(
"UPDATE api_keys SET revoked = TRUE "
"WHERE id = :k AND user_id = :u AND revoked = FALSE "
"RETURNING id"
),
{"k": key_id, "u": user.user_id},
)
await session.commit()
if result.first() is None:
raise HTTPException(status_code=404, detail="key not found or already revoked")
return {"id": str(key_id), "revoked": True}
@router.get("/api/v1/b2b/keys/{key_id}/usage")
async def get_key_usage(
session: AsyncSessionDep,
user: CurrentUserDep,
key_id: uuid.UUID,
) -> dict[str, object]:
"""Return per-month usage for a specific API key owned by the user."""
key_result = await session.execute(
text(
"SELECT name, rate_limit_rps, monthly_quota, monthly_used, resets_at "
"FROM api_keys WHERE id = :k AND user_id = :u"
),
{"k": key_id, "u": user.user_id},
)
key_row = key_result.first()
if key_row is None:
raise HTTPException(status_code=404, detail="key not found")
monthly_result = await session.execute(
text(
"SELECT DATE_TRUNC('month', created_at) AS month, COUNT(*) "
"FROM api_key_requests "
"WHERE api_key_id = :k "
"GROUP BY month ORDER BY month DESC"
),
{"k": key_id},
)
return {
"key_id": str(key_id),
"name": key_row[0],
"rate_limit_rps": key_row[1],
"monthly_quota": key_row[2],
"monthly_used": key_row[3],
"resets_at": key_row[4].isoformat() if key_row[4] else None,
"monthly_requests": [
{"month": row[0].isoformat() if row[0] else None, "requests": int(row[1])}
for row in monthly_result.all()
],
}

26
src/contract_check/api/routes/documents.py Normal file
View file

@ -0,0 +1,26 @@
"""Document upload endpoint: reserve credit, store in MinIO, enqueue to RabbitMQ."""
from __future__ import annotations
from typing import Annotated
from fastapi import APIRouter, File, UploadFile, status
from ...core.logging import get_logger
from ..deps import AsyncSessionDep, CurrentUserDep, PublisherDep, StorageDep
from ..services import upload_and_enqueue
log = get_logger(__name__)
router = APIRouter(tags=["documents"])
@router.post("/api/v1/documents", status_code=status.HTTP_202_ACCEPTED)
async def upload_document(
session: AsyncSessionDep,
storage: StorageDep,
publisher: PublisherDep,
user: CurrentUserDep,
file: Annotated[UploadFile, File()] = ..., # type: ignore[assignment]
) -> dict[str, object]:
return await upload_and_enqueue(session, storage, publisher, user.user_id, file)

24
src/contract_check/api/routes/health.py Normal file
View file

@ -0,0 +1,24 @@
"""Health endpoints: /healthz (liveness) and /readyz (readiness)."""
from __future__ import annotations
from fastapi import APIRouter
from sqlalchemy import text
from ..deps import AsyncSessionDep
router = APIRouter(tags=["health"])
@router.get("/healthz")
async def healthz() -> dict[str, str]:
return {"status": "ok"}
@router.get("/readyz")
async def readyz(session: AsyncSessionDep) -> dict[str, str]:
try:
await session.execute(text("SELECT 1"))
except Exception as exc:
return {"status": "not_ready", "reason": f"db: {exc}"}
return {"status": "ok"}

18
src/contract_check/api/routes/me.py Normal file
View file

@ -0,0 +1,18 @@
"""User profile endpoint (credits balance)."""
from __future__ import annotations
from fastapi import APIRouter
from ..deps import AsyncSessionDep, CurrentUserDep, get_credits
router = APIRouter(tags=["me"])
@router.get("/api/v1/me")
async def me(
session: AsyncSessionDep,
user: CurrentUserDep,
) -> dict[str, object]:
credits = await get_credits(session, user.user_id)
return {"telegram_id": user.telegram_id, "credits_left": credits}

13
src/contract_check/api/routes/metrics.py Normal file
View file

@ -0,0 +1,13 @@
"""Prometheus metrics endpoint."""
from __future__ import annotations
from fastapi import APIRouter, Response
from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
router = APIRouter(tags=["metrics"])
@router.get("/metrics")
async def metrics() -> Response:
return Response(generate_latest(), media_type=CONTENT_TYPE_LATEST)

42
src/contract_check/api/routes/reports.py Normal file
View file

@ -0,0 +1,42 @@
"""Report polling endpoint."""
from __future__ import annotations
from uuid import UUID
from fastapi import APIRouter, HTTPException
from ..deps import AsyncSessionDep, CurrentUserDep, fetch_document_status
router = APIRouter(tags=["reports"])
@router.get("/api/v1/reports/{document_id}")
async def get_report(
session: AsyncSessionDep,
document_id: UUID,
user: CurrentUserDep,
) -> dict[str, object]:
row = await fetch_document_status(session, document_id, user.user_id)
if row is None:
raise HTTPException(status_code=404, detail="report not found")
status = row["status"]
if status != "done":
return {
"document_id": str(document_id),
"status": status,
"stage": row["stage"],
}
return {
"document_id": str(document_id),
"status": status,
"filename": row["filename"],
"markdown": row["markdown"],
"findings": row["content_json"],
"model_used": row["model_used"],
"prompt_tokens": row["prompt_tokens"],
"eval_tokens": row["eval_tokens"],
"latency_ms": row["latency_ms"],
}

149
src/contract_check/api/services.py Normal file
View file

@ -0,0 +1,149 @@
"""Shared api upload/enqueue service (used by Telegram and B2B adapters)."""
from __future__ import annotations
import uuid
from typing import TYPE_CHECKING
from fastapi import HTTPException
from sqlalchemy import text
from ..core.analysis.extractor import SUPPORTED_SUFFIXES
from ..core.credits import reserve_credit
from ..core.logging import get_logger, new_correlation_id
from ..core.metrics import credits_reserved, documents_uploaded
from ..core.mq.messages import DocumentUploaded
from ..core.mq.topology import RK_EXTRACT
from ..core.s3 import original_key
if TYPE_CHECKING:
from fastapi import UploadFile
from sqlalchemy.ext.asyncio import AsyncSession
from ..core.mq.publisher import Publisher
from ..core.s3.port import Storage
log = get_logger(__name__)
def _content_type_from_suffix(suffix: str) -> str:
return {
".pdf": "application/pdf",
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
}.get(suffix, "application/octet-stream")
async def upload_and_enqueue(
session: AsyncSession,
storage: Storage,
publisher: Publisher,
user_id: uuid.UUID,
file: UploadFile,
) -> dict[str, object]:
"""Reserve credit, store file in MinIO, create document/job rows, publish to queue.
Returns 202 payload: {document_id, correlation_id, credits_left}.
Raises HTTPException on validation, credit, storage, DB, or MQ failure.
"""
if file.filename is None:
raise HTTPException(status_code=400, detail="filename is required")
suffix = file.filename.lower().split(".")[-1] if "." in file.filename else ""
suffix = f".{suffix}" if suffix else ""
if suffix not in SUPPORTED_SUFFIXES:
raise HTTPException(
status_code=400,
detail=f"unsupported format {suffix!r}; supported: {sorted(SUPPORTED_SUFFIXES)}",
)
if not await reserve_credit(session, user_id):
raise HTTPException(status_code=402, detail="no credits available")
await session.commit()
credits_reserved.inc()
document_id = uuid.uuid4()
correlation_id = new_correlation_id()
s3_key = original_key(str(user_id), str(document_id), suffix)
content_type = file.content_type or _content_type_from_suffix(suffix)
try:
data = await file.read()
if len(data) == 0:
raise HTTPException(status_code=400, detail="empty file")
await storage.put(s3_key, data, content_type=content_type)
except HTTPException:
raise
except Exception as exc:
log.error("s3_upload_failed", document_id=str(document_id), error=str(exc))
await session.execute(
text("UPDATE users SET credits_left = credits_left + 1 WHERE id = :u"),
{"u": user_id},
)
await session.commit()
raise HTTPException(status_code=500, detail="failed to store document") from exc
try:
await session.execute(
text(
"INSERT INTO documents (id, user_id, s3_key, filename, mime, bytes, status) "
"VALUES (:id, :uid, :s3, :fn, :mime, :bytes, 'queued')"
),
{
"id": document_id,
"uid": user_id,
"s3": s3_key,
"fn": file.filename,
"mime": content_type,
"bytes": len(data),
},
)
await session.execute(
text(
"INSERT INTO jobs (document_id, correlation_id, queue, status) "
"VALUES (:did, :cid, 'extract', 'pending')"
),
{"did": document_id, "cid": correlation_id},
)
await session.commit()
except Exception as exc:
log.error("db_enqueue_failed", document_id=str(document_id), error=str(exc))
await session.execute(
text("UPDATE users SET credits_left = credits_left + 1 WHERE id = :u"),
{"u": user_id},
)
await session.commit()
raise HTTPException(status_code=500, detail="failed to enqueue document") from exc
try:
msg = DocumentUploaded(
correlation_id=uuid.UUID(correlation_id),
document_id=document_id,
user_id=user_id,
s3_key=s3_key,
filename=file.filename,
mime=content_type,
)
await publisher.publish(msg, routing_key=RK_EXTRACT)
documents_uploaded.inc()
except Exception as exc:
log.error("mq_publish_failed", document_id=str(document_id), error=str(exc))
await session.execute(
text("UPDATE users SET credits_left = credits_left + 1 WHERE id = :u"),
{"u": user_id},
)
await session.execute(
text("UPDATE documents SET status = 'failed', stage = 'publish_failed' WHERE id = :id"),
{"id": document_id},
)
await session.commit()
raise HTTPException(status_code=500, detail="failed to publish job") from exc
credits = await session.execute(
text("SELECT credits_left FROM users WHERE id = :u"),
{"u": user_id},
)
return {
"document_id": str(document_id),
"correlation_id": str(correlation_id),
"credits_left": int(credits.scalar_one()),
}

6
src/contract_check/bot/__init__.py Normal file
View file

@ -0,0 +1,6 @@
"""Telegram-bot adapter (aiogram 3, HTTP-only).
The leanest service image: speaks to the api exclusively over HTTP and touches
no infrastructure directly (no DB/S3/MQ/LLM). The hexagonal boundary is enforced
by `tests/unit/test_bot_boundary.py`. See docs/ARCHITECTURE.md §4, §17.
"""

63
src/contract_check/bot/__main__.py Normal file
View file

@ -0,0 +1,63 @@
"""Bot entrypoint: configure logging, wire the api client, start aiogram polling."""
from __future__ import annotations
import asyncio
import signal
from aiogram import Bot, Dispatcher
from ..core.logging import bind_context, configure_logging, get_logger
from .client import ApiClient
from .config import get_bot_settings
from .handlers import router as bot_router
log = get_logger(__name__)
async def main() -> None:
settings = get_bot_settings()
configure_logging(
settings.log_level,
json_output=settings.json_logs,
service="bot",
env=settings.env,
)
bind_context(service="bot", env=settings.env)
api = ApiClient(settings)
await api.start()
bot = Bot(token=settings.bot_token)
dp = Dispatcher(api=api, settings=settings)
dp.include_router(bot_router)
me = await bot.get_me()
log.info("bot_started", username=me.username, api_url=settings.api_url)
loop = asyncio.get_running_loop()
stop_event = asyncio.Event()
for sig in (signal.SIGINT, signal.SIGTERM):
try:
loop.add_signal_handler(sig, stop_event.set)
except NotImplementedError:
pass
polling = asyncio.create_task(dp.start_polling(bot, handle_signals=False, polling_timeout=30))
stopper = asyncio.create_task(stop_event.wait())
try:
await asyncio.wait({polling, stopper}, return_when=asyncio.FIRST_COMPLETED)
finally:
if not polling.done():
polling.cancel()
if not stopper.done():
stopper.cancel()
await dp.stop_polling()
await bot.session.close()
await api.aclose()
log.info("bot_stopped")
if __name__ == "__main__":
asyncio.run(main())

199
src/contract_check/bot/client.py Normal file
View file

@ -0,0 +1,199 @@
"""HTTP client to the contract-check api.
The bot touches the api exclusively over HTTP (hexagonal boundary). Every
request carries the service-token bearer and a per-interaction `X-Correlation-ID`
so api/worker logs line up under one id across the whole pipeline
(docs/ARCHITECTURE.md §13).
"""
from __future__ import annotations
from dataclasses import dataclass
import httpx
from ..core.logging import get_logger
from .config import BotSettings
log = get_logger(__name__)
DISCLAIMER_MARKER = "Дисклеймер"
FALLBACK_DISCLAIMER = (
"\n\n> ⚠️ **Дисклеймер.** Это первичный скрининг, а не юридическая консультация. "
"Находки могут содержать ошибки. Перед подписанием договор проверяет юрист."
)
def ensure_disclaimer(markdown: str) -> str:
"""Return markdown guaranteed to carry a disclaimer (append only if missing).
The api's `render_markdown` already appends one; this is the defensive net
required by docs/ARCHITECTURE.md §17 ("bot appends if api omitted").
"""
if DISCLAIMER_MARKER in markdown:
return markdown
return markdown.rstrip() + FALLBACK_DISCLAIMER
class ApiError(Exception):
"""Unexpected api response."""
def __init__(self, status_code: int, detail: str) -> None:
super().__init__(f"api error {status_code}: {detail}")
self.status_code = status_code
self.detail = detail
class NoCreditsError(ApiError):
"""api returned 402 — the user has no credits left."""
class UnsupportedFormatError(ApiError):
"""api returned 400 — bad mime/size/extension."""
@dataclass(slots=True)
class UploadResult:
document_id: str
correlation_id: str
credits_left: int
@dataclass(slots=True)
class ReportStatus:
document_id: str
status: str
stage: str | None
markdown: str | None = None
filename: str | None = None
def _extract_detail(response: httpx.Response) -> str:
try:
body = response.json()
except Exception:
return response.text
detail = body.get("detail") if isinstance(body, dict) else None
return detail if isinstance(detail, str) else str(detail)
class ApiClient:
"""Thin httpx wrapper around the api endpoints the bot needs.
The bot authenticates adapter-level calls (POST /auth/telegram/bot) with its
service token. All user-scoped calls carry a user JWT obtained from that
endpoint, not a raw telegram_id query parameter.
"""
def __init__(self, settings: BotSettings) -> None:
self._settings = settings
self._client: httpx.AsyncClient | None = None
self._token_cache: dict[int, str] = {}
async def start(self) -> None:
if self._client is None:
self._client = httpx.AsyncClient(
base_url=self._settings.api_url,
timeout=httpx.Timeout(30.0, connect=5.0),
)
async def aclose(self) -> None:
if self._client is not None:
await self._client.aclose()
self._client = None
@property
def client(self) -> httpx.AsyncClient:
if self._client is None:
raise RuntimeError("ApiClient not started; call start() first")
return self._client
def _service_headers(self, correlation_id: str) -> dict[str, str]:
return {
"Authorization": f"Bearer {self._settings.bot_service_token}",
"X-Correlation-ID": correlation_id,
}
def _user_headers(self, telegram_id: int, correlation_id: str) -> dict[str, str]:
token = self._token_cache.get(telegram_id)
if not token:
raise RuntimeError("No JWT cached for telegram_id; call login() first")
return {
"Authorization": f"Bearer {token}",
"X-Correlation-ID": correlation_id,
}
async def login(self, telegram_id: int, correlation_id: str) -> None:
"""Exchange a verified telegram_id for a user JWT and cache it."""
r = await self.client.post(
"/api/v1/auth/telegram/bot",
json={"telegram_id": telegram_id},
headers=self._service_headers(correlation_id),
)
if r.status_code != 200:
raise ApiError(r.status_code, _extract_detail(r))
body = r.json()
token = body.get("access_token")
if not isinstance(token, str):
raise ApiError(500, "auth response missing access_token")
self._token_cache[telegram_id] = token
def forget(self, telegram_id: int) -> None:
self._token_cache.pop(telegram_id, None)
async def get_credits(self, telegram_id: int, correlation_id: str) -> int:
r = await self.client.get(
"/api/v1/me",
headers=self._user_headers(telegram_id, correlation_id),
)
if r.status_code != 200:
raise ApiError(r.status_code, _extract_detail(r))
return int(r.json().get("credits_left", 0))
async def upload_document(
self,
telegram_id: int,
correlation_id: str,
filename: str,
data: bytes,
content_type: str,
) -> UploadResult:
files = {"file": (filename, data, content_type)}
r = await self.client.post(
"/api/v1/documents",
files=files,
headers=self._user_headers(telegram_id, correlation_id),
)
if r.status_code == 202:
body = r.json()
return UploadResult(
document_id=body["document_id"],
correlation_id=body["correlation_id"],
credits_left=int(body.get("credits_left", 0)),
)
detail = _extract_detail(r)
if r.status_code == 402:
raise NoCreditsError(r.status_code, detail)
if r.status_code == 400:
raise UnsupportedFormatError(r.status_code, detail)
raise ApiError(r.status_code, detail)
async def get_report(
self, telegram_id: int, correlation_id: str, document_id: str
) -> ReportStatus:
r = await self.client.get(
f"/api/v1/reports/{document_id}",
headers=self._user_headers(telegram_id, correlation_id),
)
if r.status_code == 404:
raise ApiError(404, "report not found")
if r.status_code != 200:
raise ApiError(r.status_code, _extract_detail(r))
body = r.json()
return ReportStatus(
document_id=body["document_id"],
status=body["status"],
stage=body.get("stage"),
markdown=body.get("markdown"),
filename=body.get("filename"),
)

53
src/contract_check/bot/config.py Normal file
View file

@ -0,0 +1,53 @@
"""Bot-only configuration (pydantic-settings).
Deliberately independent of `core.config.Settings`, which carries the DB/MQ/S3
secrets the bot must never need (docs/ARCHITECTURE.md §11). The bot image installs
no DB/MQ/S3 driver, so requiring those env vars would be wrong; the only secrets
the bot holds are its Telegram token and the service-token bearer used to call
the api.
"""
from __future__ import annotations
from functools import lru_cache
from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
class BotSettings(BaseSettings):
"""Telegram-bot adapter settings. See docs/ARCHITECTURE.md §11 (Bot)."""
model_config = SettingsConfigDict(
env_file=".env",
env_file_encoding="utf-8",
extra="ignore",
case_sensitive=False,
)
env: str = "dev"
log_level: str = "INFO"
bot_token: str = Field(..., description="Telegram bot token from @BotFather")
api_url: str = Field(
default="http://api:8000",
description="Base URL of the api service (HTTP-only target).",
)
bot_service_token: str = Field(
...,
description="Bearer token authenticating the bot against the api (service_tokens).",
)
poll_interval_initial: float = 1.0
poll_interval_max: float = 5.0
poll_timeout: float = 300.0
long_message_threshold: int = 4096
@property
def json_logs(self) -> bool:
return self.env != "dev"
@lru_cache
def get_bot_settings() -> BotSettings:
return BotSettings() # type: ignore[call-arg]

262
src/contract_check/bot/handlers.py Normal file
View file

@ -0,0 +1,262 @@
"""aiogram 3 handlers: /start, document upload → poll → deliver report.
HTTP-only (docs/ARCHITECTURE.md §17). No DB/S3/MQ/LLM imports — the boundary is
enforced statically by `tests/unit/test_bot_boundary.py`.
Flow:
/start → GET /api/v1/me → greeting + credit balance.
PDF/DOCX → forward multipart to POST /api/v1/documents (Bearer) →
react to 402 (no credit) / 400 (bad format) / 202 (accepted) →
poll GET /api/v1/reports/{id} with backoff, surfacing `stage` →
on done send the report (attachment if >4096 chars, disclaimer
guaranteed); on failed notify the user.
"""
from __future__ import annotations
import asyncio
import io
from aiogram import Bot, F, Router
from aiogram.filters import Command
from aiogram.types import BufferedInputFile, Document, Message
from ..core.logging import get_logger, new_correlation_id
from .client import (
ApiClient,
ApiError,
NoCreditsError,
ReportStatus,
UnsupportedFormatError,
ensure_disclaimer,
)
from .config import BotSettings
log = get_logger(__name__)
router = Router(name="contract-check-bot")
_SUPPORTED_SUFFIXES = (".pdf", ".docx")
_CONTENT_TYPES = {
".pdf": "application/pdf",
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
}
_STAGE_LABELS: dict[str, str] = {
"queued": "Документ принят. В очереди на обработку…",
"extracting": "Извлекаю текст из документа…",
"ocr": "Документ похож на скан — распознаю страницы (OCR)…",
"analyzing": "Анализирую риски по чек-листу…",
}
def _suffix(name: str) -> str:
dot = name.rfind(".")
return name[dot:].lower() if dot >= 0 else ""
def _user_id(message: Message) -> int:
return message.from_user.id if message.from_user else 0
@router.message(Command("start", "help"))
async def cmd_start(message: Message, api: ApiClient) -> None:
cid = new_correlation_id()
tg = _user_id(message)
try:
await api.login(tg, cid)
credits = await api.get_credits(tg, cid)
except ApiError as exc:
log.error("me_failed", correlation_id=cid, error=str(exc))
await message.answer(
"Привет! Я «Контракт-чек» — скрининг рисков в договорах.\n"
"Не удалось связаться с сервисом, попробуйте позже."
)
return
await message.answer(
"Привет! Я «Контракт-чек» — первичный скрининг рисков в договорах "
"по ГК РФ / ГК РБ.\n\n"
"Пришлите PDF или DOCX договор — я проверю его по чек-листу и пришлю "
"отчёт с рисками и рекомендациями.\n\n"
f"Осталось проверок: {credits}."
)
@router.message(F.document)
async def handle_document(message: Message, api: ApiClient, settings: BotSettings) -> None:
cid = new_correlation_id()
tg = _user_id(message)
document: Document | None = message.document
if document is None or document.file_name is None:
await message.answer("Пришлите файл договора (PDF или DOCX).")
return
suffix = _suffix(document.file_name)
if suffix not in _SUPPORTED_SUFFIXES:
await message.answer(
f"Поддерживаются только форматы PDF и DOCX. Получил: {suffix or 'без расширения'}."
)
return
status_msg = await message.answer("Скачиваю файл…")
bot = message.bot
if bot is None:
await _edit(status_msg, "Внутренняя ошибка: бот недоступен.")
return
data = await _download(bot, document.file_id, cid)
if data is None:
await _edit(status_msg, "Не удалось скачать файл. Попробуйте ещё раз.")
return
content_type = document.mime_type or _CONTENT_TYPES[suffix]
try:
await api.login(tg, cid)
upload = await api.upload_document(tg, cid, document.file_name, data, content_type)
except NoCreditsError:
await _edit(status_msg, "У вас закончились проверки. Пополните баланс, чтобы продолжить.")
return
except UnsupportedFormatError:
await _edit(status_msg, "Сервис не принял формат файла. Пришлите корректный PDF или DOCX.")
return
except ApiError as exc:
log.error(
"upload_failed",
correlation_id=cid,
status=exc.status_code,
detail=exc.detail,
)
await _edit(status_msg, "Не удалось отправить документ на анализ. Попробуйте позже.")
return
log.info(
"document_uploaded",
correlation_id=cid,
document_id=upload.document_id,
credits_left=upload.credits_left,
)
await _edit(status_msg, _STAGE_LABELS["queued"])
await _poll_and_deliver(
api=api,
settings=settings,
bot=bot,
chat_id=message.chat.id,
status_msg=status_msg,
correlation_id=cid,
telegram_id=tg,
document_id=upload.document_id,
filename=document.file_name,
)
async def _download(bot: Bot, file_id: str, correlation_id: str) -> bytes | None:
try:
buf = io.BytesIO()
await bot.download(file_id, destination=buf)
except Exception as exc:
log.error("tg_download_failed", correlation_id=correlation_id, error=str(exc))
return None
return buf.getvalue()
async def _poll_and_deliver(
*,
api: ApiClient,
settings: BotSettings,
bot: Bot,
chat_id: int,
status_msg: Message,
correlation_id: str,
telegram_id: int,
document_id: str,
filename: str,
) -> None:
interval = settings.poll_interval_initial
elapsed = 0.0
last_label = _STAGE_LABELS["queued"]
while elapsed < settings.poll_timeout:
try:
report = await api.get_report(telegram_id, correlation_id, document_id)
except ApiError as exc:
log.warning(
"report_poll_failed",
correlation_id=correlation_id,
status=exc.status_code,
detail=exc.detail,
)
await _sleep(elapsed, settings.poll_timeout, interval)
elapsed += interval
interval = _next_interval(interval, settings.poll_interval_max)
continue
if report.status == "done" and report.markdown is not None:
await _deliver_report(bot, chat_id, report, filename, settings.long_message_threshold)
return
if report.status == "failed":
await _edit(
status_msg,
"Не удалось обработать документ. Проверка списана не будет — "
"попробуйте другой файл.",
)
return
label = _stage_label(report)
if label != last_label:
await _edit(status_msg, label)
last_label = label
if not await _sleep(elapsed, settings.poll_timeout, interval):
break
elapsed += interval
interval = _next_interval(interval, settings.poll_interval_max)
await _edit(
status_msg,
"Анализ занимает больше обычного. Попробуйте прислать документ ещё раз через минуту.",
)
def _next_interval(current: float, cap: float) -> float:
return min(current * 1.5, cap)
async def _sleep(elapsed: float, timeout: float, interval: float) -> bool:
"""Sleep `interval`, but never past the overall `timeout`. False if at budget end."""
remaining = timeout - elapsed
if remaining <= 0:
return False
await asyncio.sleep(min(interval, remaining))
return True
def _stage_label(report: ReportStatus) -> str:
stage = report.stage or report.status
return _STAGE_LABELS.get(stage, _STAGE_LABELS.get(report.status, "Обрабатываю…"))
async def _deliver_report(
bot: Bot,
chat_id: int,
report: ReportStatus,
filename: str,
threshold: int,
) -> None:
markdown = ensure_disclaimer(report.markdown or "")
if len(markdown) <= threshold:
await bot.send_message(chat_id, markdown)
return
safe_name = filename.rsplit(".", 1)[0] if "." in filename else filename
attachment = BufferedInputFile(markdown.encode("utf-8"), filename=f"{safe_name}-report.md")
await bot.send_document(
chat_id,
attachment,
caption="Отчёт получился объёмным — отправляю как файл.",
)
async def _edit(message: Message, text: str) -> None:
try:
await message.edit_text(text)
except Exception as exc:
log.debug("edit_skipped", error=str(exc))

20
src/contract_check/core/__init__.py Normal file
View file

@ -0,0 +1,20 @@
"""«Контракт-чек» core — shared domain package.
Imported by every service image (api, worker-extract, worker-analyze) and the
prototype benchmark. Adapters (bot) deliberately do NOT import this package
beyond config/logging (hexagonal boundary — see docs/ARCHITECTURE.md §4).
Subpackages:
config — typed env config (pydantic-settings)
logging — structlog JSON + correlation_id propagation
telemetry — OpenTelemetry init
sentry — Sentry init
metrics — Prometheus counters/histograms + metrics HTTP server
db — SQLAlchemy 2 models (6 tables), async session, enums
mq — RabbitMQ topology, publisher (confirms), consumer (retry/DLQ)
s3 — Storage port + MinIO adapter
llm — LLM provider port + Ollama Cloud adapter + factory
analysis — extractor, chunker, checklist, report_schema, analyzer, ocr
credits — reserve/refund billing invariants
tokens — service-token hashing/verification (no FastAPI here)
"""

4
src/contract_check/core/analysis/__init__.py Normal file
View file

@ -0,0 +1,4 @@
"""Analysis domain — document text extraction, chunking, checklist, report
schema, OCR, and the analyzer (prompt building, finding merge/dedupe/sort,
markdown rendering). Imported by the LLM adapter and the prototype benchmark.
"""

129
src/contract_check/core/analysis/analyzer.py Normal file
View file

@ -0,0 +1,129 @@
"""Analyzer — prompt building, finding merge/dedupe/sort, markdown rendering.
Pure logic (no LLM, no DB, no S3). Shared by the Ollama Cloud adapter
(chunk fan-out → merge), the worker-analyze handler (render final report), and
the stage-0 prototype benchmark. Kept dependency-free so it cannot form an
import cycle with `core.llm`.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from pathlib import Path
from ..logging import is_debug_enabled
from .checklist import CHECKLIST
from .report_schema import Finding
DISCLAIMER = (
"> ⚠️ **Дисклеймер.** Это первичный скрининг на основе открытой языковой "
"модели, а не юридическая консультация. Находки могут содержать ошибки и "
"не учитывают полную картину отношений сторон. Перед подписанием договор "
"обязательно проверяет квалифицированный юрист. Текст договора может "
"обрабатываться вне РБ/РФ (cloud-инференс)."
)
@dataclass(slots=True)
class RenderMetrics:
"""Aggregate run metrics, attached to every rendered report."""
chars: int = 0
chunks: int = 0
findings: int = 0
prompt_tokens: int = 0
eval_tokens: int = 0
latency_sec: float = 0.0
models_used: set[str] = field(default_factory=set)
fell_back: bool = False
repaired: bool = False
broken_json: int = 0
@property
def total_tokens(self) -> int:
return self.prompt_tokens + self.eval_tokens
def build_user_prompt(chunk: str) -> str:
return (
"Проанализируй договор по чек-листу и верни находки в JSON по схеме.\n\n"
"=== ТЕКСТ ДОГОВОРА (фрагмент) ===\n"
f"{chunk}\n"
"=== КОНЕЦ ТЕКСТА ===\n\n"
'Верни JSON: {"findings": [ ... ]}. Только реальные риски, с цитатами '
"и ссылкой на пункт оригинала."
)
def dedupe_findings(findings: list[Finding]) -> list[Finding]:
seen: set[tuple[str, str]] = set()
out: list[Finding] = []
for f in findings:
key = (f.checklist_id, f.quote.strip().lower()[:200])
if key in seen:
continue
seen.add(key)
out.append(f)
return out
def sort_findings(findings: list[Finding]) -> list[Finding]:
order = {item.id: i for i, item in enumerate(CHECKLIST)}
sev_rank = {"high": 0, "medium": 1, "low": 2}
return sorted(
findings,
key=lambda f: (order.get(f.checklist_id, 999), sev_rank.get(f.severity, 9)),
)
def checklist_title(checklist_id: str) -> str:
for item in CHECKLIST:
if item.id == checklist_id:
return item.title
return checklist_id
def render_markdown(findings: list[Finding], source: str, metrics: RenderMetrics) -> str:
lines: list[str] = []
lines.append(f"# Отчёт по договору: `{Path(source).name}`\n")
lines.append(f"**Найдено рисков:** {len(findings)}\n")
if not findings:
lines.append(
"_Явных рисков по чек-листу не найдено. Это не юридическое заключение "
"об отсутствии рисков — отдельные формулировки стоит проверить юристу._\n"
)
else:
sev_emoji = {"high": "🔴 Высокий", "medium": "🟡 Средний", "low": "🟢 Низкий"}
for i, f in enumerate(findings, start=1):
title = checklist_title(f.checklist_id)
lines.append(f"## {i}. {title} — {sev_emoji.get(f.severity, f.severity)}\n")
lines.append(f"**Пункт договора:** {f.section_ref}\n")
lines.append("**Цитата из оригинала:**")
lines.append(f"> {f.quote}\n")
lines.append(f"**Риск:** {f.risk or checklist_title(f.checklist_id)}\n")
if f.recommendation:
lines.append(f"**Рекомендация:** {f.recommendation}\n")
lines.append("---\n")
lines.append(DISCLAIMER + "\n")
if is_debug_enabled():
lines.append("\n<details><summary>Метрики прогона</summary>\n")
lines.append("```")
lines.append(f"chars: {metrics.chars}")
lines.append(f"chunks: {metrics.chunks}")
lines.append(f"findings: {metrics.findings}")
lines.append(
f"tokens total: {metrics.total_tokens} "
f"(prompt {metrics.prompt_tokens} + eval {metrics.eval_tokens})"
)
lines.append(f"latency sec: {metrics.latency_sec:.1f}")
lines.append(f"models: {', '.join(sorted(metrics.models_used)) or '-'}")
lines.append(f"fell back: {metrics.fell_back}")
lines.append(
f"repaired: {metrics.repaired} (broken-json attempts: {metrics.broken_json})"
)
lines.append("```")
lines.append("</details>\n")
return "\n".join(lines)

108
src/contract_check/core/analysis/checklist.py Normal file
View file

@ -0,0 +1,108 @@
"""Чек-лист анализа договора (v1).
Один список, без БД. Источник: docs/IMPLEMENTATION_PLAN.md «Чек-лист пунктов анализа».
Каждый пункт — то, что модель ищет в тексте договора.
"""
from __future__ import annotations
from dataclasses import dataclass
@dataclass(frozen=True, slots=True)
class ChecklistItem:
id: str
title: str
description: str
CHECKLIST: list[ChecklistItem] = [
ChecklistItem(
id="penalties",
title="Неустойки / штрафы",
description=(
"Размер неустойки или штрафа, её соразмерность, право стороны "
"взимать её в одностороннем порядке, двойные санкции за одно нарушение."
),
),
ChecklistItem(
id="jurisdiction",
title="Подсудность",
description=(
"Рассмотрение споров в «своём» регионе одной из сторон, обход "
"потребительского / претензионного порядка, суд чужого государства."
),
),
ChecklistItem(
id="payment_terms",
title="Сроки и условия оплаты",
description=(
"Размытые сроки, авансы без возврата, право сдвигать срок оплаты "
"или изменять цену, просрочка и её последствия."
),
),
ChecklistItem(
id="intellectual_property",
title="Интеллектуальные права (IP)",
description=(
"Кому отходят исключительные права на результаты, отчуждение "
"сверх необходимого, лицензия шире цели договора, служебные произведения."
),
),
ChecklistItem(
id="unilateral_changes",
title="Одностороннее изменение условий",
description=(
"Право одной стороны менять цену, тариф, объём, условия "
"оказания без согласия второй стороны."
),
),
ChecklistItem(
id="warranties",
title="Гарантии и их срок",
description=(
"Срок гарантии, порядок предъявления требований по гарантии, "
"исключения из гарантии, бремя доказывания недостатков."
),
),
ChecklistItem(
id="force_majeure",
title="Форс-мажор",
description=(
"Узкий перечень обстоятельств, исключение «экономических» форс-мажоров, "
"срок уведомления и бремя доказывания форс-мажора."
),
),
ChecklistItem(
id="vat",
title="НДС",
description=(
"Включён НДС в цену или начисляется сверх, кто несёт налоговую "
"нагрузку, право сторон менять режим налогообложения."
),
),
ChecklistItem(
id="liability",
title="Ответственность сторон",
description=(
"Кап (cap) ответственности, исключения косвенных/упущенных убытков, "
"асимметрия ответственности между сторонами."
),
),
ChecklistItem(
id="termination",
title="Расторжение",
description=(
"Условия и срок уведомления о расторжении, право одностороннего "
"отказа, штрафы за досрочное расторжение, возврат аванса."
),
),
]
def checklist_for_prompt() -> str:
"""Render the checklist as a numbered list for the LLM prompt."""
lines = []
for i, item in enumerate(CHECKLIST, start=1):
lines.append(f"{i}. [{item.id}] {item.title} — {item.description}")
return "\n".join(lines)

69
src/contract_check/core/analysis/chunker.py Normal file
View file

@ -0,0 +1,69 @@
"""Разбиение длинного текста на чанки под контекст модели.
Граница — по абзацам / предложениям, чтобы не рвать смысл. Оставляем чанки
одинакового размера по символам; перекрытий нет (риски локальны в пределах пункта).
"""
from __future__ import annotations
import re
_SENTENCE_END = re.compile(r"(?<=[.!?…])\s+")
def chunk_text(text: str, max_chars: int = 10000) -> list[str]:
"""Разрезать text на куски ≤ max_chars с сохранением границ предложений."""
if max_chars <= 0:
raise ValueError("max_chars must be positive")
if len(text) <= max_chars:
return [text]
units: list[str] = []
for para in text.split("\n"):
para = para.strip()
if not para:
continue
if len(para) <= max_chars:
units.append(para)
else:
units.extend(_split_sentences(para, max_chars))
chunks: list[str] = []
buf = ""
for unit in units:
candidate = f"{buf}\n{unit}" if buf else unit
if len(candidate) <= max_chars:
buf = candidate
else:
if buf:
chunks.append(buf)
if len(unit) > max_chars:
for i in range(0, len(unit), max_chars):
chunks.append(unit[i : i + max_chars])
buf = ""
else:
buf = unit
if buf:
chunks.append(buf)
return chunks
def _split_sentences(para: str, max_chars: int) -> list[str]:
parts = _SENTENCE_END.split(para)
out: list[str] = []
buf = ""
for s in parts:
candidate = f"{buf} {s}".strip() if buf else s
if len(candidate) <= max_chars:
buf = candidate
else:
if buf:
out.append(buf)
if len(s) <= max_chars:
buf = s
else:
out.extend(s[i : i + max_chars] for i in range(0, len(s), max_chars))
buf = ""
if buf:
out.append(buf)
return out

80
src/contract_check/core/analysis/extractor.py Normal file
View file

@ -0,0 +1,80 @@
"""Экстрактор текста: PDF/DOCX → str.
PDF (с текстовым слоем) — pymupdf. DOCX — python-docx.
Сканы здесь не обрабатываются (это OCR, см. core.analysis.ocr); пустой/битый
файл → ExtractionError. Лibraries imported lazily so the analyzer package can
be imported without pymupdf/python-docx installed.
"""
from __future__ import annotations
from pathlib import Path
from ..logging import get_logger
log = get_logger(__name__)
SUPPORTED_SUFFIXES = {".pdf", ".docx"}
class ExtractionError(Exception):
"""Файл пуст, бит или формат не поддерживается."""
def extract_text(path: str | Path) -> str:
"""Достать текст из PDF/DOCX. Поднять ExtractionError, если пусто/бито."""
p = Path(path)
if not p.exists():
raise ExtractionError(f"Файл не найден: {p}")
if p.suffix.lower() == ".pdf":
text = _extract_pdf(p)
elif p.suffix.lower() == ".docx":
text = _extract_docx(p)
else:
raise ExtractionError(
f"Неподдерживаемый формат '{p.suffix}'. Поддерживаются: "
f"{', '.join(sorted(SUPPORTED_SUFFIXES))}."
)
stripped = text.strip()
if len(stripped) < 100:
raise ExtractionError(
f"Извлечено слишком мало текста ({len(stripped)} симв.). "
"Возможно, это скан — нужен OCR, либо файл повреждён."
)
log.info("extracted", file=p.name, chars=len(stripped))
return stripped
def _extract_pdf(p: Path) -> str:
import pymupdf
try:
doc = pymupdf.open(p)
except Exception as exc: # pymupdf кидает разные типы
raise ExtractionError(f"Не удалось открыть PDF {p.name}: {exc}") from exc
parts: list[str] = []
try:
for page_index in range(doc.page_count):
page_text = doc[page_index].get_text("text")
if page_text:
parts.append(page_text)
finally:
doc.close()
return "\n".join(parts).strip()
def _extract_docx(p: Path) -> str:
from docx import Document
try:
doc = Document(str(p))
except Exception as exc:
raise ExtractionError(f"Не удалось открыть DOCX {p.name}: {exc}") from exc
parts = [para.text for para in doc.paragraphs if para.text.strip()]
for table in doc.tables:
for row in table.rows:
cells = [c.text.strip() for c in row.cells if c.text.strip()]
if cells:
parts.append(" | ".join(cells))
return "\n".join(parts).strip()

65
src/contract_check/core/analysis/ocr.py Normal file
View file

@ -0,0 +1,65 @@
"""OCR fallback for scanned PDFs (Tesseract, rus+eng).
Triggered by the extract worker only when `extract_text` returns < 100 chars
(a strong scan signal). Rasterizes each PDF page via pymupdf and runs
pytesseract. Both libs are imported lazily so this module imports cleanly even
in images without tesseract/pymupdf installed (the bot/analyze images).
Yandex Vision is a future alternative behind the same `ocr_pdf(path)` entry.
"""
from __future__ import annotations
from pathlib import Path
from ..logging import get_logger
from .extractor import ExtractionError
log = get_logger(__name__)
class OCRError(Exception):
"""OCR failed (Tesseract unavailable, rasterization failed, no text)."""
def ocr_pdf(path: str | Path, *, lang: str = "rus+eng") -> str:
"""Rasterize and OCR a PDF → plaintext. Raises OCRError on failure."""
p = Path(path)
try:
import pymupdf
import pytesseract
from PIL import Image
except ImportError as exc:
raise OCRError(f"OCR backend not installed: {exc.name}") from exc
try:
doc = pymupdf.open(p)
except Exception as exc: # pymupdf кидает разные типы
raise OCRError(f"Не удалось открыть PDF для OCR {p.name}: {exc}") from exc
parts: list[str] = []
try:
for page_index in range(doc.page_count):
page = doc[page_index]
pix = page.get_pixmap(dpi=300)
img = Image.frombytes("RGB", (pix.width, pix.height), pix.samples)
text = pytesseract.image_to_string(img, lang=lang)
if text:
parts.append(text)
except pytesseract.pytesseract.TesseractError as exc:
# Tesseract itself failed (missing language data, etc.) — surface as
# an OCR infra failure so the retry/DLQ path can refund appropriately.
raise OCRError(f"Tesseract engine failed for {p.name}: {exc}") from exc
except Exception as exc:
raise OCRError(f"OCR failed for {p.name}: {exc}") from exc
finally:
doc.close()
stripped = "\n".join(parts).strip()
if len(stripped) < 100:
raise ExtractionError(
f"OCR тоже дал мало текста ({len(stripped)} симв.). "
"Файл, видимо, не содержит распознаваемого текста."
)
log.info("ocr_done", file=p.name, chars=len(stripped))
return stripped

62
src/contract_check/core/analysis/report_schema.py Normal file
View file

@ -0,0 +1,62 @@
"""Pydantic schema of the analysis report.
Used both for client-side validation (repair-loop in the LLM adapter) and for
generating the JSON-schema handed to Ollama via `format: <schema>`.
Open models routinely ignore the requested field names and emit their own
(e.g. ``risk_type`` instead of ``checklist_id``, ``description`` instead of
``risk``, and they often omit ``recommendation`` entirely). To keep the
repair-loop quiet we accept the common model aliases on input while exposing
the canonical names to the rest of the codebase.
"""
from __future__ import annotations
from typing import Literal
from pydantic import AliasChoices, BaseModel, ConfigDict, Field
Severity = Literal["high", "medium", "low"]
class Finding(BaseModel):
model_config = ConfigDict(populate_by_name=True, extra="ignore")
checklist_id: str = Field(
...,
validation_alias=AliasChoices("checklist_id", "risk_type", "type", "category"),
description="id пункта чек-листа, напр. 'penalties', 'jurisdiction'.",
)
severity: Severity = Field(
...,
description="high / medium / low — уровень риска для стороны клиента.",
)
quote: str = Field(
...,
validation_alias=AliasChoices("quote", "citation", "text"),
description="Дословная цитата из оригинала договора (без пересказа).",
)
section_ref: str = Field(
...,
validation_alias=AliasChoices("section_ref", "section", "clause", "point"),
description="Номер/название пункта договора, чтобы проверить одним кликом.",
)
risk: str = Field(
default="",
validation_alias=AliasChoices("risk", "description", "risk_description", "risk_text"),
description="В чём конкретно риск для стороны клиента.",
)
recommendation: str = Field(
default="",
validation_alias=AliasChoices("recommendation", "recommendation_text", "advice"),
description="Что предложить изменить / уточнить в договоре.",
)
class ReportPayload(BaseModel):
model_config = ConfigDict(extra="ignore")
findings: list[Finding] = Field(
default_factory=list,
description="Только реальные находки. Если по пункту риска нет — не включаем.",
)

26
src/contract_check/core/api_keys.py Normal file
View file

@ -0,0 +1,26 @@
"""B2B API-key utilities: generation, hashing, verification.
Just like service tokens, the raw key is shown only once; we store SHA-256.
"""
from __future__ import annotations
import hashlib
import hmac
import secrets
def generate_api_key() -> str:
"""Mint a new opaque B2B API key (~43 URL-safe chars)."""
return secrets.token_urlsafe(32)
def hash_api_key(key: str) -> str:
"""SHA-256 hex digest of an API key (store this, never the raw key)."""
return hashlib.sha256(key.encode("utf-8")).hexdigest()
def verify_api_key(key: str, key_hash: str) -> bool:
"""Constant-time check that `key` matches the stored `key_hash`."""
digest = hash_api_key(key)
return hmac.compare_digest(digest, key_hash)

267
src/contract_check/core/auth.py Normal file
View file

@ -0,0 +1,267 @@
"""User authentication helpers: JWT signing/verification and Telegram identity checks.
This module is part of `core` and may be imported by the API. It does NOT depend
on DB/S3/MQ — only on pydantic-settings + python-jose-style JWT via PyJWT +
standard library hmac/hashlib.
Supports three Telegram identity sources:
1. Bot adapter: the bot already got a verified `telegram_id` from Telegram and
uses a service token to exchange it for a user JWT.
2. Telegram Login Widget: web callback payload signed by Telegram.
3. Telegram Mini App: `initData` signed by Telegram.
All sources issue the same JWT containing `sub` (user UUID) and `telegram_id`.
"""
from __future__ import annotations
import datetime as dt
import hashlib
import hmac
import secrets
import uuid
from dataclasses import dataclass
from typing import Any
from urllib.parse import parse_qsl
import jwt
from .config import get_settings
from .logging import get_logger
log = get_logger(__name__)
JWT_TYPE_ACCESS = "access"
class AuthError(Exception):
"""Raised when an identity proof cannot be verified or a token is invalid."""
class TokenExpiredError(AuthError):
"""JWT has expired."""
class TokenInvalidError(AuthError):
"""JWT is malformed or signature is bad."""
@dataclass(slots=True)
class UserIdentity:
"""Verified user identity returned by Telegram identity checks."""
telegram_id: int
@dataclass(slots=True)
class AccessTokenClaims:
"""Payload we put into (and expect from) an access JWT."""
sub: uuid.UUID # user_id
telegram_id: int
type: str
exp: int | None = None # unix seconds
def to_dict(self) -> dict[str, Any]:
return {
"sub": str(self.sub),
"telegram_id": self.telegram_id,
"type": self.type,
"exp": self.exp,
}
@classmethod
def from_dict(cls, payload: dict[str, Any]) -> AccessTokenClaims:
return cls(
sub=uuid.UUID(str(payload["sub"])),
telegram_id=int(payload["telegram_id"]),
type=str(payload.get("type", JWT_TYPE_ACCESS)),
exp=payload.get("exp"),
)
def _jwt_secret() -> str:
return get_settings().jwt_secret
def _jwt_algorithm() -> str:
return get_settings().jwt_algorithm
def _jwt_access_ttl() -> dt.timedelta:
return dt.timedelta(minutes=get_settings().jwt_access_ttl_minutes)
def create_access_token(user_id: uuid.UUID, telegram_id: int) -> str:
"""Sign a fresh access JWT for a verified user."""
settings = get_settings()
now = dt.datetime.now(tz=dt.UTC)
claims = AccessTokenClaims(
sub=user_id,
telegram_id=telegram_id,
type=JWT_TYPE_ACCESS,
)
payload = claims.to_dict()
payload.update(
{
"iat": int(now.timestamp()),
"exp": int((now + _jwt_access_ttl()).timestamp()),
"iss": settings.otel_service_name or "contract-check",
"aud": "contract-check",
}
)
token: str = jwt.encode(
payload,
key=_jwt_secret(),
algorithm=_jwt_algorithm(),
)
return token
def verify_access_token(token: str) -> AccessTokenClaims:
"""Verify an access JWT and return its claims.
Raises TokenInvalidError / TokenExpiredError on failure.
"""
try:
payload = jwt.decode(
token,
key=_jwt_secret(),
algorithms=[_jwt_algorithm()],
audience="contract-check",
options={
"require": ["sub", "telegram_id", "exp", "iat"],
"verify_aud": True,
},
)
except jwt.ExpiredSignatureError as exc:
raise TokenExpiredError("token expired") from exc
except jwt.InvalidTokenError as exc:
raise TokenInvalidError("invalid token") from exc
if payload.get("type") != JWT_TYPE_ACCESS:
raise TokenInvalidError("unexpected token type")
try:
return AccessTokenClaims.from_dict(payload)
except (KeyError, ValueError, TypeError) as exc:
raise TokenInvalidError("malformed token claims") from exc
def _telegram_secret_key(bot_token: str) -> bytes:
"""Telegram uses HMAC_SHA256(BOT_TOKEN, 'WebAppData') as the signing key."""
return hmac.new(
bot_token.encode("utf-8"),
b"WebAppData",
hashlib.sha256,
).digest()
def _constant_time_compare(a: str, b: str) -> bool:
return secrets.compare_digest(a.encode("utf-8"), b.encode("utf-8"))
def verify_telegram_web_payload(payload: dict[str, Any], bot_token: str) -> UserIdentity:
"""Verify a Telegram Login Widget callback payload.
Reference: https://core.telegram.org/widgets/login
"""
if not bot_token:
raise AuthError("telegram_bot_token is not configured")
received_hash = payload.get("hash")
if not isinstance(received_hash, str) or not received_hash:
raise AuthError("missing hash")
auth_date = payload.get("auth_date")
if not isinstance(auth_date, (int, str)):
raise AuthError("missing auth_date")
try:
auth_date_int = int(auth_date)
except ValueError as exc:
raise AuthError("invalid auth_date") from exc
# Reject payloads older than 24 hours to limit replay window.
now = int(dt.datetime.now(tz=dt.UTC).timestamp())
if now - auth_date_int > 24 * 60 * 60:
raise AuthError("telegram auth payload expired")
# Build data-check-string from all fields except hash, sorted by key.
data_check_fields = sorted((k, v) for k, v in payload.items() if k != "hash" and v is not None)
data_check_string = "\n".join(f"{k}={v}" for k, v in data_check_fields)
expected_hash = hmac.new(
_telegram_secret_key(bot_token),
data_check_string.encode("utf-8"),
hashlib.sha256,
).hexdigest()
if not _constant_time_compare(received_hash, expected_hash):
raise AuthError("telegram signature mismatch")
telegram_id = payload.get("id")
if not isinstance(telegram_id, int) or telegram_id <= 0:
raise AuthError("missing telegram id")
return UserIdentity(telegram_id=telegram_id)
def verify_telegram_miniapp_init_data(init_data: str, bot_token: str) -> UserIdentity:
"""Verify Telegram Mini App `initData` and extract the user identity.
`initData` is a query-string-like string that contains a `hash` parameter
and (optionally) a JSON `user` parameter. Telegram signs the full string
excluding `hash` using HMAC_SHA256(bot_token, "WebAppData").
Reference: https://core.telegram.org/bots/webapps#validating-data-received-via-the-mini-app
"""
if not bot_token:
raise AuthError("telegram_bot_token is not configured")
if not isinstance(init_data, str) or "=" not in init_data:
raise AuthError("invalid init_data")
params = dict(parse_qsl(init_data, keep_blank_values=True))
received_hash = params.pop("hash", None)
if not received_hash:
raise AuthError("missing hash")
data_check_string = "\n".join(f"{k}={v}" for k, v in sorted(params.items()))
expected_hash = hmac.new(
_telegram_secret_key(bot_token),
data_check_string.encode("utf-8"),
hashlib.sha256,
).hexdigest()
if not _constant_time_compare(received_hash, expected_hash):
raise AuthError("telegram signature mismatch")
user_json = params.get("user")
if not user_json:
raise AuthError("missing user in init_data")
import json
try:
user = json.loads(user_json)
except json.JSONDecodeError as exc:
raise AuthError("invalid user json") from exc
telegram_id = user.get("id")
if not isinstance(telegram_id, int) or telegram_id <= 0:
raise AuthError("missing telegram id")
return UserIdentity(telegram_id=telegram_id)
def verify_bot_identity(telegram_id: int) -> UserIdentity:
"""Identity proof used by the trusted bot adapter.
The bot receives `message.from_user.id` directly from Telegram; here we just
validate it is a positive integer. The API caller (the bot) is authenticated
separately via its service token.
"""
if not isinstance(telegram_id, int) or telegram_id <= 0:
raise AuthError("invalid telegram_id")
return UserIdentity(telegram_id=telegram_id)

123
src/contract_check/core/config.py Normal file
View file

@ -0,0 +1,123 @@
"""Typed configuration via pydantic-settings (12-factor).
Base `Settings` carries everything shared across services. A service imports
this and reads only the fields it needs. All values come from environment
variables (or `.env`); nothing is hardcoded.
Per-service entrypoints may subclass `Settings` to add their own fields
(e.g. the bot adds `BOT_TOKEN`, `API_URL`). See docs/ARCHITECTURE.md §11 for the
full env reference.
"""
from __future__ import annotations
from functools import lru_cache
from typing import Literal
from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
Env = Literal["dev", "staging", "prod"]
RefundPolicy = Literal["all", "infra_only"]
class Settings(BaseSettings):
"""Application configuration. See docs/ARCHITECTURE.md §11."""
model_config = SettingsConfigDict(
env_file=".env",
env_file_encoding="utf-8",
extra="ignore",
case_sensitive=False,
)
# --- runtime ---
env: Env = "dev"
log_level: str = "INFO"
# --- datastores / brokers ---
database_url: str = Field(..., description="async DSN: postgresql+asyncpg://...")
redis_url: str = "redis://redis:6379/0"
rabbitmq_url: str = Field(..., description="amqp://user:pass@host:5672//")
# --- RabbitMQ tuning ---
mq_prefetch_extract: int = 1
mq_prefetch_analyze: int = 3
mq_max_attempts: int = 5
mq_retry_base_ms: int = 2000
# --- object storage (MinIO) ---
s3_endpoint_url: str
s3_access_key: str
s3_secret_key: str
s3_bucket: str = "contract-check-docs"
s3_region: str = "us-east-1"
s3_server_side_encryption: bool = False
doc_retention_days: int = 7
text_retention_days: int = 30
# --- billing ---
refund_policy: RefundPolicy = "all"
# --- observability (empty disables) ---
sentry_dsn: str = ""
otel_exporter_otlp_endpoint: str = ""
otel_service_name: str = "contract-check"
# --- LLM provider ---
llm_provider: str = "ollama_cloud"
ollama_host: str = ""
ollama_api_key: str = ""
ollama_model: str = "qwen2.5:14b"
ollama_fallback_model: str = "qwen2.5:7b"
ollama_temperature: float = 0.2
ollama_num_predict: int = 3072
ollama_timeout: float = 120.0
ollama_max_concurrency: int = 3
# --- API ---
api_host: str = "0.0.0.0"
api_port: int = 8000
api_metrics_port: int = 9100
b2b_default_rate_limit_rps: int = 3
# --- auth (JWT + Telegram identity verification) ---
telegram_bot_token: str = Field(
"", description="Telegram bot token; used to verify Login Widget / Mini App signatures"
)
jwt_secret: str = Field(..., description="HS256 secret for signing user JWTs")
jwt_algorithm: str = "HS256"
jwt_access_ttl_minutes: int = 24 * 60 # 24 hours default; tune per env
# --- logging ---
log_format: str = "json" # json | console
# --- analysis ---
chunk_size_chars: int = 10000
@property
def json_logs(self) -> bool:
"""JSON logs in staging/prod, pretty console in dev.
Override with LOG_FORMAT=json|console.
"""
fmt = self.log_format.lower()
if fmt == "console":
return False
if fmt == "json":
return True
return self.env != "dev"
@property
def log_format_value(self) -> str:
"""Resolved log format: json in prod/staging unless explicitly console."""
fmt = self.log_format.lower()
if fmt in ("json", "console"):
return fmt
return "json" if self.env != "dev" else "console"
@lru_cache
def get_settings() -> Settings:
"""Cached settings singleton. Call `get_settings.cache_clear()` to reset."""
return Settings() # type: ignore[call-arg]

84
src/contract_check/core/credits.py Normal file
View file

@ -0,0 +1,84 @@
"""Credits & refund policy — billing invariants.
Reserve-on-enqueue is sacred: a credit moves ONLY on `POST /documents` in the
api, atomically (never below zero). Refund is sacred: idempotent via the
`documents.refunded` flag (a retried/DLQ message can never double-refund).
`refund_credit` honours the runtime policy `REFUND_POLICY`:
- "all" → refund on any terminal failure
- "infra_only" → refund everything EXCEPT user-garbage input (extraction_failed)
See docs/ARCHITECTURE.md §8. Callers own the transaction (commit after these return).
"""
from __future__ import annotations
import uuid
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from .db.enums import FailureClass, RefundPolicyLike
NON_REFUNDABLE_INFRA_ONLY: frozenset[str] = frozenset({"extraction_failed"})
def should_refund(failure_class: FailureClass | str, policy: RefundPolicyLike) -> bool:
"""Pure policy decision: would this failure be refunded under `policy`?
Under `infra_only`, user-garbage `extraction_failed` is NOT refunded
(the user pays for undetectable garbage). Everything else is.
"""
if policy == "infra_only" and failure_class in NON_REFUNDABLE_INFRA_ONLY:
return False
return True
async def reserve_credit(session: AsyncSession, user_id: uuid.UUID) -> bool:
"""Atomically decrement credits_left by 1. Returns False if none available.
Never lets the balance go negative: the `WHERE credits_left > 0` guard makes
concurrent reservations race-safe — exactly one of N wins the row.
"""
result = await session.execute(
text(
"UPDATE users SET credits_left = credits_left - 1 "
"WHERE id = :u AND credits_left > 0 "
"RETURNING credits_left"
),
{"u": user_id},
)
return result.first() is not None
async def refund_credit(
session: AsyncSession,
document_id: uuid.UUID,
failure_class: FailureClass | str,
policy: RefundPolicyLike,
) -> bool:
"""Refund one credit for a failed document, once. Returns False if skipped.
Idempotent: a second call (retried message, requeue) finds `refunded = TRUE`
and returns False without crediting again. Under `infra_only`, user-garbage
`extraction_failed` is NOT refunded (the user pays for undetectable garbage).
"""
if not should_refund(failure_class, policy):
return False
result = await session.execute(
text(
"UPDATE users SET credits_left = credits_left + 1 "
"WHERE id = (SELECT user_id FROM documents "
" WHERE id = :d AND refunded = FALSE) "
"RETURNING id"
),
{"d": document_id},
)
if result.first() is None:
return False # already refunded (idempotent) or document missing
await session.execute(
text("UPDATE documents SET refunded = TRUE WHERE id = :d"),
{"d": document_id},
)
return True

5
src/contract_check/core/db/__init__.py Normal file
View file

@ -0,0 +1,5 @@
"""Database layer — SQLAlchemy 2 models (6 tables), async session, status enums.
See docs/ARCHITECTURE.md §7 for the full schema and rationale. status/queue/adapter
columns are TEXT + CHECK (not Postgres enums) so migrations stay additive.
"""

67
src/contract_check/core/db/enums.py Normal file
View file

@ -0,0 +1,67 @@
"""Status / class string constants (stored as TEXT+CHECK in Postgres).
Kept as plain Python tuples + Literal aliases (not Postgres ENUMs) so every
migration is additive — matches the convention in docs/IMPLEMENTATION_PLAN.md §1.2.
"""
from __future__ import annotations
from typing import Literal
# documents.status
DocStatus = Literal["queued", "extracting", "ocr", "analyzing", "done", "failed"]
DOC_STATUSES: tuple[str, ...] = (
"queued",
"extracting",
"ocr",
"analyzing",
"done",
"failed",
)
DOC_TERMINAL: tuple[str, ...] = ("done", "failed")
# jobs.status
JobStatus = Literal["pending", "running", "retrying", "dlq", "done"]
JOB_STATUSES: tuple[str, ...] = ("pending", "running", "retrying", "dlq", "done")
# jobs.queue
QueueName = Literal["extract", "analyze"]
QUEUE_NAMES: tuple[str, ...] = ("extract", "analyze")
# service_tokens.adapter
AdapterName = Literal["bot", "web", "cli"]
ADAPTER_NAMES: tuple[str, ...] = ("bot", "web", "cli")
# jobs.last_failure_class — drives the refund policy (see core/credits.py)
FailureClass = Literal[
"extraction_failed",
"ocr_failed",
"llm_quota",
"llm_invalid_output",
"llm_timeout",
"infra",
"unknown",
]
FAILURE_CLASSES: tuple[str, ...] = (
"extraction_failed",
"ocr_failed",
"llm_quota",
"llm_invalid_output",
"llm_timeout",
"infra",
"unknown",
)
# invoices.status (stub table — no ЮKassa logic this refactor)
InvoiceStatus = Literal["draft", "pending", "succeeded", "cancelled", "refunded"]
INVOICE_STATUSES: tuple[str, ...] = (
"draft",
"pending",
"succeeded",
"cancelled",
"refunded",
)
# refund policy switch (REFUND_POLICY env)
RefundPolicyLike = Literal["all", "infra_only"]
REFUND_POLICIES: tuple[str, ...] = ("all", "infra_only")

333
src/contract_check/core/db/models.py Normal file
View file

@ -0,0 +1,333 @@
"""SQLAlchemy 2 declarative models — the 6 production tables.
Tables: users, documents, reports, jobs, service_tokens, invoices (stub).
See docs/ARCHITECTURE.md §7. UUIDs default to gen_random_uuid() server-side
(built into Postgres 13+, no extension needed for pg16).
"""
from __future__ import annotations
import datetime as dt
import uuid
from sqlalchemy import (
BigInteger,
Boolean,
CheckConstraint,
DateTime,
ForeignKey,
Index,
Integer,
String,
Text,
UniqueConstraint,
func,
text,
)
from sqlalchemy.dialects.postgresql import JSONB, UUID
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
class Base(DeclarativeBase):
"""Declarative base for all models."""
def _now() -> dt.datetime:
return dt.datetime.now(tz=dt.UTC)
class User(Base):
__tablename__ = "users"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
primary_key=True,
server_default=text("gen_random_uuid()"),
)
telegram_id: Mapped[int | None] = mapped_column(BigInteger, unique=True)
created_at: Mapped[dt.datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
credits_left: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
__table_args__ = (CheckConstraint("credits_left >= 0", name="users_credits_nonneg"),)
documents: Mapped[list[Document]] = relationship(
back_populates="user", cascade="all, delete-orphan"
)
api_keys: Mapped[list[ApiKey]] = relationship(
back_populates="user", cascade="all, delete-orphan"
)
class Document(Base):
__tablename__ = "documents"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
primary_key=True,
server_default=text("gen_random_uuid()"),
)
user_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("users.id", ondelete="CASCADE"),
nullable=False,
)
s3_key: Mapped[str] = mapped_column(Text, nullable=False)
extracted_s3_key: Mapped[str | None] = mapped_column(Text)
filename: Mapped[str] = mapped_column(Text, nullable=False)
mime: Mapped[str] = mapped_column(Text, nullable=False)
bytes_: Mapped[int] = mapped_column("bytes", BigInteger, nullable=False, default=0)
status: Mapped[str] = mapped_column(
String,
nullable=False,
default="queued",
server_default=text("'queued'"),
)
stage: Mapped[str | None] = mapped_column(Text)
refunded: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default=text("false")
)
created_at: Mapped[dt.datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[dt.datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=_now
)
user: Mapped[User] = relationship(back_populates="documents")
report: Mapped[Report | None] = relationship(
back_populates="document", cascade="all, delete-orphan", uselist=False
)
jobs: Mapped[list[Job]] = relationship(back_populates="document", cascade="all, delete-orphan")
api_key_requests: Mapped[list[ApiKeyRequest]] = relationship(
back_populates="document", cascade="all, delete-orphan"
)
__table_args__ = (
CheckConstraint(
"status IN ('queued','extracting','ocr','analyzing','done','failed')",
name="documents_status_check",
),
Index("documents_user_created_idx", "user_id", "created_at"),
Index("documents_status_idx", "status"),
)
class Report(Base):
__tablename__ = "reports"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
primary_key=True,
server_default=text("gen_random_uuid()"),
)
document_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("documents.id", ondelete="CASCADE"),
nullable=False,
unique=True,
)
content_json: Mapped[dict[str, object]] = mapped_column(JSONB, nullable=False)
markdown: Mapped[str] = mapped_column(Text, nullable=False)
model_used: Mapped[str | None] = mapped_column(Text)
prompt_tokens: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
eval_tokens: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
latency_ms: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
created_at: Mapped[dt.datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
document: Mapped[Document] = relationship(back_populates="report")
class Job(Base):
__tablename__ = "jobs"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
primary_key=True,
server_default=text("gen_random_uuid()"),
)
document_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("documents.id", ondelete="CASCADE"),
nullable=False,
)
correlation_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), nullable=False)
queue: Mapped[str] = mapped_column(String, nullable=False)
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=5)
last_failure_class: Mapped[str | None] = mapped_column(Text)
last_error: Mapped[str | None] = mapped_column(Text)
dlq: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default=text("false")
)
status: Mapped[str] = mapped_column(
String,
nullable=False,
default="pending",
server_default=text("'pending'"),
)
created_at: Mapped[dt.datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[dt.datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=_now
)
document: Mapped[Document] = relationship(back_populates="jobs")
__table_args__ = (
CheckConstraint("queue IN ('extract','analyze')", name="jobs_queue_check"),
CheckConstraint(
"status IN ('pending','running','retrying','dlq','done')",
name="jobs_status_check",
),
Index("jobs_correlation_idx", "correlation_id"),
Index("jobs_document_idx", "document_id"),
)
class ServiceToken(Base):
__tablename__ = "service_tokens"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
primary_key=True,
server_default=text("gen_random_uuid()"),
)
name: Mapped[str] = mapped_column(Text, unique=True, nullable=False)
token_hash: Mapped[str] = mapped_column(Text, nullable=False)
adapter: Mapped[str] = mapped_column(String, nullable=False)
revoked: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default=text("false")
)
created_at: Mapped[dt.datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
last_used_at: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True))
__table_args__ = (
CheckConstraint("adapter IN ('bot','web','cli')", name="service_tokens_adapter_check"),
)
class ApiKey(Base):
"""B2B API key: opaque bearer secret hashed via SHA-256."""
__tablename__ = "api_keys"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
primary_key=True,
server_default=text("gen_random_uuid()"),
)
user_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("users.id", ondelete="CASCADE"),
nullable=False,
)
name: Mapped[str] = mapped_column(Text, nullable=False)
key_hash: Mapped[str] = mapped_column(Text, nullable=False)
rate_limit_rps: Mapped[int] = mapped_column(
Integer, nullable=False, default=3, server_default=text("3")
)
monthly_quota: Mapped[int | None] = mapped_column(Integer)
monthly_used: Mapped[int] = mapped_column(
Integer, nullable=False, default=0, server_default=text("0")
)
resets_at: Mapped[dt.datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
revoked: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default=text("false")
)
created_at: Mapped[dt.datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
last_used_at: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True))
user: Mapped[User] = relationship(back_populates="api_keys")
requests: Mapped[list[ApiKeyRequest]] = relationship(
back_populates="api_key", cascade="all, delete-orphan"
)
__table_args__ = (
CheckConstraint("rate_limit_rps > 0", name="api_keys_rate_limit_positive"),
CheckConstraint("monthly_used >= 0", name="api_keys_monthly_used_nonneg"),
CheckConstraint(
"monthly_quota IS NULL OR monthly_quota >= 0",
name="api_keys_monthly_quota_nonneg",
),
Index("api_keys_user_idx", "user_id"),
Index("api_keys_hash_idx", "key_hash"),
UniqueConstraint("user_id", "name", name="api_keys_user_name_unique"),
)
class ApiKeyRequest(Base):
"""One B2B API call = one request row for usage/dashboard."""
__tablename__ = "api_key_requests"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
primary_key=True,
server_default=text("gen_random_uuid()"),
)
api_key_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("api_keys.id", ondelete="CASCADE"),
nullable=False,
)
document_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("documents.id", ondelete="CASCADE"),
nullable=False,
)
created_at: Mapped[dt.datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
api_key: Mapped[ApiKey] = relationship(back_populates="requests")
document: Mapped[Document] = relationship(back_populates="api_key_requests")
__table_args__ = (
Index("api_key_requests_key_created_idx", "api_key_id", text("created_at DESC")),
)
class Invoice(Base):
"""STUB — schema forward-compatible for ЮKassa. No payment logic yet."""
__tablename__ = "invoices"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
primary_key=True,
server_default=text("gen_random_uuid()"),
)
user_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("users.id", ondelete="CASCADE"),
nullable=False,
)
amount: Mapped[int] = mapped_column(Integer, nullable=False) # kopecks
status: Mapped[str] = mapped_column(
String, nullable=False, default="draft", server_default=text("'draft'")
)
provider: Mapped[str | None] = mapped_column(Text)
external_id: Mapped[str | None] = mapped_column(Text)
created_at: Mapped[dt.datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
paid_at: Mapped[dt.datetime | None] = mapped_column(DateTime(timezone=True))
__table_args__ = (
CheckConstraint(
"status IN ('draft','pending','succeeded','cancelled','refunded')",
name="invoices_status_check",
),
Index("invoices_user_idx", "user_id", "created_at"),
)

43
src/contract_check/core/db/session.py Normal file
View file

@ -0,0 +1,43 @@
"""Async engine + session factory.
All services share the same engine construction. The api uses
`get_session()` as a FastAPI dependency (wired in Step 2); workers build
sessions directly from the factory. See docs/ARCHITECTURE.md §7.
"""
from __future__ import annotations
from collections.abc import AsyncIterator
from typing import Any
from sqlalchemy.ext.asyncio import (
AsyncEngine,
AsyncSession,
async_sessionmaker,
create_async_engine,
)
from ..config import get_settings
def create_engine(url: str | None = None, **kwargs: Any) -> AsyncEngine:
"""Build an async engine from DATABASE_URL (or an explicit url)."""
url = url or get_settings().database_url
return create_async_engine(url, pool_pre_ping=True, **kwargs)
def create_session_factory(
engine: AsyncEngine | None = None,
) -> async_sessionmaker[AsyncSession]:
"""Build an async_sessionmaker bound to `engine` (or a fresh one)."""
engine = engine or create_engine()
return async_sessionmaker(engine, expire_on_commit=False)
async def get_session(
factory: async_sessionmaker[AsyncSession] | None = None,
) -> AsyncIterator[AsyncSession]:
"""FastAPI/yield-style session dependency."""
factory = factory or create_session_factory()
async with factory() as session:
yield session

17
src/contract_check/core/errors.py Normal file
View file

@ -0,0 +1,17 @@
"""Shared exception markers used across service boundaries.
These are intentionally lightweight and cross-cutting so that a RabbitMQ
consumer (core/mq) can recognize a terminal failure raised by an adapter
(core/llm) without creating a circular dependency.
"""
from __future__ import annotations
class TerminalError(Exception):
"""Raised for failures that should not be retried by the MQ consumer.
Subclasses indicate a permanent or configuration-level problem (e.g. an
unreachable host, missing model, or invalid credentials). The consumer will
route the message straight to the DLQ instead of retrying.
"""

12
src/contract_check/core/llm/__init__.py Normal file
View file

@ -0,0 +1,12 @@
"""LLM provider layer — abstract port + adapters.
The first and only realization is Ollama Cloud (ported from the stage-0
prototype). Future providers (self-hosted Ollama, GigaChat, YandexGPT) plug
in here without touching analysis code. See docs/ARCHITECTURE.md §9.
"""
from __future__ import annotations
from .port import AnalysisResult, LLMProvider
__all__ = ["LLMProvider", "AnalysisResult"]

32
src/contract_check/core/llm/factory.py Normal file
View file

@ -0,0 +1,32 @@
"""LLM provider factory — selects an adapter by `LLM_PROVIDER` (env).
Add new providers here (self-hosted Ollama, GigaChat, YandexGPT) and register
a case. The analysis code never changes.
"""
from __future__ import annotations
from ..config import Settings
from .ollama_cloud import OllamaCloudProvider
from .port import LLMProvider
def build_llm_provider(settings: Settings) -> LLMProvider:
"""Build the configured LLM provider. Defaults to Ollama Cloud."""
match settings.llm_provider:
case "ollama_cloud":
return OllamaCloudProvider(
host=settings.ollama_host,
api_key=settings.ollama_api_key,
model=settings.ollama_model,
fallback_model=settings.ollama_fallback_model or None,
temperature=settings.ollama_temperature,
num_predict=settings.ollama_num_predict,
timeout=settings.ollama_timeout,
max_concurrency=settings.ollama_max_concurrency,
chunk_size=settings.chunk_size_chars,
)
case _:
raise ValueError(
f"unknown LLM_PROVIDER={settings.llm_provider!r} (expected: 'ollama_cloud')"
)

465
src/contract_check/core/llm/ollama_cloud.py Normal file
View file

@ -0,0 +1,465 @@
"""Ollama Cloud adapter — first realization of LLMProvider.
Ported from the stage-0 prototype `llm_client.py` (bearer httpx, `format:
json-schema`, 5xx backoff, 429→fallback, pydantic repair-loop) and wrapped to
implement `LLMProvider.analyze(text, *, checklist)` with chunk fan-out,
de-duplication, and severity sort. The worker maps the raised exceptions to a
FailureClass (see core/mq/consumer.py + worker_analyze/handler.py).
"""
from __future__ import annotations
import asyncio
import json
import time
from dataclasses import dataclass
from types import TracebackType
from typing import Any, Self
import httpx
from pydantic import BaseModel, ValidationError
from ..analysis.analyzer import build_user_prompt, dedupe_findings, sort_findings
from ..analysis.checklist import checklist_for_prompt
from ..analysis.chunker import chunk_text
from ..analysis.report_schema import ReportPayload
from ..errors import TerminalError
from ..logging import get_logger
from .port import AnalysisResult
log = get_logger(__name__)
# How much of a payload/response we keep in log lines. Full contracts can be
# large; we log enough to debug schema/JSON issues without spamming or leaking
# entire documents at info/warning level.
_LOG_MAX_PAYLOAD_CHARS = 1200
REPAIR_SYSTEM = (
"Твой предыдущий ответ не прошёл валидацию JSON-схемы. "
"Верни СТРОГО валидный JSON, соответствующий схеме, без пояснений и markdown. "
"Цитаты — дословные из исходного текста договора."
)
SYSTEM_PROMPT = (
"Ты — помощник первичного юридического скрининга договоров по праву "
"РФ / РБ (ГК РФ, ГК РБ). Твоя задача — найти РИСКИ для стороны, которая "
"обращается за анализом (считай её «нашим клиентом»), по чек-листу ниже.\n\n"
"Жёсткие правила:\n"
"1. Анализируй ТОЛЬКО текст договора в этом сообщении. Не придумывай пункты.\n"
"2. На каждую находку ОБЯЗАТЕЛЬНО: дословная цитата из оригинала (поле quote) "
"и номер/название пункта договора (поле section_ref), чтобы сверить одним кликом.\n"
"3. Если по какому-то пункту чек-листа риска нет — НЕ включай его в findings.\n"
"4. severity: high — существенный риск (штрафы, потеря прав, чужая подсудность); "
"medium — стоит уточнить; low — мелочь на заметку.\n"
"5. Возвращай ТОЛЬКО JSON по схеме, без markdown и пояснений вне JSON.\n\n"
"Чек-лист пунктов анализа:\n{checklist}"
)
class LLMError(Exception):
"""Unrecoverable LLM failure (after all retries)."""
class LLMQuotaError(LLMError):
"""429 / quota on both primary and fallback — refundable failure."""
class LLMUnavailableError(LLMError):
"""Ollama server unreachable or returns non-200 status — retryable."""
class LLMConfigError(LLMError, TerminalError):
"""Misconfigured Ollama host/model/endpoint — terminal, do not retry."""
class _QuotaSignal(Exception):
"""Internal: 429 triggers fallback within the same call."""
@dataclass(slots=True)
class _ChatResult:
data: BaseModel
model_used: str
fell_back: bool
repaired: bool
attempts: int
prompt_tokens: int
eval_tokens: int
latency_sec: float
class OllamaCloudProvider:
"""LLMProvider backed by Ollama Cloud (hosted open models)."""
def __init__(
self,
*,
host: str,
api_key: str,
model: str,
fallback_model: str | None = None,
temperature: float = 0.2,
num_predict: int = 3072,
timeout: float = 120.0,
max_concurrency: int = 3,
chunk_size: int = 10000,
) -> None:
self._chunk_size = chunk_size
self._model = model
self._fallback = fallback_model or None
self._temperature = temperature
self._num_predict = num_predict
self._retries = 3
self._sem = asyncio.Semaphore(max(1, max_concurrency))
self._client = httpx.AsyncClient(
base_url=host.rstrip("/"),
headers={
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
},
timeout=httpx.Timeout(timeout, connect=10.0),
)
async def aclose(self) -> None:
await self._client.aclose()
async def __aenter__(self) -> Self:
return self
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc: BaseException | None,
tb: TracebackType | None,
) -> None:
await self.aclose()
# ── public: LLMProvider.analyze ──────────────────────────────────────────
async def analyze(self, text: str, *, checklist: str | None = None) -> AnalysisResult:
checklist = checklist or checklist_for_prompt()
chunks = chunk_text(text, max_chars=self._chunk_size)
system_msg = SYSTEM_PROMPT.format(checklist=checklist)
async def one(chunk: str) -> _ChatResult:
async with self._sem:
return await self._run_with_fallback(
messages=[
{"role": "system", "content": system_msg},
{"role": "user", "content": build_user_prompt(chunk)},
],
schema_model=ReportPayload,
)
results = await asyncio.gather(*(one(c) for c in chunks))
findings = []
prompt_tokens = eval_tokens = 0
latency = 0.0
models_used: set[str] = set()
fell_back = repaired = False
for r in results:
payload: ReportPayload = r.data # type: ignore[assignment]
findings.extend(payload.findings)
prompt_tokens += r.prompt_tokens
eval_tokens += r.eval_tokens
latency = max(latency, r.latency_sec)
models_used.add(r.model_used)
fell_back |= r.fell_back
repaired |= r.repaired
findings = sort_findings(dedupe_findings(findings))
return AnalysisResult(
findings=findings,
model_used=", ".join(sorted(models_used)),
fell_back=fell_back,
repaired=repaired,
prompt_tokens=prompt_tokens,
eval_tokens=eval_tokens,
latency_sec=latency,
models_used=models_used,
)
# ── internals (ported from prototype llm_client.py) ─────────────────────
async def _run_with_fallback(
self,
messages: list[dict[str, str]],
schema_model: type[BaseModel],
) -> _ChatResult:
schema = schema_model.model_json_schema()
try:
return await self._run_repair_loop(messages, schema_model, schema, self._model)
except _QuotaSignal:
if self._fallback and self._model != self._fallback:
log.warning("quota_429_fallback", primary=self._model, fallback=self._fallback)
return await self._run_repair_loop(messages, schema_model, schema, self._fallback)
raise LLMQuotaError(f"429/quota on {self._model} and no usable fallback") from None
async def _run_repair_loop(
self,
messages: list[dict[str, str]],
schema_model: type[BaseModel],
schema: dict[str, Any],
model: str,
) -> _ChatResult:
started = time.perf_counter()
attempts = 0
repaired = False
current = list(messages)
while True:
attempts += 1
content, usage = await self._post_chat(current, model, schema)
cleaned = _strip_markdown_fences(content)
try:
data = _parse_and_validate(cleaned, schema_model)
except (ValidationError, ValueError) as exc:
error_summary = _format_parse_error(exc)
log.warning(
"invalid_json_repair",
model=model,
error=type(exc).__name__,
attempt=attempts,
content_preview=cleaned[:_LOG_MAX_PAYLOAD_CHARS],
content_type=type(content).__name__,
content_len=len(content),
**error_summary,
)
if attempts <= 1:
repaired = True
current = [
{"role": "system", "content": REPAIR_SYSTEM},
{"role": "user", "content": messages[-1]["content"]},
{"role": "assistant", "content": cleaned[:4000]},
{"role": "user", "content": "Верни только валидный JSON по схеме."},
]
continue
raise LLMError(f"Model {model} returned invalid JSON after repair: {exc}") from exc
return _ChatResult(
data=data,
model_used=model,
fell_back=(model != self._model),
repaired=repaired,
attempts=attempts,
prompt_tokens=usage[0],
eval_tokens=usage[1],
latency_sec=time.perf_counter() - started,
)
async def _post_chat(
self,
messages: list[dict[str, str]],
model: str,
schema: dict[str, Any],
) -> tuple[str, tuple[int, int]]:
payload = {
"model": model,
"messages": messages,
"format": schema,
"stream": False,
"options": {
"temperature": self._temperature,
"num_predict": self._num_predict,
},
}
log.debug(
"ollama_request",
model=model,
messages=_safe_messages(messages, _LOG_MAX_PAYLOAD_CHARS),
schema=_safe_schema(schema),
)
last_exc: Exception | None = None
for attempt in range(1, self._retries + 1):
try:
resp = await self._client.post("/api/chat", json=payload)
except httpx.ConnectError as exc:
# ConnectError means the configured host is not listening.
# Retrying will not fix a wrong host, so fail terminal immediately.
raise LLMConfigError(
f"Cannot connect to Ollama server at configured host: {exc}. "
"Verify OLLAMA_HOST points to a running Ollama server."
) from exc
except httpx.TimeoutException as exc:
last_exc = exc
log.warning(
"ollama_timeout",
model=model,
attempt=attempt,
error=f"{type(exc).__name__}: {exc}",
request_messages=_safe_messages(messages, _LOG_MAX_PAYLOAD_CHARS),
)
if attempt < self._retries:
await _backoff(attempt)
continue
raise LLMUnavailableError(
f"Ollama request timed out after {self._retries} attempts: {exc}"
) from exc
except httpx.HTTPError as exc:
last_exc = exc
log.warning(
"ollama_http_error",
model=model,
attempt=attempt,
error=f"{type(exc).__name__}: {exc}",
request_messages=_safe_messages(messages, _LOG_MAX_PAYLOAD_CHARS),
)
if attempt < self._retries:
await _backoff(attempt)
continue
raise LLMUnavailableError(
f"HTTP error calling Ollama after {self._retries} attempts: {exc}"
) from exc
if resp.status_code == 429:
raise _QuotaSignal(f"429 Too Many Requests (model={model})")
if resp.status_code >= 500:
if attempt < self._retries:
log.warning(
"http_5xx_retry",
status=resp.status_code,
model=model,
attempt=attempt,
response_text=resp.text[:500],
)
await _backoff(attempt)
continue
raise LLMUnavailableError(f"HTTP {resp.status_code} from Ollama: {resp.text[:500]}")
if resp.status_code == 404:
raise LLMConfigError(
f"Ollama endpoint not found (404): {resp.text[:500]}. "
"Verify OLLAMA_HOST points to a running Ollama server."
)
if resp.status_code >= 400:
raise LLMError(f"HTTP {resp.status_code} from Ollama: {resp.text[:500]}")
try:
body = resp.json()
except Exception as exc:
log.error(
"ollama_response_not_json",
model=model,
response_text=resp.text[:_LOG_MAX_PAYLOAD_CHARS],
error=f"{type(exc).__name__}: {exc}",
)
raise LLMError(f"Invalid JSON response from Ollama: {exc}") from exc
raw_content = body.get("message", {}).get("content", "")
content = _normalize_content(raw_content).strip()
prompt_tokens = int(body.get("prompt_eval_count", 0) or 0)
eval_tokens = int(body.get("eval_count", 0) or 0)
if not content:
log.error(
"ollama_empty_content",
model=model,
raw_content_type=type(raw_content).__name__,
body=_safe_body(body, _LOG_MAX_PAYLOAD_CHARS),
)
raise LLMError("Empty message content from Ollama")
log.debug(
"ollama_response",
model=model,
content=content[:_LOG_MAX_PAYLOAD_CHARS],
prompt_tokens=prompt_tokens,
eval_tokens=eval_tokens,
)
return content, (prompt_tokens, eval_tokens)
raise LLMUnavailableError(f"Exhausted retries: {last_exc}")
def _parse_and_validate(content: str, schema_model: type[BaseModel]) -> BaseModel:
data = json.loads(content)
return schema_model.model_validate(data)
def _safe_messages(messages: list[dict[str, str]], max_chars: int) -> list[dict[str, str]]:
"""Return a copy of messages with long contents truncated for logging."""
out: list[dict[str, str]] = []
for msg in messages:
content = msg.get("content", "")
out.append(
{
"role": msg.get("role", "unknown"),
"content": content[:max_chars] + ("..." if len(content) > max_chars else ""),
}
)
return out
def _safe_schema(schema: dict[str, Any]) -> dict[str, Any]:
"""Return a schema safe for logs: title/properties preserved, no full examples."""
# Schema objects are small; log as-is but drop any possible long examples.
safe: dict[str, Any] = {
k: v for k, v in schema.items() if k not in ("examples", "$defs", "definitions")
}
# Limit property list for very large schemas.
properties = safe.get("properties")
if isinstance(properties, dict) and len(properties) > 30:
safe["properties"] = {k: properties[k] for k in list(properties.keys())[:30]}
safe["_properties_truncated"] = True
return safe
def _safe_body(body: dict[str, Any], max_chars: int) -> dict[str, Any]:
"""Return a copy of the Ollama chat response body with long strings truncated."""
safe: dict[str, Any] = {}
for k, v in body.items():
if isinstance(v, bytes):
safe[k] = _normalize_content(v)[:max_chars]
elif isinstance(v, str):
safe[k] = v[:max_chars] + ("..." if len(v) > max_chars else "")
elif isinstance(v, dict):
safe[k] = _safe_body(v, max_chars)
else:
safe[k] = v
return safe
def _normalize_content(value: object) -> str:
"""Turn whatever Ollama put in message.content into a clean str.
Ollama Cloud occasionally returns ``bytes`` instead of ``str``.
``str(bytes)`` produces a Python repr like "b'...'" which breaks JSON
parsing. We decode UTF-8 (with fallback) and keep plain strings untouched.
"""
if isinstance(value, str):
return value
if isinstance(value, bytes):
try:
return value.decode("utf-8")
except UnicodeDecodeError:
return value.decode("utf-8", errors="replace")
if value is None:
return ""
return str(value)
def _strip_markdown_fences(text: str) -> str:
"""Remove ```json ... ``` or ``` ... ``` wrappers that models often emit."""
stripped = text.strip()
if stripped.startswith("```"):
# Drop the opening fence (possibly with 'json' label).
first_newline = stripped.find("\n")
if first_newline != -1:
stripped = stripped[first_newline + 1 :]
else:
stripped = stripped[3:]
stripped = stripped.rstrip()
if stripped.endswith("```"):
stripped = stripped[:-3].rstrip()
return stripped
def _format_parse_error(exc: BaseException) -> dict[str, object]:
"""Return a small, log-safe summary of a JSON/Pydantic parse failure."""
if isinstance(exc, ValidationError):
return {
"validation_errors": [
{"loc": list(e.get("loc", [])), "msg": e.get("msg", ""), "type": e.get("type", "")}
for e in exc.errors()
],
}
return {"parse_error": str(exc)[:500]}
async def _backoff(attempt: int) -> None:
await asyncio.sleep(min(8.0, 0.5 * (2 ** (attempt - 1))))

34
src/contract_check/core/llm/port.py Normal file
View file

@ -0,0 +1,34 @@
"""LLM provider port + AnalysisResult contract.
`analyze(text, *, checklist)` returns the merged, de-duplicated, severity-
sorted findings for a whole contract (the provider handles chunking + fan-out
internally). Failures bubble as exceptions the worker classifies into a
FailureClass for the refund policy.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Protocol, runtime_checkable
from ..analysis.report_schema import Finding
@dataclass(slots=True)
class AnalysisResult:
findings: list[Finding]
model_used: str = ""
fell_back: bool = False
repaired: bool = False
prompt_tokens: int = 0
eval_tokens: int = 0
latency_sec: float = 0.0
# aggregate metrics; populated by adapters
models_used: set[str] = field(default_factory=set)
@runtime_checkable
class LLMProvider(Protocol):
async def analyze(self, text: str, *, checklist: str) -> AnalysisResult: ...
async def aclose(self) -> None: ...

231
src/contract_check/core/logging.py Normal file
View file

@ -0,0 +1,231 @@
"""Structured logging — structlog JSON + correlation_id propagation.
The correlation_id is the spine of every log line in the event-driven system.
It is set:
- in the api by middleware (from `X-Correlation-ID` header or minted),
- in workers by the consumer base (from the RabbitMQ message header
`x-correlation-id`),
so a single upload's logs trace api → rabbit → worker-extract →
worker-analyze → DB under one id.
Additional global context (service, env, version) is bound at startup and
inherited by every logger. Per-request/job attributes can be added with
`bind_context(**extra)`.
"""
from __future__ import annotations
import contextvars
import logging
import os
import sys
import traceback
import uuid
import structlog
from structlog.stdlib import BoundLogger
from structlog.types import EventDict, WrappedLogger
correlation_id_var: contextvars.ContextVar[str] = contextvars.ContextVar(
"correlation_id", default=""
)
service_var: contextvars.ContextVar[str] = contextvars.ContextVar("service", default="")
env_var: contextvars.ContextVar[str] = contextvars.ContextVar("env", default="")
def set_correlation_id(value: str | None) -> None:
"""Set the correlation id for the current async context."""
correlation_id_var.set(value or "")
def get_correlation_id() -> str:
"""Return the current correlation id ("" if unset)."""
return correlation_id_var.get()
def new_correlation_id() -> str:
"""Mint a new correlation id and bind it to the current context."""
cid = str(uuid.uuid4())
set_correlation_id(cid)
return cid
def set_service(value: str | None) -> None:
"""Bind the service name to the current logging context."""
service_var.set(value or "")
def set_env(value: str | None) -> None:
"""Bind the runtime environment to the current logging context."""
env_var.set(value or "")
def bind_context(**kwargs: object) -> None:
"""Bind extra key/value pairs to the current structlog context.
These appear in every subsequent log line in this context until cleared.
"""
structlog.contextvars.bind_contextvars(**{k: v for k, v in kwargs.items() if v is not None})
def clear_context() -> None:
"""Clear all structlog contextvars (use with care, mostly for tests)."""
structlog.contextvars.clear_contextvars()
correlation_id_var.set("")
service_var.set("")
env_var.set("")
def _inject_static_context(
_logger: WrappedLogger, _method_name: str, event_dict: EventDict
) -> EventDict:
cid = correlation_id_var.get()
if cid:
event_dict["correlation_id"] = cid
service = service_var.get()
if service:
event_dict["service"] = service
env = env_var.get()
if env:
event_dict["env"] = env
return event_dict
def _add_version(_logger: WrappedLogger, _method_name: str, event_dict: EventDict) -> EventDict:
version = os.getenv("APP_VERSION", "unknown")
if version:
event_dict["version"] = version
return event_dict
def _format_exc_info(_logger: WrappedLogger, _method_name: str, event_dict: EventDict) -> EventDict:
"""Attach a full, structured exception payload when an exception is in flight.
Unlike the default structlog processor this never truncates the traceback
and always separates exception class, message, and traceback so log parsers
can group by exception_type.
"""
exc_info = event_dict.get("exc_info", False)
if not exc_info:
return event_dict
if isinstance(exc_info, bool):
exc_info = sys.exc_info()
if exc_info is None or exc_info == (None, None, None):
event_dict.pop("exc_info", None)
return event_dict
exc_type, exc_value, exc_tb = exc_info
if exc_value is None:
event_dict.pop("exc_info", None)
return event_dict
event_dict["exception_type"] = exc_type.__name__ if exc_type else "UnknownException"
event_dict["exception_message"] = str(exc_value)
event_dict["exception_module"] = (
exc_type.__module__ if exc_type and exc_type.__module__ != "builtins" else None
)
event_dict["exception_traceback"] = "".join(
traceback.format_exception(exc_type, exc_value, exc_tb)
)
event_dict["exc_info"] = False
return event_dict
def _filter_sensitive_keys(
_logger: WrappedLogger, _method_name: str, event_dict: EventDict
) -> EventDict:
"""Redact obvious secrets from log payloads when they leak into keyword args."""
sensitive = {"api_key", "token", "secret", "password", "authorization"}
for key in event_dict:
if any(s in key.lower() for s in sensitive):
value = event_dict[key]
if isinstance(value, str) and value:
event_dict[key] = value[:4] + "***"
return event_dict
def configure_logging(
level: str = "INFO",
*,
json_output: bool = True,
service: str = "contract-check",
env: str = "dev",
) -> None:
"""Configure structlog + stdlib logging.
json_output=True (staging/prod) → JSON renderer; False (dev) → colored console.
service and env are bound globally so every log line carries them.
"""
set_service(service)
set_env(env)
processors: list[structlog.types.Processor] = [
structlog.contextvars.merge_contextvars,
structlog.processors.add_log_level,
structlog.processors.TimeStamper(fmt="iso", utc=True),
_inject_static_context,
_add_version,
structlog.processors.StackInfoRenderer(),
_format_exc_info,
_filter_sensitive_keys,
]
processors.append(
structlog.processors.JSONRenderer(sort_keys=True)
if json_output
else structlog.dev.ConsoleRenderer(colors=True, pad_event=False)
)
structlog.configure(
processors=processors,
wrapper_class=structlog.make_filtering_bound_logger(_level_to_int(level)),
context_class=dict,
logger_factory=structlog.PrintLoggerFactory(file=sys.stderr),
cache_logger_on_first_use=True,
)
# Route stdlib logging through structlog so third-party libs match our format.
stdlib_handler = logging.StreamHandler(sys.stderr)
stdlib_handler.setFormatter(
structlog.stdlib.ProcessorFormatter(
processor=processors[-1],
foreign_pre_chain=processors[:-1],
)
)
root_logger = logging.getLogger()
root_logger.handlers.clear()
root_logger.addHandler(stdlib_handler)
root_logger.setLevel(_level_to_int(level))
def _level_to_int(level: str) -> int:
return getattr(logging, level.upper(), logging.INFO)
def get_logger(name: str | None = None) -> BoundLogger:
"""Return a bound structlog logger."""
logger = structlog.get_logger(name)
return logger # type: ignore[no-any-return]
def is_debug_enabled() -> bool:
"""True when DEBUG-level records are emitted under the current configuration.
structlog's filtering bound logger exposes no ``isEnabledFor``; both it and
the stdlib root logger are set to the same level in :func:`configure_logging`,
so the root logger is the authoritative level source. Call this instead of
touching stdlib ``logging`` from feature code.
"""
return logging.getLogger().isEnabledFor(logging.DEBUG)
def log_error(
logger: BoundLogger,
event: str,
exc: BaseException | None = None,
**kwargs: object,
) -> None:
"""Structured error helper: captures full traceback and exception fields."""
if exc is not None:
kwargs["exc_info"] = (type(exc), exc, exc.__traceback__)
logger.error(event, **kwargs)

86
src/contract_check/core/metrics.py Normal file
View file

@ -0,0 +1,86 @@
"""Prometheus metrics registry + a background metrics HTTP server.
The api exposes `/metrics` itself (FastAPI route). Workers (which don't serve
HTTP for their main job) call `start_metrics_server(port)` to run a tiny
prometheus_client HTTP listener in a background thread on :9101/:9102.
This module imports `prometheus_client` — only obs-enabled services use it.
"""
from __future__ import annotations
import threading
from prometheus_client import Counter, Gauge, Histogram, start_http_server
from .logging import get_logger
log = get_logger(__name__)
_started: dict[int, threading.Thread] = {}
# ── counters ─────────────────────────────────────────────────────────────────
documents_uploaded = Counter(
"contract_check_documents_uploaded_total",
"Documents accepted by the api and enqueued.",
)
credits_reserved = Counter(
"contract_check_credits_reserved_total",
"Credits reserved on enqueue (successful).",
)
credits_refunded = Counter(
"contract_check_credits_refunded_total",
"Credits refunded on terminal failure.",
["policy"], # all | infra_only
)
mq_published = Counter(
"contract_check_mq_published_total",
"Messages published to RabbitMQ.",
["queue"], # extract | analyze
)
mq_failed = Counter(
"contract_check_mq_failed_total",
"Jobs that hit a failure class.",
["queue", "failure_class"],
)
llm_tokens = Counter(
"contract_check_llm_tokens_total",
"LLM tokens consumed by the analyze worker.",
["kind"], # prompt | eval
)
llm_fell_back = Counter(
"contract_check_llm_fell_back_total",
"Times the LLM provider switched to the fallback model.",
)
redis_connected = Gauge(
"contract_check_redis_connected",
"Whether the api successfully connected to Redis at startup.",
)
# ── histograms ───────────────────────────────────────────────────────────────
http_request_duration = Histogram(
"contract_check_http_request_duration_seconds",
"HTTP request latency.",
["method", "path", "status"],
)
extract_duration = Histogram(
"contract_check_extract_duration_seconds",
"Document extraction (extract.q handler) latency.",
)
analyze_duration = Histogram(
"contract_check_analyze_duration_seconds",
"Document analysis (analyze.q handler) latency.",
)
def start_metrics_server(port: int) -> None:
"""Start a prometheus_client HTTP listener on `port` (idempotent per port).
Used by workers; the api serves /metrics itself instead.
"""
if port in _started:
return
start_http_server(port)
log.info("metrics_server_started", port=port)

6
src/contract_check/core/mq/__init__.py Normal file
View file

@ -0,0 +1,6 @@
"""RabbitMQ layer — topology, message schemas, publisher, consumer base.
See docs/ARCHITECTURE.md §5 for the full topology spec (direct exchange, pipeline
fan-out extract.q → analyze.q, quorum main queues, classic TTL retry queues,
quorum DLQs). All declarations are idempotent and run on every service start.
"""

282
src/contract_check/core/mq/consumer.py Normal file
View file

@ -0,0 +1,282 @@
"""RabbitMQ consumer base with retry/DLQ mechanics.
Subclasses (worker-extract, worker-analyze, Steps 3–4) implement:
- `handle(payload)` — the per-message work; raise on failure
- `classify(exc)` — map an exception to a FailureClass (default unknown)
- `on_failure(...)` — record attempts/last_failure_class (default no-op)
- `on_dlq(...)` — terminal: status=failed + refund (default no-op)
Retry is MANUAL (plugin-free, exponential): on a classified failure the base
publishes a copy of the message to the retry exchange with `expiration =
base * 2^attempt` and `x-attempt` incremented, then acks the original. When
the retry queue's TTL expires it dead-letters back to the main queue. After
`max_attempts`, the message is published to the matching DLQ instead.
A body that fails pydantic validation is sent straight to the DLQ with
failure_class="infra" (poison message). See docs/ARCHITECTURE.md §5.
"""
from __future__ import annotations
import asyncio
from typing import TYPE_CHECKING, Any
from aio_pika import DeliveryMode, Message, connect_robust
from pydantic import ValidationError
from ..db.enums import FailureClass
from ..errors import TerminalError
from ..logging import get_logger, set_correlation_id
from .messages import PipelineMessage
from .topology import (
DLQ_FOR,
EXCHANGE_RETRY,
H_ATTEMPT,
H_CORRELATION_ID,
H_ORIGIN,
declare_all,
ensure_retry_queues_lazy,
)
if TYPE_CHECKING:
from aio_pika.abc import (
AbstractChannel,
AbstractConnection,
AbstractIncomingMessage,
AbstractRobustConnection,
)
log = get_logger(__name__)
def _is_terminal(exc: BaseException) -> bool:
"""Return True for errors that will not be fixed by RabbitMQ retries."""
return isinstance(exc, TerminalError)
class Consumer[MsgT: PipelineMessage]:
"""Base RabbitMQ consumer bound to one main queue."""
#: subclass declares which queue + routing key + message model it owns
queue: str = ""
routing_key: str = ""
message_model: type[PipelineMessage] = PipelineMessage
def __init__(
self,
url: str,
*,
origin: str,
prefetch: int,
max_attempts: int,
retry_base_ms: int,
) -> None:
self._url = url
self._origin = origin
self._prefetch = prefetch
self._max_attempts = max(1, max_attempts)
self._retry_base_ms = max(1, retry_base_ms)
self._conn: AbstractRobustConnection | None = None
self._channel: AbstractChannel | None = None
self._stop = asyncio.Event()
# ── subclass hooks ───────────────────────────────────────────────────────
async def handle(self, payload: MsgT) -> None:
raise NotImplementedError
def classify(self, exc: BaseException) -> FailureClass:
return "unknown"
async def on_failure(
self, payload: MsgT, failure_class: FailureClass, attempt: int, error: str
) -> None:
"""Per-failure hook (update jobs.attempts / last_failure_class)."""
async def on_dlq(self, payload: MsgT, failure_class: FailureClass, error: str) -> None:
"""Terminal hook (set status=failed + refund)."""
# ── lifecycle ────────────────────────────────────────────────────────────
async def connect(self) -> None:
log.info("consumer_connecting", queue=self.queue, origin=self._origin)
# Ensure retry queues are lazy on existing deployments before declaring.
# Fresh deployments will get x-queue-mode:lazy from declare_all below.
try:
ensure_retry_queues_lazy(self._url)
except Exception as exc: # noqa: BLE001
log.warning(
"retry_lazy_policy_failed",
queue=self.queue,
origin=self._origin,
error=f"{type(exc).__name__}: {exc}",
)
self._conn = await connect_robust(self._url)
def _on_connection_close(
conn: AbstractConnection | None, exc: BaseException | None
) -> None:
if exc is not None:
log.warning(
"rabbitmq_connection_lost",
queue=self.queue,
origin=self._origin,
error=f"{type(exc).__name__}: {exc}",
)
else:
log.info(
"rabbitmq_connection_closed",
queue=self.queue,
origin=self._origin,
)
self._conn.close_callbacks.add(_on_connection_close)
channel = await self._conn.channel()
self._channel = channel
await channel.set_qos(prefetch_count=self._prefetch)
await declare_all(channel)
log.info(
"consumer_connected",
queue=self.queue,
origin=self._origin,
prefetch=self._prefetch,
)
async def run(self) -> None:
if self._channel is None:
await self.connect()
assert self._channel is not None
queue = await self._channel.get_queue(self.queue, ensure=False)
await queue.consume(self._on_message)
await self._stop.wait()
async def stop(self) -> None:
self._stop.set()
if self._conn is not None:
try:
await self._conn.close()
except Exception as exc: # noqa: BLE001
log.warning(
"consumer_close_error",
queue=self.queue,
origin=self._origin,
error=f"{type(exc).__name__}: {exc}",
)
# ── internals ────────────────────────────────────────────────────────────
async def _on_message(self, message: AbstractIncomingMessage) -> None:
headers: dict[str, Any] = message.headers or {}
cid = str(headers.get(H_CORRELATION_ID, message.correlation_id or ""))
set_correlation_id(cid)
attempt = int(str(headers.get(H_ATTEMPT, 0) or "0"))
payload = self._parse(message.body)
if payload is None:
await self._to_dlq(message, "infra", "invalid message body", attempt=0)
return
try:
await self.handle(payload)
await message.ack()
return
except Exception as exc: # noqa: BLE001 — classified below
failure_class = self.classify(exc)
error = f"{type(exc).__name__}: {exc}"
new_attempt = attempt + 1
await self.on_failure(payload, failure_class, new_attempt, error)
if _is_terminal(exc):
log.warning(
"message_terminal_dlq",
queue=self.queue,
correlation_id=cid,
failure_class=failure_class,
)
await self.on_dlq(payload, failure_class, error)
await self._to_dlq(message, failure_class, error, attempt=new_attempt)
return
if new_attempt >= self._max_attempts:
log.warning(
"message_max_attempts_dlq",
queue=self.queue,
correlation_id=cid,
attempt=new_attempt,
failure_class=failure_class,
)
await self.on_dlq(payload, failure_class, error)
await self._to_dlq(message, failure_class, error, attempt=new_attempt)
else:
await self._to_retry(message, payload, new_attempt)
await message.ack()
def _parse(self, body: bytes) -> MsgT | None:
try:
return self.message_model.model_validate_json(body) # type: ignore[return-value]
except ValidationError as exc:
log.error("message_parse_failed", error=str(exc), body=body[:500])
return None
async def _to_retry(
self, message: AbstractIncomingMessage, payload: MsgT, new_attempt: int
) -> None:
assert self._channel is not None
retry_rk = f"retry.{self.routing_key}"
expiration_ms = int(self._retry_base_ms * (2 ** (new_attempt - 1)))
body = payload.next_attempt().model_dump_json().encode("utf-8")
headers: dict[str, Any] = {
H_CORRELATION_ID: str(payload.correlation_id),
H_ATTEMPT: new_attempt,
H_ORIGIN: self._origin,
}
retry_exchange = await self._channel.get_exchange(EXCHANGE_RETRY, ensure=False)
await retry_exchange.publish(
Message(
body,
content_type="application/json",
delivery_mode=DeliveryMode.PERSISTENT,
correlation_id=str(payload.correlation_id),
headers=headers,
expiration=expiration_ms,
),
routing_key=retry_rk,
)
log.info(
"message_retrying",
queue=self.queue,
correlation_id=str(payload.correlation_id),
attempt=new_attempt,
delay_ms=expiration_ms,
)
async def _to_dlq(
self,
message: AbstractIncomingMessage,
failure_class: FailureClass,
error: str,
*,
attempt: int,
) -> None:
assert self._channel is not None
dlq = DLQ_FOR.get(self.queue, f"{self.queue}.dlq")
headers = dict(message.headers or {})
headers.update(
{
H_CORRELATION_ID: str(message.correlation_id or headers.get(H_CORRELATION_ID, "")),
H_ATTEMPT: attempt,
"x-failure-class": failure_class,
"x-failure-error": error[:1000],
}
)
await self._channel.default_exchange.publish(
Message(
message.body,
content_type=message.content_type or "application/json",
delivery_mode=DeliveryMode.PERSISTENT,
correlation_id=message.correlation_id,
headers=headers,
),
routing_key=dlq,
)
await message.ack()
log.warning("message_to_dlq", dlq=dlq, failure_class=failure_class)

49
src/contract_check/core/mq/messages.py Normal file
View file

@ -0,0 +1,49 @@
"""Message payloads for the RabbitMQ pipeline (pydantic v2).
Validated on consume; a body that fails validation goes straight to the DLQ
with failure_class=infra. Every message carries `correlation_id` (also placed
in RabbitMQ header `x-correlation-id` by the publisher/consumer) so logs and
traces trace api → rabbit → workers under one id.
"""
from __future__ import annotations
import uuid
from typing import Self
from pydantic import BaseModel, Field
class PipelineMessage(BaseModel):
"""Base: every pipeline message has these."""
correlation_id: uuid.UUID
document_id: uuid.UUID
user_id: uuid.UUID
attempt: int = Field(default=0, ge=0)
def next_attempt(self) -> Self:
return self.model_copy(update={"attempt": self.attempt + 1})
class DocumentUploaded(PipelineMessage):
"""api → contracts.x[extract] → worker-extract.
`s3_key` points at the raw uploaded blob (`users/{uid}/docs/{did}.{ext}`).
"""
s3_key: str
filename: str
mime: str
class DocumentExtracted(PipelineMessage):
"""worker-extract → contracts.x[analyze] → worker-analyze.
`extracted_s3_key` points at the extracted plaintext
(`users/{uid}/docs/{did}.txt`).
"""
extracted_s3_key: str
char_count: int
ocr_used: bool

81
src/contract_check/core/mq/publisher.py Normal file
View file

@ -0,0 +1,81 @@
"""RabbitMQ publisher with publisher confirms.
The api publishes `DocumentUploaded` on upload; worker-extract publishes
`DocumentExtracted` on success. Publisher confirms are ON: a publish that is
not confirmed raises, so a paid job's message can never silently vanish (the
api fails the HTTP request instead of returning a false 202).
Every published message carries the correlation_id in both `message_id` and
the `x-correlation-id` header for log/trace propagation.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from aio_pika import DeliveryMode, Message, connect_robust
from ..logging import get_logger
from .messages import PipelineMessage
from .topology import EXCHANGE_MAIN, H_ATTEMPT, H_CORRELATION_ID, H_ORIGIN, declare_all
if TYPE_CHECKING:
from aio_pika.abc import AbstractChannel, AbstractExchange, AbstractRobustConnection
log = get_logger(__name__)
class Publisher:
"""Robust RabbitMQ publisher bound to the main exchange."""
def __init__(self, url: str, *, origin: str = "api") -> None:
self._url = url
self._origin = origin
self._conn: AbstractRobustConnection | None = None
self._channel: AbstractChannel | None = None
self._exchange: AbstractExchange | None = None
async def connect(self) -> None:
self._conn = await connect_robust(self._url)
channel = await self._conn.channel(publisher_confirms=True)
self._channel = channel
await declare_all(channel)
self._exchange = await channel.declare_exchange(EXCHANGE_MAIN, durable=True)
log.info("publisher_connected", url=self._url, origin=self._origin)
async def publish(self, message: PipelineMessage, routing_key: str) -> None:
"""Publish a PipelineMessage to the main exchange. Raises on no-confirm."""
if self._exchange is None:
raise RuntimeError("Publisher not connected; call connect() first")
body = message.model_dump_json().encode("utf-8")
amqp = Message(
body,
content_type="application/json",
delivery_mode=DeliveryMode.PERSISTENT,
correlation_id=str(message.correlation_id),
message_id=str(message.correlation_id),
headers={
H_CORRELATION_ID: str(message.correlation_id),
H_ATTEMPT: message.attempt,
H_ORIGIN: self._origin,
},
)
await self._exchange.publish(amqp, routing_key=routing_key)
log.debug(
"published",
routing_key=routing_key,
correlation_id=str(message.correlation_id),
attempt=message.attempt,
)
async def close(self) -> None:
if self._conn is not None:
await self._conn.close()
self._conn = self._channel = self._exchange = None
async def __aenter__(self) -> Publisher:
await self.connect()
return self
async def __aexit__(self, *exc: object) -> None:
await self.close()

206
src/contract_check/core/mq/topology.py Normal file
View file

@ -0,0 +1,206 @@
"""RabbitMQ topology — exchange/queue/routing-key names + idempotent declare.
Layout (docs/ARCHITECTURE.md §5):
contracts.x (direct)
├─ extract ─► extract.q (quorum; consumed by worker-extract)
└─ analyze ─► analyze.q (quorum; consumed by worker-analyze)
contracts.retry.x (direct) ← DLX target of main queues on nack
├─ retry.extract ─► extract.retry.q (classic; per-msg TTL; DLX→contracts.x[extract])
└─ retry.analyze ─► analyze.retry.q (classic; per-msg TTL; DLX→contracts.x[analyze])
extract.dlq / analyze.dlq (quorum; manual requeue after max attempts)
Retry is plugin-free: the main queue's DLX routes nacked messages to the retry
exchange; the retry queue's per-message TTL, on expiry, dead-letters back to
the main exchange → the message re-enters the main quorum queue.
The retry queues are additionally forced to `queue-mode: lazy` via a RabbitMQ
policy. Without lazy mode, classic queues keep per-message TTL messages in RAM
and do not expire them while idle, so retries can sit forever. The policy is
idempotent and applies to existing queues without redeclaring them.
"""
from __future__ import annotations
import json
import urllib.error
import urllib.request
from typing import TYPE_CHECKING, Any
from urllib.parse import urlparse
if TYPE_CHECKING:
from aio_pika.abc import AbstractChannel
# ── exchanges ────────────────────────────────────────────────────────────────
EXCHANGE_MAIN = "contracts.x"
EXCHANGE_RETRY = "contracts.retry.x"
# ── queues ───────────────────────────────────────────────────────────────────
QUEUE_EXTRACT = "extract.q"
QUEUE_ANALYZE = "analyze.q"
QUEUE_EXTRACT_RETRY = "extract.retry.q"
QUEUE_ANALYZE_RETRY = "analyze.retry.q"
QUEUE_EXTRACT_DLQ = "extract.dlq"
QUEUE_ANALYZE_DLQ = "analyze.dlq"
# ── routing keys ─────────────────────────────────────────────────────────────
RK_EXTRACT = "extract"
RK_ANALYZE = "analyze"
RK_RETRY_EXTRACT = "retry.extract"
RK_RETRY_ANALYZE = "retry.analyze"
# ── message headers ──────────────────────────────────────────────────────────
H_CORRELATION_ID = "x-correlation-id"
H_ATTEMPT = "x-attempt"
H_ORIGIN = "x-origin"
# Queue type args. Quorum queues are HA-ready (persisted; replicate when the
# RabbitMQ cluster grows to 3 nodes — no app code change). Retry queues are
# classic transient delay slots (safe to lose; the originating job is acked).
_QUORUM_ARGS: dict[str, Any] = {"x-queue-type": "quorum"}
_RETRY_QUEUE_FOR: dict[str, tuple[str, str, str]] = {
# main_queue: (retry_exchange, retry_routing_key, main_routing_key_back)
QUEUE_EXTRACT: (EXCHANGE_RETRY, RK_RETRY_EXTRACT, RK_EXTRACT),
QUEUE_ANALYZE: (EXCHANGE_RETRY, RK_RETRY_ANALYZE, RK_ANALYZE),
}
# Reverse lookup: queue name → its DLQ name.
DLQ_FOR: dict[str, str] = {
QUEUE_EXTRACT: QUEUE_EXTRACT_DLQ,
QUEUE_ANALYZE: QUEUE_ANALYZE_DLQ,
}
async def declare_all(channel: AbstractChannel) -> None:
"""Idempotently declare every exchange, queue, and binding."""
main_exchange = await channel.declare_exchange(EXCHANGE_MAIN, durable=True)
retry_exchange = await channel.declare_exchange(EXCHANGE_RETRY, durable=True)
# Main quorum queues: DLX → retry exchange.
for queue, retry_rk in (
(QUEUE_EXTRACT, RK_RETRY_EXTRACT),
(QUEUE_ANALYZE, RK_RETRY_ANALYZE),
):
q = await channel.declare_queue(
queue,
durable=True,
arguments={
**_QUORUM_ARGS,
"x-dead-letter-exchange": EXCHANGE_RETRY,
"x-dead-letter-routing-key": retry_rk,
},
)
await q.bind(main_exchange, routing_key=retry_rk.replace("retry.", ""))
# Classic retry queues: DLX → main exchange (back to the main routing key).
# The retry queues are kept lazy via the policy set by
# ensure_retry_queues_lazy() so that per-message TTL expires even when no
# consumer is attached. Including x-queue-mode here would break upgrades
# because RabbitMQ treats it as a queue argument that cannot be changed on
# an existing queue.
for retry_queue, main_rk in (
(QUEUE_EXTRACT_RETRY, RK_EXTRACT),
(QUEUE_ANALYZE_RETRY, RK_ANALYZE),
):
rq = await channel.declare_queue(
retry_queue,
durable=True,
arguments={
"x-dead-letter-exchange": EXCHANGE_MAIN,
"x-dead-letter-routing-key": main_rk,
},
)
await rq.bind(retry_exchange, routing_key=f"retry.{main_rk}")
# Quorum DLQs (no bindings; published to directly via default exchange).
for dlq in (QUEUE_EXTRACT_DLQ, QUEUE_ANALYZE_DLQ):
await channel.declare_queue(dlq, durable=True, arguments=_QUORUM_ARGS)
# ── lazy-mode policy for existing (non-lazy) retry queues ───────────────────
# RabbitMQ classic queues only expire per-message TTL when the message reaches
# the head of the queue. In default mode the queue may keep messages in RAM
# indefinitely while idle; lazy mode forces paging to disk and enables expiry.
# This policy is applied in addition to the x-queue-mode declaration above so
# that already-created retry queues are corrected without being deleted.
_RETRY_QUEUE_POLICY_NAME = "contract_check_retry_lazy"
_RETRY_QUEUE_POLICY_PATTERN = "^.+\\.retry\\.q$"
_RETRY_QUEUE_POLICY_DEF = {"queue-mode": "lazy"}
def _management_url_from_amqp(amqp_url: str) -> str | None:
"""Derive a likely RabbitMQ management URL from the AMQP URL.
Returns None when the URL cannot be parsed into something usable.
"""
try:
parsed = urlparse(amqp_url)
except ValueError:
return None
if not parsed.hostname:
return None
scheme = "https" if parsed.scheme.endswith("s") else "http"
port = parsed.port or (5671 if scheme == "https" else 5672)
# Common management port mapping: 5672 -> 15672, 5671 -> 15671.
mgmt_port = 15671 if port == 5671 else 15672
return f"{scheme}://{parsed.hostname}:{mgmt_port}"
def _amqp_credentials(amqp_url: str) -> tuple[str, str] | None:
"""Return (username, password) from the AMQP URL, or None if absent."""
try:
parsed = urlparse(amqp_url)
except ValueError:
return None
if not parsed.username or parsed.password is None:
return None
return parsed.username, parsed.password
def ensure_retry_queues_lazy(amqp_url: str) -> None:
"""Set the lazy-mode policy for *.retry.q queues via the management API.
This is best-effort: if the management plugin is not reachable the
declaration-time x-queue-mode still protects fresh deployments.
"""
mgmt_url = _management_url_from_amqp(amqp_url)
creds = _amqp_credentials(amqp_url)
if not mgmt_url or not creds:
return
import base64
username, password = creds
auth = base64.b64encode(f"{username}:{password}".encode()).decode()
payload = json.dumps(
{
"pattern": _RETRY_QUEUE_POLICY_PATTERN,
"apply-to": "queues",
"definition": _RETRY_QUEUE_POLICY_DEF,
"priority": 0,
}
).encode("utf-8")
url = f"{mgmt_url}/api/policies/%2F/{_RETRY_QUEUE_POLICY_NAME}"
req = urllib.request.Request(
url,
data=payload,
method="PUT",
headers={
"Authorization": f"Basic {auth}",
"Content-Type": "application/json",
},
)
try:
with urllib.request.urlopen(req, timeout=10) as resp:
if resp.status in (201, 204):
return
except urllib.error.HTTPError as exc:
# 204 No Content is the success response for PUT on existing policy.
if exc.code == 204:
return
except Exception:
# Management plugin may be disabled; declaration-time x-queue-mode is
# the fallback for fresh deployments.
pass

131
src/contract_check/core/rate_limit.py Normal file
View file

@ -0,0 +1,131 @@
"""Token-bucket rate limiter with Redis backend.
Default B2B limit mirrors Ollama Pro concurrency (3 req/s) so burst traffic
from API clients cannot outrun the LLM provider. Per-key override lives in
`api_keys.rate_limit_rps`.
"""
from __future__ import annotations
import time
from dataclasses import dataclass
from typing import Any, Protocol
from .logging import get_logger
log = get_logger(__name__)
class RateLimiterUnavailable(Exception):
"""Raised when the rate-limiter backend cannot be reached."""
class RateLimiter(Protocol):
"""Backend-agnostic rate limiter interface."""
async def allow(self, key: str, limit_rps: int) -> RateLimitResult: ...
@dataclass(frozen=True)
class RateLimitResult:
allowed: bool
retry_after_sec: float | None
class RedisRateLimiter:
"""Token-bucket rate limiter backed by Redis (atomic Lua script)."""
# Capacity = limit_rps tokens, refill rate = limit_rps tokens/second.
# This allows a burst of `limit_rps` requests and then throttles to the
# same sustained rate.
_SCRIPT = """
local key = KEYS[1]
local capacity = tonumber(ARGV[1])
local refill_rate = tonumber(ARGV[2])
local now = tonumber(ARGV[3])
local cost = 1
local bucket = redis.call("HMGET", key, "tokens", "last_refill")
local tokens = tonumber(bucket[1])
local last_refill = tonumber(bucket[2])
if tokens == nil then
tokens = capacity
last_refill = now
end
local elapsed = now - last_refill
tokens = math.min(capacity, tokens + elapsed * refill_rate)
if tokens >= cost then
tokens = tokens - cost
redis.call("HMSET", key, "tokens", tokens, "last_refill", now)
redis.call("EXPIRE", key, 3600)
return {1, 0}
else
local deficit = cost - tokens
local retry_after = deficit / refill_rate
redis.call("HMSET", key, "tokens", tokens, "last_refill", now)
redis.call("EXPIRE", key, 3600)
return {0, retry_after}
end
"""
def __init__(self, redis: Any) -> None:
self._redis = redis
self._script_sha: str | None = None
async def _ensure_script(self) -> str:
if self._script_sha is None:
self._script_sha = await self._redis.script_load(self._SCRIPT)
return self._script_sha
async def allow(self, key: str, limit_rps: int) -> RateLimitResult:
if limit_rps <= 0:
# Defensive: a misconfigured key should not block all traffic.
log.warning("rate_limit_misconfigured", key=key, limit_rps=limit_rps)
return RateLimitResult(allowed=True, retry_after_sec=None)
capacity = float(limit_rps)
refill_rate = float(limit_rps)
now = time.monotonic()
try:
sha = await self._ensure_script()
raw = await self._redis.evalsha(sha, 1, key, str(capacity), str(refill_rate), str(now))
except Exception as exc: # pragma: no cover - Redis unavailable
log.error("rate_limit_redis_error", key=key, error=str(exc))
raise RateLimiterUnavailable("rate limiter backend unavailable") from exc
# Redis returns a list of integers/floats as strings; unpack.
allowed = int(raw[0]) == 1
retry_after = None if allowed else float(raw[1])
return RateLimitResult(allowed=allowed, retry_after_sec=retry_after)
class MemoryRateLimiter:
"""In-memory token bucket for unit tests and dev without Redis."""
def __init__(self) -> None:
self._buckets: dict[str, tuple[float, float]] = {}
async def allow(self, key: str, limit_rps: int) -> RateLimitResult:
if limit_rps <= 0:
# Match RedisRateLimiter: misconfigured limits fail open.
log.warning("rate_limit_misconfigured", key=key, limit_rps=limit_rps)
return RateLimitResult(allowed=True, retry_after_sec=None)
capacity = float(limit_rps)
refill_rate = float(limit_rps)
now = time.monotonic()
tokens, last_refill = self._buckets.get(key, (capacity, now))
tokens = min(capacity, tokens + (now - last_refill) * refill_rate)
if tokens >= 1.0:
self._buckets[key] = (tokens - 1.0, now)
return RateLimitResult(allowed=True, retry_after_sec=None)
retry_after = (1.0 - tokens) / refill_rate
self._buckets[key] = (tokens, now)
return RateLimitResult(allowed=False, retry_after_sec=retry_after)

19
src/contract_check/core/redis_client.py Normal file
View file

@ -0,0 +1,19 @@
"""Async Redis client factory.
Used for rate-limiting and future sessions. NOT a job queue — RabbitMQ owns that.
"""
from __future__ import annotations
from typing import Any
def get_redis_client(redis_url: str) -> Any:
"""Create a new async Redis client from a Redis DSN.
Callers own the connection lifecycle. The api creates one client at startup
and stores it in app.state.redis.
"""
from redis.asyncio import Redis as AsyncRedis
return AsyncRedis.from_url(redis_url, decode_responses=True)

39
src/contract_check/core/s3/__init__.py Normal file
View file

@ -0,0 +1,39 @@
"""Object storage layer — Storage port + MinIO adapter + key builders.
See docs/ARCHITECTURE.md §6. Adapters never import this; only the api and workers
do. Layout: users/{uid}/docs/{did}.{ext} (raw) and {did}.txt (extracted).
The MinIO SDK is sync; the adapter offloads every call to a thread so the
async event loop is never blocked.
"""
from __future__ import annotations
import urllib.parse
from .port import ObjectInfo, Storage
__all__ = ["Storage", "ObjectInfo", "original_key", "extracted_key", "key_user_prefix"]
def original_key(user_id: str, document_id: str, ext: str) -> str:
"""Raw upload object key: `users/{uid}/docs/{did}.{ext}`."""
ext = ext.lstrip(".")
return f"users/{user_id}/docs/{document_id}.{ext}"
def extracted_key(user_id: str, document_id: str) -> str:
"""Extracted-plaintext object key: `users/{uid}/docs/{did}.txt`."""
return f"users/{user_id}/docs/{document_id}.txt"
def key_user_prefix(user_id: str) -> str:
"""Prefix to enumerate/purge all of a user's objects."""
return f"users/{user_id}/docs/"
def endpoint_host_port(endpoint_url: str) -> tuple[str, bool]:
"""Split an S3_ENDPOINT_URL into (host:port, secure) for the MinIO client."""
parsed = urllib.parse.urlparse(endpoint_url)
secure = parsed.scheme == "https"
netloc = parsed.netloc or parsed.path
return netloc, secure

103
src/contract_check/core/s3/minio_storage.py Normal file
View file

@ -0,0 +1,103 @@
"""MinIO adapter implementing the Storage port.
Sync `minio` SDK calls are wrapped in `asyncio.to_thread` so the async loop
in the api/workers never blocks on network I/O. `ensure_bucket()` is the lazy
safety net behind the `minio-init` compose service.
"""
from __future__ import annotations
import asyncio
import io
from typing import Any
from minio import Minio
from ..logging import get_logger
from .port import ObjectInfo
log = get_logger(__name__)
class MinioStorage:
"""Async-friendly MinIO storage adapter."""
def __init__(
self,
endpoint: str,
access_key: str,
secret_key: str,
bucket: str,
*,
secure: bool = False,
region: str = "us-east-1",
) -> None:
self._bucket = bucket
self._client: Any = Minio(
endpoint,
access_key=access_key,
secret_key=secret_key,
secure=secure,
region=region,
)
@classmethod
def from_endpoint_url(
cls,
endpoint_url: str,
access_key: str,
secret_key: str,
bucket: str,
*,
region: str = "us-east-1",
) -> MinioStorage:
from . import endpoint_host_port
host, secure = endpoint_host_port(endpoint_url)
return cls(host, access_key, secret_key, bucket, secure=secure, region=region)
async def ensure_bucket(self) -> None:
await asyncio.to_thread(self._ensure_bucket_sync)
def _ensure_bucket_sync(self) -> None:
if not self._client.bucket_exists(self._bucket):
self._client.make_bucket(self._bucket)
log.info("bucket_created", bucket=self._bucket)
async def put(
self, key: str, data: bytes, *, content_type: str = "application/octet-stream"
) -> ObjectInfo:
return await asyncio.to_thread(self._put_sync, key, data, content_type)
def _put_sync(self, key: str, data: bytes, content_type: str) -> ObjectInfo:
stream = io.BytesIO(data)
result = self._client.put_object(
self._bucket, key, stream, length=len(data), content_type=content_type
)
return ObjectInfo(key=key, size=len(data), etag=result.etag if result else None)
async def get(self, key: str) -> bytes:
data: bytes = await asyncio.to_thread(self._get_sync, key)
return data
def _get_sync(self, key: str) -> bytes:
response = self._client.get_object(self._bucket, key)
try:
data: bytes = response.read()
return data
finally:
response.close()
response.release_conn()
async def delete(self, key: str) -> None:
await asyncio.to_thread(self._client.remove_object, self._bucket, key)
async def stat(self, key: str) -> ObjectInfo | None:
return await asyncio.to_thread(self._stat_sync, key)
def _stat_sync(self, key: str) -> ObjectInfo | None:
try:
stat = self._client.stat_object(self._bucket, key)
except Exception:
return None
return ObjectInfo(key=key, size=stat.size, etag=stat.etag)

31
src/contract_check/core/s3/port.py Normal file
View file

@ -0,0 +1,31 @@
"""Storage port (hexagonal). The MinIO adapter and any future adapter
(Selectel, in-memory for tests) implement this Protocol."""
from __future__ import annotations
from dataclasses import dataclass
from typing import Protocol, runtime_checkable
@dataclass(slots=True)
class ObjectInfo:
key: str
size: int
etag: str | None
@runtime_checkable
class Storage(Protocol):
"""Object storage port used by the api and both workers."""
async def ensure_bucket(self) -> None: ...
async def put(
self, key: str, data: bytes, *, content_type: str = "application/octet-stream"
) -> ObjectInfo: ...
async def get(self, key: str) -> bytes: ...
async def delete(self, key: str) -> None: ...
async def stat(self, key: str) -> ObjectInfo | None: ...

31
src/contract_check/core/sentry.py Normal file
View file

@ -0,0 +1,31 @@
"""Sentry initialization helper.
`init_sentry()` is called by each service entrypoint. No-op when `SENTRY_DSN`
is empty. Imports `sentry_sdk` — only obs-enabled services use it.
"""
from __future__ import annotations
from .config import get_settings
from .logging import get_logger
log = get_logger(__name__)
def init_sentry(service_name: str | None = None) -> None:
"""Initialize Sentry SDK. Safe to call when no DSN is configured."""
settings = get_settings()
dsn = settings.sentry_dsn
if not dsn:
return
import sentry_sdk
traces_sample_rate = 1.0 if settings.env == "dev" else 0.1
sentry_sdk.init(
dsn=dsn,
environment=settings.env,
traces_sample_rate=traces_sample_rate,
send_default_pii=False,
)
log.info("sentry_initialized", service=service_name or settings.otel_service_name)

73
src/contract_check/core/telemetry.py Normal file
View file

@ -0,0 +1,73 @@
"""OpenTelemetry initialization.
Sets up a tracer exporting OTLP to the collector when
`OTEL_EXPORTER_OTLP_ENDPOINT` is set; otherwise no-op. httpx auto-instrumentation
is wired so Ollama Cloud calls appear as spans. Service entrypoints call
`setup_telemetry()` early and `shutdown_telemetry()` on exit.
This module imports `opentelemetry` — only services in the `obs` group import it
(see docs/ARCHITECTURE.md §4). mypy resolves it via the dev group.
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from .config import get_settings
from .logging import get_logger
if TYPE_CHECKING:
from opentelemetry.sdk.trace import TracerProvider
log = get_logger(__name__)
_provider: TracerProvider | None = None
def setup_telemetry(service_name: str | None = None) -> None:
"""Initialize OTel tracing. Safe to call multiple times / when disabled."""
global _provider
settings = get_settings()
endpoint = settings.otel_exporter_otlp_endpoint
if not endpoint:
log.debug("otel_disabled", reason="no endpoint configured")
return
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.trace import set_tracer_provider
resource = Resource.create({"service.name": service_name or settings.otel_service_name})
provider = TracerProvider(resource=resource)
provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint)))
set_tracer_provider(provider)
_provider = provider
try:
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
HTTPXClientInstrumentor().instrument()
except Exception as exc: # pragma: no cover - optional instrumentor
log.warning("otel_httpx_instrument_failed", error=str(exc))
log.info("otel_initialized", endpoint=endpoint)
def shutdown_telemetry() -> None:
"""Flush and shut down the tracer provider if it was initialized."""
global _provider
if _provider is None:
return
try:
_provider.shutdown()
finally:
_provider = None
def get_tracer(name: str | None = None) -> Any:
"""Return a tracer. Returns a no-op-safe tracer when OTel is absent."""
from opentelemetry import trace
return trace.get_tracer(name or "contract_check")

41
src/contract_check/core/tokens.py Normal file
View file

@ -0,0 +1,41 @@
"""Service-token hashing/verification (no FastAPI here — pure crypto helpers).
The api turns `Authorization: Bearer <token>` into a lookup: hash the bearer,
match against `service_tokens.token_hash` where `revoked = FALSE`. Per-adapter
tokens are revocable and auditable (docs/ARCHITECTURE.md §18/Q18). The FastAPI
dependency that uses these lives in `api/deps.py` (Step 2).
"""
from __future__ import annotations
import hashlib
import hmac
import secrets
from .db.enums import ADAPTER_NAMES, AdapterName
def generate_token() -> str:
"""Mint a new opaque bearer secret (URL-safe, ~43 chars)."""
return secrets.token_urlsafe(32)
def hash_token(token: str) -> str:
"""SHA-256 hex digest of a bearer token (store this, never the raw token)."""
return hashlib.sha256(token.encode("utf-8")).hexdigest()
def verify_token(token: str, token_hash: str) -> bool:
"""Constant-time check that `token` matches the stored `token_hash`."""
digest = hash_token(token)
return hmac.compare_digest(digest, token_hash)
def is_valid_adapter(adapter: str) -> bool:
return adapter in ADAPTER_NAMES
def assert_adapter(adapter: str) -> AdapterName:
if not is_valid_adapter(adapter):
raise ValueError(f"invalid adapter {adapter!r}; expected one of {ADAPTER_NAMES}")
return adapter # type: ignore[return-value]

158
src/contract_check/prototype/__init__.py Normal file
View file

@ -0,0 +1,158 @@
"""«Контракт-чек» — Stage 0 prototype (standalone benchmark CLI).
Preserved on purpose (docs/ARCHITECTURE.md Q44): a run-once end-to-end CLI for
go/no-go measurements against a real contract — PDF/DOCX → text → chunks →
Ollama Cloud → markdown report + metrics. It does NOT use Postgres/RabbitMQ/
MinIO; it builds an OllamaCloudProvider directly from OLLAMA_* env vars.
python -m contract_check.prototype path/to/contract.pdf
python -m contract_check.prototype contract.docx --out report.md
python -m contract_check.prototype contract.pdf --json metrics.json
Не заменяет юриста. Это первичный скрининг рисков.
"""
from __future__ import annotations
import argparse
import asyncio
import json
import sys
from pathlib import Path
from pydantic_settings import BaseSettings, SettingsConfigDict
from ..core.analysis.analyzer import RenderMetrics, render_markdown
from ..core.analysis.checklist import checklist_for_prompt
from ..core.analysis.extractor import ExtractionError, extract_text
from ..core.llm.ollama_cloud import LLMError, OllamaCloudProvider
from ..core.logging import configure_logging
class PrototypeSettings(BaseSettings):
"""OLLAMA_* config only — the prototype needs no DB/MQ/S3."""
model_config = SettingsConfigDict(env_file=".env", extra="ignore", case_sensitive=False)
ollama_host: str = ""
ollama_api_key: str = ""
ollama_model: str = "qwen2.5:14b"
ollama_fallback_model: str = "qwen2.5:7b"
ollama_temperature: float = 0.2
ollama_num_predict: int = 3072
ollama_timeout: float = 120.0
ollama_max_concurrency: int = 3
chunk_size_chars: int = 10000
def _build_provider(s: PrototypeSettings) -> OllamaCloudProvider:
if not s.ollama_host or not s.ollama_api_key:
sys.exit(
"ERROR: OLLAMA_HOST и OLLAMA_API_KEY обязательны. "
"Скопируй .env.example → .env и заполни (см. ollama.com/cloud)."
)
return OllamaCloudProvider(
host=s.ollama_host,
api_key=s.ollama_api_key,
model=s.ollama_model,
fallback_model=s.ollama_fallback_model or None,
temperature=s.ollama_temperature,
num_predict=s.ollama_num_predict,
timeout=s.ollama_timeout,
max_concurrency=s.ollama_max_concurrency,
chunk_size=s.chunk_size_chars,
)
async def run(path: Path, out: Path | None, json_out: Path | None) -> int:
settings = PrototypeSettings()
started = asyncio.get_event_loop().time()
try:
text = extract_text(path)
except ExtractionError as exc:
print(f"ERROR извлечения: {exc}", file=sys.stderr)
return 2
provider = _build_provider(settings)
try:
async with provider:
try:
result = await provider.analyze(text, checklist=checklist_for_prompt())
except LLMError as exc:
print(f"ERROR LLM: {exc}", file=sys.stderr)
return 3
finally:
latency = asyncio.get_event_loop().time() - started
metrics = RenderMetrics(
chars=len(text),
findings=len(result.findings),
prompt_tokens=result.prompt_tokens,
eval_tokens=result.eval_tokens,
latency_sec=max(result.latency_sec, latency),
models_used=result.models_used,
fell_back=result.fell_back,
repaired=result.repaired,
)
md = render_markdown(result.findings, str(path), metrics)
if out:
out.write_text(md, encoding="utf-8")
print(f"Отчёт сохранён: {out}")
else:
print(md)
if json_out:
json_out.write_text(
json.dumps(
{
"file": path.name,
"findings": [f.model_dump() for f in result.findings],
"metrics": {
"chars": metrics.chars,
"findings": metrics.findings,
"prompt_tokens": metrics.prompt_tokens,
"eval_tokens": metrics.eval_tokens,
"total_tokens": metrics.total_tokens,
"latency_sec": round(metrics.latency_sec, 2),
"models_used": sorted(metrics.models_used),
"fell_back": metrics.fell_back,
"repaired": metrics.repaired,
},
},
ensure_ascii=False,
indent=2,
),
encoding="utf-8",
)
print(f"Метрики/JSON сохранены: {json_out}")
return 0
def main() -> None:
parser = argparse.ArgumentParser(
prog="contract-check",
description="«Контракт-чек» — первичный скрининг рисков договора (Stage 0).",
)
parser.add_argument("file", help="Путь к PDF/DOCX договору.")
parser.add_argument("--out", "-o", help="Записать markdown-отчёт в файл.")
parser.add_argument("--json", help="Дополнительно записать находки + метрики.")
parser.add_argument("-v", "--verbose", action="store_true", help="Подробное логирование.")
args = parser.parse_args()
configure_logging(
level="DEBUG" if args.verbose else "WARNING",
json_output=False,
service="contract-check-prototype",
)
rc = asyncio.run(
run(
Path(args.file),
Path(args.out) if args.out else None,
Path(args.json) if args.json else None,
)
)
sys.exit(rc)

6
src/contract_check/prototype/__main__.py Normal file
View file

@ -0,0 +1,6 @@
"""Stage-0 prototype entrypoint: `python -m contract_check.prototype <file>`."""
from . import main
if __name__ == "__main__":
main()

0
src/contract_check/worker_analyze/__init__.py Normal file
View file

62
src/contract_check/worker_analyze/__main__.py Normal file
View file

@ -0,0 +1,62 @@
"""worker-analyze entrypoint: connects to RabbitMQ and runs the analyze consumer."""
from __future__ import annotations
import asyncio
import signal
from ..core.config import get_settings
from ..core.logging import bind_context, configure_logging, get_logger
from ..core.metrics import start_metrics_server
from ..core.sentry import init_sentry
from ..core.telemetry import setup_telemetry, shutdown_telemetry
from .consumer import AnalyzeConsumer
log = get_logger(__name__)
async def main() -> None:
settings = get_settings()
configure_logging(
settings.log_level,
json_output=settings.json_logs,
service="worker-analyze",
env=settings.env,
)
bind_context(service="worker-analyze", env=settings.env)
init_sentry("worker-analyze")
setup_telemetry("worker-analyze")
start_metrics_server(9102)
consumer = AnalyzeConsumer(
url=settings.rabbitmq_url,
origin="worker-analyze",
prefetch=settings.mq_prefetch_analyze,
max_attempts=settings.mq_max_attempts,
retry_base_ms=settings.mq_retry_base_ms,
)
await consumer.connect()
loop = asyncio.get_running_loop()
stop_event = asyncio.Event()
for sig in (signal.SIGINT, signal.SIGTERM):
loop.add_signal_handler(sig, stop_event.set)
consumer_task = asyncio.create_task(consumer.run())
stop_task = asyncio.create_task(stop_event.wait())
log.info("worker_analyze_started", prefetch=settings.mq_prefetch_analyze)
try:
await asyncio.wait(
{consumer_task, stop_task},
return_when=asyncio.FIRST_COMPLETED,
)
finally:
await consumer.stop()
shutdown_telemetry()
if __name__ == "__main__":
asyncio.run(main())

67
src/contract_check/worker_analyze/consumer.py Normal file
View file

@ -0,0 +1,67 @@
"""worker-analyze consumer: wires the analyze handler into the RabbitMQ base."""
from __future__ import annotations
from ..core.db.enums import FailureClass
from ..core.db.session import create_session_factory
from ..core.llm.port import LLMProvider
from ..core.logging import get_logger
from ..core.metrics import analyze_duration
from ..core.mq.consumer import Consumer
from ..core.mq.messages import DocumentExtracted
from .handler import AnalyzeHandler
log = get_logger(__name__)
class AnalyzeConsumer(Consumer[DocumentExtracted]):
"""Consumes `analyze.q`, runs the LLM analysis, persists the report."""
queue: str = "analyze.q"
routing_key: str = "analyze"
message_model = DocumentExtracted
def __init__(
self,
url: str,
*,
origin: str,
prefetch: int,
max_attempts: int,
retry_base_ms: int,
provider: LLMProvider | None = None,
) -> None:
super().__init__(
url,
origin=origin,
prefetch=prefetch,
max_attempts=max_attempts,
retry_base_ms=retry_base_ms,
)
self._session_factory = create_session_factory()
self._handler = AnalyzeHandler(session_factory=self._session_factory, provider=provider)
def classify(self, exc: BaseException) -> FailureClass:
return self._handler.classify(exc)
@analyze_duration.time()
async def handle(self, payload: DocumentExtracted) -> None:
await self._handler.handle(payload)
async def on_failure(
self,
payload: DocumentExtracted,
failure_class: FailureClass,
attempt: int,
error: str,
) -> None:
await self._handler.on_failure(payload, failure_class, attempt, error)
async def on_dlq(
self, payload: DocumentExtracted, failure_class: FailureClass, error: str
) -> None:
await self._handler.on_terminal_failure(payload, failure_class, error)
async def stop(self) -> None:
await super().stop()
await self._handler.aclose()

257
src/contract_check/worker_analyze/handler.py Normal file
View file

@ -0,0 +1,257 @@
"""Analyze worker handler: download extracted text -> LLM analyze -> save report.
The handler is intentionally separate from the consumer so it can be tested
in-process without spinning up a real RabbitMQ consumer. It owns:
- idempotency checks against documents.status
- status transitions (analyzing -> done | failed)
- MinIO download of the extracted plaintext
- LLM analysis via core.llm (provider port; chunking/repair live in the adapter)
- rendering + persisting the report (JSONB + markdown, disclaimer guaranteed)
- updating the jobs row, recording failure class, and refund-on-DLQ.
"""
from __future__ import annotations
import json
import uuid
from typing import TYPE_CHECKING
from sqlalchemy import text
from ..core.analysis.analyzer import RenderMetrics, render_markdown
from ..core.analysis.checklist import checklist_for_prompt
from ..core.config import get_settings
from ..core.credits import refund_credit
from ..core.db.enums import DOC_TERMINAL, FailureClass
from ..core.llm.factory import build_llm_provider
from ..core.llm.port import LLMProvider
from ..core.logging import get_logger
from ..core.metrics import analyze_duration, llm_fell_back, llm_tokens, mq_failed
from ..core.mq.messages import DocumentExtracted
from ..core.s3.minio_storage import MinioStorage
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
log = get_logger(__name__)
class AnalyzeHandler:
"""Business logic for worker-analyze."""
def __init__(
self,
*,
session_factory: async_sessionmaker[AsyncSession],
provider: LLMProvider | None = None,
) -> None:
self._session_factory = session_factory
self._settings = get_settings()
self._storage = MinioStorage.from_endpoint_url(
endpoint_url=self._settings.s3_endpoint_url,
access_key=self._settings.s3_access_key,
secret_key=self._settings.s3_secret_key,
bucket=self._settings.s3_bucket,
region=self._settings.s3_region,
)
self._provider: LLMProvider | None = provider
self._owns_provider = provider is None
async def _provider_instance(self) -> LLMProvider:
if self._provider is None:
self._provider = build_llm_provider(self._settings)
return self._provider
async def aclose(self) -> None:
if self._provider is not None and self._owns_provider:
await self._provider.aclose()
async def _db_state(
self, session: AsyncSession, document_id: uuid.UUID
) -> tuple[str | None, str | None]:
row = await session.execute(
text("SELECT status, filename FROM documents WHERE id = :d FOR UPDATE"),
{"d": document_id},
)
result = row.first()
if result is None:
return None, None
return (None if result[0] is None else str(result[0]), str(result[1]))
async def handle(self, payload: DocumentExtracted) -> None:
async with self._session_factory() as session:
status, filename = await self._db_state(session, payload.document_id)
if status is None:
log.warning("document_not_found", document_id=str(payload.document_id))
return
if status in DOC_TERMINAL:
log.info("document_already_terminal", status=status)
return
await session.execute(
text("UPDATE documents SET status = 'analyzing', stage = 'llm' WHERE id = :d"),
{"d": payload.document_id},
)
await session.execute(
text(
"UPDATE jobs SET status = 'running', attempts = attempts + 1 "
"WHERE document_id = :d AND queue = 'analyze'"
),
{"d": payload.document_id},
)
await session.commit()
text_bytes = await self._storage.get(payload.extracted_s3_key)
contract_text = text_bytes.decode("utf-8")
source_name = filename or payload.extracted_s3_key
provider = await self._provider_instance()
with analyze_duration.time():
result = await provider.analyze(contract_text, checklist=checklist_for_prompt())
llm_tokens.labels(kind="prompt").inc(result.prompt_tokens)
llm_tokens.labels(kind="eval").inc(result.eval_tokens)
if result.fell_back:
llm_fell_back.inc()
chunks = max(
1,
(len(contract_text) + self._settings.chunk_size_chars - 1)
// self._settings.chunk_size_chars,
)
metrics = RenderMetrics(
chars=len(contract_text),
chunks=chunks,
findings=len(result.findings),
prompt_tokens=result.prompt_tokens,
eval_tokens=result.eval_tokens,
latency_sec=result.latency_sec,
models_used=set(result.models_used),
fell_back=result.fell_back,
repaired=result.repaired,
)
markdown = render_markdown(result.findings, source_name, metrics)
content_json = json.dumps(
{"findings": [f.model_dump() for f in result.findings]},
ensure_ascii=False,
)
async with self._session_factory() as session:
await session.execute(
text(
"INSERT INTO reports "
"(document_id, content_json, markdown, model_used, "
" prompt_tokens, eval_tokens, latency_ms) "
"VALUES (:d, CAST(:content AS jsonb), :md, :model, :pt, :et, :lat) "
"ON CONFLICT (document_id) DO UPDATE SET "
" content_json = EXCLUDED.content_json, "
" markdown = EXCLUDED.markdown, "
" model_used = EXCLUDED.model_used, "
" prompt_tokens = EXCLUDED.prompt_tokens, "
" eval_tokens = EXCLUDED.eval_tokens, "
" latency_ms = EXCLUDED.latency_ms"
),
{
"d": payload.document_id,
"content": content_json,
"md": markdown,
"model": result.model_used,
"pt": result.prompt_tokens,
"et": result.eval_tokens,
"lat": int(result.latency_sec * 1000),
},
)
await session.execute(
text("UPDATE documents SET status = 'done', stage = 'done' WHERE id = :d"),
{"d": payload.document_id},
)
await session.execute(
text(
"UPDATE jobs SET status = 'done' WHERE document_id = :d AND queue = 'analyze'"
),
{"d": payload.document_id},
)
await session.commit()
log.info(
"analyze_success",
document_id=str(payload.document_id),
findings=len(result.findings),
)
def classify(self, exc: BaseException) -> FailureClass:
from ..core.llm.ollama_cloud import (
LLMConfigError,
LLMError,
LLMQuotaError,
LLMUnavailableError,
)
if isinstance(exc, LLMQuotaError):
return "llm_quota"
if isinstance(exc, LLMConfigError):
return "infra"
if isinstance(exc, LLMUnavailableError):
return "llm_timeout"
if isinstance(exc, LLMError):
return "llm_invalid_output"
return "infra"
async def on_failure(
self,
payload: DocumentExtracted,
failure_class: FailureClass,
attempt: int,
error: str,
) -> None:
mq_failed.labels(queue="analyze", failure_class=failure_class).inc()
async with self._session_factory() as session:
await session.execute(
text(
"UPDATE jobs SET status = 'retrying', attempts = :a, "
"last_failure_class = :fc, last_error = :err "
"WHERE document_id = :d AND queue = 'analyze'"
),
{
"a": attempt,
"fc": failure_class,
"err": error[:1000],
"d": payload.document_id,
},
)
await session.commit()
async def on_terminal_failure(
self, payload: DocumentExtracted, failure_class: FailureClass, error: str
) -> None:
mq_failed.labels(queue="analyze", failure_class=failure_class).inc()
async with self._session_factory() as session:
await session.execute(
text(
"UPDATE jobs SET status = 'dlq', dlq = TRUE, "
"last_failure_class = :fc, last_error = :err "
"WHERE document_id = :d AND queue = 'analyze'"
),
{
"fc": failure_class,
"err": error[:1000],
"d": payload.document_id,
},
)
await session.execute(
text("UPDATE documents SET status = 'failed', stage = :stage WHERE id = :d"),
{"stage": failure_class, "d": payload.document_id},
)
await refund_credit(
session,
payload.document_id,
failure_class,
self._settings.refund_policy,
)
await session.commit()
log.warning(
"analyze_terminal_failure",
document_id=str(payload.document_id),
failure_class=failure_class,
)

0
src/contract_check/worker_extract/__init__.py Normal file
View file

62
src/contract_check/worker_extract/__main__.py Normal file
View file

@ -0,0 +1,62 @@
"""worker-extract entrypoint: connects to RabbitMQ and runs the extract consumer."""
from __future__ import annotations
import asyncio
import signal
from ..core.config import get_settings
from ..core.logging import bind_context, configure_logging, get_logger
from ..core.metrics import start_metrics_server
from ..core.sentry import init_sentry
from ..core.telemetry import setup_telemetry, shutdown_telemetry
from .consumer import ExtractConsumer
log = get_logger(__name__)
async def main() -> None:
settings = get_settings()
configure_logging(
settings.log_level,
json_output=settings.json_logs,
service="worker-extract",
env=settings.env,
)
bind_context(service="worker-extract", env=settings.env)
init_sentry("worker-extract")
setup_telemetry("worker-extract")
start_metrics_server(9101)
consumer = ExtractConsumer(
url=settings.rabbitmq_url,
origin="worker-extract",
prefetch=settings.mq_prefetch_extract,
max_attempts=settings.mq_max_attempts,
retry_base_ms=settings.mq_retry_base_ms,
)
await consumer.connect()
loop = asyncio.get_running_loop()
stop_event = asyncio.Event()
for sig in (signal.SIGINT, signal.SIGTERM):
loop.add_signal_handler(sig, stop_event.set)
consumer_task = asyncio.create_task(consumer.run())
stop_task = asyncio.create_task(stop_event.wait())
log.info("worker_extract_started", prefetch=settings.mq_prefetch_extract)
try:
await asyncio.wait(
{consumer_task, stop_task},
return_when=asyncio.FIRST_COMPLETED,
)
finally:
await consumer.stop()
shutdown_telemetry()
if __name__ == "__main__":
asyncio.run(main())

61
src/contract_check/worker_extract/consumer.py Normal file
View file

@ -0,0 +1,61 @@
"""worker-extract consumer: wires the extract handler into the RabbitMQ base."""
from __future__ import annotations
from ..core.db.enums import FailureClass
from ..core.db.session import create_session_factory
from ..core.logging import get_logger
from ..core.metrics import extract_duration
from ..core.mq.consumer import Consumer
from ..core.mq.messages import DocumentUploaded
from ..core.mq.topology import RK_ANALYZE
from .handler import ExtractHandler
log = get_logger(__name__)
class ExtractConsumer(Consumer[DocumentUploaded]):
"""Consumes `extract.q`, extracts text/OCR, publishes to `analyze.q`."""
queue: str = "extract.q"
routing_key: str = "extract"
message_model = DocumentUploaded
def __init__(
self,
url: str,
*,
origin: str,
prefetch: int,
max_attempts: int,
retry_base_ms: int,
) -> None:
super().__init__(
url,
origin=origin,
prefetch=prefetch,
max_attempts=max_attempts,
retry_base_ms=retry_base_ms,
)
self._session_factory = create_session_factory()
self._handler = ExtractHandler(
session_factory=self._session_factory,
publish_routing_key=RK_ANALYZE,
)
def classify(self, exc: BaseException) -> FailureClass:
return self._handler.classify(exc)
@extract_duration.time()
async def handle(self, payload: DocumentUploaded) -> None:
await self._handler.handle(payload)
async def on_failure(
self, payload: DocumentUploaded, failure_class: FailureClass, attempt: int, error: str
) -> None:
await self._handler.on_failure(payload, failure_class, attempt, error)
async def on_dlq(
self, payload: DocumentUploaded, failure_class: FailureClass, error: str
) -> None:
await self._handler.on_terminal_failure(payload, failure_class, error)

26
src/contract_check/worker_extract/extract_document.py Normal file
View file

@ -0,0 +1,26 @@
"""Facade re-exporting extraction + OCR helpers for the worker."""
from __future__ import annotations
from ..core.analysis.extractor import ExtractionError, extract_text
from ..core.analysis.ocr import OCRError, ocr_pdf
__all__ = ["ExtractionError", "OCRError", "extract_document"]
def extract_document(path: str) -> str:
"""Try direct extraction; fall back to OCR for likely scans.
Mirrors docs/ARCHITECTURE.md §16 worker-extract pseudocode:
1. extract_text (PDF text layer / DOCX)
2. if ExtractionError due to short text, try OCR
3. if OCR still short, raise ExtractionError (failure_class extraction_failed)
"""
try:
return extract_text(path)
except ExtractionError:
# Likely scan or image-only PDF; attempt OCR once.
return ocr_pdf(path)
__all__.append("extract_document")

240
src/contract_check/worker_extract/handler.py Normal file
View file

@ -0,0 +1,240 @@
"""Extract worker handler: download blob → extract/OCR → upload text → publish.
The handler is intentionally separate from the consumer so it can be tested
in-process without spinning up a real RabbitMQ consumer. It owns:
- idempotency checks against documents.status
- status transitions (queued → extracting → ocr)
- MinIO download/upload
- extraction/OCR via core.analysis
- publishing DocumentExtracted to analyze.q
- updating the jobs row, recording failure class, and refund-on-DLQ.
"""
from __future__ import annotations
import asyncio
import tempfile
import uuid
from pathlib import Path
from typing import TYPE_CHECKING
from sqlalchemy import text
from ..core.config import get_settings
from ..core.credits import refund_credit
from ..core.db.enums import DOC_TERMINAL, FailureClass
from ..core.logging import get_logger
from ..core.metrics import extract_duration, mq_failed, mq_published
from ..core.mq.messages import DocumentExtracted, DocumentUploaded
from ..core.mq.publisher import Publisher
from ..core.s3 import extracted_key
from ..core.s3.minio_storage import MinioStorage
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
log = get_logger(__name__)
class ExtractHandler:
"""Business logic for worker-extract."""
def __init__(
self,
*,
session_factory: async_sessionmaker[AsyncSession],
publish_routing_key: str = "analyze",
) -> None:
self._session_factory = session_factory
self._publish_routing_key = publish_routing_key
self._settings = get_settings()
self._storage = MinioStorage.from_endpoint_url(
endpoint_url=self._settings.s3_endpoint_url,
access_key=self._settings.s3_access_key,
secret_key=self._settings.s3_secret_key,
bucket=self._settings.s3_bucket,
region=self._settings.s3_region,
)
self._publisher: Publisher | None = None
async def _publisher_instance(self) -> Publisher:
if self._publisher is None:
self._publisher = Publisher(self._settings.rabbitmq_url, origin="worker-extract")
await self._publisher.connect()
return self._publisher
async def _db_status(self, session: AsyncSession, document_id: uuid.UUID) -> str | None:
row = await session.execute(
text("SELECT status FROM documents WHERE id = :d FOR UPDATE"),
{"d": document_id},
)
result = row.first()
return str(result[0]) if result else None
async def handle(self, payload: DocumentUploaded) -> None:
async with self._session_factory() as session:
current = await self._db_status(session, payload.document_id)
if current is None:
log.warning("document_not_found", document_id=str(payload.document_id))
return
if current in DOC_TERMINAL:
log.info("document_already_terminal", status=current)
return
await session.execute(
text(
"UPDATE documents SET status = 'extracting', stage = 'downloading' "
"WHERE id = :d"
),
{"d": payload.document_id},
)
await session.execute(
text(
"UPDATE jobs SET status = 'running', attempts = attempts + 1 "
"WHERE document_id = :d AND queue = 'extract'"
),
{"d": payload.document_id},
)
await session.commit()
with extract_duration.time():
extracted_text, ocr_used = await self._extract_text(payload)
ext_key = extracted_key(str(payload.user_id), str(payload.document_id))
text_bytes = extracted_text.encode("utf-8")
await self._storage.put(ext_key, text_bytes, content_type="text/plain; charset=utf-8")
msg = DocumentExtracted(
correlation_id=payload.correlation_id,
document_id=payload.document_id,
user_id=payload.user_id,
extracted_s3_key=ext_key,
char_count=len(extracted_text),
ocr_used=ocr_used,
attempt=payload.attempt,
)
publisher = await self._publisher_instance()
await publisher.publish(msg, routing_key=self._publish_routing_key)
mq_published.labels(queue="analyze").inc()
async with self._session_factory() as session:
await session.execute(
text(
"UPDATE documents SET status = 'analyzing', stage = 'queued_analyze', "
"extracted_s3_key = :key WHERE id = :d"
),
{"key": ext_key, "d": payload.document_id},
)
await session.execute(
text(
"UPDATE jobs SET status = 'done' WHERE document_id = :d AND queue = 'extract'"
),
{"d": payload.document_id},
)
await session.commit()
log.info(
"extract_success",
document_id=str(payload.document_id),
char_count=len(extracted_text),
)
async def _extract_text(self, payload: DocumentUploaded) -> tuple[str, bool]:
from ..core.analysis.extractor import ExtractionError, extract_text
from ..core.analysis.ocr import ocr_pdf
data = await self._storage.get(payload.s3_key)
suffix = Path(payload.filename).suffix.lower()
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:
tmp.write(data)
tmp_path = Path(tmp.name)
try:
try:
text = await asyncio.to_thread(extract_text, str(tmp_path))
return text, False
except ExtractionError:
await self._update_stage(payload.document_id, "ocr")
text = await asyncio.to_thread(ocr_pdf, str(tmp_path))
return text, True
finally:
tmp_path.unlink(missing_ok=True)
async def _update_stage(self, document_id: uuid.UUID, stage: str) -> None:
from sqlalchemy import text
async with self._session_factory() as session:
await session.execute(
text("UPDATE documents SET stage = :stage WHERE id = :d"),
{"stage": stage, "d": document_id},
)
await session.commit()
def classify(self, exc: BaseException) -> FailureClass:
from ..core.analysis.extractor import ExtractionError
from ..core.analysis.ocr import OCRError
if isinstance(exc, ExtractionError):
return "extraction_failed"
if isinstance(exc, OCRError):
return "ocr_failed"
return "infra"
async def on_failure(
self, payload: DocumentUploaded, failure_class: FailureClass, attempt: int, error: str
) -> None:
from sqlalchemy import text
mq_failed.labels(queue="extract", failure_class=failure_class).inc()
async with self._session_factory() as session:
await session.execute(
text(
"UPDATE jobs SET status = 'retrying', attempts = :a, "
"last_failure_class = :fc, last_error = :err "
"WHERE document_id = :d AND queue = 'extract'"
),
{
"a": attempt,
"fc": failure_class,
"err": error[:1000],
"d": payload.document_id,
},
)
await session.commit()
async def on_terminal_failure(
self, payload: DocumentUploaded, failure_class: FailureClass, error: str
) -> None:
mq_failed.labels(queue="extract", failure_class=failure_class).inc()
async with self._session_factory() as session:
await session.execute(
text(
"UPDATE jobs SET status = 'dlq', dlq = TRUE, "
"last_failure_class = :fc, last_error = :err "
"WHERE document_id = :d AND queue = 'extract'"
),
{
"fc": failure_class,
"err": error[:1000],
"d": payload.document_id,
},
)
await session.execute(
text("UPDATE documents SET status = 'failed', stage = :stage WHERE id = :d"),
{"stage": failure_class, "d": payload.document_id},
)
await refund_credit(
session,
payload.document_id,
failure_class,
self._settings.refund_policy,
)
await session.commit()
log.warning(
"extract_terminal_failure",
document_id=str(payload.document_id),
failure_class=failure_class,
)

38
srv/api/Dockerfile Normal file
View file

@ -0,0 +1,38 @@
# syntax=docker/dockerfile:1
# ─── Stage 1: build deps + project into a venv via uv ─────────────────────────
FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim AS builder
ENV UV_COMPILE_BYTECODE=1 \
UV_LINK_MODE=copy \
UV_PYTHON_DOWNLOADS=never \
UV_PROJECT_ENVIRONMENT=/app/.venv
WORKDIR /app
COPY pyproject.toml uv.lock ./
COPY README.md ./
COPY alembic.ini ./
COPY migrations ./migrations
COPY src ./src
# Install only the api group (core + db/mq/s3/obs + fastapi/uvicorn).
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --group api --no-install-project && \
uv pip install --no-deps .
# ─── Stage 2: lean runtime ─────────────────────────────────────────────────
FROM python:3.13-slim
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PATH=/app/.venv/bin:$PATH
WORKDIR /app
COPY --from=builder /app/.venv /app/.venv
COPY alembic.ini ./
COPY migrations ./migrations
EXPOSE 8000 9100
CMD ["python", "-m", "contract_check.api"]

34
srv/bot/Dockerfile Normal file
View file

@ -0,0 +1,34 @@
# syntax=docker/dockerfile:1
# ─── Stage 1: build deps + project into a venv via uv ─────────────────────────
FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim AS builder
ENV UV_COMPILE_BYTECODE=1 \
UV_LINK_MODE=copy \
UV_PYTHON_DOWNLOADS=never \
UV_PROJECT_ENVIRONMENT=/app/.venv
WORKDIR /app
COPY pyproject.toml uv.lock ./
COPY README.md ./
COPY src ./src
# Install only the bot group (core + aiogram). No DB/MQ/S3/LLM/OCR drivers —
# this is the leanest service image; it speaks HTTP to the api exclusively.
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --group bot --no-install-project && \
uv pip install --no-deps .
# ─── Stage 2: lean runtime ─────────────────────────────────────────────────
FROM python:3.13-slim
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PATH=/app/.venv/bin:$PATH
WORKDIR /app
COPY --from=builder /app/.venv /app/.venv
CMD ["python", "-m", "contract_check.bot"]

35
srv/prototype/Dockerfile Normal file
View file

@ -0,0 +1,35 @@
# syntax=docker/dockerfile:1
# ─── Stage 1: build deps + project into a venv via uv ─────────────────────────
# Standalone stage-0 benchmark CLI (PDF/DOCX → text → Ollama Cloud → markdown).
# No DB/MQ/S3 — kept as a reference benchmark image, not a production service.
FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim AS builder
ENV UV_COMPILE_BYTECODE=1 \
UV_LINK_MODE=copy \
UV_PYTHON_DOWNLOADS=never \
UV_PROJECT_ENVIRONMENT=/app/.venv
WORKDIR /app
COPY pyproject.toml uv.lock ./
COPY README.md ./
COPY src ./src
# Install only the prototype group (core + pymupdf/python-docx).
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --group prototype --no-install-project && \
uv pip install --no-deps .
# ─── Stage 2: lean runtime ─────────────────────────────────────────────────
FROM python:3.13-slim
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PATH=/app/.venv/bin:$PATH
WORKDIR /app
COPY --from=builder /app/.venv /app/.venv
CMD ["python", "-m", "contract_check.prototype"]

35
srv/worker-analyze/Dockerfile Normal file
View file

@ -0,0 +1,35 @@
# syntax=docker/dockerfile:1
# ─── Stage 1: build deps + project into a venv via uv ─────────────────────────
FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim AS builder
ENV UV_COMPILE_BYTECODE=1 \
UV_LINK_MODE=copy \
UV_PYTHON_DOWNLOADS=never \
UV_PROJECT_ENVIRONMENT=/app/.venv
WORKDIR /app
COPY pyproject.toml uv.lock ./
COPY README.md ./
COPY src ./src
# Install only the analyze group (core + db/mq/s3/obs + otel-httpx). No
# tesseract/pymupdf — this image is the leanest LLM/I/O worker.
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --group analyze --no-install-project && \
uv pip install --no-deps .
# ─── Stage 2: lean runtime ─────────────────────────────────────────────────
FROM python:3.13-slim
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PATH=/app/.venv/bin:$PATH
WORKDIR /app
COPY --from=builder /app/.venv /app/.venv
EXPOSE 9102
CMD ["python", "-m", "contract_check.worker_analyze"]

42
srv/worker-extract/Dockerfile Normal file
View file

@ -0,0 +1,42 @@
# syntax=docker/dockerfile:1
# ─── Stage 1: build deps + project into a venv via uv ─────────────────────────
FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim AS builder
ENV UV_COMPILE_BYTECODE=1 \
UV_LINK_MODE=copy \
UV_PYTHON_DOWNLOADS=never \
UV_PROJECT_ENVIRONMENT=/app/.venv
WORKDIR /app
COPY pyproject.toml uv.lock ./
COPY README.md ./
COPY src ./src
# Install only the extract group (core + db/mq/s3/obs + pymupdf/tesseract).
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --group extract --no-install-project && \
uv pip install --no-deps .
# ─── Stage 2: runtime with tesseract-ocr + language packs ────────────────────
FROM python:3.13-slim
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PATH=/app/.venv/bin:$PATH
WORKDIR /app
# Tesseract and Russian/English language data for scanned PDFs.
RUN apt-get update && apt-get install -y --no-install-recommends \
tesseract-ocr \
tesseract-ocr-rus \
tesseract-ocr-eng \
fonts-dejavu-core \
&& rm -rf /var/lib/apt/lists/*
COPY --from=builder /app/.venv /app/.venv
EXPOSE 9101
CMD ["python", "-m", "contract_check.worker_extract"]

18
tests/conftest.py Normal file
View file

@ -0,0 +1,18 @@
"""Shared pytest fixtures.
Unit tests stay fast and dependency-free (no DB/MQ/S3). Integration tests use
testcontainers and are marked `@pytest.mark.integration` (deselected by the
default `pytest -q` run).
"""
from __future__ import annotations
import os
# Default test env: prevent Settings() from failing on required fields when no
# .env is present. Individual tests that build Settings override as needed.
os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://x:x@localhost:5432/x")
os.environ.setdefault("RABBITMQ_URL", "amqp://x:x@localhost:5672//")
os.environ.setdefault("S3_ENDPOINT_URL", "http://localhost:9000")
os.environ.setdefault("S3_ACCESS_KEY", "test")
os.environ.setdefault("S3_SECRET_KEY", "test")

140
tests/integration/conftest.py Normal file
View file

@ -0,0 +1,140 @@
"""Integration-test fixtures using the running Docker Compose infrastructure.
Run `docker compose up -d` before executing integration tests. Migrations run
once per session; a service token is seeded so adapter-style endpoints can
authenticate.
"""
from __future__ import annotations
import os
import subprocess
import sys
from collections.abc import Iterator
from pathlib import Path
from typing import TYPE_CHECKING
import httpx
import pytest
import pytest_asyncio
from asgi_lifespan import LifespanManager
from sqlalchemy import text
from contract_check.api.app import create_app
from contract_check.core.config import get_settings
from contract_check.core.db.session import create_session_factory
from contract_check.core.tokens import hash_token
if TYPE_CHECKING:
pass
pytestmark = pytest.mark.integration
_BOT_SERVICE_TOKEN = "it-test-bot-token"
_DB_URL = "postgresql+asyncpg://contract_check:contract_check@localhost:15432/contract_check"
_AMQP_URL = "amqp://contract_check:contract_check@localhost:5672//"
_S3_URL = "http://localhost:9000"
def _set_env() -> None:
env_vars = {
"DATABASE_URL": _DB_URL,
"RABBITMQ_URL": _AMQP_URL,
"S3_ENDPOINT_URL": _S3_URL,
"S3_ACCESS_KEY": "contract_check",
"S3_SECRET_KEY": "contract_check",
"S3_BUCKET": "contract-check-docs",
"REDIS_URL": "redis://localhost:17379/0",
"OLLAMA_HOST": "http://localhost",
"OLLAMA_API_KEY": "test",
"JWT_SECRET": "it-test-jwt-secret-not-for-production",
"JWT_ALGORITHM": "HS256",
"JWT_ACCESS_TTL_MINUTES": "1440",
"TELEGRAM_BOT_TOKEN": "it-test-bot-token:it-test-secret",
}
for k, v in env_vars.items():
os.environ[k] = v
@pytest.fixture(scope="session")
def infra() -> Iterator[dict[str, str]]:
_set_env()
get_settings.cache_clear()
repo_root = Path(__file__).resolve().parents[2]
subprocess.run(
[sys.executable, "-m", "alembic", "-c", str(repo_root / "alembic.ini"), "upgrade", "head"],
cwd=str(repo_root),
env={**os.environ},
check=True,
capture_output=False,
)
factory = create_session_factory()
async def _seed() -> None:
async with factory() as session:
await session.execute(
text(
"INSERT INTO service_tokens (name, token_hash, adapter) "
"VALUES (:name, :hash, 'bot') "
"ON CONFLICT (name) DO UPDATE SET "
" token_hash = EXCLUDED.token_hash, revoked = FALSE"
),
{"name": "bot-test", "hash": hash_token(_BOT_SERVICE_TOKEN)},
)
await session.commit()
import asyncio
asyncio.run(_seed())
yield {
"database_url": _DB_URL,
"rabbitmq_url": _AMQP_URL,
"s3_endpoint_url": _S3_URL,
"s3_access_key": "contract_check",
"s3_secret_key": "contract_check",
"token": _BOT_SERVICE_TOKEN,
}
get_settings.cache_clear()
@pytest_asyncio.fixture
async def client(infra: dict[str, str]) -> httpx.AsyncClient:
get_settings.cache_clear()
_set_env()
get_settings()
app = create_app()
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=app), base_url="http://test"
) as c:
async with LifespanManager(app):
yield c
get_settings.cache_clear()
@pytest.fixture
def auth_header(infra: dict[str, str]) -> dict[str, str]:
return {"Authorization": f"Bearer {infra['token']}"}
async def user_token(client: httpx.AsyncClient, infra: dict[str, str], telegram_id: int) -> str:
"""Call /auth/telegram/bot and return a user JWT for the given telegram_id."""
r = await client.post(
"/api/v1/auth/telegram/bot",
json={"telegram_id": telegram_id},
headers={"Authorization": f"Bearer {infra['token']}"},
)
assert r.status_code == 200, f"auth failed: {r.status_code} {r.text}"
return r.json()["access_token"]
@pytest.fixture
async def db_session():
factory = create_session_factory()
async with factory() as session:
yield session
await session.close()

265
tests/integration/test_analyze_worker.py Normal file
View file

@ -0,0 +1,265 @@
"""Integration tests for worker-analyze.
Uses the same Docker Compose infra as test_upload_pipeline / test_extract_worker:
real Postgres, RabbitMQ, MinIO. The LLM is respx-mocked at a fake Ollama host.
Scenarios:
1. Happy path: DocumentExtracted -> report saved (JSONB+markdown), status=done,
disclaimer present, jobs row done.
2. Terminal failure: LLM quota error -> on_terminal_failure refunds + DLQ state.
"""
from __future__ import annotations
import json
import uuid
from typing import TYPE_CHECKING
import httpx
import pytest
import respx
from sqlalchemy import text
from contract_check.core.db.session import create_session_factory
from contract_check.core.llm.ollama_cloud import LLMQuotaError, OllamaCloudProvider
from contract_check.core.mq.messages import DocumentExtracted
from contract_check.core.s3 import extracted_key
from contract_check.core.s3.minio_storage import MinioStorage
from contract_check.worker_analyze.handler import AnalyzeHandler
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
pytestmark = pytest.mark.integration
HOST = "https://ollama.test"
URL = f"{HOST}/api/chat"
_VALID_FINDINGS = {
"findings": [
{
"checklist_id": "penalties",
"severity": "high",
"quote": "Штраф 0,5% за каждый день просрочки",
"section_ref": "п. 6.3",
"risk": "Высокая неустойка",
"recommendation": "Ограничить cap",
}
]
}
def _chat_body(content: str, model: str = "qwen2.5:14b") -> dict[str, object]:
return {
"model": model,
"message": {"content": content},
"prompt_eval_count": 10,
"eval_count": 20,
}
def _storage(infra: dict[str, str]) -> MinioStorage:
return MinioStorage.from_endpoint_url(
endpoint_url=infra["s3_endpoint_url"],
access_key=infra["s3_access_key"],
secret_key=infra["s3_secret_key"],
bucket="contract-check-docs",
)
def _session_factory(infra: dict[str, str]) -> async_sessionmaker[AsyncSession]:
return create_session_factory()
def _provider() -> OllamaCloudProvider:
return OllamaCloudProvider(
host=HOST,
api_key="key",
model="qwen2.5:14b",
fallback_model=None,
max_concurrency=1,
chunk_size=10000,
)
async def _seed_document(
infra: dict[str, str], *, telegram_id: int, credits: int, document_id: uuid.UUID
) -> tuple[uuid.UUID, int, str, str]:
"""Insert user + a post-extraction document; return (user_id, text, ext_key, s3_key)."""
sess = _session_factory(infra)
store = _storage(infra)
async with sess() as session:
result = await session.execute(
text(
"INSERT INTO users (telegram_id, credits_left) VALUES (:t, :c) "
"ON CONFLICT (telegram_id) DO UPDATE SET credits_left = :c "
"RETURNING id, credits_left"
),
{"t": telegram_id, "c": credits},
)
row = result.first()
assert row is not None
user_id, initial_credits = row
await session.commit()
ext_key = extracted_key(str(user_id), str(document_id))
s3_key = f"users/{user_id}/docs/{document_id}.pdf"
contract_text = "Договор поставки. " * 50
await store.put(
ext_key, contract_text.encode("utf-8"), content_type="text/plain; charset=utf-8"
)
async with sess() as session:
await session.execute(
text(
"INSERT INTO documents "
"(id, user_id, s3_key, extracted_s3_key, filename, mime, bytes, "
" status, stage) "
"VALUES (:id, :uid, :s3, :ext, :fn, :mime, :bytes, "
" 'analyzing', 'queued_analyze')"
),
{
"id": document_id,
"uid": user_id,
"s3": s3_key,
"ext": ext_key,
"fn": "contract.pdf",
"mime": "application/pdf",
"bytes": len(contract_text),
},
)
await session.execute(
text(
"INSERT INTO jobs (document_id, correlation_id, queue, status) "
"VALUES (:did, :cid, 'analyze', 'pending')"
),
{"did": document_id, "cid": uuid.uuid4()},
)
await session.commit()
return user_id, initial_credits, ext_key, contract_text
async def test_analyze_worker_saves_report_and_marks_done(
infra: dict[str, str],
) -> None:
sess = _session_factory(infra)
document_id = uuid.uuid4()
correlation_id = uuid.uuid4()
user_id, _initial, ext_key, contract_text = await _seed_document(
infra, telegram_id=222_333_444, credits=5, document_id=document_id
)
async with _provider() as provider:
with respx.mock(base_url=HOST) as mock:
mock.post(URL).mock(
return_value=httpx.Response(200, json=_chat_body(json.dumps(_VALID_FINDINGS)))
)
await AnalyzeHandler(session_factory=sess, provider=provider).handle(
DocumentExtracted(
correlation_id=correlation_id,
document_id=document_id,
user_id=user_id,
extracted_s3_key=ext_key,
char_count=len(contract_text),
ocr_used=False,
)
)
async with sess() as session:
report = await session.execute(
text(
"SELECT markdown, model_used, prompt_tokens, eval_tokens "
"FROM reports WHERE document_id = :d"
),
{"d": document_id},
)
row = report.first()
assert row is not None
markdown, model_used, prompt_tokens, eval_tokens = row
assert "Дисклеймер" in markdown
assert "Штраф 0,5%" in markdown
assert model_used == "qwen2.5:14b"
assert prompt_tokens == 10
assert eval_tokens == 20
doc = await session.execute(
text("SELECT status, stage FROM documents WHERE id = :d"),
{"d": document_id},
)
status, stage = doc.one()
assert status == "done"
assert stage == "done"
job = await session.execute(
text("SELECT status FROM jobs WHERE document_id = :d AND queue = 'analyze'"),
{"d": document_id},
)
assert job.scalar_one() == "done"
async def test_analyze_worker_terminal_failure_refunds_and_dlqs(
infra: dict[str, str],
) -> None:
sess = _session_factory(infra)
document_id = uuid.uuid4()
correlation_id = uuid.uuid4()
user_id, initial_credits, ext_key, contract_text = await _seed_document(
infra, telegram_id=555_666_777, credits=3, document_id=document_id
)
async with _provider() as provider:
handler = AnalyzeHandler(session_factory=sess, provider=provider)
with respx.mock(base_url=HOST) as mock:
mock.post(URL).mock(return_value=httpx.Response(429, json={"error": "quota"}))
with pytest.raises(LLMQuotaError):
await handler.handle(
DocumentExtracted(
correlation_id=correlation_id,
document_id=document_id,
user_id=user_id,
extracted_s3_key=ext_key,
char_count=len(contract_text),
ocr_used=False,
)
)
await handler.on_terminal_failure(
DocumentExtracted(
correlation_id=correlation_id,
document_id=document_id,
user_id=user_id,
extracted_s3_key=ext_key,
char_count=len(contract_text),
ocr_used=False,
),
"llm_quota",
"test terminal failure",
)
async with sess() as session:
doc = await session.execute(
text("SELECT status, refunded FROM documents WHERE id = :d"),
{"d": document_id},
)
status, refunded = doc.one()
assert status == "failed"
assert refunded is True
job = await session.execute(
text(
"SELECT status, dlq, last_failure_class FROM jobs "
"WHERE document_id = :d AND queue = 'analyze'"
),
{"d": document_id},
)
j_status, j_dlq, j_class = job.one()
assert j_status == "dlq"
assert j_dlq is True
assert j_class == "llm_quota"
credits = await session.execute(
text("SELECT credits_left FROM users WHERE id = :u"),
{"u": user_id},
)
assert credits.scalar_one() == initial_credits + 1

168
tests/integration/test_auth_flow.py Normal file
View file

@ -0,0 +1,168 @@
"""Auth integration tests: Telegram identity sources issue a common JWT.
Run against the Docker Compose infrastructure (`docker compose up -d`).
"""
from __future__ import annotations
import hashlib
import hmac
import time
from urllib.parse import urlencode
import httpx
import pytest
from sqlalchemy import text
pytestmark = pytest.mark.integration
_BOT_TOKEN = "it-test-bot-token:it-test-secret"
_SECRET_KEY = hmac.new(
_BOT_TOKEN.encode("utf-8"),
b"WebAppData",
hashlib.sha256,
).digest()
def _make_web_payload(telegram_id: int) -> dict[str, object]:
auth_date = int(time.time())
data = {"id": telegram_id, "first_name": "Integration", "auth_date": auth_date}
data_check_string = "\n".join(f"{k}={v}" for k, v in sorted(data.items()))
data["hash"] = hmac.new(
_SECRET_KEY,
data_check_string.encode("utf-8"),
hashlib.sha256,
).hexdigest()
return data
def _make_init_data(telegram_id: int) -> str:
auth_date = int(time.time())
user_json = f'{{"id":{telegram_id},"first_name":"Integration"}}'
params = {"user": user_json, "auth_date": str(auth_date)}
data_check_string = "\n".join(f"{k}={v}" for k, v in sorted(params.items()))
params["hash"] = hmac.new(
_SECRET_KEY,
data_check_string.encode("utf-8"),
hashlib.sha256,
).hexdigest()
return urlencode(params)
async def test_auth_telegram_bot_creates_user_and_issues_jwt(
client: httpx.AsyncClient,
infra: dict[str, str],
db_session,
) -> None:
telegram_id = 123_456_789
r = await client.post(
"/api/v1/auth/telegram/bot",
json={"telegram_id": telegram_id},
headers={"Authorization": f"Bearer {infra['token']}"},
)
assert r.status_code == 200
body = r.json()
assert body["token_type"] == "bearer"
assert body["telegram_id"] == telegram_id
assert body["access_token"]
# User row created.
result = await db_session.execute(
text("SELECT id, telegram_id FROM users WHERE telegram_id = :t"),
{"t": telegram_id},
)
row = result.first()
assert row is not None
assert str(row[0]) == body["user_id"]
# JWT works on /me.
me = await client.get(
"/api/v1/me",
headers={"Authorization": f"Bearer {body['access_token']}"},
)
assert me.status_code == 200
assert me.json()["telegram_id"] == telegram_id
async def test_auth_telegram_web_verifies_widget_payload(
client: httpx.AsyncClient,
infra: dict[str, str],
) -> None:
telegram_id = 222_333_444
r = await client.post(
"/api/v1/auth/telegram/web",
json=_make_web_payload(telegram_id),
)
assert r.status_code == 200
body = r.json()
assert body["telegram_id"] == telegram_id
assert body["access_token"]
async def test_auth_telegram_miniapp_verifies_init_data(
client: httpx.AsyncClient,
infra: dict[str, str],
) -> None:
telegram_id = 333_444_555
r = await client.post(
"/api/v1/auth/telegram/miniapp",
json={"init_data": _make_init_data(telegram_id)},
)
assert r.status_code == 200
body = r.json()
assert body["telegram_id"] == telegram_id
assert body["access_token"]
async def test_auth_telegram_web_rejects_bad_hash(
client: httpx.AsyncClient,
infra: dict[str, str],
) -> None:
payload = _make_web_payload(444_555_666)
payload["hash"] = "0" * 64
r = await client.post("/api/v1/auth/telegram/web", json=payload)
assert r.status_code == 401
async def test_protected_endpoint_rejects_missing_token(
client: httpx.AsyncClient,
) -> None:
r = await client.get("/api/v1/me")
assert r.status_code == 401
async def test_protected_endpoint_rejects_service_token(
client: httpx.AsyncClient,
infra: dict[str, str],
) -> None:
r = await client.get(
"/api/v1/me",
headers={"Authorization": f"Bearer {infra['token']}"},
)
assert r.status_code == 401
async def test_auth_me_introspects_jwt(
client: httpx.AsyncClient,
infra: dict[str, str],
) -> None:
telegram_id = 555_666_777
r = await client.post(
"/api/v1/auth/telegram/bot",
json={"telegram_id": telegram_id},
headers={"Authorization": f"Bearer {infra['token']}"},
)
token = r.json()["access_token"]
introspect = await client.get(
"/api/v1/auth/me",
headers={"Authorization": f"Bearer {token}"},
)
assert introspect.status_code == 200
body = introspect.json()
assert body["telegram_id"] == telegram_id
assert body["type"] == "access"

202
tests/integration/test_b2b_api.py Normal file
View file

@ -0,0 +1,202 @@
"""B2B API integration tests: API-key auth, upload, report polling, management.
Run against the Docker Compose infrastructure (`docker compose up -d`).
"""
from __future__ import annotations
import uuid
from pathlib import Path
import aio_pika
import httpx
import pytest
from tests.integration.conftest import user_token
pytestmark = pytest.mark.integration
@pytest.fixture
def pdf_bytes(tmp_path: Path) -> bytes:
import pymupdf
doc = pymupdf.open()
page = doc.new_page()
page.insert_text((72, 72), "Договор. Стороны обязуются.")
path = tmp_path / "contract.pdf"
doc.save(str(path))
doc.close()
return path.read_bytes()
@pytest.fixture
async def api_key_client(
client: httpx.AsyncClient,
db_session, # noqa: ANN001
infra: dict[str, str],
) -> tuple[httpx.AsyncClient, str, int]:
"""Create a user, authenticate, create an API key, and return (client, key, telegram_id)."""
from sqlalchemy import text
from contract_check.core.api_keys import hash_api_key
telegram_id = 999000999
await db_session.execute(
text(
"INSERT INTO users (telegram_id, credits_left) VALUES (:t, 10) "
"ON CONFLICT (telegram_id) DO UPDATE SET credits_left = 10"
),
{"t": telegram_id},
)
# Clean any leftover key from a previous interrupted run.
await db_session.execute(
text(
"DELETE FROM api_keys "
"WHERE user_id = (SELECT id FROM users WHERE telegram_id = :t) "
"AND name = :name"
),
{"t": telegram_id, "name": "integration-test-key"},
)
await db_session.commit()
token = await user_token(client, infra, telegram_id)
create_resp = await client.post(
"/api/v1/b2b/keys",
headers={"Authorization": f"Bearer {token}"},
json={"name": "integration-test-key"},
)
assert create_resp.status_code == 201
body = create_resp.json()
raw_key = body["api_key"]
# Confirm hash is stored.
result = await db_session.execute(
text("SELECT id FROM api_keys WHERE key_hash = :h"),
{"h": hash_api_key(raw_key)},
)
assert result.first() is not None
return client, raw_key, telegram_id
async def test_b2b_analyze_upload_returns_202_and_enqueues(
api_key_client: tuple[httpx.AsyncClient, str, int],
pdf_bytes: bytes,
infra: dict[str, str],
) -> None:
client, api_key, _telegram_id = api_key_client
response = await client.post(
"/api/v1/analyze",
headers={"X-API-Key": api_key},
files={"file": ("contract.pdf", pdf_bytes, "application/pdf")},
)
assert response.status_code == 202
body = response.json()
document_id = uuid.UUID(body["document_id"])
correlation_id = uuid.UUID(body["correlation_id"])
assert body["credits_left"] == 9
# Verify a DocumentUploaded message landed on extract.q (drain leftovers).
connection = await aio_pika.connect_robust(infra["rabbitmq_url"])
try:
channel = await connection.channel()
queue = await channel.get_queue("extract.q", ensure=False)
deadline = 10.0
found = False
import time
while deadline > 0:
start = time.monotonic()
message = await queue.get(timeout=deadline)
await message.ack()
msg_body = message.body.decode("utf-8")
if str(document_id) in msg_body:
assert str(correlation_id) in msg_body
found = True
break
deadline -= time.monotonic() - start
assert found, "expected DocumentUploaded message not found in extract.q"
finally:
await connection.close()
async def test_b2b_get_report_before_ready_returns_status(
api_key_client: tuple[httpx.AsyncClient, str, int],
pdf_bytes: bytes,
) -> None:
client, api_key, _telegram_id = api_key_client
upload = await client.post(
"/api/v1/analyze",
headers={"X-API-Key": api_key},
files={"file": ("contract.pdf", pdf_bytes, "application/pdf")},
)
assert upload.status_code == 202
document_id = upload.json()["document_id"]
poll = await client.get(
f"/api/v1/b2b/reports/{document_id}",
headers={"X-API-Key": api_key},
)
assert poll.status_code == 200
body = poll.json()
assert body["document_id"] == document_id
assert body["status"] == "queued"
assert "stage" in body
async def test_b2b_missing_api_key_returns_401(client: httpx.AsyncClient) -> None:
response = await client.get("/api/v1/b2b/usage")
assert response.status_code == 401
async def test_b2b_invalid_api_key_returns_401(client: httpx.AsyncClient) -> None:
response = await client.get(
"/api/v1/b2b/usage",
headers={"X-API-Key": "clearly-invalid-key"},
)
assert response.status_code == 401
async def test_b2b_key_management_requires_user_jwt(
client: httpx.AsyncClient,
) -> None:
response = await client.post(
"/api/v1/b2b/keys",
json={"name": "no-auth-key"},
)
assert response.status_code == 401
async def test_b2b_revoke_key_blocks_usage(
api_key_client: tuple[httpx.AsyncClient, str, int],
infra: dict[str, str],
pdf_bytes: bytes,
) -> None:
client, api_key, telegram_id = api_key_client
token = await user_token(client, infra, telegram_id)
# List keys to find id.
list_resp = await client.get(
"/api/v1/b2b/keys",
headers={"Authorization": f"Bearer {token}"},
)
assert list_resp.status_code == 200
key_id = list_resp.json()[0]["id"]
revoke = await client.post(
f"/api/v1/b2b/keys/{key_id}/revoke",
headers={"Authorization": f"Bearer {token}"},
)
assert revoke.status_code == 200
# Revoked key cannot upload.
response = await client.post(
"/api/v1/analyze",
headers={"X-API-Key": api_key},
files={"file": ("contract.pdf", pdf_bytes, "application/pdf")},
)
assert response.status_code == 401

137
tests/integration/test_credits_db.py Normal file
View file

@ -0,0 +1,137 @@
"""Credits DB integration tests (requires real Postgres via testcontainers).
Marked `integration`; not run by the default fast suite. Verifies the atomic
reserve, idempotent refund, and the race-safety of `reserve_credit`.
"""
from __future__ import annotations
import asyncio
import uuid
import pytest
import pytest_asyncio
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
from testcontainers.community.postgres import PostgresContainer
from contract_check.core.credits import refund_credit, reserve_credit
pytestmark = pytest.mark.integration
@pytest_asyncio.fixture
async def pg_session():
with PostgresContainer("postgres:16-alpine") as pg:
url = pg.get_connection_url().replace("psycopg2", "asyncpg")
engine = create_async_engine(url)
# Create minimal schema in-memory (Postgres handles UUID/JSONB fine).
async with engine.begin() as conn:
await conn.execute(text("CREATE EXTENSION IF NOT EXISTS pgcrypto"))
await conn.execute(
text(
"CREATE TABLE users ("
"id UUID PRIMARY KEY DEFAULT gen_random_uuid(),"
"credits_left INT NOT NULL DEFAULT 0 CHECK (credits_left >= 0)"
")"
)
)
await conn.execute(
text(
"CREATE TABLE documents ("
"id UUID PRIMARY KEY DEFAULT gen_random_uuid(),"
"user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,"
"refunded BOOLEAN NOT NULL DEFAULT FALSE"
")"
)
)
async with engine.connect() as conn:
async_session = AsyncSession(bind=conn)
yield async_session
await engine.dispose()
async def insert_user(session: AsyncSession, credits: int = 5) -> uuid.UUID:
result = await session.execute(
text("INSERT INTO users (credits_left) VALUES (:c) RETURNING id"),
{"c": credits},
)
user_id = result.scalar_one()
await session.commit()
return user_id
async def insert_doc(session: AsyncSession, user_id: uuid.UUID) -> uuid.UUID:
result = await session.execute(
text("INSERT INTO documents (user_id, refunded) VALUES (:u, FALSE) RETURNING id"),
{"u": user_id},
)
doc_id = result.scalar_one()
await session.commit()
return doc_id
async def credit_balance(session: AsyncSession, user_id: uuid.UUID) -> int:
result = await session.execute(
text("SELECT credits_left FROM users WHERE id = :u"), {"u": user_id}
)
return int(result.scalar_one())
async def test_reserve_credit_decrements_once(pg_session: AsyncSession) -> None:
u = await insert_user(pg_session, credits=2)
ok = await reserve_credit(pg_session, u)
assert ok is True
await pg_session.commit()
assert await credit_balance(pg_session, u) == 1
async def test_reserve_credit_rejects_when_zero(pg_session: AsyncSession) -> None:
u = await insert_user(pg_session, credits=0)
ok = await reserve_credit(pg_session, u)
assert ok is False
async def test_reserve_credit_never_goes_negative(pg_session: AsyncSession) -> None:
u = await insert_user(pg_session, credits=1)
results = await asyncio.gather(
reserve_credit(pg_session, u),
reserve_credit(pg_session, u),
reserve_credit(pg_session, u),
)
# The atomic UPDATE WHERE credits_left>0 serializes concurrent attempts.
assert sum(1 for r in results if r) == 1
await pg_session.commit()
assert await credit_balance(pg_session, u) == 0
async def test_refund_credit_idempotent(pg_session: AsyncSession) -> None:
u = await insert_user(pg_session, credits=0)
d = await insert_doc(pg_session, u)
ok1 = await refund_credit(pg_session, d, "llm_quota", "all")
assert ok1 is True
await pg_session.commit()
assert await credit_balance(pg_session, u) == 1
ok2 = await refund_credit(pg_session, d, "llm_quota", "all")
assert ok2 is False # already refunded
await pg_session.commit()
assert await credit_balance(pg_session, u) == 1
async def test_refund_credit_respects_infra_only(pg_session: AsyncSession) -> None:
u = await insert_user(pg_session, credits=0)
d = await insert_doc(pg_session, u)
ok = await refund_credit(pg_session, d, "extraction_failed", "infra_only")
assert ok is False # user pays for garbage
await pg_session.commit()
assert await credit_balance(pg_session, u) == 0
# LLM failure is refunded under infra_only.
d2 = await insert_doc(pg_session, u)
ok2 = await refund_credit(pg_session, d2, "llm_quota", "infra_only")
assert ok2 is True
await pg_session.commit()
assert await credit_balance(pg_session, u) == 1

369
tests/integration/test_extract_worker.py Normal file
View file

@ -0,0 +1,369 @@
"""Integration tests for worker-extract.
Uses the same Docker Compose infra as test_upload_pipeline:
- real Postgres, RabbitMQ, MinIO
- alembic migrations + seeded service token (via conftest.py infra fixture)
Scenarios:
1. Happy path: DocumentUploaded → extracted text on analyze.q + status=analyzing.
2. DOCX extraction also works end-to-end.
3. Terminal failure path: bad PDF → extraction_failed → refund + document failed.
"""
from __future__ import annotations
import json
import time
import uuid
from pathlib import Path
from typing import TYPE_CHECKING
import aio_pika
import pytest
from contract_check.core.db.session import create_session_factory
from contract_check.core.mq.messages import DocumentUploaded
from contract_check.core.s3 import original_key
from contract_check.core.s3.minio_storage import MinioStorage
from contract_check.worker_extract.handler import ExtractHandler
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
pytestmark = pytest.mark.integration
def storage(infra: dict[str, str]) -> MinioStorage:
return MinioStorage.from_endpoint_url(
endpoint_url=infra["s3_endpoint_url"],
access_key=infra["s3_access_key"],
secret_key=infra["s3_secret_key"],
bucket="contract-check-docs",
)
def session_factory(infra: dict[str, str]) -> async_sessionmaker[AsyncSession]:
return create_session_factory()
@pytest.fixture
def pdf_bytes(tmp_path: Path) -> bytes:
import pymupdf
doc = pymupdf.open()
page = doc.new_page()
long_text = (
"Договор. Стороны обязуются выполнять условия. "
"Сторона А обязуется передать товар. "
"Сторона Б обязуется оплатить товар в течение десяти банковских дней. "
"Ответственность сторон ограничена суммой договора. "
"Споры подлежат рассмотрению в арбитражном суде города Москвы."
)
page.insert_htmlbox(
page.rect,
f'<p style="font-family:DejaVu Sans;font-size:14px">{long_text}</p>',
)
path = tmp_path / "contract.pdf"
doc.save(str(path))
doc.close()
return path.read_bytes()
@pytest.fixture
def docx_bytes(tmp_path: Path) -> bytes:
from docx import Document
doc = Document()
long_text = (
"Договор. Стороны обязуются выполнять условия. "
"Сторона А обязуется передать товар. "
"Сторона Б обязуется оплатить товар в течение десяти банковских дней. "
"Ответственность сторон ограничена суммой договора. "
"Споры подлежат рассмотрению в арбитражном суде города Москвы."
)
doc.add_paragraph(long_text)
path = tmp_path / "contract.docx"
doc.save(str(path))
return path.read_bytes()
async def test_extract_worker_pdf_uploads_text_and_publishes_analyze(
infra: dict[str, str],
pdf_bytes: bytes,
) -> None:
from sqlalchemy import text
telegram_id = 111_222_333
user_id: uuid.UUID | None = None
sess_factory = session_factory(infra)
store = storage(infra)
async with sess_factory() as session:
result = await session.execute(
text(
"INSERT INTO users (telegram_id, credits_left) VALUES (:t, 5) "
"ON CONFLICT (telegram_id) DO UPDATE SET credits_left = 5 "
"RETURNING id"
),
{"t": telegram_id},
)
user_id = result.scalar_one()
await session.commit()
assert user_id is not None
document_id = uuid.uuid4()
correlation_id = uuid.uuid4()
s3_key = original_key(str(user_id), str(document_id), ".pdf")
await store.put(s3_key, pdf_bytes, content_type="application/pdf")
async with sess_factory() as session:
await session.execute(
text(
"INSERT INTO documents (id, user_id, s3_key, filename, mime, bytes, status) "
"VALUES (:id, :uid, :s3, :fn, :mime, :bytes, 'queued')"
),
{
"id": document_id,
"uid": user_id,
"s3": s3_key,
"fn": "contract.pdf",
"mime": "application/pdf",
"bytes": len(pdf_bytes),
},
)
await session.execute(
text(
"INSERT INTO jobs (document_id, correlation_id, queue, status) "
"VALUES (:did, :cid, 'extract', 'pending')"
),
{"did": document_id, "cid": correlation_id},
)
await session.commit()
handler = ExtractHandler(session_factory=sess_factory)
await handler.handle(
DocumentUploaded(
correlation_id=correlation_id,
document_id=document_id,
user_id=user_id,
s3_key=s3_key,
filename="contract.pdf",
mime="application/pdf",
)
)
extracted_key_path = f"users/{user_id}/docs/{document_id}.txt"
text_data = await store.get(extracted_key_path)
assert "Договор" in text_data.decode("utf-8")
connection = await aio_pika.connect_robust(infra["rabbitmq_url"])
try:
channel = await connection.channel()
queue = await channel.get_queue("analyze.q", ensure=False)
deadline = 10.0
found = False
while deadline > 0:
start = time.monotonic()
message = await queue.get(timeout=deadline)
await message.ack()
body = json.loads(message.body.decode("utf-8"))
if body["document_id"] == str(document_id):
assert body["correlation_id"] == str(correlation_id)
assert body["extracted_s3_key"] == extracted_key_path
assert body["char_count"] > 0
assert body["ocr_used"] is False
found = True
break
deadline -= time.monotonic() - start
assert found, "expected DocumentExtracted message not found in analyze.q"
finally:
await connection.close()
async with sess_factory() as session:
res = await session.execute(
text("SELECT status, stage, extracted_s3_key FROM documents WHERE id = :d"),
{"d": document_id},
)
row = res.first()
assert row is not None
status, stage, ext_key = row
assert status == "analyzing"
assert stage == "queued_analyze"
assert ext_key == extracted_key_path
async def test_extract_worker_docx_uploads_text(
infra: dict[str, str],
docx_bytes: bytes,
) -> None:
from sqlalchemy import text
telegram_id = 444_555_666
sess_factory = session_factory(infra)
store = storage(infra)
async with sess_factory() as session:
result = await session.execute(
text(
"INSERT INTO users (telegram_id, credits_left) VALUES (:t, 5) "
"ON CONFLICT (telegram_id) DO UPDATE SET credits_left = 5 "
"RETURNING id"
),
{"t": telegram_id},
)
user_id = result.scalar_one()
await session.commit()
document_id = uuid.uuid4()
correlation_id = uuid.uuid4()
s3_key = original_key(str(user_id), str(document_id), ".docx")
await store.put(
s3_key,
docx_bytes,
content_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
)
async with sess_factory() as session:
await session.execute(
text(
"INSERT INTO documents (id, user_id, s3_key, filename, mime, bytes, status) "
"VALUES (:id, :uid, :s3, :fn, :mime, :bytes, 'queued')"
),
{
"id": document_id,
"uid": user_id,
"s3": s3_key,
"fn": "contract.docx",
"mime": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"bytes": len(docx_bytes),
},
)
await session.execute(
text(
"INSERT INTO jobs (document_id, correlation_id, queue, status) "
"VALUES (:did, :cid, 'extract', 'pending')"
),
{"did": document_id, "cid": correlation_id},
)
await session.commit()
handler = ExtractHandler(session_factory=sess_factory)
await handler.handle(
DocumentUploaded(
correlation_id=correlation_id,
document_id=document_id,
user_id=user_id,
s3_key=s3_key,
filename="contract.docx",
mime="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
)
)
extracted_key_path = f"users/{user_id}/docs/{document_id}.txt"
text_data = await store.get(extracted_key_path)
assert "Стороны" in text_data.decode("utf-8")
async with sess_factory() as session:
row = await session.execute(
text("SELECT status FROM documents WHERE id = :d"),
{"d": document_id},
)
assert row.scalar_one() == "analyzing"
async def test_extract_worker_terminal_failure_refunds_and_dlqs(
infra: dict[str, str],
) -> None:
from sqlalchemy import text
telegram_id = 777_888_999
sess_factory = session_factory(infra)
store = storage(infra)
async with sess_factory() as session:
result = await session.execute(
text(
"INSERT INTO users (telegram_id, credits_left) VALUES (:t, 3) "
"ON CONFLICT (telegram_id) DO UPDATE SET credits_left = 3 "
"RETURNING id, credits_left"
),
{"t": telegram_id},
)
row = result.first()
assert row is not None
user_id, initial_credits = row
await session.commit()
document_id = uuid.uuid4()
correlation_id = uuid.uuid4()
s3_key = original_key(str(user_id), str(document_id), ".pdf")
await store.put(s3_key, b"not a pdf", content_type="application/pdf")
async with sess_factory() as session:
await session.execute(
text(
"INSERT INTO documents (id, user_id, s3_key, filename, mime, bytes, status) "
"VALUES (:id, :uid, :s3, :fn, :mime, :bytes, 'queued')"
),
{
"id": document_id,
"uid": user_id,
"s3": s3_key,
"fn": "bad.pdf",
"mime": "application/pdf",
"bytes": 9,
},
)
await session.execute(
text(
"INSERT INTO jobs (document_id, correlation_id, queue, status, max_attempts) "
"VALUES (:did, :cid, 'extract', 'pending', 2)"
),
{"did": document_id, "cid": correlation_id},
)
await session.commit()
handler = ExtractHandler(session_factory=sess_factory)
from contract_check.core.analysis.extractor import ExtractionError
from contract_check.core.analysis.ocr import OCRError
# The handler may raise ExtractionError or OCRError depending on local
# tesseract data; either way the terminal-failure path refunds the credit.
with pytest.raises((ExtractionError, OCRError)):
await handler.handle(
DocumentUploaded(
correlation_id=correlation_id,
document_id=document_id,
user_id=user_id,
s3_key=s3_key,
filename="bad.pdf",
mime="application/pdf",
)
)
await handler.on_terminal_failure(
DocumentUploaded(
correlation_id=correlation_id,
document_id=document_id,
user_id=user_id,
s3_key=s3_key,
filename="bad.pdf",
mime="application/pdf",
),
"extraction_failed",
"test terminal failure",
)
async with sess_factory() as session:
res = await session.execute(
text("SELECT status, refunded FROM documents WHERE id = :d"),
{"d": document_id},
)
row = res.first()
assert row is not None
status, refunded = row
assert status == "failed"
assert refunded is True
credits = await session.execute(
text("SELECT credits_left FROM users WHERE id = :u"),
{"u": user_id},
)
assert credits.scalar_one() == initial_credits + 1

130
tests/integration/test_upload_pipeline.py Normal file
View file

@ -0,0 +1,130 @@
"""Upload pipeline integration test: POST /api/v1/documents → MinIO + RabbitMQ + credits.
Verifies the api:
- accepts a multipart PDF upload with a service token and telegram_id,
- reserves a credit,
- stores the blob in MinIO under the expected key,
- publishes a `DocumentUploaded` message to the `extract.q` queue,
- returns 202 with document_id/correlation_id/credits_left.
"""
from __future__ import annotations
import json
import uuid
from pathlib import Path
from typing import TYPE_CHECKING
import aio_pika
import httpx
import pytest
from tests.integration.conftest import user_token
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession
pytestmark = pytest.mark.integration
@pytest.fixture
def pdf_bytes(tmp_path: Path) -> bytes:
# The real test requires pymupdf to create a minimal PDF.
import pymupdf
doc = pymupdf.open()
page = doc.new_page()
long_text = (
"Договор. Стороны обязуются выполнять условия. "
"Сторона А обязуется передать товар. "
"Сторона Б обязуется оплатить товар в течение десяти банковских дней. "
"Ответственность сторон ограничена суммой договора. "
"Споры подлежат рассмотрению в арбитражном суде города Москвы."
)
page.insert_htmlbox(
page.rect,
f'<p style="font-family:DejaVu Sans;font-size:14px">{long_text}</p>',
)
path = tmp_path / "contract.pdf"
doc.save(str(path))
doc.close()
return path.read_bytes()
async def test_upload_document_reserves_credit_and_enqueues(
client: httpx.AsyncClient,
infra: dict[str, str],
pdf_bytes: bytes,
db_session: AsyncSession,
) -> None:
from sqlalchemy import text
telegram_id = 123456789
await db_session.execute(
text(
"INSERT INTO users (telegram_id, credits_left) VALUES (:t, 5) "
"ON CONFLICT (telegram_id) DO UPDATE SET credits_left = 5"
),
{"t": telegram_id},
)
await db_session.commit()
token = await user_token(client, infra, telegram_id)
response = await client.post( # type: ignore[misc]
"/api/v1/documents",
headers={"Authorization": f"Bearer {token}"},
files={"file": ("contract.pdf", pdf_bytes, "application/pdf")},
)
assert response.status_code == 202
body = response.json()
document_id = uuid.UUID(body["document_id"])
correlation_id = uuid.UUID(body["correlation_id"])
assert body["credits_left"] == 4 # 5 - 1
# Verify a DocumentUploaded message landed on extract.q.
# When worker-extract is running it consumes quickly; in that case we
# verify the pipeline completes by checking analyze.q instead.
import time
connection = await aio_pika.connect_robust(infra["rabbitmq_url"])
try:
channel = await connection.channel()
deadline = 10.0
found = False
try:
queue = await channel.get_queue("extract.q", ensure=False)
while deadline > 0:
start = time.monotonic()
message = await queue.get(timeout=deadline)
await message.ack()
msg_body = json.loads(message.body.decode("utf-8"))
if msg_body["document_id"] == str(document_id):
assert msg_body["correlation_id"] == str(correlation_id)
assert msg_body["user_id"] # any valid UUID
assert msg_body["s3_key"].endswith(".pdf")
assert msg_body["mime"] == "application/pdf"
found = True
break
deadline -= time.monotonic() - start
except aio_pika.exceptions.QueueEmpty:
# Worker already consumed it; assert end-to-end completion.
analyze_queue = await channel.get_queue("analyze.q", ensure=False)
# Use a consumer iterator so we don't depend on quorum-queue
# basic_get behaviour returning stale/empty results under concurrency.
async with analyze_queue.iterator() as queue_iter:
async for message in queue_iter:
async with message.process():
msg_body = json.loads(message.body.decode("utf-8"))
if msg_body["document_id"] == str(document_id):
assert msg_body["correlation_id"] == str(correlation_id)
assert msg_body["extracted_s3_key"].endswith(".txt")
assert msg_body["char_count"] > 0
found = True
break
if found:
break
if time.monotonic() - start > deadline:
break
assert found, "expected pipeline message not found on extract.q or analyze.q"
finally:
await connection.close()

198
tests/unit/test_auth.py Normal file
View file

@ -0,0 +1,198 @@
"""Unit tests for core.auth: JWT signing/verification and Telegram identity checks.
These tests do not touch the database; they exercise only the crypto helpers.
"""
from __future__ import annotations
import datetime as dt
import hashlib
import hmac
import uuid
from urllib.parse import urlencode
import pytest
from contract_check.core.auth import (
AuthError,
TokenExpiredError,
TokenInvalidError,
create_access_token,
verify_access_token,
verify_bot_identity,
verify_telegram_miniapp_init_data,
verify_telegram_web_payload,
)
from contract_check.core.config import get_settings
_BOT_TOKEN = "123456789:ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghi"
_SECRET_KEY = hmac.new(
_BOT_TOKEN.encode("utf-8"),
b"WebAppData",
hashlib.sha256,
).digest()
def _make_web_payload(telegram_id: int, auth_date: int | None = None) -> dict[str, object]:
if auth_date is None:
auth_date = int(dt.datetime.now(tz=dt.UTC).timestamp())
data = {
"id": telegram_id,
"first_name": "Test",
"auth_date": auth_date,
}
data_check_string = "\n".join(f"{k}={v}" for k, v in sorted(data.items()))
data["hash"] = hmac.new(
_SECRET_KEY,
data_check_string.encode("utf-8"),
hashlib.sha256,
).hexdigest()
return data
def _make_init_data(telegram_id: int, auth_date: int | None = None) -> str:
if auth_date is None:
auth_date = int(dt.datetime.now(tz=dt.UTC).timestamp())
user_json = f'{{"id":{telegram_id},"first_name":"Test"}}'
params = {"user": user_json, "auth_date": str(auth_date), "chat_type": "private"}
data_check_string = "\n".join(f"{k}={v}" for k, v in sorted(params.items()))
params["hash"] = hmac.new(
_SECRET_KEY,
data_check_string.encode("utf-8"),
hashlib.sha256,
).hexdigest()
return urlencode(params)
@pytest.fixture(autouse=True)
def _clear_settings_cache(monkeypatch: pytest.MonkeyPatch) -> None:
get_settings.cache_clear()
monkeypatch.setenv("JWT_SECRET", "unit-test-secret-do-not-use-in-prod")
monkeypatch.setenv("JWT_ALGORITHM", "HS256")
monkeypatch.setenv("JWT_ACCESS_TTL_MINUTES", "1440")
get_settings.cache_clear()
yield
get_settings.cache_clear()
def test_create_and_verify_access_token() -> None:
user_id = uuid.uuid4()
telegram_id = 42
token = create_access_token(user_id, telegram_id)
claims = verify_access_token(token)
assert claims.sub == user_id
assert claims.telegram_id == telegram_id
assert claims.type == "access"
def test_verify_token_rejects_tampered_signature() -> None:
token = create_access_token(uuid.uuid4(), 42)
# Flip a bit in the middle of the payload; this breaks the signature reliably.
tampered = token[:-10] + ("A" if token[-10] != "A" else "B") + token[-9:]
with pytest.raises(TokenInvalidError):
verify_access_token(tampered)
def test_verify_token_rejects_expired_token(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("JWT_ACCESS_TTL_MINUTES", "-1")
get_settings.cache_clear()
token = create_access_token(uuid.uuid4(), 42)
with pytest.raises(TokenExpiredError):
verify_access_token(token)
def test_verify_token_rejects_wrong_audience() -> None:
import jwt
user_id = uuid.uuid4()
token = jwt.encode(
{
"sub": str(user_id),
"telegram_id": 42,
"type": "access",
"exp": int((dt.datetime.now(tz=dt.UTC) + dt.timedelta(hours=1)).timestamp()),
"iat": int(dt.datetime.now(tz=dt.UTC).timestamp()),
"aud": "wrong-audience",
},
key=get_settings().jwt_secret,
algorithm="HS256",
)
with pytest.raises(TokenInvalidError):
verify_access_token(token)
def test_verify_token_rejects_missing_telegram_id() -> None:
import jwt
user_id = uuid.uuid4()
token = jwt.encode(
{
"sub": str(user_id),
"type": "access",
"exp": int((dt.datetime.now(tz=dt.UTC) + dt.timedelta(hours=1)).timestamp()),
"iat": int(dt.datetime.now(tz=dt.UTC).timestamp()),
"aud": "contract-check",
},
key=get_settings().jwt_secret,
algorithm="HS256",
)
with pytest.raises(TokenInvalidError):
verify_access_token(token)
def test_verify_telegram_web_payload_valid() -> None:
payload = _make_web_payload(12345)
identity = verify_telegram_web_payload(payload, _BOT_TOKEN)
assert identity.telegram_id == 12345
def test_verify_telegram_web_payload_bad_hash() -> None:
payload = _make_web_payload(12345)
payload["hash"] = "0" * 64
with pytest.raises(AuthError, match="signature mismatch"):
verify_telegram_web_payload(payload, _BOT_TOKEN)
def test_verify_telegram_web_payload_expired() -> None:
old_auth_date = int(dt.datetime.now(tz=dt.UTC).timestamp()) - 25 * 60 * 60
payload = _make_web_payload(12345, auth_date=old_auth_date)
with pytest.raises(AuthError, match="expired"):
verify_telegram_web_payload(payload, _BOT_TOKEN)
def test_verify_telegram_web_payload_missing_bot_token() -> None:
with pytest.raises(AuthError, match="not configured"):
verify_telegram_web_payload(_make_web_payload(1), "")
def test_verify_telegram_miniapp_init_data_valid() -> None:
init_data = _make_init_data(67890)
identity = verify_telegram_miniapp_init_data(init_data, _BOT_TOKEN)
assert identity.telegram_id == 67890
def test_verify_telegram_miniapp_init_data_bad_hash() -> None:
init_data = _make_init_data(67890)
init_data = init_data[:-64] + "0" * 64
with pytest.raises(AuthError, match="signature mismatch"):
verify_telegram_miniapp_init_data(init_data, _BOT_TOKEN)
def test_verify_telegram_miniapp_init_data_missing_user() -> None:
# Sign an empty data-check-string; hash is valid, but there is no user param.
empty_hash = hmac.new(_SECRET_KEY, b"", hashlib.sha256).hexdigest()
bad = f"hash={empty_hash}"
with pytest.raises(AuthError, match="missing user"):
verify_telegram_miniapp_init_data(bad, _BOT_TOKEN)
def test_verify_bot_identity_valid() -> None:
identity = verify_bot_identity(111222)
assert identity.telegram_id == 111222
def test_verify_bot_identity_invalid() -> None:
with pytest.raises(AuthError):
verify_bot_identity(-1)
with pytest.raises(AuthError):
verify_bot_identity(0)

Some files were not shown because too many files have changed in this diff Show more