admin-git/DealDocumentScreening
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Prescreen stage was added. Docs synced. Some refactoring of worker

Browse source 
analyzer. Lint fix.
This commit is contained in:
febux 2026-08-17 20:49:29 +03:00
parent 3828b46985
commit c3d7f03038
77 changed files with 7192 additions and 184 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
.env.example
View file

@ -53,6 +53,19 @@ OLLAMA_TEMPERATURE=0.2
OLLAMA_NUM_PREDICT=3072
OLLAMA_TIMEOUT=120
OLLAMA_MAX_CONCURRENCY=3
# --- YandexGPT (alternative provider) ---
# Set LLM_PROVIDER=yandex_gpt to use. Get API key & folder ID from Yandex Cloud.
YANDEXGPT_API_KEY=replace-me
YANDEXGPT_FOLDER_ID=replace-me
YANDEXGPT_MODEL=yandexgpt-lite # yandexgpt / yandexgpt-lite / yandexgpt-preview etc.
YANDEXGPT_FALLBACK_MODEL= # empty disables fallback
YANDEXGPT_BASE_URL=https://llm.api.cloud.yandex.net
YANDEXGPT_COMPLETION_PATH=/foundationModels/v1/completion
YANDEXGPT_TEMPERATURE=0.2
YANDEXGPT_MAX_TOKENS=3072
YANDEXGPT_TIMEOUT=120
YANDEXGPT_MAX_CONCURRENCY=3
CHUNK_SIZE_CHARS=10000
# --- API (FastAPI) ---
@ -82,6 +95,16 @@ PASSWORD_RESET_TTL_MINUTES=60
PASSWORD_MIN_LENGTH=8
WEB_APP_BASE_URL=http://localhost:5173 # SPA base — used to build reset links
# --- Prescreen stage (hybrid heuristic + optional LLM fallback) ---
PRESCREEN_ENABLED=true
PRESCREEN_AUTO_APPROVE=false
PRESCREEN_CONFIDENCE_THRESHOLD=0.75
PRESCREEN_HIGH_VALUE_THRESHOLD=100000
PRESCREEN_LLM_FALLBACK_ENABLED=false # kill-switch; set true after burn-in
PRESCREEN_LLM_FALLBACK_THRESHOLD=0.75 # <= confidence threshold
PRESCREEN_LLM_MAX_CHARS=20000 # cost-control cap for fallback
PRESCREEN_KEEP_REGEX=false # temporary rollback to regex-v1
# --- SMTP (notification transport; empty host → dev logger) ---
SMTP_HOST=
SMTP_PORT=587

4
.github/workflows/ci.yml vendored
View file

@ -35,8 +35,8 @@ jobs:
- name: Ruff format check
run: uv run ruff format --check src tests
- name: Mypy
run: uv run mypy src
- name: Ty type check
run: uv run ty check src
test-unit:
name: Unit tests

13
.pre-commit-config.yaml
View file

@ -18,6 +18,13 @@ repos:
- repo: local
hooks:
- id: isort
name: isort
entry: uv run isort
language: system
types: [python]
require_serial: true
- id: ruff-check
name: ruff check (autofix)
entry: uv run ruff check --fix --exit-non-zero-on-fix
@ -32,9 +39,9 @@ repos:
types: [python]
require_serial: true
- id: mypy
name: mypy (src)
entry: uv run mypy src
- id: ty
name: ty (src)
entry: uv run ty check src
language: system
types: [python]
pass_filenames: false

19
Makefile
View file

@ -1,7 +1,7 @@
# «Контракт-чек» — everyday commands (uv + docker)
# Usage: make <target> (see `make help`)
.PHONY: help install lint typecheck test test-unit test-integration migrate \
.PHONY: help install lint lint-fix isort isort-check typecheck test test-unit test-integration migrate \
infra-up infra-down infra-logs services-up services-down services-logs \
api api-logs bot bot-logs worker-extract worker-analyze worker-notify \
seed-token jwt-secret jwt-token jwt-verify health shell-api shell-bot \
@ -20,12 +20,23 @@ help: ## Show available commands
install: ## Sync dev dependencies (uv)
uv sync --group dev
lint: ## Run ruff linter + import sorter
lint: ## Run ruff linter + format check
uv run ruff check src tests
uv run ruff format --check src tests
typecheck: ## Run mypy type checker
uv run mypy src
lint-fix: ## Run ruff autofix + format + isort fix
uv run ruff check --fix src tests
uv run ruff format src tests
uv run isort src tests
isort: ## Run isort (sort imports in-place)
uv run isort src tests
isort-check: ## Run isort in check-only mode
uv run isort --check-only src tests
typecheck: ## Run ty type checker
uv run ty check src
test: ## Run all tests (unit + integration)
uv run pytest

81
docker-compose.yml
View file

@ -162,6 +162,16 @@ services:
OLLAMA_NUM_PREDICT: ${OLLAMA_NUM_PREDICT:-3072}
OLLAMA_TIMEOUT: ${OLLAMA_TIMEOUT:-120}
OLLAMA_MAX_CONCURRENCY: ${OLLAMA_MAX_CONCURRENCY:-3}
YANDEXGPT_API_KEY: ${YANDEXGPT_API_KEY:-}
YANDEXGPT_FOLDER_ID: ${YANDEXGPT_FOLDER_ID:-}
YANDEXGPT_MODEL: ${YANDEXGPT_MODEL:-yandexgpt-lite}
YANDEXGPT_FALLBACK_MODEL: ${YANDEXGPT_FALLBACK_MODEL:-}
YANDEXGPT_BASE_URL: ${YANDEXGPT_BASE_URL:-https://llm.api.cloud.yandex.net}
YANDEXGPT_COMPLETION_PATH: ${YANDEXGPT_COMPLETION_PATH:-/foundationModels/v1/completion}
YANDEXGPT_TEMPERATURE: ${YANDEXGPT_TEMPERATURE:-0.2}
YANDEXGPT_MAX_TOKENS: ${YANDEXGPT_MAX_TOKENS:-3072}
YANDEXGPT_TIMEOUT: ${YANDEXGPT_TIMEOUT:-120}
YANDEXGPT_MAX_CONCURRENCY: ${YANDEXGPT_MAX_CONCURRENCY:-3}
CHUNK_SIZE_CHARS: ${CHUNK_SIZE_CHARS:-10000}
API_HOST: ${API_HOST:-0.0.0.0}
API_PORT: ${API_PORT:-8000}
@ -277,6 +287,16 @@ services:
OLLAMA_NUM_PREDICT: ${OLLAMA_NUM_PREDICT:-3072}
OLLAMA_TIMEOUT: ${OLLAMA_TIMEOUT:-120}
OLLAMA_MAX_CONCURRENCY: ${OLLAMA_MAX_CONCURRENCY:-3}
YANDEXGPT_API_KEY: ${YANDEXGPT_API_KEY:-}
YANDEXGPT_FOLDER_ID: ${YANDEXGPT_FOLDER_ID:-}
YANDEXGPT_MODEL: ${YANDEXGPT_MODEL:-yandexgpt-lite}
YANDEXGPT_FALLBACK_MODEL: ${YANDEXGPT_FALLBACK_MODEL:-}
YANDEXGPT_BASE_URL: ${YANDEXGPT_BASE_URL:-https://llm.api.cloud.yandex.net}
YANDEXGPT_COMPLETION_PATH: ${YANDEXGPT_COMPLETION_PATH:-/foundationModels/v1/completion}
YANDEXGPT_TEMPERATURE: ${YANDEXGPT_TEMPERATURE:-0.2}
YANDEXGPT_MAX_TOKENS: ${YANDEXGPT_MAX_TOKENS:-3072}
YANDEXGPT_TIMEOUT: ${YANDEXGPT_TIMEOUT:-120}
YANDEXGPT_MAX_CONCURRENCY: ${YANDEXGPT_MAX_CONCURRENCY:-3}
CHUNK_SIZE_CHARS: ${CHUNK_SIZE_CHARS:-10000}
TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:-}
JWT_SECRET: ${JWT_SECRET}
@ -285,6 +305,67 @@ services:
ports:
- "9102:9102"
worker-prescreen:
profiles: ["services"]
build:
context: .
dockerfile: srv/worker-prescreen/Dockerfile
container_name: contract_check-worker-prescreen
restart: unless-stopped
depends_on:
postgres:
condition: service_healthy
rabbitmq:
condition: service_healthy
minio-init:
condition: service_completed_successfully
environment:
ENV: ${ENV:-dev}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-contract_check}:${POSTGRES_PASSWORD:-contract_check}@postgres:5432/${POSTGRES_DB:-contract_check}
RABBITMQ_URL: amqp://${RABBITMQ_USER:-contract_check}:${RABBITMQ_PASS:-contract_check}@rabbitmq:5672/${RABBITMQ_VHOST:-/}
S3_ENDPOINT_URL: http://minio:9000
S3_ACCESS_KEY: ${S3_ACCESS_KEY:-contract_check}
S3_SECRET_KEY: ${S3_SECRET_KEY:-contract_check}
S3_BUCKET: ${S3_BUCKET:-contract-check-docs}
S3_REGION: ${S3_REGION:-us-east-1}
REFUND_POLICY: ${REFUND_POLICY:-all}
SENTRY_DSN: ${SENTRY_DSN:-}
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-}
OTEL_SERVICE_NAME: worker-prescreen
MQ_PREFETCH_PRESCREEN: ${MQ_PREFETCH_PRESCREEN:-1}
MQ_MAX_ATTEMPTS: ${MQ_MAX_ATTEMPTS:-5}
MQ_RETRY_BASE_MS: ${MQ_RETRY_BASE_MS:-2000}
PRESCREEN_ENABLED: ${PRESCREEN_ENABLED:-true}
PRESCREEN_AUTO_APPROVE: ${PRESCREEN_AUTO_APPROVE:-false}
PRESCREEN_CONFIDENCE_THRESHOLD: ${PRESCREEN_CONFIDENCE_THRESHOLD:-0.75}
PRESCREEN_HIGH_VALUE_THRESHOLD: ${PRESCREEN_HIGH_VALUE_THRESHOLD:-100000}
LLM_PROVIDER: ${LLM_PROVIDER:-ollama_cloud}
OLLAMA_HOST: ${OLLAMA_HOST:-}
OLLAMA_API_KEY: ${OLLAMA_API_KEY:-}
OLLAMA_MODEL: ${OLLAMA_MODEL:-qwen2.5:14b}
OLLAMA_FALLBACK_MODEL: ${OLLAMA_FALLBACK_MODEL:-qwen2.5:7b}
OLLAMA_TEMPERATURE: ${OLLAMA_TEMPERATURE:-0.2}
OLLAMA_NUM_PREDICT: ${OLLAMA_NUM_PREDICT:-3072}
OLLAMA_TIMEOUT: ${OLLAMA_TIMEOUT:-120}
OLLAMA_MAX_CONCURRENCY: ${OLLAMA_MAX_CONCURRENCY:-3}
YANDEXGPT_API_KEY: ${YANDEXGPT_API_KEY:-}
YANDEXGPT_FOLDER_ID: ${YANDEXGPT_FOLDER_ID:-}
YANDEXGPT_MODEL: ${YANDEXGPT_MODEL:-yandexgpt-lite}
YANDEXGPT_FALLBACK_MODEL: ${YANDEXGPT_FALLBACK_MODEL:-}
YANDEXGPT_BASE_URL: ${YANDEXGPT_BASE_URL:-https://llm.api.cloud.yandex.net}
YANDEXGPT_COMPLETION_PATH: ${YANDEXGPT_COMPLETION_PATH:-/foundationModels/v1/completion}
YANDEXGPT_TEMPERATURE: ${YANDEXGPT_TEMPERATURE:-0.2}
YANDEXGPT_MAX_TOKENS: ${YANDEXGPT_MAX_TOKENS:-3072}
YANDEXGPT_TIMEOUT: ${YANDEXGPT_TIMEOUT:-120}
YANDEXGPT_MAX_CONCURRENCY: ${YANDEXGPT_MAX_CONCURRENCY:-3}
CHUNK_SIZE_CHARS: ${CHUNK_SIZE_CHARS:-10000}
JWT_SECRET: ${JWT_SECRET}
JWT_ALGORITHM: ${JWT_ALGORITHM:-HS256}
JWT_ACCESS_TTL_MINUTES: ${JWT_ACCESS_TTL_MINUTES:-1440}
ports:
- "9104:9104"
worker-notify:
profiles: ["services"]
build:

331
docs/ARCHITECTURE.md
View file

@ -9,13 +9,15 @@
>
> **One-line summary:** take the stage-0 prototype (PDF/DOCX → text → Ollama
> Cloud → markdown report) and rebuild it as a production, event-driven,
> multi-container application: a FastAPI **api**, two RabbitMQ-consuming
> **workers** (extract/OCR and LLM-analyze, split for CPU vs I/O profiles),
> a Telegram **bot** adapter, backed by **Postgres + RabbitMQ + MinIO + Redis**,
> observable via **structlog + Sentry + Prometheus/Grafana + OpenTelemetry**,
> behind **Nginx + certbot**. Monorepo, five fine-tuned Docker images
> (`srv/` — api + two workers + bot + prototype), shared `contract_check.core` domain package. Compose now, k8s-ready later.
> HA-*ready* (quorum queues, WAL-archiving Postgres), not HA-*running* on one
> multi-container application: a FastAPI **api**, three RabbitMQ-consuming
> **workers** (extract/OCR, deterministic prescreen, and LLM-analyze, split for
> CPU vs I/O profiles), a Telegram **bot** adapter, backed by
> **Postgres + RabbitMQ + MinIO + Redis**, observable via
> **structlog + Sentry + Prometheus/Grafana + OpenTelemetry**, behind
> **Nginx + certbot**. Monorepo, six fine-tuned Docker images
> (`srv/` — api + three workers + bot + prototype), shared `contract_check.core`
> domain package. Compose now, k8s-ready later.
> HA-*ready* (quorum queues, WAL archiving Postgres), not HA-*running* on one
> VPS — see §10.
---
@ -39,9 +41,11 @@ The owner has explicitly pivoted away from all three:
Additional decisions locked during the architecture questionnaire:
- **Two worker pools**, not one: `worker-extract` (pymupdf + Tesseract, CPU)
and `worker-analyze` (LLM, I/O). This pairs naturally with the pipeline
topology and lets each pool scale independently.
- **Three worker pools**, not one: `worker-extract` (pymupdf + Tesseract, CPU),
`worker-prescreen` (hybrid heuristic → optional LLM-fallback metadata extraction, CPU but
tiny), and `worker-analyze` (LLM, I/O). The prescreen stage routes low-risk
contracts to manual/auto-approval and high-risk contracts to deep LLM analysis,
protecting the expensive LLM worker from trivial documents.
- **LLM behind a provider interface** (`core/llm/port.py`), Ollama Cloud as
the first realization — swappable later for self-hosted Ollama / GigaChat.
- **Refund policy is a runtime switch** (`REFUND_POLICY=all|infra_only`) with
@ -73,10 +77,14 @@ HTTP clients to the api and touch nothing but the api.
│ owns: Postgres, MinIO, ▼
│ Redis, credits, tokens extract.q ──► worker-extract
│ (core) (quorum) (core, CPU: pymupdf+tesseract)
▼ │ publish document.extracted
▼ │ publish PrescreenRequested
Postgres ▼
MinIO ◄──── read/write ────► analyze.q ──► worker-analyze
Redis (rate limit/sess) (quorum) (core, I/O: LLM provider)
MinIO ◄──── read/write ────► prescreen.q ──► worker-prescreen
Redis (rate limit/sess) (quorum) (hybrid heuristic router)
│ publish AnalyzeRequested
▼
analyze.q ──► worker-analyze
(quorum) (core, I/O: LLM provider)
│ save Report, status=done
▼
Postgres
@ -103,11 +111,11 @@ Control plane:
| Area | Decision | Replaces |
|---|---|---|
| Repo | Monorepo, shared `contract_check.core`, 5 Dockerfiles in `srv/<svc>/` | one-image `MODE` dispatch |
| Scope now | api + worker-extract + worker-analyze + bot (+ prototype benchmark) | (web/payments later) |
| Repo | Monorepo, shared `contract_check.core`, 6 Dockerfiles in `srv/<svc>/` | one-image `MODE` dispatch |
| Scope now | api + worker-extract + worker-prescreen + worker-analyze + bot (+ prototype benchmark) | (web/payments later) |
| Queue | RabbitMQ, direct exchange, pipeline fan-out | arq + Redis |
| Retry | TTL retry queues, exponential backoff, final DLQ | — |
| Workers | Two pools (extract/OCR + analyze/LLM) | single arq worker |
| Workers | Three pools (extract/OCR + prescreen + analyze/LLM) | single arq worker |
| Storage | MinIO, `users/{uid}/docs/{did}.{ext}` | Selectel S3 |
| Upload | Proxy through API (multipart) | — |
| Doc retention | TTL purge of raw docs after N days | — |
@ -115,7 +123,7 @@ Control plane:
| Auth | Per-adapter `service_tokens`, revocable | single `SERVICE_TOKEN` |
| Report delivery | Polling now (fine-grained stage), SSE/webhook later | — |
| Sync `/analyze` | No | — |
| Doc status | Fine-grained `queued→extracting→ocr→analyzing→done\|failed` | coarse status |
| Doc status | Fine-grained `queued→extracting→prescreening→ocr→analyzing→done\|failed` | coarse status |
| Extra tables | jobs, service_tokens, invoices(stub) | 3-table plan |
| Report storage | JSONB + markdown column | — |
| Durability | HA-*ready* (quorum queues, WAL archive) | — |
@ -199,14 +207,19 @@ DealDocumentScreening/
│ │ │ ├── port.py (LLMProvider Protocol + AnalysisResult dataclass)
│ │ │ ├── ollama_cloud.py (ports prototype llm_client: repair-loop + 429 fallback)
│ │ │ └── factory.py (provider selection by LLM_PROVIDER env)
│ │ ├── analysis/
│ │ ├── extraction/ (new — DocumentExtractor port + adapters)
│ │ │ ├── port.py (ExtractedDocument, DocumentExtractor Protocol, errors)
│ │ │ ├── factory.py (mime/suffix → adapter; SUPPORTED_SUFFIXES gate)
│ │ │ ├── adapters/ (pdf_pymupdf, docx_mammoth, rtf_striprtf, txt_chardet, ocr_tesseract)
│ │ │ └── __init__.py (extract_document orchestrator with OCR fallback)
│ │ ├── analysis/ (existing; now consumed by prototype + ocr adapter)
│ │ │ ├── __init__.py
│ │ │ ├── extractor.py (← from contract_check/extractor.py)
│ │ │ ├── chunker.py (← from contract_check/chunker.py)
│ │ │ ├── extractor.py (← from contract_check/extractor.py; prototype use)
│ │ │ ├── chunker.py (← from contract_check/chunker.py; + chunk_markdown)
│ │ │ ├── checklist.py (← from contract_check/checklist.py)
│ │ │ ├── report_schema.py (← from contract_check/report_schema.py)
│ │ │ ├── ocr.py (new — pytesseract via pymupdf rasterize, <100 chars trigger)
│ │ │ └── analyzer.py (new — orchestrate chunk→LLM→merge→dedupe→sort→markdown)
│ │ │ ├── ocr.py (pytesseract via pymupdf rasterize)
│ │ │ └── analyzer.py (orchestrate chunk→LLM→merge→dedupe→sort→markdown)
│ │ ├── credits.py (reserve_credit atomic UPDATE; refund_credit idempotent + policy)
│ │ └── tokens.py (service-token hash/verify; FastAPI dependency)
│ ├── api/ (new — FastAPI image)
@ -226,13 +239,23 @@ DealDocumentScreening/
│ ├── worker_extract/ (new — CPU image: pymupdf + tesseract)
│ │ ├── __init__.py
│ │ ├── __main__.py (entrypoint: start consumer + metrics server)
│ │ ├── handler.py (handle DocumentUploaded: MinIO dl→extract→(ocr)→upload text→publish)
│ │ ├── extract_document.py (extraction/OCR orchestration helper)
│ │ ├── handler.py (handle DocumentUploaded: MinIO dl → extraction factory → OCR fallback → upload .txt (markdown) → publish PrescreenRequested)
│ │ └── consumer.py (wire handler into core.mq.consumer base)
│ ├── worker_prescreen/ (new — lean CPU image: hybrid heuristic + optional LLM fallback)
│ │ ├── __init__.py
│ │ ├── __main__.py (entrypoint: metrics server on :9104)
│ │ ├── config.py (PrescreenSettings, prefetch tunable)
│ │ ├── handler.py (handle PrescreenRequested: download text → hybrid extract → route → persist prescreen_results → publish AnalyzeRequested/auto-approve/manual-review)
│ │ ├── consumer.py (wire handler into core.mq.consumer base)
│ │ ├── extractor.py (PrescreenContractMeta + MetadataExtractor protocol + compat shim)
│ │ ├── extractor_heuristic.py (Stage 1: deterministic keyword/positional extractor, regex-free)
│ │ ├── extractor_llm.py (Stage 2: dict → validated PrescreenContractMeta)
│ │ ├── extractor_hybrid.py (orchestrator: heuristic → optional LLM fallback)
│ │ └── router.py (decision logic: auto_approve / manual_review / deep_analysis)
│ ├── worker_analyze/ (new — I/O image: LLM provider)
│ │ ├── __init__.py
│ │ ├── __main__.py
│ │ ├── handler.py (handle DocumentExtracted: text dl→chunk→LLM→validate→save report→done)
│ │ ├── handler.py (handle AnalyzeRequested: text dl→chunk→LLM (with prescreen context)→validate→save report→done)
│ │ └── consumer.py
│ ├── bot/ (new — aiogram adapter, HTTP-only)
│ │ ├── __init__.py
@ -247,7 +270,11 @@ DealDocumentScreening/
├── conftest.py (fixtures: testcontainers pg/rabbit/minio)
├── unit/
│ ├── test_chunker.py
│ ├── test_extraction_adapters.py
│ ├── test_extraction_factory.py
│ ├── test_extractor.py
│ ├── test_prescreen_extractor.py
│ ├── test_prescreen_router.py
│ ├── test_checklist_report.py
│ ├── test_llm_ollama_cloud.py (respx: 200 / 429-fallback / repair)
│ ├── test_credits.py
@ -257,9 +284,9 @@ DealDocumentScreening/
│ └── test_bot_boundary.py (static AST: bot must not import core.db/s3/llm/mq/credits)
└── integration/
├── conftest.py (seed service-token + api-key; testcontainers wiring)
├── test_upload_pipeline.py (POST /documents → message on extract.q)
├── test_extract_worker.py (DocumentUploaded → DocumentExtracted published)
├── test_analyze_worker.py (DocumentExtracted → Report saved, status done)
├── test_upload_pipeline.py (POST /documents → message on extract.q or prescreen.q)
├── test_extract_worker.py (DocumentUploaded → PrescreenRequested published)
├── test_analyze_worker.py (AnalyzeRequested → Report saved, status done)
├── test_credits_db.py (reserve/refund against real Postgres)
└── test_b2b_api.py (X-API-Key: 202 → poll report; 401/429 branches)
```
@ -267,7 +294,7 @@ DealDocumentScreening/
**Import rule (hexagonal boundary, enforced in review):**
- `core/*` may import anything.
- `api/*`, `worker_extract/*`, `worker_analyze/*` import only `core/*`.
- `api/*`, `worker_extract/*`, `worker_prescreen/*`, `worker_analyze/*` import only `core/*`.
- `bot/*` imports only `httpx` + its own modules. **It must NOT import
`core.db`, `core.s3`, `core.llm`, `core.mq`, `core.credits`.** It speaks to
the api over HTTP only. (Add a ruff/flake8 import-forbidden rule or a unit
@ -286,18 +313,21 @@ service start).
| Name | Type | Purpose |
|---|---|---|
| `contracts.x` | direct | Main exchange. Routing keys: `extract`, `analyze`. |
| `contracts.retry.x` | direct | DLX target of main queues; routing keys: `retry.extract`, `retry.analyze`. |
| `contracts.x` | direct | Main exchange. Routing keys: `extract`, `prescreen`, `analyze`. |
| `contracts.retry.x` | direct | DLX target of main queues; routing keys: `retry.extract`, `retry.prescreen`, `retry.analyze`. |
### Queues
| Name | Type | Args | Bound (exchange / rk) | Consumers |
|---|---|---|---|---|
| `extract.q` | **quorum** | `x-dead-letter-exchange=contracts.retry.x`, `x-dead-letter-routing-key=retry.extract` | contracts.x / `extract` | worker-extract |
| `prescreen.q` | **quorum** | `x-dead-letter-exchange=contracts.retry.x`, `x-dead-letter-routing-key=retry.prescreen` | contracts.x / `prescreen` | worker-prescreen |
| `analyze.q` | **quorum** | `x-dead-letter-exchange=contracts.retry.x`, `x-dead-letter-routing-key=retry.analyze` | contracts.x / `analyze` | worker-analyze |
| `extract.retry.q` | classic | `x-dead-letter-exchange=contracts.x`, `x-dead-letter-routing-key=extract`; per-message `expiration` | contracts.retry.x / `retry.extract` | none (delay slot) |
| `prescreen.retry.q` | classic | `x-dead-letter-exchange=contracts.x`, `x-dead-letter-routing-key=prescreen`; per-message `expiration` | contracts.retry.x / `retry.prescreen` | none (delay slot) |
| `analyze.retry.q` | classic | `x-dead-letter-exchange=contracts.x`, `x-dead-letter-routing-key=analyze`; per-message `expiration` | contracts.retry.x / `retry.analyze` | none (delay slot) |
| `extract.dlq` | **quorum** | — | — | none (manual requeue) |
| `prescreen.dlq` | **quorum** | — | — | none (manual requeue) |
| `analyze.dlq` | **quorum** | — | — | none (manual requeue) |
### Flow & retry mechanics
@ -306,13 +336,26 @@ service start).
(persistent, `correlation_id` UUID in both message id and
`headers["x-correlation-id"]`).
2. **worker-extract** consumes from `extract.q` with `prefetch=1..N` (config,
default 1 — extraction is CPU-bound). On success: upload extracted text
to MinIO, publish `DocumentExtracted` to `contracts.x` rk `analyze`,
ack. On failure: see retry below.
3. **worker-analyze** consumes from `analyze.q` with `prefetch=3` (mirrors
default 1 — extraction is CPU-bound). On success: upload extracted **Markdown**
to MinIO (`text/markdown`), publish `PrescreenRequested` to `contracts.x` rk
`prescreen` (`is_structured`/`has_tables` metadata included), ack. On failure:
see retry below.
3. **worker-prescreen** consumes from `prescreen.q` with `prefetch=1`. It runs a
**hybrid two-stage extractor**: Stage 1 is a deterministic regex-free heuristic
over keyword dictionaries and positional windows; Stage 2 is an LLM fallback
(`extract_prescreen` on `core/llm/port.py`) triggered only when heuristic
confidence is below `PRESCREEN_LLM_FALLBACK_THRESHOLD` and the kill-switch
`PRESCREEN_LLM_FALLBACK_ENABLED` is true. It computes a weighted
`confidence_score`, routes to `auto_approve` (lightweight report, disabled by
default), `manual_review` (terminal DB state for future admin SPA), or
`deep_analysis`. For `deep_analysis` it publishes `AnalyzeRequested` to
`contracts.x` rk `analyze` carrying `prescreen_meta`. On failure: retry.
The v1 regex implementation is kept behind `PRESCREEN_KEEP_REGEX=true` for one
release as a rollback safety net.
4. **worker-analyze** consumes from `analyze.q` with `prefetch=3` (mirrors
Ollama Pro concurrency). On success: save report, `status=done`, ack.
On failure: retry.
4. **Retry (nack):** the base consumer does **not** use `basic.nack(requeue=True)`
5. **Retry (nack):** the base consumer does **not** use `basic.nack(requeue=True)`
(instant re-redelivery, no backoff). Instead it publishes a copy of the
message to the matching `*.retry.q` with:
- `headers["x-attempt"] = attempt + 1`
@ -320,20 +363,22 @@ service start).
`BASE_DELAY_MS=2000` → 2s, 4s, 8s, 16s, 32s)
- then `basic_ack` the original.
When the retry queue's per-message TTL expires, its DLX bounces the
message back to `contracts.x` with rk `extract`/`analyze` → re-enters the
message back to `contracts.x` with rk `extract`/`prescreen`/`analyze` → re-enters the
main quorum queue. Clean, plugin-free, exponential.
5. **Poison (max attempts):** when `headers["x-attempt"] >= MAX_ATTEMPTS`
(default 5), the consumer publishes to `extract.dlq` / `analyze.dlq`
instead of retry, acks the original, sets `jobs.dlq=true`,
6. **Poison (max attempts):** when `headers["x-attempt"] >= MAX_ATTEMPTS`
(default 5), the consumer publishes to `extract.dlq` / `prescreen.dlq` /
`analyze.dlq` instead of retry, acks the original, sets `jobs.dlq=true`,
`jobs.last_failure_class`, transitions `documents.status=failed`, and
calls `refund_credit(document_id, failure_class)` per policy (§8).
6. **Manual requeue:** a CLI script `scripts/mq_requeue.py` (or mgmt UI)
7. **Manual requeue:** a CLI script `scripts/mq_requeue.py` (or mgmt UI)
moves a DLQ message back to the main exchange with reset attempt.
### Prefetch & concurrency
- worker-extract: RabbitMQ prefetch caps concurrent extraction jobs (CPU
bound; default 1, tune to CPU count).
- worker-prescreen: prefetch=1; Stage 1 heuristic is cheap but routing decisions must be
ordered per document.
- worker-analyze: prefetch caps concurrent LLM jobs (default 3, mirrors
Ollama Pro). **Additionally** keep the in-process `asyncio.Semaphore`
from the prototype `llm_client` — prefetch caps *jobs*, the semaphore
@ -368,13 +413,67 @@ class DocumentExtracted(BaseModel):
extracted_s3_key: str # users/{uid}/docs/{did}.txt
char_count: int
ocr_used: bool
is_structured: bool = False
has_tables: bool = False
attempt: int = 0
class PrescreenRequested(BaseModel):
correlation_id: UUID
document_id: UUID
user_id: UUID
text_s3_key: str # users/{uid}/docs/{did}.txt
filename: str
char_count: int = 0
ocr_used: bool = False
is_structured: bool = False
has_tables: bool = False
attempt: int = 0
class PrescreenCompleted(BaseModel):
correlation_id: UUID
document_id: UUID
user_id: UUID
text_s3_key: str
filename: str
prescreened_at: str
contract_type: str | None = None
party_a: str | None = None
party_b: str | None = None
total_amount: float | None = None
currency: str | None = None
start_date: str | None = None
end_date: str | None = None
has_penalty_clause: bool | None = None
has_termination_clause: bool | None = None
has_arbitration: bool | None = None
confidence_score: float | None = None
routing_decision: str # auto_approve | manual_review | deep_analysis
auto_summary: str | None = None
auto_findings: list[dict] = []
extractor_version: str = "heuristic-v2" # heuristic-v2 | hybrid-llm-v1
attempt: int = 0
class AnalyzeRequested(BaseModel):
correlation_id: UUID
document_id: UUID
user_id: UUID
extracted_s3_key: str
filename: str
char_count: int = 0
ocr_used: bool = False
is_structured: bool = False
has_tables: bool = False
prescreen_meta: PrescreenCompleted | None = None
attempt: int = 0
```
RabbitMQ `headers`: `x-correlation-id`, `x-attempt`, `x-origin` (api |
worker-extract). `content_type=application/json`, `delivery_mode=2`
(persistent). Validate on consume with the pydantic model; on validation
error → `.dlq` immediately with `failure_class=infra`.
worker-extract | worker-prescreen). `content_type=application/json`,
`delivery_mode=2` (persistent). Validate on consume with the pydantic model;
on validation error → `.dlq` immediately with `failure_class=infra`.
---
@ -434,6 +533,8 @@ Migrations (hand-written, async `env.py`):
- `0002` `api_keys` + `api_key_requests` (B2B)
- `0003` webUI auth columns on `users` (`email`, `password_hash`, reset tokens, `is_active`)
- `0004` `users.role` for the admin panel (`'user'|'admin'`, default `'user'`)
- `0005` Telegram bot hardening columns on `users`
- `0006` `prescreen_results` table; expand `documents.status` and `jobs.queue` enums
```sql
-- users
@ -464,7 +565,7 @@ CREATE TABLE documents (
mime TEXT NOT NULL,
bytes BIGINT NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'queued'
CHECK (status IN ('queued','extracting','ocr','analyzing','done','failed')),
CHECK (status IN ('queued','extracting','prescreening','ocr','analyzing','done','failed')),
stage TEXT, -- sub-stage / human label
refunded BOOLEAN NOT NULL DEFAULT FALSE,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
@ -483,15 +584,48 @@ CREATE TABLE reports (
prompt_tokens INTEGER NOT NULL DEFAULT 0,
eval_tokens INTEGER NOT NULL DEFAULT 0,
latency_ms INTEGER NOT NULL DEFAULT 0,
prescreen_result_id UUID REFERENCES prescreen_results(id) ON DELETE SET NULL,
prescreen_meta JSONB,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
-- jobs (RabbitMQ correlation; one document has up to 2 jobs: extract + analyze)
-- prescreen_results (added by 0006)
CREATE TABLE prescreen_results (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
correlation_id UUID NOT NULL,
contract_type TEXT,
party_a TEXT,
party_b TEXT,
total_amount NUMERIC(18, 2),
currency TEXT,
start_date DATE,
end_date DATE,
has_penalty_clause BOOLEAN,
has_termination_clause BOOLEAN,
has_arbitration BOOLEAN,
confidence_score NUMERIC(4, 3) CHECK (confidence_score >= 0 AND confidence_score <= 1),
routing_decision TEXT NOT NULL DEFAULT 'manual_review'
CHECK (routing_decision IN ('auto_approve','manual_review','deep_analysis')),
prescreened_at TIMESTAMPTZ NOT NULL DEFAULT now(),
processing_ms INTEGER,
extractor_version TEXT NOT NULL DEFAULT 'heuristic-v2'
CHECK (extractor_version IN ('regex-v1','heuristic-v2','hybrid-llm-v1')),
auto_summary TEXT,
auto_findings JSONB NOT NULL DEFAULT '[]',
error_message TEXT,
retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0)
);
CREATE INDEX prescreen_results_document_idx ON prescreen_results (document_id);
CREATE INDEX prescreen_results_routing_idx ON prescreen_results (routing_decision);
CREATE INDEX prescreen_results_confidence_idx ON prescreen_results (confidence_score);
-- jobs (RabbitMQ correlation; one document has up to 3 jobs: extract + prescreen + analyze)
CREATE TABLE jobs (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
correlation_id UUID NOT NULL,
queue TEXT NOT NULL CHECK (queue IN ('extract','analyze')),
queue TEXT NOT NULL CHECK (queue IN ('extract','prescreen','analyze')),
attempts INTEGER NOT NULL DEFAULT 0,
max_attempts INTEGER NOT NULL DEFAULT 5,
last_failure_class TEXT,
@ -610,13 +744,17 @@ Set on the `jobs.last_failure_class` column. `REFUND_POLICY` env selects
---
## 9. LLM provider port + Ollama Cloud adapter
## 9. LLM provider port + adapters
The prototype's `llm_client.py` (bearer httpx, `format: json-schema`,
5xx backoff, 429→fallback, repair-loop) is excellent. It becomes the first
adapter behind a port so we can later swap in self-hosted Ollama,
GigaChat, or YandexGPT without touching `analyzer.py`.
Registered providers are selected by `LLM_PROVIDER`:
- `ollama_cloud` (default)
- `yandex_gpt`
### Port (`core/llm/port.py`)
```python
@ -636,7 +774,9 @@ class AnalysisResult:
class LLMProvider(Protocol):
async def analyze(self, text: str, *, checklist: str) -> AnalysisResult: ...
async def analyze(
self, text: str, *, checklist: str, extra_context: str = ""
) -> AnalysisResult: ...
async def aclose(self) -> None: ...
```
@ -646,13 +786,15 @@ Move the existing `OllamaCloudClient` + `ChatResult` + `_QuotaError` +
`_run_with_fallback` + `_run_repair_loop` + `_post_chat` verbatim, then add
an `analyze(text, *, checklist)` method that:
1. chunks the text via `core.analysis.chunker.chunk_text`,
2. builds the system/user prompts (from `prototype.SYSTEM_PROMPT` +
1. chunks the text via `core.analysis.chunker.chunk_markdown` (heading-aware when
the extractor produced structured Markdown; falls back to plain chunking),
2. appends `extra_context` (prescreen metadata for `AnalyzeRequested`) to each user prompt,
3. builds the system/user prompts (from `prototype.SYSTEM_PROMPT` +
`build_user_prompt`, lifted into `core/analysis/analyzer.py`),
3. fans out chunk requests under the existing `asyncio.Semaphore`,
4. merges/dedupes/sorts findings (lift `dedupe_findings` + `sort_findings`
4. fans out chunk requests under the existing `asyncio.Semaphore`,
5. merges/dedupes/sorts findings (lift `dedupe_findings` + `sort_findings`
into `core/analysis/analyzer.py`),
5. returns `AnalysisResult`.
6. returns `AnalysisResult`.
Map adapter-internal failures to `FailureClass`:
@ -668,11 +810,13 @@ def build_llm_provider(settings) -> LLMProvider:
match settings.llm_provider:
case "ollama_cloud":
return OllamaCloudProvider(settings)
case "yandex_gpt":
return YandexGPTProvider(settings)
case _:
raise ValueError(f"unknown LLM_PROVIDER={settings.llm_provider!r}")
```
`LLM_PROVIDER` env (default `ollama_cloud`). Future providers register here.
`LLM_PROVIDER` env (default `ollama_cloud`). Additional providers register here.
---
@ -686,11 +830,11 @@ change.
### RabbitMQ — quorum queues
- `extract.q`, `analyze.q`, `extract.dlq`, `analyze.dlq` are `x-queue-type:
quorum`. Quorum queues persist every message to disk and use Raft. On a
single node they are durable; the moment a 3-node RabbitMQ cluster is
added (compose: `rabbitmq@rmq1/2/3`), the same queues replicate with no
application code change.
- `extract.q`, `prescreen.q`, `analyze.q`, `extract.dlq`, `prescreen.dlq`,
`analyze.dlq` are `x-queue-type: quorum`. Quorum queues persist every message
to disk and use Raft. On a single node they are durable; the moment a 3-node
RabbitMQ cluster is added (compose: `rabbitmq@rmq1/2/3`), the same queues
replicate with no application code change.
- Retry queues are `classic` (transient delay slots; safe to lose on
catastrophic failure — the originating message is already acked and
tracked in `jobs`).
@ -747,6 +891,7 @@ None of 1–5 requires touching `core/` application code — only compose/infra.
| `RABBITMQ_URL` | (required) | `amqp://guest:guest@rabbitmq:5672//` |
| `MQ_PREFETCH_EXTRACT` | `1` | CPU-bound |
| `MQ_PREFETCH_ANALYZE` | `3` | mirrors Ollama Pro concurrency |
| `MQ_PREFETCH_PRESCREEN` | `1` | deterministic heuristic is tiny but keep ordered |
| `MQ_MAX_ATTEMPTS` | `5` | before DLQ |
| `MQ_RETRY_BASE_MS` | `2000` | exponential base |
| `S3_ENDPOINT_URL` | (required) | MinIO URL |
@ -765,7 +910,7 @@ None of 1–5 requires touching `core/` application code — only compose/infra.
| Env | Default | Notes |
|---|---|---|
| `LLM_PROVIDER` | `ollama_cloud` | factory selection |
| `LLM_PROVIDER` | `ollama_cloud` | factory selection: `ollama_cloud`, `yandex_gpt` |
| `OLLAMA_HOST` | (required if provider=ollama_cloud) | `https://ollama.com` for Ollama Cloud; `http://localhost:11434` for self-hosted |
| `OLLAMA_API_KEY` | (required) | bearer |
| `OLLAMA_MODEL` | `qwen2.5:14b` | primary; must be a model available at the configured host |
@ -775,6 +920,29 @@ None of 1–5 requires touching `core/` application code — only compose/infra.
| `OLLAMA_TIMEOUT` | `120` | seconds |
| `OLLAMA_MAX_CONCURRENCY` | `3` | in-process semaphore (per analyze worker) |
| `CHUNK_SIZE_CHARS` | `10000` | chunker |
| `YANDEXGPT_API_KEY` | (required if provider=yandex_gpt) | API key for Yandex Cloud Foundation Models |
| `YANDEXGPT_FOLDER_ID` | (required if provider=yandex_gpt) | Yandex Cloud folder ID (`gpt://<folder_id>/<model>`) |
| `YANDEXGPT_MODEL` | `yandexgpt-lite` | primary YandexGPT model URI name |
| `YANDEXGPT_FALLBACK_MODEL` | (empty) | optional fallback model name |
| `YANDEXGPT_BASE_URL` | `https://llm.api.cloud.yandex.net` | completion API base URL |
| `YANDEXGPT_COMPLETION_PATH` | `/foundationModels/v1/completion` | completion API path |
| `YANDEXGPT_TEMPERATURE` | `0.2` | |
| `YANDEXGPT_MAX_TOKENS` | `3072` | max completion tokens |
| `YANDEXGPT_TIMEOUT` | `120` | seconds |
| `YANDEXGPT_MAX_CONCURRENCY` | `3` | in-process semaphore (per analyze worker) |
### Prescreen
| Env | Default | Notes |
|---|---|---|
| `PRESCREEN_ENABLED` | `true` | Run the prescreen stage between extraction and analysis. |
| `PRESCREEN_AUTO_APPROVE` | `false` | Allow auto-approval of low-risk contracts (disabled until accuracy proven). |
| `PRESCREEN_CONFIDENCE_THRESHOLD` | `0.75` | Minimum weighted field-coverage score to route away from `manual_review`. |
| `PRESCREEN_HIGH_VALUE_THRESHOLD` | `100000` | Amount (in contract currency) above which contract is always sent to deep analysis. |
| `PRESCREEN_LLM_FALLBACK_ENABLED` | `false` | Kill-switch for the Stage-2 LLM fallback in the hybrid extractor. Flip after burn-in. |
| `PRESCREEN_LLM_FALLBACK_THRESHOLD` | `0.75` | Heuristic confidence below this triggers the LLM fallback (keep ≤ confidence threshold). |
| `PRESCREEN_LLM_MAX_CHARS` | `20000` | Cap on characters sent to the LLM fallback (cost control). |
| `PRESCREEN_KEEP_REGEX` | `false` | Temporary: run the legacy regex-v1 extractor via `extract_contract_meta` shim. |
### API
@ -852,15 +1020,16 @@ Per-service differences (`uv` uses PEP 735 dependency-groups — see `pyproject.
| Dockerfile | Extra installed | Runtime apt | CMD | Expose |
|---|---|---|---|---|
| `srv/api/Dockerfile` | `--group api` | none | `python -m contract_check.api` | 8000, 9100 |
| `srv/worker-extract/Dockerfile` | `--group extract` | tesseract-ocr, -rus, -eng | `python -m contract_check.worker_extract` | 9101 |
| `srv/worker-extract/Dockerfile` | `--group extract` | tesseract-ocr, -rus, -eng, libmagic1 | `python -m contract_check.worker_extract` | 9101 |
| `srv/worker-prescreen/Dockerfile` | `--group prescreen` | none | `python -m contract_check.worker_prescreen` | 9104 |
| `srv/worker-analyze/Dockerfile` | `--group analyze` | none | `python -m contract_check.worker_analyze` | 9102 |
| `srv/bot/Dockerfile` | `--group bot` | none | `python -m contract_check.bot` | — |
| `srv/prototype/Dockerfile` | `--group prototype` | none | `python -m contract_check.prototype` | — |
The bot image is the leanest (no DB driver, no S3 client, no pymupdf). The
analyze image has httpx but no tesseract/pymupdf. The extract image is the
heaviest (tesseract + language packs). This is the "fine-tuned deps per
service" payoff.
heaviest (tesseract + language packs + libmagic1). This is the "fine-tuned deps
per service" payoff.
### pyproject.toml dependency-groups (actual — PEP 735)
@ -894,15 +1063,19 @@ api = [{ include-group = "db" }, { include-group = "mq" },
"opentelemetry-instrumentation-asgi>=0.45b0"]
extract = [{ include-group = "db" }, { include-group = "mq" },
{ include-group = "s3" }, { include-group = "obs" },
"pymupdf>=1.24", "pytesseract>=0.3.10", "pillow>=10"]
"pymupdf>=1.24", "pytesseract>=0.3.10", "pillow>=10",
"mammoth>=1.8", "striprtf>=0.0.26", "chardet>=5.2",
"python-magic>=0.4.27"]
analyze = [{ include-group = "db" }, { include-group = "mq" },
{ include-group = "s3" }, { include-group = "obs" },
"opentelemetry-instrumentation-httpx>=0.45b0"]
prescreen = [{ include-group = "db" }, { include-group = "mq" },
{ include-group = "s3" }, { include-group = "obs" }]
bot = ["aiogram>=3.4"]
prototype = ["pymupdf>=1.24", "python-docx>=1.1"]
dev = [{ include-group = "api" }, { include-group = "extract" },
{ include-group = "analyze" }, { include-group = "bot" },
{ include-group = "prototype" },
{ include-group = "prescreen" }, { include-group = "analyze" },
{ include-group = "bot" }, { include-group = "prototype" },
"pytest>=8", "pytest-asyncio>=0.23", "respx>=0.21", "ruff>=0.5",
"mypy>=1.10", "anyio>=4", "aiosqlite>=0.20",
"testcontainers[rabbitmq,postgres,minio]>=4", "asgi-lifespan>=2.1.0"]
@ -927,6 +1100,7 @@ services:
api: build srv/api/Dockerfile; depends_on pg/rabbit/minio-init healthy;
healthcheck /healthz; ports 8000,9100
worker-extract: build srv/worker-extract/Dockerfile; depends_on pg/rabbit/minio-init healthy; 9101
worker-prescreen: build srv/worker-prescreen/Dockerfile; depends_on pg/rabbit/minio-init healthy; 9104
worker-analyze: build srv/worker-analyze/Dockerfile; depends_on pg/rabbit/minio-init healthy; 9102
bot: build srv/bot/Dockerfile; depends_on api healthy (NOT pg/rabbit)
@ -973,11 +1147,16 @@ enforcing the adapter boundary even in dependency ordering.
- api: `http_requests_total`, `http_request_duration_seconds`,
`documents_uploaded_total`, `credits_reserved_total`, `mq_publish_total`.
- worker-extract: `extract_jobs_total`, `extract_duration_seconds`,
`ocr_used_total`, `extract_failures_total{class=...}`, expose on `:9101`.
- worker-analyze: `analyze_jobs_total`, `analyze_duration_seconds`,
- worker-extract: `extract_duration_seconds{format=...}`,
`extraction_total{format,structured}`, expose on `:9101`.
- worker-prescreen: `prescreen_duration_seconds{decision=...}`,
`prescreen_runs_total{decision,contract_type}`, `prescreen_confidence`,
`prescreen_fallback_runs_total{outcome=used|failed|skipped|disabled}`,
`prescreen_extraction_stage_seconds{stage=heuristic|llm}`,
expose on `:9104`.
- worker-analyze: `analyze_duration_seconds`,
`llm_tokens_total{kind=prompt|eval}`, `llm_fell_back_total`,
`refund_total{policy=...}`, expose on `:9102`.
expose on `:9102`.
- Workers run `prometheus_client.start_http_server(port)` in a background
thread alongside the async consumer.
@ -1000,7 +1179,9 @@ enforcing the adapter boundary even in dependency ordering.
### Unit (`tests/unit/`, fast, no I/O)
- `test_chunker.py`, `test_extractor.py` — port existing, parametrized.
- `test_chunker.py` — `chunk_text` + `chunk_markdown`.
- `test_extraction_adapters.py`, `test_extraction_factory.py` — adapters and factory.
- `test_extractor.py` — legacy prototype extractor path remains valid.
- `test_llm_ollama_cloud.py` — **respx** mocks: 200 happy, 429→fallback,
invalid-JSON→repair→success, invalid-JSON→repair→fail, timeout.
- `test_credits_policy.py` — `reserve_credit` never negative; `refund_credit`
@ -1020,9 +1201,9 @@ enforcing the adapter boundary even in dependency ordering.
credit reserved, blob in MinIO, `DocumentUploaded` lands on `extract.q`
(assert via `aio-pika` consumer).
- `test_extract_worker.py` — publish `DocumentUploaded`, run
worker-extract handler in-process → `DocumentExtracted` published,
extracted text in MinIO, `documents.status` advanced.
- `test_analyze_worker.py` — publish `DocumentExtracted` with respx-mocked
worker-extract handler in-process → `PrescreenRequested` published,
extracted **Markdown** in MinIO, `documents.status=advanced to prescreening`.
- `test_analyze_worker.py` — publish `AnalyzeRequested` with respx-mocked
Ollama, run worker-analyze handler → report row in Postgres with
`status=done`, markdown contains disclaimer.
- `test_retry_and_dlq.py` — force Ollama 500 repeatedly → message cycles

116
docs/PHASE1_HANDOFF.md Normal file
View file

@ -0,0 +1,116 @@
# Phase 1 Handoff — Extraction Layer Refactor
> Scope: implement the structured-extraction port from `document-extraction-spec.md`,
> preserving the hexagonal architecture and keeping the existing pipeline intact.
> Completed: 2026-08-16.
## Goal
Replace flat-text extraction (`core.analysis.extractor.extract_text`) with a
format-aware extraction **port** that returns Markdown with preserved structure
(headings from DOCX, tables from PDF). The rest of the pipeline (chunker,
worker-analyze, RabbitMQ contracts) remains unchanged except for Markdown-aware
chunking and additive metadata in `DocumentExtracted`.
## New / modified files
### New: `core/extraction/` hexagonal package
| File | Purpose |
|---|---|
| `src/contract_check/core/extraction/port.py` | `DocumentExtractor` Protocol, `ExtractedDocument` DTO, `UnsupportedFormatError`, `ExtractionFailedError`, `MIN_TEXT_CHARS` |
| `src/contract_check/core/extraction/factory.py` | `ExtractorFactory`, `detect_format()`, `SUPPORTED_SUFFIXES`; routes bytes/mime/suffix → adapter. Uses `python-magic` for content sniffing, suffix as fallback. |
| `src/contract_check/core/extraction/adapters/pdf_pymupdf.py` | `PyMuPDFExtractor` — text + `page.find_tables()` → Markdown pipes. Interleaves tables and text blocks by bbox so cell text is not duplicated. `is_structured=True` iff tables found. |
| `src/contract_check/core/extraction/adapters/docx_mammoth.py` | `MammothDocxExtractor` — `mammoth.convert_to_markdown()`. Headings/lists/tables become Markdown; `is_structured=True` on headings/tables/lists. |
| `src/contract_check/core/extraction/adapters/rtf_striprtf.py` | `RtfExtractor` — `striprtf` to plaintext. `is_structured=False`. |
| `src/contract_check/core/extraction/adapters/txt_chardet.py` | `TxtExtractor` — `chardet` encoding detection, decode, metadata includes `encoding`. |
| `src/contract_check/core/extraction/adapters/ocr_tesseract.py` | `TesseractOcrExtractor` — OCR for scanned PDFs and raster images (PNG/JPG/TIFF), reusing the same pytesseract/pymupdf logic as `core.analysis.ocr`. |
| `src/contract_check/core/extraction/__init__.py` | `extract_document()` orchestrator: adapter → extract → OCR fallback for short PDFs. |
| `src/contract_check/core/extraction/adapters/__init__.py` | Re-exports of all adapter classes. |
### Modified worker / pipeline
| File | Change |
|---|---|
| `src/contract_check/worker_extract/handler.py` | Bytes-based extraction via `core.extraction.extract_document()`. No temp files. Uploaded text now `text/markdown; charset=utf-8`. `DocumentExtracted` includes `is_structured`/`has_tables`. Duration histogram labeled by `format`. Classification updated for new errors. |
| `src/contract_check/worker_extract/consumer.py` | Removed `@extract_duration.time()` decorator — labeled timing moved into the handler where the format is known. |
| `src/contract_check/core/mq/messages.py` | `DocumentExtracted` added `is_structured: bool = False` and `has_tables: bool = False` (backward-compatible defaults). |
| `src/contract_check/core/metrics.py` | `extract_duration` now has `format` label. Added `extraction_total{format,structured}` counter. |
| `src/contract_check/core/analysis/chunker.py` | Added `chunk_markdown()` — splits on ATX headings, keeps headings attached to their body, falls back to `chunk_text()` for unstructured text. |
| `src/contract_check/core/llm/ollama_cloud.py` | Analyzer now uses `chunk_markdown(text)` instead of `chunk_text(text)` so heading boundaries are preserved when the extractor produced structured Markdown. |
| `src/contract_check/api/services.py` | Upload gate now imports `SUPPORTED_SUFFIXES` from `core.extraction`. `_content_type_from_suffix()` extended for RTF/TXT/CSV/images. |
| `src/contract_check/bot/handlers.py` | Bot extension gate now uses the same `SUPPORTED_SUFFIXES` set and content-type map. |
| `src/contract_check/worker_extract/extract_document.py` | **Deleted**. Superseded by `core.extraction.extract_document()`. |
### Dependencies / Docker
| File | Change |
|---|---|
| `pyproject.toml` | `extract` group adds `mammoth`, `striprtf`, `chardet`, `python-magic`. Mypy ignore list extended for these libs. |
| `srv/worker-extract/Dockerfile` | Adds `libmagic1` apt package (required by `python-magic`). |
| `uv.lock` | Regenerated. |
### Documentation
| File | Change |
|---|---|
| `docs/ARCHITECTURE.md` | Layout updated (`core/extraction/` added, `worker_extract/extract_document.py` removed, new tests listed). Pipeline notes that extracted Markdown is uploaded. `DocumentExtracted` schema updated. Metrics table updated. pyproject `extract` group updated. worker-extract Dockerfile apt row updated. |
| `src/contract_check/api/routes/README.md` | Upload endpoint extension list updated. |
### Tests
| File | Change |
|---|---|
| `tests/unit/test_extraction_adapters.py` | New: PDF text-only/table, DOCX heading, RTF, TXT chardet, OCR error paths, DTO defaults. |
| `tests/unit/test_extraction_factory.py` | New: suffix/mime/magic routing, unsupported formats, end-to-end `extract_document()`, OCR fallback behavior. |
| `tests/unit/test_chunker.py` | New: `chunk_markdown` heading preservation, packing, oversized-section split, preamble handling, fallback. |
| `tests/integration/test_extract_worker.py` | Expected exceptions updated: `ExtractionFailedError` is now a possible raised error alongside `OCRError` (the failure-class mapping remains `extraction_failed` / `ocr_failed`). |
## Behavior changes
- **Format support** (upload-gate accepted): `.pdf`, `.docx`, `.rtf`, `.txt`, `.csv`, `.png`, `.jpg`, `.jpeg`, `.tif`, `.tiff`.
- **Output format**: extracted text is now Markdown; stored with content-type `text/markdown; charset=utf-8`.
- **Backward compatibility**: `DocumentExtracted` new fields have defaults, so in-flight messages during deploy are valid.
- **Refund policy unchanged**: `ExtractionFailedError` → `extraction_failed` (not refundable under `infra_only`); `OCRError` → `ocr_failed` (refundable).
- **Chunking**: `worker-analyze` uses heading-aware `chunk_markdown()` when structure exists; otherwise identical to previous `chunk_text()`.
## Verification
```bash
make lint # ruff check + format check — passed
make typecheck # mypy src — passed (92 files)
make test-unit # 132 passed, 32 deselected integration tests
```
Integration tests (`tests/integration/test_extract_worker.py` and
`tests/integration/test_upload_pipeline.py`) passed with the live Docker stack.
The full `make test-integration` suite is slow because it exercises the real LLM
path; the extraction-specific integration subset is green.
## Deployment notes
1. The `worker-extract` Docker image must be rebuilt because:
- new apt package `libmagic1`
- new Python deps `mammoth`, `striprtf`, `chardet`, `python-magic`
2. Existing running workers should be recreated (`docker compose --profile services up -d --build worker-extract`).
3. No DB migration required — changes are additive to message schema and object storage content.
## Rollback
- The new code is additive; old messages with fewer `DocumentExtracted` fields
still validate. If rollback is needed, the previous worker-extract image
continues to work, but extracted objects would be stored as `text/plain`
instead of `text/markdown`.
## Deviations from the spec
- `ExtractedDocument.attachments: list[bytes]` was omitted — there is no consumer
or storage design for extracted images yet.
- Heavy adapters (`marker`, `paddleocr`, `easyocr`, `img2table`, `email`) remain
deferred per the spec.
## Next step
Phase 2: prescreen stage between `worker-extract` and `worker-analyze`. Based
on `SPIKE_PHASE0.md`, this will use a deterministic regex+pydantic extractor for
RU/BY contracts instead of the Needle 2 model (English-only in testing).

153
docs/PHASE2_HANDOFF.md Normal file
View file

@ -0,0 +1,153 @@
# Phase 2 Handoff — Prescreen Stage
> Scope: insert a deterministic prescreen stage between extraction and LLM
> analysis for Russian/Belarusian contracts.
> Completed: 2026-08-16.
## Goal
Short-circuit obvious low-risk contracts and protect the expensive LLM worker
from trivial documents. Use a deterministic regex+pydantic extractor because the
Needle 2 (cactus-needle) model is English-only and unusable for RU/BY contracts
(see `docs/SPIKE_PHASE0.md`).
## What changed
### DB schema (`migrations/versions/0006_prescreen.py`)
- New `prescreen_results` table with contract metadata, field-coverage score,
routing decision, and lightweight auto-approve output.
- `documents.status` adds `'prescreening'`.
- `jobs.queue` adds `'prescreen'`.
- `reports` gains `prescreen_result_id` FK and `prescreen_meta` JSONB.
### Models (`src/contract_check/core/db/`)
- `models.py`: added `PrescreenResult` model and relationships.
- `enums.py`: `DocStatus` and `QueueName` literals updated.
### RabbitMQ topology (`core/mq/topology.py`)
- Added `prescreen.q`, `prescreen.retry.q`, `prescreen.dlq`.
- Added routing keys `prescreen` / `retry.prescreen`.
### Messages (`core/mq/messages.py`)
- `PrescreenRequested` — worker-extract → prescreen.q.
- `PrescreenCompleted` — prescreen result, persisted and forwarded.
- `AnalyzeRequested` — worker-prescreen → analyze.q, carries `prescreen_meta`.
- `DocumentExtracted` kept for backward compatibility but no longer published.
### New service: `worker_prescreen/`
| File | Purpose |
|---|---|
| `extractor.py` | Regex extractor for RU/BY contracts; returns `PrescreenContractMeta` + confidence score. |
| `router.py` | Routing decision: `auto_approve` / `manual_review` / `deep_analysis`. `auto_approve` is disabled by default. |
| `handler.py` | Download Markdown, run extractor, persist `prescreen_results`, publish next message. |
| `consumer.py` | RabbitMQ consumer for `prescreen.q`. |
| `config.py` | `PrescreenSettings` with prefetch tunable. |
| `__main__.py` | Entrypoint, metrics server on `:9104`. |
### Wired existing services
- `worker_extract/handler.py`: now publishes `PrescreenRequested` to `prescreen.q`
and sets `documents.status = 'prescreening'`.
- `worker_analyze/consumer.py` + `handler.py`: now consumes `AnalyzeRequested`,
receives `prescreen_meta`, appends it to the LLM prompt as known contract
fields.
- `core/llm/port.py` + `ollama_cloud.py`: `analyze()` accepts `extra_context: str`
and injects it into each chunk's user prompt.
### Configuration (`core/config.py`)
New envs:
- `PRESCREEN_ENABLED` (default `true`)
- `PRESCREEN_AUTO_APPROVE` (default `false`)
- `PRESCREEN_CONFIDENCE_THRESHOLD` (default `0.75`)
- `PRESCREEN_HIGH_VALUE_THRESHOLD` (default `100000`)
### Metrics (`core/metrics.py`)
- `prescreen_duration_seconds{decision}`
- `prescreen_runs_total{decision,contract_type}`
- `prescreen_confidence` histogram
### Docker / compose
- New `srv/worker-prescreen/Dockerfile` (lean, no tesseract).
- Added `worker-prescreen` service to `docker-compose.yml` on port `9104`.
### Dependencies (`pyproject.toml`)
- Added `prescreen` dependency group (core + db/mq/s3/obs).
- Added to `dev` group.
### Tests
- Unit: `tests/unit/test_prescreen_extractor.py`, `tests/unit/test_prescreen_router.py`.
- Integration: updated `test_extract_worker.py`, `test_analyze_worker.py`,
`test_upload_pipeline.py`, `test_b2b_api.py` for the new pipeline.
### Docs
- Updated `docs/ARCHITECTURE.md` with prescreen topology, messages, schema,
metrics, and Docker/compose rows.
## Behavior
1. Upload → `extract.q` (unchanged).
2. worker-extract uploads Markdown to MinIO, publishes `PrescreenRequested`.
3. worker-prescreen:
- extracts contract type, parties, amount, currency, dates, penalty/
termination/arbitration clauses via regex
- `confidence_score = matched_fields / total_fields`
- routes:
- `deep_analysis` if high value, penalty clause, or arbitration
- `manual_review` if low confidence, missing parties, or auto_approve disabled
- `auto_approve` only when `PRESCREEN_AUTO_APPROVE=true` and low risk
4. `deep_analysis` → `AnalyzeRequested` → worker-analyze with prescreen context.
5. `manual_review` → terminal DB state, no LLM call, credit **not** refunded
(this is an intentional routing outcome, not a failure).
6. `auto_approve` → lightweight report row, `status=done`.
## Verification
```bash
make lint # passed
make typecheck # passed (99 files)
make test-unit # 147 passed, 32 deselected
make test-integration # 32 passed, 147 deselected (with live workers stopped to avoid races)
```
## Deployment notes
1. Run migration: `make migrate` (or `docker compose --profile services run --rm api alembic upgrade head`).
2. Rebuild images: `docker compose --profile services up -d --build`.
3. New worker-prescreen must be started; worker-extract and worker-analyze
images also changed.
4. No application code outside core/workers was changed; the API just returns
the new `prescreening` status string.
## Deviations from the original prescreen spec
- **Extractor:** regex instead of Needle 2. Rationale in `docs/SPIKE_PHASE0.md`.
- **No separate credits split:** the credit is still reserved once on upload;
prescreen is treated as part of the same paid job.
- **No admin/web SPA yet:** `manual_review` documents are terminal in the DB
until the admin panel lands.
## Next step
Phase 3 is storage modernization evaluation (RustFS watch). The current stack
keeps MinIO as the production default; RustFS is tracked as a future option.
Alternatively, continue with the backlog: admin/web SPA, ЮKassa payments, heavy
OCR adapters.
---
**Recommended immediate follow-up:** end-to-end smoke test in Docker with all
services running (`make services-up`) and a real PDF/DOCX upload through the
API or bot, verifying that `worker-prescreen` routes to `analyze.q` or
`manual_review` correctly.

439
docs/PHASES_2_PLUS_ROADMAP.md Normal file
View file

@ -0,0 +1,439 @@
# Implementation Roadmap — Phase 2 (Prescreen) & Remaining Work
> Based on `document-extraction-spec.md`, `needle-prescreen-integration.md`,
> and the Phase 0 spike report (`docs/SPIKE_PHASE0.md`).
> Current checkpoint: Phase 1 extraction refactor is complete (`docs/PHASE1_HANDOFF.md`).
## Status overview
| Phase | Scope | Status |
|---|---|---|
| Phase 0 | Needle/RustFS spikes | Done (`docs/SPIKE_PHASE0.md`) |
| Phase 1 | Extraction layer refactor | Done (`docs/PHASE1_HANDOFF.md`) |
| Phase 2 | Prescreen stage | **Planned below** |
| Phase 3 | Storage modernization / RustFS watch | **Planned below** |
| Follow-up | Admin/web, payments, heavy OCR | **Backlog** |
## Phase 2 — Prescreen stage between extract and analyze
### Why
Insert a fast, local decision layer after extraction and before the expensive LLM
analysis so that:
- obvious low-risk contracts can short-circuit to a lightweight report
- high-risk / high-value contracts are routed to deep LLM analysis
- the heavy LLM worker is no longer the only path for every document
- per-stage queue metrics, retry, and DLQ are clean and independent
### Key correction from the original prescreen spec
The original spec assumed **Needle 2** (`cactus-needle`) as the prescreen
extractor. The Phase 0 spike proved the base model is English-only and fails on
Russian contracts with `confidence=0.0`. Fine-tuning would also disable the
calibrated confidence head the routing design depends on.
Therefore Phase 2 uses a **deterministic regex+pydantic extractor** for RU/BY
contracts. Contract boilerplate is highly templated, so regex gives:
- zero marginal cost
- no hallucination risk
- missing fields naturally map to `manual_review`
- `confidence_score` becomes a deterministic **field-coverage ratio**
A small-LLM fallback can be added later for fields regex misses.
### 2.1 RabbitMQ topology changes
Add to `core/mq/topology.py` using the existing direct-exchange + TTL retry/DLQ
pattern:
```text
contracts.x
├─ extract ─► extract.q (existing)
├─ prescreen ─► prescreen.q (NEW)
└─ analyze ─► analyze.q (existing)
contracts.retry.x
├─ retry.prescreen ─► prescreen.retry.q (NEW classic delay queue, DLX→contracts.x[prescreen])
prescreen.dlq (NEW quorum)
```
Constants: `QUEUE_PRESCREEN`, `RK_PRESCREEN`, update `_RETRY_QUEUE_FOR`,
`DLQ_FOR`, `declare_all()`.
### 2.2 Database migration `0006_prescreen.py`
New table `prescreen_results`:
```sql
CREATE TABLE prescreen_results (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
correlation_id UUID NOT NULL,
-- extracted metadata
contract_type VARCHAR(64),
party_a TEXT,
party_b TEXT,
total_amount DECIMAL(18, 2),
currency VARCHAR(8),
start_date DATE,
end_date DATE,
has_penalty_clause BOOLEAN,
has_termination_clause BOOLEAN,
has_arbitration BOOLEAN,
confidence_score DECIMAL(4, 3), -- coverage ratio 0.000–1.000
-- routing
routing_decision VARCHAR(32) NOT NULL, -- auto_approve | manual_review | deep_analysis
-- metrics / audit
prescreened_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
processing_ms INTEGER,
extractor_version VARCHAR(32) DEFAULT 'regex-v1',
-- auto-approve lightweight output
auto_summary TEXT,
auto_findings JSONB DEFAULT '[]',
-- retry tracking
error_message TEXT,
retry_count INTEGER DEFAULT 0
);
CREATE INDEX idx_prescreen_document ON prescreen_results(document_id);
CREATE INDEX idx_prescreen_routing ON prescreen_results(routing_decision);
CREATE INDEX idx_prescreen_confidence ON prescreen_results(confidence_score);
```
Also:
```sql
ALTER TYPE document_status ADD VALUE 'prescreening' AFTER 'extracted';
ALTER TYPE jobs.queue ADD VALUE 'prescreen' AFTER 'analyze';
ALTER TABLE reports
ADD COLUMN prescreen_result_id UUID REFERENCES prescreen_results(id),
ADD COLUMN prescreen_meta JSONB DEFAULT NULL;
```
Use `TEXT+CHECK` instead of Postgres enums if the project convention prefers
additive migrations (the existing schema uses both). Decision to make during
implementation.
### 2.3 Message schema additions
Add to `core/mq/messages.py`:
```python
class PrescreenRequested(PipelineMessage):
"""worker-extract → contracts.x[prescreen] → worker-prescreen."""
text_s3_key: str
filename: str
class PrescreenCompleted(PipelineMessage):
"""worker-prescreen result; also the payload forwarded to analyze.q."""
text_s3_key: str
prescreened_at: datetime
contract_type: str | None
party_a: str | None
party_b: str | None
total_amount: float | None
currency: str | None
start_date: str | None
end_date: str | None
has_penalty_clause: bool | None
has_termination_clause: bool | None
has_arbitration: bool | None
confidence_score: float | None # 0..1 coverage ratio
routing_decision: Literal["auto_approve", "manual_review", "deep_analysis"]
auto_summary: str | None
auto_findings: list[dict] = Field(default_factory=list)
error_message: str | None
class AnalyzeRequested(PipelineMessage):
"""worker-prescreen → contracts.x[analyze] → worker-analyze."""
text_s3_key: str
filename: str
prescreen_meta: PrescreenCompleted
```
Keep `DocumentExtracted` for backward compatibility; `worker-extract` may still
publish it for observability, but the prescreen stage is the main forward path.
### 2.4 New service: `worker-prescreen`
Directory:
```text
src/contract_check/worker_prescreen/
__init__.py
__main__.py
consumer.py
handler.py
router.py
extractor.py # regex+pydantic RU/BY contract extractor
config.py
```
#### `extractor.py`
Deterministic extractor built with `re` + Pydantic model. Target fields:
```python
class PrescreenContractMeta(BaseModel):
contract_type: str | None
party_a: str | None
party_b: str | None
total_amount: float | None
currency: str | None
start_date: str | None
end_date: str | None
has_penalty_clause: bool | None
has_termination_clause: bool | None
has_arbitration: bool | None
```
Heuristics:
- `contract_type`: map first matched keyword (`договор поставки`, `договор оказания услуг`, `договор подряда`, `договор аренды`, `договор купли-продажи`, etc.)
- `party_a` / `party_b`: extract first two legal entities after patterns like
`Общество с ограниченной ответственностью «(.*?)»`, `Акционерное общество «(.*?)»`, `Индивидуальный предприниматель (.*?)`.
- `total_amount`: find `составляет ([\d\s,.]+) (рублей|руб|USD|EUR|€|\$)` and parse as float.
- `currency`: normalize to `RUB`, `USD`, `EUR`, `BYN`.
- `start_date` / `end_date`: match `с (\d{2}\.\d{2}\.\d{4})`, `от (\d{2}\.\d{2}\.\d{4})`,
`действует с (\d{2}\.\d{2}\.\d{4}) по (\d{2}\.\d{2}\.\d{4})`.
- `has_penalty_clause`: presence of `неустойка`, `штраф`, `пеня`, `0,1%`/`% за каждый день просрочки`.
- `has_termination_clause`: presence of `расторгнуть`, `расторжение`, `одностороннему порядке`.
- `has_arbitration`: presence of `арбитражный суд`, `Арбитражный суд`, `Международный коммерческий арбитражный суд`.
`confidence_score = matched_fields / total_fields`.
Test on synthetic and a few real contracts; tune false-positive/negative rate.
#### `router.py`
```python
class PrescreenRouter:
def __init__(
self,
confidence_threshold: float = 0.75,
high_value_threshold: float = 100_000,
) -> None:
...
def decide(self, meta: PrescreenContractMeta) -> str:
if confidence < threshold or missing mandatory fields:
return "manual_review"
if total_amount > high_value_threshold:
return "deep_analysis"
if has_penalty_clause or has_arbitration:
return "deep_analysis"
return "auto_approve"
```
`auto_approve` is **disabled by default** via env `PRESCREEN_AUTO_APPROVE=false`
until accuracy is proven; when disabled, `auto_approve` decisions are mapped
to `manual_review`.
#### `handler.py`
- idempotency check on `documents.status`
- set `status = 'prescreening'`, `jobs.status = 'running'`
- download extracted Markdown from MinIO
- run regex extractor in thread pool (`asyncio.to_thread`)
- route
- persist `prescreen_results`
- publish next message:
- `deep_analysis` → `contracts.x[analyze]` with `AnalyzeRequested`
- `auto_approve` → `contracts.x[report.completed]` with lightweight summary
(only when `PRESCREEN_AUTO_APPROVE=true`)
- `manual_review` → `contracts.x[review.manual]`; no further processing
- ack
#### `consumer.py`
`Consumer[PrescreenRequested]` for `prescreen.q`, wired like `AnalyzeConsumer`.
#### `config.py`
- `PRESCREEN_ENABLED` — if false, handler immediately publishes
`AnalyzeRequested` (passthrough, preserving analytics)
- `PRESCREEN_AUTO_APPROVE`
- `PRESCREEN_CONFIDENCE_THRESHOLD`
- `PRESCREEN_HIGH_VALUE_THRESHOLD`
### 2.5 Dockerfile + compose
- `srv/worker-prescreen/Dockerfile`: lean image, no tesseract/pymupdf, only core
deps + regex engine.
- Add `worker-prescreen` to `docker-compose.yml` profile `services`.
- Memory limit 128M (regex is tiny).
### 2.6 Changes to existing services
#### `worker-extract`
After publishing `DocumentExtracted` (backward compat), also publish
`PrescreenRequested`:
```python
await publisher.publish(
PrescreenRequested(
correlation_id=...,
document_id=...,
user_id=...,
text_s3_key=extracted_key,
filename=payload.filename,
),
routing_key=RK_PRESCREEN,
)
```
Gate behind `PRESCREEN_ENABLED` (default true after Phase 2).
#### `worker-analyze`
Change consumer to `AnalyzeRequested`:
```python
class AnalyzeConsumer(Consumer[AnalyzeRequested]):
queue = "analyze.q"
message_model = AnalyzeRequested
```
Enrich prompt with `prescreen_meta` (§6.3 of the original spec).
#### `api`
- Update document status endpoint to include prescreen block.
- Status lifecycle: `queued → extracting → prescreening → analyzing | manual_review | done`.
- Optional: expose `?mode=fast` vs `?mode=full` for B2B.
### 2.7 Metrics
Add to `core/metrics.py` and expose from `worker-prescreen`:
- `prescreen_requests_total{source}`
- `prescreen_duration_seconds{decision}`
- `prescreen_routing_decisions_total{decision,contract_type}`
- `prescreen_confidence_distribution`
- `prescreen_errors_total{error_type}`
- `prescreen_dlq_messages_total`
### 2.8 Tests
- Unit: `tests/unit/test_prescreen_extractor.py` — regex patterns on synthetic
contracts, coverage scoring.
- Unit: `tests/unit/test_prescreen_router.py` — routing matrix with all flag
combinations.
- Integration: `tests/integration/test_prescreen_worker.py`:
- `PrescreenRequested` → `AnalyzeRequested` for `deep_analysis`
- `PrescreenRequested` → `review.manual` for `manual_review`
- bypass mode (`PRESCREEN_ENABLED=false`) → passthrough `AnalyzeRequested`
- Update `test_extract_worker.py` to assert `PrescreenRequested` is also
published.
- Update `test_analyze_worker.py` to consume `AnalyzeRequested` and include
prescreen metadata.
### 2.9 DoD
```bash
make lint && make typecheck && make test-unit
make test-integration
```
## Phase 3 — Storage watch / RustFS evaluation
### Goal
Keep MinIO as the production default but maintain a credible migration path to
RustFS when its Lifecycle + KMS features reach GA. No application code changes
are required because S3 is already behind `core.s3.port.py`.
### Why not switch now
From the Phase 0 RustFS spike:
- Lifecycle/KMS/distributed mode are marked **"Under Testing"** upstream.
- SSE-S3 objects written by MinIO are **not readable by RustFS** today — a
migration would require re-putting objects through the app.
- RustFS is weeks old; MinIO is battle-tested for this exact compose setup.
- Real advantages exist: Apache 2.0 license (vs MinIO AGPL), no telemetry,
RF data-sovereignty friendly.
### Actions
1. Add an optional `docker-compose.rustfs.yml` overlay for local experiments.
2. Run the full integration suite against RustFS quarterly or when upstream
announces Lifecycle/KMS GA.
3. If/when adopted:
- pin exact RustFS image tag
- set `S3_SERVER_SIDE_ENCRYPTION=false` during transition
- re-encrypt objects via the app after cutover
- update `minio-init` service or replace with `rustfs-init`
## Backlog / follow-up work (not tied to Phase 2)
| Item | Rationale |
|---|---|
| Admin/web SPA | Deferred by original architecture scope. |
| ЮKassa payments | Stub `invoices` table exists; payment logic deferred. |
| Heavy OCR adapters (marker, paddleocr, easyocr, img2table) | Spec deferral. Consider separate `worker-extract-heavy` queue or cloud API. |
| Email extractor (MSG/EML) | Spec deferral; only relevant for B2B corporate email ingestion. |
| Multi-tenancy | Explicit non-goal of original refactor. Do not add `tenant_id` until orgs are a real requirement. |
| Fine-tuned RU Needle model | Revisit if `cactus-needle` ships a multilingual or RU-tuned base with calibrated confidence. |
## Appendix: original spec alignment
### `document-extraction-spec.md`
| Requirement | Phase 1 status |
|---|---|
| `DocumentExtractor` Protocol / ABC | Done (`core/extraction/port.py`) |
| `ExtractedDocument` dataclass / Pydantic | Done |
| `ExtractorFactory` with MIME detection | Done |
| `DocxExtractor` (`mammoth`) | Done |
| `RtfExtractor` (`striprtf`) | Done |
| `EncodingDetector` (`chardet`) | Done |
| `PyMuPDFExtractor` + `has_tables` | Done |
| `TesseractOCRAdapter` + `is_structured=False` | Done |
| Metrics by format | Done |
| Update architecture docs | Done |
| Heavy adapters deferred | Backlog |
### `needle-prescreen-integration.md`
| Requirement | Phase 2 plan above |
|---|---|
| Separate `prescreen.q` stage | Yes |
| `PrescreenRequested` / `PrescreenCompleted` messages | Yes, corrected to extend `PipelineMessage` |
| `prescreen_results` table | Yes, adjusted columns |
| `worker-prescreen` service | Yes, regex-based instead of Needle |
| Routing decisions | Yes, conservative default |
| `worker-extract` publishes prescreen | Yes |
| `worker-analyze` consumes `AnalyzeRequested` | Yes |
| API status endpoint enriched | Yes |
| Dockerfile/compose for prescreen worker | Yes |
| Rollback / bypass flag | Yes |
| Multi-tenant / credit split | **Rejected** — not in current scope; integer credit model stays |
## Ordering recommendation
1. Phase 2 DB migration + RabbitMQ topology.
2. Phase 2 messages + `worker_prescreen` skeleton (no-op passthrough first).
3. Phase 2 regex extractor + router + tests.
4. Wire `worker-extract` → `prescreen.q` and `worker-analyze` → `analyze.q`.
5. End-to-end integration test.
6. Phase 3 RustFS spike only after upstream GA announcement.
---
**Next action:** implement Phase 2 step 1 (migration + topology) if approved.

242
docs/PRESCREEN_HYBRID_REFACTOR_PLAN.md Normal file
View file

@ -0,0 +1,242 @@
# Refactor Plan: Prescreen Worker → Hybrid Extraction (Heuristic + LLM Fallback)
> Status: delivered through Phase 4; Phase 5 rollout pending merge
> Scope: `src/contract_check/worker_prescreen/`, `src/contract_check/core/llm/`, `src/contract_check/core/config.py`, `src/contract_check/core/metrics.py`
> Supersedes: the regex-only extractor described in `worker_prescreen/extractor.py` (regex-v1)
---
## 1. Goal
Replace the monolithic regex extractor (`extractor.py`, ~260 lines of Russian/BY legal
regexes) with a **hybrid two-stage extractor**:
1. **Stage 1 — Heuristic (deterministic, zero-cost):** keyword dictionaries,
positional windows, sentence scanning, per-field confidence weights.
No regex — plain string operations only.
2. **Stage 2 — LLM fallback (only when Stage 1 confidence is low):** reuse the
existing `core/llm` provider infrastructure (Ollama Cloud / YandexGPT) with a
JSON-schema-constrained extraction prompt.
Non-goals:
- No changes to routing semantics (`router.py` thresholds/decisions stay as-is).
- No DB schema migration (`prescreen_results.extractor_version` already exists,
`String(32)` fits `"heuristic-v2"` / `"llm-v1"`).
- No changes to MQ topology, messages (`PrescreenCompleted` shape unchanged),
or downstream `worker_analyze`.
- Auto-approve stays disabled by default.
## 2. Current State
```
worker_prescreen/
├── consumer.py # consumes prescreen.q → handler
├── handler.py # Stage 4: asyncio.to_thread(extract_contract_meta, text)
├── extractor.py # PrescreenContractMeta + 10 regex patterns ← TO REPLACE
├── router.py # decides auto_approve|manual_review|deep_analysis
└── config.py
```
Problems:
- Regexes are brittle (line-noise, OCR artifacts, whitespace variants) and hard
to extend (every new contract type = new regex).
- No recovery path: low-confidence extraction always ends in `manual_review`.
- `prescreen_results.extractor_version` is never written by the handler INSERT —
it silently relies on the DB default `'regex-v1'`.
## 3. Target Architecture
```
┌────────────────────────────────────────────┐
│ handler.py │
│ Stage 4: meta = await extractor.extract() │
└───────────────┬────────────────────────────┘
│
HybridMetaExtractor (orchestrator)
│
┌────────────────────┴──────────────────────┐
▼ ▼
HeuristicExtractor (sync, to_thread) LLMPrescreenExtractor
keyword + positional + sentence wraps core/llm provider
scanning, confidence-weighted extract_prescreen(text)
│ │
confidence ≥ threshold? ──── no ────► run LLM, validate JSON,
│ yes merge over heuristic meta
▼
PrescreenContractMeta → router.decide() → persist (with extractor_version)
```
### 3.1 New module layout (flat, matches existing package conventions)
```
worker_prescreen/
├── extractor.py # PrescreenContractMeta (model, unchanged shape)
│ # + MetadataExtractor protocol + re-export shim
├── extractor_heuristic.py# HeuristicExtractor (Stage 1)
├── extractor_llm.py # LLMPrescreenExtractor (Stage 2, dict → pydantic validation)
├── extractor_hybrid.py # HybridMetaExtractor (threshold + merge + kill-switch)
├── handler.py # Stage 4 rewired; INSERT gains extractor_version
└── router.py # unchanged
```
### 3.2 Heuristic stage design (regex-free)
| Field | Method | Notes |
|---|---|---|
| `contract_type` | Phrase dictionary match on normalized text | `{"supply": ["договор поставки", "договор купли-продажи", ...], ...}` — lowercase + whitespace-collapse once, then `in` checks |
| `party_a/b` | Positional: scan only the header window (text up to first `1.` / `ПРЕДМЕТ` heading, capped ~1500 chars); token-scan for entity-form tokens (ООО, ИП, АО, …) then capture until quote `»`/`"` close or line end | Replaces `_ENTITY_RE` |
| `total_amount` + `currency` | Trigger-word scan (`составляет`, `стоимость`, `цена`, …) then manual digit-window parse (`_scan_number` helper walking chars, handling spaces/commas); currency via token lookup in the following ~50 chars | Replaces `_AMOUNT_RE` |
| `start/end_date` | Trigger-word scan (`действует с`, `с … по …`) then `_parse_ddmmyyyy` manual splitter (already regex-free in v1 — keep) | Replaces `_DATE_RE`/`_END_DATE_RE` |
| boolean clauses | Sentence segmentation via `str.split` on `. ` / `\n`, keyword membership per sentence (`неустойка`, `штраф`, `расторгнуть`, `арбитражн`, …) | Replaces `_PENALTY_RE` etc. |
Confidence scoring becomes **weighted** instead of flat field coverage:
```python
FIELD_WEIGHTS = {
"contract_type": 0.25,
"party_a": 0.15,
"party_b": 0.15,
"total_amount": 0.15,
"currency": 0.05,
"start_date": 0.10,
"end_date": 0.05,
"has_penalty_clause": 0.05,
"has_termination_clause": 0.05,
"has_arbitration": 0.05,
} # sums to 1.0
```
Optional per-field method bonus (exact phrase match = full weight, positional
window hit = full weight, fuzzy tail hit = ×0.7) — start simple, weights
constant, tune later with real data.
### 3.3 LLM fallback design
- **Protocol extension** (`core/llm/port.py`):
`async def extract_prescreen(self, text: str) -> dict[str, Any]` — returns raw
JSON dict; **no import of worker_prescreen** (keeps layering clean).
Implemented by both `OllamaCloudProvider` and `YandexGPTProvider` as a thin
wrapper over their existing JSON-chat + repair-loop machinery
(`_chat_json` / `responseFormat=json_schema`) with a dedicated
`PRESCREEN_SYSTEM` prompt: extract the 10 meta fields, cite nothing, JSON only.
- **Worker-side wrapper** (`extractor_llm.py`):
`dict` → `PrescreenContractMeta` via pydantic (rejects hallucinated fields,
re-`None`s unknown enum values), then recomputes weighted confidence.
- **Input cap:** only the first `prescreen_llm_max_chars` (default 20 000) chars
are sent — meta lives in the header for templated contracts; cost control.
- **Merge rule:** LLM values override heuristic `None`s and low-confidence
fields; boolean clauses become OR(heuristic, llm) — both sources are
presence-checks, false positives are cheap, false negatives route wrong.
### 3.4 Failure semantics
- LLM fallback error (quota, timeout, invalid JSON after repair):
**do not fail the message.** Log + keep heuristic meta,
`extractor_version="heuristic-v2"`, record the error in
`prescreen_results.auto_findings` (`{"llm_fallback_error": "..."}`) and the
`prescreen_fallback_runs_total{outcome="failed"}` counter.
- Heuristic stage is pure string ops — its only failure mode is `None` fields,
which is already handled by low confidence → fallback / manual_review.
### 3.5 Config additions (`core/config.py`, `--- prescreen stage ---`)
```python
prescreen_llm_fallback_enabled: bool = Field(default=False, ...) # kill-switch; flip to True after burn-in
prescreen_llm_fallback_threshold: float = Field(default=0.75, ...) # ≤ router threshold
prescreen_llm_max_chars: int = Field(default=20_000, ...)
```
### 3.6 Metrics additions (`core/metrics.py`)
```python
prescreen_fallback_runs = Counter(
"contract_check_prescreen_fallback_runs_total",
"...", labelnames=["outcome"], # used | failed | skipped | disabled
)
prescreen_extraction_stage = Histogram(
"contract_check_prescreen_extraction_stage_seconds",
"...", labelnames=["stage"], # heuristic | llm
)
```
`extractor_version` values: `"regex-v1"` (legacy shim flag), `"heuristic-v2"`
(LLM not run / disabled / failed) and `"hybrid-llm-v1"` (LLM result merged).
## 4. Implementation Phases
Each phase is independently shippable and reverted by config/env flag.
### Phase 1 — Port + heuristic extractor (no behavior change for routing)
- [x] `extractor.py`: keep `PrescreenContractMeta` + weighted `_score_confidence`;
add `MetadataExtractor` protocol (`def extract(text: str) -> PrescreenContractMeta`);
keep `extract_contract_meta` as a delegating shim so
`tests/unit/test_prescreen_extractor.py` and `worker_prescreen/__init__.py`
keep importing it.
- [x] New `extractor_heuristic.py` implementing all helpers from §3.2
(`_scan_number`, `_find_after_trigger`, header-window splitter,
sentence splitter, dictionaries as module-level constants).
- [x] Port existing unit-test fixtures (`SIMPLE_SUPPLY`, `MINIMAL`, boolean-flag
parametrize) onto `HeuristicExtractor`; they must pass with identical
expected values.
- [x] `extractor_version` shim reports `"heuristic-v2"`.
### Phase 2 — LLM `extract_prescreen` on providers
- [x] Extend `LLMProvider` protocol + both adapters
(`ollama_cloud.py`, `yandex_gpt.py`): new `PRESCREEN_SYSTEM` prompt,
JSON schema for the 10 fields, reuse repair loop, truncate input to
`chunk_size`-independent small cap.
- [x] Unit tests with a fake transport (both adapters already have this pattern):
valid dict, invalid enum → `None`, malformed JSON → repair once → fail.
- [x] `extractor_llm.py`: dict → `PrescreenContractMeta` validation +
weighted confidence.
### Phase 3 — Orchestrator + handler wiring
- [x] `extractor_hybrid.py`: threshold check, LLM call (async), merge rules,
error swallowing, extractor_version selection, kill-switch.
- [x] `handler.py` Stage 4: replace `asyncio.to_thread(extract_contract_meta, …)`
with `await self._extractor.extract(contract_text)` where
`self._extractor` is injectable (constructor arg, defaults to hybrid) —
mirrors the existing `provider` injection pattern in `AnalyzeHandler`.
- [x] Handler Stage 6 INSERT: bind `extractor_version`.
- [x] Config + metrics from §3.5/§3.6.
### Phase 4 — Tests + verification
- [x] Unit: orchestrator matrix — high confidence skips LLM; low confidence
merges; LLM failure → heuristic result + `outcome="failed"`; disabled →
`outcome="disabled"`.
- [x] Integration (`tests/integration/test_prescreen_worker.py` pattern):
run handler in-process with a stub provider; assert
`prescreen_results.extractor_version` persisted, routing unchanged.
- [x] Backfill spot-check: replayed 8 RU/BY-style contract samples through
regex-v1 vs heuristic-v2. Confidence deltas: all neutral or better,
heuristic fixes regex under-parsing on NBSP/noise and captures fuller
party names. Record below in PR description.
- [x] `ruff check . && mypy src && pytest tests/unit tests/integration -k prescreen`.
(ruff/mypy findings outside the touched files are from pre-existing
uncommitted changes; prescreen tests pass.)
### Phase 5 — Rollout
- [ ] Merge with `prescreen_llm_fallback_enabled=false` (pure heuristic).
- [ ] Observe `prescreen_fallback_runs` / `prescreen_confidence` for a few days.
- [x] Document in `docs/ARCHITECTURE.md` prescreen section + `.env.example`
completed. The regex notes in `SPIKE_PHASE0.md` are no longer referenced
from the extractor docstring. Staging/prod enablement remains an
operational step after merge.
## 5. Risks & Mitigations
| Risk | Mitigation |
|---|---|
| Heuristic drops accuracy vs regex on edge templates | Port 100% of v1 unit tests before deleting regex; keep `extractor.py` regex code one release behind a flag (`PRESCREEN_KEEP_REGEX=true` env, temporary) |
| LLM hallucinates fields | pydantic validation whitelists contract_type enums; numeric/date parse checks; booleans only OR-merged |
| LLM latency blows up prescreen SLA | 20k char cap, existing provider timeouts, `prescreen_extraction_stage` histogram; fallback failures never block the pipeline |
| Cost creep on fallback rate | `prescreen_fallback_runs_total` alert; threshold tunable without deploy via env |
| Layering violation (core ← worker import) | Protocol returns plain `dict`; pydantic model stays in `worker_prescreen` |
## 6. Rollback
1. Config: `PRESCREEN_LLM_FALLBACK_ENABLED=false` → deterministic heuristic only.
2. Full: revert merge — no DB migration to undo; `extractor_version` strings are
informational only.

103
docs/SPIKE_PHASE0.md Normal file
View file

@ -0,0 +1,103 @@
# Phase 0 Spike Report — Needle 2 prescreen & RustFS storage
> Date: 2026-08-16. Evidence from live testing in isolated environments
> (`/tmp/opencode/needle-spike`, ephemeral `rustfs/rustfs:latest` container).
> Gates the Phase 1 (extraction refactor) and Phase 2 (prescreen stage) plans.
## Spike 1: cactus-needle (Needle 2) for RU contract prescreen — **NOT VIABLE**
### What was verified
- `cactus-needle==2.0.5` is real (Cactus Compute, Apache-2.0, first release
2026-08-10). Installs cleanly on Python 3.13. The spec's claims hold:
14MB engine (fetched once from HF, cached offline), `needle.extract(text,
PydanticModel)`, calibrated `confidence` on responses, `.cact` LoRA
fine-tuning pipeline. Note: PyPI package name is **`cactus-needle`**
(`import needle`); bare `needle` on PyPI is an unrelated dead 2017 CSS
testing tool.
- Engine session RAM measured at 37.8MB (simple schema) to 119.7MB (larger
schema) — fine for a 256M container. Python deps pull the JAX/Flax/optax
stack (~200MB+ image impact) because fine-tuning support ships in the core
dependency list.
### Results (base weights, `buffer_size` default)
| Case | Result |
|---|---|
| EN invoice (README example) | Perfect extraction, `confidence=0.9484` |
| EN off-topic sentence | Correctly refused: empty call `[]`, `confidence=0.7427` |
| RU contract, 2.5K chars | **Total hallucination**: parties → "Supplier"/"Supplier", amount → 12.0 USD (real: 1 234 567,80 RUB), dates → 2024 defaults. `confidence=0.0` |
| RU contract, 43K chars | Same failure mode, different hallucinations. `confidence=0.0` |
| RU garbage text | **Not refused** — fabricated "Party A/Party B/150 USD". `confidence=0.0` |
### Verdict
The base model is **English-centric; it does not understand Russian**. The
calibrated confidence head behaves correctly (0.0 = "I can't read this"), so a
naive deployment would have been *safe* (100% → manual_review) but useless.
Fine-tuning cannot rescue this for v1:
1. Calibrated confidence is **disabled on tuned weights** (reports `None`) —
and the spec's entire routing design hinges on confidence gating.
2. The model failed even trivial verbatim copy tasks (could not copy
«ООО «Ромашка»» from adjacent text) — this is tokenizer/training coverage,
not prompt tuning.
3. Building a RU legal extraction dataset + LoRA pipeline is its own project.
### Plan adjustment (Phase 2)
- `worker-prescreen` ships with a **deterministic regex+pydantic extractor**
instead of Needle: RU/BY contract boilerplate is highly templated
(«договор поставки», «именуемое в дальнейшем», «сумма договора составляет
N рублей», «неустойка», «арбитражный суд», DD.MM.YYYY ranges). Zero marginal
cost, no hallucination, p95 ≈ ms. Absence of a match = `None` →
`manual_review`, which matches the routing philosophy.
- `confidence_score` is redefined as a deterministic **field-coverage score**
(fraction of target fields extracted), not model calibration. Router
thresholds operate on coverage.
- `needle_client.py` / `model_weights` / `NEEDLE_*` env are dropped from the
plan; keep `prescreen_results.extractor_version` for future engine swaps.
- Optional fast-follow (not v1): small-LLM prescreen via the existing
`core/llm` port for fields regex missed.
- Revisit Needle only if/when a multilingual or RU-tuned base ships with
calibration intact.
## Spike 2: RustFS as MinIO alternative — **VIABLE, but stay on MinIO for now**
Tested with the project's own `MinioStorage` adapter (minio SDK) against
`rustfs/rustfs:latest`, single node, named volume.
| Check | Result |
|---|---|
| Health (`/minio/health/live`) | 200 — MinIO-compatible probe works |
| bucket create/exists, put/get/stat/delete | OK (incl. Cyrillic UTF-8 payloads) |
| `set/get/delete_bucket_lifecycle` (ILM config API) | OK — rule round-trips |
| **ILM expiration behavior** (`RUSTFS_ILM_DEBUG_DAY_SECS=2`, rule days=1) | **Object under `users/` purged in ~10s; control object outside prefix untouched** |
| Perf (4KB put+get ×20) | ~2.0ms/op |
| Docker ergonomics | Runs as UID 10001 — bind mounts must be chowned; named volumes are the easy path |
### Verdict
The critical blocker (152-ФЗ TTL purge via ILM) **works in practice** — the
"Under Testing" label upstream refers to CI gating, not absence. Remaining
reasons to keep MinIO as the default for production:
1. Project is weeks old (1.0.0-rc era); lifecycle, distributed mode, and KMS
are all still marked "Under Testing" upstream.
2. **SSE migration trap**: objects written by MinIO with SSE-S3/KMS/C are not
readable by RustFS — a later migration requires re-putting objects through
the app (we run SSE-S3=off in dev, on in prod per `S3_SERVER_SIDE_ENCRYPTION`).
3. No operational history; MinIO is battle-tested for this exact compose setup.
Because everything S3-shaped is already behind `core/s3/port.py`, switching
later is a compose + env change, not a code change. Action item: revisit when
RustFS marks Lifecycle + KMS GA (tracked in ARCHITECTURE notes, not code).
## Phase gate outcomes
- **Phase 1 (extraction refactor): GO** — unchanged.
- **Phase 2 (prescreen stage): GO with amendments** — regex extractor instead
of Needle; coverage-based confidence; no needle deps/env; conservative
routing (`PRESCREEN_AUTO_APPROVE=false` default) as already planned.
- **Storage: no change** — MinIO stays; RustFS is a credible fast-follow.

169
migrations/versions/0006_prescreen.py Normal file
View file

@ -0,0 +1,169 @@
"""Add prescreen_results table and expand status/queue enums.
Revision ID: 0006
Revises: 0005
Create Date: 2026-08-16
Adds the prescreen stage between extract and analyze:
- documents.status gains 'prescreening'
- jobs.queue gains 'prescreen'
- new prescreen_results table stores deterministic regex extraction output
and routing decision
- reports gains FK + meta column for downstream enrichment
"""
from __future__ import annotations
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision: str = "0006"
down_revision: str | None = "0005"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
# ── prescreen_results ────────────────────────────────────────────────────
op.create_table(
"prescreen_results",
sa.Column(
"id",
postgresql.UUID(as_uuid=True),
primary_key=True,
server_default=sa.text("gen_random_uuid()"),
),
sa.Column(
"document_id",
postgresql.UUID(as_uuid=True),
sa.ForeignKey("documents.id", ondelete="CASCADE"),
nullable=False,
index=True,
),
sa.Column(
"correlation_id",
postgresql.UUID(as_uuid=True),
nullable=False,
),
sa.Column("contract_type", sa.String(64), nullable=True),
sa.Column("party_a", sa.Text, nullable=True),
sa.Column("party_b", sa.Text, nullable=True),
sa.Column("total_amount", sa.Numeric(18, 2), nullable=True),
sa.Column("currency", sa.String(8), nullable=True),
sa.Column("start_date", sa.Date, nullable=True),
sa.Column("end_date", sa.Date, nullable=True),
sa.Column("has_penalty_clause", sa.Boolean, nullable=True),
sa.Column("has_termination_clause", sa.Boolean, nullable=True),
sa.Column("has_arbitration", sa.Boolean, nullable=True),
sa.Column(
"confidence_score",
sa.Numeric(4, 3),
nullable=True,
comment="Field-coverage ratio 0.000-1.000",
),
sa.Column(
"routing_decision",
sa.String(32),
nullable=False,
server_default=sa.text("'manual_review'"),
),
sa.Column(
"prescreened_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.Column("processing_ms", sa.Integer, nullable=True),
sa.Column(
"extractor_version",
sa.String(32),
nullable=False,
server_default=sa.text("'regex-v1'"),
),
sa.Column("auto_summary", sa.Text, nullable=True),
sa.Column(
"auto_findings", postgresql.JSONB, nullable=False, server_default=sa.text("'[]'")
),
sa.Column("error_message", sa.Text, nullable=True),
sa.Column(
"retry_count",
sa.Integer,
nullable=False,
server_default=sa.text("0"),
),
sa.CheckConstraint(
"routing_decision IN ('auto_approve','manual_review','deep_analysis')",
name="prescreen_results_routing_decision_check",
),
sa.CheckConstraint(
"confidence_score IS NULL OR (confidence_score >= 0 AND confidence_score <= 1)",
name="prescreen_results_confidence_check",
),
sa.CheckConstraint("retry_count >= 0", name="prescreen_results_retry_count_nonneg"),
)
op.create_index("prescreen_results_routing_idx", "prescreen_results", ["routing_decision"])
op.create_index("prescreen_results_confidence_idx", "prescreen_results", ["confidence_score"])
# ── reports enrichment from prescreen ────────────────────────────────────
op.add_column(
"reports",
sa.Column(
"prescreen_result_id",
postgresql.UUID(as_uuid=True),
sa.ForeignKey("prescreen_results.id", ondelete="SET NULL"),
nullable=True,
),
)
op.add_column(
"reports",
sa.Column(
"prescreen_meta",
postgresql.JSONB,
nullable=True,
comment="Serialized PrescreenCompleted metadata used by the analyzer",
),
)
# ── expand additive TEXT+CHECK enums ─────────────────────────────────────
# Drop the old narrower constraints first so the new definitions can reuse
# the same names.
op.drop_constraint("documents_status_check", "documents", type_="check")
op.create_check_constraint(
"documents_status_check",
"documents",
"status IN ('queued','extracting','prescreening','ocr','analyzing','done','failed')",
)
op.drop_constraint("jobs_queue_check", "jobs", type_="check")
op.create_check_constraint(
"jobs_queue_check",
"jobs",
"queue IN ('extract','prescreen','analyze')",
)
def downgrade() -> None:
op.drop_constraint("jobs_queue_check", "jobs", type_="check")
op.create_check_constraint(
"jobs_queue_check",
"jobs",
"queue IN ('extract','analyze')",
)
op.drop_constraint("documents_status_check", "documents", type_="check")
op.create_check_constraint(
"documents_status_check",
"documents",
"status IN ('queued','extracting','ocr','analyzing','done','failed')",
)
op.drop_column("reports", "prescreen_meta")
op.drop_column("reports", "prescreen_result_id")
op.drop_index("prescreen_results_confidence_idx", table_name="prescreen_results")
op.drop_index("prescreen_results_routing_idx", table_name="prescreen_results")
op.drop_table("prescreen_results")

30
migrations/versions/0007_jobs_unique_constraint.py Normal file
View file

@ -0,0 +1,30 @@
"""Add unique constraint on jobs.document_id + queue for ON CONFLICT support.
Revision ID: 0007
Revises: 0006
Create Date: 2026-08-16
Adds a unique constraint on (document_id, queue) to support ON CONFLICT clauses
in worker handlers that need idempotent job tracking per queue.
"""
from __future__ import annotations
from collections.abc import Sequence
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "0007"
down_revision: str | None = "0006"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
# Add unique constraint on (document_id, queue) to support ON CONFLICT
op.create_unique_constraint("jobs_document_queue_unique", "jobs", ["document_id", "queue"])
def downgrade() -> None:
op.drop_constraint("jobs_document_queue_unique", "jobs", type_="unique")

42
migrations/versions/0008_add_manual_review_status.py Normal file
View file

@ -0,0 +1,42 @@
"""add_manual_review_status
Revision ID: 0008
Revises: 0007
Create Date: 2026-08-16 21:55:02.604147
"""
from __future__ import annotations
from collections.abc import Sequence
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "0008"
down_revision: str | None = "0007"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
# Drop old check constraint and add new one with manual_review status
op.execute("ALTER TABLE documents DROP CONSTRAINT documents_status_check")
op.execute(
"ALTER TABLE documents ADD CONSTRAINT documents_status_check "
"CHECK (status IN ('queued','extracting','prescreening','ocr','analyzing','done','failed','manual_review'))"
)
# Update existing documents with failed status but manual_review stage to manual_review status
op.execute(
"UPDATE documents SET status = 'manual_review' WHERE status = 'failed' AND stage = 'manual_review'"
)
def downgrade() -> None:
# Update manual_review status back to failed
op.execute("UPDATE documents SET status = 'failed' WHERE status = 'manual_review'")
# Drop new constraint and restore old one
op.execute("ALTER TABLE documents DROP CONSTRAINT documents_status_check")
op.execute(
"ALTER TABLE documents ADD CONSTRAINT documents_status_check "
"CHECK (status IN ('queued','extracting','prescreening','ocr','analyzing','done','failed'))"
)

34
pyproject.toml
View file

@ -85,6 +85,10 @@ extract = [
"pymupdf>=1.24",
"pytesseract>=0.3.10",
"pillow>=10",
"mammoth>=1.8",
"striprtf>=0.0.26",
"chardet>=5.2",
"python-magic>=0.4.27",
]
analyze = [
{ include-group = "db" },
@ -124,6 +128,8 @@ dev = [
"testcontainers[rabbitmq,postgres,minio]>=4",
"asgi-lifespan>=2.1.0",
"pre-commit>=4.6.2",
"isort>=5.13",
"ty>=0.0.72",
]
[tool.ruff]
@ -132,28 +138,14 @@ target-version = "py313"
src = ["src", "tests"]
[tool.ruff.lint]
select = ["E4", "E7", "E9", "F", "I", "UP", "B"]
select = ["E4", "E7", "E9", "F", "UP", "B"]
[tool.mypy]
python_version = "3.13"
strict = true
packages = ["contract_check"]
# Third-party libs without complete type stubs. Our own code stays fully typed.
disallow_untyped_calls = false
[[tool.mypy.overrides]]
module = [
"aio_pika.*",
"minio.*",
"prometheus_client.*",
"sentry_sdk.*",
"opentelemetry.*",
"pymupdf.*",
"docx.*",
"pytesseract.*",
"PIL.*",
]
ignore_missing_imports = true
[tool.isort]
profile = "black"
line_length = 100
src_paths = ["src", "tests"]
known_first_party = "contract_check"
skip_glob = ["migrations/*"]
[tool.pytest.ini_options]
testpaths = ["tests"]

2
src/contract_check/__init__.py
View file

@ -8,4 +8,6 @@ Service packages (api/, worker_extract/, worker_analyze/, bot/) land in
Steps 2–5. Importable from each service's own Docker image.
"""
from __future__ import annotations
__version__ = "0.1.0"

2
src/contract_check/__main__.py
View file

@ -1,5 +1,7 @@
"""Top-level entrypoint: `python -m contract_check <file>` → stage-0 prototype."""
from __future__ import annotations
from .prototype import main
if __name__ == "__main__":

2
src/contract_check/api/admin/__init__.py
View file

@ -4,3 +4,5 @@ Reuses the api's auth (JWT + users.role) and DB session. The panel is a set of
HTML routes for managing users (and, later, subscriptions). It is gated behind
``WEB_ADMIN_ENABLED`` and the ``users.role = 'admin'`` check in ``auth.py``.
"""
from __future__ import annotations

2
src/contract_check/api/app.py
View file

@ -5,6 +5,8 @@ initiated document analysis. Adapters (bot, future web/cli) call it over HTTP.
See docs/ARCHITECTURE.md §15.
"""
from __future__ import annotations
from contextlib import asynccontextmanager
from typing import Any

2
src/contract_check/api/routes/__init__.py
View file

@ -1 +1,3 @@
"""API routes package."""
from __future__ import annotations

2
src/contract_check/api/routes/b2b.py
View file

@ -37,7 +37,7 @@ async def analyze_document(
session: AsyncSessionDep,
storage: StorageDep,
publisher: PublisherDep,
file: Annotated[UploadFile, File()] = ..., # type: ignore[assignment]
file: Annotated[UploadFile, File()],
) -> dict[str, object]:
"""Upload a document for analysis using a B2B API key.

2
src/contract_check/api/routes/documents.py
View file

@ -21,6 +21,6 @@ async def upload_document(
storage: StorageDep,
publisher: PublisherDep,
user: CurrentUserDep,
file: Annotated[UploadFile, File()] = ..., # type: ignore[assignment]
file: Annotated[UploadFile, File()],
) -> dict[str, object]:
return await upload_and_enqueue(session, storage, publisher, user.user_id, file)

2
src/contract_check/bot/__init__.py
View file

@ -4,3 +4,5 @@ The leanest service image: speaks to the api exclusively over HTTP and touches
no infrastructure directly (no DB/S3/MQ/LLM). The hexagonal boundary is enforced
by `tests/unit/test_bot_boundary.py`. See docs/ARCHITECTURE.md §4, §17.
"""
from __future__ import annotations

2
src/contract_check/core/__init__.py
View file

@ -18,3 +18,5 @@ Subpackages:
credits — reserve/refund billing invariants
tokens — service-token hashing/verification (no FastAPI here)
"""
from __future__ import annotations

2
src/contract_check/core/analysis/__init__.py
View file

@ -2,3 +2,5 @@
schema, OCR, and the analyzer (prompt building, finding merge/dedupe/sort,
markdown rendering). Imported by the LLM adapter and the prototype benchmark.
"""
from __future__ import annotations

8
src/contract_check/core/analysis/chunker.py
View file

@ -67,3 +67,11 @@ def _split_sentences(para: str, max_chars: int) -> list[str]:
if buf:
out.append(buf)
return out
def chunk_markdown(text: str, max_chars: int = 10000) -> list[str]:
"""Alias for chunk_text; same semantics but explicitly for Markdown input.
Preserves paragraph/sentence boundaries so headings and lists stay intact.
"""
return chunk_text(text, max_chars)

9
src/contract_check/core/config.py
View file

@ -139,6 +139,15 @@ class Settings(BaseSettings):
# --- analysis ---
chunk_size_chars: int = 10000
# --- prescreen ---
prescreen_enabled: bool = False
prescreen_confidence_threshold: float = 0.65
prescreen_high_value_threshold: float = 500_000.0
prescreen_auto_approve: bool = False
prescreen_llm_fallback_enabled: bool = True
prescreen_llm_fallback_threshold: float = 0.55
prescreen_llm_max_chars: int = 20_000
@property
def json_logs(self) -> bool:
"""JSON logs in staging/prod, pretty console in dev.

2
src/contract_check/core/db/__init__.py
View file

@ -3,3 +3,5 @@
See docs/ARCHITECTURE.md §7 for the full schema and rationale. status/queue/adapter
columns are TEXT + CHECK (not Postgres enums) so migrations stay additive.
"""
from __future__ import annotations

69
src/contract_check/core/extraction/__init__.py Normal file
View file

@ -0,0 +1,69 @@
"""Extraction layer: port + adapters + factory + the OCR-fallback orchestrator.
This is the structured successor to `core.analysis.extractor` (which stays for
the prototype CLI). New formats are added by writing an adapter and registering
it in the factory — no domain/pipeline changes.
"""
from __future__ import annotations
from ..analysis.ocr import OCRError as _OCRError # re-export for classify()
from .adapters.docx_mammoth import MammothDocxExtractor
from .adapters.ocr_tesseract import TesseractOcrExtractor
from .adapters.pdf_pymupdf import PyMuPDFExtractor
from .adapters.rtf_striprtf import RtfExtractor
from .adapters.txt_chardet import TxtExtractor
from .factory import SUPPORTED_SUFFIXES, ExtractorFactory, detect_format, get_factory
from .port import (
MIN_TEXT_CHARS,
DocumentExtractor,
ExtractedDocument,
ExtractionFailedError,
UnsupportedFormatError,
)
__all__ = [
"MIN_TEXT_CHARS",
"SUPPORTED_SUFFIXES",
"DocumentExtractor",
"ExtractedDocument",
"ExtractionFailedError",
"UnsupportedFormatError",
"ExtractorFactory",
"MammothDocxExtractor",
"PyMuPDFExtractor",
"RtfExtractor",
"TesseractOcrExtractor",
"TxtExtractor",
"detect_format",
"extract_document",
"get_factory",
]
# Re-exported under its canonical name (imported by worker_extract.handler.classify).
OCRError = _OCRError
def extract_document(data: bytes, *, mime: str = "", filename: str = "") -> ExtractedDocument:
"""High-level entry: pick adapter → extract → OCR fallback for scans.
Mirrors the historical extract-then-OCR flow, now format-aware:
1. Resolve the adapter (magic bytes > reported mime > filename suffix).
2. Extract. Images go straight to OCR (there is no text layer).
3. If a PDF yields too little text (likely a scan), retry once with the
Tesseract adapter; its result carries ``is_scan=True`` metadata.
"""
factory = get_factory()
fmt = detect_format(data, mime=mime, filename=filename)
extractor = factory.get_extractor(data, mime=mime, filename=filename)
try:
document = extractor.extract(data)
except ExtractionFailedError:
if fmt != "pdf":
raise
ocr = TesseractOcrExtractor()
document = ocr.extract(data)
document.metadata["format"] = "pdf"
document.metadata.setdefault("is_scan", True)
return document

17
src/contract_check/core/extraction/adapters/__init__.py Normal file
View file

@ -0,0 +1,17 @@
"""Format adapters for the extraction port. One module per library."""
from __future__ import annotations
from .docx_mammoth import MammothDocxExtractor
from .ocr_tesseract import TesseractOcrExtractor
from .pdf_pymupdf import PyMuPDFExtractor
from .rtf_striprtf import RtfExtractor
from .txt_chardet import TxtExtractor
__all__ = [
"MammothDocxExtractor",
"PyMuPDFExtractor",
"RtfExtractor",
"TesseractOcrExtractor",
"TxtExtractor",
]

49
src/contract_check/core/extraction/adapters/docx_mammoth.py Normal file
View file

@ -0,0 +1,49 @@
"""DOCX adapter: mammoth → Markdown (headings/lists/tables out of the box).
mammoth maps Word styles semantically (Heading 1 → `#`, List Bullet → `- `),
which is exactly what the chunker wants. Images become `![](...)` data-URI
noise for our use case; they are stripped because the LLM cannot see them.
"""
from __future__ import annotations
import re
from ...logging import get_logger
from ..port import MIN_TEXT_CHARS, ExtractedDocument, ExtractionFailedError
log = get_logger(__name__)
_IMAGE_RE = re.compile(r"!\[[^\]]*\]\([^)]*\)")
_STRUCTURED_RE = re.compile(r"^(#{1,6} .*|[-*+] |\d+\. |\|)", re.MULTILINE)
class MammothDocxExtractor:
"""Extract DOCX → structured Markdown. is_structured=True on md elements."""
def extract(self, data: bytes) -> ExtractedDocument:
import io
import mammoth
try:
with io.BytesIO(data) as fileobj:
result = mammoth.convert_to_markdown(fileobj)
except Exception as exc:
raise ExtractionFailedError(f"Не удалось открыть DOCX: {exc}") from exc
markdown = _IMAGE_RE.sub("", result.value).strip()
if len(markdown) < MIN_TEXT_CHARS:
raise ExtractionFailedError(
f"Извлечено слишком мало текста из DOCX ({len(markdown)} симв.)."
)
if result.messages:
log.debug("docx_mammoth_warnings", count=len(result.messages))
is_structured = bool(_STRUCTURED_RE.search(markdown))
log.info("docx_extracted", chars=len(markdown), structured=is_structured)
return ExtractedDocument(
markdown=markdown,
is_structured=is_structured,
metadata={"format": "docx", "has_tables": "|" in markdown},
)

105
src/contract_check/core/extraction/adapters/ocr_tesseract.py Normal file
View file

@ -0,0 +1,105 @@
"""Tesseract OCR adapter: scanned PDFs and raster images → plaintext.
Always is_structured=False (OCR recovers characters, not layout). Raises
`OCRError` (engine failure — retryable class `ocr_failed`) and
`ExtractionFailedError` (recognised but empty — user garbage class), matching
the semantics of core.analysis.ocr so the worker's classify() is unchanged.
"""
from __future__ import annotations
import io
from ...analysis.ocr import OCRError
from ...logging import get_logger
from ..port import MIN_TEXT_CHARS, ExtractedDocument, ExtractionFailedError
log = get_logger(__name__)
_PDF_MAGIC = b"%PDF"
class TesseractOcrExtractor:
"""OCR PDFs (rasterized per page) or PNG/JPG/TIFF via pytesseract."""
def __init__(self, *, lang: str = "rus+eng") -> None:
self._lang = lang
def extract(self, data: bytes) -> ExtractedDocument:
if data.lstrip()[:4].startswith(_PDF_MAGIC):
markdown, pages = self._ocr_pdf(data)
source = "pdf"
else:
markdown, pages = self._ocr_image(data)
source = "image"
stripped = markdown.strip()
if len(stripped) < MIN_TEXT_CHARS:
raise ExtractionFailedError(
f"OCR дал слишком мало текста ({len(stripped)} симв.). "
"Файл, видимо, не содержит распознаваемого текста."
)
log.info("ocr_extracted", source=source, pages=pages, chars=len(stripped))
return ExtractedDocument(
markdown=stripped,
is_structured=False,
metadata={"format": source, "pages": pages, "is_scan": True},
)
def _ocr_pdf(self, data: bytes) -> tuple[str, int]:
try:
import pymupdf
import pytesseract
from PIL import Image
except ImportError as exc:
raise OCRError(f"OCR backend not installed: {exc.name}") from exc
try:
doc = pymupdf.open(stream=data, filetype="pdf")
except Exception as exc:
raise OCRError(f"Не удалось открыть PDF для OCR: {exc}") from exc
parts: list[str] = []
pages = 0
try:
pages = doc.page_count
for index in range(pages):
pix = doc[index].get_pixmap(dpi=300)
img = Image.frombytes("RGB", (pix.width, pix.height), pix.samples)
parts.append(self._image_to_string(img))
except pytesseract.pytesseract.TesseractError as exc:
raise OCRError(f"Tesseract engine failed: {exc}") from exc
except OCRError:
raise
except Exception as exc:
raise OCRError(f"OCR failed: {exc}") from exc
finally:
doc.close()
return "\n".join(parts), pages
def _ocr_image(self, data: bytes) -> tuple[str, int]:
try:
import pytesseract
from PIL import Image, ImageSequence
except ImportError as exc:
raise OCRError(f"OCR backend not installed: {exc.name}") from exc
try:
img = Image.open(io.BytesIO(data))
frames = [frame.copy() for frame in ImageSequence.Iterator(img)]
except Exception as exc:
raise OCRError(f"Не удалось открыть изображение: {exc}") from exc
parts: list[str] = []
try:
for frame in frames:
parts.append(self._image_to_string(frame))
except pytesseract.pytesseract.TesseractError as exc:
raise OCRError(f"Tesseract engine failed: {exc}") from exc
return "\n".join(parts), len(frames)
def _image_to_string(self, img: object) -> str:
import pytesseract
text: str = pytesseract.image_to_string(img, lang=self._lang)
return text

106
src/contract_check/core/extraction/adapters/pdf_pymupdf.py Normal file
View file

@ -0,0 +1,106 @@
"""PDF adapter: pymupdf text + table detection → Markdown.
Per page: tables found by `page.find_tables()` are rendered as Markdown pipe
tables; regular text blocks whose bbox intersects a detected table are skipped
so cell text is not duplicated. Blocks and tables are interleaved by vertical
position to preserve reading order.
`find_tables()` needs ruling/heading lines, so borderless tables may fall back
to plain text — acceptable: the text is still present, only unpiped.
"""
from __future__ import annotations
from typing import Any
from ...logging import get_logger
from ..port import MIN_TEXT_CHARS, ExtractedDocument, ExtractionFailedError
log = get_logger(__name__)
# Tables below this shape are usually detection noise (stray rules), not data.
_MIN_TABLE_ROWS = 2
_MIN_TABLE_COLS = 2
class PyMuPDFExtractor:
"""Extract from digital (text-layer) PDFs. is_structured=True iff tables."""
def extract(self, data: bytes) -> ExtractedDocument:
import pymupdf
try:
doc = pymupdf.open(stream=data, filetype="pdf")
except Exception as exc: # pymupdf raises assorted types
raise ExtractionFailedError(f"Не удалось открыть PDF: {exc}") from exc
pages: list[str] = []
has_tables = False
page_count = 0
try:
page_count = doc.page_count
for index in range(page_count):
page_md, page_tables = _page_to_markdown(doc[index])
has_tables |= page_tables
if page_md.strip():
pages.append(page_md)
finally:
doc.close()
markdown = "\n\n".join(pages).strip()
if len(markdown) < MIN_TEXT_CHARS:
raise ExtractionFailedError(
f"Извлечено слишком мало текста ({len(markdown)} симв.). "
"Возможно, это скан — нужен OCR, либо файл повреждён."
)
log.info("pdf_extracted", pages=page_count, chars=len(markdown), tables=has_tables)
return ExtractedDocument(
markdown=markdown,
is_structured=has_tables,
metadata={"format": "pdf", "pages": page_count, "has_tables": has_tables},
)
def _page_to_markdown(page: Any) -> tuple[str, bool]:
"""One page → (markdown, has_tables). Tables replace the text inside them."""
import pymupdf
finder = page.find_tables()
table_items: list[tuple[float, str]] = []
table_boxes: list[Any] = []
for table in finder.tables:
rows = table.extract()
if len(rows) < _MIN_TABLE_ROWS or min(len(r) for r in rows) < _MIN_TABLE_COLS:
continue
table_items.append((table.bbox[1], _rows_to_markdown(rows)))
table_boxes.append(pymupdf.Rect(table.bbox))
text_items: list[tuple[float, str]] = []
for block in page.get_text("blocks"):
x0, y0, x1, y1, content, _block_no, block_type = block
if block_type != 0: # 0 = text block; skip images
continue
rect = pymupdf.Rect(x0, y0, x1, y1)
if any(rect.intersects(tb) for tb in table_boxes):
continue # cell text already rendered via the pipe table
text = content.strip()
if text:
text_items.append((y0, text))
items = sorted(table_items + text_items, key=lambda item: item[0])
return "\n\n".join(md for _y, md in items), bool(table_items)
def _rows_to_markdown(rows: list[list[str | None]]) -> str:
"""Table rows → Markdown pipes. First row is the header row."""
lines: list[str] = []
for i, row in enumerate(rows):
cells = [_escape_cell(cell) for cell in row]
lines.append(f"| {' | '.join(cells)} |")
if i == 0:
lines.append(f"|{'|'.join([' --- '] * len(cells))}|")
return "\n".join(lines)
def _escape_cell(cell: str | None) -> str:
return (cell or "").replace("\n", " ").replace("|", "\\|").strip()

44
src/contract_check/core/extraction/adapters/rtf_striprtf.py Normal file
View file

@ -0,0 +1,44 @@
"""RTF adapter: striprtf → plain text (no structure recovery).
RTF encodes structure in control words that striprtf discards; output is
plain paragraphs. is_structured stays False — the chunker falls back to
sentence splitting.
"""
from __future__ import annotations
from ...logging import get_logger
from ..port import MIN_TEXT_CHARS, ExtractedDocument, ExtractionFailedError
log = get_logger(__name__)
_RTF_MAGIC = b"{\\rtf"
class RtfExtractor:
"""Extract RTF → plaintext via striprtf (zero-dependency lib)."""
def extract(self, data: bytes) -> ExtractedDocument:
from striprtf.striprtf import rtf_to_text
if not data.lstrip()[:5].startswith(_RTF_MAGIC):
raise ExtractionFailedError("Файл не похож на RTF (нет заголовка {\\rtf).")
# RTF bodies are 7-bit ASCII with \uN escapes for other scripts;
# latin-1 never fails to decode and striprtf resolves the escapes.
try:
text = rtf_to_text(data.decode("latin-1"), errors="replace")
except Exception as exc:
raise ExtractionFailedError(f"Не удалось разобрать RTF: {exc}") from exc
stripped = text.strip()
if len(stripped) < MIN_TEXT_CHARS:
raise ExtractionFailedError(
f"Извлечено слишком мало текста из RTF ({len(stripped)} симв.)."
)
log.info("rtf_extracted", chars=len(stripped))
return ExtractedDocument(
markdown=stripped,
is_structured=False,
metadata={"format": "rtf"},
)

44
src/contract_check/core/extraction/adapters/txt_chardet.py Normal file
View file

@ -0,0 +1,44 @@
"""TXT/CSV adapter: chardet encoding detection → plaintext.
Legacy СНГ documents arrive as windows-1251 more often than UTF-8; decoding
with the wrong codec produces mojibake that the LLM cannot analyse. chardet
picks the codec, `errors="replace"` guards against misdetection.
"""
from __future__ import annotations
from ...logging import get_logger
from ..port import MIN_TEXT_CHARS, ExtractedDocument, ExtractionFailedError
log = get_logger(__name__)
class TxtExtractor:
"""Detect encoding, decode, return plaintext (never structured)."""
def extract(self, data: bytes) -> ExtractedDocument:
import chardet
detection = chardet.detect(data)
encoding = detection.get("encoding") or "utf-8"
confidence = float(detection.get("confidence") or 0.0)
encoding = encoding.lower() # chardet reports "Windows-1251" etc.
if encoding.lower() in ("ascii", "us-ascii"):
encoding = "utf-8" # ASCII is a UTF-8 subset; normalize
try:
text = data.decode(encoding, errors="replace")
except LookupError: # unknown codec name from chardet
text = data.decode("utf-8", errors="replace")
encoding = "utf-8"
stripped = text.strip()
if len(stripped) < MIN_TEXT_CHARS:
raise ExtractionFailedError(f"Извлечено слишком мало текста ({len(stripped)} симв.).")
log.info("txt_extracted", chars=len(stripped), encoding=encoding, confidence=confidence)
return ExtractedDocument(
markdown=stripped,
is_structured=False,
metadata={"format": "txt", "encoding": encoding},
)

145
src/contract_check/core/extraction/factory.py Normal file
View file

@ -0,0 +1,145 @@
"""ExtractorFactory: bytes (+optional mime/filename) → the right adapter.
Detection precedence (first hit wins):
1. Content sniffing via python-magic (when importable — the worker image
installs libmagic1). Catches mislabeled files, e.g. a PDF renamed .docx.
`application/zip` is deliberately NOT mapped: DOCX is a zip and the magic
cannot distinguish it from any other zip, so mime/suffix must decide.
2. The MIME type reported by the uploader.
3. The filename suffix.
Heavy adapters (marker, paddleocr, easyocr, img2table, email parsers) are
out of scope for now — the factory raises UnsupportedFormatError for their
formats and the pipeline refunds per policy.
"""
from __future__ import annotations
from pathlib import Path
from typing import Any
from .adapters.docx_mammoth import MammothDocxExtractor
from .adapters.ocr_tesseract import TesseractOcrExtractor
from .adapters.pdf_pymupdf import PyMuPDFExtractor
from .adapters.rtf_striprtf import RtfExtractor
from .adapters.txt_chardet import TxtExtractor
from .port import DocumentExtractor, UnsupportedFormatError
# Canonical upload gate (mirrored by the api and the bot adapter).
SUPPORTED_SUFFIXES = {
".pdf",
".docx",
".rtf",
".txt",
".csv",
".png",
".jpg",
".jpeg",
".tif",
".tiff",
}
# Canonical format ids → suffix(es). `image` covers every raster suffix.
_SUFFIX_TO_FORMAT = {
".pdf": "pdf",
".docx": "docx",
".rtf": "rtf",
".txt": "txt",
".csv": "txt",
".png": "image",
".jpg": "image",
".jpeg": "image",
".tif": "image",
".tiff": "image",
}
# Uploader-reported MIME → format. DOCX has many vendor spellings.
_MIME_TO_FORMAT = {
"application/pdf": "pdf",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
"application/msword": "rtf", # legacy .doc is rejected; mime may leak in for rtf
"application/rtf": "rtf",
"text/rtf": "rtf",
"text/plain": "txt",
"text/csv": "txt",
"image/png": "image",
"image/jpeg": "image",
"image/tiff": "image",
}
def detect_format(data: bytes, *, mime: str = "", filename: str = "") -> str:
"""Resolve a canonical format id (`pdf|docx|rtf|txt|image`) or raise."""
magic_mime = _sniff_mime(data)
for candidate in (magic_mime, mime.split(";")[0].strip().lower()):
if candidate:
fmt = _MIME_TO_FORMAT.get(candidate)
if fmt:
return fmt
suffix = Path(filename).suffix.lower()
fmt = _SUFFIX_TO_FORMAT.get(suffix)
if fmt:
return fmt
raise UnsupportedFormatError(
f"Неподдерживаемый формат (mime={mime!r}, suffix={suffix!r}). "
f"Поддерживаются: {', '.join(sorted(SUPPORTED_SUFFIXES))}."
)
def _sniff_mime(data: bytes) -> str:
"""Content-based MIME via python-magic; empty string when unavailable."""
try:
import magic
except ImportError:
return ""
try:
magic_module: Any = magic
detect = magic_module.detect_from_content(data)
return str(detect.mime_type)
except Exception:
return ""
class ExtractorFactory:
"""Builds/caches one adapter instance per format (all stateless)."""
def __init__(self) -> None:
self._extractors: dict[str, DocumentExtractor] = {}
def get_extractor(
self, data: bytes, *, mime: str = "", filename: str = ""
) -> DocumentExtractor:
return self._for_format(detect_format(data, mime=mime, filename=filename))
def _for_format(self, fmt: str) -> DocumentExtractor:
if fmt not in self._extractors:
self._extractors[fmt] = self._build(fmt)
return self._extractors[fmt]
@staticmethod
def _build(fmt: str) -> DocumentExtractor:
if fmt == "pdf":
return PyMuPDFExtractor()
if fmt == "docx":
return MammothDocxExtractor()
if fmt == "rtf":
return RtfExtractor()
if fmt == "txt":
return TxtExtractor()
if fmt == "image":
return TesseractOcrExtractor()
raise UnsupportedFormatError(f"Нет адаптера для формата {fmt!r}.")
_factory: ExtractorFactory | None = None
def get_factory() -> ExtractorFactory:
"""Process-wide singleton (adapters are stateless; lazy imports inside)."""
global _factory
if _factory is None:
_factory = ExtractorFactory()
return _factory

53
src/contract_check/core/extraction/port.py Normal file
View file

@ -0,0 +1,53 @@
"""Extraction port: one contract for every format adapter (docs/SPEC phase 1).
Adapters turn raw bytes into structured Markdown (`ExtractedDocument`), so the
LLM sees heading hierarchy and tables as pipes, and the chunker can split on
headings instead of arbitrary character counts. Heavy adapters (marker,
paddleocr, …) are deliberately deferred — see docs/ARCHITECTURE.md.
Deviation from the original spec: `attachments: list[bytes]` is omitted —
nothing consumes extracted images yet and storing them in MinIO is undesigned.
Add it when a consumer exists.
"""
from __future__ import annotations
from typing import Any, Protocol, runtime_checkable
from pydantic import BaseModel, Field
# Mirrors the historical "<100 chars ⇒ probably a scan / garbage" heuristic
# from core.analysis.extractor. Kept as the single source of truth.
MIN_TEXT_CHARS = 100
class UnsupportedFormatError(Exception):
"""No adapter can handle the input format (terminal, user garbage)."""
class ExtractionFailedError(Exception):
"""Extraction produced too little text or the file is corrupt (terminal).
For PDFs the orchestrator catches this and retries once with OCR before
letting it escape (see core/extraction/__init__.py).
"""
class ExtractedDocument(BaseModel):
"""Result of extraction. `metadata` keys are advisory per adapter.
Conventional keys: `format` (canonical format id, always set by the
factory-backed flow), `pages`, `has_tables`, `is_scan`, `encoding`,
`source_mime`.
"""
markdown: str
is_structured: bool = False
metadata: dict[str, Any] = Field(default_factory=dict)
@runtime_checkable
class DocumentExtractor(Protocol):
"""Port: bytes in, structured Markdown out."""
def extract(self, data: bytes) -> ExtractedDocument: ...

49
src/contract_check/core/llm/ollama_cloud.py
View file

@ -14,7 +14,7 @@ import json
import time
from dataclasses import dataclass
from types import TracebackType
from typing import Any, Self
from typing import Any, Self, cast
import httpx
from pydantic import BaseModel, ValidationError
@ -26,6 +26,7 @@ from ..analysis.report_schema import ReportPayload
from ..errors import TerminalError
from ..logging import get_logger
from .port import AnalysisResult
from .prescreen import PRESCREEN_MAX_CHARS_DEFAULT, PRESCREEN_SYSTEM, PrescreenExtraction
log = get_logger(__name__)
@ -77,8 +78,8 @@ class _QuotaSignal(Exception):
@dataclass(slots=True)
class _ChatResult:
data: BaseModel
class _ChatResult[DataT: BaseModel]:
data: DataT
model_used: str
fell_back: bool
repaired: bool
@ -103,8 +104,10 @@ class OllamaCloudProvider:
timeout: float = 120.0,
max_concurrency: int = 3,
chunk_size: int = 10000,
prescreen_max_chars: int = PRESCREEN_MAX_CHARS_DEFAULT,
) -> None:
self._chunk_size = chunk_size
self._prescreen_max_chars = prescreen_max_chars
self._model = model
self._fallback = fallback_model or None
self._temperature = temperature
@ -140,7 +143,7 @@ class OllamaCloudProvider:
chunks = chunk_text(text, max_chars=self._chunk_size)
system_msg = SYSTEM_PROMPT.format(checklist=checklist)
async def one(chunk: str) -> _ChatResult:
async def one(chunk: str) -> _ChatResult[ReportPayload]:
async with self._sem:
return await self._run_with_fallback(
messages=[
@ -158,8 +161,7 @@ class OllamaCloudProvider:
models_used: set[str] = set()
fell_back = repaired = False
for r in results:
payload: ReportPayload = r.data # type: ignore[assignment]
findings.extend(payload.findings)
findings.extend(r.data.findings)
prompt_tokens += r.prompt_tokens
eval_tokens += r.eval_tokens
latency = max(latency, r.latency_sec)
@ -179,12 +181,30 @@ class OllamaCloudProvider:
models_used=models_used,
)
async def extract_prescreen(
self, text: str, *, max_chars: int | None = None
) -> dict[str, object]:
"""Extract prescreen metadata from a contract text.
Returns a permissive dict validated loosely by PrescreenExtraction.
"""
max_chars = max_chars or self._prescreen_max_chars
truncated = text[:max_chars]
result = await self._run_with_fallback(
messages=[
{"role": "system", "content": PRESCREEN_SYSTEM},
{"role": "user", "content": truncated},
],
schema_model=PrescreenExtraction,
)
return result.data.model_dump() # type: ignore[return-type]
# ── internals (ported from prototype llm_client.py) ─────────────────────
async def _run_with_fallback(
async def _run_with_fallback[T: BaseModel](
self,
messages: list[dict[str, str]],
schema_model: type[BaseModel],
) -> _ChatResult:
schema_model: type[T],
) -> _ChatResult[T]:
schema = schema_model.model_json_schema()
try:
return await self._run_repair_loop(messages, schema_model, schema, self._model)
@ -194,13 +214,13 @@ class OllamaCloudProvider:
return await self._run_repair_loop(messages, schema_model, schema, self._fallback)
raise LLMQuotaError(f"429/quota on {self._model} and no usable fallback") from None
async def _run_repair_loop(
async def _run_repair_loop[T: BaseModel](
self,
messages: list[dict[str, str]],
schema_model: type[BaseModel],
schema_model: type[T],
schema: dict[str, Any],
model: str,
) -> _ChatResult:
) -> _ChatResult[T]:
started = time.perf_counter()
attempts = 0
repaired = False
@ -233,7 +253,9 @@ class OllamaCloudProvider:
]
continue
raise LLMError(f"Model {model} returned invalid JSON after repair: {exc}") from exc
return _ChatResult(
return cast(
_ChatResult[T],
_ChatResult(
data=data,
model_used=model,
fell_back=(model != self._model),
@ -242,6 +264,7 @@ class OllamaCloudProvider:
prompt_tokens=usage[0],
eval_tokens=usage[1],
latency_sec=time.perf_counter() - started,
),
)
async def _post_chat(

4
src/contract_check/core/llm/port.py
View file

@ -31,4 +31,8 @@ class AnalysisResult:
class LLMProvider(Protocol):
async def analyze(self, text: str, *, checklist: str) -> AnalysisResult: ...
async def extract_prescreen(
self, text: str, *, max_chars: int | None = None
) -> dict[str, object]: ...
async def aclose(self) -> None: ...

52
src/contract_check/core/llm/prescreen.py Normal file
View file

@ -0,0 +1,52 @@
"""Prescreen-extraction prompt + JSON schema model shared by LLM providers.
Layering: this module must NOT import `worker_prescreen` — providers return a
plain dict; the worker-side wrapper (`worker_prescreen/extractor_llm.py`)
validates enums/dates/amounts against `PrescreenContractMeta` (plan §3.3).
"""
from __future__ import annotations
from pydantic import BaseModel, ConfigDict
PRESCREEN_SYSTEM = (
"Ты — точный извлекатель метаданных из договоров (право РФ/РБ). "
"Из приведённого текста договора извлеки строго следующие поля:\n"
"- contract_type: одна из: supply | services | contract_work | lease | nda\n"
"- party_a, party_b: точные наименования сторон как в тексте\n"
"- total_amount: сумма договора числом (без пробелов и разделителей)\n"
"- currency: код валюты: RUB | BYN | USD | EUR\n"
"- start_date, end_date: даты в формате ISO YYYY-MM-DD\n"
"- has_penalty_clause, has_termination_clause, has_arbitration: true | false\n"
"Жёсткие правила:\n"
"1. Если поле отсутствует в тексте — верни для него null.\n"
"2. Не придумывай значения и не цитируй текст.\n"
"3. Верни ТОЛЬКО JSON по схеме, без markdown и пояснений."
)
# Default cap on characters sent to the fallback (meta lives in the header of
# templated contracts; cost control). Wired from settings.prescreen_llm_max_chars.
PRESCREEN_MAX_CHARS_DEFAULT = 20_000
class PrescreenExtraction(BaseModel):
"""Permissive wire model: worker-side validation whitelists enums.
extra="ignore" rejects hallucinated fields at the provider boundary.
"""
model_config = ConfigDict(extra="ignore")
contract_type: str | None = None
party_a: str | None = None
party_b: str | None = None
total_amount: float | None = None
currency: str | None = None
start_date: str | None = None
end_date: str | None = None
has_penalty_clause: bool | None = None
has_termination_clause: bool | None = None
has_arbitration: bool | None = None
__all__ = ["PRESCREEN_MAX_CHARS_DEFAULT", "PRESCREEN_SYSTEM", "PrescreenExtraction"]

568
src/contract_check/core/llm/yandex_gpt.py Normal file
View file

@ -0,0 +1,568 @@
"""YandexGPT adapter — LLMProvider for Yandex Cloud Foundation Models.
Implements the same `analyze(text, *, checklist)` contract as
`OllamaCloudProvider`: chunk fan-out, de-duplication, severity sort, JSON
schema repair loop, and usage/latency aggregation.
YandexGPT API reference:
- Base: `https://llm.api.cloud.yandex.net/foundationModels/v1/completion`
- IAM or API-key auth in `Authorization: Api-Key <key>` or `Bearer <iam>`.
- `responseFormat.type = "JSON_OBJECT"` with a JSON schema in `json_schema`.
- `messages` are role+text; system text is sent as the first `system` message.
- Usage is returned in `usage.{inputTextTokens,completionTokens,totalTokens}`.
"""
from __future__ import annotations
import asyncio
import json
import time
from dataclasses import dataclass
from types import TracebackType
from typing import Any, Self, cast
import httpx
from pydantic import BaseModel, ValidationError
from ..analysis.analyzer import build_user_prompt, dedupe_findings, sort_findings
from ..analysis.checklist import checklist_for_prompt
from ..analysis.chunker import chunk_markdown
from ..analysis.report_schema import ReportPayload
from ..errors import TerminalError
from ..logging import get_logger
from .port import AnalysisResult
from .prescreen import PRESCREEN_MAX_CHARS_DEFAULT, PRESCREEN_SYSTEM, PrescreenExtraction
log = get_logger(__name__)
_LOG_MAX_PAYLOAD_CHARS = 1200
YANDEX_LLM_BASE_URL = "https://llm.api.cloud.yandex.net"
YANDEX_COMPLETION_PATH = "/foundationModels/v1/completion"
REPAIR_SYSTEM = (
"Твой предыдущий ответ не прошёл валидацию JSON-схемы. "
"Верни СТРОГО валидный JSON, соответствующий схеме, без пояснений и markdown. "
"Цитаты — дословные из исходного текста договора."
)
SYSTEM_PROMPT = (
"Ты — помощник первичного юридического скрининга договоров по праву "
"РФ / РБ (ГК РФ, ГК РБ). Твоя задача — найти РИСКИ для стороны, которая "
"обращается за анализом (считай её «нашим клиентом»), по чек-листу ниже.\n\n"
"Жёсткие правила:\n"
"1. Анализируй ТОЛЬКО текст договора в этом сообщении. Не придумывай пункты.\n"
"2. На каждую находку ОБЯЗАТЕЛЬНО: дословная цитата из оригинала (поле quote) "
"и номер/название пункта договора (поле section_ref), чтобы сверить одним кликом.\n"
"3. Если по какому-то пункту чек-листа риска нет — НЕ включай его в findings.\n"
"4. severity: high — существенный риск (штрафы, потеря прав, чужая подсудность); "
"medium — стоит уточнить; low — мелочь на заметку.\n"
"5. Возвращай ТОЛЬКО JSON по схеме, без markdown и пояснений вне JSON.\n\n"
"Схема ответа (строго соблюдай имена полей):\n"
"{{\n"
' "findings": [\n'
" {{\n"
' "checklist_id": "<id пункта чек-листа: penalties, jurisdiction, payment_terms и т.д.>",\n'
' "severity": "high | medium | low",\n'
' "quote": "<дословная цитата из договора>",\n'
' "section_ref": "<номер/название пункта договора>",\n'
' "risk": "<в чём конкретно риск для клиента (не путать с checklist_id)>",\n'
' "recommendation": "<что предложить изменить>"\n'
" }}\n"
" ]\n"
"}}\n\n"
"Важно: поле checklist_id должно содержать только id из чек-листа в квадратных скобках, "
"например 'penalties' или 'jurisdiction'. Не пиши туда номер пункта договора — "
"номер пункта идёт в section_ref. Не используй поле risk для id пункта чек-листа.\n\n"
"Чек-лист пунктов анализа:\n{checklist}"
)
class LLMError(Exception):
"""Unrecoverable LLM failure (after all retries)."""
class LLMQuotaError(LLMError):
"""Quota/rate-limit on both primary and fallback — refundable failure."""
class LLMUnavailableError(LLMError):
"""Yandex API unreachable or returns non-200 status — retryable."""
class LLMConfigError(LLMError, TerminalError):
"""Misconfigured folder ID / API key / endpoint — terminal, do not retry."""
class _QuotaSignal(Exception):
"""Internal: 429 triggers fallback within the same call."""
type _ChatResultData = ReportPayload | PrescreenExtraction
@dataclass(slots=True)
class _ChatResult[DataT: BaseModel]:
data: DataT
model_used: str
fell_back: bool
repaired: bool
attempts: int
prompt_tokens: int
eval_tokens: int
latency_sec: float
class YandexGPTProvider:
"""LLMProvider backed by Yandex Cloud Foundation Models (YandexGPT)."""
def __init__(
self,
*,
api_key: str,
folder_id: str,
model: str = "yandexgpt-lite",
fallback_model: str | None = None,
base_url: str = YANDEX_LLM_BASE_URL,
completion_path: str = YANDEX_COMPLETION_PATH,
temperature: float = 0.2,
max_tokens: int = 3072,
timeout: float = 120.0,
max_concurrency: int = 3,
chunk_size: int = 10000,
prescreen_max_chars: int = PRESCREEN_MAX_CHARS_DEFAULT,
) -> None:
stripped_key = (api_key or "").strip()
if not stripped_key:
raise LLMConfigError(
"YandexGPT API key is empty. "
"Set YANDEXGPT_API_KEY (or the api_key argument) and restart the service."
)
if not (folder_id or "").strip():
raise LLMConfigError(
"YandexGPT folder ID is empty. "
"Set YANDEXGPT_FOLDER_ID (or the folder_id argument) and restart the service."
)
self._api_key = stripped_key
self._folder_id = folder_id.strip()
self._model = model
self._fallback = fallback_model or None
self._base_url = base_url.rstrip("/")
self._completion_path = completion_path
self._temperature = temperature
self._max_tokens = max_tokens
self._retries = 3
self._chunk_size = chunk_size
self._prescreen_max_chars = prescreen_max_chars
self._sem = asyncio.Semaphore(max(1, max_concurrency))
self._client = httpx.AsyncClient(
base_url=self._base_url,
headers={
"Authorization": f"Api-Key {api_key}",
"Content-Type": "application/json",
"x-folder-id": folder_id,
},
timeout=httpx.Timeout(timeout, connect=10.0),
)
async def aclose(self) -> None:
await self._client.aclose()
async def __aenter__(self) -> Self:
return self
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc: BaseException | None,
tb: TracebackType | None,
) -> None:
await self.aclose()
# ── public: LLMProvider.analyze ─────────────────────────────────────────
async def analyze(
self, text: str, *, checklist: str | None = None, extra_context: str = ""
) -> AnalysisResult:
checklist = checklist or checklist_for_prompt()
chunks = chunk_markdown(text, max_chars=self._chunk_size)
system_msg = SYSTEM_PROMPT.format(checklist=checklist)
def user_prompt(chunk: str) -> str:
prompt = build_user_prompt(chunk)
if extra_context:
return prompt + "\n\n" + extra_context
return prompt
async def one(chunk: str) -> _ChatResult:
async with self._sem:
return await self._run_with_fallback(
messages=[
{"role": "system", "text": system_msg},
{"role": "user", "text": user_prompt(chunk)},
],
schema_model=ReportPayload,
)
results = await asyncio.gather(*(one(c) for c in chunks))
findings: list[Any] = []
prompt_tokens = eval_tokens = 0
latency = 0.0
models_used: set[str] = set()
fell_back = repaired = False
for r in results:
findings.extend(r.data.findings)
prompt_tokens += r.prompt_tokens
eval_tokens += r.eval_tokens
latency = max(latency, r.latency_sec)
models_used.add(r.model_used)
fell_back |= r.fell_back
repaired |= r.repaired
findings = sort_findings(dedupe_findings(findings))
return AnalysisResult(
findings=findings,
model_used=", ".join(sorted(models_used)),
fell_back=fell_back,
repaired=repaired,
prompt_tokens=prompt_tokens,
eval_tokens=eval_tokens,
latency_sec=latency,
models_used=models_used,
)
# ── public: LLMProvider.extract_prescreen ────────────────────────────────
async def extract_prescreen(self, text: str, *, max_chars: int | None = None) -> dict[str, Any]:
"""Stage-2 prescreen fallback: JSON-schema-constrained field extraction.
Single call over the first `prescreen_max_chars` chars; reuses the
repair loop + quota fallback machinery.
"""
limit = max_chars or self._prescreen_max_chars
result = await self._run_with_fallback(
messages=[
{"role": "system", "text": PRESCREEN_SYSTEM},
{"role": "user", "text": text[:limit]},
],
schema_model=PrescreenExtraction,
)
return result.data.model_dump() # type: ignore[return-type]
# ── internals ────────────────────────────────────────────────────────────
async def _run_with_fallback[T: BaseModel](
self,
messages: list[dict[str, str]],
schema_model: type[T],
) -> _ChatResult[T]:
schema = schema_model.model_json_schema()
try:
return await self._run_repair_loop(messages, schema_model, schema, self._model)
except _QuotaSignal:
if self._fallback and self._model != self._fallback:
log.warning("quota_429_fallback", primary=self._model, fallback=self._fallback)
return await self._run_repair_loop(messages, schema_model, schema, self._fallback)
raise LLMQuotaError(f"429/quota on {self._model} and no usable fallback") from None
async def _run_repair_loop[T: BaseModel](
self,
messages: list[dict[str, str]],
schema_model: type[T],
schema: dict[str, Any],
model: str,
) -> _ChatResult[T]:
started = time.perf_counter()
attempts = 0
repaired = False
current = list(messages)
while True:
attempts += 1
content, usage = await self._post_completion(current, model, schema)
cleaned = _strip_markdown_fences(content)
try:
data = _parse_and_validate(cleaned, schema_model)
except (ValidationError, ValueError) as exc:
error_summary = _format_parse_error(exc)
log.warning(
"invalid_json_repair",
model=model,
error=type(exc).__name__,
attempt=attempts,
content_preview=cleaned[:_LOG_MAX_PAYLOAD_CHARS],
content_type=type(content).__name__,
content_len=len(content),
**error_summary,
)
if attempts <= 1:
repaired = True
current = [
{"role": "system", "text": REPAIR_SYSTEM},
{"role": "user", "text": messages[-1]["text"]},
{"role": "assistant", "text": cleaned[:4000]},
{"role": "user", "text": "Верни только валидный JSON по схеме."},
]
continue
raise LLMError(f"Model {model} returned invalid JSON after repair: {exc}") from exc
return cast(
_ChatResult[T],
_ChatResult(
data=data,
model_used=model,
fell_back=(model != self._model),
repaired=repaired,
attempts=attempts,
prompt_tokens=usage[0],
eval_tokens=usage[1],
latency_sec=time.perf_counter() - started,
),
)
def _build_payload(
self,
messages: list[dict[str, str]],
model: str,
schema: dict[str, Any],
) -> dict[str, Any]:
return {
"modelUri": f"gpt://{self._folder_id}/{model}",
"completionOptions": {
"stream": False,
"temperature": self._temperature,
"maxTokens": self._max_tokens,
"responseFormat": {
"type": "JSON_OBJECT",
"json_schema": {"schema": schema},
},
},
"messages": messages,
}
async def _post_completion(
self,
messages: list[dict[str, str]],
model: str,
schema: dict[str, Any],
) -> tuple[str, tuple[int, int]]:
payload = self._build_payload(messages, model, schema)
log.debug(
"yandexgpt_request",
model=model,
folder_id=self._folder_id,
messages=_safe_messages(messages, _LOG_MAX_PAYLOAD_CHARS),
schema=_safe_schema(schema),
)
last_exc: Exception | None = None
url = self._completion_path
for attempt in range(1, self._retries + 1):
try:
resp = await self._client.post(url, json=payload)
except httpx.ConnectError as exc:
raise LLMConfigError(
f"Cannot connect to Yandex API at configured host: {exc}. "
"Verify YANDEXGPT_BASE_URL or network reachability."
) from exc
except httpx.TimeoutException as exc:
last_exc = exc
log.warning(
"yandexgpt_timeout",
model=model,
attempt=attempt,
error=f"{type(exc).__name__}: {exc}",
request_messages=_safe_messages(messages, _LOG_MAX_PAYLOAD_CHARS),
)
if attempt < self._retries:
await _backoff(attempt)
continue
raise LLMUnavailableError(
f"YandexGPT request timed out after {self._retries} attempts: {exc}"
) from exc
except httpx.HTTPError as exc:
last_exc = exc
log.warning(
"yandexgpt_http_error",
model=model,
attempt=attempt,
error=f"{type(exc).__name__}: {exc}",
request_messages=_safe_messages(messages, _LOG_MAX_PAYLOAD_CHARS),
)
if attempt < self._retries:
await _backoff(attempt)
continue
raise LLMUnavailableError(
f"HTTP error calling YandexGPT after {self._retries} attempts: {exc}"
) from exc
if resp.status_code == 429:
raise _QuotaSignal(f"429 Too Many Requests (model={model})")
if resp.status_code == 401:
raise LLMConfigError(
f"YandexGPT authentication failed (401): {resp.text[:500]}. "
"Verify YANDEXGPT_API_KEY is valid and YANDEXGPT_FOLDER_ID is correct."
)
if resp.status_code == 403:
raise LLMConfigError(
f"YandexGPT authorization failed (403): {resp.text[:500]}. "
"Verify the service account/API key has the ai.languageModels.user role."
)
if resp.status_code == 400:
raise LLMConfigError(
f"YandexGPT bad request (400): {resp.text[:500]}. "
"Verify YANDEXGPT_MODEL name and folder ID."
)
if resp.status_code == 404:
raise LLMConfigError(
f"YandexGPT endpoint not found (404): {resp.text[:500]}. "
"Verify YANDEXGPT_BASE_URL / YANDEXGPT_COMPLETION_PATH."
)
if resp.status_code >= 500:
if attempt < self._retries:
log.warning(
"http_5xx_retry",
status=resp.status_code,
model=model,
attempt=attempt,
response_text=resp.text[:500],
)
await _backoff(attempt)
continue
raise LLMUnavailableError(
f"HTTP {resp.status_code} from YandexGPT: {resp.text[:500]}"
)
if resp.status_code >= 400:
raise LLMError(f"HTTP {resp.status_code} from YandexGPT: {resp.text[:500]}")
try:
body = resp.json()
except Exception as exc:
log.error(
"yandexgpt_response_not_json",
model=model,
response_text=resp.text[:_LOG_MAX_PAYLOAD_CHARS],
error=f"{type(exc).__name__}: {exc}",
)
raise LLMError(f"Invalid JSON response from YandexGPT: {exc}") from exc
result = body.get("result", {})
alternatives = result.get("alternatives", [])
if not alternatives:
log.error(
"yandexgpt_empty_alternatives",
model=model,
body=_safe_body(body, _LOG_MAX_PAYLOAD_CHARS),
)
raise LLMError("Empty alternatives list from YandexGPT")
raw_content = alternatives[0].get("message", {}).get("text", "")
content = _normalize_content(raw_content).strip()
usage = result.get("usage", {})
prompt_tokens = int(usage.get("inputTextTokens", 0) or 0)
eval_tokens = int(usage.get("completionTokens", 0) or 0)
if not content:
log.error(
"yandexgpt_empty_content",
model=model,
raw_content_type=type(raw_content).__name__,
body=_safe_body(body, _LOG_MAX_PAYLOAD_CHARS),
)
raise LLMError("Empty message content from YandexGPT")
log.debug(
"yandexgpt_response",
model=model,
content=content[:_LOG_MAX_PAYLOAD_CHARS],
prompt_tokens=prompt_tokens,
eval_tokens=eval_tokens,
)
return content, (prompt_tokens, eval_tokens)
raise LLMUnavailableError(f"Exhausted retries: {last_exc}")
def _parse_and_validate(content: str, schema_model: type[BaseModel]) -> BaseModel:
data = json.loads(content)
return schema_model.model_validate(data)
def _safe_messages(messages: list[dict[str, str]], max_chars: int) -> list[dict[str, str]]:
"""Return a copy of messages with long contents truncated for logging."""
out: list[dict[str, str]] = []
for msg in messages:
text = msg.get("text", "")
out.append(
{
"role": msg.get("role", "unknown"),
"text": text[:max_chars] + ("..." if len(text) > max_chars else ""),
}
)
return out
def _safe_schema(schema: dict[str, Any]) -> dict[str, Any]:
"""Return a schema safe for logs: title/properties preserved, no full examples."""
safe: dict[str, Any] = {
k: v for k, v in schema.items() if k not in ("examples", "$defs", "definitions")
}
properties = safe.get("properties")
if isinstance(properties, dict) and len(properties) > 30:
safe["properties"] = {k: properties[k] for k in list(properties.keys())[:30]}
safe["_properties_truncated"] = True
return safe
def _safe_body(body: dict[str, Any], max_chars: int) -> dict[str, Any]:
"""Return a copy of the YandexGPT response body with long strings truncated."""
safe: dict[str, Any] = {}
for k, v in body.items():
if isinstance(v, bytes):
safe[k] = _normalize_content(v)[:max_chars]
elif isinstance(v, str):
safe[k] = v[:max_chars] + ("..." if len(v) > max_chars else "")
elif isinstance(v, dict):
safe[k] = _safe_body(v, max_chars)
else:
safe[k] = v
return safe
def _normalize_content(value: object) -> str:
"""Turn whatever the model put in message.text into a clean str."""
if isinstance(value, str):
return value
if isinstance(value, bytes):
try:
return value.decode("utf-8")
except UnicodeDecodeError:
return value.decode("utf-8", errors="replace")
if value is None:
return ""
return str(value)
def _strip_markdown_fences(text: str) -> str:
"""Remove ```json ... ``` or ``` ... ``` wrappers that models often emit."""
stripped = text.strip()
if stripped.startswith("```"):
first_newline = stripped.find("\n")
if first_newline != -1:
stripped = stripped[first_newline + 1 :]
else:
stripped = stripped[3:]
stripped = stripped.rstrip()
if stripped.endswith("```"):
stripped = stripped[:-3].rstrip()
return stripped
def _format_parse_error(exc: BaseException) -> dict[str, object]:
"""Return a small, log-safe summary of a JSON/Pydantic parse failure."""
if isinstance(exc, ValidationError):
return {
"validation_errors": [
{"loc": list(e.get("loc", [])), "msg": e.get("msg", ""), "type": e.get("type", "")}
for e in exc.errors()
],
}
return {"parse_error": str(exc)[:500]}
async def _backoff(attempt: int) -> None:
await asyncio.sleep(min(8.0, 0.5 * (2 ** (attempt - 1))))

144
src/contract_check/core/logging.py
View file

@ -11,6 +11,14 @@ worker-analyze → DB under one id.
Additional global context (service, env, version) is bound at startup and
inherited by every logger. Per-request/job attributes can be added with
`bind_context(**extra)`.
Conventions for service authors:
- event names are snake_case.
- log at the start of every significant stage with the relevant ids
(document_id, user_id, s3_key) so a stuck job can be traced.
- log success once per handled message with decision/outcome metrics.
- log failures with exc_info=True so full traceback is captured.
- use `log_error()` for structured exception capture.
"""
from __future__ import annotations
@ -19,6 +27,7 @@ import contextvars
import logging
import os
import sys
import time
import traceback
import uuid
@ -229,3 +238,138 @@ def log_error(
if exc is not None:
kwargs["exc_info"] = (type(exc), exc, exc.__traceback__)
logger.error(event, **kwargs)
class Timer:
"""Context manager that records elapsed wall time for an operation.
Logs start at entry, success at exit, and failure when an exception is raised.
"""
def __init__(self, logger: BoundLogger, operation: str, **kwargs: object) -> None:
self.logger = logger
self.operation = operation
self.kwargs = kwargs
self.start_time: float = 0.0
self.end_time: float = 0.0
def __enter__(self) -> Timer:
self.start_time = time.perf_counter()
self.logger.debug(f"{self.operation}_started", **self.kwargs)
return self
def __exit__(self, exc_type, exc_val, exc_tb) -> None: # type: ignore[no-untyped-def]
self.end_time = time.perf_counter()
duration_ms = (self.end_time - self.start_time) * 1000
if exc_type is None:
self.logger.info(
f"{self.operation}_complete",
duration_ms=round(duration_ms, 2),
**self.kwargs,
)
else:
self.logger.error(
f"{self.operation}_failed",
duration_ms=round(duration_ms, 2),
exception_type=exc_type.__name__ if exc_type else None,
**self.kwargs,
)
def log_db_operation(
logger: BoundLogger,
operation: str,
table: str,
**kwargs: object,
) -> None:
"""Structured DB operation log."""
logger.info(f"db_{operation}_{table}", **kwargs)
def log_external_call(
logger: BoundLogger,
service: str,
operation: str,
**kwargs: object,
) -> None:
"""Structured external service call log."""
logger.info(
f"external_call_{service}_{operation}",
**kwargs,
)
def log_message_processing(
logger: BoundLogger,
state: str,
queue: str,
correlation_id: str,
**kwargs: object,
) -> None:
"""Log that a worker is processing a broker message."""
logger.info(
"message_processing",
state=state,
queue=queue,
correlation_id=correlation_id,
**kwargs,
)
def log_performance_metric(
logger: BoundLogger,
metric: str,
value: float,
unit: str,
**kwargs: object,
) -> None:
"""Log a numeric performance/funnel metric."""
logger.info(
"performance_metric",
metric=metric,
value=round(value, 4),
unit=unit,
**kwargs,
)
def log_stage_complete(
logger: BoundLogger,
stage: str,
duration_ms: float,
**kwargs: object,
) -> None:
"""Log completion of a named processing stage."""
logger.info(
f"stage_{stage}_complete",
duration_ms=round(duration_ms, 2),
**kwargs,
)
def log_stage_failure(
logger: BoundLogger,
stage: str,
exc: BaseException,
**kwargs: object,
) -> None:
"""Log failure of a named processing stage."""
logger.error(
f"stage_{stage}_failure",
error=f"{type(exc).__name__}: {exc}",
**kwargs,
)
def log_stage_progress(
logger: BoundLogger,
stage: str,
label: str,
**kwargs: object,
) -> None:
"""Log progress of a long-running named processing stage."""
logger.info(
f"stage_{stage}_progress",
label=label,
**kwargs,
)

25
src/contract_check/core/metrics.py
View file

@ -77,6 +77,31 @@ notify_duration = Histogram(
"contract_check_notify_duration_seconds",
"Notification send (notify.q handler) latency.",
)
prescreen_duration = Histogram(
"contract_check_prescreen_duration_seconds",
"Document prescreen (prescreen.q handler) latency.",
["decision"],
)
prescreen_runs = Counter(
"contract_check_prescreen_runs_total",
"Prescreen executions by outcome.",
["decision", "contract_type"],
)
prescreen_confidence = Histogram(
"contract_check_prescreen_confidence_distribution",
"Distribution of prescreen extraction confidence scores.",
buckets=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
)
prescreen_extraction_stage = Histogram(
"contract_check_prescreen_extraction_stage_seconds",
"Prescreen extraction stage latency.",
["stage"], # heuristic | llm
)
prescreen_fallback_runs = Counter(
"contract_check_prescreen_fallback_runs_total",
"Times the hybrid prescreen fell back to LLM stage.",
["outcome"], # disabled | skipped | used | failed
)
def start_metrics_server(port: int) -> None:

2
src/contract_check/core/mq/__init__.py
View file

@ -4,3 +4,5 @@ See docs/ARCHITECTURE.md §5 for the full topology spec (direct exchange, pipeli
fan-out extract.q → analyze.q, quorum main queues, classic TTL retry queues,
quorum DLQs). All declarations are idempotent and run on every service start.
"""
from __future__ import annotations

11
src/contract_check/core/mq/consumer.py
View file

@ -19,10 +19,10 @@ failure_class="infra" (poison message). See docs/ARCHITECTURE.md §5.
from __future__ import annotations
import asyncio
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, cast
from aio_pika import DeliveryMode, Message, connect_robust
from pydantic import BaseModel, ValidationError
from pydantic import ValidationError
from ..db.enums import FailureClass
from ..errors import TerminalError
@ -66,7 +66,7 @@ class Consumer[MsgT: RetryableMessage]:
#: subclass declares which queue + routing key + message model it owns
queue: str = ""
routing_key: str = ""
message_model: type[BaseModel] = PipelineMessage
message_model: type[RetryableMessage] = PipelineMessage
#: retry exchange used when nacking (defaults to the contracts retry exchange;
#: notify pipeline overrides with EXCHANGE_NOTIFY_RETRY).
retry_exchange: str = EXCHANGE_RETRY
@ -221,7 +221,7 @@ class Consumer[MsgT: RetryableMessage]:
def _parse(self, body: bytes) -> MsgT | None:
try:
return self.message_model.model_validate_json(body) # type: ignore[return-value]
return cast(MsgT, self.message_model.model_validate_json(body))
except ValidationError as exc:
log.error("message_parse_failed", error=str(exc), body=body[:500])
return None
@ -232,7 +232,8 @@ class Consumer[MsgT: RetryableMessage]:
assert self._channel is not None
retry_rk = f"retry.{self.routing_key}"
expiration_ms = int(self._retry_base_ms * (2 ** (new_attempt - 1)))
body = payload.next_attempt().model_dump_json().encode("utf-8")
retryable = cast(RetryableMessage, payload)
body = cast(MsgT, retryable.next_attempt()).model_dump_json().encode("utf-8")
headers: dict[str, Any] = {
H_CORRELATION_ID: str(payload.correlation_id),
H_ATTEMPT: new_attempt,

57
src/contract_check/core/mq/messages.py
View file

@ -49,6 +49,63 @@ class DocumentExtracted(PipelineMessage):
ocr_used: bool
class PrescreenRequested(PipelineMessage):
"""worker-extract → contracts.x[prescreen] → worker-prescreen.
`text_s3_key` points at the extracted plaintext that should be
prescreened for contract metadata before deep analysis.
"""
text_s3_key: str
filename: str
char_count: int = 0
is_structured: bool = False
has_tables: bool = False
class AnalyzeRequested(PipelineMessage):
"""worker-prescreen → contracts.x[analyze] → worker-analyze.
Inherits the same shape as DocumentExtracted but carries additional
structured metadata discovered during prescreening.
"""
extracted_s3_key: str
filename: str
char_count: int
ocr_used: bool
is_structured: bool = False
has_tables: bool = False
prescreen_meta: dict | None = None
class PrescreenCompleted(PipelineMessage):
"""worker-prescreen outcome message.
Carries the extracted metadata, routing decision, and any auto-generated
summary/findings. Published to the next stage (deep_analysis,
manual_review, or auto_approve).
"""
text_s3_key: str
filename: str
prescreened_at: str
contract_type: str | None = None
party_a: str | None = None
party_b: str | None = None
total_amount: float | None = None
currency: str | None = None
start_date: str | None = None
end_date: str | None = None
has_penalty_clause: bool | None = None
has_termination_clause: bool | None = None
has_arbitration: bool | None = None
confidence_score: float = Field(default=0.0, ge=0.0, le=1.0)
routing_decision: Literal["auto_approve", "manual_review", "deep_analysis"]
auto_summary: str | None = None
auto_findings: list[dict] = Field(default_factory=list)
# ── notifications ──────────────────────────────────────────────────────────────
# The notify pipeline is decoupled from the contracts pipeline: it shares the
# retry/DLQ mechanics but lives on its own exchange/queue (see topology.py).

4
src/contract_check/core/mq/topology.py
View file

@ -42,10 +42,13 @@ EXCHANGE_NOTIFY_RETRY = "notify.retry.x"
# ── queues ───────────────────────────────────────────────────────────────────
QUEUE_EXTRACT = "extract.q"
QUEUE_ANALYZE = "analyze.q"
QUEUE_PRESCREEN = "prescreen.q"
QUEUE_EXTRACT_RETRY = "extract.retry.q"
QUEUE_ANALYZE_RETRY = "analyze.retry.q"
QUEUE_PRESCREEN_RETRY = "prescreen.retry.q"
QUEUE_EXTRACT_DLQ = "extract.dlq"
QUEUE_ANALYZE_DLQ = "analyze.dlq"
QUEUE_PRESCREEN_DLQ = "prescreen.dlq"
QUEUE_NOTIFY = "notify.q"
QUEUE_NOTIFY_RETRY = "notify.retry.q"
QUEUE_NOTIFY_DLQ = "notify.dlq"
@ -53,6 +56,7 @@ QUEUE_NOTIFY_DLQ = "notify.dlq"
# ── routing keys ─────────────────────────────────────────────────────────────
RK_EXTRACT = "extract"
RK_ANALYZE = "analyze"
RK_PRESCREEN = "prescreen"
RK_RETRY_EXTRACT = "retry.extract"
RK_RETRY_ANALYZE = "retry.analyze"
RK_NOTIFY = "notify"

4
src/contract_check/core/s3/minio_storage.py
View file

@ -19,6 +19,10 @@ from .port import ObjectInfo
log = get_logger(__name__)
class MinioStorageError(Exception):
"""Storage operation failed in MinIO."""
class MinioStorage:
"""Async-friendly MinIO storage adapter."""

3
src/contract_check/core/tokens.py
View file

@ -11,6 +11,7 @@ from __future__ import annotations
import hashlib
import hmac
import secrets
from typing import cast
from .db.enums import ADAPTER_NAMES, AdapterName
@ -38,4 +39,4 @@ def is_valid_adapter(adapter: str) -> bool:
def assert_adapter(adapter: str) -> AdapterName:
if not is_valid_adapter(adapter):
raise ValueError(f"invalid adapter {adapter!r}; expected one of {ADAPTER_NAMES}")
return adapter # type: ignore[return-value]
return cast(AdapterName, adapter)

2
src/contract_check/prototype/__main__.py
View file

@ -1,5 +1,7 @@
"""Stage-0 prototype entrypoint: `python -m contract_check.prototype <file>`."""
from __future__ import annotations
from . import main
if __name__ == "__main__":

45
src/contract_check/worker_extract/handler.py
View file

@ -25,8 +25,13 @@ from ..core.credits import refund_credit
from ..core.db.enums import DOC_TERMINAL, FailureClass
from ..core.logging import get_logger
from ..core.metrics import extract_duration, mq_failed, mq_published
from ..core.mq.messages import DocumentExtracted, DocumentUploaded
from ..core.mq.messages import (
DocumentExtracted,
DocumentUploaded,
PrescreenRequested,
)
from ..core.mq.publisher import Publisher
from ..core.mq.topology import RK_ANALYZE, RK_PRESCREEN
from ..core.s3 import extracted_key
from ..core.s3.minio_storage import MinioStorage
@ -43,7 +48,7 @@ class ExtractHandler:
self,
*,
session_factory: async_sessionmaker[AsyncSession],
publish_routing_key: str = "analyze",
publish_routing_key: str = RK_ANALYZE,
) -> None:
self._session_factory = session_factory
self._publish_routing_key = publish_routing_key
@ -105,6 +110,24 @@ class ExtractHandler:
await self._storage.put(ext_key, text_bytes, content_type="text/plain; charset=utf-8")
publisher = await self._publisher_instance()
if self._settings.prescreen_enabled:
msg = PrescreenRequested(
correlation_id=payload.correlation_id,
document_id=payload.document_id,
user_id=payload.user_id,
text_s3_key=ext_key,
filename=payload.filename,
char_count=len(extracted_text),
is_structured=False,
has_tables=False,
attempt=payload.attempt,
)
publish_rk = RK_PRESCREEN
queue_metric = "prescreen"
doc_status = "prescreening"
doc_stage = "queued_prescreen"
else:
msg = DocumentExtracted(
correlation_id=payload.correlation_id,
document_id=payload.document_id,
@ -114,18 +137,26 @@ class ExtractHandler:
ocr_used=ocr_used,
attempt=payload.attempt,
)
publish_rk = RK_ANALYZE
queue_metric = "analyze"
doc_status = "analyzing"
doc_stage = "queued_analyze"
publisher = await self._publisher_instance()
await publisher.publish(msg, routing_key=self._publish_routing_key)
mq_published.labels(queue="analyze").inc()
await publisher.publish(msg, routing_key=publish_rk)
mq_published.labels(queue=queue_metric).inc()
async with self._session_factory() as session:
await session.execute(
text(
"UPDATE documents SET status = 'analyzing', stage = 'queued_analyze', "
"UPDATE documents SET status = :status, stage = :stage, "
"extracted_s3_key = :key WHERE id = :d"
),
{"key": ext_key, "d": payload.document_id},
{
"status": doc_status,
"stage": doc_stage,
"key": ext_key,
"d": payload.document_id,
},
)
await session.execute(
text(

29
src/contract_check/worker_prescreen/__init__.py Normal file
View file

@ -0,0 +1,29 @@
"""worker-prescreen package."""
from __future__ import annotations
from .config import PrescreenSettings
from .extractor import (
ExtractionResult,
MetadataExtractor,
PrescreenContractMeta,
extract_contract_meta,
)
from .extractor_heuristic import HeuristicExtractor
from .extractor_hybrid import HybridMetaExtractor
from .extractor_llm import LLMPrescreenExtractor
from .handler import PrescreenHandler
from .router import PrescreenRouter
__all__ = [
"PrescreenSettings",
"PrescreenContractMeta",
"ExtractionResult",
"MetadataExtractor",
"extract_contract_meta",
"HeuristicExtractor",
"HybridMetaExtractor",
"LLMPrescreenExtractor",
"PrescreenRouter",
"PrescreenHandler",
]

62
src/contract_check/worker_prescreen/__main__.py Normal file
View file

@ -0,0 +1,62 @@
"""worker-prescreen entrypoint: connects to RabbitMQ and runs the prescreen consumer."""
from __future__ import annotations
import asyncio
import signal
from ..core.config import get_settings
from ..core.logging import bind_context, configure_logging, get_logger
from ..core.metrics import start_metrics_server
from ..core.sentry import init_sentry
from ..core.telemetry import setup_telemetry, shutdown_telemetry
from .consumer import PrescreenConsumer
log = get_logger(__name__)
async def main() -> None:
settings = get_settings()
configure_logging(
settings.log_level,
json_output=settings.json_logs,
service="worker-prescreen",
env=settings.env,
)
bind_context(service="worker-prescreen", env=settings.env)
init_sentry("worker-prescreen")
setup_telemetry("worker-prescreen")
start_metrics_server(9104)
consumer = PrescreenConsumer(
url=settings.rabbitmq_url,
origin="worker-prescreen",
prefetch=1,
max_attempts=settings.mq_max_attempts,
retry_base_ms=settings.mq_retry_base_ms,
)
await consumer.connect()
loop = asyncio.get_running_loop()
stop_event = asyncio.Event()
for sig in (signal.SIGINT, signal.SIGTERM):
loop.add_signal_handler(sig, stop_event.set)
consumer_task = asyncio.create_task(consumer.run())
stop_task = asyncio.create_task(stop_event.wait())
log.info("worker_prescreen_started", prefetch=1)
try:
await asyncio.wait(
{consumer_task, stop_task},
return_when=asyncio.FIRST_COMPLETED,
)
finally:
await consumer.stop()
shutdown_telemetry()
if __name__ == "__main__":
asyncio.run(main())

21
src/contract_check/worker_prescreen/config.py Normal file
View file

@ -0,0 +1,21 @@
"""Prescreen worker settings (extends the base settings with prescreen tuning)."""
from __future__ import annotations
from pydantic import Field
from ..core.config import Settings
class PrescreenSettings(Settings):
"""Settings for worker-prescreen.
Inherits all base env and adds the prescreen tunables from core.config.
They are already declared on `Settings`, so this class just provides a
typed alias used by the worker.
"""
mq_prefetch_prescreen: int = Field(
default=1,
description="Prefetch count for prescreen.q consumer.",
)

66
src/contract_check/worker_prescreen/consumer.py Normal file
View file

@ -0,0 +1,66 @@
"""worker-prescreen consumer: wires the prescreen handler into the RabbitMQ base."""
from __future__ import annotations
from ..core.db.enums import FailureClass
from ..core.db.session import create_session_factory
from ..core.logging import get_logger
from ..core.metrics import prescreen_duration
from ..core.mq.consumer import Consumer
from ..core.mq.messages import PrescreenRequested
from ..core.mq.topology import RK_ANALYZE
from .handler import PrescreenHandler
log = get_logger(__name__)
class PrescreenConsumer(Consumer[PrescreenRequested]):
"""Consumes `prescreen.q`, extracts contract metadata, routes to analyze.q."""
queue: str = "prescreen.q"
routing_key: str = "prescreen"
message_model = PrescreenRequested
def __init__(
self,
url: str,
*,
origin: str,
prefetch: int,
max_attempts: int,
retry_base_ms: int,
) -> None:
super().__init__(
url,
origin=origin,
prefetch=prefetch,
max_attempts=max_attempts,
retry_base_ms=retry_base_ms,
)
self._session_factory = create_session_factory()
self._handler = PrescreenHandler(
session_factory=self._session_factory,
publish_routing_key=RK_ANALYZE,
)
def classify(self, exc: BaseException) -> FailureClass:
return self._handler.classify(exc)
@prescreen_duration.labels(decision="unknown").time()
async def handle(self, payload: PrescreenRequested) -> None:
await self._handler.handle(payload)
async def on_failure(
self, payload: PrescreenRequested, failure_class: FailureClass, attempt: int, error: str
) -> None:
await self._handler.on_failure(payload, failure_class, attempt, error)
async def on_dlq(
self, payload: PrescreenRequested, failure_class: FailureClass, error: str
) -> None:
await self._handler.on_terminal_failure(payload, failure_class, error)
async def stop(self) -> None:
await super().stop()
# Release the hybrid extractor's owned LLM provider, if any.
await self._handler.aclose()

333
src/contract_check/worker_prescreen/extractor.py Normal file
View file

@ -0,0 +1,333 @@
"""Prescreen metadata model, extractor protocol, and compatibility shim.
This module is the shared kernel for the hybrid two-stage extractor described
in docs/PRESCREEN_HYBRID_REFACTOR_PLAN.md:
- `PrescreenContractMeta` — the 10-field pydantic model (shape unchanged).
- `FIELD_WEIGHTS` / `_score_confidence` — weighted confidence scoring.
- `ExtractionResult` / `MetadataExtractor` — the async contract the handler
consumes (`HybridMetaExtractor` is the default implementation).
- `extract_contract_meta` — synchronous delegating shim kept for legacy
callers and tests.
The v1 regex implementation is retained one release behind the
`PRESCREEN_KEEP_REGEX=true` env flag as a rollback safety net (plan §5).
Stage 1 lives in `extractor_heuristic.py`, Stage 2 in `extractor_llm.py`, and
the orchestrator in `extractor_hybrid.py`.
"""
from __future__ import annotations
import os
import re
from datetime import date
from typing import Protocol, runtime_checkable
from pydantic import BaseModel, Field, field_validator
class PrescreenContractMeta(BaseModel):
"""Fields extracted deterministically from a contract."""
contract_type: str | None = None
party_a: str | None = None
party_b: str | None = None
total_amount: float | None = None
currency: str | None = None
start_date: str | None = None # ISO YYYY-MM-DD
end_date: str | None = None
has_penalty_clause: bool | None = None
has_termination_clause: bool | None = None
has_arbitration: bool | None = None
confidence_score: float = Field(default=0.0, ge=0.0, le=1.0)
@field_validator("currency")
@classmethod
def _upper_currency(cls, v: str | None) -> str | None:
return v.upper() if v else None
# Weighted field coverage (plan §3.2 weights, nominal sum ≈ 1.0; the scorer
# clamps at 1.0): critical identity fields weigh more than boolean clause
# flags. Tunable with real data later.
FIELD_WEIGHTS: dict[str, float] = {
"contract_type": 0.25,
"party_a": 0.15,
"party_b": 0.15,
"total_amount": 0.15,
"currency": 0.05,
"start_date": 0.10,
"end_date": 0.05,
"has_penalty_clause": 0.05,
"has_termination_clause": 0.05,
"has_arbitration": 0.05,
}
def _score_confidence(meta: PrescreenContractMeta) -> float:
"""Weighted coverage: sum of weights of non-None fields."""
total = sum(w for name, w in FIELD_WEIGHTS.items() if getattr(meta, name) is not None)
return round(min(total, 1.0), 3)
def _parse_iso(date_str: str | None) -> str | None:
"""Parse a dd.mm.yyyy / dd/mm.yyyy string into ISO; None on failure."""
if not date_str:
return None
date_str = date_str.strip()
for sep in (".", "/"):
if sep in date_str:
parts = date_str.split(sep)
if len(parts) == 3:
try:
d = date(int(parts[2]), int(parts[1]), int(parts[0]))
return d.isoformat()
except ValueError:
return None
return None
class ExtractionResult(BaseModel):
"""Outcome of a hybrid extraction run.
`extractor_version` is persisted to prescreen_results.extractor_version:
"heuristic-v2" (LLM not run / disabled / failed) or "hybrid-llm-v1"
(LLM result merged). `llm_fallback_error` is surfaced through
prescreen_results.auto_findings when the fallback failed.
"""
meta: PrescreenContractMeta
extractor_version: str
llm_fallback_error: str | None = None
@runtime_checkable
class MetadataExtractor(Protocol):
"""Anything the prescreen handler can call to turn text into meta."""
async def extract(self, text: str) -> ExtractionResult: ...
# ── legacy regex-v1 implementation (PRESCREEN_KEEP_REGEX=true) ───────────────
# Legal entity forms — abbreviated and full — optionally quoted.
_ENTITY_RE = re.compile(
r"((?:"
r"ООО|ОБЩЕСТВО\s+С\s+ОГРАНИЧЕННОЙ\s+ОТВЕТСТВЕННОСТЬЮ|"
r"АО|АКЦИОНЕРНОЕ\s+ОБЩЕСТВО|"
r"ПАО|ПУБЛИЧНОЕ\s+АКЦИОНЕРНОЕ\s+ОБЩЕСТВО|"
r"ЗАО|ЗАКРЫТОЕ\s+АКЦИОНЕРНОЕ\s+ОБЩЕСТВО|"
r"ИП|ИНДИВИДУАЛЬНЫЙ\s+ПРЕДПРИНИМАТЕЛЬ|"
r"ОДО|ОБЩЕСТВО\s+С\s+ДОПОЛНИТЕЛЬНОЙ\s+ОТВЕТСТВЕННОСТЬЮ|"
r"ЧУП|ЧАСТНОЕ\s+УНИТАРНОЕ\s+ПРЕДПРИЯТИЕ"
r")\s*"
r"(?:«[^»]+»|\"[^\"]+\"|[А-ЯA-Z][А-Яа-яA-Za-z\-]+(?:\s+[А-ЯA-Z][а-яa-z]+)?)?)",
re.IGNORECASE,
)
_CONTRACT_TYPE_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
(
"supply",
re.compile(
r"договор\s+(?:купли[-\s]продажи|поставки)",
re.IGNORECASE,
),
),
(
"services",
re.compile(
r"договор\s+(?:оказания\s+услуг|возмездного\s+оказания\s+услуг)",
re.IGNORECASE,
),
),
(
"contract_work",
re.compile(
r"договор\s+(?:подряда|строительного\s+подряда)",
re.IGNORECASE,
),
),
(
"lease",
re.compile(
r"договор\s+(?:аренды|лизинга|субаренды)",
re.IGNORECASE,
),
),
(
"nda",
re.compile(
r"договор\s+о\s+(?:неразглашении|конфиденциальности)",
re.IGNORECASE,
),
),
]
_AMOUNT_RE = re.compile(
r"(?:"
r"(?:составляет|цен[ае]|стоимостью|оплат[ае]?|общая\s+стоимость|сумма)"
r"\s*)"
r"(?:(\d{1,3}(?:\s?\d{3})+)|(\d+(?:[.,]\d+)?))\s*"
r"(?:руб(?:лей|ля)?|₽|RUB|BYN|Br|USD|\$|EUR|€)?",
re.IGNORECASE,
)
_CURRENCY_NORMALIZE: dict[str, str] = {
"руб": "RUB",
"рублей": "RUB",
"рубля": "RUB",
"₽": "RUB",
"byn": "BYN",
"br": "BYN",
"usd": "USD",
"$": "USD",
"eur": "EUR",
"€": "EUR",
}
_DATE_RE = re.compile(
r"(?:действует\s+)?(?:с|от)\s+(\d{2}[./]\d{2}[./]\d{4})"
r"(?:.*?по\s+(\d{2}[./]\d{2}[./]\d{4}))?",
re.IGNORECASE,
)
_END_DATE_RE = re.compile(
r"(?:по|до)\s+(\d{2}[./]\d{2}[./]\d{4})",
re.IGNORECASE,
)
_PENALTY_RE = re.compile(
r"(?:неустойка|штраф|пеня|0[,.]\d+%|\d+%\s+за\s+кажды?й?\s+день\s+просрочки)",
re.IGNORECASE,
)
_TERMINATION_RE = re.compile(
r"(?:расторгнуть|расторжение|односторонн\w+\s+порядке|отказаться\s+от\s+исполнения)",
re.IGNORECASE,
)
_ARBITRATION_RE = re.compile(
r"(?:арбитражн\w+\s+суд|международн\w+\s+коммерческ\w+\s+арбитражн\w+\s+суд)",
re.IGNORECASE,
)
LEGACY_EXTRACTOR_VERSION = "regex-v1"
# Fields that count toward confidence coverage (legacy flat scoring).
_META_FIELDS: tuple[str, ...] = (
"contract_type",
"party_a",
"party_b",
"total_amount",
"currency",
"start_date",
"end_date",
"has_penalty_clause",
"has_termination_clause",
"has_arbitration",
)
def _extract_contract_type_regex(text: str) -> str | None:
for name, pattern in _CONTRACT_TYPE_PATTERNS:
if pattern.search(text):
return name
return None
def _extract_parties_regex(text: str) -> tuple[str | None, str | None]:
matches = _ENTITY_RE.findall(text)
if not matches:
return None, None
# Clean whitespace from each match; drop empty/over-generic matches.
cleaned = []
for m in matches:
s = m.strip()
if not s or s.lower() in {"ооо", "ао", "пао", "зао", "ип", "одо", "чуп"}:
continue
cleaned.append(s)
if len(cleaned) >= 2:
return cleaned[0], cleaned[1]
return (cleaned[0], None) if cleaned else (None, None)
def _extract_amount_regex(text: str) -> tuple[float | None, str | None]:
for match in _AMOUNT_RE.finditer(text):
group = match.group(1) or match.group(2)
if not group:
continue
raw = group.replace(" ", "").replace(",", ".")
try:
value = float(raw)
except ValueError:
continue
# Currency symbol/word within the next ~50 chars after the number.
tail = text[match.end() : match.end() + 50]
tail_lower = tail.lower()
currency = None
for token, code in _CURRENCY_NORMALIZE.items():
if token.lower() in tail_lower or token.lower() in match.group(0).lower():
currency = code
break
# Default to RUB if no currency token was found but the trigger was Russian.
if currency is None and any(tok in match.group(0).lower() for tok in ("руб", "₽")):
currency = "RUB"
return value, currency
return None, None
def _extract_dates_regex(text: str) -> tuple[str | None, str | None]:
for match in _DATE_RE.finditer(text):
start = _parse_iso(match.group(1))
end = _parse_iso(match.group(2)) if match.group(2) else None
return start, end
# No start date — maybe only an end date is mentioned.
end_match = _END_DATE_RE.search(text)
if end_match:
return None, _parse_iso(end_match.group(1))
return None, None
def _extract_contract_meta_regex(text: str) -> PrescreenContractMeta:
"""regex-v1 implementation, kept behind PRESCREEN_KEEP_REGEX (plan §5)."""
text = text.replace("\n", " ")
party_a, party_b = _extract_parties_regex(text)
amount, currency = _extract_amount_regex(text)
start, end = _extract_dates_regex(text)
meta = PrescreenContractMeta(
contract_type=_extract_contract_type_regex(text),
party_a=party_a,
party_b=party_b,
total_amount=amount,
currency=currency,
start_date=start,
end_date=end,
has_penalty_clause=None if not text else _PENALTY_RE.search(text) is not None,
has_termination_clause=None if not text else _TERMINATION_RE.search(text) is not None,
has_arbitration=None if not text else _ARBITRATION_RE.search(text) is not None,
)
matched = sum(1 for field in _META_FIELDS if getattr(meta, field) is not None)
meta.confidence_score = round(matched / len(_META_FIELDS), 3) if _META_FIELDS else 0.0
return meta
def _keep_regex() -> bool:
return os.environ.get("PRESCREEN_KEEP_REGEX", "").strip().lower() in {"1", "true", "yes", "on"}
def extract_contract_meta(text: str) -> PrescreenContractMeta:
"""Synchronous shim: heuristic-v2 by default, regex-v1 behind the flag."""
if _keep_regex():
return _extract_contract_meta_regex(text)
from .extractor_heuristic import HeuristicExtractor
return HeuristicExtractor().extract(text)
__all__ = [
"ExtractionResult",
"FIELD_WEIGHTS",
"LEGACY_EXTRACTOR_VERSION",
"MetadataExtractor",
"PrescreenContractMeta",
"extract_contract_meta",
]

401
src/contract_check/worker_prescreen/extractor_heuristic.py Normal file
View file

@ -0,0 +1,401 @@
"""Stage 1 — heuristic deterministic extractor (pure string operations, no regex).
Keyword dictionaries, positional windows, sentence scanning, and weighted
confidence — see docs/PRESCREEN_HYBRID_REFACTOR_PLAN.md §3.2. Zero marginal
cost, no hallucination risk; low-confidence results fall through to the LLM
stage (extractor_hybrid.py).
"""
from __future__ import annotations
from ..core.logging import get_logger
from .extractor import PrescreenContractMeta, _parse_iso, _score_confidence
log = get_logger(__name__)
EXTRACTOR_VERSION = "heuristic-v2"
# ── dictionaries (module-level constants) ────────────────────────────────────
_CONTRACT_TYPE_PHRASES: tuple[tuple[str, tuple[str, ...]], ...] = (
("supply", ("договор поставки", "договор купли-продажи", "договор купли продажи")),
("services", ("договор возмездного оказания услуг", "договор оказания услуг")),
("contract_work", ("договор строительного подряда", "договор подряда")),
("lease", ("договор субаренды", "договор аренды", "договор лизинга")),
("nda", ("договор о неразглашении", "договор о конфиденциальности")),
)
# Legal entity forms (lowercase); longest-first so the scanner is greedy.
_ENTITY_FORMS: tuple[str, ...] = (
"общество с ограниченной ответственностью",
"общество с дополнительной ответственностью",
"публичное акционерное общество",
"закрытое акционерное общество",
"акционерное общество",
"частное унитарное предприятие",
"индивидуальный предприниматель",
"ооо",
"пао",
"зао",
"одо",
"чуп",
"ао",
"ип",
)
# Amount trigger words (lowercase substrings; a digit run must follow).
_AMOUNT_TRIGGERS: tuple[str, ...] = (
"общая стоимость",
"составляет",
"стоимостью",
"цена",
"цене",
"оплат",
"сумма",
)
_CURRENCY_TOKENS: dict[str, str] = {
"руб": "RUB",
"рублей": "RUB",
"рубля": "RUB",
"₽": "RUB",
"byn": "BYN",
"br": "BYN",
"usd": "USD",
"$": "USD",
"eur": "EUR",
"€": "EUR",
}
# Boolean-clause keyword stems (lowercase substring membership per sentence).
_PENALTY_KEYWORDS: tuple[str, ...] = ("неустойк", "штраф", "пеня", "пени")
_TERMINATION_KEYWORDS: tuple[str, ...] = ("расторг", "односторонн", "отказаться от исполнения")
_ARBITRATION_KEYWORDS: tuple[str, ...] = ("арбитражн",)
_HEADER_CAP_CHARS = 1500
_CURRENCY_TAIL_CHARS = 50
_MAX_PARTY_NAME_TOKENS = 8
_MAX_PARTIES = 2
_SPACES = " \t\u00a0\u202f"
_QUOTE_PAIRS = {"«": "»", "“": "”", '"': '"'}
# ── text helpers ─────────────────────────────────────────────────────────────
def _collapse(text: str) -> str:
"""Collapse every whitespace run (incl. newlines and NBSP) to one space."""
return " ".join(text.split())
def _skip_spaces(s: str, i: int) -> int:
while i < len(s) and s[i] in _SPACES:
i += 1
return i
def _split_sentences(text: str) -> list[str]:
"""Sentence segmentation via str.split on '. ' and newlines (plan §3.2)."""
parts: list[str] = []
for chunk in text.split("\n"):
parts.extend(chunk.split(". "))
return parts
def _find_closing_quote(s: str, open_idx: int) -> int:
close = _QUOTE_PAIRS.get(s[open_idx], s[open_idx])
return s.find(close, open_idx + 1)
# ── contract type ────────────────────────────────────────────────────────────
def _extract_contract_type(lower: str) -> str | None:
"""Exact phrase dictionary match on the normalized text."""
for name, phrases in _CONTRACT_TYPE_PHRASES:
for phrase in phrases:
if phrase in lower:
return name
return None
# ── parties (positional: header window only) ─────────────────────────────────
def _header_window(text: str) -> str:
"""Text up to the first numbered/ПРЕДМЕТ heading, capped at ~1500 chars."""
lines: list[str] = []
size = 0
for line in text.splitlines():
stripped = line.strip()
lower = stripped.lower()
if lines and (lower.startswith("1.") or lower.startswith("1 ") or "предмет" in lower):
break
lines.append(line)
size += len(line) + 1
if size >= _HEADER_CAP_CHARS:
break
return "\n".join(lines)
def _is_name_token(token: str) -> bool:
if not token:
return False
# Initials like «И.И.» / «П.»
if len(token) <= 3 and token[0].isupper() and token.endswith("."):
return True
return token[0].isupper()
def _capture_party_name(header: str, start: int, form_end: int) -> tuple[str, int] | None:
"""Capture a party name after an entity-form token at `start`.
Quoted names run from the form through the closing quote; unquoted names
consume following Capitalized words until a lowercase word, punctuation,
or line end.
"""
j = _skip_spaces(header, form_end)
if j < len(header) and header[j] in _QUOTE_PAIRS:
close = _find_closing_quote(header, j)
if close != -1:
return header[start : close + 1].strip(), close + 1
end = form_end
tokens = 0
j = _skip_spaces(header, form_end)
while j < len(header) and tokens < _MAX_PARTY_NAME_TOKENS:
line_end = header.find("\n", j)
segment_end = line_end if line_end != -1 else len(header)
k = j
while k < segment_end and header[k] not in _SPACES:
k += 1
token = header[j:k]
if not _is_name_token(token):
break
end = k
tokens += 1
j = _skip_spaces(header, k)
if j >= len(header) or header[j] == "\n":
break
name = header[start:end].strip()
if not name:
return None
return name, max(end, form_end)
def _extract_parties(header: str) -> tuple[str | None, str | None]:
"""Token-scan the header window for entity forms; capture party names."""
lower = header.lower()
n = len(header)
found: list[str] = []
i = 0
while i < n and len(found) < _MAX_PARTIES:
form: str | None = None
for candidate in _ENTITY_FORMS:
end = i + len(candidate)
if not lower.startswith(candidate, i):
continue
before_ok = i == 0 or not lower[i - 1].isalnum()
after_ok = end >= n or not lower[end].isalnum()
if before_ok and after_ok:
form = candidate
break
if form is None:
i += 1
continue
captured = _capture_party_name(header, i, i + len(form))
if captured is None:
i += len(form)
continue
name, capture_end = captured
# Drop bare forms without a name (e.g. a lone «ООО»).
if name.lower() != form:
if name not in found:
found.append(name)
i = max(capture_end, i + len(form))
if len(found) >= 2:
return found[0], found[1]
if len(found) == 1:
return found[0], None
return None, None
# ── amount + currency (trigger-word scan + manual digit parse) ───────────────
def _scan_number(s: str, i: int) -> tuple[float, int] | None:
"""Walk chars at s[i] parsing digits, space-separated thousands, decimals."""
n = len(s)
j = i
int_part: list[str] = []
dec_part: list[str] = []
while j < n and s[j].isdigit():
int_part.append(s[j])
j += 1
if not int_part:
return None
while j < n:
ch = s[j]
if ch in ".," and j + 1 < n and s[j + 1].isdigit() and not dec_part:
j += 1
while j < n and s[j].isdigit():
dec_part.append(s[j])
j += 1
break # a decimal part terminates the number
if ch in _SPACES:
# Thousands separator: space + exactly 3 digits, not part of a longer run.
k = j + 1
group = ""
while k < n and s[k].isdigit() and len(group) < 3:
group += s[k]
k += 1
if len(group) == 3 and (k >= n or not s[k].isdigit()):
int_part.append(group)
j = k
continue
break
raw = "".join(int_part) + ("." + "".join(dec_part) if dec_part else "")
return float(raw), j
def _find_currency(lower: str, start: int, end: int) -> str | None:
window = lower[start : min(end, len(lower))]
for token, code in _CURRENCY_TOKENS.items():
if token in window:
return code
return None
def _find_earliest_trigger(lower: str, start: int) -> tuple[int, int] | None:
best: tuple[int, int] | None = None
for trigger in _AMOUNT_TRIGGERS:
idx = lower.find(trigger, start)
if idx != -1 and (best is None or idx < best[0]):
best = (idx, idx + len(trigger))
return best
def _extract_amount(collapsed: str, lower: str) -> tuple[float | None, str | None]:
pos = 0
while True:
hit = _find_earliest_trigger(lower, pos)
if hit is None:
return None, None
trig_start, trig_end = hit
j = _skip_spaces(collapsed, trig_end)
if j < len(collapsed) and collapsed[j].isdigit():
parsed = _scan_number(collapsed, j)
if parsed is not None:
value, num_end = parsed
return value, _find_currency(lower, trig_start, num_end + _CURRENCY_TAIL_CHARS)
pos = trig_start + 1
# ── dates (trigger tokens + manual dd.mm.yyyy splitter) ──────────────────────
def _parse_ddmmyyyy(s: str, i: int) -> str | None:
"""Parse dd.mm.yyyy / dd/mm.yyyy at s[i]; None when not a date boundary."""
if i > 0 and s[i - 1].isdigit():
return None
if i + 10 > len(s):
return None
if not (s[i].isdigit() and s[i + 1].isdigit()):
return None
if s[i + 2] not in "./":
return None
if not (s[i + 3].isdigit() and s[i + 4].isdigit()):
return None
if s[i + 5] not in "./":
return None
if not all(s[k].isdigit() for k in (i + 6, i + 7, i + 8, i + 9)):
return None
if i + 10 < len(s) and s[i + 10].isdigit():
return None
return _parse_iso(s[i : i + 10])
def _find_date_after_token(s: str, token: str, start: int) -> tuple[str, int, int] | None:
"""Find `token` as a standalone word followed by a dd.mm.yyyy date.
Returns (iso_date, date_start, scan_end) or None.
"""
pos = start
n = len(s)
while pos < n:
idx = s.find(token, pos)
if idx == -1:
return None
before_ok = idx == 0 or not s[idx - 1].isalnum()
j = idx + len(token)
if before_ok and j < n and s[j] in _SPACES:
j = _skip_spaces(s, j)
iso = _parse_ddmmyyyy(s, j)
if iso is not None:
return iso, j, j + 10
pos = idx + 1
return None
def _extract_dates(lower_collapsed: str) -> tuple[str | None, str | None]:
start_hit = None
for token in ("с", "от"):
hit = _find_date_after_token(lower_collapsed, token, 0)
if hit is not None and (start_hit is None or hit[1] < start_hit[1]):
start_hit = hit
if start_hit is not None:
# v1 semantics: end date = «по <date>» anywhere after the start date.
end_hit = _find_date_after_token(lower_collapsed, "по", start_hit[2])
return start_hit[0], end_hit[0] if end_hit is not None else None
for token in ("по", "до"):
hit = _find_date_after_token(lower_collapsed, token, 0)
if hit is not None:
return None, hit[0]
return None, None
# ── boolean clauses (sentence scanning) ──────────────────────────────────────
def _has_clause(sentences: list[str], keywords: tuple[str, ...]) -> bool:
return any(keyword in sentence for sentence in sentences for keyword in keywords)
# ── Stage 1 extractor ────────────────────────────────────────────────────────
class HeuristicExtractor:
"""Deterministic keyword/positional extractor (Stage 1, sync, zero cost)."""
extractor_version = EXTRACTOR_VERSION
def extract(self, text: str) -> PrescreenContractMeta:
collapsed = _collapse(text)
lower = collapsed.lower()
header = _header_window(text)
sentences = [s.lower() for s in _split_sentences(text)] if text else []
party_a, party_b = _extract_parties(header)
amount, currency = _extract_amount(collapsed, lower)
start_date, end_date = _extract_dates(lower)
meta = PrescreenContractMeta(
contract_type=_extract_contract_type(lower),
party_a=party_a,
party_b=party_b,
total_amount=amount,
currency=currency,
start_date=start_date,
end_date=end_date,
has_penalty_clause=None if not text else _has_clause(sentences, _PENALTY_KEYWORDS),
has_termination_clause=(
None if not text else _has_clause(sentences, _TERMINATION_KEYWORDS)
),
has_arbitration=None if not text else _has_clause(sentences, _ARBITRATION_KEYWORDS),
)
meta.confidence_score = _score_confidence(meta)
return meta
__all__ = ["EXTRACTOR_VERSION", "HeuristicExtractor"]

149
src/contract_check/worker_prescreen/extractor_hybrid.py Normal file
View file

@ -0,0 +1,149 @@
"""Hybrid two-stage orchestrator for prescreen metadata extraction.
Stage 1 — HeuristicExtractor (sync, via to_thread): deterministic, zero cost.
Stage 2 — LLMPrescreenExtractor: runs only when the kill-switch is off AND the
heuristic confidence is below `prescreen_llm_fallback_threshold`.
Failure semantics (plan §3.4): an LLM fallback error never fails the message —
the heuristic meta is kept, `extractor_version="heuristic-v2"`, and the error
is recorded on the result for `prescreen_results.auto_findings` plus the
`prescreen_fallback_runs_total{outcome="failed"}` counter.
"""
from __future__ import annotations
import asyncio
from typing import Any
from ..core.config import get_settings
from ..core.llm.factory import build_llm_provider
from ..core.llm.port import LLMProvider
from ..core.logging import get_logger
from ..core.metrics import prescreen_extraction_stage, prescreen_fallback_runs
from .extractor import ExtractionResult, PrescreenContractMeta, _score_confidence
from .extractor_heuristic import EXTRACTOR_VERSION as HEURISTIC_VERSION
from .extractor_heuristic import HeuristicExtractor
from .extractor_llm import EXTRACTOR_VERSION as HYBRID_LLM_VERSION
from .extractor_llm import LLMPrescreenExtractor
log = get_logger(__name__)
_SCALAR_FIELDS = (
"contract_type",
"party_a",
"party_b",
"total_amount",
"currency",
"start_date",
"end_date",
)
_BOOL_FIELDS = (
"has_penalty_clause",
"has_termination_clause",
"has_arbitration",
)
def _merge(heuristic: PrescreenContractMeta, llm: PrescreenContractMeta) -> PrescreenContractMeta:
"""LLM values override heuristic Nones; boolean clauses OR-merge.
Booleans from both stages are presence-checks — false positives are cheap,
false negatives route wrong (plan §3.3).
"""
update: dict[str, Any] = {}
for field in _SCALAR_FIELDS:
llm_value = getattr(llm, field)
if llm_value is not None:
update[field] = llm_value
for field in _BOOL_FIELDS:
h_value, l_value = getattr(heuristic, field), getattr(llm, field)
if h_value is None:
if l_value is not None:
update[field] = l_value
elif l_value is not None:
update[field] = bool(h_value or l_value)
merged = heuristic.model_copy(update=update)
merged.confidence_score = _score_confidence(merged)
return merged
class HybridMetaExtractor:
"""Default MetadataExtractor: heuristic first, LLM only when confidence is low."""
def __init__(
self,
*,
provider: LLMProvider | None = None,
fallback_enabled: bool | None = None,
fallback_threshold: float | None = None,
heuristic: HeuristicExtractor | None = None,
) -> None:
settings = get_settings()
self._heuristic = heuristic or HeuristicExtractor()
self._fallback_enabled = (
settings.prescreen_llm_fallback_enabled
if fallback_enabled is None
else fallback_enabled
)
self._fallback_threshold = (
settings.prescreen_llm_fallback_threshold
if fallback_threshold is None
else fallback_threshold
)
self._provider = provider
self._owns_provider = provider is None
self._llm: LLMPrescreenExtractor | None = None
async def _llm_extractor(self) -> LLMPrescreenExtractor:
if self._llm is None:
if self._provider is None:
self._provider = build_llm_provider(get_settings())
self._llm = LLMPrescreenExtractor(self._provider)
return self._llm
async def aclose(self) -> None:
if self._provider is not None and self._owns_provider:
await self._provider.aclose()
async def extract(self, text: str) -> ExtractionResult:
with prescreen_extraction_stage.labels(stage="heuristic").time():
meta = await asyncio.to_thread(self._heuristic.extract, text)
if not self._fallback_enabled:
prescreen_fallback_runs.labels(outcome="disabled").inc()
return ExtractionResult(meta=meta, extractor_version=HEURISTIC_VERSION)
if meta.confidence_score >= self._fallback_threshold:
prescreen_fallback_runs.labels(outcome="skipped").inc()
return ExtractionResult(meta=meta, extractor_version=HEURISTIC_VERSION)
try:
llm_extractor = await self._llm_extractor()
with prescreen_extraction_stage.labels(stage="llm").time():
llm_meta = await llm_extractor.extract(text)
except Exception as exc: # noqa: BLE001 — never fail the message (plan §3.4)
prescreen_fallback_runs.labels(outcome="failed").inc()
error = f"{type(exc).__name__}: {exc}"
log.warning(
"prescreen_llm_fallback_failed",
error=error,
heuristic_confidence=meta.confidence_score,
)
return ExtractionResult(
meta=meta,
extractor_version=HEURISTIC_VERSION,
llm_fallback_error=error,
)
prescreen_fallback_runs.labels(outcome="used").inc()
merged = _merge(meta, llm_meta)
log.info(
"prescreen_llm_fallback_merged",
heuristic_confidence=meta.confidence_score,
merged_confidence=merged.confidence_score,
)
return ExtractionResult(meta=merged, extractor_version=HYBRID_LLM_VERSION)
__all__ = ["HybridMetaExtractor"]

100
src/contract_check/worker_prescreen/extractor_llm.py Normal file
View file

@ -0,0 +1,100 @@
"""Stage 2 — LLM fallback extractor: provider dict → validated contract meta.
The provider layer returns a plain, permissive dict (core/llm/prescreen.py).
This wrapper whitelists contract_type enums, re-`None`s hallucinated or
malformed values (numeric/date parse checks), and recomputes the weighted
confidence — pydantic model stays in worker_prescreen (plan §3.3).
"""
from __future__ import annotations
from datetime import date
from typing import TYPE_CHECKING, Any
from ..core.logging import get_logger
from .extractor import PrescreenContractMeta, _parse_iso, _score_confidence
if TYPE_CHECKING:
from ..core.llm.port import LLMProvider
log = get_logger(__name__)
EXTRACTOR_VERSION = "hybrid-llm-v1"
VALID_CONTRACT_TYPES = frozenset({"supply", "services", "contract_work", "lease", "nda"})
VALID_CURRENCIES = frozenset({"RUB", "BYN", "USD", "EUR"})
def _clean_str(value: Any) -> str | None:
if not isinstance(value, str):
return None
stripped = value.strip()
return stripped or None
def _clean_amount(value: Any) -> float | None:
if isinstance(value, bool) or not isinstance(value, int | float):
return None
amount = float(value)
return amount if amount >= 0 else None
def _clean_date(value: Any) -> str | None:
raw = _clean_str(value)
if raw is None:
return None
try:
return date.fromisoformat(raw).isoformat()
except ValueError:
return _parse_iso(raw) # accept dd.mm.yyyy / dd/mm.yyyy too
def _clean_bool(value: Any) -> bool | None:
return value if isinstance(value, bool) else None
def validate_raw(raw: dict[str, Any]) -> PrescreenContractMeta:
"""Whitelist/normalize a raw LLM dict; hallucinated values become None."""
contract_type = _clean_str(raw.get("contract_type"))
currency = _clean_str(raw.get("currency"))
contract_type_normalized = contract_type.lower() if contract_type else None
currency_normalized = currency.upper() if currency else None
meta = PrescreenContractMeta(
contract_type=(
contract_type_normalized if contract_type_normalized in VALID_CONTRACT_TYPES else None
),
party_a=_clean_str(raw.get("party_a")),
party_b=_clean_str(raw.get("party_b")),
total_amount=_clean_amount(raw.get("total_amount")),
currency=(currency_normalized if currency_normalized in VALID_CURRENCIES else None),
start_date=_clean_date(raw.get("start_date")),
end_date=_clean_date(raw.get("end_date")),
has_penalty_clause=_clean_bool(raw.get("has_penalty_clause")),
has_termination_clause=_clean_bool(raw.get("has_termination_clause")),
has_arbitration=_clean_bool(raw.get("has_arbitration")),
)
meta.confidence_score = _score_confidence(meta)
return meta
class LLMPrescreenExtractor:
"""MetadataExtractor over an LLMProvider's `extract_prescreen`."""
extractor_version = EXTRACTOR_VERSION
def __init__(self, provider: LLMProvider) -> None:
self._provider = provider
async def extract(self, text: str) -> PrescreenContractMeta:
raw = await self._provider.extract_prescreen(text)
meta = validate_raw(raw)
log.info(
"prescreen_llm_extraction_done",
confidence=meta.confidence_score,
contract_type=meta.contract_type,
)
return meta
__all__ = ["EXTRACTOR_VERSION", "LLMPrescreenExtractor", "validate_raw"]

614
src/contract_check/worker_prescreen/handler.py Normal file
View file

@ -0,0 +1,614 @@
"""Prescreen worker handler: download extracted Markdown → hybrid extract → route → publish.
The handler is intentionally separate from the consumer so it can be tested
in-process without spinning up a real RabbitMQ consumer. It owns:
- idempotency checks against documents.status
- status transitions (extracting → prescreening → analyzing | manual_review | done)
- MinIO download of the extracted Markdown
- hybrid contract metadata extraction (heuristic Stage 1 + optional LLM Stage 2)
- routing decision + persisting prescreen_results (incl. extractor_version)
- publishing the next message (deep_analysis → analyze.q, auto_approve →
report.completed, manual_review → a review queue marker)
- updating the jobs row, recording failure class, and refund-on-DLQ.
"""
from __future__ import annotations
import json
import time
import uuid
from datetime import UTC, date, datetime
from typing import TYPE_CHECKING
from sqlalchemy import text
from ..core.config import get_settings
from ..core.credits import refund_credit
from ..core.db.enums import DOC_TERMINAL, FailureClass
from ..core.logging import (
Timer,
get_logger,
log_db_operation,
log_external_call,
log_message_processing,
log_performance_metric,
log_stage_complete,
log_stage_failure,
log_stage_progress,
)
from ..core.metrics import (
mq_failed,
mq_published,
prescreen_confidence,
prescreen_duration,
prescreen_runs,
)
from ..core.mq.messages import AnalyzeRequested, PrescreenCompleted, PrescreenRequested
from ..core.mq.publisher import Publisher
from ..core.mq.topology import RK_ANALYZE
from ..core.s3.minio_storage import MinioStorage
from .extractor import ExtractionResult, MetadataExtractor
from .extractor_hybrid import HybridMetaExtractor
from .router import PrescreenRouter, RoutingDecision
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from .extractor import PrescreenContractMeta
log = get_logger(__name__)
def _as_date(value: str | None) -> date | None:
"""prescreen_results date columns are DATE; meta carries ISO strings."""
return date.fromisoformat(value) if value else None
class PrescreenHandler:
"""Business logic for worker-prescreen."""
def __init__(
self,
*,
session_factory: async_sessionmaker[AsyncSession],
publish_routing_key: str = RK_ANALYZE,
extractor: MetadataExtractor | None = None,
) -> None:
self._session_factory = session_factory
self._publish_routing_key = publish_routing_key
self._settings = get_settings()
self._storage = MinioStorage.from_endpoint_url(
endpoint_url=self._settings.s3_endpoint_url,
access_key=self._settings.s3_access_key,
secret_key=self._settings.s3_secret_key,
bucket=self._settings.s3_bucket,
region=self._settings.s3_region,
)
self._router = PrescreenRouter(
confidence_threshold=self._settings.prescreen_confidence_threshold,
high_value_threshold=self._settings.prescreen_high_value_threshold,
auto_approve_enabled=self._settings.prescreen_auto_approve,
)
self._extractor: MetadataExtractor = extractor or HybridMetaExtractor()
self._publisher: Publisher | None = None
async def aclose(self) -> None:
"""Release owned resources (LLM provider behind the hybrid extractor)."""
if isinstance(self._extractor, HybridMetaExtractor):
await self._extractor.aclose()
async def _publisher_instance(self) -> Publisher:
if self._publisher is None:
self._publisher = Publisher(self._settings.rabbitmq_url, origin="worker-prescreen")
await self._publisher.connect()
return self._publisher
async def _db_status(self, session: AsyncSession, document_id: uuid.UUID) -> str | None:
row = await session.execute(
text("SELECT status FROM documents WHERE id = :d FOR UPDATE"),
{"d": document_id},
)
result = row.first()
return str(result[0]) if result else None
async def handle(self, payload: PrescreenRequested) -> None:
started_at = time.perf_counter()
log_message_processing(
log,
"started",
"prescreen.q",
correlation_id=str(payload.correlation_id),
document_id=str(payload.document_id),
user_id=str(payload.user_id),
text_s3_key=payload.text_s3_key,
filename=payload.filename,
attempt=payload.attempt,
)
# Stage 1: Database validation
with Timer(log, "prescreen_db_validation", document_id=str(payload.document_id)):
async with self._session_factory() as session:
current = await self._db_status(session, payload.document_id)
if current is None:
log.warning(
"document_not_found",
document_id=str(payload.document_id),
user_id=str(payload.user_id),
text_s3_key=payload.text_s3_key,
)
return
if current in DOC_TERMINAL:
log.info(
"document_already_terminal",
document_id=str(payload.document_id),
status=current,
)
return
# Stage 2: Update document status and job tracking
with Timer(log, "prescreen_status_update", document_id=str(payload.document_id)):
async with self._session_factory() as session:
await session.execute(
text(
"UPDATE documents SET status = 'prescreening', stage = 'extracting_meta' "
"WHERE id = :d"
),
{"d": payload.document_id},
)
await session.execute(
text(
"INSERT INTO jobs (document_id, correlation_id, queue, status) "
"VALUES (:d, :cid, 'prescreen', 'running') "
"ON CONFLICT (document_id, queue) DO NOTHING"
),
{"d": payload.document_id, "cid": payload.correlation_id},
)
await session.execute(
text(
"UPDATE jobs SET status = 'running', attempts = attempts + 1 "
"WHERE document_id = :d AND queue = 'prescreen'"
),
{"d": payload.document_id},
)
await session.commit()
log_db_operation(
log,
"update",
"documents",
document_id=str(payload.document_id),
status="prescreening",
stage="extracting_meta",
)
# Stage 3: Download text from S3
text_bytes: bytes
contract_text: str
with Timer(
log,
"prescreen_text_download",
document_id=str(payload.document_id),
text_s3_key=payload.text_s3_key,
):
try:
text_bytes = await self._storage.get(payload.text_s3_key)
contract_text = text_bytes.decode("utf-8")
log_external_call(
log,
"minio",
f"get:{payload.text_s3_key}",
bytes_size=len(text_bytes),
)
except Exception as exc:
log_stage_failure(
log,
"prescreen_text_download",
exc,
document_id=str(payload.document_id),
text_s3_key=payload.text_s3_key,
)
raise
log_stage_progress(
log,
"prescreen_text_loaded",
"text_downloaded",
document_id=str(payload.document_id),
text_s3_key=payload.text_s3_key,
char_count=len(contract_text),
)
# Stage 4: Extract contract metadata (heuristic Stage 1 + optional LLM Stage 2)
meta: PrescreenContractMeta
extraction: ExtractionResult
with Timer(log, "prescreen_meta_extraction", document_id=str(payload.document_id)):
try:
extraction = await self._extractor.extract(contract_text)
meta = extraction.meta
log_performance_metric(
log,
"extraction_confidence",
meta.confidence_score,
"ratio",
document_id=str(payload.document_id),
extractor_version=extraction.extractor_version,
llm_fallback_error=extraction.llm_fallback_error,
extracted_fields=sum(
1 for field in self._meta_field_names() if getattr(meta, field) is not None
),
total_fields=len(self._meta_field_names()),
)
except Exception as exc:
log_stage_failure(
log, "prescreen_meta_extraction", exc, document_id=str(payload.document_id)
)
raise
# Stage 5: Make routing decision
decision: RoutingDecision
with Timer(log, "prescreen_routing_decision", document_id=str(payload.document_id)):
decision = self._router.decide(meta)
log_stage_progress(
log,
"prescreen_routing_decision",
"decision_made",
document_id=str(payload.document_id),
decision=decision,
confidence=meta.confidence_score,
)
processing_ms = int((time.perf_counter() - started_at) * 1000)
prescreened_at = datetime.now(UTC)
prescreened_at_str = prescreened_at.isoformat()
completed = PrescreenCompleted(
correlation_id=payload.correlation_id,
document_id=payload.document_id,
user_id=payload.user_id,
text_s3_key=payload.text_s3_key,
filename=payload.filename,
prescreened_at=prescreened_at_str,
contract_type=meta.contract_type,
party_a=meta.party_a,
party_b=meta.party_b,
total_amount=meta.total_amount,
currency=meta.currency,
start_date=meta.start_date,
end_date=meta.end_date,
has_penalty_clause=meta.has_penalty_clause,
has_termination_clause=meta.has_termination_clause,
has_arbitration=meta.has_arbitration,
confidence_score=meta.confidence_score,
routing_decision=decision,
auto_summary=self._router.summarize(meta, decision)
if decision == "auto_approve"
else None,
# A failed LLM fallback is recorded here (plan §3.4) — never fatal.
auto_findings=[{"llm_fallback_error": extraction.llm_fallback_error}]
if extraction.llm_fallback_error
else [],
attempt=payload.attempt,
)
log_stage_complete(
log,
"prescreen_extraction",
processing_ms,
document_id=str(payload.document_id),
decision=decision,
confidence=meta.confidence_score,
contract_type=meta.contract_type,
extractor_version=extraction.extractor_version,
matched_fields=[
field for field in self._meta_field_names() if getattr(meta, field) is not None
],
missing_fields=[
field for field in self._meta_field_names() if getattr(meta, field) is None
],
)
# Stage 6: Persist prescreen result
prescreen_result_id: uuid.UUID | None = None
with Timer(log, "prescreen_result_persistence", document_id=str(payload.document_id)):
try:
async with self._session_factory() as session:
result = await session.execute(
text(
"INSERT INTO prescreen_results "
"(document_id, correlation_id, contract_type, party_a, party_b, "
" total_amount, currency, start_date, end_date, has_penalty_clause, "
" has_termination_clause, has_arbitration, confidence_score, "
" routing_decision, prescreened_at, processing_ms, auto_summary, "
" auto_findings, extractor_version) "
"VALUES (:d, :cid, :ct, :pa, :pb, :ta, :cur, :sd, :ed, :hpc, :htc, :ha, "
" :cs, :rd, :psa, :ms, :asum, :af, :ev) "
"RETURNING id"
),
{
"d": payload.document_id,
"cid": payload.correlation_id,
"ct": completed.contract_type,
"pa": completed.party_a,
"pb": completed.party_b,
"ta": completed.total_amount,
"cur": completed.currency,
"sd": _as_date(completed.start_date),
"ed": _as_date(completed.end_date),
"hpc": completed.has_penalty_clause,
"htc": completed.has_termination_clause,
"ha": completed.has_arbitration,
"cs": completed.confidence_score,
"rd": completed.routing_decision,
"psa": prescreened_at,
"ms": processing_ms,
"asum": completed.auto_summary,
"af": json.dumps(completed.auto_findings),
"ev": extraction.extractor_version,
},
)
prescreen_result_id = result.scalar()
await session.commit()
log_db_operation(
log,
"insert",
"prescreen_results",
document_id=str(payload.document_id),
prescreen_result_id=str(prescreen_result_id),
)
except Exception as exc:
log_stage_failure(
log, "prescreen_result_persistence", exc, document_id=str(payload.document_id)
)
raise
prescreen_duration.labels(decision=decision).observe(processing_ms / 1000.0)
prescreen_runs.labels(
decision=decision, contract_type=meta.contract_type or "unknown"
).inc()
prescreen_confidence.observe(meta.confidence_score)
# Stage 7: Publish next message or complete
publisher = await self._publisher_instance()
next_stage: str
with Timer(
log,
"prescreen_next_step_routing",
document_id=str(payload.document_id),
decision=decision,
):
if decision == "deep_analysis":
next_msg = AnalyzeRequested(
correlation_id=payload.correlation_id,
document_id=payload.document_id,
user_id=payload.user_id,
extracted_s3_key=payload.text_s3_key,
filename=payload.filename,
char_count=payload.char_count,
ocr_used=False,
is_structured=payload.is_structured,
has_tables=payload.has_tables,
prescreen_meta=completed.model_dump(),
attempt=payload.attempt,
)
await publisher.publish(next_msg, routing_key=self._publish_routing_key)
mq_published.labels(queue="analyze").inc()
next_stage = "queued_analyze"
doc_status = "analyzing"
log_stage_progress(
log,
"prescreen_next_step_routing",
"published_to_analyze",
document_id=str(payload.document_id),
routing_key=self._publish_routing_key,
)
elif decision == "manual_review":
# For now, manual_review is a terminal state persisted in the DB.
# A future admin/web SPA can pick these up.
doc_status = "manual_review"
next_stage = "manual_review"
log_stage_progress(
log,
"prescreen_next_step_routing",
"queued_manual_review",
document_id=str(payload.document_id),
)
else: # auto_approve
# Auto-approval is disabled by default; if reached, write a lightweight
# report and complete the document.
await self._write_auto_approved_report(
session_factory=self._session_factory,
payload=payload,
completed=completed,
prescreen_result_id=prescreen_result_id, # type: ignore[arg-type]
)
doc_status = "done"
next_stage = "auto_approved"
log_stage_progress(
log,
"prescreen_next_step_routing",
"auto_approved",
document_id=str(payload.document_id),
)
# Stage 8: Final status update
with Timer(log, "prescreen_final_status_update", document_id=str(payload.document_id)):
async with self._session_factory() as session:
await session.execute(
text("UPDATE documents SET status = :status, stage = :stage WHERE id = :d"),
{"status": doc_status, "stage": next_stage, "d": payload.document_id},
)
await session.execute(
text(
"UPDATE jobs SET status = 'done' WHERE document_id = :d AND queue = 'prescreen'"
),
{"d": payload.document_id},
)
await session.commit()
log_db_operation(
log,
"update",
"documents",
document_id=str(payload.document_id),
final_status=doc_status,
final_stage=next_stage,
)
log_message_processing(
log,
"completed",
"prescreen.q",
correlation_id=str(payload.correlation_id),
document_id=str(payload.document_id),
user_id=str(payload.user_id),
decision=decision,
confidence=meta.confidence_score,
contract_type=meta.contract_type,
processing_ms=processing_ms,
next_stage=next_stage,
prescreen_result_id=str(prescreen_result_id) if prescreen_result_id else None,
)
async def _write_auto_approved_report(
self,
*,
session_factory: async_sessionmaker[AsyncSession],
payload: PrescreenRequested,
completed: PrescreenCompleted,
prescreen_result_id: uuid.UUID,
) -> None:
content: dict[str, object] = {
"findings": [
{
"title": "Автоматическая проверка договора",
"severity": "low",
"explanation": completed.auto_summary,
"recommendation": "Договор признан низкорисковым. Перед подписанием рекомендуется визуальная сверка.",
"citations": [],
}
]
}
markdown = (
f"# Автоматическая проверка: {completed.contract_type or 'договор'}\n\n"
f"{completed.auto_summary}\n\n"
"**Решение:** одобрено автоматически (низкий риск).\n"
"**Рекомендация:** визуальная сверка перед подписанием.\n\n"
"---\n\n"
"*Это автоматический отчёт без полного анализа ИИ.*"
)
async with session_factory() as session:
await session.execute(
text(
"INSERT INTO reports "
"(document_id, content_json, markdown, model_used, prompt_tokens, "
" eval_tokens, latency_ms, prescreen_result_id, prescreen_meta) "
"VALUES (:d, CAST(:content AS jsonb), :md, :model, 0, 0, 0, :prid, :pm) "
"ON CONFLICT (document_id) DO UPDATE SET "
" content_json = EXCLUDED.content_json, "
" markdown = EXCLUDED.markdown, "
" model_used = EXCLUDED.model_used, "
" prompt_tokens = EXCLUDED.prompt_tokens, "
" eval_tokens = EXCLUDED.eval_tokens, "
" latency_ms = EXCLUDED.latency_ms, "
" prescreen_result_id = EXCLUDED.prescreen_result_id, "
" prescreen_meta = EXCLUDED.prescreen_meta"
),
{
"d": payload.document_id,
"content": json.dumps(content),
"md": markdown,
"model": "prescreen-auto",
"prid": prescreen_result_id,
"pm": completed.model_dump(),
},
)
await session.commit()
def classify(self, exc: BaseException) -> FailureClass:
from ..core.extraction import UnsupportedFormatError
from ..core.s3.minio_storage import MinioStorageError
if isinstance(exc, MinioStorageError):
return "infra"
if isinstance(exc, (UnicodeDecodeError, UnsupportedFormatError)):
return "extraction_failed"
return "infra"
@staticmethod
def _meta_field_names() -> tuple[str, ...]:
return (
"contract_type",
"party_a",
"party_b",
"total_amount",
"currency",
"start_date",
"end_date",
"has_penalty_clause",
"has_termination_clause",
"has_arbitration",
)
async def on_failure(
self, payload: PrescreenRequested, failure_class: FailureClass, attempt: int, error: str
) -> None:
mq_failed.labels(queue="prescreen", failure_class=failure_class).inc()
log.warning(
"prescreen_retrying",
document_id=str(payload.document_id),
user_id=str(payload.user_id),
attempt=attempt,
failure_class=failure_class,
error=error,
)
async with self._session_factory() as session:
await session.execute(
text(
"UPDATE jobs SET status = 'retrying', attempts = :a, "
"last_failure_class = :fc, last_error = :err "
"WHERE document_id = :d AND queue = 'prescreen'"
),
{
"a": attempt,
"fc": failure_class,
"err": error[:1000],
"d": payload.document_id,
},
)
await session.commit()
async def on_terminal_failure(
self, payload: PrescreenRequested, failure_class: FailureClass, error: str
) -> None:
mq_failed.labels(queue="prescreen", failure_class=failure_class).inc()
async with self._session_factory() as session:
await session.execute(
text(
"UPDATE jobs SET status = 'dlq', dlq = TRUE, "
"last_failure_class = :fc, last_error = :err "
"WHERE document_id = :d AND queue = 'prescreen'"
),
{
"fc": failure_class,
"err": error[:1000],
"d": payload.document_id,
},
)
await session.execute(
text("UPDATE documents SET status = 'failed', stage = :stage WHERE id = :d"),
{"stage": failure_class, "d": payload.document_id},
)
await refund_credit(
session,
payload.document_id,
failure_class,
self._settings.refund_policy,
)
await session.commit()
log.warning(
"prescreen_terminal_failure",
document_id=str(payload.document_id),
user_id=str(payload.user_id),
failure_class=failure_class,
error=error,
)

67
src/contract_check/worker_prescreen/router.py Normal file
View file

@ -0,0 +1,67 @@
"""Routing decision logic for prescreen results.
Conservative by default: auto_approve is disabled until proven by accuracy tests.
When disabled, any auto_approve decision is remapped to manual_review.
"""
from __future__ import annotations
from typing import Literal
from ..core.logging import get_logger
from .extractor import PrescreenContractMeta
log = get_logger(__name__)
type RoutingDecision = Literal["auto_approve", "manual_review", "deep_analysis"]
class PrescreenRouter:
"""Decide where a prescreened document goes next."""
def __init__(
self,
*,
confidence_threshold: float = 0.75,
high_value_threshold: float = 100_000.0,
auto_approve_enabled: bool = False,
) -> None:
self._confidence_threshold = confidence_threshold
self._high_value_threshold = high_value_threshold
self._auto_approve_enabled = auto_approve_enabled
def decide(self, meta: PrescreenContractMeta) -> RoutingDecision:
"""Return one of auto_approve | manual_review | deep_analysis."""
confidence = meta.confidence_score
# Mandatory fields for any automatic routing.
if (
meta.contract_type is None
or meta.party_a is None
or meta.party_b is None
or confidence < self._confidence_threshold
):
return "manual_review"
# High-value or risky clauses → deep LLM analysis.
if (
(meta.total_amount is not None and meta.total_amount >= self._high_value_threshold)
or meta.has_penalty_clause
or meta.has_arbitration
):
return "deep_analysis"
# Conservative default: do not auto-approve until explicitly enabled.
if not self._auto_approve_enabled:
return "manual_review"
return "auto_approve"
def summarize(self, meta: PrescreenContractMeta, decision: RoutingDecision) -> str:
"""Short human-readable summary for auto_approve reports."""
return (
f"{meta.contract_type or 'contract'}: "
f"{meta.party_a or '?'} ↔ {meta.party_b or '?'}, "
f"{meta.total_amount or '—'} {meta.currency or ''} "
f"(confidence {meta.confidence_score:.0%}, routed {decision})."
)

4
srv/worker-extract/Dockerfile
View file

@ -28,12 +28,14 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
WORKDIR /app
# Tesseract and Russian/English language data for scanned PDFs.
# Tesseract and Russian/English language data for scanned PDFs; libmagic1
# backs python-magic content sniffing in the extraction factory.
RUN apt-get update && apt-get install -y --no-install-recommends \
tesseract-ocr \
tesseract-ocr-rus \
tesseract-ocr-eng \
fonts-dejavu-core \
libmagic1 \
&& rm -rf /var/lib/apt/lists/*
COPY --from=builder /app/.venv /app/.venv

34
srv/worker-prescreen/Dockerfile Normal file
View file

@ -0,0 +1,34 @@
# syntax=docker/dockerfile:1
# ─── Stage 1: build deps + project into a venv via uv ─────────────────────────
FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim AS builder
ENV UV_COMPILE_BYTECODE=1 \
UV_LINK_MODE=copy \
UV_PYTHON_DOWNLOADS=never \
UV_PROJECT_ENVIRONMENT=/app/.venv
WORKDIR /app
COPY pyproject.toml uv.lock ./
COPY README.md ./
COPY src ./src
# Install only the prescreen group (core + db/mq/s3/obs + regex engine).
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --group prescreen --no-install-project && \
uv pip install --no-deps .
# ─── Stage 2: lean runtime ─────────────────────────────────────────────────
FROM python:3.13-slim
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PATH=/app/.venv/bin:$PATH
WORKDIR /app
COPY --from=builder /app/.venv /app/.venv
EXPOSE 9104
CMD ["python", "-m", "contract_check.worker_prescreen"]

1
tests/integration/test_b2b_api.py
View file

@ -13,6 +13,7 @@ from pathlib import Path
import aio_pika
import httpx
import pytest
from tests.integration.conftest import user_token
pytestmark = pytest.mark.integration

283
tests/integration/test_prescreen_worker.py Normal file
View file

@ -0,0 +1,283 @@
"""Integration tests for worker-prescreen (hybrid extraction).
Uses the same Docker Compose infra as test_analyze_worker: real Postgres,
RabbitMQ, MinIO; the LLM is stubbed at the provider port.
Scenarios (plan Phase 4):
1. Heuristic-only (fallback disabled by default): prescreen_results row
persisted with extractor_version='heuristic-v2', routing unchanged.
2. Fallback enabled + low-confidence text + stub provider: merged meta
persisted with extractor_version='hybrid-llm-v1'.
3. Fallback failure: heuristic result kept, error recorded in auto_findings.
"""
from __future__ import annotations
import json
import uuid
from typing import TYPE_CHECKING, Any
import pytest
from sqlalchemy import text
from contract_check.core.db.session import create_session_factory
from contract_check.core.mq.messages import PrescreenRequested
from contract_check.core.s3 import extracted_key
from contract_check.core.s3.minio_storage import MinioStorage
from contract_check.worker_prescreen.extractor_heuristic import HeuristicExtractor
from contract_check.worker_prescreen.extractor_hybrid import HybridMetaExtractor
from contract_check.worker_prescreen.handler import PrescreenHandler
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
pytestmark = pytest.mark.integration
FULL_CONTRACT = """\
ДОГОВОР ПОСТАВКИ № 42
г. Москва
Общество с ограниченной ответственностью «Продавец»,
именуемое в дальнейшем «Поставщик», с одной стороны, и
Общество с ограниченной ответственностью «Покупатель»,
с другой стороны, заключили настоящий договор.
1. Предмет договора
Поставщик обязуется передать в собственность Покупателю товар.
2. Цена и порядок расчётов
2.1. Общая стоимость товара составляет 1 250 000 рублей.
3. Срок действия договора
3.1. Договор вступает в силу с 01.09.2025 и действует по 31.08.2026.
4. Ответственность сторон
4.1. За просрочку поставки Поставщик уплачивает неустойку.
5. Порядок разрешения споров
5.1. Споры подлежат рассмотрению в Арбитражном суде г. Москвы.
"""
LOW_CONF_TEXT = "Скан документа без распознанных реквизитов."
# The compose stack may have live workers consuming analyze.q; publish test
# messages to an unrouted key so nothing downstream races our assertions.
_DEAD_ROUTING_KEY = "it-test-unrouted"
class StubProvider:
"""LLMProvider stub returning a fixed prescreen dict (or raising)."""
def __init__(self, payload: dict[str, Any] | Exception) -> None:
self._payload = payload
async def analyze(self, text: str, *, checklist: str, extra_context: str = "") -> Any:
raise NotImplementedError
async def extract_prescreen(self, text: str) -> dict[str, Any]:
if isinstance(self._payload, Exception):
raise self._payload
return self._payload
async def aclose(self) -> None:
pass
LLM_DICT: dict[str, Any] = {
"contract_type": "services",
"party_a": "ИП Петров Петр Петрович",
"party_b": "ООО «Клиент»",
"total_amount": 50000.0,
"currency": "RUB",
"has_penalty_clause": True,
}
def _storage(infra: dict[str, str]) -> MinioStorage:
return MinioStorage.from_endpoint_url(
endpoint_url=infra["s3_endpoint_url"],
access_key=infra["s3_access_key"],
secret_key=infra["s3_secret_key"],
bucket="contract-check-docs",
)
async def _seed_document(
infra: dict[str, str], *, telegram_id: int, document_id: uuid.UUID, contract_text: str
) -> tuple[uuid.UUID, str]:
"""Insert user + post-extraction document; return (user_id, text_s3_key)."""
sess = create_session_factory()
store = _storage(infra)
async with sess() as session:
result = await session.execute(
text(
"INSERT INTO users (telegram_id, credits_left) VALUES (:t, 5) "
"ON CONFLICT (telegram_id) DO UPDATE SET credits_left = 5 "
"RETURNING id"
),
{"t": telegram_id},
)
user_id = result.scalar_one()
await session.commit()
ext_key = extracted_key(str(user_id), str(document_id))
await store.put(
ext_key, contract_text.encode("utf-8"), content_type="text/plain; charset=utf-8"
)
async with sess() as session:
await session.execute(
text(
"INSERT INTO documents "
"(id, user_id, s3_key, extracted_s3_key, filename, mime, bytes, "
" status, stage) "
"VALUES (:id, :uid, :s3, :ext, 'contract.pdf', 'application/pdf', :bytes, "
" 'prescreening', 'extracted')"
),
{
"id": document_id,
"uid": user_id,
"s3": f"users/{user_id}/docs/{document_id}.pdf",
"ext": ext_key,
"bytes": len(contract_text),
},
)
await session.commit()
return user_id, ext_key
def _requested(
payload_doc: uuid.UUID, user_id: uuid.UUID, ext_key: str, text: str
) -> PrescreenRequested:
return PrescreenRequested(
correlation_id=uuid.uuid4(),
document_id=payload_doc,
user_id=user_id,
text_s3_key=ext_key,
filename="contract.pdf",
char_count=len(text),
)
async def _prescreen_row(sess: async_sessionmaker[AsyncSession], document_id: uuid.UUID) -> Any:
async with sess() as session:
result = await session.execute(
text(
"SELECT contract_type, party_a, total_amount, currency, confidence_score, "
" routing_decision, extractor_version, auto_findings "
"FROM prescreen_results WHERE document_id = :d"
),
{"d": document_id},
)
return result.one()
async def test_prescreen_heuristic_persists_version_and_routes(
infra: dict[str, str],
) -> None:
sess = create_session_factory()
document_id = uuid.uuid4()
user_id, ext_key = await _seed_document(
infra, telegram_id=777_111_222, document_id=document_id, contract_text=FULL_CONTRACT
)
handler = PrescreenHandler(session_factory=sess, publish_routing_key=_DEAD_ROUTING_KEY)
try:
await handler.handle(_requested(document_id, user_id, ext_key, FULL_CONTRACT))
finally:
await handler.aclose()
row = await _prescreen_row(sess, document_id)
(ctype, party_a, amount, currency, confidence, decision, version, findings) = row
assert ctype == "supply"
assert "Продавец" in party_a
assert amount == 1250000.0
assert currency == "RUB"
assert confidence == 1.0
assert decision == "deep_analysis" # high-value + penalty + arbitration
assert version == "heuristic-v2" # fallback disabled by default
assert findings == []
async with sess() as session:
doc = await session.execute(
text("SELECT status, stage FROM documents WHERE id = :d"), {"d": document_id}
)
status, stage = doc.one()
assert status == "analyzing"
assert stage == "queued_analyze"
async def test_prescreen_llm_fallback_merged_and_persisted(
infra: dict[str, str],
) -> None:
sess = create_session_factory()
document_id = uuid.uuid4()
user_id, ext_key = await _seed_document(
infra, telegram_id=777_333_444, document_id=document_id, contract_text=LOW_CONF_TEXT
)
handler = PrescreenHandler(
session_factory=sess,
publish_routing_key=_DEAD_ROUTING_KEY,
extractor=HybridMetaExtractor(
provider=StubProvider(LLM_DICT),
fallback_enabled=True,
fallback_threshold=0.99,
heuristic=HeuristicExtractor(),
),
)
await handler.handle(_requested(document_id, user_id, ext_key, LOW_CONF_TEXT))
row = await _prescreen_row(sess, document_id)
(ctype, party_a, amount, currency, confidence, decision, version, findings) = row
assert ctype == "services" # recovered by the LLM stage
assert party_a == "ИП Петров Петр Петрович"
assert amount == 50000.0
assert currency == "RUB"
assert confidence >= 0.75
assert decision == "deep_analysis" # has_penalty_clause OR-merged to True
assert version == "hybrid-llm-v1"
assert findings == []
async def test_prescreen_llm_failure_records_error_not_fatal(
infra: dict[str, str],
) -> None:
sess = create_session_factory()
document_id = uuid.uuid4()
user_id, ext_key = await _seed_document(
infra, telegram_id=777_555_666, document_id=document_id, contract_text=LOW_CONF_TEXT
)
handler = PrescreenHandler(
session_factory=sess,
publish_routing_key=_DEAD_ROUTING_KEY,
extractor=HybridMetaExtractor(
provider=StubProvider(RuntimeError("quota exceeded")),
fallback_enabled=True,
fallback_threshold=0.99,
heuristic=HeuristicExtractor(),
),
)
# Must not raise: the heuristic result survives the LLM failure.
await handler.handle(_requested(document_id, user_id, ext_key, LOW_CONF_TEXT))
row = await _prescreen_row(sess, document_id)
(ctype, _party_a, _amount, _cur, confidence, decision, version, findings) = row
assert ctype is None
assert confidence < 0.75
assert decision == "manual_review" # routing semantics unchanged
assert version == "heuristic-v2"
parsed = findings if isinstance(findings, list) else json.loads(findings)
assert parsed and "llm_fallback_error" in parsed[0]
assert "quota exceeded" in parsed[0]["llm_fallback_error"]
async with sess() as session:
doc = await session.execute(
text("SELECT status, stage FROM documents WHERE id = :d"), {"d": document_id}
)
status, stage = doc.one()
assert status == "manual_review"
assert stage == "manual_review"

1
tests/integration/test_upload_pipeline.py
View file

@ -18,6 +18,7 @@ from typing import TYPE_CHECKING
import aio_pika
import httpx
import pytest
from tests.integration.conftest import user_token
if TYPE_CHECKING:

102
tests/unit/test_extract_handler.py Normal file
View file

@ -0,0 +1,102 @@
"""Unit tests for worker-extract handler.
Covers the `PRESCREEN_ENABLED` toggle: when true it publishes
`PrescreenRequested` to `prescreen.q` and marks `prescreening`; when false it
bypasses the prescreen stage, publishes `AnalyzeRequested` to `analyze.q`,
and marks `analyzing`.
"""
from __future__ import annotations
import uuid
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
from contract_check.core.extraction import ExtractedDocument
from contract_check.core.mq.messages import DocumentExtracted, DocumentUploaded, PrescreenRequested
from contract_check.worker_extract.handler import ExtractHandler
def _fake_session_factory() -> Any:
session = MagicMock()
session.execute = AsyncMock()
session.commit = AsyncMock()
session.__aenter__ = AsyncMock(return_value=session)
session.__aexit__ = AsyncMock(return_value=False)
return MagicMock(return_value=session), session
def _uploaded() -> DocumentUploaded:
return DocumentUploaded(
correlation_id=uuid.uuid4(),
document_id=uuid.uuid4(),
user_id=uuid.uuid4(),
s3_key="users/u/docs/d.pdf",
filename="contract.pdf",
mime="application/pdf",
)
def _extracted() -> ExtractedDocument:
return ExtractedDocument(
markdown="Договор поставки. Сумма 1000 рублей.",
metadata={"format": "pdf", "is_scan": False, "has_tables": False},
is_structured=True,
)
@pytest.fixture
def handler(monkeypatch: pytest.MonkeyPatch) -> tuple[ExtractHandler, Any, Any]:
session_factory, session = _fake_session_factory()
h = ExtractHandler(session_factory=session_factory)
# Stub S3 put and the extraction core.
h._storage = MagicMock(put=AsyncMock())
monkeypatch.setattr(h, "_extract_text", AsyncMock(return_value=(_extracted().markdown, False)))
# Stub publisher.
publisher = MagicMock(publish=AsyncMock())
h._publisher = publisher
# DB status is non-terminal.
monkeypatch.setattr(h, "_db_status", AsyncMock(return_value="queued"))
return h, session, publisher
@pytest.mark.asyncio
async def test_prescreen_enabled_publishes_to_prescreen(
handler: tuple[ExtractHandler, Any, Any],
) -> None:
h, session, publisher = handler
h._settings.prescreen_enabled = True
await h.handle(_uploaded())
assert publisher.publish.call_count == 1
msg, *_ = publisher.publish.call_args.args
assert isinstance(msg, PrescreenRequested)
assert publisher.publish.call_args.kwargs["routing_key"] == "prescreen"
status_params = session.execute.call_args_list[-2].args[1]
assert status_params["status"] == "prescreening"
@pytest.mark.asyncio
async def test_prescreen_disabled_bypasses_to_analyze(
handler: tuple[ExtractHandler, Any, Any],
) -> None:
h, session, publisher = handler
h._settings.prescreen_enabled = False
await h.handle(_uploaded())
assert publisher.publish.call_count == 1
msg, *_ = publisher.publish.call_args.args
assert isinstance(msg, DocumentExtracted)
assert publisher.publish.call_args.kwargs["routing_key"] == "analyze"
status_params = session.execute.call_args_list[-2].args[1]
assert status_params["status"] == "analyzing"

195
tests/unit/test_extraction_adapters.py Normal file
View file

@ -0,0 +1,195 @@
"""Unit tests for the extraction adapters (core/extraction/adapters/*).
Real files are generated in-memory per format (pymupdf/python-docx) so no
binary fixtures live in the repo. The OCR adapter's engine-dependent path is
covered by the integration suite; here we only assert the error contract.
"""
from __future__ import annotations
import pytest
from contract_check.core.extraction import (
ExtractedDocument,
ExtractionFailedError,
MammothDocxExtractor,
PyMuPDFExtractor,
RtfExtractor,
TesseractOcrExtractor,
TxtExtractor,
)
# ── shared fixture builders ──────────────────────────────────────────────────
PARAGRAPH = (
"Стороны обязуются выполнять условия договора. "
"Сторона А обязуется передать товар в срок. "
"Сторона Б обязуется оплатить товар в течение десяти банковских дней. "
"Ответственность сторон ограничена суммой договора. "
"Споры подлежат рассмотрению в арбитражном суде города Москвы. "
)
def build_pdf_bytes(*, with_table: bool = False) -> bytes:
import pymupdf
doc = pymupdf.open()
page = doc.new_page()
if with_table:
page.insert_htmlbox(
page.rect,
f'<p style="font-family:DejaVu Sans;font-size:12px">{PARAGRAPH}</p>',
)
x0, y0, cw, ch = 50, 200, 150, 30
data = [["Услуга", "Цена"], ["Консультация", "5000"], ["Аудит", "15000"]]
for r in range(len(data) + 1):
page.draw_line(pymupdf.Point(x0, y0 + r * ch), pymupdf.Point(x0 + 2 * cw, y0 + r * ch))
for c in range(3):
page.draw_line(pymupdf.Point(x0 + c * cw, y0), pymupdf.Point(x0 + c * cw, y0 + 3 * ch))
for r, row in enumerate(data):
for c, val in enumerate(row):
rect = pymupdf.Rect(
x0 + c * cw + 3, y0 + r * ch + 3, x0 + (c + 1) * cw - 3, y0 + (r + 1) * ch - 3
)
page.insert_htmlbox(
rect, f'<span style="font-family:DejaVu Sans;font-size:10px">{val}</span>'
)
else:
page.insert_htmlbox(
page.rect, f'<p style="font-family:DejaVu Sans;font-size:12px">{PARAGRAPH}</p>'
)
data = doc.tobytes()
doc.close()
return data
def build_docx_bytes() -> bytes:
import io
from docx import Document
doc = Document()
doc.add_heading("Договор поставки", level=1)
doc.add_paragraph(PARAGRAPH)
doc.add_paragraph(PARAGRAPH)
buf = io.BytesIO()
doc.save(buf)
return buf.getvalue()
# ── PDF (pymupdf) ────────────────────────────────────────────────────────────
def test_pdf_text_only() -> None:
result = PyMuPDFExtractor().extract(build_pdf_bytes())
assert "арбитражном суде" in result.markdown
assert result.is_structured is False
assert result.metadata["format"] == "pdf"
assert result.metadata["has_tables"] is False
def test_pdf_table_becomes_markdown_pipes() -> None:
result = PyMuPDFExtractor().extract(build_pdf_bytes(with_table=True))
assert result.is_structured is True
assert result.metadata["has_tables"] is True
# Header row + separator + data rows as pipes; no cell-text duplication.
assert "| Услуга | Цена |" in result.markdown
assert "| Консультация | 5000 |" in result.markdown
assert result.markdown.count("Консультация") == 1
def test_pdf_too_short_raises() -> None:
import pymupdf
doc = pymupdf.open()
doc.new_page()
data = doc.tobytes()
doc.close()
with pytest.raises(ExtractionFailedError, match="слишком мало"):
PyMuPDFExtractor().extract(data)
def test_pdf_garbage_raises() -> None:
with pytest.raises(ExtractionFailedError):
PyMuPDFExtractor().extract(b"definitely not a pdf")
# ── DOCX (mammoth) ───────────────────────────────────────────────────────────
def test_docx_heading_preserved() -> None:
result = MammothDocxExtractor().extract(build_docx_bytes())
assert result.markdown.startswith("# Договор поставки")
assert result.is_structured is True
assert "арбитражном суде" in result.markdown
def test_docx_garbage_raises() -> None:
with pytest.raises(ExtractionFailedError):
MammothDocxExtractor().extract(b"not a zip")
# ── RTF (striprtf) ───────────────────────────────────────────────────────────
def test_rtf_extracted_as_plain_text() -> None:
body = " ".join(f"Clause {i}: the parties agree to the terms herein." for i in range(30))
data = r"{\rtf1\ansi\deff0 " + body.replace("\n", r"\par ") + "}"
result = RtfExtractor().extract(data.encode("ascii"))
assert "the parties agree" in result.markdown
assert result.is_structured is False
def test_rtf_missing_header_raises() -> None:
with pytest.raises(ExtractionFailedError, match="RTF"):
RtfExtractor().extract(b"just some text that is long enough " * 10)
# ── TXT (chardet) ────────────────────────────────────────────────────────────
def test_txt_windows1251_detected() -> None:
data = (PARAGRAPH * 10).encode("windows-1251")
result = TxtExtractor().extract(data)
assert "арбитражном суде" in result.markdown # decoded, not mojibake
assert result.metadata["encoding"] == "windows-1251"
assert result.is_structured is False
def test_txt_utf8() -> None:
result = TxtExtractor().extract((PARAGRAPH * 5).encode("utf-8"))
assert "Стороны" in result.markdown
assert result.metadata["encoding"] in ("utf-8", "utf-8-sig")
def test_txt_too_short_raises() -> None:
with pytest.raises(ExtractionFailedError):
TxtExtractor().extract("коротко".encode())
# ── OCR (tesseract) — error contract only; happy path is integration ─────────
# Both cases fail at open/decode, before the engine is invoked, so they run
# regardless of whether the tesseract binary is installed.
def test_ocr_garbage_pdf_raises() -> None:
from contract_check.core.analysis.ocr import OCRError
with pytest.raises((OCRError, ExtractionFailedError)):
TesseractOcrExtractor().extract(b"%PDF-1.4 garbage")
def test_ocr_garbage_image_raises() -> None:
from contract_check.core.analysis.ocr import OCRError
with pytest.raises((OCRError, ExtractionFailedError)):
TesseractOcrExtractor().extract(b"\x89PNG\r\n\x1a\n not really a png")
# ── ExtractedDocument DTO ────────────────────────────────────────────────────
def test_extracted_document_defaults() -> None:
doc = ExtractedDocument(markdown="x")
assert doc.is_structured is False
assert doc.metadata == {}

116
tests/unit/test_extraction_factory.py Normal file
View file

@ -0,0 +1,116 @@
"""Unit tests for ExtractorFactory + detect_format + the OCR-fallback flow."""
from __future__ import annotations
import pytest
from contract_check.core.extraction import (
UnsupportedFormatError,
detect_format,
extract_document,
get_factory,
)
from tests.unit.test_extraction_adapters import PARAGRAPH, build_docx_bytes, build_pdf_bytes
# ── detect_format: suffix / mime / magic precedence ──────────────────────────
@pytest.mark.parametrize(
("filename", "expected"),
[
("contract.pdf", "pdf"),
("contract.PDF", "pdf"),
("contract.docx", "docx"),
("contract.rtf", "rtf"),
("contract.txt", "txt"),
("data.csv", "txt"),
("scan.png", "image"),
("scan.jpg", "image"),
("scan.jpeg", "image"),
("scan.tiff", "image"),
],
)
def test_detect_by_suffix(filename: str, expected: str) -> None:
assert detect_format(b"", filename=filename) == expected
def test_detect_by_mime_when_suffix_unknown() -> None:
assert detect_format(b"", mime="application/pdf", filename="contract") == "pdf"
assert detect_format(b"", mime="image/png", filename="photo") == "image"
def test_magic_beats_wrong_suffix() -> None:
# A real PDF mislabeled .docx must route to the PDF adapter.
pytest.importorskip("magic")
assert detect_format(build_pdf_bytes(), mime="", filename="contract.docx") == "pdf"
def test_unsupported_format_raises() -> None:
with pytest.raises(UnsupportedFormatError):
detect_format(b"", filename="archive.zip")
binary_soup = bytes(range(256)) * 4 # deliberately not decodable text
with pytest.raises(UnsupportedFormatError):
detect_format(binary_soup, mime="application/x-msdownload", filename="doc.exe")
def test_docx_zip_magic_does_not_shadow_suffix() -> None:
# DOCX is a zip; python-magic reports application/zip which the factory
# deliberately does not map — the suffix must decide.
pytest.importorskip("magic")
assert detect_format(build_docx_bytes(), filename="contract.docx") == "docx"
# ── factory ──────────────────────────────────────────────────────────────────
def test_factory_returns_per_format_adapters() -> None:
factory = get_factory()
assert type(factory.get_extractor(b"", filename="a.pdf")).__name__ == "PyMuPDFExtractor"
assert type(factory.get_extractor(b"", filename="a.docx")).__name__ == "MammothDocxExtractor"
assert type(factory.get_extractor(b"", filename="a.rtf")).__name__ == "RtfExtractor"
assert type(factory.get_extractor(b"", filename="a.txt")).__name__ == "TxtExtractor"
assert type(factory.get_extractor(b"", filename="a.png")).__name__ == "TesseractOcrExtractor"
# ── extract_document: end-to-end over bytes ──────────────────────────────────
def test_extract_document_pdf() -> None:
result = extract_document(build_pdf_bytes(), filename="c.pdf")
assert result.metadata["format"] == "pdf"
assert "арбитражном суде" in result.markdown
def test_extract_document_docx() -> None:
result = extract_document(build_docx_bytes(), filename="c.docx")
assert result.markdown.startswith("# ")
assert result.is_structured is True
def test_extract_document_txt_cp1251() -> None:
result = extract_document((PARAGRAPH * 10).encode("windows-1251"), filename="c.txt")
assert "Стороны" in result.markdown
assert result.metadata["format"] == "txt"
def test_extract_document_unsupported() -> None:
with pytest.raises(UnsupportedFormatError):
extract_document(b"PK\x03\x04 whatever", filename="c.zip")
def test_extract_document_pdf_scan_falls_back_to_ocr() -> None:
# A PDF with no text layer triggers the OCR fallback inside the
# orchestrator. Without the tesseract engine the fallback raises
# OCRError (retryable) — with the engine, the integration suite covers
# the happy path.
import pymupdf
from contract_check.core.analysis.ocr import OCRError
from contract_check.core.extraction import ExtractionFailedError
doc = pymupdf.open()
doc.new_page() # blank page: no text at all
data = doc.tobytes()
doc.close()
with pytest.raises((OCRError, ExtractionFailedError)):
extract_document(data, filename="scan.pdf")

164
tests/unit/test_llm_prescreen_extraction.py Normal file
View file

@ -0,0 +1,164 @@
"""extract_prescreen provider tests (respx-mocked, both adapters).
Covers (plan Phase 2): valid dict passthrough, prescreen_max_chars input cap,
malformed JSON → repair once → success, and malformed after repair → LLMError.
"""
from __future__ import annotations
import json
from typing import Any
import httpx
import pytest
import respx
from contract_check.core.llm.ollama_cloud import LLMError, OllamaCloudProvider
from contract_check.core.llm.yandex_gpt import YandexGPTProvider
OLLAMA_HOST = "https://ollama.test"
OLLAMA_URL = f"{OLLAMA_HOST}/api/chat"
YANDEX_HOST = "https://llm.test"
YANDEX_URL = f"{YANDEX_HOST}/foundationModels/v1/completion"
VALID_PRESCREEN: dict[str, Any] = {
"contract_type": "supply",
"party_a": "ООО «Альфа»",
"party_b": "ООО «Бета»",
"total_amount": 1250000.0,
"currency": "RUB",
"start_date": "2025-09-01",
"end_date": "2026-08-31",
"has_penalty_clause": True,
"has_termination_clause": False,
"has_arbitration": True,
}
def _ollama_body(content: str) -> dict[str, object]:
return {"message": {"content": content}, "prompt_eval_count": 5, "eval_count": 7}
def _yandex_body(content: str) -> dict[str, object]:
return {
"result": {
"alternatives": [{"message": {"role": "assistant", "text": content}}],
"usage": {"inputTextTokens": 5, "completionTokens": 7, "totalTokens": 12},
}
}
# ── Ollama Cloud ─────────────────────────────────────────────────────────────
async def test_ollama_extract_prescreen_returns_dict() -> None:
provider = OllamaCloudProvider(host=OLLAMA_HOST, api_key="key", model="m")
async with provider:
with respx.mock(base_url=OLLAMA_HOST) as mock:
route = mock.post(OLLAMA_URL).mock(
return_value=httpx.Response(200, json=_ollama_body(json.dumps(VALID_PRESCREEN)))
)
result = await provider.extract_prescreen("Договор поставки...")
assert route.call_count == 1
assert result == VALID_PRESCREEN
async def test_ollama_extract_prescreen_truncates_input() -> None:
provider = OllamaCloudProvider(
host=OLLAMA_HOST, api_key="key", model="m", prescreen_max_chars=100
)
async with provider:
with respx.mock(base_url=OLLAMA_HOST) as mock:
route = mock.post(OLLAMA_URL).mock(
return_value=httpx.Response(200, json=_ollama_body(json.dumps(VALID_PRESCREEN)))
)
await provider.extract_prescreen("договор " * 500)
body = json.loads(route.calls.last.request.read())
user_msg = body["messages"][1]["content"]
assert len(user_msg) == 100
assert body["messages"][0]["role"] == "system"
assert "метаданных" in body["messages"][0]["content"]
async def test_ollama_extract_prescreen_repairs_invalid_json() -> None:
provider = OllamaCloudProvider(host=OLLAMA_HOST, api_key="key", model="m")
async with provider:
with respx.mock(base_url=OLLAMA_HOST) as mock:
route = mock.post(OLLAMA_URL).mock(
side_effect=[
httpx.Response(200, json=_ollama_body("not json {")),
httpx.Response(200, json=_ollama_body(json.dumps(VALID_PRESCREEN))),
]
)
result = await provider.extract_prescreen("Договор...")
assert route.call_count == 2
assert result == VALID_PRESCREEN
async def test_ollama_extract_prescreen_invalid_after_repair_raises() -> None:
provider = OllamaCloudProvider(host=OLLAMA_HOST, api_key="key", model="m")
async with provider:
with respx.mock(base_url=OLLAMA_HOST) as mock:
mock.post(OLLAMA_URL).mock(
return_value=httpx.Response(200, json=_ollama_body("still not json"))
)
with pytest.raises(LLMError):
await provider.extract_prescreen("Договор...")
async def test_ollama_extract_prescreen_ignores_extra_fields() -> None:
"""Hallucinated fields are dropped by the wire model (extra=ignore)."""
payload = {**VALID_PRESCREEN, "risk_score": "high", "summary": "безопасен"}
provider = OllamaCloudProvider(host=OLLAMA_HOST, api_key="key", model="m")
async with provider:
with respx.mock(base_url=OLLAMA_HOST) as mock:
mock.post(OLLAMA_URL).mock(
return_value=httpx.Response(200, json=_ollama_body(json.dumps(payload)))
)
result = await provider.extract_prescreen("Договор...")
assert "risk_score" not in result
assert result["contract_type"] == "supply"
# ── YandexGPT ────────────────────────────────────────────────────────────────
async def test_yandex_extract_prescreen_returns_dict() -> None:
provider = YandexGPTProvider(api_key="key", folder_id="b1", base_url=YANDEX_HOST)
async with provider:
with respx.mock(base_url=YANDEX_HOST) as mock:
route = mock.post(YANDEX_URL).mock(
return_value=httpx.Response(200, json=_yandex_body(json.dumps(VALID_PRESCREEN)))
)
result = await provider.extract_prescreen("Договор поставки...")
assert route.call_count == 1
assert result == VALID_PRESCREEN
async def test_yandex_extract_prescreen_truncates_input() -> None:
provider = YandexGPTProvider(
api_key="key", folder_id="b1", base_url=YANDEX_HOST, prescreen_max_chars=100
)
async with provider:
with respx.mock(base_url=YANDEX_HOST) as mock:
route = mock.post(YANDEX_URL).mock(
return_value=httpx.Response(200, json=_yandex_body(json.dumps(VALID_PRESCREEN)))
)
await provider.extract_prescreen("договор " * 500)
body = json.loads(route.calls.last.request.read())
assert len(body["messages"][1]["text"]) == 100
async def test_yandex_extract_prescreen_repairs_invalid_json() -> None:
provider = YandexGPTProvider(api_key="key", folder_id="b1", base_url=YANDEX_HOST)
async with provider:
with respx.mock(base_url=YANDEX_HOST) as mock:
route = mock.post(YANDEX_URL).mock(
side_effect=[
httpx.Response(200, json=_yandex_body("{broken")),
httpx.Response(200, json=_yandex_body(json.dumps(VALID_PRESCREEN))),
]
)
result = await provider.extract_prescreen("Договор...")
assert route.call_count == 2
assert result == VALID_PRESCREEN

294
tests/unit/test_llm_yandex_gpt.py Normal file
View file

@ -0,0 +1,294 @@
"""YandexGPT adapter unit tests (respx-mocked).
Covers: 200 happy path, 429→fallback, invalid-JSON→repair→success,
auth/config errors, and the provider returning a merged AnalysisResult.
No network.
"""
from __future__ import annotations
import json
import httpx
import pytest
import respx
from contract_check.core.analysis.report_schema import ReportPayload
from contract_check.core.llm.yandex_gpt import (
LLMConfigError,
LLMError,
LLMQuotaError,
LLMUnavailableError,
YandexGPTProvider,
)
HOST = "https://llm.test"
URL = f"{HOST}/foundationModels/v1/completion"
FOLDER_ID = "b1folder"
MODEL = "yandexgpt-lite"
FALLBACK_MODEL = "yandexgpt"
VALID_PAYLOAD = ReportPayload(
findings=[
{
"checklist_id": "penalties",
"severity": "high",
"quote": "Штраф 0,5% за каждый день просрочки",
"section_ref": "п. 6.3",
"risk": "Высокая неустойка",
"recommendation": "Ограничить cap",
}
]
)
def _completion_body(content: str, model: str = MODEL) -> dict[str, object]:
return {
"result": {
"alternatives": [{"message": {"role": "assistant", "text": content}}],
"usage": {
"inputTextTokens": 100,
"completionTokens": 20,
"totalTokens": 120,
},
"modelVersion": model,
}
}
def _provider(fallback: str | None = FALLBACK_MODEL) -> YandexGPTProvider:
return YandexGPTProvider(
api_key="key",
folder_id=FOLDER_ID,
model=MODEL,
fallback_model=fallback,
base_url=HOST,
completion_path="/foundationModels/v1/completion",
max_concurrency=1,
chunk_size=10000,
)
@pytest.mark.asyncio
async def test_happy_path_returns_findings() -> None:
async with _provider() as provider:
with respx.mock(base_url=HOST) as mock:
route = mock.post("/foundationModels/v1/completion")
route.mock(
return_value=httpx.Response(
200, json=_completion_body(VALID_PAYLOAD.model_dump_json())
)
)
result = await provider.analyze("договор " * 200)
assert len(result.findings) == 1
assert result.findings[0].checklist_id == "penalties"
assert result.findings[0].severity == "high"
assert result.prompt_tokens == 100
assert result.eval_tokens == 20
assert result.fell_back is False
assert result.model_used == MODEL
@pytest.mark.asyncio
async def test_429_falls_back_to_secondary_model() -> None:
async with _provider() as provider:
with respx.mock(base_url=HOST) as mock:
route = mock.post("/foundationModels/v1/completion")
route.mock(
side_effect=[
httpx.Response(429, json={"error": "quota"}),
httpx.Response(
200,
json=_completion_body(
VALID_PAYLOAD.model_dump_json(), model=FALLBACK_MODEL
),
),
]
)
result = await provider.analyze("договор " * 200)
assert result.fell_back is True
assert FALLBACK_MODEL in result.models_used
@pytest.mark.asyncio
async def test_invalid_json_repaired_on_second_attempt() -> None:
async with _provider() as provider:
with respx.mock(base_url=HOST) as mock:
route = mock.post("/foundationModels/v1/completion")
route.mock(
side_effect=[
httpx.Response(200, json=_completion_body("not valid json {")),
httpx.Response(200, json=_completion_body(VALID_PAYLOAD.model_dump_json())),
]
)
result = await provider.analyze("договор " * 200)
assert result.repaired is True
assert len(result.findings) == 1
@pytest.mark.asyncio
async def test_model_aliases_validate_without_repair() -> None:
"""Models often emit risk_type/description and omit recommendation."""
aliased = json.dumps(
{
"findings": [
{
"risk_type": "penalties",
"severity": "medium",
"description": "Высокая неустойка.",
"quote": "Штраф 0,5% за каждый день просрочки",
"section_ref": "п. 6.3",
}
]
}
)
async with _provider() as provider:
with respx.mock(base_url=HOST) as mock:
route = mock.post("/foundationModels/v1/completion")
route.mock(return_value=httpx.Response(200, json=_completion_body(aliased)))
result = await provider.analyze("договор " * 200)
assert route.call_count == 1
assert result.repaired is False
assert result.findings[0].checklist_id == "penalties"
assert result.findings[0].risk == "Высокая неустойка."
assert result.findings[0].recommendation == ""
@pytest.mark.asyncio
async def test_429_with_no_fallback_raises_quota() -> None:
provider = YandexGPTProvider(
api_key="key",
folder_id=FOLDER_ID,
model=MODEL,
fallback_model=None,
base_url=HOST,
completion_path="/foundationModels/v1/completion",
)
async with provider:
with respx.mock(base_url=HOST) as mock:
mock.post("/foundationModels/v1/completion").mock(
return_value=httpx.Response(429, json={"error": "quota"})
)
with pytest.raises(LLMQuotaError):
await provider.analyze("договор " * 200)
@pytest.mark.asyncio
async def test_invalid_json_after_repair_raises_llm_error() -> None:
async with _provider() as provider:
with respx.mock(base_url=HOST) as mock:
mock.post("/foundationModels/v1/completion").mock(
return_value=httpx.Response(200, json=_completion_body("still not json"))
)
with pytest.raises(LLMError):
await provider.analyze("договор " * 200)
@pytest.mark.asyncio
async def test_short_text_single_chunk_one_request() -> None:
async with _provider() as provider:
with respx.mock(base_url=HOST) as mock:
route = mock.post("/foundationModels/v1/completion")
route.mock(
return_value=httpx.Response(
200, json=_completion_body(VALID_PAYLOAD.model_dump_json())
)
)
await provider.analyze("короткий договор")
assert route.call_count == 1
@pytest.mark.asyncio
async def test_multi_chunk_merges_findings() -> None:
findings = json.loads(VALID_PAYLOAD.model_dump_json())["findings"]
async with _provider() as provider:
with respx.mock(base_url=HOST) as mock:
mock.post("/foundationModels/v1/completion").mock(
return_value=httpx.Response(
200,
json=_completion_body(json.dumps({"findings": findings + findings})),
)
)
result = await provider.analyze("договор " * 3000, checklist="x")
assert len(result.findings) == 1
@pytest.mark.asyncio
async def test_401_raises_config_error() -> None:
async with _provider() as provider:
with respx.mock(base_url=HOST) as mock:
route = mock.post("/foundationModels/v1/completion")
route.mock(return_value=httpx.Response(401, json={"error": "unauthorized"}))
with pytest.raises(LLMConfigError):
await provider.analyze("договор " * 200)
assert route.call_count == 1
@pytest.mark.asyncio
async def test_400_raises_config_error() -> None:
async with _provider() as provider:
with respx.mock(base_url=HOST) as mock:
route = mock.post("/foundationModels/v1/completion")
route.mock(return_value=httpx.Response(400, json={"error": "bad request"}))
with pytest.raises(LLMConfigError):
await provider.analyze("договор " * 200)
assert route.call_count == 1
@pytest.mark.asyncio
async def test_404_raises_config_error() -> None:
async with _provider() as provider:
with respx.mock(base_url=HOST) as mock:
route = mock.post("/foundationModels/v1/completion")
route.mock(return_value=httpx.Response(404, text="Not Found"))
with pytest.raises(LLMConfigError):
await provider.analyze("договор " * 200)
assert route.call_count == 1
@pytest.mark.asyncio
async def test_connect_error_raises_config_error_immediately() -> None:
async with _provider() as provider:
with respx.mock(base_url=HOST) as mock:
route = mock.post("/foundationModels/v1/completion")
route.mock(side_effect=httpx.ConnectError("Connection refused"))
with pytest.raises(LLMConfigError):
await provider.analyze("договор " * 200)
assert route.call_count == 1
@pytest.mark.asyncio
async def test_503_retries_then_raises_unavailable() -> None:
async with _provider() as provider:
with respx.mock(base_url=HOST) as mock:
route = mock.post("/foundationModels/v1/completion")
route.mock(return_value=httpx.Response(503, text="Unavailable"))
with pytest.raises(LLMUnavailableError):
await provider.analyze("договор " * 200)
assert route.call_count == 3
@pytest.mark.asyncio
async def test_request_payload_contains_model_uri_and_json_schema() -> None:
async with _provider(fallback=None) as provider:
with respx.mock(base_url=HOST) as mock:
route = mock.post("/foundationModels/v1/completion")
route.mock(
return_value=httpx.Response(
200, json=_completion_body(VALID_PAYLOAD.model_dump_json())
)
)
await provider.analyze("короткий договор", checklist="test checklist")
request = route.calls[0].request
body = json.loads(request.content)
assert body["modelUri"] == f"gpt://{FOLDER_ID}/{MODEL}"
assert body["completionOptions"]["responseFormat"]["type"] == "JSON_OBJECT"
assert "schema" in body["completionOptions"]["responseFormat"]["json_schema"]
assert body["messages"][0]["role"] == "system"
assert "test checklist" in body["messages"][0]["text"]
assert request.headers["authorization"] == "Api-Key key"
assert request.headers["x-folder-id"] == FOLDER_ID

113
tests/unit/test_prescreen_extractor.py Normal file
View file

@ -0,0 +1,113 @@
"""Unit tests for the deterministic regex prescreen extractor."""
from __future__ import annotations
import pytest
from contract_check.worker_prescreen.extractor import extract_contract_meta
SIMPLE_SUPPLY = """\
ДОГОВОР ПОСТАВКИ № 42
г. Москва
Общество с ограниченной ответственностью «Продавец»,
именуемое в дальнейшем «Поставщик», в лице директора Иванова И.И.,
с одной стороны, и
Общество с ограниченной ответственностью «Покупатель»,
именуемое в дальнейшем «Покупатель», с другой стороны,
заключили настоящий договор о нижеследующем.
1. Предмет договора
Поставщик обязуется передать в собственность Покупателю товар.
2. Цена и порядок расчётов
2.1. Общая стоимость товара составляет 1 250 000 (один миллион двести пятьдесят тысяч) рублей.
3. Срок действия договора
3.1. Настоящий договор вступает в силу с 01.09.2025 и действует по 31.08.2026.
4. Ответственность сторон
4.1. За просрочку поставки Поставщик уплачивает неустойку в размере 0,1% от стоимости.
5. Порядок разрешения споров
5.1. Споры подлежат рассмотрению в Арбитражном суде г. Москвы.
"""
MINIMAL = """\
Договор оказания услуг между ИП Петров Петр Петрович и ООО «Клиент».
Стоимость услуг составляет 50 000 рублей.
"""
def test_supply_extraction() -> None:
meta = extract_contract_meta(SIMPLE_SUPPLY)
assert meta.contract_type == "supply"
assert "Продавец" in (meta.party_a or "")
assert "Покупатель" in (meta.party_b or "")
assert meta.total_amount == 1250000.0
assert meta.currency == "RUB"
assert meta.start_date == "2025-09-01"
assert meta.end_date == "2026-08-31"
assert meta.has_penalty_clause is True
assert meta.has_termination_clause is False
assert meta.has_arbitration is True
assert meta.confidence_score > 0.6
def test_minimal_extraction() -> None:
meta = extract_contract_meta(MINIMAL)
assert meta.contract_type == "services"
assert "ИП Петров" in (meta.party_a or "")
assert "ООО «Клиент»" in (meta.party_b or "")
assert meta.total_amount == 50000.0
assert meta.currency == "RUB"
def test_confidence_score_bounds() -> None:
meta = extract_contract_meta("")
assert meta.confidence_score == 0.0
assert 0.0 <= meta.confidence_score <= 1.0
def test_party_extraction_handles_whitespace() -> None:
text = "Договор между ООО «Альфа» и ООО «Бета»"
meta = extract_contract_meta(text)
assert "Альфа" in (meta.party_a or "")
assert "Бета" in (meta.party_b or "")
@pytest.mark.parametrize(
("clause", "expected"),
[
("Стороны вправе расторгнуть договор в одностороннем порядке", "has_termination_clause"),
("В случае нарушения Покупатель уплачивает штраф", "has_penalty_clause"),
("Споры рассматриваются в Арбитражном суде", "has_arbitration"),
],
)
def test_boolean_flags(clause: str, expected: str) -> None:
meta = extract_contract_meta(f"Договор поставки. {clause}.")
assert getattr(meta, expected) is True
def test_shim_defaults_to_heuristic_v2() -> None:
meta = extract_contract_meta(SIMPLE_SUPPLY)
assert meta.confidence_score > 0.6
def test_keep_regex_flag_routes_to_regex_v1(monkeypatch: pytest.MonkeyPatch) -> None:
"""PRESCREEN_KEEP_REGEX=true keeps the legacy regex path alive (plan §5)."""
from contract_check.worker_prescreen.extractor import _extract_contract_meta_regex
monkeypatch.setenv("PRESCREEN_KEEP_REGEX", "true")
meta = extract_contract_meta(SIMPLE_SUPPLY)
legacy = _extract_contract_meta_regex(SIMPLE_SUPPLY.replace("\n", " "))
# Identical extraction semantics on the ported fixtures.
assert meta.model_dump() == legacy.model_dump()
monkeypatch.setenv("PRESCREEN_KEEP_REGEX", "false")
assert extract_contract_meta(SIMPLE_SUPPLY).model_dump() == meta.model_dump()

191
tests/unit/test_prescreen_extractor_heuristic.py Normal file
View file

@ -0,0 +1,191 @@
"""Unit tests for the Stage-1 heuristic extractor (regex-free port).
Ports 100% of the regex-v1 fixtures from test_prescreen_extractor.py onto
HeuristicExtractor (plan Phase 1) and adds coverage for the string helpers
(`_scan_number`, header window, trigger scans).
"""
from __future__ import annotations
import pytest
from contract_check.worker_prescreen.extractor_heuristic import (
EXTRACTOR_VERSION,
HeuristicExtractor,
_header_window,
_scan_number,
)
SIMPLE_SUPPLY = """\
ДОГОВОР ПОСТАВКИ № 42
г. Москва
Общество с ограниченной ответственностью «Продавец»,
именуемое в дальнейшем «Поставщик», в лице директора Иванова И.И.,
с одной стороны, и
Общество с ограниченной ответственностью «Покупатель»,
именуемое в дальнейшем «Покупатель», с другой стороны,
заключили настоящий договор о нижеследующем.
1. Предмет договора
Поставщик обязуется передать в собственность Покупателю товар.
2. Цена и порядок расчётов
2.1. Общая стоимость товара составляет 1 250 000 (один миллион двести пятьдесят тысяч) рублей.
3. Срок действия договора
3.1. Настоящий договор вступает в силу с 01.09.2025 и действует по 31.08.2026.
4. Ответственность сторон
4.1. За просрочку поставки Поставщик уплачивает неустойку в размере 0,1% от стоимости.
5. Порядок разрешения споров
5.1. Споры подлежат рассмотрению в Арбитражном суде г. Москвы.
"""
MINIMAL = """\
Договор оказания услуг между ИП Петров Петр Петрович и ООО «Клиент».
Стоимость услуг составляет 50 000 рублей.
"""
@pytest.fixture
def extractor() -> HeuristicExtractor:
return HeuristicExtractor()
def test_version_is_heuristic_v2() -> None:
assert EXTRACTOR_VERSION == "heuristic-v2"
assert HeuristicExtractor.extractor_version == "heuristic-v2"
def test_supply_extraction(extractor: HeuristicExtractor) -> None:
meta = extractor.extract(SIMPLE_SUPPLY)
assert meta.contract_type == "supply"
assert "Продавец" in (meta.party_a or "")
assert "Покупатель" in (meta.party_b or "")
assert meta.total_amount == 1250000.0
assert meta.currency == "RUB"
assert meta.start_date == "2025-09-01"
assert meta.end_date == "2026-08-31"
assert meta.has_penalty_clause is True
assert meta.has_termination_clause is False
assert meta.has_arbitration is True
assert meta.confidence_score > 0.6
def test_minimal_extraction(extractor: HeuristicExtractor) -> None:
meta = extractor.extract(MINIMAL)
assert meta.contract_type == "services"
assert "ИП Петров" in (meta.party_a or "")
assert "ООО «Клиент»" in (meta.party_b or "")
assert meta.total_amount == 50000.0
assert meta.currency == "RUB"
def test_confidence_score_bounds(extractor: HeuristicExtractor) -> None:
meta = extractor.extract("")
assert meta.confidence_score == 0.0
assert 0.0 <= meta.confidence_score <= 1.0
def test_party_extraction_handles_whitespace(extractor: HeuristicExtractor) -> None:
text = "Договор между ООО «Альфа» и ООО «Бета»"
meta = extractor.extract(text)
assert "Альфа" in (meta.party_a or "")
assert "Бета" in (meta.party_b or "")
@pytest.mark.parametrize(
("clause", "expected"),
[
("Стороны вправе расторгнуть договор в одностороннем порядке", "has_termination_clause"),
("В случае нарушения Покупатель уплачивает штраф", "has_penalty_clause"),
("Споры рассматриваются в Арбитражном суде", "has_arbitration"),
],
)
def test_boolean_flags(clause: str, expected: str, extractor: HeuristicExtractor) -> None:
meta = extractor.extract(f"Договор поставки. {clause}.")
assert getattr(meta, expected) is True
# ── helper coverage ──────────────────────────────────────────────────────────
@pytest.mark.parametrize(
("raw", "expected"),
[
("1 250 000 рублей", 1250000.0),
("1250000", 1250000.0),
("50 000 руб", 50000.0),
("0,1% от стоимости", 0.1),
("1 250 000,50", 1250000.5),
("999", 999.0),
],
)
def test_scan_number(raw: str, expected: float) -> None:
parsed = _scan_number(raw, 0)
assert parsed is not None
assert parsed[0] == expected
def test_scan_number_rejects_non_digits() -> None:
assert _scan_number("abc", 0) is None
def test_scan_number_nbsp_thousands() -> None:
parsed = _scan_number("1\u00a0250\u00a0000", 0)
assert parsed is not None
assert parsed[0] == 1250000.0
def test_header_window_cuts_at_numbered_heading() -> None:
text = "ДОГОВОР № 1\nООО «Альфа» и ООО «Бета»\n1. Предмет договора\nООО «Гамма»"
window = _header_window(text)
assert "Альфа" in window
assert "Бета" in window
assert "Гамма" not in window
def test_parties_only_scanned_in_header(extractor: HeuristicExtractor) -> None:
text = (
"ДОГОВОР № 1\nООО «Альфа» и ООО «Бета»\n\n"
"1. Предмет договора\nПоставщик обязуется...\n\n"
"10. Реквизиты сторон\nООО «Гамма»"
)
meta = extractor.extract(text)
assert "Альфа" in (meta.party_a or "")
assert "Бета" in (meta.party_b or "")
def test_end_date_only_via_do_fallback(extractor: HeuristicExtractor) -> None:
meta = extractor.extract("Соглашение действует до 31.12.2026.")
assert meta.start_date is None
assert meta.end_date == "2026-12-31"
def test_start_date_via_ot_token(extractor: HeuristicExtractor) -> None:
meta = extractor.extract("Договор заключён от 15.03.2024.")
assert meta.start_date == "2024-03-15"
def test_amount_skips_trigger_without_digits(extractor: HeuristicExtractor) -> None:
meta = extractor.extract("Общая стоимость товара определяется расчётом. Сумма 300 BYN.")
assert meta.total_amount == 300.0
assert meta.currency == "BYN"
def test_no_false_entity_match_inside_words(extractor: HeuristicExtractor) -> None:
meta = extractor.extract("Ипотека оформлена. Сумма 100 рублей.")
assert meta.party_a is None
assert meta.party_b is None
def test_bare_entity_form_without_name_dropped(extractor: HeuristicExtractor) -> None:
meta = extractor.extract("Договор между ООО и АО.\nСумма 100 рублей.")
assert meta.party_a is None
assert meta.party_b is None

161
tests/unit/test_prescreen_extractor_hybrid.py Normal file
View file

@ -0,0 +1,161 @@
"""HybridMetaExtractor orchestrator matrix (plan Phase 4).
- high confidence → LLM skipped
- low confidence → LLM merged, version hybrid-llm-v1
- LLM failure → heuristic result kept + outcome=failed + error recorded
- fallback disabled → outcome=disabled, no provider call
- boolean merge semantics: OR across stages
"""
from __future__ import annotations
from typing import Any
from contract_check.core.metrics import prescreen_fallback_runs
from contract_check.worker_prescreen.extractor_heuristic import HeuristicExtractor
from contract_check.worker_prescreen.extractor_hybrid import HybridMetaExtractor
HIGH_CONF_TEXT = """\
ДОГОВОР ПОСТАВКИ № 42
г. Москва
Общество с ограниченной ответственностью «Продавец» и
Общество с ограниченной ответственностью «Покупатель» заключили договор.
1. Предмет договора
2.1. Стоимость товара составляет 1 250 000 рублей.
3.1. Договор вступает в силу с 01.09.2025 и действует по 31.08.2026.
4.1. Неустойка 0,1%.
5.1. Споры — в Арбитражном суде г. Москвы.
"""
LOW_CONF_TEXT = "Образец документа без опознаваемых полей."
class StubProvider:
def __init__(self, payload: dict[str, Any] | Exception) -> None:
self._payload = payload
self.calls = 0
async def analyze(self, text: str, *, checklist: str, extra_context: str = "") -> Any:
raise NotImplementedError
async def extract_prescreen(self, text: str) -> dict[str, Any]:
self.calls += 1
if isinstance(self._payload, Exception):
raise self._payload
return self._payload
async def aclose(self) -> None:
pass
LLM_DICT: dict[str, Any] = {
"contract_type": "services",
"party_a": "ИП Петров",
"party_b": "ООО «Клиент»",
"total_amount": 50000.0,
"currency": "RUB",
"has_penalty_clause": True,
}
def _counter(outcome: str) -> int:
return prescreen_fallback_runs.labels(outcome=outcome)._value.get() # type: ignore[no-any-return]
def _hybrid(
provider: StubProvider, *, enabled: bool, threshold: float = 0.75
) -> HybridMetaExtractor:
return HybridMetaExtractor(
provider=provider,
fallback_enabled=enabled,
fallback_threshold=threshold,
heuristic=HeuristicExtractor(),
)
async def test_disabled_never_calls_llm() -> None:
provider = StubProvider(LLM_DICT)
before = _counter("disabled")
result = await _hybrid(provider, enabled=False).extract(LOW_CONF_TEXT)
assert provider.calls == 0
assert result.extractor_version == "heuristic-v2"
assert result.llm_fallback_error is None
assert _counter("disabled") == before + 1
async def test_high_confidence_skips_llm() -> None:
provider = StubProvider(LLM_DICT)
before = _counter("skipped")
result = await _hybrid(provider, enabled=True).extract(HIGH_CONF_TEXT)
assert provider.calls == 0
assert result.extractor_version == "heuristic-v2"
assert result.meta.confidence_score >= 0.75
assert _counter("skipped") == before + 1
async def test_low_confidence_merges_llm() -> None:
provider = StubProvider(LLM_DICT)
before = _counter("used")
result = await _hybrid(provider, enabled=True).extract(LOW_CONF_TEXT)
assert provider.calls == 1
assert result.extractor_version == "hybrid-llm-v1"
assert result.llm_fallback_error is None
meta = result.meta
assert meta.contract_type == "services"
assert meta.party_a == "ИП Петров"
assert meta.party_b == "ООО «Клиент»"
assert meta.total_amount == 50000.0
assert meta.confidence_score >= 0.75
assert _counter("used") == before + 1
async def test_llm_failure_keeps_heuristic_result() -> None:
provider = StubProvider(RuntimeError("boom"))
before = _counter("failed")
result = await _hybrid(provider, enabled=True).extract(LOW_CONF_TEXT)
assert provider.calls == 1
assert result.extractor_version == "heuristic-v2"
assert result.llm_fallback_error is not None
assert "RuntimeError" in result.llm_fallback_error
assert result.meta.contract_type is None # heuristic result untouched
assert _counter("failed") == before + 1
async def test_llm_overrides_only_heuristic_nones() -> None:
provider = StubProvider({**LLM_DICT, "contract_type": None})
result = await _hybrid(provider, enabled=True).extract(
"Соглашение. ООО «Ромашка» действует.\nСумма 100 рублей."
)
# Heuristic found nothing for type; LLM had None too → stays None.
assert result.meta.contract_type is None
assert result.extractor_version == "hybrid-llm-v1"
async def test_booleans_or_merged() -> None:
# Heuristic: no clause words → False. LLM: penalty True → OR → True.
provider = StubProvider({**LLM_DICT, "has_penalty_clause": True})
result = await _hybrid(provider, enabled=True).extract(LOW_CONF_TEXT)
assert result.meta.has_penalty_clause is True
async def test_heuristic_boolean_survives_llm_false() -> None:
text = LOW_CONF_TEXT + " Стороны вправе расторгнуть договор."
provider = StubProvider({"has_termination_clause": False})
result = await _hybrid(provider, enabled=True).extract(text)
assert result.meta.has_termination_clause is True
async def test_threshold_boundary_equal_skips_llm() -> None:
provider = StubProvider(LLM_DICT)
# LOW_CONF_TEXT scores 0.0; threshold 0.0 → 0.0 >= 0.0 → skipped.
result = await _hybrid(provider, enabled=True, threshold=0.0).extract(LOW_CONF_TEXT)
assert provider.calls == 0
assert result.extractor_version == "heuristic-v2"
async def test_heuristic_only_failure_never_raised() -> None:
# The whole pipeline never raises from extraction stages on LLM errors.
provider = StubProvider(RuntimeError("quota exceeded"))
result = await _hybrid(provider, enabled=True).extract(HIGH_CONF_TEXT)
assert result.meta.confidence_score >= 0.75

113
tests/unit/test_prescreen_extractor_llm.py Normal file
View file

@ -0,0 +1,113 @@
"""Unit tests for the Stage-2 worker-side wrapper (dict → validated meta)."""
from __future__ import annotations
from typing import Any
from contract_check.worker_prescreen.extractor_llm import (
EXTRACTOR_VERSION,
LLMPrescreenExtractor,
validate_raw,
)
VALID: dict[str, Any] = {
"contract_type": "supply",
"party_a": "ООО «Альфа»",
"party_b": "ООО «Бета»",
"total_amount": 1250000,
"currency": "RUB",
"start_date": "2025-09-01",
"end_date": "2026-08-31",
"has_penalty_clause": True,
"has_termination_clause": False,
"has_arbitration": True,
}
def test_valid_dict_passes_through() -> None:
meta = validate_raw(VALID)
assert meta.contract_type == "supply"
assert meta.party_a == "ООО «Альфа»"
assert meta.total_amount == 1250000.0
assert meta.currency == "RUB"
assert meta.start_date == "2025-09-01"
assert meta.has_penalty_clause is True
assert meta.has_arbitration is True
assert meta.confidence_score == 1.0
def test_invalid_enum_becomes_none() -> None:
meta = validate_raw({**VALID, "contract_type": "lease-agreement"})
assert meta.contract_type is None
def test_contract_type_normalized_to_lower() -> None:
meta = validate_raw({**VALID, "contract_type": "LEASE"})
assert meta.contract_type == "lease"
def test_invalid_currency_becomes_none() -> None:
meta = validate_raw({**VALID, "currency": "CHF"})
assert meta.currency is None
def test_currency_normalized_to_upper() -> None:
meta = validate_raw({**VALID, "currency": "byn"})
assert meta.currency == "BYN"
def test_invalid_iso_date_becomes_none() -> None:
meta = validate_raw({**VALID, "start_date": "сентябрь 2025"})
assert meta.start_date is None
def test_ddmmyyyy_date_accepted() -> None:
meta = validate_raw({**VALID, "start_date": "01.09.2025"})
assert meta.start_date == "2025-09-01"
def test_impossible_date_becomes_none() -> None:
meta = validate_raw({**VALID, "start_date": "32.13.2025"})
assert meta.start_date is None
def test_negative_amount_becomes_none() -> None:
meta = validate_raw({**VALID, "total_amount": -5})
assert meta.total_amount is None
def test_non_bool_flags_become_none() -> None:
meta = validate_raw({**VALID, "has_penalty_clause": "да"})
assert meta.has_penalty_clause is None
def test_empty_strings_become_none() -> None:
meta = validate_raw({**VALID, "party_a": " "})
assert meta.party_a is None
def test_extract_version_constant() -> None:
assert EXTRACTOR_VERSION == "hybrid-llm-v1"
async def test_extractor_calls_provider_and_scores() -> None:
class StubProvider:
def __init__(self) -> None:
self.received: list[str] = []
async def analyze(self, text: str, *, checklist: str, extra_context: str = "") -> object:
raise NotImplementedError
async def extract_prescreen(self, text: str) -> dict[str, Any]:
self.received.append(text)
return dict(VALID)
async def aclose(self) -> None:
pass
provider = StubProvider()
extractor = LLMPrescreenExtractor(provider)
meta = await extractor.extract("Договор поставки...")
assert provider.received == ["Договор поставки..."]
assert meta.contract_type == "supply"
assert meta.confidence_score == 1.0

94
tests/unit/test_prescreen_router.py Normal file
View file

@ -0,0 +1,94 @@
"""Unit tests for the prescreen routing decision logic."""
from __future__ import annotations
import pytest
from contract_check.worker_prescreen.extractor import PrescreenContractMeta
from contract_check.worker_prescreen.router import PrescreenRouter
@pytest.fixture
def router() -> PrescreenRouter:
return PrescreenRouter(
confidence_threshold=0.75,
high_value_threshold=100_000.0,
auto_approve_enabled=False,
)
@pytest.fixture
def complete_meta() -> PrescreenContractMeta:
return PrescreenContractMeta(
contract_type="supply",
party_a="ООО «А»",
party_b="ООО «Б»",
total_amount=50000.0,
currency="RUB",
start_date="2025-01-01",
end_date="2025-12-31",
has_penalty_clause=False,
has_termination_clause=False,
has_arbitration=False,
confidence_score=0.8,
)
def test_high_value_routes_to_deep_analysis(
router: PrescreenRouter, complete_meta: PrescreenContractMeta
) -> None:
complete_meta.total_amount = 250_000.0
assert router.decide(complete_meta) == "deep_analysis"
def test_penalty_clause_routes_to_deep_analysis(
router: PrescreenRouter, complete_meta: PrescreenContractMeta
) -> None:
complete_meta.has_penalty_clause = True
assert router.decide(complete_meta) == "deep_analysis"
def test_arbitration_routes_to_deep_analysis(
router: PrescreenRouter, complete_meta: PrescreenContractMeta
) -> None:
complete_meta.has_arbitration = True
assert router.decide(complete_meta) == "deep_analysis"
def test_low_confidence_routes_to_manual_review(
router: PrescreenRouter, complete_meta: PrescreenContractMeta
) -> None:
complete_meta.confidence_score = 0.5
assert router.decide(complete_meta) == "manual_review"
def test_missing_party_routes_to_manual_review(
router: PrescreenRouter, complete_meta: PrescreenContractMeta
) -> None:
complete_meta.party_b = None
assert router.decide(complete_meta) == "manual_review"
def test_auto_approve_disabled_by_default(
router: PrescreenRouter, complete_meta: PrescreenContractMeta
) -> None:
# Low value, no risk clauses, high confidence — but auto_approve is off.
assert router.decide(complete_meta) == "manual_review"
def test_auto_approve_when_enabled(complete_meta: PrescreenContractMeta) -> None:
enabled_router = PrescreenRouter(
confidence_threshold=0.75,
high_value_threshold=100_000.0,
auto_approve_enabled=True,
)
assert enabled_router.decide(complete_meta) == "auto_approve"
def test_summarize_returns_non_empty_string(
router: PrescreenRouter, complete_meta: PrescreenContractMeta
) -> None:
summary = router.summarize(complete_meta, "auto_approve")
assert "supply" in summary
assert "ООО «А»" in summary
assert "50000" in summary

128
uv.lock generated
View file

@ -483,6 +483,41 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" },
]
[[package]]
name = "chardet"
version = "7.6.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/b1/51/cd61c567092a6cec796144510a68aff158ebfc1df82950a45bae65f28413/chardet-7.6.0.tar.gz", hash = "sha256:93d9df6089ded42ed1fe9f57e272c0b74bd0464d45c0c7d50f09f26f31105c3c", size = 914462, upload-time = "2026-08-14T20:36:59.305Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/a3/29/16a7419edfbd60e901e6a797cbc3e038cb2a81903bc16c029db755f0156f/chardet-7.6.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:57e6846cc13ce1ff59979f4ec9da770c57e12aa99046073f632de5a51d9a6f20", size = 1088449, upload-time = "2026-08-14T20:36:34.723Z" },
{ url = "https://files.pythonhosted.org/packages/1d/36/3a14b0f8ddeb302f157281ca656a3ce6874b78e2d6af03682f520b487245/chardet-7.6.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:089e3bb81a0a07e94f15461ded9f9ee66d349615b1a9fd557d4de1003e2fc12e", size = 1065126, upload-time = "2026-08-14T20:36:36.21Z" },
{ url = "https://files.pythonhosted.org/packages/d2/4c/f59a39c2bfe4ac99baba8da842e8d2ea0b84dff7ba53a96e7ad8c71602d7/chardet-7.6.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:43ea433e43a23c55e8e17f3fad1e07f5cfe5450c73124b95b0d849c21ad379ee", size = 1482492, upload-time = "2026-08-14T20:36:37.649Z" },
{ url = "https://files.pythonhosted.org/packages/bd/eb/93e8036681157f2217a18769927a035984526e6dbd5e91f28a375ca41c14/chardet-7.6.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2b5d31f9b7f793e15e81cca877e7ccd72bffffa2a3443a9d47be9dfee84fad69", size = 1514185, upload-time = "2026-08-14T20:36:39.119Z" },
{ url = "https://files.pythonhosted.org/packages/10/04/0066d7ab2c135e404a6fa166bb7fa49d1c7bf7af07b86ed95b1c48a348c7/chardet-7.6.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c54b6a8d3b219560fa5cf4c28df932c37471afe047afdc152067104e741f38c1", size = 1454420, upload-time = "2026-08-14T20:36:40.549Z" },
{ url = "https://files.pythonhosted.org/packages/6c/9f/e965f1d9eddfb86cf118f980d329cbee43c4ac4b562448b369a6b6ef36af/chardet-7.6.0-cp313-cp313-win_amd64.whl", hash = "sha256:b3b4c96c4df93899b3c8b9e8159e06b1f55c66d7ca384d91481108e251a06eb0", size = 1159067, upload-time = "2026-08-14T20:36:41.816Z" },
{ url = "https://files.pythonhosted.org/packages/cf/78/e14991c9487277ed7d006fff58f3084ac792ed94cced081788abb95df70c/chardet-7.6.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c6061adf247ab5dda173b67010e13904c6071717660c7c8077fb50aca362b264", size = 1087678, upload-time = "2026-08-14T20:36:43.2Z" },
{ url = "https://files.pythonhosted.org/packages/7a/40/0f95e04cb1820e0a582cd6d86bbf26be8302a94ccf330f8ba5f69735389d/chardet-7.6.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fc1e1571321baf8927582fe34363ad7f02279f11c8c2839c14b4c76894148db6", size = 1065310, upload-time = "2026-08-14T20:36:44.489Z" },
{ url = "https://files.pythonhosted.org/packages/3b/a1/04404dcc9d6e1253b02583e562d2c7ddca50a7e91f460d62eff7e1d07c92/chardet-7.6.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8900f6c7cf6b015b17a51767cc6144689059ba1cdceaa383d29eb037ac28579e", size = 1485028, upload-time = "2026-08-14T20:36:45.797Z" },
{ url = "https://files.pythonhosted.org/packages/f4/88/360064c4c7d9d0664561dae03b74c871d2f5332b329f5c99f1c997fb869a/chardet-7.6.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cedbc584789eb2edfde20fd03669972a833ce6019e60014ae613f9bfc440e8e3", size = 1510242, upload-time = "2026-08-14T20:36:47.206Z" },
{ url = "https://files.pythonhosted.org/packages/43/4c/302869fa1c69a5a41e4e78782680b12552ffd4286c3ae3c839b7b8df53d4/chardet-7.6.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d5dc835e40e0e09c2c3eab43731a8b5127834f42786dda09ba2f4b699ccd527a", size = 1456456, upload-time = "2026-08-14T20:36:48.668Z" },
{ url = "https://files.pythonhosted.org/packages/7c/a9/ff4fef15ed25fc3f945a3b981ae0f43c8559b3fbedb40267e59e583d105b/chardet-7.6.0-cp314-cp314-win_amd64.whl", hash = "sha256:0f304de7041afaec0195ad6464937cd112392002e9d72ed15d55f20a9abd3a13", size = 1159103, upload-time = "2026-08-14T20:36:50.107Z" },
{ url = "https://files.pythonhosted.org/packages/31/44/94e6f89485ce6c630e8fec6d388e3fa747e58b7246b0cc8c5c532ba98650/chardet-7.6.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a4f0a368ad04d5def08bdfaa17c7e15e71552f93923dc2aa9b2f7d9dee02fbb6", size = 1120302, upload-time = "2026-08-14T20:36:51.484Z" },
{ url = "https://files.pythonhosted.org/packages/e0/cd/8e68ba12f13aaf4ee82d54ef8a1f83d62942e7a6bc1e579dd2d530a3e26e/chardet-7.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:75d6c3a4d2046d49e83d2d2206eb073a1f390743e856d90c1bbc19949b26acf4", size = 1093157, upload-time = "2026-08-14T20:36:52.987Z" },
{ url = "https://files.pythonhosted.org/packages/f3/f6/6a35342b9efa69dcceab0ea8966571c6442a59c336bf460f0a2af95ed234/chardet-7.6.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:459e2b1c98f9a86a4698112aa42dffa802bbbff883c1ff144071f87224125862", size = 1521213, upload-time = "2026-08-14T20:36:54.812Z" },
{ url = "https://files.pythonhosted.org/packages/df/8c/8f09bfabbaedae45caa72b996e96d5e3f6436dfddc55c1311ffa2f6bd7d2/chardet-7.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bdb6f03107b7ace3f44e0edd91aa24456ee558787df265cc19daf45785b31c7", size = 1528427, upload-time = "2026-08-14T20:36:56.224Z" },
{ url = "https://files.pythonhosted.org/packages/0c/3d/2540627b193112e08b8045f3cad9295570866208555aa6625b63cc69c6cd/chardet-7.6.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:0c44a32da32cc8b23d6b20d98ace15ec7600950e4955d1bf5ab1f849b0187fdb", size = 1087374, upload-time = "2026-08-14T21:04:14.994Z" },
{ url = "https://files.pythonhosted.org/packages/fe/de/dac4f550cde73c4732b4916e72ec489ae36933fbbd1697e4122d3ae2e9dd/chardet-7.6.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:7bbc8a9652c7f859c593847f220c1d264f25749369abb1a267b404ee8cceb209", size = 1064688, upload-time = "2026-08-14T21:04:16.806Z" },
{ url = "https://files.pythonhosted.org/packages/06/45/f4f3f288496797d2cf1d1e9036f920ed2b4c79787b21c42a3314f4a6d532/chardet-7.6.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:83512a475a2f3886166aa0bca1bbb39343a4eb3186dd5532127d6f2591d09118", size = 1486320, upload-time = "2026-08-14T21:04:18.535Z" },
{ url = "https://files.pythonhosted.org/packages/c0/81/3ca30c16e6c6015b737fca22d9342957cc617d8a65329d3e963ad754a31b/chardet-7.6.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:271ab71ec1be61dbbce0436de0848895c03eae051c379e937a39573d9ce403d9", size = 1515374, upload-time = "2026-08-14T21:04:20.724Z" },
{ url = "https://files.pythonhosted.org/packages/6e/98/163a75b8b3372a6b6503f5f5a4154a222ac135c7a706b5d176f8e4b237a2/chardet-7.6.0-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:da86fc1b40ff5996fbb5e4c2d2dca770eac2c893cef157dacc050b8b4d929846", size = 1469504, upload-time = "2026-08-14T21:04:22.352Z" },
{ url = "https://files.pythonhosted.org/packages/d7/4f/3ad1b1bd27ae7f0d3d13cf711366525cd781a9e067950d8a09aa280916cb/chardet-7.6.0-cp315-cp315-win_amd64.whl", hash = "sha256:b73f277c1ac09c4f8076c4214b816c7aa78a0a2f0cb7156742f4303f856bedc3", size = 1158551, upload-time = "2026-08-14T21:04:24.259Z" },
{ url = "https://files.pythonhosted.org/packages/8d/92/22a609c68c5123ebdccd8fe1a80e423609012ce20166de19a2ed3955b2f7/chardet-7.6.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:61238d5945b36af9a2ad13494f8969b7deb3c3b4abe223e54670c064e73f5328", size = 1119186, upload-time = "2026-08-14T21:04:25.761Z" },
{ url = "https://files.pythonhosted.org/packages/29/35/75a90143e4200e197f4f6cf5895d379e22bc785d0c7711120df18fa5347c/chardet-7.6.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:f2ec3c78cc6b54bf8e091ec4ee885473078b5d7ef18ab1b01c86ae1e98bf88f7", size = 1092596, upload-time = "2026-08-14T21:04:27.884Z" },
{ url = "https://files.pythonhosted.org/packages/d1/2c/b6d5f47d878c46e04ae6fcb58e0969467925bc0819b333c682e0c41bbc5c/chardet-7.6.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:167d7ba3ee08b654e36d7b43ebd9a36606c9a12e2fabdb361757a095ca3b7e3d", size = 1516644, upload-time = "2026-08-14T21:04:29.624Z" },
{ url = "https://files.pythonhosted.org/packages/25/d2/2bc3f1066c6f27bc3fa3a98dfd8a2a9c1e969a9d6bdc1edcd35278791fc4/chardet-7.6.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f14f46ef1977e41ce1f4814ca6984cea7f8b6baf8cbc6626ef7bf3d13cf7ea13", size = 1533221, upload-time = "2026-08-14T21:04:31.39Z" },
{ url = "https://files.pythonhosted.org/packages/cf/6e/5a0b348fa4cd7847567a28c6e697ccf58391960bfd13a6e7473ee23ca2f2/chardet-7.6.0-py3-none-any.whl", hash = "sha256:4076d795897ce45239825956a1334e134322ecc4bfe84dbb12acd5390de0fbc1", size = 680279, upload-time = "2026-08-14T20:36:57.763Z" },
]
[[package]]
name = "charset-normalizer"
version = "3.4.9"
@ -543,6 +578,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" },
]
[[package]]
name = "cobble"
version = "0.1.4"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/54/7a/a507c709be2c96e1bb6102eb7b7f4026c5e5e223ef7d745a17d239e9d844/cobble-0.1.4.tar.gz", hash = "sha256:de38be1539992c8a06e569630717c485a5f91be2192c461ea2b220607dfa78aa", size = 3805, upload-time = "2024-06-01T18:11:09.528Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/d5/e1/3714a2f371985215c219c2a70953d38e3eed81ef165aed061d21de0e998b/cobble-0.1.4-py3-none-any.whl", hash = "sha256:36c91b1655e599fd428e2b95fdd5f0da1ca2e9f1abb0bc871dec21a0e78a2b44", size = 3984, upload-time = "2024-06-01T18:11:07.911Z" },
]
[[package]]
name = "colorama"
version = "0.4.6"
@ -618,9 +662,12 @@ dev = [
{ name = "argon2-cffi" },
{ name = "asgi-lifespan" },
{ name = "asyncpg" },
{ name = "chardet" },
{ name = "email-validator" },
{ name = "fastapi" },
{ name = "isort" },
{ name = "jinja2" },
{ name = "mammoth" },
{ name = "minio" },
{ name = "mypy" },
{ name = "opentelemetry-exporter-otlp" },
@ -637,19 +684,24 @@ dev = [
{ name = "pytest" },
{ name = "pytest-asyncio" },
{ name = "python-docx" },
{ name = "python-magic" },
{ name = "python-multipart" },
{ name = "redis" },
{ name = "respx" },
{ name = "ruff" },
{ name = "sentry-sdk" },
{ name = "sqlalchemy" },
{ name = "striprtf" },
{ name = "testcontainers", extra = ["minio", "rabbitmq"] },
{ name = "ty" },
{ name = "uvicorn", extra = ["standard"] },
]
extract = [
{ name = "aio-pika" },
{ name = "alembic" },
{ name = "asyncpg" },
{ name = "chardet" },
{ name = "mammoth" },
{ name = "minio" },
{ name = "opentelemetry-exporter-otlp" },
{ name = "opentelemetry-sdk" },
@ -657,8 +709,10 @@ extract = [
{ name = "prometheus-client" },
{ name = "pymupdf" },
{ name = "pytesseract" },
{ name = "python-magic" },
{ name = "sentry-sdk" },
{ name = "sqlalchemy" },
{ name = "striprtf" },
]
mq = [
{ name = "aio-pika" },
@ -751,9 +805,12 @@ dev = [
{ name = "argon2-cffi", specifier = ">=23.1" },
{ name = "asgi-lifespan", specifier = ">=2.1.0" },
{ name = "asyncpg", specifier = ">=0.29" },
{ name = "chardet", specifier = ">=5.2" },
{ name = "email-validator", specifier = ">=2.1" },
{ name = "fastapi", specifier = ">=0.110" },
{ name = "isort", specifier = ">=5.13" },
{ name = "jinja2", specifier = ">=3.1" },
{ name = "mammoth", specifier = ">=1.8" },
{ name = "minio", specifier = ">=7.2" },
{ name = "mypy", specifier = ">=1.10" },
{ name = "opentelemetry-exporter-otlp", specifier = ">=1.24" },
@ -770,19 +827,24 @@ dev = [
{ name = "pytest", specifier = ">=8" },
{ name = "pytest-asyncio", specifier = ">=0.23" },
{ name = "python-docx", specifier = ">=1.1" },
{ name = "python-magic", specifier = ">=0.4.27" },
{ name = "python-multipart", specifier = ">=0.0.9" },
{ name = "redis", specifier = ">=5.0" },
{ name = "respx", specifier = ">=0.21" },
{ name = "ruff", specifier = ">=0.5" },
{ name = "sentry-sdk", specifier = ">=2" },
{ name = "sqlalchemy", specifier = ">=2.0" },
{ name = "striprtf", specifier = ">=0.0.26" },
{ name = "testcontainers", extras = ["rabbitmq", "postgres", "minio"], specifier = ">=4" },
{ name = "ty", specifier = ">=0.0.72" },
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.29" },
]
extract = [
{ name = "aio-pika", specifier = ">=9.4" },
{ name = "alembic", specifier = ">=1.13" },
{ name = "asyncpg", specifier = ">=0.29" },
{ name = "chardet", specifier = ">=5.2" },
{ name = "mammoth", specifier = ">=1.8" },
{ name = "minio", specifier = ">=7.2" },
{ name = "opentelemetry-exporter-otlp", specifier = ">=1.24" },
{ name = "opentelemetry-sdk", specifier = ">=1.24" },
@ -790,8 +852,10 @@ extract = [
{ name = "prometheus-client", specifier = ">=0.20" },
{ name = "pymupdf", specifier = ">=1.24" },
{ name = "pytesseract", specifier = ">=0.3.10" },
{ name = "python-magic", specifier = ">=0.4.27" },
{ name = "sentry-sdk", specifier = ">=2" },
{ name = "sqlalchemy", specifier = ">=2.0" },
{ name = "striprtf", specifier = ">=0.0.26" },
]
mq = [{ name = "aio-pika", specifier = ">=9.4" }]
notify = [
@ -1229,6 +1293,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" },
]
[[package]]
name = "isort"
version = "8.0.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/ef/7c/ec4ab396d31b3b395e2e999c8f46dec78c5e29209fac49d1f4dace04041d/isort-8.0.1.tar.gz", hash = "sha256:171ac4ff559cdc060bcfff550bc8404a486fee0caab245679c2abe7cb253c78d", size = 769592, upload-time = "2026-02-28T10:08:20.685Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/3e/95/c7c34aa53c16353c56d0b802fba48d5f5caa2cdee7958acbcb795c830416/isort-8.0.1-py3-none-any.whl", hash = "sha256:28b89bc70f751b559aeca209e6120393d43fbe2490de0559662be7a9787e3d75", size = 89733, upload-time = "2026-02-28T10:08:19.466Z" },
]
[[package]]
name = "jinja2"
version = "3.1.6"
@ -1410,6 +1483,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/a5/54/12ed58d458474aaab5c3d180173e745a4fe131bb330370596876d19ff60f/mako-1.4.1-py3-none-any.whl", hash = "sha256:a359d9a94a541213958742b2698d0a7757bb83551767bc468a74b9905aba9617", size = 80010, upload-time = "2026-08-05T06:10:58.248Z" },
]
[[package]]
name = "mammoth"
version = "1.12.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "cobble" },
]
sdist = { url = "https://files.pythonhosted.org/packages/46/92/aca6c4208b3a8e56a788b224e6c9b7bb70fc68e6641124fcf784008338e1/mammoth-1.12.1.tar.gz", hash = "sha256:40521e02568583b671d9976b0fbeed50be734b40f73ca8e1939ab7146bf69bbe", size = 53797, upload-time = "2026-08-09T14:11:20.096Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/be/b4/e7907b278386ffec0cf631db93836e02de9a6386add4653a52d7ac6b43ae/mammoth-1.12.1-py2.py3-none-any.whl", hash = "sha256:2af047e3e796faa25740112310ddf11f8de2a24c96dc57de3c87dfd7cb6543b3", size = 55091, upload-time = "2026-08-09T14:11:18.854Z" },
]
[[package]]
name = "markupsafe"
version = "3.0.3"
@ -2272,6 +2357,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" },
]
[[package]]
name = "python-magic"
version = "0.4.27"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/da/db/0b3e28ac047452d079d375ec6798bf76a036a08182dbb39ed38116a49130/python-magic-0.4.27.tar.gz", hash = "sha256:c1ba14b08e4a5f5c31a302b7721239695b2f0f058d125bd5ce1ee36b9d9d3c3b", size = 14677, upload-time = "2022-06-07T20:16:59.508Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/6c/73/9f872cb81fc5c3bb48f7227872c28975f998f3e7c2b1c16e95e6432bbb90/python_magic-0.4.27-py2.py3-none-any.whl", hash = "sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3", size = 13840, upload-time = "2022-06-07T20:16:57.763Z" },
]
[[package]]
name = "python-multipart"
version = "0.0.32"
@ -2462,6 +2556,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/75/0a/67e95f21498de41433babf7b1db0eeab449eb58872dfb831b27747a70fd0/starlette-1.4.1-py3-none-any.whl", hash = "sha256:7d078e0fbefae0d2cecfb80a799d6fb84b1c0c6acd4f14ac79d17d0e7ec27f19", size = 74019, upload-time = "2026-08-05T15:17:24.357Z" },
]
[[package]]
name = "striprtf"
version = "0.0.32"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/86/f9/b3ecbd2e0abb9c29edd2913482f0c5df47a098221444ccd50cc5cd3f9b91/striprtf-0.0.32.tar.gz", hash = "sha256:7f375a375d99a2700842173168c90c9b545cb244241ffc5d868ed9f6baf9155f", size = 7571, upload-time = "2026-04-27T15:42:37.15Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/c3/cf/4157959aa36a567faab32a5fb078904b76654bdd86a3d7f8fba21ed89d84/striprtf-0.0.32-py3-none-any.whl", hash = "sha256:9f695fea9662cb4ffccf2b56286b3e77028c0a61e87258719584e681e0144869", size = 8044, upload-time = "2026-04-27T15:42:35.948Z" },
]
[[package]]
name = "structlog"
version = "26.1.0"
@ -2495,6 +2598,31 @@ rabbitmq = [
{ name = "pika" },
]
[[package]]
name = "ty"
version = "0.0.72"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/d5/df/656e684bafb13c1d146e7d5b5f3e7978ca177232acc84998ff36427e9462/ty-0.0.72.tar.gz", hash = "sha256:ec2b8066b618df18cab4cb8e992f8da45d360332acb23fa34df7fa29cd1b9d3a", size = 6654939, upload-time = "2026-08-14T21:35:42.612Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/e2/3b/f51461239a4e66565d4b362f97a3b55fe7fdba2e944068341f87c62f6743/ty-0.0.72-py3-none-linux_armv6l.whl", hash = "sha256:fda86db153ffd85ee52000cf175d6a3f1c0223772cf7c5b6f726200bf92c7b44", size = 12621989, upload-time = "2026-08-14T21:35:01.676Z" },
{ url = "https://files.pythonhosted.org/packages/ca/fb/79ddf683affc679ca856f3510b5640ec3a88a842ba5f654f5d4bc78f1786/ty-0.0.72-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ceb944c612529b9023acfdc9cf4c0dcbb722549f9d17d46baecd1141baf01d7f", size = 12233910, upload-time = "2026-08-14T21:35:04.334Z" },
{ url = "https://files.pythonhosted.org/packages/5d/45/10562a0d84802158db8fa4ec46de54aa9fdcecdeeaabbfe3639ae7042b66/ty-0.0.72-py3-none-macosx_11_0_arm64.whl", hash = "sha256:108d76218333d6c092e5f1cebf8e9b06f25738613a0236a28e2dd47c936ee52c", size = 12084108, upload-time = "2026-08-14T21:35:06.686Z" },
{ url = "https://files.pythonhosted.org/packages/a1/dc/1fe1aef8d697e3509face271a5331700c7aa1d1e44a4b622707bdfa41d4b/ty-0.0.72-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f3943f186f741a2499a31053872169250c9264a9a49684920e48d8fcf4ef4f5", size = 12132640, upload-time = "2026-08-14T21:35:09.305Z" },
{ url = "https://files.pythonhosted.org/packages/14/46/41ceb265e96969487311a2014bd0e53abb4fbc1395efb2ebe411fcb4db62/ty-0.0.72-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cf283c07dc3cc52ca48a3ad8ab100fb5aec3aebbd03ef6a12d5f910b8e596fc5", size = 12402489, upload-time = "2026-08-14T21:35:11.555Z" },
{ url = "https://files.pythonhosted.org/packages/2b/45/30bf43cb4fd505c5c2dd30fda27dde5f05208686cd21217adec77c954204/ty-0.0.72-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:95f3b6462c38f9f115d10cee21f47fedf715fcf2040daf36eef210359300bc7c", size = 13130835, upload-time = "2026-08-14T21:35:13.746Z" },
{ url = "https://files.pythonhosted.org/packages/31/2f/03bba754d2613f640df168335c41f83f41db150bb515839c60d80e3a7880/ty-0.0.72-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:30caf658feb8ffb250d9e9e47107657a78f5f3425c227df1664d8df2ebe38880", size = 13590392, upload-time = "2026-08-14T21:35:16.839Z" },
{ url = "https://files.pythonhosted.org/packages/04/c7/03c67f00e63005ec41585653dc3096064570b1e6273742baae2798cd242f/ty-0.0.72-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:27bdc012ddfbeec8948e4a6036c0dc39ac7cf2c8ec7c7d48dc7d2fd56d57b399", size = 13309629, upload-time = "2026-08-14T21:35:19.169Z" },
{ url = "https://files.pythonhosted.org/packages/c1/df/102d3b264eb7f2a58dd11952f229bb5150bb5668d176a6154976a6675981/ty-0.0.72-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:802c5970a77d7739e6f499921fbb6984fb7ad8a31d95e1ff42fd46f3642e4f3b", size = 12734028, upload-time = "2026-08-14T21:35:22.099Z" },
{ url = "https://files.pythonhosted.org/packages/61/85/d0737c8c54d0ba67366ddfb9f31d88edf0b02299e65923e6945ae60ebcb5/ty-0.0.72-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:47dce65114fdc615c68ca0edb393b433df0956447e4267df0e264137a789598d", size = 13174832, upload-time = "2026-08-14T21:35:24.71Z" },
{ url = "https://files.pythonhosted.org/packages/1e/31/497f5a96c36d9b586ab6afe0574986835c6fd5b835a89773d2bec4711b49/ty-0.0.72-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:325144fa07e2675d0faa337fcc864213c272a499eb0cfe5bde2fdc62282d27bc", size = 12215005, upload-time = "2026-08-14T21:35:26.892Z" },
{ url = "https://files.pythonhosted.org/packages/df/7d/46e65b17b4966c7cd0140f134380d33d8e84fe6efccd761533ce793dc502/ty-0.0.72-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a5c9f15d0f58e43707d8848274be1821a0ef408eccb8aa7dda28a4a9eddf7640", size = 12421298, upload-time = "2026-08-14T21:35:29.301Z" },
{ url = "https://files.pythonhosted.org/packages/08/2a/12ada4ec17700b3cb1d4fd3bc3e5b1852df9e6885288429318cade87b3c1/ty-0.0.72-py3-none-musllinux_1_2_i686.whl", hash = "sha256:8ee508d64b381871529cc22c412b41071bf5e908b7aa5d66a38f3f6b2573a806", size = 12669242, upload-time = "2026-08-14T21:35:31.444Z" },
{ url = "https://files.pythonhosted.org/packages/1c/1a/4692536880790fb550ed6d44a6096778dc71bb112f2c6d615cebb01a57e5/ty-0.0.72-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:3699e2ec7921d44da79d6b089f7bf239b2cc53c4e45a5a38430adc34ee9e9a55", size = 12988199, upload-time = "2026-08-14T21:35:33.749Z" },
{ url = "https://files.pythonhosted.org/packages/9a/0d/f5e5a50322e9c45865e7b7a428ba6cd6527387cf0f2472492ac3cf746243/ty-0.0.72-py3-none-win32.whl", hash = "sha256:f25f72a67bd36cd247707c4784e52fad0b6b4f42a1b7dd14804110fa95c486ed", size = 11939708, upload-time = "2026-08-14T21:35:36.006Z" },
{ url = "https://files.pythonhosted.org/packages/3f/4e/8af3534b2e4214e6184a5a59c34101e94a68d578f081f97b995866bab1bf/ty-0.0.72-py3-none-win_amd64.whl", hash = "sha256:cdeee869341717e1736cea2e2d7856738c6957c320f584ed2f68c8f90100d2f5", size = 12643876, upload-time = "2026-08-14T21:35:38.141Z" },
{ url = "https://files.pythonhosted.org/packages/ff/ea/a2606e654c7276bd08586391a2525b0af3f3bf60228a8c57b2d248f273f9/ty-0.0.72-py3-none-win_arm64.whl", hash = "sha256:1bd3ac3ed4424a6d6990a85dc388556aea012bd752de21349a84b685951de0d8", size = 12394857, upload-time = "2026-08-14T21:35:40.277Z" },
]
[[package]]
name = "typing-extensions"
version = "4.16.0"