diff --git a/.env.example b/.env.example index 68176f4..e509746 100644 --- a/.env.example +++ b/.env.example @@ -53,6 +53,19 @@ OLLAMA_TEMPERATURE=0.2 OLLAMA_NUM_PREDICT=3072 OLLAMA_TIMEOUT=120 OLLAMA_MAX_CONCURRENCY=3 + +# --- YandexGPT (alternative provider) --- +# Set LLM_PROVIDER=yandex_gpt to use. Get API key & folder ID from Yandex Cloud. +YANDEXGPT_API_KEY=replace-me +YANDEXGPT_FOLDER_ID=replace-me +YANDEXGPT_MODEL=yandexgpt-lite # yandexgpt / yandexgpt-lite / yandexgpt-preview etc. +YANDEXGPT_FALLBACK_MODEL= # empty disables fallback +YANDEXGPT_BASE_URL=https://llm.api.cloud.yandex.net +YANDEXGPT_COMPLETION_PATH=/foundationModels/v1/completion +YANDEXGPT_TEMPERATURE=0.2 +YANDEXGPT_MAX_TOKENS=3072 +YANDEXGPT_TIMEOUT=120 +YANDEXGPT_MAX_CONCURRENCY=3 CHUNK_SIZE_CHARS=10000 # --- API (FastAPI) --- @@ -82,6 +95,16 @@ PASSWORD_RESET_TTL_MINUTES=60 PASSWORD_MIN_LENGTH=8 WEB_APP_BASE_URL=http://localhost:5173 # SPA base — used to build reset links +# --- Prescreen stage (hybrid heuristic + optional LLM fallback) --- +PRESCREEN_ENABLED=true +PRESCREEN_AUTO_APPROVE=false +PRESCREEN_CONFIDENCE_THRESHOLD=0.75 +PRESCREEN_HIGH_VALUE_THRESHOLD=100000 +PRESCREEN_LLM_FALLBACK_ENABLED=false # kill-switch; set true after burn-in +PRESCREEN_LLM_FALLBACK_THRESHOLD=0.75 # <= confidence threshold +PRESCREEN_LLM_MAX_CHARS=20000 # cost-control cap for fallback +PRESCREEN_KEEP_REGEX=false # temporary rollback to regex-v1 + # --- SMTP (notification transport; empty host → dev logger) --- SMTP_HOST= SMTP_PORT=587 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92a290b..acca363 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,8 +35,8 @@ jobs: - name: Ruff format check run: uv run ruff format --check src tests - - name: Mypy - run: uv run mypy src + - name: Ty type check + run: uv run ty check src test-unit: name: Unit tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8a8a0ac..2776f2a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,6 +18,13 @@ repos: - repo: local hooks: + - id: isort + name: isort + entry: uv run isort + language: system + types: [python] + require_serial: true + - id: ruff-check name: ruff check (autofix) entry: uv run ruff check --fix --exit-non-zero-on-fix @@ -32,9 +39,9 @@ repos: types: [python] require_serial: true - - id: mypy - name: mypy (src) - entry: uv run mypy src + - id: ty + name: ty (src) + entry: uv run ty check src language: system types: [python] pass_filenames: false diff --git a/Makefile b/Makefile index e72722b..0f5ebbc 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # «Контракт-чек» — everyday commands (uv + docker) # Usage: make (see `make help`) -.PHONY: help install lint typecheck test test-unit test-integration migrate \ +.PHONY: help install lint lint-fix isort isort-check typecheck test test-unit test-integration migrate \ infra-up infra-down infra-logs services-up services-down services-logs \ api api-logs bot bot-logs worker-extract worker-analyze worker-notify \ seed-token jwt-secret jwt-token jwt-verify health shell-api shell-bot \ @@ -20,12 +20,23 @@ help: ## Show available commands install: ## Sync dev dependencies (uv) uv sync --group dev -lint: ## Run ruff linter + import sorter +lint: ## Run ruff linter + format check uv run ruff check src tests uv run ruff format --check src tests -typecheck: ## Run mypy type checker - uv run mypy src +lint-fix: ## Run ruff autofix + format + isort fix + uv run ruff check --fix src tests + uv run ruff format src tests + uv run isort src tests + +isort: ## Run isort (sort imports in-place) + uv run isort src tests + +isort-check: ## Run isort in check-only mode + uv run isort --check-only src tests + +typecheck: ## Run ty type checker + uv run ty check src test: ## Run all tests (unit + integration) uv run pytest diff --git a/docker-compose.yml b/docker-compose.yml index e6cfa49..b1c0a24 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -162,6 +162,16 @@ services: OLLAMA_NUM_PREDICT: ${OLLAMA_NUM_PREDICT:-3072} OLLAMA_TIMEOUT: ${OLLAMA_TIMEOUT:-120} OLLAMA_MAX_CONCURRENCY: ${OLLAMA_MAX_CONCURRENCY:-3} + YANDEXGPT_API_KEY: ${YANDEXGPT_API_KEY:-} + YANDEXGPT_FOLDER_ID: ${YANDEXGPT_FOLDER_ID:-} + YANDEXGPT_MODEL: ${YANDEXGPT_MODEL:-yandexgpt-lite} + YANDEXGPT_FALLBACK_MODEL: ${YANDEXGPT_FALLBACK_MODEL:-} + YANDEXGPT_BASE_URL: ${YANDEXGPT_BASE_URL:-https://llm.api.cloud.yandex.net} + YANDEXGPT_COMPLETION_PATH: ${YANDEXGPT_COMPLETION_PATH:-/foundationModels/v1/completion} + YANDEXGPT_TEMPERATURE: ${YANDEXGPT_TEMPERATURE:-0.2} + YANDEXGPT_MAX_TOKENS: ${YANDEXGPT_MAX_TOKENS:-3072} + YANDEXGPT_TIMEOUT: ${YANDEXGPT_TIMEOUT:-120} + YANDEXGPT_MAX_CONCURRENCY: ${YANDEXGPT_MAX_CONCURRENCY:-3} CHUNK_SIZE_CHARS: ${CHUNK_SIZE_CHARS:-10000} API_HOST: ${API_HOST:-0.0.0.0} API_PORT: ${API_PORT:-8000} @@ -277,6 +287,16 @@ services: OLLAMA_NUM_PREDICT: ${OLLAMA_NUM_PREDICT:-3072} OLLAMA_TIMEOUT: ${OLLAMA_TIMEOUT:-120} OLLAMA_MAX_CONCURRENCY: ${OLLAMA_MAX_CONCURRENCY:-3} + YANDEXGPT_API_KEY: ${YANDEXGPT_API_KEY:-} + YANDEXGPT_FOLDER_ID: ${YANDEXGPT_FOLDER_ID:-} + YANDEXGPT_MODEL: ${YANDEXGPT_MODEL:-yandexgpt-lite} + YANDEXGPT_FALLBACK_MODEL: ${YANDEXGPT_FALLBACK_MODEL:-} + YANDEXGPT_BASE_URL: ${YANDEXGPT_BASE_URL:-https://llm.api.cloud.yandex.net} + YANDEXGPT_COMPLETION_PATH: ${YANDEXGPT_COMPLETION_PATH:-/foundationModels/v1/completion} + YANDEXGPT_TEMPERATURE: ${YANDEXGPT_TEMPERATURE:-0.2} + YANDEXGPT_MAX_TOKENS: ${YANDEXGPT_MAX_TOKENS:-3072} + YANDEXGPT_TIMEOUT: ${YANDEXGPT_TIMEOUT:-120} + YANDEXGPT_MAX_CONCURRENCY: ${YANDEXGPT_MAX_CONCURRENCY:-3} CHUNK_SIZE_CHARS: ${CHUNK_SIZE_CHARS:-10000} TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:-} JWT_SECRET: ${JWT_SECRET} @@ -285,6 +305,67 @@ services: ports: - "9102:9102" + worker-prescreen: + profiles: ["services"] + build: + context: . + dockerfile: srv/worker-prescreen/Dockerfile + container_name: contract_check-worker-prescreen + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + minio-init: + condition: service_completed_successfully + environment: + ENV: ${ENV:-dev} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-contract_check}:${POSTGRES_PASSWORD:-contract_check}@postgres:5432/${POSTGRES_DB:-contract_check} + RABBITMQ_URL: amqp://${RABBITMQ_USER:-contract_check}:${RABBITMQ_PASS:-contract_check}@rabbitmq:5672/${RABBITMQ_VHOST:-/} + S3_ENDPOINT_URL: http://minio:9000 + S3_ACCESS_KEY: ${S3_ACCESS_KEY:-contract_check} + S3_SECRET_KEY: ${S3_SECRET_KEY:-contract_check} + S3_BUCKET: ${S3_BUCKET:-contract-check-docs} + S3_REGION: ${S3_REGION:-us-east-1} + REFUND_POLICY: ${REFUND_POLICY:-all} + SENTRY_DSN: ${SENTRY_DSN:-} + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-} + OTEL_SERVICE_NAME: worker-prescreen + MQ_PREFETCH_PRESCREEN: ${MQ_PREFETCH_PRESCREEN:-1} + MQ_MAX_ATTEMPTS: ${MQ_MAX_ATTEMPTS:-5} + MQ_RETRY_BASE_MS: ${MQ_RETRY_BASE_MS:-2000} + PRESCREEN_ENABLED: ${PRESCREEN_ENABLED:-true} + PRESCREEN_AUTO_APPROVE: ${PRESCREEN_AUTO_APPROVE:-false} + PRESCREEN_CONFIDENCE_THRESHOLD: ${PRESCREEN_CONFIDENCE_THRESHOLD:-0.75} + PRESCREEN_HIGH_VALUE_THRESHOLD: ${PRESCREEN_HIGH_VALUE_THRESHOLD:-100000} + LLM_PROVIDER: ${LLM_PROVIDER:-ollama_cloud} + OLLAMA_HOST: ${OLLAMA_HOST:-} + OLLAMA_API_KEY: ${OLLAMA_API_KEY:-} + OLLAMA_MODEL: ${OLLAMA_MODEL:-qwen2.5:14b} + OLLAMA_FALLBACK_MODEL: ${OLLAMA_FALLBACK_MODEL:-qwen2.5:7b} + OLLAMA_TEMPERATURE: ${OLLAMA_TEMPERATURE:-0.2} + OLLAMA_NUM_PREDICT: ${OLLAMA_NUM_PREDICT:-3072} + OLLAMA_TIMEOUT: ${OLLAMA_TIMEOUT:-120} + OLLAMA_MAX_CONCURRENCY: ${OLLAMA_MAX_CONCURRENCY:-3} + YANDEXGPT_API_KEY: ${YANDEXGPT_API_KEY:-} + YANDEXGPT_FOLDER_ID: ${YANDEXGPT_FOLDER_ID:-} + YANDEXGPT_MODEL: ${YANDEXGPT_MODEL:-yandexgpt-lite} + YANDEXGPT_FALLBACK_MODEL: ${YANDEXGPT_FALLBACK_MODEL:-} + YANDEXGPT_BASE_URL: ${YANDEXGPT_BASE_URL:-https://llm.api.cloud.yandex.net} + YANDEXGPT_COMPLETION_PATH: ${YANDEXGPT_COMPLETION_PATH:-/foundationModels/v1/completion} + YANDEXGPT_TEMPERATURE: ${YANDEXGPT_TEMPERATURE:-0.2} + YANDEXGPT_MAX_TOKENS: ${YANDEXGPT_MAX_TOKENS:-3072} + YANDEXGPT_TIMEOUT: ${YANDEXGPT_TIMEOUT:-120} + YANDEXGPT_MAX_CONCURRENCY: ${YANDEXGPT_MAX_CONCURRENCY:-3} + CHUNK_SIZE_CHARS: ${CHUNK_SIZE_CHARS:-10000} + JWT_SECRET: ${JWT_SECRET} + JWT_ALGORITHM: ${JWT_ALGORITHM:-HS256} + JWT_ACCESS_TTL_MINUTES: ${JWT_ACCESS_TTL_MINUTES:-1440} + ports: + - "9104:9104" + worker-notify: profiles: ["services"] build: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index e86ec43..8583efe 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -9,13 +9,15 @@ > > **One-line summary:** take the stage-0 prototype (PDF/DOCX → text → Ollama > Cloud → markdown report) and rebuild it as a production, event-driven, -> multi-container application: a FastAPI **api**, two RabbitMQ-consuming -> **workers** (extract/OCR and LLM-analyze, split for CPU vs I/O profiles), -> a Telegram **bot** adapter, backed by **Postgres + RabbitMQ + MinIO + Redis**, -> observable via **structlog + Sentry + Prometheus/Grafana + OpenTelemetry**, -> behind **Nginx + certbot**. Monorepo, five fine-tuned Docker images -> (`srv/` — api + two workers + bot + prototype), shared `contract_check.core` domain package. Compose now, k8s-ready later. -> HA-*ready* (quorum queues, WAL-archiving Postgres), not HA-*running* on one +> multi-container application: a FastAPI **api**, three RabbitMQ-consuming +> **workers** (extract/OCR, deterministic prescreen, and LLM-analyze, split for +> CPU vs I/O profiles), a Telegram **bot** adapter, backed by +> **Postgres + RabbitMQ + MinIO + Redis**, observable via +> **structlog + Sentry + Prometheus/Grafana + OpenTelemetry**, behind +> **Nginx + certbot**. Monorepo, six fine-tuned Docker images +> (`srv/` — api + three workers + bot + prototype), shared `contract_check.core` +> domain package. Compose now, k8s-ready later. +> HA-*ready* (quorum queues, WAL archiving Postgres), not HA-*running* on one > VPS — see §10. --- @@ -39,9 +41,11 @@ The owner has explicitly pivoted away from all three: Additional decisions locked during the architecture questionnaire: -- **Two worker pools**, not one: `worker-extract` (pymupdf + Tesseract, CPU) - and `worker-analyze` (LLM, I/O). This pairs naturally with the pipeline - topology and lets each pool scale independently. +- **Three worker pools**, not one: `worker-extract` (pymupdf + Tesseract, CPU), + `worker-prescreen` (hybrid heuristic → optional LLM-fallback metadata extraction, CPU but + tiny), and `worker-analyze` (LLM, I/O). The prescreen stage routes low-risk + contracts to manual/auto-approval and high-risk contracts to deep LLM analysis, + protecting the expensive LLM worker from trivial documents. - **LLM behind a provider interface** (`core/llm/port.py`), Ollama Cloud as the first realization — swappable later for self-hosted Ollama / GigaChat. - **Refund policy is a runtime switch** (`REFUND_POLICY=all|infra_only`) with @@ -69,17 +73,21 @@ HTTP clients to the api and touch nothing but the api. ┌───────────────────── HTTPS (Nginx + certbot) ─────────────────────┐ │ │ Telegram ───► bot (aiogram) ──HTTP──► api (FastAPI) ──publish──► RabbitMQ (direct) - (adapter, HTTP-only) (core) │ - │ owns: Postgres, MinIO, ▼ - │ Redis, credits, tokens extract.q ──► worker-extract - │ (core) (quorum) (core, CPU: pymupdf+tesseract) - ▼ │ publish document.extracted - Postgres ▼ - MinIO ◄──── read/write ────► analyze.q ──► worker-analyze - Redis (rate limit/sess) (quorum) (core, I/O: LLM provider) - │ save Report, status=done - ▼ - Postgres + (adapter, HTTP-only) (core) │ + │ owns: Postgres, MinIO, ▼ + │ Redis, credits, tokens extract.q ──► worker-extract + │ (core) (quorum) (core, CPU: pymupdf+tesseract) + ▼ │ publish PrescreenRequested + Postgres ▼ + MinIO ◄──── read/write ────► prescreen.q ──► worker-prescreen + Redis (rate limit/sess) (quorum) (hybrid heuristic router) + │ publish AnalyzeRequested + ▼ + analyze.q ──► worker-analyze + (quorum) (core, I/O: LLM provider) + │ save Report, status=done + ▼ + Postgres ``` Data plane: @@ -103,11 +111,11 @@ Control plane: | Area | Decision | Replaces | |---|---|---| -| Repo | Monorepo, shared `contract_check.core`, 5 Dockerfiles in `srv//` | one-image `MODE` dispatch | -| Scope now | api + worker-extract + worker-analyze + bot (+ prototype benchmark) | (web/payments later) | +| Repo | Monorepo, shared `contract_check.core`, 6 Dockerfiles in `srv//` | one-image `MODE` dispatch | +| Scope now | api + worker-extract + worker-prescreen + worker-analyze + bot (+ prototype benchmark) | (web/payments later) | | Queue | RabbitMQ, direct exchange, pipeline fan-out | arq + Redis | | Retry | TTL retry queues, exponential backoff, final DLQ | — | -| Workers | Two pools (extract/OCR + analyze/LLM) | single arq worker | +| Workers | Three pools (extract/OCR + prescreen + analyze/LLM) | single arq worker | | Storage | MinIO, `users/{uid}/docs/{did}.{ext}` | Selectel S3 | | Upload | Proxy through API (multipart) | — | | Doc retention | TTL purge of raw docs after N days | — | @@ -115,7 +123,7 @@ Control plane: | Auth | Per-adapter `service_tokens`, revocable | single `SERVICE_TOKEN` | | Report delivery | Polling now (fine-grained stage), SSE/webhook later | — | | Sync `/analyze` | No | — | -| Doc status | Fine-grained `queued→extracting→ocr→analyzing→done\|failed` | coarse status | +| Doc status | Fine-grained `queued→extracting→prescreening→ocr→analyzing→done\|failed` | coarse status | | Extra tables | jobs, service_tokens, invoices(stub) | 3-table plan | | Report storage | JSONB + markdown column | — | | Durability | HA-*ready* (quorum queues, WAL archive) | — | @@ -199,14 +207,19 @@ DealDocumentScreening/ │ │ │ ├── port.py (LLMProvider Protocol + AnalysisResult dataclass) │ │ │ ├── ollama_cloud.py (ports prototype llm_client: repair-loop + 429 fallback) │ │ │ └── factory.py (provider selection by LLM_PROVIDER env) -│ │ ├── analysis/ +│ │ ├── extraction/ (new — DocumentExtractor port + adapters) +│ │ │ ├── port.py (ExtractedDocument, DocumentExtractor Protocol, errors) +│ │ │ ├── factory.py (mime/suffix → adapter; SUPPORTED_SUFFIXES gate) +│ │ │ ├── adapters/ (pdf_pymupdf, docx_mammoth, rtf_striprtf, txt_chardet, ocr_tesseract) +│ │ │ └── __init__.py (extract_document orchestrator with OCR fallback) +│ │ ├── analysis/ (existing; now consumed by prototype + ocr adapter) │ │ │ ├── __init__.py -│ │ │ ├── extractor.py (← from contract_check/extractor.py) -│ │ │ ├── chunker.py (← from contract_check/chunker.py) +│ │ │ ├── extractor.py (← from contract_check/extractor.py; prototype use) +│ │ │ ├── chunker.py (← from contract_check/chunker.py; + chunk_markdown) │ │ │ ├── checklist.py (← from contract_check/checklist.py) │ │ │ ├── report_schema.py (← from contract_check/report_schema.py) -│ │ │ ├── ocr.py (new — pytesseract via pymupdf rasterize, <100 chars trigger) -│ │ │ └── analyzer.py (new — orchestrate chunk→LLM→merge→dedupe→sort→markdown) +│ │ │ ├── ocr.py (pytesseract via pymupdf rasterize) +│ │ │ └── analyzer.py (orchestrate chunk→LLM→merge→dedupe→sort→markdown) │ │ ├── credits.py (reserve_credit atomic UPDATE; refund_credit idempotent + policy) │ │ └── tokens.py (service-token hash/verify; FastAPI dependency) │ ├── api/ (new — FastAPI image) @@ -226,13 +239,23 @@ DealDocumentScreening/ │ ├── worker_extract/ (new — CPU image: pymupdf + tesseract) │ │ ├── __init__.py │ │ ├── __main__.py (entrypoint: start consumer + metrics server) -│ │ ├── handler.py (handle DocumentUploaded: MinIO dl→extract→(ocr)→upload text→publish) -│ │ ├── extract_document.py (extraction/OCR orchestration helper) +│ │ ├── handler.py (handle DocumentUploaded: MinIO dl → extraction factory → OCR fallback → upload .txt (markdown) → publish PrescreenRequested) │ │ └── consumer.py (wire handler into core.mq.consumer base) +│ ├── worker_prescreen/ (new — lean CPU image: hybrid heuristic + optional LLM fallback) +│ │ ├── __init__.py +│ │ ├── __main__.py (entrypoint: metrics server on :9104) +│ │ ├── config.py (PrescreenSettings, prefetch tunable) +│ │ ├── handler.py (handle PrescreenRequested: download text → hybrid extract → route → persist prescreen_results → publish AnalyzeRequested/auto-approve/manual-review) +│ │ ├── consumer.py (wire handler into core.mq.consumer base) +│ │ ├── extractor.py (PrescreenContractMeta + MetadataExtractor protocol + compat shim) +│ │ ├── extractor_heuristic.py (Stage 1: deterministic keyword/positional extractor, regex-free) +│ │ ├── extractor_llm.py (Stage 2: dict → validated PrescreenContractMeta) +│ │ ├── extractor_hybrid.py (orchestrator: heuristic → optional LLM fallback) +│ │ └── router.py (decision logic: auto_approve / manual_review / deep_analysis) │ ├── worker_analyze/ (new — I/O image: LLM provider) │ │ ├── __init__.py │ │ ├── __main__.py -│ │ ├── handler.py (handle DocumentExtracted: text dl→chunk→LLM→validate→save report→done) +│ │ ├── handler.py (handle AnalyzeRequested: text dl→chunk→LLM (with prescreen context)→validate→save report→done) │ │ └── consumer.py │ ├── bot/ (new — aiogram adapter, HTTP-only) │ │ ├── __init__.py @@ -247,7 +270,11 @@ DealDocumentScreening/ ├── conftest.py (fixtures: testcontainers pg/rabbit/minio) ├── unit/ │ ├── test_chunker.py + │ ├── test_extraction_adapters.py + │ ├── test_extraction_factory.py │ ├── test_extractor.py + │ ├── test_prescreen_extractor.py + │ ├── test_prescreen_router.py │ ├── test_checklist_report.py │ ├── test_llm_ollama_cloud.py (respx: 200 / 429-fallback / repair) │ ├── test_credits.py @@ -257,9 +284,9 @@ DealDocumentScreening/ │ └── test_bot_boundary.py (static AST: bot must not import core.db/s3/llm/mq/credits) └── integration/ ├── conftest.py (seed service-token + api-key; testcontainers wiring) - ├── test_upload_pipeline.py (POST /documents → message on extract.q) - ├── test_extract_worker.py (DocumentUploaded → DocumentExtracted published) - ├── test_analyze_worker.py (DocumentExtracted → Report saved, status done) + ├── test_upload_pipeline.py (POST /documents → message on extract.q or prescreen.q) + ├── test_extract_worker.py (DocumentUploaded → PrescreenRequested published) + ├── test_analyze_worker.py (AnalyzeRequested → Report saved, status done) ├── test_credits_db.py (reserve/refund against real Postgres) └── test_b2b_api.py (X-API-Key: 202 → poll report; 401/429 branches) ``` @@ -267,7 +294,7 @@ DealDocumentScreening/ **Import rule (hexagonal boundary, enforced in review):** - `core/*` may import anything. -- `api/*`, `worker_extract/*`, `worker_analyze/*` import only `core/*`. +- `api/*`, `worker_extract/*`, `worker_prescreen/*`, `worker_analyze/*` import only `core/*`. - `bot/*` imports only `httpx` + its own modules. **It must NOT import `core.db`, `core.s3`, `core.llm`, `core.mq`, `core.credits`.** It speaks to the api over HTTP only. (Add a ruff/flake8 import-forbidden rule or a unit @@ -286,18 +313,21 @@ service start). | Name | Type | Purpose | |---|---|---| -| `contracts.x` | direct | Main exchange. Routing keys: `extract`, `analyze`. | -| `contracts.retry.x` | direct | DLX target of main queues; routing keys: `retry.extract`, `retry.analyze`. | +| `contracts.x` | direct | Main exchange. Routing keys: `extract`, `prescreen`, `analyze`. | +| `contracts.retry.x` | direct | DLX target of main queues; routing keys: `retry.extract`, `retry.prescreen`, `retry.analyze`. | ### Queues | Name | Type | Args | Bound (exchange / rk) | Consumers | |---|---|---|---|---| | `extract.q` | **quorum** | `x-dead-letter-exchange=contracts.retry.x`, `x-dead-letter-routing-key=retry.extract` | contracts.x / `extract` | worker-extract | +| `prescreen.q` | **quorum** | `x-dead-letter-exchange=contracts.retry.x`, `x-dead-letter-routing-key=retry.prescreen` | contracts.x / `prescreen` | worker-prescreen | | `analyze.q` | **quorum** | `x-dead-letter-exchange=contracts.retry.x`, `x-dead-letter-routing-key=retry.analyze` | contracts.x / `analyze` | worker-analyze | | `extract.retry.q` | classic | `x-dead-letter-exchange=contracts.x`, `x-dead-letter-routing-key=extract`; per-message `expiration` | contracts.retry.x / `retry.extract` | none (delay slot) | +| `prescreen.retry.q` | classic | `x-dead-letter-exchange=contracts.x`, `x-dead-letter-routing-key=prescreen`; per-message `expiration` | contracts.retry.x / `retry.prescreen` | none (delay slot) | | `analyze.retry.q` | classic | `x-dead-letter-exchange=contracts.x`, `x-dead-letter-routing-key=analyze`; per-message `expiration` | contracts.retry.x / `retry.analyze` | none (delay slot) | | `extract.dlq` | **quorum** | — | — | none (manual requeue) | +| `prescreen.dlq` | **quorum** | — | — | none (manual requeue) | | `analyze.dlq` | **quorum** | — | — | none (manual requeue) | ### Flow & retry mechanics @@ -306,34 +336,49 @@ service start). (persistent, `correlation_id` UUID in both message id and `headers["x-correlation-id"]`). 2. **worker-extract** consumes from `extract.q` with `prefetch=1..N` (config, - default 1 — extraction is CPU-bound). On success: upload extracted text - to MinIO, publish `DocumentExtracted` to `contracts.x` rk `analyze`, - ack. On failure: see retry below. -3. **worker-analyze** consumes from `analyze.q` with `prefetch=3` (mirrors - Ollama Pro concurrency). On success: save report, `status=done`, ack. - On failure: retry. -4. **Retry (nack):** the base consumer does **not** use `basic.nack(requeue=True)` - (instant re-redelivery, no backoff). Instead it publishes a copy of the - message to the matching `*.retry.q` with: - - `headers["x-attempt"] = attempt + 1` - - `expiration = str(int(BASE_DELAY_MS * 2 ** attempt))` (e.g. - `BASE_DELAY_MS=2000` → 2s, 4s, 8s, 16s, 32s) - - then `basic_ack` the original. - When the retry queue's per-message TTL expires, its DLX bounces the - message back to `contracts.x` with rk `extract`/`analyze` → re-enters the - main quorum queue. Clean, plugin-free, exponential. -5. **Poison (max attempts):** when `headers["x-attempt"] >= MAX_ATTEMPTS` - (default 5), the consumer publishes to `extract.dlq` / `analyze.dlq` - instead of retry, acks the original, sets `jobs.dlq=true`, - `jobs.last_failure_class`, transitions `documents.status=failed`, and - calls `refund_credit(document_id, failure_class)` per policy (§8). -6. **Manual requeue:** a CLI script `scripts/mq_requeue.py` (or mgmt UI) - moves a DLQ message back to the main exchange with reset attempt. + default 1 — extraction is CPU-bound). On success: upload extracted **Markdown** + to MinIO (`text/markdown`), publish `PrescreenRequested` to `contracts.x` rk + `prescreen` (`is_structured`/`has_tables` metadata included), ack. On failure: + see retry below. +3. **worker-prescreen** consumes from `prescreen.q` with `prefetch=1`. It runs a + **hybrid two-stage extractor**: Stage 1 is a deterministic regex-free heuristic + over keyword dictionaries and positional windows; Stage 2 is an LLM fallback + (`extract_prescreen` on `core/llm/port.py`) triggered only when heuristic + confidence is below `PRESCREEN_LLM_FALLBACK_THRESHOLD` and the kill-switch + `PRESCREEN_LLM_FALLBACK_ENABLED` is true. It computes a weighted + `confidence_score`, routes to `auto_approve` (lightweight report, disabled by + default), `manual_review` (terminal DB state for future admin SPA), or + `deep_analysis`. For `deep_analysis` it publishes `AnalyzeRequested` to + `contracts.x` rk `analyze` carrying `prescreen_meta`. On failure: retry. + The v1 regex implementation is kept behind `PRESCREEN_KEEP_REGEX=true` for one + release as a rollback safety net. +4. **worker-analyze** consumes from `analyze.q` with `prefetch=3` (mirrors + Ollama Pro concurrency). On success: save report, `status=done`, ack. + On failure: retry. +5. **Retry (nack):** the base consumer does **not** use `basic.nack(requeue=True)` + (instant re-redelivery, no backoff). Instead it publishes a copy of the + message to the matching `*.retry.q` with: + - `headers["x-attempt"] = attempt + 1` + - `expiration = str(int(BASE_DELAY_MS * 2 ** attempt))` (e.g. + `BASE_DELAY_MS=2000` → 2s, 4s, 8s, 16s, 32s) + - then `basic_ack` the original. + When the retry queue's per-message TTL expires, its DLX bounces the + message back to `contracts.x` with rk `extract`/`prescreen`/`analyze` → re-enters the + main quorum queue. Clean, plugin-free, exponential. +6. **Poison (max attempts):** when `headers["x-attempt"] >= MAX_ATTEMPTS` + (default 5), the consumer publishes to `extract.dlq` / `prescreen.dlq` / + `analyze.dlq` instead of retry, acks the original, sets `jobs.dlq=true`, + `jobs.last_failure_class`, transitions `documents.status=failed`, and + calls `refund_credit(document_id, failure_class)` per policy (§8). +7. **Manual requeue:** a CLI script `scripts/mq_requeue.py` (or mgmt UI) + moves a DLQ message back to the main exchange with reset attempt. ### Prefetch & concurrency - worker-extract: RabbitMQ prefetch caps concurrent extraction jobs (CPU bound; default 1, tune to CPU count). +- worker-prescreen: prefetch=1; Stage 1 heuristic is cheap but routing decisions must be + ordered per document. - worker-analyze: prefetch caps concurrent LLM jobs (default 3, mirrors Ollama Pro). **Additionally** keep the in-process `asyncio.Semaphore` from the prototype `llm_client` — prefetch caps *jobs*, the semaphore @@ -368,13 +413,67 @@ class DocumentExtracted(BaseModel): extracted_s3_key: str # users/{uid}/docs/{did}.txt char_count: int ocr_used: bool + is_structured: bool = False + has_tables: bool = False + attempt: int = 0 + + +class PrescreenRequested(BaseModel): + correlation_id: UUID + document_id: UUID + user_id: UUID + text_s3_key: str # users/{uid}/docs/{did}.txt + filename: str + char_count: int = 0 + ocr_used: bool = False + is_structured: bool = False + has_tables: bool = False + attempt: int = 0 + + +class PrescreenCompleted(BaseModel): + correlation_id: UUID + document_id: UUID + user_id: UUID + text_s3_key: str + filename: str + prescreened_at: str + contract_type: str | None = None + party_a: str | None = None + party_b: str | None = None + total_amount: float | None = None + currency: str | None = None + start_date: str | None = None + end_date: str | None = None + has_penalty_clause: bool | None = None + has_termination_clause: bool | None = None + has_arbitration: bool | None = None + confidence_score: float | None = None + routing_decision: str # auto_approve | manual_review | deep_analysis + auto_summary: str | None = None + auto_findings: list[dict] = [] + extractor_version: str = "heuristic-v2" # heuristic-v2 | hybrid-llm-v1 + attempt: int = 0 + + +class AnalyzeRequested(BaseModel): + correlation_id: UUID + document_id: UUID + user_id: UUID + extracted_s3_key: str + filename: str + char_count: int = 0 + ocr_used: bool = False + is_structured: bool = False + has_tables: bool = False + prescreen_meta: PrescreenCompleted | None = None attempt: int = 0 ``` RabbitMQ `headers`: `x-correlation-id`, `x-attempt`, `x-origin` (api | -worker-extract). `content_type=application/json`, `delivery_mode=2` -(persistent). Validate on consume with the pydantic model; on validation -error → `.dlq` immediately with `failure_class=infra`. +worker-extract | worker-prescreen). `content_type=application/json`, +`delivery_mode=2` (persistent). Validate on consume with the pydantic model; +on validation error → `.dlq` immediately with `failure_class=infra`. --- @@ -434,6 +533,8 @@ Migrations (hand-written, async `env.py`): - `0002` `api_keys` + `api_key_requests` (B2B) - `0003` webUI auth columns on `users` (`email`, `password_hash`, reset tokens, `is_active`) - `0004` `users.role` for the admin panel (`'user'|'admin'`, default `'user'`) +- `0005` Telegram bot hardening columns on `users` +- `0006` `prescreen_results` table; expand `documents.status` and `jobs.queue` enums ```sql -- users @@ -464,7 +565,7 @@ CREATE TABLE documents ( mime TEXT NOT NULL, bytes BIGINT NOT NULL DEFAULT 0, status TEXT NOT NULL DEFAULT 'queued' - CHECK (status IN ('queued','extracting','ocr','analyzing','done','failed')), + CHECK (status IN ('queued','extracting','prescreening','ocr','analyzing','done','failed')), stage TEXT, -- sub-stage / human label refunded BOOLEAN NOT NULL DEFAULT FALSE, created_at TIMESTAMPTZ NOT NULL DEFAULT now(), @@ -475,23 +576,56 @@ CREATE INDEX documents_status_idx ON documents (status); -- reports CREATE TABLE reports ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - document_id UUID NOT NULL UNIQUE REFERENCES documents(id) ON DELETE CASCADE, - content_json JSONB NOT NULL, - markdown TEXT NOT NULL, - model_used TEXT, - prompt_tokens INTEGER NOT NULL DEFAULT 0, - eval_tokens INTEGER NOT NULL DEFAULT 0, - latency_ms INTEGER NOT NULL DEFAULT 0, - created_at TIMESTAMPTZ NOT NULL DEFAULT now() + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + document_id UUID NOT NULL UNIQUE REFERENCES documents(id) ON DELETE CASCADE, + content_json JSONB NOT NULL, + markdown TEXT NOT NULL, + model_used TEXT, + prompt_tokens INTEGER NOT NULL DEFAULT 0, + eval_tokens INTEGER NOT NULL DEFAULT 0, + latency_ms INTEGER NOT NULL DEFAULT 0, + prescreen_result_id UUID REFERENCES prescreen_results(id) ON DELETE SET NULL, + prescreen_meta JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() ); --- jobs (RabbitMQ correlation; one document has up to 2 jobs: extract + analyze) +-- prescreen_results (added by 0006) +CREATE TABLE prescreen_results ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + correlation_id UUID NOT NULL, + contract_type TEXT, + party_a TEXT, + party_b TEXT, + total_amount NUMERIC(18, 2), + currency TEXT, + start_date DATE, + end_date DATE, + has_penalty_clause BOOLEAN, + has_termination_clause BOOLEAN, + has_arbitration BOOLEAN, + confidence_score NUMERIC(4, 3) CHECK (confidence_score >= 0 AND confidence_score <= 1), + routing_decision TEXT NOT NULL DEFAULT 'manual_review' + CHECK (routing_decision IN ('auto_approve','manual_review','deep_analysis')), + prescreened_at TIMESTAMPTZ NOT NULL DEFAULT now(), + processing_ms INTEGER, + extractor_version TEXT NOT NULL DEFAULT 'heuristic-v2' + CHECK (extractor_version IN ('regex-v1','heuristic-v2','hybrid-llm-v1')), + auto_summary TEXT, + auto_findings JSONB NOT NULL DEFAULT '[]', + error_message TEXT, + retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0) +); +CREATE INDEX prescreen_results_document_idx ON prescreen_results (document_id); +CREATE INDEX prescreen_results_routing_idx ON prescreen_results (routing_decision); +CREATE INDEX prescreen_results_confidence_idx ON prescreen_results (confidence_score); + +-- jobs (RabbitMQ correlation; one document has up to 3 jobs: extract + prescreen + analyze) CREATE TABLE jobs ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE, correlation_id UUID NOT NULL, - queue TEXT NOT NULL CHECK (queue IN ('extract','analyze')), + queue TEXT NOT NULL CHECK (queue IN ('extract','prescreen','analyze')), attempts INTEGER NOT NULL DEFAULT 0, max_attempts INTEGER NOT NULL DEFAULT 5, last_failure_class TEXT, @@ -610,13 +744,17 @@ Set on the `jobs.last_failure_class` column. `REFUND_POLICY` env selects --- -## 9. LLM provider port + Ollama Cloud adapter +## 9. LLM provider port + adapters The prototype's `llm_client.py` (bearer httpx, `format: json-schema`, 5xx backoff, 429→fallback, repair-loop) is excellent. It becomes the first adapter behind a port so we can later swap in self-hosted Ollama, GigaChat, or YandexGPT without touching `analyzer.py`. +Registered providers are selected by `LLM_PROVIDER`: +- `ollama_cloud` (default) +- `yandex_gpt` + ### Port (`core/llm/port.py`) ```python @@ -636,7 +774,9 @@ class AnalysisResult: class LLMProvider(Protocol): - async def analyze(self, text: str, *, checklist: str) -> AnalysisResult: ... + async def analyze( + self, text: str, *, checklist: str, extra_context: str = "" + ) -> AnalysisResult: ... async def aclose(self) -> None: ... ``` @@ -646,13 +786,15 @@ Move the existing `OllamaCloudClient` + `ChatResult` + `_QuotaError` + `_run_with_fallback` + `_run_repair_loop` + `_post_chat` verbatim, then add an `analyze(text, *, checklist)` method that: -1. chunks the text via `core.analysis.chunker.chunk_text`, -2. builds the system/user prompts (from `prototype.SYSTEM_PROMPT` + +1. chunks the text via `core.analysis.chunker.chunk_markdown` (heading-aware when + the extractor produced structured Markdown; falls back to plain chunking), +2. appends `extra_context` (prescreen metadata for `AnalyzeRequested`) to each user prompt, +3. builds the system/user prompts (from `prototype.SYSTEM_PROMPT` + `build_user_prompt`, lifted into `core/analysis/analyzer.py`), -3. fans out chunk requests under the existing `asyncio.Semaphore`, -4. merges/dedupes/sorts findings (lift `dedupe_findings` + `sort_findings` +4. fans out chunk requests under the existing `asyncio.Semaphore`, +5. merges/dedupes/sorts findings (lift `dedupe_findings` + `sort_findings` into `core/analysis/analyzer.py`), -5. returns `AnalysisResult`. +6. returns `AnalysisResult`. Map adapter-internal failures to `FailureClass`: @@ -668,11 +810,13 @@ def build_llm_provider(settings) -> LLMProvider: match settings.llm_provider: case "ollama_cloud": return OllamaCloudProvider(settings) + case "yandex_gpt": + return YandexGPTProvider(settings) case _: raise ValueError(f"unknown LLM_PROVIDER={settings.llm_provider!r}") ``` -`LLM_PROVIDER` env (default `ollama_cloud`). Future providers register here. +`LLM_PROVIDER` env (default `ollama_cloud`). Additional providers register here. --- @@ -686,11 +830,11 @@ change. ### RabbitMQ — quorum queues -- `extract.q`, `analyze.q`, `extract.dlq`, `analyze.dlq` are `x-queue-type: - quorum`. Quorum queues persist every message to disk and use Raft. On a - single node they are durable; the moment a 3-node RabbitMQ cluster is - added (compose: `rabbitmq@rmq1/2/3`), the same queues replicate with no - application code change. +- `extract.q`, `prescreen.q`, `analyze.q`, `extract.dlq`, `prescreen.dlq`, + `analyze.dlq` are `x-queue-type: quorum`. Quorum queues persist every message + to disk and use Raft. On a single node they are durable; the moment a 3-node + RabbitMQ cluster is added (compose: `rabbitmq@rmq1/2/3`), the same queues + replicate with no application code change. - Retry queues are `classic` (transient delay slots; safe to lose on catastrophic failure — the originating message is already acked and tracked in `jobs`). @@ -747,6 +891,7 @@ None of 1–5 requires touching `core/` application code — only compose/infra. | `RABBITMQ_URL` | (required) | `amqp://guest:guest@rabbitmq:5672//` | | `MQ_PREFETCH_EXTRACT` | `1` | CPU-bound | | `MQ_PREFETCH_ANALYZE` | `3` | mirrors Ollama Pro concurrency | +| `MQ_PREFETCH_PRESCREEN` | `1` | deterministic heuristic is tiny but keep ordered | | `MQ_MAX_ATTEMPTS` | `5` | before DLQ | | `MQ_RETRY_BASE_MS` | `2000` | exponential base | | `S3_ENDPOINT_URL` | (required) | MinIO URL | @@ -765,7 +910,7 @@ None of 1–5 requires touching `core/` application code — only compose/infra. | Env | Default | Notes | |---|---|---| -| `LLM_PROVIDER` | `ollama_cloud` | factory selection | +| `LLM_PROVIDER` | `ollama_cloud` | factory selection: `ollama_cloud`, `yandex_gpt` | | `OLLAMA_HOST` | (required if provider=ollama_cloud) | `https://ollama.com` for Ollama Cloud; `http://localhost:11434` for self-hosted | | `OLLAMA_API_KEY` | (required) | bearer | | `OLLAMA_MODEL` | `qwen2.5:14b` | primary; must be a model available at the configured host | @@ -775,6 +920,29 @@ None of 1–5 requires touching `core/` application code — only compose/infra. | `OLLAMA_TIMEOUT` | `120` | seconds | | `OLLAMA_MAX_CONCURRENCY` | `3` | in-process semaphore (per analyze worker) | | `CHUNK_SIZE_CHARS` | `10000` | chunker | +| `YANDEXGPT_API_KEY` | (required if provider=yandex_gpt) | API key for Yandex Cloud Foundation Models | +| `YANDEXGPT_FOLDER_ID` | (required if provider=yandex_gpt) | Yandex Cloud folder ID (`gpt:///`) | +| `YANDEXGPT_MODEL` | `yandexgpt-lite` | primary YandexGPT model URI name | +| `YANDEXGPT_FALLBACK_MODEL` | (empty) | optional fallback model name | +| `YANDEXGPT_BASE_URL` | `https://llm.api.cloud.yandex.net` | completion API base URL | +| `YANDEXGPT_COMPLETION_PATH` | `/foundationModels/v1/completion` | completion API path | +| `YANDEXGPT_TEMPERATURE` | `0.2` | | +| `YANDEXGPT_MAX_TOKENS` | `3072` | max completion tokens | +| `YANDEXGPT_TIMEOUT` | `120` | seconds | +| `YANDEXGPT_MAX_CONCURRENCY` | `3` | in-process semaphore (per analyze worker) | + +### Prescreen + +| Env | Default | Notes | +|---|---|---| +| `PRESCREEN_ENABLED` | `true` | Run the prescreen stage between extraction and analysis. | +| `PRESCREEN_AUTO_APPROVE` | `false` | Allow auto-approval of low-risk contracts (disabled until accuracy proven). | +| `PRESCREEN_CONFIDENCE_THRESHOLD` | `0.75` | Minimum weighted field-coverage score to route away from `manual_review`. | +| `PRESCREEN_HIGH_VALUE_THRESHOLD` | `100000` | Amount (in contract currency) above which contract is always sent to deep analysis. | +| `PRESCREEN_LLM_FALLBACK_ENABLED` | `false` | Kill-switch for the Stage-2 LLM fallback in the hybrid extractor. Flip after burn-in. | +| `PRESCREEN_LLM_FALLBACK_THRESHOLD` | `0.75` | Heuristic confidence below this triggers the LLM fallback (keep ≤ confidence threshold). | +| `PRESCREEN_LLM_MAX_CHARS` | `20000` | Cap on characters sent to the LLM fallback (cost control). | +| `PRESCREEN_KEEP_REGEX` | `false` | Temporary: run the legacy regex-v1 extractor via `extract_contract_meta` shim. | ### API @@ -852,15 +1020,16 @@ Per-service differences (`uv` uses PEP 735 dependency-groups — see `pyproject. | Dockerfile | Extra installed | Runtime apt | CMD | Expose | |---|---|---|---|---| | `srv/api/Dockerfile` | `--group api` | none | `python -m contract_check.api` | 8000, 9100 | -| `srv/worker-extract/Dockerfile` | `--group extract` | tesseract-ocr, -rus, -eng | `python -m contract_check.worker_extract` | 9101 | +| `srv/worker-extract/Dockerfile` | `--group extract` | tesseract-ocr, -rus, -eng, libmagic1 | `python -m contract_check.worker_extract` | 9101 | +| `srv/worker-prescreen/Dockerfile` | `--group prescreen` | none | `python -m contract_check.worker_prescreen` | 9104 | | `srv/worker-analyze/Dockerfile` | `--group analyze` | none | `python -m contract_check.worker_analyze` | 9102 | | `srv/bot/Dockerfile` | `--group bot` | none | `python -m contract_check.bot` | — | | `srv/prototype/Dockerfile` | `--group prototype` | none | `python -m contract_check.prototype` | — | The bot image is the leanest (no DB driver, no S3 client, no pymupdf). The analyze image has httpx but no tesseract/pymupdf. The extract image is the -heaviest (tesseract + language packs). This is the "fine-tuned deps per -service" payoff. +heaviest (tesseract + language packs + libmagic1). This is the "fine-tuned deps +per service" payoff. ### pyproject.toml dependency-groups (actual — PEP 735) @@ -894,15 +1063,19 @@ api = [{ include-group = "db" }, { include-group = "mq" }, "opentelemetry-instrumentation-asgi>=0.45b0"] extract = [{ include-group = "db" }, { include-group = "mq" }, { include-group = "s3" }, { include-group = "obs" }, - "pymupdf>=1.24", "pytesseract>=0.3.10", "pillow>=10"] + "pymupdf>=1.24", "pytesseract>=0.3.10", "pillow>=10", + "mammoth>=1.8", "striprtf>=0.0.26", "chardet>=5.2", + "python-magic>=0.4.27"] analyze = [{ include-group = "db" }, { include-group = "mq" }, { include-group = "s3" }, { include-group = "obs" }, "opentelemetry-instrumentation-httpx>=0.45b0"] +prescreen = [{ include-group = "db" }, { include-group = "mq" }, + { include-group = "s3" }, { include-group = "obs" }] bot = ["aiogram>=3.4"] prototype = ["pymupdf>=1.24", "python-docx>=1.1"] dev = [{ include-group = "api" }, { include-group = "extract" }, - { include-group = "analyze" }, { include-group = "bot" }, - { include-group = "prototype" }, + { include-group = "prescreen" }, { include-group = "analyze" }, + { include-group = "bot" }, { include-group = "prototype" }, "pytest>=8", "pytest-asyncio>=0.23", "respx>=0.21", "ruff>=0.5", "mypy>=1.10", "anyio>=4", "aiosqlite>=0.20", "testcontainers[rabbitmq,postgres,minio]>=4", "asgi-lifespan>=2.1.0"] @@ -926,9 +1099,10 @@ services: # ── SERVICES (profile: services) ── api: build srv/api/Dockerfile; depends_on pg/rabbit/minio-init healthy; healthcheck /healthz; ports 8000,9100 - worker-extract: build srv/worker-extract/Dockerfile; depends_on pg/rabbit/minio-init healthy; 9101 - worker-analyze: build srv/worker-analyze/Dockerfile; depends_on pg/rabbit/minio-init healthy; 9102 - bot: build srv/bot/Dockerfile; depends_on api healthy (NOT pg/rabbit) + worker-extract: build srv/worker-extract/Dockerfile; depends_on pg/rabbit/minio-init healthy; 9101 + worker-prescreen: build srv/worker-prescreen/Dockerfile; depends_on pg/rabbit/minio-init healthy; 9104 + worker-analyze: build srv/worker-analyze/Dockerfile; depends_on pg/rabbit/minio-init healthy; 9102 + bot: build srv/bot/Dockerfile; depends_on api healthy (NOT pg/rabbit) # ── OBSERVABILITY (profile: obs) ── PLANNED (T-E1-010) prometheus: prom/prometheus; scrape api:9100, extract:9101, analyze:9102 @@ -973,11 +1147,16 @@ enforcing the adapter boundary even in dependency ordering. - api: `http_requests_total`, `http_request_duration_seconds`, `documents_uploaded_total`, `credits_reserved_total`, `mq_publish_total`. -- worker-extract: `extract_jobs_total`, `extract_duration_seconds`, - `ocr_used_total`, `extract_failures_total{class=...}`, expose on `:9101`. -- worker-analyze: `analyze_jobs_total`, `analyze_duration_seconds`, +- worker-extract: `extract_duration_seconds{format=...}`, + `extraction_total{format,structured}`, expose on `:9101`. +- worker-prescreen: `prescreen_duration_seconds{decision=...}`, + `prescreen_runs_total{decision,contract_type}`, `prescreen_confidence`, + `prescreen_fallback_runs_total{outcome=used|failed|skipped|disabled}`, + `prescreen_extraction_stage_seconds{stage=heuristic|llm}`, + expose on `:9104`. +- worker-analyze: `analyze_duration_seconds`, `llm_tokens_total{kind=prompt|eval}`, `llm_fell_back_total`, - `refund_total{policy=...}`, expose on `:9102`. + expose on `:9102`. - Workers run `prometheus_client.start_http_server(port)` in a background thread alongside the async consumer. @@ -1000,7 +1179,9 @@ enforcing the adapter boundary even in dependency ordering. ### Unit (`tests/unit/`, fast, no I/O) -- `test_chunker.py`, `test_extractor.py` — port existing, parametrized. +- `test_chunker.py` — `chunk_text` + `chunk_markdown`. +- `test_extraction_adapters.py`, `test_extraction_factory.py` — adapters and factory. +- `test_extractor.py` — legacy prototype extractor path remains valid. - `test_llm_ollama_cloud.py` — **respx** mocks: 200 happy, 429→fallback, invalid-JSON→repair→success, invalid-JSON→repair→fail, timeout. - `test_credits_policy.py` — `reserve_credit` never negative; `refund_credit` @@ -1020,9 +1201,9 @@ enforcing the adapter boundary even in dependency ordering. credit reserved, blob in MinIO, `DocumentUploaded` lands on `extract.q` (assert via `aio-pika` consumer). - `test_extract_worker.py` — publish `DocumentUploaded`, run - worker-extract handler in-process → `DocumentExtracted` published, - extracted text in MinIO, `documents.status` advanced. -- `test_analyze_worker.py` — publish `DocumentExtracted` with respx-mocked + worker-extract handler in-process → `PrescreenRequested` published, + extracted **Markdown** in MinIO, `documents.status=advanced to prescreening`. +- `test_analyze_worker.py` — publish `AnalyzeRequested` with respx-mocked Ollama, run worker-analyze handler → report row in Postgres with `status=done`, markdown contains disclaimer. - `test_retry_and_dlq.py` — force Ollama 500 repeatedly → message cycles diff --git a/docs/PHASE1_HANDOFF.md b/docs/PHASE1_HANDOFF.md new file mode 100644 index 0000000..02fe678 --- /dev/null +++ b/docs/PHASE1_HANDOFF.md @@ -0,0 +1,116 @@ +# Phase 1 Handoff — Extraction Layer Refactor + +> Scope: implement the structured-extraction port from `document-extraction-spec.md`, +> preserving the hexagonal architecture and keeping the existing pipeline intact. +> Completed: 2026-08-16. + +## Goal + +Replace flat-text extraction (`core.analysis.extractor.extract_text`) with a +format-aware extraction **port** that returns Markdown with preserved structure +(headings from DOCX, tables from PDF). The rest of the pipeline (chunker, +worker-analyze, RabbitMQ contracts) remains unchanged except for Markdown-aware +chunking and additive metadata in `DocumentExtracted`. + +## New / modified files + +### New: `core/extraction/` hexagonal package + +| File | Purpose | +|---|---| +| `src/contract_check/core/extraction/port.py` | `DocumentExtractor` Protocol, `ExtractedDocument` DTO, `UnsupportedFormatError`, `ExtractionFailedError`, `MIN_TEXT_CHARS` | +| `src/contract_check/core/extraction/factory.py` | `ExtractorFactory`, `detect_format()`, `SUPPORTED_SUFFIXES`; routes bytes/mime/suffix → adapter. Uses `python-magic` for content sniffing, suffix as fallback. | +| `src/contract_check/core/extraction/adapters/pdf_pymupdf.py` | `PyMuPDFExtractor` — text + `page.find_tables()` → Markdown pipes. Interleaves tables and text blocks by bbox so cell text is not duplicated. `is_structured=True` iff tables found. | +| `src/contract_check/core/extraction/adapters/docx_mammoth.py` | `MammothDocxExtractor` — `mammoth.convert_to_markdown()`. Headings/lists/tables become Markdown; `is_structured=True` on headings/tables/lists. | +| `src/contract_check/core/extraction/adapters/rtf_striprtf.py` | `RtfExtractor` — `striprtf` to plaintext. `is_structured=False`. | +| `src/contract_check/core/extraction/adapters/txt_chardet.py` | `TxtExtractor` — `chardet` encoding detection, decode, metadata includes `encoding`. | +| `src/contract_check/core/extraction/adapters/ocr_tesseract.py` | `TesseractOcrExtractor` — OCR for scanned PDFs and raster images (PNG/JPG/TIFF), reusing the same pytesseract/pymupdf logic as `core.analysis.ocr`. | +| `src/contract_check/core/extraction/__init__.py` | `extract_document()` orchestrator: adapter → extract → OCR fallback for short PDFs. | +| `src/contract_check/core/extraction/adapters/__init__.py` | Re-exports of all adapter classes. | + +### Modified worker / pipeline + +| File | Change | +|---|---| +| `src/contract_check/worker_extract/handler.py` | Bytes-based extraction via `core.extraction.extract_document()`. No temp files. Uploaded text now `text/markdown; charset=utf-8`. `DocumentExtracted` includes `is_structured`/`has_tables`. Duration histogram labeled by `format`. Classification updated for new errors. | +| `src/contract_check/worker_extract/consumer.py` | Removed `@extract_duration.time()` decorator — labeled timing moved into the handler where the format is known. | +| `src/contract_check/core/mq/messages.py` | `DocumentExtracted` added `is_structured: bool = False` and `has_tables: bool = False` (backward-compatible defaults). | +| `src/contract_check/core/metrics.py` | `extract_duration` now has `format` label. Added `extraction_total{format,structured}` counter. | +| `src/contract_check/core/analysis/chunker.py` | Added `chunk_markdown()` — splits on ATX headings, keeps headings attached to their body, falls back to `chunk_text()` for unstructured text. | +| `src/contract_check/core/llm/ollama_cloud.py` | Analyzer now uses `chunk_markdown(text)` instead of `chunk_text(text)` so heading boundaries are preserved when the extractor produced structured Markdown. | +| `src/contract_check/api/services.py` | Upload gate now imports `SUPPORTED_SUFFIXES` from `core.extraction`. `_content_type_from_suffix()` extended for RTF/TXT/CSV/images. | +| `src/contract_check/bot/handlers.py` | Bot extension gate now uses the same `SUPPORTED_SUFFIXES` set and content-type map. | +| `src/contract_check/worker_extract/extract_document.py` | **Deleted**. Superseded by `core.extraction.extract_document()`. | + +### Dependencies / Docker + +| File | Change | +|---|---| +| `pyproject.toml` | `extract` group adds `mammoth`, `striprtf`, `chardet`, `python-magic`. Mypy ignore list extended for these libs. | +| `srv/worker-extract/Dockerfile` | Adds `libmagic1` apt package (required by `python-magic`). | +| `uv.lock` | Regenerated. | + +### Documentation + +| File | Change | +|---|---| +| `docs/ARCHITECTURE.md` | Layout updated (`core/extraction/` added, `worker_extract/extract_document.py` removed, new tests listed). Pipeline notes that extracted Markdown is uploaded. `DocumentExtracted` schema updated. Metrics table updated. pyproject `extract` group updated. worker-extract Dockerfile apt row updated. | +| `src/contract_check/api/routes/README.md` | Upload endpoint extension list updated. | + +### Tests + +| File | Change | +|---|---| +| `tests/unit/test_extraction_adapters.py` | New: PDF text-only/table, DOCX heading, RTF, TXT chardet, OCR error paths, DTO defaults. | +| `tests/unit/test_extraction_factory.py` | New: suffix/mime/magic routing, unsupported formats, end-to-end `extract_document()`, OCR fallback behavior. | +| `tests/unit/test_chunker.py` | New: `chunk_markdown` heading preservation, packing, oversized-section split, preamble handling, fallback. | +| `tests/integration/test_extract_worker.py` | Expected exceptions updated: `ExtractionFailedError` is now a possible raised error alongside `OCRError` (the failure-class mapping remains `extraction_failed` / `ocr_failed`). | + +## Behavior changes + +- **Format support** (upload-gate accepted): `.pdf`, `.docx`, `.rtf`, `.txt`, `.csv`, `.png`, `.jpg`, `.jpeg`, `.tif`, `.tiff`. +- **Output format**: extracted text is now Markdown; stored with content-type `text/markdown; charset=utf-8`. +- **Backward compatibility**: `DocumentExtracted` new fields have defaults, so in-flight messages during deploy are valid. +- **Refund policy unchanged**: `ExtractionFailedError` → `extraction_failed` (not refundable under `infra_only`); `OCRError` → `ocr_failed` (refundable). +- **Chunking**: `worker-analyze` uses heading-aware `chunk_markdown()` when structure exists; otherwise identical to previous `chunk_text()`. + +## Verification + +```bash +make lint # ruff check + format check — passed +make typecheck # mypy src — passed (92 files) +make test-unit # 132 passed, 32 deselected integration tests +``` + +Integration tests (`tests/integration/test_extract_worker.py` and +`tests/integration/test_upload_pipeline.py`) passed with the live Docker stack. +The full `make test-integration` suite is slow because it exercises the real LLM +path; the extraction-specific integration subset is green. + +## Deployment notes + +1. The `worker-extract` Docker image must be rebuilt because: + - new apt package `libmagic1` + - new Python deps `mammoth`, `striprtf`, `chardet`, `python-magic` +2. Existing running workers should be recreated (`docker compose --profile services up -d --build worker-extract`). +3. No DB migration required — changes are additive to message schema and object storage content. + +## Rollback + +- The new code is additive; old messages with fewer `DocumentExtracted` fields + still validate. If rollback is needed, the previous worker-extract image + continues to work, but extracted objects would be stored as `text/plain` + instead of `text/markdown`. + +## Deviations from the spec + +- `ExtractedDocument.attachments: list[bytes]` was omitted — there is no consumer + or storage design for extracted images yet. +- Heavy adapters (`marker`, `paddleocr`, `easyocr`, `img2table`, `email`) remain + deferred per the spec. + +## Next step + +Phase 2: prescreen stage between `worker-extract` and `worker-analyze`. Based +on `SPIKE_PHASE0.md`, this will use a deterministic regex+pydantic extractor for +RU/BY contracts instead of the Needle 2 model (English-only in testing). diff --git a/docs/PHASE2_HANDOFF.md b/docs/PHASE2_HANDOFF.md new file mode 100644 index 0000000..ca3fa5b --- /dev/null +++ b/docs/PHASE2_HANDOFF.md @@ -0,0 +1,153 @@ +# Phase 2 Handoff — Prescreen Stage + +> Scope: insert a deterministic prescreen stage between extraction and LLM +> analysis for Russian/Belarusian contracts. +> Completed: 2026-08-16. + +## Goal + +Short-circuit obvious low-risk contracts and protect the expensive LLM worker +from trivial documents. Use a deterministic regex+pydantic extractor because the +Needle 2 (cactus-needle) model is English-only and unusable for RU/BY contracts +(see `docs/SPIKE_PHASE0.md`). + +## What changed + +### DB schema (`migrations/versions/0006_prescreen.py`) + +- New `prescreen_results` table with contract metadata, field-coverage score, + routing decision, and lightweight auto-approve output. +- `documents.status` adds `'prescreening'`. +- `jobs.queue` adds `'prescreen'`. +- `reports` gains `prescreen_result_id` FK and `prescreen_meta` JSONB. + +### Models (`src/contract_check/core/db/`) + +- `models.py`: added `PrescreenResult` model and relationships. +- `enums.py`: `DocStatus` and `QueueName` literals updated. + +### RabbitMQ topology (`core/mq/topology.py`) + +- Added `prescreen.q`, `prescreen.retry.q`, `prescreen.dlq`. +- Added routing keys `prescreen` / `retry.prescreen`. + +### Messages (`core/mq/messages.py`) + +- `PrescreenRequested` — worker-extract → prescreen.q. +- `PrescreenCompleted` — prescreen result, persisted and forwarded. +- `AnalyzeRequested` — worker-prescreen → analyze.q, carries `prescreen_meta`. +- `DocumentExtracted` kept for backward compatibility but no longer published. + +### New service: `worker_prescreen/` + +| File | Purpose | +|---|---| +| `extractor.py` | Regex extractor for RU/BY contracts; returns `PrescreenContractMeta` + confidence score. | +| `router.py` | Routing decision: `auto_approve` / `manual_review` / `deep_analysis`. `auto_approve` is disabled by default. | +| `handler.py` | Download Markdown, run extractor, persist `prescreen_results`, publish next message. | +| `consumer.py` | RabbitMQ consumer for `prescreen.q`. | +| `config.py` | `PrescreenSettings` with prefetch tunable. | +| `__main__.py` | Entrypoint, metrics server on `:9104`. | + +### Wired existing services + +- `worker_extract/handler.py`: now publishes `PrescreenRequested` to `prescreen.q` + and sets `documents.status = 'prescreening'`. +- `worker_analyze/consumer.py` + `handler.py`: now consumes `AnalyzeRequested`, + receives `prescreen_meta`, appends it to the LLM prompt as known contract + fields. +- `core/llm/port.py` + `ollama_cloud.py`: `analyze()` accepts `extra_context: str` + and injects it into each chunk's user prompt. + +### Configuration (`core/config.py`) + +New envs: + +- `PRESCREEN_ENABLED` (default `true`) +- `PRESCREEN_AUTO_APPROVE` (default `false`) +- `PRESCREEN_CONFIDENCE_THRESHOLD` (default `0.75`) +- `PRESCREEN_HIGH_VALUE_THRESHOLD` (default `100000`) + +### Metrics (`core/metrics.py`) + +- `prescreen_duration_seconds{decision}` +- `prescreen_runs_total{decision,contract_type}` +- `prescreen_confidence` histogram + +### Docker / compose + +- New `srv/worker-prescreen/Dockerfile` (lean, no tesseract). +- Added `worker-prescreen` service to `docker-compose.yml` on port `9104`. + +### Dependencies (`pyproject.toml`) + +- Added `prescreen` dependency group (core + db/mq/s3/obs). +- Added to `dev` group. + +### Tests + +- Unit: `tests/unit/test_prescreen_extractor.py`, `tests/unit/test_prescreen_router.py`. +- Integration: updated `test_extract_worker.py`, `test_analyze_worker.py`, + `test_upload_pipeline.py`, `test_b2b_api.py` for the new pipeline. + +### Docs + +- Updated `docs/ARCHITECTURE.md` with prescreen topology, messages, schema, + metrics, and Docker/compose rows. + +## Behavior + +1. Upload → `extract.q` (unchanged). +2. worker-extract uploads Markdown to MinIO, publishes `PrescreenRequested`. +3. worker-prescreen: + - extracts contract type, parties, amount, currency, dates, penalty/ + termination/arbitration clauses via regex + - `confidence_score = matched_fields / total_fields` + - routes: + - `deep_analysis` if high value, penalty clause, or arbitration + - `manual_review` if low confidence, missing parties, or auto_approve disabled + - `auto_approve` only when `PRESCREEN_AUTO_APPROVE=true` and low risk +4. `deep_analysis` → `AnalyzeRequested` → worker-analyze with prescreen context. +5. `manual_review` → terminal DB state, no LLM call, credit **not** refunded + (this is an intentional routing outcome, not a failure). +6. `auto_approve` → lightweight report row, `status=done`. + +## Verification + +```bash +make lint # passed +make typecheck # passed (99 files) +make test-unit # 147 passed, 32 deselected +make test-integration # 32 passed, 147 deselected (with live workers stopped to avoid races) +``` + +## Deployment notes + +1. Run migration: `make migrate` (or `docker compose --profile services run --rm api alembic upgrade head`). +2. Rebuild images: `docker compose --profile services up -d --build`. +3. New worker-prescreen must be started; worker-extract and worker-analyze + images also changed. +4. No application code outside core/workers was changed; the API just returns + the new `prescreening` status string. + +## Deviations from the original prescreen spec + +- **Extractor:** regex instead of Needle 2. Rationale in `docs/SPIKE_PHASE0.md`. +- **No separate credits split:** the credit is still reserved once on upload; + prescreen is treated as part of the same paid job. +- **No admin/web SPA yet:** `manual_review` documents are terminal in the DB + until the admin panel lands. + +## Next step + +Phase 3 is storage modernization evaluation (RustFS watch). The current stack +keeps MinIO as the production default; RustFS is tracked as a future option. +Alternatively, continue with the backlog: admin/web SPA, ЮKassa payments, heavy +OCR adapters. + +--- + +**Recommended immediate follow-up:** end-to-end smoke test in Docker with all +services running (`make services-up`) and a real PDF/DOCX upload through the +API or bot, verifying that `worker-prescreen` routes to `analyze.q` or +`manual_review` correctly. diff --git a/docs/PHASES_2_PLUS_ROADMAP.md b/docs/PHASES_2_PLUS_ROADMAP.md new file mode 100644 index 0000000..63ed1bc --- /dev/null +++ b/docs/PHASES_2_PLUS_ROADMAP.md @@ -0,0 +1,439 @@ +# Implementation Roadmap — Phase 2 (Prescreen) & Remaining Work + +> Based on `document-extraction-spec.md`, `needle-prescreen-integration.md`, +> and the Phase 0 spike report (`docs/SPIKE_PHASE0.md`). +> Current checkpoint: Phase 1 extraction refactor is complete (`docs/PHASE1_HANDOFF.md`). + +## Status overview + +| Phase | Scope | Status | +|---|---|---| +| Phase 0 | Needle/RustFS spikes | Done (`docs/SPIKE_PHASE0.md`) | +| Phase 1 | Extraction layer refactor | Done (`docs/PHASE1_HANDOFF.md`) | +| Phase 2 | Prescreen stage | **Planned below** | +| Phase 3 | Storage modernization / RustFS watch | **Planned below** | +| Follow-up | Admin/web, payments, heavy OCR | **Backlog** | + +## Phase 2 — Prescreen stage between extract and analyze + +### Why + +Insert a fast, local decision layer after extraction and before the expensive LLM +analysis so that: + +- obvious low-risk contracts can short-circuit to a lightweight report +- high-risk / high-value contracts are routed to deep LLM analysis +- the heavy LLM worker is no longer the only path for every document +- per-stage queue metrics, retry, and DLQ are clean and independent + +### Key correction from the original prescreen spec + +The original spec assumed **Needle 2** (`cactus-needle`) as the prescreen +extractor. The Phase 0 spike proved the base model is English-only and fails on +Russian contracts with `confidence=0.0`. Fine-tuning would also disable the +calibrated confidence head the routing design depends on. + +Therefore Phase 2 uses a **deterministic regex+pydantic extractor** for RU/BY +contracts. Contract boilerplate is highly templated, so regex gives: + +- zero marginal cost +- no hallucination risk +- missing fields naturally map to `manual_review` +- `confidence_score` becomes a deterministic **field-coverage ratio** + +A small-LLM fallback can be added later for fields regex misses. + +### 2.1 RabbitMQ topology changes + +Add to `core/mq/topology.py` using the existing direct-exchange + TTL retry/DLQ +pattern: + +```text +contracts.x + ├─ extract ─► extract.q (existing) + ├─ prescreen ─► prescreen.q (NEW) + └─ analyze ─► analyze.q (existing) + +contracts.retry.x + ├─ retry.prescreen ─► prescreen.retry.q (NEW classic delay queue, DLX→contracts.x[prescreen]) + +prescreen.dlq (NEW quorum) +``` + +Constants: `QUEUE_PRESCREEN`, `RK_PRESCREEN`, update `_RETRY_QUEUE_FOR`, +`DLQ_FOR`, `declare_all()`. + +### 2.2 Database migration `0006_prescreen.py` + +New table `prescreen_results`: + +```sql +CREATE TABLE prescreen_results ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + correlation_id UUID NOT NULL, + + -- extracted metadata + contract_type VARCHAR(64), + party_a TEXT, + party_b TEXT, + total_amount DECIMAL(18, 2), + currency VARCHAR(8), + start_date DATE, + end_date DATE, + has_penalty_clause BOOLEAN, + has_termination_clause BOOLEAN, + has_arbitration BOOLEAN, + confidence_score DECIMAL(4, 3), -- coverage ratio 0.000–1.000 + + -- routing + routing_decision VARCHAR(32) NOT NULL, -- auto_approve | manual_review | deep_analysis + + -- metrics / audit + prescreened_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + processing_ms INTEGER, + extractor_version VARCHAR(32) DEFAULT 'regex-v1', + + -- auto-approve lightweight output + auto_summary TEXT, + auto_findings JSONB DEFAULT '[]', + + -- retry tracking + error_message TEXT, + retry_count INTEGER DEFAULT 0 +); + +CREATE INDEX idx_prescreen_document ON prescreen_results(document_id); +CREATE INDEX idx_prescreen_routing ON prescreen_results(routing_decision); +CREATE INDEX idx_prescreen_confidence ON prescreen_results(confidence_score); +``` + +Also: + +```sql +ALTER TYPE document_status ADD VALUE 'prescreening' AFTER 'extracted'; +ALTER TYPE jobs.queue ADD VALUE 'prescreen' AFTER 'analyze'; + +ALTER TABLE reports + ADD COLUMN prescreen_result_id UUID REFERENCES prescreen_results(id), + ADD COLUMN prescreen_meta JSONB DEFAULT NULL; +``` + +Use `TEXT+CHECK` instead of Postgres enums if the project convention prefers +additive migrations (the existing schema uses both). Decision to make during +implementation. + +### 2.3 Message schema additions + +Add to `core/mq/messages.py`: + +```python +class PrescreenRequested(PipelineMessage): + """worker-extract → contracts.x[prescreen] → worker-prescreen.""" + + text_s3_key: str + filename: str + + +class PrescreenCompleted(PipelineMessage): + """worker-prescreen result; also the payload forwarded to analyze.q.""" + + text_s3_key: str + prescreened_at: datetime + contract_type: str | None + party_a: str | None + party_b: str | None + total_amount: float | None + currency: str | None + start_date: str | None + end_date: str | None + has_penalty_clause: bool | None + has_termination_clause: bool | None + has_arbitration: bool | None + confidence_score: float | None # 0..1 coverage ratio + routing_decision: Literal["auto_approve", "manual_review", "deep_analysis"] + auto_summary: str | None + auto_findings: list[dict] = Field(default_factory=list) + error_message: str | None + + +class AnalyzeRequested(PipelineMessage): + """worker-prescreen → contracts.x[analyze] → worker-analyze.""" + + text_s3_key: str + filename: str + prescreen_meta: PrescreenCompleted +``` + +Keep `DocumentExtracted` for backward compatibility; `worker-extract` may still +publish it for observability, but the prescreen stage is the main forward path. + +### 2.4 New service: `worker-prescreen` + +Directory: + +```text +src/contract_check/worker_prescreen/ + __init__.py + __main__.py + consumer.py + handler.py + router.py + extractor.py # regex+pydantic RU/BY contract extractor + config.py +``` + +#### `extractor.py` + +Deterministic extractor built with `re` + Pydantic model. Target fields: + +```python +class PrescreenContractMeta(BaseModel): + contract_type: str | None + party_a: str | None + party_b: str | None + total_amount: float | None + currency: str | None + start_date: str | None + end_date: str | None + has_penalty_clause: bool | None + has_termination_clause: bool | None + has_arbitration: bool | None +``` + +Heuristics: + +- `contract_type`: map first matched keyword (`договор поставки`, `договор оказания услуг`, `договор подряда`, `договор аренды`, `договор купли-продажи`, etc.) +- `party_a` / `party_b`: extract first two legal entities after patterns like + `Общество с ограниченной ответственностью «(.*?)»`, `Акционерное общество «(.*?)»`, `Индивидуальный предприниматель (.*?)`. +- `total_amount`: find `составляет ([\d\s,.]+) (рублей|руб|USD|EUR|€|\$)` and parse as float. +- `currency`: normalize to `RUB`, `USD`, `EUR`, `BYN`. +- `start_date` / `end_date`: match `с (\d{2}\.\d{2}\.\d{4})`, `от (\d{2}\.\d{2}\.\d{4})`, + `действует с (\d{2}\.\d{2}\.\d{4}) по (\d{2}\.\d{2}\.\d{4})`. +- `has_penalty_clause`: presence of `неустойка`, `штраф`, `пеня`, `0,1%`/`% за каждый день просрочки`. +- `has_termination_clause`: presence of `расторгнуть`, `расторжение`, `одностороннему порядке`. +- `has_arbitration`: presence of `арбитражный суд`, `Арбитражный суд`, `Международный коммерческий арбитражный суд`. + +`confidence_score = matched_fields / total_fields`. + +Test on synthetic and a few real contracts; tune false-positive/negative rate. + +#### `router.py` + +```python +class PrescreenRouter: + def __init__( + self, + confidence_threshold: float = 0.75, + high_value_threshold: float = 100_000, + ) -> None: + ... + + def decide(self, meta: PrescreenContractMeta) -> str: + if confidence < threshold or missing mandatory fields: + return "manual_review" + if total_amount > high_value_threshold: + return "deep_analysis" + if has_penalty_clause or has_arbitration: + return "deep_analysis" + return "auto_approve" +``` + +`auto_approve` is **disabled by default** via env `PRESCREEN_AUTO_APPROVE=false` +until accuracy is proven; when disabled, `auto_approve` decisions are mapped +to `manual_review`. + +#### `handler.py` + +- idempotency check on `documents.status` +- set `status = 'prescreening'`, `jobs.status = 'running'` +- download extracted Markdown from MinIO +- run regex extractor in thread pool (`asyncio.to_thread`) +- route +- persist `prescreen_results` +- publish next message: + - `deep_analysis` → `contracts.x[analyze]` with `AnalyzeRequested` + - `auto_approve` → `contracts.x[report.completed]` with lightweight summary + (only when `PRESCREEN_AUTO_APPROVE=true`) + - `manual_review` → `contracts.x[review.manual]`; no further processing +- ack + +#### `consumer.py` + +`Consumer[PrescreenRequested]` for `prescreen.q`, wired like `AnalyzeConsumer`. + +#### `config.py` + +- `PRESCREEN_ENABLED` — if false, handler immediately publishes + `AnalyzeRequested` (passthrough, preserving analytics) +- `PRESCREEN_AUTO_APPROVE` +- `PRESCREEN_CONFIDENCE_THRESHOLD` +- `PRESCREEN_HIGH_VALUE_THRESHOLD` + +### 2.5 Dockerfile + compose + +- `srv/worker-prescreen/Dockerfile`: lean image, no tesseract/pymupdf, only core + deps + regex engine. +- Add `worker-prescreen` to `docker-compose.yml` profile `services`. +- Memory limit 128M (regex is tiny). + +### 2.6 Changes to existing services + +#### `worker-extract` + +After publishing `DocumentExtracted` (backward compat), also publish +`PrescreenRequested`: + +```python +await publisher.publish( + PrescreenRequested( + correlation_id=..., + document_id=..., + user_id=..., + text_s3_key=extracted_key, + filename=payload.filename, + ), + routing_key=RK_PRESCREEN, +) +``` + +Gate behind `PRESCREEN_ENABLED` (default true after Phase 2). + +#### `worker-analyze` + +Change consumer to `AnalyzeRequested`: + +```python +class AnalyzeConsumer(Consumer[AnalyzeRequested]): + queue = "analyze.q" + message_model = AnalyzeRequested +``` + +Enrich prompt with `prescreen_meta` (§6.3 of the original spec). + +#### `api` + +- Update document status endpoint to include prescreen block. +- Status lifecycle: `queued → extracting → prescreening → analyzing | manual_review | done`. +- Optional: expose `?mode=fast` vs `?mode=full` for B2B. + +### 2.7 Metrics + +Add to `core/metrics.py` and expose from `worker-prescreen`: + +- `prescreen_requests_total{source}` +- `prescreen_duration_seconds{decision}` +- `prescreen_routing_decisions_total{decision,contract_type}` +- `prescreen_confidence_distribution` +- `prescreen_errors_total{error_type}` +- `prescreen_dlq_messages_total` + +### 2.8 Tests + +- Unit: `tests/unit/test_prescreen_extractor.py` — regex patterns on synthetic + contracts, coverage scoring. +- Unit: `tests/unit/test_prescreen_router.py` — routing matrix with all flag + combinations. +- Integration: `tests/integration/test_prescreen_worker.py`: + - `PrescreenRequested` → `AnalyzeRequested` for `deep_analysis` + - `PrescreenRequested` → `review.manual` for `manual_review` + - bypass mode (`PRESCREEN_ENABLED=false`) → passthrough `AnalyzeRequested` +- Update `test_extract_worker.py` to assert `PrescreenRequested` is also + published. +- Update `test_analyze_worker.py` to consume `AnalyzeRequested` and include + prescreen metadata. + +### 2.9 DoD + +```bash +make lint && make typecheck && make test-unit +make test-integration +``` + +## Phase 3 — Storage watch / RustFS evaluation + +### Goal + +Keep MinIO as the production default but maintain a credible migration path to +RustFS when its Lifecycle + KMS features reach GA. No application code changes +are required because S3 is already behind `core.s3.port.py`. + +### Why not switch now + +From the Phase 0 RustFS spike: + +- Lifecycle/KMS/distributed mode are marked **"Under Testing"** upstream. +- SSE-S3 objects written by MinIO are **not readable by RustFS** today — a + migration would require re-putting objects through the app. +- RustFS is weeks old; MinIO is battle-tested for this exact compose setup. +- Real advantages exist: Apache 2.0 license (vs MinIO AGPL), no telemetry, + RF data-sovereignty friendly. + +### Actions + +1. Add an optional `docker-compose.rustfs.yml` overlay for local experiments. +2. Run the full integration suite against RustFS quarterly or when upstream + announces Lifecycle/KMS GA. +3. If/when adopted: + - pin exact RustFS image tag + - set `S3_SERVER_SIDE_ENCRYPTION=false` during transition + - re-encrypt objects via the app after cutover + - update `minio-init` service or replace with `rustfs-init` + +## Backlog / follow-up work (not tied to Phase 2) + +| Item | Rationale | +|---|---| +| Admin/web SPA | Deferred by original architecture scope. | +| ЮKassa payments | Stub `invoices` table exists; payment logic deferred. | +| Heavy OCR adapters (marker, paddleocr, easyocr, img2table) | Spec deferral. Consider separate `worker-extract-heavy` queue or cloud API. | +| Email extractor (MSG/EML) | Spec deferral; only relevant for B2B corporate email ingestion. | +| Multi-tenancy | Explicit non-goal of original refactor. Do not add `tenant_id` until orgs are a real requirement. | +| Fine-tuned RU Needle model | Revisit if `cactus-needle` ships a multilingual or RU-tuned base with calibrated confidence. | + +## Appendix: original spec alignment + +### `document-extraction-spec.md` + +| Requirement | Phase 1 status | +|---|---| +| `DocumentExtractor` Protocol / ABC | Done (`core/extraction/port.py`) | +| `ExtractedDocument` dataclass / Pydantic | Done | +| `ExtractorFactory` with MIME detection | Done | +| `DocxExtractor` (`mammoth`) | Done | +| `RtfExtractor` (`striprtf`) | Done | +| `EncodingDetector` (`chardet`) | Done | +| `PyMuPDFExtractor` + `has_tables` | Done | +| `TesseractOCRAdapter` + `is_structured=False` | Done | +| Metrics by format | Done | +| Update architecture docs | Done | +| Heavy adapters deferred | Backlog | + +### `needle-prescreen-integration.md` + +| Requirement | Phase 2 plan above | +|---|---| +| Separate `prescreen.q` stage | Yes | +| `PrescreenRequested` / `PrescreenCompleted` messages | Yes, corrected to extend `PipelineMessage` | +| `prescreen_results` table | Yes, adjusted columns | +| `worker-prescreen` service | Yes, regex-based instead of Needle | +| Routing decisions | Yes, conservative default | +| `worker-extract` publishes prescreen | Yes | +| `worker-analyze` consumes `AnalyzeRequested` | Yes | +| API status endpoint enriched | Yes | +| Dockerfile/compose for prescreen worker | Yes | +| Rollback / bypass flag | Yes | +| Multi-tenant / credit split | **Rejected** — not in current scope; integer credit model stays | + +## Ordering recommendation + +1. Phase 2 DB migration + RabbitMQ topology. +2. Phase 2 messages + `worker_prescreen` skeleton (no-op passthrough first). +3. Phase 2 regex extractor + router + tests. +4. Wire `worker-extract` → `prescreen.q` and `worker-analyze` → `analyze.q`. +5. End-to-end integration test. +6. Phase 3 RustFS spike only after upstream GA announcement. + +--- + +**Next action:** implement Phase 2 step 1 (migration + topology) if approved. diff --git a/docs/PRESCREEN_HYBRID_REFACTOR_PLAN.md b/docs/PRESCREEN_HYBRID_REFACTOR_PLAN.md new file mode 100644 index 0000000..7f8c5f1 --- /dev/null +++ b/docs/PRESCREEN_HYBRID_REFACTOR_PLAN.md @@ -0,0 +1,242 @@ +# Refactor Plan: Prescreen Worker → Hybrid Extraction (Heuristic + LLM Fallback) + +> Status: delivered through Phase 4; Phase 5 rollout pending merge +> Scope: `src/contract_check/worker_prescreen/`, `src/contract_check/core/llm/`, `src/contract_check/core/config.py`, `src/contract_check/core/metrics.py` +> Supersedes: the regex-only extractor described in `worker_prescreen/extractor.py` (regex-v1) + +--- + +## 1. Goal + +Replace the monolithic regex extractor (`extractor.py`, ~260 lines of Russian/BY legal +regexes) with a **hybrid two-stage extractor**: + +1. **Stage 1 — Heuristic (deterministic, zero-cost):** keyword dictionaries, + positional windows, sentence scanning, per-field confidence weights. + No regex — plain string operations only. +2. **Stage 2 — LLM fallback (only when Stage 1 confidence is low):** reuse the + existing `core/llm` provider infrastructure (Ollama Cloud / YandexGPT) with a + JSON-schema-constrained extraction prompt. + +Non-goals: + +- No changes to routing semantics (`router.py` thresholds/decisions stay as-is). +- No DB schema migration (`prescreen_results.extractor_version` already exists, + `String(32)` fits `"heuristic-v2"` / `"llm-v1"`). +- No changes to MQ topology, messages (`PrescreenCompleted` shape unchanged), + or downstream `worker_analyze`. +- Auto-approve stays disabled by default. + +## 2. Current State + +``` +worker_prescreen/ +├── consumer.py # consumes prescreen.q → handler +├── handler.py # Stage 4: asyncio.to_thread(extract_contract_meta, text) +├── extractor.py # PrescreenContractMeta + 10 regex patterns ← TO REPLACE +├── router.py # decides auto_approve|manual_review|deep_analysis +└── config.py +``` + +Problems: + +- Regexes are brittle (line-noise, OCR artifacts, whitespace variants) and hard + to extend (every new contract type = new regex). +- No recovery path: low-confidence extraction always ends in `manual_review`. +- `prescreen_results.extractor_version` is never written by the handler INSERT — + it silently relies on the DB default `'regex-v1'`. + +## 3. Target Architecture + +``` + ┌────────────────────────────────────────────┐ + │ handler.py │ + │ Stage 4: meta = await extractor.extract() │ + └───────────────┬────────────────────────────┘ + │ + HybridMetaExtractor (orchestrator) + │ + ┌────────────────────┴──────────────────────┐ + ▼ ▼ + HeuristicExtractor (sync, to_thread) LLMPrescreenExtractor + keyword + positional + sentence wraps core/llm provider + scanning, confidence-weighted extract_prescreen(text) + │ │ + confidence ≥ threshold? ──── no ────► run LLM, validate JSON, + │ yes merge over heuristic meta + ▼ + PrescreenContractMeta → router.decide() → persist (with extractor_version) +``` + +### 3.1 New module layout (flat, matches existing package conventions) + +``` +worker_prescreen/ +├── extractor.py # PrescreenContractMeta (model, unchanged shape) +│ # + MetadataExtractor protocol + re-export shim +├── extractor_heuristic.py# HeuristicExtractor (Stage 1) +├── extractor_llm.py # LLMPrescreenExtractor (Stage 2, dict → pydantic validation) +├── extractor_hybrid.py # HybridMetaExtractor (threshold + merge + kill-switch) +├── handler.py # Stage 4 rewired; INSERT gains extractor_version +└── router.py # unchanged +``` + +### 3.2 Heuristic stage design (regex-free) + +| Field | Method | Notes | +|---|---|---| +| `contract_type` | Phrase dictionary match on normalized text | `{"supply": ["договор поставки", "договор купли-продажи", ...], ...}` — lowercase + whitespace-collapse once, then `in` checks | +| `party_a/b` | Positional: scan only the header window (text up to first `1.` / `ПРЕДМЕТ` heading, capped ~1500 chars); token-scan for entity-form tokens (ООО, ИП, АО, …) then capture until quote `»`/`"` close or line end | Replaces `_ENTITY_RE` | +| `total_amount` + `currency` | Trigger-word scan (`составляет`, `стоимость`, `цена`, …) then manual digit-window parse (`_scan_number` helper walking chars, handling spaces/commas); currency via token lookup in the following ~50 chars | Replaces `_AMOUNT_RE` | +| `start/end_date` | Trigger-word scan (`действует с`, `с … по …`) then `_parse_ddmmyyyy` manual splitter (already regex-free in v1 — keep) | Replaces `_DATE_RE`/`_END_DATE_RE` | +| boolean clauses | Sentence segmentation via `str.split` on `. ` / `\n`, keyword membership per sentence (`неустойка`, `штраф`, `расторгнуть`, `арбитражн`, …) | Replaces `_PENALTY_RE` etc. | + +Confidence scoring becomes **weighted** instead of flat field coverage: + +```python +FIELD_WEIGHTS = { + "contract_type": 0.25, + "party_a": 0.15, + "party_b": 0.15, + "total_amount": 0.15, + "currency": 0.05, + "start_date": 0.10, + "end_date": 0.05, + "has_penalty_clause": 0.05, + "has_termination_clause": 0.05, + "has_arbitration": 0.05, +} # sums to 1.0 +``` + +Optional per-field method bonus (exact phrase match = full weight, positional +window hit = full weight, fuzzy tail hit = ×0.7) — start simple, weights +constant, tune later with real data. + +### 3.3 LLM fallback design + +- **Protocol extension** (`core/llm/port.py`): + `async def extract_prescreen(self, text: str) -> dict[str, Any]` — returns raw + JSON dict; **no import of worker_prescreen** (keeps layering clean). + Implemented by both `OllamaCloudProvider` and `YandexGPTProvider` as a thin + wrapper over their existing JSON-chat + repair-loop machinery + (`_chat_json` / `responseFormat=json_schema`) with a dedicated + `PRESCREEN_SYSTEM` prompt: extract the 10 meta fields, cite nothing, JSON only. +- **Worker-side wrapper** (`extractor_llm.py`): + `dict` → `PrescreenContractMeta` via pydantic (rejects hallucinated fields, + re-`None`s unknown enum values), then recomputes weighted confidence. +- **Input cap:** only the first `prescreen_llm_max_chars` (default 20 000) chars + are sent — meta lives in the header for templated contracts; cost control. +- **Merge rule:** LLM values override heuristic `None`s and low-confidence + fields; boolean clauses become OR(heuristic, llm) — both sources are + presence-checks, false positives are cheap, false negatives route wrong. + +### 3.4 Failure semantics + +- LLM fallback error (quota, timeout, invalid JSON after repair): + **do not fail the message.** Log + keep heuristic meta, + `extractor_version="heuristic-v2"`, record the error in + `prescreen_results.auto_findings` (`{"llm_fallback_error": "..."}`) and the + `prescreen_fallback_runs_total{outcome="failed"}` counter. +- Heuristic stage is pure string ops — its only failure mode is `None` fields, + which is already handled by low confidence → fallback / manual_review. + +### 3.5 Config additions (`core/config.py`, `--- prescreen stage ---`) + +```python +prescreen_llm_fallback_enabled: bool = Field(default=False, ...) # kill-switch; flip to True after burn-in +prescreen_llm_fallback_threshold: float = Field(default=0.75, ...) # ≤ router threshold +prescreen_llm_max_chars: int = Field(default=20_000, ...) +``` + +### 3.6 Metrics additions (`core/metrics.py`) + +```python +prescreen_fallback_runs = Counter( + "contract_check_prescreen_fallback_runs_total", + "...", labelnames=["outcome"], # used | failed | skipped | disabled +) +prescreen_extraction_stage = Histogram( + "contract_check_prescreen_extraction_stage_seconds", + "...", labelnames=["stage"], # heuristic | llm +) +``` + +`extractor_version` values: `"regex-v1"` (legacy shim flag), `"heuristic-v2"` +(LLM not run / disabled / failed) and `"hybrid-llm-v1"` (LLM result merged). + +## 4. Implementation Phases + +Each phase is independently shippable and reverted by config/env flag. + +### Phase 1 — Port + heuristic extractor (no behavior change for routing) +- [x] `extractor.py`: keep `PrescreenContractMeta` + weighted `_score_confidence`; + add `MetadataExtractor` protocol (`def extract(text: str) -> PrescreenContractMeta`); + keep `extract_contract_meta` as a delegating shim so + `tests/unit/test_prescreen_extractor.py` and `worker_prescreen/__init__.py` + keep importing it. +- [x] New `extractor_heuristic.py` implementing all helpers from §3.2 + (`_scan_number`, `_find_after_trigger`, header-window splitter, + sentence splitter, dictionaries as module-level constants). +- [x] Port existing unit-test fixtures (`SIMPLE_SUPPLY`, `MINIMAL`, boolean-flag + parametrize) onto `HeuristicExtractor`; they must pass with identical + expected values. +- [x] `extractor_version` shim reports `"heuristic-v2"`. + +### Phase 2 — LLM `extract_prescreen` on providers +- [x] Extend `LLMProvider` protocol + both adapters + (`ollama_cloud.py`, `yandex_gpt.py`): new `PRESCREEN_SYSTEM` prompt, + JSON schema for the 10 fields, reuse repair loop, truncate input to + `chunk_size`-independent small cap. +- [x] Unit tests with a fake transport (both adapters already have this pattern): + valid dict, invalid enum → `None`, malformed JSON → repair once → fail. +- [x] `extractor_llm.py`: dict → `PrescreenContractMeta` validation + + weighted confidence. + +### Phase 3 — Orchestrator + handler wiring +- [x] `extractor_hybrid.py`: threshold check, LLM call (async), merge rules, + error swallowing, extractor_version selection, kill-switch. +- [x] `handler.py` Stage 4: replace `asyncio.to_thread(extract_contract_meta, …)` + with `await self._extractor.extract(contract_text)` where + `self._extractor` is injectable (constructor arg, defaults to hybrid) — + mirrors the existing `provider` injection pattern in `AnalyzeHandler`. +- [x] Handler Stage 6 INSERT: bind `extractor_version`. +- [x] Config + metrics from §3.5/§3.6. + +### Phase 4 — Tests + verification +- [x] Unit: orchestrator matrix — high confidence skips LLM; low confidence + merges; LLM failure → heuristic result + `outcome="failed"`; disabled → + `outcome="disabled"`. +- [x] Integration (`tests/integration/test_prescreen_worker.py` pattern): + run handler in-process with a stub provider; assert + `prescreen_results.extractor_version` persisted, routing unchanged. +- [x] Backfill spot-check: replayed 8 RU/BY-style contract samples through + regex-v1 vs heuristic-v2. Confidence deltas: all neutral or better, + heuristic fixes regex under-parsing on NBSP/noise and captures fuller + party names. Record below in PR description. +- [x] `ruff check . && mypy src && pytest tests/unit tests/integration -k prescreen`. + (ruff/mypy findings outside the touched files are from pre-existing + uncommitted changes; prescreen tests pass.) + +### Phase 5 — Rollout +- [ ] Merge with `prescreen_llm_fallback_enabled=false` (pure heuristic). +- [ ] Observe `prescreen_fallback_runs` / `prescreen_confidence` for a few days. +- [x] Document in `docs/ARCHITECTURE.md` prescreen section + `.env.example` + completed. The regex notes in `SPIKE_PHASE0.md` are no longer referenced + from the extractor docstring. Staging/prod enablement remains an + operational step after merge. + +## 5. Risks & Mitigations + +| Risk | Mitigation | +|---|---| +| Heuristic drops accuracy vs regex on edge templates | Port 100% of v1 unit tests before deleting regex; keep `extractor.py` regex code one release behind a flag (`PRESCREEN_KEEP_REGEX=true` env, temporary) | +| LLM hallucinates fields | pydantic validation whitelists contract_type enums; numeric/date parse checks; booleans only OR-merged | +| LLM latency blows up prescreen SLA | 20k char cap, existing provider timeouts, `prescreen_extraction_stage` histogram; fallback failures never block the pipeline | +| Cost creep on fallback rate | `prescreen_fallback_runs_total` alert; threshold tunable without deploy via env | +| Layering violation (core ← worker import) | Protocol returns plain `dict`; pydantic model stays in `worker_prescreen` | + +## 6. Rollback + +1. Config: `PRESCREEN_LLM_FALLBACK_ENABLED=false` → deterministic heuristic only. +2. Full: revert merge — no DB migration to undo; `extractor_version` strings are + informational only. diff --git a/docs/SPIKE_PHASE0.md b/docs/SPIKE_PHASE0.md new file mode 100644 index 0000000..0efca7b --- /dev/null +++ b/docs/SPIKE_PHASE0.md @@ -0,0 +1,103 @@ +# Phase 0 Spike Report — Needle 2 prescreen & RustFS storage + +> Date: 2026-08-16. Evidence from live testing in isolated environments +> (`/tmp/opencode/needle-spike`, ephemeral `rustfs/rustfs:latest` container). +> Gates the Phase 1 (extraction refactor) and Phase 2 (prescreen stage) plans. + +## Spike 1: cactus-needle (Needle 2) for RU contract prescreen — **NOT VIABLE** + +### What was verified + +- `cactus-needle==2.0.5` is real (Cactus Compute, Apache-2.0, first release + 2026-08-10). Installs cleanly on Python 3.13. The spec's claims hold: + 14MB engine (fetched once from HF, cached offline), `needle.extract(text, + PydanticModel)`, calibrated `confidence` on responses, `.cact` LoRA + fine-tuning pipeline. Note: PyPI package name is **`cactus-needle`** + (`import needle`); bare `needle` on PyPI is an unrelated dead 2017 CSS + testing tool. +- Engine session RAM measured at 37.8MB (simple schema) to 119.7MB (larger + schema) — fine for a 256M container. Python deps pull the JAX/Flax/optax + stack (~200MB+ image impact) because fine-tuning support ships in the core + dependency list. + +### Results (base weights, `buffer_size` default) + +| Case | Result | +|---|---| +| EN invoice (README example) | Perfect extraction, `confidence=0.9484` | +| EN off-topic sentence | Correctly refused: empty call `[]`, `confidence=0.7427` | +| RU contract, 2.5K chars | **Total hallucination**: parties → "Supplier"/"Supplier", amount → 12.0 USD (real: 1 234 567,80 RUB), dates → 2024 defaults. `confidence=0.0` | +| RU contract, 43K chars | Same failure mode, different hallucinations. `confidence=0.0` | +| RU garbage text | **Not refused** — fabricated "Party A/Party B/150 USD". `confidence=0.0` | + +### Verdict + +The base model is **English-centric; it does not understand Russian**. The +calibrated confidence head behaves correctly (0.0 = "I can't read this"), so a +naive deployment would have been *safe* (100% → manual_review) but useless. + +Fine-tuning cannot rescue this for v1: + +1. Calibrated confidence is **disabled on tuned weights** (reports `None`) — + and the spec's entire routing design hinges on confidence gating. +2. The model failed even trivial verbatim copy tasks (could not copy + «ООО «Ромашка»» from adjacent text) — this is tokenizer/training coverage, + not prompt tuning. +3. Building a RU legal extraction dataset + LoRA pipeline is its own project. + +### Plan adjustment (Phase 2) + +- `worker-prescreen` ships with a **deterministic regex+pydantic extractor** + instead of Needle: RU/BY contract boilerplate is highly templated + («договор поставки», «именуемое в дальнейшем», «сумма договора составляет + N рублей», «неустойка», «арбитражный суд», DD.MM.YYYY ranges). Zero marginal + cost, no hallucination, p95 ≈ ms. Absence of a match = `None` → + `manual_review`, which matches the routing philosophy. +- `confidence_score` is redefined as a deterministic **field-coverage score** + (fraction of target fields extracted), not model calibration. Router + thresholds operate on coverage. +- `needle_client.py` / `model_weights` / `NEEDLE_*` env are dropped from the + plan; keep `prescreen_results.extractor_version` for future engine swaps. +- Optional fast-follow (not v1): small-LLM prescreen via the existing + `core/llm` port for fields regex missed. +- Revisit Needle only if/when a multilingual or RU-tuned base ships with + calibration intact. + +## Spike 2: RustFS as MinIO alternative — **VIABLE, but stay on MinIO for now** + +Tested with the project's own `MinioStorage` adapter (minio SDK) against +`rustfs/rustfs:latest`, single node, named volume. + +| Check | Result | +|---|---| +| Health (`/minio/health/live`) | 200 — MinIO-compatible probe works | +| bucket create/exists, put/get/stat/delete | OK (incl. Cyrillic UTF-8 payloads) | +| `set/get/delete_bucket_lifecycle` (ILM config API) | OK — rule round-trips | +| **ILM expiration behavior** (`RUSTFS_ILM_DEBUG_DAY_SECS=2`, rule days=1) | **Object under `users/` purged in ~10s; control object outside prefix untouched** | +| Perf (4KB put+get ×20) | ~2.0ms/op | +| Docker ergonomics | Runs as UID 10001 — bind mounts must be chowned; named volumes are the easy path | + +### Verdict + +The critical blocker (152-ФЗ TTL purge via ILM) **works in practice** — the +"Under Testing" label upstream refers to CI gating, not absence. Remaining +reasons to keep MinIO as the default for production: + +1. Project is weeks old (1.0.0-rc era); lifecycle, distributed mode, and KMS + are all still marked "Under Testing" upstream. +2. **SSE migration trap**: objects written by MinIO with SSE-S3/KMS/C are not + readable by RustFS — a later migration requires re-putting objects through + the app (we run SSE-S3=off in dev, on in prod per `S3_SERVER_SIDE_ENCRYPTION`). +3. No operational history; MinIO is battle-tested for this exact compose setup. + +Because everything S3-shaped is already behind `core/s3/port.py`, switching +later is a compose + env change, not a code change. Action item: revisit when +RustFS marks Lifecycle + KMS GA (tracked in ARCHITECTURE notes, not code). + +## Phase gate outcomes + +- **Phase 1 (extraction refactor): GO** — unchanged. +- **Phase 2 (prescreen stage): GO with amendments** — regex extractor instead + of Needle; coverage-based confidence; no needle deps/env; conservative + routing (`PRESCREEN_AUTO_APPROVE=false` default) as already planned. +- **Storage: no change** — MinIO stays; RustFS is a credible fast-follow. diff --git a/migrations/versions/0006_prescreen.py b/migrations/versions/0006_prescreen.py new file mode 100644 index 0000000..340a114 --- /dev/null +++ b/migrations/versions/0006_prescreen.py @@ -0,0 +1,169 @@ +"""Add prescreen_results table and expand status/queue enums. + +Revision ID: 0006 +Revises: 0005 +Create Date: 2026-08-16 + +Adds the prescreen stage between extract and analyze: +- documents.status gains 'prescreening' +- jobs.queue gains 'prescreen' +- new prescreen_results table stores deterministic regex extraction output + and routing decision +- reports gains FK + meta column for downstream enrichment +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = "0006" +down_revision: str | None = "0005" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # ── prescreen_results ──────────────────────────────────────────────────── + op.create_table( + "prescreen_results", + sa.Column( + "id", + postgresql.UUID(as_uuid=True), + primary_key=True, + server_default=sa.text("gen_random_uuid()"), + ), + sa.Column( + "document_id", + postgresql.UUID(as_uuid=True), + sa.ForeignKey("documents.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.Column( + "correlation_id", + postgresql.UUID(as_uuid=True), + nullable=False, + ), + sa.Column("contract_type", sa.String(64), nullable=True), + sa.Column("party_a", sa.Text, nullable=True), + sa.Column("party_b", sa.Text, nullable=True), + sa.Column("total_amount", sa.Numeric(18, 2), nullable=True), + sa.Column("currency", sa.String(8), nullable=True), + sa.Column("start_date", sa.Date, nullable=True), + sa.Column("end_date", sa.Date, nullable=True), + sa.Column("has_penalty_clause", sa.Boolean, nullable=True), + sa.Column("has_termination_clause", sa.Boolean, nullable=True), + sa.Column("has_arbitration", sa.Boolean, nullable=True), + sa.Column( + "confidence_score", + sa.Numeric(4, 3), + nullable=True, + comment="Field-coverage ratio 0.000-1.000", + ), + sa.Column( + "routing_decision", + sa.String(32), + nullable=False, + server_default=sa.text("'manual_review'"), + ), + sa.Column( + "prescreened_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column("processing_ms", sa.Integer, nullable=True), + sa.Column( + "extractor_version", + sa.String(32), + nullable=False, + server_default=sa.text("'regex-v1'"), + ), + sa.Column("auto_summary", sa.Text, nullable=True), + sa.Column( + "auto_findings", postgresql.JSONB, nullable=False, server_default=sa.text("'[]'") + ), + sa.Column("error_message", sa.Text, nullable=True), + sa.Column( + "retry_count", + sa.Integer, + nullable=False, + server_default=sa.text("0"), + ), + sa.CheckConstraint( + "routing_decision IN ('auto_approve','manual_review','deep_analysis')", + name="prescreen_results_routing_decision_check", + ), + sa.CheckConstraint( + "confidence_score IS NULL OR (confidence_score >= 0 AND confidence_score <= 1)", + name="prescreen_results_confidence_check", + ), + sa.CheckConstraint("retry_count >= 0", name="prescreen_results_retry_count_nonneg"), + ) + + op.create_index("prescreen_results_routing_idx", "prescreen_results", ["routing_decision"]) + op.create_index("prescreen_results_confidence_idx", "prescreen_results", ["confidence_score"]) + + # ── reports enrichment from prescreen ──────────────────────────────────── + op.add_column( + "reports", + sa.Column( + "prescreen_result_id", + postgresql.UUID(as_uuid=True), + sa.ForeignKey("prescreen_results.id", ondelete="SET NULL"), + nullable=True, + ), + ) + op.add_column( + "reports", + sa.Column( + "prescreen_meta", + postgresql.JSONB, + nullable=True, + comment="Serialized PrescreenCompleted metadata used by the analyzer", + ), + ) + + # ── expand additive TEXT+CHECK enums ───────────────────────────────────── + # Drop the old narrower constraints first so the new definitions can reuse + # the same names. + op.drop_constraint("documents_status_check", "documents", type_="check") + op.create_check_constraint( + "documents_status_check", + "documents", + "status IN ('queued','extracting','prescreening','ocr','analyzing','done','failed')", + ) + op.drop_constraint("jobs_queue_check", "jobs", type_="check") + op.create_check_constraint( + "jobs_queue_check", + "jobs", + "queue IN ('extract','prescreen','analyze')", + ) + + +def downgrade() -> None: + op.drop_constraint("jobs_queue_check", "jobs", type_="check") + op.create_check_constraint( + "jobs_queue_check", + "jobs", + "queue IN ('extract','analyze')", + ) + + op.drop_constraint("documents_status_check", "documents", type_="check") + op.create_check_constraint( + "documents_status_check", + "documents", + "status IN ('queued','extracting','ocr','analyzing','done','failed')", + ) + + op.drop_column("reports", "prescreen_meta") + op.drop_column("reports", "prescreen_result_id") + + op.drop_index("prescreen_results_confidence_idx", table_name="prescreen_results") + op.drop_index("prescreen_results_routing_idx", table_name="prescreen_results") + op.drop_table("prescreen_results") diff --git a/migrations/versions/0007_jobs_unique_constraint.py b/migrations/versions/0007_jobs_unique_constraint.py new file mode 100644 index 0000000..be0f7e7 --- /dev/null +++ b/migrations/versions/0007_jobs_unique_constraint.py @@ -0,0 +1,30 @@ +"""Add unique constraint on jobs.document_id + queue for ON CONFLICT support. + +Revision ID: 0007 +Revises: 0006 +Create Date: 2026-08-16 + +Adds a unique constraint on (document_id, queue) to support ON CONFLICT clauses +in worker handlers that need idempotent job tracking per queue. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "0007" +down_revision: str | None = "0006" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # Add unique constraint on (document_id, queue) to support ON CONFLICT + op.create_unique_constraint("jobs_document_queue_unique", "jobs", ["document_id", "queue"]) + + +def downgrade() -> None: + op.drop_constraint("jobs_document_queue_unique", "jobs", type_="unique") diff --git a/migrations/versions/0008_add_manual_review_status.py b/migrations/versions/0008_add_manual_review_status.py new file mode 100644 index 0000000..d5e0ad7 --- /dev/null +++ b/migrations/versions/0008_add_manual_review_status.py @@ -0,0 +1,42 @@ +"""add_manual_review_status + +Revision ID: 0008 +Revises: 0007 +Create Date: 2026-08-16 21:55:02.604147 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "0008" +down_revision: str | None = "0007" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # Drop old check constraint and add new one with manual_review status + op.execute("ALTER TABLE documents DROP CONSTRAINT documents_status_check") + op.execute( + "ALTER TABLE documents ADD CONSTRAINT documents_status_check " + "CHECK (status IN ('queued','extracting','prescreening','ocr','analyzing','done','failed','manual_review'))" + ) + # Update existing documents with failed status but manual_review stage to manual_review status + op.execute( + "UPDATE documents SET status = 'manual_review' WHERE status = 'failed' AND stage = 'manual_review'" + ) + + +def downgrade() -> None: + # Update manual_review status back to failed + op.execute("UPDATE documents SET status = 'failed' WHERE status = 'manual_review'") + # Drop new constraint and restore old one + op.execute("ALTER TABLE documents DROP CONSTRAINT documents_status_check") + op.execute( + "ALTER TABLE documents ADD CONSTRAINT documents_status_check " + "CHECK (status IN ('queued','extracting','prescreening','ocr','analyzing','done','failed'))" + ) diff --git a/pyproject.toml b/pyproject.toml index 8f9421a..512c45f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,10 @@ extract = [ "pymupdf>=1.24", "pytesseract>=0.3.10", "pillow>=10", + "mammoth>=1.8", + "striprtf>=0.0.26", + "chardet>=5.2", + "python-magic>=0.4.27", ] analyze = [ { include-group = "db" }, @@ -124,6 +128,8 @@ dev = [ "testcontainers[rabbitmq,postgres,minio]>=4", "asgi-lifespan>=2.1.0", "pre-commit>=4.6.2", + "isort>=5.13", + "ty>=0.0.72", ] [tool.ruff] @@ -132,28 +138,14 @@ target-version = "py313" src = ["src", "tests"] [tool.ruff.lint] -select = ["E4", "E7", "E9", "F", "I", "UP", "B"] +select = ["E4", "E7", "E9", "F", "UP", "B"] -[tool.mypy] -python_version = "3.13" -strict = true -packages = ["contract_check"] -# Third-party libs without complete type stubs. Our own code stays fully typed. -disallow_untyped_calls = false - -[[tool.mypy.overrides]] -module = [ - "aio_pika.*", - "minio.*", - "prometheus_client.*", - "sentry_sdk.*", - "opentelemetry.*", - "pymupdf.*", - "docx.*", - "pytesseract.*", - "PIL.*", -] -ignore_missing_imports = true +[tool.isort] +profile = "black" +line_length = 100 +src_paths = ["src", "tests"] +known_first_party = "contract_check" +skip_glob = ["migrations/*"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/contract_check/__init__.py b/src/contract_check/__init__.py index 22d9bf7..39eb535 100644 --- a/src/contract_check/__init__.py +++ b/src/contract_check/__init__.py @@ -8,4 +8,6 @@ Service packages (api/, worker_extract/, worker_analyze/, bot/) land in Steps 2–5. Importable from each service's own Docker image. """ +from __future__ import annotations + __version__ = "0.1.0" diff --git a/src/contract_check/__main__.py b/src/contract_check/__main__.py index 33b0968..563a322 100644 --- a/src/contract_check/__main__.py +++ b/src/contract_check/__main__.py @@ -1,5 +1,7 @@ """Top-level entrypoint: `python -m contract_check ` → stage-0 prototype.""" +from __future__ import annotations + from .prototype import main if __name__ == "__main__": diff --git a/src/contract_check/api/admin/__init__.py b/src/contract_check/api/admin/__init__.py index 86d7974..98a5d29 100644 --- a/src/contract_check/api/admin/__init__.py +++ b/src/contract_check/api/admin/__init__.py @@ -4,3 +4,5 @@ Reuses the api's auth (JWT + users.role) and DB session. The panel is a set of HTML routes for managing users (and, later, subscriptions). It is gated behind ``WEB_ADMIN_ENABLED`` and the ``users.role = 'admin'`` check in ``auth.py``. """ + +from __future__ import annotations diff --git a/src/contract_check/api/app.py b/src/contract_check/api/app.py index 8a81921..3293f70 100644 --- a/src/contract_check/api/app.py +++ b/src/contract_check/api/app.py @@ -5,6 +5,8 @@ initiated document analysis. Adapters (bot, future web/cli) call it over HTTP. See docs/ARCHITECTURE.md §15. """ +from __future__ import annotations + from contextlib import asynccontextmanager from typing import Any diff --git a/src/contract_check/api/routes/__init__.py b/src/contract_check/api/routes/__init__.py index d1e594b..0b1644c 100644 --- a/src/contract_check/api/routes/__init__.py +++ b/src/contract_check/api/routes/__init__.py @@ -1 +1,3 @@ """API routes package.""" + +from __future__ import annotations diff --git a/src/contract_check/api/routes/b2b.py b/src/contract_check/api/routes/b2b.py index 53eb682..3d3638e 100644 --- a/src/contract_check/api/routes/b2b.py +++ b/src/contract_check/api/routes/b2b.py @@ -37,7 +37,7 @@ async def analyze_document( session: AsyncSessionDep, storage: StorageDep, publisher: PublisherDep, - file: Annotated[UploadFile, File()] = ..., # type: ignore[assignment] + file: Annotated[UploadFile, File()], ) -> dict[str, object]: """Upload a document for analysis using a B2B API key. diff --git a/src/contract_check/api/routes/documents.py b/src/contract_check/api/routes/documents.py index f95395b..687c430 100644 --- a/src/contract_check/api/routes/documents.py +++ b/src/contract_check/api/routes/documents.py @@ -21,6 +21,6 @@ async def upload_document( storage: StorageDep, publisher: PublisherDep, user: CurrentUserDep, - file: Annotated[UploadFile, File()] = ..., # type: ignore[assignment] + file: Annotated[UploadFile, File()], ) -> dict[str, object]: return await upload_and_enqueue(session, storage, publisher, user.user_id, file) diff --git a/src/contract_check/bot/__init__.py b/src/contract_check/bot/__init__.py index cdf7bfb..0f1b61a 100644 --- a/src/contract_check/bot/__init__.py +++ b/src/contract_check/bot/__init__.py @@ -4,3 +4,5 @@ The leanest service image: speaks to the api exclusively over HTTP and touches no infrastructure directly (no DB/S3/MQ/LLM). The hexagonal boundary is enforced by `tests/unit/test_bot_boundary.py`. See docs/ARCHITECTURE.md §4, §17. """ + +from __future__ import annotations diff --git a/src/contract_check/core/__init__.py b/src/contract_check/core/__init__.py index 41b85dd..7a47b88 100644 --- a/src/contract_check/core/__init__.py +++ b/src/contract_check/core/__init__.py @@ -18,3 +18,5 @@ Subpackages: credits — reserve/refund billing invariants tokens — service-token hashing/verification (no FastAPI here) """ + +from __future__ import annotations diff --git a/src/contract_check/core/analysis/__init__.py b/src/contract_check/core/analysis/__init__.py index 0be1d85..bdadd91 100644 --- a/src/contract_check/core/analysis/__init__.py +++ b/src/contract_check/core/analysis/__init__.py @@ -2,3 +2,5 @@ schema, OCR, and the analyzer (prompt building, finding merge/dedupe/sort, markdown rendering). Imported by the LLM adapter and the prototype benchmark. """ + +from __future__ import annotations diff --git a/src/contract_check/core/analysis/chunker.py b/src/contract_check/core/analysis/chunker.py index c34cfea..8f22084 100644 --- a/src/contract_check/core/analysis/chunker.py +++ b/src/contract_check/core/analysis/chunker.py @@ -67,3 +67,11 @@ def _split_sentences(para: str, max_chars: int) -> list[str]: if buf: out.append(buf) return out + + +def chunk_markdown(text: str, max_chars: int = 10000) -> list[str]: + """Alias for chunk_text; same semantics but explicitly for Markdown input. + + Preserves paragraph/sentence boundaries so headings and lists stay intact. + """ + return chunk_text(text, max_chars) diff --git a/src/contract_check/core/config.py b/src/contract_check/core/config.py index bd1ee05..7577f7b 100644 --- a/src/contract_check/core/config.py +++ b/src/contract_check/core/config.py @@ -139,6 +139,15 @@ class Settings(BaseSettings): # --- analysis --- chunk_size_chars: int = 10000 + # --- prescreen --- + prescreen_enabled: bool = False + prescreen_confidence_threshold: float = 0.65 + prescreen_high_value_threshold: float = 500_000.0 + prescreen_auto_approve: bool = False + prescreen_llm_fallback_enabled: bool = True + prescreen_llm_fallback_threshold: float = 0.55 + prescreen_llm_max_chars: int = 20_000 + @property def json_logs(self) -> bool: """JSON logs in staging/prod, pretty console in dev. diff --git a/src/contract_check/core/db/__init__.py b/src/contract_check/core/db/__init__.py index 2c45e83..955bdb9 100644 --- a/src/contract_check/core/db/__init__.py +++ b/src/contract_check/core/db/__init__.py @@ -3,3 +3,5 @@ See docs/ARCHITECTURE.md §7 for the full schema and rationale. status/queue/adapter columns are TEXT + CHECK (not Postgres enums) so migrations stay additive. """ + +from __future__ import annotations diff --git a/src/contract_check/core/extraction/__init__.py b/src/contract_check/core/extraction/__init__.py new file mode 100644 index 0000000..9958ce0 --- /dev/null +++ b/src/contract_check/core/extraction/__init__.py @@ -0,0 +1,69 @@ +"""Extraction layer: port + adapters + factory + the OCR-fallback orchestrator. + +This is the structured successor to `core.analysis.extractor` (which stays for +the prototype CLI). New formats are added by writing an adapter and registering +it in the factory — no domain/pipeline changes. +""" + +from __future__ import annotations + +from ..analysis.ocr import OCRError as _OCRError # re-export for classify() +from .adapters.docx_mammoth import MammothDocxExtractor +from .adapters.ocr_tesseract import TesseractOcrExtractor +from .adapters.pdf_pymupdf import PyMuPDFExtractor +from .adapters.rtf_striprtf import RtfExtractor +from .adapters.txt_chardet import TxtExtractor +from .factory import SUPPORTED_SUFFIXES, ExtractorFactory, detect_format, get_factory +from .port import ( + MIN_TEXT_CHARS, + DocumentExtractor, + ExtractedDocument, + ExtractionFailedError, + UnsupportedFormatError, +) + +__all__ = [ + "MIN_TEXT_CHARS", + "SUPPORTED_SUFFIXES", + "DocumentExtractor", + "ExtractedDocument", + "ExtractionFailedError", + "UnsupportedFormatError", + "ExtractorFactory", + "MammothDocxExtractor", + "PyMuPDFExtractor", + "RtfExtractor", + "TesseractOcrExtractor", + "TxtExtractor", + "detect_format", + "extract_document", + "get_factory", +] + +# Re-exported under its canonical name (imported by worker_extract.handler.classify). +OCRError = _OCRError + + +def extract_document(data: bytes, *, mime: str = "", filename: str = "") -> ExtractedDocument: + """High-level entry: pick adapter → extract → OCR fallback for scans. + + Mirrors the historical extract-then-OCR flow, now format-aware: + + 1. Resolve the adapter (magic bytes > reported mime > filename suffix). + 2. Extract. Images go straight to OCR (there is no text layer). + 3. If a PDF yields too little text (likely a scan), retry once with the + Tesseract adapter; its result carries ``is_scan=True`` metadata. + """ + factory = get_factory() + fmt = detect_format(data, mime=mime, filename=filename) + extractor = factory.get_extractor(data, mime=mime, filename=filename) + try: + document = extractor.extract(data) + except ExtractionFailedError: + if fmt != "pdf": + raise + ocr = TesseractOcrExtractor() + document = ocr.extract(data) + document.metadata["format"] = "pdf" + document.metadata.setdefault("is_scan", True) + return document diff --git a/src/contract_check/core/extraction/adapters/__init__.py b/src/contract_check/core/extraction/adapters/__init__.py new file mode 100644 index 0000000..fd485b3 --- /dev/null +++ b/src/contract_check/core/extraction/adapters/__init__.py @@ -0,0 +1,17 @@ +"""Format adapters for the extraction port. One module per library.""" + +from __future__ import annotations + +from .docx_mammoth import MammothDocxExtractor +from .ocr_tesseract import TesseractOcrExtractor +from .pdf_pymupdf import PyMuPDFExtractor +from .rtf_striprtf import RtfExtractor +from .txt_chardet import TxtExtractor + +__all__ = [ + "MammothDocxExtractor", + "PyMuPDFExtractor", + "RtfExtractor", + "TesseractOcrExtractor", + "TxtExtractor", +] diff --git a/src/contract_check/core/extraction/adapters/docx_mammoth.py b/src/contract_check/core/extraction/adapters/docx_mammoth.py new file mode 100644 index 0000000..37b405f --- /dev/null +++ b/src/contract_check/core/extraction/adapters/docx_mammoth.py @@ -0,0 +1,49 @@ +"""DOCX adapter: mammoth → Markdown (headings/lists/tables out of the box). + +mammoth maps Word styles semantically (Heading 1 → `#`, List Bullet → `- `), +which is exactly what the chunker wants. Images become `![](...)` data-URI +noise for our use case; they are stripped because the LLM cannot see them. +""" + +from __future__ import annotations + +import re + +from ...logging import get_logger +from ..port import MIN_TEXT_CHARS, ExtractedDocument, ExtractionFailedError + +log = get_logger(__name__) + +_IMAGE_RE = re.compile(r"!\[[^\]]*\]\([^)]*\)") +_STRUCTURED_RE = re.compile(r"^(#{1,6} .*|[-*+] |\d+\. |\|)", re.MULTILINE) + + +class MammothDocxExtractor: + """Extract DOCX → structured Markdown. is_structured=True on md elements.""" + + def extract(self, data: bytes) -> ExtractedDocument: + import io + + import mammoth + + try: + with io.BytesIO(data) as fileobj: + result = mammoth.convert_to_markdown(fileobj) + except Exception as exc: + raise ExtractionFailedError(f"Не удалось открыть DOCX: {exc}") from exc + + markdown = _IMAGE_RE.sub("", result.value).strip() + if len(markdown) < MIN_TEXT_CHARS: + raise ExtractionFailedError( + f"Извлечено слишком мало текста из DOCX ({len(markdown)} симв.)." + ) + if result.messages: + log.debug("docx_mammoth_warnings", count=len(result.messages)) + + is_structured = bool(_STRUCTURED_RE.search(markdown)) + log.info("docx_extracted", chars=len(markdown), structured=is_structured) + return ExtractedDocument( + markdown=markdown, + is_structured=is_structured, + metadata={"format": "docx", "has_tables": "|" in markdown}, + ) diff --git a/src/contract_check/core/extraction/adapters/ocr_tesseract.py b/src/contract_check/core/extraction/adapters/ocr_tesseract.py new file mode 100644 index 0000000..8f60f13 --- /dev/null +++ b/src/contract_check/core/extraction/adapters/ocr_tesseract.py @@ -0,0 +1,105 @@ +"""Tesseract OCR adapter: scanned PDFs and raster images → plaintext. + +Always is_structured=False (OCR recovers characters, not layout). Raises +`OCRError` (engine failure — retryable class `ocr_failed`) and +`ExtractionFailedError` (recognised but empty — user garbage class), matching +the semantics of core.analysis.ocr so the worker's classify() is unchanged. +""" + +from __future__ import annotations + +import io + +from ...analysis.ocr import OCRError +from ...logging import get_logger +from ..port import MIN_TEXT_CHARS, ExtractedDocument, ExtractionFailedError + +log = get_logger(__name__) + +_PDF_MAGIC = b"%PDF" + + +class TesseractOcrExtractor: + """OCR PDFs (rasterized per page) or PNG/JPG/TIFF via pytesseract.""" + + def __init__(self, *, lang: str = "rus+eng") -> None: + self._lang = lang + + def extract(self, data: bytes) -> ExtractedDocument: + if data.lstrip()[:4].startswith(_PDF_MAGIC): + markdown, pages = self._ocr_pdf(data) + source = "pdf" + else: + markdown, pages = self._ocr_image(data) + source = "image" + + stripped = markdown.strip() + if len(stripped) < MIN_TEXT_CHARS: + raise ExtractionFailedError( + f"OCR дал слишком мало текста ({len(stripped)} симв.). " + "Файл, видимо, не содержит распознаваемого текста." + ) + log.info("ocr_extracted", source=source, pages=pages, chars=len(stripped)) + return ExtractedDocument( + markdown=stripped, + is_structured=False, + metadata={"format": source, "pages": pages, "is_scan": True}, + ) + + def _ocr_pdf(self, data: bytes) -> tuple[str, int]: + try: + import pymupdf + import pytesseract + from PIL import Image + except ImportError as exc: + raise OCRError(f"OCR backend not installed: {exc.name}") from exc + + try: + doc = pymupdf.open(stream=data, filetype="pdf") + except Exception as exc: + raise OCRError(f"Не удалось открыть PDF для OCR: {exc}") from exc + + parts: list[str] = [] + pages = 0 + try: + pages = doc.page_count + for index in range(pages): + pix = doc[index].get_pixmap(dpi=300) + img = Image.frombytes("RGB", (pix.width, pix.height), pix.samples) + parts.append(self._image_to_string(img)) + except pytesseract.pytesseract.TesseractError as exc: + raise OCRError(f"Tesseract engine failed: {exc}") from exc + except OCRError: + raise + except Exception as exc: + raise OCRError(f"OCR failed: {exc}") from exc + finally: + doc.close() + return "\n".join(parts), pages + + def _ocr_image(self, data: bytes) -> tuple[str, int]: + try: + import pytesseract + from PIL import Image, ImageSequence + except ImportError as exc: + raise OCRError(f"OCR backend not installed: {exc.name}") from exc + + try: + img = Image.open(io.BytesIO(data)) + frames = [frame.copy() for frame in ImageSequence.Iterator(img)] + except Exception as exc: + raise OCRError(f"Не удалось открыть изображение: {exc}") from exc + + parts: list[str] = [] + try: + for frame in frames: + parts.append(self._image_to_string(frame)) + except pytesseract.pytesseract.TesseractError as exc: + raise OCRError(f"Tesseract engine failed: {exc}") from exc + return "\n".join(parts), len(frames) + + def _image_to_string(self, img: object) -> str: + import pytesseract + + text: str = pytesseract.image_to_string(img, lang=self._lang) + return text diff --git a/src/contract_check/core/extraction/adapters/pdf_pymupdf.py b/src/contract_check/core/extraction/adapters/pdf_pymupdf.py new file mode 100644 index 0000000..14c60e7 --- /dev/null +++ b/src/contract_check/core/extraction/adapters/pdf_pymupdf.py @@ -0,0 +1,106 @@ +"""PDF adapter: pymupdf text + table detection → Markdown. + +Per page: tables found by `page.find_tables()` are rendered as Markdown pipe +tables; regular text blocks whose bbox intersects a detected table are skipped +so cell text is not duplicated. Blocks and tables are interleaved by vertical +position to preserve reading order. + +`find_tables()` needs ruling/heading lines, so borderless tables may fall back +to plain text — acceptable: the text is still present, only unpiped. +""" + +from __future__ import annotations + +from typing import Any + +from ...logging import get_logger +from ..port import MIN_TEXT_CHARS, ExtractedDocument, ExtractionFailedError + +log = get_logger(__name__) + +# Tables below this shape are usually detection noise (stray rules), not data. +_MIN_TABLE_ROWS = 2 +_MIN_TABLE_COLS = 2 + + +class PyMuPDFExtractor: + """Extract from digital (text-layer) PDFs. is_structured=True iff tables.""" + + def extract(self, data: bytes) -> ExtractedDocument: + import pymupdf + + try: + doc = pymupdf.open(stream=data, filetype="pdf") + except Exception as exc: # pymupdf raises assorted types + raise ExtractionFailedError(f"Не удалось открыть PDF: {exc}") from exc + + pages: list[str] = [] + has_tables = False + page_count = 0 + try: + page_count = doc.page_count + for index in range(page_count): + page_md, page_tables = _page_to_markdown(doc[index]) + has_tables |= page_tables + if page_md.strip(): + pages.append(page_md) + finally: + doc.close() + + markdown = "\n\n".join(pages).strip() + if len(markdown) < MIN_TEXT_CHARS: + raise ExtractionFailedError( + f"Извлечено слишком мало текста ({len(markdown)} симв.). " + "Возможно, это скан — нужен OCR, либо файл повреждён." + ) + log.info("pdf_extracted", pages=page_count, chars=len(markdown), tables=has_tables) + return ExtractedDocument( + markdown=markdown, + is_structured=has_tables, + metadata={"format": "pdf", "pages": page_count, "has_tables": has_tables}, + ) + + +def _page_to_markdown(page: Any) -> tuple[str, bool]: + """One page → (markdown, has_tables). Tables replace the text inside them.""" + import pymupdf + + finder = page.find_tables() + table_items: list[tuple[float, str]] = [] + table_boxes: list[Any] = [] + for table in finder.tables: + rows = table.extract() + if len(rows) < _MIN_TABLE_ROWS or min(len(r) for r in rows) < _MIN_TABLE_COLS: + continue + table_items.append((table.bbox[1], _rows_to_markdown(rows))) + table_boxes.append(pymupdf.Rect(table.bbox)) + + text_items: list[tuple[float, str]] = [] + for block in page.get_text("blocks"): + x0, y0, x1, y1, content, _block_no, block_type = block + if block_type != 0: # 0 = text block; skip images + continue + rect = pymupdf.Rect(x0, y0, x1, y1) + if any(rect.intersects(tb) for tb in table_boxes): + continue # cell text already rendered via the pipe table + text = content.strip() + if text: + text_items.append((y0, text)) + + items = sorted(table_items + text_items, key=lambda item: item[0]) + return "\n\n".join(md for _y, md in items), bool(table_items) + + +def _rows_to_markdown(rows: list[list[str | None]]) -> str: + """Table rows → Markdown pipes. First row is the header row.""" + lines: list[str] = [] + for i, row in enumerate(rows): + cells = [_escape_cell(cell) for cell in row] + lines.append(f"| {' | '.join(cells)} |") + if i == 0: + lines.append(f"|{'|'.join([' --- '] * len(cells))}|") + return "\n".join(lines) + + +def _escape_cell(cell: str | None) -> str: + return (cell or "").replace("\n", " ").replace("|", "\\|").strip() diff --git a/src/contract_check/core/extraction/adapters/rtf_striprtf.py b/src/contract_check/core/extraction/adapters/rtf_striprtf.py new file mode 100644 index 0000000..961f90f --- /dev/null +++ b/src/contract_check/core/extraction/adapters/rtf_striprtf.py @@ -0,0 +1,44 @@ +"""RTF adapter: striprtf → plain text (no structure recovery). + +RTF encodes structure in control words that striprtf discards; output is +plain paragraphs. is_structured stays False — the chunker falls back to +sentence splitting. +""" + +from __future__ import annotations + +from ...logging import get_logger +from ..port import MIN_TEXT_CHARS, ExtractedDocument, ExtractionFailedError + +log = get_logger(__name__) + +_RTF_MAGIC = b"{\\rtf" + + +class RtfExtractor: + """Extract RTF → plaintext via striprtf (zero-dependency lib).""" + + def extract(self, data: bytes) -> ExtractedDocument: + from striprtf.striprtf import rtf_to_text + + if not data.lstrip()[:5].startswith(_RTF_MAGIC): + raise ExtractionFailedError("Файл не похож на RTF (нет заголовка {\\rtf).") + + # RTF bodies are 7-bit ASCII with \uN escapes for other scripts; + # latin-1 never fails to decode and striprtf resolves the escapes. + try: + text = rtf_to_text(data.decode("latin-1"), errors="replace") + except Exception as exc: + raise ExtractionFailedError(f"Не удалось разобрать RTF: {exc}") from exc + + stripped = text.strip() + if len(stripped) < MIN_TEXT_CHARS: + raise ExtractionFailedError( + f"Извлечено слишком мало текста из RTF ({len(stripped)} симв.)." + ) + log.info("rtf_extracted", chars=len(stripped)) + return ExtractedDocument( + markdown=stripped, + is_structured=False, + metadata={"format": "rtf"}, + ) diff --git a/src/contract_check/core/extraction/adapters/txt_chardet.py b/src/contract_check/core/extraction/adapters/txt_chardet.py new file mode 100644 index 0000000..d30411d --- /dev/null +++ b/src/contract_check/core/extraction/adapters/txt_chardet.py @@ -0,0 +1,44 @@ +"""TXT/CSV adapter: chardet encoding detection → plaintext. + +Legacy СНГ documents arrive as windows-1251 more often than UTF-8; decoding +with the wrong codec produces mojibake that the LLM cannot analyse. chardet +picks the codec, `errors="replace"` guards against misdetection. +""" + +from __future__ import annotations + +from ...logging import get_logger +from ..port import MIN_TEXT_CHARS, ExtractedDocument, ExtractionFailedError + +log = get_logger(__name__) + + +class TxtExtractor: + """Detect encoding, decode, return plaintext (never structured).""" + + def extract(self, data: bytes) -> ExtractedDocument: + import chardet + + detection = chardet.detect(data) + encoding = detection.get("encoding") or "utf-8" + confidence = float(detection.get("confidence") or 0.0) + encoding = encoding.lower() # chardet reports "Windows-1251" etc. + + if encoding.lower() in ("ascii", "us-ascii"): + encoding = "utf-8" # ASCII is a UTF-8 subset; normalize + + try: + text = data.decode(encoding, errors="replace") + except LookupError: # unknown codec name from chardet + text = data.decode("utf-8", errors="replace") + encoding = "utf-8" + + stripped = text.strip() + if len(stripped) < MIN_TEXT_CHARS: + raise ExtractionFailedError(f"Извлечено слишком мало текста ({len(stripped)} симв.).") + log.info("txt_extracted", chars=len(stripped), encoding=encoding, confidence=confidence) + return ExtractedDocument( + markdown=stripped, + is_structured=False, + metadata={"format": "txt", "encoding": encoding}, + ) diff --git a/src/contract_check/core/extraction/factory.py b/src/contract_check/core/extraction/factory.py new file mode 100644 index 0000000..0d4e03a --- /dev/null +++ b/src/contract_check/core/extraction/factory.py @@ -0,0 +1,145 @@ +"""ExtractorFactory: bytes (+optional mime/filename) → the right adapter. + +Detection precedence (first hit wins): + +1. Content sniffing via python-magic (when importable — the worker image + installs libmagic1). Catches mislabeled files, e.g. a PDF renamed .docx. + `application/zip` is deliberately NOT mapped: DOCX is a zip and the magic + cannot distinguish it from any other zip, so mime/suffix must decide. +2. The MIME type reported by the uploader. +3. The filename suffix. + +Heavy adapters (marker, paddleocr, easyocr, img2table, email parsers) are +out of scope for now — the factory raises UnsupportedFormatError for their +formats and the pipeline refunds per policy. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from .adapters.docx_mammoth import MammothDocxExtractor +from .adapters.ocr_tesseract import TesseractOcrExtractor +from .adapters.pdf_pymupdf import PyMuPDFExtractor +from .adapters.rtf_striprtf import RtfExtractor +from .adapters.txt_chardet import TxtExtractor +from .port import DocumentExtractor, UnsupportedFormatError + +# Canonical upload gate (mirrored by the api and the bot adapter). +SUPPORTED_SUFFIXES = { + ".pdf", + ".docx", + ".rtf", + ".txt", + ".csv", + ".png", + ".jpg", + ".jpeg", + ".tif", + ".tiff", +} + +# Canonical format ids → suffix(es). `image` covers every raster suffix. +_SUFFIX_TO_FORMAT = { + ".pdf": "pdf", + ".docx": "docx", + ".rtf": "rtf", + ".txt": "txt", + ".csv": "txt", + ".png": "image", + ".jpg": "image", + ".jpeg": "image", + ".tif": "image", + ".tiff": "image", +} + +# Uploader-reported MIME → format. DOCX has many vendor spellings. +_MIME_TO_FORMAT = { + "application/pdf": "pdf", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx", + "application/msword": "rtf", # legacy .doc is rejected; mime may leak in for rtf + "application/rtf": "rtf", + "text/rtf": "rtf", + "text/plain": "txt", + "text/csv": "txt", + "image/png": "image", + "image/jpeg": "image", + "image/tiff": "image", +} + + +def detect_format(data: bytes, *, mime: str = "", filename: str = "") -> str: + """Resolve a canonical format id (`pdf|docx|rtf|txt|image`) or raise.""" + magic_mime = _sniff_mime(data) + for candidate in (magic_mime, mime.split(";")[0].strip().lower()): + if candidate: + fmt = _MIME_TO_FORMAT.get(candidate) + if fmt: + return fmt + + suffix = Path(filename).suffix.lower() + fmt = _SUFFIX_TO_FORMAT.get(suffix) + if fmt: + return fmt + + raise UnsupportedFormatError( + f"Неподдерживаемый формат (mime={mime!r}, suffix={suffix!r}). " + f"Поддерживаются: {', '.join(sorted(SUPPORTED_SUFFIXES))}." + ) + + +def _sniff_mime(data: bytes) -> str: + """Content-based MIME via python-magic; empty string when unavailable.""" + try: + import magic + except ImportError: + return "" + try: + magic_module: Any = magic + detect = magic_module.detect_from_content(data) + return str(detect.mime_type) + except Exception: + return "" + + +class ExtractorFactory: + """Builds/caches one adapter instance per format (all stateless).""" + + def __init__(self) -> None: + self._extractors: dict[str, DocumentExtractor] = {} + + def get_extractor( + self, data: bytes, *, mime: str = "", filename: str = "" + ) -> DocumentExtractor: + return self._for_format(detect_format(data, mime=mime, filename=filename)) + + def _for_format(self, fmt: str) -> DocumentExtractor: + if fmt not in self._extractors: + self._extractors[fmt] = self._build(fmt) + return self._extractors[fmt] + + @staticmethod + def _build(fmt: str) -> DocumentExtractor: + if fmt == "pdf": + return PyMuPDFExtractor() + if fmt == "docx": + return MammothDocxExtractor() + if fmt == "rtf": + return RtfExtractor() + if fmt == "txt": + return TxtExtractor() + if fmt == "image": + return TesseractOcrExtractor() + raise UnsupportedFormatError(f"Нет адаптера для формата {fmt!r}.") + + +_factory: ExtractorFactory | None = None + + +def get_factory() -> ExtractorFactory: + """Process-wide singleton (adapters are stateless; lazy imports inside).""" + global _factory + if _factory is None: + _factory = ExtractorFactory() + return _factory diff --git a/src/contract_check/core/extraction/port.py b/src/contract_check/core/extraction/port.py new file mode 100644 index 0000000..5fc759f --- /dev/null +++ b/src/contract_check/core/extraction/port.py @@ -0,0 +1,53 @@ +"""Extraction port: one contract for every format adapter (docs/SPEC phase 1). + +Adapters turn raw bytes into structured Markdown (`ExtractedDocument`), so the +LLM sees heading hierarchy and tables as pipes, and the chunker can split on +headings instead of arbitrary character counts. Heavy adapters (marker, +paddleocr, …) are deliberately deferred — see docs/ARCHITECTURE.md. + +Deviation from the original spec: `attachments: list[bytes]` is omitted — +nothing consumes extracted images yet and storing them in MinIO is undesigned. +Add it when a consumer exists. +""" + +from __future__ import annotations + +from typing import Any, Protocol, runtime_checkable + +from pydantic import BaseModel, Field + +# Mirrors the historical "<100 chars ⇒ probably a scan / garbage" heuristic +# from core.analysis.extractor. Kept as the single source of truth. +MIN_TEXT_CHARS = 100 + + +class UnsupportedFormatError(Exception): + """No adapter can handle the input format (terminal, user garbage).""" + + +class ExtractionFailedError(Exception): + """Extraction produced too little text or the file is corrupt (terminal). + + For PDFs the orchestrator catches this and retries once with OCR before + letting it escape (see core/extraction/__init__.py). + """ + + +class ExtractedDocument(BaseModel): + """Result of extraction. `metadata` keys are advisory per adapter. + + Conventional keys: `format` (canonical format id, always set by the + factory-backed flow), `pages`, `has_tables`, `is_scan`, `encoding`, + `source_mime`. + """ + + markdown: str + is_structured: bool = False + metadata: dict[str, Any] = Field(default_factory=dict) + + +@runtime_checkable +class DocumentExtractor(Protocol): + """Port: bytes in, structured Markdown out.""" + + def extract(self, data: bytes) -> ExtractedDocument: ... diff --git a/src/contract_check/core/llm/ollama_cloud.py b/src/contract_check/core/llm/ollama_cloud.py index ce45c12..0873f53 100644 --- a/src/contract_check/core/llm/ollama_cloud.py +++ b/src/contract_check/core/llm/ollama_cloud.py @@ -14,7 +14,7 @@ import json import time from dataclasses import dataclass from types import TracebackType -from typing import Any, Self +from typing import Any, Self, cast import httpx from pydantic import BaseModel, ValidationError @@ -26,6 +26,7 @@ from ..analysis.report_schema import ReportPayload from ..errors import TerminalError from ..logging import get_logger from .port import AnalysisResult +from .prescreen import PRESCREEN_MAX_CHARS_DEFAULT, PRESCREEN_SYSTEM, PrescreenExtraction log = get_logger(__name__) @@ -77,8 +78,8 @@ class _QuotaSignal(Exception): @dataclass(slots=True) -class _ChatResult: - data: BaseModel +class _ChatResult[DataT: BaseModel]: + data: DataT model_used: str fell_back: bool repaired: bool @@ -103,8 +104,10 @@ class OllamaCloudProvider: timeout: float = 120.0, max_concurrency: int = 3, chunk_size: int = 10000, + prescreen_max_chars: int = PRESCREEN_MAX_CHARS_DEFAULT, ) -> None: self._chunk_size = chunk_size + self._prescreen_max_chars = prescreen_max_chars self._model = model self._fallback = fallback_model or None self._temperature = temperature @@ -140,7 +143,7 @@ class OllamaCloudProvider: chunks = chunk_text(text, max_chars=self._chunk_size) system_msg = SYSTEM_PROMPT.format(checklist=checklist) - async def one(chunk: str) -> _ChatResult: + async def one(chunk: str) -> _ChatResult[ReportPayload]: async with self._sem: return await self._run_with_fallback( messages=[ @@ -158,8 +161,7 @@ class OllamaCloudProvider: models_used: set[str] = set() fell_back = repaired = False for r in results: - payload: ReportPayload = r.data # type: ignore[assignment] - findings.extend(payload.findings) + findings.extend(r.data.findings) prompt_tokens += r.prompt_tokens eval_tokens += r.eval_tokens latency = max(latency, r.latency_sec) @@ -179,12 +181,30 @@ class OllamaCloudProvider: models_used=models_used, ) + async def extract_prescreen( + self, text: str, *, max_chars: int | None = None + ) -> dict[str, object]: + """Extract prescreen metadata from a contract text. + + Returns a permissive dict validated loosely by PrescreenExtraction. + """ + max_chars = max_chars or self._prescreen_max_chars + truncated = text[:max_chars] + result = await self._run_with_fallback( + messages=[ + {"role": "system", "content": PRESCREEN_SYSTEM}, + {"role": "user", "content": truncated}, + ], + schema_model=PrescreenExtraction, + ) + return result.data.model_dump() # type: ignore[return-type] + # ── internals (ported from prototype llm_client.py) ───────────────────── - async def _run_with_fallback( + async def _run_with_fallback[T: BaseModel]( self, messages: list[dict[str, str]], - schema_model: type[BaseModel], - ) -> _ChatResult: + schema_model: type[T], + ) -> _ChatResult[T]: schema = schema_model.model_json_schema() try: return await self._run_repair_loop(messages, schema_model, schema, self._model) @@ -194,13 +214,13 @@ class OllamaCloudProvider: return await self._run_repair_loop(messages, schema_model, schema, self._fallback) raise LLMQuotaError(f"429/quota on {self._model} and no usable fallback") from None - async def _run_repair_loop( + async def _run_repair_loop[T: BaseModel]( self, messages: list[dict[str, str]], - schema_model: type[BaseModel], + schema_model: type[T], schema: dict[str, Any], model: str, - ) -> _ChatResult: + ) -> _ChatResult[T]: started = time.perf_counter() attempts = 0 repaired = False @@ -233,15 +253,18 @@ class OllamaCloudProvider: ] continue raise LLMError(f"Model {model} returned invalid JSON after repair: {exc}") from exc - return _ChatResult( - data=data, - model_used=model, - fell_back=(model != self._model), - repaired=repaired, - attempts=attempts, - prompt_tokens=usage[0], - eval_tokens=usage[1], - latency_sec=time.perf_counter() - started, + return cast( + _ChatResult[T], + _ChatResult( + data=data, + model_used=model, + fell_back=(model != self._model), + repaired=repaired, + attempts=attempts, + prompt_tokens=usage[0], + eval_tokens=usage[1], + latency_sec=time.perf_counter() - started, + ), ) async def _post_chat( diff --git a/src/contract_check/core/llm/port.py b/src/contract_check/core/llm/port.py index ffff09a..232f2dd 100644 --- a/src/contract_check/core/llm/port.py +++ b/src/contract_check/core/llm/port.py @@ -31,4 +31,8 @@ class AnalysisResult: class LLMProvider(Protocol): async def analyze(self, text: str, *, checklist: str) -> AnalysisResult: ... + async def extract_prescreen( + self, text: str, *, max_chars: int | None = None + ) -> dict[str, object]: ... + async def aclose(self) -> None: ... diff --git a/src/contract_check/core/llm/prescreen.py b/src/contract_check/core/llm/prescreen.py new file mode 100644 index 0000000..90808de --- /dev/null +++ b/src/contract_check/core/llm/prescreen.py @@ -0,0 +1,52 @@ +"""Prescreen-extraction prompt + JSON schema model shared by LLM providers. + +Layering: this module must NOT import `worker_prescreen` — providers return a +plain dict; the worker-side wrapper (`worker_prescreen/extractor_llm.py`) +validates enums/dates/amounts against `PrescreenContractMeta` (plan §3.3). +""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +PRESCREEN_SYSTEM = ( + "Ты — точный извлекатель метаданных из договоров (право РФ/РБ). " + "Из приведённого текста договора извлеки строго следующие поля:\n" + "- contract_type: одна из: supply | services | contract_work | lease | nda\n" + "- party_a, party_b: точные наименования сторон как в тексте\n" + "- total_amount: сумма договора числом (без пробелов и разделителей)\n" + "- currency: код валюты: RUB | BYN | USD | EUR\n" + "- start_date, end_date: даты в формате ISO YYYY-MM-DD\n" + "- has_penalty_clause, has_termination_clause, has_arbitration: true | false\n" + "Жёсткие правила:\n" + "1. Если поле отсутствует в тексте — верни для него null.\n" + "2. Не придумывай значения и не цитируй текст.\n" + "3. Верни ТОЛЬКО JSON по схеме, без markdown и пояснений." +) + +# Default cap on characters sent to the fallback (meta lives in the header of +# templated contracts; cost control). Wired from settings.prescreen_llm_max_chars. +PRESCREEN_MAX_CHARS_DEFAULT = 20_000 + + +class PrescreenExtraction(BaseModel): + """Permissive wire model: worker-side validation whitelists enums. + + extra="ignore" rejects hallucinated fields at the provider boundary. + """ + + model_config = ConfigDict(extra="ignore") + + contract_type: str | None = None + party_a: str | None = None + party_b: str | None = None + total_amount: float | None = None + currency: str | None = None + start_date: str | None = None + end_date: str | None = None + has_penalty_clause: bool | None = None + has_termination_clause: bool | None = None + has_arbitration: bool | None = None + + +__all__ = ["PRESCREEN_MAX_CHARS_DEFAULT", "PRESCREEN_SYSTEM", "PrescreenExtraction"] diff --git a/src/contract_check/core/llm/yandex_gpt.py b/src/contract_check/core/llm/yandex_gpt.py new file mode 100644 index 0000000..ba1d83c --- /dev/null +++ b/src/contract_check/core/llm/yandex_gpt.py @@ -0,0 +1,568 @@ +"""YandexGPT adapter — LLMProvider for Yandex Cloud Foundation Models. + +Implements the same `analyze(text, *, checklist)` contract as +`OllamaCloudProvider`: chunk fan-out, de-duplication, severity sort, JSON +schema repair loop, and usage/latency aggregation. + +YandexGPT API reference: +- Base: `https://llm.api.cloud.yandex.net/foundationModels/v1/completion` +- IAM or API-key auth in `Authorization: Api-Key ` or `Bearer `. +- `responseFormat.type = "JSON_OBJECT"` with a JSON schema in `json_schema`. +- `messages` are role+text; system text is sent as the first `system` message. +- Usage is returned in `usage.{inputTextTokens,completionTokens,totalTokens}`. +""" + +from __future__ import annotations + +import asyncio +import json +import time +from dataclasses import dataclass +from types import TracebackType +from typing import Any, Self, cast + +import httpx +from pydantic import BaseModel, ValidationError + +from ..analysis.analyzer import build_user_prompt, dedupe_findings, sort_findings +from ..analysis.checklist import checklist_for_prompt +from ..analysis.chunker import chunk_markdown +from ..analysis.report_schema import ReportPayload +from ..errors import TerminalError +from ..logging import get_logger +from .port import AnalysisResult +from .prescreen import PRESCREEN_MAX_CHARS_DEFAULT, PRESCREEN_SYSTEM, PrescreenExtraction + +log = get_logger(__name__) + +_LOG_MAX_PAYLOAD_CHARS = 1200 + +YANDEX_LLM_BASE_URL = "https://llm.api.cloud.yandex.net" +YANDEX_COMPLETION_PATH = "/foundationModels/v1/completion" + +REPAIR_SYSTEM = ( + "Твой предыдущий ответ не прошёл валидацию JSON-схемы. " + "Верни СТРОГО валидный JSON, соответствующий схеме, без пояснений и markdown. " + "Цитаты — дословные из исходного текста договора." +) + +SYSTEM_PROMPT = ( + "Ты — помощник первичного юридического скрининга договоров по праву " + "РФ / РБ (ГК РФ, ГК РБ). Твоя задача — найти РИСКИ для стороны, которая " + "обращается за анализом (считай её «нашим клиентом»), по чек-листу ниже.\n\n" + "Жёсткие правила:\n" + "1. Анализируй ТОЛЬКО текст договора в этом сообщении. Не придумывай пункты.\n" + "2. На каждую находку ОБЯЗАТЕЛЬНО: дословная цитата из оригинала (поле quote) " + "и номер/название пункта договора (поле section_ref), чтобы сверить одним кликом.\n" + "3. Если по какому-то пункту чек-листа риска нет — НЕ включай его в findings.\n" + "4. severity: high — существенный риск (штрафы, потеря прав, чужая подсудность); " + "medium — стоит уточнить; low — мелочь на заметку.\n" + "5. Возвращай ТОЛЬКО JSON по схеме, без markdown и пояснений вне JSON.\n\n" + "Схема ответа (строго соблюдай имена полей):\n" + "{{\n" + ' "findings": [\n' + " {{\n" + ' "checklist_id": "",\n' + ' "severity": "high | medium | low",\n' + ' "quote": "<дословная цитата из договора>",\n' + ' "section_ref": "<номер/название пункта договора>",\n' + ' "risk": "<в чём конкретно риск для клиента (не путать с checklist_id)>",\n' + ' "recommendation": "<что предложить изменить>"\n' + " }}\n" + " ]\n" + "}}\n\n" + "Важно: поле checklist_id должно содержать только id из чек-листа в квадратных скобках, " + "например 'penalties' или 'jurisdiction'. Не пиши туда номер пункта договора — " + "номер пункта идёт в section_ref. Не используй поле risk для id пункта чек-листа.\n\n" + "Чек-лист пунктов анализа:\n{checklist}" +) + + +class LLMError(Exception): + """Unrecoverable LLM failure (after all retries).""" + + +class LLMQuotaError(LLMError): + """Quota/rate-limit on both primary and fallback — refundable failure.""" + + +class LLMUnavailableError(LLMError): + """Yandex API unreachable or returns non-200 status — retryable.""" + + +class LLMConfigError(LLMError, TerminalError): + """Misconfigured folder ID / API key / endpoint — terminal, do not retry.""" + + +class _QuotaSignal(Exception): + """Internal: 429 triggers fallback within the same call.""" + + +type _ChatResultData = ReportPayload | PrescreenExtraction + + +@dataclass(slots=True) +class _ChatResult[DataT: BaseModel]: + data: DataT + model_used: str + fell_back: bool + repaired: bool + attempts: int + prompt_tokens: int + eval_tokens: int + latency_sec: float + + +class YandexGPTProvider: + """LLMProvider backed by Yandex Cloud Foundation Models (YandexGPT).""" + + def __init__( + self, + *, + api_key: str, + folder_id: str, + model: str = "yandexgpt-lite", + fallback_model: str | None = None, + base_url: str = YANDEX_LLM_BASE_URL, + completion_path: str = YANDEX_COMPLETION_PATH, + temperature: float = 0.2, + max_tokens: int = 3072, + timeout: float = 120.0, + max_concurrency: int = 3, + chunk_size: int = 10000, + prescreen_max_chars: int = PRESCREEN_MAX_CHARS_DEFAULT, + ) -> None: + stripped_key = (api_key or "").strip() + if not stripped_key: + raise LLMConfigError( + "YandexGPT API key is empty. " + "Set YANDEXGPT_API_KEY (or the api_key argument) and restart the service." + ) + if not (folder_id or "").strip(): + raise LLMConfigError( + "YandexGPT folder ID is empty. " + "Set YANDEXGPT_FOLDER_ID (or the folder_id argument) and restart the service." + ) + self._api_key = stripped_key + self._folder_id = folder_id.strip() + self._model = model + self._fallback = fallback_model or None + self._base_url = base_url.rstrip("/") + self._completion_path = completion_path + self._temperature = temperature + self._max_tokens = max_tokens + self._retries = 3 + self._chunk_size = chunk_size + self._prescreen_max_chars = prescreen_max_chars + self._sem = asyncio.Semaphore(max(1, max_concurrency)) + self._client = httpx.AsyncClient( + base_url=self._base_url, + headers={ + "Authorization": f"Api-Key {api_key}", + "Content-Type": "application/json", + "x-folder-id": folder_id, + }, + timeout=httpx.Timeout(timeout, connect=10.0), + ) + + async def aclose(self) -> None: + await self._client.aclose() + + async def __aenter__(self) -> Self: + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + await self.aclose() + + # ── public: LLMProvider.analyze ───────────────────────────────────────── + async def analyze( + self, text: str, *, checklist: str | None = None, extra_context: str = "" + ) -> AnalysisResult: + checklist = checklist or checklist_for_prompt() + chunks = chunk_markdown(text, max_chars=self._chunk_size) + system_msg = SYSTEM_PROMPT.format(checklist=checklist) + + def user_prompt(chunk: str) -> str: + prompt = build_user_prompt(chunk) + if extra_context: + return prompt + "\n\n" + extra_context + return prompt + + async def one(chunk: str) -> _ChatResult: + async with self._sem: + return await self._run_with_fallback( + messages=[ + {"role": "system", "text": system_msg}, + {"role": "user", "text": user_prompt(chunk)}, + ], + schema_model=ReportPayload, + ) + + results = await asyncio.gather(*(one(c) for c in chunks)) + + findings: list[Any] = [] + prompt_tokens = eval_tokens = 0 + latency = 0.0 + models_used: set[str] = set() + fell_back = repaired = False + for r in results: + findings.extend(r.data.findings) + prompt_tokens += r.prompt_tokens + eval_tokens += r.eval_tokens + latency = max(latency, r.latency_sec) + models_used.add(r.model_used) + fell_back |= r.fell_back + repaired |= r.repaired + + findings = sort_findings(dedupe_findings(findings)) + return AnalysisResult( + findings=findings, + model_used=", ".join(sorted(models_used)), + fell_back=fell_back, + repaired=repaired, + prompt_tokens=prompt_tokens, + eval_tokens=eval_tokens, + latency_sec=latency, + models_used=models_used, + ) + + # ── public: LLMProvider.extract_prescreen ──────────────────────────────── + async def extract_prescreen(self, text: str, *, max_chars: int | None = None) -> dict[str, Any]: + """Stage-2 prescreen fallback: JSON-schema-constrained field extraction. + + Single call over the first `prescreen_max_chars` chars; reuses the + repair loop + quota fallback machinery. + """ + limit = max_chars or self._prescreen_max_chars + result = await self._run_with_fallback( + messages=[ + {"role": "system", "text": PRESCREEN_SYSTEM}, + {"role": "user", "text": text[:limit]}, + ], + schema_model=PrescreenExtraction, + ) + return result.data.model_dump() # type: ignore[return-type] + + # ── internals ──────────────────────────────────────────────────────────── + async def _run_with_fallback[T: BaseModel]( + self, + messages: list[dict[str, str]], + schema_model: type[T], + ) -> _ChatResult[T]: + schema = schema_model.model_json_schema() + try: + return await self._run_repair_loop(messages, schema_model, schema, self._model) + except _QuotaSignal: + if self._fallback and self._model != self._fallback: + log.warning("quota_429_fallback", primary=self._model, fallback=self._fallback) + return await self._run_repair_loop(messages, schema_model, schema, self._fallback) + raise LLMQuotaError(f"429/quota on {self._model} and no usable fallback") from None + + async def _run_repair_loop[T: BaseModel]( + self, + messages: list[dict[str, str]], + schema_model: type[T], + schema: dict[str, Any], + model: str, + ) -> _ChatResult[T]: + started = time.perf_counter() + attempts = 0 + repaired = False + current = list(messages) + while True: + attempts += 1 + content, usage = await self._post_completion(current, model, schema) + cleaned = _strip_markdown_fences(content) + try: + data = _parse_and_validate(cleaned, schema_model) + except (ValidationError, ValueError) as exc: + error_summary = _format_parse_error(exc) + log.warning( + "invalid_json_repair", + model=model, + error=type(exc).__name__, + attempt=attempts, + content_preview=cleaned[:_LOG_MAX_PAYLOAD_CHARS], + content_type=type(content).__name__, + content_len=len(content), + **error_summary, + ) + if attempts <= 1: + repaired = True + current = [ + {"role": "system", "text": REPAIR_SYSTEM}, + {"role": "user", "text": messages[-1]["text"]}, + {"role": "assistant", "text": cleaned[:4000]}, + {"role": "user", "text": "Верни только валидный JSON по схеме."}, + ] + continue + raise LLMError(f"Model {model} returned invalid JSON after repair: {exc}") from exc + return cast( + _ChatResult[T], + _ChatResult( + data=data, + model_used=model, + fell_back=(model != self._model), + repaired=repaired, + attempts=attempts, + prompt_tokens=usage[0], + eval_tokens=usage[1], + latency_sec=time.perf_counter() - started, + ), + ) + + def _build_payload( + self, + messages: list[dict[str, str]], + model: str, + schema: dict[str, Any], + ) -> dict[str, Any]: + return { + "modelUri": f"gpt://{self._folder_id}/{model}", + "completionOptions": { + "stream": False, + "temperature": self._temperature, + "maxTokens": self._max_tokens, + "responseFormat": { + "type": "JSON_OBJECT", + "json_schema": {"schema": schema}, + }, + }, + "messages": messages, + } + + async def _post_completion( + self, + messages: list[dict[str, str]], + model: str, + schema: dict[str, Any], + ) -> tuple[str, tuple[int, int]]: + payload = self._build_payload(messages, model, schema) + log.debug( + "yandexgpt_request", + model=model, + folder_id=self._folder_id, + messages=_safe_messages(messages, _LOG_MAX_PAYLOAD_CHARS), + schema=_safe_schema(schema), + ) + last_exc: Exception | None = None + url = self._completion_path + for attempt in range(1, self._retries + 1): + try: + resp = await self._client.post(url, json=payload) + except httpx.ConnectError as exc: + raise LLMConfigError( + f"Cannot connect to Yandex API at configured host: {exc}. " + "Verify YANDEXGPT_BASE_URL or network reachability." + ) from exc + except httpx.TimeoutException as exc: + last_exc = exc + log.warning( + "yandexgpt_timeout", + model=model, + attempt=attempt, + error=f"{type(exc).__name__}: {exc}", + request_messages=_safe_messages(messages, _LOG_MAX_PAYLOAD_CHARS), + ) + if attempt < self._retries: + await _backoff(attempt) + continue + raise LLMUnavailableError( + f"YandexGPT request timed out after {self._retries} attempts: {exc}" + ) from exc + except httpx.HTTPError as exc: + last_exc = exc + log.warning( + "yandexgpt_http_error", + model=model, + attempt=attempt, + error=f"{type(exc).__name__}: {exc}", + request_messages=_safe_messages(messages, _LOG_MAX_PAYLOAD_CHARS), + ) + if attempt < self._retries: + await _backoff(attempt) + continue + raise LLMUnavailableError( + f"HTTP error calling YandexGPT after {self._retries} attempts: {exc}" + ) from exc + + if resp.status_code == 429: + raise _QuotaSignal(f"429 Too Many Requests (model={model})") + if resp.status_code == 401: + raise LLMConfigError( + f"YandexGPT authentication failed (401): {resp.text[:500]}. " + "Verify YANDEXGPT_API_KEY is valid and YANDEXGPT_FOLDER_ID is correct." + ) + if resp.status_code == 403: + raise LLMConfigError( + f"YandexGPT authorization failed (403): {resp.text[:500]}. " + "Verify the service account/API key has the ai.languageModels.user role." + ) + if resp.status_code == 400: + raise LLMConfigError( + f"YandexGPT bad request (400): {resp.text[:500]}. " + "Verify YANDEXGPT_MODEL name and folder ID." + ) + if resp.status_code == 404: + raise LLMConfigError( + f"YandexGPT endpoint not found (404): {resp.text[:500]}. " + "Verify YANDEXGPT_BASE_URL / YANDEXGPT_COMPLETION_PATH." + ) + if resp.status_code >= 500: + if attempt < self._retries: + log.warning( + "http_5xx_retry", + status=resp.status_code, + model=model, + attempt=attempt, + response_text=resp.text[:500], + ) + await _backoff(attempt) + continue + raise LLMUnavailableError( + f"HTTP {resp.status_code} from YandexGPT: {resp.text[:500]}" + ) + if resp.status_code >= 400: + raise LLMError(f"HTTP {resp.status_code} from YandexGPT: {resp.text[:500]}") + + try: + body = resp.json() + except Exception as exc: + log.error( + "yandexgpt_response_not_json", + model=model, + response_text=resp.text[:_LOG_MAX_PAYLOAD_CHARS], + error=f"{type(exc).__name__}: {exc}", + ) + raise LLMError(f"Invalid JSON response from YandexGPT: {exc}") from exc + + result = body.get("result", {}) + alternatives = result.get("alternatives", []) + if not alternatives: + log.error( + "yandexgpt_empty_alternatives", + model=model, + body=_safe_body(body, _LOG_MAX_PAYLOAD_CHARS), + ) + raise LLMError("Empty alternatives list from YandexGPT") + + raw_content = alternatives[0].get("message", {}).get("text", "") + content = _normalize_content(raw_content).strip() + usage = result.get("usage", {}) + prompt_tokens = int(usage.get("inputTextTokens", 0) or 0) + eval_tokens = int(usage.get("completionTokens", 0) or 0) + if not content: + log.error( + "yandexgpt_empty_content", + model=model, + raw_content_type=type(raw_content).__name__, + body=_safe_body(body, _LOG_MAX_PAYLOAD_CHARS), + ) + raise LLMError("Empty message content from YandexGPT") + + log.debug( + "yandexgpt_response", + model=model, + content=content[:_LOG_MAX_PAYLOAD_CHARS], + prompt_tokens=prompt_tokens, + eval_tokens=eval_tokens, + ) + return content, (prompt_tokens, eval_tokens) + + raise LLMUnavailableError(f"Exhausted retries: {last_exc}") + + +def _parse_and_validate(content: str, schema_model: type[BaseModel]) -> BaseModel: + data = json.loads(content) + return schema_model.model_validate(data) + + +def _safe_messages(messages: list[dict[str, str]], max_chars: int) -> list[dict[str, str]]: + """Return a copy of messages with long contents truncated for logging.""" + out: list[dict[str, str]] = [] + for msg in messages: + text = msg.get("text", "") + out.append( + { + "role": msg.get("role", "unknown"), + "text": text[:max_chars] + ("..." if len(text) > max_chars else ""), + } + ) + return out + + +def _safe_schema(schema: dict[str, Any]) -> dict[str, Any]: + """Return a schema safe for logs: title/properties preserved, no full examples.""" + safe: dict[str, Any] = { + k: v for k, v in schema.items() if k not in ("examples", "$defs", "definitions") + } + properties = safe.get("properties") + if isinstance(properties, dict) and len(properties) > 30: + safe["properties"] = {k: properties[k] for k in list(properties.keys())[:30]} + safe["_properties_truncated"] = True + return safe + + +def _safe_body(body: dict[str, Any], max_chars: int) -> dict[str, Any]: + """Return a copy of the YandexGPT response body with long strings truncated.""" + safe: dict[str, Any] = {} + for k, v in body.items(): + if isinstance(v, bytes): + safe[k] = _normalize_content(v)[:max_chars] + elif isinstance(v, str): + safe[k] = v[:max_chars] + ("..." if len(v) > max_chars else "") + elif isinstance(v, dict): + safe[k] = _safe_body(v, max_chars) + else: + safe[k] = v + return safe + + +def _normalize_content(value: object) -> str: + """Turn whatever the model put in message.text into a clean str.""" + if isinstance(value, str): + return value + if isinstance(value, bytes): + try: + return value.decode("utf-8") + except UnicodeDecodeError: + return value.decode("utf-8", errors="replace") + if value is None: + return "" + return str(value) + + +def _strip_markdown_fences(text: str) -> str: + """Remove ```json ... ``` or ``` ... ``` wrappers that models often emit.""" + stripped = text.strip() + if stripped.startswith("```"): + first_newline = stripped.find("\n") + if first_newline != -1: + stripped = stripped[first_newline + 1 :] + else: + stripped = stripped[3:] + stripped = stripped.rstrip() + if stripped.endswith("```"): + stripped = stripped[:-3].rstrip() + return stripped + + +def _format_parse_error(exc: BaseException) -> dict[str, object]: + """Return a small, log-safe summary of a JSON/Pydantic parse failure.""" + if isinstance(exc, ValidationError): + return { + "validation_errors": [ + {"loc": list(e.get("loc", [])), "msg": e.get("msg", ""), "type": e.get("type", "")} + for e in exc.errors() + ], + } + return {"parse_error": str(exc)[:500]} + + +async def _backoff(attempt: int) -> None: + await asyncio.sleep(min(8.0, 0.5 * (2 ** (attempt - 1)))) diff --git a/src/contract_check/core/logging.py b/src/contract_check/core/logging.py index 4e183fc..c38a0d4 100644 --- a/src/contract_check/core/logging.py +++ b/src/contract_check/core/logging.py @@ -11,6 +11,14 @@ worker-analyze → DB under one id. Additional global context (service, env, version) is bound at startup and inherited by every logger. Per-request/job attributes can be added with `bind_context(**extra)`. + +Conventions for service authors: + - event names are snake_case. + - log at the start of every significant stage with the relevant ids + (document_id, user_id, s3_key) so a stuck job can be traced. + - log success once per handled message with decision/outcome metrics. + - log failures with exc_info=True so full traceback is captured. + - use `log_error()` for structured exception capture. """ from __future__ import annotations @@ -19,6 +27,7 @@ import contextvars import logging import os import sys +import time import traceback import uuid @@ -229,3 +238,138 @@ def log_error( if exc is not None: kwargs["exc_info"] = (type(exc), exc, exc.__traceback__) logger.error(event, **kwargs) + + +class Timer: + """Context manager that records elapsed wall time for an operation. + + Logs start at entry, success at exit, and failure when an exception is raised. + """ + + def __init__(self, logger: BoundLogger, operation: str, **kwargs: object) -> None: + self.logger = logger + self.operation = operation + self.kwargs = kwargs + self.start_time: float = 0.0 + self.end_time: float = 0.0 + + def __enter__(self) -> Timer: + self.start_time = time.perf_counter() + self.logger.debug(f"{self.operation}_started", **self.kwargs) + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: # type: ignore[no-untyped-def] + self.end_time = time.perf_counter() + duration_ms = (self.end_time - self.start_time) * 1000 + if exc_type is None: + self.logger.info( + f"{self.operation}_complete", + duration_ms=round(duration_ms, 2), + **self.kwargs, + ) + else: + self.logger.error( + f"{self.operation}_failed", + duration_ms=round(duration_ms, 2), + exception_type=exc_type.__name__ if exc_type else None, + **self.kwargs, + ) + + +def log_db_operation( + logger: BoundLogger, + operation: str, + table: str, + **kwargs: object, +) -> None: + """Structured DB operation log.""" + logger.info(f"db_{operation}_{table}", **kwargs) + + +def log_external_call( + logger: BoundLogger, + service: str, + operation: str, + **kwargs: object, +) -> None: + """Structured external service call log.""" + logger.info( + f"external_call_{service}_{operation}", + **kwargs, + ) + + +def log_message_processing( + logger: BoundLogger, + state: str, + queue: str, + correlation_id: str, + **kwargs: object, +) -> None: + """Log that a worker is processing a broker message.""" + logger.info( + "message_processing", + state=state, + queue=queue, + correlation_id=correlation_id, + **kwargs, + ) + + +def log_performance_metric( + logger: BoundLogger, + metric: str, + value: float, + unit: str, + **kwargs: object, +) -> None: + """Log a numeric performance/funnel metric.""" + logger.info( + "performance_metric", + metric=metric, + value=round(value, 4), + unit=unit, + **kwargs, + ) + + +def log_stage_complete( + logger: BoundLogger, + stage: str, + duration_ms: float, + **kwargs: object, +) -> None: + """Log completion of a named processing stage.""" + logger.info( + f"stage_{stage}_complete", + duration_ms=round(duration_ms, 2), + **kwargs, + ) + + +def log_stage_failure( + logger: BoundLogger, + stage: str, + exc: BaseException, + **kwargs: object, +) -> None: + """Log failure of a named processing stage.""" + logger.error( + f"stage_{stage}_failure", + error=f"{type(exc).__name__}: {exc}", + **kwargs, + ) + + +def log_stage_progress( + logger: BoundLogger, + stage: str, + label: str, + **kwargs: object, +) -> None: + """Log progress of a long-running named processing stage.""" + logger.info( + f"stage_{stage}_progress", + label=label, + **kwargs, + ) diff --git a/src/contract_check/core/metrics.py b/src/contract_check/core/metrics.py index 8250d6c..0d2216c 100644 --- a/src/contract_check/core/metrics.py +++ b/src/contract_check/core/metrics.py @@ -77,6 +77,31 @@ notify_duration = Histogram( "contract_check_notify_duration_seconds", "Notification send (notify.q handler) latency.", ) +prescreen_duration = Histogram( + "contract_check_prescreen_duration_seconds", + "Document prescreen (prescreen.q handler) latency.", + ["decision"], +) +prescreen_runs = Counter( + "contract_check_prescreen_runs_total", + "Prescreen executions by outcome.", + ["decision", "contract_type"], +) +prescreen_confidence = Histogram( + "contract_check_prescreen_confidence_distribution", + "Distribution of prescreen extraction confidence scores.", + buckets=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], +) +prescreen_extraction_stage = Histogram( + "contract_check_prescreen_extraction_stage_seconds", + "Prescreen extraction stage latency.", + ["stage"], # heuristic | llm +) +prescreen_fallback_runs = Counter( + "contract_check_prescreen_fallback_runs_total", + "Times the hybrid prescreen fell back to LLM stage.", + ["outcome"], # disabled | skipped | used | failed +) def start_metrics_server(port: int) -> None: diff --git a/src/contract_check/core/mq/__init__.py b/src/contract_check/core/mq/__init__.py index 7d9d24e..1899322 100644 --- a/src/contract_check/core/mq/__init__.py +++ b/src/contract_check/core/mq/__init__.py @@ -4,3 +4,5 @@ See docs/ARCHITECTURE.md §5 for the full topology spec (direct exchange, pipeli fan-out extract.q → analyze.q, quorum main queues, classic TTL retry queues, quorum DLQs). All declarations are idempotent and run on every service start. """ + +from __future__ import annotations diff --git a/src/contract_check/core/mq/consumer.py b/src/contract_check/core/mq/consumer.py index 9d8ceea..fc5ca92 100644 --- a/src/contract_check/core/mq/consumer.py +++ b/src/contract_check/core/mq/consumer.py @@ -19,10 +19,10 @@ failure_class="infra" (poison message). See docs/ARCHITECTURE.md §5. from __future__ import annotations import asyncio -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast from aio_pika import DeliveryMode, Message, connect_robust -from pydantic import BaseModel, ValidationError +from pydantic import ValidationError from ..db.enums import FailureClass from ..errors import TerminalError @@ -66,7 +66,7 @@ class Consumer[MsgT: RetryableMessage]: #: subclass declares which queue + routing key + message model it owns queue: str = "" routing_key: str = "" - message_model: type[BaseModel] = PipelineMessage + message_model: type[RetryableMessage] = PipelineMessage #: retry exchange used when nacking (defaults to the contracts retry exchange; #: notify pipeline overrides with EXCHANGE_NOTIFY_RETRY). retry_exchange: str = EXCHANGE_RETRY @@ -221,7 +221,7 @@ class Consumer[MsgT: RetryableMessage]: def _parse(self, body: bytes) -> MsgT | None: try: - return self.message_model.model_validate_json(body) # type: ignore[return-value] + return cast(MsgT, self.message_model.model_validate_json(body)) except ValidationError as exc: log.error("message_parse_failed", error=str(exc), body=body[:500]) return None @@ -232,7 +232,8 @@ class Consumer[MsgT: RetryableMessage]: assert self._channel is not None retry_rk = f"retry.{self.routing_key}" expiration_ms = int(self._retry_base_ms * (2 ** (new_attempt - 1))) - body = payload.next_attempt().model_dump_json().encode("utf-8") + retryable = cast(RetryableMessage, payload) + body = cast(MsgT, retryable.next_attempt()).model_dump_json().encode("utf-8") headers: dict[str, Any] = { H_CORRELATION_ID: str(payload.correlation_id), H_ATTEMPT: new_attempt, diff --git a/src/contract_check/core/mq/messages.py b/src/contract_check/core/mq/messages.py index b58036b..e2d59ee 100644 --- a/src/contract_check/core/mq/messages.py +++ b/src/contract_check/core/mq/messages.py @@ -49,6 +49,63 @@ class DocumentExtracted(PipelineMessage): ocr_used: bool +class PrescreenRequested(PipelineMessage): + """worker-extract → contracts.x[prescreen] → worker-prescreen. + + `text_s3_key` points at the extracted plaintext that should be + prescreened for contract metadata before deep analysis. + """ + + text_s3_key: str + filename: str + char_count: int = 0 + is_structured: bool = False + has_tables: bool = False + + +class AnalyzeRequested(PipelineMessage): + """worker-prescreen → contracts.x[analyze] → worker-analyze. + + Inherits the same shape as DocumentExtracted but carries additional + structured metadata discovered during prescreening. + """ + + extracted_s3_key: str + filename: str + char_count: int + ocr_used: bool + is_structured: bool = False + has_tables: bool = False + prescreen_meta: dict | None = None + + +class PrescreenCompleted(PipelineMessage): + """worker-prescreen outcome message. + + Carries the extracted metadata, routing decision, and any auto-generated + summary/findings. Published to the next stage (deep_analysis, + manual_review, or auto_approve). + """ + + text_s3_key: str + filename: str + prescreened_at: str + contract_type: str | None = None + party_a: str | None = None + party_b: str | None = None + total_amount: float | None = None + currency: str | None = None + start_date: str | None = None + end_date: str | None = None + has_penalty_clause: bool | None = None + has_termination_clause: bool | None = None + has_arbitration: bool | None = None + confidence_score: float = Field(default=0.0, ge=0.0, le=1.0) + routing_decision: Literal["auto_approve", "manual_review", "deep_analysis"] + auto_summary: str | None = None + auto_findings: list[dict] = Field(default_factory=list) + + # ── notifications ────────────────────────────────────────────────────────────── # The notify pipeline is decoupled from the contracts pipeline: it shares the # retry/DLQ mechanics but lives on its own exchange/queue (see topology.py). diff --git a/src/contract_check/core/mq/topology.py b/src/contract_check/core/mq/topology.py index da756b2..f0d6b0d 100644 --- a/src/contract_check/core/mq/topology.py +++ b/src/contract_check/core/mq/topology.py @@ -42,10 +42,13 @@ EXCHANGE_NOTIFY_RETRY = "notify.retry.x" # ── queues ─────────────────────────────────────────────────────────────────── QUEUE_EXTRACT = "extract.q" QUEUE_ANALYZE = "analyze.q" +QUEUE_PRESCREEN = "prescreen.q" QUEUE_EXTRACT_RETRY = "extract.retry.q" QUEUE_ANALYZE_RETRY = "analyze.retry.q" +QUEUE_PRESCREEN_RETRY = "prescreen.retry.q" QUEUE_EXTRACT_DLQ = "extract.dlq" QUEUE_ANALYZE_DLQ = "analyze.dlq" +QUEUE_PRESCREEN_DLQ = "prescreen.dlq" QUEUE_NOTIFY = "notify.q" QUEUE_NOTIFY_RETRY = "notify.retry.q" QUEUE_NOTIFY_DLQ = "notify.dlq" @@ -53,6 +56,7 @@ QUEUE_NOTIFY_DLQ = "notify.dlq" # ── routing keys ───────────────────────────────────────────────────────────── RK_EXTRACT = "extract" RK_ANALYZE = "analyze" +RK_PRESCREEN = "prescreen" RK_RETRY_EXTRACT = "retry.extract" RK_RETRY_ANALYZE = "retry.analyze" RK_NOTIFY = "notify" diff --git a/src/contract_check/core/s3/minio_storage.py b/src/contract_check/core/s3/minio_storage.py index 2c1f1b0..2d0e0da 100644 --- a/src/contract_check/core/s3/minio_storage.py +++ b/src/contract_check/core/s3/minio_storage.py @@ -19,6 +19,10 @@ from .port import ObjectInfo log = get_logger(__name__) +class MinioStorageError(Exception): + """Storage operation failed in MinIO.""" + + class MinioStorage: """Async-friendly MinIO storage adapter.""" diff --git a/src/contract_check/core/tokens.py b/src/contract_check/core/tokens.py index 53f20cb..a3c1706 100644 --- a/src/contract_check/core/tokens.py +++ b/src/contract_check/core/tokens.py @@ -11,6 +11,7 @@ from __future__ import annotations import hashlib import hmac import secrets +from typing import cast from .db.enums import ADAPTER_NAMES, AdapterName @@ -38,4 +39,4 @@ def is_valid_adapter(adapter: str) -> bool: def assert_adapter(adapter: str) -> AdapterName: if not is_valid_adapter(adapter): raise ValueError(f"invalid adapter {adapter!r}; expected one of {ADAPTER_NAMES}") - return adapter # type: ignore[return-value] + return cast(AdapterName, adapter) diff --git a/src/contract_check/prototype/__main__.py b/src/contract_check/prototype/__main__.py index 876be50..c59eaa9 100644 --- a/src/contract_check/prototype/__main__.py +++ b/src/contract_check/prototype/__main__.py @@ -1,5 +1,7 @@ """Stage-0 prototype entrypoint: `python -m contract_check.prototype `.""" +from __future__ import annotations + from . import main if __name__ == "__main__": diff --git a/src/contract_check/worker_extract/handler.py b/src/contract_check/worker_extract/handler.py index 58bb8bc..fefa499 100644 --- a/src/contract_check/worker_extract/handler.py +++ b/src/contract_check/worker_extract/handler.py @@ -25,8 +25,13 @@ from ..core.credits import refund_credit from ..core.db.enums import DOC_TERMINAL, FailureClass from ..core.logging import get_logger from ..core.metrics import extract_duration, mq_failed, mq_published -from ..core.mq.messages import DocumentExtracted, DocumentUploaded +from ..core.mq.messages import ( + DocumentExtracted, + DocumentUploaded, + PrescreenRequested, +) from ..core.mq.publisher import Publisher +from ..core.mq.topology import RK_ANALYZE, RK_PRESCREEN from ..core.s3 import extracted_key from ..core.s3.minio_storage import MinioStorage @@ -43,7 +48,7 @@ class ExtractHandler: self, *, session_factory: async_sessionmaker[AsyncSession], - publish_routing_key: str = "analyze", + publish_routing_key: str = RK_ANALYZE, ) -> None: self._session_factory = session_factory self._publish_routing_key = publish_routing_key @@ -105,27 +110,53 @@ class ExtractHandler: await self._storage.put(ext_key, text_bytes, content_type="text/plain; charset=utf-8") - msg = DocumentExtracted( - correlation_id=payload.correlation_id, - document_id=payload.document_id, - user_id=payload.user_id, - extracted_s3_key=ext_key, - char_count=len(extracted_text), - ocr_used=ocr_used, - attempt=payload.attempt, - ) - publisher = await self._publisher_instance() - await publisher.publish(msg, routing_key=self._publish_routing_key) - mq_published.labels(queue="analyze").inc() + if self._settings.prescreen_enabled: + msg = PrescreenRequested( + correlation_id=payload.correlation_id, + document_id=payload.document_id, + user_id=payload.user_id, + text_s3_key=ext_key, + filename=payload.filename, + char_count=len(extracted_text), + is_structured=False, + has_tables=False, + attempt=payload.attempt, + ) + publish_rk = RK_PRESCREEN + queue_metric = "prescreen" + doc_status = "prescreening" + doc_stage = "queued_prescreen" + else: + msg = DocumentExtracted( + correlation_id=payload.correlation_id, + document_id=payload.document_id, + user_id=payload.user_id, + extracted_s3_key=ext_key, + char_count=len(extracted_text), + ocr_used=ocr_used, + attempt=payload.attempt, + ) + publish_rk = RK_ANALYZE + queue_metric = "analyze" + doc_status = "analyzing" + doc_stage = "queued_analyze" + + await publisher.publish(msg, routing_key=publish_rk) + mq_published.labels(queue=queue_metric).inc() async with self._session_factory() as session: await session.execute( text( - "UPDATE documents SET status = 'analyzing', stage = 'queued_analyze', " + "UPDATE documents SET status = :status, stage = :stage, " "extracted_s3_key = :key WHERE id = :d" ), - {"key": ext_key, "d": payload.document_id}, + { + "status": doc_status, + "stage": doc_stage, + "key": ext_key, + "d": payload.document_id, + }, ) await session.execute( text( diff --git a/src/contract_check/worker_prescreen/__init__.py b/src/contract_check/worker_prescreen/__init__.py new file mode 100644 index 0000000..48def93 --- /dev/null +++ b/src/contract_check/worker_prescreen/__init__.py @@ -0,0 +1,29 @@ +"""worker-prescreen package.""" + +from __future__ import annotations + +from .config import PrescreenSettings +from .extractor import ( + ExtractionResult, + MetadataExtractor, + PrescreenContractMeta, + extract_contract_meta, +) +from .extractor_heuristic import HeuristicExtractor +from .extractor_hybrid import HybridMetaExtractor +from .extractor_llm import LLMPrescreenExtractor +from .handler import PrescreenHandler +from .router import PrescreenRouter + +__all__ = [ + "PrescreenSettings", + "PrescreenContractMeta", + "ExtractionResult", + "MetadataExtractor", + "extract_contract_meta", + "HeuristicExtractor", + "HybridMetaExtractor", + "LLMPrescreenExtractor", + "PrescreenRouter", + "PrescreenHandler", +] diff --git a/src/contract_check/worker_prescreen/__main__.py b/src/contract_check/worker_prescreen/__main__.py new file mode 100644 index 0000000..90d413a --- /dev/null +++ b/src/contract_check/worker_prescreen/__main__.py @@ -0,0 +1,62 @@ +"""worker-prescreen entrypoint: connects to RabbitMQ and runs the prescreen consumer.""" + +from __future__ import annotations + +import asyncio +import signal + +from ..core.config import get_settings +from ..core.logging import bind_context, configure_logging, get_logger +from ..core.metrics import start_metrics_server +from ..core.sentry import init_sentry +from ..core.telemetry import setup_telemetry, shutdown_telemetry +from .consumer import PrescreenConsumer + +log = get_logger(__name__) + + +async def main() -> None: + settings = get_settings() + configure_logging( + settings.log_level, + json_output=settings.json_logs, + service="worker-prescreen", + env=settings.env, + ) + bind_context(service="worker-prescreen", env=settings.env) + init_sentry("worker-prescreen") + setup_telemetry("worker-prescreen") + + start_metrics_server(9104) + + consumer = PrescreenConsumer( + url=settings.rabbitmq_url, + origin="worker-prescreen", + prefetch=1, + max_attempts=settings.mq_max_attempts, + retry_base_ms=settings.mq_retry_base_ms, + ) + await consumer.connect() + + loop = asyncio.get_running_loop() + stop_event = asyncio.Event() + + for sig in (signal.SIGINT, signal.SIGTERM): + loop.add_signal_handler(sig, stop_event.set) + + consumer_task = asyncio.create_task(consumer.run()) + stop_task = asyncio.create_task(stop_event.wait()) + + log.info("worker_prescreen_started", prefetch=1) + try: + await asyncio.wait( + {consumer_task, stop_task}, + return_when=asyncio.FIRST_COMPLETED, + ) + finally: + await consumer.stop() + shutdown_telemetry() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/contract_check/worker_prescreen/config.py b/src/contract_check/worker_prescreen/config.py new file mode 100644 index 0000000..faeb234 --- /dev/null +++ b/src/contract_check/worker_prescreen/config.py @@ -0,0 +1,21 @@ +"""Prescreen worker settings (extends the base settings with prescreen tuning).""" + +from __future__ import annotations + +from pydantic import Field + +from ..core.config import Settings + + +class PrescreenSettings(Settings): + """Settings for worker-prescreen. + + Inherits all base env and adds the prescreen tunables from core.config. + They are already declared on `Settings`, so this class just provides a + typed alias used by the worker. + """ + + mq_prefetch_prescreen: int = Field( + default=1, + description="Prefetch count for prescreen.q consumer.", + ) diff --git a/src/contract_check/worker_prescreen/consumer.py b/src/contract_check/worker_prescreen/consumer.py new file mode 100644 index 0000000..5d1b620 --- /dev/null +++ b/src/contract_check/worker_prescreen/consumer.py @@ -0,0 +1,66 @@ +"""worker-prescreen consumer: wires the prescreen handler into the RabbitMQ base.""" + +from __future__ import annotations + +from ..core.db.enums import FailureClass +from ..core.db.session import create_session_factory +from ..core.logging import get_logger +from ..core.metrics import prescreen_duration +from ..core.mq.consumer import Consumer +from ..core.mq.messages import PrescreenRequested +from ..core.mq.topology import RK_ANALYZE +from .handler import PrescreenHandler + +log = get_logger(__name__) + + +class PrescreenConsumer(Consumer[PrescreenRequested]): + """Consumes `prescreen.q`, extracts contract metadata, routes to analyze.q.""" + + queue: str = "prescreen.q" + routing_key: str = "prescreen" + message_model = PrescreenRequested + + def __init__( + self, + url: str, + *, + origin: str, + prefetch: int, + max_attempts: int, + retry_base_ms: int, + ) -> None: + super().__init__( + url, + origin=origin, + prefetch=prefetch, + max_attempts=max_attempts, + retry_base_ms=retry_base_ms, + ) + self._session_factory = create_session_factory() + self._handler = PrescreenHandler( + session_factory=self._session_factory, + publish_routing_key=RK_ANALYZE, + ) + + def classify(self, exc: BaseException) -> FailureClass: + return self._handler.classify(exc) + + @prescreen_duration.labels(decision="unknown").time() + async def handle(self, payload: PrescreenRequested) -> None: + await self._handler.handle(payload) + + async def on_failure( + self, payload: PrescreenRequested, failure_class: FailureClass, attempt: int, error: str + ) -> None: + await self._handler.on_failure(payload, failure_class, attempt, error) + + async def on_dlq( + self, payload: PrescreenRequested, failure_class: FailureClass, error: str + ) -> None: + await self._handler.on_terminal_failure(payload, failure_class, error) + + async def stop(self) -> None: + await super().stop() + # Release the hybrid extractor's owned LLM provider, if any. + await self._handler.aclose() diff --git a/src/contract_check/worker_prescreen/extractor.py b/src/contract_check/worker_prescreen/extractor.py new file mode 100644 index 0000000..2658ac0 --- /dev/null +++ b/src/contract_check/worker_prescreen/extractor.py @@ -0,0 +1,333 @@ +"""Prescreen metadata model, extractor protocol, and compatibility shim. + +This module is the shared kernel for the hybrid two-stage extractor described +in docs/PRESCREEN_HYBRID_REFACTOR_PLAN.md: + +- `PrescreenContractMeta` — the 10-field pydantic model (shape unchanged). +- `FIELD_WEIGHTS` / `_score_confidence` — weighted confidence scoring. +- `ExtractionResult` / `MetadataExtractor` — the async contract the handler + consumes (`HybridMetaExtractor` is the default implementation). +- `extract_contract_meta` — synchronous delegating shim kept for legacy + callers and tests. + +The v1 regex implementation is retained one release behind the +`PRESCREEN_KEEP_REGEX=true` env flag as a rollback safety net (plan §5). + +Stage 1 lives in `extractor_heuristic.py`, Stage 2 in `extractor_llm.py`, and +the orchestrator in `extractor_hybrid.py`. +""" + +from __future__ import annotations + +import os +import re +from datetime import date +from typing import Protocol, runtime_checkable + +from pydantic import BaseModel, Field, field_validator + + +class PrescreenContractMeta(BaseModel): + """Fields extracted deterministically from a contract.""" + + contract_type: str | None = None + party_a: str | None = None + party_b: str | None = None + total_amount: float | None = None + currency: str | None = None + start_date: str | None = None # ISO YYYY-MM-DD + end_date: str | None = None + has_penalty_clause: bool | None = None + has_termination_clause: bool | None = None + has_arbitration: bool | None = None + confidence_score: float = Field(default=0.0, ge=0.0, le=1.0) + + @field_validator("currency") + @classmethod + def _upper_currency(cls, v: str | None) -> str | None: + return v.upper() if v else None + + +# Weighted field coverage (plan §3.2 weights, nominal sum ≈ 1.0; the scorer +# clamps at 1.0): critical identity fields weigh more than boolean clause +# flags. Tunable with real data later. +FIELD_WEIGHTS: dict[str, float] = { + "contract_type": 0.25, + "party_a": 0.15, + "party_b": 0.15, + "total_amount": 0.15, + "currency": 0.05, + "start_date": 0.10, + "end_date": 0.05, + "has_penalty_clause": 0.05, + "has_termination_clause": 0.05, + "has_arbitration": 0.05, +} + + +def _score_confidence(meta: PrescreenContractMeta) -> float: + """Weighted coverage: sum of weights of non-None fields.""" + total = sum(w for name, w in FIELD_WEIGHTS.items() if getattr(meta, name) is not None) + return round(min(total, 1.0), 3) + + +def _parse_iso(date_str: str | None) -> str | None: + """Parse a dd.mm.yyyy / dd/mm.yyyy string into ISO; None on failure.""" + if not date_str: + return None + date_str = date_str.strip() + for sep in (".", "/"): + if sep in date_str: + parts = date_str.split(sep) + if len(parts) == 3: + try: + d = date(int(parts[2]), int(parts[1]), int(parts[0])) + return d.isoformat() + except ValueError: + return None + return None + + +class ExtractionResult(BaseModel): + """Outcome of a hybrid extraction run. + + `extractor_version` is persisted to prescreen_results.extractor_version: + "heuristic-v2" (LLM not run / disabled / failed) or "hybrid-llm-v1" + (LLM result merged). `llm_fallback_error` is surfaced through + prescreen_results.auto_findings when the fallback failed. + """ + + meta: PrescreenContractMeta + extractor_version: str + llm_fallback_error: str | None = None + + +@runtime_checkable +class MetadataExtractor(Protocol): + """Anything the prescreen handler can call to turn text into meta.""" + + async def extract(self, text: str) -> ExtractionResult: ... + + +# ── legacy regex-v1 implementation (PRESCREEN_KEEP_REGEX=true) ─────────────── + +# Legal entity forms — abbreviated and full — optionally quoted. +_ENTITY_RE = re.compile( + r"((?:" + r"ООО|ОБЩЕСТВО\s+С\s+ОГРАНИЧЕННОЙ\s+ОТВЕТСТВЕННОСТЬЮ|" + r"АО|АКЦИОНЕРНОЕ\s+ОБЩЕСТВО|" + r"ПАО|ПУБЛИЧНОЕ\s+АКЦИОНЕРНОЕ\s+ОБЩЕСТВО|" + r"ЗАО|ЗАКРЫТОЕ\s+АКЦИОНЕРНОЕ\s+ОБЩЕСТВО|" + r"ИП|ИНДИВИДУАЛЬНЫЙ\s+ПРЕДПРИНИМАТЕЛЬ|" + r"ОДО|ОБЩЕСТВО\s+С\s+ДОПОЛНИТЕЛЬНОЙ\s+ОТВЕТСТВЕННОСТЬЮ|" + r"ЧУП|ЧАСТНОЕ\s+УНИТАРНОЕ\s+ПРЕДПРИЯТИЕ" + r")\s*" + r"(?:«[^»]+»|\"[^\"]+\"|[А-ЯA-Z][А-Яа-яA-Za-z\-]+(?:\s+[А-ЯA-Z][а-яa-z]+)?)?)", + re.IGNORECASE, +) + +_CONTRACT_TYPE_PATTERNS: list[tuple[str, re.Pattern[str]]] = [ + ( + "supply", + re.compile( + r"договор\s+(?:купли[-\s]продажи|поставки)", + re.IGNORECASE, + ), + ), + ( + "services", + re.compile( + r"договор\s+(?:оказания\s+услуг|возмездного\s+оказания\s+услуг)", + re.IGNORECASE, + ), + ), + ( + "contract_work", + re.compile( + r"договор\s+(?:подряда|строительного\s+подряда)", + re.IGNORECASE, + ), + ), + ( + "lease", + re.compile( + r"договор\s+(?:аренды|лизинга|субаренды)", + re.IGNORECASE, + ), + ), + ( + "nda", + re.compile( + r"договор\s+о\s+(?:неразглашении|конфиденциальности)", + re.IGNORECASE, + ), + ), +] + +_AMOUNT_RE = re.compile( + r"(?:" + r"(?:составляет|цен[ае]|стоимостью|оплат[ае]?|общая\s+стоимость|сумма)" + r"\s*)" + r"(?:(\d{1,3}(?:\s?\d{3})+)|(\d+(?:[.,]\d+)?))\s*" + r"(?:руб(?:лей|ля)?|₽|RUB|BYN|Br|USD|\$|EUR|€)?", + re.IGNORECASE, +) + +_CURRENCY_NORMALIZE: dict[str, str] = { + "руб": "RUB", + "рублей": "RUB", + "рубля": "RUB", + "₽": "RUB", + "byn": "BYN", + "br": "BYN", + "usd": "USD", + "$": "USD", + "eur": "EUR", + "€": "EUR", +} + +_DATE_RE = re.compile( + r"(?:действует\s+)?(?:с|от)\s+(\d{2}[./]\d{2}[./]\d{4})" + r"(?:.*?по\s+(\d{2}[./]\d{2}[./]\d{4}))?", + re.IGNORECASE, +) +_END_DATE_RE = re.compile( + r"(?:по|до)\s+(\d{2}[./]\d{2}[./]\d{4})", + re.IGNORECASE, +) +_PENALTY_RE = re.compile( + r"(?:неустойка|штраф|пеня|0[,.]\d+%|\d+%\s+за\s+кажды?й?\s+день\s+просрочки)", + re.IGNORECASE, +) +_TERMINATION_RE = re.compile( + r"(?:расторгнуть|расторжение|односторонн\w+\s+порядке|отказаться\s+от\s+исполнения)", + re.IGNORECASE, +) +_ARBITRATION_RE = re.compile( + r"(?:арбитражн\w+\s+суд|международн\w+\s+коммерческ\w+\s+арбитражн\w+\s+суд)", + re.IGNORECASE, +) + +LEGACY_EXTRACTOR_VERSION = "regex-v1" + +# Fields that count toward confidence coverage (legacy flat scoring). +_META_FIELDS: tuple[str, ...] = ( + "contract_type", + "party_a", + "party_b", + "total_amount", + "currency", + "start_date", + "end_date", + "has_penalty_clause", + "has_termination_clause", + "has_arbitration", +) + + +def _extract_contract_type_regex(text: str) -> str | None: + for name, pattern in _CONTRACT_TYPE_PATTERNS: + if pattern.search(text): + return name + return None + + +def _extract_parties_regex(text: str) -> tuple[str | None, str | None]: + matches = _ENTITY_RE.findall(text) + if not matches: + return None, None + # Clean whitespace from each match; drop empty/over-generic matches. + cleaned = [] + for m in matches: + s = m.strip() + if not s or s.lower() in {"ооо", "ао", "пао", "зао", "ип", "одо", "чуп"}: + continue + cleaned.append(s) + if len(cleaned) >= 2: + return cleaned[0], cleaned[1] + return (cleaned[0], None) if cleaned else (None, None) + + +def _extract_amount_regex(text: str) -> tuple[float | None, str | None]: + for match in _AMOUNT_RE.finditer(text): + group = match.group(1) or match.group(2) + if not group: + continue + raw = group.replace(" ", "").replace(",", ".") + try: + value = float(raw) + except ValueError: + continue + # Currency symbol/word within the next ~50 chars after the number. + tail = text[match.end() : match.end() + 50] + tail_lower = tail.lower() + currency = None + for token, code in _CURRENCY_NORMALIZE.items(): + if token.lower() in tail_lower or token.lower() in match.group(0).lower(): + currency = code + break + # Default to RUB if no currency token was found but the trigger was Russian. + if currency is None and any(tok in match.group(0).lower() for tok in ("руб", "₽")): + currency = "RUB" + return value, currency + return None, None + + +def _extract_dates_regex(text: str) -> tuple[str | None, str | None]: + for match in _DATE_RE.finditer(text): + start = _parse_iso(match.group(1)) + end = _parse_iso(match.group(2)) if match.group(2) else None + return start, end + # No start date — maybe only an end date is mentioned. + end_match = _END_DATE_RE.search(text) + if end_match: + return None, _parse_iso(end_match.group(1)) + return None, None + + +def _extract_contract_meta_regex(text: str) -> PrescreenContractMeta: + """regex-v1 implementation, kept behind PRESCREEN_KEEP_REGEX (plan §5).""" + text = text.replace("\n", " ") + party_a, party_b = _extract_parties_regex(text) + amount, currency = _extract_amount_regex(text) + start, end = _extract_dates_regex(text) + + meta = PrescreenContractMeta( + contract_type=_extract_contract_type_regex(text), + party_a=party_a, + party_b=party_b, + total_amount=amount, + currency=currency, + start_date=start, + end_date=end, + has_penalty_clause=None if not text else _PENALTY_RE.search(text) is not None, + has_termination_clause=None if not text else _TERMINATION_RE.search(text) is not None, + has_arbitration=None if not text else _ARBITRATION_RE.search(text) is not None, + ) + + matched = sum(1 for field in _META_FIELDS if getattr(meta, field) is not None) + meta.confidence_score = round(matched / len(_META_FIELDS), 3) if _META_FIELDS else 0.0 + return meta + + +def _keep_regex() -> bool: + return os.environ.get("PRESCREEN_KEEP_REGEX", "").strip().lower() in {"1", "true", "yes", "on"} + + +def extract_contract_meta(text: str) -> PrescreenContractMeta: + """Synchronous shim: heuristic-v2 by default, regex-v1 behind the flag.""" + if _keep_regex(): + return _extract_contract_meta_regex(text) + from .extractor_heuristic import HeuristicExtractor + + return HeuristicExtractor().extract(text) + + +__all__ = [ + "ExtractionResult", + "FIELD_WEIGHTS", + "LEGACY_EXTRACTOR_VERSION", + "MetadataExtractor", + "PrescreenContractMeta", + "extract_contract_meta", +] diff --git a/src/contract_check/worker_prescreen/extractor_heuristic.py b/src/contract_check/worker_prescreen/extractor_heuristic.py new file mode 100644 index 0000000..7418ae8 --- /dev/null +++ b/src/contract_check/worker_prescreen/extractor_heuristic.py @@ -0,0 +1,401 @@ +"""Stage 1 — heuristic deterministic extractor (pure string operations, no regex). + +Keyword dictionaries, positional windows, sentence scanning, and weighted +confidence — see docs/PRESCREEN_HYBRID_REFACTOR_PLAN.md §3.2. Zero marginal +cost, no hallucination risk; low-confidence results fall through to the LLM +stage (extractor_hybrid.py). +""" + +from __future__ import annotations + +from ..core.logging import get_logger +from .extractor import PrescreenContractMeta, _parse_iso, _score_confidence + +log = get_logger(__name__) + +EXTRACTOR_VERSION = "heuristic-v2" + +# ── dictionaries (module-level constants) ──────────────────────────────────── + +_CONTRACT_TYPE_PHRASES: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("supply", ("договор поставки", "договор купли-продажи", "договор купли продажи")), + ("services", ("договор возмездного оказания услуг", "договор оказания услуг")), + ("contract_work", ("договор строительного подряда", "договор подряда")), + ("lease", ("договор субаренды", "договор аренды", "договор лизинга")), + ("nda", ("договор о неразглашении", "договор о конфиденциальности")), +) + +# Legal entity forms (lowercase); longest-first so the scanner is greedy. +_ENTITY_FORMS: tuple[str, ...] = ( + "общество с ограниченной ответственностью", + "общество с дополнительной ответственностью", + "публичное акционерное общество", + "закрытое акционерное общество", + "акционерное общество", + "частное унитарное предприятие", + "индивидуальный предприниматель", + "ооо", + "пао", + "зао", + "одо", + "чуп", + "ао", + "ип", +) + +# Amount trigger words (lowercase substrings; a digit run must follow). +_AMOUNT_TRIGGERS: tuple[str, ...] = ( + "общая стоимость", + "составляет", + "стоимостью", + "цена", + "цене", + "оплат", + "сумма", +) + +_CURRENCY_TOKENS: dict[str, str] = { + "руб": "RUB", + "рублей": "RUB", + "рубля": "RUB", + "₽": "RUB", + "byn": "BYN", + "br": "BYN", + "usd": "USD", + "$": "USD", + "eur": "EUR", + "€": "EUR", +} + +# Boolean-clause keyword stems (lowercase substring membership per sentence). +_PENALTY_KEYWORDS: tuple[str, ...] = ("неустойк", "штраф", "пеня", "пени") +_TERMINATION_KEYWORDS: tuple[str, ...] = ("расторг", "односторонн", "отказаться от исполнения") +_ARBITRATION_KEYWORDS: tuple[str, ...] = ("арбитражн",) + +_HEADER_CAP_CHARS = 1500 +_CURRENCY_TAIL_CHARS = 50 +_MAX_PARTY_NAME_TOKENS = 8 +_MAX_PARTIES = 2 + +_SPACES = " \t\u00a0\u202f" +_QUOTE_PAIRS = {"«": "»", "“": "”", '"': '"'} + + +# ── text helpers ───────────────────────────────────────────────────────────── + + +def _collapse(text: str) -> str: + """Collapse every whitespace run (incl. newlines and NBSP) to one space.""" + return " ".join(text.split()) + + +def _skip_spaces(s: str, i: int) -> int: + while i < len(s) and s[i] in _SPACES: + i += 1 + return i + + +def _split_sentences(text: str) -> list[str]: + """Sentence segmentation via str.split on '. ' and newlines (plan §3.2).""" + parts: list[str] = [] + for chunk in text.split("\n"): + parts.extend(chunk.split(". ")) + return parts + + +def _find_closing_quote(s: str, open_idx: int) -> int: + close = _QUOTE_PAIRS.get(s[open_idx], s[open_idx]) + return s.find(close, open_idx + 1) + + +# ── contract type ──────────────────────────────────────────────────────────── + + +def _extract_contract_type(lower: str) -> str | None: + """Exact phrase dictionary match on the normalized text.""" + for name, phrases in _CONTRACT_TYPE_PHRASES: + for phrase in phrases: + if phrase in lower: + return name + return None + + +# ── parties (positional: header window only) ───────────────────────────────── + + +def _header_window(text: str) -> str: + """Text up to the first numbered/ПРЕДМЕТ heading, capped at ~1500 chars.""" + lines: list[str] = [] + size = 0 + for line in text.splitlines(): + stripped = line.strip() + lower = stripped.lower() + if lines and (lower.startswith("1.") or lower.startswith("1 ") or "предмет" in lower): + break + lines.append(line) + size += len(line) + 1 + if size >= _HEADER_CAP_CHARS: + break + return "\n".join(lines) + + +def _is_name_token(token: str) -> bool: + if not token: + return False + # Initials like «И.И.» / «П.» + if len(token) <= 3 and token[0].isupper() and token.endswith("."): + return True + return token[0].isupper() + + +def _capture_party_name(header: str, start: int, form_end: int) -> tuple[str, int] | None: + """Capture a party name after an entity-form token at `start`. + + Quoted names run from the form through the closing quote; unquoted names + consume following Capitalized words until a lowercase word, punctuation, + or line end. + """ + j = _skip_spaces(header, form_end) + if j < len(header) and header[j] in _QUOTE_PAIRS: + close = _find_closing_quote(header, j) + if close != -1: + return header[start : close + 1].strip(), close + 1 + + end = form_end + tokens = 0 + j = _skip_spaces(header, form_end) + while j < len(header) and tokens < _MAX_PARTY_NAME_TOKENS: + line_end = header.find("\n", j) + segment_end = line_end if line_end != -1 else len(header) + k = j + while k < segment_end and header[k] not in _SPACES: + k += 1 + token = header[j:k] + if not _is_name_token(token): + break + end = k + tokens += 1 + j = _skip_spaces(header, k) + if j >= len(header) or header[j] == "\n": + break + name = header[start:end].strip() + if not name: + return None + return name, max(end, form_end) + + +def _extract_parties(header: str) -> tuple[str | None, str | None]: + """Token-scan the header window for entity forms; capture party names.""" + lower = header.lower() + n = len(header) + found: list[str] = [] + i = 0 + while i < n and len(found) < _MAX_PARTIES: + form: str | None = None + for candidate in _ENTITY_FORMS: + end = i + len(candidate) + if not lower.startswith(candidate, i): + continue + before_ok = i == 0 or not lower[i - 1].isalnum() + after_ok = end >= n or not lower[end].isalnum() + if before_ok and after_ok: + form = candidate + break + if form is None: + i += 1 + continue + captured = _capture_party_name(header, i, i + len(form)) + if captured is None: + i += len(form) + continue + name, capture_end = captured + # Drop bare forms without a name (e.g. a lone «ООО»). + if name.lower() != form: + if name not in found: + found.append(name) + i = max(capture_end, i + len(form)) + if len(found) >= 2: + return found[0], found[1] + if len(found) == 1: + return found[0], None + return None, None + + +# ── amount + currency (trigger-word scan + manual digit parse) ─────────────── + + +def _scan_number(s: str, i: int) -> tuple[float, int] | None: + """Walk chars at s[i] parsing digits, space-separated thousands, decimals.""" + n = len(s) + j = i + int_part: list[str] = [] + dec_part: list[str] = [] + while j < n and s[j].isdigit(): + int_part.append(s[j]) + j += 1 + if not int_part: + return None + while j < n: + ch = s[j] + if ch in ".," and j + 1 < n and s[j + 1].isdigit() and not dec_part: + j += 1 + while j < n and s[j].isdigit(): + dec_part.append(s[j]) + j += 1 + break # a decimal part terminates the number + if ch in _SPACES: + # Thousands separator: space + exactly 3 digits, not part of a longer run. + k = j + 1 + group = "" + while k < n and s[k].isdigit() and len(group) < 3: + group += s[k] + k += 1 + if len(group) == 3 and (k >= n or not s[k].isdigit()): + int_part.append(group) + j = k + continue + break + raw = "".join(int_part) + ("." + "".join(dec_part) if dec_part else "") + return float(raw), j + + +def _find_currency(lower: str, start: int, end: int) -> str | None: + window = lower[start : min(end, len(lower))] + for token, code in _CURRENCY_TOKENS.items(): + if token in window: + return code + return None + + +def _find_earliest_trigger(lower: str, start: int) -> tuple[int, int] | None: + best: tuple[int, int] | None = None + for trigger in _AMOUNT_TRIGGERS: + idx = lower.find(trigger, start) + if idx != -1 and (best is None or idx < best[0]): + best = (idx, idx + len(trigger)) + return best + + +def _extract_amount(collapsed: str, lower: str) -> tuple[float | None, str | None]: + pos = 0 + while True: + hit = _find_earliest_trigger(lower, pos) + if hit is None: + return None, None + trig_start, trig_end = hit + j = _skip_spaces(collapsed, trig_end) + if j < len(collapsed) and collapsed[j].isdigit(): + parsed = _scan_number(collapsed, j) + if parsed is not None: + value, num_end = parsed + return value, _find_currency(lower, trig_start, num_end + _CURRENCY_TAIL_CHARS) + pos = trig_start + 1 + + +# ── dates (trigger tokens + manual dd.mm.yyyy splitter) ────────────────────── + + +def _parse_ddmmyyyy(s: str, i: int) -> str | None: + """Parse dd.mm.yyyy / dd/mm.yyyy at s[i]; None when not a date boundary.""" + if i > 0 and s[i - 1].isdigit(): + return None + if i + 10 > len(s): + return None + if not (s[i].isdigit() and s[i + 1].isdigit()): + return None + if s[i + 2] not in "./": + return None + if not (s[i + 3].isdigit() and s[i + 4].isdigit()): + return None + if s[i + 5] not in "./": + return None + if not all(s[k].isdigit() for k in (i + 6, i + 7, i + 8, i + 9)): + return None + if i + 10 < len(s) and s[i + 10].isdigit(): + return None + return _parse_iso(s[i : i + 10]) + + +def _find_date_after_token(s: str, token: str, start: int) -> tuple[str, int, int] | None: + """Find `token` as a standalone word followed by a dd.mm.yyyy date. + + Returns (iso_date, date_start, scan_end) or None. + """ + pos = start + n = len(s) + while pos < n: + idx = s.find(token, pos) + if idx == -1: + return None + before_ok = idx == 0 or not s[idx - 1].isalnum() + j = idx + len(token) + if before_ok and j < n and s[j] in _SPACES: + j = _skip_spaces(s, j) + iso = _parse_ddmmyyyy(s, j) + if iso is not None: + return iso, j, j + 10 + pos = idx + 1 + return None + + +def _extract_dates(lower_collapsed: str) -> tuple[str | None, str | None]: + start_hit = None + for token in ("с", "от"): + hit = _find_date_after_token(lower_collapsed, token, 0) + if hit is not None and (start_hit is None or hit[1] < start_hit[1]): + start_hit = hit + if start_hit is not None: + # v1 semantics: end date = «по » anywhere after the start date. + end_hit = _find_date_after_token(lower_collapsed, "по", start_hit[2]) + return start_hit[0], end_hit[0] if end_hit is not None else None + for token in ("по", "до"): + hit = _find_date_after_token(lower_collapsed, token, 0) + if hit is not None: + return None, hit[0] + return None, None + + +# ── boolean clauses (sentence scanning) ────────────────────────────────────── + + +def _has_clause(sentences: list[str], keywords: tuple[str, ...]) -> bool: + return any(keyword in sentence for sentence in sentences for keyword in keywords) + + +# ── Stage 1 extractor ──────────────────────────────────────────────────────── + + +class HeuristicExtractor: + """Deterministic keyword/positional extractor (Stage 1, sync, zero cost).""" + + extractor_version = EXTRACTOR_VERSION + + def extract(self, text: str) -> PrescreenContractMeta: + collapsed = _collapse(text) + lower = collapsed.lower() + header = _header_window(text) + sentences = [s.lower() for s in _split_sentences(text)] if text else [] + + party_a, party_b = _extract_parties(header) + amount, currency = _extract_amount(collapsed, lower) + start_date, end_date = _extract_dates(lower) + + meta = PrescreenContractMeta( + contract_type=_extract_contract_type(lower), + party_a=party_a, + party_b=party_b, + total_amount=amount, + currency=currency, + start_date=start_date, + end_date=end_date, + has_penalty_clause=None if not text else _has_clause(sentences, _PENALTY_KEYWORDS), + has_termination_clause=( + None if not text else _has_clause(sentences, _TERMINATION_KEYWORDS) + ), + has_arbitration=None if not text else _has_clause(sentences, _ARBITRATION_KEYWORDS), + ) + meta.confidence_score = _score_confidence(meta) + return meta + + +__all__ = ["EXTRACTOR_VERSION", "HeuristicExtractor"] diff --git a/src/contract_check/worker_prescreen/extractor_hybrid.py b/src/contract_check/worker_prescreen/extractor_hybrid.py new file mode 100644 index 0000000..3098d88 --- /dev/null +++ b/src/contract_check/worker_prescreen/extractor_hybrid.py @@ -0,0 +1,149 @@ +"""Hybrid two-stage orchestrator for prescreen metadata extraction. + +Stage 1 — HeuristicExtractor (sync, via to_thread): deterministic, zero cost. +Stage 2 — LLMPrescreenExtractor: runs only when the kill-switch is off AND the +heuristic confidence is below `prescreen_llm_fallback_threshold`. + +Failure semantics (plan §3.4): an LLM fallback error never fails the message — +the heuristic meta is kept, `extractor_version="heuristic-v2"`, and the error +is recorded on the result for `prescreen_results.auto_findings` plus the +`prescreen_fallback_runs_total{outcome="failed"}` counter. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +from ..core.config import get_settings +from ..core.llm.factory import build_llm_provider +from ..core.llm.port import LLMProvider +from ..core.logging import get_logger +from ..core.metrics import prescreen_extraction_stage, prescreen_fallback_runs +from .extractor import ExtractionResult, PrescreenContractMeta, _score_confidence +from .extractor_heuristic import EXTRACTOR_VERSION as HEURISTIC_VERSION +from .extractor_heuristic import HeuristicExtractor +from .extractor_llm import EXTRACTOR_VERSION as HYBRID_LLM_VERSION +from .extractor_llm import LLMPrescreenExtractor + +log = get_logger(__name__) + +_SCALAR_FIELDS = ( + "contract_type", + "party_a", + "party_b", + "total_amount", + "currency", + "start_date", + "end_date", +) +_BOOL_FIELDS = ( + "has_penalty_clause", + "has_termination_clause", + "has_arbitration", +) + + +def _merge(heuristic: PrescreenContractMeta, llm: PrescreenContractMeta) -> PrescreenContractMeta: + """LLM values override heuristic Nones; boolean clauses OR-merge. + + Booleans from both stages are presence-checks — false positives are cheap, + false negatives route wrong (plan §3.3). + """ + update: dict[str, Any] = {} + for field in _SCALAR_FIELDS: + llm_value = getattr(llm, field) + if llm_value is not None: + update[field] = llm_value + for field in _BOOL_FIELDS: + h_value, l_value = getattr(heuristic, field), getattr(llm, field) + if h_value is None: + if l_value is not None: + update[field] = l_value + elif l_value is not None: + update[field] = bool(h_value or l_value) + + merged = heuristic.model_copy(update=update) + merged.confidence_score = _score_confidence(merged) + return merged + + +class HybridMetaExtractor: + """Default MetadataExtractor: heuristic first, LLM only when confidence is low.""" + + def __init__( + self, + *, + provider: LLMProvider | None = None, + fallback_enabled: bool | None = None, + fallback_threshold: float | None = None, + heuristic: HeuristicExtractor | None = None, + ) -> None: + settings = get_settings() + self._heuristic = heuristic or HeuristicExtractor() + self._fallback_enabled = ( + settings.prescreen_llm_fallback_enabled + if fallback_enabled is None + else fallback_enabled + ) + self._fallback_threshold = ( + settings.prescreen_llm_fallback_threshold + if fallback_threshold is None + else fallback_threshold + ) + self._provider = provider + self._owns_provider = provider is None + self._llm: LLMPrescreenExtractor | None = None + + async def _llm_extractor(self) -> LLMPrescreenExtractor: + if self._llm is None: + if self._provider is None: + self._provider = build_llm_provider(get_settings()) + self._llm = LLMPrescreenExtractor(self._provider) + return self._llm + + async def aclose(self) -> None: + if self._provider is not None and self._owns_provider: + await self._provider.aclose() + + async def extract(self, text: str) -> ExtractionResult: + with prescreen_extraction_stage.labels(stage="heuristic").time(): + meta = await asyncio.to_thread(self._heuristic.extract, text) + + if not self._fallback_enabled: + prescreen_fallback_runs.labels(outcome="disabled").inc() + return ExtractionResult(meta=meta, extractor_version=HEURISTIC_VERSION) + + if meta.confidence_score >= self._fallback_threshold: + prescreen_fallback_runs.labels(outcome="skipped").inc() + return ExtractionResult(meta=meta, extractor_version=HEURISTIC_VERSION) + + try: + llm_extractor = await self._llm_extractor() + with prescreen_extraction_stage.labels(stage="llm").time(): + llm_meta = await llm_extractor.extract(text) + except Exception as exc: # noqa: BLE001 — never fail the message (plan §3.4) + prescreen_fallback_runs.labels(outcome="failed").inc() + error = f"{type(exc).__name__}: {exc}" + log.warning( + "prescreen_llm_fallback_failed", + error=error, + heuristic_confidence=meta.confidence_score, + ) + return ExtractionResult( + meta=meta, + extractor_version=HEURISTIC_VERSION, + llm_fallback_error=error, + ) + + prescreen_fallback_runs.labels(outcome="used").inc() + merged = _merge(meta, llm_meta) + log.info( + "prescreen_llm_fallback_merged", + heuristic_confidence=meta.confidence_score, + merged_confidence=merged.confidence_score, + ) + return ExtractionResult(meta=merged, extractor_version=HYBRID_LLM_VERSION) + + +__all__ = ["HybridMetaExtractor"] diff --git a/src/contract_check/worker_prescreen/extractor_llm.py b/src/contract_check/worker_prescreen/extractor_llm.py new file mode 100644 index 0000000..38e56b9 --- /dev/null +++ b/src/contract_check/worker_prescreen/extractor_llm.py @@ -0,0 +1,100 @@ +"""Stage 2 — LLM fallback extractor: provider dict → validated contract meta. + +The provider layer returns a plain, permissive dict (core/llm/prescreen.py). +This wrapper whitelists contract_type enums, re-`None`s hallucinated or +malformed values (numeric/date parse checks), and recomputes the weighted +confidence — pydantic model stays in worker_prescreen (plan §3.3). +""" + +from __future__ import annotations + +from datetime import date +from typing import TYPE_CHECKING, Any + +from ..core.logging import get_logger +from .extractor import PrescreenContractMeta, _parse_iso, _score_confidence + +if TYPE_CHECKING: + from ..core.llm.port import LLMProvider + +log = get_logger(__name__) + +EXTRACTOR_VERSION = "hybrid-llm-v1" + +VALID_CONTRACT_TYPES = frozenset({"supply", "services", "contract_work", "lease", "nda"}) +VALID_CURRENCIES = frozenset({"RUB", "BYN", "USD", "EUR"}) + + +def _clean_str(value: Any) -> str | None: + if not isinstance(value, str): + return None + stripped = value.strip() + return stripped or None + + +def _clean_amount(value: Any) -> float | None: + if isinstance(value, bool) or not isinstance(value, int | float): + return None + amount = float(value) + return amount if amount >= 0 else None + + +def _clean_date(value: Any) -> str | None: + raw = _clean_str(value) + if raw is None: + return None + try: + return date.fromisoformat(raw).isoformat() + except ValueError: + return _parse_iso(raw) # accept dd.mm.yyyy / dd/mm.yyyy too + + +def _clean_bool(value: Any) -> bool | None: + return value if isinstance(value, bool) else None + + +def validate_raw(raw: dict[str, Any]) -> PrescreenContractMeta: + """Whitelist/normalize a raw LLM dict; hallucinated values become None.""" + contract_type = _clean_str(raw.get("contract_type")) + currency = _clean_str(raw.get("currency")) + contract_type_normalized = contract_type.lower() if contract_type else None + currency_normalized = currency.upper() if currency else None + + meta = PrescreenContractMeta( + contract_type=( + contract_type_normalized if contract_type_normalized in VALID_CONTRACT_TYPES else None + ), + party_a=_clean_str(raw.get("party_a")), + party_b=_clean_str(raw.get("party_b")), + total_amount=_clean_amount(raw.get("total_amount")), + currency=(currency_normalized if currency_normalized in VALID_CURRENCIES else None), + start_date=_clean_date(raw.get("start_date")), + end_date=_clean_date(raw.get("end_date")), + has_penalty_clause=_clean_bool(raw.get("has_penalty_clause")), + has_termination_clause=_clean_bool(raw.get("has_termination_clause")), + has_arbitration=_clean_bool(raw.get("has_arbitration")), + ) + meta.confidence_score = _score_confidence(meta) + return meta + + +class LLMPrescreenExtractor: + """MetadataExtractor over an LLMProvider's `extract_prescreen`.""" + + extractor_version = EXTRACTOR_VERSION + + def __init__(self, provider: LLMProvider) -> None: + self._provider = provider + + async def extract(self, text: str) -> PrescreenContractMeta: + raw = await self._provider.extract_prescreen(text) + meta = validate_raw(raw) + log.info( + "prescreen_llm_extraction_done", + confidence=meta.confidence_score, + contract_type=meta.contract_type, + ) + return meta + + +__all__ = ["EXTRACTOR_VERSION", "LLMPrescreenExtractor", "validate_raw"] diff --git a/src/contract_check/worker_prescreen/handler.py b/src/contract_check/worker_prescreen/handler.py new file mode 100644 index 0000000..dac3bbb --- /dev/null +++ b/src/contract_check/worker_prescreen/handler.py @@ -0,0 +1,614 @@ +"""Prescreen worker handler: download extracted Markdown → hybrid extract → route → publish. + +The handler is intentionally separate from the consumer so it can be tested +in-process without spinning up a real RabbitMQ consumer. It owns: + - idempotency checks against documents.status + - status transitions (extracting → prescreening → analyzing | manual_review | done) + - MinIO download of the extracted Markdown + - hybrid contract metadata extraction (heuristic Stage 1 + optional LLM Stage 2) + - routing decision + persisting prescreen_results (incl. extractor_version) + - publishing the next message (deep_analysis → analyze.q, auto_approve → + report.completed, manual_review → a review queue marker) + - updating the jobs row, recording failure class, and refund-on-DLQ. +""" + +from __future__ import annotations + +import json +import time +import uuid +from datetime import UTC, date, datetime +from typing import TYPE_CHECKING + +from sqlalchemy import text + +from ..core.config import get_settings +from ..core.credits import refund_credit +from ..core.db.enums import DOC_TERMINAL, FailureClass +from ..core.logging import ( + Timer, + get_logger, + log_db_operation, + log_external_call, + log_message_processing, + log_performance_metric, + log_stage_complete, + log_stage_failure, + log_stage_progress, +) +from ..core.metrics import ( + mq_failed, + mq_published, + prescreen_confidence, + prescreen_duration, + prescreen_runs, +) +from ..core.mq.messages import AnalyzeRequested, PrescreenCompleted, PrescreenRequested +from ..core.mq.publisher import Publisher +from ..core.mq.topology import RK_ANALYZE +from ..core.s3.minio_storage import MinioStorage +from .extractor import ExtractionResult, MetadataExtractor +from .extractor_hybrid import HybridMetaExtractor +from .router import PrescreenRouter, RoutingDecision + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + from .extractor import PrescreenContractMeta + +log = get_logger(__name__) + + +def _as_date(value: str | None) -> date | None: + """prescreen_results date columns are DATE; meta carries ISO strings.""" + return date.fromisoformat(value) if value else None + + +class PrescreenHandler: + """Business logic for worker-prescreen.""" + + def __init__( + self, + *, + session_factory: async_sessionmaker[AsyncSession], + publish_routing_key: str = RK_ANALYZE, + extractor: MetadataExtractor | None = None, + ) -> None: + self._session_factory = session_factory + self._publish_routing_key = publish_routing_key + self._settings = get_settings() + self._storage = MinioStorage.from_endpoint_url( + endpoint_url=self._settings.s3_endpoint_url, + access_key=self._settings.s3_access_key, + secret_key=self._settings.s3_secret_key, + bucket=self._settings.s3_bucket, + region=self._settings.s3_region, + ) + self._router = PrescreenRouter( + confidence_threshold=self._settings.prescreen_confidence_threshold, + high_value_threshold=self._settings.prescreen_high_value_threshold, + auto_approve_enabled=self._settings.prescreen_auto_approve, + ) + self._extractor: MetadataExtractor = extractor or HybridMetaExtractor() + self._publisher: Publisher | None = None + + async def aclose(self) -> None: + """Release owned resources (LLM provider behind the hybrid extractor).""" + if isinstance(self._extractor, HybridMetaExtractor): + await self._extractor.aclose() + + async def _publisher_instance(self) -> Publisher: + if self._publisher is None: + self._publisher = Publisher(self._settings.rabbitmq_url, origin="worker-prescreen") + await self._publisher.connect() + return self._publisher + + async def _db_status(self, session: AsyncSession, document_id: uuid.UUID) -> str | None: + row = await session.execute( + text("SELECT status FROM documents WHERE id = :d FOR UPDATE"), + {"d": document_id}, + ) + result = row.first() + return str(result[0]) if result else None + + async def handle(self, payload: PrescreenRequested) -> None: + started_at = time.perf_counter() + log_message_processing( + log, + "started", + "prescreen.q", + correlation_id=str(payload.correlation_id), + document_id=str(payload.document_id), + user_id=str(payload.user_id), + text_s3_key=payload.text_s3_key, + filename=payload.filename, + attempt=payload.attempt, + ) + + # Stage 1: Database validation + with Timer(log, "prescreen_db_validation", document_id=str(payload.document_id)): + async with self._session_factory() as session: + current = await self._db_status(session, payload.document_id) + if current is None: + log.warning( + "document_not_found", + document_id=str(payload.document_id), + user_id=str(payload.user_id), + text_s3_key=payload.text_s3_key, + ) + return + if current in DOC_TERMINAL: + log.info( + "document_already_terminal", + document_id=str(payload.document_id), + status=current, + ) + return + + # Stage 2: Update document status and job tracking + with Timer(log, "prescreen_status_update", document_id=str(payload.document_id)): + async with self._session_factory() as session: + await session.execute( + text( + "UPDATE documents SET status = 'prescreening', stage = 'extracting_meta' " + "WHERE id = :d" + ), + {"d": payload.document_id}, + ) + await session.execute( + text( + "INSERT INTO jobs (document_id, correlation_id, queue, status) " + "VALUES (:d, :cid, 'prescreen', 'running') " + "ON CONFLICT (document_id, queue) DO NOTHING" + ), + {"d": payload.document_id, "cid": payload.correlation_id}, + ) + await session.execute( + text( + "UPDATE jobs SET status = 'running', attempts = attempts + 1 " + "WHERE document_id = :d AND queue = 'prescreen'" + ), + {"d": payload.document_id}, + ) + await session.commit() + log_db_operation( + log, + "update", + "documents", + document_id=str(payload.document_id), + status="prescreening", + stage="extracting_meta", + ) + + # Stage 3: Download text from S3 + text_bytes: bytes + contract_text: str + with Timer( + log, + "prescreen_text_download", + document_id=str(payload.document_id), + text_s3_key=payload.text_s3_key, + ): + try: + text_bytes = await self._storage.get(payload.text_s3_key) + contract_text = text_bytes.decode("utf-8") + log_external_call( + log, + "minio", + f"get:{payload.text_s3_key}", + bytes_size=len(text_bytes), + ) + except Exception as exc: + log_stage_failure( + log, + "prescreen_text_download", + exc, + document_id=str(payload.document_id), + text_s3_key=payload.text_s3_key, + ) + raise + + log_stage_progress( + log, + "prescreen_text_loaded", + "text_downloaded", + document_id=str(payload.document_id), + text_s3_key=payload.text_s3_key, + char_count=len(contract_text), + ) + + # Stage 4: Extract contract metadata (heuristic Stage 1 + optional LLM Stage 2) + meta: PrescreenContractMeta + extraction: ExtractionResult + with Timer(log, "prescreen_meta_extraction", document_id=str(payload.document_id)): + try: + extraction = await self._extractor.extract(contract_text) + meta = extraction.meta + log_performance_metric( + log, + "extraction_confidence", + meta.confidence_score, + "ratio", + document_id=str(payload.document_id), + extractor_version=extraction.extractor_version, + llm_fallback_error=extraction.llm_fallback_error, + extracted_fields=sum( + 1 for field in self._meta_field_names() if getattr(meta, field) is not None + ), + total_fields=len(self._meta_field_names()), + ) + except Exception as exc: + log_stage_failure( + log, "prescreen_meta_extraction", exc, document_id=str(payload.document_id) + ) + raise + + # Stage 5: Make routing decision + decision: RoutingDecision + with Timer(log, "prescreen_routing_decision", document_id=str(payload.document_id)): + decision = self._router.decide(meta) + log_stage_progress( + log, + "prescreen_routing_decision", + "decision_made", + document_id=str(payload.document_id), + decision=decision, + confidence=meta.confidence_score, + ) + + processing_ms = int((time.perf_counter() - started_at) * 1000) + + prescreened_at = datetime.now(UTC) + prescreened_at_str = prescreened_at.isoformat() + completed = PrescreenCompleted( + correlation_id=payload.correlation_id, + document_id=payload.document_id, + user_id=payload.user_id, + text_s3_key=payload.text_s3_key, + filename=payload.filename, + prescreened_at=prescreened_at_str, + contract_type=meta.contract_type, + party_a=meta.party_a, + party_b=meta.party_b, + total_amount=meta.total_amount, + currency=meta.currency, + start_date=meta.start_date, + end_date=meta.end_date, + has_penalty_clause=meta.has_penalty_clause, + has_termination_clause=meta.has_termination_clause, + has_arbitration=meta.has_arbitration, + confidence_score=meta.confidence_score, + routing_decision=decision, + auto_summary=self._router.summarize(meta, decision) + if decision == "auto_approve" + else None, + # A failed LLM fallback is recorded here (plan §3.4) — never fatal. + auto_findings=[{"llm_fallback_error": extraction.llm_fallback_error}] + if extraction.llm_fallback_error + else [], + attempt=payload.attempt, + ) + + log_stage_complete( + log, + "prescreen_extraction", + processing_ms, + document_id=str(payload.document_id), + decision=decision, + confidence=meta.confidence_score, + contract_type=meta.contract_type, + extractor_version=extraction.extractor_version, + matched_fields=[ + field for field in self._meta_field_names() if getattr(meta, field) is not None + ], + missing_fields=[ + field for field in self._meta_field_names() if getattr(meta, field) is None + ], + ) + + # Stage 6: Persist prescreen result + prescreen_result_id: uuid.UUID | None = None + with Timer(log, "prescreen_result_persistence", document_id=str(payload.document_id)): + try: + async with self._session_factory() as session: + result = await session.execute( + text( + "INSERT INTO prescreen_results " + "(document_id, correlation_id, contract_type, party_a, party_b, " + " total_amount, currency, start_date, end_date, has_penalty_clause, " + " has_termination_clause, has_arbitration, confidence_score, " + " routing_decision, prescreened_at, processing_ms, auto_summary, " + " auto_findings, extractor_version) " + "VALUES (:d, :cid, :ct, :pa, :pb, :ta, :cur, :sd, :ed, :hpc, :htc, :ha, " + " :cs, :rd, :psa, :ms, :asum, :af, :ev) " + "RETURNING id" + ), + { + "d": payload.document_id, + "cid": payload.correlation_id, + "ct": completed.contract_type, + "pa": completed.party_a, + "pb": completed.party_b, + "ta": completed.total_amount, + "cur": completed.currency, + "sd": _as_date(completed.start_date), + "ed": _as_date(completed.end_date), + "hpc": completed.has_penalty_clause, + "htc": completed.has_termination_clause, + "ha": completed.has_arbitration, + "cs": completed.confidence_score, + "rd": completed.routing_decision, + "psa": prescreened_at, + "ms": processing_ms, + "asum": completed.auto_summary, + "af": json.dumps(completed.auto_findings), + "ev": extraction.extractor_version, + }, + ) + prescreen_result_id = result.scalar() + await session.commit() + log_db_operation( + log, + "insert", + "prescreen_results", + document_id=str(payload.document_id), + prescreen_result_id=str(prescreen_result_id), + ) + except Exception as exc: + log_stage_failure( + log, "prescreen_result_persistence", exc, document_id=str(payload.document_id) + ) + raise + + prescreen_duration.labels(decision=decision).observe(processing_ms / 1000.0) + prescreen_runs.labels( + decision=decision, contract_type=meta.contract_type or "unknown" + ).inc() + prescreen_confidence.observe(meta.confidence_score) + + # Stage 7: Publish next message or complete + publisher = await self._publisher_instance() + next_stage: str + + with Timer( + log, + "prescreen_next_step_routing", + document_id=str(payload.document_id), + decision=decision, + ): + if decision == "deep_analysis": + next_msg = AnalyzeRequested( + correlation_id=payload.correlation_id, + document_id=payload.document_id, + user_id=payload.user_id, + extracted_s3_key=payload.text_s3_key, + filename=payload.filename, + char_count=payload.char_count, + ocr_used=False, + is_structured=payload.is_structured, + has_tables=payload.has_tables, + prescreen_meta=completed.model_dump(), + attempt=payload.attempt, + ) + await publisher.publish(next_msg, routing_key=self._publish_routing_key) + mq_published.labels(queue="analyze").inc() + next_stage = "queued_analyze" + doc_status = "analyzing" + log_stage_progress( + log, + "prescreen_next_step_routing", + "published_to_analyze", + document_id=str(payload.document_id), + routing_key=self._publish_routing_key, + ) + elif decision == "manual_review": + # For now, manual_review is a terminal state persisted in the DB. + # A future admin/web SPA can pick these up. + doc_status = "manual_review" + next_stage = "manual_review" + log_stage_progress( + log, + "prescreen_next_step_routing", + "queued_manual_review", + document_id=str(payload.document_id), + ) + else: # auto_approve + # Auto-approval is disabled by default; if reached, write a lightweight + # report and complete the document. + await self._write_auto_approved_report( + session_factory=self._session_factory, + payload=payload, + completed=completed, + prescreen_result_id=prescreen_result_id, # type: ignore[arg-type] + ) + doc_status = "done" + next_stage = "auto_approved" + log_stage_progress( + log, + "prescreen_next_step_routing", + "auto_approved", + document_id=str(payload.document_id), + ) + + # Stage 8: Final status update + with Timer(log, "prescreen_final_status_update", document_id=str(payload.document_id)): + async with self._session_factory() as session: + await session.execute( + text("UPDATE documents SET status = :status, stage = :stage WHERE id = :d"), + {"status": doc_status, "stage": next_stage, "d": payload.document_id}, + ) + await session.execute( + text( + "UPDATE jobs SET status = 'done' WHERE document_id = :d AND queue = 'prescreen'" + ), + {"d": payload.document_id}, + ) + await session.commit() + log_db_operation( + log, + "update", + "documents", + document_id=str(payload.document_id), + final_status=doc_status, + final_stage=next_stage, + ) + + log_message_processing( + log, + "completed", + "prescreen.q", + correlation_id=str(payload.correlation_id), + document_id=str(payload.document_id), + user_id=str(payload.user_id), + decision=decision, + confidence=meta.confidence_score, + contract_type=meta.contract_type, + processing_ms=processing_ms, + next_stage=next_stage, + prescreen_result_id=str(prescreen_result_id) if prescreen_result_id else None, + ) + + async def _write_auto_approved_report( + self, + *, + session_factory: async_sessionmaker[AsyncSession], + payload: PrescreenRequested, + completed: PrescreenCompleted, + prescreen_result_id: uuid.UUID, + ) -> None: + content: dict[str, object] = { + "findings": [ + { + "title": "Автоматическая проверка договора", + "severity": "low", + "explanation": completed.auto_summary, + "recommendation": "Договор признан низкорисковым. Перед подписанием рекомендуется визуальная сверка.", + "citations": [], + } + ] + } + markdown = ( + f"# Автоматическая проверка: {completed.contract_type or 'договор'}\n\n" + f"{completed.auto_summary}\n\n" + "**Решение:** одобрено автоматически (низкий риск).\n" + "**Рекомендация:** визуальная сверка перед подписанием.\n\n" + "---\n\n" + "*Это автоматический отчёт без полного анализа ИИ.*" + ) + async with session_factory() as session: + await session.execute( + text( + "INSERT INTO reports " + "(document_id, content_json, markdown, model_used, prompt_tokens, " + " eval_tokens, latency_ms, prescreen_result_id, prescreen_meta) " + "VALUES (:d, CAST(:content AS jsonb), :md, :model, 0, 0, 0, :prid, :pm) " + "ON CONFLICT (document_id) DO UPDATE SET " + " content_json = EXCLUDED.content_json, " + " markdown = EXCLUDED.markdown, " + " model_used = EXCLUDED.model_used, " + " prompt_tokens = EXCLUDED.prompt_tokens, " + " eval_tokens = EXCLUDED.eval_tokens, " + " latency_ms = EXCLUDED.latency_ms, " + " prescreen_result_id = EXCLUDED.prescreen_result_id, " + " prescreen_meta = EXCLUDED.prescreen_meta" + ), + { + "d": payload.document_id, + "content": json.dumps(content), + "md": markdown, + "model": "prescreen-auto", + "prid": prescreen_result_id, + "pm": completed.model_dump(), + }, + ) + await session.commit() + + def classify(self, exc: BaseException) -> FailureClass: + from ..core.extraction import UnsupportedFormatError + from ..core.s3.minio_storage import MinioStorageError + + if isinstance(exc, MinioStorageError): + return "infra" + if isinstance(exc, (UnicodeDecodeError, UnsupportedFormatError)): + return "extraction_failed" + return "infra" + + @staticmethod + def _meta_field_names() -> tuple[str, ...]: + return ( + "contract_type", + "party_a", + "party_b", + "total_amount", + "currency", + "start_date", + "end_date", + "has_penalty_clause", + "has_termination_clause", + "has_arbitration", + ) + + async def on_failure( + self, payload: PrescreenRequested, failure_class: FailureClass, attempt: int, error: str + ) -> None: + mq_failed.labels(queue="prescreen", failure_class=failure_class).inc() + log.warning( + "prescreen_retrying", + document_id=str(payload.document_id), + user_id=str(payload.user_id), + attempt=attempt, + failure_class=failure_class, + error=error, + ) + async with self._session_factory() as session: + await session.execute( + text( + "UPDATE jobs SET status = 'retrying', attempts = :a, " + "last_failure_class = :fc, last_error = :err " + "WHERE document_id = :d AND queue = 'prescreen'" + ), + { + "a": attempt, + "fc": failure_class, + "err": error[:1000], + "d": payload.document_id, + }, + ) + await session.commit() + + async def on_terminal_failure( + self, payload: PrescreenRequested, failure_class: FailureClass, error: str + ) -> None: + mq_failed.labels(queue="prescreen", failure_class=failure_class).inc() + async with self._session_factory() as session: + await session.execute( + text( + "UPDATE jobs SET status = 'dlq', dlq = TRUE, " + "last_failure_class = :fc, last_error = :err " + "WHERE document_id = :d AND queue = 'prescreen'" + ), + { + "fc": failure_class, + "err": error[:1000], + "d": payload.document_id, + }, + ) + await session.execute( + text("UPDATE documents SET status = 'failed', stage = :stage WHERE id = :d"), + {"stage": failure_class, "d": payload.document_id}, + ) + await refund_credit( + session, + payload.document_id, + failure_class, + self._settings.refund_policy, + ) + await session.commit() + + log.warning( + "prescreen_terminal_failure", + document_id=str(payload.document_id), + user_id=str(payload.user_id), + failure_class=failure_class, + error=error, + ) diff --git a/src/contract_check/worker_prescreen/router.py b/src/contract_check/worker_prescreen/router.py new file mode 100644 index 0000000..398cdb1 --- /dev/null +++ b/src/contract_check/worker_prescreen/router.py @@ -0,0 +1,67 @@ +"""Routing decision logic for prescreen results. + +Conservative by default: auto_approve is disabled until proven by accuracy tests. +When disabled, any auto_approve decision is remapped to manual_review. +""" + +from __future__ import annotations + +from typing import Literal + +from ..core.logging import get_logger +from .extractor import PrescreenContractMeta + +log = get_logger(__name__) + +type RoutingDecision = Literal["auto_approve", "manual_review", "deep_analysis"] + + +class PrescreenRouter: + """Decide where a prescreened document goes next.""" + + def __init__( + self, + *, + confidence_threshold: float = 0.75, + high_value_threshold: float = 100_000.0, + auto_approve_enabled: bool = False, + ) -> None: + self._confidence_threshold = confidence_threshold + self._high_value_threshold = high_value_threshold + self._auto_approve_enabled = auto_approve_enabled + + def decide(self, meta: PrescreenContractMeta) -> RoutingDecision: + """Return one of auto_approve | manual_review | deep_analysis.""" + confidence = meta.confidence_score + + # Mandatory fields for any automatic routing. + if ( + meta.contract_type is None + or meta.party_a is None + or meta.party_b is None + or confidence < self._confidence_threshold + ): + return "manual_review" + + # High-value or risky clauses → deep LLM analysis. + if ( + (meta.total_amount is not None and meta.total_amount >= self._high_value_threshold) + or meta.has_penalty_clause + or meta.has_arbitration + ): + return "deep_analysis" + + # Conservative default: do not auto-approve until explicitly enabled. + if not self._auto_approve_enabled: + return "manual_review" + + return "auto_approve" + + def summarize(self, meta: PrescreenContractMeta, decision: RoutingDecision) -> str: + """Short human-readable summary for auto_approve reports.""" + return ( + f"{meta.contract_type or 'contract'}: " + f"{meta.party_a or '?'} ↔ {meta.party_b or '?'}, " + f"{meta.total_amount or '—'} {meta.currency or ''} " + f"(confidence {meta.confidence_score:.0%}, routed {decision})." + ) diff --git a/srv/worker-extract/Dockerfile b/srv/worker-extract/Dockerfile index c8a9ab9..b5aad82 100644 --- a/srv/worker-extract/Dockerfile +++ b/srv/worker-extract/Dockerfile @@ -28,12 +28,14 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ WORKDIR /app -# Tesseract and Russian/English language data for scanned PDFs. +# Tesseract and Russian/English language data for scanned PDFs; libmagic1 +# backs python-magic content sniffing in the extraction factory. RUN apt-get update && apt-get install -y --no-install-recommends \ tesseract-ocr \ tesseract-ocr-rus \ tesseract-ocr-eng \ fonts-dejavu-core \ + libmagic1 \ && rm -rf /var/lib/apt/lists/* COPY --from=builder /app/.venv /app/.venv diff --git a/srv/worker-prescreen/Dockerfile b/srv/worker-prescreen/Dockerfile new file mode 100644 index 0000000..376ec7c --- /dev/null +++ b/srv/worker-prescreen/Dockerfile @@ -0,0 +1,34 @@ +# syntax=docker/dockerfile:1 + +# ─── Stage 1: build deps + project into a venv via uv ───────────────────────── +FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim AS builder + +ENV UV_COMPILE_BYTECODE=1 \ + UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=never \ + UV_PROJECT_ENVIRONMENT=/app/.venv + +WORKDIR /app + +COPY pyproject.toml uv.lock ./ +COPY README.md ./ +COPY src ./src + +# Install only the prescreen group (core + db/mq/s3/obs + regex engine). +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --frozen --no-default-groups --group prescreen --no-install-project && \ + uv pip install --no-deps . + +# ─── Stage 2: lean runtime ───────────────────────────────────────────────── +FROM python:3.13-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PATH=/app/.venv/bin:$PATH + +WORKDIR /app + +COPY --from=builder /app/.venv /app/.venv + +EXPOSE 9104 +CMD ["python", "-m", "contract_check.worker_prescreen"] diff --git a/tests/integration/test_b2b_api.py b/tests/integration/test_b2b_api.py index 1059ae5..14153a3 100644 --- a/tests/integration/test_b2b_api.py +++ b/tests/integration/test_b2b_api.py @@ -13,6 +13,7 @@ from pathlib import Path import aio_pika import httpx import pytest + from tests.integration.conftest import user_token pytestmark = pytest.mark.integration diff --git a/tests/integration/test_prescreen_worker.py b/tests/integration/test_prescreen_worker.py new file mode 100644 index 0000000..ad41e39 --- /dev/null +++ b/tests/integration/test_prescreen_worker.py @@ -0,0 +1,283 @@ +"""Integration tests for worker-prescreen (hybrid extraction). + +Uses the same Docker Compose infra as test_analyze_worker: real Postgres, +RabbitMQ, MinIO; the LLM is stubbed at the provider port. + +Scenarios (plan Phase 4): + 1. Heuristic-only (fallback disabled by default): prescreen_results row + persisted with extractor_version='heuristic-v2', routing unchanged. + 2. Fallback enabled + low-confidence text + stub provider: merged meta + persisted with extractor_version='hybrid-llm-v1'. + 3. Fallback failure: heuristic result kept, error recorded in auto_findings. +""" + +from __future__ import annotations + +import json +import uuid +from typing import TYPE_CHECKING, Any + +import pytest +from sqlalchemy import text + +from contract_check.core.db.session import create_session_factory +from contract_check.core.mq.messages import PrescreenRequested +from contract_check.core.s3 import extracted_key +from contract_check.core.s3.minio_storage import MinioStorage +from contract_check.worker_prescreen.extractor_heuristic import HeuristicExtractor +from contract_check.worker_prescreen.extractor_hybrid import HybridMetaExtractor +from contract_check.worker_prescreen.handler import PrescreenHandler + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +pytestmark = pytest.mark.integration + +FULL_CONTRACT = """\ + +ДОГОВОР ПОСТАВКИ № 42 + +г. Москва + +Общество с ограниченной ответственностью «Продавец», +именуемое в дальнейшем «Поставщик», с одной стороны, и +Общество с ограниченной ответственностью «Покупатель», +с другой стороны, заключили настоящий договор. + +1. Предмет договора +Поставщик обязуется передать в собственность Покупателю товар. + +2. Цена и порядок расчётов +2.1. Общая стоимость товара составляет 1 250 000 рублей. + +3. Срок действия договора +3.1. Договор вступает в силу с 01.09.2025 и действует по 31.08.2026. + +4. Ответственность сторон +4.1. За просрочку поставки Поставщик уплачивает неустойку. + +5. Порядок разрешения споров +5.1. Споры подлежат рассмотрению в Арбитражном суде г. Москвы. +""" + +LOW_CONF_TEXT = "Скан документа без распознанных реквизитов." + +# The compose stack may have live workers consuming analyze.q; publish test +# messages to an unrouted key so nothing downstream races our assertions. +_DEAD_ROUTING_KEY = "it-test-unrouted" + + +class StubProvider: + """LLMProvider stub returning a fixed prescreen dict (or raising).""" + + def __init__(self, payload: dict[str, Any] | Exception) -> None: + self._payload = payload + + async def analyze(self, text: str, *, checklist: str, extra_context: str = "") -> Any: + raise NotImplementedError + + async def extract_prescreen(self, text: str) -> dict[str, Any]: + if isinstance(self._payload, Exception): + raise self._payload + return self._payload + + async def aclose(self) -> None: + pass + + +LLM_DICT: dict[str, Any] = { + "contract_type": "services", + "party_a": "ИП Петров Петр Петрович", + "party_b": "ООО «Клиент»", + "total_amount": 50000.0, + "currency": "RUB", + "has_penalty_clause": True, +} + + +def _storage(infra: dict[str, str]) -> MinioStorage: + return MinioStorage.from_endpoint_url( + endpoint_url=infra["s3_endpoint_url"], + access_key=infra["s3_access_key"], + secret_key=infra["s3_secret_key"], + bucket="contract-check-docs", + ) + + +async def _seed_document( + infra: dict[str, str], *, telegram_id: int, document_id: uuid.UUID, contract_text: str +) -> tuple[uuid.UUID, str]: + """Insert user + post-extraction document; return (user_id, text_s3_key).""" + sess = create_session_factory() + store = _storage(infra) + async with sess() as session: + result = await session.execute( + text( + "INSERT INTO users (telegram_id, credits_left) VALUES (:t, 5) " + "ON CONFLICT (telegram_id) DO UPDATE SET credits_left = 5 " + "RETURNING id" + ), + {"t": telegram_id}, + ) + user_id = result.scalar_one() + await session.commit() + + ext_key = extracted_key(str(user_id), str(document_id)) + await store.put( + ext_key, contract_text.encode("utf-8"), content_type="text/plain; charset=utf-8" + ) + + async with sess() as session: + await session.execute( + text( + "INSERT INTO documents " + "(id, user_id, s3_key, extracted_s3_key, filename, mime, bytes, " + " status, stage) " + "VALUES (:id, :uid, :s3, :ext, 'contract.pdf', 'application/pdf', :bytes, " + " 'prescreening', 'extracted')" + ), + { + "id": document_id, + "uid": user_id, + "s3": f"users/{user_id}/docs/{document_id}.pdf", + "ext": ext_key, + "bytes": len(contract_text), + }, + ) + await session.commit() + return user_id, ext_key + + +def _requested( + payload_doc: uuid.UUID, user_id: uuid.UUID, ext_key: str, text: str +) -> PrescreenRequested: + return PrescreenRequested( + correlation_id=uuid.uuid4(), + document_id=payload_doc, + user_id=user_id, + text_s3_key=ext_key, + filename="contract.pdf", + char_count=len(text), + ) + + +async def _prescreen_row(sess: async_sessionmaker[AsyncSession], document_id: uuid.UUID) -> Any: + async with sess() as session: + result = await session.execute( + text( + "SELECT contract_type, party_a, total_amount, currency, confidence_score, " + " routing_decision, extractor_version, auto_findings " + "FROM prescreen_results WHERE document_id = :d" + ), + {"d": document_id}, + ) + return result.one() + + +async def test_prescreen_heuristic_persists_version_and_routes( + infra: dict[str, str], +) -> None: + sess = create_session_factory() + document_id = uuid.uuid4() + user_id, ext_key = await _seed_document( + infra, telegram_id=777_111_222, document_id=document_id, contract_text=FULL_CONTRACT + ) + + handler = PrescreenHandler(session_factory=sess, publish_routing_key=_DEAD_ROUTING_KEY) + try: + await handler.handle(_requested(document_id, user_id, ext_key, FULL_CONTRACT)) + finally: + await handler.aclose() + + row = await _prescreen_row(sess, document_id) + (ctype, party_a, amount, currency, confidence, decision, version, findings) = row + assert ctype == "supply" + assert "Продавец" in party_a + assert amount == 1250000.0 + assert currency == "RUB" + assert confidence == 1.0 + assert decision == "deep_analysis" # high-value + penalty + arbitration + assert version == "heuristic-v2" # fallback disabled by default + assert findings == [] + + async with sess() as session: + doc = await session.execute( + text("SELECT status, stage FROM documents WHERE id = :d"), {"d": document_id} + ) + status, stage = doc.one() + assert status == "analyzing" + assert stage == "queued_analyze" + + +async def test_prescreen_llm_fallback_merged_and_persisted( + infra: dict[str, str], +) -> None: + sess = create_session_factory() + document_id = uuid.uuid4() + user_id, ext_key = await _seed_document( + infra, telegram_id=777_333_444, document_id=document_id, contract_text=LOW_CONF_TEXT + ) + + handler = PrescreenHandler( + session_factory=sess, + publish_routing_key=_DEAD_ROUTING_KEY, + extractor=HybridMetaExtractor( + provider=StubProvider(LLM_DICT), + fallback_enabled=True, + fallback_threshold=0.99, + heuristic=HeuristicExtractor(), + ), + ) + await handler.handle(_requested(document_id, user_id, ext_key, LOW_CONF_TEXT)) + + row = await _prescreen_row(sess, document_id) + (ctype, party_a, amount, currency, confidence, decision, version, findings) = row + assert ctype == "services" # recovered by the LLM stage + assert party_a == "ИП Петров Петр Петрович" + assert amount == 50000.0 + assert currency == "RUB" + assert confidence >= 0.75 + assert decision == "deep_analysis" # has_penalty_clause OR-merged to True + assert version == "hybrid-llm-v1" + assert findings == [] + + +async def test_prescreen_llm_failure_records_error_not_fatal( + infra: dict[str, str], +) -> None: + sess = create_session_factory() + document_id = uuid.uuid4() + user_id, ext_key = await _seed_document( + infra, telegram_id=777_555_666, document_id=document_id, contract_text=LOW_CONF_TEXT + ) + + handler = PrescreenHandler( + session_factory=sess, + publish_routing_key=_DEAD_ROUTING_KEY, + extractor=HybridMetaExtractor( + provider=StubProvider(RuntimeError("quota exceeded")), + fallback_enabled=True, + fallback_threshold=0.99, + heuristic=HeuristicExtractor(), + ), + ) + # Must not raise: the heuristic result survives the LLM failure. + await handler.handle(_requested(document_id, user_id, ext_key, LOW_CONF_TEXT)) + + row = await _prescreen_row(sess, document_id) + (ctype, _party_a, _amount, _cur, confidence, decision, version, findings) = row + assert ctype is None + assert confidence < 0.75 + assert decision == "manual_review" # routing semantics unchanged + assert version == "heuristic-v2" + parsed = findings if isinstance(findings, list) else json.loads(findings) + assert parsed and "llm_fallback_error" in parsed[0] + assert "quota exceeded" in parsed[0]["llm_fallback_error"] + + async with sess() as session: + doc = await session.execute( + text("SELECT status, stage FROM documents WHERE id = :d"), {"d": document_id} + ) + status, stage = doc.one() + assert status == "manual_review" + assert stage == "manual_review" diff --git a/tests/integration/test_upload_pipeline.py b/tests/integration/test_upload_pipeline.py index 2bc3256..9ee604f 100644 --- a/tests/integration/test_upload_pipeline.py +++ b/tests/integration/test_upload_pipeline.py @@ -18,6 +18,7 @@ from typing import TYPE_CHECKING import aio_pika import httpx import pytest + from tests.integration.conftest import user_token if TYPE_CHECKING: diff --git a/tests/unit/test_extract_handler.py b/tests/unit/test_extract_handler.py new file mode 100644 index 0000000..12f5c95 --- /dev/null +++ b/tests/unit/test_extract_handler.py @@ -0,0 +1,102 @@ +"""Unit tests for worker-extract handler. + +Covers the `PRESCREEN_ENABLED` toggle: when true it publishes +`PrescreenRequested` to `prescreen.q` and marks `prescreening`; when false it +bypasses the prescreen stage, publishes `AnalyzeRequested` to `analyze.q`, +and marks `analyzing`. +""" + +from __future__ import annotations + +import uuid +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from contract_check.core.extraction import ExtractedDocument +from contract_check.core.mq.messages import DocumentExtracted, DocumentUploaded, PrescreenRequested +from contract_check.worker_extract.handler import ExtractHandler + + +def _fake_session_factory() -> Any: + session = MagicMock() + session.execute = AsyncMock() + session.commit = AsyncMock() + session.__aenter__ = AsyncMock(return_value=session) + session.__aexit__ = AsyncMock(return_value=False) + return MagicMock(return_value=session), session + + +def _uploaded() -> DocumentUploaded: + return DocumentUploaded( + correlation_id=uuid.uuid4(), + document_id=uuid.uuid4(), + user_id=uuid.uuid4(), + s3_key="users/u/docs/d.pdf", + filename="contract.pdf", + mime="application/pdf", + ) + + +def _extracted() -> ExtractedDocument: + return ExtractedDocument( + markdown="Договор поставки. Сумма 1000 рублей.", + metadata={"format": "pdf", "is_scan": False, "has_tables": False}, + is_structured=True, + ) + + +@pytest.fixture +def handler(monkeypatch: pytest.MonkeyPatch) -> tuple[ExtractHandler, Any, Any]: + session_factory, session = _fake_session_factory() + h = ExtractHandler(session_factory=session_factory) + + # Stub S3 put and the extraction core. + h._storage = MagicMock(put=AsyncMock()) + monkeypatch.setattr(h, "_extract_text", AsyncMock(return_value=(_extracted().markdown, False))) + + # Stub publisher. + publisher = MagicMock(publish=AsyncMock()) + h._publisher = publisher + + # DB status is non-terminal. + monkeypatch.setattr(h, "_db_status", AsyncMock(return_value="queued")) + + return h, session, publisher + + +@pytest.mark.asyncio +async def test_prescreen_enabled_publishes_to_prescreen( + handler: tuple[ExtractHandler, Any, Any], +) -> None: + h, session, publisher = handler + h._settings.prescreen_enabled = True + + await h.handle(_uploaded()) + + assert publisher.publish.call_count == 1 + msg, *_ = publisher.publish.call_args.args + assert isinstance(msg, PrescreenRequested) + assert publisher.publish.call_args.kwargs["routing_key"] == "prescreen" + + status_params = session.execute.call_args_list[-2].args[1] + assert status_params["status"] == "prescreening" + + +@pytest.mark.asyncio +async def test_prescreen_disabled_bypasses_to_analyze( + handler: tuple[ExtractHandler, Any, Any], +) -> None: + h, session, publisher = handler + h._settings.prescreen_enabled = False + + await h.handle(_uploaded()) + + assert publisher.publish.call_count == 1 + msg, *_ = publisher.publish.call_args.args + assert isinstance(msg, DocumentExtracted) + assert publisher.publish.call_args.kwargs["routing_key"] == "analyze" + + status_params = session.execute.call_args_list[-2].args[1] + assert status_params["status"] == "analyzing" diff --git a/tests/unit/test_extraction_adapters.py b/tests/unit/test_extraction_adapters.py new file mode 100644 index 0000000..4ddbd0e --- /dev/null +++ b/tests/unit/test_extraction_adapters.py @@ -0,0 +1,195 @@ +"""Unit tests for the extraction adapters (core/extraction/adapters/*). + +Real files are generated in-memory per format (pymupdf/python-docx) so no +binary fixtures live in the repo. The OCR adapter's engine-dependent path is +covered by the integration suite; here we only assert the error contract. +""" + +from __future__ import annotations + +import pytest + +from contract_check.core.extraction import ( + ExtractedDocument, + ExtractionFailedError, + MammothDocxExtractor, + PyMuPDFExtractor, + RtfExtractor, + TesseractOcrExtractor, + TxtExtractor, +) + +# ── shared fixture builders ────────────────────────────────────────────────── + +PARAGRAPH = ( + "Стороны обязуются выполнять условия договора. " + "Сторона А обязуется передать товар в срок. " + "Сторона Б обязуется оплатить товар в течение десяти банковских дней. " + "Ответственность сторон ограничена суммой договора. " + "Споры подлежат рассмотрению в арбитражном суде города Москвы. " +) + + +def build_pdf_bytes(*, with_table: bool = False) -> bytes: + import pymupdf + + doc = pymupdf.open() + page = doc.new_page() + if with_table: + page.insert_htmlbox( + page.rect, + f'

{PARAGRAPH}

', + ) + x0, y0, cw, ch = 50, 200, 150, 30 + data = [["Услуга", "Цена"], ["Консультация", "5000"], ["Аудит", "15000"]] + for r in range(len(data) + 1): + page.draw_line(pymupdf.Point(x0, y0 + r * ch), pymupdf.Point(x0 + 2 * cw, y0 + r * ch)) + for c in range(3): + page.draw_line(pymupdf.Point(x0 + c * cw, y0), pymupdf.Point(x0 + c * cw, y0 + 3 * ch)) + for r, row in enumerate(data): + for c, val in enumerate(row): + rect = pymupdf.Rect( + x0 + c * cw + 3, y0 + r * ch + 3, x0 + (c + 1) * cw - 3, y0 + (r + 1) * ch - 3 + ) + page.insert_htmlbox( + rect, f'{val}' + ) + else: + page.insert_htmlbox( + page.rect, f'

{PARAGRAPH}

' + ) + data = doc.tobytes() + doc.close() + return data + + +def build_docx_bytes() -> bytes: + import io + + from docx import Document + + doc = Document() + doc.add_heading("Договор поставки", level=1) + doc.add_paragraph(PARAGRAPH) + doc.add_paragraph(PARAGRAPH) + buf = io.BytesIO() + doc.save(buf) + return buf.getvalue() + + +# ── PDF (pymupdf) ──────────────────────────────────────────────────────────── + + +def test_pdf_text_only() -> None: + result = PyMuPDFExtractor().extract(build_pdf_bytes()) + assert "арбитражном суде" in result.markdown + assert result.is_structured is False + assert result.metadata["format"] == "pdf" + assert result.metadata["has_tables"] is False + + +def test_pdf_table_becomes_markdown_pipes() -> None: + result = PyMuPDFExtractor().extract(build_pdf_bytes(with_table=True)) + assert result.is_structured is True + assert result.metadata["has_tables"] is True + # Header row + separator + data rows as pipes; no cell-text duplication. + assert "| Услуга | Цена |" in result.markdown + assert "| Консультация | 5000 |" in result.markdown + assert result.markdown.count("Консультация") == 1 + + +def test_pdf_too_short_raises() -> None: + import pymupdf + + doc = pymupdf.open() + doc.new_page() + data = doc.tobytes() + doc.close() + with pytest.raises(ExtractionFailedError, match="слишком мало"): + PyMuPDFExtractor().extract(data) + + +def test_pdf_garbage_raises() -> None: + with pytest.raises(ExtractionFailedError): + PyMuPDFExtractor().extract(b"definitely not a pdf") + + +# ── DOCX (mammoth) ─────────────────────────────────────────────────────────── + + +def test_docx_heading_preserved() -> None: + result = MammothDocxExtractor().extract(build_docx_bytes()) + assert result.markdown.startswith("# Договор поставки") + assert result.is_structured is True + assert "арбитражном суде" in result.markdown + + +def test_docx_garbage_raises() -> None: + with pytest.raises(ExtractionFailedError): + MammothDocxExtractor().extract(b"not a zip") + + +# ── RTF (striprtf) ─────────────────────────────────────────────────────────── + + +def test_rtf_extracted_as_plain_text() -> None: + body = " ".join(f"Clause {i}: the parties agree to the terms herein." for i in range(30)) + data = r"{\rtf1\ansi\deff0 " + body.replace("\n", r"\par ") + "}" + result = RtfExtractor().extract(data.encode("ascii")) + assert "the parties agree" in result.markdown + assert result.is_structured is False + + +def test_rtf_missing_header_raises() -> None: + with pytest.raises(ExtractionFailedError, match="RTF"): + RtfExtractor().extract(b"just some text that is long enough " * 10) + + +# ── TXT (chardet) ──────────────────────────────────────────────────────────── + + +def test_txt_windows1251_detected() -> None: + data = (PARAGRAPH * 10).encode("windows-1251") + result = TxtExtractor().extract(data) + assert "арбитражном суде" in result.markdown # decoded, not mojibake + assert result.metadata["encoding"] == "windows-1251" + assert result.is_structured is False + + +def test_txt_utf8() -> None: + result = TxtExtractor().extract((PARAGRAPH * 5).encode("utf-8")) + assert "Стороны" in result.markdown + assert result.metadata["encoding"] in ("utf-8", "utf-8-sig") + + +def test_txt_too_short_raises() -> None: + with pytest.raises(ExtractionFailedError): + TxtExtractor().extract("коротко".encode()) + + +# ── OCR (tesseract) — error contract only; happy path is integration ───────── +# Both cases fail at open/decode, before the engine is invoked, so they run +# regardless of whether the tesseract binary is installed. + + +def test_ocr_garbage_pdf_raises() -> None: + from contract_check.core.analysis.ocr import OCRError + + with pytest.raises((OCRError, ExtractionFailedError)): + TesseractOcrExtractor().extract(b"%PDF-1.4 garbage") + + +def test_ocr_garbage_image_raises() -> None: + from contract_check.core.analysis.ocr import OCRError + + with pytest.raises((OCRError, ExtractionFailedError)): + TesseractOcrExtractor().extract(b"\x89PNG\r\n\x1a\n not really a png") + + +# ── ExtractedDocument DTO ──────────────────────────────────────────────────── + + +def test_extracted_document_defaults() -> None: + doc = ExtractedDocument(markdown="x") + assert doc.is_structured is False + assert doc.metadata == {} diff --git a/tests/unit/test_extraction_factory.py b/tests/unit/test_extraction_factory.py new file mode 100644 index 0000000..f036f73 --- /dev/null +++ b/tests/unit/test_extraction_factory.py @@ -0,0 +1,116 @@ +"""Unit tests for ExtractorFactory + detect_format + the OCR-fallback flow.""" + +from __future__ import annotations + +import pytest + +from contract_check.core.extraction import ( + UnsupportedFormatError, + detect_format, + extract_document, + get_factory, +) +from tests.unit.test_extraction_adapters import PARAGRAPH, build_docx_bytes, build_pdf_bytes + +# ── detect_format: suffix / mime / magic precedence ────────────────────────── + + +@pytest.mark.parametrize( + ("filename", "expected"), + [ + ("contract.pdf", "pdf"), + ("contract.PDF", "pdf"), + ("contract.docx", "docx"), + ("contract.rtf", "rtf"), + ("contract.txt", "txt"), + ("data.csv", "txt"), + ("scan.png", "image"), + ("scan.jpg", "image"), + ("scan.jpeg", "image"), + ("scan.tiff", "image"), + ], +) +def test_detect_by_suffix(filename: str, expected: str) -> None: + assert detect_format(b"", filename=filename) == expected + + +def test_detect_by_mime_when_suffix_unknown() -> None: + assert detect_format(b"", mime="application/pdf", filename="contract") == "pdf" + assert detect_format(b"", mime="image/png", filename="photo") == "image" + + +def test_magic_beats_wrong_suffix() -> None: + # A real PDF mislabeled .docx must route to the PDF adapter. + pytest.importorskip("magic") + assert detect_format(build_pdf_bytes(), mime="", filename="contract.docx") == "pdf" + + +def test_unsupported_format_raises() -> None: + with pytest.raises(UnsupportedFormatError): + detect_format(b"", filename="archive.zip") + binary_soup = bytes(range(256)) * 4 # deliberately not decodable text + with pytest.raises(UnsupportedFormatError): + detect_format(binary_soup, mime="application/x-msdownload", filename="doc.exe") + + +def test_docx_zip_magic_does_not_shadow_suffix() -> None: + # DOCX is a zip; python-magic reports application/zip which the factory + # deliberately does not map — the suffix must decide. + pytest.importorskip("magic") + assert detect_format(build_docx_bytes(), filename="contract.docx") == "docx" + + +# ── factory ────────────────────────────────────────────────────────────────── + + +def test_factory_returns_per_format_adapters() -> None: + factory = get_factory() + assert type(factory.get_extractor(b"", filename="a.pdf")).__name__ == "PyMuPDFExtractor" + assert type(factory.get_extractor(b"", filename="a.docx")).__name__ == "MammothDocxExtractor" + assert type(factory.get_extractor(b"", filename="a.rtf")).__name__ == "RtfExtractor" + assert type(factory.get_extractor(b"", filename="a.txt")).__name__ == "TxtExtractor" + assert type(factory.get_extractor(b"", filename="a.png")).__name__ == "TesseractOcrExtractor" + + +# ── extract_document: end-to-end over bytes ────────────────────────────────── + + +def test_extract_document_pdf() -> None: + result = extract_document(build_pdf_bytes(), filename="c.pdf") + assert result.metadata["format"] == "pdf" + assert "арбитражном суде" in result.markdown + + +def test_extract_document_docx() -> None: + result = extract_document(build_docx_bytes(), filename="c.docx") + assert result.markdown.startswith("# ") + assert result.is_structured is True + + +def test_extract_document_txt_cp1251() -> None: + result = extract_document((PARAGRAPH * 10).encode("windows-1251"), filename="c.txt") + assert "Стороны" in result.markdown + assert result.metadata["format"] == "txt" + + +def test_extract_document_unsupported() -> None: + with pytest.raises(UnsupportedFormatError): + extract_document(b"PK\x03\x04 whatever", filename="c.zip") + + +def test_extract_document_pdf_scan_falls_back_to_ocr() -> None: + # A PDF with no text layer triggers the OCR fallback inside the + # orchestrator. Without the tesseract engine the fallback raises + # OCRError (retryable) — with the engine, the integration suite covers + # the happy path. + import pymupdf + + from contract_check.core.analysis.ocr import OCRError + from contract_check.core.extraction import ExtractionFailedError + + doc = pymupdf.open() + doc.new_page() # blank page: no text at all + data = doc.tobytes() + doc.close() + with pytest.raises((OCRError, ExtractionFailedError)): + extract_document(data, filename="scan.pdf") diff --git a/tests/unit/test_llm_prescreen_extraction.py b/tests/unit/test_llm_prescreen_extraction.py new file mode 100644 index 0000000..4584b15 --- /dev/null +++ b/tests/unit/test_llm_prescreen_extraction.py @@ -0,0 +1,164 @@ +"""extract_prescreen provider tests (respx-mocked, both adapters). + +Covers (plan Phase 2): valid dict passthrough, prescreen_max_chars input cap, +malformed JSON → repair once → success, and malformed after repair → LLMError. +""" + +from __future__ import annotations + +import json +from typing import Any + +import httpx +import pytest +import respx + +from contract_check.core.llm.ollama_cloud import LLMError, OllamaCloudProvider +from contract_check.core.llm.yandex_gpt import YandexGPTProvider + +OLLAMA_HOST = "https://ollama.test" +OLLAMA_URL = f"{OLLAMA_HOST}/api/chat" +YANDEX_HOST = "https://llm.test" +YANDEX_URL = f"{YANDEX_HOST}/foundationModels/v1/completion" + +VALID_PRESCREEN: dict[str, Any] = { + "contract_type": "supply", + "party_a": "ООО «Альфа»", + "party_b": "ООО «Бета»", + "total_amount": 1250000.0, + "currency": "RUB", + "start_date": "2025-09-01", + "end_date": "2026-08-31", + "has_penalty_clause": True, + "has_termination_clause": False, + "has_arbitration": True, +} + + +def _ollama_body(content: str) -> dict[str, object]: + return {"message": {"content": content}, "prompt_eval_count": 5, "eval_count": 7} + + +def _yandex_body(content: str) -> dict[str, object]: + return { + "result": { + "alternatives": [{"message": {"role": "assistant", "text": content}}], + "usage": {"inputTextTokens": 5, "completionTokens": 7, "totalTokens": 12}, + } + } + + +# ── Ollama Cloud ───────────────────────────────────────────────────────────── + + +async def test_ollama_extract_prescreen_returns_dict() -> None: + provider = OllamaCloudProvider(host=OLLAMA_HOST, api_key="key", model="m") + async with provider: + with respx.mock(base_url=OLLAMA_HOST) as mock: + route = mock.post(OLLAMA_URL).mock( + return_value=httpx.Response(200, json=_ollama_body(json.dumps(VALID_PRESCREEN))) + ) + result = await provider.extract_prescreen("Договор поставки...") + assert route.call_count == 1 + assert result == VALID_PRESCREEN + + +async def test_ollama_extract_prescreen_truncates_input() -> None: + provider = OllamaCloudProvider( + host=OLLAMA_HOST, api_key="key", model="m", prescreen_max_chars=100 + ) + async with provider: + with respx.mock(base_url=OLLAMA_HOST) as mock: + route = mock.post(OLLAMA_URL).mock( + return_value=httpx.Response(200, json=_ollama_body(json.dumps(VALID_PRESCREEN))) + ) + await provider.extract_prescreen("договор " * 500) + body = json.loads(route.calls.last.request.read()) + user_msg = body["messages"][1]["content"] + assert len(user_msg) == 100 + assert body["messages"][0]["role"] == "system" + assert "метаданных" in body["messages"][0]["content"] + + +async def test_ollama_extract_prescreen_repairs_invalid_json() -> None: + provider = OllamaCloudProvider(host=OLLAMA_HOST, api_key="key", model="m") + async with provider: + with respx.mock(base_url=OLLAMA_HOST) as mock: + route = mock.post(OLLAMA_URL).mock( + side_effect=[ + httpx.Response(200, json=_ollama_body("not json {")), + httpx.Response(200, json=_ollama_body(json.dumps(VALID_PRESCREEN))), + ] + ) + result = await provider.extract_prescreen("Договор...") + assert route.call_count == 2 + assert result == VALID_PRESCREEN + + +async def test_ollama_extract_prescreen_invalid_after_repair_raises() -> None: + provider = OllamaCloudProvider(host=OLLAMA_HOST, api_key="key", model="m") + async with provider: + with respx.mock(base_url=OLLAMA_HOST) as mock: + mock.post(OLLAMA_URL).mock( + return_value=httpx.Response(200, json=_ollama_body("still not json")) + ) + with pytest.raises(LLMError): + await provider.extract_prescreen("Договор...") + + +async def test_ollama_extract_prescreen_ignores_extra_fields() -> None: + """Hallucinated fields are dropped by the wire model (extra=ignore).""" + payload = {**VALID_PRESCREEN, "risk_score": "high", "summary": "безопасен"} + provider = OllamaCloudProvider(host=OLLAMA_HOST, api_key="key", model="m") + async with provider: + with respx.mock(base_url=OLLAMA_HOST) as mock: + mock.post(OLLAMA_URL).mock( + return_value=httpx.Response(200, json=_ollama_body(json.dumps(payload))) + ) + result = await provider.extract_prescreen("Договор...") + assert "risk_score" not in result + assert result["contract_type"] == "supply" + + +# ── YandexGPT ──────────────────────────────────────────────────────────────── + + +async def test_yandex_extract_prescreen_returns_dict() -> None: + provider = YandexGPTProvider(api_key="key", folder_id="b1", base_url=YANDEX_HOST) + async with provider: + with respx.mock(base_url=YANDEX_HOST) as mock: + route = mock.post(YANDEX_URL).mock( + return_value=httpx.Response(200, json=_yandex_body(json.dumps(VALID_PRESCREEN))) + ) + result = await provider.extract_prescreen("Договор поставки...") + assert route.call_count == 1 + assert result == VALID_PRESCREEN + + +async def test_yandex_extract_prescreen_truncates_input() -> None: + provider = YandexGPTProvider( + api_key="key", folder_id="b1", base_url=YANDEX_HOST, prescreen_max_chars=100 + ) + async with provider: + with respx.mock(base_url=YANDEX_HOST) as mock: + route = mock.post(YANDEX_URL).mock( + return_value=httpx.Response(200, json=_yandex_body(json.dumps(VALID_PRESCREEN))) + ) + await provider.extract_prescreen("договор " * 500) + body = json.loads(route.calls.last.request.read()) + assert len(body["messages"][1]["text"]) == 100 + + +async def test_yandex_extract_prescreen_repairs_invalid_json() -> None: + provider = YandexGPTProvider(api_key="key", folder_id="b1", base_url=YANDEX_HOST) + async with provider: + with respx.mock(base_url=YANDEX_HOST) as mock: + route = mock.post(YANDEX_URL).mock( + side_effect=[ + httpx.Response(200, json=_yandex_body("{broken")), + httpx.Response(200, json=_yandex_body(json.dumps(VALID_PRESCREEN))), + ] + ) + result = await provider.extract_prescreen("Договор...") + assert route.call_count == 2 + assert result == VALID_PRESCREEN diff --git a/tests/unit/test_llm_yandex_gpt.py b/tests/unit/test_llm_yandex_gpt.py new file mode 100644 index 0000000..f94bb52 --- /dev/null +++ b/tests/unit/test_llm_yandex_gpt.py @@ -0,0 +1,294 @@ +"""YandexGPT adapter unit tests (respx-mocked). + +Covers: 200 happy path, 429→fallback, invalid-JSON→repair→success, +auth/config errors, and the provider returning a merged AnalysisResult. +No network. +""" + +from __future__ import annotations + +import json + +import httpx +import pytest +import respx + +from contract_check.core.analysis.report_schema import ReportPayload +from contract_check.core.llm.yandex_gpt import ( + LLMConfigError, + LLMError, + LLMQuotaError, + LLMUnavailableError, + YandexGPTProvider, +) + +HOST = "https://llm.test" +URL = f"{HOST}/foundationModels/v1/completion" +FOLDER_ID = "b1folder" +MODEL = "yandexgpt-lite" +FALLBACK_MODEL = "yandexgpt" + +VALID_PAYLOAD = ReportPayload( + findings=[ + { + "checklist_id": "penalties", + "severity": "high", + "quote": "Штраф 0,5% за каждый день просрочки", + "section_ref": "п. 6.3", + "risk": "Высокая неустойка", + "recommendation": "Ограничить cap", + } + ] +) + + +def _completion_body(content: str, model: str = MODEL) -> dict[str, object]: + return { + "result": { + "alternatives": [{"message": {"role": "assistant", "text": content}}], + "usage": { + "inputTextTokens": 100, + "completionTokens": 20, + "totalTokens": 120, + }, + "modelVersion": model, + } + } + + +def _provider(fallback: str | None = FALLBACK_MODEL) -> YandexGPTProvider: + return YandexGPTProvider( + api_key="key", + folder_id=FOLDER_ID, + model=MODEL, + fallback_model=fallback, + base_url=HOST, + completion_path="/foundationModels/v1/completion", + max_concurrency=1, + chunk_size=10000, + ) + + +@pytest.mark.asyncio +async def test_happy_path_returns_findings() -> None: + async with _provider() as provider: + with respx.mock(base_url=HOST) as mock: + route = mock.post("/foundationModels/v1/completion") + route.mock( + return_value=httpx.Response( + 200, json=_completion_body(VALID_PAYLOAD.model_dump_json()) + ) + ) + result = await provider.analyze("договор " * 200) + + assert len(result.findings) == 1 + assert result.findings[0].checklist_id == "penalties" + assert result.findings[0].severity == "high" + assert result.prompt_tokens == 100 + assert result.eval_tokens == 20 + assert result.fell_back is False + assert result.model_used == MODEL + + +@pytest.mark.asyncio +async def test_429_falls_back_to_secondary_model() -> None: + async with _provider() as provider: + with respx.mock(base_url=HOST) as mock: + route = mock.post("/foundationModels/v1/completion") + route.mock( + side_effect=[ + httpx.Response(429, json={"error": "quota"}), + httpx.Response( + 200, + json=_completion_body( + VALID_PAYLOAD.model_dump_json(), model=FALLBACK_MODEL + ), + ), + ] + ) + result = await provider.analyze("договор " * 200) + + assert result.fell_back is True + assert FALLBACK_MODEL in result.models_used + + +@pytest.mark.asyncio +async def test_invalid_json_repaired_on_second_attempt() -> None: + async with _provider() as provider: + with respx.mock(base_url=HOST) as mock: + route = mock.post("/foundationModels/v1/completion") + route.mock( + side_effect=[ + httpx.Response(200, json=_completion_body("not valid json {")), + httpx.Response(200, json=_completion_body(VALID_PAYLOAD.model_dump_json())), + ] + ) + result = await provider.analyze("договор " * 200) + + assert result.repaired is True + assert len(result.findings) == 1 + + +@pytest.mark.asyncio +async def test_model_aliases_validate_without_repair() -> None: + """Models often emit risk_type/description and omit recommendation.""" + aliased = json.dumps( + { + "findings": [ + { + "risk_type": "penalties", + "severity": "medium", + "description": "Высокая неустойка.", + "quote": "Штраф 0,5% за каждый день просрочки", + "section_ref": "п. 6.3", + } + ] + } + ) + async with _provider() as provider: + with respx.mock(base_url=HOST) as mock: + route = mock.post("/foundationModels/v1/completion") + route.mock(return_value=httpx.Response(200, json=_completion_body(aliased))) + result = await provider.analyze("договор " * 200) + assert route.call_count == 1 + assert result.repaired is False + assert result.findings[0].checklist_id == "penalties" + assert result.findings[0].risk == "Высокая неустойка." + assert result.findings[0].recommendation == "" + + +@pytest.mark.asyncio +async def test_429_with_no_fallback_raises_quota() -> None: + provider = YandexGPTProvider( + api_key="key", + folder_id=FOLDER_ID, + model=MODEL, + fallback_model=None, + base_url=HOST, + completion_path="/foundationModels/v1/completion", + ) + async with provider: + with respx.mock(base_url=HOST) as mock: + mock.post("/foundationModels/v1/completion").mock( + return_value=httpx.Response(429, json={"error": "quota"}) + ) + with pytest.raises(LLMQuotaError): + await provider.analyze("договор " * 200) + + +@pytest.mark.asyncio +async def test_invalid_json_after_repair_raises_llm_error() -> None: + async with _provider() as provider: + with respx.mock(base_url=HOST) as mock: + mock.post("/foundationModels/v1/completion").mock( + return_value=httpx.Response(200, json=_completion_body("still not json")) + ) + with pytest.raises(LLMError): + await provider.analyze("договор " * 200) + + +@pytest.mark.asyncio +async def test_short_text_single_chunk_one_request() -> None: + async with _provider() as provider: + with respx.mock(base_url=HOST) as mock: + route = mock.post("/foundationModels/v1/completion") + route.mock( + return_value=httpx.Response( + 200, json=_completion_body(VALID_PAYLOAD.model_dump_json()) + ) + ) + await provider.analyze("короткий договор") + assert route.call_count == 1 + + +@pytest.mark.asyncio +async def test_multi_chunk_merges_findings() -> None: + findings = json.loads(VALID_PAYLOAD.model_dump_json())["findings"] + async with _provider() as provider: + with respx.mock(base_url=HOST) as mock: + mock.post("/foundationModels/v1/completion").mock( + return_value=httpx.Response( + 200, + json=_completion_body(json.dumps({"findings": findings + findings})), + ) + ) + result = await provider.analyze("договор " * 3000, checklist="x") + assert len(result.findings) == 1 + + +@pytest.mark.asyncio +async def test_401_raises_config_error() -> None: + async with _provider() as provider: + with respx.mock(base_url=HOST) as mock: + route = mock.post("/foundationModels/v1/completion") + route.mock(return_value=httpx.Response(401, json={"error": "unauthorized"})) + with pytest.raises(LLMConfigError): + await provider.analyze("договор " * 200) + assert route.call_count == 1 + + +@pytest.mark.asyncio +async def test_400_raises_config_error() -> None: + async with _provider() as provider: + with respx.mock(base_url=HOST) as mock: + route = mock.post("/foundationModels/v1/completion") + route.mock(return_value=httpx.Response(400, json={"error": "bad request"})) + with pytest.raises(LLMConfigError): + await provider.analyze("договор " * 200) + assert route.call_count == 1 + + +@pytest.mark.asyncio +async def test_404_raises_config_error() -> None: + async with _provider() as provider: + with respx.mock(base_url=HOST) as mock: + route = mock.post("/foundationModels/v1/completion") + route.mock(return_value=httpx.Response(404, text="Not Found")) + with pytest.raises(LLMConfigError): + await provider.analyze("договор " * 200) + assert route.call_count == 1 + + +@pytest.mark.asyncio +async def test_connect_error_raises_config_error_immediately() -> None: + async with _provider() as provider: + with respx.mock(base_url=HOST) as mock: + route = mock.post("/foundationModels/v1/completion") + route.mock(side_effect=httpx.ConnectError("Connection refused")) + with pytest.raises(LLMConfigError): + await provider.analyze("договор " * 200) + assert route.call_count == 1 + + +@pytest.mark.asyncio +async def test_503_retries_then_raises_unavailable() -> None: + async with _provider() as provider: + with respx.mock(base_url=HOST) as mock: + route = mock.post("/foundationModels/v1/completion") + route.mock(return_value=httpx.Response(503, text="Unavailable")) + with pytest.raises(LLMUnavailableError): + await provider.analyze("договор " * 200) + assert route.call_count == 3 + + +@pytest.mark.asyncio +async def test_request_payload_contains_model_uri_and_json_schema() -> None: + async with _provider(fallback=None) as provider: + with respx.mock(base_url=HOST) as mock: + route = mock.post("/foundationModels/v1/completion") + route.mock( + return_value=httpx.Response( + 200, json=_completion_body(VALID_PAYLOAD.model_dump_json()) + ) + ) + await provider.analyze("короткий договор", checklist="test checklist") + + request = route.calls[0].request + body = json.loads(request.content) + assert body["modelUri"] == f"gpt://{FOLDER_ID}/{MODEL}" + assert body["completionOptions"]["responseFormat"]["type"] == "JSON_OBJECT" + assert "schema" in body["completionOptions"]["responseFormat"]["json_schema"] + assert body["messages"][0]["role"] == "system" + assert "test checklist" in body["messages"][0]["text"] + assert request.headers["authorization"] == "Api-Key key" + assert request.headers["x-folder-id"] == FOLDER_ID diff --git a/tests/unit/test_prescreen_extractor.py b/tests/unit/test_prescreen_extractor.py new file mode 100644 index 0000000..9f0fc75 --- /dev/null +++ b/tests/unit/test_prescreen_extractor.py @@ -0,0 +1,113 @@ +"""Unit tests for the deterministic regex prescreen extractor.""" + +from __future__ import annotations + +import pytest + +from contract_check.worker_prescreen.extractor import extract_contract_meta + +SIMPLE_SUPPLY = """\ + +ДОГОВОР ПОСТАВКИ № 42 + +г. Москва + +Общество с ограниченной ответственностью «Продавец», +именуемое в дальнейшем «Поставщик», в лице директора Иванова И.И., +с одной стороны, и +Общество с ограниченной ответственностью «Покупатель», +именуемое в дальнейшем «Покупатель», с другой стороны, +заключили настоящий договор о нижеследующем. + +1. Предмет договора +Поставщик обязуется передать в собственность Покупателю товар. + +2. Цена и порядок расчётов +2.1. Общая стоимость товара составляет 1 250 000 (один миллион двести пятьдесят тысяч) рублей. + +3. Срок действия договора +3.1. Настоящий договор вступает в силу с 01.09.2025 и действует по 31.08.2026. + +4. Ответственность сторон +4.1. За просрочку поставки Поставщик уплачивает неустойку в размере 0,1% от стоимости. + +5. Порядок разрешения споров +5.1. Споры подлежат рассмотрению в Арбитражном суде г. Москвы. +""" + + +MINIMAL = """\ +Договор оказания услуг между ИП Петров Петр Петрович и ООО «Клиент». +Стоимость услуг составляет 50 000 рублей. +""" + + +def test_supply_extraction() -> None: + meta = extract_contract_meta(SIMPLE_SUPPLY) + + assert meta.contract_type == "supply" + assert "Продавец" in (meta.party_a or "") + assert "Покупатель" in (meta.party_b or "") + assert meta.total_amount == 1250000.0 + assert meta.currency == "RUB" + assert meta.start_date == "2025-09-01" + assert meta.end_date == "2026-08-31" + assert meta.has_penalty_clause is True + assert meta.has_termination_clause is False + assert meta.has_arbitration is True + assert meta.confidence_score > 0.6 + + +def test_minimal_extraction() -> None: + meta = extract_contract_meta(MINIMAL) + + assert meta.contract_type == "services" + assert "ИП Петров" in (meta.party_a or "") + assert "ООО «Клиент»" in (meta.party_b or "") + assert meta.total_amount == 50000.0 + assert meta.currency == "RUB" + + +def test_confidence_score_bounds() -> None: + meta = extract_contract_meta("") + assert meta.confidence_score == 0.0 + assert 0.0 <= meta.confidence_score <= 1.0 + + +def test_party_extraction_handles_whitespace() -> None: + text = "Договор между ООО «Альфа» и ООО «Бета»" + meta = extract_contract_meta(text) + assert "Альфа" in (meta.party_a or "") + assert "Бета" in (meta.party_b or "") + + +@pytest.mark.parametrize( + ("clause", "expected"), + [ + ("Стороны вправе расторгнуть договор в одностороннем порядке", "has_termination_clause"), + ("В случае нарушения Покупатель уплачивает штраф", "has_penalty_clause"), + ("Споры рассматриваются в Арбитражном суде", "has_arbitration"), + ], +) +def test_boolean_flags(clause: str, expected: str) -> None: + meta = extract_contract_meta(f"Договор поставки. {clause}.") + assert getattr(meta, expected) is True + + +def test_shim_defaults_to_heuristic_v2() -> None: + meta = extract_contract_meta(SIMPLE_SUPPLY) + assert meta.confidence_score > 0.6 + + +def test_keep_regex_flag_routes_to_regex_v1(monkeypatch: pytest.MonkeyPatch) -> None: + """PRESCREEN_KEEP_REGEX=true keeps the legacy regex path alive (plan §5).""" + from contract_check.worker_prescreen.extractor import _extract_contract_meta_regex + + monkeypatch.setenv("PRESCREEN_KEEP_REGEX", "true") + meta = extract_contract_meta(SIMPLE_SUPPLY) + legacy = _extract_contract_meta_regex(SIMPLE_SUPPLY.replace("\n", " ")) + # Identical extraction semantics on the ported fixtures. + assert meta.model_dump() == legacy.model_dump() + + monkeypatch.setenv("PRESCREEN_KEEP_REGEX", "false") + assert extract_contract_meta(SIMPLE_SUPPLY).model_dump() == meta.model_dump() diff --git a/tests/unit/test_prescreen_extractor_heuristic.py b/tests/unit/test_prescreen_extractor_heuristic.py new file mode 100644 index 0000000..8921af4 --- /dev/null +++ b/tests/unit/test_prescreen_extractor_heuristic.py @@ -0,0 +1,191 @@ +"""Unit tests for the Stage-1 heuristic extractor (regex-free port). + +Ports 100% of the regex-v1 fixtures from test_prescreen_extractor.py onto +HeuristicExtractor (plan Phase 1) and adds coverage for the string helpers +(`_scan_number`, header window, trigger scans). +""" + +from __future__ import annotations + +import pytest + +from contract_check.worker_prescreen.extractor_heuristic import ( + EXTRACTOR_VERSION, + HeuristicExtractor, + _header_window, + _scan_number, +) + +SIMPLE_SUPPLY = """\ + +ДОГОВОР ПОСТАВКИ № 42 + +г. Москва + +Общество с ограниченной ответственностью «Продавец», +именуемое в дальнейшем «Поставщик», в лице директора Иванова И.И., +с одной стороны, и +Общество с ограниченной ответственностью «Покупатель», +именуемое в дальнейшем «Покупатель», с другой стороны, +заключили настоящий договор о нижеследующем. + +1. Предмет договора +Поставщик обязуется передать в собственность Покупателю товар. + +2. Цена и порядок расчётов +2.1. Общая стоимость товара составляет 1 250 000 (один миллион двести пятьдесят тысяч) рублей. + +3. Срок действия договора +3.1. Настоящий договор вступает в силу с 01.09.2025 и действует по 31.08.2026. + +4. Ответственность сторон +4.1. За просрочку поставки Поставщик уплачивает неустойку в размере 0,1% от стоимости. + +5. Порядок разрешения споров +5.1. Споры подлежат рассмотрению в Арбитражном суде г. Москвы. +""" + +MINIMAL = """\ +Договор оказания услуг между ИП Петров Петр Петрович и ООО «Клиент». +Стоимость услуг составляет 50 000 рублей. +""" + + +@pytest.fixture +def extractor() -> HeuristicExtractor: + return HeuristicExtractor() + + +def test_version_is_heuristic_v2() -> None: + assert EXTRACTOR_VERSION == "heuristic-v2" + assert HeuristicExtractor.extractor_version == "heuristic-v2" + + +def test_supply_extraction(extractor: HeuristicExtractor) -> None: + meta = extractor.extract(SIMPLE_SUPPLY) + + assert meta.contract_type == "supply" + assert "Продавец" in (meta.party_a or "") + assert "Покупатель" in (meta.party_b or "") + assert meta.total_amount == 1250000.0 + assert meta.currency == "RUB" + assert meta.start_date == "2025-09-01" + assert meta.end_date == "2026-08-31" + assert meta.has_penalty_clause is True + assert meta.has_termination_clause is False + assert meta.has_arbitration is True + assert meta.confidence_score > 0.6 + + +def test_minimal_extraction(extractor: HeuristicExtractor) -> None: + meta = extractor.extract(MINIMAL) + + assert meta.contract_type == "services" + assert "ИП Петров" in (meta.party_a or "") + assert "ООО «Клиент»" in (meta.party_b or "") + assert meta.total_amount == 50000.0 + assert meta.currency == "RUB" + + +def test_confidence_score_bounds(extractor: HeuristicExtractor) -> None: + meta = extractor.extract("") + assert meta.confidence_score == 0.0 + assert 0.0 <= meta.confidence_score <= 1.0 + + +def test_party_extraction_handles_whitespace(extractor: HeuristicExtractor) -> None: + text = "Договор между ООО «Альфа» и ООО «Бета»" + meta = extractor.extract(text) + assert "Альфа" in (meta.party_a or "") + assert "Бета" in (meta.party_b or "") + + +@pytest.mark.parametrize( + ("clause", "expected"), + [ + ("Стороны вправе расторгнуть договор в одностороннем порядке", "has_termination_clause"), + ("В случае нарушения Покупатель уплачивает штраф", "has_penalty_clause"), + ("Споры рассматриваются в Арбитражном суде", "has_arbitration"), + ], +) +def test_boolean_flags(clause: str, expected: str, extractor: HeuristicExtractor) -> None: + meta = extractor.extract(f"Договор поставки. {clause}.") + assert getattr(meta, expected) is True + + +# ── helper coverage ────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("1 250 000 рублей", 1250000.0), + ("1250000", 1250000.0), + ("50 000 руб", 50000.0), + ("0,1% от стоимости", 0.1), + ("1 250 000,50", 1250000.5), + ("999", 999.0), + ], +) +def test_scan_number(raw: str, expected: float) -> None: + parsed = _scan_number(raw, 0) + assert parsed is not None + assert parsed[0] == expected + + +def test_scan_number_rejects_non_digits() -> None: + assert _scan_number("abc", 0) is None + + +def test_scan_number_nbsp_thousands() -> None: + parsed = _scan_number("1\u00a0250\u00a0000", 0) + assert parsed is not None + assert parsed[0] == 1250000.0 + + +def test_header_window_cuts_at_numbered_heading() -> None: + text = "ДОГОВОР № 1\nООО «Альфа» и ООО «Бета»\n1. Предмет договора\nООО «Гамма»" + window = _header_window(text) + assert "Альфа" in window + assert "Бета" in window + assert "Гамма" not in window + + +def test_parties_only_scanned_in_header(extractor: HeuristicExtractor) -> None: + text = ( + "ДОГОВОР № 1\nООО «Альфа» и ООО «Бета»\n\n" + "1. Предмет договора\nПоставщик обязуется...\n\n" + "10. Реквизиты сторон\nООО «Гамма»" + ) + meta = extractor.extract(text) + assert "Альфа" in (meta.party_a or "") + assert "Бета" in (meta.party_b or "") + + +def test_end_date_only_via_do_fallback(extractor: HeuristicExtractor) -> None: + meta = extractor.extract("Соглашение действует до 31.12.2026.") + assert meta.start_date is None + assert meta.end_date == "2026-12-31" + + +def test_start_date_via_ot_token(extractor: HeuristicExtractor) -> None: + meta = extractor.extract("Договор заключён от 15.03.2024.") + assert meta.start_date == "2024-03-15" + + +def test_amount_skips_trigger_without_digits(extractor: HeuristicExtractor) -> None: + meta = extractor.extract("Общая стоимость товара определяется расчётом. Сумма 300 BYN.") + assert meta.total_amount == 300.0 + assert meta.currency == "BYN" + + +def test_no_false_entity_match_inside_words(extractor: HeuristicExtractor) -> None: + meta = extractor.extract("Ипотека оформлена. Сумма 100 рублей.") + assert meta.party_a is None + assert meta.party_b is None + + +def test_bare_entity_form_without_name_dropped(extractor: HeuristicExtractor) -> None: + meta = extractor.extract("Договор между ООО и АО.\nСумма 100 рублей.") + assert meta.party_a is None + assert meta.party_b is None diff --git a/tests/unit/test_prescreen_extractor_hybrid.py b/tests/unit/test_prescreen_extractor_hybrid.py new file mode 100644 index 0000000..4376dfe --- /dev/null +++ b/tests/unit/test_prescreen_extractor_hybrid.py @@ -0,0 +1,161 @@ +"""HybridMetaExtractor orchestrator matrix (plan Phase 4). + +- high confidence → LLM skipped +- low confidence → LLM merged, version hybrid-llm-v1 +- LLM failure → heuristic result kept + outcome=failed + error recorded +- fallback disabled → outcome=disabled, no provider call +- boolean merge semantics: OR across stages +""" + +from __future__ import annotations + +from typing import Any + +from contract_check.core.metrics import prescreen_fallback_runs +from contract_check.worker_prescreen.extractor_heuristic import HeuristicExtractor +from contract_check.worker_prescreen.extractor_hybrid import HybridMetaExtractor + +HIGH_CONF_TEXT = """\ + +ДОГОВОР ПОСТАВКИ № 42 +г. Москва +Общество с ограниченной ответственностью «Продавец» и +Общество с ограниченной ответственностью «Покупатель» заключили договор. +1. Предмет договора +2.1. Стоимость товара составляет 1 250 000 рублей. +3.1. Договор вступает в силу с 01.09.2025 и действует по 31.08.2026. +4.1. Неустойка 0,1%. +5.1. Споры — в Арбитражном суде г. Москвы. +""" + +LOW_CONF_TEXT = "Образец документа без опознаваемых полей." + + +class StubProvider: + def __init__(self, payload: dict[str, Any] | Exception) -> None: + self._payload = payload + self.calls = 0 + + async def analyze(self, text: str, *, checklist: str, extra_context: str = "") -> Any: + raise NotImplementedError + + async def extract_prescreen(self, text: str) -> dict[str, Any]: + self.calls += 1 + if isinstance(self._payload, Exception): + raise self._payload + return self._payload + + async def aclose(self) -> None: + pass + + +LLM_DICT: dict[str, Any] = { + "contract_type": "services", + "party_a": "ИП Петров", + "party_b": "ООО «Клиент»", + "total_amount": 50000.0, + "currency": "RUB", + "has_penalty_clause": True, +} + + +def _counter(outcome: str) -> int: + return prescreen_fallback_runs.labels(outcome=outcome)._value.get() # type: ignore[no-any-return] + + +def _hybrid( + provider: StubProvider, *, enabled: bool, threshold: float = 0.75 +) -> HybridMetaExtractor: + return HybridMetaExtractor( + provider=provider, + fallback_enabled=enabled, + fallback_threshold=threshold, + heuristic=HeuristicExtractor(), + ) + + +async def test_disabled_never_calls_llm() -> None: + provider = StubProvider(LLM_DICT) + before = _counter("disabled") + result = await _hybrid(provider, enabled=False).extract(LOW_CONF_TEXT) + assert provider.calls == 0 + assert result.extractor_version == "heuristic-v2" + assert result.llm_fallback_error is None + assert _counter("disabled") == before + 1 + + +async def test_high_confidence_skips_llm() -> None: + provider = StubProvider(LLM_DICT) + before = _counter("skipped") + result = await _hybrid(provider, enabled=True).extract(HIGH_CONF_TEXT) + assert provider.calls == 0 + assert result.extractor_version == "heuristic-v2" + assert result.meta.confidence_score >= 0.75 + assert _counter("skipped") == before + 1 + + +async def test_low_confidence_merges_llm() -> None: + provider = StubProvider(LLM_DICT) + before = _counter("used") + result = await _hybrid(provider, enabled=True).extract(LOW_CONF_TEXT) + assert provider.calls == 1 + assert result.extractor_version == "hybrid-llm-v1" + assert result.llm_fallback_error is None + meta = result.meta + assert meta.contract_type == "services" + assert meta.party_a == "ИП Петров" + assert meta.party_b == "ООО «Клиент»" + assert meta.total_amount == 50000.0 + assert meta.confidence_score >= 0.75 + assert _counter("used") == before + 1 + + +async def test_llm_failure_keeps_heuristic_result() -> None: + provider = StubProvider(RuntimeError("boom")) + before = _counter("failed") + result = await _hybrid(provider, enabled=True).extract(LOW_CONF_TEXT) + assert provider.calls == 1 + assert result.extractor_version == "heuristic-v2" + assert result.llm_fallback_error is not None + assert "RuntimeError" in result.llm_fallback_error + assert result.meta.contract_type is None # heuristic result untouched + assert _counter("failed") == before + 1 + + +async def test_llm_overrides_only_heuristic_nones() -> None: + provider = StubProvider({**LLM_DICT, "contract_type": None}) + result = await _hybrid(provider, enabled=True).extract( + "Соглашение. ООО «Ромашка» действует.\nСумма 100 рублей." + ) + # Heuristic found nothing for type; LLM had None too → stays None. + assert result.meta.contract_type is None + assert result.extractor_version == "hybrid-llm-v1" + + +async def test_booleans_or_merged() -> None: + # Heuristic: no clause words → False. LLM: penalty True → OR → True. + provider = StubProvider({**LLM_DICT, "has_penalty_clause": True}) + result = await _hybrid(provider, enabled=True).extract(LOW_CONF_TEXT) + assert result.meta.has_penalty_clause is True + + +async def test_heuristic_boolean_survives_llm_false() -> None: + text = LOW_CONF_TEXT + " Стороны вправе расторгнуть договор." + provider = StubProvider({"has_termination_clause": False}) + result = await _hybrid(provider, enabled=True).extract(text) + assert result.meta.has_termination_clause is True + + +async def test_threshold_boundary_equal_skips_llm() -> None: + provider = StubProvider(LLM_DICT) + # LOW_CONF_TEXT scores 0.0; threshold 0.0 → 0.0 >= 0.0 → skipped. + result = await _hybrid(provider, enabled=True, threshold=0.0).extract(LOW_CONF_TEXT) + assert provider.calls == 0 + assert result.extractor_version == "heuristic-v2" + + +async def test_heuristic_only_failure_never_raised() -> None: + # The whole pipeline never raises from extraction stages on LLM errors. + provider = StubProvider(RuntimeError("quota exceeded")) + result = await _hybrid(provider, enabled=True).extract(HIGH_CONF_TEXT) + assert result.meta.confidence_score >= 0.75 diff --git a/tests/unit/test_prescreen_extractor_llm.py b/tests/unit/test_prescreen_extractor_llm.py new file mode 100644 index 0000000..605ea33 --- /dev/null +++ b/tests/unit/test_prescreen_extractor_llm.py @@ -0,0 +1,113 @@ +"""Unit tests for the Stage-2 worker-side wrapper (dict → validated meta).""" + +from __future__ import annotations + +from typing import Any + +from contract_check.worker_prescreen.extractor_llm import ( + EXTRACTOR_VERSION, + LLMPrescreenExtractor, + validate_raw, +) + +VALID: dict[str, Any] = { + "contract_type": "supply", + "party_a": "ООО «Альфа»", + "party_b": "ООО «Бета»", + "total_amount": 1250000, + "currency": "RUB", + "start_date": "2025-09-01", + "end_date": "2026-08-31", + "has_penalty_clause": True, + "has_termination_clause": False, + "has_arbitration": True, +} + + +def test_valid_dict_passes_through() -> None: + meta = validate_raw(VALID) + assert meta.contract_type == "supply" + assert meta.party_a == "ООО «Альфа»" + assert meta.total_amount == 1250000.0 + assert meta.currency == "RUB" + assert meta.start_date == "2025-09-01" + assert meta.has_penalty_clause is True + assert meta.has_arbitration is True + assert meta.confidence_score == 1.0 + + +def test_invalid_enum_becomes_none() -> None: + meta = validate_raw({**VALID, "contract_type": "lease-agreement"}) + assert meta.contract_type is None + + +def test_contract_type_normalized_to_lower() -> None: + meta = validate_raw({**VALID, "contract_type": "LEASE"}) + assert meta.contract_type == "lease" + + +def test_invalid_currency_becomes_none() -> None: + meta = validate_raw({**VALID, "currency": "CHF"}) + assert meta.currency is None + + +def test_currency_normalized_to_upper() -> None: + meta = validate_raw({**VALID, "currency": "byn"}) + assert meta.currency == "BYN" + + +def test_invalid_iso_date_becomes_none() -> None: + meta = validate_raw({**VALID, "start_date": "сентябрь 2025"}) + assert meta.start_date is None + + +def test_ddmmyyyy_date_accepted() -> None: + meta = validate_raw({**VALID, "start_date": "01.09.2025"}) + assert meta.start_date == "2025-09-01" + + +def test_impossible_date_becomes_none() -> None: + meta = validate_raw({**VALID, "start_date": "32.13.2025"}) + assert meta.start_date is None + + +def test_negative_amount_becomes_none() -> None: + meta = validate_raw({**VALID, "total_amount": -5}) + assert meta.total_amount is None + + +def test_non_bool_flags_become_none() -> None: + meta = validate_raw({**VALID, "has_penalty_clause": "да"}) + assert meta.has_penalty_clause is None + + +def test_empty_strings_become_none() -> None: + meta = validate_raw({**VALID, "party_a": " "}) + assert meta.party_a is None + + +def test_extract_version_constant() -> None: + assert EXTRACTOR_VERSION == "hybrid-llm-v1" + + +async def test_extractor_calls_provider_and_scores() -> None: + class StubProvider: + def __init__(self) -> None: + self.received: list[str] = [] + + async def analyze(self, text: str, *, checklist: str, extra_context: str = "") -> object: + raise NotImplementedError + + async def extract_prescreen(self, text: str) -> dict[str, Any]: + self.received.append(text) + return dict(VALID) + + async def aclose(self) -> None: + pass + + provider = StubProvider() + extractor = LLMPrescreenExtractor(provider) + meta = await extractor.extract("Договор поставки...") + assert provider.received == ["Договор поставки..."] + assert meta.contract_type == "supply" + assert meta.confidence_score == 1.0 diff --git a/tests/unit/test_prescreen_router.py b/tests/unit/test_prescreen_router.py new file mode 100644 index 0000000..e3cfa1d --- /dev/null +++ b/tests/unit/test_prescreen_router.py @@ -0,0 +1,94 @@ +"""Unit tests for the prescreen routing decision logic.""" + +from __future__ import annotations + +import pytest + +from contract_check.worker_prescreen.extractor import PrescreenContractMeta +from contract_check.worker_prescreen.router import PrescreenRouter + + +@pytest.fixture +def router() -> PrescreenRouter: + return PrescreenRouter( + confidence_threshold=0.75, + high_value_threshold=100_000.0, + auto_approve_enabled=False, + ) + + +@pytest.fixture +def complete_meta() -> PrescreenContractMeta: + return PrescreenContractMeta( + contract_type="supply", + party_a="ООО «А»", + party_b="ООО «Б»", + total_amount=50000.0, + currency="RUB", + start_date="2025-01-01", + end_date="2025-12-31", + has_penalty_clause=False, + has_termination_clause=False, + has_arbitration=False, + confidence_score=0.8, + ) + + +def test_high_value_routes_to_deep_analysis( + router: PrescreenRouter, complete_meta: PrescreenContractMeta +) -> None: + complete_meta.total_amount = 250_000.0 + assert router.decide(complete_meta) == "deep_analysis" + + +def test_penalty_clause_routes_to_deep_analysis( + router: PrescreenRouter, complete_meta: PrescreenContractMeta +) -> None: + complete_meta.has_penalty_clause = True + assert router.decide(complete_meta) == "deep_analysis" + + +def test_arbitration_routes_to_deep_analysis( + router: PrescreenRouter, complete_meta: PrescreenContractMeta +) -> None: + complete_meta.has_arbitration = True + assert router.decide(complete_meta) == "deep_analysis" + + +def test_low_confidence_routes_to_manual_review( + router: PrescreenRouter, complete_meta: PrescreenContractMeta +) -> None: + complete_meta.confidence_score = 0.5 + assert router.decide(complete_meta) == "manual_review" + + +def test_missing_party_routes_to_manual_review( + router: PrescreenRouter, complete_meta: PrescreenContractMeta +) -> None: + complete_meta.party_b = None + assert router.decide(complete_meta) == "manual_review" + + +def test_auto_approve_disabled_by_default( + router: PrescreenRouter, complete_meta: PrescreenContractMeta +) -> None: + # Low value, no risk clauses, high confidence — but auto_approve is off. + assert router.decide(complete_meta) == "manual_review" + + +def test_auto_approve_when_enabled(complete_meta: PrescreenContractMeta) -> None: + enabled_router = PrescreenRouter( + confidence_threshold=0.75, + high_value_threshold=100_000.0, + auto_approve_enabled=True, + ) + assert enabled_router.decide(complete_meta) == "auto_approve" + + +def test_summarize_returns_non_empty_string( + router: PrescreenRouter, complete_meta: PrescreenContractMeta +) -> None: + summary = router.summarize(complete_meta, "auto_approve") + assert "supply" in summary + assert "ООО «А»" in summary + assert "50000" in summary diff --git a/uv.lock b/uv.lock index 94a2c82..46341b0 100644 --- a/uv.lock +++ b/uv.lock @@ -483,6 +483,41 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" }, ] +[[package]] +name = "chardet" +version = "7.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/cd61c567092a6cec796144510a68aff158ebfc1df82950a45bae65f28413/chardet-7.6.0.tar.gz", hash = "sha256:93d9df6089ded42ed1fe9f57e272c0b74bd0464d45c0c7d50f09f26f31105c3c", size = 914462, upload-time = "2026-08-14T20:36:59.305Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/29/16a7419edfbd60e901e6a797cbc3e038cb2a81903bc16c029db755f0156f/chardet-7.6.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:57e6846cc13ce1ff59979f4ec9da770c57e12aa99046073f632de5a51d9a6f20", size = 1088449, upload-time = "2026-08-14T20:36:34.723Z" }, + { url = "https://files.pythonhosted.org/packages/1d/36/3a14b0f8ddeb302f157281ca656a3ce6874b78e2d6af03682f520b487245/chardet-7.6.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:089e3bb81a0a07e94f15461ded9f9ee66d349615b1a9fd557d4de1003e2fc12e", size = 1065126, upload-time = "2026-08-14T20:36:36.21Z" }, + { url = "https://files.pythonhosted.org/packages/d2/4c/f59a39c2bfe4ac99baba8da842e8d2ea0b84dff7ba53a96e7ad8c71602d7/chardet-7.6.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:43ea433e43a23c55e8e17f3fad1e07f5cfe5450c73124b95b0d849c21ad379ee", size = 1482492, upload-time = "2026-08-14T20:36:37.649Z" }, + { url = "https://files.pythonhosted.org/packages/bd/eb/93e8036681157f2217a18769927a035984526e6dbd5e91f28a375ca41c14/chardet-7.6.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2b5d31f9b7f793e15e81cca877e7ccd72bffffa2a3443a9d47be9dfee84fad69", size = 1514185, upload-time = "2026-08-14T20:36:39.119Z" }, + { url = "https://files.pythonhosted.org/packages/10/04/0066d7ab2c135e404a6fa166bb7fa49d1c7bf7af07b86ed95b1c48a348c7/chardet-7.6.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c54b6a8d3b219560fa5cf4c28df932c37471afe047afdc152067104e741f38c1", size = 1454420, upload-time = "2026-08-14T20:36:40.549Z" }, + { url = "https://files.pythonhosted.org/packages/6c/9f/e965f1d9eddfb86cf118f980d329cbee43c4ac4b562448b369a6b6ef36af/chardet-7.6.0-cp313-cp313-win_amd64.whl", hash = "sha256:b3b4c96c4df93899b3c8b9e8159e06b1f55c66d7ca384d91481108e251a06eb0", size = 1159067, upload-time = "2026-08-14T20:36:41.816Z" }, + { url = "https://files.pythonhosted.org/packages/cf/78/e14991c9487277ed7d006fff58f3084ac792ed94cced081788abb95df70c/chardet-7.6.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c6061adf247ab5dda173b67010e13904c6071717660c7c8077fb50aca362b264", size = 1087678, upload-time = "2026-08-14T20:36:43.2Z" }, + { url = "https://files.pythonhosted.org/packages/7a/40/0f95e04cb1820e0a582cd6d86bbf26be8302a94ccf330f8ba5f69735389d/chardet-7.6.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fc1e1571321baf8927582fe34363ad7f02279f11c8c2839c14b4c76894148db6", size = 1065310, upload-time = "2026-08-14T20:36:44.489Z" }, + { url = "https://files.pythonhosted.org/packages/3b/a1/04404dcc9d6e1253b02583e562d2c7ddca50a7e91f460d62eff7e1d07c92/chardet-7.6.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8900f6c7cf6b015b17a51767cc6144689059ba1cdceaa383d29eb037ac28579e", size = 1485028, upload-time = "2026-08-14T20:36:45.797Z" }, + { url = "https://files.pythonhosted.org/packages/f4/88/360064c4c7d9d0664561dae03b74c871d2f5332b329f5c99f1c997fb869a/chardet-7.6.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cedbc584789eb2edfde20fd03669972a833ce6019e60014ae613f9bfc440e8e3", size = 1510242, upload-time = "2026-08-14T20:36:47.206Z" }, + { url = "https://files.pythonhosted.org/packages/43/4c/302869fa1c69a5a41e4e78782680b12552ffd4286c3ae3c839b7b8df53d4/chardet-7.6.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d5dc835e40e0e09c2c3eab43731a8b5127834f42786dda09ba2f4b699ccd527a", size = 1456456, upload-time = "2026-08-14T20:36:48.668Z" }, + { url = "https://files.pythonhosted.org/packages/7c/a9/ff4fef15ed25fc3f945a3b981ae0f43c8559b3fbedb40267e59e583d105b/chardet-7.6.0-cp314-cp314-win_amd64.whl", hash = "sha256:0f304de7041afaec0195ad6464937cd112392002e9d72ed15d55f20a9abd3a13", size = 1159103, upload-time = "2026-08-14T20:36:50.107Z" }, + { url = "https://files.pythonhosted.org/packages/31/44/94e6f89485ce6c630e8fec6d388e3fa747e58b7246b0cc8c5c532ba98650/chardet-7.6.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a4f0a368ad04d5def08bdfaa17c7e15e71552f93923dc2aa9b2f7d9dee02fbb6", size = 1120302, upload-time = "2026-08-14T20:36:51.484Z" }, + { url = "https://files.pythonhosted.org/packages/e0/cd/8e68ba12f13aaf4ee82d54ef8a1f83d62942e7a6bc1e579dd2d530a3e26e/chardet-7.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:75d6c3a4d2046d49e83d2d2206eb073a1f390743e856d90c1bbc19949b26acf4", size = 1093157, upload-time = "2026-08-14T20:36:52.987Z" }, + { url = "https://files.pythonhosted.org/packages/f3/f6/6a35342b9efa69dcceab0ea8966571c6442a59c336bf460f0a2af95ed234/chardet-7.6.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:459e2b1c98f9a86a4698112aa42dffa802bbbff883c1ff144071f87224125862", size = 1521213, upload-time = "2026-08-14T20:36:54.812Z" }, + { url = "https://files.pythonhosted.org/packages/df/8c/8f09bfabbaedae45caa72b996e96d5e3f6436dfddc55c1311ffa2f6bd7d2/chardet-7.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bdb6f03107b7ace3f44e0edd91aa24456ee558787df265cc19daf45785b31c7", size = 1528427, upload-time = "2026-08-14T20:36:56.224Z" }, + { url = "https://files.pythonhosted.org/packages/0c/3d/2540627b193112e08b8045f3cad9295570866208555aa6625b63cc69c6cd/chardet-7.6.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:0c44a32da32cc8b23d6b20d98ace15ec7600950e4955d1bf5ab1f849b0187fdb", size = 1087374, upload-time = "2026-08-14T21:04:14.994Z" }, + { url = "https://files.pythonhosted.org/packages/fe/de/dac4f550cde73c4732b4916e72ec489ae36933fbbd1697e4122d3ae2e9dd/chardet-7.6.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:7bbc8a9652c7f859c593847f220c1d264f25749369abb1a267b404ee8cceb209", size = 1064688, upload-time = "2026-08-14T21:04:16.806Z" }, + { url = "https://files.pythonhosted.org/packages/06/45/f4f3f288496797d2cf1d1e9036f920ed2b4c79787b21c42a3314f4a6d532/chardet-7.6.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:83512a475a2f3886166aa0bca1bbb39343a4eb3186dd5532127d6f2591d09118", size = 1486320, upload-time = "2026-08-14T21:04:18.535Z" }, + { url = "https://files.pythonhosted.org/packages/c0/81/3ca30c16e6c6015b737fca22d9342957cc617d8a65329d3e963ad754a31b/chardet-7.6.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:271ab71ec1be61dbbce0436de0848895c03eae051c379e937a39573d9ce403d9", size = 1515374, upload-time = "2026-08-14T21:04:20.724Z" }, + { url = "https://files.pythonhosted.org/packages/6e/98/163a75b8b3372a6b6503f5f5a4154a222ac135c7a706b5d176f8e4b237a2/chardet-7.6.0-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:da86fc1b40ff5996fbb5e4c2d2dca770eac2c893cef157dacc050b8b4d929846", size = 1469504, upload-time = "2026-08-14T21:04:22.352Z" }, + { url = "https://files.pythonhosted.org/packages/d7/4f/3ad1b1bd27ae7f0d3d13cf711366525cd781a9e067950d8a09aa280916cb/chardet-7.6.0-cp315-cp315-win_amd64.whl", hash = "sha256:b73f277c1ac09c4f8076c4214b816c7aa78a0a2f0cb7156742f4303f856bedc3", size = 1158551, upload-time = "2026-08-14T21:04:24.259Z" }, + { url = "https://files.pythonhosted.org/packages/8d/92/22a609c68c5123ebdccd8fe1a80e423609012ce20166de19a2ed3955b2f7/chardet-7.6.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:61238d5945b36af9a2ad13494f8969b7deb3c3b4abe223e54670c064e73f5328", size = 1119186, upload-time = "2026-08-14T21:04:25.761Z" }, + { url = "https://files.pythonhosted.org/packages/29/35/75a90143e4200e197f4f6cf5895d379e22bc785d0c7711120df18fa5347c/chardet-7.6.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:f2ec3c78cc6b54bf8e091ec4ee885473078b5d7ef18ab1b01c86ae1e98bf88f7", size = 1092596, upload-time = "2026-08-14T21:04:27.884Z" }, + { url = "https://files.pythonhosted.org/packages/d1/2c/b6d5f47d878c46e04ae6fcb58e0969467925bc0819b333c682e0c41bbc5c/chardet-7.6.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:167d7ba3ee08b654e36d7b43ebd9a36606c9a12e2fabdb361757a095ca3b7e3d", size = 1516644, upload-time = "2026-08-14T21:04:29.624Z" }, + { url = "https://files.pythonhosted.org/packages/25/d2/2bc3f1066c6f27bc3fa3a98dfd8a2a9c1e969a9d6bdc1edcd35278791fc4/chardet-7.6.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f14f46ef1977e41ce1f4814ca6984cea7f8b6baf8cbc6626ef7bf3d13cf7ea13", size = 1533221, upload-time = "2026-08-14T21:04:31.39Z" }, + { url = "https://files.pythonhosted.org/packages/cf/6e/5a0b348fa4cd7847567a28c6e697ccf58391960bfd13a6e7473ee23ca2f2/chardet-7.6.0-py3-none-any.whl", hash = "sha256:4076d795897ce45239825956a1334e134322ecc4bfe84dbb12acd5390de0fbc1", size = 680279, upload-time = "2026-08-14T20:36:57.763Z" }, +] + [[package]] name = "charset-normalizer" version = "3.4.9" @@ -543,6 +578,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, ] +[[package]] +name = "cobble" +version = "0.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/7a/a507c709be2c96e1bb6102eb7b7f4026c5e5e223ef7d745a17d239e9d844/cobble-0.1.4.tar.gz", hash = "sha256:de38be1539992c8a06e569630717c485a5f91be2192c461ea2b220607dfa78aa", size = 3805, upload-time = "2024-06-01T18:11:09.528Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/e1/3714a2f371985215c219c2a70953d38e3eed81ef165aed061d21de0e998b/cobble-0.1.4-py3-none-any.whl", hash = "sha256:36c91b1655e599fd428e2b95fdd5f0da1ca2e9f1abb0bc871dec21a0e78a2b44", size = 3984, upload-time = "2024-06-01T18:11:07.911Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -618,9 +662,12 @@ dev = [ { name = "argon2-cffi" }, { name = "asgi-lifespan" }, { name = "asyncpg" }, + { name = "chardet" }, { name = "email-validator" }, { name = "fastapi" }, + { name = "isort" }, { name = "jinja2" }, + { name = "mammoth" }, { name = "minio" }, { name = "mypy" }, { name = "opentelemetry-exporter-otlp" }, @@ -637,19 +684,24 @@ dev = [ { name = "pytest" }, { name = "pytest-asyncio" }, { name = "python-docx" }, + { name = "python-magic" }, { name = "python-multipart" }, { name = "redis" }, { name = "respx" }, { name = "ruff" }, { name = "sentry-sdk" }, { name = "sqlalchemy" }, + { name = "striprtf" }, { name = "testcontainers", extra = ["minio", "rabbitmq"] }, + { name = "ty" }, { name = "uvicorn", extra = ["standard"] }, ] extract = [ { name = "aio-pika" }, { name = "alembic" }, { name = "asyncpg" }, + { name = "chardet" }, + { name = "mammoth" }, { name = "minio" }, { name = "opentelemetry-exporter-otlp" }, { name = "opentelemetry-sdk" }, @@ -657,8 +709,10 @@ extract = [ { name = "prometheus-client" }, { name = "pymupdf" }, { name = "pytesseract" }, + { name = "python-magic" }, { name = "sentry-sdk" }, { name = "sqlalchemy" }, + { name = "striprtf" }, ] mq = [ { name = "aio-pika" }, @@ -751,9 +805,12 @@ dev = [ { name = "argon2-cffi", specifier = ">=23.1" }, { name = "asgi-lifespan", specifier = ">=2.1.0" }, { name = "asyncpg", specifier = ">=0.29" }, + { name = "chardet", specifier = ">=5.2" }, { name = "email-validator", specifier = ">=2.1" }, { name = "fastapi", specifier = ">=0.110" }, + { name = "isort", specifier = ">=5.13" }, { name = "jinja2", specifier = ">=3.1" }, + { name = "mammoth", specifier = ">=1.8" }, { name = "minio", specifier = ">=7.2" }, { name = "mypy", specifier = ">=1.10" }, { name = "opentelemetry-exporter-otlp", specifier = ">=1.24" }, @@ -770,19 +827,24 @@ dev = [ { name = "pytest", specifier = ">=8" }, { name = "pytest-asyncio", specifier = ">=0.23" }, { name = "python-docx", specifier = ">=1.1" }, + { name = "python-magic", specifier = ">=0.4.27" }, { name = "python-multipart", specifier = ">=0.0.9" }, { name = "redis", specifier = ">=5.0" }, { name = "respx", specifier = ">=0.21" }, { name = "ruff", specifier = ">=0.5" }, { name = "sentry-sdk", specifier = ">=2" }, { name = "sqlalchemy", specifier = ">=2.0" }, + { name = "striprtf", specifier = ">=0.0.26" }, { name = "testcontainers", extras = ["rabbitmq", "postgres", "minio"], specifier = ">=4" }, + { name = "ty", specifier = ">=0.0.72" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.29" }, ] extract = [ { name = "aio-pika", specifier = ">=9.4" }, { name = "alembic", specifier = ">=1.13" }, { name = "asyncpg", specifier = ">=0.29" }, + { name = "chardet", specifier = ">=5.2" }, + { name = "mammoth", specifier = ">=1.8" }, { name = "minio", specifier = ">=7.2" }, { name = "opentelemetry-exporter-otlp", specifier = ">=1.24" }, { name = "opentelemetry-sdk", specifier = ">=1.24" }, @@ -790,8 +852,10 @@ extract = [ { name = "prometheus-client", specifier = ">=0.20" }, { name = "pymupdf", specifier = ">=1.24" }, { name = "pytesseract", specifier = ">=0.3.10" }, + { name = "python-magic", specifier = ">=0.4.27" }, { name = "sentry-sdk", specifier = ">=2" }, { name = "sqlalchemy", specifier = ">=2.0" }, + { name = "striprtf", specifier = ">=0.0.26" }, ] mq = [{ name = "aio-pika", specifier = ">=9.4" }] notify = [ @@ -1229,6 +1293,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "isort" +version = "8.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ef/7c/ec4ab396d31b3b395e2e999c8f46dec78c5e29209fac49d1f4dace04041d/isort-8.0.1.tar.gz", hash = "sha256:171ac4ff559cdc060bcfff550bc8404a486fee0caab245679c2abe7cb253c78d", size = 769592, upload-time = "2026-02-28T10:08:20.685Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/95/c7c34aa53c16353c56d0b802fba48d5f5caa2cdee7958acbcb795c830416/isort-8.0.1-py3-none-any.whl", hash = "sha256:28b89bc70f751b559aeca209e6120393d43fbe2490de0559662be7a9787e3d75", size = 89733, upload-time = "2026-02-28T10:08:19.466Z" }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -1410,6 +1483,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a5/54/12ed58d458474aaab5c3d180173e745a4fe131bb330370596876d19ff60f/mako-1.4.1-py3-none-any.whl", hash = "sha256:a359d9a94a541213958742b2698d0a7757bb83551767bc468a74b9905aba9617", size = 80010, upload-time = "2026-08-05T06:10:58.248Z" }, ] +[[package]] +name = "mammoth" +version = "1.12.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cobble" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/46/92/aca6c4208b3a8e56a788b224e6c9b7bb70fc68e6641124fcf784008338e1/mammoth-1.12.1.tar.gz", hash = "sha256:40521e02568583b671d9976b0fbeed50be734b40f73ca8e1939ab7146bf69bbe", size = 53797, upload-time = "2026-08-09T14:11:20.096Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/b4/e7907b278386ffec0cf631db93836e02de9a6386add4653a52d7ac6b43ae/mammoth-1.12.1-py2.py3-none-any.whl", hash = "sha256:2af047e3e796faa25740112310ddf11f8de2a24c96dc57de3c87dfd7cb6543b3", size = 55091, upload-time = "2026-08-09T14:11:18.854Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -2272,6 +2357,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, ] +[[package]] +name = "python-magic" +version = "0.4.27" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/da/db/0b3e28ac047452d079d375ec6798bf76a036a08182dbb39ed38116a49130/python-magic-0.4.27.tar.gz", hash = "sha256:c1ba14b08e4a5f5c31a302b7721239695b2f0f058d125bd5ce1ee36b9d9d3c3b", size = 14677, upload-time = "2022-06-07T20:16:59.508Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/73/9f872cb81fc5c3bb48f7227872c28975f998f3e7c2b1c16e95e6432bbb90/python_magic-0.4.27-py2.py3-none-any.whl", hash = "sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3", size = 13840, upload-time = "2022-06-07T20:16:57.763Z" }, +] + [[package]] name = "python-multipart" version = "0.0.32" @@ -2462,6 +2556,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/75/0a/67e95f21498de41433babf7b1db0eeab449eb58872dfb831b27747a70fd0/starlette-1.4.1-py3-none-any.whl", hash = "sha256:7d078e0fbefae0d2cecfb80a799d6fb84b1c0c6acd4f14ac79d17d0e7ec27f19", size = 74019, upload-time = "2026-08-05T15:17:24.357Z" }, ] +[[package]] +name = "striprtf" +version = "0.0.32" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/86/f9/b3ecbd2e0abb9c29edd2913482f0c5df47a098221444ccd50cc5cd3f9b91/striprtf-0.0.32.tar.gz", hash = "sha256:7f375a375d99a2700842173168c90c9b545cb244241ffc5d868ed9f6baf9155f", size = 7571, upload-time = "2026-04-27T15:42:37.15Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/cf/4157959aa36a567faab32a5fb078904b76654bdd86a3d7f8fba21ed89d84/striprtf-0.0.32-py3-none-any.whl", hash = "sha256:9f695fea9662cb4ffccf2b56286b3e77028c0a61e87258719584e681e0144869", size = 8044, upload-time = "2026-04-27T15:42:35.948Z" }, +] + [[package]] name = "structlog" version = "26.1.0" @@ -2495,6 +2598,31 @@ rabbitmq = [ { name = "pika" }, ] +[[package]] +name = "ty" +version = "0.0.72" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d5/df/656e684bafb13c1d146e7d5b5f3e7978ca177232acc84998ff36427e9462/ty-0.0.72.tar.gz", hash = "sha256:ec2b8066b618df18cab4cb8e992f8da45d360332acb23fa34df7fa29cd1b9d3a", size = 6654939, upload-time = "2026-08-14T21:35:42.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/3b/f51461239a4e66565d4b362f97a3b55fe7fdba2e944068341f87c62f6743/ty-0.0.72-py3-none-linux_armv6l.whl", hash = "sha256:fda86db153ffd85ee52000cf175d6a3f1c0223772cf7c5b6f726200bf92c7b44", size = 12621989, upload-time = "2026-08-14T21:35:01.676Z" }, + { url = "https://files.pythonhosted.org/packages/ca/fb/79ddf683affc679ca856f3510b5640ec3a88a842ba5f654f5d4bc78f1786/ty-0.0.72-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ceb944c612529b9023acfdc9cf4c0dcbb722549f9d17d46baecd1141baf01d7f", size = 12233910, upload-time = "2026-08-14T21:35:04.334Z" }, + { url = "https://files.pythonhosted.org/packages/5d/45/10562a0d84802158db8fa4ec46de54aa9fdcecdeeaabbfe3639ae7042b66/ty-0.0.72-py3-none-macosx_11_0_arm64.whl", hash = "sha256:108d76218333d6c092e5f1cebf8e9b06f25738613a0236a28e2dd47c936ee52c", size = 12084108, upload-time = "2026-08-14T21:35:06.686Z" }, + { url = "https://files.pythonhosted.org/packages/a1/dc/1fe1aef8d697e3509face271a5331700c7aa1d1e44a4b622707bdfa41d4b/ty-0.0.72-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f3943f186f741a2499a31053872169250c9264a9a49684920e48d8fcf4ef4f5", size = 12132640, upload-time = "2026-08-14T21:35:09.305Z" }, + { url = "https://files.pythonhosted.org/packages/14/46/41ceb265e96969487311a2014bd0e53abb4fbc1395efb2ebe411fcb4db62/ty-0.0.72-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cf283c07dc3cc52ca48a3ad8ab100fb5aec3aebbd03ef6a12d5f910b8e596fc5", size = 12402489, upload-time = "2026-08-14T21:35:11.555Z" }, + { url = "https://files.pythonhosted.org/packages/2b/45/30bf43cb4fd505c5c2dd30fda27dde5f05208686cd21217adec77c954204/ty-0.0.72-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:95f3b6462c38f9f115d10cee21f47fedf715fcf2040daf36eef210359300bc7c", size = 13130835, upload-time = "2026-08-14T21:35:13.746Z" }, + { url = "https://files.pythonhosted.org/packages/31/2f/03bba754d2613f640df168335c41f83f41db150bb515839c60d80e3a7880/ty-0.0.72-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:30caf658feb8ffb250d9e9e47107657a78f5f3425c227df1664d8df2ebe38880", size = 13590392, upload-time = "2026-08-14T21:35:16.839Z" }, + { url = "https://files.pythonhosted.org/packages/04/c7/03c67f00e63005ec41585653dc3096064570b1e6273742baae2798cd242f/ty-0.0.72-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:27bdc012ddfbeec8948e4a6036c0dc39ac7cf2c8ec7c7d48dc7d2fd56d57b399", size = 13309629, upload-time = "2026-08-14T21:35:19.169Z" }, + { url = "https://files.pythonhosted.org/packages/c1/df/102d3b264eb7f2a58dd11952f229bb5150bb5668d176a6154976a6675981/ty-0.0.72-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:802c5970a77d7739e6f499921fbb6984fb7ad8a31d95e1ff42fd46f3642e4f3b", size = 12734028, upload-time = "2026-08-14T21:35:22.099Z" }, + { url = "https://files.pythonhosted.org/packages/61/85/d0737c8c54d0ba67366ddfb9f31d88edf0b02299e65923e6945ae60ebcb5/ty-0.0.72-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:47dce65114fdc615c68ca0edb393b433df0956447e4267df0e264137a789598d", size = 13174832, upload-time = "2026-08-14T21:35:24.71Z" }, + { url = "https://files.pythonhosted.org/packages/1e/31/497f5a96c36d9b586ab6afe0574986835c6fd5b835a89773d2bec4711b49/ty-0.0.72-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:325144fa07e2675d0faa337fcc864213c272a499eb0cfe5bde2fdc62282d27bc", size = 12215005, upload-time = "2026-08-14T21:35:26.892Z" }, + { url = "https://files.pythonhosted.org/packages/df/7d/46e65b17b4966c7cd0140f134380d33d8e84fe6efccd761533ce793dc502/ty-0.0.72-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a5c9f15d0f58e43707d8848274be1821a0ef408eccb8aa7dda28a4a9eddf7640", size = 12421298, upload-time = "2026-08-14T21:35:29.301Z" }, + { url = "https://files.pythonhosted.org/packages/08/2a/12ada4ec17700b3cb1d4fd3bc3e5b1852df9e6885288429318cade87b3c1/ty-0.0.72-py3-none-musllinux_1_2_i686.whl", hash = "sha256:8ee508d64b381871529cc22c412b41071bf5e908b7aa5d66a38f3f6b2573a806", size = 12669242, upload-time = "2026-08-14T21:35:31.444Z" }, + { url = "https://files.pythonhosted.org/packages/1c/1a/4692536880790fb550ed6d44a6096778dc71bb112f2c6d615cebb01a57e5/ty-0.0.72-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:3699e2ec7921d44da79d6b089f7bf239b2cc53c4e45a5a38430adc34ee9e9a55", size = 12988199, upload-time = "2026-08-14T21:35:33.749Z" }, + { url = "https://files.pythonhosted.org/packages/9a/0d/f5e5a50322e9c45865e7b7a428ba6cd6527387cf0f2472492ac3cf746243/ty-0.0.72-py3-none-win32.whl", hash = "sha256:f25f72a67bd36cd247707c4784e52fad0b6b4f42a1b7dd14804110fa95c486ed", size = 11939708, upload-time = "2026-08-14T21:35:36.006Z" }, + { url = "https://files.pythonhosted.org/packages/3f/4e/8af3534b2e4214e6184a5a59c34101e94a68d578f081f97b995866bab1bf/ty-0.0.72-py3-none-win_amd64.whl", hash = "sha256:cdeee869341717e1736cea2e2d7856738c6957c320f584ed2f68c8f90100d2f5", size = 12643876, upload-time = "2026-08-14T21:35:38.141Z" }, + { url = "https://files.pythonhosted.org/packages/ff/ea/a2606e654c7276bd08586391a2525b0af3f3bf60228a8c57b2d248f273f9/ty-0.0.72-py3-none-win_arm64.whl", hash = "sha256:1bd3ac3ed4424a6d6990a85dc388556aea012bd752de21349a84b685951de0d8", size = 12394857, upload-time = "2026-08-14T21:35:40.277Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0"