diff --git a/README.md b/README.md index f1a2302..afa6cee 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,6 @@ Telegram ──► bot (aiogram, HTTP-only) ──HTTP──► api (FastAPI) ``` src/contract_check/ - __main__.py # указывает на prototype (stage-0 CLI сохранён) core/ # общий домен (импортируется каждым сервисом) config.py logging.py telemetry.py sentry.py metrics.py errors.py credits.py tokens.py api_keys.py rate_limit.py redis_client.py @@ -91,10 +90,9 @@ src/contract_check/ scheduler.py __main__.py bot/ # aiogram-адаптер (самый «тощий» образ: только core.logging) client.py config.py handlers.py rate_limit.py __main__.py - prototype/ # stage-0 standalone CLI (бенчмарк go/no-go) docs/ # документация проекта (README остаётся в корне) srv/ # Dockerfile-ы (один на сервис, deps заточены) - api/ worker-extract/ worker-prescreen/ worker-analyze/ worker-notify/ worker-billing/ bot/ prototype/ + api/ worker-extract/ worker-prescreen/ worker-analyze/ worker-notify/ worker-billing/ bot/ migrations/ # alembic (async): 0001_initial … 0011_user_profiles_billing tests/ conftest.py @@ -110,7 +108,7 @@ tests/ admin_billing Makefile # повседневные команды (make help) docker-compose.yml # default = инфра; --profile services = стек; --profile edge = nginx+certbot -pyproject.toml # hatchling + PEP 735 dependency-groups (db/mq/s3/obs/api/extract/prescreen/analyze/notify/billing/bot/prototype/dev) +pyproject.toml # hatchling + PEP 735 dependency-groups (db/mq/s3/obs/api/extract/prescreen/analyze/notify/billing/bot/dev) .env.example # полный список env (см. docs/ARCHITECTURE.md §11) rustfs-spike/ # артефакты спайка RustFS (docs/SPIKE_PHASE0.md) ``` @@ -155,15 +153,6 @@ docker compose --profile bot up -d --build # + bot (можно запус RabbitMQ AMQP `5672` / UI `15672`, MinIO `9000` / console `9001`, api `8000` / metrics `9100`, worker metrics: extract `9101`, analyze `9102`, notify `9103`, prescreen `9104`, billing `9105`, edge `80`/`443`. -### Stage-0 прототип (бенчмарк) - -```bash -uv sync --group prototype -uv run python -m contract_check prototype contract.pdf # отчёт в stdout -uv run contract-check contract.pdf -o report.md # или консольная команда -uv run python -m contract_check prototype contract.pdf --json metrics.json # + метрики go/no-go -``` - ## API (кратко) JSON-роуты под `/api/v1`; серверный admin UI — по `/admin/*` (FastAPI + Jinja2 + HTMX). Auth зависит от роута: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 845cdb6..2f9aa93 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -113,8 +113,8 @@ Control plane: | Area | Decision | Replaces | |---|---|---| -| Repo | Monorepo, shared `contract_check.core`, 7 Dockerfiles in `srv//` | one-image `MODE` dispatch | -| Scope now | api + worker-extract + worker-prescreen + worker-analyze + worker-billing + worker-notify + bot (+ prototype benchmark) | (web later) | +| Repo | Monorepo, shared `contract_check.core`, 6 Dockerfiles in `srv//` | one-image `MODE` dispatch | +| Scope now | api + worker-extract + worker-prescreen + worker-analyze + worker-billing + worker-notify + bot | (web later) | | Queue | RabbitMQ, direct exchange, pipeline fan-out | arq + Redis | | Retry | TTL retry queues, exponential backoff, final DLQ | — | | Workers | Three pools (extract/OCR + prescreen + analyze/LLM) | single arq worker | @@ -134,7 +134,7 @@ Control plane: | Refund | Policy switch `all\|infra_only` + failure classes | refund-all | | Edge | Nginx + certbot | — | | Deploy | Compose now, k8s-ready later | — | -| Prototype | Kept as standalone benchmark | — | +| Prototype | Removed (stage-0 standalone benchmark no longer needed) | Kept as standalone benchmark | | Tests | pytest+respx unit + testcontainers integration | — | | Python | **3.13** (was 3.14) — wheel availability | py3.14 | | Landing | Incremental, green per step | — | @@ -160,8 +160,7 @@ DealDocumentScreening/ │ ├── api/Dockerfile (new) │ ├── worker-extract/Dockerfile (new, tesseract layer) │ ├── worker-analyze/Dockerfile (new, lean) -│ ├── bot/Dockerfile (new) -│ └── prototype/Dockerfile (new, preserves stage-0 CLI) +│ └── bot/Dockerfile (new) ├── deploy/ (PLANNED — not yet built; lands with §20 steps 6 / E1-009..010) │ ├── nginx/ │ │ ├── templates/contract-check.conf.template (reverse proxy + TLS) @@ -215,9 +214,9 @@ DealDocumentScreening/ │ │ │ ├── factory.py (mime/suffix → adapter; SUPPORTED_SUFFIXES gate) │ │ │ ├── adapters/ (pdf_pymupdf, docx_mammoth, rtf_striprtf, txt_chardet, ocr_tesseract) │ │ │ └── __init__.py (extract_document orchestrator with OCR fallback) -│ │ ├── analysis/ (existing; now consumed by prototype + ocr adapter) +│ │ ├── analysis/ (existing; now consumed by LLM adapter + ocr adapter) │ │ │ ├── __init__.py -│ │ │ ├── extractor.py (← from contract_check/extractor.py; prototype use) +│ │ │ ├── extractor.py (← from contract_check/extractor.py; worker_extract use) │ │ │ ├── chunker.py (← from contract_check/chunker.py; + chunk_markdown) │ │ │ ├── checklist.py (← from contract_check/checklist.py) │ │ │ ├── report_schema.py (← from contract_check/report_schema.py) @@ -260,15 +259,12 @@ DealDocumentScreening/ │ │ ├── __main__.py │ │ ├── handler.py (handle AnalyzeRequested: text dl→chunk→LLM (with prescreen context)→validate→save report→done) │ │ └── consumer.py -│ ├── bot/ (new — aiogram adapter, HTTP-only) -│ │ ├── __init__.py -│ │ ├── __main__.py -│ │ ├── config.py (BotSettings: BOT_TOKEN, API_URL, BOT_SERVICE_TOKEN, poll tuning) -│ │ ├── client.py (ApiClient: typed httpx wrapper; ApiError/NoCreditsError/...) -│ │ └── handlers.py (/start, doc upload→POST api, poll→send report) -│ └── prototype/ (moved from contract_check/prototype.py — kept as benchmark) +│ └── bot/ (new — aiogram adapter, HTTP-only) │ ├── __init__.py -│ └── __main__.py (stage-0 CLI, imports updated to core.analysis.*) +│ ├── __main__.py +│ ├── config.py (BotSettings: BOT_TOKEN, API_URL, BOT_SERVICE_TOKEN, poll tuning) +│ ├── client.py (ApiClient: typed httpx wrapper; ApiError/NoCreditsError/...) +│ └── handlers.py (/start, doc upload→POST api, poll→send report) └── tests/ ├── conftest.py (fixtures: testcontainers pg/rabbit/minio) ├── unit/ @@ -302,7 +298,7 @@ DealDocumentScreening/ `core.db`, `core.s3`, `core.llm`, `core.mq`, `core.credits`.** It speaks to the api over HTTP only. (Add a ruff/flake8 import-forbidden rule or a unit test that asserts this.) -- `prototype/*` imports `core.analysis.*` but is otherwise standalone (no DB/MQ). + --- @@ -1218,7 +1214,6 @@ Per-service differences (`uv` uses PEP 735 dependency-groups — see `pyproject. | `srv/worker-prescreen/Dockerfile` | `--group prescreen` | none | `python -m contract_check.worker_prescreen` | 9104 | | `srv/worker-analyze/Dockerfile` | `--group analyze` | none | `python -m contract_check.worker_analyze` | 9102 | | `srv/bot/Dockerfile` | `--group bot` | none | `python -m contract_check.bot` | — | -| `srv/prototype/Dockerfile` | `--group prototype` | none | `python -m contract_check.prototype` | — | The bot image is the leanest (no DB driver, no S3 client, no pymupdf). The analyze image has httpx but no tesseract/pymupdf. The extract image is the @@ -1266,10 +1261,9 @@ analyze = [{ include-group = "db" }, { include-group = "mq" }, prescreen = [{ include-group = "db" }, { include-group = "mq" }, { include-group = "s3" }, { include-group = "obs" }] bot = ["aiogram>=3.4"] -prototype = ["pymupdf>=1.24", "python-docx>=1.1"] dev = [{ include-group = "api" }, { include-group = "extract" }, { include-group = "prescreen" }, { include-group = "analyze" }, - { include-group = "bot" }, { include-group = "prototype" }, + { include-group = "bot" }, "pytest>=8", "pytest-asyncio>=0.23", "respx>=0.21", "ruff>=0.5", "mypy>=1.10", "anyio>=4", "aiosqlite>=0.20", "testcontainers[rabbitmq,postgres,minio]>=4", "asgi-lifespan>=2.1.0"] @@ -1427,7 +1421,7 @@ enforcing the adapter boundary even in dependency ordering. - `test_chunker.py` — `chunk_text` + `chunk_markdown`. - `test_extraction_adapters.py`, `test_extraction_factory.py` — adapters and factory. -- `test_extractor.py` — legacy prototype extractor path remains valid. +- `test_extractor.py` — `core.analysis.extractor` text extraction path. - `test_llm_ollama_cloud.py` — **respx** mocks: 200 happy, 429→fallback, invalid-JSON→repair→success, invalid-JSON→repair→fail, timeout. - `test_credits_policy.py` — `reserve_credit` never negative; `refund_credit` @@ -1701,6 +1695,7 @@ Each step is a verifiable unit. Do not start step N+1 until N is green - Alembic: init + initial migration (6 tables). - Migrate `prototype.py` → `prototype/__main__.py` (update imports to `core.analysis.*`). +- *Later removed:* stage-0 `prototype/` standalone CLI deleted as no longer needed. - **DoD**: `docker compose up` infra healthy; `alembic upgrade head` + `downgrade base` clean; `ruff && mypy && pytest` green (unit tests ported for chunker/extractor/credits/messages). diff --git a/docs/IMPLEMENTATION_PLAN.md b/docs/IMPLEMENTATION_PLAN.md index 9018892..30b4fd6 100644 --- a/docs/IMPLEMENTATION_PLAN.md +++ b/docs/IMPLEMENTATION_PLAN.md @@ -128,11 +128,11 @@ prompt с чек-листом → Ollama Cloud (format: json) → отчёт в ``` contract_check/ ├── pyproject.toml # hatchling, deps (core + adapters) -├── docker-compose.yml # postgres, redis, api, worker, bot (+ profile app = prototype) +├── docker-compose.yml # postgres, redis, api, workers, bot (profiles: services, bot, edge) ├── .env.example ├── src/contract_check/ │ ├── main.py # диспетчер: MODE=api|worker|bot|cli -│ ├── prototype.py # stage-0 standalone (in-process LLM) — НЕ в hexagonal +│ ├── prototype.py # stage-0 standalone (in-process LLM) — удалён, больше не нужен │ │ # ── ЯДРО (application core): владеет БД/S3/кредитами/LLM ── │ ├── api.py # FastAPI: /documents, /reports/{id}, /me, /healthz │ ├── worker.py # arq analyze_document(doc_id): S3→extract→OCR→analyze→Report diff --git a/docs/TICKETS.md b/docs/TICKETS.md index 162dbc9..59d60eb 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -2,8 +2,8 @@ > **Актуальная архитектура:** `ARCHITECTURE.md` (supersedes `IMPLEMENTATION_PLAN.md` для этапов 1+). > Состояние кода на момент синхронизации: реализованы `api/` (вкл. B2B-роуты, web-auth, passkeys, magic links), -> `core/`, `worker_extract/`, `worker_prescreen/`, `worker_analyze/`, `worker_notify/`, `bot/`, `prototype/`; -> 7 Dockerfile-ов в `srv/`; `docker-compose.yml` разводит профили `services` и `edge`; +> `core/`, `worker_extract/`, `worker_prescreen/`, `worker_analyze/`, `worker_notify/`, `bot/`; +> 6 Dockerfile-ов в `srv/`; `docker-compose.yml` разводит профили `services` и `edge`; > миграции `0001_initial` … `0010_passkeys_magic_links`; unit- и интеграционные тесты зелёные. > DoD: `ruff check src tests`, `ruff format --check src tests`, `uv run ty check src`, `pytest` зелёные; > `.env.example` актуален. @@ -19,7 +19,7 @@ | Компонент | Статус | Доказательство / пробел | |-----------|--------|---------------------------| -| Stage 0 prototype | done | `src/contract_check/prototype/` работает; `tests/unit/test_checklist_report.py`, `test_chunker.py`, `test_extractor.py` зелёные. | +| Stage 0 prototype | removed | Удалён: standalone CLI больше не нужен; `tests/unit/test_checklist_report.py`, `test_chunker.py`, `test_extractor.py` зелёные. | | `core/` — общий домен | done | `db/` (models, session, enums, repositories), `mq/`, `s3/`, `llm/`, `analysis/`, `extraction/`, `notifications/`, `security/`, `credits.py`, `tokens.py`, `api_keys.py`, `rate_limit.py`, `redis_client.py`, `config.py`, `logging.py`, `metrics.py`, `telemetry.py`, `sentry.py`, `passkeys.py`. | | Миграции / БД | done | `0001_initial.py` (6 таблиц) … `0010_passkeys_magic_links.py` (passkeys, magic links, user name). | | `api/` — FastAPI ядро | done | `POST /api/v1/documents`, `GET /api/v1/reports/{id}`, `GET /api/v1/me`, `/healthz`, `/readyz`, `/metrics`; user JWT auth (`/api/v1/auth/telegram/*`, `/api/v1/auth/{register,login,...}`, passkeys, magic links); B2B `/api/v1/analyze`; `/admin/*`; reserve-on-enqueue (`services.py`). | @@ -28,7 +28,7 @@ | `worker_analyze/` | done | `consumer.py`/`handler.py`/`__main__.py`: consume `DocumentExtracted` → LLM → Report → `status=done`; refund-on-DLQ по политике. | | `worker_notify/` | done | `consumer.py`/`handler.py`/`__main__.py`: consume `NotificationMessage` → SMTP (password reset, magic link) или dev-лог при пустом `SMTP_HOST`. | | `bot/` — Telegram adapter | done | `client.py`/`config.py`/`handlers.py`/`__main__.py`: `/start`, upload→`POST /documents`, poll→deliver; граница импортов проверяется `tests/unit/test_bot_boundary.py`. | -| Dockerfile-ы | done | `srv/{api,worker-extract,worker-prescreen,worker-analyze,worker-notify,bot,prototype}/Dockerfile` — все 7 (deps-группы PEP 735 заточены на сервис). | +| Dockerfile-ы | done | `srv/{api,worker-extract,worker-prescreen,worker-analyze,worker-notify,bot}/Dockerfile` — все 6 (deps-группы PEP 735 заточены на сервис). | | Docker Compose | done | Инфра (default) + профиль `services` (api + worker-ы) + профиль `bot` (Telegram-адаптер, можно на отдельном хосте) + профиль `edge` (nginx+certbot) с `depends_on: service_healthy`. Профили `obs`/`observer` — позже. | | Observability | in_progress | Prometheus-метрики (`/metrics`), Sentry, OpenTelemetry SDK — в коде. Полный стек Prom/Grafana/Tempo/OTel-collector — позже. | | Stage 3 — B2B API | done | `api/routes/b2b.py`, `core/api_keys.py`, `core/rate_limit.py`, `core/redis_client.py`, миграция `0002_api_keys.py`, `tests/integration/test_b2b_api.py`, `tests/unit/test_rate_limit.py`; `X-API-Key` auth + token-bucket rate-limit. | diff --git a/pyproject.toml b/pyproject.toml index 4e44f79..db8075c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,11 +23,6 @@ dependencies = [ "httpx[http2]>=0.27", ] -[project.scripts] -# Stage-0 benchmark CLI (standalone, kept). Production services have their own -# entrypoints via `python -m contract_check.`. -contract-check = "contract_check.prototype:main" - [tool.hatch.build.targets.wheel] packages = ["src/contract_check"] @@ -39,7 +34,6 @@ packages = ["src/contract_check"] # worker-extract → core + db/mq/s3/obs + pymupdf/tesseract # worker-analyze → core + db/mq/s3/obs + otel-httpx # bot → core + aiogram (leanest image: no db/mq/s3/llm/tesseract) -# prototype → core + pymupdf/python-docx (benchmark CLI) # dev → everything needed to lint/typecheck/test locally # ───────────────────────────────────────────────────────────────────────────── [dependency-groups] @@ -123,17 +117,12 @@ billing = [ "opentelemetry-instrumentation-httpx>=0.45b0", "pyjwt>=2.8", ] -prototype = [ - "pymupdf>=1.24", - "python-docx>=1.1", -] dev = [ { include-group = "api" }, { include-group = "extract" }, { include-group = "analyze" }, { include-group = "bot" }, { include-group = "notify" }, - { include-group = "prototype" }, "pytest>=8", "pytest-asyncio>=0.23", "respx>=0.21", @@ -183,7 +172,6 @@ source = ["src/contract_check"] branch = true omit = [ "*/migrations/*", - "*/prototype/*", "*/__main__.py", ] diff --git a/src/contract_check/__init__.py b/src/contract_check/__init__.py index 39eb535..08a7eab 100644 --- a/src/contract_check/__init__.py +++ b/src/contract_check/__init__.py @@ -1,11 +1,12 @@ """«Контракт-чек» — AI-скрининг рисков в договорах (СНГ / ГК РФ / ГК РБ). -Production refactor (see docs/ARCHITECTURE.md): +Production architecture (see docs/ARCHITECTURE.md): - core/ shared domain (config, db, mq, s3, llm, analysis, credits, tokens) - - prototype/ stage-0 standalone benchmark CLI (preserved) + - api/ FastAPI service + - worker_*/ RabbitMQ consumers (extract, prescreen, analyze, billing, notify) + - bot/ Telegram bot adapter -Service packages (api/, worker_extract/, worker_analyze/, bot/) land in -Steps 2–5. Importable from each service's own Docker image. +Each service is importable from its own Docker image. """ from __future__ import annotations diff --git a/src/contract_check/__main__.py b/src/contract_check/__main__.py deleted file mode 100644 index cad87b9..0000000 --- a/src/contract_check/__main__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Top-level entrypoint: `python -m contract_check ` → stage-0 prototype.""" - -from __future__ import annotations - -from src.contract_check.prototype import main - -if __name__ == "__main__": - main() diff --git a/src/contract_check/api/admin/auth.py b/src/contract_check/api/admin/auth.py index 741b10d..4b8c06d 100644 --- a/src/contract_check/api/admin/auth.py +++ b/src/contract_check/api/admin/auth.py @@ -98,9 +98,6 @@ async def require_admin(request: Request, session: AsyncSessionDep) -> AdminUser return admin -AdminUserDep = AdminUser - - def require_htmx(request: Request) -> None: """Dependency: only accept mutating requests coming from HTMX. diff --git a/src/contract_check/api/deps.py b/src/contract_check/api/deps.py index e3b312b..70afe21 100644 --- a/src/contract_check/api/deps.py +++ b/src/contract_check/api/deps.py @@ -81,9 +81,6 @@ def get_redis(request: Request) -> Any: return redis -RedisDep = Annotated[Any, Depends(get_redis)] - - def get_refresh_store(request: Request) -> RefreshTokenStore: """Build a RefreshTokenStore from the app-state Redis client. @@ -166,19 +163,6 @@ async def get_or_create_user_for_telegram( return user -async def get_or_create_user_by_id(session: AsyncSession, user_id: UUID) -> User | None: - """Fetch an existing user by UUID. Returns None if not found.""" - user = await UserRepository(session).get_by_id(user_id) - if user is None: - return None - return User( - id=user.id, - telegram_id=user.telegram_id, - created_at=user.created_at, - credits_left=user.credits_left, - ) - - async def bind_telegram_to_user( session: AsyncSession, user_id: UUID, diff --git a/src/contract_check/api/services.py b/src/contract_check/api/services.py index d1db446..0fceae9 100644 --- a/src/contract_check/api/services.py +++ b/src/contract_check/api/services.py @@ -39,6 +39,14 @@ def _content_type_from_suffix(suffix: str) -> str: return { ".pdf": "application/pdf", ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ".rtf": "application/rtf", + ".txt": "text/plain", + ".csv": "text/csv", + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".tif": "image/tiff", + ".tiff": "image/tiff", }.get(suffix, "application/octet-stream") diff --git a/src/contract_check/bot/client.py b/src/contract_check/bot/client.py index 8a7546b..1f3d529 100644 --- a/src/contract_check/bot/client.py +++ b/src/contract_check/bot/client.py @@ -159,9 +159,6 @@ class ApiClient: raise ApiError(500, "auth response missing access_token") self._token_cache[telegram_id] = token - def forget(self, telegram_id: int) -> None: - self._token_cache.pop(telegram_id, None) - async def get_credits(self, telegram_id: int, correlation_id: str) -> int: r = await self.client.get( "/api/v1/me", diff --git a/src/contract_check/bot/handlers.py b/src/contract_check/bot/handlers.py index 62c5b34..3163788 100644 --- a/src/contract_check/bot/handlers.py +++ b/src/contract_check/bot/handlers.py @@ -48,6 +48,14 @@ router = Router(name="contract-check-bot") _CONTENT_TYPES = { ".pdf": "application/pdf", ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ".rtf": "application/rtf", + ".txt": "text/plain", + ".csv": "text/csv", + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".tif": "image/tiff", + ".tiff": "image/tiff", } _STAGE_LABELS: dict[str, str] = { "queued": "Документ принят. В очереди на обработку…", diff --git a/src/contract_check/core/__init__.py b/src/contract_check/core/__init__.py index 7a47b88..436ea3f 100644 --- a/src/contract_check/core/__init__.py +++ b/src/contract_check/core/__init__.py @@ -1,8 +1,8 @@ """«Контракт-чек» core — shared domain package. -Imported by every service image (api, worker-extract, worker-analyze) and the -prototype benchmark. Adapters (bot) deliberately do NOT import this package -beyond config/logging (hexagonal boundary — see docs/ARCHITECTURE.md §4). +Imported by every service image (api, worker-extract, worker-analyze). +Adapters (bot) deliberately do NOT import this package beyond config/logging +(hexagonal boundary — see docs/ARCHITECTURE.md §4). Subpackages: config — typed env config (pydantic-settings) diff --git a/src/contract_check/core/analysis/__init__.py b/src/contract_check/core/analysis/__init__.py index bdadd91..e3c00fc 100644 --- a/src/contract_check/core/analysis/__init__.py +++ b/src/contract_check/core/analysis/__init__.py @@ -1,6 +1,6 @@ """Analysis domain — document text extraction, chunking, checklist, report schema, OCR, and the analyzer (prompt building, finding merge/dedupe/sort, -markdown rendering). Imported by the LLM adapter and the prototype benchmark. +markdown rendering). Imported by the LLM adapter and worker-analyze. """ from __future__ import annotations diff --git a/src/contract_check/core/analysis/analyzer.py b/src/contract_check/core/analysis/analyzer.py index 210bacc..2e5b356 100644 --- a/src/contract_check/core/analysis/analyzer.py +++ b/src/contract_check/core/analysis/analyzer.py @@ -1,9 +1,8 @@ """Analyzer — prompt building, finding merge/dedupe/sort, markdown rendering. Pure logic (no LLM, no DB, no S3). Shared by the Ollama Cloud adapter -(chunk fan-out → merge), the worker-analyze handler (render final report), and -the stage-0 prototype benchmark. Kept dependency-free so it cannot form an -import cycle with `core.llm`. +(chunk fan-out → merge) and the worker-analyze handler (render final report). +Kept dependency-free so it cannot form an import cycle with `core.llm`. """ from __future__ import annotations diff --git a/src/contract_check/core/config.py b/src/contract_check/core/config.py index a4f9092..54e3023 100644 --- a/src/contract_check/core/config.py +++ b/src/contract_check/core/config.py @@ -241,14 +241,6 @@ class Settings(BaseSettings): return True return self.env != "dev" - @property - def log_format_value(self) -> str: - """Resolved log format: json in prod/staging unless explicitly console.""" - fmt = self.log_format.lower() - if fmt in ("json", "console"): - return fmt - return "json" if self.env != "dev" else "console" - @lru_cache def get_settings() -> Settings: diff --git a/src/contract_check/core/db/repositories/api_keys.py b/src/contract_check/core/db/repositories/api_keys.py index 3093646..ab3a78f 100644 --- a/src/contract_check/core/db/repositories/api_keys.py +++ b/src/contract_check/core/db/repositories/api_keys.py @@ -99,16 +99,6 @@ class ApiKeyRepository: {"k": api_key_id}, ) - async def count_recent_requests(self, api_key_id: uuid.UUID, since: dt.datetime) -> int: - result = await self._session.execute( - text( - "SELECT count(*) FROM api_key_requests " - "WHERE api_key_id = :k AND created_at > :since" - ), - {"k": api_key_id, "since": since}, - ) - return int(result.scalar_one()) - async def get_by_id(self, api_key_id: uuid.UUID) -> ApiKey | None: return await self._session.get(ApiKey, api_key_id) @@ -191,23 +181,6 @@ class ApiKeyRepository: for r in result.all() ] - async def count_usage_by_key( - self, - api_key_id: uuid.UUID, - since: dt.datetime | None = None, - ) -> int: - where_parts = ["api_key_id = :k"] - params: dict[str, Any] = {"k": api_key_id} - if since is not None: - where_parts.append("created_at > :since") - params["since"] = since - where = " AND ".join(where_parts) - result = await self._session.execute( - text(f"SELECT count(*) FROM api_key_requests WHERE {where}"), - params, - ) - return int(result.scalar_one()) - async def count_requests_this_month(self, api_key_id: uuid.UUID) -> int: """Count requests since the start of the current calendar month.""" result = await self._session.execute( diff --git a/src/contract_check/core/db/repositories/invoices.py b/src/contract_check/core/db/repositories/invoices.py index 38d68aa..c85f55a 100644 --- a/src/contract_check/core/db/repositories/invoices.py +++ b/src/contract_check/core/db/repositories/invoices.py @@ -65,14 +65,6 @@ class InvoicesRepository: row = result.mappings().first() return dict(row) if row is not None else None - async def get_by_external_id(self, external_id: str) -> InvoiceRow | None: - result = await self._session.execute( - text("SELECT * FROM invoices WHERE external_id = :e"), - {"e": external_id}, - ) - row = result.mappings().first() - return dict(row) if row is not None else None - async def list_for_user( self, user_id: uuid.UUID, *, limit: int = 50, offset: int = 0 ) -> list[InvoiceRow]: diff --git a/src/contract_check/core/db/repositories/jobs.py b/src/contract_check/core/db/repositories/jobs.py index 26cd2de..77079c5 100644 --- a/src/contract_check/core/db/repositories/jobs.py +++ b/src/contract_check/core/db/repositories/jobs.py @@ -144,32 +144,3 @@ class JobRepository: created_at=row[10], updated_at=row[11], ) - - async def count_active_by_document(self, document_id: uuid.UUID) -> int: - result = await self._session.execute( - text( - "SELECT count(*) FROM jobs " - "WHERE document_id = :d AND status IN ('pending','running','retrying')" - ), - {"d": document_id}, - ) - return int(result.scalar_one()) - - async def count_dlq_by_document(self, document_id: uuid.UUID) -> int: - result = await self._session.execute( - text("SELECT count(*) FROM jobs WHERE document_id = :d AND dlq = TRUE"), - {"d": document_id}, - ) - return int(result.scalar_one()) - - async def list_for_document( - self, document_id: uuid.UUID, *, limit: int = 100, offset: int = 0 - ) -> list[Job]: - result = await self._session.execute( - text( - "SELECT * FROM jobs WHERE document_id = :d " - "ORDER BY created_at DESC LIMIT :limit OFFSET :offset" - ), - {"d": document_id, "limit": limit, "offset": offset}, - ) - return [Job(**dict(row)) for row in result.mappings().all()] diff --git a/src/contract_check/core/db/repositories/passkeys.py b/src/contract_check/core/db/repositories/passkeys.py index 10f9e25..5201917 100644 --- a/src/contract_check/core/db/repositories/passkeys.py +++ b/src/contract_check/core/db/repositories/passkeys.py @@ -16,17 +16,6 @@ class PasskeyRepository: def __init__(self, session: AsyncSession) -> None: self._session = session - async def get_by_id_for_user( - self, passkey_id: uuid.UUID, user_id: uuid.UUID - ) -> PasskeyCredential | None: - result = await self._session.execute( - select(PasskeyCredential).where( - PasskeyCredential.id == passkey_id, - PasskeyCredential.user_id == user_id, - ) - ) - return result.scalars().first() - async def delete_by_id_for_user(self, passkey_id: uuid.UUID, user_id: uuid.UUID) -> bool: result = await self._session.execute( delete(PasskeyCredential).where( diff --git a/src/contract_check/core/db/repositories/prescreen_results.py b/src/contract_check/core/db/repositories/prescreen_results.py index 7987254..e325965 100644 --- a/src/contract_check/core/db/repositories/prescreen_results.py +++ b/src/contract_check/core/db/repositories/prescreen_results.py @@ -84,15 +84,3 @@ class PrescreenResultRepository: async def get_by_id(self, prescreen_result_id: uuid.UUID) -> PrescreenResult | None: return await self._session.get(PrescreenResult, prescreen_result_id) - - async def list_for_document( - self, document_id: uuid.UUID, *, limit: int = 10 - ) -> list[PrescreenResult]: - result = await self._session.execute( - text( - "SELECT * FROM prescreen_results WHERE document_id = :d " - "ORDER BY prescreened_at DESC LIMIT :limit" - ), - {"d": document_id, "limit": limit}, - ) - return [PrescreenResult(**dict(row)) for row in result.mappings().all()] diff --git a/src/contract_check/core/db/repositories/reports.py b/src/contract_check/core/db/repositories/reports.py index 0afaf90..997c917 100644 --- a/src/contract_check/core/db/repositories/reports.py +++ b/src/contract_check/core/db/repositories/reports.py @@ -109,26 +109,6 @@ class ReportRepository: assert row is not None return row[0] - async def set_prescreen_result( - self, - document_id: uuid.UUID, - *, - prescreen_result_id: uuid.UUID, - prescreen_meta: dict[str, Any] | None = None, - ) -> None: - """Attach an existing prescreen result to a report.""" - await self._session.execute( - text( - "UPDATE reports SET prescreen_result_id = :prid, prescreen_meta = :pm " - "WHERE document_id = :d" - ), - { - "d": document_id, - "prid": prescreen_result_id, - "pm": None if prescreen_meta is None else json.dumps(prescreen_meta), - }, - ) - async def list_for_user( self, user_id: uuid.UUID, *, limit: int = 100, offset: int = 0 ) -> list[Report]: diff --git a/src/contract_check/core/db/repositories/users.py b/src/contract_check/core/db/repositories/users.py index 97f9d0f..9811343 100644 --- a/src/contract_check/core/db/repositories/users.py +++ b/src/contract_check/core/db/repositories/users.py @@ -26,10 +26,6 @@ class UserRepository: async def get_by_id(self, user_id: uuid.UUID) -> User | None: return await self._session.get(User, user_id) - async def get_by_id_or_none(self, user_id: uuid.UUID) -> User | None: - """Alias used by auth dependency; returns None instead of raising.""" - return await self.get_by_id(user_id) - async def get_by_email(self, email: str) -> User | None: result = await self._session.execute(select(User).where(User.email == email)) return result.scalars().first() @@ -228,12 +224,6 @@ class UserRepository: {"h": token_hash, "e": expires_at, "u": user_id}, ) - async def set_name(self, user_id: uuid.UUID, name: str) -> None: - await self._session.execute( - text("UPDATE users SET name = :n WHERE id = :u"), - {"n": name, "u": user_id}, - ) - async def set_active(self, user_id: uuid.UUID, active: bool) -> None: await self._session.execute( text("UPDATE users SET is_active = :a WHERE id = :u"), @@ -280,12 +270,6 @@ class UserRepository: row = result.first() return bool(row[0]) if row else False - async def set_billing_hold(self, user_id: uuid.UUID, hold: bool) -> None: - await self._session.execute( - text("UPDATE users SET billing_hold = :h WHERE id = :u"), - {"h": hold, "u": user_id}, - ) - async def set_magic_link_token( self, user_id: uuid.UUID, diff --git a/src/contract_check/core/extraction/__init__.py b/src/contract_check/core/extraction/__init__.py index 361e631..4352bd0 100644 --- a/src/contract_check/core/extraction/__init__.py +++ b/src/contract_check/core/extraction/__init__.py @@ -1,8 +1,8 @@ """Extraction layer: port + adapters + factory + the OCR-fallback orchestrator. -This is the structured successor to `core.analysis.extractor` (which stays for -the prototype CLI). New formats are added by writing an adapter and registering -it in the factory — no domain/pipeline changes. +This is the structured successor to `core.analysis.extractor`. New formats are +added by writing an adapter and registering it in the factory — no domain/pipeline +changes. """ from __future__ import annotations diff --git a/src/contract_check/core/llm/__init__.py b/src/contract_check/core/llm/__init__.py index 4cd366e..c10ffc3 100644 --- a/src/contract_check/core/llm/__init__.py +++ b/src/contract_check/core/llm/__init__.py @@ -1,8 +1,8 @@ """LLM provider layer — abstract port + adapters. -The first and only realization is Ollama Cloud (ported from the stage-0 -prototype). Future providers (self-hosted Ollama, GigaChat, YandexGPT) plug -in here without touching analysis code. See docs/ARCHITECTURE.md §9. +The first realization is Ollama Cloud. Future providers (self-hosted Ollama, +GigaChat, YandexGPT) plug in here without touching analysis code. +See docs/ARCHITECTURE.md §9. """ from __future__ import annotations diff --git a/src/contract_check/core/llm/ollama_cloud.py b/src/contract_check/core/llm/ollama_cloud.py index 6de44ad..c38d547 100644 --- a/src/contract_check/core/llm/ollama_cloud.py +++ b/src/contract_check/core/llm/ollama_cloud.py @@ -1,10 +1,10 @@ """Ollama Cloud adapter — first realization of LLMProvider. -Ported from the stage-0 prototype `llm_client.py` (bearer httpx, `format: -json-schema`, 5xx backoff, 429→fallback, pydantic repair-loop) and wrapped to -implement `LLMProvider.analyze(text, *, checklist)` with chunk fan-out, -de-duplication, and severity sort. The worker maps the raised exceptions to a -FailureClass (see core/mq/consumer.py + worker_analyze/handler.py). +Bearer-httpx Ollama Cloud client (`format: json-schema`, 5xx backoff, +429→fallback, pydantic repair-loop) wrapped to implement +`LLMProvider.analyze(text, *, checklist)` with chunk fan-out, de-duplication, +and severity sort. The worker maps the raised exceptions to a FailureClass +(see core/mq/consumer.py + worker_analyze/handler.py). """ from __future__ import annotations @@ -196,7 +196,7 @@ class OllamaCloudProvider: ) return result.data.model_dump() # type: ignore[return-type] - # ── internals (ported from prototype llm_client.py) ───────────────────── + # ── internals ────────────────────────────────────────────────────────── async def _run_with_fallback[T: BaseModel]( self, messages: list[dict[str, str]], diff --git a/src/contract_check/core/logging.py b/src/contract_check/core/logging.py index 9cfaba2..6ebcdcf 100644 --- a/src/contract_check/core/logging.py +++ b/src/contract_check/core/logging.py @@ -18,7 +18,6 @@ Conventions for service authors: (document_id, user_id, s3_key) so a stuck job can be traced. - log success once per handled message with decision/outcome metrics. - log failures with exc_info=True so full traceback is captured. - - use `log_error()` for structured exception capture. """ from __future__ import annotations @@ -47,11 +46,6 @@ def set_correlation_id(value: str | None) -> None: correlation_id_var.set(value or "") -def get_correlation_id() -> str: - """Return the current correlation id ("" if unset).""" - return correlation_id_var.get() - - def new_correlation_id() -> str: """Mint a new correlation id and bind it to the current context.""" cid = str(uuid.uuid4()) @@ -77,14 +71,6 @@ def bind_context(**kwargs: object) -> None: structlog.contextvars.bind_contextvars(**{k: v for k, v in kwargs.items() if v is not None}) -def clear_context() -> None: - """Clear all structlog contextvars (use with care, mostly for tests).""" - structlog.contextvars.clear_contextvars() - correlation_id_var.set("") - service_var.set("") - env_var.set("") - - def _inject_static_context( _logger: WrappedLogger, _method_name: str, event_dict: EventDict ) -> EventDict: @@ -233,18 +219,6 @@ def is_debug_enabled() -> bool: return logging.getLogger().isEnabledFor(logging.DEBUG) -def log_error( - logger: BoundLogger, - event: str, - exc: BaseException | None = None, - **kwargs: object, -) -> None: - """Structured error helper: captures full traceback and exception fields.""" - if exc is not None: - kwargs["exc_info"] = (type(exc), exc, exc.__traceback__) - logger.error(event, **kwargs) - - class Timer: """Context manager that records elapsed wall time for an operation. @@ -263,7 +237,7 @@ class Timer: self.logger.debug(f"{self.operation}_started", **self.kwargs) return self - def __exit__(self, exc_type, exc_val, exc_tb) -> None: # type: ignore[no-untyped-def] + def __exit__(self, exc_type, _exc_val, exc_tb) -> None: # type: ignore[no-untyped-def] self.end_time = time.perf_counter() duration_ms = (self.end_time - self.start_time) * 1000 if exc_type is None: diff --git a/src/contract_check/core/metrics.py b/src/contract_check/core/metrics.py index f2b3d69..6111119 100644 --- a/src/contract_check/core/metrics.py +++ b/src/contract_check/core/metrics.py @@ -98,6 +98,12 @@ http_request_duration = Histogram( extract_duration = Histogram( "contract_check_extract_duration_seconds", "Document extraction (extract.q handler) latency.", + ["format"], +) +extraction_total = Counter( + "contract_check_extraction_total", + "Document extractions by format and structure.", + ["format", "structured"], ) analyze_duration = Histogram( "contract_check_analyze_duration_seconds", diff --git a/src/contract_check/core/mq/messages.py b/src/contract_check/core/mq/messages.py index 78ba065..3cf554f 100644 --- a/src/contract_check/core/mq/messages.py +++ b/src/contract_check/core/mq/messages.py @@ -40,13 +40,15 @@ class DocumentUploaded(PipelineMessage): class DocumentExtracted(PipelineMessage): """worker-extract → contracts.x[analyze] → worker-analyze. - `extracted_s3_key` points at the extracted plaintext + `extracted_s3_key` points at the extracted Markdown (`users/{uid}/docs/{did}.txt`). """ extracted_s3_key: str char_count: int ocr_used: bool + is_structured: bool = False + has_tables: bool = False class PrescreenRequested(PipelineMessage): diff --git a/src/contract_check/prototype/__init__.py b/src/contract_check/prototype/__init__.py deleted file mode 100644 index ff1ae8c..0000000 --- a/src/contract_check/prototype/__init__.py +++ /dev/null @@ -1,158 +0,0 @@ -"""«Контракт-чек» — Stage 0 prototype (standalone benchmark CLI). - -Preserved on purpose (docs/ARCHITECTURE.md Q44): a run-once end-to-end CLI for -go/no-go measurements against a real contract — PDF/DOCX → text → chunks → -Ollama Cloud → markdown report + metrics. It does NOT use Postgres/RabbitMQ/ -MinIO; it builds an OllamaCloudProvider directly from OLLAMA_* env vars. - - python -m contract_check.prototype path/to/contract.pdf - python -m contract_check.prototype contract.docx --out report.md - python -m contract_check.prototype contract.pdf --json metrics.json - -Не заменяет юриста. Это первичный скрининг рисков. -""" - -from __future__ import annotations - -import argparse -import asyncio -import json -import sys -from pathlib import Path - -from pydantic_settings import BaseSettings, SettingsConfigDict - -from src.contract_check.core.analysis.analyzer import RenderMetrics, render_markdown -from src.contract_check.core.analysis.checklist import checklist_for_prompt -from src.contract_check.core.analysis.extractor import ExtractionError, extract_text -from src.contract_check.core.llm.ollama_cloud import LLMError, OllamaCloudProvider -from src.contract_check.core.logging import configure_logging - - -class PrototypeSettings(BaseSettings): - """OLLAMA_* config only — the prototype needs no DB/MQ/S3.""" - - model_config = SettingsConfigDict(env_file=".env", extra="ignore", case_sensitive=False) - - ollama_host: str = "" - ollama_api_key: str = "" - ollama_model: str = "qwen2.5:14b" - ollama_fallback_model: str = "qwen2.5:7b" - ollama_temperature: float = 0.2 - ollama_num_predict: int = 3072 - ollama_timeout: float = 120.0 - ollama_max_concurrency: int = 3 - chunk_size_chars: int = 10000 - - -def _build_provider(s: PrototypeSettings) -> OllamaCloudProvider: - if not s.ollama_host or not s.ollama_api_key: - sys.exit( - "ERROR: OLLAMA_HOST и OLLAMA_API_KEY обязательны. " - "Скопируй .env.example → .env и заполни (см. ollama.com/cloud)." - ) - return OllamaCloudProvider( - host=s.ollama_host, - api_key=s.ollama_api_key, - model=s.ollama_model, - fallback_model=s.ollama_fallback_model or None, - temperature=s.ollama_temperature, - num_predict=s.ollama_num_predict, - timeout=s.ollama_timeout, - max_concurrency=s.ollama_max_concurrency, - chunk_size=s.chunk_size_chars, - ) - - -async def run(path: Path, out: Path | None, json_out: Path | None) -> int: - settings = PrototypeSettings() - started = asyncio.get_event_loop().time() - - try: - text = extract_text(path) - except ExtractionError as exc: - print(f"ERROR извлечения: {exc}", file=sys.stderr) - return 2 - - provider = _build_provider(settings) - try: - async with provider: - try: - result = await provider.analyze(text, checklist=checklist_for_prompt()) - except LLMError as exc: - print(f"ERROR LLM: {exc}", file=sys.stderr) - return 3 - finally: - latency = asyncio.get_event_loop().time() - started - - metrics = RenderMetrics( - chars=len(text), - findings=len(result.findings), - prompt_tokens=result.prompt_tokens, - eval_tokens=result.eval_tokens, - latency_sec=max(result.latency_sec, latency), - models_used=result.models_used, - fell_back=result.fell_back, - repaired=result.repaired, - ) - md = render_markdown(result.findings, str(path), metrics) - - if out: - out.write_text(md, encoding="utf-8") - print(f"Отчёт сохранён: {out}") - else: - print(md) - - if json_out: - json_out.write_text( - json.dumps( - { - "file": path.name, - "findings": [f.model_dump() for f in result.findings], - "metrics": { - "chars": metrics.chars, - "findings": metrics.findings, - "prompt_tokens": metrics.prompt_tokens, - "eval_tokens": metrics.eval_tokens, - "total_tokens": metrics.total_tokens, - "latency_sec": round(metrics.latency_sec, 2), - "models_used": sorted(metrics.models_used), - "fell_back": metrics.fell_back, - "repaired": metrics.repaired, - }, - }, - ensure_ascii=False, - indent=2, - ), - encoding="utf-8", - ) - print(f"Метрики/JSON сохранены: {json_out}") - - return 0 - - -def main() -> None: - parser = argparse.ArgumentParser( - prog="contract-check", - description="«Контракт-чек» — первичный скрининг рисков договора (Stage 0).", - ) - parser.add_argument("file", help="Путь к PDF/DOCX договору.") - parser.add_argument("--out", "-o", help="Записать markdown-отчёт в файл.") - parser.add_argument("--json", help="Дополнительно записать находки + метрики.") - parser.add_argument("-v", "--verbose", action="store_true", help="Подробное логирование.") - args = parser.parse_args() - - configure_logging( - level="DEBUG" if args.verbose else "WARNING", - json_output=False, - service="contract-check-prototype", - ) - - rc = asyncio.run( - run( - Path(args.file), - Path(args.out) if args.out else None, - Path(args.json) if args.json else None, - ) - ) - sys.exit(rc) diff --git a/src/contract_check/prototype/__main__.py b/src/contract_check/prototype/__main__.py deleted file mode 100644 index 41b90ea..0000000 --- a/src/contract_check/prototype/__main__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Stage-0 prototype entrypoint: `python -m contract_check.prototype `.""" - -from __future__ import annotations - -from src.contract_check.prototype import main - -if __name__ == "__main__": - main() diff --git a/src/contract_check/worker_extract/consumer.py b/src/contract_check/worker_extract/consumer.py index 37abb3d..7bd1c90 100644 --- a/src/contract_check/worker_extract/consumer.py +++ b/src/contract_check/worker_extract/consumer.py @@ -5,7 +5,6 @@ from __future__ import annotations from src.contract_check.core.db.enums import FailureClass from src.contract_check.core.db.session import create_session_factory from src.contract_check.core.logging import get_logger -from src.contract_check.core.metrics import extract_duration from src.contract_check.core.mq.consumer import Consumer from src.contract_check.core.mq.messages import DocumentUploaded from src.contract_check.core.mq.topology import RK_ANALYZE @@ -46,7 +45,6 @@ class ExtractConsumer(Consumer[DocumentUploaded]): def classify(self, exc: BaseException) -> FailureClass: return self._handler.classify(exc) - @extract_duration.time() async def handle(self, payload: DocumentUploaded) -> None: await self._handler.handle(payload) diff --git a/src/contract_check/worker_extract/extract_document.py b/src/contract_check/worker_extract/extract_document.py deleted file mode 100644 index ac42404..0000000 --- a/src/contract_check/worker_extract/extract_document.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Facade re-exporting extraction + OCR helpers for the worker.""" - -from __future__ import annotations - -from src.contract_check.core.analysis.extractor import ExtractionError, extract_text -from src.contract_check.core.analysis.ocr import OCRError, ocr_pdf - -__all__ = ["ExtractionError", "OCRError", "extract_document"] - - -def extract_document(path: str) -> str: - """Try direct extraction; fall back to OCR for likely scans. - - Mirrors docs/ARCHITECTURE.md §16 worker-extract pseudocode: - 1. extract_text (PDF text layer / DOCX) - 2. if ExtractionError due to short text, try OCR - 3. if OCR still short, raise ExtractionError (failure_class extraction_failed) - """ - try: - return extract_text(path) - except ExtractionError: - # Likely scan or image-only PDF; attempt OCR once. - return ocr_pdf(path) - - -__all__.append("extract_document") diff --git a/src/contract_check/worker_extract/handler.py b/src/contract_check/worker_extract/handler.py index 178d83e..e2a53bb 100644 --- a/src/contract_check/worker_extract/handler.py +++ b/src/contract_check/worker_extract/handler.py @@ -3,9 +3,9 @@ The handler is intentionally separate from the consumer so it can be tested in-process without spinning up a real RabbitMQ consumer. It owns: - idempotency checks against documents.status - - status transitions (queued → extracting → ocr) + - status transitions (queued → extracting) - MinIO download/upload - - extraction/OCR via core.analysis + - extraction/OCR via core.extraction - publishing DocumentExtracted to analyze.q - updating the jobs row, recording failure class, and refund-on-DLQ. """ @@ -13,9 +13,7 @@ in-process without spinning up a real RabbitMQ consumer. It owns: from __future__ import annotations import asyncio -import tempfile import uuid -from pathlib import Path from typing import TYPE_CHECKING from src.contract_check.core.billing.quota import release_document_slot @@ -23,8 +21,19 @@ from src.contract_check.core.config import get_settings from src.contract_check.core.credits import refund_credit from src.contract_check.core.db.enums import DOC_TERMINAL, FailureClass from src.contract_check.core.db.repositories import DocumentRepository, JobRepository +from src.contract_check.core.extraction import ( + ExtractionFailedError, + UnsupportedFormatError, + detect_format, + extract_document, +) from src.contract_check.core.logging import get_logger -from src.contract_check.core.metrics import extract_duration, mq_failed, mq_published +from src.contract_check.core.metrics import ( + extract_duration, + extraction_total, + mq_failed, + mq_published, +) from src.contract_check.core.mq.messages import ( DocumentExtracted, DocumentUploaded, @@ -38,6 +47,8 @@ from src.contract_check.core.s3.minio_storage import MinioStorage if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + from src.contract_check.core.extraction import ExtractedDocument + log = get_logger(__name__) @@ -87,13 +98,17 @@ class ExtractHandler: await jobs.claim_start(payload.document_id, "extract") await session.commit() - with extract_duration.time(): - extracted_text, ocr_used = await self._extract_text(payload) + document = await self._extract_document(payload) + extracted_text = document.markdown + fmt = document.metadata.get("format", "unknown") + ocr_used = document.metadata.get("is_scan", False) + is_structured = document.is_structured + has_tables = document.metadata.get("has_tables", False) ext_key = extracted_key(str(payload.user_id), str(payload.document_id)) text_bytes = extracted_text.encode("utf-8") - await self._storage.put(ext_key, text_bytes, content_type="text/plain; charset=utf-8") + await self._storage.put(ext_key, text_bytes, content_type="text/markdown; charset=utf-8") publisher = await self._publisher_instance() if self._settings.prescreen_enabled: @@ -104,8 +119,8 @@ class ExtractHandler: text_s3_key=ext_key, filename=payload.filename, char_count=len(extracted_text), - is_structured=False, - has_tables=False, + is_structured=is_structured, + has_tables=has_tables, attempt=payload.attempt, ) publish_rk = RK_PRESCREEN @@ -120,6 +135,8 @@ class ExtractHandler: extracted_s3_key=ext_key, char_count=len(extracted_text), ocr_used=ocr_used, + is_structured=is_structured, + has_tables=has_tables, attempt=payload.attempt, ) publish_rk = RK_ANALYZE @@ -145,40 +162,29 @@ class ExtractHandler: log.info( "extract_success", document_id=str(payload.document_id), + format=fmt, + structured=is_structured, + tables=has_tables, char_count=len(extracted_text), ) - async def _extract_text(self, payload: DocumentUploaded) -> tuple[str, bool]: - from src.contract_check.core.analysis.extractor import ExtractionError, extract_text - from src.contract_check.core.analysis.ocr import ocr_pdf - + async def _extract_document(self, payload: DocumentUploaded) -> ExtractedDocument: data = await self._storage.get(payload.s3_key) - suffix = Path(payload.filename).suffix.lower() - with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp: - tmp.write(data) - tmp_path = Path(tmp.name) + fmt = detect_format(data, mime=payload.mime, filename=payload.filename) - try: - try: - text = await asyncio.to_thread(extract_text, str(tmp_path)) - return text, False - except ExtractionError: - await self._update_stage(payload.document_id, "ocr") - text = await asyncio.to_thread(ocr_pdf, str(tmp_path)) - return text, True - finally: - tmp_path.unlink(missing_ok=True) + def _extract() -> ExtractedDocument: + return extract_document(data, mime=payload.mime, filename=payload.filename) - async def _update_stage(self, document_id: uuid.UUID, stage: str) -> None: - async with self._session_factory() as session: - await DocumentRepository(session).update_stage(document_id, stage=stage) - await session.commit() + with extract_duration.labels(format=fmt).time(): + document = await asyncio.to_thread(_extract) + + extraction_total.labels(format=fmt, structured=str(document.is_structured).lower()).inc() + return document def classify(self, exc: BaseException) -> FailureClass: - from src.contract_check.core.analysis.extractor import ExtractionError from src.contract_check.core.analysis.ocr import OCRError - if isinstance(exc, ExtractionError): + if isinstance(exc, (ExtractionFailedError, UnsupportedFormatError)): return "extraction_failed" if isinstance(exc, OCRError): return "ocr_failed" diff --git a/srv/prototype/Dockerfile b/srv/prototype/Dockerfile deleted file mode 100644 index 5c3d2a2..0000000 --- a/srv/prototype/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -# syntax=docker/dockerfile:1 - -# ─── Stage 1: build deps + project into a venv via uv ───────────────────────── -# Standalone stage-0 benchmark CLI (PDF/DOCX → text → Ollama Cloud → markdown). -# No DB/MQ/S3 — kept as a reference benchmark image, not a production service. -FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim AS builder - -ENV UV_COMPILE_BYTECODE=1 \ - UV_LINK_MODE=copy \ - UV_PYTHON_DOWNLOADS=never \ - UV_PROJECT_ENVIRONMENT=/app/.venv - -WORKDIR /app - -COPY pyproject.toml uv.lock ./ -COPY README.md ./ - -# Install only the prototype group (core + pymupdf/python-docx). -RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-default-groups --group prototype --no-install-project - -# ─── Stage 2: lean runtime ───────────────────────────────────────────────── -FROM python:3.13-slim - -ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - PATH=/app/.venv/bin:$PATH - -WORKDIR /app - -COPY --from=builder /app/.venv /app/.venv - -# App source: internal imports use the absolute `src.contract_check.*` form. -COPY src ./src - -CMD ["python", "-m", "src.contract_check.prototype"] diff --git a/tests/integration/test_extract_worker.py b/tests/integration/test_extract_worker.py index ae32d60..419e604 100644 --- a/tests/integration/test_extract_worker.py +++ b/tests/integration/test_extract_worker.py @@ -329,12 +329,12 @@ async def test_extract_worker_terminal_failure_refunds_and_dlqs( await session.commit() handler = ExtractHandler(session_factory=sess_factory) - from contract_check.core.analysis.extractor import ExtractionError from contract_check.core.analysis.ocr import OCRError + from contract_check.core.extraction import ExtractionFailedError - # The handler may raise ExtractionError or OCRError depending on local + # The handler may raise ExtractionFailedError or OCRError depending on local # tesseract data; either way the terminal-failure path refunds the credit. - with pytest.raises((ExtractionError, OCRError)): + with pytest.raises((ExtractionFailedError, OCRError)): await handler.handle( DocumentUploaded( correlation_id=correlation_id, diff --git a/tests/unit/test_chunker.py b/tests/unit/test_chunker.py index a3fc65a..08e4b9d 100644 --- a/tests/unit/test_chunker.py +++ b/tests/unit/test_chunker.py @@ -1,4 +1,4 @@ -"""Chunker unit tests (ported from the prototype smoke test + extra cases).""" +"""Chunker unit tests.""" from __future__ import annotations diff --git a/tests/unit/test_extract_handler.py b/tests/unit/test_extract_handler.py index 12f5c95..58e6cb1 100644 --- a/tests/unit/test_extract_handler.py +++ b/tests/unit/test_extract_handler.py @@ -54,7 +54,7 @@ def handler(monkeypatch: pytest.MonkeyPatch) -> tuple[ExtractHandler, Any, Any]: # Stub S3 put and the extraction core. h._storage = MagicMock(put=AsyncMock()) - monkeypatch.setattr(h, "_extract_text", AsyncMock(return_value=(_extracted().markdown, False))) + monkeypatch.setattr(h, "_extract_document", AsyncMock(return_value=_extracted())) # Stub publisher. publisher = MagicMock(publish=AsyncMock()) diff --git a/tests/unit/test_extraction_adapters.py b/tests/unit/test_extraction_adapters.py index 4ddbd0e..eda0d09 100644 --- a/tests/unit/test_extraction_adapters.py +++ b/tests/unit/test_extraction_adapters.py @@ -1,12 +1,14 @@ """Unit tests for the extraction adapters (core/extraction/adapters/*). -Real files are generated in-memory per format (pymupdf/python-docx) so no -binary fixtures live in the repo. The OCR adapter's engine-dependent path is -covered by the integration suite; here we only assert the error contract. +Real files are generated in-memory per format (pymupdf/python-docx/PIL) so no +binary fixtures live in the repo. The OCR adapter's engine-dependent happy path +is covered when tesseract is available; error-contract tests always run. """ from __future__ import annotations +from pathlib import Path + import pytest from contract_check.core.extraction import ( @@ -167,9 +169,51 @@ def test_txt_too_short_raises() -> None: TxtExtractor().extract("коротко".encode()) -# ── OCR (tesseract) — error contract only; happy path is integration ───────── -# Both cases fail at open/decode, before the engine is invoked, so they run -# regardless of whether the tesseract binary is installed. +# ── OCR (tesseract) ───────────────────────────────────────────────────────── + + +def _tesseract_available() -> bool: + import os + import shutil + + if shutil.which("tesseract") is None: + return False + # The default adapter uses rus+eng; skip if traineddata is missing. + tessdata = os.environ.get("TESSDATA_PREFIX", "/usr/share/tessdata") + return os.path.exists(os.path.join(tessdata, "eng.traineddata")) + + +def build_png_bytes(tmp_path: Path) -> bytes: + from PIL import Image, ImageDraw, ImageFont + + text = ( + "Contract. The parties agree to the terms herein. " + "Party A shall deliver the goods within ten business days. " + "Party B shall pay the invoice within thirty days. " + "Liability is limited to the contract value. " + "Disputes shall be resolved in arbitration in Moscow." + ) + font_path = "/usr/share/fonts/noto/NotoSans-Regular.ttf" + try: + font = ImageFont.truetype(font_path, 24) + except OSError: + font = ImageFont.load_default() + + img = Image.new("RGB", (1200, 400), color="white") + draw = ImageDraw.Draw(img) + draw.text((20, 20), text, fill="black", font=font) + path = tmp_path / "contract.png" + img.save(path, format="PNG") + return path.read_bytes() + + +@pytest.mark.skipif(not _tesseract_available(), reason="tesseract not installed") +def test_ocr_image_extracts_text(tmp_path: Path) -> None: + data = build_png_bytes(tmp_path) + result = TesseractOcrExtractor().extract(data) + assert "Contract" in result.markdown + assert result.metadata["format"] == "image" + assert result.metadata["pages"] == 1 def test_ocr_garbage_pdf_raises() -> None: diff --git a/tests/unit/test_extraction_factory.py b/tests/unit/test_extraction_factory.py index f036f73..e902248 100644 --- a/tests/unit/test_extraction_factory.py +++ b/tests/unit/test_extraction_factory.py @@ -60,6 +60,21 @@ def test_docx_zip_magic_does_not_shadow_suffix() -> None: assert detect_format(build_docx_bytes(), filename="contract.docx") == "docx" +@pytest.mark.parametrize( + ("filename", "mime"), + [ + ("scan.png", "image/png"), + ("scan.jpg", "image/jpeg"), + ("scan.jpeg", "image/jpeg"), + ("scan.tif", "image/tiff"), + ("scan.tiff", "image/tiff"), + ], +) +def test_image_formats_detected(filename: str, mime: str) -> None: + assert detect_format(b"", filename=filename) == "image" + assert detect_format(b"", mime=mime, filename="scan.bin") == "image" + + # ── factory ────────────────────────────────────────────────────────────────── @@ -70,6 +85,10 @@ def test_factory_returns_per_format_adapters() -> None: assert type(factory.get_extractor(b"", filename="a.rtf")).__name__ == "RtfExtractor" assert type(factory.get_extractor(b"", filename="a.txt")).__name__ == "TxtExtractor" assert type(factory.get_extractor(b"", filename="a.png")).__name__ == "TesseractOcrExtractor" + assert type(factory.get_extractor(b"", filename="a.jpg")).__name__ == "TesseractOcrExtractor" + assert type(factory.get_extractor(b"", filename="a.jpeg")).__name__ == "TesseractOcrExtractor" + assert type(factory.get_extractor(b"", filename="a.tif")).__name__ == "TesseractOcrExtractor" + assert type(factory.get_extractor(b"", filename="a.tiff")).__name__ == "TesseractOcrExtractor" # ── extract_document: end-to-end over bytes ────────────────────────────────── diff --git a/uv.lock b/uv.lock index 8dd2bad..a811bb9 100644 --- a/uv.lock +++ b/uv.lock @@ -730,7 +730,6 @@ dev = [ { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, - { name = "python-docx" }, { name = "python-magic" }, { name = "python-multipart" }, { name = "redis" }, @@ -795,10 +794,6 @@ prescreen = [ { name = "sentry-sdk" }, { name = "sqlalchemy" }, ] -prototype = [ - { name = "pymupdf" }, - { name = "python-docx" }, -] s3 = [ { name = "minio" }, ] @@ -899,7 +894,6 @@ dev = [ { name = "pytest", specifier = ">=8" }, { name = "pytest-asyncio", specifier = ">=0.23" }, { name = "pytest-cov", specifier = ">=7.1.0" }, - { name = "python-docx", specifier = ">=1.1" }, { name = "python-magic", specifier = ">=0.4.27" }, { name = "python-multipart", specifier = ">=0.0.9" }, { name = "redis", specifier = ">=5.0" }, @@ -962,10 +956,6 @@ prescreen = [ { name = "sentry-sdk", specifier = ">=2" }, { name = "sqlalchemy", specifier = ">=2.0" }, ] -prototype = [ - { name = "pymupdf", specifier = ">=1.24" }, - { name = "python-docx", specifier = ">=1.1" }, -] s3 = [{ name = "minio", specifier = ">=7.2" }] [[package]] @@ -1471,68 +1461,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] -[[package]] -name = "lxml" -version = "6.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/05/3b/aab6728cae887456f409b4d75e8a01856e4f04bd510de38052a47768b680/lxml-6.1.1.tar.gz", hash = "sha256:ba96ae44888e0185281e937633a743ea90d5a196c6000f82565ebb0580012d40", size = 4197430, upload-time = "2026-05-18T19:19:06.424Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a5/eb/7e6f37c5584ccbb2ff267f56fd0339016938c1c8684cfefab9b33ffc2f36/lxml-6.1.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:68a9198d0fc122d14bb76837de9aa80cf84caed990b5b237f532ed87d3706736", size = 8559780, upload-time = "2026-05-18T19:17:57.661Z" }, - { url = "https://files.pythonhosted.org/packages/a1/36/587c2521cf23a2cd6c9c22108aa7528f683a1f195ed7ccd23a4b1786ad36/lxml-6.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7d47866cb32fb503450b6edc9df355d10dc49836af2e89901bd6ac6b0896d9d9", size = 4618006, upload-time = "2026-05-18T19:18:04.452Z" }, - { url = "https://files.pythonhosted.org/packages/6e/ca/ab7bfe2bf4c972af5e7878262845ead3a24a929a9b04bc11c7c1ece6c82a/lxml-6.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb7c9811bfaa8b1ed5ed319f5d370dfbcaa59d52ea64be2a5a85e18195930354", size = 4924139, upload-time = "2026-05-18T19:19:04.873Z" }, - { url = "https://files.pythonhosted.org/packages/6b/55/a0c72851dfee5ecc689f949723a73dea457758912542cb955b108eaf0d8f/lxml-6.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:762ff394d5bd56da0cf034a23dcce4e13923f15321a2adfa2ac00201dc6d3fca", size = 5082329, upload-time = "2026-05-18T19:19:09.728Z" }, - { url = "https://files.pythonhosted.org/packages/f0/b6/0608f7d61a3b96cc67e5648a3d906e31a5082093e10e7be65b3886289938/lxml-6.1.1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a088f287f7d8275a33c07f2cac6c50b9319309a0200a39e7e75d80c707723099", size = 4993564, upload-time = "2026-05-18T19:19:13.608Z" }, - { url = "https://files.pythonhosted.org/packages/4c/66/ae227524b066d29d55bf0b453d93d2d793c40218657d643dcbbca13b8faf/lxml-6.1.1-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e902da4b04e6b52e5893900d4b8ab46068f75f3561f01bf1080957f9fd932ed6", size = 5613467, upload-time = "2026-05-18T19:19:16.228Z" }, - { url = "https://files.pythonhosted.org/packages/a6/76/dbe4a00b50385e40194231dcfe5a12c059de7cf90e89c83407d2b085b719/lxml-6.1.1-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1d4962d4c66bf830a7e59ed6cfc17d148149898a3aefa8ec6e59763e6e3ed085", size = 5228304, upload-time = "2026-05-18T19:19:19.354Z" }, - { url = "https://files.pythonhosted.org/packages/1c/01/00b1b8442ed2041793336868ba0b9ea4b13d7da7c085c6404c207a63bf79/lxml-6.1.1-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:581d4c8ae690a6609e64862dd6b7c2489635c2d13907fc2b20f2bc200ff1d21e", size = 5341607, upload-time = "2026-05-18T19:19:22.297Z" }, - { url = "https://files.pythonhosted.org/packages/63/36/1ad29931e9a4638bb707869f01d423a6c815f82152138d1a40dfcfde2b95/lxml-6.1.1-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:876e1ff5930ed8bf295ec5ef9a8155e9b6b1876bbf1deed8b3a8069311875a8f", size = 4700168, upload-time = "2026-05-18T19:19:25.133Z" }, - { url = "https://files.pythonhosted.org/packages/3c/d1/a9536cecf9be18a0dc72d32bead283a2332d1ffebd2dd3ac70ce444686e5/lxml-6.1.1-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9eb9b5a968f6e0f6d640092a567e14529ff8cea2e29d00da6f78a79fa49f013c", size = 5232487, upload-time = "2026-05-18T19:19:28.603Z" }, - { url = "https://files.pythonhosted.org/packages/0e/77/b4fb1e03bf5d130e879214d3100092e386418807fb74dd0adc4b0a48f351/lxml-6.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:aa49e06d94aba782c6a02eecb7e507969e7e7a41b267f1b359bb35585f295d5b", size = 5044231, upload-time = "2026-05-18T19:18:42.246Z" }, - { url = "https://files.pythonhosted.org/packages/26/4c/d00daeeb0a5530c4028a9232aa1b93db3ef4ed2158c116ea73c79a9765b3/lxml-6.1.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:70cdfd80589d59e43e18005dd7244e8895e93db8ab6a620b7e23df5445a4e3d2", size = 4769450, upload-time = "2026-05-18T19:18:48.013Z" }, - { url = "https://files.pythonhosted.org/packages/ed/6a/715a3a8d156ce42f29cf014706f5410c2ff3b02267774110fc23266409fe/lxml-6.1.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:aad9aa39483ed8ec44d6d2e59e5b98a0d80676ef0d92f44bfc374836111f62f5", size = 5635874, upload-time = "2026-05-18T19:18:51.914Z" }, - { url = "https://files.pythonhosted.org/packages/45/37/0544bc21dde2a88f3a17b504e6fc79c0e01d25a33c2f6079724e9e72b9c7/lxml-6.1.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:d49514be2f28d895c38cf9d2b72d7b9a07d00314519f456c0b50b53cfcf4c785", size = 5223987, upload-time = "2026-05-18T19:18:59.715Z" }, - { url = "https://files.pythonhosted.org/packages/4d/f8/f6a5e8185bcb28c2befae3d31f8e3df3b811cb0f47746517a81279fcafe1/lxml-6.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:47402e62c52ff5988c1e8c6c63177f5708bccf48e366dea4e3dcf1e645e04947", size = 5250276, upload-time = "2026-05-18T19:19:03.834Z" }, - { url = "https://files.pythonhosted.org/packages/c7/f2/1a2b9f1b7a49d45495369be7ef9ad05b262930f2eab3e3145706fca8083f/lxml-6.1.1-cp313-cp313-win32.whl", hash = "sha256:3483644525531e1d5762b0c44a8e18b6efba321b6dcf8a8952de10b037618bca", size = 3596903, upload-time = "2026-05-18T19:17:29.863Z" }, - { url = "https://files.pythonhosted.org/packages/e6/99/f4ffb024f238eec2131aaa09f3278fb6129cf892741bf68e1fc1afb8c100/lxml-6.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:a10bd2fd62e8ce916ececb342f348f190724a098c1faa056fdfb2a22ad5e8660", size = 3995869, upload-time = "2026-05-18T19:18:02.596Z" }, - { url = "https://files.pythonhosted.org/packages/d1/53/70eb8c5c6037f27448f1e3c54ebede9545a801ae63f0a7254afca4fe8e45/lxml-6.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:424aa57aca0897eb922aef34395bd1289b3b6f04e6bae20ea123c0c7e333cffc", size = 3658490, upload-time = "2026-05-19T19:22:53.846Z" }, - { url = "https://files.pythonhosted.org/packages/13/e2/2e325795566de01d0d7c3bb57d3c370616b2d07b01214e84eec5d3b10963/lxml-6.1.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:19b7ab10b210b0b3ad7985d9ac4eb66ab09a90b20fe6e2f7ba55d01a234345d0", size = 8577146, upload-time = "2026-05-18T19:18:17.765Z" }, - { url = "https://files.pythonhosted.org/packages/93/cf/5630b5e4be7d2e6bee8efe83865c925221103cf0221303b104ce134b01e2/lxml-6.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c08e5c694306507275f2290073350c4f32e383db15213b2c69e7ff39c1193840", size = 4623866, upload-time = "2026-05-18T19:18:30.669Z" }, - { url = "https://files.pythonhosted.org/packages/d2/51/3904907c063451cf8d4a5c9fe0cad95fa1f4ec57f4e3884fa0731bd7a305/lxml-6.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:74a9717fd0d82effef5c2854f0d917231d5324b5a3eb7275c43ac9fa32f97a14", size = 4950022, upload-time = "2026-05-18T19:19:31.958Z" }, - { url = "https://files.pythonhosted.org/packages/94/cd/9c7611a51c37a2830928405817cc5d56a97f64fab83cc3f628748b135749/lxml-6.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:efe0374196335f93b53269acd811b944f2e6bdc88e8894f214bd636455484909", size = 5086695, upload-time = "2026-05-18T19:19:34.764Z" }, - { url = "https://files.pythonhosted.org/packages/da/d6/24e3b5906abb0b674ff2ae195bc3ce59708df2bcd17cf17703b2d7dd643a/lxml-6.1.1-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac931cdc9442c1763b8a8f6cd62c0c938737eafc5be75eff88df55fc73bc0d00", size = 5031642, upload-time = "2026-05-18T19:19:37.771Z" }, - { url = "https://files.pythonhosted.org/packages/2d/db/6ec54f99019838bff54785c51da07f189eb4676861c5f2730962b0d8d665/lxml-6.1.1-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:aee395f5d0927f947758b4ec119fd5fc8ec71f07a1c5c52077b30b04c0fa6955", size = 5647338, upload-time = "2026-05-18T19:19:40.553Z" }, - { url = "https://files.pythonhosted.org/packages/42/3d/ef4dcfffd22d27a61805d8ed9f7fb888495bc6aa88648fa07c1eaa5586b6/lxml-6.1.1-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9395002973c827b3ed67db77e6ec09f092919a587022174554096a269378fb13", size = 5239528, upload-time = "2026-05-18T19:19:43.657Z" }, - { url = "https://files.pythonhosted.org/packages/62/bb/37fb3f0dff146bdcfa78eec47879273820b2a0bf350ec236ce14bd0b1c26/lxml-6.1.1-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:73bc2086f141224ebddb7fc5c6a36ca58b31b94b561e1dfe8e073e3270fad1e7", size = 5350730, upload-time = "2026-05-18T19:19:46.307Z" }, - { url = "https://files.pythonhosted.org/packages/90/42/43253f168388df4fae1f38c01df36ddb9bee39e2048167b54cdcbae85ea3/lxml-6.1.1-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:3779def59032b81e44a5f70096ef6bf2082f8d901937dca354474ba09782e245", size = 4697530, upload-time = "2026-05-18T19:19:49.889Z" }, - { url = "https://files.pythonhosted.org/packages/eb/a8/c5a8504f81bbdfc8e7094c2c850cdb4ed6777fc4d5ddd9e5ab819f3b0d54/lxml-6.1.1-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:86c89b9d55ebf820ad7c90bc533410f0d098054f293351f10603c0c46ff598f5", size = 5250670, upload-time = "2026-05-18T19:19:53.199Z" }, - { url = "https://files.pythonhosted.org/packages/77/b7/c7e76ab18744d75e21f320ebf9ff9d1ceae2b54dd431ea5a64caf26c9672/lxml-6.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19607c6bbff2a44cf3fe8250abccd20942d3462473e0a721d01d379ed017e462", size = 5084485, upload-time = "2026-05-18T19:19:08.422Z" }, - { url = "https://files.pythonhosted.org/packages/31/31/b35c53f8ef7b7c31cacd23d3638652fff7bcd1deb6eedb709ab43b685908/lxml-6.1.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:c6ed5141a5c7507cf3ee76bd363b0d6f801e3321adc35b5d825a23115faa5465", size = 4737635, upload-time = "2026-05-18T19:19:12.321Z" }, - { url = "https://files.pythonhosted.org/packages/d9/06/31f23c813a7fe8e0cb1b175e915b08c9bf4e86d225b210feadbdbe519667/lxml-6.1.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:62aeb7e85b5d60320b9d77eef2e773994e2c0ce10121b277e0a19804e1654a5a", size = 5670681, upload-time = "2026-05-18T19:19:15.001Z" }, - { url = "https://files.pythonhosted.org/packages/1a/bc/ce619bccc89b1fd9ad8a8e1330ee3f3beff9f2ff95b712d7bbcdd6e22fc3/lxml-6.1.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b1b963fd8f5caa68e99dfae060d54de1fe9cba899b8718b44a00cdca53c3e590", size = 5238229, upload-time = "2026-05-18T19:19:18.131Z" }, - { url = "https://files.pythonhosted.org/packages/2f/5d/b329acbbedc0b619ebc2be6cf7ee9ed07e80892c88d4dfd612c33805789a/lxml-6.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:63876be28efefa04a1df615b46770e82042cce445cfdce55160522f57b231ccb", size = 5264191, upload-time = "2026-05-18T19:19:21.118Z" }, - { url = "https://files.pythonhosted.org/packages/d6/85/be36fb1425b30db3c3f9df75fe86343ebffb79e6320bd7f588e25bfeac39/lxml-6.1.1-cp314-cp314-win32.whl", hash = "sha256:7f7a92e8583f06b1fd49d01158143b8461cfcd135dcb10ec807270a3051bd603", size = 3657202, upload-time = "2026-05-18T19:17:39.509Z" }, - { url = "https://files.pythonhosted.org/packages/b8/ce/3cf9a827342269f54d405a6202397de63f07c69cbd6ce7d183a3f0cba1e9/lxml-6.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:b2d444f2e66624d68e9c6b211e28a76e22fff5fcabcfff4deac18b529b7d4137", size = 4064497, upload-time = "2026-05-18T19:18:14.662Z" }, - { url = "https://files.pythonhosted.org/packages/d9/3e/1a957bde8f0760039e627f94699f82caa782c9d838d86c3d28245ee67212/lxml-6.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:3fd9728a2735fda14f4e8235830c86b539e9661e849665bf926d3f867943b4bf", size = 3741991, upload-time = "2026-05-19T19:22:59.111Z" }, - { url = "https://files.pythonhosted.org/packages/78/b2/00ed55b3a2efa4658fb795c38d1090ec9b3e8a6c3683d4441fa517f09c3b/lxml-6.1.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:787b2496d0dbe8cd180984e8d29e3a6f76e7ea34db781cb3bd55e4ba1ef8b4ee", size = 8827545, upload-time = "2026-05-18T19:18:41.193Z" }, - { url = "https://files.pythonhosted.org/packages/c0/73/74573db19baa618d5f266f2407898b087ff6927115b00b71e5fc1b700847/lxml-6.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2c8daa471358dc2d6fcf02165e80ec68f77871a286df95bc5cc3816153b0fd2c", size = 4735736, upload-time = "2026-05-18T19:18:46.761Z" }, - { url = "https://files.pythonhosted.org/packages/16/02/6f7061f4f95f51e545d48e87647c54791d204a4e881be4156e7a26ba5338/lxml-6.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:acd7d70b64c0aae0c7922cca83d288a16f5f6da523637697872253415269baef", size = 4970291, upload-time = "2026-05-18T19:19:56.215Z" }, - { url = "https://files.pythonhosted.org/packages/b0/02/55fc057d8283427dea7d6edb102e7a840239c77a64a983d92f62a304c0e9/lxml-6.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4f0dd2f01f9f8a89f565d000e03abcf0a13d692a346c8d22f628d49af098777a", size = 5102822, upload-time = "2026-05-18T19:19:59.223Z" }, - { url = "https://files.pythonhosted.org/packages/e4/48/8e1cf78d89d66850121d9255a2a24414c98f775da93b90cf976956c24b14/lxml-6.1.1-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b7e8a14c8634bf6f7a568634cb395305a6d964aeb5b7ee32248094bed3a7e2c", size = 5027923, upload-time = "2026-05-18T19:20:01.549Z" }, - { url = "https://files.pythonhosted.org/packages/ed/00/0632a0647612c8af24d26997b3b961397daa9d5b2581444805933629a4cb/lxml-6.1.1-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:86281fbdd6a8162756f8d603f37e3435bfa38043adb79c6dc6a2dfee065e7525", size = 5595843, upload-time = "2026-05-18T19:20:03.93Z" }, - { url = "https://files.pythonhosted.org/packages/bc/86/ab008a7dc360711b66858d61c80a5979a70a09f2aa2b05d9698df80b803d/lxml-6.1.1-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5d7152ec39ca7c402d8fb9bad86140a15b9503bd0c54484e3f1bbe3dd37ceca", size = 5224515, upload-time = "2026-05-18T19:20:06.381Z" }, - { url = "https://files.pythonhosted.org/packages/75/c6/2702ff375e728e34f56d9a45339a9cf7e4427e917f542225242d63a05afa/lxml-6.1.1-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:88d8cb75b9d82858497a5393e3c63cfbf03035225e4b35a49ed7ccb151e4dc0e", size = 5312511, upload-time = "2026-05-18T19:20:09.308Z" }, - { url = "https://files.pythonhosted.org/packages/b7/57/a5807c98f87a86f10ef9ffab35516df7c0f0c4b6d5d33e9f608ab9c04a31/lxml-6.1.1-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f64ec5397ea6a41fc1b4af0380d79b44a755b5531dcaccd9940fb260dca93038", size = 4639206, upload-time = "2026-05-18T19:20:11.704Z" }, - { url = "https://files.pythonhosted.org/packages/1f/e1/8a0a2c35734812395f4da4eaf33748a7e5705bfb2a58b128da764339d5ec/lxml-6.1.1-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d34bbf07dbc7ca5970671b1512e928991fb5e9d95365636c9b2d8b4f53af405e", size = 5232404, upload-time = "2026-05-18T19:20:14.064Z" }, - { url = "https://files.pythonhosted.org/packages/c2/e2/0e6a4dd5ad84d01d99aa7bae7cfefd4a760a0e0f8176818241de17d9b6c0/lxml-6.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:17e0e18d4ad8adbd0399291bc44845b69d9dd68439a3cdebdf35ff902ec05072", size = 5083769, upload-time = "2026-05-18T19:19:23.758Z" }, - { url = "https://files.pythonhosted.org/packages/a0/7e/161f33d463f6ffc1c7679104b65086dea120080d49dde4d238f015aaee2f/lxml-6.1.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:3ab541146f1f6968c462d6c2ac495148e8cdba2f8347700b2141b6ec5a75bf52", size = 4758936, upload-time = "2026-05-18T19:19:27.256Z" }, - { url = "https://files.pythonhosted.org/packages/f1/fb/2369825e3f6ca99305bf9f7b7085fda91c8b0922a89e54d900974aa3ef85/lxml-6.1.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2a0217714657e023ef4293500f65aa20fce6164c8fd6b08fa5bd4a859fb14b9b", size = 5620296, upload-time = "2026-05-18T19:19:29.993Z" }, - { url = "https://files.pythonhosted.org/packages/30/90/d61e383146f74c5ab683947ea14dc7b82778838ab9b95ea73a23b60d0191/lxml-6.1.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:05a82eb6e1530a64f26225b55cbd178113bd0b5af1c2b625f25e5296742c26d2", size = 5228598, upload-time = "2026-05-18T19:19:33.523Z" }, - { url = "https://files.pythonhosted.org/packages/76/2d/2dafd8149e94b05bb070690efd5bb2680720681e03ff03fc57d2b70a1105/lxml-6.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9e36f163528fc50cbef305f02a5fd66d404edf7049cdaff211dbc2cba5a7013e", size = 5247845, upload-time = "2026-05-18T19:19:36.649Z" }, - { url = "https://files.pythonhosted.org/packages/ce/68/b30e913340c380ddac9580c6e6230991fc37240ec4f64704833e4f3e2769/lxml-6.1.1-cp314-cp314t-win32.whl", hash = "sha256:649dda677cf3bd6ac9ae14007ba0c824ded8ce5808b53fc7431d9140399118c1", size = 3897345, upload-time = "2026-05-18T19:17:33.562Z" }, - { url = "https://files.pythonhosted.org/packages/3c/4e/9eb2af5335545f9fbcd7af57bcf87c6025d31eaa31b14ec184a6c8675328/lxml-6.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:793033d6c5cdf33a573f910d9bea14ef8f5771820411d118da8e1182edb53d5e", size = 4393350, upload-time = "2026-05-18T19:18:10.076Z" }, - { url = "https://files.pythonhosted.org/packages/7f/2c/0f1e93c636720e8a3eb59af2bfda99d98b55891e1c53bc30c2e0e865f01b/lxml-6.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:58bb955caba94e467d2a96da17660d2d704e0675894cba21ab8a775b8621fd1c", size = 3817223, upload-time = "2026-05-19T19:22:56.823Z" }, -] - [[package]] name = "magic-filter" version = "1.0.12" @@ -2388,19 +2316,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/71/45/689603d04b3bb8d7faa00f25c24acef993aab7813b3dbbfc472a459ab0b5/python_discovery-1.5.2-py3-none-any.whl", hash = "sha256:3e338c2d0f15dfaeea57493f4c2c6caebe0e998ea815c30ae8bf8ee21f1112d3", size = 38350, upload-time = "2026-08-12T14:05:25.113Z" }, ] -[[package]] -name = "python-docx" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "lxml" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a9/f7/eddfe33871520adab45aaa1a71f0402a2252050c14c7e3009446c8f4701c/python_docx-1.2.0.tar.gz", hash = "sha256:7bc9d7b7d8a69c9c02ca09216118c86552704edc23bac179283f2e38f86220ce", size = 5723256, upload-time = "2025-06-16T20:46:27.921Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d0/00/1e03a4989fa5795da308cd774f05b704ace555a70f9bf9d3be057b680bcf/python_docx-1.2.0-py3-none-any.whl", hash = "sha256:3fd478f3250fbbbfd3b94fe1e985955737c145627498896a8a6bf81f4baf66c7", size = 252987, upload-time = "2025-06-16T20:46:22.506Z" }, -] - [[package]] name = "python-dotenv" version = "1.2.3"