diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000..a7af44a --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,123 @@ +# Test-only infrastructure for integration tests. +# +# Mirrors the real infra (postgres + redis + rabbitmq + minio) but uses +# dedicated container names and host ports so it can coexist with the +# development stack (`docker-compose.yml`). Nothing in this file runs +# application services — only the data stores required by +# tests/integration/conftest.py. +# +# Usage: +# docker compose -f docker-compose.test.yml up -d --wait +# uv run pytest -m integration +# docker compose -f docker-compose.test.yml down -v +# +# The conftest.py fixture `infra` brings this file up automatically before the +# first integration test and tears it down after the session. + +services: + postgres-test: + image: postgres:18-alpine + restart: "no" + command: + - "postgres" + - "-c" + - "wal_level=replica" + - "-c" + - "archive_mode=on" + - "-c" + - "archive_command=test ! -f /walarchive/%f && cp %p /walarchive/%f" + environment: + POSTGRES_USER: contract_check + POSTGRES_PASSWORD: contract_check + POSTGRES_DB: contract_check + volumes: + - pgdata-test:/var/lib/postgresql + - pgwal-test:/walarchive + ports: + - "25432:5432" + healthcheck: + test: + - CMD-SHELL + - "pg_isready -U contract_check -d contract_check" + interval: 5s + timeout: 3s + retries: 10 + + redis-test: + image: redis:8-alpine + restart: "no" + command: ["redis-server", "--appendonly", "yes"] + volumes: + - redisdata-test:/data + ports: + - "27379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + + rabbitmq-test: + image: rabbitmq:4-management-alpine + restart: "no" + environment: + RABBITMQ_DEFAULT_USER: contract_check + RABBITMQ_DEFAULT_PASS: contract_check + RABBITMQ_DEFAULT_VHOST: / + volumes: + - rabbitmq-test:/var/lib/rabbitmq + ports: + - "6672:5672" # AMQP + - "25672:15672" # management UI (http://localhost:25672) + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 15s + + minio-test: + image: minio/minio:latest + restart: "no" + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: contract_check + MINIO_ROOT_PASSWORD: contract_check + volumes: + - minio-test:/data + ports: + - "10000:9000" # S3 API + - "10001:9001" # console (http://localhost:10001) + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:9000/minio/health/ready"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 10s + + minio-init-test: + image: minio/mc:latest + depends_on: + minio-test: + condition: service_healthy + entrypoint: /bin/sh + command: + - -c + - | + set -e + mc alias set local http://minio-test:9000 contract_check contract_check + mc mb --ignore-existing local/contract-check-docs-test + mc anonymous set none local/contract-check-docs-test || true + mc ilm rule add --expire-days 7 local/contract-check-docs-test || true + echo "bucket contract-check-docs-test ready" + environment: + MINIO_ROOT_USER: contract_check + MINIO_ROOT_PASSWORD: contract_check + restart: "no" + +volumes: + pgdata-test: + pgwal-test: + redisdata-test: + rabbitmq-test: + minio-test: diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index a9c294a..d327e47 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,15 +1,21 @@ -"""Integration-test fixtures using the running Docker Compose infrastructure. +"""Integration-test fixtures using an isolated Docker Compose infrastructure. -Run `docker compose up -d` before executing integration tests. Migrations run -once per session; a service token is seeded so adapter-style endpoints can -authenticate. +The test infra (postgres + redis + rabbitmq + minio) is brought up automatically +by the session-scoped `infra` fixture and torn down after the session. This +keeps integration tests from racing against the development service workers +that also consume RabbitMQ queues. + +To run against an already-running external infra instead, set +``INTEGRATION_TEST_USE_EXTERNAL_INFRA=1`` before invoking pytest. """ from __future__ import annotations +import json import os import subprocess import sys +import time from collections.abc import Iterator from pathlib import Path from typing import TYPE_CHECKING @@ -32,9 +38,113 @@ pytestmark = pytest.mark.integration _BOT_SERVICE_TOKEN = "it-test-bot-token" -_DB_URL = "postgresql+asyncpg://contract_check:contract_check@localhost:15432/contract_check" -_AMQP_URL = "amqp://contract_check:contract_check@localhost:5672//" -_S3_URL = "http://localhost:9000" +# Ports must match docker-compose.test.yml. +_TEST_DB_URL = "postgresql+asyncpg://contract_check:contract_check@localhost:25432/contract_check" +_TEST_AMQP_URL = "amqp://contract_check:contract_check@localhost:6672//" +_TEST_S3_URL = "http://localhost:10000" + +_DB_URL = _TEST_DB_URL +_AMQP_URL = _TEST_AMQP_URL +_S3_URL = _TEST_S3_URL + +_USE_EXTERNAL_INFRA = os.environ.get("INTEGRATION_TEST_USE_EXTERNAL_INFRA", "0") == "1" + + +def _repo_root() -> Path: + return Path(__file__).resolve().parents[2] + + +def _run(cmd: list[str], *, check: bool = True) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + cmd, + cwd=str(_repo_root()), + env={**os.environ}, + check=False, + capture_output=True, + text=True, + ) + if check and result.returncode != 0: + raise subprocess.CalledProcessError( + result.returncode, + cmd, + output=result.stdout, + stderr=result.stderr, + ) + return result + + +def _docker_compose_test(args: list[str]) -> list[str]: + # Use a dedicated project name so the test stack is independent from the + # development stack. Without this, `docker compose` treats the running dev + # services as "orphans" and returns a non-zero exit code. + return [ + "docker", + "compose", + "-p", + "contract-check-test", + "-f", + str(_repo_root() / "docker-compose.test.yml"), + *args, + ] + + +def _wait_for_healthy(service: str, *, deadline_seconds: int = 60) -> None: + """Poll `docker compose ps` until `service` reports healthy.""" + start = time.monotonic() + while time.monotonic() - start < deadline_seconds: + result = _run( + _docker_compose_test(["ps", service, "--format", "json"]), + check=False, + ) + if result.returncode == 0 and result.stdout: + # `docker compose ps --format json` emits one JSON object per line. + for line in result.stdout.strip().splitlines(): + try: + info = json.loads(line) + except json.JSONDecodeError: + continue + if info.get("Health") == "healthy": + return + if info.get("State") == "exited" and info.get("ExitCode") == 0: + # One-shot containers are also fine. + return + time.sleep(1) + raise RuntimeError(f"service {service} did not become healthy within {deadline_seconds}s") + + +def _start_test_infra() -> None: + if _USE_EXTERNAL_INFRA: + return + + # Bring up the test stack without --wait: minio-init-test is a one-shot + # container that exits after creating the bucket, which makes --wait fail. + _run(_docker_compose_test(["up", "-d"])) + + # Wait for every persistent service to be healthy. + for service in ("postgres-test", "redis-test", "rabbitmq-test", "minio-test"): + _wait_for_healthy(service) + + # postgres healthcheck can still race with alembic, so explicitly wait for + # the DB to accept connections. + _run( + _docker_compose_test( + [ + "exec", + "-T", + "postgres-test", + "sh", + "-c", + "until pg_isready -U contract_check -d contract_check; do sleep 1; done", + ] + ) + ) + + +def _stop_test_infra() -> None: + if _USE_EXTERNAL_INFRA: + return + # Use --volumes to wipe test data between runs. Never use -v for external infra. + _run(_docker_compose_test(["down", "-v"]), check=False) def _set_env() -> None: @@ -44,8 +154,8 @@ def _set_env() -> None: "S3_ENDPOINT_URL": _S3_URL, "S3_ACCESS_KEY": "contract_check", "S3_SECRET_KEY": "contract_check", - "S3_BUCKET": "contract-check-docs", - "REDIS_URL": "redis://localhost:17379/0", + "S3_BUCKET": "contract-check-docs-test", + "REDIS_URL": "redis://localhost:27379/0", "OLLAMA_HOST": "http://localhost", "OLLAMA_API_KEY": "test", "JWT_SECRET": "it-test-jwt-secret-not-for-production", @@ -67,13 +177,19 @@ def infra() -> Iterator[dict[str, str]]: _set_env() get_settings.cache_clear() - repo_root = Path(__file__).resolve().parents[2] - subprocess.run( - [sys.executable, "-m", "alembic", "-c", str(repo_root / "alembic.ini"), "upgrade", "head"], - cwd=str(repo_root), - env={**os.environ}, + _start_test_infra() + + _run( + [ + sys.executable, + "-m", + "alembic", + "-c", + str(_repo_root() / "alembic.ini"), + "upgrade", + "head", + ], check=True, - capture_output=False, ) factory = create_session_factory() @@ -101,10 +217,12 @@ def infra() -> Iterator[dict[str, str]]: "s3_endpoint_url": _S3_URL, "s3_access_key": "contract_check", "s3_secret_key": "contract_check", + "s3_bucket": "contract-check-docs-test", "token": _BOT_SERVICE_TOKEN, } get_settings.cache_clear() + _stop_test_infra() @pytest_asyncio.fixture diff --git a/tests/integration/test_admin_review.py b/tests/integration/test_admin_review.py index 5495144..d15b62b 100644 --- a/tests/integration/test_admin_review.py +++ b/tests/integration/test_admin_review.py @@ -35,7 +35,7 @@ def _storage(infra: dict[str, str]) -> MinioStorage: endpoint_url=infra["s3_endpoint_url"], access_key=infra["s3_access_key"], secret_key=infra["s3_secret_key"], - bucket="contract-check-docs", + bucket=infra["s3_bucket"], ) diff --git a/tests/integration/test_analyze_worker.py b/tests/integration/test_analyze_worker.py index 6b3af81..07e5b64 100644 --- a/tests/integration/test_analyze_worker.py +++ b/tests/integration/test_analyze_worker.py @@ -63,7 +63,7 @@ def _storage(infra: dict[str, str]) -> MinioStorage: endpoint_url=infra["s3_endpoint_url"], access_key=infra["s3_access_key"], secret_key=infra["s3_secret_key"], - bucket="contract-check-docs", + bucket=infra["s3_bucket"], ) diff --git a/tests/integration/test_extract_worker.py b/tests/integration/test_extract_worker.py index 419e604..a275c1b 100644 --- a/tests/integration/test_extract_worker.py +++ b/tests/integration/test_extract_worker.py @@ -39,7 +39,7 @@ def storage(infra: dict[str, str]) -> MinioStorage: endpoint_url=infra["s3_endpoint_url"], access_key=infra["s3_access_key"], secret_key=infra["s3_secret_key"], - bucket="contract-check-docs", + bucket=infra["s3_bucket"], ) diff --git a/tests/integration/test_prescreen_worker.py b/tests/integration/test_prescreen_worker.py index 6b2f67d..5502d51 100644 --- a/tests/integration/test_prescreen_worker.py +++ b/tests/integration/test_prescreen_worker.py @@ -100,7 +100,7 @@ def _storage(infra: dict[str, str]) -> MinioStorage: endpoint_url=infra["s3_endpoint_url"], access_key=infra["s3_access_key"], secret_key=infra["s3_secret_key"], - bucket="contract-check-docs", + bucket=infra["s3_bucket"], ) diff --git a/tests/integration/test_review_actions.py b/tests/integration/test_review_actions.py index 777daf9..8d0692a 100644 --- a/tests/integration/test_review_actions.py +++ b/tests/integration/test_review_actions.py @@ -83,7 +83,7 @@ def _storage(infra: dict[str, str]) -> MinioStorage: endpoint_url=infra["s3_endpoint_url"], access_key=infra["s3_access_key"], secret_key=infra["s3_secret_key"], - bucket="contract-check-docs", + bucket=infra["s3_bucket"], )