226 lines
7.9 KiB
Python
226 lines
7.9 KiB
Python
"""Ollama Cloud adapter unit tests (respx-mocked).
|
|
|
|
Covers: 200 happy path, 429→fallback, invalid-JSON→repair→success, and the
|
|
provider returning a merged AnalysisResult. No network.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import httpx
|
|
import pytest
|
|
import respx
|
|
|
|
from contract_check.core.analysis.report_schema import ReportPayload
|
|
from contract_check.core.llm.ollama_cloud import (
|
|
LLMConfigError,
|
|
LLMError,
|
|
LLMQuotaError,
|
|
LLMUnavailableError,
|
|
OllamaCloudProvider,
|
|
)
|
|
|
|
HOST = "https://ollama.test"
|
|
URL = f"{HOST}/api/chat"
|
|
|
|
VALID_PAYLOAD = ReportPayload(
|
|
findings=[
|
|
{
|
|
"checklist_id": "penalties",
|
|
"severity": "high",
|
|
"quote": "Штраф 0,5% за каждый день просрочки",
|
|
"section_ref": "п. 6.3",
|
|
"risk": "Высокая неустойка",
|
|
"recommendation": "Ограничить cap",
|
|
}
|
|
]
|
|
)
|
|
|
|
|
|
def _chat_body(content: str, model: str = "qwen2.5:14b") -> dict[str, object]:
|
|
return {
|
|
"model": model,
|
|
"message": {"content": content},
|
|
"prompt_eval_count": 10,
|
|
"eval_count": 20,
|
|
}
|
|
|
|
|
|
def _provider() -> OllamaCloudProvider:
|
|
return OllamaCloudProvider(
|
|
host=HOST,
|
|
api_key="key",
|
|
model="qwen2.5:14b",
|
|
fallback_model="qwen2.5:7b",
|
|
max_concurrency=1,
|
|
chunk_size=10000,
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_happy_path_returns_findings() -> None:
|
|
async with _provider() as provider:
|
|
with respx.mock(base_url=HOST) as mock:
|
|
mock.post(URL).mock(
|
|
return_value=httpx.Response(200, json=_chat_body(VALID_PAYLOAD.model_dump_json()))
|
|
)
|
|
result = await provider.analyze("договор " * 200)
|
|
|
|
assert len(result.findings) == 1
|
|
assert result.findings[0].checklist_id == "penalties"
|
|
assert result.findings[0].severity == "high"
|
|
assert result.prompt_tokens == 10
|
|
assert result.eval_tokens == 20
|
|
assert result.fell_back is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_429_falls_back_to_secondary_model() -> None:
|
|
async with _provider() as provider:
|
|
with respx.mock(base_url=HOST) as mock:
|
|
route = mock.post(URL)
|
|
route.mock(
|
|
side_effect=[
|
|
httpx.Response(429, json={"error": "quota"}),
|
|
httpx.Response(
|
|
200, json=_chat_body(VALID_PAYLOAD.model_dump_json(), model="qwen2.5:7b")
|
|
),
|
|
]
|
|
)
|
|
result = await provider.analyze("договор " * 200)
|
|
|
|
assert result.fell_back is True
|
|
assert "qwen2.5:7b" in result.models_used
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_invalid_json_repaired_on_second_attempt() -> None:
|
|
async with _provider() as provider:
|
|
with respx.mock(base_url=HOST) as mock:
|
|
mock.post(URL).mock(
|
|
side_effect=[
|
|
httpx.Response(200, json=_chat_body("not valid json {")),
|
|
httpx.Response(200, json=_chat_body(VALID_PAYLOAD.model_dump_json())),
|
|
]
|
|
)
|
|
result = await provider.analyze("договор " * 200)
|
|
|
|
assert result.repaired is True
|
|
assert len(result.findings) == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_model_aliases_validate_without_repair() -> None:
|
|
"""Models often emit risk_type/description and omit recommendation.
|
|
|
|
The schema accepts these aliases, so no repair loop should fire.
|
|
"""
|
|
aliased = json.dumps(
|
|
{
|
|
"findings": [
|
|
{
|
|
"risk_type": "penalties",
|
|
"severity": "medium",
|
|
"description": "Высокая неустойка.",
|
|
"quote": "Штраф 0,5% за каждый день просрочки",
|
|
"section_ref": "п. 6.3",
|
|
}
|
|
]
|
|
}
|
|
)
|
|
async with _provider() as provider:
|
|
with respx.mock(base_url=HOST) as mock:
|
|
route = mock.post(URL).mock(return_value=httpx.Response(200, json=_chat_body(aliased)))
|
|
result = await provider.analyze("договор " * 200)
|
|
assert route.call_count == 1
|
|
assert result.repaired is False
|
|
assert result.findings[0].checklist_id == "penalties"
|
|
assert result.findings[0].risk == "Высокая неустойка."
|
|
assert result.findings[0].recommendation == ""
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_429_with_no_fallback_raises_quota() -> None:
|
|
provider = OllamaCloudProvider(
|
|
host=HOST, api_key="key", model="qwen2.5:14b", fallback_model=None
|
|
)
|
|
async with provider:
|
|
with respx.mock(base_url=HOST) as mock:
|
|
mock.post(URL).mock(return_value=httpx.Response(429, json={"error": "quota"}))
|
|
with pytest.raises(LLMQuotaError):
|
|
await provider.analyze("договор " * 200)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_invalid_json_after_repair_raises_llm_error() -> None:
|
|
async with _provider() as provider:
|
|
with respx.mock(base_url=HOST) as mock:
|
|
mock.post(URL).mock(return_value=httpx.Response(200, json=_chat_body("still not json")))
|
|
with pytest.raises(LLMError):
|
|
await provider.analyze("договор " * 200)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_short_text_single_chunk_one_request() -> None:
|
|
"""Short text (< chunk_size) produces exactly one LLM call."""
|
|
async with _provider() as provider:
|
|
with respx.mock(base_url=HOST) as mock:
|
|
route = mock.post(URL).mock(
|
|
return_value=httpx.Response(200, json=_chat_body(VALID_PAYLOAD.model_dump_json()))
|
|
)
|
|
await provider.analyze("короткий договор")
|
|
assert route.call_count == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_multi_chunk_merges_findings() -> None:
|
|
"""Two chunks → two calls → findings merged and deduped."""
|
|
findings = json.loads(VALID_PAYLOAD.model_dump_json())["findings"]
|
|
async with _provider() as provider:
|
|
with respx.mock(base_url=HOST) as mock:
|
|
mock.post(URL).mock(
|
|
return_value=httpx.Response(
|
|
200,
|
|
json=_chat_body(json.dumps({"findings": findings + findings})),
|
|
)
|
|
)
|
|
result = await provider.analyze("договор " * 3000, checklist="x")
|
|
# dedupe collapses identical (checklist_id, quote) pairs → 1 finding
|
|
assert len(result.findings) == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_404_endpoint_missing_raises_config_error() -> None:
|
|
"""A 404 from Ollama is a terminal config error, not a retryable one."""
|
|
async with _provider() as provider:
|
|
with respx.mock(base_url=HOST) as mock:
|
|
route = mock.post(URL)
|
|
route.mock(return_value=httpx.Response(404, text="Not Found"))
|
|
with pytest.raises(LLMConfigError):
|
|
await provider.analyze("договор " * 200)
|
|
assert route.call_count == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_connect_error_raises_config_error_immediately() -> None:
|
|
"""A connection error is terminal: the configured host is wrong/down."""
|
|
async with _provider() as provider:
|
|
with respx.mock(base_url=HOST) as mock:
|
|
route = mock.post(URL)
|
|
route.mock(side_effect=httpx.ConnectError("Connection refused"))
|
|
with pytest.raises(LLMConfigError):
|
|
await provider.analyze("договор " * 200)
|
|
assert route.call_count == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_503_retries_then_raises_unavailable() -> None:
|
|
"""5xx errors retry and eventually surface as LLMUnavailableError."""
|
|
async with _provider() as provider:
|
|
with respx.mock(base_url=HOST) as mock:
|
|
route = mock.post(URL)
|
|
route.mock(return_value=httpx.Response(503, text="Unavailable"))
|
|
with pytest.raises(LLMUnavailableError):
|
|
await provider.analyze("договор " * 200)
|
|
assert route.call_count == 3
|