From 1e051f3730909fa120c27379f5df94bbb51f709f Mon Sep 17 00:00:00 2001 From: febux Date: Wed, 2 Sep 2026 23:32:47 +0300 Subject: [PATCH] Grafana stack was added. Update docs. --- .env.example | 8 ++ Makefile | 31 +++++++ .../contract-check-http.conf.template | 24 +++++ .../templates/contract-check.conf.template | 24 +++++ .../grafana/dashboards/logs.json | 90 +++++++++++++++++++ .../provisioning/dashboards/dashboards.yaml | 12 +++ .../provisioning/datasources/datasources.yaml | 16 ++++ deploy/observability/loki-config.yaml | 43 +++++++++ deploy/observability/promtail-config.yaml | 28 ++++++ deploy/vps/front.conf | 3 +- docker-compose.yml | 53 +++++++++++ docs/ARCHITECTURE.md | 45 +++++++--- docs/DEPLOY.md | 23 ++++- docs/TICKETS.md | 20 +++-- tests/conftest.py | 52 +++++++++++ 15 files changed, 452 insertions(+), 20 deletions(-) create mode 100644 deploy/observability/grafana/dashboards/logs.json create mode 100644 deploy/observability/grafana/provisioning/dashboards/dashboards.yaml create mode 100644 deploy/observability/grafana/provisioning/datasources/datasources.yaml create mode 100644 deploy/observability/loki-config.yaml create mode 100644 deploy/observability/promtail-config.yaml diff --git a/.env.example b/.env.example index 6c58a52..c1c136b 100644 --- a/.env.example +++ b/.env.example @@ -51,6 +51,14 @@ SENTRY_DSN= OTEL_EXPORTER_OTLP_ENDPOINT= # e.g. http://otel-collector:4317 OTEL_SERVICE_NAME=contract-check +# --- Grafana / Loki (profile: obs) --- +GRAFANA_PORT=3000 +GRAFANA_ADMIN_USER=admin +GRAFANA_ADMIN_PASSWORD=admin +# External URL of the project domain. Grafana is also served under /grafana via nginx. +# Examples: http://localhost:3000 (dev direct), https://contract-check.example.com +GRAFANA_ROOT_URL=http://localhost:3000 + # --- LLM provider (abstract port; first realization = Ollama Cloud) --- # For Ollama Cloud use https://ollama.com (not api.ollama.com). Models must be # available on the chosen host — cloud models differ from local Ollama models. diff --git a/Makefile b/Makefile index 9415323..99d812a 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ .PHONY: help install lint lint-fix isort isort-check typecheck test test-unit test-integration migrate \ infra-up infra-down infra-logs services-up services-down services-logs \ + services-obs obs-up obs-down obs-logs obs-url obs-reset \ api api-logs bot bot-logs worker-extract worker-analyze worker-notify \ seed-token jwt-secret jwt-token jwt-verify health shell-api shell-bot \ shell-db admin-promote admin-list clean @@ -77,9 +78,35 @@ services-logs: ## Tail all service logs services-ps: ## Show running containers docker compose --profile services ps +services-obs: ## Start services + observability (logs) + docker compose --profile services --profile obs up -d --build --remove-orphans + services-nginx: ## Start with nginx reverse proxy docker compose --profile services --profile edge up -d --build --remove-orphans +# ───────────────────────────────────────────────────────────────────────────── +# Docker: observability (Grafana + Loki logs) +# ───────────────────────────────────────────────────────────────────────────── +obs-up: ## Start observability containers (needs infra or services running) + docker compose --profile obs up -d --build --remove-orphans + +obs-down: ## Stop observability containers + docker compose --profile obs down + +obs-logs: ## Tail observability containers logs + docker compose --profile obs logs -f + +obs-url: ## Print Grafana URL and default credentials + @echo "Direct: http://localhost:$${GRAFANA_PORT:-3000}" + @echo "Via nginx: $${GRAFANA_ROOT_URL:-http://localhost}/grafana/" + @echo "User: $${GRAFANA_ADMIN_USER:-admin}" + @echo "Password: $${GRAFANA_ADMIN_PASSWORD:-admin}" + @echo "Dashboard: $${GRAFANA_ROOT_URL:-http://localhost}/grafana/d/contract-check-logs" + +obs-reset: ## Reset Grafana and Loki volumes (wipes dashboards/logs data) + docker compose --profile obs down -v + rm -rf deploy/observability/grafana/dashboards/*.json.tmp + # ───────────────────────────────────────────────────────────────────────────── # Docker: nginx edge (reverse proxy) # ───────────────────────────────────────────────────────────────────────────── @@ -220,4 +247,8 @@ clean: ## Remove containers, volumes, caches # ───────────────────────────────────────────────────────────────────────────── dev: install infra-up migrate services-up ## Bootstrap full dev environment +dev-obs: install infra-up migrate services-obs ## Bootstrap full dev environment with observability + stop: services-down infra-down ## Stop everything + +stop-obs: obs-down services-down infra-down ## Stop everything including observability diff --git a/deploy/nginx/templates/contract-check-http.conf.template b/deploy/nginx/templates/contract-check-http.conf.template index fb9b199..f3a3633 100644 --- a/deploy/nginx/templates/contract-check-http.conf.template +++ b/deploy/nginx/templates/contract-check-http.conf.template @@ -63,6 +63,30 @@ server { proxy_read_timeout 60s; } + # Grafana / Loki: exposed under /grafana so everything lives on one domain. + # In production restrict access here or via firewall (basic auth / VPN / SSO). + location /grafana/ { + # Variable forces dynamic DNS resolution for `grafana`. + set $grafana http://grafana:3000; + proxy_pass $grafana; + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Correlation-Id $http_x_correlation_id; + + # Grafana needs the subpath so it can build correct links/redirects. + proxy_set_header X-Forwarded-Prefix /grafana; + + proxy_connect_timeout 10s; + proxy_send_timeout 30s; + proxy_read_timeout 60s; + proxy_buffering off; + } + location / { return 404; } diff --git a/deploy/nginx/templates/contract-check.conf.template b/deploy/nginx/templates/contract-check.conf.template index 1c4f15d..53fa744 100644 --- a/deploy/nginx/templates/contract-check.conf.template +++ b/deploy/nginx/templates/contract-check.conf.template @@ -108,6 +108,30 @@ server { proxy_read_timeout 60s; } + # Grafana / Loki: exposed under /grafana so everything lives on one domain. + # In production restrict access here or via firewall (basic auth / VPN / SSO). + location /grafana/ { + # Variable forces dynamic DNS resolution for `grafana`. + set $grafana http://grafana:3000; + proxy_pass $grafana; + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Correlation-Id $http_x_correlation_id; + + # Grafana needs the subpath so it can build correct links/redirects. + proxy_set_header X-Forwarded-Prefix /grafana; + + proxy_connect_timeout 10s; + proxy_send_timeout 30s; + proxy_read_timeout 60s; + proxy_buffering off; + } + # Static web SPA (future). Placeholder: serve pre-built static assets. # location / { # root /usr/share/nginx/html; diff --git a/deploy/observability/grafana/dashboards/logs.json b/deploy/observability/grafana/dashboards/logs.json new file mode 100644 index 0000000..1cfe338 --- /dev/null +++ b/deploy/observability/grafana/dashboards/logs.json @@ -0,0 +1,90 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { "type": "loki", "uid": "loki" }, + "enable": true, + "iconColor": "red", + "name": "Log events", + "target": { "expr": "{service=~\".+\"} |~ \"(?i)error|exception|fatal|panic\"" }, + "type": "logs" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "description": "All application logs from the docker-compose stack. Filter by service or search for a correlation_id.", + "gridPos": { "h": 20, "w": 24, "x": 0, "y": 0 }, + "id": 1, + "options": { + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true, + "prettifyLogMessage": true, + "enableLogDetails": true, + "dedupStrategy": "none" + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "{service=~\"${service:regex}\"} |= \"$query\"", + "refId": "A" + } + ], + "title": "Application logs", + "type": "logs" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "style": "dark", + "tags": ["logs", "loki"], + "templating": { + "list": [ + { + "current": { "selected": false, "text": ["All"], "value": ["$__all"] }, + "datasource": { "type": "loki", "uid": "loki" }, + "definition": "label_values(service)", + "hide": 0, + "includeAll": true, + "label": "Service", + "multi": true, + "name": "service", + "options": [], + "query": "label_values(service)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { "selected": false, "text": "", "value": "" }, + "hide": 0, + "label": "Search", + "name": "query", + "options": [ + { "selected": true, "text": "", "value": "" } + ], + "query": "", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { "from": "now-15m", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Contract Check — Logs", + "uid": "contract-check-logs", + "version": 1, + "weekStart": "" +} diff --git a/deploy/observability/grafana/provisioning/dashboards/dashboards.yaml b/deploy/observability/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..50d4aa9 --- /dev/null +++ b/deploy/observability/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: default + orgId: 1 + folder: Contract Check + type: file + disableDeletion: false + updateIntervalSeconds: 10 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards diff --git a/deploy/observability/grafana/provisioning/datasources/datasources.yaml b/deploy/observability/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..eb8f750 --- /dev/null +++ b/deploy/observability/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,16 @@ +apiVersion: 1 + +datasources: + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + isDefault: false + editable: false + jsonData: + maxLines: 1000 + derivedFields: + - name: correlation_id + matcherRegex: '"correlation_id":\s*"([^"]+)"' + url: /explore?left={"datasource":"Loki","queries":[{"expr":"{service=\"$service\"} |= \"$__value\""}]} + urlDisplayLabel: 'Trace by correlation_id' diff --git a/deploy/observability/loki-config.yaml b/deploy/observability/loki-config.yaml new file mode 100644 index 0000000..d6515db --- /dev/null +++ b/deploy/observability/loki-config.yaml @@ -0,0 +1,43 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: warn + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + reject_old_samples: true + reject_old_samples_max_age: 168h + allow_structured_metadata: true + +analytics: + reporting_enabled: false diff --git a/deploy/observability/promtail-config.yaml b/deploy/observability/promtail-config.yaml new file mode 100644 index 0000000..45ba7bc --- /dev/null +++ b/deploy/observability/promtail-config.yaml @@ -0,0 +1,28 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + log_level: warn + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + batchwait: 1s + batchsize: 1048576 + +scrape_configs: + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + relabel_configs: + - source_labels: [__meta_docker_container_name] + regex: '/(.*)' + target_label: container + - source_labels: [__meta_docker_container_label_com_docker_compose_service] + target_label: service + - source_labels: [__meta_docker_container_label_com_docker_compose_project] + target_label: project + - source_labels: [__meta_docker_container_label_com_docker_compose_profile] + target_label: profile diff --git a/deploy/vps/front.conf b/deploy/vps/front.conf index c56f34b..ed57f64 100644 --- a/deploy/vps/front.conf +++ b/deploy/vps/front.conf @@ -37,7 +37,8 @@ server { } # ── «Контракт-чек»: exact path set of its edge (contract-check-http.conf.template) - location ~ ^/(healthz|readyz|metrics|api/|admin/|docs|openapi\.json) { + # Grafana is also routed through the edge container under /grafana. + location ~ ^/(healthz|readyz|metrics|api/|admin/|docs|openapi\.json|grafana/) { proxy_pass http://contract_check_edge; proxy_http_version 1.1; proxy_set_header Host $host; diff --git a/docker-compose.yml b/docker-compose.yml index 2c088dc..9e7105e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -368,6 +368,57 @@ services: sleep 12h & wait $${!} done + # ── OBSERVABILITY (profile: obs) ─────────────────────────────────────────── + # Minimal Grafana + Loki logs stack. Promtail scrapes all compose container + # logs via the local Docker socket. See deploy/observability/ for configs. + loki: + profiles: ["obs"] + image: grafana/loki + container_name: contract_check-loki + restart: unless-stopped + command: -config.file=/etc/loki/loki-config.yaml + volumes: + - loki-data:/loki + - ./deploy/observability/loki-config.yaml:/etc/loki/loki-config.yaml:ro + ports: + - "13100:3100" + + promtail: + profiles: ["obs"] + image: grafana/promtail + container_name: contract_check-promtail + restart: unless-stopped + command: -config.file=/etc/promtail/promtail-config.yaml + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./deploy/observability/promtail-config.yaml:/etc/promtail/promtail-config.yaml:ro + + grafana: + profiles: ["obs"] + image: grafana/grafana + container_name: contract_check-grafana + restart: unless-stopped + volumes: + - grafana-data:/var/lib/grafana + - ./deploy/observability/grafana/provisioning:/etc/grafana/provisioning:ro + - ./deploy/observability/grafana/dashboards:/var/lib/grafana/dashboards:ro + ports: + - "${GRAFANA_PORT:-3000}:3000" + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin} + GF_USERS_ALLOW_SIGN_UP: "false" + # When served under a subpath via nginx, Grafana must know the root URL. + GF_SERVER_ROOT_URL: "${GRAFANA_ROOT_URL:-http://localhost:3000}/grafana/" + GF_SERVER_SERVE_FROM_SUB_PATH: "true" + GF_INSTALL_PLUGINS: "" + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:3000/api/health || exit 1"] + interval: 10s + timeout: 3s + retries: 10 + start_period: 30s + volumes: pgdata: pgwal: @@ -376,3 +427,5 @@ volumes: minio: certbot-data: certbot-webroot: + loki-data: + grafana-data: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 05e65d8..eba363e 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1090,6 +1090,10 @@ None of 1–5 requires touching `core/` application code — only compose/infra. | `SENTRY_DSN` | (empty) | if set, init sentry | | `OTEL_EXPORTER_OTLP_ENDPOINT` | (empty) | OTel collector | | `OTEL_SERVICE_NAME` | per-service | overridden in each service settings | +| `GRAFANA_PORT` | `3000` | host port for the Grafana UI (profile `obs`) | +| `GRAFANA_ADMIN_USER` | `admin` | initial Grafana admin user | +| `GRAFANA_ADMIN_PASSWORD` | `admin` | initial Grafana admin password | +| `GRAFANA_ROOT_URL` | `http://localhost:3000` | external Grafana URL for callbacks | ### LLM @@ -1280,6 +1284,9 @@ services: rabbitmq: rabbitmq:4-management-alpine, healthcheck, volume; AMQP 5672, mgmt UI 15672 minio: minio/minio, healthcheck, volume, console on :9001; S3 API :9000 minio-init: one-shot: mc alias + mb + ilm rule; depends_on minio healthy + loki: grafana/loki; filesystem-backed single-node log store + promtail: grafana/promtail; ships Docker container logs into Loki + grafana: grafana/grafana; provisioned Loki datasource + logs dashboard # ── SERVICES (profile: services) ── api: build srv/api/Dockerfile; depends_on pg/rabbit/minio-init healthy; @@ -1289,17 +1296,17 @@ services: worker-analyze: build srv/worker-analyze/Dockerfile; depends_on pg/rabbit/minio-init healthy; 9102 bot: build srv/bot/Dockerfile; depends_on api healthy (NOT pg/rabbit) - # ── OBSERVABILITY (profile: obs) ── PLANNED (T-E1-010) - prometheus: prom/prometheus; scrape api:9100, extract:9101, analyze:9102 - grafana: grafana/grafana; provisioned datasources + starter dashboards - otel-collector: otel/opentelemetry-collector-contrib; receives OTLP - tempo: grafana/tempo; trace storage + # ── OBSERVABILITY (profile: obs) ── + # Logs are live (Grafana + Loki + Promtail). Metrics/traces still planned. + loki: grafana/loki; filesystem-backed single-node log store + promtail: grafana/promtail; ships Docker container logs into Loki + grafana: grafana/grafana; provisioned Loki datasource + logs dashboard # ── EDGE (profile: edge) ── PLANNED (T-E1-009) nginx: nginx:alpine; reverse proxy → api; TLS via certbot certbot: certbot/certbot; renew cron sidecar -volumes: { pgdata, pgwal, redisdata, rabbitmq, minio } +volumes: { pgdata, pgwal, redisdata, rabbitmq, minio, loki-data, grafana-data } ``` `bot` depends on `api` healthy (not on infra) — it speaks HTTP to the api, @@ -1320,6 +1327,9 @@ enforcing the adapter boundary even in dependency ordering. contextvar before handling. So a single upload's logs trace api → rabbit → worker-extract → worker-analyze → DB under one ID. - OTel baggage/span context propagates the same ID for distributed traces. +- The same `correlation_id` is parsed by the Loki datasource as a derived field, + so you can click through from any log line to every other log line with the + same id. ### Sentry (`core/sentry.py`) @@ -1345,11 +1355,26 @@ enforcing the adapter boundary even in dependency ordering. - Workers run `prometheus_client.start_http_server(port)` in a background thread alongside the async consumer. -### Grafana +### Grafana + Loki -- Provisioned Prometheus + Tempo datasources. -- Starter dashboard: queue depth (`rabbitmq_queue_messages`), job duration - histogram, LLM tokens/min, refund rate, 429/fallback rate. +- `obs` profile adds `loki`, `promtail`, `grafana` to compose. +- Promtail discovers all compose containers via the Docker socket and pushes + their stdout/stderr to Loki; labels include `service`, `project`, `profile`, + and `container`. +- Logs retain the JSON format emitted by structlog in `staging`/`prod` and the + pretty console format in `dev`; Loki stores the raw line. +- Provisioned datasource `Loki` and a starter `Contract Check — Logs` + dashboard at `http://localhost:${GRAFANA_PORT:-3000}/d/contract-check-logs`. +- Dashboard has a `service` filter and a free-text search box; error words are + highlighted as annotations. Use correlation_id values to trace one upload + across `api` → `worker-extract` → `worker-prescreen` → `worker-analyze`. + +### Prometheus + Tempo (planned) + +- Prometheus will scrape `:9100`/`:9101`/`:9102` metrics exposed by the services. +- Tempo will receive OTLP traces via the OpenTelemetry collector. +- Starter dashboards for queue depth, job latency, LLM tokens/min, refund rate, + and 429/fallback rate will ship with the metrics/traces datasources. ### OpenTelemetry (`core/telemetry.py`) diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index 02af46f..fb277d1 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -187,8 +187,8 @@ make shell-db # UPDATE plans SET price_kopecks = 59000 WHERE code = 'lite'; |---|---| | *(default)* | `postgres`, `redis`, `rabbitmq`, `minio`, `minio-init` | | `services` | `api`, `worker-extract`, `worker-prescreen`, `worker-analyze`, `worker-billing`, `worker-notify`, `bot` | -| `obs` *(планируется)* | `prometheus`, `grafana`, `otel-collector`, `tempo` | -| `edge` *(планируется)* | `nginx`, `certbot` | +| `obs` | `loki`, `promtail`, `grafana` (logs only; metrics/traces — позже) | +| `edge` | `nginx`, `certbot` | ```bash # Инфра: @@ -197,8 +197,11 @@ docker compose up -d # + сервисы (сборка + запуск): docker compose --profile services up -d --build -# + observability + edge (когда будут готовы конфиги в deploy/): -# docker compose --profile services --profile obs --profile edge up -d --build +# + observability (логи): +docker compose --profile services --profile obs up -d --build + +# + edge (nginx reverse proxy + TLS; deploy/nginx готов): +docker compose --profile services --profile obs --profile edge up -d --build ``` ### 4.2 depends_on и healthchecks @@ -593,6 +596,18 @@ curl https://contract-check.example.com/healthz - `/healthz`, `/readyz` - `/metrics` (открыт наружу; закройте файрволом или уберите приватный скрейпинг) - `/webhook/*` → `/api/v1/webhooks/` +- `/grafana/*` → `grafana:3000` (когда поднят профиль `obs`) + +Grafana под путём `/grafana`: +```bash +# Открыть логи через основной домен (нужен профиль obs + edge) +docker compose --profile services --profile obs --profile edge up -d +# https://contract-check.example.com/grafana/d/contract-check-logs +``` + +> **Безопасность:** Grafana под `/grafana` доступна всем, у кого есть доступ к домену. +> На проде добавьте basic auth, IP whitelist или вынесите Grafana на отдельный +> поддомен с отдельным ingress/VPN. ### 13.3 Обновление сертификата diff --git a/docs/TICKETS.md b/docs/TICKETS.md index 426bce9..cb148e3 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -121,7 +121,7 @@ consume `DocumentExtracted` из `analyze.q` → скачать `.txt` → chunk `docker-compose.yml`: default-профиль = инфра (`postgres`, `redis`, `rabbitmq`, `minio`, `minio-init`); профиль `services` = `api`, `worker-extract`, `worker-prescreen`, `worker-analyze`, `worker-notify`, `bot` с `depends_on: service_healthy`; -профиль `edge` = nginx+certbot. Все 7 Dockerfile-ов в `srv/`. Профиль `obs` — позже (T-E1-010). +профиль `edge` = nginx+certbot. Все 7 Dockerfile-ов в `srv/`. Профиль `obs` добавляет `loki` + `promtail` + `grafana` для логов (сделано в T-E1-010a). ### T-E1-008 — Оплата (ЮKassa) и пополнение кредитов **Статус:** done · **Оценка:** M · **Зависимости:** T-E1-003, T-E1-006 @@ -136,10 +136,20 @@ state-machine, авто-возвраты с клавбэком и billing-hold, `deploy/nginx/templates/contract-check.conf.template`, `deploy/nginx/certbot-init.sh`, `DEPLOY.md §13`, `docker-compose.yml` профиль `edge`. -### T-E1-010 — Мониторинг (Prometheus/Grafana/Tempo/Sentry) -**Статус:** todo · **Оценка:** M · **Зависимости:** T-E1-007 +### T-E1-010a — Мониторинг: логи (Grafana + Loki + Promtail) +**Статус:** done · **Оценка:** S · **Зависимости:** T-E1-007 -`deploy/observability/` + compose профиль `obs`. Дашборды: queue depth, job latency, LLM tokens, credits. +`deploy/observability/loki-config.yaml`, `promtail-config.yaml`, `grafana/provisioning/{datasources,dashboards}/`, +`grafana/dashboards/logs.json`; профиль `obs` в `docker-compose.yml` поднимает `loki`, `promtail`, `grafana`. +Promtail забирает логи всех compose-контейнеров через Docker socket и пушит в Loki; +доступен дашборд `Contract Check — Logs` с фильтром по `service` и поиском по `correlation_id`. + +### T-E1-010b — Мониторинг: метрики + трейсы (Prometheus + Tempo + OTel collector + Sentry) +**Статус:** todo · **Оценка:** M · **Зависимости:** T-E1-010a + +Добавить `prometheus`, `tempo`, `otel-collector` в профиль `obs`. Prometheus scrape `:9100`/`:9101`/`:9102`. +OTel collector принимает OTLP и маршрутизирует в Tempo. Дашборды: queue depth, job latency, LLM tokens, credits. +Sentry уже инициализируется в коде при наличии `SENTRY_DSN`. --- @@ -216,7 +226,7 @@ Seed API key в `tests/integration/conftest.py`. 2. **E1.001–007** — ядро + БД + FastAPI `api` + worker-extract + worker-analyze + bot + Docker/compose (done). 3. **E3** — B2B API (done: миграция, rate-limit, auth, роуты, тесты). 4. **E1.008** — оплата ЮKassa (done — трек `.scratch/user-profile-billing/`, тикеты 001–019: профили, планы/подписки, платежи, возвраты, worker-billing, админ-панель). -5. **E1.009–010** — деплой/edge (Nginx/certbot — done) и observability (Prometheus/Grafana/Tempo/Sentry) (todo). +5. **E1.009–010** — деплой/edge (Nginx/certbot — done) и observability: логи Grafana+Loki+Promtail — done, метрики/трейсы Prometheus/Tempo/OTel — todo. 6. **E2** — веб (todo; React SPA; Login Widget и подписки уже реализованы в рамках E1.008-трека). **Ближайшие работы:** E1.010 (observability), затем E2 (React SPA поверх готового API). diff --git a/tests/conftest.py b/tests/conftest.py index a209ff9..573d59e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,11 @@ the running Docker Compose infrastructure and are marked `@pytest.mark.integrati from __future__ import annotations +import importlib.abc +import importlib.util import os +import sys +import types # Default test env: prevent Settings() from failing on required fields when no # .env is present. Individual tests that build Settings override as needed. @@ -17,3 +21,51 @@ os.environ.setdefault("S3_ENDPOINT_URL", "http://localhost:9000") os.environ.setdefault("S3_ACCESS_KEY", "test") os.environ.setdefault("S3_SECRET_KEY", "test") os.environ.setdefault("JWT_SECRET", "test") + + +# `src/` is on sys.path for production imports (`src.contract_check.*`), but +# tests use the installed package path (`contract_check.*`). When both import +# paths are used in the same process, Python creates two module objects for +# every shared module. Metrics use the global Prometheus registry and crash +# with DuplicateTimeseries. This import hook redirects `src.contract_check.*` +# to the already-loaded `contract_check.*` modules when they exist. +class _SrcRedirectFinder(importlib.abc.MetaPathFinder): + def find_spec( + self, + fullname: str, + path: object = None, + target: object = None, + ) -> importlib.util.ModuleSpec | None: + if not fullname.startswith("src.contract_check"): + return None + target_name = fullname[4:] # strip leading "src." + if target_name in sys.modules: + return importlib.util.spec_from_loader(fullname, self) + # The `contract_check.*` package is already loaded (tests use that + # path), but a production source file references the `src.contract_check.*` + # path. Eagerly load the target through the `contract_check.*` path and + # redirect so both module names point to the same object. + if "contract_check" in sys.modules: + try: + __import__(target_name) + except Exception: + return None + if target_name in sys.modules: + return importlib.util.spec_from_loader(fullname, self) + return None + + def create_module( + self, + spec: importlib.util.ModuleSpec, # noqa: ARG002 + ) -> types.ModuleType | None: + target_name = spec.name[4:] + return sys.modules.get(target_name) + + def exec_module( + self, + module: types.ModuleType, # noqa: ARG002 + ) -> None: + return None + + +sys.meta_path.insert(0, _SrcRedirectFinder())