Grafana stack was added. Update docs.

2026-09-02 23:32:47 +03:00 · 2026-09-02 23:32:47 +03:00 · 1e051f3730
commit 1e051f3730
parent ef39c2793c
15 changed files with 452 additions and 20 deletions
--- a/.env.example
+++ b/.env.example
@ -51,6 +51,14 @@ SENTRY_DSN=
 OTEL_EXPORTER_OTLP_ENDPOINT=  # e.g. http://otel-collector:4317
 OTEL_SERVICE_NAME=contract-check

+# --- Grafana / Loki (profile: obs) ---
+GRAFANA_PORT=3000
+GRAFANA_ADMIN_USER=admin
+GRAFANA_ADMIN_PASSWORD=admin
+# External URL of the project domain. Grafana is also served under /grafana via nginx.
+# Examples: http://localhost:3000  (dev direct), https://contract-check.example.com
+GRAFANA_ROOT_URL=http://localhost:3000
+
 # --- LLM provider (abstract port; first realization = Ollama Cloud) ---
 # For Ollama Cloud use https://ollama.com (not api.ollama.com). Models must be
 # available on the chosen host — cloud models differ from local Ollama models.
--- a/31
+++ b/31
@ -3,6 +3,7 @@

 .PHONY: help install lint lint-fix isort isort-check typecheck test test-unit test-integration migrate \
        infra-up infra-down infra-logs services-up services-down services-logs \
+        services-obs obs-up obs-down obs-logs obs-url obs-reset \
        api api-logs bot bot-logs worker-extract worker-analyze worker-notify \
        seed-token jwt-secret jwt-token jwt-verify health shell-api shell-bot \
        shell-db admin-promote admin-list clean
@ -77,9 +78,35 @@ services-logs:  ## Tail all service logs
 services-ps:  ## Show running containers
 	docker compose --profile services ps

+services-obs:  ## Start services + observability (logs)
+	docker compose --profile services --profile obs up -d --build --remove-orphans
+
 services-nginx:  ## Start with nginx reverse proxy
 	docker compose --profile services --profile edge up -d --build --remove-orphans

+# ─────────────────────────────────────────────────────────────────────────────
+# Docker: observability (Grafana + Loki logs)
+# ─────────────────────────────────────────────────────────────────────────────
+obs-up:  ## Start observability containers (needs infra or services running)
+	docker compose --profile obs up -d --build --remove-orphans
+
+obs-down:  ## Stop observability containers
+	docker compose --profile obs down
+
+obs-logs:  ## Tail observability containers logs
+	docker compose --profile obs logs -f
+
+obs-url:  ## Print Grafana URL and default credentials
+	@echo "Direct:     http://localhost:$${GRAFANA_PORT:-3000}"
+	@echo "Via nginx:  $${GRAFANA_ROOT_URL:-http://localhost}/grafana/"
+	@echo "User:       $${GRAFANA_ADMIN_USER:-admin}"
+	@echo "Password:   $${GRAFANA_ADMIN_PASSWORD:-admin}"
+	@echo "Dashboard:  $${GRAFANA_ROOT_URL:-http://localhost}/grafana/d/contract-check-logs"
+
+obs-reset:  ## Reset Grafana and Loki volumes (wipes dashboards/logs data)
+	docker compose --profile obs down -v
+	rm -rf deploy/observability/grafana/dashboards/*.json.tmp
+
 # ─────────────────────────────────────────────────────────────────────────────
 # Docker: nginx edge (reverse proxy)
 # ─────────────────────────────────────────────────────────────────────────────
@ -220,4 +247,8 @@ clean:  ## Remove containers, volumes, caches
 # ─────────────────────────────────────────────────────────────────────────────
 dev: install infra-up migrate services-up  ## Bootstrap full dev environment

+dev-obs: install infra-up migrate services-obs  ## Bootstrap full dev environment with observability
+
 stop: services-down infra-down  ## Stop everything
+
+stop-obs: obs-down services-down infra-down  ## Stop everything including observability
--- a/deploy/nginx/templates/contract-check-http.conf.template
+++ b/deploy/nginx/templates/contract-check-http.conf.template
@ -63,6 +63,30 @@ server {
        proxy_read_timeout    60s;
    }

+    # Grafana / Loki: exposed under /grafana so everything lives on one domain.
+    # In production restrict access here or via firewall (basic auth / VPN / SSO).
+    location /grafana/ {
+        # Variable forces dynamic DNS resolution for `grafana`.
+        set $grafana http://grafana:3000;
+        proxy_pass $grafana;
+        proxy_http_version 1.1;
+
+        proxy_set_header Host              $host;
+        proxy_set_header X-Real-IP         $remote_addr;
+        proxy_set_header X-Forwarded-For   $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+        proxy_set_header X-Forwarded-Host  $host;
+        proxy_set_header X-Correlation-Id  $http_x_correlation_id;
+
+        # Grafana needs the subpath so it can build correct links/redirects.
+        proxy_set_header X-Forwarded-Prefix /grafana;
+
+        proxy_connect_timeout 10s;
+        proxy_send_timeout    30s;
+        proxy_read_timeout    60s;
+        proxy_buffering       off;
+    }
+
    location / {
        return 404;
    }
--- a/deploy/nginx/templates/contract-check.conf.template
+++ b/deploy/nginx/templates/contract-check.conf.template
@ -108,6 +108,30 @@ server {
        proxy_read_timeout    60s;
    }

+    # Grafana / Loki: exposed under /grafana so everything lives on one domain.
+    # In production restrict access here or via firewall (basic auth / VPN / SSO).
+    location /grafana/ {
+        # Variable forces dynamic DNS resolution for `grafana`.
+        set $grafana http://grafana:3000;
+        proxy_pass $grafana;
+        proxy_http_version 1.1;
+
+        proxy_set_header Host              $host;
+        proxy_set_header X-Real-IP         $remote_addr;
+        proxy_set_header X-Forwarded-For   $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-Host  $host;
+        proxy_set_header X-Correlation-Id  $http_x_correlation_id;
+
+        # Grafana needs the subpath so it can build correct links/redirects.
+        proxy_set_header X-Forwarded-Prefix /grafana;
+
+        proxy_connect_timeout 10s;
+        proxy_send_timeout    30s;
+        proxy_read_timeout    60s;
+        proxy_buffering       off;
+    }
+
    # Static web SPA (future). Placeholder: serve pre-built static assets.
    # location / {
    #     root /usr/share/nginx/html;
--- a/deploy/observability/grafana/dashboards/logs.json
+++ b/deploy/observability/grafana/dashboards/logs.json
@ -0,0 +1,90 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": { "type": "loki", "uid": "loki" },
+        "enable": true,
+        "iconColor": "red",
+        "name": "Log events",
+        "target": { "expr": "{service=~\".+\"} |~ \"(?i)error|exception|fatal|panic\"" },
+        "type": "logs"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": { "type": "loki", "uid": "loki" },
+      "description": "All application logs from the docker-compose stack. Filter by service or search for a correlation_id.",
+      "gridPos": { "h": 20, "w": 24, "x": 0, "y": 0 },
+      "id": 1,
+      "options": {
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": true,
+        "prettifyLogMessage": true,
+        "enableLogDetails": true,
+        "dedupStrategy": "none"
+      },
+      "targets": [
+        {
+          "datasource": { "type": "loki", "uid": "loki" },
+          "expr": "{service=~\"${service:regex}\"} |= \"$query\"",
+          "refId": "A"
+        }
+      ],
+      "title": "Application logs",
+      "type": "logs"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 38,
+  "style": "dark",
+  "tags": ["logs", "loki"],
+  "templating": {
+    "list": [
+      {
+        "current": { "selected": false, "text": ["All"], "value": ["$__all"] },
+        "datasource": { "type": "loki", "uid": "loki" },
+        "definition": "label_values(service)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Service",
+        "multi": true,
+        "name": "service",
+        "options": [],
+        "query": "label_values(service)",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      },
+      {
+        "current": { "selected": false, "text": "", "value": "" },
+        "hide": 0,
+        "label": "Search",
+        "name": "query",
+        "options": [
+          { "selected": true, "text": "", "value": "" }
+        ],
+        "query": "",
+        "skipUrlSync": false,
+        "type": "textbox"
+      }
+    ]
+  },
+  "time": { "from": "now-15m", "to": "now" },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Contract Check — Logs",
+  "uid": "contract-check-logs",
+  "version": 1,
+  "weekStart": ""
+}
--- a/deploy/observability/grafana/provisioning/dashboards/dashboards.yaml
+++ b/deploy/observability/grafana/provisioning/dashboards/dashboards.yaml
@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: default
+    orgId: 1
+    folder: Contract Check
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/dashboards
--- a/deploy/observability/grafana/provisioning/datasources/datasources.yaml
+++ b/deploy/observability/grafana/provisioning/datasources/datasources.yaml
@ -0,0 +1,16 @@
+apiVersion: 1
+
+datasources:
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    isDefault: false
+    editable: false
+    jsonData:
+      maxLines: 1000
+      derivedFields:
+        - name: correlation_id
+          matcherRegex: '"correlation_id":\s*"([^"]+)"'
+          url: /explore?left={"datasource":"Loki","queries":[{"expr":"{service=\"$service\"} |= \"$__value\""}]}
+          urlDisplayLabel: 'Trace by correlation_id'
--- a/deploy/observability/loki-config.yaml
+++ b/deploy/observability/loki-config.yaml
@ -0,0 +1,43 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+  log_level: warn
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /loki
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  allow_structured_metadata: true
+
+analytics:
+  reporting_enabled: false
--- a/deploy/observability/promtail-config.yaml
+++ b/deploy/observability/promtail-config.yaml
@ -0,0 +1,28 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+  log_level: warn
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+    batchwait: 1s
+    batchsize: 1048576
+
+scrape_configs:
+  - job_name: docker
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 5s
+    relabel_configs:
+      - source_labels: [__meta_docker_container_name]
+        regex: '/(.*)'
+        target_label: container
+      - source_labels: [__meta_docker_container_label_com_docker_compose_service]
+        target_label: service
+      - source_labels: [__meta_docker_container_label_com_docker_compose_project]
+        target_label: project
+      - source_labels: [__meta_docker_container_label_com_docker_compose_profile]
+        target_label: profile
--- a/deploy/vps/front.conf
+++ b/deploy/vps/front.conf
@ -37,7 +37,8 @@ server {
    }

    # ── «Контракт-чек»: exact path set of its edge (contract-check-http.conf.template)
-    location ~ ^/(healthz|readyz|metrics|api/|admin/|docs|openapi\.json) {
+    # Grafana is also routed through the edge container under /grafana.
+    location ~ ^/(healthz|readyz|metrics|api/|admin/|docs|openapi\.json|grafana/) {
        proxy_pass http://contract_check_edge;
        proxy_http_version 1.1;
        proxy_set_header Host              $host;
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -368,6 +368,57 @@ services:
          sleep 12h & wait $${!}
        done

+  # ── OBSERVABILITY (profile: obs) ───────────────────────────────────────────
+  # Minimal Grafana + Loki logs stack. Promtail scrapes all compose container
+  # logs via the local Docker socket. See deploy/observability/ for configs.
+  loki:
+    profiles: ["obs"]
+    image: grafana/loki
+    container_name: contract_check-loki
+    restart: unless-stopped
+    command: -config.file=/etc/loki/loki-config.yaml
+    volumes:
+      - loki-data:/loki
+      - ./deploy/observability/loki-config.yaml:/etc/loki/loki-config.yaml:ro
+    ports:
+      - "13100:3100"
+
+  promtail:
+    profiles: ["obs"]
+    image: grafana/promtail
+    container_name: contract_check-promtail
+    restart: unless-stopped
+    command: -config.file=/etc/promtail/promtail-config.yaml
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ./deploy/observability/promtail-config.yaml:/etc/promtail/promtail-config.yaml:ro
+
+  grafana:
+    profiles: ["obs"]
+    image: grafana/grafana
+    container_name: contract_check-grafana
+    restart: unless-stopped
+    volumes:
+      - grafana-data:/var/lib/grafana
+      - ./deploy/observability/grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./deploy/observability/grafana/dashboards:/var/lib/grafana/dashboards:ro
+    ports:
+      - "${GRAFANA_PORT:-3000}:3000"
+    environment:
+      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
+      GF_USERS_ALLOW_SIGN_UP: "false"
+      # When served under a subpath via nginx, Grafana must know the root URL.
+      GF_SERVER_ROOT_URL: "${GRAFANA_ROOT_URL:-http://localhost:3000}/grafana/"
+      GF_SERVER_SERVE_FROM_SUB_PATH: "true"
+      GF_INSTALL_PLUGINS: ""
+    healthcheck:
+      test: ["CMD-SHELL", "curl -sf http://localhost:3000/api/health || exit 1"]
+      interval: 10s
+      timeout: 3s
+      retries: 10
+      start_period: 30s
+
 volumes:
  pgdata:
  pgwal:
@ -376,3 +427,5 @@ volumes:
  minio:
  certbot-data:
  certbot-webroot:
+  loki-data:
+  grafana-data:
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@ -1090,6 +1090,10 @@ None of 1–5 requires touching `core/` application code — only compose/infra.
 | `SENTRY_DSN` | (empty) | if set, init sentry |
 | `OTEL_EXPORTER_OTLP_ENDPOINT` | (empty) | OTel collector |
 | `OTEL_SERVICE_NAME` | per-service | overridden in each service settings |
+| `GRAFANA_PORT` | `3000` | host port for the Grafana UI (profile `obs`) |
+| `GRAFANA_ADMIN_USER` | `admin` | initial Grafana admin user |
+| `GRAFANA_ADMIN_PASSWORD` | `admin` | initial Grafana admin password |
+| `GRAFANA_ROOT_URL` | `http://localhost:3000` | external Grafana URL for callbacks |

 ### LLM

@ -1280,6 +1284,9 @@ services:
  rabbitmq:        rabbitmq:4-management-alpine, healthcheck, volume; AMQP 5672, mgmt UI 15672
  minio:           minio/minio, healthcheck, volume, console on :9001; S3 API :9000
  minio-init:      one-shot: mc alias + mb + ilm rule; depends_on minio healthy
+  loki:            grafana/loki; filesystem-backed single-node log store
+  promtail:        grafana/promtail; ships Docker container logs into Loki
+  grafana:         grafana/grafana; provisioned Loki datasource + logs dashboard

  # ── SERVICES (profile: services) ──
  api:             build srv/api/Dockerfile; depends_on pg/rabbit/minio-init healthy;
@ -1289,17 +1296,17 @@ services:
   worker-analyze:  build srv/worker-analyze/Dockerfile; depends_on pg/rabbit/minio-init healthy; 9102
   bot:             build srv/bot/Dockerfile; depends_on api healthy (NOT pg/rabbit)

-  # ── OBSERVABILITY (profile: obs) ── PLANNED (T-E1-010)
-  prometheus:      prom/prometheus; scrape api:9100, extract:9101, analyze:9102
-  grafana:         grafana/grafana; provisioned datasources + starter dashboards
-  otel-collector:  otel/opentelemetry-collector-contrib; receives OTLP
-  tempo:           grafana/tempo; trace storage
+  # ── OBSERVABILITY (profile: obs) ──
+  # Logs are live (Grafana + Loki + Promtail). Metrics/traces still planned.
+  loki:            grafana/loki; filesystem-backed single-node log store
+  promtail:        grafana/promtail; ships Docker container logs into Loki
+  grafana:         grafana/grafana; provisioned Loki datasource + logs dashboard

  # ── EDGE (profile: edge) ── PLANNED (T-E1-009)
  nginx:           nginx:alpine; reverse proxy → api; TLS via certbot
  certbot:         certbot/certbot; renew cron sidecar

-volumes: { pgdata, pgwal, redisdata, rabbitmq, minio }
+volumes: { pgdata, pgwal, redisdata, rabbitmq, minio, loki-data, grafana-data }
 ```

 `bot` depends on `api` healthy (not on infra) — it speaks HTTP to the api,
@ -1320,6 +1327,9 @@ enforcing the adapter boundary even in dependency ordering.
  contextvar before handling. So a single upload's logs trace
  api → rabbit → worker-extract → worker-analyze → DB under one ID.
 - OTel baggage/span context propagates the same ID for distributed traces.
+- The same `correlation_id` is parsed by the Loki datasource as a derived field,
+  so you can click through from any log line to every other log line with the
+  same id.

 ### Sentry (`core/sentry.py`)

@ -1345,11 +1355,26 @@ enforcing the adapter boundary even in dependency ordering.
 - Workers run `prometheus_client.start_http_server(port)` in a background
  thread alongside the async consumer.

-### Grafana
+### Grafana + Loki

- Provisioned Prometheus + Tempo datasources.
- Starter dashboard: queue depth (`rabbitmq_queue_messages`), job duration
-  histogram, LLM tokens/min, refund rate, 429/fallback rate.
+- `obs` profile adds `loki`, `promtail`, `grafana` to compose.
+- Promtail discovers all compose containers via the Docker socket and pushes
+  their stdout/stderr to Loki; labels include `service`, `project`, `profile`,
+  and `container`.
+- Logs retain the JSON format emitted by structlog in `staging`/`prod` and the
+  pretty console format in `dev`; Loki stores the raw line.
+- Provisioned datasource `Loki` and a starter `Contract Check — Logs`
+  dashboard at `http://localhost:${GRAFANA_PORT:-3000}/d/contract-check-logs`.
+- Dashboard has a `service` filter and a free-text search box; error words are
+  highlighted as annotations. Use correlation_id values to trace one upload
+  across `api` → `worker-extract` → `worker-prescreen` → `worker-analyze`.
+
+### Prometheus + Tempo (planned)
+
+- Prometheus will scrape `:9100`/`:9101`/`:9102` metrics exposed by the services.
+- Tempo will receive OTLP traces via the OpenTelemetry collector.
+- Starter dashboards for queue depth, job latency, LLM tokens/min, refund rate,
+  and 429/fallback rate will ship with the metrics/traces datasources.

 ### OpenTelemetry (`core/telemetry.py`)

--- a/docs/DEPLOY.md
+++ b/docs/DEPLOY.md
@ -187,8 +187,8 @@ make shell-db   # UPDATE plans SET price_kopecks = 59000 WHERE code = 'lite';
 |---|---|
 | *(default)* | `postgres`, `redis`, `rabbitmq`, `minio`, `minio-init` |
 | `services` | `api`, `worker-extract`, `worker-prescreen`, `worker-analyze`, `worker-billing`, `worker-notify`, `bot` |
-| `obs` *(планируется)* | `prometheus`, `grafana`, `otel-collector`, `tempo` |
-| `edge` *(планируется)* | `nginx`, `certbot` |
+| `obs` | `loki`, `promtail`, `grafana` (logs only; metrics/traces — позже) |
+| `edge` | `nginx`, `certbot` |

 ```bash
 # Инфра:
@ -197,8 +197,11 @@ docker compose up -d
 # + сервисы (сборка + запуск):
 docker compose --profile services up -d --build

-# + observability + edge (когда будут готовы конфиги в deploy/):
-# docker compose --profile services --profile obs --profile edge up -d --build
+# + observability (логи):
+docker compose --profile services --profile obs up -d --build
+
+# + edge (nginx reverse proxy + TLS; deploy/nginx готов):
+docker compose --profile services --profile obs --profile edge up -d --build
 ```

 ### 4.2 depends_on и healthchecks
@ -593,6 +596,18 @@ curl https://contract-check.example.com/healthz
 - `/healthz`, `/readyz`
 - `/metrics` (открыт наружу; закройте файрволом или уберите приватный скрейпинг)
 - `/webhook/*` → `/api/v1/webhooks/`
+- `/grafana/*` → `grafana:3000` (когда поднят профиль `obs`)
+
+Grafana под путём `/grafana`:
+```bash
+# Открыть логи через основной домен (нужен профиль obs + edge)
+docker compose --profile services --profile obs --profile edge up -d
+# https://contract-check.example.com/grafana/d/contract-check-logs
+```
+
+> **Безопасность:** Grafana под `/grafana` доступна всем, у кого есть доступ к домену.
+> На проде добавьте basic auth, IP whitelist или вынесите Grafana на отдельный
+> поддомен с отдельным ingress/VPN.

 ### 13.3 Обновление сертификата

--- a/docs/TICKETS.md
+++ b/docs/TICKETS.md
@ -121,7 +121,7 @@ consume `DocumentExtracted` из `analyze.q` → скачать `.txt` → chunk

 `docker-compose.yml`: default-профиль = инфра (`postgres`, `redis`, `rabbitmq`, `minio`, `minio-init`);
 профиль `services` = `api`, `worker-extract`, `worker-prescreen`, `worker-analyze`, `worker-notify`, `bot` с `depends_on: service_healthy`;
-профиль `edge` = nginx+certbot. Все 7 Dockerfile-ов в `srv/`. Профиль `obs` — позже (T-E1-010).
+профиль `edge` = nginx+certbot. Все 7 Dockerfile-ов в `srv/`. Профиль `obs` добавляет `loki` + `promtail` + `grafana` для логов (сделано в T-E1-010a).

 ### T-E1-008 — Оплата (ЮKassa) и пополнение кредитов
 **Статус:** done · **Оценка:** M · **Зависимости:** T-E1-003, T-E1-006
@ -136,10 +136,20 @@ state-machine, авто-возвраты с клавбэком и billing-hold,

 `deploy/nginx/templates/contract-check.conf.template`, `deploy/nginx/certbot-init.sh`, `DEPLOY.md §13`, `docker-compose.yml` профиль `edge`.

-### T-E1-010 — Мониторинг (Prometheus/Grafana/Tempo/Sentry)
-**Статус:** todo · **Оценка:** M · **Зависимости:** T-E1-007
+### T-E1-010a — Мониторинг: логи (Grafana + Loki + Promtail)
+**Статус:** done · **Оценка:** S · **Зависимости:** T-E1-007

-`deploy/observability/` + compose профиль `obs`. Дашборды: queue depth, job latency, LLM tokens, credits.
+`deploy/observability/loki-config.yaml`, `promtail-config.yaml`, `grafana/provisioning/{datasources,dashboards}/`,
+`grafana/dashboards/logs.json`; профиль `obs` в `docker-compose.yml` поднимает `loki`, `promtail`, `grafana`.
+Promtail забирает логи всех compose-контейнеров через Docker socket и пушит в Loki;
+доступен дашборд `Contract Check — Logs` с фильтром по `service` и поиском по `correlation_id`.
+
+### T-E1-010b — Мониторинг: метрики + трейсы (Prometheus + Tempo + OTel collector + Sentry)
+**Статус:** todo · **Оценка:** M · **Зависимости:** T-E1-010a
+
+Добавить `prometheus`, `tempo`, `otel-collector` в профиль `obs`. Prometheus scrape `:9100`/`:9101`/`:9102`.
+OTel collector принимает OTLP и маршрутизирует в Tempo. Дашборды: queue depth, job latency, LLM tokens, credits.
+Sentry уже инициализируется в коде при наличии `SENTRY_DSN`.

 ---

@ -216,7 +226,7 @@ Seed API key в `tests/integration/conftest.py`.
 2. **E1.001–007** — ядро + БД + FastAPI `api` + worker-extract + worker-analyze + bot + Docker/compose (done).
 3. **E3** — B2B API (done: миграция, rate-limit, auth, роуты, тесты).
 4. **E1.008** — оплата ЮKassa (done — трек `.scratch/user-profile-billing/`, тикеты 001–019: профили, планы/подписки, платежи, возвраты, worker-billing, админ-панель).
-5. **E1.009–010** — деплой/edge (Nginx/certbot — done) и observability (Prometheus/Grafana/Tempo/Sentry) (todo).
+5. **E1.009–010** — деплой/edge (Nginx/certbot — done) и observability: логи Grafana+Loki+Promtail — done, метрики/трейсы Prometheus/Tempo/OTel — todo.
 6. **E2** — веб (todo; React SPA; Login Widget и подписки уже реализованы в рамках E1.008-трека).

 **Ближайшие работы:** E1.010 (observability), затем E2 (React SPA поверх готового API).
--- a/tests/conftest.py
+++ b/tests/conftest.py
@ -7,7 +7,11 @@ the running Docker Compose infrastructure and are marked `@pytest.mark.integrati

 from __future__ import annotations

+import importlib.abc
+import importlib.util
 import os
+import sys
+import types

 # Default test env: prevent Settings() from failing on required fields when no
 # .env is present. Individual tests that build Settings override as needed.
@ -17,3 +21,51 @@ os.environ.setdefault("S3_ENDPOINT_URL", "http://localhost:9000")
 os.environ.setdefault("S3_ACCESS_KEY", "test")
 os.environ.setdefault("S3_SECRET_KEY", "test")
 os.environ.setdefault("JWT_SECRET", "test")
+
+
+# `src/` is on sys.path for production imports (`src.contract_check.*`), but
+# tests use the installed package path (`contract_check.*`). When both import
+# paths are used in the same process, Python creates two module objects for
+# every shared module. Metrics use the global Prometheus registry and crash
+# with DuplicateTimeseries. This import hook redirects `src.contract_check.*`
+# to the already-loaded `contract_check.*` modules when they exist.
+class _SrcRedirectFinder(importlib.abc.MetaPathFinder):
+    def find_spec(
+        self,
+        fullname: str,
+        path: object = None,
+        target: object = None,
+    ) -> importlib.util.ModuleSpec | None:
+        if not fullname.startswith("src.contract_check"):
+            return None
+        target_name = fullname[4:]  # strip leading "src."
+        if target_name in sys.modules:
+            return importlib.util.spec_from_loader(fullname, self)
+        # The `contract_check.*` package is already loaded (tests use that
+        # path), but a production source file references the `src.contract_check.*`
+        # path. Eagerly load the target through the `contract_check.*` path and
+        # redirect so both module names point to the same object.
+        if "contract_check" in sys.modules:
+            try:
+                __import__(target_name)
+            except Exception:
+                return None
+            if target_name in sys.modules:
+                return importlib.util.spec_from_loader(fullname, self)
+        return None
+
+    def create_module(
+        self,
+        spec: importlib.util.ModuleSpec,  # noqa: ARG002
+    ) -> types.ModuleType | None:
+        target_name = spec.name[4:]
+        return sys.modules.get(target_name)
+
+    def exec_module(
+        self,
+        module: types.ModuleType,  # noqa: ARG002
+    ) -> None:
+        return None
+
+
+sys.meta_path.insert(0, _SrcRedirectFinder())