diff --git a/.env.example b/.env.example index e2a9e63..df25da9 100644 --- a/.env.example +++ b/.env.example @@ -49,9 +49,12 @@ REFUND_FULL_USAGE_THRESHOLD=0.20 # ≤20% consumed + within window ⇒ full ref # --- Observability (leave empty to disable) --- SENTRY_DSN= # OTLP endpoint for traces/metrics/logs. -# profile: observer → http://openobserve:5081 (OTLP/gRPC) +# profile: observer → http://openobserve:5080/api/default (OTLP/HTTP) # profile: obs → configure your own otel-collector/tempo, or leave empty OTEL_EXPORTER_OTLP_ENDPOINT= +# Required when sending to OpenObserve. Generate with: echo -n 'user:pass' | base64 +# Default value is for root@example.com:Complexpass#123 (percent-encoded space). +OTEL_EXPORTER_OTLP_HEADERS=Authorization=Basic%20cm9vdEBleGFtcGxlLmNvbTpDb21wbGV4cGFzcyMxMjM= OTEL_SERVICE_NAME=contract-check # --- OpenObserve (profile: observer) --- diff --git a/src/contract_check/core/telemetry.py b/src/contract_check/core/telemetry.py index 277b70d..a69d5d8 100644 --- a/src/contract_check/core/telemetry.py +++ b/src/contract_check/core/telemetry.py @@ -1,49 +1,87 @@ """OpenTelemetry initialization. -Sets up a tracer exporting OTLP to the collector when +Sets up OTLP exporters for traces, logs, and metrics when `OTEL_EXPORTER_OTLP_ENDPOINT` is set; otherwise no-op. httpx auto-instrumentation is wired so Ollama Cloud calls appear as spans. Service entrypoints call `setup_telemetry()` early and `shutdown_telemetry()` on exit. +Auth headers are taken from `OTEL_EXPORTER_OTLP_HEADERS` by the OTLP exporters +(no code-level parsing needed). + This module imports `opentelemetry` — only services in the `obs` group import it (see docs/ARCHITECTURE.md §4). mypy resolves it via the dev group. """ from __future__ import annotations +import logging from typing import TYPE_CHECKING, Any from src.contract_check.core.config import get_settings from src.contract_check.core.logging import get_logger if TYPE_CHECKING: + from opentelemetry.sdk._logs import LoggerProvider + from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.trace import TracerProvider log = get_logger(__name__) -_provider: TracerProvider | None = None +_trace_provider: TracerProvider | None = None +_log_provider: LoggerProvider | None = None +_meter_provider: MeterProvider | None = None def setup_telemetry(service_name: str | None = None) -> None: - """Initialize OTel tracing. Safe to call multiple times / when disabled.""" - global _provider + """Initialize OTel tracing/logging/metrics. Safe to call when disabled.""" + global _trace_provider, _log_provider, _meter_provider settings = get_settings() endpoint = settings.otel_exporter_otlp_endpoint if not endpoint: log.debug("otel_disabled", reason="no endpoint configured") return + from opentelemetry._logs import set_logger_provider + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.metrics import set_meter_provider + from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor from opentelemetry.trace import set_tracer_provider resource = Resource.create({"service.name": service_name or settings.otel_service_name}) - provider = TracerProvider(resource=resource) - provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint))) - set_tracer_provider(provider) - _provider = provider + + # Traces + trace_provider = TracerProvider(resource=resource) + trace_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint))) + set_tracer_provider(trace_provider) + _trace_provider = trace_provider + + # Logs (stdlib logging → OTLP). Structlog's PrintLogger still writes to stderr; + # this captures stdlib/third-party logs and anything routed through the root logger. + log_provider = LoggerProvider(resource=resource) + log_provider.add_log_record_processor( + BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint)) + ) + set_logger_provider(log_provider) + _log_provider = log_provider + + otel_log_handler = LoggingHandler(logger_provider=log_provider) + otel_log_handler.setLevel(logging.INFO) + logging.getLogger().addHandler(otel_log_handler) + + # Metrics infrastructure (OTel metrics → OTLP). Existing prometheus_client + # metrics are still scraped by Prometheus; use remote_write for those. + metric_reader = PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=endpoint)) + meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader]) + set_meter_provider(meter_provider) + _meter_provider = meter_provider try: from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor @@ -53,17 +91,26 @@ def setup_telemetry(service_name: str | None = None) -> None: log.warning("otel_httpx_instrument_failed", error=str(exc)) log.info("otel_initialized", endpoint=endpoint) + # Stdlib log so the OTel handler exports a verification record too. + logging.getLogger("contract_check.telemetry").info( + "OpenTelemetry initialized for %s at %s", + service_name or settings.otel_service_name, + endpoint, + ) def shutdown_telemetry() -> None: - """Flush and shut down the tracer provider if it was initialized.""" - global _provider - if _provider is None: - return - try: - _provider.shutdown() - finally: - _provider = None + """Flush and shut down all OTel providers if initialized.""" + global _trace_provider, _log_provider, _meter_provider + for provider in (_trace_provider, _log_provider, _meter_provider): + if provider is not None: + try: + provider.shutdown() + except Exception as exc: # pragma: no cover - shutdown must not raise + log.warning("otel_shutdown_failed", error=str(exc)) + _trace_provider = None + _log_provider = None + _meter_provider = None def get_tracer(name: str | None = None) -> Any: @@ -71,3 +118,10 @@ def get_tracer(name: str | None = None) -> Any: from opentelemetry import trace return trace.get_tracer(name or "contract_check") + + +def get_meter(name: str | None = None) -> Any: + """Return a meter. Returns a no-op-safe meter when OTel is absent.""" + from opentelemetry import metrics + + return metrics.get_meter(name or "contract_check")