DealDocumentScreening/deploy/observability/grafana/dashboards/services.json

674 lines
22 KiB
JSON

{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": { "type": "loki", "uid": "loki" },
"enable": true,
"iconColor": "red",
"name": "Errors / exceptions",
"target": { "expr": "{service=~\".+\"} |~ \"(?i)error|exception|fatal|panic\"" },
"type": "logs"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
"id": 1,
"panels": [],
"title": "Service health",
"type": "row"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": {
"mappings": [
{ "options": { "0": { "color": "red", "text": "Down" } }, "type": "value" },
{ "options": { "1": { "color": "green", "text": "Up" } }, "type": "value" }
],
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": 0 }, { "color": "green", "value": 1 }] },
"unit": "none"
},
"overrides": []
},
"gridPos": { "h": 4, "w": 24, "x": 0, "y": 1 },
"id": 2,
"options": {
"colorMode": "background",
"graphMode": "none",
"orientation": "auto",
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.0.0",
"targets": [
{
"expr": "up{job=~\"$service\"}",
"legendFormat": "{{ job }}",
"refId": "A"
}
],
"title": "Service up",
"type": "stat"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
"id": 3,
"panels": [],
"title": "API",
"type": "row"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"unit": "reqps"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
"id": 4,
"options": {
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "none" }
},
"targets": [
{
"expr": "sum by (path, status) (rate(contract_check_http_request_duration_seconds_count{job=\"api\"}[5m]))",
"legendFormat": "{{ path }} {{ status }}",
"refId": "A"
}
],
"title": "API request rate by path/status",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"unit": "s"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
"id": 5,
"options": {
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "none" }
},
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (le, path) (rate(contract_check_http_request_duration_seconds_bucket{job=\"api\"}[5m])))",
"legendFormat": "p50 {{ path }}",
"refId": "A"
},
{
"expr": "histogram_quantile(0.95, sum by (le, path) (rate(contract_check_http_request_duration_seconds_bucket{job=\"api\"}[5m])))",
"legendFormat": "p95 {{ path }}",
"refId": "B"
},
{
"expr": "histogram_quantile(0.99, sum by (le, path) (rate(contract_check_http_request_duration_seconds_bucket{job=\"api\"}[5m])))",
"legendFormat": "p99 {{ path }}",
"refId": "C"
}
],
"title": "API latency",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false },
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 0.01 }] },
"unit": "percentunit"
},
"overrides": []
},
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 },
"id": 6,
"options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, "showHeader": true, "sortBy": [] },
"pluginVersion": "11.0.0",
"targets": [
{
"expr": "sum by (path, status) (rate(contract_check_http_request_duration_seconds_count{job=\"api\",status=~\"5..|4..\"}[5m])) / sum by (path, status) (rate(contract_check_http_request_duration_seconds_count{job=\"api\"}[5m]))",
"format": "table",
"instant": true,
"refId": "A"
}
],
"title": "API error rate by path/status",
"transformations": [
{ "id": "organize", "options": { "excludeByName": { "Time": true }, "indexByName": {}, "renameByName": {} } }
],
"type": "table"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"unit": "short"
},
"overrides": []
},
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 },
"id": 7,
"options": {
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "none" }
},
"targets": [
{
"expr": "rate(contract_check_documents_uploaded_total[5m])",
"legendFormat": "Documents uploaded/sec",
"refId": "A"
},
{
"expr": "rate(contract_check_credits_reserved_total[5m])",
"legendFormat": "Credits reserved/sec",
"refId": "B"
}
],
"title": "Business throughput",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 },
"id": 8,
"panels": [],
"title": "Workers",
"type": "row"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"unit": "ops"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 },
"id": 9,
"options": {
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "none" }
},
"targets": [
{
"expr": "sum by (job) (rate(contract_check_extract_duration_seconds_count[5m]))",
"legendFormat": "extract",
"refId": "A"
},
{
"expr": "sum by (job) (rate(contract_check_analyze_duration_seconds_count[5m]))",
"legendFormat": "analyze",
"refId": "B"
},
{
"expr": "sum by (decision) (rate(contract_check_prescreen_duration_seconds_count[5m]))",
"legendFormat": "prescreen {{ decision }}",
"refId": "C"
},
{
"expr": "sum by (job) (rate(contract_check_notify_duration_seconds_count[5m]))",
"legendFormat": "notify",
"refId": "D"
}
],
"title": "Worker throughput",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"unit": "s"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 },
"id": 10,
"options": {
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "none" }
},
"targets": [
{
"expr": "histogram_quantile(0.95, sum by (le) (rate(contract_check_extract_duration_seconds_bucket[5m])))",
"legendFormat": "extract p95",
"refId": "A"
},
{
"expr": "histogram_quantile(0.95, sum by (le) (rate(contract_check_analyze_duration_seconds_bucket[5m])))",
"legendFormat": "analyze p95",
"refId": "B"
},
{
"expr": "histogram_quantile(0.95, sum by (le, decision) (rate(contract_check_prescreen_duration_seconds_bucket[5m])))",
"legendFormat": "prescreen p95 {{ decision }}",
"refId": "C"
},
{
"expr": "histogram_quantile(0.95, sum by (le) (rate(contract_check_notify_duration_seconds_bucket[5m])))",
"legendFormat": "notify p95",
"refId": "D"
}
],
"title": "Worker p95 latency",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"unit": "short"
},
"overrides": []
},
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 30 },
"id": 11,
"options": {
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "none" }
},
"targets": [
{
"expr": "sum by (queue) (rate(contract_check_mq_published_total[5m]))",
"legendFormat": "published {{ queue }}",
"refId": "A"
},
{
"expr": "sum by (queue, failure_class) (rate(contract_check_mq_failed_total[5m]))",
"legendFormat": "failed {{ queue }}/{{ failure_class }}",
"refId": "B"
}
],
"title": "Queue publish / fail rate",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"unit": "short"
},
"overrides": []
},
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 30 },
"id": 12,
"options": {
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "none" }
},
"targets": [
{
"expr": "rate(contract_check_llm_tokens_total[5m])",
"legendFormat": "LLM tokens/sec {{ kind }}",
"refId": "A"
},
{
"expr": "rate(contract_check_llm_fell_back_total[5m])",
"legendFormat": "LLM fallback/sec",
"refId": "B"
}
],
"title": "LLM usage",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 37 },
"id": 13,
"panels": [],
"title": "Nginx edge",
"type": "row"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"unit": "reqps"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 38 },
"id": 14,
"options": {
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "none" }
},
"targets": [
{
"expr": "rate(nginx_http_requests_total{job=\"nginx\"}[5m])",
"legendFormat": "Requests/sec",
"refId": "A"
}
],
"title": "Nginx request rate",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"unit": "short"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 38 },
"id": 15,
"options": {
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
"tooltip": { "mode": "multi", "sort": "none" }
},
"targets": [
{
"expr": "nginx_connections_active{job=\"nginx\"}",
"legendFormat": "Active",
"refId": "A"
},
{
"expr": "nginx_connections_reading{job=\"nginx\"}",
"legendFormat": "Reading",
"refId": "B"
},
{
"expr": "nginx_connections_writing{job=\"nginx\"}",
"legendFormat": "Writing",
"refId": "C"
},
{
"expr": "nginx_connections_waiting{job=\"nginx\"}",
"legendFormat": "Waiting",
"refId": "D"
}
],
"title": "Nginx connections",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 46 },
"id": 16,
"panels": [],
"title": "Logs",
"type": "row"
},
{
"datasource": { "type": "loki", "uid": "loki" },
"description": "Application logs filtered by the selected service. Search for correlation_id, error, etc.",
"gridPos": { "h": 12, "w": 24, "x": 0, "y": 47 },
"id": 17,
"options": {
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true,
"prettifyLogMessage": true,
"enableLogDetails": true,
"dedupStrategy": "none"
},
"targets": [
{
"expr": "{service=~\"${service:regex}\"} |= \"$query\"",
"refId": "A"
}
],
"title": "Service logs",
"type": "logs"
}
],
"refresh": "5s",
"schemaVersion": 39,
"tags": ["services", "prometheus", "loki"],
"templating": {
"list": [
{
"current": { "selected": false, "text": ["All"], "value": ["$__all"] },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"definition": "label_values(up, job)",
"hide": 0,
"includeAll": true,
"label": "Service",
"multi": true,
"name": "service",
"options": [],
"query": "label_values(up, job)",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
},
{
"current": { "selected": false, "text": "", "value": "" },
"hide": 0,
"label": "Search",
"name": "query",
"options": [{ "selected": true, "text": "", "value": "" }],
"query": "",
"skipUrlSync": false,
"type": "textbox"
}
]
},
"time": { "from": "now-15m", "to": "now" },
"timepicker": {},
"timezone": "browser",
"title": "Contract Check — Services",
"uid": "contract-check-services",
"version": 1,
"weekStart": ""
}