Prometheus was added. Dashboard Grafana was updated.
This commit is contained in:
febux
parent
1add4353db
commit
97189c97b6
6 changed files with 797 additions and 2 deletions
|
|
@ -87,6 +87,15 @@ server {
|
|||
proxy_buffering off;
|
||||
}
|
||||
|
||||
location /stub_status {
|
||||
stub_status on;
|
||||
allow 10.0.0.0/8;
|
||||
allow 172.16.0.0/12;
|
||||
allow 192.168.0.0/16;
|
||||
allow 127.0.0.1;
|
||||
deny all;
|
||||
}
|
||||
|
||||
location / {
|
||||
return 404;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -142,4 +142,13 @@ server {
|
|||
location / {
|
||||
return 404;
|
||||
}
|
||||
|
||||
location /stub_status {
|
||||
stub_status on;
|
||||
allow 10.0.0.0/8;
|
||||
allow 172.16.0.0/12;
|
||||
allow 192.168.0.0/16;
|
||||
allow 127.0.0.1;
|
||||
deny all;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
674
deploy/observability/grafana/dashboards/services.json
Normal file
674
deploy/observability/grafana/dashboards/services.json
Normal file
|
|
@ -0,0 +1,674 @@
|
|||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"enable": true,
|
||||
"iconColor": "red",
|
||||
"name": "Errors / exceptions",
|
||||
"target": { "expr": "{service=~\".+\"} |~ \"(?i)error|exception|fatal|panic\"" },
|
||||
"type": "logs"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"id": 1,
|
||||
"panels": [],
|
||||
"title": "Service health",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"mappings": [
|
||||
{ "options": { "0": { "color": "red", "text": "Down" } }, "type": "value" },
|
||||
{ "options": { "1": { "color": "green", "text": "Up" } }, "type": "value" }
|
||||
],
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": 0 }, { "color": "green", "value": 1 }] },
|
||||
"unit": "none"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 4, "w": 24, "x": 0, "y": 1 },
|
||||
"id": 2,
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "11.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "up{job=~\"$service\"}",
|
||||
"legendFormat": "{{ job }}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Service up",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
|
||||
"id": 3,
|
||||
"panels": [],
|
||||
"title": "API",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": { "group": "A", "mode": "none" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
|
||||
"id": 4,
|
||||
"options": {
|
||||
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "none" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (path, status) (rate(contract_check_http_request_duration_seconds_count{job=\"api\"}[5m]))",
|
||||
"legendFormat": "{{ path }} {{ status }}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "API request rate by path/status",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": { "group": "A", "mode": "none" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
|
||||
"id": 5,
|
||||
"options": {
|
||||
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "none" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum by (le, path) (rate(contract_check_http_request_duration_seconds_bucket{job=\"api\"}[5m])))",
|
||||
"legendFormat": "p50 {{ path }}",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by (le, path) (rate(contract_check_http_request_duration_seconds_bucket{job=\"api\"}[5m])))",
|
||||
"legendFormat": "p95 {{ path }}",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum by (le, path) (rate(contract_check_http_request_duration_seconds_bucket{job=\"api\"}[5m])))",
|
||||
"legendFormat": "p99 {{ path }}",
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"title": "API latency",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "thresholds" },
|
||||
"custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 0.01 }] },
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 },
|
||||
"id": 6,
|
||||
"options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, "showHeader": true, "sortBy": [] },
|
||||
"pluginVersion": "11.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (path, status) (rate(contract_check_http_request_duration_seconds_count{job=\"api\",status=~\"5..|4..\"}[5m])) / sum by (path, status) (rate(contract_check_http_request_duration_seconds_count{job=\"api\"}[5m]))",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "API error rate by path/status",
|
||||
"transformations": [
|
||||
{ "id": "organize", "options": { "excludeByName": { "Time": true }, "indexByName": {}, "renameByName": {} } }
|
||||
],
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": { "group": "A", "mode": "none" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 },
|
||||
"id": 7,
|
||||
"options": {
|
||||
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "none" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(contract_check_documents_uploaded_total[5m])",
|
||||
"legendFormat": "Documents uploaded/sec",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "rate(contract_check_credits_reserved_total[5m])",
|
||||
"legendFormat": "Credits reserved/sec",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Business throughput",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 },
|
||||
"id": 8,
|
||||
"panels": [],
|
||||
"title": "Workers",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": { "group": "A", "mode": "none" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 },
|
||||
"id": 9,
|
||||
"options": {
|
||||
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "none" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (job) (rate(contract_check_extract_duration_seconds_count[5m]))",
|
||||
"legendFormat": "extract",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum by (job) (rate(contract_check_analyze_duration_seconds_count[5m]))",
|
||||
"legendFormat": "analyze",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "sum by (decision) (rate(contract_check_prescreen_duration_seconds_count[5m]))",
|
||||
"legendFormat": "prescreen {{ decision }}",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"expr": "sum by (job) (rate(contract_check_notify_duration_seconds_count[5m]))",
|
||||
"legendFormat": "notify",
|
||||
"refId": "D"
|
||||
}
|
||||
],
|
||||
"title": "Worker throughput",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": { "group": "A", "mode": "none" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 },
|
||||
"id": 10,
|
||||
"options": {
|
||||
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "none" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(contract_check_extract_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "extract p95",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(contract_check_analyze_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "analyze p95",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by (le, decision) (rate(contract_check_prescreen_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "prescreen p95 {{ decision }}",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(contract_check_notify_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "notify p95",
|
||||
"refId": "D"
|
||||
}
|
||||
],
|
||||
"title": "Worker p95 latency",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": { "group": "A", "mode": "none" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 30 },
|
||||
"id": 11,
|
||||
"options": {
|
||||
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "none" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (queue) (rate(contract_check_mq_published_total[5m]))",
|
||||
"legendFormat": "published {{ queue }}",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum by (queue, failure_class) (rate(contract_check_mq_failed_total[5m]))",
|
||||
"legendFormat": "failed {{ queue }}/{{ failure_class }}",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Queue publish / fail rate",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": { "group": "A", "mode": "none" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 30 },
|
||||
"id": 12,
|
||||
"options": {
|
||||
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "none" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(contract_check_llm_tokens_total[5m])",
|
||||
"legendFormat": "LLM tokens/sec {{ kind }}",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "rate(contract_check_llm_fell_back_total[5m])",
|
||||
"legendFormat": "LLM fallback/sec",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "LLM usage",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 37 },
|
||||
"id": 13,
|
||||
"panels": [],
|
||||
"title": "Nginx edge",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": { "group": "A", "mode": "none" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 38 },
|
||||
"id": 14,
|
||||
"options": {
|
||||
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "none" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(nginx_http_requests_total{job=\"nginx\"}[5m])",
|
||||
"legendFormat": "Requests/sec",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Nginx request rate",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": { "type": "linear" },
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": { "group": "A", "mode": "none" },
|
||||
"thresholdsStyle": { "mode": "off" }
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 38 },
|
||||
"id": 15,
|
||||
"options": {
|
||||
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true },
|
||||
"tooltip": { "mode": "multi", "sort": "none" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "nginx_connections_active{job=\"nginx\"}",
|
||||
"legendFormat": "Active",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "nginx_connections_reading{job=\"nginx\"}",
|
||||
"legendFormat": "Reading",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "nginx_connections_writing{job=\"nginx\"}",
|
||||
"legendFormat": "Writing",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"expr": "nginx_connections_waiting{job=\"nginx\"}",
|
||||
"legendFormat": "Waiting",
|
||||
"refId": "D"
|
||||
}
|
||||
],
|
||||
"title": "Nginx connections",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 46 },
|
||||
"id": 16,
|
||||
"panels": [],
|
||||
"title": "Logs",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"description": "Application logs filtered by the selected service. Search for correlation_id, error, etc.",
|
||||
"gridPos": { "h": 12, "w": 24, "x": 0, "y": 47 },
|
||||
"id": 17,
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": true,
|
||||
"prettifyLogMessage": true,
|
||||
"enableLogDetails": true,
|
||||
"dedupStrategy": "none"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "{service=~\"${service:regex}\"} |= \"$query\"",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Service logs",
|
||||
"type": "logs"
|
||||
}
|
||||
],
|
||||
"refresh": "5s",
|
||||
"schemaVersion": 39,
|
||||
"tags": ["services", "prometheus", "loki"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": { "selected": false, "text": ["All"], "value": ["$__all"] },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"definition": "label_values(up, job)",
|
||||
"hide": 0,
|
||||
"includeAll": true,
|
||||
"label": "Service",
|
||||
"multi": true,
|
||||
"name": "service",
|
||||
"options": [],
|
||||
"query": "label_values(up, job)",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"current": { "selected": false, "text": "", "value": "" },
|
||||
"hide": 0,
|
||||
"label": "Search",
|
||||
"name": "query",
|
||||
"options": [{ "selected": true, "text": "", "value": "" }],
|
||||
"query": "",
|
||||
"skipUrlSync": false,
|
||||
"type": "textbox"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": { "from": "now-15m", "to": "now" },
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "Contract Check — Services",
|
||||
"uid": "contract-check-services",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
|
|
@ -5,6 +5,7 @@ datasources:
|
|||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
uid: loki
|
||||
isDefault: false
|
||||
editable: false
|
||||
jsonData:
|
||||
|
|
@ -14,3 +15,16 @@ datasources:
|
|||
matcherRegex: '"correlation_id":\s*"([^"]+)"'
|
||||
url: /explore?left={"datasource":"Loki","queries":[{"expr":"{service=\"$service\"} |= \"$__value\""}]}
|
||||
urlDisplayLabel: 'Trace by correlation_id'
|
||||
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
uid: prometheus
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
timeInterval: 15s
|
||||
httpMethod: POST
|
||||
manageAlerts: false
|
||||
prometheusType: Prometheus
|
||||
|
|
|
|||
51
deploy/observability/prometheus/prometheus.yml
Normal file
51
deploy/observability/prometheus/prometheus.yml
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: [localhost:9090]
|
||||
|
||||
- job_name: api
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets: [api:8000]
|
||||
labels:
|
||||
service: api
|
||||
|
||||
- job_name: worker-extract
|
||||
static_configs:
|
||||
- targets: [worker-extract:9101]
|
||||
labels:
|
||||
service: worker-extract
|
||||
|
||||
- job_name: worker-analyze
|
||||
static_configs:
|
||||
- targets: [worker-analyze:9102]
|
||||
labels:
|
||||
service: worker-analyze
|
||||
|
||||
- job_name: worker-prescreen
|
||||
static_configs:
|
||||
- targets: [worker-prescreen:9104]
|
||||
labels:
|
||||
service: worker-prescreen
|
||||
|
||||
- job_name: worker-notify
|
||||
static_configs:
|
||||
- targets: [worker-notify:9103]
|
||||
labels:
|
||||
service: worker-notify
|
||||
|
||||
- job_name: worker-billing
|
||||
static_configs:
|
||||
- targets: [worker-billing:9105]
|
||||
labels:
|
||||
service: worker-billing
|
||||
|
||||
- job_name: nginx
|
||||
static_configs:
|
||||
- targets: [nginx-exporter:9113]
|
||||
labels:
|
||||
service: nginx
|
||||
|
|
@ -368,9 +368,46 @@ services:
|
|||
sleep 12h & wait $${!}
|
||||
done
|
||||
|
||||
nginx-exporter:
|
||||
profiles: ["edge"]
|
||||
image: nginx/nginx-prometheus-exporter:latest
|
||||
container_name: contract_check-nginx-exporter
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- "-nginx.scrape-uri=http://nginx:80/stub_status"
|
||||
depends_on:
|
||||
nginx:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "${NGINX_EXPORTER_PORT:-9113}:9113"
|
||||
|
||||
# ── OBSERVABILITY (profile: obs) ───────────────────────────────────────────
|
||||
# Minimal Grafana + Loki logs stack. Promtail scrapes all compose container
|
||||
# logs via the local Docker socket. See deploy/observability/ for configs.
|
||||
# Grafana + Loki logs + Prometheus metrics. Promtail scrapes all compose
|
||||
# container logs via the local Docker socket. See deploy/observability/.
|
||||
prometheus:
|
||||
profiles: ["obs"]
|
||||
image: prom/prometheus:latest
|
||||
container_name: contract_check-prometheus
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- "--config.file=/etc/prometheus/prometheus.yml"
|
||||
- "--storage.tsdb.path=/prometheus"
|
||||
- "--storage.tsdb.retention.time=15d"
|
||||
- "--web.console.libraries=/usr/share/prometheus/console_libraries"
|
||||
- "--web.console.templates=/usr/share/prometheus/consoles"
|
||||
- "--web.enable-lifecycle"
|
||||
volumes:
|
||||
- prometheus-data:/prometheus
|
||||
- ./deploy/observability/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
ports:
|
||||
- "${PROMETHEUS_PORT:-9090}:9090"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://localhost:9090/-/healthy || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
start_period: 15s
|
||||
|
||||
loki:
|
||||
profiles: ["obs"]
|
||||
image: grafana/loki
|
||||
|
|
@ -429,3 +466,4 @@ volumes:
|
|||
certbot-webroot:
|
||||
loki-data:
|
||||
grafana-data:
|
||||
prometheus-data:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue