From 8880167738ca5594ae8c03f724f26b8859ceb85c Mon Sep 17 00:00:00 2001 From: tithakka Date: Thu, 23 Jul 2026 14:03:51 -0500 Subject: [PATCH 1/3] HYPERFLEET-1361 - feat: Add Grafana dashboard for API and DB metrics --- CHANGELOG.md | 1 + charts/Chart.yaml | 2 +- charts/README.md | 2 +- charts/dashboards/hyperfleet-api.json | 1197 +++++++++++++++++++++++++ docs/metrics.md | 21 +- 5 files changed, 1212 insertions(+), 11 deletions(-) create mode 100644 charts/dashboards/hyperfleet-api.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b09e2f1..3864f261 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Grafana dashboard for API and database metrics (`charts/dashboards/hyperfleet-api.json`) — covers HTTP request rate/latency, reconciliation pending/stuck gauges, DB query duration/errors, connection pool, and build info ([#TBD](https://github.com/openshift-hyperfleet/hyperfleet-api/pull/TBD)) - JWT authentication handler using `golang-jwt/jwt/v5` and `MicahParks/keyfunc/v3` with RS256 validation, configurable issuer and audience, and JWKS key rotation support ([#120](https://github.com/openshift-hyperfleet/hyperfleet-api/pull/120)) - Hard deletion for Clusters and NodePools: resources and their adapter statuses are permanently removed from the database once all required adapters report `Finalized=True` and no child resources remain ([#119](https://github.com/openshift-hyperfleet/hyperfleet-api/pull/119)) - `Finalized` condition aggregation with `WaitingForChildResources` intermediate state when all adapters are finalized but child node pools still exist ([#119](https://github.com/openshift-hyperfleet/hyperfleet-api/pull/119)) diff --git a/charts/Chart.yaml b/charts/Chart.yaml index 50903267..61bdc1d4 100644 --- a/charts/Chart.yaml +++ b/charts/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: hyperfleet-api description: HyperFleet API - Cluster Lifecycle Management Service type: application -version: 1.1.0 +version: 1.2.0 appVersion: "0.0.0-dev" maintainers: - name: HyperFleet Team diff --git a/charts/README.md b/charts/README.md index 4a1f2f56..a902a58c 100644 --- a/charts/README.md +++ b/charts/README.md @@ -1,6 +1,6 @@ # hyperfleet-api -![Version: 1.1.0](https://img.shields.io/badge/Version-1.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.0.0-dev](https://img.shields.io/badge/AppVersion-0.0.0--dev-informational?style=flat-square) +![Version: 1.2.0](https://img.shields.io/badge/Version-1.2.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.0.0-dev](https://img.shields.io/badge/AppVersion-0.0.0--dev-informational?style=flat-square) HyperFleet API - Cluster Lifecycle Management Service diff --git a/charts/dashboards/hyperfleet-api.json b/charts/dashboards/hyperfleet-api.json new file mode 100644 index 00000000..7c988840 --- /dev/null +++ b/charts/dashboards/hyperfleet-api.json @@ -0,0 +1,1197 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "last", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (code) (rate(hyperfleet_api_requests_total[$__rate_interval]))", + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "title": "Request Rate by Status Code", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "1": { + "color": "green", + "index": 0, + "text": "Active" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "hyperfleet_api_build_info", + "legendFormat": "Build: {{version}}", + "refId": "A" + } + ], + "title": "API Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "last", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(hyperfleet_api_request_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(hyperfleet_api_request_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(hyperfleet_api_request_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C" + } + ], + "title": "Request Duration (Percentiles)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "last", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (path) (rate(hyperfleet_api_requests_total{code=~\"5..\"}[$__rate_interval]))", + "legendFormat": "{{path}}", + "refId": "A" + } + ], + "title": "Server Error Rate by Path", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "last", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (resource_type, is_delete) (rate(hyperfleet_api_reconciliation_started_total[$__rate_interval]))", + "legendFormat": "{{resource_type}} (delete={{is_delete}})", + "refId": "A" + } + ], + "title": "Reconciliation Started Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "last", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max by (resource_type, is_delete) (hyperfleet_api_resource_pending_reconciliation)", + "legendFormat": "{{resource_type}} (delete={{is_delete}})", + "refId": "A" + } + ], + "title": "Resources Pending Reconciliation", + "description": "Live-query gauge — absent series means zero resources pending for that combination.", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "last", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max by (resource_type, is_delete) (hyperfleet_api_resource_pending_reconciliation_stuck)", + "legendFormat": "{{resource_type}} (delete={{is_delete}})", + "refId": "A" + } + ], + "title": "Resources Stuck", + "description": "Resources pending reconciliation beyond the stuck threshold (default 10m). Absent series means no stuck resources.", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "last", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max by (resource_type, is_delete) (hyperfleet_api_resource_pending_reconciliation_stuck_duration_seconds)", + "legendFormat": "{{resource_type}} (delete={{is_delete}})", + "refId": "A" + } + ], + "title": "Max Stuck Duration", + "description": "Maximum time any resource has been stuck pending reconciliation, by resource type.", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "last", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(hyperfleet_db_query_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(hyperfleet_db_query_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(hyperfleet_db_query_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C" + } + ], + "title": "DB Query Duration (Percentiles)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "last", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (error_type) (rate(hyperfleet_db_errors_total[$__rate_interval]))", + "legendFormat": "{{error_type}}", + "refId": "A" + } + ], + "title": "DB Error Rate by Type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "last", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(hyperfleet_db_connections_open)", + "legendFormat": "Open", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(hyperfleet_db_connections_in_use)", + "legendFormat": "In Use", + "refId": "B" + } + ], + "title": "DB Connection Pool", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 12, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Requests/sec" + } + ] + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "topk(10, sum by (path) (rate(hyperfleet_api_requests_total[$__rate_interval])))", + "format": "table", + "instant": true, + "refId": "A" + } + ], + "title": "Top Paths by Request Rate", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": {}, + "renameByName": { + "Value": "Requests/sec", + "path": "Path" + } + } + } + ], + "type": "table" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "hyperfleet", + "api", + "observability" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "HyperFleet API Metrics", + "uid": "hyperfleet-api", + "version": 1, + "weekStart": "" +} diff --git a/docs/metrics.md b/docs/metrics.md index 7ad6e695..38bf668a 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -400,15 +400,18 @@ This creates a `monitoring.googleapis.com/v1/PodMonitoring` resource that config ## Grafana Dashboard -Example dashboard JSON for HyperFleet API monitoring is available in the architecture repository. Key panels to include: - -1. **Request Rate** - Total requests per second over time -2. **Error Rate** - Percentage of 5xx responses -3. **Latency Distribution** - P50, P90, P99 latencies -4. **Request Duration Heatmap** - Visual distribution of request times -5. **Top Endpoints** - Most frequently accessed paths -6. **Memory Usage** - Resident memory over time -7. **Goroutines** - Goroutine count over time +A pre-built Grafana dashboard is shipped at `charts/dashboards/hyperfleet-api.json` (relative to the repository root). It covers all `hyperfleet_api_*` and `hyperfleet_db_*` metrics across 12 panels: HTTP request rate by status code, build info (API status), request duration percentiles, server error rate by path, reconciliation started rate, resources pending reconciliation, resources stuck, max stuck duration, DB query duration percentiles, DB error rate by type, DB connection pool, and top paths by request rate. + +### Importing + +1. Navigate to **Grafana > Dashboards > Import** +2. Upload `charts/dashboards/hyperfleet-api.json` +3. Select your Prometheus datasource from the dropdown +4. Click **Import** + +The dashboard uses a parameterized `datasource` variable (no hardcoded UIDs) so it works across environments. Reconciliation gauge panels (pending, stuck) show "No data" when no resources match — this is expected behavior, not a missing datasource. + +Automated provisioning via `hyperfleet-infra` is tracked in [HYPERFLEET-1363](https://issues.redhat.com/browse/HYPERFLEET-1363). ## Related Documentation From 0f9a69a5420f911581132a27abe8bcd4a6ab9d83 Mon Sep 17 00:00:00 2001 From: tithakka Date: Thu, 30 Jul 2026 09:50:13 -0500 Subject: [PATCH 2/3] HYPERFLEET-1361 - feat: Revert version and improve dashboard --- charts/Chart.yaml | 2 +- charts/README.md | 2 +- charts/dashboards/hyperfleet-api.json | 12 ++++++------ docs/metrics.md | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/charts/Chart.yaml b/charts/Chart.yaml index 61bdc1d4..50903267 100644 --- a/charts/Chart.yaml +++ b/charts/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: hyperfleet-api description: HyperFleet API - Cluster Lifecycle Management Service type: application -version: 1.2.0 +version: 1.1.0 appVersion: "0.0.0-dev" maintainers: - name: HyperFleet Team diff --git a/charts/README.md b/charts/README.md index a902a58c..4a1f2f56 100644 --- a/charts/README.md +++ b/charts/README.md @@ -1,6 +1,6 @@ # hyperfleet-api -![Version: 1.2.0](https://img.shields.io/badge/Version-1.2.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.0.0-dev](https://img.shields.io/badge/AppVersion-0.0.0--dev-informational?style=flat-square) +![Version: 1.1.0](https://img.shields.io/badge/Version-1.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.0.0-dev](https://img.shields.io/badge/AppVersion-0.0.0--dev-informational?style=flat-square) HyperFleet API - Cluster Lifecycle Management Service diff --git a/charts/dashboards/hyperfleet-api.json b/charts/dashboards/hyperfleet-api.json index 7c988840..9899ab9c 100644 --- a/charts/dashboards/hyperfleet-api.json +++ b/charts/dashboards/hyperfleet-api.json @@ -561,13 +561,13 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max by (resource_type, is_delete) (hyperfleet_api_resource_pending_reconciliation)", + "expr": "max by (resource_type, is_delete) (hyperfleet_api_resource_pending_reconciliation) or vector(0)", "legendFormat": "{{resource_type}} (delete={{is_delete}})", "refId": "A" } ], "title": "Resources Pending Reconciliation", - "description": "Live-query gauge — absent series means zero resources pending for that combination.", + "description": "Live-query gauge — falls back to 0 when no resources are pending for that combination.", "type": "timeseries" }, { @@ -653,13 +653,13 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max by (resource_type, is_delete) (hyperfleet_api_resource_pending_reconciliation_stuck)", + "expr": "max by (resource_type, is_delete) (hyperfleet_api_resource_pending_reconciliation_stuck) or vector(0)", "legendFormat": "{{resource_type}} (delete={{is_delete}})", "refId": "A" } ], "title": "Resources Stuck", - "description": "Resources pending reconciliation beyond the stuck threshold (default 10m). Absent series means no stuck resources.", + "description": "Resources pending reconciliation beyond the stuck threshold (default 10m). Falls back to 0 when no resources are stuck.", "type": "timeseries" }, { @@ -745,13 +745,13 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max by (resource_type, is_delete) (hyperfleet_api_resource_pending_reconciliation_stuck_duration_seconds)", + "expr": "max by (resource_type, is_delete) (hyperfleet_api_resource_pending_reconciliation_stuck_duration_seconds) or vector(0)", "legendFormat": "{{resource_type}} (delete={{is_delete}})", "refId": "A" } ], "title": "Max Stuck Duration", - "description": "Maximum time any resource has been stuck pending reconciliation, by resource type.", + "description": "Maximum time any resource has been stuck pending reconciliation, by resource type. Falls back to 0 when no resources are stuck.", "type": "timeseries" }, { diff --git a/docs/metrics.md b/docs/metrics.md index 38bf668a..e8254ee1 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -409,7 +409,7 @@ A pre-built Grafana dashboard is shipped at `charts/dashboards/hyperfleet-api.js 3. Select your Prometheus datasource from the dropdown 4. Click **Import** -The dashboard uses a parameterized `datasource` variable (no hardcoded UIDs) so it works across environments. Reconciliation gauge panels (pending, stuck) show "No data" when no resources match — this is expected behavior, not a missing datasource. +The dashboard uses a parameterized `datasource` variable (no hardcoded UIDs) so it works across environments. Reconciliation gauge panels (pending, stuck, max stuck duration) fall back to 0 when no resources match the query. Automated provisioning via `hyperfleet-infra` is tracked in [HYPERFLEET-1363](https://issues.redhat.com/browse/HYPERFLEET-1363). From 381e7f43bc33485866e5cb3fd11c175cf24adf8c Mon Sep 17 00:00:00 2001 From: tithakka Date: Thu, 30 Jul 2026 10:37:42 -0500 Subject: [PATCH 3/3] HYPERFLEET-1361 - feat: fix coderabbit suggestion --- charts/dashboards/hyperfleet-api.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/dashboards/hyperfleet-api.json b/charts/dashboards/hyperfleet-api.json index 9899ab9c..16324696 100644 --- a/charts/dashboards/hyperfleet-api.json +++ b/charts/dashboards/hyperfleet-api.json @@ -561,7 +561,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max by (resource_type, is_delete) (hyperfleet_api_resource_pending_reconciliation) or vector(0)", + "expr": "max by (resource_type, is_delete) (hyperfleet_api_resource_pending_reconciliation) or on() vector(0)", "legendFormat": "{{resource_type}} (delete={{is_delete}})", "refId": "A" } @@ -653,7 +653,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max by (resource_type, is_delete) (hyperfleet_api_resource_pending_reconciliation_stuck) or vector(0)", + "expr": "max by (resource_type, is_delete) (hyperfleet_api_resource_pending_reconciliation_stuck) or on() vector(0)", "legendFormat": "{{resource_type}} (delete={{is_delete}})", "refId": "A" } @@ -745,7 +745,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max by (resource_type, is_delete) (hyperfleet_api_resource_pending_reconciliation_stuck_duration_seconds) or vector(0)", + "expr": "max by (resource_type, is_delete) (hyperfleet_api_resource_pending_reconciliation_stuck_duration_seconds) or on() vector(0)", "legendFormat": "{{resource_type}} (delete={{is_delete}})", "refId": "A" }