From a9bb6ff4fca559df8d58722048c1b605272c7e00 Mon Sep 17 00:00:00 2001
From: planpal <planpal@local>
Date: Wed, 2 Sep 2026 15:35:29 +0700
Subject: [PATCH] monitoring: scrape prod app metrics + Grafana PlanPal Prod
 dashboard

---
 planpalto-helm/dashboards/planpal-prod.json   | 150 ++++++++++++++++++
 planpalto-helm/templates/dashboard.yaml       |  14 ++
 planpalto-helm/templates/servicemonitors.yaml |  48 ++++++
 planpalto-helm/values-prod.yaml               |   4 +
 planpalto-helm/values.yaml                    |   9 ++
 5 files changed, 225 insertions(+)
 create mode 100644 planpalto-helm/dashboards/planpal-prod.json
 create mode 100644 planpalto-helm/templates/dashboard.yaml
 create mode 100644 planpalto-helm/templates/servicemonitors.yaml

diff --git a/planpalto-helm/dashboards/planpal-prod.json b/planpalto-helm/dashboards/planpal-prod.json
new file mode 100644
index 0000000..277586b
--- /dev/null
+++ b/planpalto-helm/dashboards/planpal-prod.json
@@ -0,0 +1,150 @@
+{
+  "annotations": { "list": [] },
+  "editable": true,
+  "graphTooltip": 1,
+  "title": "PlanPal Prod",
+  "uid": "planpal-prod",
+  "tags": ["planpal"],
+  "time": { "from": "now-1h", "to": "now" },
+  "refresh": "30s",
+  "templating": {
+    "list": [
+      {
+        "name": "namespace",
+        "type": "constant",
+        "query": "planpal-prod",
+        "current": { "text": "planpal-prod", "value": "planpal-prod" },
+        "hide": 2
+      }
+    ]
+  },
+  "panels": [
+    {
+      "type": "row", "title": "HTTP golden signals (backend)",
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
+    },
+    {
+      "type": "timeseries", "title": "Request rate (req/s) by route",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 },
+      "fieldConfig": { "defaults": { "unit": "reqps" } },
+      "targets": [
+        {
+          "expr": "sum by (route) (rate(http_request_duration_seconds_count{namespace=\"$namespace\"}[5m]))",
+          "legendFormat": "{{route}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries", "title": "Latency p95 (s) by route",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 },
+      "fieldConfig": { "defaults": { "unit": "s" } },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.95, sum by (le, route) (rate(http_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])))",
+          "legendFormat": "{{route}}"
+        }
+      ]
+    },
+    {
+      "type": "stat", "title": "Error rate % (5xx)",
+      "gridPos": { "h": 6, "w": 8, "x": 0, "y": 9 },
+      "fieldConfig": { "defaults": { "unit": "percent", "thresholds": { "steps": [
+        { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 5 } ] } } },
+      "targets": [
+        {
+          "expr": "100 * sum(rate(http_request_duration_seconds_count{namespace=\"$namespace\",status=~\"5..\"}[5m])) / clamp_min(sum(rate(http_request_duration_seconds_count{namespace=\"$namespace\"}[5m])), 1)",
+          "legendFormat": "5xx %"
+        }
+      ]
+    },
+    {
+      "type": "stat", "title": "Requests/s (total)",
+      "gridPos": { "h": 6, "w": 8, "x": 8, "y": 9 },
+      "fieldConfig": { "defaults": { "unit": "reqps" } },
+      "targets": [
+        { "expr": "sum(rate(http_request_duration_seconds_count{namespace=\"$namespace\"}[5m]))" }
+      ]
+    },
+    {
+      "type": "timeseries", "title": "Requests by status class",
+      "gridPos": { "h": 6, "w": 8, "x": 16, "y": 9 },
+      "fieldConfig": { "defaults": { "unit": "reqps" } },
+      "targets": [
+        {
+          "expr": "sum by (status) (rate(http_request_duration_seconds_count{namespace=\"$namespace\"}[5m]))",
+          "legendFormat": "{{status}}"
+        }
+      ]
+    },
+    {
+      "type": "row", "title": "Database pool",
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 }
+    },
+    {
+      "type": "timeseries", "title": "DB pool connections (active / idle / max)",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
+      "targets": [
+        { "expr": "db_pool_connections_active{namespace=\"$namespace\"}", "legendFormat": "active" },
+        { "expr": "db_pool_connections_idle{namespace=\"$namespace\"}", "legendFormat": "idle" },
+        { "expr": "db_pool_connections_max{namespace=\"$namespace\"}", "legendFormat": "max" }
+      ]
+    },
+    {
+      "type": "gauge", "title": "DB pool saturation %",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
+      "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "thresholds": { "steps": [
+        { "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 } ] } } },
+      "targets": [
+        { "expr": "100 * sum(db_pool_connections_active{namespace=\"$namespace\"}) / clamp_min(sum(db_pool_connections_max{namespace=\"$namespace\"}), 1)" }
+      ]
+    },
+    {
+      "type": "row", "title": "Pod health (all planpal-prod workloads)",
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }
+    },
+    {
+      "type": "timeseries", "title": "CPU cores by pod",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 25 },
+      "fieldConfig": { "defaults": { "unit": "short" } },
+      "targets": [
+        {
+          "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\",container!=\"\",pod!=\"\"}[5m]))",
+          "legendFormat": "{{pod}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries", "title": "Memory (working set) by pod",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 25 },
+      "fieldConfig": { "defaults": { "unit": "bytes" } },
+      "targets": [
+        {
+          "expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"$namespace\",container!=\"\",pod!=\"\"})",
+          "legendFormat": "{{pod}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries", "title": "Pod restarts (increase, 1h)",
+      "gridPos": { "h": 6, "w": 12, "x": 0, "y": 33 },
+      "targets": [
+        {
+          "expr": "sum by (pod) (increase(kube_pod_container_status_restarts_total{namespace=\"$namespace\"}[1h]))",
+          "legendFormat": "{{pod}}"
+        }
+      ]
+    },
+    {
+      "type": "stat", "title": "Ready pods",
+      "gridPos": { "h": 6, "w": 12, "x": 12, "y": 33 },
+      "targets": [
+        {
+          "expr": "sum by (deployment) (kube_deployment_status_replicas_ready{namespace=\"$namespace\"})",
+          "legendFormat": "{{deployment}}"
+        }
+      ]
+    }
+  ],
+  "schemaVersion": 39,
+  "version": 1
+}
diff --git a/planpalto-helm/templates/dashboard.yaml b/planpalto-helm/templates/dashboard.yaml
new file mode 100644
index 0000000..52a9888
--- /dev/null
+++ b/planpalto-helm/templates/dashboard.yaml
@@ -0,0 +1,14 @@
+{{- if .Values.monitoring.enabled }}
+# Grafana auto-imports any ConfigMap labeled grafana_dashboard=1 (sidecar watches
+# all namespaces). The dashboard JSON lives in dashboards/planpal-prod.json.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: planpal-prod-dashboard
+  namespace: {{ include "planpal.namespace" . }}
+  labels:
+    grafana_dashboard: "1"
+data:
+  planpal-prod.json: |-
+{{ .Files.Get "dashboards/planpal-prod.json" | indent 4 }}
+{{- end }}
diff --git a/planpalto-helm/templates/servicemonitors.yaml b/planpalto-helm/templates/servicemonitors.yaml
new file mode 100644
index 0000000..0dd6e9d
--- /dev/null
+++ b/planpalto-helm/templates/servicemonitors.yaml
@@ -0,0 +1,48 @@
+{{- if .Values.monitoring.enabled }}
+{{- /*
+  For every app that declares a container port named "metrics", create:
+   - a headless <name>-metrics Service exposing that port
+   - a ServiceMonitor (labeled for the Prometheus release) scraping it
+  Frontend/redis/nats have no metrics port, so they are skipped automatically.
+*/ -}}
+{{- range $name, $app := .Values.apps }}
+{{- range $app.ports }}
+{{- if eq .name "metrics" }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ $name }}-metrics
+  namespace: {{ include "planpal.namespace" $ }}
+  labels:
+    app: {{ $name }}
+    planpal-metrics: "true"
+spec:
+  clusterIP: None   # headless; ServiceMonitor scrapes the pods behind it
+  selector: { app: {{ $name }} }
+  ports:
+  - name: metrics
+    port: {{ .containerPort }}
+    targetPort: {{ .containerPort }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ $name }}
+  namespace: {{ include "planpal.namespace" $ }}
+  labels:
+    # Prometheus (kube-prometheus-stack) only discovers ServiceMonitors with this label.
+    release: {{ $.Values.monitoring.prometheusRelease }}
+spec:
+  namespaceSelector:
+    matchNames: [{{ include "planpal.namespace" $ }}]
+  selector:
+    matchLabels: { app: {{ $name }}, planpal-metrics: "true" }
+  endpoints:
+  - port: metrics
+    interval: {{ $.Values.monitoring.scrapeInterval }}
+    path: /metrics
+---
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
diff --git a/planpalto-helm/values-prod.yaml b/planpalto-helm/values-prod.yaml
index 2a2c3bc..f6ce892 100644
--- a/planpalto-helm/values-prod.yaml
+++ b/planpalto-helm/values-prod.yaml
@@ -24,3 +24,7 @@ ingress:
 
 seedJob:
   enabled: true
+
+# Scrape prod app metrics into Prometheus + ship the Grafana dashboard.
+monitoring:
+  enabled: true
diff --git a/planpalto-helm/values.yaml b/planpalto-helm/values.yaml
index a9e928d..dfe96bc 100644
--- a/planpalto-helm/values.yaml
+++ b/planpalto-helm/values.yaml
@@ -118,6 +118,15 @@ externalSecrets:
     seed-env: planpal/seed-env
     frontend-env: planpal/frontend
 
+# Prometheus scraping + Grafana dashboard. Off by default; prod turns it on.
+# Needs kube-prometheus-stack on the cluster (ServiceMonitor CRD + Grafana sidecar).
+monitoring:
+  enabled: false
+  # ServiceMonitor is only discovered by Prometheus if it carries this label.
+  # kube-prometheus-stack default release name is "monitoring".
+  prometheusRelease: monitoring
+  scrapeInterval: 30s
+
 # Admin seed one-shot Job.
 seedJob:
   enabled: true