monitoring: scrape prod app metrics + Grafana PlanPal Prod dashboard
This commit is contained in:
planpal
parent
77b579a40f
commit
a9bb6ff4fc
150
planpalto-helm/dashboards/planpal-prod.json
Normal file
150
planpalto-helm/dashboards/planpal-prod.json
Normal file
@ -0,0 +1,150 @@
|
|||||||
|
{
|
||||||
|
"annotations": { "list": [] },
|
||||||
|
"editable": true,
|
||||||
|
"graphTooltip": 1,
|
||||||
|
"title": "PlanPal Prod",
|
||||||
|
"uid": "planpal-prod",
|
||||||
|
"tags": ["planpal"],
|
||||||
|
"time": { "from": "now-1h", "to": "now" },
|
||||||
|
"refresh": "30s",
|
||||||
|
"templating": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"name": "namespace",
|
||||||
|
"type": "constant",
|
||||||
|
"query": "planpal-prod",
|
||||||
|
"current": { "text": "planpal-prod", "value": "planpal-prod" },
|
||||||
|
"hide": 2
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"type": "row", "title": "HTTP golden signals (backend)",
|
||||||
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "timeseries", "title": "Request rate (req/s) by route",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "reqps" } },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum by (route) (rate(http_request_duration_seconds_count{namespace=\"$namespace\"}[5m]))",
|
||||||
|
"legendFormat": "{{route}}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "timeseries", "title": "Latency p95 (s) by route",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "s" } },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "histogram_quantile(0.95, sum by (le, route) (rate(http_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])))",
|
||||||
|
"legendFormat": "{{route}}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "stat", "title": "Error rate % (5xx)",
|
||||||
|
"gridPos": { "h": 6, "w": 8, "x": 0, "y": 9 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "percent", "thresholds": { "steps": [
|
||||||
|
{ "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 5 } ] } } },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "100 * sum(rate(http_request_duration_seconds_count{namespace=\"$namespace\",status=~\"5..\"}[5m])) / clamp_min(sum(rate(http_request_duration_seconds_count{namespace=\"$namespace\"}[5m])), 1)",
|
||||||
|
"legendFormat": "5xx %"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "stat", "title": "Requests/s (total)",
|
||||||
|
"gridPos": { "h": 6, "w": 8, "x": 8, "y": 9 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "reqps" } },
|
||||||
|
"targets": [
|
||||||
|
{ "expr": "sum(rate(http_request_duration_seconds_count{namespace=\"$namespace\"}[5m]))" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "timeseries", "title": "Requests by status class",
|
||||||
|
"gridPos": { "h": 6, "w": 8, "x": 16, "y": 9 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "reqps" } },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum by (status) (rate(http_request_duration_seconds_count{namespace=\"$namespace\"}[5m]))",
|
||||||
|
"legendFormat": "{{status}}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "row", "title": "Database pool",
|
||||||
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "timeseries", "title": "DB pool connections (active / idle / max)",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
|
||||||
|
"targets": [
|
||||||
|
{ "expr": "db_pool_connections_active{namespace=\"$namespace\"}", "legendFormat": "active" },
|
||||||
|
{ "expr": "db_pool_connections_idle{namespace=\"$namespace\"}", "legendFormat": "idle" },
|
||||||
|
{ "expr": "db_pool_connections_max{namespace=\"$namespace\"}", "legendFormat": "max" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "gauge", "title": "DB pool saturation %",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "thresholds": { "steps": [
|
||||||
|
{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 } ] } } },
|
||||||
|
"targets": [
|
||||||
|
{ "expr": "100 * sum(db_pool_connections_active{namespace=\"$namespace\"}) / clamp_min(sum(db_pool_connections_max{namespace=\"$namespace\"}), 1)" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "row", "title": "Pod health (all planpal-prod workloads)",
|
||||||
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "timeseries", "title": "CPU cores by pod",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 25 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "short" } },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\",container!=\"\",pod!=\"\"}[5m]))",
|
||||||
|
"legendFormat": "{{pod}}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "timeseries", "title": "Memory (working set) by pod",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 25 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "bytes" } },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"$namespace\",container!=\"\",pod!=\"\"})",
|
||||||
|
"legendFormat": "{{pod}}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "timeseries", "title": "Pod restarts (increase, 1h)",
|
||||||
|
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 33 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum by (pod) (increase(kube_pod_container_status_restarts_total{namespace=\"$namespace\"}[1h]))",
|
||||||
|
"legendFormat": "{{pod}}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "stat", "title": "Ready pods",
|
||||||
|
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 33 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum by (deployment) (kube_deployment_status_replicas_ready{namespace=\"$namespace\"})",
|
||||||
|
"legendFormat": "{{deployment}}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"schemaVersion": 39,
|
||||||
|
"version": 1
|
||||||
|
}
|
||||||
14
planpalto-helm/templates/dashboard.yaml
Normal file
14
planpalto-helm/templates/dashboard.yaml
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
{{- if .Values.monitoring.enabled }}
|
||||||
|
# Grafana auto-imports any ConfigMap labeled grafana_dashboard=1 (sidecar watches
|
||||||
|
# all namespaces). The dashboard JSON lives in dashboards/planpal-prod.json.
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: planpal-prod-dashboard
|
||||||
|
namespace: {{ include "planpal.namespace" . }}
|
||||||
|
labels:
|
||||||
|
grafana_dashboard: "1"
|
||||||
|
data:
|
||||||
|
planpal-prod.json: |-
|
||||||
|
{{ .Files.Get "dashboards/planpal-prod.json" | indent 4 }}
|
||||||
|
{{- end }}
|
||||||
48
planpalto-helm/templates/servicemonitors.yaml
Normal file
48
planpalto-helm/templates/servicemonitors.yaml
Normal file
@ -0,0 +1,48 @@
|
|||||||
|
{{- if .Values.monitoring.enabled }}
|
||||||
|
{{- /*
|
||||||
|
For every app that declares a container port named "metrics", create:
|
||||||
|
- a headless <name>-metrics Service exposing that port
|
||||||
|
- a ServiceMonitor (labeled for the Prometheus release) scraping it
|
||||||
|
Frontend/redis/nats have no metrics port, so they are skipped automatically.
|
||||||
|
*/ -}}
|
||||||
|
{{- range $name, $app := .Values.apps }}
|
||||||
|
{{- range $app.ports }}
|
||||||
|
{{- if eq .name "metrics" }}
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: {{ $name }}-metrics
|
||||||
|
namespace: {{ include "planpal.namespace" $ }}
|
||||||
|
labels:
|
||||||
|
app: {{ $name }}
|
||||||
|
planpal-metrics: "true"
|
||||||
|
spec:
|
||||||
|
clusterIP: None # headless; ServiceMonitor scrapes the pods behind it
|
||||||
|
selector: { app: {{ $name }} }
|
||||||
|
ports:
|
||||||
|
- name: metrics
|
||||||
|
port: {{ .containerPort }}
|
||||||
|
targetPort: {{ .containerPort }}
|
||||||
|
---
|
||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: ServiceMonitor
|
||||||
|
metadata:
|
||||||
|
name: {{ $name }}
|
||||||
|
namespace: {{ include "planpal.namespace" $ }}
|
||||||
|
labels:
|
||||||
|
# Prometheus (kube-prometheus-stack) only discovers ServiceMonitors with this label.
|
||||||
|
release: {{ $.Values.monitoring.prometheusRelease }}
|
||||||
|
spec:
|
||||||
|
namespaceSelector:
|
||||||
|
matchNames: [{{ include "planpal.namespace" $ }}]
|
||||||
|
selector:
|
||||||
|
matchLabels: { app: {{ $name }}, planpal-metrics: "true" }
|
||||||
|
endpoints:
|
||||||
|
- port: metrics
|
||||||
|
interval: {{ $.Values.monitoring.scrapeInterval }}
|
||||||
|
path: /metrics
|
||||||
|
---
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
@ -24,3 +24,7 @@ ingress:
|
|||||||
|
|
||||||
seedJob:
|
seedJob:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
|
||||||
|
# Scrape prod app metrics into Prometheus + ship the Grafana dashboard.
|
||||||
|
monitoring:
|
||||||
|
enabled: true
|
||||||
|
|||||||
@ -118,6 +118,15 @@ externalSecrets:
|
|||||||
seed-env: planpal/seed-env
|
seed-env: planpal/seed-env
|
||||||
frontend-env: planpal/frontend
|
frontend-env: planpal/frontend
|
||||||
|
|
||||||
|
# Prometheus scraping + Grafana dashboard. Off by default; prod turns it on.
|
||||||
|
# Needs kube-prometheus-stack on the cluster (ServiceMonitor CRD + Grafana sidecar).
|
||||||
|
monitoring:
|
||||||
|
enabled: false
|
||||||
|
# ServiceMonitor is only discovered by Prometheus if it carries this label.
|
||||||
|
# kube-prometheus-stack default release name is "monitoring".
|
||||||
|
prometheusRelease: monitoring
|
||||||
|
scrapeInterval: 30s
|
||||||
|
|
||||||
# Admin seed one-shot Job.
|
# Admin seed one-shot Job.
|
||||||
seedJob:
|
seedJob:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user