monitoring: scrape prod app metrics + Grafana PlanPal Prod dashboard
This commit is contained in:
planpal
parent
77b579a40f
commit
a9bb6ff4fc
150
planpalto-helm/dashboards/planpal-prod.json
Normal file
150
planpalto-helm/dashboards/planpal-prod.json
Normal file
@ -0,0 +1,150 @@
|
||||
{
|
||||
"annotations": { "list": [] },
|
||||
"editable": true,
|
||||
"graphTooltip": 1,
|
||||
"title": "PlanPal Prod",
|
||||
"uid": "planpal-prod",
|
||||
"tags": ["planpal"],
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"refresh": "30s",
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "namespace",
|
||||
"type": "constant",
|
||||
"query": "planpal-prod",
|
||||
"current": { "text": "planpal-prod", "value": "planpal-prod" },
|
||||
"hide": 2
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "row", "title": "HTTP golden signals (backend)",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "Request rate (req/s) by route",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps" } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (route) (rate(http_request_duration_seconds_count{namespace=\"$namespace\"}[5m]))",
|
||||
"legendFormat": "{{route}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "Latency p95 (s) by route",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 },
|
||||
"fieldConfig": { "defaults": { "unit": "s" } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by (le, route) (rate(http_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])))",
|
||||
"legendFormat": "{{route}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "stat", "title": "Error rate % (5xx)",
|
||||
"gridPos": { "h": 6, "w": 8, "x": 0, "y": 9 },
|
||||
"fieldConfig": { "defaults": { "unit": "percent", "thresholds": { "steps": [
|
||||
{ "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 5 } ] } } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "100 * sum(rate(http_request_duration_seconds_count{namespace=\"$namespace\",status=~\"5..\"}[5m])) / clamp_min(sum(rate(http_request_duration_seconds_count{namespace=\"$namespace\"}[5m])), 1)",
|
||||
"legendFormat": "5xx %"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "stat", "title": "Requests/s (total)",
|
||||
"gridPos": { "h": 6, "w": 8, "x": 8, "y": 9 },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps" } },
|
||||
"targets": [
|
||||
{ "expr": "sum(rate(http_request_duration_seconds_count{namespace=\"$namespace\"}[5m]))" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "Requests by status class",
|
||||
"gridPos": { "h": 6, "w": 8, "x": 16, "y": 9 },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps" } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (status) (rate(http_request_duration_seconds_count{namespace=\"$namespace\"}[5m]))",
|
||||
"legendFormat": "{{status}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "row", "title": "Database pool",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 }
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "DB pool connections (active / idle / max)",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
|
||||
"targets": [
|
||||
{ "expr": "db_pool_connections_active{namespace=\"$namespace\"}", "legendFormat": "active" },
|
||||
{ "expr": "db_pool_connections_idle{namespace=\"$namespace\"}", "legendFormat": "idle" },
|
||||
{ "expr": "db_pool_connections_max{namespace=\"$namespace\"}", "legendFormat": "max" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "gauge", "title": "DB pool saturation %",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
|
||||
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "thresholds": { "steps": [
|
||||
{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 } ] } } },
|
||||
"targets": [
|
||||
{ "expr": "100 * sum(db_pool_connections_active{namespace=\"$namespace\"}) / clamp_min(sum(db_pool_connections_max{namespace=\"$namespace\"}), 1)" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "row", "title": "Pod health (all planpal-prod workloads)",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "CPU cores by pod",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 25 },
|
||||
"fieldConfig": { "defaults": { "unit": "short" } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\",container!=\"\",pod!=\"\"}[5m]))",
|
||||
"legendFormat": "{{pod}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "Memory (working set) by pod",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 25 },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes" } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"$namespace\",container!=\"\",pod!=\"\"})",
|
||||
"legendFormat": "{{pod}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries", "title": "Pod restarts (increase, 1h)",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 33 },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (pod) (increase(kube_pod_container_status_restarts_total{namespace=\"$namespace\"}[1h]))",
|
||||
"legendFormat": "{{pod}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "stat", "title": "Ready pods",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 33 },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (deployment) (kube_deployment_status_replicas_ready{namespace=\"$namespace\"})",
|
||||
"legendFormat": "{{deployment}}"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"schemaVersion": 39,
|
||||
"version": 1
|
||||
}
|
||||
14
planpalto-helm/templates/dashboard.yaml
Normal file
14
planpalto-helm/templates/dashboard.yaml
Normal file
@ -0,0 +1,14 @@
|
||||
{{- if .Values.monitoring.enabled }}
|
||||
# Grafana auto-imports any ConfigMap labeled grafana_dashboard=1 (sidecar watches
|
||||
# all namespaces). The dashboard JSON lives in dashboards/planpal-prod.json.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: planpal-prod-dashboard
|
||||
namespace: {{ include "planpal.namespace" . }}
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
planpal-prod.json: |-
|
||||
{{ .Files.Get "dashboards/planpal-prod.json" | indent 4 }}
|
||||
{{- end }}
|
||||
48
planpalto-helm/templates/servicemonitors.yaml
Normal file
48
planpalto-helm/templates/servicemonitors.yaml
Normal file
@ -0,0 +1,48 @@
|
||||
{{- if .Values.monitoring.enabled }}
|
||||
{{- /*
|
||||
For every app that declares a container port named "metrics", create:
|
||||
- a headless <name>-metrics Service exposing that port
|
||||
- a ServiceMonitor (labeled for the Prometheus release) scraping it
|
||||
Frontend/redis/nats have no metrics port, so they are skipped automatically.
|
||||
*/ -}}
|
||||
{{- range $name, $app := .Values.apps }}
|
||||
{{- range $app.ports }}
|
||||
{{- if eq .name "metrics" }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ $name }}-metrics
|
||||
namespace: {{ include "planpal.namespace" $ }}
|
||||
labels:
|
||||
app: {{ $name }}
|
||||
planpal-metrics: "true"
|
||||
spec:
|
||||
clusterIP: None # headless; ServiceMonitor scrapes the pods behind it
|
||||
selector: { app: {{ $name }} }
|
||||
ports:
|
||||
- name: metrics
|
||||
port: {{ .containerPort }}
|
||||
targetPort: {{ .containerPort }}
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: {{ $name }}
|
||||
namespace: {{ include "planpal.namespace" $ }}
|
||||
labels:
|
||||
# Prometheus (kube-prometheus-stack) only discovers ServiceMonitors with this label.
|
||||
release: {{ $.Values.monitoring.prometheusRelease }}
|
||||
spec:
|
||||
namespaceSelector:
|
||||
matchNames: [{{ include "planpal.namespace" $ }}]
|
||||
selector:
|
||||
matchLabels: { app: {{ $name }}, planpal-metrics: "true" }
|
||||
endpoints:
|
||||
- port: metrics
|
||||
interval: {{ $.Values.monitoring.scrapeInterval }}
|
||||
path: /metrics
|
||||
---
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@ -24,3 +24,7 @@ ingress:
|
||||
|
||||
seedJob:
|
||||
enabled: true
|
||||
|
||||
# Scrape prod app metrics into Prometheus + ship the Grafana dashboard.
|
||||
monitoring:
|
||||
enabled: true
|
||||
|
||||
@ -118,6 +118,15 @@ externalSecrets:
|
||||
seed-env: planpal/seed-env
|
||||
frontend-env: planpal/frontend
|
||||
|
||||
# Prometheus scraping + Grafana dashboard. Off by default; prod turns it on.
|
||||
# Needs kube-prometheus-stack on the cluster (ServiceMonitor CRD + Grafana sidecar).
|
||||
monitoring:
|
||||
enabled: false
|
||||
# ServiceMonitor is only discovered by Prometheus if it carries this label.
|
||||
# kube-prometheus-stack default release name is "monitoring".
|
||||
prometheusRelease: monitoring
|
||||
scrapeInterval: 30s
|
||||
|
||||
# Admin seed one-shot Job.
|
||||
seedJob:
|
||||
enabled: true
|
||||
|
||||
Loading…
Reference in New Issue
Block a user