feat(metrics): Add node and kube metrics

manifests/grafana/database-backup.yaml

@@ -0,0 +1,10 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: grafana-db-backup
+  namespace: monitoring
+spec:
+  schedule: "0 0 0 * * *"
+  backupOwnerReference: self
+  cluster:
+    name: grafana-db

manifests/grafana/kustomization.yaml

@@ -7,5 +7,6 @@ resources:
   - ingress.yaml
   - servicemonitor.yaml
   - database.yaml
+  - database-backup.yaml
   - datasources.yaml
   - deployment.yaml

manifests/kube-state-metrics/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: monitoring
+
+resources:
+  - git@github.com:kubernetes/kube-state-metrics/examples/standard?ref=v2.14.0
+  - servicemonitor.yaml

manifests/kube-state-metrics/servicemonitor.yaml

@@ -0,0 +1,13 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: kube-state-metrics
+  namespace: monitoring
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: exporter
+      app.kubernetes.io/name: kube-state-metrics
+  endpoints:
+    - port: http-metrics
+      path: /metrics

manifests/kustomization.yaml

@@ -10,3 +10,4 @@ resources:
   - prometheus
   - grafana
   - node-exporter
+  - kube-state-metrics

manifests/node-exporter/daemonset.yaml

@@ -22,8 +22,7 @@ spec:
         - --path.sysfs=/host/sys
         - --path.rootfs=/host/root
         - --no-collector.wifi
-        - --no-collector.hwmon
+        - --collector.filesystem.ignored-mount-points=^/(dev|proc|var/lib/docker/.+|var/lib/kubelet/pods/.+)($|/)
-        - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/pods/.+)($|/)
         - --collector.netclass.ignored-devices=^(veth.*)$
         name: node-exporter
         image: prom/node-exporter

manifests/node-exporter/servicemonitor.yaml

@@ -11,3 +11,6 @@ spec:
   endpoints:
     - port: node-exporter
       path: /metrics
+      relabelings:
+      - sourceLabels: [__meta_kubernetes_pod_node_name]
+        targetLabel: instance

manifests/prometheus/prometheus.yaml

@@ -13,6 +13,10 @@ spec:
       memory: 300Mi
   storage:
     volumeClaimTemplate:
+      metadata:
+        labels:
+          recurring-job.longhorn.io/source: enabled
+          recurring-job-group.longhorn.io/standard-pvc: enabled
       spec:
         storageClassName: redundant-storage-class
         resources: