From 441a1d723568042480331a6e1aabcbf9300a25ad Mon Sep 17 00:00:00 2001 From: Tanguy Herbron Date: Wed, 5 Jul 2023 15:10:47 +0200 Subject: [PATCH] feat(logs): Add promtail and loki for log aggregation --- manifests/grafana/datasources.yaml | 2 +- manifests/kustomization.yaml | 2 + manifests/loki/configmap.yaml | 64 ++++++++++++++++++++++ manifests/loki/deployment.yaml | 28 ++++++++++ manifests/loki/kustomization.yaml | 7 +++ manifests/loki/service.yaml | 13 +++++ manifests/promtail/clusterrole.yaml | 14 +++++ manifests/promtail/clusterrolebinding.yaml | 13 +++++ manifests/promtail/configmap.yaml | 53 ++++++++++++++++++ manifests/promtail/daemonset.yaml | 48 ++++++++++++++++ manifests/promtail/kustomization.yaml | 9 +++ manifests/promtail/serviceaccount.yaml | 5 ++ 12 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 manifests/loki/configmap.yaml create mode 100644 manifests/loki/deployment.yaml create mode 100644 manifests/loki/kustomization.yaml create mode 100644 manifests/loki/service.yaml create mode 100644 manifests/promtail/clusterrole.yaml create mode 100644 manifests/promtail/clusterrolebinding.yaml create mode 100644 manifests/promtail/configmap.yaml create mode 100644 manifests/promtail/daemonset.yaml create mode 100644 manifests/promtail/kustomization.yaml create mode 100644 manifests/promtail/serviceaccount.yaml diff --git a/manifests/grafana/datasources.yaml b/manifests/grafana/datasources.yaml index 7db5886..addc07e 100644 --- a/manifests/grafana/datasources.yaml +++ b/manifests/grafana/datasources.yaml @@ -10,7 +10,7 @@ data: - name: Loki type: loki access: proxy - url: "http://loki:3100" + url: "http://loki-svc:3100" version: 1 isDefault: true - name: Prometheus diff --git a/manifests/kustomization.yaml b/manifests/kustomization.yaml index 78b4da9..8522193 100644 --- a/manifests/kustomization.yaml +++ b/manifests/kustomization.yaml @@ -4,3 +4,5 @@ kind: Kustomization resources: - namespace.yaml - grafana + - promtail + - loki diff --git a/manifests/loki/configmap.yaml b/manifests/loki/configmap.yaml new file mode 100644 index 0000000..6001822 --- /dev/null +++ b/manifests/loki/configmap.yaml @@ -0,0 +1,64 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: loki-config + namespace: monitoring +data: + loki-config.yaml: | + auth_enabled: false + + server: + http_listen_port: 3100 + grpc_listen_port: 9096 + + common: + instance_addr: 127.0.0.1 + + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + + query_range: + results_cache: + cache: + embedded_cache: + enabled: true + + max_size_mb: 100 + + schema_config: + configs: + - from: 2020-10-24 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h + + + ruler: + alertmanager_url: http://localhost:9093 + + # By default, Loki will send anonymous, but uniquely-identifiable usage and configuration + # analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ + + # + # Statistics help us better understand how Loki is used, and they show us performance + + # levels for most users. This helps us prioritize features and documentation. + # For more information on what's sent, look at + # https://github.com/grafana/loki/blob/main/pkg/usagestats/stats.go + # Refer to the buildReport method to see what goes into a report. + # + # If you would like to disable reporting, uncomment the following lines: + #analytics: + # reporting_enabled: false + + diff --git a/manifests/loki/deployment.yaml b/manifests/loki/deployment.yaml new file mode 100644 index 0000000..7c1ea2a --- /dev/null +++ b/manifests/loki/deployment.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: loki + namespace: monitoring +spec: + selector: + matchLabels: + app: loki + template: + metadata: + labels: + app: loki + spec: + containers: + - name: loki + image: grafana/loki:2.8.2 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 3100 + volumeMounts: + - name: loki-config-volume + mountPath: /mnt/config/loki-config.yaml + subPath: loki-config.yaml + volumes: + - name: loki-config-volume + configMap: + name: loki-config diff --git a/manifests/loki/kustomization.yaml b/manifests/loki/kustomization.yaml new file mode 100644 index 0000000..19ef0b2 --- /dev/null +++ b/manifests/loki/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - configmap.yaml + - service.yaml + - deployment.yaml diff --git a/manifests/loki/service.yaml b/manifests/loki/service.yaml new file mode 100644 index 0000000..6ca5c92 --- /dev/null +++ b/manifests/loki/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: loki-svc + namespace: monitoring +spec: + ports: + - name: http + port: 3100 + protocol: TCP + targetPort: 3100 + selector: + app: loki diff --git a/manifests/promtail/clusterrole.yaml b/manifests/promtail/clusterrole.yaml new file mode 100644 index 0000000..2d3bdbb --- /dev/null +++ b/manifests/promtail/clusterrole.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: promtail-clusterrole +rules: + - apiGroups: [""] + resources: + - nodes + - services + - pods + verbs: + - get + - watch + - list diff --git a/manifests/promtail/clusterrolebinding.yaml b/manifests/promtail/clusterrolebinding.yaml new file mode 100644 index 0000000..0f3dac6 --- /dev/null +++ b/manifests/promtail/clusterrolebinding.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: promtail-clusterrolebinding + namespace: monitoring +subjects: + - kind: ServiceAccount + name: promtail-serviceaccount + namespace: monitoring +roleRef: + kind: ClusterRole + name: promtail-clusterrole + apiGroup: rbac.authorization.k8s.io diff --git a/manifests/promtail/configmap.yaml b/manifests/promtail/configmap.yaml new file mode 100644 index 0000000..6e7f4ea --- /dev/null +++ b/manifests/promtail/configmap.yaml @@ -0,0 +1,53 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: promtail-config + namespace: monitoring +data: + promtail.yaml: | + server: + http_listen_port: 9080 + grpc_listen_port: 0 + clients: + - url: http://loki-svc:3100/loki/api/v1/push + positions: + filename: /tmp/positions.yaml + target_config: + sync_period: 10s + scrape_configs: + - job_name: pod-logs + kubernetes_sd_configs: + - role: pod + pipeline_stages: + - docker: {} + relabel_configs: + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_pod_name + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container + - replacement: /var/log/pods/*$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ diff --git a/manifests/promtail/daemonset.yaml b/manifests/promtail/daemonset.yaml new file mode 100644 index 0000000..b02ce48 --- /dev/null +++ b/manifests/promtail/daemonset.yaml @@ -0,0 +1,48 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: promtail-daemonset + namespace: monitoring +spec: + selector: + matchLabels: + name: promtail + template: + metadata: + labels: + name: promtail + spec: + serviceAccount: promtail-serviceaccount + containers: + - name: promtail-container + image: grafana/promtail + args: + - -config.file=/etc/promtail/promtail.yaml + env: + - name: 'HOSTNAME' # needed when using kubernetes_sd_configs + valueFrom: + fieldRef: + fieldPath: 'spec.nodeName' + volumeMounts: + - name: logs + mountPath: /var/log + - name: promtail-config + mountPath: /etc/promtail + - mountPath: /var/lib/docker/containers + name: varlibdockercontainers + readOnly: true + tolerations: + - key: type + operator: Equal + value: services + effect: NoSchedule + volumes: + - name: logs + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: promtail-config + configMap: + name: promtail-config diff --git a/manifests/promtail/kustomization.yaml b/manifests/promtail/kustomization.yaml new file mode 100644 index 0000000..f1bca4f --- /dev/null +++ b/manifests/promtail/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - clusterrole.yaml + - serviceaccount.yaml + - clusterrolebinding.yaml + - configmap.yaml + - daemonset.yaml diff --git a/manifests/promtail/serviceaccount.yaml b/manifests/promtail/serviceaccount.yaml new file mode 100644 index 0000000..643dccc --- /dev/null +++ b/manifests/promtail/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: promtail-serviceaccount + namespace: monitoring