From 93b25589b0d6ebf3472c9e2a21bbb3e5ec462574 Mon Sep 17 00:00:00 2001 From: Tanguy Herbron Date: Sat, 21 Dec 2024 22:27:07 +0100 Subject: [PATCH] feat(traefik): Add traefik ingress controller configuration --- traefik/flannel.yaml | 230 ++++++++++++++++++ traefik/rbac/clusterrole.yaml | 22 +- traefik/rbac/clusterrolebinding.yaml | 2 - traefik/rbac/serviceaccount.yaml | 2 - traefik/traefik-external/deployment.yaml | 27 +- traefik/traefik-external/kustomization.yaml | 3 +- traefik/traefik-external/loadbalancer.yaml | 28 +++ traefik/traefik-external/pvc.yaml | 7 +- .../traefik-external/service-dashboard.yaml | 1 - traefik/traefik-external/service.yaml | 18 +- traefik/traefik-external/servicemonitor.yaml | 3 +- traefik/traefik-internal/config-map-dev.yaml | 7 + traefik/traefik-internal/deployment.yaml | 17 +- traefik/traefik-internal/kustomization.yaml | 3 +- traefik/traefik-internal/loadbalancer.yaml | 22 ++ traefik/traefik-internal/pvc.yaml | 7 +- traefik/traefik-internal/service.yaml | 13 +- traefik/traefik-internal/servicemonitor.yaml | 4 +- 18 files changed, 346 insertions(+), 70 deletions(-) create mode 100644 traefik/flannel.yaml create mode 100644 traefik/traefik-external/loadbalancer.yaml create mode 100644 traefik/traefik-internal/config-map-dev.yaml create mode 100644 traefik/traefik-internal/loadbalancer.yaml diff --git a/traefik/flannel.yaml b/traefik/flannel.yaml new file mode 100644 index 0000000..2c4437d --- /dev/null +++ b/traefik/flannel.yaml @@ -0,0 +1,230 @@ +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: psp.flannel.unprivileged + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default + seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default + apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default + apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default +spec: + privileged: false + volumes: + - configMap + - secret + - emptyDir + - hostPath + allowedHostPaths: + - pathPrefix: "/etc/cni/net.d" + - pathPrefix: "/etc/kube-flannel" + - pathPrefix: "/run/flannel" + readOnlyRootFilesystem: false + # Users and groups + runAsUser: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + fsGroup: + rule: RunAsAny + # Privilege Escalation + allowPrivilegeEscalation: false + defaultAllowPrivilegeEscalation: false + # Capabilities + allowedCapabilities: ['NET_ADMIN'] + defaultAddCapabilities: [] + requiredDropCapabilities: [] + # Host namespaces + hostPID: false + hostIPC: false + hostNetwork: true + hostPorts: + - min: 0 + max: 65535 + # SELinux + seLinux: + # SELinux is unused in CaaSP + rule: 'RunAsAny' +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: flannel +rules: + - apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: ['psp.flannel.unprivileged'] + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: flannel +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flannel +subjects: +- kind: ServiceAccount + name: flannel + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount + +metadata: + name: flannel + namespace: kube-system +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-flannel-cfg + namespace: kube-system + labels: + tier: node + app: flannel +data: + cni-conf.json: | + { + "name": "cbr0", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "flannel", + "delegate": { + "hairpinMode": true, + "isDefaultGateway": true + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + } + ] + } + net-conf.json: | + { + "Network": "${flannel_cidr}", + "Backend": { + "Type": "vxlan" + } + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-flannel-ds-amd64 + namespace: kube-system + labels: + tier: node + app: flannel +spec: + selector: + matchLabels: + app: flannel + template: + metadata: + labels: + tier: node + app: flannel + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + - key: kubernetes.io/arch + operator: In + + values: + - amd64 + hostNetwork: true + tolerations: + - operator: Exists + effect: NoSchedule + serviceAccountName: flannel + initContainers: + - name: install-cni + image: quay.io/coreos/flannel:v0.12.0-amd64 + command: + - cp + args: + - -f + - /etc/kube-flannel/cni-conf.json + - /etc/cni/net.d/10-flannel.conflist + volumeMounts: + - name: cni + mountPath: /etc/cni/net.d + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + containers: + - name: kube-flannel + image: quay.io/coreos/flannel:v0.12.0-amd64 + command: + - /opt/bin/flanneld + args: + - --ip-masq=false + - --kube-subnet-mgr + - --iface=${interface} + resources: + requests: + cpu: "50m" + memory: "50Mi" + limits: + cpu: "50m" + memory: "50Mi" + securityContext: + privileged: false + capabilities: + add: ["NET_ADMIN"] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: run + mountPath: /run/flannel + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + volumes: + - name: run + hostPath: + path: /run/flannel + - name: cni + hostPath: + path: /etc/cni/net.d + - name: flannel-cfg + configMap: + name: kube-flannel-cfg +--- diff --git a/traefik/rbac/clusterrole.yaml b/traefik/rbac/clusterrole.yaml index 3f316f3..cea2b91 100644 --- a/traefik/rbac/clusterrole.yaml +++ b/traefik/rbac/clusterrole.yaml @@ -1,5 +1,3 @@ ---- -# Source: traefik/templates/rbac/clusterrole.yaml kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -12,12 +10,19 @@ rules: - "" resources: - services - - endpoints - secrets + - nodes verbs: - get - list - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch - apiGroups: - extensions - networking.k8s.io @@ -36,17 +41,18 @@ rules: verbs: - update - apiGroups: - - traefik.containo.us + - traefik.io resources: - - ingressroutes - - ingressroutetcps - - ingressrouteudps - middlewares - middlewaretcps + - ingressroutes + - traefikservices + - ingressroutetcps + - ingressrouteudps - tlsoptions - tlsstores - - traefikservices - serverstransports + - serverstransporttcps verbs: - get - list diff --git a/traefik/rbac/clusterrolebinding.yaml b/traefik/rbac/clusterrolebinding.yaml index e35e2b2..c58a420 100644 --- a/traefik/rbac/clusterrolebinding.yaml +++ b/traefik/rbac/clusterrolebinding.yaml @@ -14,5 +14,3 @@ roleRef: subjects: - kind: ServiceAccount name: traefik - namespace: default - diff --git a/traefik/rbac/serviceaccount.yaml b/traefik/rbac/serviceaccount.yaml index b2e4ed6..8834ddf 100644 --- a/traefik/rbac/serviceaccount.yaml +++ b/traefik/rbac/serviceaccount.yaml @@ -5,5 +5,3 @@ metadata: labels: app.kubernetes.io/name: traefik app.kubernetes.io/instance: traefik - annotations: - diff --git a/traefik/traefik-external/deployment.yaml b/traefik/traefik-external/deployment.yaml index 0825f2e..36de030 100644 --- a/traefik/traefik-external/deployment.yaml +++ b/traefik/traefik-external/deployment.yaml @@ -1,5 +1,4 @@ --- -# Source: traefik/templates/deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: @@ -7,9 +6,8 @@ metadata: labels: app.kubernetes.io/name: traefik app.kubernetes.io/instance: traefik - annotations: spec: - replicas: 1 + replicas: 2 selector: matchLabels: app.kubernetes.io/name: traefik @@ -30,11 +28,18 @@ spec: app.kubernetes.io/name: traefik app.kubernetes.io/instance: traefik spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: traefik serviceAccountName: traefik terminationGracePeriodSeconds: 60 - hostNetwork: false + hostNetwork: true containers: - - image: "traefik:2.8.4" + - image: "traefik:3.2.1" imagePullPolicy: IfNotPresent name: traefik resources: @@ -89,13 +94,20 @@ spec: - name: tmp mountPath: /tmp args: - - "--global.checknewversion" - "--entrypoints.admin.address=:8080/tcp" - "--entrypoints.metrics.address=:9100/tcp" - "--entrypoints.minecrafttcp.address=:25565/tcp" - "--entrypoints.traefik.address=:9000/tcp" - "--entrypoints.web.address=:8000/tcp" - "--entrypoints.websecure.address=:8443/tcp" + #- "--entryPoints.web.proxyProtocol.insecure=true" + #- "--entryPoints.web.forwardedHeaders.insecure=true" + - "--entryPoints.web.proxyProtocol.trustedIPs=10.20.0.0/24,51.15.80.73/32,192.168.113.0/24" + - "--entryPoints.web.forwardedHeaders.trustedIPs=10.20.0.0/24,51.15.80.73/32,192.168.113.0/24" + #- "--entryPoints.websecure.proxyProtocol.insecure=true" + #- "--entryPoints.websecure.forwardedHeaders.insecure=true" + - "--entryPoints.websecure.proxyProtocol.trustedIPs=10.20.0.0/24,51.15.80.73/32,192.168.113.0/24" + - "--entryPoints.websecure.forwardedHeaders.trustedIPs=10.20.0.0/24,51.15.80.73/32,192.168.113.0/24" - "--api.dashboard=true" - "--ping=true" - "--metrics.prometheus=true" @@ -108,8 +120,7 @@ spec: - "--accesslog=true" - "--entrypoints.websecure.http.tls=true" - "--entrypoints.websecure.http.tls.certresolver=letsencrypt" - - "--entrypoints.websecure.http.tls.domains[0].main=beta.halia.dev" - - "--entrypoints.websecure.http.tls.domains[0].sans=*.beta.halia.dev" + - "--entrypoints.websecure.http.tls.domains[0].sans=*.halis.io" - "--certificatesresolvers.letsencrypt.acme.tlschallenge=true" - "--certificatesresolvers.letsencrypt.acme.dnschallenge=true" - "--certificatesresolvers.letsencrypt.acme.dnschallenge.provider=ovh" diff --git a/traefik/traefik-external/kustomization.yaml b/traefik/traefik-external/kustomization.yaml index a876e32..082e64d 100644 --- a/traefik/traefik-external/kustomization.yaml +++ b/traefik/traefik-external/kustomization.yaml @@ -5,4 +5,5 @@ resources: - pvc.yaml - deployment.yaml - service.yaml - - servicemonitor.yaml + - loadbalancer.yaml + #- servicemonitor.yaml diff --git a/traefik/traefik-external/loadbalancer.yaml b/traefik/traefik-external/loadbalancer.yaml new file mode 100644 index 0000000..af38954 --- /dev/null +++ b/traefik/traefik-external/loadbalancer.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Service +metadata: + name: traefik-external + labels: + app.kubernetes.io/name: traefik-external + app.kubernetes.io/instance: traefik-external +spec: + type: LoadBalancer + loadBalancerIP: 51.15.80.73 + loadBalancerSourceRanges: + - 0.0.0.0/0 + selector: + app.kubernetes.io/name: traefik + app.kubernetes.io/instance: traefik + ports: + - port: 25565 + name: minecrafttcp + targetPort: "minecrafttcp" + protocol: TCP + - port: 80 + name: web + targetPort: "web" + protocol: TCP + - port: 443 + name: websecure + targetPort: "websecure" + protocol: TCP diff --git a/traefik/traefik-external/pvc.yaml b/traefik/traefik-external/pvc.yaml index 31dbb00..05155ea 100644 --- a/traefik/traefik-external/pvc.yaml +++ b/traefik/traefik-external/pvc.yaml @@ -1,18 +1,15 @@ --- -# Source: traefik/templates/pvc.yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: traefik - annotations: - helm.sh/resource-policy: keep labels: app.kubernetes.io/name: traefik app.kubernetes.io/instance: traefik spec: accessModes: - - "ReadWriteOnce" + - "ReadWriteMany" resources: requests: storage: "128Mi" - storageClassName: "local-path" + storageClassName: "redundant-storage-class" diff --git a/traefik/traefik-external/service-dashboard.yaml b/traefik/traefik-external/service-dashboard.yaml index d8884a3..4871f22 100644 --- a/traefik/traefik-external/service-dashboard.yaml +++ b/traefik/traefik-external/service-dashboard.yaml @@ -2,7 +2,6 @@ apiVersion: v1 kind: Service metadata: name: traefik-dashboard-svc - namespace: default spec: ports: - name: admin diff --git a/traefik/traefik-external/service.yaml b/traefik/traefik-external/service.yaml index 1abf0b4..ca6933f 100644 --- a/traefik/traefik-external/service.yaml +++ b/traefik/traefik-external/service.yaml @@ -1,32 +1,16 @@ ---- -# Source: traefik/templates/service.yaml apiVersion: v1 kind: Service metadata: - name: traefik-external + name: traefik-external-metrics labels: app.kubernetes.io/name: traefik-external app.kubernetes.io/instance: traefik-external annotations: spec: - type: LoadBalancer - loadBalancerIP: 51.15.80.73 selector: app.kubernetes.io/name: traefik app.kubernetes.io/instance: traefik ports: - - port: 25565 - name: minecrafttcp - targetPort: "minecrafttcp" - protocol: TCP - - port: 80 # Change port here to accomodate for internal only services - name: web - targetPort: "web" - protocol: TCP - - port: 443 - name: websecure - targetPort: "websecure" - protocol: TCP - port: 9100 name: metrics targetPort: "metrics" diff --git a/traefik/traefik-external/servicemonitor.yaml b/traefik/traefik-external/servicemonitor.yaml index 523bf56..aaea8cf 100644 --- a/traefik/traefik-external/servicemonitor.yaml +++ b/traefik/traefik-external/servicemonitor.yaml @@ -2,8 +2,7 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: name: traefik-external - labels: - team: core + namespace: traefik spec: selector: matchLabels: diff --git a/traefik/traefik-internal/config-map-dev.yaml b/traefik/traefik-internal/config-map-dev.yaml new file mode 100644 index 0000000..4bdd96c --- /dev/null +++ b/traefik/traefik-internal/config-map-dev.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: general-purpose-dev-config + +data: + lb-ip: 10.10.0.32 diff --git a/traefik/traefik-internal/deployment.yaml b/traefik/traefik-internal/deployment.yaml index f89abab..f9f451e 100644 --- a/traefik/traefik-internal/deployment.yaml +++ b/traefik/traefik-internal/deployment.yaml @@ -1,5 +1,4 @@ --- -# Source: traefik/templates/deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: @@ -7,9 +6,8 @@ metadata: labels: app.kubernetes.io/name: traefik-inter app.kubernetes.io/instance: traefik-inter - annotations: spec: - replicas: 1 + replicas: 2 selector: matchLabels: app.kubernetes.io/name: traefik-inter @@ -30,11 +28,18 @@ spec: app.kubernetes.io/name: traefik-inter app.kubernetes.io/instance: traefik-inter spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: traefik-inter serviceAccountName: traefik terminationGracePeriodSeconds: 60 hostNetwork: false containers: - - image: "traefik:2.8.4" + - image: "traefik:3.2.1" imagePullPolicy: IfNotPresent name: traefik-inter resources: @@ -92,6 +97,10 @@ spec: - "--entrypoints.traefik.address=:9000/tcp" - "--entrypoints.web.address=:8000/tcp" - "--entrypoints.websecure.address=:8443/tcp" + - "--entryPoints.web.proxyProtocol.insecure" + - "--entryPoints.web.forwardedHeaders.insecure" + - "--entryPoints.websecure.proxyProtocol.insecure" + - "--entryPoints.websecure.forwardedHeaders.insecure" - "--api.dashboard=true" - "--api.insecure=true" - "--ping=true" diff --git a/traefik/traefik-internal/kustomization.yaml b/traefik/traefik-internal/kustomization.yaml index a876e32..082e64d 100644 --- a/traefik/traefik-internal/kustomization.yaml +++ b/traefik/traefik-internal/kustomization.yaml @@ -5,4 +5,5 @@ resources: - pvc.yaml - deployment.yaml - service.yaml - - servicemonitor.yaml + - loadbalancer.yaml + #- servicemonitor.yaml diff --git a/traefik/traefik-internal/loadbalancer.yaml b/traefik/traefik-internal/loadbalancer.yaml new file mode 100644 index 0000000..f1d0015 --- /dev/null +++ b/traefik/traefik-internal/loadbalancer.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: traefik-internal + labels: + app.kubernetes.io/name: traefik-inter + app.kubernetes.io/instance: traefik-inter +spec: + type: LoadBalancer + loadBalancerIP: 10.10.0.16 + selector: + app.kubernetes.io/name: traefik-inter + app.kubernetes.io/instance: traefik-inter + ports: + - port: 80 + name: web + targetPort: "web" + protocol: TCP + - port: 443 + name: websecure + targetPort: "websecure" + protocol: TCP diff --git a/traefik/traefik-internal/pvc.yaml b/traefik/traefik-internal/pvc.yaml index d4b1360..6800aa2 100644 --- a/traefik/traefik-internal/pvc.yaml +++ b/traefik/traefik-internal/pvc.yaml @@ -1,18 +1,15 @@ --- -# Source: traefik/templates/pvc.yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: traefik-inter - annotations: - helm.sh/resource-policy: keep labels: app.kubernetes.io/name: traefik-inter app.kubernetes.io/instance: traefik-inter spec: accessModes: - - "ReadWriteOnce" + - "ReadWriteMany" resources: requests: storage: "128Mi" - storageClassName: "local-path" + storageClassName: "redundant-storage-class" diff --git a/traefik/traefik-internal/service.yaml b/traefik/traefik-internal/service.yaml index 478f665..d03c0ec 100644 --- a/traefik/traefik-internal/service.yaml +++ b/traefik/traefik-internal/service.yaml @@ -1,26 +1,15 @@ apiVersion: v1 kind: Service metadata: - name: traefik-internal - namespace: default + name: traefik-internal-metrics labels: app.kubernetes.io/name: traefik-inter app.kubernetes.io/instance: traefik-inter spec: - type: LoadBalancer - loadBalancerIP: 192.168.56.101 selector: app.kubernetes.io/name: traefik-inter app.kubernetes.io/instance: traefik-inter ports: - - port: 80 - name: web - targetPort: "web" - protocol: TCP - - port: 443 - name: websecure - targetPort: "websecure" - protocol: TCP - port: 9100 name: metrics targetPort: "metrics" diff --git a/traefik/traefik-internal/servicemonitor.yaml b/traefik/traefik-internal/servicemonitor.yaml index 90920df..9ab1904 100644 --- a/traefik/traefik-internal/servicemonitor.yaml +++ b/traefik/traefik-internal/servicemonitor.yaml @@ -2,12 +2,12 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: name: traefik-inter - labels: - team: core + namespace: traefik spec: selector: matchLabels: app.kubernetes.io/name: traefik-inter + app.kubernetes.io/instance: traefik-inter endpoints: - port: metrics path: /metrics