feat(traefik): Add traefik ingress controller configuration

traefik/flannel.yaml

@@ -0,0 +1,230 @@
+---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: psp.flannel.unprivileged
+  annotations:
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
+    seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
+    apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
+    apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
+spec:
+  privileged: false
+  volumes:
+    - configMap
+    - secret
+    - emptyDir
+    - hostPath
+  allowedHostPaths:
+    - pathPrefix: "/etc/cni/net.d"
+    - pathPrefix: "/etc/kube-flannel"
+    - pathPrefix: "/run/flannel"
+  readOnlyRootFilesystem: false
+  # Users and groups
+  runAsUser:
+    rule: RunAsAny
+  supplementalGroups:
+    rule: RunAsAny
+  fsGroup:
+    rule: RunAsAny
+  # Privilege Escalation
+  allowPrivilegeEscalation: false
+  defaultAllowPrivilegeEscalation: false
+  # Capabilities
+  allowedCapabilities: ['NET_ADMIN']
+  defaultAddCapabilities: []
+  requiredDropCapabilities: []
+  # Host namespaces
+  hostPID: false
+  hostIPC: false
+  hostNetwork: true
+  hostPorts:
+  - min: 0
+    max: 65535
+  # SELinux
+  seLinux:
+    # SELinux is unused in CaaSP
+    rule: 'RunAsAny'
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: flannel
+rules:
+  - apiGroups: ['extensions']
+    resources: ['podsecuritypolicies']
+    verbs: ['use']
+    resourceNames: ['psp.flannel.unprivileged']
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/status
+    verbs:
+      - patch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: flannel
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: flannel
+subjects:
+- kind: ServiceAccount
+  name: flannel
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+
+metadata:
+  name: flannel
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: kube-flannel-cfg
+  namespace: kube-system
+  labels:
+    tier: node
+    app: flannel
+data:
+  cni-conf.json: |
+    {
+      "name": "cbr0",
+      "cniVersion": "0.3.1",
+      "plugins": [
+        {
+          "type": "flannel",
+          "delegate": {
+            "hairpinMode": true,
+            "isDefaultGateway": true
+          }
+        },
+        {
+          "type": "portmap",
+          "capabilities": {
+            "portMappings": true
+          }
+        }
+      ]
+    }
+  net-conf.json: |
+    {
+      "Network": "${flannel_cidr}",
+      "Backend": {
+        "Type": "vxlan"
+      }
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-flannel-ds-amd64
+  namespace: kube-system
+  labels:
+    tier: node
+    app: flannel
+spec:
+  selector:
+    matchLabels:
+      app: flannel
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: flannel
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/os
+                    operator: In
+                    values:
+                      - linux
+                  - key: kubernetes.io/arch
+                    operator: In
+
+                    values:
+                      - amd64
+      hostNetwork: true
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: flannel
+      initContainers:
+      - name: install-cni
+        image: quay.io/coreos/flannel:v0.12.0-amd64
+        command:
+        - cp
+        args:
+        - -f
+        - /etc/kube-flannel/cni-conf.json
+        - /etc/cni/net.d/10-flannel.conflist
+        volumeMounts:
+        - name: cni
+          mountPath: /etc/cni/net.d
+        - name: flannel-cfg
+          mountPath: /etc/kube-flannel/
+      containers:
+      - name: kube-flannel
+        image: quay.io/coreos/flannel:v0.12.0-amd64
+        command:
+        - /opt/bin/flanneld
+        args:
+        - --ip-masq=false
+        - --kube-subnet-mgr
+        - --iface=${interface}
+        resources:
+          requests:
+            cpu: "50m"
+            memory: "50Mi"
+          limits:
+            cpu: "50m"
+            memory: "50Mi"
+        securityContext:
+          privileged: false
+          capabilities:
+            add: ["NET_ADMIN"]
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        volumeMounts:
+        - name: run
+          mountPath: /run/flannel
+        - name: flannel-cfg
+          mountPath: /etc/kube-flannel/
+      volumes:
+        - name: run
+          hostPath:
+            path: /run/flannel
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: flannel-cfg
+          configMap:
+            name: kube-flannel-cfg
+---

traefik/rbac/clusterrole.yaml

@@ -1,5 +1,3 @@
----
-# Source: traefik/templates/rbac/clusterrole.yaml
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -12,12 +10,19 @@ rules:
       - ""
     resources:
       - services
-      - endpoints
       - secrets
+      - nodes
     verbs:
       - get
       - list
       - watch
+  - apiGroups:
+      - discovery.k8s.io
+    resources:
+      - endpointslices
+    verbs:
+      - list
+      - watch
   - apiGroups:
       - extensions
       - networking.k8s.io
@@ -36,17 +41,18 @@ rules:
     verbs:
       - update
   - apiGroups:
-      - traefik.containo.us
+      - traefik.io
     resources:
-      - ingressroutes
-      - ingressroutetcps
-      - ingressrouteudps
       - middlewares
       - middlewaretcps
+      - ingressroutes
+      - traefikservices
+      - ingressroutetcps
+      - ingressrouteudps
       - tlsoptions
       - tlsstores
-      - traefikservices
       - serverstransports
+      - serverstransporttcps
     verbs:
       - get
       - list

traefik/rbac/clusterrolebinding.yaml

@@ -14,5 +14,3 @@ roleRef:
 subjects:
   - kind: ServiceAccount
     name: traefik
-    namespace: default
-

traefik/rbac/serviceaccount.yaml

@@ -5,5 +5,3 @@ metadata:
   labels:
     app.kubernetes.io/name: traefik
     app.kubernetes.io/instance: traefik
-  annotations:
-

traefik/traefik-external/deployment.yaml

@@ -1,5 +1,4 @@
 ---
-# Source: traefik/templates/deployment.yaml
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -7,9 +6,8 @@ metadata:
   labels:
     app.kubernetes.io/name: traefik
     app.kubernetes.io/instance: traefik
-  annotations:
 spec:
-  replicas: 1
+  replicas: 2
   selector:
     matchLabels:
       app.kubernetes.io/name: traefik
@@ -30,11 +28,18 @@ spec:
         app.kubernetes.io/name: traefik
         app.kubernetes.io/instance: traefik
     spec:
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: traefik
       serviceAccountName: traefik
       terminationGracePeriodSeconds: 60
-      hostNetwork: false
+      hostNetwork: true
       containers:
-      - image: "traefik:2.8.4"
+      - image: "traefik:3.2.1"
         imagePullPolicy: IfNotPresent
         name: traefik
         resources:
@@ -89,13 +94,20 @@ spec:
           - name: tmp
             mountPath: /tmp
         args:
-          - "--global.checknewversion"
           - "--entrypoints.admin.address=:8080/tcp"
           - "--entrypoints.metrics.address=:9100/tcp"
           - "--entrypoints.minecrafttcp.address=:25565/tcp"
           - "--entrypoints.traefik.address=:9000/tcp"
           - "--entrypoints.web.address=:8000/tcp"
           - "--entrypoints.websecure.address=:8443/tcp"
+          #- "--entryPoints.web.proxyProtocol.insecure=true"
+          #- "--entryPoints.web.forwardedHeaders.insecure=true"
+          - "--entryPoints.web.proxyProtocol.trustedIPs=10.20.0.0/24,51.15.80.73/32,192.168.113.0/24"
+          - "--entryPoints.web.forwardedHeaders.trustedIPs=10.20.0.0/24,51.15.80.73/32,192.168.113.0/24"
+          #- "--entryPoints.websecure.proxyProtocol.insecure=true"
+          #- "--entryPoints.websecure.forwardedHeaders.insecure=true"
+          - "--entryPoints.websecure.proxyProtocol.trustedIPs=10.20.0.0/24,51.15.80.73/32,192.168.113.0/24"
+          - "--entryPoints.websecure.forwardedHeaders.trustedIPs=10.20.0.0/24,51.15.80.73/32,192.168.113.0/24"
           - "--api.dashboard=true"
           - "--ping=true"
           - "--metrics.prometheus=true"
@@ -108,8 +120,7 @@ spec:
           - "--accesslog=true"
           - "--entrypoints.websecure.http.tls=true"
           - "--entrypoints.websecure.http.tls.certresolver=letsencrypt"
-          - "--entrypoints.websecure.http.tls.domains[0].main=beta.halia.dev"
+          - "--entrypoints.websecure.http.tls.domains[0].sans=*.halis.io"
-          - "--entrypoints.websecure.http.tls.domains[0].sans=*.beta.halia.dev"
           - "--certificatesresolvers.letsencrypt.acme.tlschallenge=true"
           - "--certificatesresolvers.letsencrypt.acme.dnschallenge=true"
           - "--certificatesresolvers.letsencrypt.acme.dnschallenge.provider=ovh"

traefik/traefik-external/kustomization.yaml

@@ -5,4 +5,5 @@ resources:
   - pvc.yaml
   - deployment.yaml
   - service.yaml
-  - servicemonitor.yaml
+  - loadbalancer.yaml
+  #- servicemonitor.yaml

traefik/traefik-external/loadbalancer.yaml

@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: traefik-external
+  labels:
+    app.kubernetes.io/name: traefik-external
+    app.kubernetes.io/instance: traefik-external
+spec:
+  type: LoadBalancer
+  loadBalancerIP: 51.15.80.73
+  loadBalancerSourceRanges:
+    - 0.0.0.0/0
+  selector:
+    app.kubernetes.io/name: traefik
+    app.kubernetes.io/instance: traefik
+  ports:
+  - port: 25565
+    name: minecrafttcp
+    targetPort: "minecrafttcp"
+    protocol: TCP
+  - port: 80
+    name: web
+    targetPort: "web"
+    protocol: TCP
+  - port: 443
+    name: websecure
+    targetPort: "websecure"
+    protocol: TCP

traefik/traefik-external/pvc.yaml

@@ -1,18 +1,15 @@
 ---
-# Source: traefik/templates/pvc.yaml
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
   name: traefik
-  annotations:
-    helm.sh/resource-policy: keep
   labels:
     app.kubernetes.io/name: traefik
     app.kubernetes.io/instance: traefik
 spec:
   accessModes:
-    - "ReadWriteOnce"
+    - "ReadWriteMany"
   resources:
     requests:
       storage: "128Mi"
-  storageClassName: "local-path"
+  storageClassName: "redundant-storage-class"

traefik/traefik-external/service-dashboard.yaml

@@ -2,7 +2,6 @@ apiVersion: v1
 kind: Service
 metadata:
     name: traefik-dashboard-svc
-    namespace: default
 spec:
     ports:
         - name: admin

traefik/traefik-external/service.yaml

@@ -1,32 +1,16 @@
----
-# Source: traefik/templates/service.yaml
 apiVersion: v1
 kind: Service
 metadata:
-  name: traefik-external
+  name: traefik-external-metrics
   labels:
     app.kubernetes.io/name: traefik-external
     app.kubernetes.io/instance: traefik-external
   annotations:
 spec:
-  type: LoadBalancer
-  loadBalancerIP: 51.15.80.73
   selector:
     app.kubernetes.io/name: traefik
     app.kubernetes.io/instance: traefik
   ports:
-  - port: 25565
-    name: minecrafttcp
-    targetPort: "minecrafttcp"
-    protocol: TCP
-  - port: 80 # Change port here to accomodate for internal only services
-    name: web
-    targetPort: "web"
-    protocol: TCP
-  - port: 443
-    name: websecure
-    targetPort: "websecure"
-    protocol: TCP
   - port: 9100
     name: metrics
     targetPort: "metrics"

traefik/traefik-external/servicemonitor.yaml

@@ -2,8 +2,7 @@ apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
   name: traefik-external
-  labels:
+  namespace: traefik
-    team: core
 spec:
   selector:
     matchLabels:

traefik/traefik-internal/config-map-dev.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: general-purpose-dev-config
+
+data:
+  lb-ip: 10.10.0.32

traefik/traefik-internal/deployment.yaml

@@ -1,5 +1,4 @@
 ---
-# Source: traefik/templates/deployment.yaml
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -7,9 +6,8 @@ metadata:
   labels:
     app.kubernetes.io/name: traefik-inter
     app.kubernetes.io/instance: traefik-inter
-  annotations:
 spec:
-  replicas: 1
+  replicas: 2
   selector:
     matchLabels:
       app.kubernetes.io/name: traefik-inter
@@ -30,11 +28,18 @@ spec:
         app.kubernetes.io/name: traefik-inter
         app.kubernetes.io/instance: traefik-inter
     spec:
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: traefik-inter
       serviceAccountName: traefik
       terminationGracePeriodSeconds: 60
       hostNetwork: false
       containers:
-      - image: "traefik:2.8.4"
+      - image: "traefik:3.2.1"
         imagePullPolicy: IfNotPresent
         name: traefik-inter
         resources:
@@ -92,6 +97,10 @@ spec:
           - "--entrypoints.traefik.address=:9000/tcp"
           - "--entrypoints.web.address=:8000/tcp"
           - "--entrypoints.websecure.address=:8443/tcp"
+          - "--entryPoints.web.proxyProtocol.insecure"
+          - "--entryPoints.web.forwardedHeaders.insecure"
+          - "--entryPoints.websecure.proxyProtocol.insecure"
+          - "--entryPoints.websecure.forwardedHeaders.insecure"
           - "--api.dashboard=true"
           - "--api.insecure=true"
           - "--ping=true"

traefik/traefik-internal/kustomization.yaml

@@ -5,4 +5,5 @@ resources:
   - pvc.yaml
   - deployment.yaml
   - service.yaml
-  - servicemonitor.yaml
+  - loadbalancer.yaml
+  #- servicemonitor.yaml

traefik/traefik-internal/loadbalancer.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: traefik-internal
+  labels:
+    app.kubernetes.io/name: traefik-inter
+    app.kubernetes.io/instance: traefik-inter
+spec:
+  type: LoadBalancer
+  loadBalancerIP: 10.10.0.16
+  selector:
+    app.kubernetes.io/name: traefik-inter
+    app.kubernetes.io/instance: traefik-inter
+  ports:
+  - port: 80
+    name: web
+    targetPort: "web"
+    protocol: TCP
+  - port: 443
+    name: websecure
+    targetPort: "websecure"
+    protocol: TCP

traefik/traefik-internal/pvc.yaml

@@ -1,18 +1,15 @@
 ---
-# Source: traefik/templates/pvc.yaml
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
   name: traefik-inter
-  annotations:
-    helm.sh/resource-policy: keep
   labels:
     app.kubernetes.io/name: traefik-inter
     app.kubernetes.io/instance: traefik-inter
 spec:
   accessModes:
-    - "ReadWriteOnce"
+    - "ReadWriteMany"
   resources:
     requests:
       storage: "128Mi"
-  storageClassName: "local-path"
+  storageClassName: "redundant-storage-class"

traefik/traefik-internal/service.yaml

@@ -1,26 +1,15 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: traefik-internal
+  name: traefik-internal-metrics
-  namespace: default
   labels:
     app.kubernetes.io/name: traefik-inter
     app.kubernetes.io/instance: traefik-inter
 spec:
-  type: LoadBalancer
-  loadBalancerIP: 192.168.56.101
   selector:
     app.kubernetes.io/name: traefik-inter
     app.kubernetes.io/instance: traefik-inter
   ports:
-  - port: 80
-    name: web
-    targetPort: "web"
-    protocol: TCP
-  - port: 443
-    name: websecure
-    targetPort: "websecure"
-    protocol: TCP
   - port: 9100
     name: metrics
     targetPort: "metrics"

traefik/traefik-internal/servicemonitor.yaml

@@ -2,12 +2,12 @@ apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
   name: traefik-inter
-  labels:
+  namespace: traefik
-    team: core
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: traefik-inter
+      app.kubernetes.io/instance: traefik-inter
   endpoints:
     - port: metrics
       path: /metrics