From 0a4137368872abbdf99c558e3f08241933ab1cd0 Mon Sep 17 00:00:00 2001 From: Tanguy Herbron Date: Sun, 20 Nov 2022 03:38:22 +0100 Subject: [PATCH] feat(ingress): Add additional traefik for internal only access A new traefik deployment has been added, and two ingressclasses have been set for the new instance, as well as the previous one. This allows the network to be split in two, one for external access and another for internal access. Each traefik deployment is connected to a loadbalancer requesting the IP necessary for each type of access. --- README.md | 15 +- traefik/{rbac => 0-rbac}/clusterrole.yaml | 0 .../{rbac => 0-rbac}/clusterrolebinding.yaml | 0 traefik/{rbac => 0-rbac}/serviceaccount.yaml | 0 .../{ => 1-traefik-external}/deployment.yaml | 0 traefik/{ => 1-traefik-external}/pvc.yaml | 0 .../service.yaml} | 2 +- traefik/1-traefik-internal/deployment.yaml | 146 ++++++++++++++++++ traefik/1-traefik-internal/pvc.yaml | 18 +++ .../service.yaml} | 14 +- traefik/dashboard-hook-ingressroute.yaml | 22 --- traefik/dashboard.yaml | 13 -- 12 files changed, 181 insertions(+), 49 deletions(-) rename traefik/{rbac => 0-rbac}/clusterrole.yaml (100%) rename traefik/{rbac => 0-rbac}/clusterrolebinding.yaml (100%) rename traefik/{rbac => 0-rbac}/serviceaccount.yaml (100%) rename traefik/{ => 1-traefik-external}/deployment.yaml (100%) rename traefik/{ => 1-traefik-external}/pvc.yaml (100%) rename traefik/{service-external.yaml => 1-traefik-external/service.yaml} (91%) create mode 100644 traefik/1-traefik-internal/deployment.yaml create mode 100644 traefik/1-traefik-internal/pvc.yaml rename traefik/{service-internal.yaml => 1-traefik-internal/service.yaml} (58%) delete mode 100644 traefik/dashboard-hook-ingressroute.yaml delete mode 100644 traefik/dashboard.yaml diff --git a/README.md b/README.md index ccc3e78..30a709a 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ # K3s cluster +/!\ CHECK PORT RULES FOR EXPOSING INTERNAL SERVICES + | Name | Usage | Accessibility | Host | DB type | Additional data | Backup configuration | Loki integration | Prometheus integration | Status | |-------------------------|--------------------------------------|---------------|--------------|------------|----------------------|----------------------|------------------|------------------------|-----------------------------------| -| Traefik | Reverse proxy and load balancer | Public* | Socrates | - | - | - | Configured | Configured | Completed | +| Traefik | Reverse proxy and load balancer | Public* | Socrates | - | - | - | Configured | Configured | Completed5 | | Vaultwarden | Password manager | Public | Pythagoras-b | MariaDB | - | 4AM K8s CronJob | Configured | Not available | Completed | | Gitlab | Version control system | Public | Pythagoras-b | PostgreSQL | User created content | 5AM internal CronJob | Configured | Configured | Completed4 | | Prometheus | Metrics aggregator | Private | Pythagoras-b | TBD | - | Not configured | Configured | Configured | Partial | @@ -32,6 +34,7 @@ 2 Missing configuration for NAS volume mounting (over network)
3 Missing Longhorn scheduling for saving media_store and secret management
4 Backup management is not handled by k3s but by an internal cronjob rule (Change image name when putting to production)
+5 Missing dashboard configuration
## Backup management ### Databases @@ -57,7 +60,7 @@ longhorn - Change host/deployment specific variables to use environment variables - Write CI/CD pipeline to create environment loaded files - Write CI/CD pipeline to deploy cluster -- ~~Setup internal traefik with nodeport as reverse proxy for internal only services~~ Done through internal LB +- ~~Setup internal traefik with nodeport as reverse proxy for internal only services~~ Done through double ingress class and LB - ~~Setup DB container sidecars for automated backups to Longhorn volume~~ - Setup secrets configuration through CI/CD variable injection - Explore permission issues when issuing OVH API keys (not working for wildcard and `beta.halia.dev` subdomain) @@ -103,5 +106,11 @@ This ansible script could create one (or more) additional client(s) depending on Cf : https://docs.k3s.io/advanced#auto-deploying-manifests ### Development domains -To access a service publicly when developping, the domain name should be *.beta.halia.dev +To access a service publicly when developing, the domain name should be *.beta.halia.dev To only expose a service internally, the domain name should be *.k3s.beta + +### Ingresses +To split between external and internal services, two traefik ingresses are implemented through the `ingressclass` annotation. +`traefik-external` will only allow external access to a given service, while `traefik-internal` restrict to an internal only access. + + diff --git a/traefik/rbac/clusterrole.yaml b/traefik/0-rbac/clusterrole.yaml similarity index 100% rename from traefik/rbac/clusterrole.yaml rename to traefik/0-rbac/clusterrole.yaml diff --git a/traefik/rbac/clusterrolebinding.yaml b/traefik/0-rbac/clusterrolebinding.yaml similarity index 100% rename from traefik/rbac/clusterrolebinding.yaml rename to traefik/0-rbac/clusterrolebinding.yaml diff --git a/traefik/rbac/serviceaccount.yaml b/traefik/0-rbac/serviceaccount.yaml similarity index 100% rename from traefik/rbac/serviceaccount.yaml rename to traefik/0-rbac/serviceaccount.yaml diff --git a/traefik/deployment.yaml b/traefik/1-traefik-external/deployment.yaml similarity index 100% rename from traefik/deployment.yaml rename to traefik/1-traefik-external/deployment.yaml diff --git a/traefik/pvc.yaml b/traefik/1-traefik-external/pvc.yaml similarity index 100% rename from traefik/pvc.yaml rename to traefik/1-traefik-external/pvc.yaml diff --git a/traefik/service-external.yaml b/traefik/1-traefik-external/service.yaml similarity index 91% rename from traefik/service-external.yaml rename to traefik/1-traefik-external/service.yaml index d1c17b6..6830721 100644 --- a/traefik/service-external.yaml +++ b/traefik/1-traefik-external/service.yaml @@ -26,7 +26,7 @@ items: name: minecrafttcp targetPort: "minecrafttcp" protocol: TCP - - port: 80 + - port: 80 # Change port here to accomodate for internal only services name: web targetPort: "web" protocol: TCP diff --git a/traefik/1-traefik-internal/deployment.yaml b/traefik/1-traefik-internal/deployment.yaml new file mode 100644 index 0000000..c1893df --- /dev/null +++ b/traefik/1-traefik-internal/deployment.yaml @@ -0,0 +1,146 @@ +--- +# Source: traefik/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: traefik-inter + labels: + app.kubernetes.io/name: traefik-inter + app.kubernetes.io/instance: traefik-inter + annotations: +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: traefik-inter + app.kubernetes.io/instance: traefik-inter + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + minReadySeconds: 0 + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/path: "/metrics" + prometheus.io/port: "9100" + labels: + app.kubernetes.io/name: traefik-inter + app.kubernetes.io/instance: traefik-inter + spec: + serviceAccountName: traefik + terminationGracePeriodSeconds: 60 + hostNetwork: false + containers: + - image: "traefik:2.8.4" + imagePullPolicy: IfNotPresent + name: traefik-inter + resources: + readinessProbe: + httpGet: + path: /ping + port: 9000 + failureThreshold: 1 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + livenessProbe: + httpGet: + path: /ping + port: 9000 + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + ports: + - name: "admin" + containerPort: 8080 + protocol: "TCP" + - name: "metrics" + containerPort: 9100 + protocol: "TCP" + - name: "traefik" + containerPort: 9000 + protocol: "TCP" + - name: "web" + containerPort: 8000 + protocol: "TCP" + - name: "websecure" + containerPort: 8443 + protocol: "TCP" + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + volumeMounts: + - name: data + mountPath: /certs + - name: tmp + mountPath: /tmp + args: + - "--global.checknewversion" + - "--entrypoints.admin.address=:8080/tcp" + - "--entrypoints.metrics.address=:9100/tcp" + - "--entrypoints.traefik.address=:9000/tcp" + - "--entrypoints.web.address=:8000/tcp" + - "--entrypoints.websecure.address=:8443/tcp" + - "--api.dashboard=true" + - "--api.insecure=true" + - "--ping=true" + - "--metrics.prometheus=true" + - "--metrics.prometheus.entrypoint=metrics" + - "--providers.kubernetescrd" + - "--providers.kubernetesingress" + - "--providers.kubernetescrd.ingressclass=traefik-inter" + - "--providers.kubernetesingress.ingressclass=traefik-inter" + - "--entrypoints.web.http.redirections.entryPoint.to=:443" + - "--entrypoints.web.http.redirections.entryPoint.scheme=https" + - "--log.level=DEBUG" + - "--accesslog=true" + - "--entrypoints.websecure.http.tls=true" + - "--entrypoints.websecure.http.tls.certresolver=letsencrypt" + - "--entrypoints.websecure.http.tls.domains[0].main=k3s.beta" + - "--entrypoints.websecure.http.tls.domains[0].sans=*.k3s.beta" + - "--certificatesresolvers.letsencrypt.acme.tlschallenge=true" + - "--certificatesresolvers.letsencrypt.acme.dnschallenge=true" + - "--certificatesresolvers.letsencrypt.acme.dnschallenge.provider=ovh" + - "--certificatesresolvers.letsencrypt.acme.dnschallenge.resolvers=1.1.1.1" + - "--certificatesresolvers.letsencrypt.acme.email=tanguy.herbron@outlook.com" + - "--certificatesresolvers.letsencrypt.acme.storage=/certs/acme.json" + env: + - name: OVH_APPLICATION_KEY + valueFrom: + secretKeyRef: + key: appKey + name: ovh-api-credentials + - name: OVH_APPLICATION_SECRET + valueFrom: + secretKeyRef: + key: appSecret + name: ovh-api-credentials + - name: OVH_CONSUMER_KEY + valueFrom: + secretKeyRef: + key: consumerKey + name: ovh-api-credentials + - name: OVH_ENDPOINT + valueFrom: + secretKeyRef: + key: endpoint + name: ovh-api-credentials + volumes: + - name: data + persistentVolumeClaim: + claimName: traefik-inter + - name: tmp + emptyDir: {} + securityContext: + fsGroup: 65532 diff --git a/traefik/1-traefik-internal/pvc.yaml b/traefik/1-traefik-internal/pvc.yaml new file mode 100644 index 0000000..d4b1360 --- /dev/null +++ b/traefik/1-traefik-internal/pvc.yaml @@ -0,0 +1,18 @@ +--- +# Source: traefik/templates/pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: traefik-inter + annotations: + helm.sh/resource-policy: keep + labels: + app.kubernetes.io/name: traefik-inter + app.kubernetes.io/instance: traefik-inter +spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "128Mi" + storageClassName: "local-path" diff --git a/traefik/service-internal.yaml b/traefik/1-traefik-internal/service.yaml similarity index 58% rename from traefik/service-internal.yaml rename to traefik/1-traefik-internal/service.yaml index 8a67e6d..793e10e 100644 --- a/traefik/service-internal.yaml +++ b/traefik/1-traefik-internal/service.yaml @@ -10,22 +10,16 @@ items: metadata: name: traefik-internal labels: - app.kubernetes.io/name: traefik-internal - helm.sh/chart: traefik-10.24.2 - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/instance: traefik + app.kubernetes.io/name: traefik-inter + app.kubernetes.io/instance: traefik-inter annotations: spec: type: LoadBalancer loadBalancerIP: 10.10.0.64 selector: - app.kubernetes.io/name: traefik - app.kubernetes.io/instance: traefik + app.kubernetes.io/name: traefik-inter + app.kubernetes.io/instance: traefik-inter ports: - - port: 25565 - name: minecrafttcp - targetPort: "minecrafttcp" - protocol: TCP - port: 80 name: web targetPort: "web" diff --git a/traefik/dashboard-hook-ingressroute.yaml b/traefik/dashboard-hook-ingressroute.yaml deleted file mode 100644 index 858b5c3..0000000 --- a/traefik/dashboard-hook-ingressroute.yaml +++ /dev/null @@ -1,22 +0,0 @@ ---- -# Source: traefik/templates/dashboard-hook-ingressroute.yaml -apiVersion: traefik.containo.us/v1alpha1 -kind: IngressRoute -metadata: - name: traefik-dashboard - annotations: - helm.sh/hook: "post-install,post-upgrade" - labels: - app.kubernetes.io/name: traefik - helm.sh/chart: traefik-10.24.2 - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/instance: traefik -spec: - entryPoints: - - traefik - routes: - - match: PathPrefix(`/dashboard`) || PathPrefix(`/api`) - kind: Rule - services: - - name: api@internal - kind: TraefikService diff --git a/traefik/dashboard.yaml b/traefik/dashboard.yaml deleted file mode 100644 index ce18f71..0000000 --- a/traefik/dashboard.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: traefik.containo.us/v1alpha1 -kind: IngressRoute -metadata: - name: traefik-dashboard -spec: - entryPoints: - - websecure - routes: - - kind: Rule - match: Host(`traefik.k3s.beta`) - services: - - name: api@internal - kind: TraefikService