diff --git a/README.md b/README.md index ccc3e78..30a709a 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ # K3s cluster +/!\ CHECK PORT RULES FOR EXPOSING INTERNAL SERVICES + | Name | Usage | Accessibility | Host | DB type | Additional data | Backup configuration | Loki integration | Prometheus integration | Status | |-------------------------|--------------------------------------|---------------|--------------|------------|----------------------|----------------------|------------------|------------------------|-----------------------------------| -| Traefik | Reverse proxy and load balancer | Public* | Socrates | - | - | - | Configured | Configured | Completed | +| Traefik | Reverse proxy and load balancer | Public* | Socrates | - | - | - | Configured | Configured | Completed5 | | Vaultwarden | Password manager | Public | Pythagoras-b | MariaDB | - | 4AM K8s CronJob | Configured | Not available | Completed | | Gitlab | Version control system | Public | Pythagoras-b | PostgreSQL | User created content | 5AM internal CronJob | Configured | Configured | Completed4 | | Prometheus | Metrics aggregator | Private | Pythagoras-b | TBD | - | Not configured | Configured | Configured | Partial | @@ -32,6 +34,7 @@ 2 Missing configuration for NAS volume mounting (over network)
3 Missing Longhorn scheduling for saving media_store and secret management
4 Backup management is not handled by k3s but by an internal cronjob rule (Change image name when putting to production)
+5 Missing dashboard configuration
## Backup management ### Databases @@ -57,7 +60,7 @@ longhorn - Change host/deployment specific variables to use environment variables - Write CI/CD pipeline to create environment loaded files - Write CI/CD pipeline to deploy cluster -- ~~Setup internal traefik with nodeport as reverse proxy for internal only services~~ Done through internal LB +- ~~Setup internal traefik with nodeport as reverse proxy for internal only services~~ Done through double ingress class and LB - ~~Setup DB container sidecars for automated backups to Longhorn volume~~ - Setup secrets configuration through CI/CD variable injection - Explore permission issues when issuing OVH API keys (not working for wildcard and `beta.halia.dev` subdomain) @@ -103,5 +106,11 @@ This ansible script could create one (or more) additional client(s) depending on Cf : https://docs.k3s.io/advanced#auto-deploying-manifests ### Development domains -To access a service publicly when developping, the domain name should be *.beta.halia.dev +To access a service publicly when developing, the domain name should be *.beta.halia.dev To only expose a service internally, the domain name should be *.k3s.beta + +### Ingresses +To split between external and internal services, two traefik ingresses are implemented through the `ingressclass` annotation. +`traefik-external` will only allow external access to a given service, while `traefik-internal` restrict to an internal only access. + + diff --git a/traefik/rbac/clusterrole.yaml b/traefik/0-rbac/clusterrole.yaml similarity index 100% rename from traefik/rbac/clusterrole.yaml rename to traefik/0-rbac/clusterrole.yaml diff --git a/traefik/rbac/clusterrolebinding.yaml b/traefik/0-rbac/clusterrolebinding.yaml similarity index 100% rename from traefik/rbac/clusterrolebinding.yaml rename to traefik/0-rbac/clusterrolebinding.yaml diff --git a/traefik/rbac/serviceaccount.yaml b/traefik/0-rbac/serviceaccount.yaml similarity index 100% rename from traefik/rbac/serviceaccount.yaml rename to traefik/0-rbac/serviceaccount.yaml diff --git a/traefik/deployment.yaml b/traefik/1-traefik-external/deployment.yaml similarity index 100% rename from traefik/deployment.yaml rename to traefik/1-traefik-external/deployment.yaml diff --git a/traefik/pvc.yaml b/traefik/1-traefik-external/pvc.yaml similarity index 100% rename from traefik/pvc.yaml rename to traefik/1-traefik-external/pvc.yaml diff --git a/traefik/service-external.yaml b/traefik/1-traefik-external/service.yaml similarity index 91% rename from traefik/service-external.yaml rename to traefik/1-traefik-external/service.yaml index d1c17b6..6830721 100644 --- a/traefik/service-external.yaml +++ b/traefik/1-traefik-external/service.yaml @@ -26,7 +26,7 @@ items: name: minecrafttcp targetPort: "minecrafttcp" protocol: TCP - - port: 80 + - port: 80 # Change port here to accomodate for internal only services name: web targetPort: "web" protocol: TCP diff --git a/traefik/1-traefik-internal/deployment.yaml b/traefik/1-traefik-internal/deployment.yaml new file mode 100644 index 0000000..c1893df --- /dev/null +++ b/traefik/1-traefik-internal/deployment.yaml @@ -0,0 +1,146 @@ +--- +# Source: traefik/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: traefik-inter + labels: + app.kubernetes.io/name: traefik-inter + app.kubernetes.io/instance: traefik-inter + annotations: +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: traefik-inter + app.kubernetes.io/instance: traefik-inter + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + minReadySeconds: 0 + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/path: "/metrics" + prometheus.io/port: "9100" + labels: + app.kubernetes.io/name: traefik-inter + app.kubernetes.io/instance: traefik-inter + spec: + serviceAccountName: traefik + terminationGracePeriodSeconds: 60 + hostNetwork: false + containers: + - image: "traefik:2.8.4" + imagePullPolicy: IfNotPresent + name: traefik-inter + resources: + readinessProbe: + httpGet: + path: /ping + port: 9000 + failureThreshold: 1 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + livenessProbe: + httpGet: + path: /ping + port: 9000 + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + ports: + - name: "admin" + containerPort: 8080 + protocol: "TCP" + - name: "metrics" + containerPort: 9100 + protocol: "TCP" + - name: "traefik" + containerPort: 9000 + protocol: "TCP" + - name: "web" + containerPort: 8000 + protocol: "TCP" + - name: "websecure" + containerPort: 8443 + protocol: "TCP" + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + volumeMounts: + - name: data + mountPath: /certs + - name: tmp + mountPath: /tmp + args: + - "--global.checknewversion" + - "--entrypoints.admin.address=:8080/tcp" + - "--entrypoints.metrics.address=:9100/tcp" + - "--entrypoints.traefik.address=:9000/tcp" + - "--entrypoints.web.address=:8000/tcp" + - "--entrypoints.websecure.address=:8443/tcp" + - "--api.dashboard=true" + - "--api.insecure=true" + - "--ping=true" + - "--metrics.prometheus=true" + - "--metrics.prometheus.entrypoint=metrics" + - "--providers.kubernetescrd" + - "--providers.kubernetesingress" + - "--providers.kubernetescrd.ingressclass=traefik-inter" + - "--providers.kubernetesingress.ingressclass=traefik-inter" + - "--entrypoints.web.http.redirections.entryPoint.to=:443" + - "--entrypoints.web.http.redirections.entryPoint.scheme=https" + - "--log.level=DEBUG" + - "--accesslog=true" + - "--entrypoints.websecure.http.tls=true" + - "--entrypoints.websecure.http.tls.certresolver=letsencrypt" + - "--entrypoints.websecure.http.tls.domains[0].main=k3s.beta" + - "--entrypoints.websecure.http.tls.domains[0].sans=*.k3s.beta" + - "--certificatesresolvers.letsencrypt.acme.tlschallenge=true" + - "--certificatesresolvers.letsencrypt.acme.dnschallenge=true" + - "--certificatesresolvers.letsencrypt.acme.dnschallenge.provider=ovh" + - "--certificatesresolvers.letsencrypt.acme.dnschallenge.resolvers=1.1.1.1" + - "--certificatesresolvers.letsencrypt.acme.email=tanguy.herbron@outlook.com" + - "--certificatesresolvers.letsencrypt.acme.storage=/certs/acme.json" + env: + - name: OVH_APPLICATION_KEY + valueFrom: + secretKeyRef: + key: appKey + name: ovh-api-credentials + - name: OVH_APPLICATION_SECRET + valueFrom: + secretKeyRef: + key: appSecret + name: ovh-api-credentials + - name: OVH_CONSUMER_KEY + valueFrom: + secretKeyRef: + key: consumerKey + name: ovh-api-credentials + - name: OVH_ENDPOINT + valueFrom: + secretKeyRef: + key: endpoint + name: ovh-api-credentials + volumes: + - name: data + persistentVolumeClaim: + claimName: traefik-inter + - name: tmp + emptyDir: {} + securityContext: + fsGroup: 65532 diff --git a/traefik/1-traefik-internal/pvc.yaml b/traefik/1-traefik-internal/pvc.yaml new file mode 100644 index 0000000..d4b1360 --- /dev/null +++ b/traefik/1-traefik-internal/pvc.yaml @@ -0,0 +1,18 @@ +--- +# Source: traefik/templates/pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: traefik-inter + annotations: + helm.sh/resource-policy: keep + labels: + app.kubernetes.io/name: traefik-inter + app.kubernetes.io/instance: traefik-inter +spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "128Mi" + storageClassName: "local-path" diff --git a/traefik/service-internal.yaml b/traefik/1-traefik-internal/service.yaml similarity index 58% rename from traefik/service-internal.yaml rename to traefik/1-traefik-internal/service.yaml index 8a67e6d..793e10e 100644 --- a/traefik/service-internal.yaml +++ b/traefik/1-traefik-internal/service.yaml @@ -10,22 +10,16 @@ items: metadata: name: traefik-internal labels: - app.kubernetes.io/name: traefik-internal - helm.sh/chart: traefik-10.24.2 - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/instance: traefik + app.kubernetes.io/name: traefik-inter + app.kubernetes.io/instance: traefik-inter annotations: spec: type: LoadBalancer loadBalancerIP: 10.10.0.64 selector: - app.kubernetes.io/name: traefik - app.kubernetes.io/instance: traefik + app.kubernetes.io/name: traefik-inter + app.kubernetes.io/instance: traefik-inter ports: - - port: 25565 - name: minecrafttcp - targetPort: "minecrafttcp" - protocol: TCP - port: 80 name: web targetPort: "web" diff --git a/traefik/dashboard-hook-ingressroute.yaml b/traefik/dashboard-hook-ingressroute.yaml deleted file mode 100644 index 858b5c3..0000000 --- a/traefik/dashboard-hook-ingressroute.yaml +++ /dev/null @@ -1,22 +0,0 @@ ---- -# Source: traefik/templates/dashboard-hook-ingressroute.yaml -apiVersion: traefik.containo.us/v1alpha1 -kind: IngressRoute -metadata: - name: traefik-dashboard - annotations: - helm.sh/hook: "post-install,post-upgrade" - labels: - app.kubernetes.io/name: traefik - helm.sh/chart: traefik-10.24.2 - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/instance: traefik -spec: - entryPoints: - - traefik - routes: - - match: PathPrefix(`/dashboard`) || PathPrefix(`/api`) - kind: Rule - services: - - name: api@internal - kind: TraefikService diff --git a/traefik/dashboard.yaml b/traefik/dashboard.yaml deleted file mode 100644 index ce18f71..0000000 --- a/traefik/dashboard.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: traefik.containo.us/v1alpha1 -kind: IngressRoute -metadata: - name: traefik-dashboard -spec: - entryPoints: - - websecure - routes: - - kind: Rule - match: Host(`traefik.k3s.beta`) - services: - - name: api@internal - kind: TraefikService