From 8a67ac83c9931d8a5d6ad1f9be5aacabc2dc7715 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 3 Jan 2026 16:11:33 +0100 Subject: [PATCH 01/57] feat: add hks chart Signed-off-by: Henrik Gerdes --- .../hks/.helmignore | 23 ++ .../hegerdes-kubernetes-sevice/hks/Chart.yaml | 24 ++ .../hks/templates/_helpers.tpl | 62 +++++ .../hks/templates/eso.yaml | 68 +++++ .../hks/templates/kube-apiserver.yaml | 249 +++++++++++++++++ .../hks/templates/kube-certs-core.yaml | 203 ++++++++++++++ .../hks/templates/kube-certs-etcd.yaml | 118 ++++++++ .../hks/templates/kube-certs-util.yaml | 32 +++ .../hks/templates/kube-confs.yaml | 172 ++++++++++++ .../templates/kube-controller-manager.yaml | 163 +++++++++++ .../hks/templates/kube-etcd.yaml | 257 ++++++++++++++++++ .../hks/templates/kube-init-ca.yml | 233 ++++++++++++++++ .../hks/templates/kube-issuer.yml | 23 ++ .../hks/templates/kube-scheduler.yaml | 133 +++++++++ .../hks/values.yaml | 0 15 files changed, 1760 insertions(+) create mode 100644 k8s-apps/hegerdes-kubernetes-sevice/hks/.helmignore create mode 100644 k8s-apps/hegerdes-kubernetes-sevice/hks/Chart.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-sevice/hks/templates/_helpers.tpl create mode 100644 k8s-apps/hegerdes-kubernetes-sevice/hks/templates/eso.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-apiserver.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-certs-core.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-certs-etcd.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-certs-util.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-confs.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-controller-manager.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-etcd.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-init-ca.yml create mode 100644 k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-issuer.yml create mode 100644 k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-scheduler.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-sevice/hks/values.yaml diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/.helmignore b/k8s-apps/hegerdes-kubernetes-sevice/hks/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-sevice/hks/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/Chart.yaml b/k8s-apps/hegerdes-kubernetes-sevice/hks/Chart.yaml new file mode 100644 index 0000000..2e01207 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-sevice/hks/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: hks +description: A Helm chart for deploying managed Kubernetes-Clusters + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.1.0" diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/_helpers.tpl b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/_helpers.tpl new file mode 100644 index 0000000..3e114b1 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "hks.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "hks.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "hks.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "hks.labels" -}} +helm.sh/chart: {{ include "hks.chart" . }} +{{ include "hks.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "hks.selectorLabels" -}} +app.kubernetes.io/name: {{ include "hks.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "hks.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "hks.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/eso.yaml b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/eso.yaml new file mode 100644 index 0000000..c0433d4 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/eso.yaml @@ -0,0 +1,68 @@ +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: kubeconfig-credentials-{{ .Release.Namespace }} + namespace: argocd +spec: + refreshInterval: 1h0m0s + secretStoreRef: + kind: SecretStore + name: secret-sync + target: + name: kubeconfig-credentials-{{ .Release.Namespace }} + template: + metadata: + labels: + argocd.argoproj.io/secret-type: cluster + hks.hegerdes.com/managed-cluster: "true" + hks.hegerdes.com/cluster-type: "downstream" + type: Opaque + data: + server: https://kube-apiserver.{{ .Release.Namespace }}:6443 + name: "{{ .Release.Namespace }}" + namespaces: kube-system,cilium,default,kubelet-serving-cert-approver + clusterResources: "true" + config: | + { + "tlsClientConfig": { + "caData": "{{ "{{ .ca | b64enc }}" }}", + "certData": "{{ "{{ .clientCert | b64enc }}" }}", + "keyData": "{{ "{{ .clientKey | b64enc }}" }}" + } + } + data: + - secretKey: ca + remoteRef: + # key: argocd-client-tls + key: kube-super-admin-client-tls + property: ca.crt + - secretKey: clientCert + remoteRef: + # key: argocd-client-tls + key: kube-super-admin-client-tls + property: tls.crt + - secretKey: clientKey + remoteRef: + # key: argocd-client-tls + key: kube-super-admin-client-tls + property: tls.key +--- +apiVersion: external-secrets.io/v1 +kind: SecretStore +metadata: + name: secret-sync-{{ .Release.Namespace }} + namespace: argocd +spec: + provider: + kubernetes: + auth: + serviceAccount: + name: external-secrets + remoteNamespace: "{{ .Release.Namespace }}" + server: + url: "https://kubernetes.default.svc.cluster.local" + caProvider: + type: ConfigMap + name: kube-root-ca.crt + key: ca.crt diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-apiserver.yaml b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-apiserver.yaml new file mode 100644 index 0000000..fd070dd --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-apiserver.yaml @@ -0,0 +1,249 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: kube-apiserver + component: kube-apiserver + tier: control-plane + name: kube-apiserver +spec: + replicas: 1 + selector: + matchLabels: + app: kube-apiserver + component: kube-apiserver + tier: control-plane + strategy: {} + template: + metadata: + labels: + app: kube-apiserver + component: kube-apiserver + tier: control-plane + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault + automountServiceAccountToken: false + enableServiceLinks: false + containers: + - name: kube-apiserver + image: registry.k8s.io/kube-apiserver:v1.35.0 + imagePullPolicy: IfNotPresent + command: + - kube-apiserver + - --secure-port=6443 + - --profiling=false + - --external-hostname={{ .Release.Namespace }}.hks.eu-central.hegerdes.com + - --enable-admission-plugins=NodeRestriction + - --enable-bootstrap-token-auth=true + - --allow-privileged=true + - --audit-log-maxage=30 + - --audit-log-maxbackup=10 + - --audit-log-maxsize=100 + - --audit-log-path=/var/log/kube-apiserver/audit.log + - --client-ca-file=/etc/kubernetes/pki/ca.crt + - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt + - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --audit-policy-file=/etc/kubernetes/extras/audit-policy.yaml + - --authentication-config=/etc/kubernetes/extras/authentication-conf.yaml + - --authorization-config=/etc/kubernetes/extras/authorization-conf.yaml + - --admission-control-config-file=/etc/kubernetes/extras/admission-control-config.yaml + - --encryption-provider-config=/etc/kubernetes/extras/encryption-config.yaml + - --encryption-provider-config-automatic-reload=true + - --etcd-servers=https://etcd:2379 + - --etcd-cafile=/etc/kubernetes/etcd/pki/ca.crt + - --etcd-certfile=/etc/kubernetes/etcd/pki/apiserver-etcd-client.crt + - --etcd-keyfile=/etc/kubernetes/etcd/pki/apiserver-etcd-client.key + - --service-account-issuer=https://kubernetes.default.svc.cluster.local + - --service-cluster-ip-range=10.96.0.0/16 + - --service-account-key-file=/etc/kubernetes/pki/sa.pub + - --service-account-signing-key-file=/etc/kubernetes/pki/sa.key + - --tls-min-version=VersionTLS13 + - --tls-cert-file=/etc/kubernetes/pki/apiserver.crt + - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key + - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt + - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key + - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt + - --requestheader-allowed-names=front-proxy-client + - --requestheader-extra-headers-prefix=X-Remote-Extra- + - --requestheader-group-headers=X-Remote-Group + - --requestheader-username-headers=X-Remote-User + resources: + requests: + cpu: 200m + memory: 200Mi + limits: + cpu: 2 + memory: 4Gi + ports: + - containerPort: 6443 + name: kube-api + protocol: TCP + securityContext: + readOnlyRootFilesystem: true + # capabilities: + # drop: [all] + + livenessProbe: + failureThreshold: 8 + httpGet: + path: /livez + port: kube-api + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 15 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: kube-api + scheme: HTTPS + periodSeconds: 1 + timeoutSeconds: 15 + startupProbe: + failureThreshold: 24 + httpGet: + path: /livez + port: kube-api + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 15 + volumeMounts: + - mountPath: /var/log/kube-apiserver/ + name: audit-log + - mountPath: /etc/kubernetes/pki/ + name: tls-kube + - mountPath: /etc/kubernetes/etcd/pki/ + name: tls-etcd + - mountPath: /etc/kubernetes/extras/ + name: kube-extra-conf + # - mountPath: /etc/ssl/certs + # name: ca-certs + # readOnly: true + # - mountPath: /etc/ca-certificates + # name: etc-ca-certificates + # readOnly: true + # - mountPath: /usr/local/share/ca-certificates + # name: usr-local-share-ca-certificates + # readOnly: true + # - mountPath: /usr/share/ca-certificates + # name: usr-share-ca-certificates + # readOnly: true + # priority: 2000001000 + # priorityClassName: system-node-critical + volumes: + - name: audit-log + persistentVolumeClaim: + claimName: kube-adit-log + - name: kube-extra-conf + projected: + sources: + - configMap: + name: kube-api-configs + items: + - key: admission-control-config.yaml + path: admission-control-config.yaml + - key: authorization-conf.yaml + path: authorization-conf.yaml + - key: authentication-conf.yaml + path: authentication-conf.yaml + - key: audit-policy.yaml + path: audit-policy.yaml + - secret: + name: kube-encryption + items: + - key: encryption-config.yaml + path: encryption-config.yaml + - name: tls-etcd + projected: + sources: + - secret: + name: kube-apiserver-etcd-client-tls + items: + - key: tls.key + path: apiserver-etcd-client.key + - key: tls.crt + path: apiserver-etcd-client.crt + - key: ca.crt + path: ca.crt + - name: tls-kube + projected: + sources: + # Serving cert for server + - secret: + name: kube-apiserver-tls + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: apiserver.crt + - key: tls.key + path: apiserver.key + # Client cert for kubelet auth + - secret: + name: kube-apiserver-kubelet-client-tls + items: + - key: tls.crt + path: apiserver-kubelet-client.crt + - key: tls.key + path: apiserver-kubelet-client.key + # Client cert for front-proxy auth + - secret: + name: kube-front-proxy-client-tls + items: + - key: tls.crt + path: front-proxy-client.crt + - key: tls.key + path: front-proxy-client.key + - key: ca.crt + path: front-proxy-ca.crt + # Sign and validate JWTs + - secret: + name: kube-sa-tls + items: + - key: tls.key + path: sa.key + - key: sa.pub + path: sa.pub +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: kube-apiserver + component: kube-apiserver + tier: control-plane + name: kube-apiserver +spec: + ports: + - port: 6443 + protocol: TCP + targetPort: kube-api + name: kube-api + selector: + app: kube-apiserver + component: kube-apiserver + tier: control-plane +--- +# persistent volume claim +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: kube-adit-log + labels: + app: kube-apiserver + component: kube-apiserver + tier: control-plane +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-certs-core.yaml b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-certs-core.yaml new file mode 100644 index 0000000..11a1fe8 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-certs-core.yaml @@ -0,0 +1,203 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: kube-apiserver +spec: + secretName: kube-apiserver-tls + commonName: kube-apiserver + duration: 2160h # 90d + renewBefore: 360h # 15d + isCA: false + usages: + - digital signature + - key encipherment + - server auth + + privateKey: + algorithm: RSA + size: 2048 + + subject: + organizationalUnits: + - controlplane-serving + organizations: + - kubernetes + + dnsNames: + - "{{ .Release.Namespace }}.hks.eu-central.hegerdes.com" + - "kube-apiserver" + - "kube-apiserver.{{ .Release.Namespace }}" + - "kube-apiserver.{{ .Release.Namespace }}.svc" + - "kube-apiserver.{{ .Release.Namespace }}.svc.cluster.local" + - "kubernetes" + - "kubernetes.default" + - "kubernetes.default.svc" + - "kubernetes.default.svc.cluster.local" + emailAddresses: + - kubernetes-pki@hegerdes.com + + issuerRef: + name: ca-kube-controlplane + kind: Issuer +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: kube-apiserver-kubelet-client +spec: + secretName: kube-apiserver-kubelet-client-tls + commonName: kube-apiserver-kubelet-client + duration: 2160h # 90d + renewBefore: 360h # 15d + isCA: false + usages: + - digital signature + - key encipherment + - client auth + + privateKey: + algorithm: RSA + size: 2048 + + subject: + organizationalUnits: + - controlplane-kubelet-auth + organizations: + - kubernetes + + dnsNames: [] + emailAddresses: + - kubernetes-pki@hegerdes.com + + issuerRef: + name: ca-kube-controlplane + kind: Issuer +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: kube-front-proxy-client-kube +spec: + secretName: kube-front-proxy-client-tls + commonName: front-proxy-client + duration: 2160h # 90d + renewBefore: 360h # 15d + isCA: false + usages: + - digital signature + - key encipherment + - client auth + privateKey: + algorithm: RSA + size: 2048 + + subject: + organizationalUnits: + - front-proxy-client + organizations: + - kubernetes + + dnsNames: [] + emailAddresses: + - kubernetes-pki@hegerdes.com + + issuerRef: + name: ca-kube-front-proxy + kind: Issuer +--- +############################ KUBECONF CERTS ############################ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: kube-scheduler-client +spec: + secretName: kube-scheduler-client-tls + commonName: system:kube-scheduler + duration: 2160h # 90d + renewBefore: 360h # 15d + isCA: false + usages: + - digital signature + - key encipherment + - client auth + privateKey: + algorithm: RSA + size: 2048 + + subject: + organizationalUnits: + - scheduler-client + organizations: + - kubernetes + + dnsNames: [] + emailAddresses: + - kubernetes-pki@hegerdes.com + + issuerRef: + name: ca-kube-controlplane + kind: Issuer +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: kube-controller-manager-client +spec: + secretName: kube-controller-manager-client-tls + commonName: system:kube-controller-manager + duration: 2160h # 90d + renewBefore: 360h # 15d + isCA: false + usages: + - digital signature + - key encipherment + - client auth + privateKey: + algorithm: RSA + size: 2048 + + subject: + organizationalUnits: + - controller-manager-client + organizations: + - kubernetes + + dnsNames: [] + emailAddresses: + - kubernetes-pki@hegerdes.com + + issuerRef: + name: ca-kube-controlplane + kind: Issuer +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: kube-super-admin-client +spec: + secretName: kube-super-admin-client-tls + commonName: kubernetes-super-admin + duration: 2160h # 90d + renewBefore: 360h # 15d + isCA: false + usages: + - digital signature + - key encipherment + - client auth + privateKey: + algorithm: RSA + size: 2048 + + subject: + organizationalUnits: + - super-admin-client + organizations: + - system:masters + + dnsNames: [] + emailAddresses: + - kubernetes-pki@hegerdes.com + + issuerRef: + name: ca-kube-controlplane + kind: Issuer diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-certs-etcd.yaml b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-certs-etcd.yaml new file mode 100644 index 0000000..cedfa09 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-certs-etcd.yaml @@ -0,0 +1,118 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: kube-apiserver-etcd-client +spec: + secretName: kube-apiserver-etcd-client-tls + commonName: kube-apiserver-etcd-client + duration: 2160h # 90d + renewBefore: 360h # 15d + isCA: false + usages: + - digital signature + - key encipherment + - client auth + + privateKey: + algorithm: RSA + size: 2048 + + subject: + organizationalUnits: + - controlplane-etcd-auth + organizations: + - kubernetes + + dnsNames: [] + emailAddresses: + - kubernetes-pki@hegerdes.com + + issuerRef: + name: ca-etcd + kind: Issuer +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: kube-etcd-peer +spec: + secretName: etcd-peer-tls + commonName: etcd + duration: 2160h # 90d + renewBefore: 360h # 15d + isCA: false + usages: + - digital signature + - key encipherment + - server auth + - client auth + + privateKey: + algorithm: RSA + size: 2048 + + subject: + organizationalUnits: + - etcd-peer + organizations: + - kubernetes + + dnsNames: + - "etcd" + - "etcd.default" + - "etcd.default.svc" + - "etcd.default.svc.cluster.local" + - "etcd.{{ .Release.Namespace }}" + - "etcd.{{ .Release.Namespace }}.svc" + - "etcd.{{ .Release.Namespace }}.svc.cluster.local" + emailAddresses: + - kubernetes-pki@hegerdes.com + + issuerRef: + name: ca-etcd + kind: Issuer +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: kube-etcd-server +spec: + secretName: etcd-server-tls + commonName: etcd + duration: 2160h # 90d + renewBefore: 360h # 15d + isCA: false + usages: + - digital signature + - key encipherment + - server auth + - client auth + + privateKey: + algorithm: RSA + size: 2048 + + subject: + organizationalUnits: + - etcd-server + organizations: + - kubernetes + + dnsNames: + - etcd + - etcd-0 + - etcd-1 + - etcd-2 + - localhost + - etcd.default + - etcd.default.svc + - etcd.default.svc.cluster.local + - "etcd.{{ .Release.Namespace }}" + - "etcd.{{ .Release.Namespace }}.svc" + - "etcd.{{ .Release.Namespace }}.svc.cluster.local" + emailAddresses: + - kubernetes-pki@hegerdes.com + + issuerRef: + name: ca-etcd + kind: Issuer diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-certs-util.yaml b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-certs-util.yaml new file mode 100644 index 0000000..b3494a3 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-certs-util.yaml @@ -0,0 +1,32 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: argocd-client +spec: + secretName: argocd-client-tls + commonName: argocd-client + duration: 2160h # 90d + renewBefore: 360h # 15d + isCA: false + usages: + - digital signature + - key encipherment + - client auth + + privateKey: + algorithm: RSA + size: 2048 + + subject: + organizationalUnits: + - argocd-auth + organizations: + - kubernetes + + dnsNames: [] + emailAddresses: + - kubernetes-pki@hegerdes.com + + issuerRef: + name: ca-kube-controlplane + kind: Issuer diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-confs.yaml b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-confs.yaml new file mode 100644 index 0000000..2933484 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-confs.yaml @@ -0,0 +1,172 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + tier: control-plane + name: kubeconf-template +data: + kubeconfig.yml: | + apiVersion: v1 + kind: Config + current-context: kubernetes@default + contexts: + - context: + cluster: default + user: default + name: kubernetes@default + clusters: + - cluster: + certificate-authority: /etc/kubernetes/ca.crt + server: https://kube-apiserver:6443 + name: default + users: + - name: default + user: + client-certificate: /etc/kubernetes/kube.crt + client-key: /etc/kubernetes/kube.key + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-api-configs +data: + authentication-conf.yaml: | + # Docs: https://kubernetes.io/docs/reference/access-authn-authz/authentication/#openid-connect-tokens + apiVersion: apiserver.config.k8s.io/v1beta1 + kind: AuthenticationConfiguration + anonymous: + enabled: true + conditions: + # kubeadm also needs to read the discovery info: + - path: /api/v1/namespaces/kube-public/configmaps/cluster-info + # for status checks: + - path: /healthz + - path: /readyz + - path: /livez + jwt: [] + authorization-conf.yaml: | + # Docs: https://kubernetes.io/docs/reference/access-authn-authz/authorization/ + apiVersion: apiserver.config.k8s.io/v1 + kind: AuthorizationConfiguration + authorizers: + - type: Node + name: node + - type: RBAC + name: rbac + audit-policy.yaml: | + apiVersion: audit.k8s.io/v1 + kind: Policy + + # Don't log the initial RequestReceived stage to cut down on volume + omitStages: + - RequestReceived + + # The rules are evaluated in order; the first matching rule applies. + rules: + # The following requests were manually identified as high-volume and low-risk,so drop them. + - level: None + users: ["system:kube-proxy"] + verbs: ["watch", "get"] + resources: + - group: "" # core + resources: ["endpoints", "services", "services/status"] + - level: None + userGroups: ["system:nodes"] + verbs: ["get", "watch"] + resources: + - group: "" # core + resources: ["nodes", "nodes/status"] + - level: None + users: + - system:kube-controller-manager + - system:kube-scheduler + - system:serviceaccount:kube-system:endpoint-controller + verbs: ["get", "update", "watch"] + namespaces: ["kube-system"] + resources: + - group: "" # core + resources: ["endpoints"] + - level: None + users: ["system:apiserver"] + verbs: ["get", "watch"] + resources: + - group: "" # core + resources: ["namespaces", "namespaces/status", "namespaces/finalize"] + # Don't log HPA fetching metrics. + - level: None + users: + - system:kube-controller-manager + verbs: ["get", "list"] + resources: + - group: "metrics.k8s.io" + # Don't log these read-only URLs. + - level: None + nonResourceURLs: + - /healthz* + - /version + - /swagger* + - "/metrics" + - "/openapi*" + # Don't log events requests. + - level: None + resources: + - group: "" # core + resources: ["events"] + # Secrets, TokenRequest and TokenReviews can contain sensitive & binary data, + # so only log at the Metadata level. + - level: Metadata + resources: + - group: "" # core + resources: ["secrets", "serviceaccounts/token"] + - group: authentication.k8s.io + resources: ["tokenreviews"] + + # Log request+response bodies for any mutating action + - level: RequestResponse + verbs: ["create", "update", "patch", "delete"] + # userGroups: [oidc:] + + # Log metadata only for all other resource requests + - level: Metadata + + # (Optional) If you have custom API groups you care about, you can add them here + # - level: RequestResponse + # resources: + # - group: "myorg.example.com" + # resources: ["widgets","gadgets"] + + admission-control-config.yaml: | + apiVersion: apiserver.config.k8s.io/v1 + kind: AdmissionConfiguration + plugins: + - name: PodSecurity # Enable the built-in Pod Security admission plugin + configuration: + apiVersion: pod-security.admission.config.k8s.io/v1 + kind: PodSecurityConfiguration + + # -------------------------------------------------------------------- + # 1. Cluster-wide defaults + # -------------------------------------------------------------------- + defaults: + # Reject any pod that violates the baseline profile + enforce: "baseline" + enforce-version: "latest" + + # Log (audit) and warn on anything that would break the stricter + # restricted profile so you can spot future hardening issues early + audit: "restricted" + audit-version: "latest" + warn: "restricted" + warn-version: "latest" + + # -------------------------------------------------------------------- + # 2. Exemptions for critical system components + # -------------------------------------------------------------------- + exemptions: + usernames: [] # (add human break-glass accounts if required) + runtimeClasses: [] # (e.g. kata-containers, gvisor, etc.) + namespaces: + - kube-system # Core control-plane add-ons + - kube-public # Cluster-info ConfigMap & bootstrap helpers + - kube-node-lease # Node heart-beat leases diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-controller-manager.yaml b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-controller-manager.yaml new file mode 100644 index 0000000..aa37991 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-controller-manager.yaml @@ -0,0 +1,163 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: kube-controller-manager + component: kube-controller-manager + tier: control-plane + name: kube-controller-manager +spec: + replicas: 1 + selector: + matchLabels: + app: kube-controller-manager + component: kube-controller-manager + tier: control-plane + strategy: {} + template: + metadata: + labels: + app: kube-controller-manager + component: kube-controller-manager + tier: control-plane + spec: + enableServiceLinks: false + containers: + - name: kube-controller-manager + image: registry.k8s.io/kube-controller-manager:v1.35.0 + imagePullPolicy: IfNotPresent + command: + - kube-controller-manager + - --cluster-name={{ .Release.Namespace }} + - --allocate-node-cidrs=true + - --cluster-cidr=10.244.0.0/16 + - --service-cluster-ip-range=10.96.0.0/16 + - --root-ca-file=/etc/kubernetes/pki/ca.crt + - --client-ca-file=/etc/kubernetes/pki/ca.crt + - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key + - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt + - --service-account-private-key-file=/etc/kubernetes/pki/sa.key + - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt + - --authentication-kubeconfig=/etc/kubernetes/kubeconfig.yml + - --authorization-kubeconfig=/etc/kubernetes/kubeconfig.yml + - --kubeconfig=/etc/kubernetes/kubeconfig.yml + - --controllers=*,bootstrapsigner,tokencleaner + - --leader-elect=true + - --profiling=false + - --tls-min-version=VersionTLS13 + - --use-service-account-credentials=true + - --leader-elect-resource-namespace=default + ports: + - containerPort: 10257 + name: controller-port + protocol: TCP + securityContext: + readOnlyRootFilesystem: true + # capabilities: + # drop: [all] + livenessProbe: + failureThreshold: 8 + httpGet: + path: /healthz + port: controller-port + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 15 + startupProbe: + failureThreshold: 24 + httpGet: + path: /healthz + port: controller-port + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 15 + resources: + requests: + cpu: 200m + memory: 50Mi + limits: + cpu: 2 + memory: 2Gi + volumeMounts: + - mountPath: /etc/kubernetes/pki + name: tls-kube + readOnly: true + - mountPath: /etc/kubernetes + name: kubeconfig + readOnly: true + - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/ + name: kubelet-plugins + readOnly: true + # priority: 2000001000 + # priorityClassName: system-node-critical + automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + volumes: + - name: kubelet-plugins + emptyDir: + sizeLimit: 100Mi + - name: kubeconfig + projected: + sources: + - secret: + name: kube-controller-manager-client-tls + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: kube.crt + - key: tls.key + path: kube.key + - configMap: + name: kubeconf-template + items: + - key: kubeconfig.yml + path: kubeconfig.yml + - name: tls-kube + projected: + sources: + - secret: + name: kube-ca-tls + items: + - key: tls.crt + path: ca.crt + - key: tls.key + path: ca.key + - secret: + name: kube-sa-tls + items: + - key: tls.key + path: sa.key + - key: sa.pub + path: sa.pub + - secret: + name: kube-front-proxy-ca-tls + items: + - key: tls.crt + path: front-proxy-ca.crt +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: kube-controller-manager + component: kube-controller-manager + tier: control-plane + name: kube-controller-manager +spec: + ports: + - port: 10257 + protocol: TCP + targetPort: controller-port + name: controller-port + selector: + app: kube-controller-manager + component: kube-controller-manager + tier: control-plane diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-etcd.yaml b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-etcd.yaml new file mode 100644 index 0000000..29773fc --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-etcd.yaml @@ -0,0 +1,257 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: etcd + labels: + app: etcd + component: etcd + tier: control-plane +spec: + serviceName: etcd + replicas: 1 + selector: + matchLabels: + app: etcd + component: etcd + tier: control-plane + template: + metadata: + labels: + app: etcd + component: etcd + tier: control-plane + spec: + hostname: etcd + enableServiceLinks: false + initContainers: + - name: create-conf + image: alpine + command: + - /scripts/generate-conf.sh + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - mountPath: /etc/etcd + name: share + - mountPath: /scripts + name: scripts + containers: + - name: etcd + image: registry.k8s.io/etcd:3.6.7-0 + command: [etcd, --config-file=/etc/etcd/etcd-conf.yaml] + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 8 + httpGet: + path: /livez + port: probe-port + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 15 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: probe-port + scheme: HTTP + periodSeconds: 1 + timeoutSeconds: 15 + startupProbe: + failureThreshold: 24 + httpGet: + path: /readyz + port: probe-port + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 15 + ports: + - containerPort: 2381 + name: probe-port + protocol: TCP + - containerPort: 2380 + name: peer-port + protocol: TCP + - containerPort: 2379 + name: client-port + protocol: TCP + resources: + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - mountPath: /var/lib/etcd + name: etcd-data + - mountPath: /etc/kubernetes/pki/etcd + name: tls-etcd + - mountPath: /etc/etcd + name: share + # priority: 2000001000 + # priorityClassName: system-node-critical + # securityContext: + # seccompProfile: + # type: RuntimeDefault + volumes: + - name: share + emptyDir: + sizeLimit: 10Mi + - name: scripts + configMap: + name: etcd-scripts + defaultMode: 0755 + - name: tls-etcd + projected: + sources: + - secret: + name: etcd-ca-tls + items: + - key: tls.crt + path: ca.crt + - secret: + name: etcd-server-tls + items: + - key: tls.crt + path: server.crt + - key: tls.key + path: server.key + - secret: + name: etcd-peer-tls + items: + - key: tls.crt + path: peer.crt + - key: tls.key + path: peer.key + volumeClaimTemplates: + - metadata: + name: etcd-data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 8Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: etcd + labels: + app: etcd + component: etcd + tier: control-plane +spec: + clusterIP: None + ports: + - name: clients + port: 2379 + targetPort: client-port + selector: + app: etcd + component: etcd + tier: control-plane +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: etcd-scripts +data: + generate-conf.sh: | + #!/bin/sh + + # This is the configuration file for the etcd server + # Full example: https://github.com/etcd-io/etcd/blob/main/etcd.conf.yml.sample + cat > /etc/etcd/etcd-conf.yaml < $WORKSPACE/encryption-config.yaml + + echo "KEY GENERATED: ENCRYPTION_KEY" + kubectl create secret generic kube-encryption \ + --from-file=encryption-config.yaml=$WORKSPACE/encryption-config.yaml \ + --from-literal=key=$ENCRYPTION_KEY + echo "SECRET CREATE: ENCRYPTION_KEY" + else + echo "ENCRYPTION_KEY is already set" + fi + + echo "Checking for file: $CA_ETCD_PATH_KEY" + if [ ! -e "$CA_ETCD_PATH_KEY" ]; then + echo "NOT FOUND: $CA_ETCD_PATH_KEY. Generating..." + openssl genrsa -out $WORKSPACE/etcd-ca.key $KEY_SIZE + openssl req -new -key $WORKSPACE/etcd-ca.key -x509 -days $CA_VALID_DAYS -batch \ + -subj "/CN=${CLUSTER_ID}/O=kubernetes/OU=pki/emailAddress=kubernetes-pki@hegerdes.com" \ + -out $WORKSPACE/ca.crt + + echo "KEY GENERATED: $CA_ETCD_PATH_KEY" + echo "SECRET CREATE: $CA_ETCD_PATH_KEY" + kubectl create secret generic etcd-ca-tls --type=kubernetes.io/tls \ + --from-file=tls.key=$WORKSPACE/etcd-ca.key \ + --from-file=tls.crt=$WORKSPACE/ca.crt + echo "SECRET CREATED: $CA_ETCD_PATH_KEY" + else + echo "FOUND: $CA_ETCD_PATH_KEY" + fi + + echo "Checking for file: $CA_CP_PATH_KEY" + if [ ! -e "$CA_CP_PATH_KEY" ]; then + echo "NOT FOUND: $CA_CP_PATH_KEY. Generating..." + openssl genrsa -out $WORKSPACE/kube-ca.key $KEY_SIZE + openssl req -new -key $WORKSPACE/kube-ca.key -x509 -days $CA_VALID_DAYS -batch \ + -subj "/CN=${CLUSTER_ID}/O=kubernetes/OU=pki/emailAddress=kubernetes-pki@hegerdes.com" \ + -out $WORKSPACE/ca.crt + + echo "KEY GENERATED: $CA_CP_PATH_KEY" + echo "SECRET CREATE: $CA_CP_PATH_KEY" + kubectl create secret generic kube-ca-tls --type=kubernetes.io/tls \ + --from-file=tls.key=$WORKSPACE/kube-ca.key \ + --from-file=tls.crt=$WORKSPACE/ca.crt + echo "SECRET CREATED: $CA_CP_PATH_KEY" + else + echo "FOUND: $CA_CP_PATH_KEY" + fi + + echo "Checking for file: $CA_PROXY_PATH_KEY" + if [ ! -e "$CA_PROXY_PATH_KEY" ]; then + echo "NOT FOUND: $CA_PROXY_PATH_KEY. Generating..." + openssl genrsa -out $WORKSPACE/front-proxy-ca.key $KEY_SIZE + openssl req -new -key $WORKSPACE/front-proxy-ca.key -x509 -days $CA_VALID_DAYS -batch \ + -subj "/CN=${CLUSTER_ID}/O=kubernetes/OU=pki/emailAddress=kubernetes-pki@hegerdes.com" \ + -out $WORKSPACE/front-proxy-ca.crt + + echo "KEY GENERATED: $CA_PROXY_PATH_KEY" + echo "SECRET CREATE: $CA_PROXY_PATH_KEY" + kubectl create secret generic kube-front-proxy-ca-tls --type=kubernetes.io/tls \ + --from-file=tls.key=$WORKSPACE/front-proxy-ca.key \ + --from-file=tls.crt=$WORKSPACE/front-proxy-ca.crt + echo "SECRET CREATED: $CA_PROXY_PATH_KEY" + else + echo "FOUND: $CA_PROXY_PATH_KEY" + fi + + echo "Checking for file: $CA_SA_PATH_KEY" + if [ ! -e "$CA_SA_PATH_KEY" ]; then + echo "NOT FOUND: $CA_SA_PATH_KEY. Generating..." + openssl genrsa -out $WORKSPACE/sa.key $KEY_SIZE + openssl req -new -key $WORKSPACE/sa.key -x509 -days $CA_VALID_DAYS -batch \ + -subj "/CN=${CLUSTER_ID}/O=kubernetes/OU=pki/emailAddress=kubernetes-pki@hegerdes.com" \ + -out $WORKSPACE/sa.crt + openssl rsa -in $WORKSPACE/sa.key -pubout -out $WORKSPACE/sa.pub + + echo "KEY GENERATED: $CA_SA_PATH_KEY" + echo "SECRET CREATE: $CA_SA_PATH_KEY" + kubectl create secret generic kube-sa-tls --type=kubernetes.io/tls \ + --from-file=tls.key=$WORKSPACE/sa.key \ + --from-file=tls.crt=$WORKSPACE/sa.crt \ + --from-file=$WORKSPACE/sa.pub + echo "SECRET CREATED: $CA_SA_PATH_KEY" + else + echo "FOUND: $CA_SA_PATH_KEY" + fi + rm -rf $WORKSPACE || true + volumeMounts: + - mountPath: /etc/kubernetes/pki + name: tls-kube + readOnly: true + - mountPath: /workspace + name: workdir + volumes: + - name: workdir + emptyDir: + sizeLimit: 10Mi + - name: tls-kube + projected: + sources: + - secret: + name: etcd-ca-tls + optional: true + items: + - key: tls.key + path: etcd-ca.key + - key: tls.crt + path: etcd-ca.crt + - secret: + name: kube-ca-tls + optional: true + items: + - key: tls.key + path: kube-ca.key + - key: tls.crt + path: kube-ca.crt + - secret: + name: kube-sa-tls + optional: true + items: + - key: tls.key + path: sa.key + - key: tls.crt + path: sa.crt + - key: sa.pub + path: sa.pub + - secret: + name: kube-front-proxy-ca-tls + optional: true + items: + - key: tls.key + path: front-proxy-ca.key + - key: tls.crt + path: front-proxy-ca.crt +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-init-ca-gen +--- +# Role granting create, get, list on secrets & configmaps +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kube-init-ca-gen +rules: + - apiGroups: [""] # core API group for Secrets & ConfigMaps + resources: ["secrets", "configmaps"] + verbs: ["create", "get", "list"] +--- +# Bind the Role to the ServiceAccount +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kube-init-ca-gen-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kube-init-ca-gen +subjects: + - kind: ServiceAccount + name: kube-init-ca-gen diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-issuer.yml b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-issuer.yml new file mode 100644 index 0000000..5ec3c52 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-issuer.yml @@ -0,0 +1,23 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: ca-kube-controlplane +spec: + ca: + secretName: kube-ca-tls +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: ca-etcd +spec: + ca: + secretName: etcd-ca-tls +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: ca-kube-front-proxy +spec: + ca: + secretName: kube-front-proxy-ca-tls diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-scheduler.yaml b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-scheduler.yaml new file mode 100644 index 0000000..220b15e --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-scheduler.yaml @@ -0,0 +1,133 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + component: kube-scheduler + tier: control-plane + app: kube-scheduler + name: kube-scheduler +spec: + replicas: 1 + selector: + matchLabels: + component: kube-scheduler + tier: control-plane + app: kube-scheduler + strategy: {} + template: + metadata: + labels: + component: kube-scheduler + tier: control-plane + app: kube-scheduler + spec: + enableServiceLinks: false + containers: + - name: kube-scheduler + image: registry.k8s.io/kube-scheduler:v1.35.0 + imagePullPolicy: IfNotPresent + command: + - kube-scheduler + - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt + - --authentication-kubeconfig=/etc/kubernetes/kubeconfig.yml + - --authorization-kubeconfig=/etc/kubernetes/kubeconfig.yml + - --kubeconfig=/etc/kubernetes/kubeconfig.yml + - --leader-elect=true + - --profiling=false + - --tls-min-version=VersionTLS13 + - --leader-elect-resource-namespace=default + ports: + - containerPort: 10259 + name: scheduler-port + protocol: TCP + securityContext: + readOnlyRootFilesystem: true + livenessProbe: + failureThreshold: 8 + httpGet: + path: /livez + port: scheduler-port + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 15 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: scheduler-port + scheme: HTTPS + periodSeconds: 1 + timeoutSeconds: 15 + resources: + requests: + cpu: 100m + startupProbe: + failureThreshold: 24 + httpGet: + path: /livez + port: scheduler-port + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 15 + volumeMounts: + - mountPath: /etc/kubernetes + name: kubeconfig + readOnly: true + - mountPath: /etc/kubernetes/pki + name: tls-kube + readOnly: true + # priority: 2000001000 + # priorityClassName: system-node-critical + securityContext: + seccompProfile: + type: RuntimeDefault + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + volumes: + - name: kubeconfig + projected: + sources: + - secret: + name: kube-scheduler-client-tls + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: kube.crt + - key: tls.key + path: kube.key + - configMap: + name: kubeconf-template + items: + - key: kubeconfig.yml + path: kubeconfig.yml + - name: tls-kube + projected: + sources: + - secret: + name: kube-front-proxy-client-tls + items: + - key: ca.crt + path: front-proxy-ca.crt +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: kube-scheduler + component: kube-scheduler + tier: control-plane + name: kube-scheduler +spec: + ports: + - port: 10259 + protocol: TCP + targetPort: scheduler-port + name: scheduler-port + selector: + app: kube-scheduler + component: kube-scheduler + tier: control-plane diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/values.yaml b/k8s-apps/hegerdes-kubernetes-sevice/hks/values.yaml new file mode 100644 index 0000000..e69de29 From 4810afc4ad5fecf0b3fa04dd6b789028238a39ed Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 3 Jan 2026 16:18:14 +0100 Subject: [PATCH 02/57] fix(hks): eso secret store ref Signed-off-by: Henrik Gerdes --- k8s-apps/hegerdes-kubernetes-sevice/hks/templates/eso.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/eso.yaml b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/eso.yaml index c0433d4..b9a7c2e 100644 --- a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/eso.yaml +++ b/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/eso.yaml @@ -8,7 +8,7 @@ spec: refreshInterval: 1h0m0s secretStoreRef: kind: SecretStore - name: secret-sync + name: secret-sync-{{ .Release.Namespace }} target: name: kubeconfig-credentials-{{ .Release.Namespace }} template: From 335fc7f30e6488806eda513c3d66c73f6e99da47 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 3 Jan 2026 16:27:19 +0100 Subject: [PATCH 03/57] feat: add demo clusters Signed-off-by: Henrik Gerdes --- k8s-apps/hegerdes-kubernetes-sevice/clusters/test1/values.yaml | 0 k8s-apps/hegerdes-kubernetes-sevice/clusters/test2/values.yaml | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 k8s-apps/hegerdes-kubernetes-sevice/clusters/test1/values.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-sevice/clusters/test2/values.yaml diff --git a/k8s-apps/hegerdes-kubernetes-sevice/clusters/test1/values.yaml b/k8s-apps/hegerdes-kubernetes-sevice/clusters/test1/values.yaml new file mode 100644 index 0000000..e69de29 diff --git a/k8s-apps/hegerdes-kubernetes-sevice/clusters/test2/values.yaml b/k8s-apps/hegerdes-kubernetes-sevice/clusters/test2/values.yaml new file mode 100644 index 0000000..e69de29 From 0cac2354fbd5fea801b752b36ac88244d32ee02e Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 3 Jan 2026 16:36:31 +0100 Subject: [PATCH 04/57] fix(hks): correct dir name Signed-off-by: Henrik Gerdes --- .../clusters/test1/values.yaml | 0 .../clusters/test2/values.yaml | 0 .../hks/.helmignore | 0 .../hks/Chart.yaml | 0 .../hks/templates/_helpers.tpl | 0 .../hks/templates/eso.yaml | 0 .../hks/templates/kube-apiserver.yaml | 0 .../hks/templates/kube-certs-core.yaml | 0 .../hks/templates/kube-certs-etcd.yaml | 0 .../hks/templates/kube-certs-util.yaml | 0 .../hks/templates/kube-confs.yaml | 0 .../hks/templates/kube-controller-manager.yaml | 0 .../hks/templates/kube-etcd.yaml | 0 .../hks/templates/kube-init-ca.yml | 0 .../hks/templates/kube-issuer.yml | 0 .../hks/templates/kube-scheduler.yaml | 0 .../hks/values.yaml | 0 17 files changed, 0 insertions(+), 0 deletions(-) rename k8s-apps/{hegerdes-kubernetes-sevice => hegerdes-kubernetes-service}/clusters/test1/values.yaml (100%) rename k8s-apps/{hegerdes-kubernetes-sevice => hegerdes-kubernetes-service}/clusters/test2/values.yaml (100%) rename k8s-apps/{hegerdes-kubernetes-sevice => hegerdes-kubernetes-service}/hks/.helmignore (100%) rename k8s-apps/{hegerdes-kubernetes-sevice => hegerdes-kubernetes-service}/hks/Chart.yaml (100%) rename k8s-apps/{hegerdes-kubernetes-sevice => hegerdes-kubernetes-service}/hks/templates/_helpers.tpl (100%) rename k8s-apps/{hegerdes-kubernetes-sevice => hegerdes-kubernetes-service}/hks/templates/eso.yaml (100%) rename k8s-apps/{hegerdes-kubernetes-sevice => hegerdes-kubernetes-service}/hks/templates/kube-apiserver.yaml (100%) rename k8s-apps/{hegerdes-kubernetes-sevice => hegerdes-kubernetes-service}/hks/templates/kube-certs-core.yaml (100%) rename k8s-apps/{hegerdes-kubernetes-sevice => hegerdes-kubernetes-service}/hks/templates/kube-certs-etcd.yaml (100%) rename k8s-apps/{hegerdes-kubernetes-sevice => hegerdes-kubernetes-service}/hks/templates/kube-certs-util.yaml (100%) rename k8s-apps/{hegerdes-kubernetes-sevice => hegerdes-kubernetes-service}/hks/templates/kube-confs.yaml (100%) rename k8s-apps/{hegerdes-kubernetes-sevice => hegerdes-kubernetes-service}/hks/templates/kube-controller-manager.yaml (100%) rename k8s-apps/{hegerdes-kubernetes-sevice => hegerdes-kubernetes-service}/hks/templates/kube-etcd.yaml (100%) rename k8s-apps/{hegerdes-kubernetes-sevice => hegerdes-kubernetes-service}/hks/templates/kube-init-ca.yml (100%) rename k8s-apps/{hegerdes-kubernetes-sevice => hegerdes-kubernetes-service}/hks/templates/kube-issuer.yml (100%) rename k8s-apps/{hegerdes-kubernetes-sevice => hegerdes-kubernetes-service}/hks/templates/kube-scheduler.yaml (100%) rename k8s-apps/{hegerdes-kubernetes-sevice => hegerdes-kubernetes-service}/hks/values.yaml (100%) diff --git a/k8s-apps/hegerdes-kubernetes-sevice/clusters/test1/values.yaml b/k8s-apps/hegerdes-kubernetes-service/clusters/test1/values.yaml similarity index 100% rename from k8s-apps/hegerdes-kubernetes-sevice/clusters/test1/values.yaml rename to k8s-apps/hegerdes-kubernetes-service/clusters/test1/values.yaml diff --git a/k8s-apps/hegerdes-kubernetes-sevice/clusters/test2/values.yaml b/k8s-apps/hegerdes-kubernetes-service/clusters/test2/values.yaml similarity index 100% rename from k8s-apps/hegerdes-kubernetes-sevice/clusters/test2/values.yaml rename to k8s-apps/hegerdes-kubernetes-service/clusters/test2/values.yaml diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/.helmignore b/k8s-apps/hegerdes-kubernetes-service/hks/.helmignore similarity index 100% rename from k8s-apps/hegerdes-kubernetes-sevice/hks/.helmignore rename to k8s-apps/hegerdes-kubernetes-service/hks/.helmignore diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/Chart.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/Chart.yaml similarity index 100% rename from k8s-apps/hegerdes-kubernetes-sevice/hks/Chart.yaml rename to k8s-apps/hegerdes-kubernetes-service/hks/Chart.yaml diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/_helpers.tpl b/k8s-apps/hegerdes-kubernetes-service/hks/templates/_helpers.tpl similarity index 100% rename from k8s-apps/hegerdes-kubernetes-sevice/hks/templates/_helpers.tpl rename to k8s-apps/hegerdes-kubernetes-service/hks/templates/_helpers.tpl diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/eso.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml similarity index 100% rename from k8s-apps/hegerdes-kubernetes-sevice/hks/templates/eso.yaml rename to k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-apiserver.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml similarity index 100% rename from k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-apiserver.yaml rename to k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-certs-core.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-core.yaml similarity index 100% rename from k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-certs-core.yaml rename to k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-core.yaml diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-certs-etcd.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-etcd.yaml similarity index 100% rename from k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-certs-etcd.yaml rename to k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-etcd.yaml diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-certs-util.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-util.yaml similarity index 100% rename from k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-certs-util.yaml rename to k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-util.yaml diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-confs.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml similarity index 100% rename from k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-confs.yaml rename to k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-controller-manager.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-controller-manager.yaml similarity index 100% rename from k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-controller-manager.yaml rename to k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-controller-manager.yaml diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-etcd.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml similarity index 100% rename from k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-etcd.yaml rename to k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-init-ca.yml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml similarity index 100% rename from k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-init-ca.yml rename to k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-issuer.yml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-issuer.yml similarity index 100% rename from k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-issuer.yml rename to k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-issuer.yml diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-scheduler.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-scheduler.yaml similarity index 100% rename from k8s-apps/hegerdes-kubernetes-sevice/hks/templates/kube-scheduler.yaml rename to k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-scheduler.yaml diff --git a/k8s-apps/hegerdes-kubernetes-sevice/hks/values.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/values.yaml similarity index 100% rename from k8s-apps/hegerdes-kubernetes-sevice/hks/values.yaml rename to k8s-apps/hegerdes-kubernetes-service/hks/values.yaml From bec3bdcf248861e42d3bd5a2ff21d490021d811e Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 3 Jan 2026 16:44:56 +0100 Subject: [PATCH 05/57] fix(hks): correct sync wave for pro job rbac Signed-off-by: Henrik Gerdes --- .../hks/templates/kube-init-ca.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml index c212d11..b48ebed 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml @@ -208,12 +208,18 @@ apiVersion: v1 kind: ServiceAccount metadata: name: kube-init-ca-gen + annotations: + argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/sync-wave: "-1" --- # Role granting create, get, list on secrets & configmaps apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: kube-init-ca-gen + annotations: + argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/sync-wave: "-1" rules: - apiGroups: [""] # core API group for Secrets & ConfigMaps resources: ["secrets", "configmaps"] @@ -224,6 +230,9 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: kube-init-ca-gen-rolebinding + annotations: + argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/sync-wave: "-1" roleRef: apiGroup: rbac.authorization.k8s.io kind: Role From 4c408f09af383be5b43d4da2d59d80fb8379f7d1 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 3 Jan 2026 16:53:39 +0100 Subject: [PATCH 06/57] feat: add hks shared res Signed-off-by: Henrik Gerdes --- .../argo-hks-appset.yaml | 39 ++++++ .../argo-hks-shared.yaml | 24 ++++ .../shared/argo-coredns.yml | 111 ++++++++++++++++++ .../shared/rbac-secret-read.yaml | 31 +++++ 4 files changed, 205 insertions(+) create mode 100644 k8s-apps/hegerdes-kubernetes-service/argo-hks-appset.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-service/argo-hks-shared.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml create mode 100644 k8s-apps/hegerdes-kubernetes-service/shared/rbac-secret-read.yaml diff --git a/k8s-apps/hegerdes-kubernetes-service/argo-hks-appset.yaml b/k8s-apps/hegerdes-kubernetes-service/argo-hks-appset.yaml new file mode 100644 index 0000000..06fbaae --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/argo-hks-appset.yaml @@ -0,0 +1,39 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: hks-appset + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: ["missingkey=error"] + generators: + - git: + repoURL: https://github.com/hegerdes/GitOps.git + revision: feat/hks + directories: + - path: k8s-apps/hegerdes-kubernetes-service/clusters/* + template: + metadata: + name: "hks-{{.path.basename}}" + labels: + name: hks + finalizers: + - resources-finalizer.argocd.argoproj.io/background + spec: + project: default + source: + repoURL: https://github.com/hegerdes/GitOps.git + targetRevision: feat/hks + # path: "{{.path.path}}" + path: k8s-apps/hegerdes-kubernetes-service/hks + directory: + recurse: false + destination: + server: https://kubernetes.default.svc + namespace: "{{.path.basename}}" + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/k8s-apps/hegerdes-kubernetes-service/argo-hks-shared.yaml b/k8s-apps/hegerdes-kubernetes-service/argo-hks-shared.yaml new file mode 100644 index 0000000..5f1b5b0 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/argo-hks-shared.yaml @@ -0,0 +1,24 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: hks-shared + namespace: argocd + labels: + name: hks +spec: + project: default + source: + repoURL: https://github.com/hegerdes/GitOps.git + targetRevision: feat/hks + path: k8s-apps/hegerdes-kubernetes-sevice/shared + directory: + recurse: false + destination: + server: https://kubernetes.default.svc + namespace: argocd + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml new file mode 100644 index 0000000..bfa95a8 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml @@ -0,0 +1,111 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: coredns + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: ["missingkey=error"] + generators: + - clusters: + selector: + matchLabels: + hks.hegerdes.com/managed-cluster: "true" + hks.hegerdes.com/cluster-type: "downstream" + template: + metadata: + name: "{{.name}}-coredns" + spec: + project: default + destination: + server: "{{.server}}" + namespace: kube-system + source: + chart: coredns + repoURL: https://coredns.github.io/helm + targetRevision: 1.* + helm: + releaseName: coredns + valuesObject: + replicaCount: 2 + priorityClassName: system-cluster-critical + isClusterService: false + customLabels: + k8s-app: kube-dns + deployment: + annotations: + configmap.reloader.stakater.com/reload: coredns + selector: + matchLabels: + k8s-app: kube-dns + prometheus: + service: + enabled: true + # Default zone is what Kubernetes recommends: + # https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/#coredns-configmap-options + servers: + - zones: + - zone: . + port: 53 + # -- expose the service on a different port + # servicePort: 5353 + # If serviceType is nodePort you can specify nodePort here + # nodePort: 30053 + # hostPort: 53 + plugins: + - name: errors + # Serves a /health endpoint on :8080, required for livenessProbe + - name: health + configBlock: |- + lameduck 10s + # Serves a /ready endpoint on :8181, required for readinessProbe + - name: ready + - name: log + parameters: "." + configBlock: |- + class error + # Required to query kubernetes API for data + - name: kubernetes + parameters: cluster.local in-addr.arpa ip6.arpa + configBlock: |- + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + # Serves a /metrics endpoint on :9153, required for serviceMonitor + - name: prometheus + parameters: :9153 + - name: forward + # parameters: . /etc/resolv.conf + # parameters: . 2a00:1098:2b::1 2a00:1098:2c::1 # nat64 + parameters: . tls://1.1.1.1 tls://[2606:4700:4700::1111]:853 tls://1.0.0.1 tls://[2606:4700:4700::1001]:853 + configBlock: |- + tls_servername tls.cloudflare-dns.com + health_check 5s + max_concurrent 1000 + - name: cache + parameters: 60 + configBlock: |- + disable success cluster.local + disable denial cluster.local + - name: loop + - name: reload + - name: loadbalance + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: "kubernetes.io/hostname" + labelSelector: + matchLabels: + k8s-app: kube-dns + info: + - name: Chart-Info CoreDNS + value: https://github.com/coredns/helm/ + syncPolicy: + automated: + prune: false + selfHeal: false + ignoreDifferences: + - kind: Deployment + group: apps + jqPathExpressions: + - .spec.template.spec.containers[].ports[].name diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/rbac-secret-read.yaml b/k8s-apps/hegerdes-kubernetes-service/shared/rbac-secret-read.yaml new file mode 100644 index 0000000..fb8df65 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/shared/rbac-secret-read.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: external-secrets + namespace: argocd +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: secret-reader +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [authorization.k8s.io] + resources: [selfsubjectrulesreviews] + verbs: [create] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: external-secrets-reader-binding +subjects: + - kind: ServiceAccount + name: external-secrets + namespace: argocd +roleRef: + kind: ClusterRole + name: secret-reader + apiGroup: rbac.authorization.k8s.io From 54fa92f400c91ee0e56e4249256bb8d3aa9d9e4e Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 3 Jan 2026 16:55:42 +0100 Subject: [PATCH 07/57] fix(hks): correct dir name Signed-off-by: Henrik Gerdes --- k8s-apps/hegerdes-kubernetes-service/argo-hks-shared.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s-apps/hegerdes-kubernetes-service/argo-hks-shared.yaml b/k8s-apps/hegerdes-kubernetes-service/argo-hks-shared.yaml index 5f1b5b0..a372667 100644 --- a/k8s-apps/hegerdes-kubernetes-service/argo-hks-shared.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/argo-hks-shared.yaml @@ -10,7 +10,7 @@ spec: source: repoURL: https://github.com/hegerdes/GitOps.git targetRevision: feat/hks - path: k8s-apps/hegerdes-kubernetes-sevice/shared + path: k8s-apps/hegerdes-kubernetes-service/shared directory: recurse: false destination: From eb642a39aebeaad1655a8db71f90ee10ff12edae Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 3 Jan 2026 17:39:06 +0100 Subject: [PATCH 08/57] feat: add ca singed serving certs to scheduler & controller Signed-off-by: Henrik Gerdes --- .../hks/templates/kube-certs-core.yaml | 76 +++++++++++++++++++ .../templates/kube-controller-manager.yaml | 11 +++ .../hks/templates/kube-scheduler.yaml | 13 ++++ 3 files changed, 100 insertions(+) diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-core.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-core.yaml index 11a1fe8..56103d3 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-core.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-core.yaml @@ -42,6 +42,82 @@ spec: --- apiVersion: cert-manager.io/v1 kind: Certificate +metadata: + name: kube-scheduler-serving +spec: + secretName: kube-scheduler-serving-tls + commonName: kube-scheduler + duration: 2160h # 90d + renewBefore: 360h # 15d + isCA: false + usages: + - digital signature + - key encipherment + - server auth + + privateKey: + algorithm: RSA + size: 2048 + + subject: + organizationalUnits: + - scheduler-serving + organizations: + - kubernetes + + dnsNames: + - localhost + - "kube-scheduler" + - "kube-scheduler.{{ .Release.Namespace }}" + - "kube-scheduler.{{ .Release.Namespace }}.svc" + - "kube-scheduler.{{ .Release.Namespace }}.svc.cluster.local" + emailAddresses: + - kubernetes-pki@hegerdes.com + + issuerRef: + name: ca-kube-controlplane + kind: Issuer +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: kube-controller-manager-serving +spec: + secretName: kube-controller-manager-serving-tls + commonName: kube-controller-manager + duration: 2160h # 90d + renewBefore: 360h # 15d + isCA: false + usages: + - digital signature + - key encipherment + - server auth + + privateKey: + algorithm: RSA + size: 2048 + + subject: + organizationalUnits: + - kube-controller-manager-serving + organizations: + - kubernetes + + dnsNames: + - localhost + - "kube-controller-manager" + - "kube-controller-manager.{{ .Release.Namespace }}" + - "kube-controller-manager.{{ .Release.Namespace }}.svc" + - "kube-controller-manager.{{ .Release.Namespace }}.svc.cluster.local" + emailAddresses: + - kubernetes-pki@hegerdes.com + + issuerRef: + name: ca-kube-controlplane + kind: Issuer +--- +apiVersion: cert-manager.io/v1 +kind: Certificate metadata: name: kube-apiserver-kubelet-client spec: diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-controller-manager.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-controller-manager.yaml index aa37991..dc49c2a 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-controller-manager.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-controller-manager.yaml @@ -30,6 +30,7 @@ spec: - kube-controller-manager - --cluster-name={{ .Release.Namespace }} - --allocate-node-cidrs=true + - --secure-port=10257 - --cluster-cidr=10.244.0.0/16 - --service-cluster-ip-range=10.96.0.0/16 - --root-ca-file=/etc/kubernetes/pki/ca.crt @@ -45,6 +46,9 @@ spec: - --leader-elect=true - --profiling=false - --tls-min-version=VersionTLS13 + - --tls-cert-file=/etc/kubernetes/pki/tls.crt + - --tls-private-key-file=/etc/kubernetes/pki/tls.key + - --client-ca-file=/etc/kubernetes/pki/ca.crt - --use-service-account-credentials=true - --leader-elect-resource-namespace=default ports: @@ -137,6 +141,13 @@ spec: path: sa.key - key: sa.pub path: sa.pub + - secret: + name: kube-controller-manager-serving-tls + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key - secret: name: kube-front-proxy-ca-tls items: diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-scheduler.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-scheduler.yaml index 220b15e..6277b67 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-scheduler.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-scheduler.yaml @@ -34,7 +34,11 @@ spec: - --kubeconfig=/etc/kubernetes/kubeconfig.yml - --leader-elect=true - --profiling=false + - --secure-port=10259 - --tls-min-version=VersionTLS13 + - --tls-cert-file=/etc/kubernetes/pki/tls.crt + - --tls-private-key-file=/etc/kubernetes/pki/tls.key + - --client-ca-file=/etc/kubernetes/pki/kube-ca.crt - --leader-elect-resource-namespace=default ports: - containerPort: 10259 @@ -112,6 +116,15 @@ spec: items: - key: ca.crt path: front-proxy-ca.crt + - secret: + name: kube-scheduler-serving-tls + items: + - key: ca.crt + path: kube-ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key --- apiVersion: v1 kind: Service From ad702a28150d0ff0206dfe8f90edd6c4739eb550 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 3 Jan 2026 19:25:38 +0100 Subject: [PATCH 09/57] feat(hks): support extra helm objects Signed-off-by: Henrik Gerdes --- .gitignore | 2 ++ .pre-commit-config.yaml | 2 +- .../argo-hks-appset.yaml | 6 ++-- .../hegerdes-kubernetes-service/argo-hks.yaml | 27 +++++++++++++++++ .../hks/templates/_helpers.tpl | 19 +++++++----- .../hks/templates/extra-manifests.yaml | 4 +++ .../hks/templates/kube-init-ca.yml | 29 ++++++++++++++----- .../hks/values.yaml | 1 + 8 files changed, 70 insertions(+), 20 deletions(-) create mode 100644 k8s-apps/hegerdes-kubernetes-service/argo-hks.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-service/hks/templates/extra-manifests.yaml diff --git a/.gitignore b/.gitignore index 6633775..fdc95a6 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ infra/hcloud-tf-k8s-talos/data/k8s-secret-external-secret.yml k8s-apps/linkerd/ infra/test *.log +k8s-apps/hegerdes-kubernetes-service/generated +k8s-apps/hegerdes-kubernetes-service/helper/generated diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0dcebfd..56e0a15 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: hooks: - id: check-yaml args: [--allow-multiple-documents] - exclude: .gitlab-ci.yml|infra/.*/cloud-init.yml + exclude: .gitlab-ci.yml|infra/.*/cloud-init.yml|k8s-apps/hegerdes-kubernetes-service/hks/templates/extra-manifests.yaml - id: check-json - id: pretty-format-json args: [--autofix, --no-sort-keys, --no-ensure-ascii] diff --git a/k8s-apps/hegerdes-kubernetes-service/argo-hks-appset.yaml b/k8s-apps/hegerdes-kubernetes-service/argo-hks-appset.yaml index 06fbaae..7cd93f2 100644 --- a/k8s-apps/hegerdes-kubernetes-service/argo-hks-appset.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/argo-hks-appset.yaml @@ -24,10 +24,10 @@ spec: source: repoURL: https://github.com/hegerdes/GitOps.git targetRevision: feat/hks - # path: "{{.path.path}}" path: k8s-apps/hegerdes-kubernetes-service/hks - directory: - recurse: false + helm: + valuesObject: + extraObjects: [] destination: server: https://kubernetes.default.svc namespace: "{{.path.basename}}" diff --git a/k8s-apps/hegerdes-kubernetes-service/argo-hks.yaml b/k8s-apps/hegerdes-kubernetes-service/argo-hks.yaml new file mode 100644 index 0000000..f4bc214 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/argo-hks.yaml @@ -0,0 +1,27 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: hks + namespace: argocd + labels: + name: hks + finalizers: + - resources-finalizer.argocd.argoproj.io/background +spec: + project: default + source: + repoURL: https://github.com/hegerdes/GitOps.git + targetRevision: feat/hks + path: k8s-apps/hegerdes-kubernetes-sevice/hks + helm: + valuesObject: + extraObjects: [] + destination: + server: https://kubernetes.default.svc + namespace: test1 + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/_helpers.tpl b/k8s-apps/hegerdes-kubernetes-service/hks/templates/_helpers.tpl index 3e114b1..4839a9e 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/_helpers.tpl +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/_helpers.tpl @@ -50,13 +50,16 @@ app.kubernetes.io/name: {{ include "hks.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} + {{/* -Create the name of the service account to use +Renders a value that contains template. +Usage: +{{ include ".render" ( dict "value" .Values.path.to.the.Value "context" $) }} */}} -{{- define "hks.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "hks.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} +{{- define ".render" -}} + {{- if typeIs "string" .value }} + {{- tpl .value .context }} + {{- else }} + {{- tpl (.value | toYaml) .context }} + {{- end }} +{{- end -}} diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/extra-manifests.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/extra-manifests.yaml new file mode 100644 index 0000000..0f48fe1 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{- range .Values.extraObjects }} +--- +{{ include ".render" (dict "value" . "context" $) }} +{{- end }} diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml index b48ebed..68cb0c9 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml @@ -13,6 +13,12 @@ spec: spec: serviceAccountName: kube-init-ca-gen restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault containers: - name: file-check image: hegerdes/debug:hks @@ -31,10 +37,8 @@ spec: - resources: - secrets - configmaps - - secretstores - - storeconfigs - - passwords - - certificates + - *.external-secrets.io + - *.cert-manager.io providers: - aescbc: keys: @@ -155,7 +159,10 @@ spec: else echo "FOUND: $CA_SA_PATH_KEY" fi - rm -rf $WORKSPACE || true + # Make sure all keys are gone from fs + rm -rf $WORKSPACE/* || true + securityContext: + readOnlyRootFilesystem: true volumeMounts: - mountPath: /etc/kubernetes/pki name: tls-kube @@ -212,7 +219,7 @@ metadata: argocd.argoproj.io/hook: PreSync argocd.argoproj.io/sync-wave: "-1" --- -# Role granting create, get, list on secrets & configmaps +# Role granting create on secrets & configmaps apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -221,9 +228,15 @@ metadata: argocd.argoproj.io/hook: PreSync argocd.argoproj.io/sync-wave: "-1" rules: - - apiGroups: [""] # core API group for Secrets & ConfigMaps + - apiGroups: [""] resources: ["secrets", "configmaps"] - verbs: ["create", "get", "list"] + verbs: ["create"] + resourceNames: + - kube-encryption + - etcd-ca-tls + - kube-ca-tls + - kube-front-proxy-ca-tls + - kube-sa-tls --- # Bind the Role to the ServiceAccount apiVersion: rbac.authorization.k8s.io/v1 diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/values.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/values.yaml index e69de29..3face71 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/values.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/values.yaml @@ -0,0 +1 @@ +extraObjects: [] From 729d8ac57d32d1587fdd8e2c0b7ae7a9e034e033 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 3 Jan 2026 21:31:27 +0100 Subject: [PATCH 10/57] fix(hks): ca init rbac Signed-off-by: Henrik Gerdes --- .../hks/templates/kube-apiserver.yaml | 1 + .../hks/templates/kube-controller-manager.yaml | 1 + .../hks/templates/kube-init-ca.yml | 6 ------ .../hks/templates/kube-scheduler.yaml | 1 + 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml index fd070dd..5c17f37 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml @@ -33,6 +33,7 @@ spec: - name: kube-apiserver image: registry.k8s.io/kube-apiserver:v1.35.0 imagePullPolicy: IfNotPresent + # Docs: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/ command: - kube-apiserver - --secure-port=6443 diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-controller-manager.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-controller-manager.yaml index dc49c2a..c09c5aa 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-controller-manager.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-controller-manager.yaml @@ -26,6 +26,7 @@ spec: - name: kube-controller-manager image: registry.k8s.io/kube-controller-manager:v1.35.0 imagePullPolicy: IfNotPresent + # Docs: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/ command: - kube-controller-manager - --cluster-name={{ .Release.Namespace }} diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml index 68cb0c9..17d03c9 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml @@ -231,12 +231,6 @@ rules: - apiGroups: [""] resources: ["secrets", "configmaps"] verbs: ["create"] - resourceNames: - - kube-encryption - - etcd-ca-tls - - kube-ca-tls - - kube-front-proxy-ca-tls - - kube-sa-tls --- # Bind the Role to the ServiceAccount apiVersion: rbac.authorization.k8s.io/v1 diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-scheduler.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-scheduler.yaml index 6277b67..300e85a 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-scheduler.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-scheduler.yaml @@ -26,6 +26,7 @@ spec: - name: kube-scheduler image: registry.k8s.io/kube-scheduler:v1.35.0 imagePullPolicy: IfNotPresent + # Docs: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-scheduler/ command: - kube-scheduler - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt From 0e75080224cd50d077c8802b474ca1edd7912ff0 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 3 Jan 2026 21:37:12 +0100 Subject: [PATCH 11/57] fix(hks): encryption conf Signed-off-by: Henrik Gerdes --- .../hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml index 17d03c9..7b4c555 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml @@ -37,8 +37,6 @@ spec: - resources: - secrets - configmaps - - *.external-secrets.io - - *.cert-manager.io providers: - aescbc: keys: From 6f6a7afe1e4ed406ffbf8d7e61992537a02fb5fd Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 3 Jan 2026 22:24:05 +0100 Subject: [PATCH 12/57] feat(hks): add tls-route Signed-off-by: Henrik Gerdes --- .../hks/templates/kube-apiserver.yaml | 31 ++++++ .../hks/templates/kube-confs.yaml | 103 ++++++++++++++++++ .../hks/templates/kube-etcd.yaml | 101 +---------------- .../hks/templates/tls-route.yaml | 17 +++ 4 files changed, 152 insertions(+), 100 deletions(-) create mode 100644 k8s-apps/hegerdes-kubernetes-service/hks/templates/tls-route.yaml diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml index 5c17f37..2e559b1 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml @@ -29,6 +29,36 @@ spec: type: RuntimeDefault automountServiceAccountToken: false enableServiceLinks: false + initContainers: + - name: etcd-check + image: curlimages/curl + command: + - sh + - -c + - | + set -eu + ETCD_ENDPOINT="https://etcd:2379/health" + CA_CERT="/etc/kubernetes/etcd/pki/ca.crt" + CLIENT_CERT="/etc/kubernetes/etcd/pki/apiserver-etcd-client.crt" + CLIENT_KEY="/etc/kubernetes/etcd/pki/apiserver-etcd-client.key" + while true; do + if curl --silent --show-error --fail --cacert "$CA_CERT" \ + --cert "$CLIENT_CERT" --key "$CLIENT_KEY" \ + "$ETCD_ENDPOINT" | grep -q '"health":.*"true"'; then + echo "etcd is ready!" + break + fi + echo "waiting for etcd..." + sleep 2 + done + volumeMounts: + - mountPath: /etc/kubernetes/etcd/pki/ + name: tls-etcd + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] containers: - name: kube-apiserver image: registry.k8s.io/kube-apiserver:v1.35.0 @@ -87,6 +117,7 @@ spec: protocol: TCP securityContext: readOnlyRootFilesystem: true + allowPrivilegeEscalation: false # capabilities: # drop: [all] diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml index 2933484..f8d325b 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml @@ -30,6 +30,8 @@ apiVersion: v1 kind: ConfigMap metadata: name: kube-api-configs + labels: + tier: control-plane data: authentication-conf.yaml: | # Docs: https://kubernetes.io/docs/reference/access-authn-authz/authentication/#openid-connect-tokens @@ -170,3 +172,104 @@ data: - kube-system # Core control-plane add-ons - kube-public # Cluster-info ConfigMap & bootstrap helpers - kube-node-lease # Node heart-beat leases +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: etcd-configs + labels: + tier: control-plane +data: + generate-conf.sh: | + #!/bin/sh + + # This is the configuration file for the etcd server + # Full example: https://github.com/etcd-io/etcd/blob/main/etcd.conf.yml.sample + cat > /etc/etcd/etcd-conf.yaml < /etc/etcd/etcd-conf.yaml < Date: Sat, 3 Jan 2026 23:03:55 +0100 Subject: [PATCH 13/57] feat(hks): secure deployments Signed-off-by: Henrik Gerdes --- .pre-commit-config.yaml | 2 +- .../hks/templates/kube-apiserver.yaml | 13 +++++++++---- .../hks/templates/kube-controller-manager.yaml | 13 ++++++++++--- .../hks/templates/kube-etcd.yaml | 5 +++++ .../hks/templates/kube-scheduler.yaml | 10 +++++++++- .../hks/templates/pdb.yaml | 14 ++++++++++++++ .../hegerdes-kubernetes-service/hks/values.yaml | 4 ++++ 7 files changed, 52 insertions(+), 9 deletions(-) create mode 100644 k8s-apps/hegerdes-kubernetes-service/hks/templates/pdb.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 56e0a15..7a0d29e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: hooks: - id: check-yaml args: [--allow-multiple-documents] - exclude: .gitlab-ci.yml|infra/.*/cloud-init.yml|k8s-apps/hegerdes-kubernetes-service/hks/templates/extra-manifests.yaml + exclude: .gitlab-ci.yml|infra/.*/cloud-init.yml|k8s-apps/hegerdes-kubernetes-service/hks/templates/extra-manifests.yaml|k8s-apps/hegerdes-kubernetes-service/hks/templates/pdb.yaml - id: check-json - id: pretty-format-json args: [--autofix, --no-sort-keys, --no-ensure-ascii] diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml index 2e559b1..bef9aaf 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml @@ -13,7 +13,12 @@ spec: app: kube-apiserver component: kube-apiserver tier: control-plane - strategy: {} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + minReadySeconds: 10 template: metadata: labels: @@ -118,9 +123,9 @@ spec: securityContext: readOnlyRootFilesystem: true allowPrivilegeEscalation: false - # capabilities: - # drop: [all] - + capabilities: + drop: [all] + add: ["NET_BIND_SERVICE"] livenessProbe: failureThreshold: 8 httpGet: diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-controller-manager.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-controller-manager.yaml index c09c5aa..d9e104b 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-controller-manager.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-controller-manager.yaml @@ -13,7 +13,12 @@ spec: app: kube-controller-manager component: kube-controller-manager tier: control-plane - strategy: {} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + minReadySeconds: 10 template: metadata: labels: @@ -58,8 +63,10 @@ spec: protocol: TCP securityContext: readOnlyRootFilesystem: true - # capabilities: - # drop: [all] + allowPrivilegeEscalation: false + capabilities: + drop: [all] + add: ["NET_BIND_SERVICE"] livenessProbe: failureThreshold: 8 httpGet: diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml index af5daa0..6f35494 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml @@ -42,6 +42,11 @@ spec: name: share - mountPath: /scripts name: scripts + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] containers: - name: etcd image: registry.k8s.io/etcd:3.6.7-0 diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-scheduler.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-scheduler.yaml index 300e85a..c002202 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-scheduler.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-scheduler.yaml @@ -13,7 +13,12 @@ spec: component: kube-scheduler tier: control-plane app: kube-scheduler - strategy: {} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + minReadySeconds: 10 template: metadata: labels: @@ -47,6 +52,9 @@ spec: protocol: TCP securityContext: readOnlyRootFilesystem: true + capabilities: + drop: [all] + add: ["NET_BIND_SERVICE"] livenessProbe: failureThreshold: 8 httpGet: diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/pdb.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/pdb.yaml new file mode 100644 index 0000000..6850ac4 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/pdb.yaml @@ -0,0 +1,14 @@ +{{- if .Values.controlplane.pdb.enabled -}} + +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: kube-apiserver +spec: + minAvailable: 1 + selector: + matchLabels: + app: kube-apiserver + component: kube-apiserver + tier: control-plane +{{- end }} diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/values.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/values.yaml index 3face71..6de7d3d 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/values.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/values.yaml @@ -1 +1,5 @@ +controlplane: + pdb: + enabled: false + extraObjects: [] From 08caf84560d5447d82860343fb30ca517985cf26 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sun, 4 Jan 2026 21:48:53 +0100 Subject: [PATCH 14/57] feat(hks): allow to set hks hostname via argo Signed-off-by: Henrik Gerdes --- k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml | 3 +++ .../hks/templates/kube-apiserver.yaml | 2 +- .../hks/templates/kube-certs-core.yaml | 2 +- .../hegerdes-kubernetes-service/hks/templates/tls-route.yaml | 2 +- k8s-apps/hegerdes-kubernetes-service/hks/values.yaml | 1 + 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml index b9a7c2e..33a2467 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml @@ -13,6 +13,9 @@ spec: name: kubeconfig-credentials-{{ .Release.Namespace }} template: metadata: + annotations: + hks.hegerdes.com/public-host: "{{ .Release.Namespace }}.{{ .Values.controlplane.domainSuffix }}" + hks.hegerdes.com/private-host: "kube-apiserver.{{ .Release.Namespace }}" labels: argocd.argoproj.io/secret-type: cluster hks.hegerdes.com/managed-cluster: "true" diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml index bef9aaf..955892d 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml @@ -73,7 +73,7 @@ spec: - kube-apiserver - --secure-port=6443 - --profiling=false - - --external-hostname={{ .Release.Namespace }}.hks.eu-central.hegerdes.com + - --external-hostname={{ .Release.Namespace }}.{{ .Values.controlplane.domainSuffix }} - --enable-admission-plugins=NodeRestriction - --enable-bootstrap-token-auth=true - --allow-privileged=true diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-core.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-core.yaml index 56103d3..558252f 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-core.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-core.yaml @@ -24,7 +24,7 @@ spec: - kubernetes dnsNames: - - "{{ .Release.Namespace }}.hks.eu-central.hegerdes.com" + - "{{ .Release.Namespace }}.{{ .Values.controlplane.domainSuffix }}" - "kube-apiserver" - "kube-apiserver.{{ .Release.Namespace }}" - "kube-apiserver.{{ .Release.Namespace }}.svc" diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/tls-route.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/tls-route.yaml index 809d205..69caaee 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/tls-route.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/tls-route.yaml @@ -5,7 +5,7 @@ metadata: spec: hostnames: - "{{ .Release.Namespace }}.localhost" - - "{{ .Release.Namespace }}.hks.eu-central.hegerdes.com" + - "{{ .Release.Namespace }}.{{ .Values.controlplane.domainSuffix }}" parentRefs: - name: hks-kube-gw namespace: nginx-gateway diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/values.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/values.yaml index 6de7d3d..275aff2 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/values.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/values.yaml @@ -1,4 +1,5 @@ controlplane: + domainSuffix: hks.eu-central.hegerdes.com pdb: enabled: false From e9a22e24c800d6361f1fcda40be5bc0534108e47 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sun, 4 Jan 2026 21:58:07 +0100 Subject: [PATCH 15/57] fix: also manage cilium-secrets ns Signed-off-by: Henrik Gerdes --- k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml index 33a2467..af5c69b 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml @@ -24,7 +24,7 @@ spec: data: server: https://kube-apiserver.{{ .Release.Namespace }}:6443 name: "{{ .Release.Namespace }}" - namespaces: kube-system,cilium,default,kubelet-serving-cert-approver + namespaces: kube-system,cilium,cilium-secrets,default,kubelet-serving-cert-approver clusterResources: "true" config: | { From b9c3a6ba68ce4628ed0f13b014ca7cb1f091444b Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Mon, 5 Jan 2026 19:12:29 +0100 Subject: [PATCH 16/57] feat(hks): ha etcd Signed-off-by: Henrik Gerdes --- .../hks/templates/kube-certs-etcd.yaml | 9 ++++--- .../hks/templates/kube-confs.yaml | 24 +++++++++++++++---- .../hks/templates/kube-etcd.yaml | 22 ++++++++++------- 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-etcd.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-etcd.yaml index cedfa09..9987501 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-etcd.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-etcd.yaml @@ -59,12 +59,15 @@ spec: dnsNames: - "etcd" - - "etcd.default" - - "etcd.default.svc" - - "etcd.default.svc.cluster.local" - "etcd.{{ .Release.Namespace }}" - "etcd.{{ .Release.Namespace }}.svc" - "etcd.{{ .Release.Namespace }}.svc.cluster.local" + - "etcd-0.etcd" + - "etcd-1.etcd" + - "etcd-2.etcd" + - "etcd-0.etcd.{{ .Release.Namespace }}" + - "etcd-1.etcd.{{ .Release.Namespace }}" + - "etcd-2.etcd.{{ .Release.Namespace }}" emailAddresses: - kubernetes-pki@hegerdes.com diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml index f8d325b..68bd2ce 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml @@ -189,7 +189,7 @@ data: # This is the configuration file for the etcd server # Human-readable name for this member. - name: $POD_NAME + name: ${POD_NAME} # Path to the data directory. data-dir: /var/lib/etcd @@ -204,10 +204,10 @@ data: election-timeout: 1000 # List of comma separated URLs to listen on for peer traffic. - listen-peer-urls: https://$POD_IP:2380 + listen-peer-urls: https://${POD_IP}:2380 # List of comma separated URLs to listen on for client traffic. - listen-client-urls: https://$POD_IP:2379 + listen-client-urls: https://${POD_IP}:2379,https://localhost:2379 # Maximum number of snapshot files to retain (0 is unlimited). max-snapshots: 200 @@ -215,11 +215,25 @@ data: # Maximum number of wal files to retain (0 is unlimited). max-wals: 200 + # Comma separated string of initial cluster configuration for bootstrapping. + # Example: initial-cluster: "infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380" + initial-cluster: "etcd-0=https://etcd-0.etcd:2380,etcd-1=https://etcd-1.etcd:2380,etcd-2=https://etcd-2.etcd:2380" + + # Initial cluster token for the etcd cluster during bootstrap. + initial-cluster-token: 'etcd-cluster' + + # Initial cluster state ('new' or 'existing'). + initial-cluster-state: 'new' + + # List of this member's peer URLs to advertise to the rest of the cluster. + # The URLs needed to be a comma-separated list. + initial-advertise-peer-urls: https://${POD_IP}:2380,https://${POD_NAME}.etcd:2380 + # List of this member's client URLs to advertise to the public. # The URLs needed to be a comma-separated list. - advertise-client-urls: https://$POD_IP:2379 + advertise-client-urls: https://${POD_NAME}.etcd:2379 - listen-metrics-urls: http://$POD_IP:2381 + listen-metrics-urls: http://${POD_IP}:2381 # Valid values include 'on', 'readonly', 'off' proxy: 'off' diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml index 6f35494..4160348 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml @@ -8,7 +8,11 @@ metadata: tier: control-plane spec: serviceName: etcd - replicas: 1 + replicas: 3 + podManagementPolicy: Parallel + updateStrategy: + rollingUpdate: + maxUnavailable: 1 selector: matchLabels: app: etcd @@ -21,7 +25,6 @@ spec: component: etcd tier: control-plane spec: - hostname: etcd enableServiceLinks: false initContainers: - name: create-conf @@ -33,6 +36,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name + - name: CLUSTER_ID + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: POD_IP valueFrom: fieldRef: @@ -99,15 +106,13 @@ spec: name: tls-etcd - mountPath: /etc/etcd name: share - # priority: 2000001000 - # priorityClassName: system-node-critical - # securityContext: - # seccompProfile: - # type: RuntimeDefault + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - name: share emptyDir: - sizeLimit: 10Mi + sizeLimit: 1Mi - name: scripts configMap: name: etcd-configs @@ -153,6 +158,7 @@ metadata: tier: control-plane spec: clusterIP: None + publishNotReadyAddresses: true ports: - name: clients port: 2379 From 5d3799b2686f9c86223317e7c7b42016867df0c7 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Mon, 5 Jan 2026 19:18:58 +0100 Subject: [PATCH 17/57] fix(hks): etcd cluster conf Signed-off-by: Henrik Gerdes --- .../hegerdes-kubernetes-service/hks/templates/kube-confs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml index 68bd2ce..fb1fca1 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml @@ -227,11 +227,11 @@ data: # List of this member's peer URLs to advertise to the rest of the cluster. # The URLs needed to be a comma-separated list. - initial-advertise-peer-urls: https://${POD_IP}:2380,https://${POD_NAME}.etcd:2380 + initial-advertise-peer-urls: https://${POD_NAME}.etcd:2380 # List of this member's client URLs to advertise to the public. # The URLs needed to be a comma-separated list. - advertise-client-urls: https://${POD_NAME}.etcd:2379 + advertise-client-urls: https://${POD_IP}:2379,https://${POD_NAME}.etcd:2379 listen-metrics-urls: http://${POD_IP}:2381 From 7be428803a4249c4d086868489bda6c41006c230 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Thu, 8 Jan 2026 19:13:08 +0100 Subject: [PATCH 18/57] feat(hks): support etcd backups Signed-off-by: Henrik Gerdes --- .../hks/templates/kube-confs.yaml | 2 +- .../hks/templates/kube-etcd.yaml | 171 +++++++++++++++++- 2 files changed, 164 insertions(+), 9 deletions(-) diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml index fb1fca1..ecddbca 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml @@ -220,7 +220,7 @@ data: initial-cluster: "etcd-0=https://etcd-0.etcd:2380,etcd-1=https://etcd-1.etcd:2380,etcd-2=https://etcd-2.etcd:2380" # Initial cluster token for the etcd cluster during bootstrap. - initial-cluster-token: 'etcd-cluster' + initial-cluster-token: etcd-${CLUSTER_ID} # Initial cluster state ('new' or 'existing'). initial-cluster-state: 'new' diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml index 4160348..43bcad7 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml @@ -26,6 +26,16 @@ spec: tier: control-plane spec: enableServiceLinks: false + automountServiceAccountToken: false + # affinity: + # podAntiAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # - labelSelector: + # matchLabels: + # app: etcd + # component: etcd + # tier: control-plane + # topologyKey: kubernetes.io/hostname initContainers: - name: create-conf image: alpine @@ -59,11 +69,16 @@ spec: image: registry.k8s.io/etcd:3.6.7-0 command: [etcd, --config-file=/etc/etcd/etcd-conf.yaml] imagePullPolicy: IfNotPresent + env: + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name livenessProbe: failureThreshold: 8 httpGet: path: /livez - port: probe-port + port: metrics scheme: HTTP initialDelaySeconds: 10 periodSeconds: 10 @@ -72,7 +87,7 @@ spec: failureThreshold: 3 httpGet: path: /readyz - port: probe-port + port: metrics scheme: HTTP periodSeconds: 1 timeoutSeconds: 15 @@ -80,20 +95,25 @@ spec: failureThreshold: 24 httpGet: path: /readyz - port: probe-port + port: metrics scheme: HTTP initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 15 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] ports: - containerPort: 2381 - name: probe-port + name: metrics protocol: TCP - containerPort: 2380 - name: peer-port + name: peer protocol: TCP - containerPort: 2379 - name: client-port + name: client protocol: TCP resources: requests: @@ -107,6 +127,9 @@ spec: - mountPath: /etc/etcd name: share securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 seccompProfile: type: RuntimeDefault volumes: @@ -160,10 +183,142 @@ spec: clusterIP: None publishNotReadyAddresses: true ports: - - name: clients + - name: client port: 2379 - targetPort: client-port + targetPort: client + - port: 2381 + name: metrics + targetPort: metrics + - port: 2380 + name: peer + targetPort: peer selector: app: etcd component: etcd tier: control-plane +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: etcd-backup +spec: + schedule: "0 */2 * * *" # every 2 hours + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + enableServiceLinks: false + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault + restartPolicy: OnFailure + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: etcd + component: etcd + tier: control-plane + topologyKey: kubernetes.io/hostname + + containers: + - name: etcd-backup + image: hegerdes/debug:hks + imagePullPolicy: IfNotPresent + env: + - name: ENCRYPTION_KEY + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ETCD_ENDPOINTS + value: "https://etcd:2379" + - name: HOME + value: /backups/home + volumeMounts: + - name: tls-etcd + mountPath: /etc/kubernetes/pki/etcd + readOnly: true + - name: backup + mountPath: /backups + command: + - /bin/sh + - -c + - | + set -euo pipefail + mkdir -p $HOME + ts=$(date +%Y%m%d%H%M%S) + SNAP="/backups/etcd-snapshot-${ts}.db" + # take snapshot + etcdctl --endpoints="${ETCD_ENDPOINTS}" \ + --cacert=/etc/kubernetes/pki/etcd/ca.crt \ + --cert=/etc/kubernetes/pki/etcd/client.crt \ + --key=/etc/kubernetes/pki/etcd/client.key \ + snapshot save "${SNAP}" + # gzip to save space + gzip -f "${SNAP}" + # Encrypt + # encrypt (passphrase from env, avoids showing on ps) + printf '%s' "$ENCRYPTION_KEY" | gpg --batch --yes --passphrase-fd 0 \ + --symmetric --cipher-algo AES256 -o "${SNAP}.gz.gpg" "${SNAP}.gz" + + # decrypt + # printf '%s' "$ENCRYPTION_KEY" | gpg --batch --yes --passphrase-fd 0 \ + # -o /data/myfile --decrypt "${SNAP}.gz.gpg" + + # remove snapshots older than 30 days + find /backups -type f -name 'etcd-snapshot-*.db.gz' -mtime +30 -print -delete + ls -lh /backups + + # decrypt + # printf '%s' "$ENCRYPTION_KEY" | gpg --batch --yes --passphrase-fd 0 \ + # -o /data/myfile --decrypt "${SNAP}.gz.gpg" + + # decompress + # gzip -d snapshot.db.gz + + # restore + # etcdutl snapshot restore snapshot.db \ + # --name metcd-0 \ + # --initial-cluster etcd-0=https://etcd-0.etcd:2380,etcd-1=https://etcd-1.etcd:2380,etcd-2=https://etcd-2.etcd:2380 \ + # --initial-cluster-token etcd-${CLUSTER_ID} \ + # --initial-advertise-peer-urls https://${POD_NAME}.etcd:2380 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + volumes: + - name: tls-etcd + projected: + sources: + - secret: + name: kube-apiserver-etcd-client-tls + items: + - key: tls.key + path: client.key + - key: tls.crt + path: client.crt + - key: ca.crt + path: ca.crt + - name: backup + persistentVolumeClaim: + claimName: etcd-backups +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: etcd-backups +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi From e0537bf07dde11c80e385971e04bfbd0b96ad892 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Thu, 8 Jan 2026 21:05:08 +0100 Subject: [PATCH 19/57] fix(hks): prefix etcd backups with cluster id Signed-off-by: Henrik Gerdes --- .../hks/templates/kube-etcd.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml index 43bcad7..3356144 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml @@ -238,6 +238,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: ClUSTER_ID + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: ETCD_ENDPOINTS value: "https://etcd:2379" - name: HOME @@ -255,7 +259,7 @@ spec: set -euo pipefail mkdir -p $HOME ts=$(date +%Y%m%d%H%M%S) - SNAP="/backups/etcd-snapshot-${ts}.db" + SNAP="/backups/etcd-snapshot-${ClUSTER_ID}-${ts}.db" # take snapshot etcdctl --endpoints="${ETCD_ENDPOINTS}" \ --cacert=/etc/kubernetes/pki/etcd/ca.crt \ @@ -265,14 +269,9 @@ spec: # gzip to save space gzip -f "${SNAP}" # Encrypt - # encrypt (passphrase from env, avoids showing on ps) printf '%s' "$ENCRYPTION_KEY" | gpg --batch --yes --passphrase-fd 0 \ --symmetric --cipher-algo AES256 -o "${SNAP}.gz.gpg" "${SNAP}.gz" - # decrypt - # printf '%s' "$ENCRYPTION_KEY" | gpg --batch --yes --passphrase-fd 0 \ - # -o /data/myfile --decrypt "${SNAP}.gz.gpg" - # remove snapshots older than 30 days find /backups -type f -name 'etcd-snapshot-*.db.gz' -mtime +30 -print -delete ls -lh /backups From d959ea989e17f8e2bbbe4bc98621364aab0e09f4 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sun, 11 Jan 2026 13:46:49 +0100 Subject: [PATCH 20/57] feat(hks): add hks manager chart Signed-off-by: Henrik Gerdes --- .pre-commit-config.yaml | 2 +- .../hks-manager/Chart.yaml | 6 + .../hks-manager/chart_schema.yaml | 37 +++ .../hks-manager/lintconf.yaml | 42 +++ .../hks-manager/templates/NOTES.txt | 23 ++ .../hks-manager/templates/_helpers.tpl | 85 ++++++ .../hks-manager/templates/deployment.yaml | 89 ++++++ .../templates/extra-manifests.yaml | 4 + .../hks-manager/templates/hpa.yaml | 28 ++ .../hks-manager/templates/httproute.yaml | 38 +++ .../hks-manager/templates/ingress.yaml | 45 +++ .../hks-manager/templates/rbac.yaml | 30 ++ .../hks-manager/templates/service.yaml | 21 ++ .../hks-manager/templates/serviceaccount.yaml | 12 + .../hks-manager/templates/servicemonitor.yaml | 83 ++++++ .../hks-manager/values.schema.json | 264 ++++++++++++++++++ .../hks-manager/values.yaml | 227 +++++++++++++++ 17 files changed, 1035 insertions(+), 1 deletion(-) create mode 100644 k8s-apps/hegerdes-kubernetes-service/hks-manager/Chart.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-service/hks-manager/chart_schema.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-service/hks-manager/lintconf.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/NOTES.txt create mode 100644 k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/_helpers.tpl create mode 100644 k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/deployment.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/extra-manifests.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/hpa.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/httproute.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/ingress.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/rbac.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/service.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/serviceaccount.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/servicemonitor.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-service/hks-manager/values.schema.json create mode 100644 k8s-apps/hegerdes-kubernetes-service/hks-manager/values.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7a0d29e..99bf5a5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: hooks: - id: check-yaml args: [--allow-multiple-documents] - exclude: .gitlab-ci.yml|infra/.*/cloud-init.yml|k8s-apps/hegerdes-kubernetes-service/hks/templates/extra-manifests.yaml|k8s-apps/hegerdes-kubernetes-service/hks/templates/pdb.yaml + exclude: .gitlab-ci.yml|infra/.*/cloud-init.yml|k8s-apps/hegerdes-kubernetes-service/hks/templates/extra-manifests.yaml|k8s-apps/hegerdes-kubernetes-service/hks/templates/pdb.yaml|k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/.* - id: check-json - id: pretty-format-json args: [--autofix, --no-sort-keys, --no-ensure-ascii] diff --git a/k8s-apps/hegerdes-kubernetes-service/hks-manager/Chart.yaml b/k8s-apps/hegerdes-kubernetes-service/hks-manager/Chart.yaml new file mode 100644 index 0000000..f893482 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/hks-manager/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +appVersion: 0.1.0 +description: A Helm chart for Kubernetes +name: hks-manager +type: application +version: 0.1.0 diff --git a/k8s-apps/hegerdes-kubernetes-service/hks-manager/chart_schema.yaml b/k8s-apps/hegerdes-kubernetes-service/hks-manager/chart_schema.yaml new file mode 100644 index 0000000..2a26d9b --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/hks-manager/chart_schema.yaml @@ -0,0 +1,37 @@ +name: str() +home: str(required=False) +version: str() +apiVersion: str() +appVersion: any(str(), num(), required=False) +description: str(required=False) +keywords: list(str(), required=False) +sources: list(str(), required=False) +maintainers: list(include('maintainer'), required=False) +dependencies: list(include('dependency'), required=False) +icon: str(required=False) +engine: str(required=False) +condition: str(required=False) +tags: str(required=False) +deprecated: bool(required=False) +kubeVersion: str(required=False) +annotations: map(str(), str(), required=False) +type: str(required=False) +--- +maintainer: + name: str() + email: str(required=False) + url: str(required=False) +--- +dependency: + name: str() + version: str() + repository: str(required=False) + condition: str(required=False) + tags: list(str(), required=False) + enabled: bool(required=False) + import-values: any(list(str()), list(include('import-value')), required=False) + alias: str(required=False) +--- +import-value: + child: str() + parent: str() diff --git a/k8s-apps/hegerdes-kubernetes-service/hks-manager/lintconf.yaml b/k8s-apps/hegerdes-kubernetes-service/hks-manager/lintconf.yaml new file mode 100644 index 0000000..90f48c8 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/hks-manager/lintconf.yaml @@ -0,0 +1,42 @@ +--- +rules: + braces: + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: -1 + max-spaces-inside-empty: -1 + brackets: + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: -1 + max-spaces-inside-empty: -1 + colons: + max-spaces-before: 0 + max-spaces-after: 1 + commas: + max-spaces-before: 0 + min-spaces-after: 1 + max-spaces-after: 1 + comments: + require-starting-space: true + min-spaces-from-content: 2 + document-end: disable + document-start: disable # No --- to start a file + empty-lines: + max: 2 + max-start: 0 + max-end: 0 + hyphens: + max-spaces-after: 1 + indentation: + spaces: consistent + indent-sequences: whatever # - list indentation will handle both indentation and without + check-multi-line-strings: false + key-duplicates: enable + line-length: disable # Lines can be any length + new-line-at-end-of-file: enable + new-lines: + type: unix + trailing-spaces: enable + truthy: + level: warning diff --git a/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/NOTES.txt b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/NOTES.txt new file mode 100644 index 0000000..a1f26f6 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/NOTES.txt @@ -0,0 +1,23 @@ +# Template by https://dev.to/harisharavindan/helm-starter-chart-m1j +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "hks-manager.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "hks-manager.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "hks-manager.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "hks-manager.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/_helpers.tpl b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/_helpers.tpl new file mode 100644 index 0000000..fd34dcc --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/_helpers.tpl @@ -0,0 +1,85 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "hks-manager.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "hks-manager.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "hks-manager.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "hks-manager.labels" -}} +helm.sh/chart: {{ include "hks-manager.chart" . }} +{{ include "hks-manager.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "hks-manager.selectorLabels" -}} +app.kubernetes.io/name: {{ include "hks-manager.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Pod annotations +*/}} +{{- define "hks-manager.pod.annotations" -}} +{{- range $k, $v := .Values.podAnnotations }} +{{- $k }}: {{ $v }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "hks-manager.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "hks-manager.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Renders a value that contains template. +Usage: +{{ include "hks-manager.render" ( dict "value" .Values.path.to.the.Value "context" $) }} +*/}} +{{- define "hks-manager.render" -}} + {{- if typeIs "string" .value }} + {{- tpl .value .context }} + {{- else }} + {{- tpl (.value | toYaml) .context }} + {{- end }} +{{- end -}} diff --git a/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/deployment.yaml b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/deployment.yaml new file mode 100644 index 0000000..f40ff8a --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/deployment.yaml @@ -0,0 +1,89 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "hks-manager.fullname" . }} + labels: + {{- include "hks-manager.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "hks-manager.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- include "hks-manager.pod.annotations" . | nindent 8 }} + labels: + {{- include "hks-manager.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "hks-manager.serviceAccountName" . }} + {{- if .Values.useUserNamespaces }} + hostUsers: true + {{- end }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + {{- with (concat .Values.podEnvs .Values.defaultEnvs) | uniq }} + {{- toYaml . | nindent 12}} + {{- end }} + - name: PORT + value: {{ .Values.podContainerPort | quote }} + ports: + - name: http + containerPort: {{ .Values.podContainerPort }} + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: http + readinessProbe: + httpGet: + path: /healthz + port: http + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.extraContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 10 }} + {{- end }} diff --git a/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/extra-manifests.yaml b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/extra-manifests.yaml new file mode 100644 index 0000000..4501047 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{- range .Values.extraDeploy }} +--- +{{ include "hks-manager.render" (dict "value" . "context" $) }} +{{- end }} diff --git a/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/hpa.yaml b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/hpa.yaml new file mode 100644 index 0000000..8eef778 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "hks-manager.fullname" . }} + labels: + {{- include "hks-manager.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "hks-manager.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/httproute.yaml b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/httproute.yaml new file mode 100644 index 0000000..8f2e8de --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/httproute.yaml @@ -0,0 +1,38 @@ +{{- if .Values.httpRoute.enabled -}} +{{- $fullName := include "hks-manager.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ $fullName }} + labels: + {{- include "hks-manager.labels" . | nindent 4 }} + {{- with .Values.httpRoute.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + parentRefs: + {{- with .Values.httpRoute.parentRefs }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.httpRoute.hostnames }} + hostnames: + {{- toYaml . | nindent 4 }} + {{- end }} + rules: + {{- range .Values.httpRoute.rules }} + {{- with .matches }} + - matches: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .filters }} + filters: + {{- toYaml . | nindent 8 }} + {{- end }} + backendRefs: + - name: {{ $fullName }} + port: {{ $svcPort }} + weight: 1 + {{- end }} +{{- end }} diff --git a/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/ingress.yaml b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/ingress.yaml new file mode 100644 index 0000000..3c00e63 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/ingress.yaml @@ -0,0 +1,45 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "hks-manager.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "hks-manager.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- end }} + {{- end }} +{{- end }} diff --git a/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/rbac.yaml b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/rbac.yaml new file mode 100644 index 0000000..59e46d3 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/rbac.yaml @@ -0,0 +1,30 @@ +{{- if .Values.rbac.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "hks-manager.labels" . | nindent 4 }} + name: {{ include "hks-manager.fullname" . }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - "*" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hks-manager.fullname" . }} + labels: + {{- include "hks-manager.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "hks-manager.serviceAccountName" . }} + namespace: {{ .Release.Namespace}} +roleRef: + kind: ClusterRole + name: {{ include "hks-manager.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/service.yaml b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/service.yaml new file mode 100644 index 0000000..d485cea --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/service.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hks-manager.fullname" . }} + labels: + {{- include "hks-manager.labels" . | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + internalTrafficPolicy: {{ .Values.service.internalTrafficPolicy }} + ipFamilyPolicy: {{ .Values.service.ipFamilyPolicy }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "hks-manager.selectorLabels" . | nindent 4 }} diff --git a/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/serviceaccount.yaml b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/serviceaccount.yaml new file mode 100644 index 0000000..d1f9532 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hks-manager.serviceAccountName" . }} + labels: + {{- include "hks-manager.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/servicemonitor.yaml b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/servicemonitor.yaml new file mode 100644 index 0000000..8651c4b --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/servicemonitor.yaml @@ -0,0 +1,83 @@ +{{- if and (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "hks-manager.fullname" . }} + labels: + {{- include "hks-manager.labels" . | nindent 4 }} + {{- with .Values.metrics.serviceMonitor.selector }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.metrics.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.metrics.serviceMonitor.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.metrics.service.portName }} + {{- with .Values.metrics.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.metrics.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + path: /metrics + {{- with .Values.metrics.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.metrics.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + honorLabels: {{ .Values.metrics.serviceMonitor.honorLabels }} + {{- with .Values.metrics.serviceMonitor.scheme }} + scheme: {{ . }} + {{- end }} + {{- with .Values.metrics.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ include "hks-manager.namespace" . }} + selector: + matchLabels: + {{- include "hks-manager.selectorLabels" (dict "context" . "component" .Values.name "name" (printf "%s-metrics" .Values.name)) | nindent 6 }} +{{- end }} +--- +{{- if .Values.metrics.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hks-manager.fullname" . }}-metrics + namespace: {{ include "hks-manager.namespace" . }} + labels: + {{- include "hks-manager.labels" (dict "context" . "component" .Values.name "name" (printf "%s-metrics" .Values.name)) | nindent 4 }} + {{- with .Values.metrics.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- if .Values.metrics.prometheusAnnotations.enabled }} + prometheus.io/port: {{ .Values.metrics.prometheusAnnotations.port | quote }} + prometheus.io/scheme: {{ .Values.metrics.prometheusAnnotations.scheme | quote}} + prometheus.io/path: {{ .Values.metrics.prometheusAnnotations.path | quote}} + prometheus.io/scrape: "true" + {{- end }} + {{- range $key, $value := .Values.metrics.service.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + type: {{ .Values.metrics.service.type }} + ipFamilyPolicy: {{ .Values.metrics.service.ipFamilyPolicy }} + ports: + - name: metrics + protocol: TCP + port: {{ .Values.metrics.service.servicePort }} + targetPort: metrics + selector: + {{- include "hks-manager.selectorLabels" (dict "context" . "name" .Values.name) | nindent 4 }} +{{- end }} diff --git a/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.schema.json b/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.schema.json new file mode 100644 index 0000000..315d766 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.schema.json @@ -0,0 +1,264 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "affinity": { + "properties": {}, + "type": "object" + }, + "autoscaling": { + "properties": { + "enabled": { + "type": "boolean" + }, + "maxReplicas": { + "type": "integer" + }, + "minReplicas": { + "type": "integer" + }, + "targetCPUUtilizationPercentage": { + "type": "integer" + } + }, + "type": "object" + }, + "commonLabels": { + "properties": {}, + "type": "object" + }, + "defaultEnvs": { + "type": "array" + }, + "extraContainers": { + "type": "array" + }, + "extraDeploy": { + "type": "array" + }, + "fullnameOverride": { + "type": "string" + }, + "httpRoute": { + "properties": { + "annotations": { + "properties": {}, + "type": "object" + }, + "enabled": { + "type": "boolean" + }, + "hostnames": { + "items": { + "type": "string" + }, + "type": "array" + }, + "parentRefs": { + "items": { + "properties": { + "name": { + "type": "string" + }, + "sectionName": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "rules": { + "items": { + "properties": { + "matches": { + "items": { + "properties": { + "path": { + "properties": { + "type": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + }, + "image": { + "properties": { + "pullPolicy": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "tag": { + "type": "string" + } + }, + "type": "object" + }, + "imagePullSecrets": { + "type": "array" + }, + "ingress": { + "properties": { + "annotations": { + "properties": {}, + "type": "object" + }, + "className": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "hosts": { + "items": { + "properties": { + "host": { + "type": "string" + }, + "paths": { + "items": { + "properties": { + "path": { + "type": "string" + }, + "pathType": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + }, + "type": "array" + }, + "tls": { + "type": "array" + } + }, + "type": "object" + }, + "initContainers": { + "type": "array" + }, + "nameOverride": { + "type": "string" + }, + "nodeSelector": { + "properties": {}, + "type": "object" + }, + "podAnnotations": { + "properties": {}, + "type": "object" + }, + "podContainerPort": { + "type": "integer" + }, + "podEnvs": { + "items": { + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "podSecurityContext": { + "properties": {}, + "type": "object" + }, + "replicaCount": { + "type": "integer" + }, + "resources": { + "properties": {}, + "type": "object" + }, + "securityContext": { + "properties": {}, + "type": "object" + }, + "service": { + "properties": { + "annotations": { + "properties": {}, + "type": "object" + }, + "port": { + "type": "integer" + }, + "prometheus": { + "properties": { + "enabled": { + "type": "boolean" + }, + "path": { + "type": "string" + }, + "port": { + "type": "integer" + }, + "scheme": { + "type": "string" + } + }, + "type": "object" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "serviceAccount": { + "properties": { + "annotations": { + "properties": {}, + "type": "object" + }, + "create": { + "type": "boolean" + }, + "name": { + "type": "string" + } + }, + "type": "object" + }, + "tolerations": { + "type": "array" + }, + "volumeMounts": { + "type": "array" + }, + "volumes": { + "type": "array" + } + }, + "type": "object" +} diff --git a/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.yaml b/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.yaml new file mode 100644 index 0000000..bad714f --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.yaml @@ -0,0 +1,227 @@ +# Default values for hks-manager. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# -- Number of replicas for that pod +replicaCount: 1 + +image: + # -- The container registry and image to use. + repository: hegerdes/debug + # -- Pull policy of that image. + pullPolicy: IfNotPresent + # -- The image tag and/or sha. + tag: hks-manager + +# -- Any repository secrets needed to pull the image. +imagePullSecrets: [] +# -- Override the application name. +nameOverride: "" +# -- Override full release name. +fullnameOverride: "" + +# -- Labels applied to all manifests. +commonLabels: {} +# -- Any additional init containers. +initContainers: [] +# -- Any additional containers. +extraContainers: [] + +serviceAccount: + # -- Specifies whether a service account should be created. + create: false + # -- Annotations to add to the service account. + annotations: {} + # -- The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template. + name: "" + +rbac: + # -- En able (Cluster)Role and (Cluster)RoleBinding creation. + enabled: true + +# -- Extra annotations for the pod. +podAnnotations: {} + +# -- App and Container note. Change also in ENVs. +podContainerPort: 8080 + +# -- List of ENVs to configure the app. +podEnvs: [] + # - name: MY_ENV + # value: "MY_VAL" + # - name: ACCESS_TOKEN + # valueFrom: + # secretKeyRef: + # key: token + # name: sec_name + +# -- List of default ENVs. No need to change +defaultEnvs: [] + +# -- If pod should use user namespaces. Must be supported by CRI. See https://kubernetes.io/docs/tasks/configure-pod-container/user-namespaces/ +useUserNamespaces: false + +# -- PodSecurity settings that will be applied to all containers. +podSecurityContext: {} + # fsGroup: 2000 + +# -- Security settings for the container. +securityContext: {} + # runAsNonRoot: true + # runAsUser: 1000 + # runAsGroup: 1000 + # allowPrivilegeEscalation: false + # capabilities: + # drop: [ALL] + # privileged: false + # readOnlyRootFilesystem: true + +# -- Resources for the container. +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +# -- Volume mount's for container. +volumeMounts: [] + +# -- Volumes where data should be persisted. +volumes: [] + # - name: config-vol + # configMap: + # name: log-config + # items: + # - key: log_level + # path: log_level + +# -- Node selector for pod. +nodeSelector: {} + +# -- Tolerations for pod. +tolerations: [] + +# -- Affinity for pod. +affinity: {} + +# -- How the service is exposed. +service: + # -- Service type. + type: ClusterIP + # -- Service and container port. + port: 8080 + # -- Annotations for the service. + annotations: {} + # -- Service traffic policy. + internalTrafficPolicy: Cluster + # -- Service IP family. + ipFamilyPolicy: SingleStack + +# -- Metrics configuration. +metrics: + # -- If metrics endpoint should be enabled. + enabled: false + service: + # -- Service type. + type: ClusterIP + # -- Service and container port. + port: 8080 + # -- Annotations for the service. + annotations: {} + # -- Service IP family + ipFamilyPolicy: SingleStack + # -- Prometheus service annotations. + prometheusAnnotations: + enabled: false + scheme: http + path: /metrics + port: 80 + + # -- ServiceMonitor configuration. + serviceMonitor: + enabled: false + interval: 30s + scrapeTimeout: 30s + honorLabels: false + scheme: http + tlsConfig: {} + selector: {} + additionalLabels: {} + annotations: {} + relabelings: [] + metricRelabelings: [] + +# -- How the service is exposed via ingress. +ingress: + # -- Ingress enabled. + enabled: false + # -- Ingress class. + className: nginx + # -- Ingress annotations. + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + # -- Hostname and path config. + - host: ingress.k8s.internal + paths: + - path: / + pathType: Prefix + # -- TLS config. + tls: [] + # - secretName: ingress.k8s.internal + # hosts: + # - ingress.k8s.internal + +# -- How the service is exposed via gateway-apis HTTPRoute. +httpRoute: + # -- HTTPRoute enabled. + enabled: false + # -- HTTPRoute annotations. + annotations: {} + # -- Which Gateways this Route is attached to + parentRefs: + - name: gateway + sectionName: http + # -- Hostnames matching HTTP header. + hostnames: + - "example.com" + # -- List of rules and filters applied. + rules: + - matches: + - path: + type: PathPrefix + value: /headers + # filters: + # - type: RequestHeaderModifier + # requestHeaderModifier: + # set: + # - name: My-Overwrite-Header + # value: this-is-the-only-value + # remove: + # - User-Agent + # - matches: + # - path: + # type: PathPrefix + # value: /echo + # headers: + # - name: version + # value: v2 + + # -- Autoscaling +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +# -- Extra manifests +extraDeploy: [] From ffcec3b712f6786bd0fedecfe5ab41f484de3612 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sun, 11 Jan 2026 14:42:04 +0100 Subject: [PATCH 21/57] feat(hks): add argo appsets for cilium & metrics Signed-off-by: Henrik Gerdes --- .../shared/argo-cilium.yaml | 85 +++++++++++++++++++ .../shared/argo-metrics.yml | 52 ++++++++++++ .../shared/gateway.yaml | 65 ++++++++++++++ 3 files changed, 202 insertions(+) create mode 100644 k8s-apps/hegerdes-kubernetes-service/shared/argo-cilium.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-service/shared/argo-metrics.yml create mode 100644 k8s-apps/hegerdes-kubernetes-service/shared/gateway.yaml diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/argo-cilium.yaml b/k8s-apps/hegerdes-kubernetes-service/shared/argo-cilium.yaml new file mode 100644 index 0000000..9d0ee25 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/shared/argo-cilium.yaml @@ -0,0 +1,85 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: cilium + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: ["missingkey=error"] + generators: + - clusters: + selector: + matchLabels: + hks.hegerdes.com/managed-cluster: "true" + hks.hegerdes.com/cluster-type: "downstream" + template: + metadata: + name: "{{.name}}-cilium" + spec: + project: default + source: + chart: cilium + repoURL: https://helm.cilium.io/ + targetRevision: 1.* + helm: + releaseName: cilium + valuesObject: + cluster: + name: "{{.name}}" + + k8sServiceHost: '{{ index .metadata.annotations "hks.hegerdes.com/public-host" }}' + k8sServicePort: 6443 + kubeProxyReplacement: true + rollOutCiliumPods: true + annotateK8sNode: true + + l7Proxy: false + envoy: + enabled: false + rollOutPods: true + operator: + rollOutPods: true + replicas: 1 + + endpointRoutes: + enabled: true + + loadBalancer: + algorithm: maglev + serviceTopology: true + + hubble: + enabled: false + rollOutPods: true + relay: + enabled: false + rollOutPods: true + ui: + enabled: false + rollOutPods: true + # Needed if tailscale services are routed + socketLB: + hostNamespaceOnly: true + + ipam: + mode: kubernetes + operator: + clusterPoolIPv4PodCIDRList: [10.244.0.0/16] + # clusterPoolIPv6PodCIDRList: ["{{ k8s_pod_network_cidr_ipv6 }}"] + + destination: + server: "{{.server}}" + namespace: kube-system + info: + - name: "Source Info" + value: "https://artifacthub.io/packages/helm/cilium/cilium" + syncPolicy: + automated: + prune: true + selfHeal: true + retry: + limit: 3 + backoff: + duration: 30s + factor: 2 + maxDuration: 5m diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/argo-metrics.yml b/k8s-apps/hegerdes-kubernetes-service/shared/argo-metrics.yml new file mode 100644 index 0000000..20e8336 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/shared/argo-metrics.yml @@ -0,0 +1,52 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: metrics + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: ["missingkey=error"] + generators: + - clusters: + selector: + matchLabels: + hks.hegerdes.com/managed-cluster: "true" + hks.hegerdes.com/cluster-type: "downstream" + template: + metadata: + name: "{{.name}}-metrics" + spec: + project: default + source: + chart: metrics-server + repoURL: https://kubernetes-sigs.github.io/metrics-server/ + targetRevision: 3.* + helm: + releaseName: metrics-server + valuesObject: + # serviceAccount: + # create: false + # rbac: + # create: false + defaultArgs: + - --cert-dir=/tmp + - --kubelet-use-node-status-port + - --metric-resolution=15s + - --profiling=false + - --kubelet-preferred-address-types=InternalDNS,InternalIP,ExternalDNS,ExternalIP + # - --kubeconfig=TODO + # - --kubelet-certificate-authority string Path to the CA to use to validate the Kubelet's serving certificates. + # - --kubelet-client-certificate string Path to a client cert file for TLS. + # - --kubelet-client-key string + destination: + server: "{{.server}}" + namespace: kube-system + info: + - name: Chart-Info Metrics-Server + value: https://artifacthub.io/packages/helm/metrics-server/metrics-server + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/gateway.yaml b/k8s-apps/hegerdes-kubernetes-service/shared/gateway.yaml new file mode 100644 index 0000000..1b98814 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/shared/gateway.yaml @@ -0,0 +1,65 @@ +# https://docs.nginx.com/nginx-gateway-fabric/traffic-management/tls-passthrough/ +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: hks-kube-gw + namespace: nginx-gateway +spec: + gatewayClassName: nginx + infrastructure: + annotations: + load-balancer.hetzner.cloud/location: nbg1 + load-balancer.hetzner.cloud/name: k8s-hks-gateway-lb + load-balancer.hetzner.cloud/network: k8s-network + load-balancer.hetzner.cloud/use-private-ip: "true" + load-balancer.hetzner.cloud/uses-proxyprotocol: "true" + parametersRef: + group: gateway.nginx.org + kind: NginxProxy + name: ngf-proxy-config + listeners: + - name: tls + protocol: TLS + port: 6443 + hostname: "*.hks.eu-central.hegerdes.com" + allowedRoutes: + namespaces: + from: All + kinds: + - kind: TLSRoute + tls: + mode: Passthrough + # certificateRefs: + # - kind: Secret + # name: kube-apiserver-tls + # namespace: test1 + # - name: test1-tls + # protocol: TLS + # port: 443 + # hostname: bar.example.com + # tls: + # certificateRefs: + # - kind: Secret + # group: "" + # name: bar-example-com-cert +# --- +# apiVersion: gateway.networking.k8s.io/v1alpha2 +# kind: TLSRoute +# metadata: +# name: test1 +# namespace: test1 +# spec: +# hostnames: +# - test1.localhost +# parentRefs: +# - name: hks-kube-gw +# namespace: nginx-gateway +# # sectionName: test1-tls +# # kind: Gateway +# # group: gateway.networking.k8s.io +# rules: +# - backendRefs: +# - name: kube-apiserver +# kind: Service +# namespace: test1 +# port: 6443 From dd69e8912a28a2bbfa2ba5c2e9826f68bba3cd93 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sun, 11 Jan 2026 14:49:24 +0100 Subject: [PATCH 22/57] fix(hks) set fsgroup for etcd Signed-off-by: Henrik Gerdes --- .../hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml index 3356144..ad8b6ba 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml @@ -130,6 +130,7 @@ spec: runAsNonRoot: true runAsUser: 1000 runAsGroup: 1000 + fsGroup: 1000 seccompProfile: type: RuntimeDefault volumes: From fcc30aaed635d2f4b0335d283abe17f27568c0ea Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sun, 11 Jan 2026 14:59:44 +0100 Subject: [PATCH 23/57] fix(hks) set fsgroup for kube-apiserver Signed-off-by: Henrik Gerdes --- .../hks/templates/kube-apiserver.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml index 955892d..2bc2a14 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml @@ -30,6 +30,7 @@ spec: runAsNonRoot: true runAsUser: 1000 runAsGroup: 1000 + fsGroup: 1000 seccompProfile: type: RuntimeDefault automountServiceAccountToken: false From 5352838d2b81262ccf33e37a55ff76908c531010 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sun, 11 Jan 2026 15:03:40 +0100 Subject: [PATCH 24/57] fix(hks) use empty dir for audit-log for now Signed-off-by: Henrik Gerdes --- .../hks/templates/kube-apiserver.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml index 2bc2a14..8d4c63b 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml @@ -178,8 +178,10 @@ spec: # priorityClassName: system-node-critical volumes: - name: audit-log - persistentVolumeClaim: - claimName: kube-adit-log + emptyDir: + sizeLimit: 265Mi + # persistentVolumeClaim: + # claimName: kube-adit-log - name: kube-extra-conf projected: sources: From c31a981f3c3d2f771812650f007531ab179d2945 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sun, 11 Jan 2026 15:15:43 +0100 Subject: [PATCH 25/57] fix(hks): turn off proxy proto Signed-off-by: Henrik Gerdes --- .../shared/gateway.yaml | 2 +- .../shared/nginx-gateway-conf.yaml | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 k8s-apps/hegerdes-kubernetes-service/shared/nginx-gateway-conf.yaml diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/gateway.yaml b/k8s-apps/hegerdes-kubernetes-service/shared/gateway.yaml index 1b98814..2ddb466 100644 --- a/k8s-apps/hegerdes-kubernetes-service/shared/gateway.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/shared/gateway.yaml @@ -12,7 +12,7 @@ spec: load-balancer.hetzner.cloud/name: k8s-hks-gateway-lb load-balancer.hetzner.cloud/network: k8s-network load-balancer.hetzner.cloud/use-private-ip: "true" - load-balancer.hetzner.cloud/uses-proxyprotocol: "true" + # load-balancer.hetzner.cloud/uses-proxyprotocol: "true" parametersRef: group: gateway.nginx.org kind: NginxProxy diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/nginx-gateway-conf.yaml b/k8s-apps/hegerdes-kubernetes-service/shared/nginx-gateway-conf.yaml new file mode 100644 index 0000000..e2eb318 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/shared/nginx-gateway-conf.yaml @@ -0,0 +1,21 @@ +apiVersion: gateway.nginx.org/v1alpha2 +kind: NginxProxy +metadata: + name: ngf-proxy-config + namespace: nginx-gateway +spec: + rewriteClientIP: + # mode: ProxyProtocol + trustedAddresses: + - type: CIDR + value: "0.0.0.0/0" + kubernetes: + daemonSet: + container: + resources: + requests: + cpu: 100m + memory: 128Mi + service: + externalTrafficPolicy: Local + type: LoadBalancer From fd3cb6c084ae33d10961894366371483a5ee175f Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sun, 11 Jan 2026 15:33:38 +0100 Subject: [PATCH 26/57] fix(hks): debug tls route Signed-off-by: Henrik Gerdes --- .../{argo-hks-appset.yaml => argo-appset-hks.yaml} | 0 .../{argo-hks.yaml => argo-hks-manager.yaml} | 4 ++-- .../hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml | 1 + .../hegerdes-kubernetes-service/hks/templates/tls-route.yaml | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) rename k8s-apps/hegerdes-kubernetes-service/{argo-hks-appset.yaml => argo-appset-hks.yaml} (100%) rename k8s-apps/hegerdes-kubernetes-service/{argo-hks.yaml => argo-hks-manager.yaml} (86%) diff --git a/k8s-apps/hegerdes-kubernetes-service/argo-hks-appset.yaml b/k8s-apps/hegerdes-kubernetes-service/argo-appset-hks.yaml similarity index 100% rename from k8s-apps/hegerdes-kubernetes-service/argo-hks-appset.yaml rename to k8s-apps/hegerdes-kubernetes-service/argo-appset-hks.yaml diff --git a/k8s-apps/hegerdes-kubernetes-service/argo-hks.yaml b/k8s-apps/hegerdes-kubernetes-service/argo-hks-manager.yaml similarity index 86% rename from k8s-apps/hegerdes-kubernetes-service/argo-hks.yaml rename to k8s-apps/hegerdes-kubernetes-service/argo-hks-manager.yaml index f4bc214..90e9072 100644 --- a/k8s-apps/hegerdes-kubernetes-service/argo-hks.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/argo-hks-manager.yaml @@ -12,13 +12,13 @@ spec: source: repoURL: https://github.com/hegerdes/GitOps.git targetRevision: feat/hks - path: k8s-apps/hegerdes-kubernetes-sevice/hks + path: k8s-apps/hegerdes-kubernetes-service/hks-manager helm: valuesObject: extraObjects: [] destination: server: https://kubernetes.default.svc - namespace: test1 + namespace: hks-manager syncPolicy: automated: prune: true diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml index ad8b6ba..976eb2f 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-etcd.yaml @@ -216,6 +216,7 @@ spec: securityContext: runAsNonRoot: true runAsUser: 1000 + fsGroup: 1000 runAsGroup: 1000 seccompProfile: type: RuntimeDefault diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/tls-route.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/tls-route.yaml index 69caaee..0ad2d52 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/tls-route.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/tls-route.yaml @@ -4,7 +4,7 @@ metadata: name: "{{ .Release.Namespace }}" spec: hostnames: - - "{{ .Release.Namespace }}.localhost" + # - "{{ .Release.Namespace }}.localhost" - "{{ .Release.Namespace }}.{{ .Values.controlplane.domainSuffix }}" parentRefs: - name: hks-kube-gw From 9aeddb6d050ea8459ceb687265b76531c5672083 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sun, 11 Jan 2026 16:11:03 +0100 Subject: [PATCH 27/57] debug(hks): test cilium tls gw Signed-off-by: Henrik Gerdes --- .../hegerdes-kubernetes-service/hks/templates/tls-route.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/tls-route.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/tls-route.yaml index 0ad2d52..de5c8f4 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/tls-route.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/tls-route.yaml @@ -7,8 +7,8 @@ spec: # - "{{ .Release.Namespace }}.localhost" - "{{ .Release.Namespace }}.{{ .Values.controlplane.domainSuffix }}" parentRefs: - - name: hks-kube-gw - namespace: nginx-gateway + - name: cilium-gateway + namespace: cilium-gateway rules: - backendRefs: - name: kube-apiserver From 6812dda3816b566d5fd2bd3a9506e8a52310bcb8 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sun, 11 Jan 2026 16:42:56 +0100 Subject: [PATCH 28/57] feat:(hks) add shared conf dir Signed-off-by: Henrik Gerdes --- .../clustes-shared/rbac.yaml | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml diff --git a/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml new file mode 100644 index 0000000..259aae5 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml @@ -0,0 +1,24 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: anonymous-read-cluster-info + namespace: kube-public +subjects: + - kind: User + name: system:anonymous + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: Role + name: read-cluster-info + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: read-cluster-info + namespace: kube-public +rules: + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["cluster-info"] + verbs: ["get", "list"] From f42075d96f464ab867298ae47ac7eed46ff83a69 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sun, 11 Jan 2026 16:45:26 +0100 Subject: [PATCH 29/57] fix(hks): also manage ns kube-public with argo Signed-off-by: Henrik Gerdes --- k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml index af5c69b..e31b958 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml @@ -24,7 +24,7 @@ spec: data: server: https://kube-apiserver.{{ .Release.Namespace }}:6443 name: "{{ .Release.Namespace }}" - namespaces: kube-system,cilium,cilium-secrets,default,kubelet-serving-cert-approver + namespaces: kube-system,kube-public,cilium,cilium-secrets,default,kubelet-serving-cert-approver clusterResources: "true" config: | { From f71ed0b9a93a6b2a27f141aade07adc9d7599bb0 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sun, 11 Jan 2026 18:15:38 +0100 Subject: [PATCH 30/57] fix(hks): use cilium gateway Signed-off-by: Henrik Gerdes --- .../hegerdes-kubernetes-service/Kube-tasks.md | 39 ++++++++ .../argo-hks-manager.yaml | 6 +- .../shared/argo-hks-conf.yaml | 39 ++++++++ .../shared/gateway.yaml | 88 ++++++++++--------- .../shared/nginx-gateway-conf.yaml | 21 ----- 5 files changed, 128 insertions(+), 65 deletions(-) create mode 100644 k8s-apps/hegerdes-kubernetes-service/Kube-tasks.md create mode 100644 k8s-apps/hegerdes-kubernetes-service/shared/argo-hks-conf.yaml delete mode 100644 k8s-apps/hegerdes-kubernetes-service/shared/nginx-gateway-conf.yaml diff --git a/k8s-apps/hegerdes-kubernetes-service/Kube-tasks.md b/k8s-apps/hegerdes-kubernetes-service/Kube-tasks.md new file mode 100644 index 0000000..4c90b71 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/Kube-tasks.md @@ -0,0 +1,39 @@ +Hennes Kubernes Service + +Tasks: + * Create deploymets for: + * kube-apiserver ✅ + * kube-controller ✅ + * etcd ✅ + * Create certs - hacky ✅ + * Ensure communication ✅ + * Refine Deployments ✅ + * Create certs - prod ✅ + * GitOps: + * HKS ✅ + * Slave Cluster + * Generate join token + * POC ✅ + * Service-Controller + * Join Worker + * Ensure Connectivity + * Worker -> CP + * CP -> Worker + * Worker Apps + * CoreDNS + * Cilium + + +```bash +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.2/cert-manager.yaml +kubectl create namespace argocd +kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/v3.2.3/manifests/install.yaml +kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.4.1/standard-install.yaml +kubectl kustomize "https://github.com/nginx/nginx-gateway-fabric/config/crd/gateway-api/experimental?ref=v2.3.0" | kubectl apply -f - --server-side + + +kaf argo-hks-shared.yaml +kaf ../../k8s-cluster-hcloud-critical/argo-external-secrets.yml +kaf ../argo-nginx-gateway-fabric.yml +kaf argo-hks-appset.yaml +``` diff --git a/k8s-apps/hegerdes-kubernetes-service/argo-hks-manager.yaml b/k8s-apps/hegerdes-kubernetes-service/argo-hks-manager.yaml index 90e9072..f02c566 100644 --- a/k8s-apps/hegerdes-kubernetes-service/argo-hks-manager.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/argo-hks-manager.yaml @@ -1,12 +1,10 @@ apiVersion: argoproj.io/v1alpha1 kind: Application metadata: - name: hks + name: hks-manager namespace: argocd labels: - name: hks - finalizers: - - resources-finalizer.argocd.argoproj.io/background + name: hks-manager spec: project: default source: diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/argo-hks-conf.yaml b/k8s-apps/hegerdes-kubernetes-service/shared/argo-hks-conf.yaml new file mode 100644 index 0000000..5a8e796 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/shared/argo-hks-conf.yaml @@ -0,0 +1,39 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: cluster-conf + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: ["missingkey=error"] + generators: + - clusters: + selector: + matchLabels: + hks.hegerdes.com/managed-cluster: "true" + hks.hegerdes.com/cluster-type: "downstream" + template: + metadata: + name: "{{.name}}-cluster-conf" + spec: + project: default + source: + repoURL: https://github.com/hegerdes/GitOps.git + targetRevision: feat/hks + path: k8s-apps/hegerdes-kubernetes-service/clustes-shared + destination: + server: "{{.server}}" + namespace: kube-system + info: + - name: "Source Info" + value: "https://github.com/hegerdes/GitOps" + syncPolicy: + automated: + prune: true + selfHeal: true + retry: + limit: 3 + backoff: + duration: 30s + factor: 2 + maxDuration: 5m diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/gateway.yaml b/k8s-apps/hegerdes-kubernetes-service/shared/gateway.yaml index 2ddb466..166113f 100644 --- a/k8s-apps/hegerdes-kubernetes-service/shared/gateway.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/shared/gateway.yaml @@ -1,22 +1,18 @@ -# https://docs.nginx.com/nginx-gateway-fabric/traffic-management/tls-passthrough/ apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: - name: hks-kube-gw - namespace: nginx-gateway + name: cilium-gateway + namespace: cilium-gateway spec: - gatewayClassName: nginx + gatewayClassName: cilium infrastructure: annotations: load-balancer.hetzner.cloud/location: nbg1 load-balancer.hetzner.cloud/name: k8s-hks-gateway-lb load-balancer.hetzner.cloud/network: k8s-network load-balancer.hetzner.cloud/use-private-ip: "true" - # load-balancer.hetzner.cloud/uses-proxyprotocol: "true" - parametersRef: - group: gateway.nginx.org - kind: NginxProxy - name: ngf-proxy-config + load-balancer.hetzner.cloud/uses-proxyprotocol: "true" + load-balancer.hetzner.cloud/ipv6-disabled: "true" listeners: - name: tls protocol: TLS @@ -29,37 +25,49 @@ spec: - kind: TLSRoute tls: mode: Passthrough - # certificateRefs: - # - kind: Secret - # name: kube-apiserver-tls - # namespace: test1 - # - name: test1-tls - # protocol: TLS - # port: 443 - # hostname: bar.example.com - # tls: - # certificateRefs: - # - kind: Secret - # group: "" - # name: bar-example-com-cert # --- -# apiVersion: gateway.networking.k8s.io/v1alpha2 -# kind: TLSRoute +# https://docs.nginx.com/nginx-gateway-fabric/traffic-management/tls-passthrough/ +# apiVersion: gateway.networking.k8s.io/v1 +# kind: Gateway # metadata: -# name: test1 -# namespace: test1 +# name: hks-kube-gw +# namespace: nginx-gateway # spec: -# hostnames: -# - test1.localhost -# parentRefs: -# - name: hks-kube-gw -# namespace: nginx-gateway -# # sectionName: test1-tls -# # kind: Gateway -# # group: gateway.networking.k8s.io -# rules: -# - backendRefs: -# - name: kube-apiserver -# kind: Service -# namespace: test1 -# port: 6443 +# gatewayClassName: nginx +# infrastructure: +# annotations: +# load-balancer.hetzner.cloud/location: nbg1 +# load-balancer.hetzner.cloud/name: k8s-hks-gateway-lb +# load-balancer.hetzner.cloud/network: k8s-network +# load-balancer.hetzner.cloud/use-private-ip: "true" +# load-balancer.hetzner.cloud/uses-proxyprotocol: "true" +# parametersRef: +# group: gateway.nginx.org +# kind: NginxProxy +# name: ngf-proxy-config +# listeners: +# - name: tls +# protocol: TLS +# port: 6443 +# hostname: "*.hks.eu-central.hegerdes.com" +# allowedRoutes: +# namespaces: +# from: All +# kinds: +# - kind: TLSRoute +# tls: +# mode: Passthrough +# # certificateRefs: +# # - kind: Secret +# # name: kube-apiserver-tls +# # namespace: test1 +# # - name: test1-tls +# # protocol: TLS +# # port: 443 +# # hostname: bar.example.com +# # tls: +# # certificateRefs: +# # - kind: Secret +# # group: "" +# # name: bar-example-com-cert +# --- diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/nginx-gateway-conf.yaml b/k8s-apps/hegerdes-kubernetes-service/shared/nginx-gateway-conf.yaml deleted file mode 100644 index e2eb318..0000000 --- a/k8s-apps/hegerdes-kubernetes-service/shared/nginx-gateway-conf.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: gateway.nginx.org/v1alpha2 -kind: NginxProxy -metadata: - name: ngf-proxy-config - namespace: nginx-gateway -spec: - rewriteClientIP: - # mode: ProxyProtocol - trustedAddresses: - - type: CIDR - value: "0.0.0.0/0" - kubernetes: - daemonSet: - container: - resources: - requests: - cpu: 100m - memory: 128Mi - service: - externalTrafficPolicy: Local - type: LoadBalancer From 38070f751cc22b79b0b77efd8bbd1d22ee36604c Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Fri, 16 Jan 2026 20:56:34 +0100 Subject: [PATCH 31/57] chore: delete test1 cluster for now Signed-off-by: Henrik Gerdes --- k8s-apps/hegerdes-kubernetes-service/clusters/test2/values.yaml | 0 .../hks-manager/templates/deployment.yaml | 2 ++ k8s-apps/hegerdes-kubernetes-service/hks-manager/values.yaml | 1 + 3 files changed, 3 insertions(+) delete mode 100644 k8s-apps/hegerdes-kubernetes-service/clusters/test2/values.yaml diff --git a/k8s-apps/hegerdes-kubernetes-service/clusters/test2/values.yaml b/k8s-apps/hegerdes-kubernetes-service/clusters/test2/values.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/deployment.yaml b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/deployment.yaml index f40ff8a..587b50e 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/deployment.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/deployment.yaml @@ -46,6 +46,8 @@ spec: {{- with (concat .Values.podEnvs .Values.defaultEnvs) | uniq }} {{- toYaml . | nindent 12}} {{- end }} + - name: HKS_CLUSTER_ZONE + value: {{ .Values.clusterZone | quote }} - name: PORT value: {{ .Values.podContainerPort | quote }} ports: diff --git a/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.yaml b/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.yaml index bad714f..87ad81a 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.yaml @@ -4,6 +4,7 @@ # -- Number of replicas for that pod replicaCount: 1 +clusterZone: hks.eu-central.hegerdes.com image: # -- The container registry and image to use. From 78bb75433d51e58adff73c8e675c4f9417cd49da Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Fri, 16 Jan 2026 21:03:37 +0100 Subject: [PATCH 32/57] fix(hks): add cilium gateway Signed-off-by: Henrik Gerdes --- k8s-apps/hegerdes-kubernetes-service/shared/gateway.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/gateway.yaml b/k8s-apps/hegerdes-kubernetes-service/shared/gateway.yaml index 166113f..a6882c6 100644 --- a/k8s-apps/hegerdes-kubernetes-service/shared/gateway.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/shared/gateway.yaml @@ -1,3 +1,8 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cilium-gateway +--- apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: From efe713d1dbd939d0ee9df40aede02d476ed81ed0 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 17 Jan 2026 19:10:31 +0100 Subject: [PATCH 33/57] fix(hks): debug manager Signed-off-by: Henrik Gerdes --- .../hegerdes-kubernetes-service/hks-manager/values.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.yaml b/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.yaml index 87ad81a..81ba633 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.yaml @@ -10,7 +10,7 @@ image: # -- The container registry and image to use. repository: hegerdes/debug # -- Pull policy of that image. - pullPolicy: IfNotPresent + pullPolicy: Always # -- The image tag and/or sha. tag: hks-manager @@ -104,7 +104,9 @@ volumes: [] # path: log_level # -- Node selector for pod. -nodeSelector: {} +# nodeSelector: {} +nodeSelector: + kubernetes.io/arch: arm64 # -- Tolerations for pod. tolerations: [] From 8f42f8c7ffc69d195b16d49218c8b01e495454b2 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 17 Jan 2026 20:00:23 +0100 Subject: [PATCH 34/57] fix(hks): add kubeadm conf Signed-off-by: Henrik Gerdes --- .../clustes-shared/cm-kubeadm-conf.yaml | 22 +++++++++++++++++++ .../clustes-shared/rbac.yaml | 15 +++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 k8s-apps/hegerdes-kubernetes-service/clustes-shared/cm-kubeadm-conf.yaml diff --git a/k8s-apps/hegerdes-kubernetes-service/clustes-shared/cm-kubeadm-conf.yaml b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/cm-kubeadm-conf.yaml new file mode 100644 index 0000000..fcfd9c6 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/cm-kubeadm-conf.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kubeadm-config + namespace: kube-system +data: + ClusterConfiguration: | + apiVersion: kubeadm.k8s.io/v1beta4 + kind: ClusterConfiguration + caCertificateValidityPeriod: 87600h0m0s + certificateValidityPeriod: 8760h0m0s + certificatesDir: /etc/kubernetes/pki + clusterName: test1 + controlPlaneEndpoint: test1.hks.eu-central.hegerdes.com:6443 + dns: {} + encryptionAlgorithm: RSA-2048 + imageRepository: registry.k8s.io + kubernetesVersion: v1.35.0 + networking: + dnsDomain: cluster.local + podSubnet: 10.244.0.0/16 + serviceSubnet: 10.96.0.0/16 diff --git a/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml index 259aae5..dc4aa59 100644 --- a/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml @@ -1,5 +1,19 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding +metadata: + name: bootstrap-kubeadm-read + namespace: kube-system +subjects: + - kind: Group + name: system:bootstrap + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: Role + name: extension-apiserver-authentication-reader + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding metadata: name: anonymous-read-cluster-info namespace: kube-public @@ -22,3 +36,4 @@ rules: resources: ["configmaps"] resourceNames: ["cluster-info"] verbs: ["get", "list"] +--- From 5f8e502db76aaac30fa55a29c2ba6ac232067ad1 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 17 Jan 2026 20:08:01 +0100 Subject: [PATCH 35/57] fix(hks): kubeadm conf rbac Signed-off-by: Henrik Gerdes --- .../clustes-shared/rbac.yaml | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml index dc4aa59..ecab602 100644 --- a/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml @@ -1,16 +1,35 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: bootstrap-kubeadm-read + name: kubeadm:nodes-kubeadm-config namespace: kube-system -subjects: - - kind: Group - name: system:bootstrap - apiGroup: rbac.authorization.k8s.io roleRef: - kind: Role - name: extension-apiserver-authentication-reader apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubeadm:nodes-kubeadm-config +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:bootstrappers:kubeadm:default-node-token + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kubeadm:nodes-kubeadm-config + namespace: kube-system +rules: + - apiGroups: + - "" + resourceNames: + - kubeadm-config + resources: + - configmaps + verbs: + - get + --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding From b56076c7113c32acf4f8c261f780f1df254533c8 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 17 Jan 2026 20:10:14 +0100 Subject: [PATCH 36/57] fix(hks): kubeadm conf rbac Signed-off-by: Henrik Gerdes --- k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml index ecab602..d9679fa 100644 --- a/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml @@ -14,6 +14,9 @@ subjects: - apiGroup: rbac.authorization.k8s.io kind: Group name: system:nodes + - apiGroup: rbac.authorization.k8s.io + kind: User + name: system:bootstrap:60ae1c --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role From 9b5c0b25798eb4c4361b594e8c6d822ef20ec051 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 17 Jan 2026 20:13:52 +0100 Subject: [PATCH 37/57] fix(hks): add kubelet conf Signed-off-by: Henrik Gerdes --- .../clustes-shared/cm-kubelet-conf.yaml | 70 +++++++++++++++++++ .../clustes-shared/rbac.yaml | 36 +++++++++- 2 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 k8s-apps/hegerdes-kubernetes-service/clustes-shared/cm-kubelet-conf.yaml diff --git a/k8s-apps/hegerdes-kubernetes-service/clustes-shared/cm-kubelet-conf.yaml b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/cm-kubelet-conf.yaml new file mode 100644 index 0000000..ca9b5fc --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/cm-kubelet-conf.yaml @@ -0,0 +1,70 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kubelet-config + namespace: kube-system +data: + kubelet: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.96.0.10 + clusterDomain: cluster.local + containerRuntimeEndpoint: unix:///run/containerd/containerd.sock + cpuManagerReconcilePeriod: 0s + crashLoopBackOff: {} + enableDebugFlagsHandler: false + enableProfilingHandler: false + evictionPressureTransitionPeriod: 0s + failCgroupV1: true + failSwapOn: true + featureGates: + ClusterTrustBundle: true + ClusterTrustBundleProjection: true + fileCheckFrequency: 0s + healthzBindAddress: 0.0.0.0 + healthzPort: 10248 + httpCheckFrequency: 0s + imageMaximumGCAge: 1h0m0s + imageMinimumGCAge: 5m0s + kind: KubeletConfiguration + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + text: + infoBufferSize: "0" + verbosity: 0 + maxParallelImagePulls: 8 + maxPods: 220 + memorySwap: {} + nodeStatusMaxImages: -1 + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + resolvConf: /etc/resolv.conf + rotateCertificates: true + runtimeRequestTimeout: 0s + seccompDefault: true + serializeImagePulls: false + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + volumeStatsAggPeriod: 5m0s diff --git a/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml index d9679fa..b714dd6 100644 --- a/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml @@ -19,6 +19,26 @@ subjects: name: system:bootstrap:60ae1c --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kubeadm:kubelet-config + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubeadm:kubelet-config +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:bootstrappers:kubeadm:default-node-token + - apiGroup: rbac.authorization.k8s.io + kind: User + name: system:bootstrap:60ae1c +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: kubeadm:nodes-kubeadm-config @@ -32,7 +52,21 @@ rules: - configmaps verbs: - get - +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kubeadm:kubelet-config + namespace: kube-system +rules: + - apiGroups: + - "" + resourceNames: + - kubelet-config + resources: + - configmaps + verbs: + - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding From 65b35711123da9da5bdbc3de1c781402ab1b9026 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 17 Jan 2026 20:18:14 +0100 Subject: [PATCH 38/57] fix(hks): add kubeadm node rbac Signed-off-by: Henrik Gerdes --- .../clustes-shared/rbac.yaml | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml index b714dd6..4caecb3 100644 --- a/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml @@ -1,4 +1,45 @@ apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeadm:get-nodes +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubeadm:get-nodes +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubeadm:get-nodes +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:bootstrappers:kubeadm:default-node-token + - apiGroup: rbac.authorization.k8s.io + kind: User + name: system:bootstrap:60ae1c +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubeadm:node-autoapprove-certificate-rotation +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: kubeadm:nodes-kubeadm-config From 8ad261af142ed008a5866fcfe4249a86ec804fc1 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 17 Jan 2026 20:32:58 +0100 Subject: [PATCH 39/57] fix(hks): add kubeadm node tls rbac Signed-off-by: Henrik Gerdes --- .../clustes-shared/rbac.yaml | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml index 4caecb3..e81c785 100644 --- a/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac.yaml @@ -12,6 +12,22 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kubeadm:node-autoapprove-bootstrap +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:certificates.k8s.io:certificatesigningrequests:nodeclient +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:bootstrappers:kubeadm:default-node-token + - apiGroup: rbac.authorization.k8s.io + kind: User + name: system:bootstrap:60ae1c +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kubeadm:get-nodes roleRef: @@ -28,6 +44,22 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kubeadm:kubelet-bootstrap +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:node-bootstrapper +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:bootstrappers:kubeadm:default-node-token + - apiGroup: rbac.authorization.k8s.io + kind: User + name: system:bootstrap:60ae1c +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kubeadm:node-autoapprove-certificate-rotation roleRef: @@ -38,6 +70,24 @@ subjects: - apiGroup: rbac.authorization.k8s.io kind: Group name: system:nodes + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kubeadm:kubeadm-certs + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubeadm:kubeadm-certs +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:bootstrappers:kubeadm:default-node-token + - apiGroup: rbac.authorization.k8s.io + kind: User + name: system:bootstrap:60ae1c --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -134,3 +184,17 @@ rules: resourceNames: ["cluster-info"] verbs: ["get", "list"] --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kubeadm:kubeadm-certs + namespace: kube-system +rules: + - apiGroups: + - "" + resourceNames: + - kubeadm-certs + resources: + - secrets + verbs: + - get From 1f0d7668ecacb66c187103ed1f5ab100e01c71a8 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 17 Jan 2026 20:58:18 +0100 Subject: [PATCH 40/57] fix(hks): set correct coredns params Signed-off-by: Henrik Gerdes --- k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml index bfa95a8..e174583 100644 --- a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml +++ b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml @@ -27,6 +27,9 @@ spec: helm: releaseName: coredns valuesObject: + service: + clusterIP: "10.96.0.10" + name: "kube-dns" replicaCount: 2 priorityClassName: system-cluster-critical isClusterService: false From 9b964ab2d97a51333cbeddd18b8174b7c12c51ab Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 17 Jan 2026 21:07:36 +0100 Subject: [PATCH 41/57] fix(hks): set coredns kube endpoint Signed-off-by: Henrik Gerdes --- k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml index e174583..4b59edf 100644 --- a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml +++ b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml @@ -71,6 +71,7 @@ spec: - name: kubernetes parameters: cluster.local in-addr.arpa ip6.arpa configBlock: |- + endpoint 'https://{{ index .metadata.annotations "hks.hegerdes.com/public-host" }}:6443' pods insecure fallthrough in-addr.arpa ip6.arpa ttl 30 From d4645ded82acf5b69e9896c1de1d87a5bdda9bf6 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 17 Jan 2026 21:10:56 +0100 Subject: [PATCH 42/57] fix(hks): coredns conf quote Signed-off-by: Henrik Gerdes --- k8s-apps/hegerdes-kubernetes-service/argo-appset-hks.yaml | 5 +++++ k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/k8s-apps/hegerdes-kubernetes-service/argo-appset-hks.yaml b/k8s-apps/hegerdes-kubernetes-service/argo-appset-hks.yaml index 7cd93f2..90eed55 100644 --- a/k8s-apps/hegerdes-kubernetes-service/argo-appset-hks.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/argo-appset-hks.yaml @@ -37,3 +37,8 @@ spec: selfHeal: true syncOptions: - CreateNamespace=true + ignoreDifferences: + - kind: StatefulSet + group: apps + jqPathExpressions: + - .spec.updateStrategy.rollingUpdate.maxUnavailable diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml index 4b59edf..5bcc6a6 100644 --- a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml +++ b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml @@ -71,7 +71,7 @@ spec: - name: kubernetes parameters: cluster.local in-addr.arpa ip6.arpa configBlock: |- - endpoint 'https://{{ index .metadata.annotations "hks.hegerdes.com/public-host" }}:6443' + endpoint https://{{ index .metadata.annotations "hks.hegerdes.com/public-host" }}:6443 pods insecure fallthrough in-addr.arpa ip6.arpa ttl 30 From 9bbc7cf2285e6fc534921b72ec7fe7dfc021ced3 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 17 Jan 2026 21:20:05 +0100 Subject: [PATCH 43/57] fix(hks): coredns create serviceaccount Signed-off-by: Henrik Gerdes --- k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml index 5bcc6a6..19e97dc 100644 --- a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml +++ b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml @@ -26,6 +26,9 @@ spec: targetRevision: 1.* helm: releaseName: coredns + serviceAccount: + create: true + name: coredns valuesObject: service: clusterIP: "10.96.0.10" From 4376b4c803e93c58859460252b3a26b778df60a3 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 17 Jan 2026 21:21:21 +0100 Subject: [PATCH 44/57] fix(hks): coredns create serviceaccount Signed-off-by: Henrik Gerdes --- .../hegerdes-kubernetes-service/shared/argo-coredns.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml index 19e97dc..1cc08d8 100644 --- a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml +++ b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml @@ -26,10 +26,10 @@ spec: targetRevision: 1.* helm: releaseName: coredns - serviceAccount: - create: true - name: coredns valuesObject: + serviceAccount: + create: true + name: coredns service: clusterIP: "10.96.0.10" name: "kube-dns" From 4fac0e0ef748a1e24251b08d982a277bbd11ba60 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 17 Jan 2026 21:25:23 +0100 Subject: [PATCH 45/57] fix(hks): coredns extra rbac Signed-off-by: Henrik Gerdes --- .../clustes-shared/rbac-dns.yaml | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac-dns.yaml diff --git a/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac-dns.yaml b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac-dns.yaml new file mode 100644 index 0000000..e0aa8db --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac-dns.yaml @@ -0,0 +1,35 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:coredns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns +subjects: + - kind: ServiceAccount + name: coredns + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:coredns +rules: + - apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch From 956b912d0e17735ccb045015abec1ffaaf3640f9 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 17 Jan 2026 21:29:17 +0100 Subject: [PATCH 46/57] fix(hks): set coredns kube endpoint Signed-off-by: Henrik Gerdes --- k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml index 1cc08d8..da41255 100644 --- a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml +++ b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml @@ -35,7 +35,7 @@ spec: name: "kube-dns" replicaCount: 2 priorityClassName: system-cluster-critical - isClusterService: false + isClusterService: true customLabels: k8s-app: kube-dns deployment: From 1ba51bbb4614b81c3186a276dcfbb8411939de92 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 17 Jan 2026 21:45:03 +0100 Subject: [PATCH 47/57] debug(hks): cordns Signed-off-by: Henrik Gerdes --- .../shared/argo-coredns.yml | 15 ++++++++------- .../shared/gateway.yaml | 11 +++++++++++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml index da41255..fc008e4 100644 --- a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml +++ b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml @@ -70,6 +70,7 @@ spec: parameters: "." configBlock: |- class error + - name: debug # Required to query kubernetes API for data - name: kubernetes parameters: cluster.local in-addr.arpa ip6.arpa @@ -97,13 +98,13 @@ spec: - name: loop - name: reload - name: loadbalance - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - topologyKey: "kubernetes.io/hostname" - labelSelector: - matchLabels: - k8s-app: kube-dns + # affinity: + # podAntiAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # - topologyKey: "kubernetes.io/hostname" + # labelSelector: + # matchLabels: + # k8s-app: kube-dns info: - name: Chart-Info CoreDNS value: https://github.com/coredns/helm/ diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/gateway.yaml b/k8s-apps/hegerdes-kubernetes-service/shared/gateway.yaml index a6882c6..e092877 100644 --- a/k8s-apps/hegerdes-kubernetes-service/shared/gateway.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/shared/gateway.yaml @@ -30,6 +30,17 @@ spec: - kind: TLSRoute tls: mode: Passthrough + + # - name: http + # allowedRoutes: + # namespaces: + # from: All + # kinds: + # - kind: HTTPRoute + # hostname: "manager.hks.eu-central" + # port: 80 + # protocol: HTTP + # --- # https://docs.nginx.com/nginx-gateway-fabric/traffic-management/tls-passthrough/ # apiVersion: gateway.networking.k8s.io/v1 From 8345d334577aefe4f3c69beb8c1ade3e21c7e1e6 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 17 Jan 2026 21:48:49 +0100 Subject: [PATCH 48/57] debug(hks): cordns Signed-off-by: Henrik Gerdes --- k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml index fc008e4..075f52b 100644 --- a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml +++ b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml @@ -69,7 +69,7 @@ spec: - name: log parameters: "." configBlock: |- - class error + class debug - name: debug # Required to query kubernetes API for data - name: kubernetes From 4b1f04a9ba57ee625de4e8e3cd9d1de7ad6f42f8 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 17 Jan 2026 22:07:13 +0100 Subject: [PATCH 49/57] debug(hks): coredns Signed-off-by: Henrik Gerdes --- .../shared/argo-cilium.yaml | 2 +- .../shared/argo-coredns.yml | 98 +++++++++---------- 2 files changed, 50 insertions(+), 50 deletions(-) diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/argo-cilium.yaml b/k8s-apps/hegerdes-kubernetes-service/shared/argo-cilium.yaml index 9d0ee25..f3e04fa 100644 --- a/k8s-apps/hegerdes-kubernetes-service/shared/argo-cilium.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/shared/argo-cilium.yaml @@ -42,7 +42,7 @@ spec: replicas: 1 endpointRoutes: - enabled: true + enabled: false loadBalancer: algorithm: maglev diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml index 075f52b..2687032 100644 --- a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml +++ b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml @@ -49,55 +49,55 @@ spec: enabled: true # Default zone is what Kubernetes recommends: # https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/#coredns-configmap-options - servers: - - zones: - - zone: . - port: 53 - # -- expose the service on a different port - # servicePort: 5353 - # If serviceType is nodePort you can specify nodePort here - # nodePort: 30053 - # hostPort: 53 - plugins: - - name: errors - # Serves a /health endpoint on :8080, required for livenessProbe - - name: health - configBlock: |- - lameduck 10s - # Serves a /ready endpoint on :8181, required for readinessProbe - - name: ready - - name: log - parameters: "." - configBlock: |- - class debug - - name: debug - # Required to query kubernetes API for data - - name: kubernetes - parameters: cluster.local in-addr.arpa ip6.arpa - configBlock: |- - endpoint https://{{ index .metadata.annotations "hks.hegerdes.com/public-host" }}:6443 - pods insecure - fallthrough in-addr.arpa ip6.arpa - ttl 30 - # Serves a /metrics endpoint on :9153, required for serviceMonitor - - name: prometheus - parameters: :9153 - - name: forward - # parameters: . /etc/resolv.conf - # parameters: . 2a00:1098:2b::1 2a00:1098:2c::1 # nat64 - parameters: . tls://1.1.1.1 tls://[2606:4700:4700::1111]:853 tls://1.0.0.1 tls://[2606:4700:4700::1001]:853 - configBlock: |- - tls_servername tls.cloudflare-dns.com - health_check 5s - max_concurrent 1000 - - name: cache - parameters: 60 - configBlock: |- - disable success cluster.local - disable denial cluster.local - - name: loop - - name: reload - - name: loadbalance + # servers: + # - zones: + # - zone: . + # port: 53 + # # -- expose the service on a different port + # # servicePort: 5353 + # # If serviceType is nodePort you can specify nodePort here + # # nodePort: 30053 + # # hostPort: 53 + # plugins: + # - name: errors + # # Serves a /health endpoint on :8080, required for livenessProbe + # - name: health + # configBlock: |- + # lameduck 10s + # # Serves a /ready endpoint on :8181, required for readinessProbe + # - name: ready + # - name: log + # parameters: "." + # configBlock: |- + # class error + # - name: debug + # # Required to query kubernetes API for data + # - name: kubernetes + # parameters: cluster.local in-addr.arpa ip6.arpa + # configBlock: |- + # endpoint https://{{ index .metadata.annotations "hks.hegerdes.com/public-host" }}:6443 + # pods insecure + # fallthrough in-addr.arpa ip6.arpa + # ttl 30 + # # Serves a /metrics endpoint on :9153, required for serviceMonitor + # - name: prometheus + # parameters: :9153 + # - name: forward + # # parameters: . /etc/resolv.conf + # # parameters: . 2a00:1098:2b::1 2a00:1098:2c::1 # nat64 + # parameters: . tls://1.1.1.1 tls://[2606:4700:4700::1111]:853 tls://1.0.0.1 tls://[2606:4700:4700::1001]:853 + # configBlock: |- + # tls_servername tls.cloudflare-dns.com + # health_check 5s + # max_concurrent 1000 + # - name: cache + # parameters: 60 + # configBlock: |- + # disable success cluster.local + # disable denial cluster.local + # - name: loop + # - name: reload + # - name: loadbalance # affinity: # podAntiAffinity: # requiredDuringSchedulingIgnoredDuringExecution: From 4aaea90b5a7619321b9215891834ccbd38f1f795 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 17 Jan 2026 23:19:14 +0100 Subject: [PATCH 50/57] feat(hks): add helm template chart Signed-off-by: Henrik Gerdes --- .pre-commit-config.yaml | 2 +- .../shared/helm-rendering/.helmignore | 23 +++++++ .../shared/helm-rendering/Chart.yaml | 24 +++++++ .../helm-rendering/templates/_helpers.tpl | 65 +++++++++++++++++++ .../templates/extra-manifests.yaml | 4 ++ .../shared/helm-rendering/values.yaml | 1 + 6 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/.helmignore create mode 100644 k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/Chart.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/templates/_helpers.tpl create mode 100644 k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/templates/extra-manifests.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/values.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 99bf5a5..59688b6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: hooks: - id: check-yaml args: [--allow-multiple-documents] - exclude: .gitlab-ci.yml|infra/.*/cloud-init.yml|k8s-apps/hegerdes-kubernetes-service/hks/templates/extra-manifests.yaml|k8s-apps/hegerdes-kubernetes-service/hks/templates/pdb.yaml|k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/.* + exclude: .gitlab-ci.yml|infra/.*/cloud-init.yml|k8s-apps/hegerdes-kubernetes-service/hks/templates/extra-manifests.yaml|k8s-apps/hegerdes-kubernetes-service/hks/templates/pdb.yaml|k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/.*|k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/templates/extra-manifests.yaml - id: check-json - id: pretty-format-json args: [--autofix, --no-sort-keys, --no-ensure-ascii] diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/.helmignore b/k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/Chart.yaml b/k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/Chart.yaml new file mode 100644 index 0000000..26768ea --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: helm-rendering +description: A Helm chart for helm templates + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.1.0" diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/templates/_helpers.tpl b/k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/templates/_helpers.tpl new file mode 100644 index 0000000..4839a9e --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/templates/_helpers.tpl @@ -0,0 +1,65 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "hks.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "hks.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "hks.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "hks.labels" -}} +helm.sh/chart: {{ include "hks.chart" . }} +{{ include "hks.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "hks.selectorLabels" -}} +app.kubernetes.io/name: {{ include "hks.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* +Renders a value that contains template. +Usage: +{{ include ".render" ( dict "value" .Values.path.to.the.Value "context" $) }} +*/}} +{{- define ".render" -}} + {{- if typeIs "string" .value }} + {{- tpl .value .context }} + {{- else }} + {{- tpl (.value | toYaml) .context }} + {{- end }} +{{- end -}} diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/templates/extra-manifests.yaml b/k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/templates/extra-manifests.yaml new file mode 100644 index 0000000..0f48fe1 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{- range .Values.extraObjects }} +--- +{{ include ".render" (dict "value" . "context" $) }} +{{- end }} diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/values.yaml b/k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/values.yaml new file mode 100644 index 0000000..3face71 --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/values.yaml @@ -0,0 +1 @@ +extraObjects: [] From e9f3708a458862f23876c462a76be33f0b090bed Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 17 Jan 2026 23:48:55 +0100 Subject: [PATCH 51/57] feat(hks): inject cluster version to argo secret Signed-off-by: Henrik Gerdes --- k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml index e31b958..43534e2 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml @@ -20,6 +20,7 @@ spec: argocd.argoproj.io/secret-type: cluster hks.hegerdes.com/managed-cluster: "true" hks.hegerdes.com/cluster-type: "downstream" + argocd.argoproj.io/auto-label-cluster-info: "true" type: Opaque data: server: https://kube-apiserver.{{ .Release.Namespace }}:6443 From ac5660b9546ad35a9214b979baa7fe97b3253d25 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sun, 18 Jan 2026 00:03:34 +0100 Subject: [PATCH 52/57] feat(hks): updates to latest working version Signed-off-by: Henrik Gerdes --- .../clustes-shared/cm-kubeadm-conf.yaml | 22 -- .../clustes-shared/rbac-dns.yaml | 35 --- .../clustes-shared/rbac-super-user.yaml | 31 +++ .../hks-manager/values.yaml | 7 +- .../shared/argo-coredns.yml | 203 ++++++++++-------- .../shared/argo-hks-conf.yaml | 37 +++- 6 files changed, 182 insertions(+), 153 deletions(-) delete mode 100644 k8s-apps/hegerdes-kubernetes-service/clustes-shared/cm-kubeadm-conf.yaml delete mode 100644 k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac-dns.yaml create mode 100644 k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac-super-user.yaml diff --git a/k8s-apps/hegerdes-kubernetes-service/clustes-shared/cm-kubeadm-conf.yaml b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/cm-kubeadm-conf.yaml deleted file mode 100644 index fcfd9c6..0000000 --- a/k8s-apps/hegerdes-kubernetes-service/clustes-shared/cm-kubeadm-conf.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: kubeadm-config - namespace: kube-system -data: - ClusterConfiguration: | - apiVersion: kubeadm.k8s.io/v1beta4 - kind: ClusterConfiguration - caCertificateValidityPeriod: 87600h0m0s - certificateValidityPeriod: 8760h0m0s - certificatesDir: /etc/kubernetes/pki - clusterName: test1 - controlPlaneEndpoint: test1.hks.eu-central.hegerdes.com:6443 - dns: {} - encryptionAlgorithm: RSA-2048 - imageRepository: registry.k8s.io - kubernetesVersion: v1.35.0 - networking: - dnsDomain: cluster.local - podSubnet: 10.244.0.0/16 - serviceSubnet: 10.96.0.0/16 diff --git a/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac-dns.yaml b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac-dns.yaml deleted file mode 100644 index e0aa8db..0000000 --- a/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac-dns.yaml +++ /dev/null @@ -1,35 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: system:coredns -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:coredns -subjects: - - kind: ServiceAccount - name: coredns - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: system:coredns -rules: - - apiGroups: - - "" - resources: - - endpoints - - services - - pods - - namespaces - verbs: - - list - - watch - - apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch diff --git a/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac-super-user.yaml b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac-super-user.yaml new file mode 100644 index 0000000..844488e --- /dev/null +++ b/k8s-apps/hegerdes-kubernetes-service/clustes-shared/rbac-super-user.yaml @@ -0,0 +1,31 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-apiserver-kubelet-client-extra +rules: + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes/proxy"] + verbs: ["get", "create"] + - apiGroups: [""] + resources: + - pods/exec + - pods/attach + - pods/portforward + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kube-apiserver-kubelet-client-extra-binding +subjects: + - kind: User + name: kube-apiserver-kubelet-client + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: kube-apiserver-kubelet-client-extra + apiGroup: rbac.authorization.k8s.io diff --git a/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.yaml b/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.yaml index 81ba633..4af5abd 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.yaml @@ -191,17 +191,18 @@ httpRoute: annotations: {} # -- Which Gateways this Route is attached to parentRefs: - - name: gateway + - name: cilium-gateway + namespace: cilium-gateway sectionName: http # -- Hostnames matching HTTP header. hostnames: - - "example.com" + - "manager.hks.eu-central" # -- List of rules and filters applied. rules: - matches: - path: type: PathPrefix - value: /headers + value: / # filters: # - type: RequestHeaderModifier # requestHeaderModifier: diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml index 2687032..32c690f 100644 --- a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml +++ b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml @@ -20,91 +20,121 @@ spec: destination: server: "{{.server}}" namespace: kube-system - source: - chart: coredns - repoURL: https://coredns.github.io/helm - targetRevision: 1.* - helm: - releaseName: coredns - valuesObject: - serviceAccount: - create: true - name: coredns - service: - clusterIP: "10.96.0.10" - name: "kube-dns" - replicaCount: 2 - priorityClassName: system-cluster-critical - isClusterService: true - customLabels: - k8s-app: kube-dns - deployment: - annotations: - configmap.reloader.stakater.com/reload: coredns - selector: - matchLabels: - k8s-app: kube-dns - prometheus: + sources: + - repoURL: https://github.com/hegerdes/GitOps.git + targetRevision: feat/hks + path: k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering + helm: + valuesObject: + extraObjects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: coredns-kubeconf + namespace: kube-system + data: + kubeconfig: | + kind: Config + apiVersion: v1 + current-context: default + contexts: + - name: default + context: + cluster: kubernetes + user: coredns + clusters: + - name: kubernetes + cluster: + server: "https://{{ index .metadata.annotations "hks.hegerdes.com/public-host" }}:6443" + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + users: + - name: coredns + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + - chart: coredns + repoURL: https://coredns.github.io/helm + targetRevision: 1.* + helm: + releaseName: coredns + valuesObject: + serviceAccount: + create: true + name: coredns service: - enabled: true - # Default zone is what Kubernetes recommends: - # https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/#coredns-configmap-options - # servers: - # - zones: - # - zone: . - # port: 53 - # # -- expose the service on a different port - # # servicePort: 5353 - # # If serviceType is nodePort you can specify nodePort here - # # nodePort: 30053 - # # hostPort: 53 - # plugins: - # - name: errors - # # Serves a /health endpoint on :8080, required for livenessProbe - # - name: health - # configBlock: |- - # lameduck 10s - # # Serves a /ready endpoint on :8181, required for readinessProbe - # - name: ready - # - name: log - # parameters: "." - # configBlock: |- - # class error - # - name: debug - # # Required to query kubernetes API for data - # - name: kubernetes - # parameters: cluster.local in-addr.arpa ip6.arpa - # configBlock: |- - # endpoint https://{{ index .metadata.annotations "hks.hegerdes.com/public-host" }}:6443 - # pods insecure - # fallthrough in-addr.arpa ip6.arpa - # ttl 30 - # # Serves a /metrics endpoint on :9153, required for serviceMonitor - # - name: prometheus - # parameters: :9153 - # - name: forward - # # parameters: . /etc/resolv.conf - # # parameters: . 2a00:1098:2b::1 2a00:1098:2c::1 # nat64 - # parameters: . tls://1.1.1.1 tls://[2606:4700:4700::1111]:853 tls://1.0.0.1 tls://[2606:4700:4700::1001]:853 - # configBlock: |- - # tls_servername tls.cloudflare-dns.com - # health_check 5s - # max_concurrent 1000 - # - name: cache - # parameters: 60 - # configBlock: |- - # disable success cluster.local - # disable denial cluster.local - # - name: loop - # - name: reload - # - name: loadbalance - # affinity: - # podAntiAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # - topologyKey: "kubernetes.io/hostname" - # labelSelector: - # matchLabels: - # k8s-app: kube-dns + clusterIP: "10.96.0.10" + name: "kube-dns" + replicaCount: 2 + extraVolumes: + - name: kubeconf-volume + configMap: + name: coredns-kubeconf + defaultMode: 420 + items: + - key: kubeconfig + path: kubeconfig + extraVolumeMounts: + - name: kubeconf-volume + mountPath: /etc/kube-auth + priorityClassName: system-cluster-critical + isClusterService: true + deployment: + annotations: + configmap.reloader.stakater.com/reload: coredns + prometheus: + service: + enabled: true + # Default zone is what Kubernetes recommends: + # https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/#coredns-configmap-options + servers: + - zones: + - zone: . + port: 53 + plugins: + - name: errors + # Serves a /health endpoint on :8080, required for livenessProbe + - name: health + configBlock: |- + lameduck 10s + # Serves a /ready endpoint on :8181, required for readinessProbe + - name: ready + - name: log + parameters: "." + configBlock: |- + class error + # Required to query kubernetes API for data + - name: kubernetes + parameters: cluster.local in-addr.arpa ip6.arpa + configBlock: |- + kubeconfig /etc/kube-auth/kubeconfig + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + # Serves a /metrics endpoint on :9153, required for serviceMonitor + - name: prometheus + parameters: :9153 + - name: forward + # parameters: . /etc/resolv.conf + # parameters: . 2a00:1098:2b::1 2a00:1098:2c::1 # nat64 + parameters: . tls://1.1.1.1 tls://[2606:4700:4700::1111]:853 tls://1.0.0.1 tls://[2606:4700:4700::1001]:853 + configBlock: |- + tls_servername tls.cloudflare-dns.com + health_check 5s + max_concurrent 1000 + - name: cache + parameters: 60 + configBlock: |- + disable success cluster.local + disable denial cluster.local + - name: loop + - name: reload + - name: loadbalance + # affinity: + # podAntiAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # - topologyKey: "kubernetes.io/hostname" + # labelSelector: + # matchLabels: + # k8s-app: kube-dns info: - name: Chart-Info CoreDNS value: https://github.com/coredns/helm/ @@ -112,8 +142,3 @@ spec: automated: prune: false selfHeal: false - ignoreDifferences: - - kind: Deployment - group: apps - jqPathExpressions: - - .spec.template.spec.containers[].ports[].name diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/argo-hks-conf.yaml b/k8s-apps/hegerdes-kubernetes-service/shared/argo-hks-conf.yaml index 5a8e796..07895c0 100644 --- a/k8s-apps/hegerdes-kubernetes-service/shared/argo-hks-conf.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/shared/argo-hks-conf.yaml @@ -12,15 +12,44 @@ spec: matchLabels: hks.hegerdes.com/managed-cluster: "true" hks.hegerdes.com/cluster-type: "downstream" + # argocd.argoproj.io/kubernetes-version: "1.35" template: metadata: name: "{{.name}}-cluster-conf" spec: project: default - source: - repoURL: https://github.com/hegerdes/GitOps.git - targetRevision: feat/hks - path: k8s-apps/hegerdes-kubernetes-service/clustes-shared + sources: + - repoURL: https://github.com/hegerdes/GitOps.git + targetRevision: feat/hks + path: k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering + helm: + valuesObject: + extraObjects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: kubeadm-config + namespace: kube-system + data: + ClusterConfiguration: | + apiVersion: kubeadm.k8s.io/v1beta4 + kind: ClusterConfiguration + caCertificateValidityPeriod: 87600h0m0s + certificateValidityPeriod: 8760h0m0s + certificatesDir: /etc/kubernetes/pki + clusterName: {{.name}} + controlPlaneEndpoint: {{ index .metadata.annotations "hks.hegerdes.com/public-host" }}:6443 + dns: {} + encryptionAlgorithm: RSA-2048 + imageRepository: registry.k8s.io + kubernetesVersion: v{{ index .metadata.labels "argocd.argoproj.io/kubernetes-version" }}.0 + networking: + dnsDomain: cluster.local + podSubnet: 10.244.0.0/16 + serviceSubnet: 10.96.0.0/16 + - repoURL: https://github.com/hegerdes/GitOps.git + targetRevision: feat/hks + path: k8s-apps/hegerdes-kubernetes-service/clustes-shared destination: server: "{{.server}}" namespace: kube-system From a857289afe6a94e2fc1f0cad8528429f87436cd4 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Wed, 21 Jan 2026 20:40:44 +0100 Subject: [PATCH 53/57] deps: bump coredns Signed-off-by: Henrik Gerdes --- k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml index 32c690f..3d2c614 100644 --- a/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml +++ b/k8s-apps/hegerdes-kubernetes-service/shared/argo-coredns.yml @@ -57,6 +57,8 @@ spec: helm: releaseName: coredns valuesObject: + image: + tag: 1.14.1 serviceAccount: create: true name: coredns From 1e6ed108849e927b1059c76789c75f92d4ad1126 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Wed, 21 Jan 2026 21:35:35 +0100 Subject: [PATCH 54/57] feat(hks): add ip ref to cluster Signed-off-by: Henrik Gerdes --- .pre-commit-config.yaml | 2 +- .../hks/templates/eso.yaml | 12 +++++++++++- .../hks/templates/pdb.yaml | 1 - .../hks/templates/tls-route.yaml | 2 ++ k8s-apps/hegerdes-kubernetes-service/hks/values.yaml | 8 ++++++++ 5 files changed, 22 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 59688b6..38635a3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: hooks: - id: check-yaml args: [--allow-multiple-documents] - exclude: .gitlab-ci.yml|infra/.*/cloud-init.yml|k8s-apps/hegerdes-kubernetes-service/hks/templates/extra-manifests.yaml|k8s-apps/hegerdes-kubernetes-service/hks/templates/pdb.yaml|k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/.*|k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/templates/extra-manifests.yaml + exclude: .gitlab-ci.yml|infra/.*/cloud-init.yml|k8s-apps/hegerdes-kubernetes-service/hks/templates/extra-manifests.yaml|k8s-apps/hegerdes-kubernetes-service/hks/templates/pdb.yaml|k8s-apps/hegerdes-kubernetes-service/hks-manager/templates/.*|k8s-apps/hegerdes-kubernetes-service/shared/helm-rendering/templates/extra-manifests.yaml|k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml|k8s-apps/hegerdes-kubernetes-service/hks/templates/tls-route.yaml - id: check-json - id: pretty-format-json args: [--autofix, --no-sort-keys, --no-ensure-ascii] diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml index 43534e2..0bddb3b 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml @@ -1,4 +1,11 @@ --- +{{- $svc := lookup "v1" "Service" .Values.controlplane.publicIpServiceRef.namespace .Values.controlplane.publicIpServiceRef.namespace -}} +{{- $ip := "unknown" -}} +{{- if and $svc $svc.status $svc.status.loadBalancer $svc.status.loadBalancer.ingress (gt (len $svc.status.loadBalancer.ingress) 0) }} + {{- $firstIngress := index $svc.status.loadBalancer.ingress 0 -}} + {{- $ip = $firstIngress.ip | default "unknown" -}} +{{- end }} + apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: @@ -14,8 +21,11 @@ spec: template: metadata: annotations: + hks.hegerdes.com/public-ipv4: {{ $ip }} hks.hegerdes.com/public-host: "{{ .Release.Namespace }}.{{ .Values.controlplane.domainSuffix }}" hks.hegerdes.com/private-host: "kube-apiserver.{{ .Release.Namespace }}" + hks.hegerdes.com/private-port: "6443" + hks.hegerdes.com/public-port: "6443" labels: argocd.argoproj.io/secret-type: cluster hks.hegerdes.com/managed-cluster: "true" @@ -25,7 +35,7 @@ spec: data: server: https://kube-apiserver.{{ .Release.Namespace }}:6443 name: "{{ .Release.Namespace }}" - namespaces: kube-system,kube-public,cilium,cilium-secrets,default,kubelet-serving-cert-approver + namespaces: kube-system,kube-public,default,cilium,cilium-secrets,kubelet-serving-cert-approver clusterResources: "true" config: | { diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/pdb.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/pdb.yaml index 6850ac4..651db9f 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/pdb.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/pdb.yaml @@ -1,5 +1,4 @@ {{- if .Values.controlplane.pdb.enabled -}} - apiVersion: policy/v1 kind: PodDisruptionBudget metadata: diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/tls-route.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/tls-route.yaml index de5c8f4..34f8510 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/tls-route.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/tls-route.yaml @@ -1,3 +1,4 @@ +{{- if .Values.controlplane.tlsRoute.enabled -}} apiVersion: gateway.networking.k8s.io/v1alpha2 kind: TLSRoute metadata: @@ -15,3 +16,4 @@ spec: kind: Service namespace: "{{ .Release.Namespace }}" port: 6443 +{{- end }} diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/values.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/values.yaml index 275aff2..9ba521f 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/values.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/values.yaml @@ -1,5 +1,13 @@ controlplane: domainSuffix: hks.eu-central.hegerdes.com + publicIp: "publicIpServiceRef" + publicIpServiceRef: + namespace: cilium-gateway + name: cilium-gateway-cilium-gateway + tlsRoute: + enabled: true + parentRefs: + pdb: enabled: false From b44ffbab5562b6e96f3a549abbb9386ce77b6326 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 24 Jan 2026 19:41:44 +0100 Subject: [PATCH 55/57] chore: update gitignore Signed-off-by: Henrik Gerdes --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index fdc95a6..2b37a9e 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ infra/test *.log k8s-apps/hegerdes-kubernetes-service/generated k8s-apps/hegerdes-kubernetes-service/helper/generated +k8s-apps/hegerdes-kubernetes-service/helper/ From 00343ab63662d689cf86284dd25fa0ddef050285 Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sat, 24 Jan 2026 22:33:25 +0100 Subject: [PATCH 56/57] feat(hks) set cluster advertise address Signed-off-by: Henrik Gerdes --- .../hks/templates/kube-apiserver.yaml | 7 +++++++ .../hks/templates/kube-confs.yaml | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml index 8d4c63b..e1f393b 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml @@ -69,12 +69,19 @@ spec: - name: kube-apiserver image: registry.k8s.io/kube-apiserver:v1.35.0 imagePullPolicy: IfNotPresent + env: + - name: ADVERTISE_IP + valueFrom: + configMapKeyRef: + key: advertiseIP + name: cluster-data # Docs: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/ command: - kube-apiserver - --secure-port=6443 - --profiling=false - --external-hostname={{ .Release.Namespace }}.{{ .Values.controlplane.domainSuffix }} + - --advertise-address=$(ADVERTISE_IP) - --enable-admission-plugins=NodeRestriction - --enable-bootstrap-token-auth=true - --allow-privileged=true diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml index ecddbca..d2ba429 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-confs.yaml @@ -28,6 +28,14 @@ data: --- apiVersion: v1 kind: ConfigMap +metadata: + name: cluster-data +data: + advertiseIP: 10.0.0.1 # placeholder + info: hks-managed-cluster +--- +apiVersion: v1 +kind: ConfigMap metadata: name: kube-api-configs labels: From 8ebc11fe599240797d2aa7e994b04ee67b1406bb Mon Sep 17 00:00:00 2001 From: Henrik Gerdes Date: Sun, 25 Jan 2026 21:44:46 +0100 Subject: [PATCH 57/57] feat: switch to hks docker repo Signed-off-by: Henrik Gerdes --- k8s-apps/hegerdes-kubernetes-service/Kube-tasks.md | 7 ++++--- .../hegerdes-kubernetes-service/argo-appset-hks.yaml | 4 ++++ .../hks-manager/values.yaml | 4 ++-- .../hegerdes-kubernetes-service/hks/templates/eso.yaml | 9 +-------- .../hks/templates/kube-apiserver.yaml | 4 +++- .../hks/templates/kube-certs-util.yaml | 4 ---- .../hks/templates/kube-controller-manager.yaml | 4 +++- .../hks/templates/kube-init-ca.yml | 10 +++++++++- .../hks/templates/kube-scheduler.yaml | 4 +++- 9 files changed, 29 insertions(+), 21 deletions(-) diff --git a/k8s-apps/hegerdes-kubernetes-service/Kube-tasks.md b/k8s-apps/hegerdes-kubernetes-service/Kube-tasks.md index 4c90b71..08569c2 100644 --- a/k8s-apps/hegerdes-kubernetes-service/Kube-tasks.md +++ b/k8s-apps/hegerdes-kubernetes-service/Kube-tasks.md @@ -28,12 +28,13 @@ Tasks: kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.2/cert-manager.yaml kubectl create namespace argocd kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/v3.2.3/manifests/install.yaml -kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.4.1/standard-install.yaml +# kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.4.1/standard-install.yaml +kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.4.1/experimental-install.yaml kubectl kustomize "https://github.com/nginx/nginx-gateway-fabric/config/crd/gateway-api/experimental?ref=v2.3.0" | kubectl apply -f - --server-side - +helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric --create-namespace -n nginx-gateway --set nginxGateway.gwAPIExperimentalFeatures.enable=true kaf argo-hks-shared.yaml kaf ../../k8s-cluster-hcloud-critical/argo-external-secrets.yml kaf ../argo-nginx-gateway-fabric.yml -kaf argo-hks-appset.yaml +kaf argo-appset-hks.yaml ``` diff --git a/k8s-apps/hegerdes-kubernetes-service/argo-appset-hks.yaml b/k8s-apps/hegerdes-kubernetes-service/argo-appset-hks.yaml index 90eed55..f3aab28 100644 --- a/k8s-apps/hegerdes-kubernetes-service/argo-appset-hks.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/argo-appset-hks.yaml @@ -42,3 +42,7 @@ spec: group: apps jqPathExpressions: - .spec.updateStrategy.rollingUpdate.maxUnavailable + - kind: ConfigMap + name: cluster-data + jqPathExpressions: + - .data.advertiseIP diff --git a/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.yaml b/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.yaml index 4af5abd..ed9acc0 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks-manager/values.yaml @@ -8,11 +8,11 @@ clusterZone: hks.eu-central.hegerdes.com image: # -- The container registry and image to use. - repository: hegerdes/debug + repository: hegerdes/hks # -- Pull policy of that image. pullPolicy: Always # -- The image tag and/or sha. - tag: hks-manager + tag: latest # -- Any repository secrets needed to pull the image. imagePullSecrets: [] diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml index 0bddb3b..b5809e8 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/eso.yaml @@ -1,11 +1,4 @@ --- -{{- $svc := lookup "v1" "Service" .Values.controlplane.publicIpServiceRef.namespace .Values.controlplane.publicIpServiceRef.namespace -}} -{{- $ip := "unknown" -}} -{{- if and $svc $svc.status $svc.status.loadBalancer $svc.status.loadBalancer.ingress (gt (len $svc.status.loadBalancer.ingress) 0) }} - {{- $firstIngress := index $svc.status.loadBalancer.ingress 0 -}} - {{- $ip = $firstIngress.ip | default "unknown" -}} -{{- end }} - apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: @@ -21,7 +14,7 @@ spec: template: metadata: annotations: - hks.hegerdes.com/public-ipv4: {{ $ip }} + hks.hegerdes.com/public-ipv4: "unknown" hks.hegerdes.com/public-host: "{{ .Release.Namespace }}.{{ .Values.controlplane.domainSuffix }}" hks.hegerdes.com/private-host: "kube-apiserver.{{ .Release.Namespace }}" hks.hegerdes.com/private-port: "6443" diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml index e1f393b..6de5dc5 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-apiserver.yaml @@ -1,11 +1,13 @@ apiVersion: apps/v1 kind: Deployment metadata: + name: kube-apiserver labels: app: kube-apiserver component: kube-apiserver tier: control-plane - name: kube-apiserver + annotations: + reloader.stakater.com/auto: "true" spec: replicas: 1 selector: diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-util.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-util.yaml index b3494a3..bb271f6 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-util.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-certs-util.yaml @@ -12,21 +12,17 @@ spec: - digital signature - key encipherment - client auth - privateKey: algorithm: RSA size: 2048 - subject: organizationalUnits: - argocd-auth organizations: - kubernetes - dnsNames: [] emailAddresses: - kubernetes-pki@hegerdes.com - issuerRef: name: ca-kube-controlplane kind: Issuer diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-controller-manager.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-controller-manager.yaml index d9e104b..d7eaf58 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-controller-manager.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-controller-manager.yaml @@ -1,11 +1,13 @@ apiVersion: apps/v1 kind: Deployment metadata: + name: kube-controller-manager labels: app: kube-controller-manager component: kube-controller-manager tier: control-plane - name: kube-controller-manager + annotations: + reloader.stakater.com/auto: "true" spec: replicas: 1 selector: diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml index 7b4c555..8e0393e 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-init-ca.yml @@ -21,7 +21,7 @@ spec: type: RuntimeDefault containers: - name: file-check - image: hegerdes/debug:hks + image: hegerdes/hks:util env: - name: ENCRYPTION_KEY valueFrom: @@ -210,6 +210,14 @@ spec: path: front-proxy-ca.crt --- apiVersion: v1 +kind: Namespace +metadata: + name: "{{ .Release.Namespace }}" + labels: + hks.hegerdes.com/cluster: "true" + hks.hegerdes.com/region: "{{ .Values.controlplane.domainSuffix }}" +--- +apiVersion: v1 kind: ServiceAccount metadata: name: kube-init-ca-gen diff --git a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-scheduler.yaml b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-scheduler.yaml index c002202..256faca 100644 --- a/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-scheduler.yaml +++ b/k8s-apps/hegerdes-kubernetes-service/hks/templates/kube-scheduler.yaml @@ -1,11 +1,13 @@ apiVersion: apps/v1 kind: Deployment metadata: + name: kube-scheduler labels: component: kube-scheduler tier: control-plane app: kube-scheduler - name: kube-scheduler + annotations: + reloader.stakater.com/auto: "true" spec: replicas: 1 selector: