From 370e0c892dfbcbff22be397226159d28e02ff580 Mon Sep 17 00:00:00 2001 From: Quirin Vetterl Date: Fri, 29 Aug 2025 14:56:25 +0200 Subject: [PATCH 1/4] Optionally start control plane in k3d --- Makefile | 28 ++++++++++++++++++++++------ control-plane/k3d.yaml | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 control-plane/k3d.yaml diff --git a/Makefile b/Makefile index 520cd053..ded9b132 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ YQ=docker run --rm -i -v $(shell pwd):/workdir mikefarah/yq:4 KINDCONFIG := $(or $(KINDCONFIG),control-plane/kind.yaml) +K3DCONFIG := $(or $(K3DCONFIG),control-plane/k3d.yaml) KUBECONFIG := $(shell pwd)/.kubeconfig METALCTL_HMAC := $(or $(METALCTL_HMAC),metal-admin) @@ -45,6 +46,11 @@ ifneq ($(K8S_VERSION),) KIND_ARGS=--image kindest/node:v$(K8S_VERSION) endif +K3D_ARGS= +ifneq ($(K8S_VERSION),) +K3D_ARGS=--image rancher/k3s:v$(K8S_VERSION)-k3s1 +endif + ifeq ($(CI),true) DOCKER_COMPOSE_RUN_ARG=--no-TTY --rm else @@ -94,12 +100,21 @@ create-proxy-registries: .PHONY: control-plane-bake control-plane-bake: - @if ! which kind > /dev/null; then echo "kind needs to be installed"; exit 1; fi - @if ! kind get clusters | grep metal-control-plane > /dev/null; then \ - kind create cluster $(KIND_ARGS) \ - --name metal-control-plane \ - --config $(KINDCONFIG) \ - --kubeconfig $(KUBECONFIG); fi + @if [ -n "$(USE_K3D)" ]; then \ + if ! which k3d > /dev/null; then echo "k3d needs to be installed"; exit 1; fi; \ + if ! k3d cluster list | grep metal-control-plane > /dev/null; then \ + k3d cluster create metal-control-plane $(K3D_ARGS) \ + --config $(K3DCONFIG); \ + fi; \ + else \ + if ! which kind > /dev/null; then echo "kind needs to be installed"; exit 1; fi; \ + if ! kind get clusters | grep metal-control-plane > /dev/null; then \ + kind create cluster $(KIND_ARGS) \ + --name metal-control-plane \ + --config $(KINDCONFIG) \ + --kubeconfig $(KUBECONFIG); \ + fi; \ + fi; $(MAKE) create-proxy-registries .PHONY: partition @@ -142,6 +157,7 @@ cleanup: cleanup-control-plane cleanup-partition .PHONY: cleanup-control-plane cleanup-control-plane: kind delete cluster --name metal-control-plane + k3d cluster delete metal-control-plane docker compose down rm -f $(KUBECONFIG) diff --git a/control-plane/k3d.yaml b/control-plane/k3d.yaml new file mode 100644 index 00000000..29115bae --- /dev/null +++ b/control-plane/k3d.yaml @@ -0,0 +1,34 @@ +apiVersion: k3d.io/v1alpha5 +kind: Simple +servers: 1 +agents: 0 +kubeAPI: + hostIP: 0.0.0.0 + hostPort: "6443" +ports: + - port: 4443:4443 + nodeFilters: + - loadbalancer + - port: 8080:8080 + nodeFilters: + - loadbalancer + - port: 4150:4150 + nodeFilters: + - loadbalancer + - port: 50051:50051 + nodeFilters: + - loadbalancer +volumes: + - volume: ${PWD}/control-plane/config-patches:/etc/containerd/certs.d + nodeFilters: + - server:0 +options: + k3s: + extraArgs: + - arg: "--kubelet-arg=max-pods=256" + nodeFilters: + - server:0 + k3d: + wait: true + timeout: 60s + disableLoadbalancer: false From d5d4ac3200c2b5fe227482d810e26f39da796142 Mon Sep 17 00:00:00 2001 From: Quirin Vetterl Date: Mon, 8 Sep 2025 15:54:57 +0200 Subject: [PATCH 2/4] Disable Traefik and servicelb in k3d --- control-plane/k3d.yaml | 43 +++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/control-plane/k3d.yaml b/control-plane/k3d.yaml index 29115bae..d12e9363 100644 --- a/control-plane/k3d.yaml +++ b/control-plane/k3d.yaml @@ -1,34 +1,55 @@ apiVersion: k3d.io/v1alpha5 kind: Simple servers: 1 -agents: 0 +agents: 1 kubeAPI: hostIP: 0.0.0.0 hostPort: "6443" ports: - port: 4443:4443 nodeFilters: - - loadbalancer + - server:* + - agents:* - port: 8080:8080 nodeFilters: - - loadbalancer + - server:* + - agents:* - port: 4150:4150 nodeFilters: - - loadbalancer + - server:* + - agents:* - port: 50051:50051 nodeFilters: - - loadbalancer -volumes: - - volume: ${PWD}/control-plane/config-patches:/etc/containerd/certs.d - nodeFilters: - - server:0 + - server:* + - agents:* options: k3s: extraArgs: - - arg: "--kubelet-arg=max-pods=256" + - arg: "--disable=traefik" + nodeFilters: + - server:* + - arg: "--disable=servicelb" nodeFilters: - - server:0 + - server:* k3d: wait: true timeout: 60s disableLoadbalancer: false +registries: + config: | + mirrors: + "docker.io": + endpoint: + - http://proxy-docker:5000 + "gcr.io": + endpoint: + - http://proxy-gcr:5000 + "ghcr.io": + endpoint: + - http://proxy-ghcr:5000 + "quay.io": + endpoint: + - http://proxy-quay:5000 + "registry.k8s.io": + endpoint: + - http://proxy-k8s:5000 From 821a3a1f0ae13f640a536d5139a5a7c639a4ad17 Mon Sep 17 00:00:00 2001 From: Quirin Vetterl Date: Mon, 8 Sep 2025 16:06:55 +0200 Subject: [PATCH 3/4] Add flavor k3s --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index ded9b132..196ef20c 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,12 @@ GARDENER_ENABLED=true # usually gardener restricts the maximum version for k8s: K8S_VERSION=1.32.5 LAB_TOPOLOGY=mini-lab.sonic.yaml +else ifeq ($(MINI_LAB_FLAVOR),k3s) +GARDENER_ENABLED=true +# usually gardener restricts the maximum version for k8s: +K8S_VERSION=1.32.5 +LAB_TOPOLOGY=mini-lab.sonic.yaml +USE_K3D=true else $(error Unknown flavor $(MINI_LAB_FLAVOR)) endif From f459ce9126489088b70cb3eebbbd0135d0b6d4ce Mon Sep 17 00:00:00 2001 From: Quirin Vetterl Date: Tue, 9 Sep 2025 09:08:15 +0200 Subject: [PATCH 4/4] Include k3s flavor in integration test --- .github/workflows/integration.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 754b0007..3b2530d3 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -95,6 +95,7 @@ jobs: flavors: - name: sonic - name: gardener + - name: k3s steps: - name: Gain back workspace permissions # https://github.com/actions/checkout/issues/211 @@ -109,6 +110,10 @@ jobs: # set if required: # DESIRED_VERSION: v0.59.0 + - name: Setup k3d + run: | + curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash + - name: Checkout uses: actions/checkout@v4