From 918d5d4b9d19052d80c2061b950bac29a727bee8 Mon Sep 17 00:00:00 2001 From: yansun1996 Date: Wed, 4 Feb 2026 21:01:35 +0000 Subject: [PATCH] [Build] Deprecate helm charts for OpenShift Signed-off-by: yansun1996 --- .gitignore | 1 - Dockerfile | 8 - Dockerfile.build | 3 + Makefile | 127 +- charts/gpu-operator-helm-k8s-v1.2.0.tgz | Bin 89775 -> 0 bytes .../openshift-patch/metadata-patch/Chart.yaml | 33 - .../metadata-patch/values.yaml | 85 - .../metadata-patch/Chart.yaml | 8 - .../metadata-patch/values.yaml | 134 - .../controller-metrics-service.yaml | 18 - .../template-patch/deployment.yaml | 201 -- .../template-patch/serviceaccount.yaml | 10 - .../validating-webhook-configuration.yaml | 51 - .../template-patch/webhook-service.yaml | 20 - .../crds/nodefeature-crd.yaml | 128 - .../crds/nodefeaturerule-crd.yaml | 330 -- .../metadata-patch/Chart.yaml | 21 - .../template-patch/config-manager-rbac.yaml | 75 - .../template-patch/deployment.yaml | 83 - .../kmm-device-plugin-rbac.yaml | 34 - .../kmm-module-loader-rbac.yaml | 34 - .../metrics-exporter-rbac-proxy-rbac.yaml | 63 - .../template-patch/metrics-exporter-rbac.yaml | 51 - .../template-patch/node-labeller-rbac.yaml | 43 - .../template-patch/nodefeaturediscovery.yaml | 124 - .../template-patch/post-delete-hook.yaml | 118 - .../template-patch/pre-delete-hook.yaml | 146 - .../template-patch/pre-upgrade-hook.yaml | 110 - .../template-patch/prometheus-k8s-rbac.yaml | 36 - .../template-patch/serviceaccount.yaml | 56 - .../template-patch/test-runner-rbac.yaml | 50 - .../template-patch/utils-container-rbac.yaml | 34 - helm-charts-openshift/.helmignore | 23 - helm-charts-openshift/Chart.lock | 9 - helm-charts-openshift/Chart.yaml | 33 - helm-charts-openshift/charts/kmm/.helmignore | 23 - helm-charts-openshift/charts/kmm/Chart.yaml | 8 - .../charts/kmm/crds/module-crd.yaml | 2604 ---------------- .../kmm/crds/nodemodulesconfig-crd.yaml | 440 --- .../charts/kmm/templates/_helpers.tpl | 62 - .../charts/kmm/templates/cluster-ca.yaml | 11 - .../templates/controller-metrics-monitor.yaml | 20 - .../templates/controller-metrics-service.yaml | 18 - .../charts/kmm/templates/deployment.yaml | 201 -- .../event-recorder-clusterrole-rbac.yaml | 16 - ...vent-recorder-clusterrolebinding-rbac.yaml | 16 - .../kmm/templates/leader-election-rbac.yaml | 50 - .../charts/kmm/templates/manager-config.yaml | 11 - .../charts/kmm/templates/manager-rbac.yaml | 231 -- .../kmm/templates/metrics-reader-rbac.yaml | 13 - .../templates/preflightvalidation-crd.yaml | 238 -- .../templates/preflightvalidationocp-crd.yaml | 247 -- .../kmm/templates/prometheus-k8s-rbac.yaml | 36 - .../charts/kmm/templates/proxy-rbac.yaml | 38 - .../charts/kmm/templates/service-ca.yaml | 12 - .../charts/kmm/templates/serviceaccount.yaml | 10 - .../validating-webhook-configuration.yaml | 51 - .../charts/kmm/templates/webhook-service.yaml | 20 - helm-charts-openshift/charts/kmm/values.yaml | 134 - helm-charts-openshift/charts/nfd/.helmignore | 23 - helm-charts-openshift/charts/nfd/Chart.yaml | 21 - .../charts/nfd/crds/nodefeature-crd.yaml | 128 - .../nfd/crds/nodefeaturediscovery-crd.yaml | 211 -- .../charts/nfd/crds/nodefeaturerule-crd.yaml | 330 -- .../charts/nfd/templates/_helpers.tpl | 62 - .../controller-manager-alerts-monitor.yaml | 19 - .../controller-manager-metrics-monitor.yaml | 20 - .../charts/nfd/templates/deployment.yaml | 85 - .../nfd/templates/leader-election-rbac.yaml | 96 - .../charts/nfd/templates/manager-config.yaml | 9 - .../nfd/templates/metrics-reader-rbac.yaml | 11 - .../charts/nfd/templates/metrics-service.yaml | 16 - .../charts/nfd/templates/operator-rbac.yaml | 279 -- .../nfd/templates/prometheus-k8s-rbac.yaml | 32 - .../charts/nfd/templates/proxy-rbac.yaml | 34 - .../charts/nfd/templates/serviceaccount.yaml | 8 - helm-charts-openshift/charts/nfd/values.yaml | 66 - .../crds/deviceconfig-crd.yaml | 1819 ----------- .../crds/remediationworkflowstatus-crd.yaml | 78 - helm-charts-openshift/templates/_helpers.tpl | 62 - .../templates/config-manager-rbac.yaml | 75 - .../templates/deployment.yaml | 83 - .../event-recorder-clusterrole-rbac.yaml | 16 - ...vent-recorder-clusterrolebinding-rbac.yaml | 16 - .../templates/kmm-device-plugin-rbac.yaml | 34 - .../templates/kmm-module-loader-rbac.yaml | 34 - .../templates/leader-election-rbac.yaml | 50 - .../templates/manager-config.yaml | 11 - .../templates/manager-rbac.yaml | 219 -- .../metrics-exporter-rbac-proxy-rbac.yaml | 63 - .../templates/metrics-exporter-rbac.yaml | 51 - .../templates/node-labeller-rbac.yaml | 43 - .../templates/nodefeaturediscovery.yaml | 124 - .../templates/post-delete-hook.yaml | 118 - .../templates/pre-delete-hook.yaml | 146 - .../templates/pre-upgrade-hook.yaml | 110 - .../templates/prometheus-k8s-rbac.yaml | 36 - .../templates/serviceaccount.yaml | 56 - .../templates/test-runner-rbac.yaml | 50 - .../templates/utils-container-rbac.yaml | 34 - helm-charts-openshift/values.yaml | 85 - helm-charts/Chart.lock | 9 - helm-charts/Chart.yaml | 33 - helm-charts/README.md | 205 -- helm-charts/charts/kmm-v1.0.0.tgz | Bin 30212 -> 0 bytes helm-charts/charts/kmm/.helmignore | 23 - helm-charts/charts/kmm/Chart.yaml | 9 - helm-charts/charts/kmm/crds/module-crd.yaml | 2700 ----------------- .../kmm/crds/nodemodulesconfig-crd.yaml | 367 --- helm-charts/charts/kmm/templates/_helpers.tpl | 62 - .../templates/controller-metrics-service.yaml | 18 - .../charts/kmm/templates/deployment.yaml | 203 -- .../event-recorder-clusterrole-rbac.yaml | 16 - ...vent-recorder-clusterrolebinding-rbac.yaml | 16 - .../kmm/templates/leader-election-rbac.yaml | 50 - .../charts/kmm/templates/manager-config.yaml | 11 - .../charts/kmm/templates/manager-rbac.yaml | 135 - .../kmm/templates/metrics-reader-rbac.yaml | 13 - .../templates/preflightvalidation-crd.yaml | 243 -- .../charts/kmm/templates/proxy-rbac.yaml | 38 - .../kmm/templates/selfsigned-issuer.yaml | 8 - .../charts/kmm/templates/serviceaccount.yaml | 10 - .../charts/kmm/templates/serving-cert.yaml | 15 - .../validating-webhook-configuration.yaml | 51 - .../charts/kmm/templates/webhook-service.yaml | 18 - helm-charts/charts/kmm/values.yaml | 133 - .../node-feature-discovery-chart-0.16.1.tgz | Bin 14942 -> 0 bytes helm-charts/crds/deviceconfig-crd.yaml | 798 ----- helm-charts/templates/_helpers.tpl | 62 - helm-charts/templates/deployment.yaml | 83 - .../event-recorder-clusterrole-rbac.yaml | 16 - ...vent-recorder-clusterrolebinding-rbac.yaml | 16 - .../templates/leader-election-rbac.yaml | 50 - helm-charts/templates/manager-config.yaml | 11 - helm-charts/templates/manager-rbac.yaml | 263 -- .../metrics-exporter-rbac-proxy-rbac.yaml | 47 - .../templates/metrics-exporter-rbac.yaml | 35 - helm-charts/templates/nfd-default-rule.yaml | 50 - helm-charts/templates/node-labeller-rbac.yaml | 35 - helm-charts/templates/post-delete-hook.yaml | 117 - helm-charts/templates/pre-delete-hook.yaml | 101 - helm-charts/templates/pre-upgrade-hook.yaml | 168 - helm-charts/templates/serviceaccount.yaml | 76 - helm-charts/templates/test-runner-rbac.yaml | 41 - helm-charts/values.yaml | 97 - 145 files changed, 17 insertions(+), 18108 deletions(-) delete mode 100644 charts/gpu-operator-helm-k8s-v1.2.0.tgz delete mode 100644 hack/openshift-patch/metadata-patch/Chart.yaml delete mode 100644 hack/openshift-patch/metadata-patch/values.yaml delete mode 100644 hack/openshift-patch/openshift-kmm-patch/metadata-patch/Chart.yaml delete mode 100644 hack/openshift-patch/openshift-kmm-patch/metadata-patch/values.yaml delete mode 100644 hack/openshift-patch/openshift-kmm-patch/template-patch/controller-metrics-service.yaml delete mode 100644 hack/openshift-patch/openshift-kmm-patch/template-patch/deployment.yaml delete mode 100644 hack/openshift-patch/openshift-kmm-patch/template-patch/serviceaccount.yaml delete mode 100644 hack/openshift-patch/openshift-kmm-patch/template-patch/validating-webhook-configuration.yaml delete mode 100644 hack/openshift-patch/openshift-kmm-patch/template-patch/webhook-service.yaml delete mode 100644 hack/openshift-patch/openshift-nfd-patch/crds/nodefeature-crd.yaml delete mode 100644 hack/openshift-patch/openshift-nfd-patch/crds/nodefeaturerule-crd.yaml delete mode 100644 hack/openshift-patch/openshift-nfd-patch/metadata-patch/Chart.yaml delete mode 100644 hack/openshift-patch/template-patch/config-manager-rbac.yaml delete mode 100644 hack/openshift-patch/template-patch/deployment.yaml delete mode 100644 hack/openshift-patch/template-patch/kmm-device-plugin-rbac.yaml delete mode 100644 hack/openshift-patch/template-patch/kmm-module-loader-rbac.yaml delete mode 100644 hack/openshift-patch/template-patch/metrics-exporter-rbac-proxy-rbac.yaml delete mode 100644 hack/openshift-patch/template-patch/metrics-exporter-rbac.yaml delete mode 100644 hack/openshift-patch/template-patch/node-labeller-rbac.yaml delete mode 100644 hack/openshift-patch/template-patch/nodefeaturediscovery.yaml delete mode 100644 hack/openshift-patch/template-patch/post-delete-hook.yaml delete mode 100644 hack/openshift-patch/template-patch/pre-delete-hook.yaml delete mode 100644 hack/openshift-patch/template-patch/pre-upgrade-hook.yaml delete mode 100644 hack/openshift-patch/template-patch/prometheus-k8s-rbac.yaml delete mode 100644 hack/openshift-patch/template-patch/serviceaccount.yaml delete mode 100644 hack/openshift-patch/template-patch/test-runner-rbac.yaml delete mode 100644 hack/openshift-patch/template-patch/utils-container-rbac.yaml delete mode 100644 helm-charts-openshift/.helmignore delete mode 100644 helm-charts-openshift/Chart.lock delete mode 100644 helm-charts-openshift/Chart.yaml delete mode 100644 helm-charts-openshift/charts/kmm/.helmignore delete mode 100644 helm-charts-openshift/charts/kmm/Chart.yaml delete mode 100644 helm-charts-openshift/charts/kmm/crds/module-crd.yaml delete mode 100644 helm-charts-openshift/charts/kmm/crds/nodemodulesconfig-crd.yaml delete mode 100644 helm-charts-openshift/charts/kmm/templates/_helpers.tpl delete mode 100644 helm-charts-openshift/charts/kmm/templates/cluster-ca.yaml delete mode 100644 helm-charts-openshift/charts/kmm/templates/controller-metrics-monitor.yaml delete mode 100644 helm-charts-openshift/charts/kmm/templates/controller-metrics-service.yaml delete mode 100644 helm-charts-openshift/charts/kmm/templates/deployment.yaml delete mode 100644 helm-charts-openshift/charts/kmm/templates/event-recorder-clusterrole-rbac.yaml delete mode 100644 helm-charts-openshift/charts/kmm/templates/event-recorder-clusterrolebinding-rbac.yaml delete mode 100644 helm-charts-openshift/charts/kmm/templates/leader-election-rbac.yaml delete mode 100644 helm-charts-openshift/charts/kmm/templates/manager-config.yaml delete mode 100644 helm-charts-openshift/charts/kmm/templates/manager-rbac.yaml delete mode 100644 helm-charts-openshift/charts/kmm/templates/metrics-reader-rbac.yaml delete mode 100644 helm-charts-openshift/charts/kmm/templates/preflightvalidation-crd.yaml delete mode 100644 helm-charts-openshift/charts/kmm/templates/preflightvalidationocp-crd.yaml delete mode 100644 helm-charts-openshift/charts/kmm/templates/prometheus-k8s-rbac.yaml delete mode 100644 helm-charts-openshift/charts/kmm/templates/proxy-rbac.yaml delete mode 100644 helm-charts-openshift/charts/kmm/templates/service-ca.yaml delete mode 100644 helm-charts-openshift/charts/kmm/templates/serviceaccount.yaml delete mode 100644 helm-charts-openshift/charts/kmm/templates/validating-webhook-configuration.yaml delete mode 100644 helm-charts-openshift/charts/kmm/templates/webhook-service.yaml delete mode 100644 helm-charts-openshift/charts/kmm/values.yaml delete mode 100644 helm-charts-openshift/charts/nfd/.helmignore delete mode 100644 helm-charts-openshift/charts/nfd/Chart.yaml delete mode 100644 helm-charts-openshift/charts/nfd/crds/nodefeature-crd.yaml delete mode 100644 helm-charts-openshift/charts/nfd/crds/nodefeaturediscovery-crd.yaml delete mode 100644 helm-charts-openshift/charts/nfd/crds/nodefeaturerule-crd.yaml delete mode 100644 helm-charts-openshift/charts/nfd/templates/_helpers.tpl delete mode 100644 helm-charts-openshift/charts/nfd/templates/controller-manager-alerts-monitor.yaml delete mode 100644 helm-charts-openshift/charts/nfd/templates/controller-manager-metrics-monitor.yaml delete mode 100644 helm-charts-openshift/charts/nfd/templates/deployment.yaml delete mode 100644 helm-charts-openshift/charts/nfd/templates/leader-election-rbac.yaml delete mode 100644 helm-charts-openshift/charts/nfd/templates/manager-config.yaml delete mode 100644 helm-charts-openshift/charts/nfd/templates/metrics-reader-rbac.yaml delete mode 100644 helm-charts-openshift/charts/nfd/templates/metrics-service.yaml delete mode 100644 helm-charts-openshift/charts/nfd/templates/operator-rbac.yaml delete mode 100644 helm-charts-openshift/charts/nfd/templates/prometheus-k8s-rbac.yaml delete mode 100644 helm-charts-openshift/charts/nfd/templates/proxy-rbac.yaml delete mode 100644 helm-charts-openshift/charts/nfd/templates/serviceaccount.yaml delete mode 100644 helm-charts-openshift/charts/nfd/values.yaml delete mode 100644 helm-charts-openshift/crds/deviceconfig-crd.yaml delete mode 100644 helm-charts-openshift/crds/remediationworkflowstatus-crd.yaml delete mode 100644 helm-charts-openshift/templates/_helpers.tpl delete mode 100644 helm-charts-openshift/templates/config-manager-rbac.yaml delete mode 100644 helm-charts-openshift/templates/deployment.yaml delete mode 100644 helm-charts-openshift/templates/event-recorder-clusterrole-rbac.yaml delete mode 100644 helm-charts-openshift/templates/event-recorder-clusterrolebinding-rbac.yaml delete mode 100644 helm-charts-openshift/templates/kmm-device-plugin-rbac.yaml delete mode 100644 helm-charts-openshift/templates/kmm-module-loader-rbac.yaml delete mode 100644 helm-charts-openshift/templates/leader-election-rbac.yaml delete mode 100644 helm-charts-openshift/templates/manager-config.yaml delete mode 100644 helm-charts-openshift/templates/manager-rbac.yaml delete mode 100644 helm-charts-openshift/templates/metrics-exporter-rbac-proxy-rbac.yaml delete mode 100644 helm-charts-openshift/templates/metrics-exporter-rbac.yaml delete mode 100644 helm-charts-openshift/templates/node-labeller-rbac.yaml delete mode 100644 helm-charts-openshift/templates/nodefeaturediscovery.yaml delete mode 100644 helm-charts-openshift/templates/post-delete-hook.yaml delete mode 100644 helm-charts-openshift/templates/pre-delete-hook.yaml delete mode 100644 helm-charts-openshift/templates/pre-upgrade-hook.yaml delete mode 100644 helm-charts-openshift/templates/prometheus-k8s-rbac.yaml delete mode 100644 helm-charts-openshift/templates/serviceaccount.yaml delete mode 100644 helm-charts-openshift/templates/test-runner-rbac.yaml delete mode 100644 helm-charts-openshift/templates/utils-container-rbac.yaml delete mode 100644 helm-charts-openshift/values.yaml delete mode 100644 helm-charts/Chart.lock delete mode 100644 helm-charts/Chart.yaml delete mode 100644 helm-charts/README.md delete mode 100644 helm-charts/charts/kmm-v1.0.0.tgz delete mode 100644 helm-charts/charts/kmm/.helmignore delete mode 100644 helm-charts/charts/kmm/Chart.yaml delete mode 100644 helm-charts/charts/kmm/crds/module-crd.yaml delete mode 100644 helm-charts/charts/kmm/crds/nodemodulesconfig-crd.yaml delete mode 100644 helm-charts/charts/kmm/templates/_helpers.tpl delete mode 100644 helm-charts/charts/kmm/templates/controller-metrics-service.yaml delete mode 100644 helm-charts/charts/kmm/templates/deployment.yaml delete mode 100644 helm-charts/charts/kmm/templates/event-recorder-clusterrole-rbac.yaml delete mode 100644 helm-charts/charts/kmm/templates/event-recorder-clusterrolebinding-rbac.yaml delete mode 100644 helm-charts/charts/kmm/templates/leader-election-rbac.yaml delete mode 100644 helm-charts/charts/kmm/templates/manager-config.yaml delete mode 100644 helm-charts/charts/kmm/templates/manager-rbac.yaml delete mode 100644 helm-charts/charts/kmm/templates/metrics-reader-rbac.yaml delete mode 100644 helm-charts/charts/kmm/templates/preflightvalidation-crd.yaml delete mode 100644 helm-charts/charts/kmm/templates/proxy-rbac.yaml delete mode 100644 helm-charts/charts/kmm/templates/selfsigned-issuer.yaml delete mode 100644 helm-charts/charts/kmm/templates/serviceaccount.yaml delete mode 100644 helm-charts/charts/kmm/templates/serving-cert.yaml delete mode 100644 helm-charts/charts/kmm/templates/validating-webhook-configuration.yaml delete mode 100644 helm-charts/charts/kmm/templates/webhook-service.yaml delete mode 100644 helm-charts/charts/kmm/values.yaml delete mode 100644 helm-charts/charts/node-feature-discovery-chart-0.16.1.tgz delete mode 100644 helm-charts/crds/deviceconfig-crd.yaml delete mode 100644 helm-charts/templates/_helpers.tpl delete mode 100644 helm-charts/templates/deployment.yaml delete mode 100644 helm-charts/templates/event-recorder-clusterrole-rbac.yaml delete mode 100644 helm-charts/templates/event-recorder-clusterrolebinding-rbac.yaml delete mode 100644 helm-charts/templates/leader-election-rbac.yaml delete mode 100644 helm-charts/templates/manager-config.yaml delete mode 100644 helm-charts/templates/manager-rbac.yaml delete mode 100644 helm-charts/templates/metrics-exporter-rbac-proxy-rbac.yaml delete mode 100644 helm-charts/templates/metrics-exporter-rbac.yaml delete mode 100644 helm-charts/templates/nfd-default-rule.yaml delete mode 100644 helm-charts/templates/node-labeller-rbac.yaml delete mode 100644 helm-charts/templates/post-delete-hook.yaml delete mode 100644 helm-charts/templates/pre-delete-hook.yaml delete mode 100644 helm-charts/templates/pre-upgrade-hook.yaml delete mode 100644 helm-charts/templates/serviceaccount.yaml delete mode 100644 helm-charts/templates/test-runner-rbac.yaml delete mode 100644 helm-charts/values.yaml diff --git a/.gitignore b/.gitignore index 87c2238c3..e60a26ecf 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,6 @@ *.tgz /*gpu-operator*.tgz /helm-charts-k8s/charts/*.tgz -/helm-charts-openshift/charts/*.tgz *.out # node app for e2e test diff --git a/Dockerfile b/Dockerfile index 76a5d4440..fce87d424 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,6 @@ COPY LICENSE LICENSE # Copy the helm charts COPY helm-charts-k8s helm-charts-k8s -COPY helm-charts-openshift helm-charts-openshift # need to decompress nfd subchart for k8s chart, in preparation for copying out CRD RUN cd helm-charts-k8s/charts && \ tar -xvzf node-feature-discovery-chart-0.16.1.tgz @@ -56,13 +55,6 @@ COPY --from=builder /opt/app-root/src/helm-charts-k8s/crds/deviceconfig-crd.yaml /opt/app-root/src/helm-charts-k8s/charts/kmm/crds/module-crd.yaml \ /opt/app-root/src/helm-charts-k8s/charts/kmm/crds/nodemodulesconfig-crd.yaml \ /opt/helm-charts-crds-k8s/ -COPY --from=builder /opt/app-root/src/helm-charts-openshift/crds/deviceconfig-crd.yaml \ - /opt/app-root/src/helm-charts-openshift/charts/nfd/crds/nodefeature-crd.yaml \ - /opt/app-root/src/helm-charts-openshift/charts/nfd/crds/nodefeaturediscovery-crd.yaml \ - /opt/app-root/src/helm-charts-openshift/charts/nfd/crds/nodefeaturerule-crd.yaml \ - /opt/app-root/src/helm-charts-openshift/charts/kmm/crds/module-crd.yaml \ - /opt/app-root/src/helm-charts-openshift/charts/kmm/crds/nodemodulesconfig-crd.yaml \ - /opt/helm-charts-crds-openshift/ RUN mkdir -p /remediation COPY --from=builder /opt/app-root/src/internal/controllers/remediation/configs /remediation/configs diff --git a/Dockerfile.build b/Dockerfile.build index a2fde2981..14c9cb6dc 100644 --- a/Dockerfile.build +++ b/Dockerfile.build @@ -30,6 +30,9 @@ RUN apt-get update -y && \ # Add Docker's official GPG key RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg +RUN wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 && \ + chmod a+x /usr/local/bin/yq + # Add Docker APT repository RUN echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu jammy stable" \ > /etc/apt/sources.list.d/docker.list diff --git a/Makefile b/Makefile index a4007f9d7..868bc22f3 100644 --- a/Makefile +++ b/Makefile @@ -47,27 +47,13 @@ UTILS_IMG ?= $(DOCKER_REGISTRY)/$(UTILS_IMAGE_NAME):$(UTILS_IMAGE_TAG) YAML_FILES=bundle/manifests/amd-gpu-operator-node-metrics_rbac.authorization.k8s.io_v1_rolebinding.yaml bundle/manifests/amd-gpu-operator.clusterserviceversion.yaml bundle/manifests/amd-gpu-operator-node-labeller_rbac.authorization.k8s.io_v1_clusterrolebinding.yaml bundle/manifests/amd-gpu-operator-node-metrics_monitoring.coreos.com_v1_servicemonitor.yaml config/samples/amd.com_deviceconfigs.yaml config/manifests/bases/amd-gpu-operator.clusterserviceversion.yaml example/deviceconfig_example.yaml config/default/kustomization.yaml CRD_YAML_FILES = deviceconfig-crd.yaml remediationworkflowstatus-crd.yaml K8S_KMM_CRD_YAML_FILES=module-crd.yaml nodemodulesconfig-crd.yaml -OPENSHIFT_KMM_CRD_YAML_FILES=module-crd.yaml nodemodulesconfig-crd.yaml -OPENSHIFT_CLUSTER_NFD_CRD_YAML_FILES=nodefeature-crd.yaml nodefeaturediscovery-crd.yaml nodefeaturerule-crd.yaml -DEFAULT_VALUES_FILES=helm-charts-k8s/values.yaml helm-charts-openshift/values.yaml hack/k8s-patch/metadata-patch/values.yaml hack/openshift-patch/metadata-patch/values.yaml - -ifdef OPENSHIFT -$(info selected openshift) -GPU_OPERATOR_CHART ?= $(shell pwd)/helm-charts-openshift/gpu-operator-helm-openshift-$(PROJECT_VERSION).tgz -KUBECTL_CMD=oc -HELM_OC_CMD=--set platform=openshift -else +DEFAULT_VALUES_FILES=helm-charts-k8s/values.yaml hack/k8s-patch/metadata-patch/values.yaml + GPU_OPERATOR_CHART ?= $(shell pwd)/helm-charts-k8s/gpu-operator-helm-k8s-$(PROJECT_VERSION).tgz -$(info selected k8s) -KUBECTL_CMD=kubectl -endif +KUBECTL_CMD ?= kubectl ifdef SKIP_NFD - ifdef OPENSHIFT - SKIP_NFD_CMD=--set nfd.enabled=false - else - SKIP_NFD_CMD=--set node-feature-discovery.enabled=false - endif + SKIP_NFD_CMD=--set node-feature-discovery.enabled=false endif ifdef SKIP_KMM @@ -177,7 +163,7 @@ docker/shell: docker-build-env ## Bring up and attach to a container that has de "cd /gpu-operator && git config --global --add safe.directory /gpu-operator && bash" .PHONY: all -all: generate manager manifests helm-k8s helm-openshift bundle-build docker-build +all: generate manager manifests helm-k8s bundle-build docker-build ##@ General @@ -203,13 +189,11 @@ update-registry: # updating registry information in yaml files sed -i -e 's|image:.*$$|image: ${IMG}|' bundle/manifests/amd-gpu-operator.clusterserviceversion.yaml sed -i -e 's|repository:.*$$|repository: ${IMAGE_TAG_BASE}|' \ - hack/k8s-patch/metadata-patch/values.yaml \ - hack/openshift-patch/metadata-patch/values.yaml + hack/k8s-patch/metadata-patch/values.yaml sed -i -e "s/newTag:.*$$/newTag: ${IMAGE_TAG}/" -e "s/tag:.*$$/tag: ${IMAGE_TAG}/" \ -e 's|newName:.*$$|newName: ${IMAGE_TAG_BASE}|' \ config/manager-base/kustomization.yaml config/manager/kustomization.yaml \ hack/k8s-patch/metadata-patch/values.yaml helm-charts-k8s/values.yaml \ - hack/openshift-patch/metadata-patch/values.yaml helm-charts-openshift/values.yaml \ example/deviceconfig_example.yaml # update operands image tags @for file in $(DEFAULT_VALUES_FILES); do \ @@ -231,8 +215,6 @@ update-version: # updating project version in manifests sed -i -e 's|appVersion:.*$$|appVersion: "${PROJECT_VERSION}"|' hack/k8s-patch/metadata-patch/Chart.yaml sed -i '0,/version:/s|version:.*|version: ${PROJECT_VERSION}|' hack/k8s-patch/metadata-patch/Chart.yaml - sed -i -e 's|appVersion:.*$$|appVersion: "${PROJECT_VERSION}"|' hack/openshift-patch/metadata-patch/Chart.yaml - sed -i '0,/version:/s|version:.*|version: ${PROJECT_VERSION}|' hack/openshift-patch/metadata-patch/Chart.yaml # updating project version in Dockerfile metadata sed -i 's/release="[^"]*"/release="${PROJECT_VERSION}"/g' Dockerfile internal/utils_container/Dockerfile sed -i 's/version="[^"]*"/version="${PROJECT_VERSION}"/g' Dockerfile internal/utils_container/Dockerfile @@ -346,15 +328,11 @@ docker-build-env: ## Build the docker shell container. fi .PHONY: helm -helm: - if [ -z ${OPENSHIFT} ]; then \ - $(MAKE) helm-k8s; \ - else \ - $(MAKE) helm-openshift; \ - fi +helm: ## Build helm charts for Kubernetes. + $(MAKE) helm-k8s .PHONY: helm-k8s -helm-k8s: helmify manifests kustomize clean-helm-k8s gen-kmm-charts-k8s ## Build helm charts for Kubernetes. +helm-k8s: helmify manifests kustomize clean-helm gen-kmm-charts $(KUSTOMIZE) build config/default | $(HELMIFY) helm-charts-k8s # Patching k8s helm chart metadata cp $(shell pwd)/hack/k8s-patch/metadata-patch/*.yaml $(shell pwd)/helm-charts-k8s/ @@ -382,34 +360,6 @@ helm-k8s: helmify manifests kustomize clean-helm-k8s gen-kmm-charts-k8s ## Build cd $(shell pwd)/helm-charts-k8s; helm dependency update; helm lint .; cd ..; helm package helm-charts-k8s/ --destination ./helm-charts-k8s mv $(shell pwd)/helm-charts-k8s/gpu-operator-charts-$(PROJECT_VERSION).tgz $(shell pwd)/helm-charts-k8s/gpu-operator-helm-k8s-$(PROJECT_VERSION).tgz -.PHONY: helm-openshift -helm-openshift: helmify manifests kustomize clean-helm-openshift gen-nfd-charts-openshift gen-kmm-charts-openshift - $(KUSTOMIZE) build config/default | $(HELMIFY) helm-charts-openshift - # Patching openshift helm chart metadata - cp $(shell pwd)/hack/openshift-patch/metadata-patch/*.yaml $(shell pwd)/helm-charts-openshift/ - # Patching openshift helm chart template - cp $(shell pwd)/hack/openshift-patch/template-patch/*.yaml $(shell pwd)/helm-charts-openshift/templates/ - # Patching openshift helm chart nfd subchart - cp $(shell pwd)/hack/openshift-patch/openshift-nfd-patch/crds/* $(shell pwd)/helm-charts-openshift/charts/nfd/crds/ - cp $(shell pwd)/hack/openshift-patch/openshift-nfd-patch/metadata-patch/* $(shell pwd)/helm-charts-openshift/charts/nfd/ - # Patching openshift helm chart kmm subchart - cp $(shell pwd)/hack/openshift-patch/openshift-kmm-patch/template-patch/* $(shell pwd)/helm-charts-openshift/charts/kmm/templates/ - cp $(shell pwd)/hack/openshift-patch/openshift-kmm-patch/metadata-patch/*.yaml $(shell pwd)/helm-charts-openshift/charts/kmm/ - # opeartor already has device-plugin rbac yaml, removing the redundant rbac yaml from subchart - rm $(shell pwd)/helm-charts-openshift/charts/kmm/templates/device-plugin-rbac.yaml - # opeartor already has module-loader rbac yaml, removing the redundant rbac yaml from subchart - rm $(shell pwd)/helm-charts-openshift/charts/kmm/templates/module-loader-rbac.yaml - cd $(shell pwd)/helm-charts-openshift; helm dependency update; helm lint .; cd ..; - mkdir $(shell pwd)/helm-charts-openshift/crds - echo "moving crd yaml files to crds folder" - @for file in $(CRD_YAML_FILES); do \ - helm template amd-gpu helm-charts-openshift -s templates/$$file > $(shell pwd)/helm-charts-openshift/crds/$$file; \ - done - rm $(shell pwd)/helm-charts-openshift/templates/*crd.yaml - echo "dependency update, lint and pack charts" - cd $(shell pwd)/helm-charts-openshift; helm dependency update; helm lint .; cd ..; helm package helm-charts-openshift/ --destination ./helm-charts-openshift - mv $(shell pwd)/helm-charts-openshift/gpu-operator-charts-$(PROJECT_VERSION).tgz $(shell pwd)/helm-charts-openshift/gpu-operator-helm-openshift-$(PROJECT_VERSION).tgz - .PHONY: bundle-build bundle-build: operator-sdk manifests kustomize ## OpenShift Build OLM bundle. rm -fr ./bundle @@ -611,64 +561,17 @@ helmify: .PHONY: helm-install helm-install: ## Deploy Helm Charts. - if [ -z ${OPENSHIFT} ]; then \ - $(MAKE) helm-install-k8s; \ - else \ - $(MAKE) helm-install-openshift; \ - fi - -.PHONY: helm-uninstall -helm-uninstall: ## Undeploy Helm Charts. - if [ -z ${OPENSHIFT} ]; then \ - $(MAKE) helm-uninstall-k8s; \ - else \ - $(MAKE) helm-uninstall-openshift; \ - fi - -helm-install-openshift: - helm install amd-gpu-operator ${GPU_OPERATOR_CHART} -n kube-amd-gpu --create-namespace ${SKIP_NFD_CMD} ${SKIP_KMM_CMD} ${HELM_OC_CMD} ${SIM_ENABLE_CMD} - -helm-uninstall-openshift: - echo "Deleting all CRs before uninstalling operator..." - ${KUBECTL_CMD} delete deviceconfigs.amd.com -n kube-amd-gpu --all - ${KUBECTL_CMD} delete remediationworkflowstatuses.amd.com -n kube-amd-gpu --all - ${KUBECTL_CMD} delete nodefeaturediscoveries.nfd.openshift.io -n kube-amd-gpu --all - echo "Uninstalling operator..." - helm uninstall amd-gpu-operator -n kube-amd-gpu - -helm-install-k8s: helm install -f helm-charts-k8s/values.yaml amd-gpu-operator ${GPU_OPERATOR_CHART} -n kube-amd-gpu --create-namespace ${SKIP_NFD_CMD} ${SKIP_KMM_CMD} ${SKIP_REMEDIATION_CONTROLLER_CMD} ${HELM_OC_CMD} ${SIM_ENABLE_CMD} ${SKIP_INSTALL_DEFAULT_CR_CMD} -helm-uninstall-k8s: +.PHONY: helm-uninstall +helm-uninstall-k8s: ## Undeploy Helm Charts. echo "Deleting all device configs before uninstalling operator..." ${KUBECTL_CMD} delete deviceconfigs.amd.com -n kube-amd-gpu --all ${KUBECTL_CMD} delete remediationworkflowstatuses.amd.com -n kube-amd-gpu --all echo "Uninstalling operator..." helm uninstall amd-gpu-operator -n kube-amd-gpu -gen-nfd-charts-openshift: - rm -rf /tmp/nfd && git clone https://github.com/openshift/cluster-nfd-operator /tmp/nfd; cd /tmp/nfd; git checkout release-4.16 - $(KUSTOMIZE) build /tmp/nfd/config/default | $(HELMIFY) helm-charts-openshift/charts/nfd - cp $(shell pwd)/hack/openshift-patch/openshift-nfd-patch/metadata-patch/Chart.yaml $(shell pwd)/helm-charts-openshift/charts/nfd/ - mkdir helm-charts-openshift/charts/nfd/crds - @for file in $(OPENSHIFT_CLUSTER_NFD_CRD_YAML_FILES); do \ - helm template amd-gpu helm-charts-openshift/charts/nfd -s templates/$$file > helm-charts-openshift/charts/nfd/crds/$$file; \ - done - rm helm-charts-openshift/charts/nfd/templates/*crd.yaml - rm -rf /tmp/nfd - -gen-kmm-charts-openshift: - rm -rf /tmp/kmm && git clone https://github.com/rh-ecosystem-edge/kernel-module-management.git /tmp/kmm; cd /tmp/kmm; git checkout release-2.3 - $(KUSTOMIZE) build /tmp/kmm/config/default | $(HELMIFY) helm-charts-openshift/charts/kmm - cp $(shell pwd)/hack/openshift-patch/openshift-kmm-patch/metadata-patch/Chart.yaml $(shell pwd)/helm-charts-openshift/charts/kmm/ - mkdir helm-charts-openshift/charts/kmm/crds - @for file in $(OPENSHIFT_KMM_CRD_YAML_FILES); do \ - helm template amd-gpu helm-charts-openshift/charts/kmm -s templates/$$file > helm-charts-openshift/charts/kmm/crds/$$file; \ - rm helm-charts-openshift/charts/kmm/templates/$$file; \ - done - rm -rf /tmp/kmm - -gen-kmm-charts-k8s: +gen-kmm-charts: ifdef JOB_ID @echo "Running in CI" $(KUSTOMIZE) build /ws/builder/kernel-module-management/config/default | $(HELMIFY) helm-charts-k8s/charts/kmm @@ -690,10 +593,8 @@ cert-manager-uninstall: ## Undeploy cert-manager. helm uninstall cert-manager -n cert-manager ${KUBECTL_CMD} delete crd issuers.cert-manager.io clusterissuers.cert-manager.io certificates.cert-manager.io certificaterequests.cert-manager.io orders.acme.cert-manager.io challenges.acme.cert-manager.io -clean-helm-openshift: - rm -rf $(shell pwd)/helm-charts-openshift - -clean-helm-k8s: +.PHONY: clean-helm +clean-helm: ## Clean up generated helm chart folder ./helm-charts-k8s. rm -rf $(shell pwd)/helm-charts-k8s copyrights: diff --git a/charts/gpu-operator-helm-k8s-v1.2.0.tgz b/charts/gpu-operator-helm-k8s-v1.2.0.tgz deleted file mode 100644 index 671760e1647caf1dc7830078d8b3307ba9c98770..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 89775 zcmZ6yV~i$D7q0uXZQHgnZQHhO+qT`)-P5*hW7@{FZ5z*dznz?uol{v!rRr~0rPjKy zbrZ!xqXYgofHVL)V<{D8Qz>~)Id2{g6E+QIGgS_29aSDqMGXy3c}+VTV+S*DHDyPB zDRVn}z}5Gs&lYzRbvROX@9B{aBks>8^YVW1+^VasYyACfuh(=w<{?3nF%}9ir)|eUY8l{Fr+@p_gK}6@VpJpwQ49W_@_}iQ4e|w< zWmsVpj*H(kzfb)7vN$***ik!xm|0AHMF+r)uABg2z66p5jdCws?uvngLMBCgf)=f-lK7J6<~W)I?&PrT#q+wH)Vc&Uy(OYDIW0YNXpEflIS&Y|?HYs@`@TwfgZ zh6Uexz;_&K@3Wa`urgab=#Uxq4w?*c1RR7&93(#?3K|fNBqFtkT`71jFCmIYnI*5RYCO3ctqa1`aqDMToy&5(ppw)%L>&LJ@tY~6W&FW=3 zf$KWLCj(AO`u%W$;pyiV&wnH)%r{noCbD5>84}WfHqYWt#_FJ7bdF%&cM$D1Prkqr z!2l&k{OCw5$^bPm@5p0>Kl;Gsz%w984GIzxXT<8jLtG-35d0-7BHMA7+usMt-wv$Z za_`!05s8Wp$|5e?Tp50YBi|#X++*q&IxrY9y9^wh&-{M3mDQw1abZv}J+V%35EEgE zp9c(3!!|%2Pqf#;p&d7tKd30Sej!zOw&rl2OaN~b>NV4V+nKty??7Zl@26UgNUkGDxG+Pu@>;54AP53R>+b4k zX3#aMrez@L0jero!xrqz$&WAeP#t>*uZWJO0~A?B5K&2k!}n$Q-kDz}0Ss{-dMtPz zJq_UCgbW~pIfALYlj;S7Cy^8=1Q#QOT#I^33oZ`L7#vw6R@lMM8wZ&Z-W3#_2Sh@P zDS4qCflUaHB%w)~Li#T!_v0XhkUZ55ge}rJu?d5}p{KGWZQ?d?r0Rr#GFiyID61=} z_L)~*7Z8iGN&ZEhJ4Q!z>j5gg8$=GQsdA17CT+uR<%og!`47 zBv&%j)A~Gvd$NpKwlgm-YUdjOLBc|gv)FxOAh^Y%H^Tf~V1;i&D?=psWS_x*?Dcoy zDL<;GNzMA^>SP3->yT6F2#O7a4fU!xK*mo*LB^v^vH`p*>3_JPg~Y(FS~K0Dl0ii*Cn546!V{+hrW2B4M%Ypc zYBLp8-bajTfBgWP_Rk_p=E%k}sDxc2OCVNsJfl`Y|I}cThl>z0+KNiW-2j`Bz!+nr zzbm|%?4SVL_#`UGLVppaBez(vWryWFk{~pQq{8&pDJ7HQvm6k_MyVgrf!G6(^>dHm z5^}L3lj&N`t#NMS^}x}nIlnc!cRS6%QtW-PDVqe)qe&8nDa*=6x_G;bVmL5kfWWr) zJQPl>7yA%+Y-KumRgrcFL(=l{c2eQ13x$x(7>Wt}J5(b(83GFn4pBUed<>FfG8P?7 zvfC~Q7jz<(3BX4>8zM3)5;9wbW;Kk9rzal!fbV$XiDh~*N)xugBKzbH)3ubGtwc_t zuJJHR`mf*Iqm^77dRUl}aaklK8|m5y#`u1bPG=2Y59~0Q`BnkVQoua&kvcYEwfF-= zcwfeOynt+1Uwu9(l$y>R*d8Fbrq|sf~r%a-06!p^WO_^`wnX3#63CxzdAm1j7SdU%rfw1R()+ zL5S@?c!s6<@Uo~dWaTo;JPzx$#*%>vO|Ux~Utd|^WJMt%0_2_xP)_#O#F$iGrOgZ5 zwLh~MAFnFxrY^TwikLjqu#vYH9gEdZwvb-lco;j-1b?0c9vug}m*Y{jsl@Mn9rpHP zlHdHLK**T<`Pa@nv#@X&wx_rolB!d}cYEv7Hp6-#lV=yx+RLseUx9$TK?|ZInUtS7 z35|VX!m{HiqQj;5=}q19c`BISf5#Qv#*PnP&#hVCLDz2$Mpau{L#o-0qDU_EN_DN0 z#GzP^V!AH5*Nk2}0@kg2L50Y4umHUq(zXN%iAmb`wV|SQ!v+ok%4}_n4mF4_wGErt z(30>+YX7$FCz#z%-EabvybB*b8Ok&PP);(%yV9BvtGwq5z*)?}d`Oq)8N}zqQ@{{M zRA4mIuY`Dn_MY2bO;&RT1MeisZ`ACd{LMH_%5r2(+cGk@}0piTr?pNnJ{V=97xFv7o9vbd|m=Nb-2ip3%~0^I@({RJG{g3#(( zII+|CVH@|$R!(Od<0d|is!Hat`(q2?;_txRbz#ZdGRkOPSsR$!i*l!Z9x*Rk_+4N> z|1)pkC$qp@B4#>uYFc$^t#prO**D6-b+te#N~~T*1oD$jfq($ zysRjL2-V#|JGMxnl&d4!IM*CI9t|I$_6OM1D#nav6;tX@)5cP%qtYABJhP*QA0KH` zfT*0G3Dk#3H}TI-Ai?$82ejaUWF?~{_Qo&)moEb=c>)WzgLpo69Xu_m>VePQ-?h&_ zyPh8o+s|9mGMh9f118sRHDyMiydaNo{ zxWO!k%}(|l?=(_=)?G_eu}_?$;Sgu@S*SBsERnd))h;sW`r90I%#f?sDNeBu^+!IP zN`hEt)=Pam#qITOj#OjUD_u$*-S?24jVQAEz4wp%8tz9VVFe14wTKRkX85eSWi!IT(PR?+R*va)t%q% z!_(LBoq=nAU{`f88 zoFXC<5|hAH|M&gD!`;{L;Q9T|f;0*U1y2jB4UE!0-OS4>r>nEu*Z!T^ZFf5x0bTGf zSuL@;?Gcu|@OrMPi|`x*4D~g6&*6dXZKGhMrF_PBKStb4Z+^i@Lyy-g@6dBi?I4go z32^7So{5d30%#QfM6us^q@$1o*IqyTK$&0ewWk|}vJC^w+(ICE2E>V|1xP}URNdo2 z*ku6^b-VIA=fSgZjGcEvL`-oFUp8IzeH>>mT{t6G1}i46zupp}ZP1 zsyI_z^|JmX*}vsLI!7gx5)3piQl?SpaJu2znPS=e%1 z?@X83l2)O~-wN!_$fSPhIf{h6$=|q zlFVk>5Tc`ugPbrHJ>zhLosI`?Gi(C4zOVCJzyHgoR(~`_VNF?z;DL4I8FUbkWtFM} z6=kRFLa&doYObqT&xrJvug%=^3k%FcyKK0U7Pf|?cSl2R_=UU{A-R!da=z>!H8BGxC9=a(;4S@goC0kA}Sg#=!Re=EtV~-$0~$dO=|S68>H3NRV+uIW>Y+P+a-I zR4DFdDHSOtxbW<~gY9fF#V(%HaWbEk%bp zg^dW#($_wESSd#+>qU)w$n(-sr6KU~r{COE=;Sl7E%u&>>GSe!7s}h^T@aOS&L`uR zs?}2OucExt*6)?u)}t387fE%CcF;vvT87?Yv%If@;B9SCPtQdDla2C?FGGdf_PM-w zV3Xh>@Z6#HQIK!?QR`Dn z05Cxd1Yv2Q$*41kw2qn9!iQx`Dr})vNGw0AhFz~G3l4Wv1Vg>1i3=mAmNmf${XWG( z2wi1JIbL-KkjEyXKFnIaG6h1j9}6kRT;JN{tfQ`p#S+kOmK*lrdcAcFqKT;Ws9Y7H z&lR5$il-dNv&ghAAsGXf{${feLet)zX0#NBEaTb-_> zX?#}oPpMrY{c4Rz>t<_7{ZA7pPlJ%@tF9p)JNa>)v+&7+Tu zM4v2~*c*7Fn0Y8=L9)DMSS;T^En3DcJ=M*+x1Fw6sh4u-D7G1;MPNVB64U#nrRMuWI!Sf+J@FRIcsJ(DY^^&mk5xM+Y86VPY5u->(*H7ua@nzoF74&09~^uF)!)*byW2ml_x_Lu zoD9A444GqeoOhSD^DV+Ow-90WU=4|UC8FL11<`l+=%6Mvu^Dek)@(DzbV4mTWbQ3N z(5yJ>0sJ^@1alI}JxBx|Oq~M<07HQ9IS2YY5{ok#(y#SXhpX+5o!g5u-&4E(rAv>` z?75F~OP@?%)LcBL5`-0$C1`h2Bu}?Kk8k&sFL5R~VjhzG^w4)bsxkK}6?62F;&&^+ zX2>Yze)AMevt)`KVFDvd%QOY92mKXD6yMc_0)1MUmZv>HV8IHIFi|!`bYI8dyA=?q zSy7)$T@7@>^Vqt*9f0H(*u3y~?c=@j#kmUn*=)E?pnv&3n)p}?{i+6k&#zB)eTxNp(FWj@g~c-*#q|5qO*28X0@n=@9AelSyt8kfa~S<&u~Flk^~^xK03TMxmc=Al|KiKrzA&Yz~8acHe+`DxE}1R zzl-xUiV&(}$WSR>gI0*rCeBE@T*{CtMg9a{3XP1fiS^+cobzMX*C@Pdd*S2S@)?SR z{1v8mIa0D&qIvzsBFJdSEo%CWn~;yE5a9A9jF}Yh3rWS0F?4ywV*<*F9GQ;(BL2+- zj2V5=)*md4q6mgm#(}F5D?SFs1OS)B%uom)N4&~)#l!(4WQ?g*@QLiB>BjC8D@qSp z2*(Ma_&xT(#S88d_747@&h~?MxRY@Pub6Nav`=oj#OQkP_hTs zu*0-BFnAW+%t8p%U%E1DXHh18*Z7E4sl-UBBo zF8XfzJ6{ERuVzPpZtZJ7{ZQT+_dp7{d#4-0G##2iWVE`e%e3o@wOi0>Hw%8!nSVhz zTYp9#HK*Cw4zGQ4ZhGsZk*fUs=uSf=UG6Ei-&h0s#KoQ$X9XNXdg!Qx$Vh4}%em=( zh;2vkxe6?rBm(;QZtJ%F6c=YQ@rr#6ckzGP=ZIGSW?nKoNgcCu+0;xW{T;Mqwx9-` z?GMu3#}V2I3wa3!?ROj~&rB=Qoeqa%1dm_DDKB;N#aw2@sG=y82Lx1G`OIKdO0{r}atMm*r^E_f7Pltnkf*4Qy6N1#aBd-v)XoSsKsK z{YACG@FBg>$ag4Ns9lbm{*-^$RWPZHz~0m8y#oc8#;1sZrsz_ygc!B6_ARV{3|_%& zv7M>9ft}z*sE(VvejHb><{Z$Sh6oZ)TH+u0;bB^yw$`f$e%VRe$A#>2Q_Sg#w^oQ< zf7Hh=r3x#TFJJ)kY#R{-LU(^4{*lfjG%!IaTyQ7J4JP`qGld`yNczpNsPlg|E|qu2msF1bm=as zjtAS(pKA>uwE**&tiYgxN$uV{vH zGdXshq&3)1Y{kQdTP0c>Y`u;@Y1oEM9A26$@7KUu)&cw~Y3?ai*lCW#>w;I4%-@=6 zpLDM5H7;ajEVm33)(SbK?laD@?`KQ?=;`GhepICfuw2bWY`-K{e1%<)r9*qmN?*)a z5$gRx71Qr7vlF-6P zJ5)tY`Nv1>4I<~}=fIaxwMh<@ICwiynGFe zi;C142`u^~YHv53%>&-_>74?-l1^jC9Wr=#3v|k&Q7Krvbu{WoyfG$b05ok4JzEx( zwIk}-IYCHNum>!Zh77{>LrTA-=hOrV6Z0(eVW^+_O@4Y{MX&1ut3U3!kNhsA&2y4f zUJh!{M-J!j=_aPP;?l~vqa5_X4SDU3-?Xw9O!&5 zo`KV6JBNs8-tRzz7mLIG)zf~$6YrX0VMSdB`jl@ZeKHl-b|>?nehN0Y3xZ-Qr;5|) z#J$5K#A)PXfo!nHmhTjAu)jzBbibm3+8YPlbOYI6;OB z#ZkAH5W4+O)q^+1uc#=2_R(~T>)6@^WTH)jB$8FSxC~HHLl1BlisrxIq~22+Q3rt< zlF?LBX1hpztwIu8)ld?8VY09fwNI?wECBESL6nncfyyjrqEaA}K!VC8ZMh&Z3g|&k zDcOQuEispXf3*aH_)UBLB`{r32Ckg7|J%QDonaN&o*YqIerDERLew}6{ypt$|=9I*E0Mq&SFeCOq5RtM;tWC?afKooR47XV%! z0jEaKdAR7fGXv)v~w^#bL zR)dQA|FlAv$x?8*De|Ed^pFKD*4How$g?Vklze=Xrzp6aoyWy8!e*@~U*>G}YH zlETn+?f?3I9*Z|Lf1qD>2|tq<=J%H>D8UPYrIMBut=HXfxa_Oyb<>T%C5cxq` z_F9AuT^Zm{^c4Y)HQz0B)Mjv_uX|$xi*G5G9f@yRVgt>|Acy%5u2Uzwr)gcx8$Adr zV_>tzttyFSXsfD`Am%uJovq@~6u7Y-ycRMl^`aPm1{56I!BCQ9Jle%8JPqOB)m%<$ zRz*1WMm1Cs$DXVt9&av}0^M#+XmWXFZz`%o?C2Om;r4b-x`leH z7%%60BFAY&f#?3n8hYP<>%do_6dMUhBlarJobSd2NPNgqMZQzgfcF1-bEa&fEI@FD9o{<}Ph|8W)2t9tT&%F}O_6RUB6Y@+lox3%?vc$yO#glXf`(|r~G#{@^^(n%~;HJ?mKu|5t zQhx5N);{uuluQzG0=%0^kvw`=KSw}<^d`GKrmDCAVB9?f+2KZq2T3T4B<#^E;g@Wpo3)L^+u1AtY#|D@QT~}IA zOoeD+qu_owl;*JlP=k09QIeYZE%vrwwZ8ZXL&&rzR2|s=MjnY&svYwv3GdAbgO5uU zKVbSsIW61h=eX0A|1(7d)gOdnxsNCJYR3|G47qAAy!F@>^!05C^m1{2`)6+-loDRN z@Kjn^LCJD3)cBVQ8=}u;1hGg^fMZaIN&V;l2QP7WJ~Z!eDu9>T#p3q@ z&&g>baD|?X>W&dDy-PBm0stc5$L86&!fUY@|H``1A`)8wswUN=-|RP8V5>lNC-0e? zUh$qs-4L=L2o){lF3A*FEs(Rgg3;%rbL>%NOo+3h< z0=iJhX0H8T!fWBr{~*!emgns)=hjPw$nDMd_O_&Wp4*>|n~ma=RmxR0!0J3Tm4f;D zyJe;PRO=ht+xknTXQ_QN&UZJ%LZb_b@~F(FCPR=N90D?`Q{K2(Y!w8ay3$E|MzrT# zYbVyf0UAwvLK-h+d~hKmZRKT$5NWaRGVXOzY##EM8CP{_`%oHuv!a@}oB(hR51KAL=1*(kf#ubbmJBdN-? zJ)+X{PZlz+65%m340phUq{ts|QA5UWBk0mTA($vWyPJRDl2BFocGN%gPjKWdD=5VY zO+!b9T|WPVw7NDQde*lE2_@5vk#hmu0s+4lg-w(kS?Z8!HkAG>m4be3Oy-7y%4CbK zHqZQ3@y(ID?k6J>`W^r8)@&}CqVf^?uF`Dz$~s6vMOG(_#WY2tDnv&k1x4ol7>QC)m;e9@)F{~Bx2O905#Yl zyc(4|kHs$@j9nt~UueXk%Iv~VEDr*)%XQVZ$`PIQt9~4q;BI7NdBz_)sXGk!IO(kg zDqI@82?~?H*VT1Wpcg(qmz*d8WafKWGkTXMnvQX#HZ61E%JL9zE)U(PhY+LgqIpbr zS<7|LI$;||cWPnMtx0F0_SqcTCQOD$XEipM6JH2S(2}iG%wua&L3Hd8C?l214I^3; z0ktO7F8S}QqIXDBb!Mj-IAM6t_RipL))wHPuo%UjLULD*Y`E{rg(CqquWpz(W3*Pk zMFTn54W)W(OFR@#Ym;V5Z5$Xh+<6`B$uw$fL;G43bQhY`O;20BnF!k2)~b!8$jjyt zoT4B+d#Pv1x|gmaTr-Dpm`PwQ*KIFXURVqOY0u3w2g|z9rQixUD&6uQI}fz9Zda43v@s%ElW#NGaOcCm2@*T zvncjqKpmwVHts9=eYI>+4!W>B1B{q8#SD!Fq6Pv~42lzxtS3(f)4C0Je79Z<%Qg&6BNO zExD+#xm%hfW6XlO>K`tzQYU~*Q9O>y5ryez`~m?6B+JI3pvwqkVfAJID0$z?Cnn#6 z0JJvm5PeTD{CB>KGvuflfOjp)9hC2CUDNvk*Jr{5ScuuJqUp}mzu$SC_(!uXkJ8ml zpTa8zE#$c2RJz^odPr+~%aa+)#}Ts2!lBrc3A)x0f+6}v3wciiLuk@A*eui*+sLpM zWJb7t)n8?8(T1K7Y-u0Xs(T=HWo*`O`b3-mowBKHR&V{a$=j}5TeMs`_O;FHf!|D2?$Jdg?3jv|Pf*GqJ2orpL8odh42PxMIq~Vn{0n@6yF`*wY-8mJYss*;*N>}cJmP-5R9$%}(jzQaRV?nIu|C&RhX4Js)_VWtbdg*U4!! z2YmYY-p(BT6zu-q`S?kM^2Q&x-e~wOtUa$qi6`Xk&~?Rf)O6LAjW$7>usG&QNS<6o zahgVU|pg>_q5M_W_i+ijF#K{&;=E0 z1?o;3cq6Zrb@Zzgh1AzlQqW%iCJU##yu4)I}BZx?2(wrf0? ziE5d{L}YX&F*>Gjt)Y|ut@yAUj+q4|3I+QtuO*j~(ghY<#UenC^{&o+6T?YvB_EnB zhmZo_)Oa*Pa+sO&!w(q+9?4WV8&!n*%!$u~_>oeJSaAg2T`fQ+ns;IRL?n<*IrSY5na=(s+~hC^C8-CV8ET$!QHMqv+J5+E4vC3z>CTe}Tl~UyT=IcLF;WN>9t{J? z2&D`WESnYehLWBWCU6ppLX%Q3E^3`a?)+Cs3G0f^8YW@_U-MqyNo+s>Ubz8@=i>m3 zm}Z7Rdfh@)NlrMgwx%+}J9$P-kJVr!+6RTCo?DwOc{O^h%2Ft}snk^mim>V!zR{68 zipRxuH+|0Ch;>Tk4+>o$-IK5;gaNHuE(gf6--yYsk5QVsfkyTG-omtG%3-9`gU9sD ze)thS>^)CZ#T=I`2w0H*5v3&F)qc15|9R6Ji z7J|>W=?$3gSV|}~D>f+!RaxPAQGb|WAInEA_tNk4N3h`A{%-bruSXH~cfS{>S9##~^?S2|{^%*w?{9l% zW{8LFKmp)Zb3ejQCjEx%Po>S|8X4HlTzb|Fj*U{vQzy=#Q>AJR%^-60@I?fEVXq?x zMuG&DS{52B$I6pCv-)i=>gm6?)t0q7lHj;k#_Z=9HHZ(ZyA?Z*%+cT(Hp9oh4#et5 zzzKco1}sV`owh-%I+Q)X3?DkoPSMGe+G#`+o!r>$e=Tp4^(=ao`#sG?Kb52KL4TJ* zmx6g+tB@YX@#7BwaZey=aW3tZ=yCGMra(t?ESFNnP~%!X^vAi{RGv-bp@{`UVPVm$ zm8ezN1`eigIm|VlSRI7}rj82pTE+s4KKI)|OYm3L=l{1*7+vQTU%_7hh z2UDfVrMt_}L{eo!Q%2S(LPe?cIH^j^_z=)v$FTBTtqf(~Q-KgHT%0l#TDdhe>e0Xy z`~w%dJ{!_w4oG(u0m(AaYv}Qj@!wrYO3$+HYi|<}D;-^X%yAh>3BfLZc6S_;Of&fn(5ccleS-A0z;!iKj1Pu`%g< z$JmvMzSXq;#g(H0y6w1eu=+HrfF>dyjV#*+$P@g|GNLiWR)h!b3NZ(1=fwXYINSwJKi0~zA4K*AC;|{JVLt*A6yk&Vy!Yf zuC{j9^`lLB(0Dl^xm9GYM&^9#6ZPuCS-0krzm+?ffmI%FG4^+za{QU4Sgqa7neyx$ zwz6I)iKeTw$Yu~0*I$$0))3!-`SAf|Xm4(0}1O-%h)!AmIfvM!p}ZKOy=j1TF4AXA!K zc7gmi@xxap#JY(UhjJAEsVbX}&9eQ&Rc`(R$5!h=L@Jom0r&OJ^rsrMQXRdk@%%C# zEH;!pRDZBGU^s=w^F{RN!%r@QU}?de3% zjFY-`nZmbRSdn0YNh$(XqbvFX(cCU)5~<4(%>qnKAYWiLwoh1`eMu3e_9)IkF}bZ$ zI`iZS>L1?zJL)V8qX`XWMOWM6s4iT%bXt}6^Q;eyTKH7=K%7r+Y@u7)fu3k0#!B1h zqF)pCyOeH}R~)gTt-4Cxc$(*()4OSVI@Fc$Df4=l+cgguHsgT)A>}lAoswV=6BE_c z|AYg1ZMmbb6-X}11fvSr77`=I!`TPI6$xWPUfRvPqPz;VQQ9po#Rz4JH;(f5LFU5Y zsd@2&6h~CRg&5`dh^R_?3-1m!Vo0TQ-<~X`yjvzpZOX=4(jIZp-bMNhi7qxrc(18{ ze`vl#`_}v8{rO;IYVYB2Y;*6oD4V5#UU&<3r412}PB{uG78)O_{HC*YSjhq>s=_`* z|4W7#DgR#^7z@;-OAMtt)<78MXh(`fSP9_8*C@GhyYO|O`_G?^@|Y}Xj^Xw;K%p+S zM6iQ?_CI*#8@pGJZO8U}g7)@nkNoQ&*>>R6btm=P%W#|Tr_Jl|OTP zTy#`^!0t7z)Gvp%caJR`vJmW-JpJ}lhi~V94S%mKoLsrNv3M)m=PKum5^Ed z7*?qNI$(q;_GqPqPwNR(d3e=(^87wwluXEF+%;fFxzL6n+d%2|^Cgk@bw42OB`OoJ zXtxM83n1oP;1SOmGf7-N=!WTmd8jMFO&6Vi8rwMtNlVu)H?3abge9ijyUonBT;YA} zDP`ZvjHScZ@9TYoVd1l}`9)5U3LE;ca+z%862~V!oa|y3%C0-Y@033rxNad}*F$2a z%61_X;2sJ|c*#LifbJSnb;mgOCws-umBHAb5Q>cfx;j#fRvV`wEsLHrMd-zab=NB_ z!_0(701hIEs*>V0m8o@$nGkYa^~b53is4gtc#wyJS=y~qT0Dt;fMiJ^KE+^dpteYo zEhotR?fvR8YyZ@S&5DQddoZ9Pd)W%pmQmmJe3&pr>tDOAUsnjF!H%1>&Z@HRsZ*Bl zKnMDRz={0juRAtG>`Kt}1KwNTX%sLts~1jb_|i(d{jFb%YOC9y)@&p_{GsrWAh2jv;`a{W~T zRlHVER^@gw&nbWcHJor*R%Db?SDOiJxUX;dC(-rk$8wJY0wNE>0q(6}L#S=O8Xq2vppd7^2~`Nt&(zGYX=4!dT^4arOIgkgyykZ)?@V;Ku`-Ia-zCZKZa_rNC(= zriDpmjo(ab5|^5xxz#Be;rZ$!p^?L33NHQ2vutoxq^Owb0_Eq+pzlZC7E)G@{NF{YAw2xSzRRmqYSSG ztk8K{2n>SmPv2(nup5hpV6i|05w3!!GN>pmr46B(MPgnRM=uZEA0vH0%6HA(E$QU?_u6+`WsYm z8-f1)zf|?L@z(s!Uu&_c2hyQo=fDZwF4{m|Vq^WcsDCvie|L63##LEfbiG zpKmS!lb20;3Z-UWV>@2#IeTmf8nnYW43X4nj z_yR}UVlIk_EC^7VX+mYlWe`YTktnH6?j(O1E>T-5gtBqCmJ@kZjoV~wZ8Dz5K5O>| z7VFzp3RK84ow)vuuCy8ck9bQbS2Uqos{YaK!)4KFRTVNFY`jOndz~vos5DNBOIFM2 z*0=nxUFMa-3$n9oW(JBGlRXU7ml?Xs528}d-$2JSV7|&Z9U;i3uO~*`9^*GhsCwO1uneOGtF8*M^*(JH&J0DamXq-}6S}^8MfN%g4D>2MT6Qa0a^WX`dfY3iq5z zW?uzeY1)l2A%hfG)kB+RvmIMmM75g&A^a7fxf7cc`YM@ZlTXXtgW z`0*ixKauUAFQ@0xsRt|9*`&Cv^`@@O6=B>?^&be|Y8m3y^A}g&`V1KNc8%|@{C^(v zCAKh1TR{RHDUDK`wFz!+tBp2V=}5GdJdP2oIIPcazoHpC;%Z56+^_nR+7`N-~9~bz4Lh#YA(e%F`*H87Z7qeA7{sl&*eLip1KpSk)*7gdD zS9jt{Xyr+dm~nQ6gTa+^pAD;PT1{`^r`uUHQt3SBLsV94yTH5SNY7D)F+h_A*WIExNr z0_{6T@(xhm6UWMOyGqJNYMaw4zkAh&4E%4H;{*5@X|U!(R|IXU!b3!Oe@V{U<>{qt z#EFO!tK5p=Y>_582heqYZ^pu?ONkvKP&t6PfQgO6gz*U}1Z8bFj}Wd_aLf*pNR}<` z2nj3L6&Yg<(n6l}jEeP-#j!umyY2d5PP@eP`N21*)(J=7j;{7?ZOyC3sxRGTFGj=v zwH&2Q>}*e2DVu4N32*l6ZEV`n(sry#)6{Y^Exk|)G%q{-rQu)OUzPXkKlngPkUzsG zEV-w3>}nIhBe?KlH;F#C-&mlw40W?CAvT@0oZv=dZUL%#`wdtUo2 zfQk&Y$zBQ$LanVu4h6-LP)e3PFK3PRpOmqOe|}iZ=!6fYH)X=8bwx8Hq#$^Zdx$Dv zp^nOTf(iqO&Nvvr#?jGL{vpy-otCe>1-*-4&7ElV3;swydw+{i8HT}aS1eotIrS!x ze+*~YOi$tY({k|hbHz-m3Hlj4kEaq*YD2`z4(M-tt-=3p)em<3j9Xw4)!W?YZ{tco zd9?imoYM>Rx0$N~aVEomup3!EK;#XpAyZ7~(hVXX{H@8F>dYqoT89}#GRwdw7^eZI znQyyaNl@>)PASbxjRb$jXmGwZ6+ZiYn;L zI$7D6bHXSPCAWW9R#nXmp2{5t&J6C42I{zDU63+0T6t{MnrJU&ddMg}wg9gV zT~67al0)puevG`+r*2LAbl$1?cX6oBfAMTKKePGEPVjN#a>mWdca4 z69F4`Rca~VGYYzrqInv;DqRWEh|;&Y?3wGI)WUQcujz6kzdM3xtF`Hq8xxyd3aAgG z4FXD`E|1y794|l=9EK7j*L(n*qta8HX_E`BM5AEj&#FwXB+R&g$pjQ~oaqEHN(E*7 z?9N7Vwcn3;HGYUc_8%i6m)zZHXiNs1VHH&^K`{!81()*Rfphmy4*ZPes)uVTXPDcU zyg-rD*Vnn}#Jb1p$eF=UtZczuO`?}vy3MKSkf%w@f;pO97%sc2*7Ssd(kY{FNJxD9 z5TauHApZV3x1SP1{{%x9$u9KFjJEprcLxF^))GqsMp7@f%hstc0JLgmKUi!7%v z73ABM4_KT|Ig|!8)WiPi_NI<$YcC{)7|xtQfr#$YO^^?%UUyqH=%8t)b#VqjClC>8v6&cU@a?g z^-0>Kbrlns^i4B8*KxBZFuOX4wn=>EQHp9XPtyH8;3OyFgEFd_3u3(7!zL{! zgY_!2L+9US$a2cMBR}@78Z@2_bY&3C|B$j2e|OM0t!+-1ShHW*#9C!mikeZ&lgXuz zr*-QS%{5T2=th$=QxGd{awD52Ix<>!cyY3U@6ByZrqj?d&MJ3zxo2pED5%W?OG`SHd!6~Kdwcsw!|_yc`@c5ro7n;(Pl)f_&k;SL zm-3t5En1HoI`2hSmjBNW+|{e!zBX0x#v=sk9TEy-*4HJq<|IX#*@xZ@ox|<#k1~Zc zshmabYc6Y83Hxo^N$EmyLNhn8l$&r~`|=NBw}p~j?7Z?13udZUg8<*LMw`Ltfj^Q>yaidOm zTEhLsuZ6}m;oI*NC7?(_^e$Is)XD$ksK-@iXsj0{xAoq!~M zq9=sRQf$(UwX0wsqD*)Uvvey!2y%c3bu4vCW7=sgx+UACNO9cEJz=7ISKIe}EJ~bU zf6PyWp7pc(nI^!K7ihi6_|WtBkB?H!UQ!KNb+NARwKjaBz}tEV0F^?!Rv{3W@YIU_ z43ujSa%8~*nRLCtry_FmNMiS~4Y-FkfwQ0h!1K9s2G zgClC(cm*K{gzBXmER1{^A$}$Ore;G{x$GHAc1ZZ|QW!?Ti28|u1p+?KyKV6#3xSI- zVb&^``FRF}h||~C|CwQ*aF~0)Db)cY5O?L^5xg_(Ml8z4{O9g4oGx0yONlOE=e@+% zCGmceY|Z5L?ColH5IrIDP<|*-EU6pU6Fz{~x+D7!vKTcPpZX6+LFtGV{rHP5%*oNL$XlK+?Z6$8VTjodyqDQS-JRIgb2q@GMqwscjOv=00Vc!N{>O180?2- z7UY%Q|FMd6zumkv)^bv+DPhaYgt@sFb*~)Q^#S3+AOsYn|A(k|0PfxU!hVZW+qP}n zwr$&XPi?oSwvAKUwoYx^_}>11_ulu-OeV>mWMz`sJISnPJ)ck(uUgX~p}38RZ7pKR z^c$|0rAe{bCo7ZONOIUj&V_WK@&f8e^WBdZ&MI=Q|9m< z-%zaF0wriZP47D5Lqw!^(PL>mkh{U&APN8G0;hq~C_#}VbIltO#Zy-dy< z#k2oiK{L*?Z@q;b=-;-P+TvAUEhEztxcz^X_n=Inv*HKC;*3syqM4Btb(v5Bv7Iqb)g23Juh@@OaCQ`#gwCO!PY9f*#USvWF^0D_ zBq3H1-{GcHYQppV0r)h~dFutt0&ENg{Uk9R{qi-CVk?axMJf^I6bI62;LPLlrOd@}dmuhjtw3rbhuXa0dfrOYG7ql&TTD;{E;I7p;KBbS{H`E?qYlR+xv zf%Np$FMzibP;E$I=^3@W{Hw5FV9OR&X`Ox^K*I_Jt0;Yuh!twbkCOlvlzRsII*T;!+$V_7 z=SyQ$leq~Oxznhq)Xs){IIu|GCZv_Q5))2QfBx!pSh*^7eBO9Y-NMd?~X=-C^>gPdG^qbeUmhED6>0c5+op;nhHZH^>_6wvbC3H2AKU#o2CqpVHUPgCcz^tAJKlz|~#6gG^MsLI5?@ zS9=?ej<=`3FO$xDXNzw!bncJ-VtTb>QLH11! zZc3^FE`C~?f&OLQbee&$!NTeAu#uG!)D;6ge4DgP+oX()P|Cnx-q*Lm7Xf_zL!P|) zB>`F%a@vyAcGz&ev-V1II7<&=N^(3Qo>ERwb&PEB;FDJs`0=ywUO%Y7eZ%SQ zDTiebRI+1%@6j?Xz6|1HhdaqLWR}>@Px-nAd^E;XhTeUG& zeE5d-r{cHTd(W-kRCx4rK&cZeby|%1d5k!M2ttL4#k#fd8JfV7#sm5aX-*KEiiS?f z8L~;5(QXwXRU$djIQH{Opc;3alW*BI?XIN=O7mca6(VEN8hQ6zr!$z9O-I#IYlX;& z`(_~@kX<6~2IDfv{WK`9I54yve{NvhQ3vF+Jbd$OxRBw%2Nq()?|=XL)5FVG;w;w0 zYTx@@RhD;tr367TMJN9r;c&ypd^&lc{fq0(auEg0k`SF)h2HENOAW`Y0vy<#=f8dS zv;*5mfej}+kkI*_rph^kEJrk{P+39ZKW?nLquOk%66%l0gtJgg=SF;D-%9+F&&tUk zbA*Uu)?9%wR?6nP-7@XBkl|epdt&0z`%k{XHW_2BHKRqWiF2AG`D~CbB^9Mh!`|@3uZJ#esj^$SM0o41Wyq znDE4T*!C`_;wgy$N>a=2=c!w#-ckaLY%(rb#!wqgMywf&X=}IZi~Ibu@T<<-<3?ry z?VJT-^7&4*?2|r6g+XLB)RJ>J@KbGCUn-VC7%QU%(sdf~<>2P~MCuwcX))Z3+cNo*fs7a_=KbtQ4sJlGGTbf=33r*ym|JY!K2tRmhdx+%qCl%4*^?AyH~2*{IBg ziJI_zfDJ|9!-KCkrP`v}#3pjNr5PiXcssG{T6W}3UBz{xReO^4&bEr2+q)gutP&rU zin3Qsoo?R)FWt!|{AhcZKAi_xks^t~u-rJnIJN9Rjb zb{kfvCOzntxD#3d{qzU$Yxd1tnZhx?PupM3;LgtM;-wopyDf#hz^YoJznwmG0T(G* z9v&>aNOv&9aN&k6RWFy6^N>g{VpN;ouAcr*uHN^1r+qa`IX*hhUpH4vcaBb`f&uLf zQY9u%s>BF9$7w`d(AKp0{%e>kVgV(I_-Q)kAN} z_L@KjR{FuTw6Agc2pPQ3{(3xCgt)}1v%HQuFoWt%97_&F# zg2`0zSF5FH5R+)wWNBa%xrxiEN{A;Q+4yGNM39}_BORAVuh~o+Z@QQ3;Me&n?nI8h z6#3?Y1d>EBvG2sf7k%5J?o!tNYM<$v?=LxOEFD>goPsEP~jTo^xg01N3W)2F=> zUADtv<1a#1p;%lemJ|}G-`<|pIh~P1E2HXsVf)YVcSxI1$RPZ zam@@fdsN30vz`Zp)6EmjogKNpFGSP)st1B}j>(vj6Le^XCkW0UDHV|`qRa?xa-Aqw z;aCeJ`^Fiqv=8K?0#jB#`;$&8>w|@UI+X3k`M{p+GfWA360jrC4U_`Z^JMp;E>X3q zIgT??yF^3w>#s7rB`va`wG2|F)?PWL_?t1x?b;upv3@exa@9^$Y5`~W{;oqx#X-mQ z9DTrFazl3t+NGT>d-R_2IO%1GkIY`08QS$}(wuBvvG1wS1ijqcoF2ZfZ;X@CrjTy% zlQj0FVdlCgD(i02a2ZK{}edl>_Js;|euJn$0ONv%3ZE zXAH<)IfYXc%2}2e)iCi!e$!Slu_B*zZfRR>(?va8+5hIuOFcGgY8AhliS%L-8^Vzt zi4cQDsoH2A(U+=s98WA{pww_AhVo9%I{W7e>-|Kb7$uUB4Vi)F@Pe6<^`QL*<_VfD z?hVS5Q8^5U6fdWim<2^{{=yLVpkXL~;>8NK%)TFPDmJipIdTSSksGFG0PW zv9n=G)omwjva8}JD{3%tN<0AOrkls4X2VOJB+2E<86yOV@C%| zmHi#I%k++$Zr!H0AVBk<1DDs3-@C-vvkA-?G<9DBCQj` zNrs3|Hbht2DtWG|ZpjJ)tU&ao^<;kA@kVHID+^gv@b@Ik`u*V7s_xkUB(W?f>2wl78C(%Pl1Z|HJS6_{zIvZ5r&+nEZR-L% zevu`KD{23+#;R|-rjV-8 zWU5(N|FK`p%3%)VX~`U&LtcYP>7O^K#KwmPrYQ>1`|Z6(lUR}4?K9zW&$*OzL!#}~ z{Isen+3stc9hxZE$V0zprZ+7p)M%uw>{eeLRh$y4!Do+===FO4ow#TdX$2n*D=g@5 zep)hs*yK8D+Y-hMx7-Sz-ekdxT3vMnjg9er+4(f|d-0c@9f_9#SEI$sZxeLC-l77r z;+4J6V#8;E?OU#%+>3BP{5cwNM9Qzil4ZWnX=r=JjO#8PWLIh0lZ@>y88vSmKMFH} z&rU+}w0&^>^4&Y(|0Y0S+Ou4a;hMnF$VGFF16Wwtlxf@oy8d0;*L(>e+@Y=(2X<q?q*Ek3J%4EP86X8&F@ah;-4W!u*qY|71XI@QlM zc!jN11bM<0s5f>Sq@QZOx>90^`g;ebz~;WD6mR`4k@G3>bI7dV3Or@gFXfO z^xV9=*E2_}+eNJBS)Yrw+#ySi+g~oM*u{UO+s#zrYa#IT>h##unEzds;(!w#fcKLe z^=V#yjI;#wS@q@RI!Jl$1{+Xti@|RsR;H^ltlZzVHY8F7w2f`4WW>OiKcjgisOrvI6;QA*%wOaD}uUn}sZ$MKhHbkPymJp9U6~9lA zr@noyMLnbVshb1N?r7P7PSygt(>qz7cV;HHTE1QoXMKTzbCIagQ}AbxVm5&=AjvA! zq9;=n__n2jpNs$AIHJaI;-egTu!p5U(__XE%Z^lF9@fC^2AbL!_ya#ui04+m2sooq zF_gm*hZ_bcL`G#ohi)HIV{yi&YmwwnXlNpEODU-Ttp_iHS+>GEZ4r*`3#nQLX5874 zDSVBB7TQ>YRtrS6JOj~TrX-`08(wJSF70crSx#6F>4;ZDt<{IPAp^9C7d6hT=s~bb zd^|arR*K|- z=;|{%Z!Q&;3@$*wq$me11R3v;kPLN#jh*rxu7i=(DIj8p^PTli3PV~L_s{C&oohbX zj73x|s_nzv1b_YJQ0i?4F7U7X)yN{3n8-j?Y7ddTU)sE-ltsa$k$=m8PR*qrlp$q& zae9FCGAQ1rA|0OjJ0UbjbxMJ&R5ZKCVLe{DhP+K*Y(D}POJG<0_ zjNDJYjhf@4N*}{q$LY=M6-&e1NZ=f`+W693@X}cuqyi`(SCks$e{!`JC7R;|64R>T zYSAufB6O{7Q&Q8BjE-qfADW7V6F9z)ST3Fq7rURh+}{9X$*lIzsp%cxv?Z!e-WqOJ zp;iBHQS+fSA7K~6T*bOyER%f(*${rH`Kh(xZ1t;g`B4^P?*K#;g5*2pj~~z zGk@-hOX{z9C`Wv>jC+_mdMgCJ2QCD88Rlzj+T^Uzj-Gujx?TsE2~o zW$O|6oN3JZJqk1pO?5-{=mRe7>|WZUPXa!=-9Ar3cpnORng*c7kra*=gq-mkMDe#M zs_*=Dh-px*sh2_F=7325N_kSxngkh^XkoF&*g}W@q7VuQGE!VI6&GCF!b@feIT5C* z=Fc53D*#?Qu<2kz^Ll3&Ni=88j_$v% z*}o5o_7Hu@NQ+e=JNP>RmB*MYc9wKdIw<9EJP9->I(j^O{E+YfF>YQ9_ZJ2?f;jZw z*HW)pXgl=e=IrVjXy9=+QuAT|=yA1c^xh~_nPK}w2ru?UnKz&;m*WVt2$Kh|aDqH6vucUi$0;U4>!|;>;LP=5IKlIxPcB^~TsDW*Np;Fm0~fP3 z=Wh!S&G37D!zy%N7DJ`ZR3!hTG#eztQol8`h9~($M;53@jmVqO3jRE}eVf`pv~yB6 zU8(c6oNBM96eJqIDnBYWXX1nOjLj4Cazam4E$2 zQfa{}S>g=sO(P5_$_RWuo`QP7-5nNp5}zeq9KoEJm`0om?#A$UzqjaN&xU!z`obu>8TuiD+ zgV5s|hR)|;@AHs)-<8`nsZDNSA}Fq@I547q!A3+WXQ^GMvMWI{z&*gS@~k6b-M{38 zM>I$pS)a@FUZTP7ywT14NwQa9qPdtz^GaM0l5wK-^g@uE9^Qz(!ZKt2qU__%7H~Q; zh9S8WJ(D1#oB`2?S3tUrQ7{g>%V!w?_TCa2xF~MmR)eQ?Q*J*HS~V%;1HFWnOAjPR z>Jnbma-`XRhzR0^;LLkF<3{__v%-hxt4N{PiF}O2~q+YWpR9J0S)L8UX;7F z$l><7?ND_q&y|oEfV}R}RvZm7r zea89pPhco*b}DSv)}|dV?{PA_vv+3QVDG%n`El0mA)WIrHag(4%h-O(mMx`e_7g5F%Ws5SV+s>GO_#MOukpg%7U5Wz+oUqp zz!YMFqr0*-NpL!Dd97-zrbETUJ!Nt?yv4tGE&yoK;yXLoecqgr&0|JL~io z&{@|Bd}S58@;m8mIZ4jeCkr;*{@mH39s=0Ad8~ftg?nX{UGq+P8Wh@Oi{4bdR zq@#g(imRNRQ^)nU-lirUUO@MUv)}iJQSQQ<9w$#*loWnLMt)h#1@l3=!zN7255A}c zf^cxB>4xq=V!~<)kS3w}cz^2%q6!cgnPe1Kz8A5JArrcqdSNXfHBfsuyLhlvU4a5g>dZ`+! zod&KWy4(TML-I3X|1cpKlsrXq$hjYl9g(cf)=Y69KLm+edQi>wR~mHF*&!(7W9F$Z z)(g2Le4l89ZsU9Ox?(S>mi?fNnz`C=(rf?VW(3hplhuwu1>)PePclaknm(wbD7zab zGC`onRnkYw55DR+Fx7G;*w%l&$@cO`^qq1ccoX2b>-0K)0~okR-|cL6@?3o0Al#=6 z1)RJ9L6^Kt0{${Hs5`4{BsXYQ@hZPa8gJ4{J$#J-dLDY&@2b3x&jP-#ucy8cw$d+t zn8ve%vjDXY;k&$EvqbvYFhcEJnfr4=ooI5W6YKQ z*LpUoF-Rj5^81(=d8Fph|Bq0e@^)>?9JxpTnCPPAn#d5_gTk2#sJ8&SC$Qei_r}T~ z0kpXr%$x!k+E(uXKNoolXn7GheCmgmF)RrZHFzWrQG z$K5t29gq%xHL{0!-8bP2&p_ZfElI~uLAW~552#nBe>e8}=;b_^iklg{ldfqyaT@B! zaO3m6>!+pIO2?Y}r~|yr1I%aqnR;1~^M5{nZhC0X19Y~wSG)l~1M8y-R-5uN4-oJ} z9CtZjZ=?juGpGYV>&|XB=Cqjl=#M*H6?L^zTrgxt3T$T9JRBXG6Co!#%7VZGO%%yF z1D@;VfFu)$#T3)nShS3)*i90Lisn!4A0LF-N$&~{Lx^EzJEE@}+EsbG8&(ovKyrCg znxN-3ImtPTK(oz)XC-SGFRYYo{DqfU34$hr(l6rbsmzh2pj7TDCzB6P533mD2CkW3 zP7H-HU3n&~kT)y0!BTnfbF*RdZU_%`A#W_DiwCUSqcn3#%`$IB-opn+_9NwS45Q=P zP&G|UhPMJ0nhYSics}5G<;#D_h&}f;6^N0FX@?*bb}DT85i0bm!%GOPLWodz4##*C zIT=LV`8mE?inY+=_ED@0BDRZf#N74Tn4q$!Qu zZYOFSUaTW#f<_9E8dkPEBWf|ZN2#P{8b}p-fUy^x^MP?%!Q8?v0cN0V zH$glGyLuf7k)=!w=9Bf;z~~^e-8+Ecb>Uzf0}?36iv{<_ASo zIGykak`&@YRx}M5Pb1(WmN+rzft1FyOEz#X`#wKS#yPPv`! zvU2}K_1^md_!>2UfC$*y0ql9+<_UZ|0eZu4;~C(;y>F{N4&osoK0X9|AD%BdFD41j z`s)PhdAm6RzB&M)?Vn8b$sr-V-vw^c1mH(#o`RX!j!cX?D+v}ow!`lNa=g^;;n?!Z zTf5FUMY@*QDYVMXVI{)rDB}2YJ?k!h1yK*7LlE^>=taD^`U_UlTJ$uYa7_ywkbOnQ z#*&9IXFB7(Fy>;MgGywkHbx@+_{KS#<8Fv#a;X2f(+Gta#qJ#n|3-3jmY)35lfUZ) zYotwIt;8nGPI_-`2|1l7=DQ{Y4N0nP<~4{Clo9Qch;X{C&f{GH4qa;)=y)20#88+q;b^9on^d04>^5_lTnxe z@%LgJ2dl@yc%D)X+LU0m+( zAKyQ_^9=^^a%DSL)Ye9?ZdB2(-nDlhGZ(X;I2ABrVn)yB#Xo9>V4+Ju;R)gD3#%T< zy&Nb8DAm<;iBdADbU;^M(xM1Aj_l9ieX%e3$djJo?*6`{MA>TZ2`3(`O*8ITfp9>8 zF0fb!vB3+cuje1Y=*Gx+Ju9xtk?>?Jg?f8HQc5JKS$F;^ZT+Ws%ZO~F9nQUs~zJA94ac z%2W8Zia_=;u&<`De`oVbQoky1s;#2hplO_i#F+Y}31;WOQUl$d-SUHfQ3VzGOpsf$ zN}zBr0tTks6O7pGwD5d+BPw29PA(xV4OMIyc;fKTX7nsp*9Og*WM$TC^|Hq1L%a_5{RToePiYvs0*Lx_;&|$A?fuY$(os z(?8-*Guovd^Z`f94;Y*8 z3^-PP=e^-NU;Y>+xQa&V)W<7nmp3nF>ojQ}^H*NOve;VbVBcXe9)MQ!mQ>rcEiPUq zg+Do0cXy>7(8_c*cANasa}%uu!|xcH%kDNGAFOr(t2=tt4wMqj;njBKH+A$YJj(J1 z?x`z{BFi&8T`5l6c6Reh%Qp1i3^>+;X(t#s22y}@li?^WmkR657Zt?%dG@>1ObogZ z^W*DWCyoR&CTe6lS2R_7Hars4H>d24vd=cn=!Xdlib}SQkzI@YI&fB!JPi-LUn z)}1*|SL1eD1e+%I?U1>@^rLi;*sye=*QAi#2IQkoosRJajlrEZC5@WLxz9%>`0}r# zeEHpNR~35aW!T?}N2<5Q1v$znIKJq1MW-EGvK!gfA*DamQYYL{nvS|#kXp$eZM?$K zv{!^0(_Wedd(spuj`&($79=y$JXS=DSEQmlf|LtnzQz7xSsbDx53Z;4sTC6%~xr zwb^%}y*mx&XSp;EU6=!(+5oU+q*7<0PC_7~r{w^T&-3A`tgn;d@e8}Jo7iTH(~ZzA z;SKhtno}cb%wP~o#QoSJA6_EozYnlmKOch3v%CCJhd*Y!EK*<7OXL(e_(Vb|byxAg zDDPeq^c!B07+G2K$S>IhrY}Kbm)Gx?encxHIA^&vNy8MRAmoWLM+;A|iJ6LhwWO2KIEriAK#BYI@vh0SU#v-J+>jzk@yaa}@ zd!o%ZW6psW9s^jlO29N&&ZX}iz&*F%MN=_X#?{S<$|Edu7jcHGn5kdl*scFZP2o&N ziXWe+a9_gF`6*6w@p8X=cFq{-H*w7YFkqYm1Ol8ITLAXOd?7=rW7J&pzc-I_a*lU? zY&{$p5afK@9KRzX9B~miLsU_jlt_a~y;NxTA!x_ql2L`kC{C0Um!g8&_qIh`fejKd zQ*WSy9eGKFm86VA<>06BOx$n_Jsjv2$zOjgf`v}P{EBpuYTm#1qCiw3kuRh&NW>9X z$$7*H#zrm0HLC;6nFh4d=Ei&(SdizLeu}U`FtzHUk&q|%W@hM!cmz(phIl7b(nTLe z{nCwQC-Df+MB(kUs3$h#RxUj~@82oOwFXW)dP%0O!&OqrA$Y)>ub~QMsS_p0L3cRwttq}$%_S#*!`Gw9WDGYzeDVoH>q!7gjsDmjm2~Sv%obvp7@v8c8z#F4Zhla><%@K93$xPNN{y&5c608b{Z}m9OPN!D6RK zY>Dn7o4}_B!D@~IlfSdoDrErBwwi^V&oo3JO39urK6AX&tu*JGnz4=PHe}3LW*-#) z3LaMaXbV&FL9d=MRW4^iNjn~eC!}NeF(7npF36l@l9Es7`LNIV6-IG3tkII9E9mSF zjX>pAozJ#s26w#RZX zuQC!QX6{DC-;4PO<3U=5q2bM&dg1pFvKnL@-rPA_J=$<;U@ullOw=};&85bt^w^rL zByGMRjh;_DZw?bd#+6329V`!~h^xG&pt-czf|Ym`mh%^o75SW-w3_?sCQbh}8-iB- z(tUZ~Rsphv5|kTsR7JTj$Y`IAj8~HFlX9?vAlhvPbsHm0_3&9_Tm->tv9hb4w$}v~ zhDkqi6x>Ux{?nMjZdtcpzwmdmQ6X&FX8OnZs6=C0j)KKA_PWw|G~ z+G?)$Z}q*%-4B;nzchu#g_$tNxuA=-0(593**#}#D6WfXSkF((?Kd8#F7pgZr1--? zQ+pu72K7Pf#NQg`>j%{lwYZEi6k@yI;Dv2&Na{IRZ22%89YyeHII5U?2hNHdh2I{s zs7O>TD?pRd5~O>v%*2>5<*cFpiQ%miXQ^+fC=7zZM)svK!0~W;;wD*?g&ZYPwr5Gz z{bz?UE#wzsmfW;*r0xAB*FmQU788`J-M%n0c(6sjxx1AJ-{+Fhq}%Y?qLMw%0%>b>Dn4;lc0`FbBxx=$F38Q|vE%2zJ6FNkl4ACDzv;cMTeDJS%|&yfZOj>sZqZ0P ze+ow*Hp4ZyZPTR77U63*&5&j$&Jl9j6J@luUo0TXssAYAzvzknZf@t&JBjbk7nUVC=rq5I?wh}uekdP{hEJ-i1^t%*u0MWE+Qy!jh^#8OjaZrEKTUgiqg<`e!de{0PG0Ip#zVn5l@hcj)p*k*{TWK(w&(jru&z!D%LKNjF`Z&xcbATzcL+*B`Ztd zt}Ye-QO7KYQ10eD2*Z1#blJKYj)KLp;6NIt04YtBy$+Tp1<`hxgvaw0Qz$rRN4@V6 z@yns*w^fB64k5nyS8&T=X^r#YIkTBuT^16PI+3?T~(M+rE}!m7j(&wK$PGR)=~{I8k)O|P6gedkYtAtQjbT(L( zBm-|qhA+|d?O+*&1~amPlJKzw5yLaG+DBCN_bBS_aSsM2Gc_VJ6XJH{eX_BOEhvPFe@3KbEz8m<^$iSxdd?KT7T_%wZ{U#%olLC z{69h4s%e!lz(Fx6X+EOs8&Cr%ptteprxQkiYanf*FALLIM220StST zn!5&KYdaD7y;4L0A|T)3zdy=QFMS}N_`u#Bjres_u2J_xN>2SH%viXG*lZ9$jgZQj zbb$>OD#>)!1TbeVMKr&%u>3IpAQ;~l(Dp!tAei6qW%?l6AQ)c;+45yngf~6n4>2X= zyjUFL7#{sGIm9tKz%kiZ7##?~@??0%o9Ge=(cs4PK&*mRvUWS$SjnF;meDj@{@7Ac zi{Q12#X4WhFq>r0eTPE;3D!l|SmRwVoCo%xFr-xR?~Dh$(D}|Ib~+eC+-e4GvI|kA z?`k9LrxBT$%>~q`oSlF{K5>Gt?!q%1cNLC`!*}e z4&+ilPCDdC{X*m1tK$fi&V%(_z!WISqld{u9&Wde%4LS%jGb2*fVBxJvlS|qE8`7j z3?_aaZi`WU=!^AIqB3S2XiVB61yRihP?wX3qGSI{370&6N2adA+HGA=jB14&+W)&TR24=2@@1#70dQwUH91d0j9f9 znF-w`9T>G~>ENLR7JXG={&*Hz($N%lo{@-!J zKk>3!|6BbznU2ARxP^Q3Pptnhyuj?KY$&UHvj#+Uj2A>*?eT6W``2m?+$9Sz1|yjw z80GJv0~&a<@TV^KaZg#|mgrYz`QPifX1wg8w8x^vBWTLJ^AL%G5{wfjEh^^B7bzk3 zv@^j#mLDFRz?)H=i!vAc74ApT7wVI%b{js~b8B(*LS(OTJLffHD9#bJU|1+h-ypwk z@p?Z63w7O=h+s*nUy7Gm)|;)@#K0`UWnWD7(Mmhm{NfB?P|{Dc&{E>BpE6bulGX;voH<RVoePADI)PT!Z5M*!Ky)T>Q+FK)6dbc|7L9>(BHwnSQ%- zaB%P*5O0v3{W&^5$zagU@Bj5VJYEIhu5A&!qm5j7# z(Nc*p=8TM3YTvp&faEtYN6_uY4k2QDsT7#X-)!-RRVe&&|anAm* zv>2EcHY1jocNSL6e248Gv6qZ_pE%n}JW&;1rV6-S3^_+*A&X%YIyt4|1=%Z3BOw{X zyHx)JhxaBG)<{CT_%pu*sM@1Y5|B1|dU*?q!4rnC*^A^v*8%F}If12N3cZpqk zJ4&HZ$polWX-lGvA!M=Op-`$Q6uLc;x?5`;W!)1gXPCjW{3Y zH(EC#XRw<*WE+QdK8Q$LoA`2Li6pkVG%VKqP0i~%`KvyI6~VDdg!ZF#71C9Al4 z_}j@7praJl?ymPrmivsx6fT?DSH1$9_uTBkTo~ff+uLi?Inl*l=V#5_{1bwXycH%xt-(P zX=7ylTNlBmsDINuZ0A10=g%K-Q`aBbwz==l)X7;;0&?OX2jm}n^H1Ag;k642ziAHF zryQGa1wo1C>ofO4N5p}HR$Py)((*5zSE<<7=^^sL^$>F_L2GH5+U-~9$koc)Qe*i= z*2j4IqhppC7Z~}iz0`7I{cY7IEY`Un17_UG;{CIdNJ)g@)mAjn)(QXtcl(EKYJiu| z!{w?~ov$784TfG^zXlg8+q|a?w!VxUGT3zLo#~y{7NL|9?~sEiL+&bNfn^pO9|`JrWKQhMy4Sn^6y?j z4C=q;aq}JTuGM?1uo9;vT^awiFZ`n*($scD6EYjY1nE`!jc=JGc&%pzg3lzoJ=2O* z{_cfV@2cB3&qr%$Wnkrp4{{-j{`Cob?Du|hJ576{DcI-A+{LuZwA10G?tjVEE~{8N zV)+6X>jxxuw)9Np@0Blq&O#4(@l+^;v9X`&g>E&M8*PIqgs0*C=9svuW*7{we`}ir z`lk8E>L@yQP2e$qtaFIWpLxjCw8uinpv+wr>7lWZDgonvDL@wHfpqV}^fqewI^ z?EXV~QyE}2gS~tw&m!D%kx|@JqlXx(Wax`dHM5@Y2CIP??t)fm_|Vuek>q;aq}Gzx zi3AV3rPB9}Iq%bzu`hqK0`lp_`z<7kdgXEI9u;P7W6Ej>a@$Jb>z5F&duge`&%`YZ(B?d?i~Gv)0O32PNIlFIqtJz_u7-xnr9HPIEbweCtDe zbt;mPn#c0Z=nX6t)NeOr4b-~MFWlVD(F8zNk^g4ZK0WHrYnY3b*F4AMDikMebX}sE zC}KLu=RD!^KSo2Gtq0zygDRtrv^&GI$z;E6JI|+i`J`h!h)heKy47=n0k)2>quJ0E zu`rvoBxcbGMZWA^X3-3~7g+sW-mccSehS$&9A#0CS`!OQQCnXvj`< zitu9UAMtHC&?XP~x2*xn_|xn8UkKhNTXVdz7dE81{h7BR@PI->=#h6Bwi! zK(_pAbkHlM_aQ~l(u`yVo$fSLVO0G0gA`ua0Y z6En$5VouIN$h^H|NEQu0!e5>5Di$$ie6+>v&unhvS1|}j;+LlFcym$tN2>(I#iP#i zaw+x%viga$w7a07H>(ahhOC~G>zVwx3kk$5X&+R9k&Ek@Lw}{q353V~;kaVg@gd?r z-D}cCeXCy;V4O3V=1I6Q0a58q-_yU*}&r$D{xF)NyuMdP8Y-9?cy{;7%=xIefE>y#{pflT{w?+XLjM zCvSA^;}<@4`QSkvLgn;=_*>cBAL!P;70g2`f(Vf?8o)CWn|~>c3C1`!Pf_L50A{OB z>a3pus|@n3-v&5V-@gf-kxr~9z`igP;|idTX%eANa8iC1sHR$000)a%Uh|R|n+C9KH;~6k8N3TTmp)tCaQ;7}okNr$YtyaE z?y_xLUAAr8wr$(C)n(hZZQHKD&iY1od~=k8%vr8{BX;a~_74#h2JOZeU+K@uL;KH6 z>ObAzidba*{N%^hWMf4fjGENj%Ayy^fEyT_TMZ}^CkR!%FmP~tJa2U)p52HRq+Q8= zqUqlB7QYy#-LwbL6zoBt&CiaIWz-L+>-h4wk~jaxI|F1}rhCIvfq`Nfu__j;u?iK?zFtZX zq3IWY&)0L^-#dirm$w^&myTuoZ`~X(cdg%+LD%Y`$EwI$;YTxXwv_7ofI2Y}$TdoQ zJe^qbv0Ej~26;$QegontQlV$+VU(#q3vqN}bJ-2D_Mb3hiTOvPMh$3eCSX|fF=wXL zbjeMepqN|Vfd+1aPQwenI?%L^*1PTs*z)G-_n-O3sYC>qU?|LnxY{W;TG{9NZO*y^ zb8JnYD@eOqia=9qyF>+gFYRsXhq*vp@nB_}yp6c%InQnS78@?8w6JYO&U8kclaaNs zp>k$gUQKsDQV3p$rpVD*-JM$4i@=#69!uu|%4aa&J5B3>=BM}{YaTRxWL57PK<$^G z)PEQEKaq60*tSPM(v8vMKPbrWt4b?d0O+QFQ$DW3?VmqR4#2oIqar++>1L%w(;QEvdbeq7mHFx){?r#2GJtQK8guwfQ#UDHsVBp zr4%RDp7_=NVPg5>`YJ0MmB|qZCe@I(<)*uiR1en&;^x+hIDCX$`uBeYwrAr$z;MbX zMh^00FYx-PD;_d;*pLMyLhKLk4Z-M^wmrcCi&m4?_UQ|Sg^@W@ZYkr2SstEOpMgK|>g&QW%{e{b5Et66FcP z>cB;AT^o+Os&NZgjTq2FSO5NJbz^DUxYRk*jtB07=eIEt9c*%2d$D45xTm~NLw z;_XWkFaGev7vF!69wX0H3$~|?aD*dEkv%kVl@oOLQ|BUgayPatiEPK1Wlk6gj(#8Z zZE40r9c=!L1T$x1aGeTtU?!RKhCCeVKaiA&F>TBGrYQ|?zn|C*BFwPyJ>Vip%eN08 zGudZgkXI2Z2&M2xe66K6hY~43r^I)+f~lxG5V{Y!76myrLye>%3rC~6UsUB}T&Uo( zu5{B`8^{BOM897pg3LBUta{-+LPST`r%Wf}VH}#j61$dTeyryMO1@?|P|sa(`}PnHS~h0uQ4CzZpq@+S;IUWhSR zbPVXg*?LkqVu@lt%QMf@%H0;SiE*lGFGO!m$R5k(Z22RvDp?+aCk4J^h;`S**l9P` zI`T5BQ$IYu5I~x_MITi1_Hr}gNx}6PBA7Q~_$zD_bx@S{8SN_bCk{@DvcJHB3_LW% z?FI|crlYERex6s~eVw(rns@mf+2N~+r>i)zonC>bbI!p?X1%9-_fPVT)I~=m^kkk@ z{)U?-{JTI>MU;U}y4g;+5_o7yNcP9j^%MGn5U&+ZC`2FV`rn_rtX4=(SXlXKKg>mu zGy_43qJ9CUGAp z@ef!dm)OPi{eE4?2M9ALglriaLTib|#7p!JInp38cVVPTQ}We52D8Xmc+@~{04qt^ z%m1SI4k#pl-EsFNNloa+JR0ONrp7di{~YDy>$r9*)S4+%sw0dm4c7jjTNN&Y<;*%@ z_AB|dT0IFA@fMEkoi-Wp!S}c=|Me);Q=dN~6F4}I+UmTDm@|T=hEnI-Nt1AvIV&n> zM=jO!m2Z10XG^Qi694pyQ%Uuz;VP8fK1(y_lhZ@hAUesDebAG9bOM7USj6mETge{4 zxU`#dQ1PilK1nqp}8X&bL)Nz?QPGP(Cs2O{^^Lk-u&aVldJM zgMrcC_eNr`Sl&L&xj%j1Lf*sx9`emuU9B0WA=DpfZ=e+@@G80Fx)$b$z^tew1lYQ5=DeW!RYc^@6Y z-;G{xb^I?KX57;~xg7e{Z@D_OhraIi_xnT7pSA@o$G#PJ4YJjru55c|Vf30hP_-KS z`lsI77w?8Q8|@lB54M6Q%b7oChi-M>Kb0pN4?XdygFiFjaw4|P{nay~S)%tpY2tJ* zax)M^`k8XvNeyPUSKP3^Z=k2ruJMfA%?Pm(9)~D((4Zqmw4<7yoL}SBURG9{&C)G; z?>8`qRtoqsL*=HBvZr}JGrC^x3jcoY&yW5qqm=i54w59kWcf*v4aPH>#c!a6YWt2) zcrYJN;IzYimyuYJ~jBh3j^*xK=wEAy*jXr=+m-c zypWdA7l_dYm2JHO%@r=~?zRCO`~jC+S3fKcyL`@?D5{3^QxWG{GJw`CzCS5h4K|LzL z;z;%R-g-&lYj3YtKhxbU-FWLIu1GfKlj8G@Q^%3PA~l_T9NkS?Ed08Ay4fr@S{M!Y z`b)g6`ai-O^}cmey*;%6Id6Rb=`KI<&d4^u??g8mzvqsic$Cg%OnMEUqQHqfD5lyj zXIawmU$vu%UJsjh5@uM&o-5|%{$29g(oAPFb{dKj{wsk`SWuwT>U*TmP*&XdF;C><{45`ryxXs3-*`Oy0gCT z1UZ1s&MB5m2-Y=6-S(D?2}I!c^bVlK*c5QgnF=au51=sMSg29>mU#7lTsy_g^%p`iG&P z4fl|3Lw^&(FE>h84;QULrpZKt&_@ie3HC(if z`G=qLx73lDi|3xi#Kh`vQzC16^j}+4i=8!4-GJm*1{5w};Ee`+iC9*~^WQ zYt8zte{@8|;hR%@UZCnK5&y1Q5$Kg2VJa5d9A1$wrEbzztqj3RVnBcbfzF6UOBP7* zVp3N`SLQ9y#1?Xdq2Glkj0t(+E{zXgl;yL@33^9l!nW|JW$r##y|I)1r^(GN4^T0n z+x-KTJ$xiPHO&Yzm*^6yqn_X44{bQbg@FJT0c+;+w7w^wnbc#~JI5GVBUL~r!!NVo z19$IN^hdnIKnMs3qji_)oYUsZP|gBw=AUiWDTba{!LjJC_G~;8WK&w z@?G`yb@`t{0uBKkFd7QekT$3jX3Y@G9S)Hx--n^(D4=Q>G&kweZnRmJN0+fp!i0#o zj*RGn8HPETPd$-bmeL3M{1@#}=y*2;v+l1mJ=5UMToTUp%+C z_mu`MWE8~IB87+OFpd0e7jYT~Gk1;GDcV8*kHpX<;KF>(PDxE%00n`Kzr%-D(5YNB z6b5w2rZ)p49U))!`dMHngiLJ^GXAQrS`1G$RTvNisAU5lg%SbncWZ2GaohIKVizU$ zFjs;o)P(2RWNF4HtX&ETE5QD!7zOYY6n!N4YEBrxzpChr&@{SnA|UN{ft4S49)?-A zwxy7F-IsQLUKtVc3^Ql3vs~cB{%s~g=qZkBm>0f$O%<%5qDdj*nJgad&eqoF%9E?= z&2_Vpfh4KMdWzIyi!ZlPK`aGMxsYc^5HcYg~Y01I4#ux&ROvyiEF*5NwW3sAf zsW;a(+gjS&)S56*!hBo1lxvpalJZlHJcr#44GB0#0h8o2#*B0d3@n9CIH_2T;E{hC z+8(^jubYQmGSVbZ@UirHT*{nl^QPr7)s)zX&M&=5M3#nQI7BP!E2mfJM$u2g+d{5f zXiIj&y@=c_YhYF?e&fgIMQyFHs@D=rZmyHvAGe^qL=KEdcoIx32s~B0f)kXCYL-i? zKK>f%v}U}hbG@R91}BLcA5=@!CzDgglFZ2lkSFGM2ZN34wSWpGFZs5QdHtU>-pYvHKqev9O|U%zpZYj7!y^r_~)qhdP*t+(=cgpU70aKz*nz3$FZdS~5T6`3YDUE0y*4`C*;%~h56=vo=jsh`3jI*~FfT9T zJ%2w=9r=D4-;DUa&v0qhya>z+8H*6wLHL3@`ter!XFLNQVqwa(DXFduD_pY6>jHTp}b>V1nk9O)NuT#0mkbu zV!Gdlv;j7xcVmxrl@_ovWXq3f!i4;Oy>G7kYSFuS8E0}Oqd`IA9D(1FX2|Vj&Io&L z;||6RTkD31ofwCL!z7#6R}2*l0g1C8Va^j=!auG*JwLA(ySx8>7h$mBz8^+-uYX?P zzwd_nbm4rQeLwGZ^?ZLqZMS{AUA^CkxL=>ZQ%YZs|LMN?;-~V0xj5L}uJD4$)C#RT+9i;i45bUR|AmRA=c#TM3C>nB-N_l%{KrZAid>M0XLmQ$@ z%ttM6l%QbBurfVRn39i}s>p=n!vJ%iX_fbLPdDB`9wa?R+$ zNDEBu#I9`wBkx4gc&W~~XB+d~;%x6oW6GKz)n8Okuv1O-*Gq;DAXST>bw>fhkzUwb%;8ss z5>CS~UJUcYn@Njwn8JBKlsUmx+Bw1U74>T1GF^sKx7}umZ5m~$>f9>dxOOy$MA=Y9 z=;eO}BLFuLqN1*;pNtwA0OJz&!e)j=UfQoq84voiLjJ5)NN;5uag%N8Yi~QkB$bFC z3kISimoWS;gCf)&G4_NYSKWM9&U!fPq>J#kfFd<0y}!`L%e9ntmXS4x`clJ=g;g~{ zu*m&UOZtg`ZeK5th&vE@WE&Pcg_>0=YK3fRnPU>^NLZ;4YPpjspV6r_DEu8_W-Fn@ z+`A&-G0WOaIKhGm>g6afG-c_Z7gE8>Rq4!C@yM6aQemt@(Q1%T(_Pv>!!31L zrOq=Xc!UB5oXIn<$l#quRr$70E}=T%JD#~|@CSon&}q|h7PxI4O9{@q)I{(-hJ}bU zgl6&XkeeSv;BE^~wK$pT7VdwjsgJ9R@#*>t0)qBuGa;pj20M!WA;ZLwxlFUnxEOPH z^ewDis;eWTkb)%321?PtceWxndzN}yI-0ZEA#e?lj!d`rDcpf6D|BPY=}qh5&;Is1fq>3w0Kkjzm4ajfMGEH-vb99=Exk#-eqUKrRi>dQ zgu1|*9lJebv%~#MJRV%c<<}EdxZ~ zsTNx9w*AI4s&QlwQil6}PN3+3QVha0bu1u-~c)xX<5T31nO8@Ha{|A#XjbGS;PR1~6C#YUFP!b^-B?fFu$HMd>oTn)f+q zomQIbLuZ7!bIc<5m7uNxD=jIj#rP_TR{3^q9-AiNYH+bM1QF0`>3&!}x`nM;G-}Fl zFbl{P2dkE1flY`Z&jmK@>pTxy6VkrHKBl|u4-0@I-3O~!iByDIkTmhd414xoP})Tr zr;Mc`DI=b>z-yxdX_##B<=M!2%Xog*>)18z`d{uVabh;r3nVD&B3fRf?CEsxeCM-y zbR#T~jh#{CnJ}S*66QWafA|K?1~I!Q^Qe3sS;i2uZNG|y#w%jP*h@xzuuoy;M&+`IFJsU@GR_=W!!ZP48>U2*gtu)Gh%;db3V+a5ibwt>C zv$LDHjVxz~4-;Ydijym02x(K^Bia2-O@9Un-r+y->5!;VqQ6saPjMX%!7ki6POTJo znu-uie_EGdp0{U0O_k!*D6plY0RmHD3+G4)EMx;h0E*c%6httexkGg^nRa~i zn=%Kl$ISc5Tcc<UF(;VPT;XBt={e<9qq)8i{1s6VNyH}_L8G>g`2w$ty}!j zx1e?nJ*A{++=!EuN2aYi2BtY$sWF?_Fkt|DLda*NM)rm>)|sml#JvG;hHY%a(-m+6-RhN|yt!FB_J9Q;w`#qkExNBP27q z5wM&L!8iaJ2Xc3Q+wul^D{S=GJy6IN zA|`)%Aph-z7ev`;6$Kk@lHq9#B0R8~de-|8^+XhG$b)RJbM86}>Z^}@kH~0dj6?P5 zYfAY>*xT3fXu0TTY$W*zdEsg~w8oV^f4(*vWWiu7QGD9gqQI&(%rJ4BA1vAe?;Ba9*P%32|wVkcH$g@r_4MKFHrX@q{*U=?m)B{P%9ErvGkh- zWyrXHT`V?&7O-La6OGf&f`Uw4C)%6`l$Bqr^Af%0kCLmAtAg}F7S4hGo^f~W2!XGO zRR@y_cO?$Oxuyu22yG(M_%9d7ALB_+4RfwHxg+UsVA=f?(b1gT8oiX2Jxu9C@r`ZG zZ$7jc<%(?MoGx<&>uoP3952b)@faQF-3=fw70TWRU&ZI7IC3!SnsAY<{odN5wE@eh zlcjjgf`H%;ru1*tF3X=5LGyw&->z$_7Oer`UDHK-7f&g=D4nEU-S3Rd2Salaq53nn zO+&e!@8jkQW?r&2_(ib>p=K?E@SB)PcRJGAB zBVc1A?g?pM$^mhLx=-$p2tq9=ZQ7u0xs7JcZ3lCyV=m&_X2E|bJ!@>{tdCp^zf}~w z^k0l(@OdB8prelqA`*mz0g?-~5E+=ad=22?^zZ3!{4V(``#wUVGJCCkX%nM0<55p6 zB+}|h5VQOiA%60>W-@_a7Y4y!#LpTWnC~RYy;w7cO$#Z!a1*;v3`?%hXtY6&GSsgD{iIkeEk&*!fKVzq`YvU z$1Y3Q-f4+0RrXH54~UD`pt4J>gQsWYPQ4u9h+K2btLQ404Qn3lS&Ocb^0IT&3gzU5 zDYV#ikR{6_;)Z5aPNp7=*o7E#j=~{93Tb1E|1jo~Bz1KK0L4vRw2yl!d`*N^BvgwP z$cYF^kSOi0>^m9_wDYYl1Y}>;gyxOH$0k+U#)=4W!}rhL-Q6*ao{#AIypaSvVlsrA zwZ9T}KAj&Hd5->Ycz)2dewa!WAgC&G)?eq%w{Z#mdz*?0dMZ6+_WO@)1ruo14-jsS%$K=afAN-QWe)>)i{Gv3!XZ z6tZI$elg;qPs`HR7|igKH7H({O&_RsY^*;_gjSdW`N0b$PEv0w?SPZ?6c&UlvGXw( zzQR7qNn6L%fFP+76E%I9){6c*mHop14SLWKeGkDeXS_IsY3~XqiVVs+LV~GMbl6q+!0I}v;0pu>I*UL=?#2= zbaCYEpVko)wTV)Z9{-bMWKs`A&w;%h0(`BsQB%kDryFfD+Sx-jb{t##68XuLuEwEn zI}&tc%~2=CDj)ODz{FQoQY!|&1kP#mI~`u`6ON zS&26YtIhg|!EuRp^o9QQHvfxoML~qM93j?1P7ZJ8bCK41BTwggWA~7kz$cWH<`q6W z9SAV8q)~&!SHmE=*j~nNnCR_WUC}x=0apUoSTaVQEgDZ#C?sMc4sF=r3g-h`kZKoO zXX&=Nq7jy2e02rxhgwD5ZXv1$ zM{xkVa6a4HDq%x&Yt^OtUlJv5Jhu#0#TVg(5UNCC;Sb{`o)aF=A!GzY?08_=u|}sW z;jw(Y^WXVcJX{1sm+VQ)#O54A1ZXw_!QA&wShr;c$fsl>d|;E(?87>W z(yXPTOVq;#)eAC!VIHCPmL^}g>ongyU;@KQ8~yS}fuGl4@N7K_OcV*CQ2||6zmh3E z#+>8VzS@s89jMs)V=#exQ8b)z`i=|HS z$O9$aN4Cqji!ELWY&S;%D~<+Qj{y8SwK~|$Ea4rZ$A3EwRrj7jiNw1p$|zLWaEvQR za<*&;0P@k1!ZiGJi4EdX%Tw1_fK>A1(u{w=ifJb$Lp31m@!Z4<@pCD6Kb9jjtTm}D z3^z37C!P;;Xp9*ZFOSlzim2&Yy~`G7_pEwIF=yamrpmnzT+pEH$u9IXmx2bY|_p}X4DKxwPOr2^1 zG-JI$QUtc=HU%bk%2X@i>FNGiZ{#^dKybic92QS?54L0qU~&&n;wFUXZYD??A#u%5 z(JDP*4s+6^x0UT4E_I;^iA#_*_aZdvH|b+sc}&5gizd8RNFmPf`4A0eCfdH~QHW@M zrht(69ScET>!xPExt$$sH0xOHdSxA{PSDmgp#nJOJY-i=S5Xy@2hVsg)wK@~)Y!-O z0hzi}ir9jBJKW{cW-5I>qI@KRWyi0!^t4+`53dB8-mBkI1FpFO{7BJ(>qh;&&Vfel zfd{Q(?Iu-F%WKcKgr`a6X!e1G`Ey~)p&tLpo&phAB)4v`mlV8 z4GpC;()xjJwPx>!Gy3s+!Hp?;5(>13fdZwB}3Cy8;w%4oEELBdJQ!{91W!nCOWCL%e#g5mx2ioBtvG}^O%Y{%|_vQb;=Z= z<Za-G4JtEM?X>o66A6^vm-0a;vN#iMZCc@~ z1>}ijLl(EWGjy7PR4T&eu95|DiF^CAki~2zTpxHDyZ(hbn=Z*m2GxF0p{L^z6(4qi zay2r^>Q)mj0YzWc?0ldqeC${b^)`#(i$WR}Q?0Y(RtpY}8Nn-!7`tpvF}L1^{68bf zM_mvLZr}6d3D1%7j$rmw`k}phZKW3(x5KtcWe`rjUP)14f#@#GKTl;Mj9^brgwY)c z(RqI8T8DtL0rof3R2{jJ+=|l`-bqN_cFwb(4#y}YB7d)P5uG&oY8JCFQo%hRQrNXd zE~zh_17iCwimUdtlT^>W{V=2}$X1ssC;&20N&pfoMA8cyBn@GpIH7e| zW_XzMe*h0LoOukUvTxBZtVR;D1v8>V<`PcWlagrzvPPJ;6H+#H3qfRS4-M!FSWTzR zRdLcYl=Gr46S1r)-+eQJ0t`K_Chwac45NLaHoM$7rzeg$fhlFCtbqBQwIn!L_60I@ zq(sg+%Oek`q0AsjDK^qU@~2+QrIEE9ml!?vZ~2&Y3zE@6?K5TeEuXb;tHQ9oXCs-| zKO}Wr^jS1o>(HR>ik_WkBcMLaf0&x*ny#qMuZ*WTT0&p-1_m{R=WaH!JxP8nc}4Rd zZDT|`X0gRAxAp!W4NTP8QcJu3e$AT@I3&@IiN(-0=Sn~?7u2hkEH5T)aBi>^CDk8} zL-AuM{0v*MoP-PtBdj$0PPA_@VWL48VjFtbvuYs72KHUrCsR~$N?ry=0O2e?E}Jz1 zwmc1pPuF4@Bx==L_NRC`?n4bvmQX?JT)pALFBdN-tO={L(K<3At}(W?1u`|Lx>(|1 zMie;l=%d})>R~TTzIsAz$n8G9K4HHDpewyTE^N9NB+=H^Fys#R#+@5*z{}sD2j`JMU)dkjVanKh7^O7^^(9y^ zQJjDn)R#Ol$K=Hdd!CF!EgJjxI3|hYo)UZWrw zY-=-}D0y&o#%MUq5^A`pqdY?*`td`vO;0>z;95|coVsq5F5C-|C*v9*`~UiGj5EaQfwrHXEqM4Tzh%|Ckoa-U_hqXIe=pcN~= zTgH6XZ;=*?jy#@Kv?IbykTgI(>0|XzunJWp_M~`g0{BqMv{&_D6JEj(7J>AHz=JO- z!`Cs%$;-dwIiAcaJCDhr$%QA8TrnC@0`^o@0bpVRVVJq9Um{_lB`>oLBumgdE*}bx zlfJfeVrA*yxo6vy=*n2@VN}A1GBLmOzKcz!bD^l?_uQMST$v!G!`ulT7s4WPkKyTzrH48;#{t^s}cFV`^pN z=dwo-E`O6Bo|Dkc#KtfXsk-TT5fU}P4~QxMvrx+rl_mFPr`8^|aUmJY zwOylLRpzAZTv`=339l={{ z;xcG&SJVXh(;gFBV?y!vj}lR1iAJg;|3h`d1r<|=k6{lWB7F-`X@&v<93%DO=4lfw z6%$qe*asU<_5KgCLgSKcG7T6!yJK0?z>9;meRX%GY}ffLy6XyMG0G`GrW8p1wYFQGk zS67s1Yqp!9dX+44t@P~bC|3jog;P5TQhM|u z{+&9L&aGQcFFGuytfukOfOw+#%ge;FYLKg3vzVN4NMyR9Vd5P!8R$I# zSvJ;o`Au+f_R#V9QdR9}+Z=F9=O%Wjh{QuMMe~Vgnp|_m>ghkr9pL2vHg~zO_ylB< zNMkk}q2+Y2|BdXK)0{5TsTT@hwf1NX55ePfs5?Dz3rzqY5da?aj9l}fU|XbQe<}Vq zoT-GM<>mMwb4t3Y^oryq2ZVSr4P0~^4Lq)MzJ7RtE17dZWRSjbV%)a^vML&+)(%Mn z=Xu=d9aq|kCS5(a+}w+%kN(8r-#3gEnAr16ts)%yq>5n4GDsE!!_S6&4U_9a^MOk% z5H@TEBE)WPZE-AJ%1HfTy-+aitePyf-S0*MEKw!ZcB&VvUWP8V+g2|z-s^@rN$K9M zCk-Pa0xDg=k6p|gigrPSAoQmuHc05{6YjBnGYMAb(bqfnk?7ZyQIroHXjx=HFxc=OBgxKH0RHGL=V%P@v*28K zj0ILAe@i)f>g7bghrx=PmP)%MBrNF;7%FQFhMoLxwCDo1O$hP1W$#vhVNggesZ^ebp zgwc~byb%mY>CWxwWtf53hbd^Vv~qm*vjIIXA0uGtDFaBTg1;+_IYm(P(z z%ZR5MXfCdAS@`YYww6ZRDF~owEX)D{@32#AOA(nRxa&zV2neWKG)rKwB z;HkJDBhy7NJlq zEsVi9Nt44$l_>VLsv-@Y!pbU)f@%i=tx$$#GrY(pB2ojq{+0I{fA~ufl`sh{Oe&rD z3-Z*;$I8&@PI3V4NDA@zJq=1-?gTtPeh(IuW(iv9b5*xgGGUwc{6OZ2GkI{5nVjF zKVKK%v7eq0Rk`WL(Zo6Z#2ja7J4%BHqx$&9e}vdy<|m%dy+?Dozo65 z>Zoo|rkQKlNsII}EtxXg1q)by>HBAKuL0uizp~f`L^iwQR8z9&8Svskrfi#?#cKZ< z>Lz>OHb<(nCoN@WV&NnKtm%a6PkYAAYS_v<7V#N^o#U)|^raynHn^DGIn`j10 zby<105SE6f+EDxI*Vq0xqtbp<1GUNGk3w8yOR#Ct@N$Xrt346VXjUTlD`c5FyhNrr z`C)#7J^2QdZ-| z5lc!ow~H=CjYF%}8_|U>!IH$?xXSI&Zk;|;L(X}QTC`LxVx$$$+wL=9I^xXn?o20^ znhs;PaIVi&Tv{u$mw+95C0^KJL z{0yDKVsxv~F;5)U>wY{JRc!qJNdC1bFUM14s6$z4*?x1de?Bw$4bc!}8l!!wL{l={LR`h~+~5}Q{A1^oj7lmy&P6MG39b=vYn0QJyd`r#?V!0N^&mdYTsC1LXgeL> zZy=C#4PD2DgD{uu@NEvY%k6HyG2YH;F<*f7f--pPSH^NoU=VZn$9c&~T3l!@rLN} zmhZp^9+pq)v=PeF6>Ygz6!SG~IEX@;IY{W^R@hY_R??y#R^Uf2og7DA<+z9XMaL1z z`#PJ2AJOSr*wO=!Ees%KL-5<*yX6*eJMT&`R=t+hrXkKOeJtx^PP> zgnY>UuxK!wd=V-!o|sWXxsfqPAZ4;w|<8p zFvBgCS#uPpPwc1dTZS=+$b81VOvLIxUdzoi%?aD3%1hoeOD~nR^0MmmN8;27Z+Swl zcNbx$<)mZ6xrRY&oed_Ix5<_bgespO@O8xDAo76xt6i3ktud8)m7kq)lZ#x=buz}` zhkkQuMRH8DvOzU$Od$`XHk618mYs|kx{>-v20pF5tavGv&#-sKk_cknlvT!1d?7TX6U8Q>0{95 zG}B2^+9QEoz4GHY1O&{s|5z7Xiw}*Z8W}TTY4Z1WMuzC%9{0dl*1jOBWre1r7 z%uAOCgNlZue<`A-ILO>;tRqm81p>$m-sTZ2O{CB~?sc)fJ`(t%v64ekV7_(xJe8g7 z$Al0%?QMwWMYouL4F^3!3;hPq`a(yhuRng7j_=yG<2PB2a6dHxBYM&&4t?L^zSG{) z07?hswX4r9Im+NT{o+CrAN`=oFBk2IbWU2_ExNh7gddhqGFr;nbU1`s)wl&5zE1kp zzmE+#N*#Tf=9T`_<+#YNvj`2?OaQbs}+dY8-lc3)av1QKV+ z$c5O@%K>7+N&aq$ML*m{)}=$VwU|P>R!nNRjSy6KF7`OYhR@E2Yg_n3t9KN^gq(nu zGrJhVkq8jn4bci6GG(z54$H`(J9%;IYEbV#?Q$yWu z9(>ang^Bd={^eKVUPb|SFm`UGXAzqEzWNjDDX91EIlxT@MH<{tcT{+k=S|t6%-|F@ z>q-lPGmpOVnemHit6%&#=A%0F0FX^&Ch!o?YkmR7ozIl{VznHNEZC z(aoNIu6MwNk*C2)tSOvqVVVQc8G#- z|9+I>a&~huoia6ti`u!*Z3Waiwmq&D%8Xk-S!MF<4Pcuo>8G9__?9w^=O*!z(^sk( zi!(fRpORLF5}(vL8AN$3p#h!WOc~QITL)vL@m6nCSW-r9xAfSyY&+EjVn?K7mloTI zg}TN&{&hQH^DI*pvYKf8%Q#(5=1e+JV^-0E3( zWKXLELNMO1KP$*lPcLXEJGAhrk#P>haPp`__i5xZ`WbCK_cf>&CCMCBx6S$v#O{qs zJXPhWF=%()O{O+TuNPDR+NTo_Gsu`T(4PK8eUq?a)jKT$a!BIdYUR8?h@krjs!=4E zfRQ{I{PHU$NunsCE+V308Kj{+5;pKSP9Ffeh?oc~9`xSHP6bp(!14mUk|bL%(+Z%V zGZz^|Icl6OCF0b)PBU(oUU5`cj zwe%B?0hib2n2-CIR3&xch{B@Nc^V;3N5m4r%j9atJ-StXknWQ3xdxc~3PH5soKBkm zOwPGl3#e zLqMGh9X+;8(UCSKG22_5$_sE;CueFuIk2w=Yivr<0McKMAu%n1LU-pK5`ACyNA@QFYWJRGkl^t(`4E!U9EwF?;2@Q| zdezJgP$K4k&jyOXKvH)#N5Mw8s4B{Pa4PBCJcIJqHtntX{b$48_3|6TxnipEy`ZPq zJ|-e+=6`7_Z@=QN~GfvoMStq+ZU-tH}*`xu~UWzX|*SCE* zbOOGyHHt7Tiq|pa!exzP=bO2XX_C@ztw(d$EFz{Ljy2nZldiI17KJ>4f6O%vyGt`0 z1TqE98WnW%G|`>xB#=&^fj4ACWa+GZu53;vUb}6!k*$jmp@)5^7skuicLLH8>;17}BXm2|;#|od*2wDr`&|Qj7ees~M*g zHvr&k7L*gm#pau~G@MPFG%uE}-maETc9YjiG(&`|mkN-j?OjqT&p?yq0*Z6#Yj2xU z6@vW=79te0(IvN*SAC{`d|aaKY5*(DK(I}uBOV;4T1XLgC8(NrnyXi;N*$MKQcJb2 zMj+s~Jl1RmCs0TG+CebK%L>%&?~q|5lc%9&!?Y-K)(d@)w*79S@P$7GN{w?^>EVFf zYSIV|qKFhGVC^6@(3x1BZ=7P@$u4L>n(PyMvcAV0SH2_(fYhgA>8|JzoiA+PFPCS& zPEo(om#}jt=YM+GWne}HdU1KWpiVnXEr_75c$cHSl(}m4$XT}rfdTy)T$xSu3qed2 zZk&2P#&WcWaMxAkm(NZ=jous+vfl1v)fckYK5ZjE*|iru30utAcr9HG>$XreFX`Se zbMkG=l}K6q+%firq+t!Oq=gtx87JD ziRV=R#qt|@-F-gUlf`(YluLYjdo|>8@bn-W!?js7b-Gyw%(EocJwm~EIl5zBKrOGy zo^OUE+767h>>V!g@6KWz*SpX`{<9UsX&;?|q3ZbdQd`sTCOMs%AYbN-wod6?&7RjudVDJb(5Zcl$4E|yEw4oGHPW;mFgZd1{7gr#}mtw4{aoWUT96xCId-b%)c9(HX&A)|a=5A+u zd)~pi2#JQ4KMXgBh9hi`u08jMTRF-cs848f5#e=rKt3Yx5W{acc(dw&R+GKtIa-jh zF497UC7?9;$MXO^)-M0~E8NJ*xg!1}Jq0wP*r`vcZ9S{5Fxgk9(+`@mK+eTu zHZ@Ln1q2rd;*FFF%2}^EeL^|UCsJiFiNaC(v;G1NVY;0qL&2DVlwr$(CZCe}Lnpm^3 zZQI7Yd7k&)x^<^|YG$gtr)K`^{{GH6A16q=uI5D-Im=KB?5dacgFn?V6Ea$NP{+Nr zHs_6Fad{&F?Va<`P>X3tnKWz>t;NG)yAkomC=_3nje}`NNW5Ex)SlF9ohY4V(l$O@v96CJ^>~(iSy5R7MVY8*?PT&D1Kj zGrE*Ih;Co!pJ8QB72ObG-#aMrNlA78iLywD6%?8`KFs^`2X;(re2Yp%K}lhrVNFM& zy22S7{mLLHhvIrZU#+fp$QYvB|6BiZ0F(g5gIpdz2?|g~Y1Oa6#XqXUi2m36?Z^$uT*gsKT*KM`&N`L{T zxM;LerTaN0zw@nsVKHZ-rl!Q>{%BPOY;p}Io7Qtm#oyqEuEjeEwH-YDt0_hKrivz$ z8E(B>s4LL7XT*HRe>fe89I@tFEM{5*dw)80B|X8x5s~uNKAV8^(iH zt^4j;$jr29bv0?eMHI z3zyU_1y688S}KeV{!AZ9tGsD1JC$m|aV+Z!c;I}93{xg!xKUq(2jW6S?yr|$MvxV| zM9c}loCi^P;>p1=ZvYWkh9M*#iPOOEl;uA(rF#?O%|e9FnXD)Yo~=<_6!sZ;-642Y8(X%P$6SGD8c;maNYeFur>JD!YS z?M9DvO5{j7DaRxMo%9XP*E*I58~na81#hT54j z(OT(-=@vv-$1Z!HsLKEnBD0^99~986a;oJki+q~PldAlJN4*Sk^kZcden!S9l9%Zv z`jVR92ywm~?u?K@>2mM}CQirbBQQbx9c_^pu@9&29hyde=Fbs+B3=WGd}`V@CPIgo zZF*;45|c^L$&GU&XKk4}-}JAm0?IC*Wmql=vaX~C2}CeTxJPg)B26h@%xJ#%Kav_d z(+`kX*=GDpAdQcqD9;VC{Z$rm=*cL zQmiKqLOP3zs?z)_Xdb@^wXCgl*p~@|Yly*MbuPdJ;G!m4@4Y$^7Ku(EqQU}xLa}qX z`Cua+#{^)zx450c!Vfk*M4GX#t*XD$Hyl^tXp|)%M*jLNWGBnV>`BPJ6+{GP0C)Cy z=Nt>a+`#XS_B-!hLuSk~v*I(U)b}i6gXxMa@IqaiXgiwF5c+4y}rm zv+484#}R1YO8Rt0eh2C3?D1z+;jewGHe`*ABuG>@e*3G+(Vnrq3Ut~E!EY0_I)GC3 zt`KJcNSKnhIX0gavP~z{*}h0#;B;hI{2MUyf))A&9_{zJix%v6S8e_1h0*6zi3^vKm&GbqStRTkavYgL z)#`>IjKIr$6TG(P6EdfRbxBSgNEawwsVKGKE z$H<(m9iL7%n%0mDQ5{-D8MB;IF+#}aJ=2qeA-t6|*Xx%XOoSZ&V%pyS1|!8h7In`Q3n9M^7Y9i_SONV?ZPZ^yD{^u{+6hwA;0_R3zoUm3vJIT`j_N` z=#&D^T&&)aR8*>IhDr%1&8gDrLIu&+quPG`j9K0}s!)}6;aIch*+hG;XkJr#scBw2 zz;1kZT@-iP_(22OPcdxVAlxdi-XL56R&NMqZ>~;k!PIQ@e+{v0#P5 zhL5Ll3X89H5Pf(E9!ZWZ7O$9XHJ@*oT<=8uZ8Ps>fTi83byV^l5wf<%_1xih^v(#wn8KQ)R1jT>0D=P~` zMkPk$hl+O@I)t*W!*_i3dypJjia0dyfq7HvIGix?-YJ6sD+7vM|y*6}|t>N-o1fO3ag}s@h;%YW~sKi2VA*YIv=pv0Q zc*1FbOX;4m;F}n-V`epXic+O}k(;$V5|}5#dAGu}xW`x(nC98g{6x^y`VhYxa={G_ zF4gwhqv`+}+ixQhLoWmA|5XZY?9iK(lOKk)C_U6MMEdTL)y*iVpi@&NwGYVKCFHei z!#FjUYbB}9|J~;QYtB*BjLqX>b%Jgk)?6a`@vi|VV@5D^m(7Em3;zx$?F_PM+WM2> zSsQs=x{LL}?y;@~n@)0&-1 zM%AEn>xcCDCf9?g`#I9rrxynedTr$+c>hl?;z%0k=2q=)Bu}@SoBjFh=?diS<#NOJ zdArnOs^S^u_u+Z1=6b_6C9B*09>-V3umZ(4)#ykR31)jo@tg|fX3-0>*Ob3QaPesl z6Zh8QJwJnUF>TMwlWxtWxNx(^{TgLqs;VhO8vB!0A#`tGD~9mB@tS7-LD@AnMDAEU z+^g=6a@_+1syRn(XR^}1lxD+1%1t+4yxOXa(#JeZ2k$-$urb1*0}eNFZ5BnwiBqpRU1w=Acvg4 zD$hh8qz+9QVlE*u1F12-h3-eAQ}1#tNeGgE64DdpH^^@As{NQutpBUS`qyiR9G`h!QHjZc1XYSfhd0JYo}s_q%C%xYg(Sxd#erfP<1 z24+6H<&lzXAJCwWKV?pAQ88queJ;e@zfE|jgZMu71c7TQQd*7pHFO{*3gLyU3etNb zfl|>V!*e!HF`pijRb>E#5#-}R$nU8FRe7})jmE2zj=s5++y^XaSk|2eYCQvgkBtOMaQC@-*4AWbCZUc$2m9 z=^Xh~dZ!pVUCZHy`+6o-&~uRr>%|32%$SMZaODV7IHSS_%@0A9)Qxh?6KKD~+_wA! z74p`K4AFn;=w;{h^kA1%9``%dWL9{1I+;+_O^AqGQXE=8RF!#--6=UH04wxe`kx== zBfD*?C`m6zPf{zYc-u(47*0FcyCiVsPK@`&!E-dLVYBJqfpla$9bt!2v1Fd;UT(`!wj4@+=K zzC7cKuwLRZu8bu{a)9xWG>hLpTOLEL>mCToixn$kkD1fGnALH$IB22Z>Ol8hI&At|Wqhlne zi!_&EM5$%%wAC~!O!%B@yVAB(ZQrfsuiw0c^RbpSVbm_9hT+J(J_v0}{Z2`+HAhdY z>Sx4{IGyjHO;0f~QE}7aremg-wm-GJF`Sx=RVZw5r(K-<#F@pixcIqDJ$nG{gXL!O zM$*w(PoBLCuF_{%xS))u^-(lN28a5cLw+hYKY4V_fTpPFd%Ew6Cj|IpBPuDrQ%J;U zr3kdp%8qYKtQ8@^EHTI}eD9@93g1x!dZU2aQG7E3hK)=EgJjot+LzdHWaFysjlgZo zT=6#+4guMRtc=X?q_1eV)NwHV^B(!3;z!#0dKbi;Uy22!5S{Ti;`?fgIT#X}O?;0* zs}q1&ei6??t5UAcg6N~n+=8cQuvvY1`PKg8g55WdzpSR%0Dm!Q8M+RqLowl*Tt{S5 z%;#!0)-w}rk8)QX?#dn}?WlHRTV~^JoD@aj>{sRdhI&mFEn!1_6V_Hu{oDb}$Sx0P zGgJC+TgN(tDYSovnILLoXb>Hy{~#zGTanf&x8BHXc&=Tz|A-#I zW6#0n7W~x3kz89{c4OO6lO9=?dzCJQ@p>TN|Gxk)3bMSgpzLghsZ}(y{^oyB3tJ7C`|Ojja+D> zZv1=;>K3;~M^>qf<6T*~O-stODV&9Mf6`L0D8Z{3evMl^AqT|-l;VsvGuTAQ?Bw?Z zAyA!I&md!NK*AK&C$SX#tAcA;Y!CJZ3<-9{iQ(+1FbhmTGOS0h1 zk(gV_lce6MhyD9K6+2jcqSdKlm~WSbhw^e>%z1tenJVoe4u|6Vorjn;XY&5b^6Xv? zjgLvC%}GN-{^qe@z%tkxm1svlzAoBh{2Mx6G%QrPwmC_d&U;;bodxaMbzc*;MyA)i z>vG1dTgiP61z@Jd0y|6f>z7B?1Ofm%zy(97&|vq)^Y6Vur?`$3$~YjeAxoQhlTpIH zCSfXHTFxv5N}|sY~HCRx2^3)~314Pr<`B<$BI;EiKO$ zG@U#CqL5P1!77*+m2~3YFYF~*0AZ7NW`Zg}Ra`F=bG}ej3@O2AGC|Jyn+Y({~1lA)z2Sv(igL2rA;rc(g=(=ZvEvIUj^*NG+1> zFM2t6@tov*u5Pa^c9~b`NwO=yr%wWu;^gG}o^KzXu07FV2nNtz-F6Mw4d{BYkA_W< z^_d6!diKl*0^UbSe-@p!3shm3|MmQgGkC|JIjY<+-qd{w%+V=fn?r6*uYuh(v;Z$J>EqsZ! zu1hhkZrY3+!y$z}@dvIIw39JAkrtdOD>$0O$n|tuHs>(fG$B7@qyzD6?57xp+^&#_ z`AlbX&}dEfe(D&&C$VPB1&d^Yx8?nH6xAv{x*W^lsIKl3cR=V|gu zbbi{KtB#t)1d)?YnQSPjTsE}64+}?XJYt3;krBmhQ^3FMv~AZz$(kq6n5eK!kHJn` z!dxXXIhC_H!?e2F50vZueNxg{9JLZjUq`7Kl)Qi;P)afr<-BcBn2r0(5azYnUsX9n zdFvnqo2I?+EyILJbnL2nSg~8>DpN!JJ^5NXRRr>KlB%iLSNknovgCy0@Uz3mZSDx~ zWU_Ye&a*?akxF7*Qnfh$+eQ0l=`NwLqBL45S|W#f)nQcIpO_dJf5uU`0^XC#Wp~;_ zF4g-too6Z$kKY=~7=)HHQ2VqaktO|Uy#^Qhw?~fZ3aaG$!NNj~B(^k%N-wQ?r1z7wi zY4V{JmQa5R-G?G5G2M^xyCiL2E1QAG=3yl1_8hQY$ZwfLAz(|p#KuRCoA+Vt=41Xf zlZdR*vNy*P=mN`Ua=D16e>U1BMH z()SnJZ*I*yUjwHFRAIl$`RRvbD6T)a1q`L8w*;%ky;hS3+9xQt+1z_bd*ruJhR;$x*1F0vQxMI3puL zaVyYoD(zMIG;VP)xzY(ck(HaIDS_!wsE_imU{0m;fdQi5BXxbQo%`3%_(aVJ+|Q?0UyI&3B*resQ}e+|g5YJn`CCc361}W$qex#-y8^-3oDz$^2JTkEgS!&dAS9uG+~t6+RU zgm2ra@w*5x3H=i2BZ10k81ajOrl=gChGbPLvUDHXYDqGf!x9qX0hX)pVH>r=uO^M! zt@!fy54TZBFm4($8EpnDJ0@gO`>6hy)~YJ`AB_{D_rNK212*563NJREUpFPq7VC@63bx+gO6PY=4jgRDE{>^T49D7kcsr)!JMRxcVFQE>|>5# zo|;_TpE{?A%r&@^c`^?aadrDM!DDk+8m+*btwT%uVPbt33m?jLt7)kVvMBchQs zy^5zBr9C1TTIB3Ok@cRd>h;wo77vm2f(ywl)OG+o(% zT2$mUT(T(uO`}yTvS!utfU1@5L;bQL9H$hq!@Je8{eu}^>%Jk}P#y9+tP*n)bn>OH z-i~xjjq~p{RqiBc$~+(%RlWo}4h(nC8C8))ybiWjo69nsxXoXC{}^m7a*FE)EF-v9 z+&4Y?Mb%RyuEi~G5>^k5r21;1ixz8*;TD@W#ugg&jQZ*^Mo&F?tp+ZmwfqO1H6x~i z9qazr=zbxMkWzijTz?&R_JQkr>H*vaF`~6=`C(#d`|Cz?Bdw8|i^)+uQ>*2+kK$fR zt%2DK+CD3+51l|E>#cyf>*gS%v-;3hrR3)M7r^$Mw#M8y)AZT&gNSr4FcG|{b%e$V zSN{Gz#O351%r!I(t9qOD!El@Ei`%*_L%5CFBmJ?VNAYdn4#B0>Usz2>SJ!wsT_`l| z&r%XmFjQJlL!TxvOX8!vGiB0;!LsOb+WkHSUiag%hbh9g{k#tqYq*u&p>orX0P{v? zteMX35#7w%zFGjU)m{>O(ubwjR=V8uXT%}d&(Pi&d?q$z!UV9;+{>{EXrd781HwPl zHwoZq143v*)uL8-+k|7!kL<*e2KqOq>%a1XRhqhXlB&j*_+5%=Q{~Nq5C};ik4-cf zJylK4cFaW=wG%v8B;CJfR00e!VRT_(u`i^w!JtsAe+9>xw=h|qkK+6EKqrvG{q-vN7hdExk7LyYN0@6%hpoY?iv~5+8pV7-2Ej(+5Fje!;R>To#i8*k%Xeqd(C!iX*tJw!^r*ymP4AF6#^se4xdP#5mqbZt8utp~-d$;kd?O!IZU0KONsjX&!oW3 zth1)peKTa^eXe5ZpHY+614xhxMP63WBifrBV}_|BPFkKVf92+w?~q|@AH_It6W6~A zR&4%l3xU0JV=!=RXW-OX$LxhnxWo05R2dCYlsitJ>x;>O|Tl1>P^| zw?7NdaT}CcZq=7Ewf=-3X2pW-zP>1+-39Zz2Yj@|mFF?`Gm?3nHvbk}f7swBc)M`0 zKp2Pp(EC^fEp&(IntIAfff0c}JX-}}hhe_L!q+Q>%zdBbG$tyAl+lQYWl;zHOr)>P z4p>?sOz*6|%4|W~Ag}c^@zNRQV*&V`YG8x)Mwn{n2z1)EDzUn~C*PMuizMv&-}wo} z&`3(VLrRo{E)EK$!R%@rr4anqPA4tlyG&%jUkQBQNy9bnc@W2y3;6xozw#?qI~Lg7 zJ>2^|Prkx4lCFvlVWJ^l*0Et)BFbGf^bUeMLYFe&YG9f2<_2pD>PgD*O%g2D)!&M} zG#(+l0Gg!doIgjKh}9}`6JjPjY`l0T4j0kBexH?biSr&C*6pE3bdu!K)>TbkORmf%^RFkarP$qKsRtOZN#CaN zqEZz!VSul3Zm-sq+qpy6@Zt^Dn)OYrj!Dbc_Rv>fkA6d_tHJxJPIxzCP?9yO_)hA& zPfnF)doP?apD!R>GxP^qe|UI3;DQddU~hwQ*W2ZK2~O6-aM#M_<_~MZJ*nc7s5c z;?~O({ZdT1OoVmQ%lX04b9R74K6zjc`6KOoclgYZPL6Chi{1FZ8zL!?T^DNcbAh3l z_cwD?wJc|bP+&abrTZ6+V!&ZOPhrc$IPJN^;7@I#(M+`XSYCzd3Q^1QBNm%N7|bi7 z@YS$N&!)zPL&*PNur@#Qc{lbUuiEM8Efdri38L3XlK(!IdWL26PcZ=~t2j9_90PV2 zT9-bUB8aU>KjeHDEYwmNBB$oW!ziz-A>Bn9j#HL5KO z%FwY!slV~|p%TL^-Ha^D`L||7hH`rAVbj9kDv+Jcyayr;?T40~~ZV4ZIR8@*yljZ$;$r(7cB~ ztx)_(c^0C8U>v+qlbNGmrU56Ns@$ z<$!viM{sD&LYju=<_!s%H5sp zl87zrk*~*BG6DhO;pMgcc$zq??e@N-`(xMrd0V+xs|)xt^z>r&0KOlG&w8UbMs&K( zlBrG4X?xId3%cuOvrs{OQFNmS@tI16EnUVIh$%Cw=WlSCPBN^rFcTba$_?=0qt};s z$dG1#=;Dz9A>w6LG82;+W=PD(BL5_D=MpI^AP*rM=yUNm|DMQK_<7^%k3=4tWRO$L zG7vdNj*tr{))~?ptjGh4<9{Y#(4#R;t=wGiSMC7e3aQ9;Mwq9+=3q69ZKmTymqIs3x`9IJTAr;Pibs ze6CEggKAV8NZJ8zCekTp&cH-L7rjut$fXw38BdZxMoSP^;2$=9yQ}i)@kXwf5D1I- zxuel<^5XM!=cb(+LuKL6dUKi;S?FBT2nJMC-j~W1$gPi+ThPITahd8Knirs}B7yI~ zF^A8}iB^}GDUc9VN>y~~jLbIqN)-9JyDi?=ceOB8MJ}<&>RpLmeesAF(AODNITBD# zuVF1Ao4>D3>~fk}x36~*HCBi?A73ytC*vc)HIw%#VTVT3(xl%t;Cim@-Y__~?U%Q{W`+Q5#* zEp*jNzC%rgQJQh92wpq(t1Y^v^Mc0OBO{@e*#xN7p-z04@FaN=iC-pjy-M>mY0=YF z);McC<7R~wJ`hs~+w6%1VaR`4c@ix2lV6Q$wZ^OBP2(u~1V@ctGD!4K=U=v z#r_G^8gV^%n`~JSVqcbWNzt9Ay}m(trg>s7<9FB!L6t*-64IC5-JQhZg#fs+|h9c zX68o;?RORMsSUGZD?LS|*S}{f{X=SI4=bcz_B=|gG|B-RDZJl({N97O9@_x;=)I>; zNP}uRP*0z?N9+F9;c=k+16WZfTJ0}mqUs*K4hzP0nx{vMFN!x_UzbA-2WEda@zqz? z9Al=nY~3BAj|~Y4tc9+kIn;3<9EV$*0$js4&{bvl8EnAA3vgfuHqbvG8Mwbn!^>%C z46>05vD3IGR>8~VrWr(K;N+s(bTKklDF_~G7+a|SksBy7wp<2Q@$T^=Sy=pOJ9`nFY)h8zoI^ib`h)0!$eVEfH@CJk2Iu1>kTAU2>ly*|^uiJ& zhakS}amRvB(MtXG8|KoBv%@YyZS{J+N$X4p@oy^l$5q|9%M;Cy?;~F~)KEn%{qmMS zA5){`=BlCBOkjxnNNQ0}-C2!tJdoph;lhR?63F0YqVnn=wy<9x5Q+uDW4(f7g;h{R zi^_fN^bEnoucMFoQ(|Efb6ot)HnudL+!jAX*Z%Y3W6dQ4lvTRj@2qRQysCPWTAS?X zX}qLs`F}_HH2ilD*V*UfmuEyub78!AR0sFpk6%ZAozvrY<|}C8p*1ZnD4VF&jn5&I zTArS2Eq^QP)UqV`$$<~ci&Pt+V_TE^E@!vE2cgL0(z5W97aOOSm(6^~m*eeodvK=n zS!%<~mN0j|0GD|*nK$7rNQc{ciL;ML;!lM+9drc}l@0x~1s?}J@QkK=1jdWX;FR(97)4#=j7(=D^3fSx6Nkz)xF6@n2vzh}N@;R+PoqtDB} z`zo|S4ilYr*GfXQ@`vjA+kltKHICeJu+C`sZ*&V_`tMcIzJ?5s4>(3_f4>caiRT<5 z&c9horaycYzW3rV3dR=at6AsEldLZ@~&~zEf0T{cEx|7g#=3f!ki&6ZJ-iPCNvm< z&~c`39(4JJ1}4YM;*#SwnbB@Ig~!Au8AB@8SD-eN4Hz9CiLY)pD)9>#`E(k(lPK}kAR+zpfFpl)OJV!}1%V#MO z^G#_M#vUH(Vs+ZNizw|rJ*rK&ZLdaRGSQ1OEeBAuTJ$#wah;Cq=Q@#vTN!!;xLOV^ z8f;8Vx za;8cxyzU;`1)5WYo>5}kQ0MMtMqR;l!>P4 z_gD5&5m=w^AhjbApeMgtQM?&xvMP?&Em5*G1Cmo`?st`R=_#y)tFnes=+HIdpRKgp zp9jf~pW&ba168BhQeZRN9TG<=>6h}or~Us`(1bE}k^fV=oycOV zy^+&NJV{>f2K3KC!%Wm}v)tjpGI1)M94th6*1xtoB`D_R;4bQ?hll^ByT|Nljy-$G zzG(#&S{N=AD2XOHda8t@yrr7mXy8+A=OxbB#rE{kbea{FBfGJ;`msofW5;MX6wptv z*r(1Y%*sd4i=E$gahXvUYbobjyE>|-%t?Q2@!+BOrdwG zXkdB?Qtle9|F|ZYs*r0foA_6h7yUZIuKQ6MPv#z(P=rGhi^|K%7^iEFZD2a_1(8-{ zP}@_+-E9^ zM!2+~!AgAV>?H5mo@JcnoF?xcK|?P$0k(9QF3px@nv^-Wb|accY`34iJrspOr`Y4A zyrptsbziPRk!Nx?16mmR=9}5@%Pa&x)uW8EP7DNk{nC-EUy=TNjka5m1+~UJj?szv zc{|)tlpafM+K)kTV|PFA&Diw-o)6LjT$g`&MXo4aT_gy61glC&gqqWVC%s*Z%@b6z z@lC2LT&E}KVeF-f>{l*b%jIq=?=gUcAECU2pN1OLKkgvM&uQ36(JaNV)g&1IM4kso z;T3owIuUk1!wq!rd??1`AJPHCMVb<$#S<8B0c5qWnLE@o7{dWSa7(>tfdPo~JRtj` zO%<$!agBS&=Uv4OUYH`qpfOH^e1`A7dw(CIzKfAi!h`_NR$ReVc>io*2xxM93jzP&H3GJjeEvof-eKFar=U{GCf#*;j^`MudDEr03b4Gkkn=KYrnvU zLgiZ?La>voVh+60MY80&ObqTbrZSpFL(#P;Am_rQ4SCovYZaj zj*pW>Mx?|l$9W1dZ|7d~LCW_f>v8F7yy3Fu) zjE*4hUGh^mBC$EOy`v%GbW}gWl!LR+_{_crT-hrmXmUq^V>%R(@Yvuf`izq9(tE>b zE+ofBuo?qDnDt)=t(N54Kw;7r^?<_I7prADc_u~<|c zo87}~kJ~l7pabz+ag|nF@Z0ht6d!>f^2}}x5r8~jGMo|n7Pd(1bV)jL1xm@Ph$+VE zYve9QzqbpKPSg9ODyy4Mo9ZdaRZ!d*J*^IYS5-RvjUKa1_N4BG`T13~#~etLhTt4w zeh+lFkOlbyS~%OIbG!G$tJWB0`06JHkr*#!(@P3 zba%3>I=U{Znk=w64VCQP5`JHTFcZfvZvXqLfG zTQANvLXFA;);635)M{B{^z^#8IsxpUliQD#JUZXkv5~VYx*CGJ+!lrR+o;rIHTa2! z75f=TV2(e|-QL2`!^*|TSsrgTpr@^!)5~jy@TOC+SO$WO{Ih1I&88)IYX;XkcG#Z5 zxWKD<>(l<_(wiv?(yghcnN_T-{rxU%0^WV#kH(Vw7U=q7=pig>E$XYqqWf*<@NxIG z^e*mdiqp^Md&rQJF$>HwB!;sCZ6GsfvyyK~wP(!S0AD$~VG&ISTa(L_J( zM-IdGk+56cUhT%-PQMM)_4FQHlDe&QD2WNJYD!GW9s5i}@Vi^6zv0Jk`EotH6h4y3 zUo5ql6qKXej*mWIUt-hpbDc0{IV=UQbiFFlrxaon8z^UdpJ z3%n^9^)rj{pn`(#%v!i;j4xf(^|xsk3bzO9TfjhiHhw1_oz^CME&<)$p~bfSQ#ezC zGKCjiMOAxH`Kt(5TJ^Q`X>VkoprolrOkHUT!9&u13!5|`7LpHVj7d|oNwvqOaCI{e z7pvl2+824H8QEljtB%x?MI1_{+F0ZtQ#{J2_H?)ojQ~(Zx<{jOk6oG0sV^(M>JK;( zBggcJj<4L^9HjLLSU;-q>0ncz{`IcIJQ!TAnU*{Zb#yy%Gyee64UPpAXV)9A*9Cv+ z$nKo^56+SXMTKeMq|#XoX0v(&7lFpR#XzA4sI?R(zbEPU3c$(E;TzuuyVe%h{t@6{Od>{7&ZcG1>N^5k@=(^w*3`mU(c&NzyWGZFiX7ge7bRSw0@ z_jU1A_IhBDIz(W7(Fn)aEqx(w81aPgJ`^0S5V2FkJ>eH?kKd8rg1HhImj!02U2+Vi z@uja5Hovrc;6qLRSVIlSUB&JEIJ5zoF{P#i1&YM)yX6a^!6noQFci*Emil+U^XYZJ zHkf?DMi0PAkQLPg#E^tDOTZo9u13{z>R;phCD5J9 z8tCY(cJq7^q`LZg1kzf6lt~1Ok&u%?5bkd>^Dls=(!c9BesGc~3K;xc&0qE-rjjlw zKUN%|QM=3A44UUY{)A%-`sKYG&f&iWwClOxV^O_aZOLT&80kWWC_xjtXY`RhD*)BT z%B_BazAhowXF%0W{I5NPd#(Old01m%{RWbEm}_hXWbbS{s33>h^d43*^#g|>j$pqb zY6CJYd|uST@MhoqZhg%=ut2~|d(*gI?s6R~-gF3h9t#Ar^PO9@uiMepYD?DB$JTr@ zMl_@$I*6<$9w1oDQ7w`?h7LIrZY_bo=1&_TJ+G7y~t=w8(taiWD4;ea>;GMZvpC zYfv%*RgZmG&^CaPOI56l2T|HRlXv8fsW@cuN$u52Kc>HI(Sws)3mOzSLRJW^>3(HJ z+~a!!#~(0s!t4KxFqyre6Ir&Kwoz$o5SrR&-tpQja{dg+1>fS%IWDuu0+Rq0Pr6md zxPP#2NO$zrpr#hLJCaMEI`A#-`ho}zbTIG@81Rbjx{Pc;-1iw!noB(yT{tI8{{xFU z)AH$g`P2U3v;Trck!$RhgCF_;|Aj?4AEDfCxNHV$Lasrac~Sw9mB zP*uF3+Rd)liVM)q;n`B>tDKg_gg@ORZ_3w{ChfOlwFdNZ4$33=UA4AR&SQS8H}-Qw zE-XVw64hh~JbCh1_Y(GG0b%Gp!D1@x$fCbZVqV_hkd8c6Pa9?R!5Bd>WBzT&6@oGU zD~oxxQ2{xlfp8Z>&kCFOs>tG8&TkEWB!6%zXDari~t39b&JA zulZb-zNi&Wze)q#ARc6OgdU*=L?Cluh@2>YBZ&9r9{9tZr3^#E$>%E&9vz&tg-=HU z2S(;{02yVzLjEs6OASKsfzhNqW($pDY_f*d&)aA2xm`uR-KdXqzsNXo+OvC5<4YAS zji7it9;OS2Z#+;HDUe?Q_F2!%x>!q_!B~Wevji%srUj_NEjWm$e^cpaX2#a2GBBc| z!GbMJG!c)_xArcv8fbz0U9<&I?$jcyA6`gAr$$YR1`RyYFk~^4+J~hEn~_m5X6m0W z5f^BzpK|p(U-yNFesK-0yYC5T|IX4IWzKn#wnP(;5aH)Tr2m*B;j zCds@_6fse$Bh9=6qy8oIrHbmQSMwV!leq-!utQqMacaxg>bd zVKut#qu)eyb2gdG4x@eTQn&mI6>Wym?OcdGclkm^tIkC`Uaf7Q!PDo^{C2k=bxu19fZY@`8~vcqb{_Q|+#(+hJnwjUv-*7b1>5G0I+mmhC=H|1VO) zk2>Tv zWRu{aC+SBuPmwS$j=nC|cB~j{4ngA8W zOOVuPVxUQ+4z+$Ko)OfS@qf{x#KFVrEjho{q_m%ep09doMS6elr-TscI`;h;b2Za4R6C#STOYqVInL71{05T6=hiSHPDESOrsv zND@nFmcf}${(Ersy=p0|@+{lCakKOo%Y8bRf0D85(Eat8Q%i%dlGoZHF@nkqt_2~o zshc%6DNi{8Z_beSe?m0TWFHMDMFLW_m*RW|f}7wpr`d|UW0s*w=%{am{^_@dCu*b9 z?Eg?TBENLe`7d3R>o z-K3s8%xw@A@5EBX=fxE0aUo5wbDPnxqm44f1_Y%Kk>AY0@O z@5*#H7dEhK7psn0Ycz1{k#7o4KyVQIbIY{O>vY6ri+GDspjP;NGq zGc5$t1AM*~>wpf%*tlMMQWnEOaZ0ocd~mEg$V!`kC0=Jp0ufpL&I$#dKc>Uf&x&Dw zOrh4*cS_cREkr@(M-iVx4B5Cr9<+kr{zQ-di5gxTbp2pk*Y~v|WG|y^^e?GGzr=IT zE%}dY*o7CwC?u4+k>OQV!qe&BtZvG77gT}0q~J*ih&{4mr9GUug4B+M!$~JV=0y_D z@DS;Q)6ZK{)L z=OsdwQhuJRijPlYkqfo2LPkmk9lvLg!WOLtW|tvaE|*T1q7FC9M~ON2;6XMloolRE@V0V!2{;SGJvC1@M_WioSXt=(?vj zw^d@lcu|eScnK2G(cwF5M{(HH>BK4Q>x{`O<|sXH0Y+AVxZ$8KeWTBaA@##&(4C_= zujzk$QKxKFpoRcyZv9Tw&dY=x+h6kKuSGvoHHiHKYhZ>i zlJwc{Zot{G_;c!=!fZuhV8443J3ZKLqZdi&f}a1^mbnPB4CpEI-iC9D%6#4%(P z$ocS<^&_?>%kMd})0AYpAyuJS68dF1FK5SN zuGAyy0|AU9R3TJ+zlghkp{61u!1%Xhd%+Zqxk_{GYE*Bg@+ zs8A?s-iH4Mt_zb;A^ZIgPxly}NfT}hIGJRUiEZ1qC$?>y6Wg|J+qP}nwvGLM=j?0$ zt^VKLRrRcOFXHQ5+B&UPW;wV(E9y?t5HzM%hPFRMK@%NTrYeru&Mx3% zGlZDBDPy*~%_M-DDVPZQ&j=S4|Ag@MDnX&D&duZVFX1EjYJ|L>#%{{GF|)Yxq5jrq z%x_aKhJxUjk$68#yd}Vj@8V;RdDhdR8lHs8D=r$4*C&CXQ|CvjKY3;%Mze4nmD6(p zSMcG zALlZkED8UFHOnUB1ls@8mgv_G3HVtna=fXU@qbTav#N;y=SX`RkPup6i6eO59S<4LlZDt6pF=xC$GfTH z4JhvFvS1iQ16Br+XEYP1l84d!({^)J*$sQ|Kh;`l#Mn}n+AiN7HG#7pFeK8@cISC^ z-RyMbs7>1yV0CmqCuqG7d5%+b5Pshi>h7U?k+0#5^Ax?@S?#XNXKJWHLLPkeL`!-p zEY0Wruonrq~+nzX`cWKNJ$(=aTmX7DLo+r z$5>B8<Hcc$*q`lD_oJ=Aqnjf*VWfUwZTyr>j|X;?)?@=#1U z_2X)UCyI9M3}9%5$autur)M6RuOJhl-I-tQ>o4952IYiI@%06}hUb5`2P80)Jb9S)7B%+M2PW)OU8)`y3P6{50uhpIPL! zEHP&6-Y>5*8)7VW)zHIVcd3CilLzJFss7XD>NX%O~Y%_ zcVT2u)cl5g%JT!sFYX#+p}T?kPa16iNTU{jjSCG|OT6YsZ?&7X4%<4p6s!jWH>g7AFjmuuZiJ7#Fm0h9RWlR^qbQt$GX`?px+ zz@|(j*kfNK0B1DkA4M`cG#b{_F?9C* zXgjFP!XU3HU-K>pM{8Q(5y3$%oft=;{%ahjsuSlcz0~((W~9}exvwL_?UxQpl_Cu5 z7HcYIXxmW9inM${JcIxkr-e-!ok6ail$-Tyt4_x{!JJEYOh$fx!8tr0i(4}ve=|Xq zM2y^712M;Az=1qRnO2$yhgqekDa}PypP|Z6KDMN*V*T$#0O;3;3R94DOiw$28bw}$ zEL%PJj~WGBj}q)-(U%;oKTwK`!Ddwxa^W!nO{v?M^Nc$4}gW%j@Jpcr2|{ zhe34UdrHYP#W;!ss8KTjHG0n6p^ZeI_iqMG+iL6ij}1n$-TGFds9Nv|6uMxX+~)^@ zM@Q_wN8Q`axn|t4^4Mro(pgm@#B>3`n_@G-Ft44e zN^Eh1;t|LVR)iD6i7I{sSMkcuqg~?ekNGI{q}Eb4D=xmJj7J9$AmIyzK6?5mF>j4-OKYdKD>dHp3tyctC=p9U+g$MsyOLnJD8t zWQ7IeRA3S_zr5_#_ZoVyyE`vD*9cKo;6zZa^@q8Mq=Va<(2s{dZp&F1L{&&4wrlQ2 z5l@Ffxvl_zp{dw;9Tu&@DxQIvKj77TIt^IN*H1je!8A*EqEmbiD^Zp(185eohJ|gt z>-7c7SYBifADP$?35;o*dCxaUO*PW!CQxl|%MqK${-*=i6aRQ1Q-lgTshc3C_o!eY z=&(?+y-r=Vo#MR4k#PUfb8FN!nc8^_7^sW&FL3*9O4zxAiJ1M}8FtTJ=SlGjM~`?? z;yqBjyhGFw+^M*FMhWwAgo5jsQ7ON3%fB4b;mP%-bac@b_O^wmUI50Gdn?Q#np^m+A39&c8HtMN3p zxpigCD#;0=dB@%{fS_0(ZZb!r*6C{Opc+zC?3I{{No$rcMdm=2L0!bgCJui=!h<8q zptx!$br^Q0gEon1`7{bT!LmCV;Ot1*ho*O;g=dJV^E7Sutt++++Usv`J=8CpiUZY1j)#e9~oq zRll*bqr(Z!cM_12(mL5f>IQ3Fyy!+4laNA?Bp0dw?%HQcr$-){{0I1hAglmXTFFis z|N2mR$hN8cR^VkmI!^rsQwHc4s&OzOZ={vN4DgK5`$(2Pixekx z8%NS0hBt5yhyYd1nMOaKA~tkAOF+&4)~55xoq%$tvUj(W@F;T`urB&lDEKxf77Zj90ka1b6x@ErubEIy-xG-r zNZyW$%vxP@UaQS48sk6Dh0J=m#mTAPpKb^Vxx^B!XwEDMns&U~kVLV6OhmQo-E4Zg zmiiYOix*HZ(mNi}Ls}!MTuaWv$0mv>W-}0dnlAfjQ+28UL2KqZ%Y$2Cn?Jt!Z%YPO z;Fc*1-o3$)R}J5*!jycB50GUa>lZ! zKwTz5<&)F$mK4(5YK~wa4h$MMm%ks8J6?;1xo77y%J`cxQOBT%&5FFFwJw94e=3h0 z1j8S7W2J`35x#aQHvD)9Urf=2X5%2O6wj06JAyT^j?+$G`hgMA4_lgC@6rj*7=y`? zk0FkpW$p$KF@U{={v^i~XP`TqOKarj4v!QRbAumNCXQ71Vfu+eHqv)%QnOZ*Tp+TUG8 zU5^Ib_%sjB$gY<9)iG{wCVb^jA(icnanMlIi~K!T@fRx54f;^1;~~l( zN*8!%;OUkU1MOTo>#qCz*CWW!g~oi zTe4xvMh)E7rM+jHJ`P0jYP;w`W{+iE{Wo>A3?(=QbGnyRG#ai=7}WE2`%vR~M3p%A zBAP_Fp}aZLNPxzfi3EWXYgK&w32t<=T|M%(L@7)`T)bHoFu@5l%h6q#_}FpKrX40H zmuCzLGIJeSJAfG_=+S|jy^LLxZIY!L{uWp1#%@o)`vx$h8|Zb9CvZ>yF{6eI|1qQX zx!M0Qqer!@`F>O`Q?~B%M;GAt1>K!_cpC<^ncFGet=*e{EDK7t&@BtPPmQd1{Zw#y z$*zy*g0fy;d~>40ZwX?{`IxYSmPj)~A1Ru$rN2m3go?4G?W0gJ-&M`-y=*JXdT=BV zrf#3oYbNdQV10AMXVzd+v+=r0{d(S6m`+0GBSViC>Uu_U-M>0AiswIRCQ1%Fu)G#= zC~FaId9$XBlq`F1KudriyVKqxq1{3!-)0c%8o#G z8FH$KikPaP8JQ}%4jz`myZvF)B#=bzPDUBKgpW!i`%Z-Q2B4+k=NleYB;jH zP|z7cnEMYsw&QqDW`v(Z@GxA^<&?l7J+7-1GpO=|FABo9Rtdz-oqD~?S!FL=9^i}S zW9t>5=?0UjW*2S&x;5VFh{`&Crfd}DD&!Zn8S|T_UxGG^yvk*5sHp5-@qH)q^-S-Z5fYY6b`tenB@19W7Q`oPuMT7N zz1z>iSpltf&iuSKx=LkF|L(`XxY8)7S|a4mElt*>MIknZW5PxPbBjk>AsuHS%zDJp zJ6P(Zzym$)VbYMK)FP&BeFQ^%izzp2W>S7sMgCqjdc%N18g>4=(rG(iLPd}m&|9E~ zkIwdw8f*8BIS=|7^8PPt$;I(eLbdu+vOB8vM<5!m9-C--Rqo`_(&m+ zC87#g%&ZB)dmkHK?^1^?mK^{1QGYV(>yy_CHe2rQ-QN)s@_#{Xa5$1O%yOg>1N)N> ziC-5wlT2GZ$H^2|fuvYCWn7i^m;CEKL;sNO{f;yi27%L*oP-tczd^^C;6uwFhqEpR z%DSb>Ps79kulOfjue0ff&9`}r?28)$#c4f;6%sgSCUL#$>6@CVXk#edOI9a;Su(x1 z%%!1yKL%QM9Q2AOgqUsKR1z0bggQ6MgF0PPRoZp1$xvC9mnxUM=IF?IGkLTc^{oVl zNV|l5y{eU-_7pwG@X)Rh$qA0gP(F_8K60uPgFt^ljoP4OX|FhXJ}RWSjZNr4=HC=0e^4=<;W-cwXq$ee@rM@+ZYP+i7Q9zmT+n5BsL* zFG9#j&*6n??;Eav*`y=GXRfjiK?d!YMYPYNSbe!lg2g!oK}B~jGW$EJwZp#kw9=*j zW@8v&rHr!OwT^HS9%Zs}&Obg4SG}r<+K^&+`kp~o*qck*C+oDMH}tQk`CZg0HHVg| z;$WIbdUEP`&S|1(4y%VzH6}+!YJfJ}!N*()@a34}rm&u75S3#6Q}seaUf$*kF=OCy z#N(7mVByz;B(L=Sp&JI=;$EFir(yP#=$>yV*sZH|!$5j|frF^|{A^v;5U++uMCt~U z>L#g^x7Uun+HK6Rd$>&R}Q ziC@=90jh=;m);{kE{#>{4Wx*k=j9ozv^r?i%?fa(PP=8=iiCWkiUYkU)$^XuuBIqm z90m3P8C-&~z80ym^i*FTW90mnWJvezR)0wMtMTZ&=i_+mK+z@yJ&Bpwqu9nDj&*CBACwZnH#LZZ#yY!+VRiyP01y&ylO^L7&k6%`6~aRU%4@fqw*Se2fupXI)>{dccd(l91J2L7Y&5>#5QQhJ%jNrz z9*^rTy|oQ<$?p9-))_x?LaZ4Quv=A}*eIMcsrY4kVbiv^3W#@YWKv~Jmqlg&5TiC+ z%#NuiV&{2(5lmq9h%$?F7gmyL>R;m{%l_@sR|D!tp!T_}TEUY6grNz*fjFG1>L#n-#c zpq{;Agiw^^?Ab;1_l;gdAm?aZYPZ!oJ#euX+jbQQ(#u`NTvrFThjwk-PHZ$tmBPRv zyp#V4!#9na7y_qG2=_b|LUU}3>Ni_+_=ug()zVf*${$TL5?2AUX&9(^i(e?^4Aj9hi4$%64L_*>dqPTr4IX}5rh zRbvJ-1t~j4dB((Im|nw#z~{pJxf0d#79+74+Kmcu=BOoo87Q|;plQXgyl}TOg8`Km z8FpCBy6BsJ^5r&N0XYfpnxFn1vHd&S&T)!h7(st95<0I4`9L=}3(Ax3CjX4DnGgHA z09>8sQ|ZX{7Div4cpn~H(SgPYLQ11b)zOPefywq(@b(h@YfUfPo11?ZMlasYyM@=| zNSfR7c%m47$+!M(Og65kJObTdO+~XPV+{AhU<>wkK|fJ7q&-7_TtR*k&RX?czrgkj zY*{?CJYNqZk*P6qjp)u86^QrE|9qJ{pven$@zN>kqfT0LQ`2rDD93l7;_;hB6ukI)RhRtD)O_ ziYezcRl1gEgR;WOL#}J79Ye3mqx+nzx|&Om{Y#V^HIKSIZPVd!R zI@6X~-uJ8q<3zch8SOXCG4ZlS|JyjVX^h6!uVVBEWzZF_B+Ll6BpdEH%*S(qTU-m7 zQ+L#1Rq-B^aQ3-VSYd7`1H3AW0`k7??B_q?7*mzWxSjq&lg~w7Xd$o)sRbBn9c{R$ z(K?i*uRL}fG1*01h8yB?^(5bLU`^RM9!K}PYHkhm2Z$<=@ZHuyMW+v^jotrNb)wyS zzMxsHV0mfX^xzCZUReKX+Z#0s&((W5$QFxS)XKp8=bqvTHb|6@VF$29<^I1d>UAHBdGpZfZkIe&k=9sX*2O2VT2)spyraYBW#b4ZOz zxOzCFvkoJmrh@uF(1y56G-}B0K>y7bz?yp)ol}l9NI@n;&nbMbbO*3=&|X7SgU4H)^D= zgm8&VQXsHdHyM?DkMC>q`1h)#iak*()=Z)SqJcwLOv11?>!wO?5e`h>B{Vvzv`=!i zJkB695@pG>e1$cWz)7kaLzWc-A-+C-7|{PCPq1e)^0JuBx16m3WHAyT=|ChRhRET< z)Ts6Kx@?IZ8xGVS3stY=S^@J_yMZomn9s3@JQ*rxuP(JezCkLNV2u}H6dSM0P#e>P z(b5#6xCpIqdRQBB3p^pALr=yOvm)n?R{<@n69R0DpKQ59kPz?H-7+v6? zr^JWk&x*A)iN@3roK=#1>M~F<3;SKBZ%InExg~n?7mRu_558Z3dAP_9XqxQGP_jF* zm#jq%#f51}z%z#G;u7AxkCk1l#(Zr%E_4z%{di8E2vmYp{oDHl3;1u;R!z9Qt%D+G zb=Lh9L?=l4msB%1r$NX#;!2=d^C8)Z|8UX#ywx4DL2{%jt<*Y?iCu-Tvymu4IC8I^~cafoCD>p;QaRVsCN)Esw-uAL|yBr4RBe%&U8FH-yAF zEzT!gQo#YOT4$JGGq`I{y;25QDv&v`M46qqoh!M#ztHHe(V! zM|wamLw(;X%{kJcI2P2DC46$OteQ8~hTy9yV-qcSIIm?wx#7Yi<=RZUaaw)R3%vY` z>utV1E`BY3Xg&uP@k&atLZPtn5W$T8ZIwa2xEVI{DVhHpxD^fNrq^(B%9j^N4>t0V zmbp(q`RrEF@1dK8LP3m!2Azmoc^`2~0`Z}4D#Va_rsA1e2=xohPN-au1OY_ z*-exWCY5Z)ZfEKZdR1(+{3ZSq|DVTYl%60^7qLTD@1LLtUw|y?c^-K^-pU%>0Z}mA zxs56?4tJa&nz}FZy<|=H)4U<`{bb$){+6<+WUguY9x!3fa+VL$5Yp=x3zxqbjnWIh zQ1mVg3Hq2l9fCpSB*Ip}*~4o1(ZtCD=*$P(H^Rc0GhO|P- z`%-s_Tc3%R(tMza5{b@6{o?#tqHOW8@c|7?G^4PUZ1W4Wk@$oJtHXoPF2pV9vTnyr z%;+BjZi&&PIvv(Q5jcH@g+t(f3r1h7=J%N(%IVgEdt--w(0yEy*8Q|1>D*~{m7Iyx zh~_#;lLUt8v4PrtekYZ2>5t?^7+JM`S^c@ugPf4vzyXe5gT?NvIPKTXT#f__t;?&6 zyw2&WYEGBS2!bp#xg&@1p@}IZOtj0-=s1Wvtc(I?`#{Z*j|nC@R^oT*ad7F7e>PsU z2s5$~vX9lWz+vD1it=D~b!It%cBce-{knjVxyoeoOs9!sU zY)3tiZd^H83wfZohd;Fx!xg*y!=VWb99Zu4%W>G*mXuUbKq1>aKLxuz=2BA zYqT4(3D(`Ldg`7FSrH~EW*bh)jP0uEfa}+O{h3}fkxLwlc-s)(!rqMTh-!`s*uv{7EH>dM=>j zfAcG!9VZv8ho8rhgVYE#FSYdy;+<|lR`NX?R7^`8(;e$ z*7A|{|7-XUSZo^Xa#lu5S<}gSXy*SzL-Z#b*5lU7HqKZl24*{$x*38^zRJowL#`2) zruUD@W=DHch5eu-YHatHXb0+1coC;L zf)Gt1rQVXrO^!JMmWGKTADzycK{pF6!}+~3Ne&LiKxV#1%8{<$gLiljDTpm&1){`% z?*cI3e~|i}SgP2rqh4e-jbw1nfRyHv{mEUUa)e3Qzp}ZH?olM*zn559>)DJx-OG3Hp=|)QPf%cvP#AJPd7O%8GPhpUyK64oX2ksEW8+4w0_^_ir znI=UKZm4>zPH@$L5tB3CZ9qTA=Yvdl2~-8Z{<3KMy=6 z@MlepE4S~~htHbF%gOLRDqU{A8C}2g!txFmnl&D#w8M*=?XCMp55%3bkd~{o#VV_7 zFj4{tOUPaL+|BU=pPnEGAjCpkP;v?9nLj;_8H9JkeQw?|Ba(hsU|pqM$dko|J9}shk$wsA%T=AcXG_tNC49I)bJ+ddHSJ8Qtvnw(cjr z_1OQ!oM2F2IjcAqZ&(mcnHJG2YMWxcU$e|LrZQTe6Xs{`)a66jd-FluK4-5tGh9b! zHXasyKpMFEE~Sscvx0jGYO`ufg_%|7;a%aVhua*JaUaoCZ{>VH^3po(KD>8GZ|w1- zs=xOxTq6WLP`l|sjp4*Fd)sjO@y!x z70DZgddS zs^+k?ITJ=KWMq*UfF>TlA6tG3v+ zab)#v*XJb=Xm$q-ZYX;rMydXi32wTdV(0@x0b=t_%KR(D`i%$UWg?~X{R>_B1$s^1 zr4#*N83nV>EM}a;4K-e;DklB9%Vnb7u9Y4}4oNkcA55`Z&sXC@PYY%uHM#W6@ehXd zVZ`jJ%M6gbM~fCj5rr>F@1IE&i)*e3C@UA-bZP8E0X;(p2SL(G`V1<~_z;0$_iT~! zq9cD4sA*goYv_YfM{lEoD_QkzzZ z+JnK2cdn7EfuD?pfR18^R=uA?zNBtap!$DG9jnW1lcfQ$d(8SxUVHHsA9+IiN#PkM z&u}$6@E?s7NNma{ni+poD?&=AU1Ta*q}8llW!l<*pdTNt>lTX2?(7w5jKy)OXMJ+j zJSH9-?nO%L^_*q|T^p4e-c#ioXAGb#^7L%YKb!h0NB z?DsFGPZMY;r ztGvOcofKv!Pn z78~nES(2dkW~xN!g9-=s;&8N3DsH_v<>=|L7bp6vH(V*uqeV+`zmMUEo;-MQ`S1pc zemp-n>-^P^tgLS4+UHUyZKiNINe=^{_`%T+l64)9>a|MVg&#YPdEsU`{7D--AGoZy za!Wk*N{au>BD)`7o6E7?EpuI#+}4=~q=Jcg}(5IJ%TY0K4vc+_ardz=3ANk;%a z>4o@Kny+DW?bz}Ho^Zc*5Y1VKyHy^U5RErX`p=HZog(b<)OK@;a*qsOcU!CHiwXBm zb90_jTo`kTmg7M4pu$kOL5t)T2wg?)Q0T+4%7m7Q{W@Kc_u%9g<^Y_C<6_njy+Hh8 z!(NaWj{}pMtRe~XnQ!VM8M|Ukzd?-@Uip6@DRYF=c5fz%L$tnE$bQ#LJEdUV)>7N{ zbf#h#Ofqk1J=VPY*1tjn*3gO?@s$VvsB&%vq-=gqm8n=)BX7$inS^ttdNsjjD@!HI zWYj5AE%ndKYN1T!Yo!YKLd8iCWi_kq&RwYNItnAdV}o*wRenpQOUB#Q$T{hQX)byM z6**ql(rusCAS-}Jx>*st>R4>Z6r|(i@t;RpXR`omekT_f9GH#RlxeE9nWLoMM>;Ri z#Ax7cr&*@QeBtvOy7Lcl@aF~3e;%pL^a$P-gXohu@)hhYpR0l*Mkxh`FZu&V7VPl{ zL=wwrEk2%(uJYI~udUzRR8Rs?Q)J9@bmBS!_o!mh`g8z~lvS;?#re27`9F_z%u3&4 zC6*ck^p>&Z*gEo~5yQ87g1RFaLpE928)UJkuoYiZ6DKa>(vNnjZO2w%4b!7W;I z9=aK(UyDkgo}pTjkvEUK;#Axm&vEty!EFY$((G9q8TOu_EWy>?`mHmBvr)X zoG7okas-9grT6LiEnwDQapl#fVQ2LKrfFf)&P8%1TzxdL;v=hFBj|>;gv)V&zj*T( z|AXGj+VbRdz>XHJ*Md}^)#PK?malgtvxhJAQTz0Xs@Vu1{i-Kvf=7;rZ=&*ek$DJ`R7zkmg}f&vzE|BWaY1^R z5eF%1jk+60mQx>9i48Eu^u6!c|2cbnLL)?@XNAs?9K!QA~YH5=goFDw7fxRMPB2HN3})@td#t zdd|HO-8l~npIEP1jEbe@;g`SUh4A-7&!=(ctu)$gTU)%YK(n`sl=gTW2a0EZqWWFY z8m>y*pSKX)$zeIWqL$z}{y7^>N(tK_i@R%K0^Rk^TZ2K~y23>}-rhqQoj>N;PN~SY z<6K4?x$i2f+U_)0RT3vsW%tuct8>>^lSO8J@5qwl{uf0AhL;JrvdBh&(Hpw;p!w|PzUW9116gg`s zQh%-7&Qorm_Zz-ik))_je7(xTiWgK!Vbz&~wE`Yt&2nlBu-+d%9Tk4HS=h!Ghja~% zon?30u0_(WYPe@jICch4_B~%#uih>uQF#Pyg(lZ z+f(#kopgXQIQ|SY0*)OL7E84JfMk`t!;koonqM3UmnS}WiPWm+PLPb?CRMO6C^Jc5 z?0`t^C(veTlR(CkvS3MSDqOC}SYcDm2~a&I-C~5ckc&`YEO-+xGfmOrhfHQS(L1t) zQwZ|QU#M5i;!5r)s(b4EJ{V%jD@v1_LTc3qQ^;H~?B>Vs)d8exZ7}JgOB@!flf9y! z8GZ@H=rn`h)&I7qM;09Ty^?96emS6DC+~IyAx}wj@_ee>#)i*LB!>yqcFdoe zO_it4kOyIz2a@TF3bt{4oi(lUGvyA&Z?k#1GxIYA07+lMf7hquQ)_|2|fBc~~4^~8FyxQBcl=5#*sgW zkqU<_hr6Y~RYvEO_{Pz38_Qcp1v2NSk6~+nzo+Wfp}oC@>_3k*prdZpqJrc(#1p0` zu0_i{5Ue+aKn9sX3_8Ts`qe7?hfR86AKTWxVPT|VmqHKKuE)?ZsWyV^~v z+K;p*cVBkZ@SUZYCV)bXDIY-5%=46{8UhxM$ut(4M;HR`p%wTQ1ViRRt65tF)ctoy zhM4Rf0a+c*O2k<6n!6($C!LFd;FSTXe4W7@jnbuC@Myw|4J)9F^yTm~{iP0Czw#eK z#1vYfrn2j;*Rc|jHO=IgnT<$sVnJih!8gueiJehM*9A&yO~5x^xEY{y%>TP1^LKWM z>wr@GR&E19T7YR+AQW|pxTmMw%p~0vC>)SK?_d!S%9Y@Nrt)U+cp<}sWNA5f&p2li zKf1|r0WFZ5g3ENyb0HFj3Lc_JXL(i=lUuqtjj$|pt`i;`#WYLjU3+~sxB^q4e(B^4 z4WGHCmCbrOcrJ+O>MB`COioGMs`fDm+lp#RGL1{G=J>TJVk^Pec`-M6NW6~rwP&|+ z9-a{bl-Jg0C5o+yPlW%@M`z3}47R>4YBSDJiEBLG4d~66#!p%<@_~MMq4PCS#Ui7n z<^m@qrQA&+ZoG3__LWo*c!n5Q3NxnWMTdgmm*~e z*OL2>3N1{o>s_aM>q*5nG%zWJ=X_ne^nl>$#McMTsOko`cz>KLyCnRa>~CobAjN9a z@BMCuwmed;mKpu&e!K0S{#yP-X%=VU0q*R9!QF#jSFDIH%U0G$Lbfv0QHvrW1lG|3 z+L$U~N|mA2!!qHn`^#w-XovrRP7?JG;acb1st@`JR9oA+Q8+8RMQ=Qmg%yO9Rplnr zS1In}E~fUx+0HtZ1U%V%vUOf(oJ&&r@0!^5tdU6SED%hfq=n9nm4S$`#WW&*wx9zD z2iu^gcTL3|^RHfud~qg<;&0XpH$=KsQu42Z3|V9#(8*f)tWvAhn0usUM^zL<12SU% z+OUm$rT2~bNl!I}9`7+E-I~T=R!d79U}?>G`k;=B8g%vCiYX4O+7sB`9!#5@46l?od7j#1e}jJfG#dGvNe>E>Gg^TA^OX^g=- zy^&uT7RIs)o|I~Se(I!Ql?|YWiGfX12 zdfZO(HBlI#%T2c64b>?1I2y_<+VrJHu}D~mu^9o}o}9A(e{?86y5+0q(my!#A5KGN zCt$x|{z-@AXb3lJ*rPx)CBs2Xadiz$29Vj$Ul|=)41^Jh`V#vPuDVJ|?y6My8FmZP zI4i$tl3n%JSpHZ|#&SuWK|2X_zzYbp$5m^E6^In~f_2n=P(2oWuW3Q_HTzy^^Oz~0 z*2tJVBU<7D3CQW>@4cmlK}{ZB`zn8rNX5XImP)1@Pau34jW%%L+YJG6XHSq%Il2bh z_+xR=(YI>FnX=;&0COE$^@=QwT~3vB_L_y3 z+G=u@XrmZ3f-=xf5t3D9?f)+w$`@9-4IdlvlCqZwB8gwINc|OqnjAsHq|actNf*j^ zKEK1R9vJUoLXjGm{CB`mH~Lzt5C0fRP;Cv7a>#k#r2a-{$bN>?iTgsrkVNVR2M_ka zW2Rl}-%1K4bFl`PR8~!Ba$SPJyHVnEgyvRONy&)F(}r#NzniuDw*D@|(*DSkRHC6- zDWadp^}lM~phDm))b(Jh`+=P)>z@x`h@%6nU5Qd%DVW*Vp4xVyF-?>zH@$r;keSnd zTtRuQYc6rGOOW=$)j%p}CG{t3w{l=o4Kk=8_z)NR9qZNFEb7dt*1NQ;$NWEjJwW;L z*>izKu!TYC>iOx?T$^%iNgPzgx#i-&d`DCyYoNd>Rfb}aSB;YjMm%;R9C@F#evGWAm}+8>+;CH5M@-S8^P%3{AmnfP*Ok1&VCT3j6!G)%9JD zv-C9|gjm%s+D1L!qXW64O4wfvYkw#H)S^tA+G+>nAxJgzV~r3K%c1mMJYiL#ty{L2 zwMCj)Z!`n1Yltf6tsmM}#v(k`$Wsno55s4;Y6}49(D~JHJpO)lwGHgv7dLk3phSjB zzvSFz;j2BINY9#)TAfj{Qr1W2GX^%P5ZYVkVGEqL%+`}4uN|L6Yn+f+k3R)LF z?z3BkXCh7l=+Y*^JlJ^=0s4A4CGO_KFfOqV z(`N`YoFoMx7EP2~=me**8fVUt#DfYC>P}!p8k$V$n2Up%NEhjf;YrSu)>vz({^05S zawXA^_)0)c$yBAQ`Y&rsmB|DmB>OZ`1}LZ0RFLYacl1SnLf*0#_OpvnG+9YkOm<~# z5$6o{ss+KdFHS5Ox7+|5-j>qainR=XtZ^zBhqXBi1iImF49v$b4{iA{(56xRCO8bo=3+b_SxU73tY)j?H)OouaqO zmMJ6m+LkF55c!|gQ*p-~2r8e7@Pj5DmjF3b`}zEEnU+Du_6qSDbcObxKSHuLTt&`Y znVFbsauhHah|$C*krB^yX;=1b>O2=ogF*;t)Vfld&kBD@mOI_GzoqKMR$_17?$T1p zbar4zj0mp^79K&0-Wq=jdWjbv01mB?j08UWxAVy3{srt&i;db0KEdDTp_TAO;91K} z8Vi>^P+>_0ZIXh%#oxjTnh`3V_?uycT+Cp)*^){8QeGBa+S-}rZ>w2s#6<<}JMzs1 z3?|A=R8Ex45#$o_H29R9--LUrA4SbM;t={ZX;QW01J}UGq6~}tOJ~on#(Hi>x<|d1 zMCGLl*PoD&!~~o|Z8I{U_8M)>hxg9l7f~2y%CdbCUtyKCkm)&2iJ;ijG?A=aoI;{q zOKtv$SnyGN1%-R$tvo=Z?5mqmR|osXT;|VcIDFx?QxU^QGcKDGWm?^vvsJA0kmq-n zdb1@Jz8aj%Y;0XV5~YDxTtb3KV+C1pLbax7gT_O}?rFF+H9rxWkzV=|OBm1QsC*S+$x&WnH%`9~w% zg#GH4;9r$Fs8irgjv5V^)IB9C8)cK3-IX%u%L8Olu_rSl2PXhFRMp}^K~|)POp2KC zs$~RFgJ0xFQuBnF&B)-52KeNNP@Q^En=XIAI`013f$2kiX+^boZW})`ENLG1HhSB~ z1S2?bu<5#k*)pQ!vlAga=f zc*QDpg6XjzuRGnJ_n5hfAJ@D68PkHF8@iwO{hs0593QtgJi4E~@7I+SI^E9KhnJgg zJCCnpy3ZrNZ|dteM=jDid9tgCsa=%uy=Bfn z#kTmi$&;?&Dh=87s7evx5K&Lnozo;Z^hNg?u0vy4Yl zgXBgZioz?&IjEY2)KxyU#EKaZ=m}5?wSgy_{(*9yxO4bc#!y?`%vhPm3E|GlvDZN* z^tF{D6@m<=<=G=h zjL#G&zb{|0lEvb1g-l!K&8D{efsui$C_PtyPq|vq3Z8_9h|-vB?3YzK7nQ5V&T`Wc zOPff!eF?-(9`FB=MBGi`rYh0O{RC_3sL-`InXmw#yI!waseI$kmp=@3stXiSo417) z9VY|qJVvRVFf@I9OoYBbW6Tqum4SIH&FPp7!rFpZ*&dEoj7TR0qP`9TYIfBpEy5d} zm*kZ@i8o`rI%cLUhannU4=;i1N&wnKrIB2f>? zr@Z7`((g4xA)!uB-@rHiuD7y87fGD#tvZZ z+0YTLq)?1>Oj8e)*^M#RARcor5d|zzl01gcLsELF@i>>Mbmh&dv#}<>^J&y%pfswa z2u~a^f&NboMF*&%>O2NB6Beacm1(*83(&Y(KzJ4;_xyU4aw5f^eL^01n!tOS>6=V* zG}u46o0-Lu{3onAUOJ<{**3C@s@|8vmWuHQJ0wy3sQ2WM)XMh~@G>}Eio76UM1SiN zPyTBz-p(8DI+a=$T{ivIFni!yylYWj`Td6}6GLOU>c^2HSHokxco&_VO-qfC!R2pD z_aa3*XdS|kwrg@-be76yku>IeT(Lr&E%5qnyq^?uiY6cD1mlO$iQDkv>-S5pUgODg zW2>a>{~ex+p#4t(P#Ul0o^sYy%hS6n+f3V1mgIpVW@}(Z)mxsQlQ%+@1*-w$+f5nz zHdt0hT#@qwWm(-hBYSbzb#VlfG(M-(!8Zw1T4`0_gyEwwNW#8{=GQa%Fw$~VDmLO_ zGE%?{-Lob5t9H(jwGW*#ot!2tpPkw-2cBG8YeUY~5;X7d+%QZZzrFlHh3M`)p<$)p zpW8ysrQli?gD0YM_2MO-$Skudw&aAoX8Ejo;fYLGpfgs48sq@_1VVBtI}KXZA?v1q zv8!yFZ;~u{5=ODSV^rMrqyLY1a2`dxA1uv(xEz1bc@I$+pyzZzJX&<_Ojj{zgFIgX zTIv}SnkQ)9uzubCl|im1C|;%krdzpSMZr_1NbsHEtEqVQM3yYZ>>27d66Y||N9o?* zN#wBZAqRB~*~KN~F`Yr8uj3#~K!1maFddKG;xwIiYL|K`gc=tzJc+)Lp>I!w&J3!7 z0g-C!8;&vERRNinC~j*a zWV+q#I<&^bZu(D?u5&%CawS}WQhB?t)G?gOWV}IH5_2@+%Yn3(1Tq~z&^RvEe&ePb zW`m|_!IW}0tKGMq1@^*Tl-@-NS_}J9o%(Q7qOL1 zWZ@+jC<~SntEL)k^m~^}RwM}W!u)m$sYG6mYZ;{_z;w&j0|+=zN{sN#6)qyL79AXA zhx@s5TnngOC=T8sp68=YamZC2bceFY33)50+@)kS8xhxob_)?l?*7Q#AG!NRQ7U^Q z=v~(B)YN3`s?W%G8J!Xaf1IXVnSaU6&08@2m%=I-I%ftpguOK@I6AY0$5II-<-bn0!oFe0U^7kVQM%Nd=&Lt_}p< za}dCuL%{UxADrg^kUWP1;@Kd`?`jXfRaHkN`Q6u|DBoIQ@d84h4Mk zXmB@=3v#npXq!U;+8hAJCLX*dHtH}u;;`RA+A}E408li80?-TvI}=3?!^0lKfMo&$ zBcE2BV>sAG1jL5Nt8`9B|BPL+2BdxHcIukf>bM-Ns7ns?TzT1iv^4a2@6Pl=@TGwV zCrrPB9gthOrgiGj4>v83it4eG9jvp&*1M~Yvbcf`=4Bu~edo*r%KTT5+AR}FxA zrwavQ2W+prvl-^_ny3zWNV37CU;yPpGc{hv4?7E+<+2?gOtoh7JRrBj(Ug8Pr5{b{ zjdDoC*sPpWF*0-&?ClLtt`;*eCL}OtCrmu3t^rG-6#S=>)!LwnA=`r~5$fIFv>_ka zqPK}$2~@RXt)|eC&akP2Mj9$gJ!8nyUbk4))pG+}!f0PNcW;+mk)$UVPX9S(e%ns6<4;SUsAQ2n3Uu)8eQYiCXUdX!*S9CYC z48&;jz10}Xx8_~f3x%H&E|e}xaChFKB@N@+@2Fyyo+I|x?c19!-$=iIYd){LYc;!y zq)fA74hqnY>whL6KqN(&I15PT?PAkk+*QVN1!?r_V`6H?etr2(+=~EWzc}ZH0KDZ$buH!Y#h|0bPUPR^=dk-iy+tV)3-CgR@b{NTHm`S31H-NhuxYkgA zkgWnv{StI>CbwBp_5lNg^#TZC?ckQXmfd&T&^rQkI5A7w zNqbOPq}tk|j4Q|+VOw-|FPbt!7nEm?x2-lrpi2H0A}w64b(k$|H6sG%1z%7|Okk7$ z6ICWNjMZ?z{ekOQd1qLUPB+uPmL^*Ch5Hw9W(&p>%Ycg~`8Ykz*_QyWGV-OgFv0&i0iqd&e zb|yLSm<7iUY~Ms-81N;WHwnko4VM9yGJneEZi6mUrmQbgt_xv@qVMrMMz0=x? z+k!DPDBQ}AY%Uj}yKd_OU$VSlhB-*jNuDvPfbk+%UUSN|w{5Pj*7VPF{*uDX@R4xT9c}v?0=_E@VgkSIlE2@RxJnGIxk73Lk>j0ntsx=&C8QgXULsSgk zw`#KCoj9v0$+%_(*hXbxeImhXQ)aXv?2|cBs>7QzR1E4cH?EmDG&-WEgfjFX$x5Z2 z8-+8V2FMd`LxE*@#ZGOO9;woEXW4WH!y!e$cnjycDG9Q32FWd+Ei}m58C2;ui)gUT z6H9Em=y3ZdmN+YDV5OuEdkM6K23L$W(`bt*miYYVJ*}fy_G2T(5@9I~t~1=CiY0bB zqmr=(+E}slva|+`5@7}gvbSOhL>1XMu5Gs0z?B2F>y8#!Eb-~=LzRG7dZauLw#@cy zAvRfYfUVZO8e*}<5_Ohh;`n>maP4PJEZRJuW%*@S2~kJE1t=Y z%4hP(3TP5fMk3E|e`Pd@vI}F0w+;tQQd=n;VI#&8OA3PQ#aQC4#^D!CWh)J_9oujA zxSN8k$yoYOy8~O6u^igIjO74UW-NVK#^4L;Kz3&=16rT4?3)RVXA}A}3cIojkHIWF z0=uv~!|?blLl4;_l5N7wpPM_tHEOAePiSeO{u~wtr2Dp70 z-}V!t7&k4!jGN-qxQv!Jd@9!Y8kmaA#S-gmFyJ9F6TJx~qhnN6J{-%)x+^j-?=qeM za&=J5BEd_+Oh)?>Hxq*!>`uxM(d=l}IGQyA%46#&p0Zg3(8F%!O#qQ2E2(=S%ZR6U zw1_p<1rJYm&>U54yd?~VZxh(W@$GwAEoc@wLoF|oO`w$n&4VZ z`;p=vDejTtt}joS;+6twSdbo&tZ?DZ)!0;WQXv)b38B`@HE>^9wa7`eY3sO<;28^? z;Kqn|@mc+0Lq!MhTXkays(5e4_@ig)mU8q=-CCZIXKJ`XrWRN`U#&Q;VFdp$IL;(g z6CBj)oQo^qTFI-8?{qVt4$UnvwCA0D9kri|frp6)F4u5V0yE#1VOmYO>w?APqrzh0 zqLr|&d{?YD8iya%$s&3l&nr$h&OO+<-Q-kRgIP6NYe8sR=Ne3RH_C&L0h-A9oQkx{ zfpJd$TDA6@znkv<)7Janrp3Pwg%V=cEfOZ=t)ywl+y#&lXN&}2v5oeaZbB9;0!Fbj zA*Zxiafql`iXLmf5Bo3oDH`Pt8V;AN2Vq8nSgl3!0{zWox4BTARpc$G*{ zN^Lp>P8dsdas@8Q5B=4-(0#E@k;&u8|dTNlwP#=*kfrJRdLhuPG=!3gXJ5zo}&5U}Bl<7mU z7HxRfDQZ+-*})OZJ7*$gB3V|cXIuI-OEXU(@oIOZBiH1CVv1;&1|JgO=X05o7hJdo zb*r7A-Hf*Y45D(3I45NenjNgj3#Yb>8`pM};M8UKHYG`S4wg5c=seHZxmkCt>B6rE zgR$L(f-ih~Vv~5lA6;+1?qcd^j4)v&PdnC3%KNSxXw7;DTv{VH!thtW4S%(3pjX1K zsg^KBpdSCqZH$v@_RTb;UKUOlNb8Y-oECJ>G&eF1Itb^!$|UqD*8l#|D@G3pKVVG2s=Ym}%M)2b(5{ zi?vsFoN(cenEiO2jN!=^r|h2{lE!0f6MNwdxks)hrfK!=*SmF7Q*ZnJ>2%36{72=33k|vvOR>V$a~lsdRP}i{1^vyx(geA zzzNvyVz=sCXS=JabdL8%U~>L-oi@O*JsavP*T+k)`zqVfTjL1eegts;g#x$-d$h&- zvbmGkecKG+#r7B?RG_z7FF&;-&1;k+&1*Gvqlh@C8 zY(HLOef-6SP2O?L%y!Jo)_*z%gNP6C$_nwv3h0jwGaUk@_(!a&VbM#d|DQ4l*@Ef zIsJLh?eIqw`*ONv3OB6c;CZn(Ly3d}hivU{Bq)gIWl-6_)--942^DFjlg^d^5VD3l z-Z=R6b2)0{DQ^AbX6uW929&0{Y|U8Pr?Sl04ZPMMIP%VH6tEW=sxK$@)#lhXGX$7U z{2H=}udZ14*}C+;ulXO$_z#_RfFhW-+`gDnHkU%JCFOkL+!zoygry`c&4wWbGfX$n zIG|mve7c?MsqoC+1NK7zEvZ2w%m8SjOV~P9ugsPzN&(zkF4Ofi5W%_52^adSFSndw zE;O621zJg&GCQwlw@t^|U>4N~%=NNuDTe4i9M*SKCo?Xl?o{^}j63ue`4_<>YWBw7 ze`^ojP=nb*8YA+RH&T;tIWdj*Hm}lg*B^KNA81+7Df6F{ zcNJu$Yu#R<X7eT*oKcAl_MXFBC8MWGFWZy?gk&c&io`tn>c6N65&DUSU zzh`G>t$)A%>YLww{i|1B|L(WnoPG10-Odun_?pO2OK&!-VqY-pYkQd6X3K;#UMQOGP~ z#c0aJ*vO=SkH`w>iR6(>xzHrPxHyb19h6FdrLU%4wlTa#FuC zW7&MHW_Wnu@o4dCd^SGYPyr5ka|7HO4bBOjr=w|JZg{U6-m}G*XJ3AO_RXu`j}~8! zUyWZq+wjGl3OZ%!=x%vV{>ZZV7b<6onT%;6%N+W&GUuXwzr2^JgP=p;NBAw{N`Jrp z_qSZ>#+9_X+IaQNWTCz5Y(&Hy75Rz_z!_WfskzG%Y3|wA2p#lAr3iy+Ms|lSUfnTG z(V_s{f?9%I z&*FM#Y_r_FBD0!7tkK)q*)Vc646rgqRcSpCVOdVVxed6Lw))`4FfK&}be%}CV1)*$ zbt?GZs=AeP>9h$J4dYb^nxTygyx7~)1znPYfzM_sy!zmSl_;GxdZ;Paj`AdOJ}(8= zOURm7@VipWLY<~;!Ln1urz2Vp%YvQKoR4520tm`@p8mSxyw>n`E1DfR4H3&@ zPKS*fg{rLGuUZAC2L1Tv=C*<6H7|H)~2v|_O^J1bQuxxYT0B+2{A#2BNzwSc7v1fdY%}59^0+& zd&^cg;q6{(v-E;?flp4sZ@I%k1(N%&q+Y>1qzAq7zHFMZ!YRro!w9l;obw? zSPAiNOiutWVZVq8aS%OWQ5bz3S*z}c`>mVabSC)aWH#}gc&#%=5t(y zU|+W&OobR!g&w&AhC)`@2ev}QiiYDW#Lv-y7z>9%S%{doP^5+USsNT}VOYF{=sAiC zlJ7TJ2gF_2Bl5!Gu@{~I`a)0qg}w+3Q5XzSC=B5^43Ce*@R*rJLJ=9_X~CnQGCVpi z!vT>Q9&_6SAvE+zA@n3r8v5Wg>;DM3-2oElhLG8=P}8s=)0i7EKCtvK zA*BZiChZeO`WOME0faAr+Qf&8ZrhHXeX0&b?;1jbggUF*MrTJPseO3bKbbLX`~e2H z_Se;puX%0jify`$`i7WnG1yysJ-Vk`^%ES6f+ZM?)(AtXeb`$rK25hCELR#rf@NN2 z)bnh{Jo6wPffo);BQY-zZ1x{?WRR<)|q+YDyk`6^_>Zqji5{Ia>FR*8SCTwC*3R`$y~k z(YjBL*8QV(|7hJmTK9iB($Z08eG02^wC*3R`$y~k(YpWhx9(?D>Dz(|1(+naeDmgX vC{robY^3>Gb9T^3ei;r0pNH4L5u58+j^$W>x#j-_00960?8?e*00IjDj8@Cc diff --git a/hack/openshift-patch/metadata-patch/Chart.yaml b/hack/openshift-patch/metadata-patch/Chart.yaml deleted file mode 100644 index 751e47ad5..000000000 --- a/hack/openshift-patch/metadata-patch/Chart.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: v2 -name: gpu-operator-charts -description: AMD GPU Operator simplifies the deployment and management of AMD Instinct GPU accelerators within Kubernetes clusters. -type: application -home: https://github.com/ROCm/gpu-operator -sources: - - https://github.com/ROCm/gpu-operator -icon: https://raw.githubusercontent.com/ROCm/k8s-device-plugin/master/helm/logo.png -maintainers: - - name: Yan Sun -keywords: - - kubernetes - - cluster - - hardware - - amd - - gpu - - ai - - deep learning - - monitoring - -kubeVersion: ">= 1.29.0-0" -version: v1.4.0 -appVersion: "v1.4.0" - -dependencies: -- name: nfd - version: v1.0.0 - repository: "file://./charts/nfd" - condition: nfd.enabled -- name: kmm - version: v1.0.0 - repository: "file://./charts/kmm" - condition: kmm.enabled \ No newline at end of file diff --git a/hack/openshift-patch/metadata-patch/values.yaml b/hack/openshift-patch/metadata-patch/values.yaml deleted file mode 100644 index 82f247da0..000000000 --- a/hack/openshift-patch/metadata-patch/values.yaml +++ /dev/null @@ -1,85 +0,0 @@ -nfd: - enabled: true # Set to false to disable nfd -kmm: - enabled: true # Set to false to disable kmm -installdefaultNFDRule: true # default NFD rule will detect amd gpu based on pci vendor ID -upgradeCRD: true # CRD will be patched as pre-upgrade hook when doing helm upgrade to current helm chart -controllerManager: - manager: - args: - - --config=controller_manager_config.yaml - containerSecurityContext: - allowPrivilegeEscalation: false - image: - repository: docker.io/rocm/gpu-operator - tag: dev - imagePullPolicy: Always - imagePullSecrets: "" - tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Equal" - value: "" - effect: "NoSchedule" - - key: "node-role.kubernetes.io/control-plane" - operator: "Equal" - value: "" - effect: "NoSchedule" - resources: - limits: - cpu: 1000m - memory: 1Gi - requests: - cpu: 100m - memory: 256Mi - nodeSelector: {} - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - replicas: 1 - serviceAccount: - annotations: {} - env: - simEnable: false -kmmDevicePlugin: - serviceAccount: - annotations: {} -kmmModuleLoader: - serviceAccount: - annotations: {} -kubernetesClusterDomain: cluster.local -managerConfig: - controllerManagerConfigYaml: |- - healthProbeBindAddress: :8081 - metricsBindAddress: 127.0.0.1:8080 - leaderElection: - enabled: true - resourceID: gpu.amd.com -metricsService: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: https - type: ClusterIP -nodeLabeller: - serviceAccount: - annotations: {} -metricsExporter: - serviceAccount: - annotations: {} -deviceConfig: - spec: - metricsExporter: - image: docker.io/rocm/device-metrics-exporter:latest - configManager: - image: docker.io/rocm/device-config-manager:latest - testRunner: - image: docker.io/rocm/test-runner:latest - commonConfig: - utilsContainer: - image: docker.io/rocm/gpu-operator-utils:latest diff --git a/hack/openshift-patch/openshift-kmm-patch/metadata-patch/Chart.yaml b/hack/openshift-patch/openshift-kmm-patch/metadata-patch/Chart.yaml deleted file mode 100644 index e85ede682..000000000 --- a/hack/openshift-patch/openshift-kmm-patch/metadata-patch/Chart.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v2 -name: kmm -description: A Helm chart for deploying Kernel Module Management for AMD GPU Operator -type: application - -kubeVersion: ">= 1.18.0-0" -version: v1.0.0 -appVersion: "v20240618-v2.1.1" \ No newline at end of file diff --git a/hack/openshift-patch/openshift-kmm-patch/metadata-patch/values.yaml b/hack/openshift-patch/openshift-kmm-patch/metadata-patch/values.yaml deleted file mode 100644 index 25ae6204c..000000000 --- a/hack/openshift-patch/openshift-kmm-patch/metadata-patch/values.yaml +++ /dev/null @@ -1,134 +0,0 @@ -controller: - manager: - args: - - --config=controller_config.yaml - containerSecurityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - env: - relatedImageMustGather: quay.io/edge-infrastructure/kernel-module-management-must-gather:release-2.1 - relatedImageSign: quay.io/edge-infrastructure/kernel-module-management-signimage:release-2.1 - relatedImageWorker: quay.io/edge-infrastructure/kernel-module-management-worker:release-2.1 - sslCertDir: /etc/pki/ca-trust/extracted/pem - image: - repository: quay.io/edge-infrastructure/kernel-module-management-operator - tag: release-2.1 - imagePullPolicy: Always - imagePullSecrets: "" - tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Equal" - value: "" - effect: "NoSchedule" - - key: "node-role.kubernetes.io/control-plane" - operator: "Equal" - value: "" - effect: "NoSchedule" - resources: - limits: - cpu: 500m - memory: 384Mi - requests: - cpu: 10m - memory: 64Mi - nodeSelector: {} - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - replicas: 1 - serviceAccount: - annotations: {} -controllerManager: - serviceAccount: - annotations: {} -controllerMetricsService: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: https - type: ClusterIP -kmmDevicePlugin: - serviceAccount: - annotations: {} -kubernetesClusterDomain: cluster.local -managerConfig: - controllerConfigYaml: |- - healthProbeBindAddress: :8081 - leaderElection: - enabled: true - resourceID: kmm.sigs.x-k8s.io - webhook: - disableHTTP2: true # CVE-2023-44487 - port: 9443 - metrics: - enableAuthnAuthz: true - disableHTTP2: true # CVE-2023-44487 - bindAddress: 0.0.0.0:8443 - secureServing: true - worker: - runAsUser: 0 - seLinuxType: spc_t - setFirmwareClassPath: /var/lib/firmware -kmmModuleLoader: - serviceAccount: - annotations: {} -nodeLabeller: - serviceAccount: - annotations: {} -webhookServer: - replicas: 1 - nodeSelector: {} - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - webhookServer: - args: - - --config=controller_config.yaml - - --enable-module - - --enable-namespace - - --enable-preflightvalidation - containerSecurityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - image: - repository: quay.io/edge-infrastructure/kernel-module-management-webhook-server - tag: latest - imagePullPolicy: Always - imagePullSecrets: "" - tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Equal" - value: "" - effect: "NoSchedule" - - key: "node-role.kubernetes.io/control-plane" - operator: "Equal" - value: "" - effect: "NoSchedule" - resources: - limits: - cpu: 500m - memory: 384Mi - requests: - cpu: 10m - memory: 64Mi -webhookService: - ports: - - port: 443 - protocol: TCP - targetPort: 9443 - type: ClusterIP diff --git a/hack/openshift-patch/openshift-kmm-patch/template-patch/controller-metrics-service.yaml b/hack/openshift-patch/openshift-kmm-patch/template-patch/controller-metrics-service.yaml deleted file mode 100644 index 4f17b470a..000000000 --- a/hack/openshift-patch/openshift-kmm-patch/template-patch/controller-metrics-service.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "kmm.fullname" . }}-controller-metrics-service - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: controller - {{- include "kmm.labels" . | nindent 4 }} -spec: - type: {{ .Values.controllerMetricsService.type }} - selector: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: controller - {{- include "kmm.selectorLabels" . | nindent 4 }} - ports: - {{- .Values.controllerMetricsService.ports | toYaml | nindent 2 }} diff --git a/hack/openshift-patch/openshift-kmm-patch/template-patch/deployment.yaml b/hack/openshift-patch/openshift-kmm-patch/template-patch/deployment.yaml deleted file mode 100644 index 4b8b2f7f6..000000000 --- a/hack/openshift-patch/openshift-kmm-patch/template-patch/deployment.yaml +++ /dev/null @@ -1,201 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "kmm.fullname" . }}-controller - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: controller - {{- include "kmm.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.controller.replicas }} - selector: - matchLabels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: controller - {{- include "kmm.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: controller - {{- include "kmm.selectorLabels" . | nindent 8 }} - annotations: - kubectl.kubernetes.io/default-container: manager - spec: - {{- with .Values.controller.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - nodeSelector: {{- toYaml .Values.controller.nodeSelector | nindent 8 }} - containers: - - args: {{- toYaml .Values.controller.manager.args | nindent 8 }} - command: - - /usr/local/bin/manager - env: - - name: RELATED_IMAGE_WORKER - value: {{ quote .Values.controller.manager.env.relatedImageWorker }} - - name: SSL_CERT_DIR - value: {{ quote .Values.controller.manager.env.sslCertDir }} - - name: OPERATOR_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: RELATED_IMAGE_MUST_GATHER - value: {{ quote .Values.controller.manager.env.relatedImageMustGather }} - - name: RELATED_IMAGE_SIGN - value: {{ quote .Values.controller.manager.env.relatedImageSign }} - - name: KUBERNETES_CLUSTER_DOMAIN - value: {{ quote .Values.kubernetesClusterDomain }} - image: {{ .Values.controller.manager.image.repository }}:{{ .Values.controller.manager.image.tag - | default .Chart.AppVersion }} - imagePullPolicy: {{ .Values.controller.manager.imagePullPolicy }} - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - ports: - - containerPort: 8443 - name: metrics - protocol: TCP - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: {{- toYaml .Values.controller.manager.resources | nindent 10 }} - securityContext: {{- toYaml .Values.controller.manager.containerSecurityContext - | nindent 10 }} - volumeMounts: - - mountPath: /etc/pki/ca-trust/extracted/pem - name: trusted-ca - readOnly: true - - mountPath: /controller_config.yaml - name: manager-config - subPath: controller_config.yaml - {{- if .Values.controller.manager.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.controller.manager.imagePullSecrets }} - {{- end}} - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - serviceAccountName: {{ include "kmm.fullname" . }}-controller - terminationGracePeriodSeconds: 10 - {{- with .Values.controller.manager.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - volumes: - - name: trusted-ca - projected: - sources: - - configMap: - items: - - key: ca-bundle.crt - path: tls-ca-bundle.pem - name: {{ include "kmm.fullname" . }}-cluster-ca - - configMap: - items: - - key: service-ca.crt - path: ocp-service-ca-bundle.pem - name: {{ include "kmm.fullname" . }}-service-ca - - configMap: - name: {{ include "kmm.fullname" . }}-manager-config - name: manager-config ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "kmm.fullname" . }}-webhook-server - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: webhook-server - {{- include "kmm.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.webhookServer.replicas }} - selector: - matchLabels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: webhook-server - {{- include "kmm.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: webhook-server - {{- include "kmm.selectorLabels" . | nindent 8 }} - annotations: - kubectl.kubernetes.io/default-container: webhook-server - spec: - {{- with .Values.webhookServer.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - nodeSelector: {{- toYaml .Values.webhookServer.nodeSelector | nindent 8 }} - containers: - - args: {{- toYaml .Values.webhookServer.webhookServer.args | nindent 8 }} - env: - - name: KUBERNETES_CLUSTER_DOMAIN - value: {{ quote .Values.kubernetesClusterDomain }} - image: {{ .Values.webhookServer.webhookServer.image.repository }}:{{ .Values.webhookServer.webhookServer.image.tag - | default .Chart.AppVersion }} - imagePullPolicy: {{ .Values.webhookServer.webhookServer.imagePullPolicy }} - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: webhook-server - ports: - - containerPort: 9443 - name: webhook-server - protocol: TCP - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: {{- toYaml .Values.webhookServer.webhookServer.resources | nindent 10 - }} - securityContext: {{- toYaml .Values.webhookServer.webhookServer.containerSecurityContext - | nindent 10 }} - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true - - mountPath: /controller_config.yaml - name: manager-config - subPath: controller_config.yaml - {{- if .Values.webhookServer.webhookServer.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.webhookServer.webhookServer.imagePullSecrets }} - {{- end}} - securityContext: - runAsNonRoot: true - serviceAccountName: {{ include "kmm.fullname" . }}-controller - terminationGracePeriodSeconds: 10 - {{- with .Values.webhookServer.webhookServer.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - volumes: - - name: cert - secret: - defaultMode: 420 - secretName: {{ include "kmm.fullname" . }}-webhook-server-cert - - configMap: - name: {{ include "kmm.fullname" . }}-manager-config - name: manager-config diff --git a/hack/openshift-patch/openshift-kmm-patch/template-patch/serviceaccount.yaml b/hack/openshift-patch/openshift-kmm-patch/template-patch/serviceaccount.yaml deleted file mode 100644 index fb58d569c..000000000 --- a/hack/openshift-patch/openshift-kmm-patch/template-patch/serviceaccount.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "kmm.fullname" . }}-controller - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - {{- toYaml .Values.controllerManager.serviceAccount.annotations | nindent 4 }} diff --git a/hack/openshift-patch/openshift-kmm-patch/template-patch/validating-webhook-configuration.yaml b/hack/openshift-patch/openshift-kmm-patch/template-patch/validating-webhook-configuration.yaml deleted file mode 100644 index 33267a878..000000000 --- a/hack/openshift-patch/openshift-kmm-patch/template-patch/validating-webhook-configuration.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: {{ include "kmm.fullname" . }}-validating-webhook-configuration - annotations: - service.beta.openshift.io/inject-cabundle: 'true' - labels: - {{- include "kmm.labels" . | nindent 4 }} -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: '{{ include "kmm.fullname" . }}-webhook-service' - namespace: '{{ .Release.Namespace }}' - path: /validate--v1-namespace - failurePolicy: Fail - name: namespace-deletion.kmm.sigs.k8s.io - namespaceSelector: - matchLabels: - kmm.node.k8s.io/contains-modules: "" - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - DELETE - resources: - - namespaces - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: '{{ include "kmm.fullname" . }}-webhook-service' - namespace: '{{ .Release.Namespace }}' - path: /validate-kmm-sigs-x-k8s-io-v1beta1-module - failurePolicy: Fail - name: vmodule.kb.io - rules: - - apiGroups: - - kmm.sigs.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - modules - sideEffects: None \ No newline at end of file diff --git a/hack/openshift-patch/openshift-kmm-patch/template-patch/webhook-service.yaml b/hack/openshift-patch/openshift-kmm-patch/template-patch/webhook-service.yaml deleted file mode 100644 index 30d8a4857..000000000 --- a/hack/openshift-patch/openshift-kmm-patch/template-patch/webhook-service.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "kmm.fullname" . }}-webhook-service - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/created-by: kernel-module-management - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} - annotations: - service.beta.openshift.io/serving-cert-secret-name: {{ include "kmm.fullname" . }}-webhook-server-cert -spec: - type: {{ .Values.webhookService.type }} - selector: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: webhook-server - {{- include "kmm.selectorLabels" . | nindent 4 }} - ports: - {{- .Values.webhookService.ports | toYaml | nindent 2 }} diff --git a/hack/openshift-patch/openshift-nfd-patch/crds/nodefeature-crd.yaml b/hack/openshift-patch/openshift-nfd-patch/crds/nodefeature-crd.yaml deleted file mode 100644 index 42496a70f..000000000 --- a/hack/openshift-patch/openshift-nfd-patch/crds/nodefeature-crd.yaml +++ /dev/null @@ -1,128 +0,0 @@ ---- -# Source: nfd/templates/nodefeature-crd.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: nodefeatures.nfd.openshift.io - annotations: - controller-gen.kubebuilder.io/version: v0.14.0 - labels: - helm.sh/chart: nfd-v1.0.0 - app.kubernetes.io/name: nfd - app.kubernetes.io/instance: amd-gpu - app.kubernetes.io/version: "v4.16" - app.kubernetes.io/managed-by: Helm -spec: - group: nfd.openshift.io - names: - kind: NodeFeature - listKind: NodeFeatureList - plural: nodefeatures - singular: nodefeature - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: |- - NodeFeature resource holds the features discovered for one node in the - cluster. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: NodeFeatureSpec describes a NodeFeature object. - properties: - features: - description: Features is the full "raw" features data that has been - discovered. - properties: - attributes: - additionalProperties: - description: AttributeFeatureSet is a set of features having string - value. - properties: - elements: - additionalProperties: - type: string - type: object - required: - - elements - type: object - type: object - flags: - additionalProperties: - description: FlagFeatureSet is a set of simple features only containing - names without values. - properties: - elements: - additionalProperties: - description: Nil is a dummy empty struct for protobuf compatibility - type: object - type: object - required: - - elements - type: object - type: object - instances: - additionalProperties: - description: InstanceFeatureSet is a set of features each of which - is an instance having multiple attributes. - properties: - elements: - items: - description: InstanceFeature represents one instance of - a complex features, e.g. a device. - properties: - attributes: - additionalProperties: - type: string - type: object - required: - - attributes - type: object - type: array - required: - - elements - type: object - type: object - required: - - attributes - - flags - - instances - type: object - labels: - additionalProperties: - type: string - description: Labels is the set of node labels that are requested to - be created. - type: object - required: - - features - type: object - required: - - spec - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/hack/openshift-patch/openshift-nfd-patch/crds/nodefeaturerule-crd.yaml b/hack/openshift-patch/openshift-nfd-patch/crds/nodefeaturerule-crd.yaml deleted file mode 100644 index e8de37783..000000000 --- a/hack/openshift-patch/openshift-nfd-patch/crds/nodefeaturerule-crd.yaml +++ /dev/null @@ -1,330 +0,0 @@ ---- -# Source: nfd/templates/nodefeaturerule-crd.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: nodefeaturerules.nfd.openshift.io - annotations: - controller-gen.kubebuilder.io/version: v0.14.0 - labels: - helm.sh/chart: nfd-v1.0.0 - app.kubernetes.io/name: nfd - app.kubernetes.io/instance: amd-gpu - app.kubernetes.io/version: "v4.16" - app.kubernetes.io/managed-by: Helm -spec: - group: nfd.openshift.io - names: - kind: NodeFeatureRule - listKind: NodeFeatureRuleList - plural: nodefeaturerules - shortNames: - - nfr - singular: nodefeaturerule - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: |- - NodeFeatureRule resource specifies a configuration for feature-based - customization of node objects, such as node labeling. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: NodeFeatureRuleSpec describes a NodeFeatureRule. - properties: - rules: - description: Rules is a list of node customization rules. - items: - description: Rule defines a rule for node customization such as labeling. - properties: - annotations: - additionalProperties: - type: string - description: Annotations to create if the rule matches. - type: object - extendedResources: - additionalProperties: - type: string - description: ExtendedResources to create if the rule matches. - type: object - labels: - additionalProperties: - type: string - description: Labels to create if the rule matches. - type: object - labelsTemplate: - description: |- - LabelsTemplate specifies a template to expand for dynamically generating - multiple labels. Data (after template expansion) must be keys with an - optional value ([=]) separated by newlines. - type: string - matchAny: - description: MatchAny specifies a list of matchers one of which - must match. - items: - description: MatchAnyElem specifies one sub-matcher of MatchAny. - properties: - matchFeatures: - description: MatchFeatures specifies a set of matcher terms - all of which must match. - items: - description: |- - FeatureMatcherTerm defines requirements against one feature set. All - requirements (specified as MatchExpressions) are evaluated against each - element in the feature set. - properties: - feature: - description: Feature is the name of the feature set - to match against. - type: string - matchExpressions: - additionalProperties: - description: |- - MatchExpression specifies an expression to evaluate against a set of input - values. It contains an operator that is applied when matching the input and - an array of values that the operator evaluates the input against. - properties: - op: - description: Op is the operator to be applied. - enum: - - In - - NotIn - - InRegexp - - Exists - - DoesNotExist - - Gt - - Lt - - GtLt - - IsTrue - - IsFalse - type: string - value: - description: |- - Value is the list of values that the operand evaluates the input - against. Value should be empty if the operator is Exists, DoesNotExist, - IsTrue or IsFalse. Value should contain exactly one element if the - operator is Gt or Lt and exactly two elements if the operator is GtLt. - In other cases Value should contain at least one element. - items: - type: string - type: array - required: - - op - type: object - description: |- - MatchExpressions is the set of per-element expressions evaluated. These - match against the value of the specified elements. - type: object - matchName: - description: |- - MatchName in an expression that is matched against the name of each - element in the feature set. - properties: - op: - description: Op is the operator to be applied. - enum: - - In - - NotIn - - InRegexp - - Exists - - DoesNotExist - - Gt - - Lt - - GtLt - - IsTrue - - IsFalse - type: string - value: - description: |- - Value is the list of values that the operand evaluates the input - against. Value should be empty if the operator is Exists, DoesNotExist, - IsTrue or IsFalse. Value should contain exactly one element if the - operator is Gt or Lt and exactly two elements if the operator is GtLt. - In other cases Value should contain at least one element. - items: - type: string - type: array - required: - - op - type: object - required: - - feature - type: object - type: array - required: - - matchFeatures - type: object - type: array - matchFeatures: - description: MatchFeatures specifies a set of matcher terms all - of which must match. - items: - description: |- - FeatureMatcherTerm defines requirements against one feature set. All - requirements (specified as MatchExpressions) are evaluated against each - element in the feature set. - properties: - feature: - description: Feature is the name of the feature set to match - against. - type: string - matchExpressions: - additionalProperties: - description: |- - MatchExpression specifies an expression to evaluate against a set of input - values. It contains an operator that is applied when matching the input and - an array of values that the operator evaluates the input against. - properties: - op: - description: Op is the operator to be applied. - enum: - - In - - NotIn - - InRegexp - - Exists - - DoesNotExist - - Gt - - Lt - - GtLt - - IsTrue - - IsFalse - type: string - value: - description: |- - Value is the list of values that the operand evaluates the input - against. Value should be empty if the operator is Exists, DoesNotExist, - IsTrue or IsFalse. Value should contain exactly one element if the - operator is Gt or Lt and exactly two elements if the operator is GtLt. - In other cases Value should contain at least one element. - items: - type: string - type: array - required: - - op - type: object - description: |- - MatchExpressions is the set of per-element expressions evaluated. These - match against the value of the specified elements. - type: object - matchName: - description: |- - MatchName in an expression that is matched against the name of each - element in the feature set. - properties: - op: - description: Op is the operator to be applied. - enum: - - In - - NotIn - - InRegexp - - Exists - - DoesNotExist - - Gt - - Lt - - GtLt - - IsTrue - - IsFalse - type: string - value: - description: |- - Value is the list of values that the operand evaluates the input - against. Value should be empty if the operator is Exists, DoesNotExist, - IsTrue or IsFalse. Value should contain exactly one element if the - operator is Gt or Lt and exactly two elements if the operator is GtLt. - In other cases Value should contain at least one element. - items: - type: string - type: array - required: - - op - type: object - required: - - feature - type: object - type: array - name: - description: Name of the rule. - type: string - taints: - description: Taints to create if the rule matches. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Required. The taint key to be applied to a - node. - type: string - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint - key. - type: string - required: - - effect - - key - type: object - type: array - vars: - additionalProperties: - type: string - description: |- - Vars is the variables to store if the rule matches. Variables do not - directly inflict any changes in the node object. However, they can be - referenced from other rules enabling more complex rule hierarchies, - without exposing intermediary output values as labels. - type: object - varsTemplate: - description: |- - VarsTemplate specifies a template to expand for dynamically generating - multiple variables. Data (after template expansion) must be keys with an - optional value ([=]) separated by newlines. - type: string - required: - - name - type: object - type: array - required: - - rules - type: object - required: - - spec - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/hack/openshift-patch/openshift-nfd-patch/metadata-patch/Chart.yaml b/hack/openshift-patch/openshift-nfd-patch/metadata-patch/Chart.yaml deleted file mode 100644 index b185f251d..000000000 --- a/hack/openshift-patch/openshift-nfd-patch/metadata-patch/Chart.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v2 -name: nfd -description: A Helm chart for deploying Cluster NFD Operator Kubernetes -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: v1.0.0 -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "v4.16" \ No newline at end of file diff --git a/hack/openshift-patch/template-patch/config-manager-rbac.yaml b/hack/openshift-patch/template-patch/config-manager-rbac.yaml deleted file mode 100644 index 0f6fc3515..000000000 --- a/hack/openshift-patch/template-patch/config-manager-rbac.yaml +++ /dev/null @@ -1,75 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-config-manager - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - list - - get - - update - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch - - update -- apiGroups: - - "apps" - resources: - - daemonsets - verbs: - - get - - list - - watch - - delete - - create - - update -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch - - delete - - create - - update -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-config-manager - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-openshift.fullname" . }}-config-manager' -subjects: -- kind: ServiceAccount - name: amd-gpu-operator-config-manager - namespace: '{{ .Release.Namespace }}' \ No newline at end of file diff --git a/hack/openshift-patch/template-patch/deployment.yaml b/hack/openshift-patch/template-patch/deployment.yaml deleted file mode 100644 index 69f2e1370..000000000 --- a/hack/openshift-patch/template-patch/deployment.yaml +++ /dev/null @@ -1,83 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-controller-manager - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - control-plane: controller-manager - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.controllerManager.replicas }} - selector: - matchLabels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - control-plane: controller-manager - {{- include "helm-charts-openshift.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - control-plane: controller-manager - {{- include "helm-charts-openshift.selectorLabels" . | nindent 8 }} - annotations: - kubectl.kubernetes.io/default-container: manager - spec: - {{- with .Values.controllerManager.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - nodeSelector: {{- toYaml .Values.controllerManager.nodeSelector | nindent 8 }} - containers: - - args: {{- toYaml .Values.controllerManager.manager.args | nindent 8 }} - env: - - name: OPERATOR_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: KUBERNETES_CLUSTER_DOMAIN - value: {{ quote .Values.kubernetesClusterDomain }} - - name: SIM_ENABLE - value: {{ quote .Values.controllerManager.env.simEnable }} - image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag - | default .Chart.AppVersion }} - imagePullPolicy: {{ .Values.controllerManager.manager.imagePullPolicy }} - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: {{- toYaml .Values.controllerManager.manager.resources | nindent 10 - }} - securityContext: {{- toYaml .Values.controllerManager.manager.containerSecurityContext - | nindent 10 }} - volumeMounts: - - mountPath: /controller_manager_config.yaml - name: manager-config - subPath: controller_manager_config.yaml - {{- if .Values.controllerManager.manager.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.controllerManager.manager.imagePullSecrets }} - {{- end}} - securityContext: - runAsNonRoot: true - serviceAccountName: {{ include "helm-charts-openshift.fullname" . }}-controller-manager - terminationGracePeriodSeconds: 10 - {{- with .Values.controllerManager.manager.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - volumes: - - configMap: - name: {{ include "helm-charts-openshift.fullname" . }}-manager-config - name: manager-config diff --git a/hack/openshift-patch/template-patch/kmm-device-plugin-rbac.yaml b/hack/openshift-patch/template-patch/kmm-device-plugin-rbac.yaml deleted file mode 100644 index e81a5133a..000000000 --- a/hack/openshift-patch/template-patch/kmm-device-plugin-rbac.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-kmm-device-plugin - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -rules: -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-kmm-device-plugin - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-openshift.fullname" . }}-kmm-device-plugin' -subjects: -- kind: ServiceAccount - name: amd-gpu-operator-kmm-device-plugin - namespace: '{{ .Release.Namespace }}' diff --git a/hack/openshift-patch/template-patch/kmm-module-loader-rbac.yaml b/hack/openshift-patch/template-patch/kmm-module-loader-rbac.yaml deleted file mode 100644 index b997dd567..000000000 --- a/hack/openshift-patch/template-patch/kmm-module-loader-rbac.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-kmm-module-loader - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -rules: -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-kmm-module-loader - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-openshift.fullname" . }}-kmm-module-loader' -subjects: -- kind: ServiceAccount - name: amd-gpu-operator-kmm-module-loader - namespace: '{{ .Release.Namespace }}' diff --git a/hack/openshift-patch/template-patch/metrics-exporter-rbac-proxy-rbac.yaml b/hack/openshift-patch/template-patch/metrics-exporter-rbac-proxy-rbac.yaml deleted file mode 100644 index 3e518c0c5..000000000 --- a/hack/openshift-patch/template-patch/metrics-exporter-rbac-proxy-rbac.yaml +++ /dev/null @@ -1,63 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-metrics-exporter-rbac-proxy - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -rules: -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - watch - - get - - list - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-metrics-exporter-rbac-proxy - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-openshift.fullname" . }}-metrics-exporter-rbac-proxy' -subjects: -- kind: ServiceAccount - name: amd-gpu-operator-metrics-exporter-rbac-proxy - namespace: '{{ .Release.Namespace }}' diff --git a/hack/openshift-patch/template-patch/metrics-exporter-rbac.yaml b/hack/openshift-patch/template-patch/metrics-exporter-rbac.yaml deleted file mode 100644 index cb94fd753..000000000 --- a/hack/openshift-patch/template-patch/metrics-exporter-rbac.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-metrics-exporter - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - watch - - get - - list - - update -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-metrics-exporter - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-openshift.fullname" . }}-metrics-exporter' -subjects: -- kind: ServiceAccount - name: amd-gpu-operator-metrics-exporter - namespace: '{{ .Release.Namespace }}' diff --git a/hack/openshift-patch/template-patch/node-labeller-rbac.yaml b/hack/openshift-patch/template-patch/node-labeller-rbac.yaml deleted file mode 100644 index bc7fd4272..000000000 --- a/hack/openshift-patch/template-patch/node-labeller-rbac.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-node-labeller - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - watch - - get - - list - - update -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-node-labeller - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "helm-charts-openshift.fullname" . }}-node-labeller -subjects: -- kind: ServiceAccount - name: amd-gpu-operator-node-labeller - namespace: '{{ .Release.Namespace }}' diff --git a/hack/openshift-patch/template-patch/nodefeaturediscovery.yaml b/hack/openshift-patch/template-patch/nodefeaturediscovery.yaml deleted file mode 100644 index b7a7c5291..000000000 --- a/hack/openshift-patch/template-patch/nodefeaturediscovery.yaml +++ /dev/null @@ -1,124 +0,0 @@ -{{- if .Values.nfd.enabled }} -apiVersion: nfd.openshift.io/v1 -kind: NodeFeatureDiscovery -metadata: - name: {{ .Release.Name }}-nfd-instance - namespace: {{ .Release.Namespace }} -spec: - #instance: "" # instance is empty by default - #labelWhiteList: "" - #extraLabelNs: - # - "example.com" - #resourceLabels: - # - "example.com/resource" - operand: - image: quay.io/openshift/origin-node-feature-discovery:4.16 - imagePullPolicy: IfNotPresent - servicePort: 12000 - workerConfig: - configData: | - core: - # labelWhiteList: - # noPublish: false - sleepInterval: 60s - # sources: [all] - # klog: - # addDirHeader: false - # alsologtostderr: false - # logBacktraceAt: - # logtostderr: true - # skipHeaders: false - # stderrthreshold: 2 - # v: 0 - # vmodule: - ## NOTE: the following options are not dynamically run-time - ## configurable and require a nfd-worker restart to take effect - ## after being changed - # logDir: - # logFile: - # logFileMaxSize: 1800 - # skipLogHeaders: false - sources: - # cpu: - # cpuid: - ## NOTE: whitelist has priority over blacklist - # attributeBlacklist: - # - "BMI1" - # - "BMI2" - # - "CLMUL" - # - "CMOV" - # - "CX16" - # - "ERMS" - # - "F16C" - # - "HTT" - # - "LZCNT" - # - "MMX" - # - "MMXEXT" - # - "NX" - # - "POPCNT" - # - "RDRAND" - # - "RDSEED" - # - "RDTSCP" - # - "SGX" - # - "SSE" - # - "SSE2" - # - "SSE3" - # - "SSE4.1" - # - "SSE4.2" - # - "SSSE3" - # attributeWhitelist: - # kernel: - # kconfigFile: "/path/to/kconfig" - # configOpts: - # - "NO_HZ" - # - "X86" - # - "DMI" - pci: - deviceClassWhitelist: - - "0200" - - "03" - - "12" - deviceLabelFields: - - "vendor" - - "device" - custom: - - name: amd-gpu - labels: - feature.node.kubernetes.io/amd-gpu: "true" - matchAny: - - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["1002"]} - device: {op: In, value: ["74a0"]} # MI300A - - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["1002"]} - device: {op: In, value: ["74a1"]} # MI300X - - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["1002"]} - device: {op: In, value: ["740f"]} # MI210 - - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["1002"]} - device: {op: In, value: ["7408"]} # MI250X - - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["1002"]} - device: {op: In, value: ["740c"]} # MI250/MI250X - - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["1002"]} - device: {op: In, value: ["738c"]} # MI100 - - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["1002"]} - device: {op: In, value: ["738e"]} # MI100 -{{- end }} \ No newline at end of file diff --git a/hack/openshift-patch/template-patch/post-delete-hook.yaml b/hack/openshift-patch/template-patch/post-delete-hook.yaml deleted file mode 100644 index 553ec78f0..000000000 --- a/hack/openshift-patch/template-patch/post-delete-hook.yaml +++ /dev/null @@ -1,118 +0,0 @@ -# Run helm uninstall with --no-hooks to bypass the post-delete hook -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-prune - namespace: {{ .Release.Namespace }} - labels: - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "0" - "helm.sh/hook": post-delete - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-prune - labels: - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "0" - "helm.sh/hook": post-delete - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded -rules: - - apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - delete - - get - - list ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-prune - labels: - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "1" - "helm.sh/hook": post-delete - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "helm-charts-openshift.fullname" . }}-prune -subjects: -- kind: ServiceAccount - name: {{ include "helm-charts-openshift.fullname" . }}-prune - namespace: {{ .Release.Namespace }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: delete-custom-resource-definitions - namespace: {{ .Release.Namespace }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "2" - # hook will be executed before helm uninstall - "helm.sh/hook": post-delete - # remove the resource created by the hook whether it succeeded or failed - "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded -spec: - backoffLimit: 0 # once the job finished first run, don't retry to create another pod - ttlSecondsAfterFinished: 60 # job info will be kept for 1 min then deleted - template: - spec: - serviceAccountName: {{ include "helm-charts-openshift.fullname" . }}-prune - containers: - - name: delete-custom-resource-definitions - image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag }} - command: - - /bin/sh - - -c - - | - if kubectl get crds deviceconfigs.amd.com > /dev/null 2>&1; then - kubectl delete crds deviceconfigs.amd.com - fi - {{- if .Values.nfd.enabled }} - if kubectl get crds nodefeatures.nfd.openshift.io > /dev/null 2>&1; then - kubectl delete crds nodefeatures.nfd.openshift.io - fi - if kubectl get crds nodefeaturediscoveries.nfd.openshift.io > /dev/null 2>&1; then - kubectl delete crds nodefeaturediscoveries.nfd.openshift.io - fi - if kubectl get crds nodefeaturerules.nfd.openshift.io > /dev/null 2>&1; then - kubectl delete crds nodefeaturerules.nfd.openshift.io - fi - if kubectl get crds noderesourcetopologies.topology.node.k8s.io > /dev/null 2>&1; then - kubectl delete crds noderesourcetopologies.topology.node.k8s.io - fi - {{- end }} - {{- if .Values.kmm.enabled }} - if kubectl get crds modules.kmm.sigs.x-k8s.io > /dev/null 2>&1; then - kubectl delete crds modules.kmm.sigs.x-k8s.io - fi - if kubectl get crds nodemodulesconfigs.kmm.sigs.x-k8s.io > /dev/null 2>&1; then - kubectl delete crds nodemodulesconfigs.kmm.sigs.x-k8s.io - fi - {{- end }} - {{- if .Values.controllerManager.manager.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.controllerManager.manager.imagePullSecrets }} - {{- end }} - {{- with .Values.controllerManager.manager.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.controllerManager.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - restartPolicy: Never diff --git a/hack/openshift-patch/template-patch/pre-delete-hook.yaml b/hack/openshift-patch/template-patch/pre-delete-hook.yaml deleted file mode 100644 index eddda8139..000000000 --- a/hack/openshift-patch/template-patch/pre-delete-hook.yaml +++ /dev/null @@ -1,146 +0,0 @@ -# Run helm uninstall with --no-hooks to bypass the pre-delete hook -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-pre-delete - namespace: {{ .Release.Namespace }} - labels: - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "0" - "helm.sh/hook": pre-delete - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-pre-delete - labels: - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "0" - "helm.sh/hook": pre-delete - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded -rules: - - apiGroups: - - amd.com - resources: - - deviceconfigs - verbs: - - get - - list - - apiGroups: - - nfd.openshift.io - resources: - - nodefeaturediscoveries - verbs: - - delete ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-pre-delete - labels: - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "1" - "helm.sh/hook": pre-delete - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "helm-charts-openshift.fullname" . }}-pre-delete -subjects: -- kind: ServiceAccount - name: {{ include "helm-charts-openshift.fullname" . }}-pre-delete - namespace: {{ .Release.Namespace }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: check-leftover-deviceconfigs - namespace: {{ .Release.Namespace }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "2" - # hook will be executed before helm uninstall - "helm.sh/hook": pre-delete - # remove the resource created by the hook whether it succeeded or failed - "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded -spec: - backoffLimit: 0 # once the job finished first run, don't retry to create another pod - ttlSecondsAfterFinished: 60 # job info will be kept for 1 min then deleted - template: - spec: - serviceAccountName: {{ include "helm-charts-openshift.fullname" . }}-pre-delete - containers: - - name: check-leftover-deviceconfigs - image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag }} - command: - - /bin/sh - - -c - - | - if kubectl get deviceconfigs -n {{ .Release.Namespace }} --no-headers | grep -q .; then - echo "DeviceConfigs resources exist. Stop uninstallation." - exit 1 - else - echo "No DeviceConfigs resources found. Proceeding with uninstallation." - exit 0 - fi - {{- if .Values.controllerManager.manager.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.controllerManager.manager.imagePullSecrets }} - {{- end}} - {{- with .Values.controllerManager.manager.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.controllerManager.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - restartPolicy: Never - ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: pre-uninstall-remove-nodefeaturediscovery - namespace: {{ .Release.Namespace }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "3" - # hook will be executed before helm uninstall - "helm.sh/hook": pre-delete - # remove the resource created by the hook whether it succeeded or failed - "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded -spec: - backoffLimit: 0 # once the job finished first run, don't retry to create another pod - ttlSecondsAfterFinished: 60 # job info will be kept for 1 min then deleted - template: - spec: - serviceAccountName: {{ include "helm-charts-openshift.fullname" . }}-pre-delete - containers: - - name: pre-uninstall-remove-nodefeaturediscovery - image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag }} - command: - - /bin/sh - - -c - - | - kubectl delete nodefeaturediscoveries --all -n {{ .Release.Namespace }} - {{- if .Values.controllerManager.manager.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.controllerManager.manager.imagePullSecrets }} - {{- end}} - {{- with .Values.controllerManager.manager.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.controllerManager.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - restartPolicy: Never diff --git a/hack/openshift-patch/template-patch/pre-upgrade-hook.yaml b/hack/openshift-patch/template-patch/pre-upgrade-hook.yaml deleted file mode 100644 index 183571749..000000000 --- a/hack/openshift-patch/template-patch/pre-upgrade-hook.yaml +++ /dev/null @@ -1,110 +0,0 @@ -{{- if .Values.upgradeCRD }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: upgrade-crd-hook-sa - annotations: - # hook will be executed before helm upgrade - "helm.sh/hook": pre-upgrade,pre-rollback - # don't cleanup the job on hook failure - "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "1" ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: upgrade-crd-hook-cluster-role - annotations: - # hook will be executed before helm upgrade - "helm.sh/hook": pre-upgrade,pre-rollback - # don't cleanup the job on hook failure - "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "1" -rules: - - apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - create - - get - - list - - watch - - patch - - update ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: upgrade-crd-hook-cluster-role-binding - annotations: - # hook will be executed before helm upgrade - "helm.sh/hook": pre-upgrade,pre-rollback - # don't cleanup the job on hook failure - "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "2" -subjects: - - kind: ServiceAccount - name: upgrade-crd-hook-sa - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: upgrade-crd-hook-cluster-role - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: upgrade-crd - namespace: {{ .Release.Namespace }} - labels: - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - # hook will be executed before helm upgrade - "helm.sh/hook": pre-upgrade,pre-rollback - # don't cleanup the job on hook failure - "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "3" -spec: - template: - metadata: - name: upgrade-crd - spec: - serviceAccountName: upgrade-crd-hook-sa - {{- if .Values.controllerManager.manager.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.controllerManager.manager.imagePullSecrets }} - {{- end }} - {{- with .Values.controllerManager.manager.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.controllerManager.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: upgrade-crd - image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag }} - imagePullPolicy: {{ .Values.controllerManager.manager.imagePullPolicy }} - command: - - /bin/sh - - -c - - | - kubectl apply -f /opt/helm-charts-crds-openshift/deviceconfig-crd.yaml - {{- if .Values.nfd.enabled }} - kubectl apply -f /opt/helm-charts-crds-openshift/nodefeature-crd.yaml - kubectl apply -f /opt/helm-charts-crds-openshift/nodefeaturediscovery-crd.yaml - kubectl apply -f /opt/helm-charts-crds-openshift/nodefeaturerule-crd.yaml - {{- end }} - {{- if .Values.kmm.enabled }} - kubectl apply -f /opt/helm-charts-crds-openshift/module-crd.yaml - kubectl apply -f /opt/helm-charts-crds-openshift/nodemodulesconfig-crd.yaml - {{- end }} - restartPolicy: OnFailure -{{- end }} -# Run helm upgrade with --no-hooks to bypass the pre-upgrade hook \ No newline at end of file diff --git a/hack/openshift-patch/template-patch/prometheus-k8s-rbac.yaml b/hack/openshift-patch/template-patch/prometheus-k8s-rbac.yaml deleted file mode 100644 index 74e6bf95e..000000000 --- a/hack/openshift-patch/template-patch/prometheus-k8s-rbac.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-prometheus-k8s - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - services - - endpoints - - pods - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-prometheus-k8s - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: '{{ include "helm-charts-openshift.fullname" . }}-prometheus-k8s' -subjects: -- kind: ServiceAccount - name: prometheus-k8s - namespace: openshift-monitoring \ No newline at end of file diff --git a/hack/openshift-patch/template-patch/serviceaccount.yaml b/hack/openshift-patch/template-patch/serviceaccount.yaml deleted file mode 100644 index 6ffc7d019..000000000 --- a/hack/openshift-patch/template-patch/serviceaccount.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-controller-manager - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - {{- toYaml .Values.controllerManager.serviceAccount.annotations | nindent 4 }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: amd-gpu-operator-kmm-device-plugin - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - {{- toYaml .Values.kmmDevicePlugin.serviceAccount.annotations | nindent 4 }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: amd-gpu-operator-kmm-module-loader - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - {{- toYaml .Values.kmmModuleLoader.serviceAccount.annotations | nindent 4 }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - # the node labeller service account name should be fixed, not templated - name: amd-gpu-operator-node-labeller - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - {{- toYaml .Values.nodeLabeller.serviceAccount.annotations | nindent 4 }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: amd-gpu-operator-metrics-exporter - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - annotations: - {{- toYaml .Values.metricsExporter.serviceAccount.annotations | nindent 4 }} diff --git a/hack/openshift-patch/template-patch/test-runner-rbac.yaml b/hack/openshift-patch/template-patch/test-runner-rbac.yaml deleted file mode 100644 index 880395387..000000000 --- a/hack/openshift-patch/template-patch/test-runner-rbac.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-test-runner - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - list - - get - - update - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - patch -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-test-runner - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-openshift.fullname" . }}-test-runner' -subjects: -- kind: ServiceAccount - name: amd-gpu-operator-test-runner - namespace: '{{ .Release.Namespace }}' diff --git a/hack/openshift-patch/template-patch/utils-container-rbac.yaml b/hack/openshift-patch/template-patch/utils-container-rbac.yaml deleted file mode 100644 index f4bccdb80..000000000 --- a/hack/openshift-patch/template-patch/utils-container-rbac.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-utils-container - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -rules: -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-utils-container - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-openshift.fullname" . }}-utils-container' -subjects: -- kind: ServiceAccount - name: amd-gpu-operator-utils-container - namespace: '{{ .Release.Namespace }}' diff --git a/helm-charts-openshift/.helmignore b/helm-charts-openshift/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/helm-charts-openshift/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/helm-charts-openshift/Chart.lock b/helm-charts-openshift/Chart.lock deleted file mode 100644 index 2c110d8fc..000000000 --- a/helm-charts-openshift/Chart.lock +++ /dev/null @@ -1,9 +0,0 @@ -dependencies: -- name: nfd - repository: file://./charts/nfd - version: v1.0.0 -- name: kmm - repository: file://./charts/kmm - version: v1.0.0 -digest: sha256:25200c34a5cc846a1275e5bf3fc637b19e909dc68de938189c5278d77d03f5ac -generated: "2026-01-28T11:30:38.160988877Z" diff --git a/helm-charts-openshift/Chart.yaml b/helm-charts-openshift/Chart.yaml deleted file mode 100644 index 751e47ad5..000000000 --- a/helm-charts-openshift/Chart.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: v2 -name: gpu-operator-charts -description: AMD GPU Operator simplifies the deployment and management of AMD Instinct GPU accelerators within Kubernetes clusters. -type: application -home: https://github.com/ROCm/gpu-operator -sources: - - https://github.com/ROCm/gpu-operator -icon: https://raw.githubusercontent.com/ROCm/k8s-device-plugin/master/helm/logo.png -maintainers: - - name: Yan Sun -keywords: - - kubernetes - - cluster - - hardware - - amd - - gpu - - ai - - deep learning - - monitoring - -kubeVersion: ">= 1.29.0-0" -version: v1.4.0 -appVersion: "v1.4.0" - -dependencies: -- name: nfd - version: v1.0.0 - repository: "file://./charts/nfd" - condition: nfd.enabled -- name: kmm - version: v1.0.0 - repository: "file://./charts/kmm" - condition: kmm.enabled \ No newline at end of file diff --git a/helm-charts-openshift/charts/kmm/.helmignore b/helm-charts-openshift/charts/kmm/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/helm-charts-openshift/charts/kmm/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/helm-charts-openshift/charts/kmm/Chart.yaml b/helm-charts-openshift/charts/kmm/Chart.yaml deleted file mode 100644 index e85ede682..000000000 --- a/helm-charts-openshift/charts/kmm/Chart.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v2 -name: kmm -description: A Helm chart for deploying Kernel Module Management for AMD GPU Operator -type: application - -kubeVersion: ">= 1.18.0-0" -version: v1.0.0 -appVersion: "v20240618-v2.1.1" \ No newline at end of file diff --git a/helm-charts-openshift/charts/kmm/crds/module-crd.yaml b/helm-charts-openshift/charts/kmm/crds/module-crd.yaml deleted file mode 100644 index 0c68ec4a3..000000000 --- a/helm-charts-openshift/charts/kmm/crds/module-crd.yaml +++ /dev/null @@ -1,2604 +0,0 @@ ---- -# Source: kmm/templates/module-crd.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: modules.kmm.sigs.x-k8s.io - annotations: - controller-gen.kubebuilder.io/version: v0.14.0 - service.beta.openshift.io/inject-cabundle: "true" - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - helm.sh/chart: kmm-v1.0.0 - app.kubernetes.io/name: kmm - app.kubernetes.io/instance: amd-gpu - app.kubernetes.io/version: "v20240618-v2.1.1" - app.kubernetes.io/managed-by: Helm -spec: - group: kmm.sigs.x-k8s.io - names: - kind: Module - listKind: ModuleList - plural: modules - singular: module - scope: Namespaced - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: Module describes how to load a module on different kernel versions - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: ModuleSpec describes how the KMM operator should deploy a Module - on those nodes that need it. - properties: - devicePlugin: - description: |- - DevicePlugin allows overriding some properties of the container that deploys the device plugin on the node. - Name is ignored and is set automatically by the KMM Operator. - properties: - container: - properties: - args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment variable present - in a Container. - properties: - name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment variable's value. - Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - optional: - description: Specify whether the ConfigMap or - its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema the FieldPath - is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the - specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the - exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the pod's - namespace - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - optional: - description: Specify whether the Secret or its - key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - image: - description: Image is the name of the container image that the - device plugin container will run. - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - volumeMounts: - description: VolumeMounts is a list of volume mounts that are - appended to the default ones. - items: - description: VolumeMount describes a mounting of a Volume - within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - type: string - name: - description: This must match the Name of a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - required: - - image - type: object - serviceAccountName: - description: |- - ServiceAccountName is the name of the ServiceAccount to use to run this pod. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ - type: string - volumes: - items: - description: Volume represents a named volume in a pod that may - be accessed by any container in the pod. - properties: - awsElasticBlockStore: - description: |- - awsElasticBlockStore represents an AWS Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - TODO: how do we prevent errors in the filesystem from compromising the machine - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - format: int32 - type: integer - readOnly: - description: |- - readOnly value true will force the readOnly setting in VolumeMounts. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: boolean - volumeID: - description: |- - volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: string - required: - - volumeID - type: object - azureDisk: - description: azureDisk represents an Azure Data Disk mount - on the host and bind mount to the pod. - properties: - cachingMode: - description: 'cachingMode is the Host Caching mode: None, - Read Only, Read Write.' - type: string - diskName: - description: diskName is the Name of the data disk in - the blob storage - type: string - diskURI: - description: diskURI is the URI of data disk in the blob - storage - type: string - fsType: - description: |- - fsType is Filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - kind: - description: 'kind expected values are Shared: multiple - blob disks per storage account Dedicated: single blob - disk per storage account Managed: azure managed data - disk (only in managed availability set). defaults to - shared' - type: string - readOnly: - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: azureFile represents an Azure File Service mount - on the host and bind mount to the pod. - properties: - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretName: - description: secretName is the name of secret that contains - Azure Storage Account Name and Key - type: string - shareName: - description: shareName is the azure share Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: cephFS represents a Ceph FS mount on the host - that shares a pod's lifetime - properties: - monitors: - description: |- - monitors is Required: Monitors is a collection of Ceph monitors - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - items: - type: string - type: array - path: - description: 'path is Optional: Used as the mounted root, - rather than the full Ceph tree, default is /' - type: string - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: boolean - secretFile: - description: |- - secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - secretRef: - description: |- - secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - user: - description: |- - user is optional: User is the rados user name, default is admin - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - required: - - monitors - type: object - cinder: - description: |- - cinder represents a cinder volume attached and mounted on kubelets host machine. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: boolean - secretRef: - description: |- - secretRef is optional: points to a secret object containing parameters used to connect - to OpenStack. - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - volumeID: - description: |- - volumeID used to identify the volume in cinder. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - required: - - volumeID - type: object - configMap: - description: configMap represents a configMap that should - populate this volume - properties: - defaultMode: - description: |- - defaultMode is optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - optional: - description: optional specify whether the ConfigMap or - its keys must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - csi: - description: csi (Container Storage Interface) represents - ephemeral storage that is handled by certain external CSI - drivers (Beta feature). - properties: - driver: - description: |- - driver is the name of the CSI driver that handles this volume. - Consult with your admin for the correct name as registered in the cluster. - type: string - fsType: - description: |- - fsType to mount. Ex. "ext4", "xfs", "ntfs". - If not provided, the empty value is passed to the associated CSI driver - which will determine the default filesystem to apply. - type: string - nodePublishSecretRef: - description: |- - nodePublishSecretRef is a reference to the secret object containing - sensitive information to pass to the CSI driver to complete the CSI - NodePublishVolume and NodeUnpublishVolume calls. - This field is optional, and may be empty if no secret is required. If the - secret object contains more than one secret, all secret references are passed. - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - readOnly: - description: |- - readOnly specifies a read-only configuration for the volume. - Defaults to false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: |- - volumeAttributes stores driver-specific properties that are passed to the CSI - driver. Consult your driver's documentation for supported values. - type: object - required: - - driver - type: object - downwardAPI: - description: downwardAPI represents downward API about the - pod that should populate this volume - properties: - defaultMode: - description: |- - Optional: mode bits to use on created files by default. Must be a - Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: Items is a list of downward API volume file - items: - description: DownwardAPIVolumeFile represents information - to create the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects a field of the pod: - only annotations, labels, name and namespace are - supported.' - properties: - apiVersion: - description: Version of the schema the FieldPath - is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in - the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path is the relative path - name of the file to be created. Must not be absolute - or contain the ''..'' path. Must be utf-8 encoded. - The first item of the relative path must not start - with ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of - the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - type: object - emptyDir: - description: |- - emptyDir represents a temporary directory that shares a pod's lifetime. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - properties: - medium: - description: |- - medium represents what type of storage medium should back this directory. - The default is "" which means to use the node's default medium. - Must be an empty string (default) or Memory. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: |- - sizeLimit is the total amount of local storage required for this EmptyDir volume. - The size limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir would be the minimum value between - the SizeLimit specified here and the sum of memory limits of all containers in a pod. - The default is nil which means that the limit is undefined. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: |- - ephemeral represents a volume that is handled by a cluster storage driver. - The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, - and deleted when the pod is removed. - - - Use this if: - a) the volume is only needed while the pod runs, - b) features of normal volumes like restoring from snapshot or capacity - tracking are needed, - c) the storage driver is specified through a storage class, and - d) the storage driver supports dynamic volume provisioning through - a PersistentVolumeClaim (see EphemeralVolumeSource for more - information on the connection between this volume type - and PersistentVolumeClaim). - - - Use PersistentVolumeClaim or one of the vendor-specific - APIs for volumes that persist for longer than the lifecycle - of an individual pod. - - - Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to - be used that way - see the documentation of the driver for - more information. - - - A pod can use both types of ephemeral volumes and - persistent volumes at the same time. - properties: - volumeClaimTemplate: - description: |- - Will be used to create a stand-alone PVC to provision the volume. - The pod in which this EphemeralVolumeSource is embedded will be the - owner of the PVC, i.e. the PVC will be deleted together with the - pod. The name of the PVC will be `-` where - `` is the name from the `PodSpec.Volumes` array - entry. Pod validation will reject the pod if the concatenated name - is not valid for a PVC (for example, too long). - - - An existing PVC with that name that is not owned by the pod - will *not* be used for the pod to avoid using an unrelated - volume by mistake. Starting the pod is then blocked until - the unrelated PVC is removed. If such a pre-created PVC is - meant to be used by the pod, the PVC has to updated with an - owner reference to the pod once the pod exists. Normally - this should not be necessary, but it may be useful when - manually reconstructing a broken cluster. - - - This field is read-only and no changes will be made by Kubernetes - to the PVC after it has been created. - - - Required, must not be nil. - properties: - metadata: - description: |- - May contain labels and annotations that will be copied into the PVC - when creating it. No other fields are allowed and will be rejected during - validation. - type: object - spec: - description: |- - The specification for the PersistentVolumeClaim. The entire content is - copied unchanged into the PVC that gets created from this - template. The same fields as in a PersistentVolumeClaim - are also valid here. - properties: - accessModes: - description: |- - accessModes contains the desired access modes the volume should have. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 - items: - type: string - type: array - dataSource: - description: |- - dataSource field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) - If the provisioner or an external controller can support the specified data source, - it will create a new volume based on the contents of the specified data source. - When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, - and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. - If the namespace is specified, then dataSourceRef will not be copied to dataSource. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource - being referenced - type: string - name: - description: Name is the name of resource - being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: |- - dataSourceRef specifies the object from which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a non-empty API group (non - core object) or a PersistentVolumeClaim object. - When this field is specified, volume binding will only succeed if the type of - the specified object matches some installed volume populator or dynamic - provisioner. - This field will replace the functionality of the dataSource field and as such - if both fields are non-empty, they must have the same value. For backwards - compatibility, when namespace isn't specified in dataSourceRef, - both fields (dataSource and dataSourceRef) will be set to the same - value automatically if one of them is empty and the other is non-empty. - When namespace is specified in dataSourceRef, - dataSource isn't set to the same value and must be empty. - There are three important differences between dataSource and dataSourceRef: - * While dataSource only allows two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim objects. - * While dataSource ignores disallowed values (dropping them), dataSourceRef - preserves all values, and generates an error if a disallowed value is - specified. - * While dataSource only allows local objects, dataSourceRef allows objects - in any namespaces. - (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. - (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource - being referenced - type: string - name: - description: Name is the name of resource - being referenced - type: string - namespace: - description: |- - Namespace is the namespace of resource being referenced - Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. - (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: |- - resources represents the minimum resources the volume should have. - If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements - that are lower than previous value but must still be higher than capacity recorded in the - status field of the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - selector: - description: selector is a label query over volumes - to consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of - label selector requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - description: |- - storageClassName is the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 - type: string - volumeAttributesClassName: - description: |- - volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. - If specified, the CSI driver will create or update the volume with the attributes defined - in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. - If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be - set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource - exists. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#volumeattributesclass - (Alpha) Using this field requires the VolumeAttributesClass feature gate to be enabled. - type: string - volumeMode: - description: |- - volumeMode defines what type of volume is required by the claim. - Value of Filesystem is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference - to the PersistentVolume backing this claim. - type: string - type: object - required: - - spec - type: object - type: object - fc: - description: fc represents a Fibre Channel resource that is - attached to a kubelet's host machine and then exposed to - the pod. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - TODO: how do we prevent errors in the filesystem from compromising the machine - type: string - lun: - description: 'lun is Optional: FC target lun number' - format: int32 - type: integer - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - targetWWNs: - description: 'targetWWNs is Optional: FC target worldwide - names (WWNs)' - items: - type: string - type: array - wwids: - description: |- - wwids Optional: FC volume world wide identifiers (wwids) - Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. - items: - type: string - type: array - type: object - flexVolume: - description: |- - flexVolume represents a generic volume resource that is - provisioned/attached using an exec based plugin. - properties: - driver: - description: driver is the name of the driver to use for - this volume. - type: string - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. - type: string - options: - additionalProperties: - type: string - description: 'options is Optional: this field holds extra - command options if any.' - type: object - readOnly: - description: |- - readOnly is Optional: defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef is Optional: secretRef is reference to the secret object containing - sensitive information to pass to the plugin scripts. This may be - empty if no secret object is specified. If the secret object - contains more than one secret, all secrets are passed to the plugin - scripts. - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - required: - - driver - type: object - flocker: - description: flocker represents a Flocker volume attached - to a kubelet's host machine. This depends on the Flocker - control service being running - properties: - datasetName: - description: |- - datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker - should be considered as deprecated - type: string - datasetUUID: - description: datasetUUID is the UUID of the dataset. This - is unique identifier of a Flocker dataset - type: string - type: object - gcePersistentDisk: - description: |- - gcePersistentDisk represents a GCE Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - properties: - fsType: - description: |- - fsType is filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - TODO: how do we prevent errors in the filesystem from compromising the machine - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - format: int32 - type: integer - pdName: - description: |- - pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: boolean - required: - - pdName - type: object - gitRepo: - description: |- - gitRepo represents a git repository at a particular revision. - DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an - EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir - into the Pod's container. - properties: - directory: - description: |- - directory is the target directory name. - Must not contain or start with '..'. If '.' is supplied, the volume directory will be the - git repository. Otherwise, if specified, the volume will contain the git repository in - the subdirectory with the given name. - type: string - repository: - description: repository is the URL - type: string - revision: - description: revision is the commit hash for the specified - revision. - type: string - required: - - repository - type: object - glusterfs: - description: |- - glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. - More info: https://examples.k8s.io/volumes/glusterfs/README.md - properties: - endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: string - path: - description: |- - path is the Glusterfs volume path. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: string - readOnly: - description: |- - readOnly here will force the Glusterfs volume to be mounted with read-only permissions. - Defaults to false. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: boolean - required: - - endpoints - - path - type: object - hostPath: - description: |- - hostPath represents a pre-existing file or directory on the host - machine that is directly exposed to the container. This is generally - used for system agents or other privileged things that are allowed - to see the host machine. Most containers will NOT need this. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - --- - TODO(jonesdl) We need to restrict who can use host directory mounts and who can/can not - mount host directories as read/write. - properties: - path: - description: |- - path of the directory on the host. - If the path is a symlink, it will follow the link to the real path. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - type: - description: |- - type for HostPath Volume - Defaults to "" - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - required: - - path - type: object - iscsi: - description: |- - iscsi represents an ISCSI Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md - properties: - chapAuthDiscovery: - description: chapAuthDiscovery defines whether support - iSCSI Discovery CHAP authentication - type: boolean - chapAuthSession: - description: chapAuthSession defines whether support iSCSI - Session CHAP authentication - type: boolean - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - TODO: how do we prevent errors in the filesystem from compromising the machine - type: string - initiatorName: - description: |- - initiatorName is the custom iSCSI Initiator Name. - If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface - : will be created for the connection. - type: string - iqn: - description: iqn is the target iSCSI Qualified Name. - type: string - iscsiInterface: - description: |- - iscsiInterface is the interface Name that uses an iSCSI transport. - Defaults to 'default' (tcp). - type: string - lun: - description: lun represents iSCSI Target Lun number. - format: int32 - type: integer - portals: - description: |- - portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - items: - type: string - type: array - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - type: boolean - secretRef: - description: secretRef is the CHAP Secret for iSCSI target - and initiator authentication - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - targetPortal: - description: |- - targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - description: |- - name of the volume. - Must be a DNS_LABEL and unique within the pod. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - nfs: - description: |- - nfs represents an NFS mount on the host that shares a pod's lifetime - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - properties: - path: - description: |- - path that is exported by the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - readOnly: - description: |- - readOnly here will force the NFS export to be mounted with read-only permissions. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: boolean - server: - description: |- - server is the hostname or IP address of the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: |- - persistentVolumeClaimVolumeSource represents a reference to a - PersistentVolumeClaim in the same namespace. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - properties: - claimName: - description: |- - claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - type: string - readOnly: - description: |- - readOnly Will force the ReadOnly setting in VolumeMounts. - Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: photonPersistentDisk represents a PhotonController - persistent disk attached and mounted on kubelets host machine - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - pdID: - description: pdID is the ID that identifies Photon Controller - persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: portworxVolume represents a portworx volume attached - and mounted on kubelets host machine - properties: - fsType: - description: |- - fSType represents the filesystem type to mount - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - volumeID: - description: volumeID uniquely identifies a Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: projected items for all in one resources secrets, - configmaps, and downward API - properties: - defaultMode: - description: |- - defaultMode are the mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - sources: - description: sources is the list of volume projections - items: - description: Projection that may be projected along - with other supported volume types - properties: - clusterTrustBundle: - description: |- - ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field - of ClusterTrustBundle objects in an auto-updating file. - - - Alpha, gated by the ClusterTrustBundleProjection feature gate. - - - ClusterTrustBundle objects can either be selected by name, or by the - combination of signer name and a label selector. - - - Kubelet performs aggressive normalization of the PEM contents written - into the pod filesystem. Esoteric PEM features such as inter-block - comments and block headers are stripped. Certificates are deduplicated. - The ordering of certificates within the file is arbitrary, and Kubelet - may change the order over time. - properties: - labelSelector: - description: |- - Select all ClusterTrustBundles that match this label selector. Only has - effect if signerName is set. Mutually-exclusive with name. If unset, - interpreted as "match nothing". If set but empty, interpreted as "match - everything". - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - name: - description: |- - Select a single ClusterTrustBundle by object name. Mutually-exclusive - with signerName and labelSelector. - type: string - optional: - description: |- - If true, don't block pod startup if the referenced ClusterTrustBundle(s) - aren't available. If using name, then the named ClusterTrustBundle is - allowed not to exist. If using signerName, then the combination of - signerName and labelSelector is allowed to match zero - ClusterTrustBundles. - type: boolean - path: - description: Relative path from the volume root - to write the bundle. - type: string - signerName: - description: |- - Select all ClusterTrustBundles that match this signer name. - Mutually-exclusive with name. The contents of all selected - ClusterTrustBundles will be unified and deduplicated. - type: string - required: - - path - type: object - configMap: - description: configMap information about the configMap - data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - optional: - description: optional specify whether the ConfigMap - or its keys must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - downwardAPI: - description: downwardAPI information about the downwardAPI - data to project - properties: - items: - description: Items is a list of DownwardAPIVolume - file - items: - description: DownwardAPIVolumeFile represents - information to create the file containing - the pod field - properties: - fieldRef: - description: 'Required: Selects a field - of the pod: only annotations, labels, - name and namespace are supported.' - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path is the relative - path name of the file to be created. - Must not be absolute or contain the - ''..'' path. Must be utf-8 encoded. - The first item of the relative path - must not start with ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container name: required - for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to - select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - type: object - secret: - description: secret information about the secret - data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - optional: - description: optional field specify whether - the Secret or its key must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - serviceAccountToken: - description: serviceAccountToken is information - about the serviceAccountToken data to project - properties: - audience: - description: |- - audience is the intended audience of the token. A recipient of a token - must identify itself with an identifier specified in the audience of the - token, and otherwise should reject the token. The audience defaults to the - identifier of the apiserver. - type: string - expirationSeconds: - description: |- - expirationSeconds is the requested duration of validity of the service - account token. As the token approaches expiration, the kubelet volume - plugin will proactively rotate the service account token. The kubelet will - start trying to rotate the token if the token is older than 80 percent of - its time to live or if the token is older than 24 hours.Defaults to 1 hour - and must be at least 10 minutes. - format: int64 - type: integer - path: - description: |- - path is the path relative to the mount point of the file to project the - token into. - type: string - required: - - path - type: object - type: object - type: array - type: object - quobyte: - description: quobyte represents a Quobyte mount on the host - that shares a pod's lifetime - properties: - group: - description: |- - group to map volume access to - Default is no group - type: string - readOnly: - description: |- - readOnly here will force the Quobyte volume to be mounted with read-only permissions. - Defaults to false. - type: boolean - registry: - description: |- - registry represents a single or multiple Quobyte Registry services - specified as a string as host:port pair (multiple entries are separated with commas) - which acts as the central registry for volumes - type: string - tenant: - description: |- - tenant owning the given Quobyte volume in the Backend - Used with dynamically provisioned Quobyte volumes, value is set by the plugin - type: string - user: - description: |- - user to map volume access to - Defaults to serivceaccount user - type: string - volume: - description: volume is a string that references an already - created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: |- - rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. - More info: https://examples.k8s.io/volumes/rbd/README.md - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - TODO: how do we prevent errors in the filesystem from compromising the machine - type: string - image: - description: |- - image is the rados image name. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - keyring: - description: |- - keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - monitors: - description: |- - monitors is a collection of Ceph monitors. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - items: - type: string - type: array - pool: - description: |- - pool is the rados pool name. - Default is rbd. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: boolean - secretRef: - description: |- - secretRef is name of the authentication secret for RBDUser. If provided - overrides keyring. - Default is nil. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - user: - description: |- - user is the rados user name. - Default is admin. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - required: - - image - - monitors - type: object - scaleIO: - description: scaleIO represents a ScaleIO persistent volume - attached and mounted on Kubernetes nodes. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". - Default is "xfs". - type: string - gateway: - description: gateway is the host address of the ScaleIO - API Gateway. - type: string - protectionDomain: - description: protectionDomain is the name of the ScaleIO - Protection Domain for the configured storage. - type: string - readOnly: - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef references to the secret for ScaleIO user and other - sensitive information. If this is not provided, Login operation will fail. - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - sslEnabled: - description: sslEnabled Flag enable/disable SSL communication - with Gateway, default false - type: boolean - storageMode: - description: |- - storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. - type: string - storagePool: - description: storagePool is the ScaleIO Storage Pool associated - with the protection domain. - type: string - system: - description: system is the name of the storage system - as configured in ScaleIO. - type: string - volumeName: - description: |- - volumeName is the name of a volume already created in the ScaleIO system - that is associated with this volume source. - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - description: |- - secret represents a secret that should populate this volume. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - properties: - defaultMode: - description: |- - defaultMode is Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values - for mode bits. Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items If unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - optional: - description: optional field specify whether the Secret - or its keys must be defined - type: boolean - secretName: - description: |- - secretName is the name of the secret in the pod's namespace to use. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - type: string - type: object - storageos: - description: storageOS represents a StorageOS volume attached - and mounted on Kubernetes nodes. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef specifies the secret to use for obtaining the StorageOS API - credentials. If not specified, default values will be attempted. - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - volumeName: - description: |- - volumeName is the human-readable name of the StorageOS volume. Volume - names are only unique within a namespace. - type: string - volumeNamespace: - description: |- - volumeNamespace specifies the scope of the volume within StorageOS. If no - namespace is specified then the Pod's namespace will be used. This allows the - Kubernetes name scoping to be mirrored within StorageOS for tighter integration. - Set VolumeName to any name to override the default behaviour. - Set to "default" if you are not using namespaces within StorageOS. - Namespaces that do not pre-exist within StorageOS will be created. - type: string - type: object - vsphereVolume: - description: vsphereVolume represents a vSphere volume attached - and mounted on kubelets host machine - properties: - fsType: - description: |- - fsType is filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - storagePolicyID: - description: storagePolicyID is the storage Policy Based - Management (SPBM) profile ID associated with the StoragePolicyName. - type: string - storagePolicyName: - description: storagePolicyName is the storage Policy Based - Management (SPBM) profile name. - type: string - volumePath: - description: volumePath is the path that identifies vSphere - volume vmdk - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - required: - - container - type: object - imageRepoSecret: - description: |- - ImageRepoSecret is an optional secret that is used to pull both the module loader and the device plugin, and - to push the resulting image from the module loader build, if enabled. - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - moduleLoader: - description: |- - ModuleLoader allows overriding some properties of the container that loads the kernel module on the node. - Name and image are ignored and are set automatically by the KMM Operator. - properties: - container: - description: Container holds the properties for the module loader - container that runs modprobe. - properties: - build: - description: Build contains build instructions. - properties: - baseImageRegistryTLS: - description: BaseImageRegistryTLS contains settings determining - how to access registries of the base images in the build-process' - Dockerfile. - properties: - insecure: - description: If Insecure is true, the operator will - be able to access a registry in an insecure (plain - HTTP) protocol. - type: boolean - insecureSkipTLSVerify: - description: If InsecureSkipTLSVerify, the operator - will accept any certificate provided by the registry. - type: boolean - type: object - buildArgs: - description: BuildArgs is an array of build variables that - are provided to the image building backend. - items: - description: BuildArg represents a build argument used - when building a container image. - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - dockerfileConfigMap: - description: ConfigMap that holds Dockerfile contents - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - kanikoParams: - description: KanikoParams is used to customize the building - process of the image. - properties: - tag: - description: Kaniko image tag to use when creating the - build Job - type: string - type: object - secrets: - description: |- - Secrets is an optional list of secrets to be made available to the build system. - Those secrets should be used for private resources such as a private Github repo. - For container registries auth use module.spec.imagePullSecret instead. - items: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - type: array - selector: - additionalProperties: - type: string - description: Selector describes on which nodes will run - the building process. - type: object - required: - - dockerfileConfigMap - type: object - containerImage: - description: ContainerImage is a top-level field - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - inTreeModuleToRemove: - description: |- - Deprecated: please use InTreeModulesToRemove. - InTreeModuleToRemove specifies one in-tree kernel module that should be removed (if present) - before loading the kernel module from the ContainerImage - type: string - inTreeModulesToRemove: - description: |- - InTreeModulesToRemove specifies any number of in-tree kernel modules that should be removed (if present) - before loading the kernel module from the ContainerImage - items: - type: string - type: array - kernelMappings: - description: |- - KernelMappings is a list of kernel mappings. - When a node's labels match Selector, then the KMM Operator will look for the first mapping that matches its - kernel version, and use the corresponding container image to run the DriverContainer. - items: - description: |- - KernelMapping pairs kernel versions with a DriverContainer image. - Kernel versions can be matched literally or using a regular expression. - properties: - build: - description: Build enables in-cluster builds for this - mapping and allows overriding the Module's build settings. - properties: - baseImageRegistryTLS: - description: BaseImageRegistryTLS contains settings - determining how to access registries of the base - images in the build-process' Dockerfile. - properties: - insecure: - description: If Insecure is true, the operator - will be able to access a registry in an insecure - (plain HTTP) protocol. - type: boolean - insecureSkipTLSVerify: - description: If InsecureSkipTLSVerify, the operator - will accept any certificate provided by the - registry. - type: boolean - type: object - buildArgs: - description: BuildArgs is an array of build variables - that are provided to the image building backend. - items: - description: BuildArg represents a build argument - used when building a container image. - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - dockerfileConfigMap: - description: ConfigMap that holds Dockerfile contents - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - kanikoParams: - description: KanikoParams is used to customize the - building process of the image. - properties: - tag: - description: Kaniko image tag to use when creating - the build Job - type: string - type: object - secrets: - description: |- - Secrets is an optional list of secrets to be made available to the build system. - Those secrets should be used for private resources such as a private Github repo. - For container registries auth use module.spec.imagePullSecret instead. - items: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - type: array - selector: - additionalProperties: - type: string - description: Selector describes on which nodes will - run the building process. - type: object - required: - - dockerfileConfigMap - type: object - containerImage: - description: ContainerImage is the name of the DriverContainer - image that should be used to deploy the module. - type: string - inTreeModuleToRemove: - description: |- - Deprecated: please use InTreeModulesToRemove. - InTreeModuleToRemove specifies one in-tree kernel module that should be removed (if present) - before loading the kernel module from the ContainerImage - type: string - inTreeModulesToRemove: - description: |- - InTreeModulesToRemove specifies any number of in-tree kernel modules that should be removed (if present) - before loading the kernel module from the ContainerImage - items: - type: string - type: array - literal: - description: Literal defines a literal target kernel version - to be matched exactly against node kernels. - type: string - regexp: - description: Regexp is a regular expression to be match - against node kernels. - type: string - registryTLS: - description: RegistryTLS set the TLS configs for accessing - the registry of the module-loader's image. - properties: - insecure: - description: If Insecure is true, the operator will - be able to access a registry in an insecure (plain - HTTP) protocol. - type: boolean - insecureSkipTLSVerify: - description: If InsecureSkipTLSVerify, the operator - will accept any certificate provided by the registry. - type: boolean - type: object - sign: - description: Sign enables in-cluster signing for this - mapping - properties: - certSecret: - description: a secret containing the public key used - to sign kernel modules for secureboot - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - filesToSign: - description: paths inside the image for the kernel - modules to sign (if ommited all kmods are signed) - items: - type: string - type: array - keySecret: - description: a secret containing the private key used - to sign kernel modules for secureboot - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - unsignedImage: - description: Image to sign, ignored if a Build is - present, required otherwise - type: string - unsignedImageRegistryTLS: - description: UnsignedImageRegistryTLS contains settings - determining how to access registries of the unsigned - image. - properties: - insecure: - description: If Insecure is true, the operator - will be able to access a registry in an insecure - (plain HTTP) protocol. - type: boolean - insecureSkipTLSVerify: - description: If InsecureSkipTLSVerify, the operator - will accept any certificate provided by the - registry. - type: boolean - type: object - required: - - certSecret - - keySecret - type: object - required: - - containerImage - type: object - minItems: 1 - type: array - modprobe: - description: Modprobe is a set of properties to customize which - module modprobe loads and with which properties. - properties: - args: - description: |- - Args is an optional list of arguments to be passed to modprobe before the name of the kernel module. - The resulting commands will be: `modprobe ${Args} module_name`. - properties: - load: - description: Load is an optional list of arguments to - be used when loading the kernel module. - items: - type: string - minItems: 1 - type: array - unload: - description: Unload is an optional list of arguments - to be used when unloading the kernel module. - items: - type: string - minItems: 1 - type: array - type: object - dirName: - default: /opt - description: |- - DirName is the root directory for modules. - It adds `-d ${DirName}` to the modprobe command-line. - type: string - firmwarePath: - description: |- - FirmwarePath is the path of the firmware(s). - The firmware(s) will be copied to the host for the kernel to find them. - type: string - moduleName: - description: |- - ModuleName is the name of the Module to be loaded. - This field can only be unset if rawArgs is set. - type: string - modulesLoadingOrder: - description: |- - ModulesLoadingOrder defines the dependency between kernel modules loading, in case - it was not created by depmod (independent kernel modules). - The list order should be: upmost module, then the module it depends on and so on. - Example: if moduleA depends on first loading moduleB, and moduleB depends on first loading moduleC - the entry should look: - ModulesLoadingOrder: - - moduleA - - moduleB - - moduleC - In order to load all 3 modules, moduleA shoud be defined in the ModuleName parameter of this struct - items: - type: string - type: array - parameters: - description: |- - Parameters is an optional list of kernel module parameters to be provided to modprobe. - They should be in the form of key=value and will be separated by spaces in the modprobe command. - The resulting loading command will be: `modprobe module_name ${Parameters}`. - items: - type: string - type: array - rawArgs: - description: |- - If RawArgs are specified, they are passed straight to the modprobe binary; all other properties in this - object are ignored. - The resulting commands will be: `modprobe ${RawArgs}`. - properties: - load: - description: Load is an optional list of arguments to - be used when loading the kernel module. - items: - type: string - minItems: 1 - type: array - unload: - description: Unload is an optional list of arguments - to be used when unloading the kernel module. - items: - type: string - minItems: 1 - type: array - type: object - type: object - registryTLS: - description: RegistryTLS set the TLS configs for accessing the - registry of the module-loader's image. - properties: - insecure: - description: If Insecure is true, the operator will be able - to access a registry in an insecure (plain HTTP) protocol. - type: boolean - insecureSkipTLSVerify: - description: If InsecureSkipTLSVerify, the operator will - accept any certificate provided by the registry. - type: boolean - type: object - sign: - description: Sign provides default kmod signing settings - properties: - certSecret: - description: a secret containing the public key used to - sign kernel modules for secureboot - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - filesToSign: - description: paths inside the image for the kernel modules - to sign (if ommited all kmods are signed) - items: - type: string - type: array - keySecret: - description: a secret containing the private key used to - sign kernel modules for secureboot - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - unsignedImage: - description: Image to sign, ignored if a Build is present, - required otherwise - type: string - unsignedImageRegistryTLS: - description: UnsignedImageRegistryTLS contains settings - determining how to access registries of the unsigned image. - properties: - insecure: - description: If Insecure is true, the operator will - be able to access a registry in an insecure (plain - HTTP) protocol. - type: boolean - insecureSkipTLSVerify: - description: If InsecureSkipTLSVerify, the operator - will accept any certificate provided by the registry. - type: boolean - type: object - required: - - certSecret - - keySecret - type: object - version: - description: |- - Version defines the current version of the kernel module being used - Used for upgrading the currently loaded kernel module to a new version - type: string - required: - - kernelMappings - - modprobe - type: object - serviceAccountName: - description: |- - ServiceAccountName is the name of the ServiceAccount to use to run this pod. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ - type: string - required: - - container - type: object - selector: - additionalProperties: - type: string - description: Selector describes on which nodes the Module should be - loaded and optionally built. - type: object - tolerations: - description: If specified, the pod's tolerations. - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - required: - - moduleLoader - - selector - type: object - status: - description: ModuleStatus defines the observed state of Module. - properties: - devicePlugin: - description: |- - DevicePlugin contains the status of the Device Plugin daemonset - if it was deployed during reconciliation - properties: - availableNumber: - description: number of the actually deployed and running pods - format: int32 - type: integer - desiredNumber: - description: number of the pods that should be deployed for daemonset - format: int32 - type: integer - nodesMatchingSelectorNumber: - description: number of nodes that are targeted by the module selector - format: int32 - type: integer - type: object - moduleLoader: - description: ModuleLoader contains the status of the ModuleLoader daemonset - properties: - availableNumber: - description: number of the actually deployed and running pods - format: int32 - type: integer - desiredNumber: - description: number of the pods that should be deployed for daemonset - format: int32 - type: integer - nodesMatchingSelectorNumber: - description: number of nodes that are targeted by the module selector - format: int32 - type: integer - type: object - required: - - moduleLoader - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/helm-charts-openshift/charts/kmm/crds/nodemodulesconfig-crd.yaml b/helm-charts-openshift/charts/kmm/crds/nodemodulesconfig-crd.yaml deleted file mode 100644 index ee3d5da51..000000000 --- a/helm-charts-openshift/charts/kmm/crds/nodemodulesconfig-crd.yaml +++ /dev/null @@ -1,440 +0,0 @@ ---- -# Source: kmm/templates/nodemodulesconfig-crd.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: nodemodulesconfigs.kmm.sigs.x-k8s.io - annotations: - controller-gen.kubebuilder.io/version: v0.14.0 - service.beta.openshift.io/inject-cabundle: "true" - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - helm.sh/chart: kmm-v1.0.0 - app.kubernetes.io/name: kmm - app.kubernetes.io/instance: amd-gpu - app.kubernetes.io/version: "v20240618-v2.1.1" - app.kubernetes.io/managed-by: Helm -spec: - group: kmm.sigs.x-k8s.io - names: - kind: NodeModulesConfig - listKind: NodeModulesConfigList - plural: nodemodulesconfigs - shortNames: - - nmc - singular: nodemodulesconfig - scope: Cluster - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: NodeModulesConfig keeps spec and state of the KMM modules on a - node. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: |- - NodeModulesConfigSpec describes the desired state of modules on the node - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - modules: - description: |- - Modules list the spec of all the modules that need to be executed - on the node - items: - properties: - config: - properties: - containerImage: - type: string - imagePullPolicy: - default: IfNotPresent - description: PullPolicy describes a policy for if/when to - pull a container image - type: string - inTreeModuleToRemove: - type: string - inTreeModulesToRemove: - items: - type: string - type: array - insecurePull: - description: When InsecurePull is true, the container image - can be pulled without TLS. - type: boolean - kernelVersion: - type: string - modprobe: - properties: - args: - description: |- - Args is an optional list of arguments to be passed to modprobe before the name of the kernel module. - The resulting commands will be: `modprobe ${Args} module_name`. - properties: - load: - description: Load is an optional list of arguments - to be used when loading the kernel module. - items: - type: string - minItems: 1 - type: array - unload: - description: Unload is an optional list of arguments - to be used when unloading the kernel module. - items: - type: string - minItems: 1 - type: array - type: object - dirName: - default: /opt - description: |- - DirName is the root directory for modules. - It adds `-d ${DirName}` to the modprobe command-line. - type: string - firmwarePath: - description: |- - FirmwarePath is the path of the firmware(s). - The firmware(s) will be copied to the host for the kernel to find them. - type: string - moduleName: - description: |- - ModuleName is the name of the Module to be loaded. - This field can only be unset if rawArgs is set. - type: string - modulesLoadingOrder: - description: |- - ModulesLoadingOrder defines the dependency between kernel modules loading, in case - it was not created by depmod (independent kernel modules). - The list order should be: upmost module, then the module it depends on and so on. - Example: if moduleA depends on first loading moduleB, and moduleB depends on first loading moduleC - the entry should look: - ModulesLoadingOrder: - - moduleA - - moduleB - - moduleC - In order to load all 3 modules, moduleA shoud be defined in the ModuleName parameter of this struct - items: - type: string - type: array - parameters: - description: |- - Parameters is an optional list of kernel module parameters to be provided to modprobe. - They should be in the form of key=value and will be separated by spaces in the modprobe command. - The resulting loading command will be: `modprobe module_name ${Parameters}`. - items: - type: string - type: array - rawArgs: - description: |- - If RawArgs are specified, they are passed straight to the modprobe binary; all other properties in this - object are ignored. - The resulting commands will be: `modprobe ${RawArgs}`. - properties: - load: - description: Load is an optional list of arguments - to be used when loading the kernel module. - items: - type: string - minItems: 1 - type: array - unload: - description: Unload is an optional list of arguments - to be used when unloading the kernel module. - items: - type: string - minItems: 1 - type: array - type: object - type: object - required: - - containerImage - - insecurePull - - kernelVersion - - modprobe - type: object - imageRepoSecret: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - name: - type: string - namespace: - type: string - serviceAccountName: - type: string - tolerations: - description: tolerations define which tolerations should be added - for every load/unload pod running on the node - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - required: - - config - - name - - namespace - - serviceAccountName - type: object - type: array - type: object - status: - description: |- - NodeModuleConfigStatus is the most recently observed status of the KMM modules on node. - It is populated by the system and is read-only. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - modules: - description: Modules contain observations about each Module's node state - status - items: - properties: - config: - properties: - containerImage: - type: string - imagePullPolicy: - default: IfNotPresent - description: PullPolicy describes a policy for if/when to - pull a container image - type: string - inTreeModuleToRemove: - type: string - inTreeModulesToRemove: - items: - type: string - type: array - insecurePull: - description: When InsecurePull is true, the container image - can be pulled without TLS. - type: boolean - kernelVersion: - type: string - modprobe: - properties: - args: - description: |- - Args is an optional list of arguments to be passed to modprobe before the name of the kernel module. - The resulting commands will be: `modprobe ${Args} module_name`. - properties: - load: - description: Load is an optional list of arguments - to be used when loading the kernel module. - items: - type: string - minItems: 1 - type: array - unload: - description: Unload is an optional list of arguments - to be used when unloading the kernel module. - items: - type: string - minItems: 1 - type: array - type: object - dirName: - default: /opt - description: |- - DirName is the root directory for modules. - It adds `-d ${DirName}` to the modprobe command-line. - type: string - firmwarePath: - description: |- - FirmwarePath is the path of the firmware(s). - The firmware(s) will be copied to the host for the kernel to find them. - type: string - moduleName: - description: |- - ModuleName is the name of the Module to be loaded. - This field can only be unset if rawArgs is set. - type: string - modulesLoadingOrder: - description: |- - ModulesLoadingOrder defines the dependency between kernel modules loading, in case - it was not created by depmod (independent kernel modules). - The list order should be: upmost module, then the module it depends on and so on. - Example: if moduleA depends on first loading moduleB, and moduleB depends on first loading moduleC - the entry should look: - ModulesLoadingOrder: - - moduleA - - moduleB - - moduleC - In order to load all 3 modules, moduleA shoud be defined in the ModuleName parameter of this struct - items: - type: string - type: array - parameters: - description: |- - Parameters is an optional list of kernel module parameters to be provided to modprobe. - They should be in the form of key=value and will be separated by spaces in the modprobe command. - The resulting loading command will be: `modprobe module_name ${Parameters}`. - items: - type: string - type: array - rawArgs: - description: |- - If RawArgs are specified, they are passed straight to the modprobe binary; all other properties in this - object are ignored. - The resulting commands will be: `modprobe ${RawArgs}`. - properties: - load: - description: Load is an optional list of arguments - to be used when loading the kernel module. - items: - type: string - minItems: 1 - type: array - unload: - description: Unload is an optional list of arguments - to be used when unloading the kernel module. - items: - type: string - minItems: 1 - type: array - type: object - type: object - required: - - containerImage - - insecurePull - - kernelVersion - - modprobe - type: object - imageRepoSecret: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - lastTransitionTime: - format: date-time - type: string - name: - type: string - namespace: - type: string - serviceAccountName: - type: string - tolerations: - description: tolerations define which tolerations should be added - for every load/unload pod running on the node - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - required: - - name - - namespace - - serviceAccountName - type: object - type: array - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/helm-charts-openshift/charts/kmm/templates/_helpers.tpl b/helm-charts-openshift/charts/kmm/templates/_helpers.tpl deleted file mode 100644 index 182641509..000000000 --- a/helm-charts-openshift/charts/kmm/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "kmm.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "kmm.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "kmm.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "kmm.labels" -}} -helm.sh/chart: {{ include "kmm.chart" . }} -{{ include "kmm.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "kmm.selectorLabels" -}} -app.kubernetes.io/name: {{ include "kmm.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "kmm.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "kmm.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/helm-charts-openshift/charts/kmm/templates/cluster-ca.yaml b/helm-charts-openshift/charts/kmm/templates/cluster-ca.yaml deleted file mode 100644 index 230d26a27..000000000 --- a/helm-charts-openshift/charts/kmm/templates/cluster-ca.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "kmm.fullname" . }}-cluster-ca - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - config.openshift.io/inject-trusted-cabundle: "true" - kmm.openshift.io/ca.type: cluster - {{- include "kmm.labels" . | nindent 4 }} -data: {} \ No newline at end of file diff --git a/helm-charts-openshift/charts/kmm/templates/controller-metrics-monitor.yaml b/helm-charts-openshift/charts/kmm/templates/controller-metrics-monitor.yaml deleted file mode 100644 index 2c5717f19..000000000 --- a/helm-charts-openshift/charts/kmm/templates/controller-metrics-monitor.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: {{ include "kmm.fullname" . }}-controller-metrics-monitor - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: controller - {{- include "kmm.labels" . | nindent 4 }} -spec: - endpoints: - - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - path: /metrics - port: https - scheme: https - tlsConfig: - insecureSkipVerify: true - selector: - matchLabels: - control-plane: controller \ No newline at end of file diff --git a/helm-charts-openshift/charts/kmm/templates/controller-metrics-service.yaml b/helm-charts-openshift/charts/kmm/templates/controller-metrics-service.yaml deleted file mode 100644 index 4f17b470a..000000000 --- a/helm-charts-openshift/charts/kmm/templates/controller-metrics-service.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "kmm.fullname" . }}-controller-metrics-service - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: controller - {{- include "kmm.labels" . | nindent 4 }} -spec: - type: {{ .Values.controllerMetricsService.type }} - selector: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: controller - {{- include "kmm.selectorLabels" . | nindent 4 }} - ports: - {{- .Values.controllerMetricsService.ports | toYaml | nindent 2 }} diff --git a/helm-charts-openshift/charts/kmm/templates/deployment.yaml b/helm-charts-openshift/charts/kmm/templates/deployment.yaml deleted file mode 100644 index 4b8b2f7f6..000000000 --- a/helm-charts-openshift/charts/kmm/templates/deployment.yaml +++ /dev/null @@ -1,201 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "kmm.fullname" . }}-controller - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: controller - {{- include "kmm.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.controller.replicas }} - selector: - matchLabels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: controller - {{- include "kmm.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: controller - {{- include "kmm.selectorLabels" . | nindent 8 }} - annotations: - kubectl.kubernetes.io/default-container: manager - spec: - {{- with .Values.controller.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - nodeSelector: {{- toYaml .Values.controller.nodeSelector | nindent 8 }} - containers: - - args: {{- toYaml .Values.controller.manager.args | nindent 8 }} - command: - - /usr/local/bin/manager - env: - - name: RELATED_IMAGE_WORKER - value: {{ quote .Values.controller.manager.env.relatedImageWorker }} - - name: SSL_CERT_DIR - value: {{ quote .Values.controller.manager.env.sslCertDir }} - - name: OPERATOR_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: RELATED_IMAGE_MUST_GATHER - value: {{ quote .Values.controller.manager.env.relatedImageMustGather }} - - name: RELATED_IMAGE_SIGN - value: {{ quote .Values.controller.manager.env.relatedImageSign }} - - name: KUBERNETES_CLUSTER_DOMAIN - value: {{ quote .Values.kubernetesClusterDomain }} - image: {{ .Values.controller.manager.image.repository }}:{{ .Values.controller.manager.image.tag - | default .Chart.AppVersion }} - imagePullPolicy: {{ .Values.controller.manager.imagePullPolicy }} - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - ports: - - containerPort: 8443 - name: metrics - protocol: TCP - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: {{- toYaml .Values.controller.manager.resources | nindent 10 }} - securityContext: {{- toYaml .Values.controller.manager.containerSecurityContext - | nindent 10 }} - volumeMounts: - - mountPath: /etc/pki/ca-trust/extracted/pem - name: trusted-ca - readOnly: true - - mountPath: /controller_config.yaml - name: manager-config - subPath: controller_config.yaml - {{- if .Values.controller.manager.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.controller.manager.imagePullSecrets }} - {{- end}} - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - serviceAccountName: {{ include "kmm.fullname" . }}-controller - terminationGracePeriodSeconds: 10 - {{- with .Values.controller.manager.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - volumes: - - name: trusted-ca - projected: - sources: - - configMap: - items: - - key: ca-bundle.crt - path: tls-ca-bundle.pem - name: {{ include "kmm.fullname" . }}-cluster-ca - - configMap: - items: - - key: service-ca.crt - path: ocp-service-ca-bundle.pem - name: {{ include "kmm.fullname" . }}-service-ca - - configMap: - name: {{ include "kmm.fullname" . }}-manager-config - name: manager-config ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "kmm.fullname" . }}-webhook-server - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: webhook-server - {{- include "kmm.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.webhookServer.replicas }} - selector: - matchLabels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: webhook-server - {{- include "kmm.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: webhook-server - {{- include "kmm.selectorLabels" . | nindent 8 }} - annotations: - kubectl.kubernetes.io/default-container: webhook-server - spec: - {{- with .Values.webhookServer.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - nodeSelector: {{- toYaml .Values.webhookServer.nodeSelector | nindent 8 }} - containers: - - args: {{- toYaml .Values.webhookServer.webhookServer.args | nindent 8 }} - env: - - name: KUBERNETES_CLUSTER_DOMAIN - value: {{ quote .Values.kubernetesClusterDomain }} - image: {{ .Values.webhookServer.webhookServer.image.repository }}:{{ .Values.webhookServer.webhookServer.image.tag - | default .Chart.AppVersion }} - imagePullPolicy: {{ .Values.webhookServer.webhookServer.imagePullPolicy }} - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: webhook-server - ports: - - containerPort: 9443 - name: webhook-server - protocol: TCP - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: {{- toYaml .Values.webhookServer.webhookServer.resources | nindent 10 - }} - securityContext: {{- toYaml .Values.webhookServer.webhookServer.containerSecurityContext - | nindent 10 }} - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true - - mountPath: /controller_config.yaml - name: manager-config - subPath: controller_config.yaml - {{- if .Values.webhookServer.webhookServer.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.webhookServer.webhookServer.imagePullSecrets }} - {{- end}} - securityContext: - runAsNonRoot: true - serviceAccountName: {{ include "kmm.fullname" . }}-controller - terminationGracePeriodSeconds: 10 - {{- with .Values.webhookServer.webhookServer.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - volumes: - - name: cert - secret: - defaultMode: 420 - secretName: {{ include "kmm.fullname" . }}-webhook-server-cert - - configMap: - name: {{ include "kmm.fullname" . }}-manager-config - name: manager-config diff --git a/helm-charts-openshift/charts/kmm/templates/event-recorder-clusterrole-rbac.yaml b/helm-charts-openshift/charts/kmm/templates/event-recorder-clusterrole-rbac.yaml deleted file mode 100644 index 6d86d628c..000000000 --- a/helm-charts-openshift/charts/kmm/templates/event-recorder-clusterrole-rbac.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "kmm.fullname" . }}-event-recorder-clusterrole - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch \ No newline at end of file diff --git a/helm-charts-openshift/charts/kmm/templates/event-recorder-clusterrolebinding-rbac.yaml b/helm-charts-openshift/charts/kmm/templates/event-recorder-clusterrolebinding-rbac.yaml deleted file mode 100644 index 21366100f..000000000 --- a/helm-charts-openshift/charts/kmm/templates/event-recorder-clusterrolebinding-rbac.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "kmm.fullname" . }}-event-recorder-clusterrolebinding - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "kmm.fullname" . }}-event-recorder-clusterrole' -subjects: -- kind: ServiceAccount - name: '{{ include "kmm.fullname" . }}-controller' - namespace: '{{ .Release.Namespace }}' \ No newline at end of file diff --git a/helm-charts-openshift/charts/kmm/templates/leader-election-rbac.yaml b/helm-charts-openshift/charts/kmm/templates/leader-election-rbac.yaml deleted file mode 100644 index d4b7df6c5..000000000 --- a/helm-charts-openshift/charts/kmm/templates/leader-election-rbac.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "kmm.fullname" . }}-leader-election-role - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "kmm.fullname" . }}-leader-election-rolebinding - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: '{{ include "kmm.fullname" . }}-leader-election-role' -subjects: -- kind: ServiceAccount - name: '{{ include "kmm.fullname" . }}-controller' - namespace: '{{ .Release.Namespace }}' \ No newline at end of file diff --git a/helm-charts-openshift/charts/kmm/templates/manager-config.yaml b/helm-charts-openshift/charts/kmm/templates/manager-config.yaml deleted file mode 100644 index 27f3a711d..000000000 --- a/helm-charts-openshift/charts/kmm/templates/manager-config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "kmm.fullname" . }}-manager-config - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -data: - controller_config.yaml: {{ .Values.managerConfig.controllerConfigYaml | toYaml - | indent 1 }} \ No newline at end of file diff --git a/helm-charts-openshift/charts/kmm/templates/manager-rbac.yaml b/helm-charts-openshift/charts/kmm/templates/manager-rbac.yaml deleted file mode 100644 index 9a111f589..000000000 --- a/helm-charts-openshift/charts/kmm/templates/manager-rbac.yaml +++ /dev/null @@ -1,231 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "kmm.fullname" . }}-manager-role - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -rules: -- apiGroups: - - apps - resources: - - daemonsets - verbs: - - create - - delete - - get - - list - - patch - - watch -- apiGroups: - - build.openshift.io - resources: - - builds - verbs: - - create - - delete - - get - - list - - patch - - watch -- apiGroups: - - cluster.open-cluster-management.io - resources: - - clusterclaims - verbs: - - create - - get - - list - - watch -- apiGroups: - - cluster.open-cluster-management.io - resourceNames: - - kernel-versions.kmm.node.kubernetes.io - resources: - - clusterclaims - verbs: - - delete - - patch - - update -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - delete - - get - - list - - patch - - watch -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get - - list - - patch - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - patch - - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - create - - delete - - get - - list - - patch - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - get - - list - - watch -- apiGroups: - - image.openshift.io - resources: - - imagestreams - verbs: - - get - - list - - watch -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - modules - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - modules/finalizers - verbs: - - update -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - modules/status - verbs: - - get - - patch - - update -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - nodemodulesconfigs - verbs: - - create - - delete - - get - - list - - patch - - watch -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - nodemodulesconfigs/finalizers - verbs: - - patch - - update -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - nodemodulesconfigs/status - verbs: - - patch -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - preflightvalidations - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - preflightvalidations/finalizers - verbs: - - update -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - preflightvalidations/status - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - preflightvalidationsocp - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - preflightvalidationsocp/finalizers - verbs: - - update -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - preflightvalidationsocp/status - verbs: - - get - - patch - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "kmm.fullname" . }}-manager-rolebinding - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "kmm.fullname" . }}-manager-role' -subjects: -- kind: ServiceAccount - name: '{{ include "kmm.fullname" . }}-controller' - namespace: '{{ .Release.Namespace }}' \ No newline at end of file diff --git a/helm-charts-openshift/charts/kmm/templates/metrics-reader-rbac.yaml b/helm-charts-openshift/charts/kmm/templates/metrics-reader-rbac.yaml deleted file mode 100644 index 2acb7127b..000000000 --- a/helm-charts-openshift/charts/kmm/templates/metrics-reader-rbac.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "kmm.fullname" . }}-metrics-reader - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -rules: -- nonResourceURLs: - - /metrics - verbs: - - get \ No newline at end of file diff --git a/helm-charts-openshift/charts/kmm/templates/preflightvalidation-crd.yaml b/helm-charts-openshift/charts/kmm/templates/preflightvalidation-crd.yaml deleted file mode 100644 index d8477bcaa..000000000 --- a/helm-charts-openshift/charts/kmm/templates/preflightvalidation-crd.yaml +++ /dev/null @@ -1,238 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: preflightvalidations.kmm.sigs.x-k8s.io - annotations: - controller-gen.kubebuilder.io/version: v0.14.0 - service.beta.openshift.io/inject-cabundle: "true" - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - name: '{{ include "kmm.fullname" . }}-webhook-service' - namespace: '{{ .Release.Namespace }}' - path: /convert - conversionReviewVersions: - - v1beta2 - - v1beta1 - group: kmm.sigs.x-k8s.io - names: - kind: PreflightValidation - listKind: PreflightValidationList - plural: preflightvalidations - shortNames: - - pfv - singular: preflightvalidation - scope: Cluster - versions: - - deprecated: true - name: v1beta1 - schema: - openAPIV3Schema: - description: PreflightValidation initiates a preflight validations for all Modules - on the current Kubernetes cluster. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: |- - PreflightValidationSpec describes the desired state of the resource, such as the kernel version - that Module CRs need to be verified against as well as the debug configuration of the logs - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - kernelVersion: - description: KernelVersion describes the kernel image that all Modules - need to be checked against. - type: string - pushBuiltImage: - description: |- - Boolean flag that determines whether images build during preflight must also - be pushed to a defined repository - type: boolean - required: - - kernelVersion - type: object - status: - description: |- - PreflightValidationStatus is the most recently observed status of the PreflightValidation. - It is populated by the system and is read-only. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - crStatuses: - additionalProperties: - properties: - lastTransitionTime: - description: |- - LastTransitionTime is the last time the CR status transitioned from one status to another. - This should be when the underlying status changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - statusReason: - description: StatusReason contains a string describing the status - source. - type: string - verificationStage: - description: |- - Current stage of the verification process: - image (image existence verification), build(build process verification) - enum: - - Image - - Build - - Sign - - Requeued - - Done - type: string - verificationStatus: - description: |- - Status of Module CR verification: true (verified), false (verification failed), - error (error during verification process), unknown (verification has not started yet) - enum: - - "True" - - "False" - type: string - required: - - lastTransitionTime - - verificationStage - - verificationStatus - type: object - description: CRStatuses contain observations about each Module's preflight - upgradability validation - type: object - type: object - type: object - served: true - storage: false - subresources: - status: {} - - name: v1beta2 - schema: - openAPIV3Schema: - description: PreflightValidation initiates a preflight validations for all Modules - on the current Kubernetes cluster. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: |- - PreflightValidationSpec describes the desired state of the resource, such as the kernel version - that Module CRs need to be verified against as well as the debug configuration of the logs - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - kernelVersion: - description: KernelVersion describes the kernel image that all Modules - need to be checked against. - type: string - pushBuiltImage: - description: |- - Boolean flag that determines whether images build during preflight must also - be pushed to a defined repository - type: boolean - required: - - kernelVersion - type: object - status: - description: |- - PreflightValidationStatus is the most recently observed status of the PreflightValidation. - It is populated by the system and is read-only. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - modules: - description: Modules contain observations about each Module's preflight - upgradability validation - items: - properties: - lastTransitionTime: - description: |- - LastTransitionTime is the last time the CR status transitioned from one status to another. - This should be when the underlying status changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - name: - description: Name is the name of the Module resource. - type: string - namespace: - description: Namespace is the namespace of the Module resource. - type: string - statusReason: - description: StatusReason contains a string describing the status - source. - type: string - verificationStage: - description: |- - Current stage of the verification process: - image (image existence verification), build(build process verification) - enum: - - Image - - Build - - Sign - - Requeued - - Done - type: string - verificationStatus: - description: |- - Status of Module CR verification: true (verified), false (verification failed), - error (error during verification process), unknown (verification has not started yet) - enum: - - "True" - - "False" - type: string - required: - - lastTransitionTime - - name - - namespace - - verificationStage - - verificationStatus - type: object - type: array - x-kubernetes-list-map-keys: - - namespace - - name - x-kubernetes-list-type: map - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] \ No newline at end of file diff --git a/helm-charts-openshift/charts/kmm/templates/preflightvalidationocp-crd.yaml b/helm-charts-openshift/charts/kmm/templates/preflightvalidationocp-crd.yaml deleted file mode 100644 index 8770b9590..000000000 --- a/helm-charts-openshift/charts/kmm/templates/preflightvalidationocp-crd.yaml +++ /dev/null @@ -1,247 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: preflightvalidationsocp.kmm.sigs.x-k8s.io - annotations: - controller-gen.kubebuilder.io/version: v0.14.0 - service.beta.openshift.io/inject-cabundle: "true" - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - name: '{{ include "kmm.fullname" . }}-webhook-service' - namespace: '{{ .Release.Namespace }}' - path: /convert - conversionReviewVersions: - - v1beta2 - - v1beta1 - group: kmm.sigs.x-k8s.io - names: - kind: PreflightValidationOCP - listKind: PreflightValidationOCPList - plural: preflightvalidationsocp - shortNames: - - pfvo - singular: preflightvalidationocp - scope: Cluster - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: PreflightValidationOCP initiates a preflight validations for all - Modules on the current OCP cluster. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: |- - PreflightValidationOCPSpec describes the desired state of the resource, such as the OCP release image - that Module CRs need to be verified against as well as the push image flag - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - pushBuiltImage: - description: |- - Boolean flag that determines whether images build during preflight must also - be pushed to a defined repository - type: boolean - releaseImage: - description: releaseImage describes the OCP release image that all Modules - need to be checked against. - type: string - useRTKernel: - description: |- - Boolean flag that determines whether the preflight should be checked with RT kernel version - instead of Full kernel version - type: boolean - required: - - releaseImage - type: object - status: - description: |- - PreflightValidationStatus is the most recently observed status of the PreflightValidation. - It is populated by the system and is read-only. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - crStatuses: - additionalProperties: - properties: - lastTransitionTime: - description: |- - LastTransitionTime is the last time the CR status transitioned from one status to another. - This should be when the underlying status changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - statusReason: - description: StatusReason contains a string describing the status - source. - type: string - verificationStage: - description: |- - Current stage of the verification process: - image (image existence verification), build(build process verification) - enum: - - Image - - Build - - Sign - - Requeued - - Done - type: string - verificationStatus: - description: |- - Status of Module CR verification: true (verified), false (verification failed), - error (error during verification process), unknown (verification has not started yet) - enum: - - "True" - - "False" - type: string - required: - - lastTransitionTime - - verificationStage - - verificationStatus - type: object - description: CRStatuses contain observations about each Module's preflight - upgradability validation - type: object - type: object - type: object - served: true - storage: false - subresources: - status: {} - - name: v1beta2 - schema: - openAPIV3Schema: - description: PreflightValidationOCP initiates a preflight validations for all - Modules on the current OCP cluster. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: |- - PreflightValidationOCPSpec describes the desired state of the resource, such as the OCP release image - that Module CRs need to be verified against as well as the push image flag - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - pushBuiltImage: - description: |- - Boolean flag that determines whether images build during preflight must also - be pushed to a defined repository - type: boolean - releaseImage: - description: releaseImage describes the OCP release image that all Modules - need to be checked against. - type: string - useRTKernel: - description: |- - Boolean flag that determines whether the preflight should be checked with RT kernel version - instead of Full kernel version - type: boolean - required: - - releaseImage - type: object - status: - description: |- - PreflightValidationStatus is the most recently observed status of the PreflightValidation. - It is populated by the system and is read-only. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - modules: - description: Modules contain observations about each Module's preflight - upgradability validation - items: - properties: - lastTransitionTime: - description: |- - LastTransitionTime is the last time the CR status transitioned from one status to another. - This should be when the underlying status changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - name: - description: Name is the name of the Module resource. - type: string - namespace: - description: Namespace is the namespace of the Module resource. - type: string - statusReason: - description: StatusReason contains a string describing the status - source. - type: string - verificationStage: - description: |- - Current stage of the verification process: - image (image existence verification), build(build process verification) - enum: - - Image - - Build - - Sign - - Requeued - - Done - type: string - verificationStatus: - description: |- - Status of Module CR verification: true (verified), false (verification failed), - error (error during verification process), unknown (verification has not started yet) - enum: - - "True" - - "False" - type: string - required: - - lastTransitionTime - - name - - namespace - - verificationStage - - verificationStatus - type: object - type: array - x-kubernetes-list-map-keys: - - namespace - - name - x-kubernetes-list-type: map - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] \ No newline at end of file diff --git a/helm-charts-openshift/charts/kmm/templates/prometheus-k8s-rbac.yaml b/helm-charts-openshift/charts/kmm/templates/prometheus-k8s-rbac.yaml deleted file mode 100644 index 24fdd1842..000000000 --- a/helm-charts-openshift/charts/kmm/templates/prometheus-k8s-rbac.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "kmm.fullname" . }}-prometheus-k8s - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - services - - endpoints - - pods - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "kmm.fullname" . }}-prometheus-k8s - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: '{{ include "kmm.fullname" . }}-prometheus-k8s' -subjects: -- kind: ServiceAccount - name: prometheus-k8s - namespace: '{{ .Release.Namespace }}' \ No newline at end of file diff --git a/helm-charts-openshift/charts/kmm/templates/proxy-rbac.yaml b/helm-charts-openshift/charts/kmm/templates/proxy-rbac.yaml deleted file mode 100644 index 6cc30bba1..000000000 --- a/helm-charts-openshift/charts/kmm/templates/proxy-rbac.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "kmm.fullname" . }}-proxy-role - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "kmm.fullname" . }}-proxy-rolebinding - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "kmm.fullname" . }}-proxy-role' -subjects: -- kind: ServiceAccount - name: '{{ include "kmm.fullname" . }}-controller' - namespace: '{{ .Release.Namespace }}' \ No newline at end of file diff --git a/helm-charts-openshift/charts/kmm/templates/service-ca.yaml b/helm-charts-openshift/charts/kmm/templates/service-ca.yaml deleted file mode 100644 index 2c1179663..000000000 --- a/helm-charts-openshift/charts/kmm/templates/service-ca.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "kmm.fullname" . }}-service-ca - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - kmm.openshift.io/ca.type: service - {{- include "kmm.labels" . | nindent 4 }} - annotations: - service.beta.openshift.io/inject-cabundle: "true" -data: {} \ No newline at end of file diff --git a/helm-charts-openshift/charts/kmm/templates/serviceaccount.yaml b/helm-charts-openshift/charts/kmm/templates/serviceaccount.yaml deleted file mode 100644 index fb58d569c..000000000 --- a/helm-charts-openshift/charts/kmm/templates/serviceaccount.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "kmm.fullname" . }}-controller - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - {{- toYaml .Values.controllerManager.serviceAccount.annotations | nindent 4 }} diff --git a/helm-charts-openshift/charts/kmm/templates/validating-webhook-configuration.yaml b/helm-charts-openshift/charts/kmm/templates/validating-webhook-configuration.yaml deleted file mode 100644 index 33267a878..000000000 --- a/helm-charts-openshift/charts/kmm/templates/validating-webhook-configuration.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: {{ include "kmm.fullname" . }}-validating-webhook-configuration - annotations: - service.beta.openshift.io/inject-cabundle: 'true' - labels: - {{- include "kmm.labels" . | nindent 4 }} -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: '{{ include "kmm.fullname" . }}-webhook-service' - namespace: '{{ .Release.Namespace }}' - path: /validate--v1-namespace - failurePolicy: Fail - name: namespace-deletion.kmm.sigs.k8s.io - namespaceSelector: - matchLabels: - kmm.node.k8s.io/contains-modules: "" - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - DELETE - resources: - - namespaces - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: '{{ include "kmm.fullname" . }}-webhook-service' - namespace: '{{ .Release.Namespace }}' - path: /validate-kmm-sigs-x-k8s-io-v1beta1-module - failurePolicy: Fail - name: vmodule.kb.io - rules: - - apiGroups: - - kmm.sigs.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - modules - sideEffects: None \ No newline at end of file diff --git a/helm-charts-openshift/charts/kmm/templates/webhook-service.yaml b/helm-charts-openshift/charts/kmm/templates/webhook-service.yaml deleted file mode 100644 index 30d8a4857..000000000 --- a/helm-charts-openshift/charts/kmm/templates/webhook-service.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "kmm.fullname" . }}-webhook-service - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/created-by: kernel-module-management - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} - annotations: - service.beta.openshift.io/serving-cert-secret-name: {{ include "kmm.fullname" . }}-webhook-server-cert -spec: - type: {{ .Values.webhookService.type }} - selector: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: webhook-server - {{- include "kmm.selectorLabels" . | nindent 4 }} - ports: - {{- .Values.webhookService.ports | toYaml | nindent 2 }} diff --git a/helm-charts-openshift/charts/kmm/values.yaml b/helm-charts-openshift/charts/kmm/values.yaml deleted file mode 100644 index 25ae6204c..000000000 --- a/helm-charts-openshift/charts/kmm/values.yaml +++ /dev/null @@ -1,134 +0,0 @@ -controller: - manager: - args: - - --config=controller_config.yaml - containerSecurityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - env: - relatedImageMustGather: quay.io/edge-infrastructure/kernel-module-management-must-gather:release-2.1 - relatedImageSign: quay.io/edge-infrastructure/kernel-module-management-signimage:release-2.1 - relatedImageWorker: quay.io/edge-infrastructure/kernel-module-management-worker:release-2.1 - sslCertDir: /etc/pki/ca-trust/extracted/pem - image: - repository: quay.io/edge-infrastructure/kernel-module-management-operator - tag: release-2.1 - imagePullPolicy: Always - imagePullSecrets: "" - tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Equal" - value: "" - effect: "NoSchedule" - - key: "node-role.kubernetes.io/control-plane" - operator: "Equal" - value: "" - effect: "NoSchedule" - resources: - limits: - cpu: 500m - memory: 384Mi - requests: - cpu: 10m - memory: 64Mi - nodeSelector: {} - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - replicas: 1 - serviceAccount: - annotations: {} -controllerManager: - serviceAccount: - annotations: {} -controllerMetricsService: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: https - type: ClusterIP -kmmDevicePlugin: - serviceAccount: - annotations: {} -kubernetesClusterDomain: cluster.local -managerConfig: - controllerConfigYaml: |- - healthProbeBindAddress: :8081 - leaderElection: - enabled: true - resourceID: kmm.sigs.x-k8s.io - webhook: - disableHTTP2: true # CVE-2023-44487 - port: 9443 - metrics: - enableAuthnAuthz: true - disableHTTP2: true # CVE-2023-44487 - bindAddress: 0.0.0.0:8443 - secureServing: true - worker: - runAsUser: 0 - seLinuxType: spc_t - setFirmwareClassPath: /var/lib/firmware -kmmModuleLoader: - serviceAccount: - annotations: {} -nodeLabeller: - serviceAccount: - annotations: {} -webhookServer: - replicas: 1 - nodeSelector: {} - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - webhookServer: - args: - - --config=controller_config.yaml - - --enable-module - - --enable-namespace - - --enable-preflightvalidation - containerSecurityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - image: - repository: quay.io/edge-infrastructure/kernel-module-management-webhook-server - tag: latest - imagePullPolicy: Always - imagePullSecrets: "" - tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Equal" - value: "" - effect: "NoSchedule" - - key: "node-role.kubernetes.io/control-plane" - operator: "Equal" - value: "" - effect: "NoSchedule" - resources: - limits: - cpu: 500m - memory: 384Mi - requests: - cpu: 10m - memory: 64Mi -webhookService: - ports: - - port: 443 - protocol: TCP - targetPort: 9443 - type: ClusterIP diff --git a/helm-charts-openshift/charts/nfd/.helmignore b/helm-charts-openshift/charts/nfd/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/helm-charts-openshift/charts/nfd/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/helm-charts-openshift/charts/nfd/Chart.yaml b/helm-charts-openshift/charts/nfd/Chart.yaml deleted file mode 100644 index b185f251d..000000000 --- a/helm-charts-openshift/charts/nfd/Chart.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v2 -name: nfd -description: A Helm chart for deploying Cluster NFD Operator Kubernetes -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: v1.0.0 -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "v4.16" \ No newline at end of file diff --git a/helm-charts-openshift/charts/nfd/crds/nodefeature-crd.yaml b/helm-charts-openshift/charts/nfd/crds/nodefeature-crd.yaml deleted file mode 100644 index 42496a70f..000000000 --- a/helm-charts-openshift/charts/nfd/crds/nodefeature-crd.yaml +++ /dev/null @@ -1,128 +0,0 @@ ---- -# Source: nfd/templates/nodefeature-crd.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: nodefeatures.nfd.openshift.io - annotations: - controller-gen.kubebuilder.io/version: v0.14.0 - labels: - helm.sh/chart: nfd-v1.0.0 - app.kubernetes.io/name: nfd - app.kubernetes.io/instance: amd-gpu - app.kubernetes.io/version: "v4.16" - app.kubernetes.io/managed-by: Helm -spec: - group: nfd.openshift.io - names: - kind: NodeFeature - listKind: NodeFeatureList - plural: nodefeatures - singular: nodefeature - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: |- - NodeFeature resource holds the features discovered for one node in the - cluster. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: NodeFeatureSpec describes a NodeFeature object. - properties: - features: - description: Features is the full "raw" features data that has been - discovered. - properties: - attributes: - additionalProperties: - description: AttributeFeatureSet is a set of features having string - value. - properties: - elements: - additionalProperties: - type: string - type: object - required: - - elements - type: object - type: object - flags: - additionalProperties: - description: FlagFeatureSet is a set of simple features only containing - names without values. - properties: - elements: - additionalProperties: - description: Nil is a dummy empty struct for protobuf compatibility - type: object - type: object - required: - - elements - type: object - type: object - instances: - additionalProperties: - description: InstanceFeatureSet is a set of features each of which - is an instance having multiple attributes. - properties: - elements: - items: - description: InstanceFeature represents one instance of - a complex features, e.g. a device. - properties: - attributes: - additionalProperties: - type: string - type: object - required: - - attributes - type: object - type: array - required: - - elements - type: object - type: object - required: - - attributes - - flags - - instances - type: object - labels: - additionalProperties: - type: string - description: Labels is the set of node labels that are requested to - be created. - type: object - required: - - features - type: object - required: - - spec - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/helm-charts-openshift/charts/nfd/crds/nodefeaturediscovery-crd.yaml b/helm-charts-openshift/charts/nfd/crds/nodefeaturediscovery-crd.yaml deleted file mode 100644 index 6d49caddc..000000000 --- a/helm-charts-openshift/charts/nfd/crds/nodefeaturediscovery-crd.yaml +++ /dev/null @@ -1,211 +0,0 @@ ---- -# Source: nfd/templates/nodefeaturediscovery-crd.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: nodefeaturediscoveries.nfd.openshift.io - annotations: - controller-gen.kubebuilder.io/version: v0.14.0 - labels: - helm.sh/chart: nfd-v1.0.0 - app.kubernetes.io/name: nfd - app.kubernetes.io/instance: amd-gpu - app.kubernetes.io/version: "v4.16" - app.kubernetes.io/managed-by: Helm -spec: - group: nfd.openshift.io - names: - kind: NodeFeatureDiscovery - listKind: NodeFeatureDiscoveryList - plural: nodefeaturediscoveries - singular: nodefeaturediscovery - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - description: NodeFeatureDiscovery is the Schema for the nodefeaturediscoveries - API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: NodeFeatureDiscoverySpec defines the desired state of NodeFeatureDiscovery - properties: - extraLabelNs: - description: |- - ExtraLabelNs defines the list of of allowed extra label namespaces - By default, only allow labels in the default `feature.node.kubernetes.io` label namespace - items: - type: string - nullable: true - type: array - instance: - description: |- - Instance name. Used to separate annotation namespaces for - multiple parallel deployments. - type: string - labelWhiteList: - description: |- - LabelWhiteList defines a regular expression - for filtering feature labels based on their name. - Each label must match against the given reqular expression in order to be published. - nullable: true - type: string - operand: - description: OperandSpec describes configuration options for the operand - properties: - image: - description: |- - Image defines the image to pull for the - NFD operand - [defaults to registry.k8s.io/nfd/node-feature-discovery] - pattern: '[a-zA-Z0-9\-]+' - type: string - imagePullPolicy: - description: |- - ImagePullPolicy defines Image pull policy for the - NFD operand image [defaults to Always] - type: string - servicePort: - description: |- - ServicePort specifies the TCP port that nfd-master - listens for incoming requests. - type: integer - type: object - prunerOnDelete: - description: |- - PruneOnDelete defines whether the NFD-master prune should be - enabled or not. If enabled, the Operator will deploy an NFD-Master prune - job that will remove all NFD labels (and other NFD-managed assets such - as annotations, extended resources and taints) from the cluster nodes. - type: boolean - resourceLabels: - description: |- - ResourceLabels defines the list of features - to be advertised as extended resources instead of labels. - items: - type: string - nullable: true - type: array - topologyUpdater: - description: |- - Deploy the NFD-Topology-Updater - NFD-Topology-Updater is a daemon responsible for examining allocated - resources on a worker node to account for resources available to be - allocated to new pod on a per-zone basis - https://kubernetes-sigs.github.io/node-feature-discovery/master/get-started/introduction.html#nfd-topology-updater - type: boolean - workerConfig: - description: |- - WorkerConfig describes configuration options for the NFD - worker. - properties: - configData: - description: BinaryData holds the NFD configuration file - type: string - required: - - configData - type: object - type: object - status: - description: NodeFeatureDiscoveryStatus defines the observed state of NodeFeatureDiscovery - properties: - conditions: - description: Conditions represents the latest available observations - of current state. - items: - description: "Condition contains details for one aspect of the current - state of this API Resource.\n---\nThis struct is intended for direct - use as an array at the field path .status.conditions. For example,\n\n\n\ttype - FooStatus struct{\n\t // Represents the observations of a foo's - current state.\n\t // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\"\n\t // +patchMergeKey=type\n\t - \ // +patchStrategy=merge\n\t // +listType=map\n\t // +listMapKey=type\n\t - \ Conditions []metav1.Condition `json:\"conditions,omitempty\" - patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t - \ // other fields\n\t}" - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: |- - type of condition in CamelCase or in foo.example.com/CamelCase. - --- - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be - useful (see .node.status.conditions), the ability to deconflict is important. - The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/helm-charts-openshift/charts/nfd/crds/nodefeaturerule-crd.yaml b/helm-charts-openshift/charts/nfd/crds/nodefeaturerule-crd.yaml deleted file mode 100644 index e8de37783..000000000 --- a/helm-charts-openshift/charts/nfd/crds/nodefeaturerule-crd.yaml +++ /dev/null @@ -1,330 +0,0 @@ ---- -# Source: nfd/templates/nodefeaturerule-crd.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: nodefeaturerules.nfd.openshift.io - annotations: - controller-gen.kubebuilder.io/version: v0.14.0 - labels: - helm.sh/chart: nfd-v1.0.0 - app.kubernetes.io/name: nfd - app.kubernetes.io/instance: amd-gpu - app.kubernetes.io/version: "v4.16" - app.kubernetes.io/managed-by: Helm -spec: - group: nfd.openshift.io - names: - kind: NodeFeatureRule - listKind: NodeFeatureRuleList - plural: nodefeaturerules - shortNames: - - nfr - singular: nodefeaturerule - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: |- - NodeFeatureRule resource specifies a configuration for feature-based - customization of node objects, such as node labeling. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: NodeFeatureRuleSpec describes a NodeFeatureRule. - properties: - rules: - description: Rules is a list of node customization rules. - items: - description: Rule defines a rule for node customization such as labeling. - properties: - annotations: - additionalProperties: - type: string - description: Annotations to create if the rule matches. - type: object - extendedResources: - additionalProperties: - type: string - description: ExtendedResources to create if the rule matches. - type: object - labels: - additionalProperties: - type: string - description: Labels to create if the rule matches. - type: object - labelsTemplate: - description: |- - LabelsTemplate specifies a template to expand for dynamically generating - multiple labels. Data (after template expansion) must be keys with an - optional value ([=]) separated by newlines. - type: string - matchAny: - description: MatchAny specifies a list of matchers one of which - must match. - items: - description: MatchAnyElem specifies one sub-matcher of MatchAny. - properties: - matchFeatures: - description: MatchFeatures specifies a set of matcher terms - all of which must match. - items: - description: |- - FeatureMatcherTerm defines requirements against one feature set. All - requirements (specified as MatchExpressions) are evaluated against each - element in the feature set. - properties: - feature: - description: Feature is the name of the feature set - to match against. - type: string - matchExpressions: - additionalProperties: - description: |- - MatchExpression specifies an expression to evaluate against a set of input - values. It contains an operator that is applied when matching the input and - an array of values that the operator evaluates the input against. - properties: - op: - description: Op is the operator to be applied. - enum: - - In - - NotIn - - InRegexp - - Exists - - DoesNotExist - - Gt - - Lt - - GtLt - - IsTrue - - IsFalse - type: string - value: - description: |- - Value is the list of values that the operand evaluates the input - against. Value should be empty if the operator is Exists, DoesNotExist, - IsTrue or IsFalse. Value should contain exactly one element if the - operator is Gt or Lt and exactly two elements if the operator is GtLt. - In other cases Value should contain at least one element. - items: - type: string - type: array - required: - - op - type: object - description: |- - MatchExpressions is the set of per-element expressions evaluated. These - match against the value of the specified elements. - type: object - matchName: - description: |- - MatchName in an expression that is matched against the name of each - element in the feature set. - properties: - op: - description: Op is the operator to be applied. - enum: - - In - - NotIn - - InRegexp - - Exists - - DoesNotExist - - Gt - - Lt - - GtLt - - IsTrue - - IsFalse - type: string - value: - description: |- - Value is the list of values that the operand evaluates the input - against. Value should be empty if the operator is Exists, DoesNotExist, - IsTrue or IsFalse. Value should contain exactly one element if the - operator is Gt or Lt and exactly two elements if the operator is GtLt. - In other cases Value should contain at least one element. - items: - type: string - type: array - required: - - op - type: object - required: - - feature - type: object - type: array - required: - - matchFeatures - type: object - type: array - matchFeatures: - description: MatchFeatures specifies a set of matcher terms all - of which must match. - items: - description: |- - FeatureMatcherTerm defines requirements against one feature set. All - requirements (specified as MatchExpressions) are evaluated against each - element in the feature set. - properties: - feature: - description: Feature is the name of the feature set to match - against. - type: string - matchExpressions: - additionalProperties: - description: |- - MatchExpression specifies an expression to evaluate against a set of input - values. It contains an operator that is applied when matching the input and - an array of values that the operator evaluates the input against. - properties: - op: - description: Op is the operator to be applied. - enum: - - In - - NotIn - - InRegexp - - Exists - - DoesNotExist - - Gt - - Lt - - GtLt - - IsTrue - - IsFalse - type: string - value: - description: |- - Value is the list of values that the operand evaluates the input - against. Value should be empty if the operator is Exists, DoesNotExist, - IsTrue or IsFalse. Value should contain exactly one element if the - operator is Gt or Lt and exactly two elements if the operator is GtLt. - In other cases Value should contain at least one element. - items: - type: string - type: array - required: - - op - type: object - description: |- - MatchExpressions is the set of per-element expressions evaluated. These - match against the value of the specified elements. - type: object - matchName: - description: |- - MatchName in an expression that is matched against the name of each - element in the feature set. - properties: - op: - description: Op is the operator to be applied. - enum: - - In - - NotIn - - InRegexp - - Exists - - DoesNotExist - - Gt - - Lt - - GtLt - - IsTrue - - IsFalse - type: string - value: - description: |- - Value is the list of values that the operand evaluates the input - against. Value should be empty if the operator is Exists, DoesNotExist, - IsTrue or IsFalse. Value should contain exactly one element if the - operator is Gt or Lt and exactly two elements if the operator is GtLt. - In other cases Value should contain at least one element. - items: - type: string - type: array - required: - - op - type: object - required: - - feature - type: object - type: array - name: - description: Name of the rule. - type: string - taints: - description: Taints to create if the rule matches. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Required. The taint key to be applied to a - node. - type: string - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint - key. - type: string - required: - - effect - - key - type: object - type: array - vars: - additionalProperties: - type: string - description: |- - Vars is the variables to store if the rule matches. Variables do not - directly inflict any changes in the node object. However, they can be - referenced from other rules enabling more complex rule hierarchies, - without exposing intermediary output values as labels. - type: object - varsTemplate: - description: |- - VarsTemplate specifies a template to expand for dynamically generating - multiple variables. Data (after template expansion) must be keys with an - optional value ([=]) separated by newlines. - type: string - required: - - name - type: object - type: array - required: - - rules - type: object - required: - - spec - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/helm-charts-openshift/charts/nfd/templates/_helpers.tpl b/helm-charts-openshift/charts/nfd/templates/_helpers.tpl deleted file mode 100644 index e90c77d92..000000000 --- a/helm-charts-openshift/charts/nfd/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "nfd.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "nfd.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "nfd.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "nfd.labels" -}} -helm.sh/chart: {{ include "nfd.chart" . }} -{{ include "nfd.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "nfd.selectorLabels" -}} -app.kubernetes.io/name: {{ include "nfd.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "nfd.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "nfd.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/helm-charts-openshift/charts/nfd/templates/controller-manager-alerts-monitor.yaml b/helm-charts-openshift/charts/nfd/templates/controller-manager-alerts-monitor.yaml deleted file mode 100644 index f492cea69..000000000 --- a/helm-charts-openshift/charts/nfd/templates/controller-manager-alerts-monitor.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ include "nfd.fullname" . }}-controller-manager-alerts-monitor - labels: - role: alert-rules - {{- include "nfd.labels" . | nindent 4 }} -spec: - groups: - - name: node-feature-discovery-operator.rules - rules: - - alert: NFDDegraded - annotations: - message: | - The Node Feature Discovery Operator is degraded. Review the "NodeFeatureDiscovery" CustomResource object for further details. - expr: nfd_degraded_info == 1 - for: 1h - labels: - severity: warning \ No newline at end of file diff --git a/helm-charts-openshift/charts/nfd/templates/controller-manager-metrics-monitor.yaml b/helm-charts-openshift/charts/nfd/templates/controller-manager-metrics-monitor.yaml deleted file mode 100644 index ac777e164..000000000 --- a/helm-charts-openshift/charts/nfd/templates/controller-manager-metrics-monitor.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: {{ include "nfd.fullname" . }}-controller-manager-metrics-monitor - labels: - control-plane: controller-manager - {{- include "nfd.labels" . | nindent 4 }} -spec: - endpoints: - - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - interval: 30s - path: /metrics - port: https - scheme: https - tlsConfig: - caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt - serverName: nfd-controller-manager-metrics-service.openshift-nfd.svc - selector: - matchLabels: - control-plane: controller-manager \ No newline at end of file diff --git a/helm-charts-openshift/charts/nfd/templates/deployment.yaml b/helm-charts-openshift/charts/nfd/templates/deployment.yaml deleted file mode 100644 index 0f730f72d..000000000 --- a/helm-charts-openshift/charts/nfd/templates/deployment.yaml +++ /dev/null @@ -1,85 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "nfd.fullname" . }}-controller-manager - labels: - control-plane: controller-manager - {{- include "nfd.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.controllerManager.replicas }} - selector: - matchLabels: - control-plane: controller-manager - {{- include "nfd.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - control-plane: controller-manager - {{- include "nfd.selectorLabels" . | nindent 8 }} - spec: - containers: - - args: {{- toYaml .Values.controllerManager.kubeRbacProxy.args | nindent 8 }} - env: - - name: KUBERNETES_CLUSTER_DOMAIN - value: {{ quote .Values.kubernetesClusterDomain }} - image: {{ .Values.controllerManager.kubeRbacProxy.image.repository }}:{{ .Values.controllerManager.kubeRbacProxy.image.tag - | default .Chart.AppVersion }} - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - resources: {{- toYaml .Values.controllerManager.kubeRbacProxy.resources | nindent - 10 }} - securityContext: {{- toYaml .Values.controllerManager.kubeRbacProxy.containerSecurityContext - | nindent 10 }} - volumeMounts: - - mountPath: /etc/secrets - name: node-feature-discovery-operator-tls - - args: {{- toYaml .Values.controllerManager.manager.args | nindent 8 }} - command: - - /node-feature-discovery-operator - env: - - name: SSL_CERT_DIR - value: {{ quote .Values.controllerManager.manager.env.sslCertDir }} - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: OPERATOR_NAME - value: {{ quote .Values.controllerManager.manager.env.operatorName }} - - name: NODE_FEATURE_DISCOVERY_IMAGE - value: {{ quote .Values.controllerManager.manager.env.nodeFeatureDiscoveryImage - }} - - name: KUBERNETES_CLUSTER_DOMAIN - value: {{ quote .Values.kubernetesClusterDomain }} - image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag - | default .Chart.AppVersion }} - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - ports: - - containerPort: 8080 - name: metrics - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: {} - securityContext: {{- toYaml .Values.controllerManager.manager.containerSecurityContext - | nindent 10 }} - serviceAccountName: {{ include "nfd.fullname" . }}-operator - terminationGracePeriodSeconds: 10 - volumes: - - name: node-feature-discovery-operator-tls - secret: - secretName: node-feature-discovery-operator-tls \ No newline at end of file diff --git a/helm-charts-openshift/charts/nfd/templates/leader-election-rbac.yaml b/helm-charts-openshift/charts/nfd/templates/leader-election-rbac.yaml deleted file mode 100644 index 1c02feb83..000000000 --- a/helm-charts-openshift/charts/nfd/templates/leader-election-rbac.yaml +++ /dev/null @@ -1,96 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "nfd.fullname" . }}-leader-election-role - labels: - {{- include "nfd.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - - coordination.k8s.io - resources: - - configmaps - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - rbac.authorization.k8s.io - resources: - - roles - - rolebindings - - clusterroles - - clusterrolebindings - verbs: - - '*' -- apiGroups: - - "" - resources: - - pods - - services - - endpoints - - persistentvolumeclaims - - events - - configmaps - - secrets - - serviceaccounts - - nodes - verbs: - - '*' -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get - - list - - watch -- apiGroups: - - apps - resources: - - deployments - - daemonsets - - replicasets - - statefulsets - verbs: - - '*' -- apiGroups: - - monitoring.coreos.com - resources: - - servicemonitors - verbs: - - get - - create -- apiGroups: - - nfd.openshift.io - resources: - - '*' - verbs: - - '*' ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "nfd.fullname" . }}-leader-election-rolebinding - labels: - {{- include "nfd.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: '{{ include "nfd.fullname" . }}-leader-election-role' -subjects: -- kind: ServiceAccount - name: '{{ include "nfd.fullname" . }}-operator' - namespace: '{{ .Release.Namespace }}' \ No newline at end of file diff --git a/helm-charts-openshift/charts/nfd/templates/manager-config.yaml b/helm-charts-openshift/charts/nfd/templates/manager-config.yaml deleted file mode 100644 index e01664d3b..000000000 --- a/helm-charts-openshift/charts/nfd/templates/manager-config.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "nfd.fullname" . }}-manager-config - labels: - {{- include "nfd.labels" . | nindent 4 }} -data: - controller_manager_config.yaml: {{ .Values.managerConfig.controllerManagerConfigYaml - | toYaml | indent 1 }} \ No newline at end of file diff --git a/helm-charts-openshift/charts/nfd/templates/metrics-reader-rbac.yaml b/helm-charts-openshift/charts/nfd/templates/metrics-reader-rbac.yaml deleted file mode 100644 index db129ad47..000000000 --- a/helm-charts-openshift/charts/nfd/templates/metrics-reader-rbac.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "nfd.fullname" . }}-metrics-reader - labels: - {{- include "nfd.labels" . | nindent 4 }} -rules: -- nonResourceURLs: - - /metrics - verbs: - - get \ No newline at end of file diff --git a/helm-charts-openshift/charts/nfd/templates/metrics-service.yaml b/helm-charts-openshift/charts/nfd/templates/metrics-service.yaml deleted file mode 100644 index 8b2f7aa02..000000000 --- a/helm-charts-openshift/charts/nfd/templates/metrics-service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "nfd.fullname" . }}-controller-manager-metrics-service - labels: - control-plane: controller-manager - {{- include "nfd.labels" . | nindent 4 }} - annotations: - service.beta.openshift.io/serving-cert-secret-name: node-feature-discovery-operator-tls -spec: - type: {{ .Values.metricsService.type }} - selector: - control-plane: controller-manager - {{- include "nfd.selectorLabels" . | nindent 4 }} - ports: - {{- .Values.metricsService.ports | toYaml | nindent 2 }} \ No newline at end of file diff --git a/helm-charts-openshift/charts/nfd/templates/operator-rbac.yaml b/helm-charts-openshift/charts/nfd/templates/operator-rbac.yaml deleted file mode 100644 index b77a5c6c5..000000000 --- a/helm-charts-openshift/charts/nfd/templates/operator-rbac.yaml +++ /dev/null @@ -1,279 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "nfd.fullname" . }}-operator - labels: - {{- include "nfd.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - update - - watch -- apiGroups: - - apps - resources: - - daemonsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - deployments - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - cert-manager.io - resources: - - certificates - verbs: - - get - - list - - watch -- apiGroups: - - cert-manager.io - resources: - - issuers - verbs: - - get - - list - - watch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - delete - - get - - list - - update - - watch -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - namespaces - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - nodes/status - verbs: - - get - - list - - patch - - update -- apiGroups: - - "" - resources: - - pods - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - nfd.openshift.io - resources: - - nodefeaturediscoveries - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - nfd.openshift.io - resources: - - nodefeaturediscoveries/finalizers - verbs: - - update -- apiGroups: - - nfd.openshift.io - resources: - - nodefeaturediscoveries/status - verbs: - - get - - patch - - update -- apiGroups: - - nfd.openshift.io - resources: - - nodefeaturerules - verbs: - - get - - list - - watch -- apiGroups: - - nfd.openshift.io - resources: - - nodefeatures - verbs: - - get - - list - - watch -- apiGroups: - - policy - resourceNames: - - nfd-worker - resources: - - podsecuritypolicies - verbs: - - use -- apiGroups: - - rbac.authorization.k8s.io - resources: - - clusterrolebindings - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - rbac.authorization.k8s.io - resources: - - clusterroles - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - rbac.authorization.k8s.io - resources: - - rolebindings - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - rbac.authorization.k8s.io - resources: - - roles - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - security.openshift.io - resources: - - securitycontextconstraints - verbs: - - create - - delete - - get - - list - - patch - - update - - use - - watch -- apiGroups: - - topology.node.k8s.io - resources: - - noderesourcetopologies - verbs: - - create - - get - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "nfd.fullname" . }}-operator - labels: - {{- include "nfd.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "nfd.fullname" . }}-operator' -subjects: -- kind: ServiceAccount - name: '{{ include "nfd.fullname" . }}-operator' - namespace: '{{ .Release.Namespace }}' \ No newline at end of file diff --git a/helm-charts-openshift/charts/nfd/templates/prometheus-k8s-rbac.yaml b/helm-charts-openshift/charts/nfd/templates/prometheus-k8s-rbac.yaml deleted file mode 100644 index 2e16fab47..000000000 --- a/helm-charts-openshift/charts/nfd/templates/prometheus-k8s-rbac.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "nfd.fullname" . }}-prometheus-k8s - labels: - {{- include "nfd.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - services - - endpoints - - pods - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "nfd.fullname" . }}-prometheus-k8s - labels: - {{- include "nfd.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: '{{ include "nfd.fullname" . }}-prometheus-k8s' -subjects: -- kind: ServiceAccount - name: prometheus-k8s - namespace: '{{ .Release.Namespace }}' \ No newline at end of file diff --git a/helm-charts-openshift/charts/nfd/templates/proxy-rbac.yaml b/helm-charts-openshift/charts/nfd/templates/proxy-rbac.yaml deleted file mode 100644 index 5e02d4279..000000000 --- a/helm-charts-openshift/charts/nfd/templates/proxy-rbac.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "nfd.fullname" . }}-proxy-role - labels: - {{- include "nfd.labels" . | nindent 4 }} -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "nfd.fullname" . }}-proxy-rolebinding - labels: - {{- include "nfd.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "nfd.fullname" . }}-proxy-role' -subjects: -- kind: ServiceAccount - name: '{{ include "nfd.fullname" . }}-operator' - namespace: '{{ .Release.Namespace }}' \ No newline at end of file diff --git a/helm-charts-openshift/charts/nfd/templates/serviceaccount.yaml b/helm-charts-openshift/charts/nfd/templates/serviceaccount.yaml deleted file mode 100644 index e615976bb..000000000 --- a/helm-charts-openshift/charts/nfd/templates/serviceaccount.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "nfd.fullname" . }}-operator - labels: - {{- include "nfd.labels" . | nindent 4 }} - annotations: - {{- toYaml .Values.operator.serviceAccount.annotations | nindent 4 }} \ No newline at end of file diff --git a/helm-charts-openshift/charts/nfd/values.yaml b/helm-charts-openshift/charts/nfd/values.yaml deleted file mode 100644 index afcb2d8f6..000000000 --- a/helm-charts-openshift/charts/nfd/values.yaml +++ /dev/null @@ -1,66 +0,0 @@ -controllerManager: - kubeRbacProxy: - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=6 - - --tls-cert-file=/etc/secrets/tls.crt - - --tls-private-key-file=/etc/secrets/tls.key - - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 - containerSecurityContext: - readOnlyRootFilesystem: true - image: - repository: gcr.io/kubebuilder/kube-rbac-proxy - tag: v0.8.0 - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 250m - memory: 64Mi - manager: - args: - - --metrics-bind-address=127.0.0.1:8080 - - --leader-elect - containerSecurityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - env: - nodeFeatureDiscoveryImage: quay.io/openshift/origin-node-feature-discovery:4.16 - operatorName: cluster-nfd-operator - sslCertDir: /etc/pki/tls/certs - image: - repository: quay.io/openshift/origin-cluster-nfd-operator - tag: "4.16" - replicas: 1 -kubernetesClusterDomain: cluster.local -managerConfig: - controllerManagerConfigYaml: |- - apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 - kind: ControllerManagerConfig - health: - healthProbeBindAddress: :8081 - metrics: - bindAddress: 127.0.0.1:8080 - webhook: - port: 9443 - leaderElection: - leaderElect: true - resourceName: 39f5e5c3.nodefeaturediscoveries.nfd.openshift.io -metricsService: - ports: - - name: https - port: 8443 - targetPort: https - type: ClusterIP -operator: - serviceAccount: - annotations: {} diff --git a/helm-charts-openshift/crds/deviceconfig-crd.yaml b/helm-charts-openshift/crds/deviceconfig-crd.yaml deleted file mode 100644 index 21bac56e5..000000000 --- a/helm-charts-openshift/crds/deviceconfig-crd.yaml +++ /dev/null @@ -1,1819 +0,0 @@ ---- -# Source: gpu-operator-charts/templates/deviceconfig-crd.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: deviceconfigs.amd.com - annotations: - controller-gen.kubebuilder.io/version: v0.17.0 - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - helm.sh/chart: gpu-operator-charts-v1.4.0 - app.kubernetes.io/name: gpu-operator-charts - app.kubernetes.io/instance: amd-gpu - app.kubernetes.io/version: "v1.4.0" - app.kubernetes.io/managed-by: Helm -spec: - group: amd.com - names: - kind: DeviceConfig - listKind: DeviceConfigList - plural: deviceconfigs - shortNames: - - gpue - singular: deviceconfig - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: DeviceConfig describes how to enable AMD GPU device - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: DeviceConfigSpec describes how the AMD GPU operator should - enable AMD GPU device for customer's use. - properties: - commonConfig: - description: common config - properties: - initContainerImage: - description: InitContainerImage is being used for the operands pods, - i.e. metrics exporter, test runner, device plugin, device config - manager and node labeller - type: string - utilsContainer: - description: UtilsContainer contains parameters to configure operator's - utils container - properties: - image: - description: Image is the image of utils container - pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$ - type: string - imagePullPolicy: - description: image pull policy for utils container - enum: - - Always - - IfNotPresent - - Never - type: string - imageRegistrySecret: - description: secret used for pull utils container image - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - type: object - type: object - configManager: - description: config manager - properties: - config: - description: config map to customize the config for config manager, - if not specified default config will be applied - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - configManagerTolerations: - description: tolerations for the device config manager DaemonSet - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - enable: - description: enable config manager, disabled by default - type: boolean - image: - description: config manager image - pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$ - type: string - imagePullPolicy: - description: image pull policy for config manager - enum: - - Always - - IfNotPresent - - Never - type: string - imageRegistrySecret: - description: config manager image registry secret used to pull/push - images - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - selector: - additionalProperties: - type: string - description: Selector describes on which nodes to enable config - manager - type: object - upgradePolicy: - description: upgrade policy for config manager daemonset - properties: - maxUnavailable: - default: 1 - description: MaxUnavailable specifies the maximum number of - Pods that can be unavailable during the update process. Applicable - for RollingUpdate only. Default value is 1. - format: int32 - type: integer - upgradeStrategy: - description: UpgradeStrategy specifies the type of the DaemonSet - update. Valid values are "RollingUpdate" (default) or "OnDelete". - enum: - - RollingUpdate - - OnDelete - type: string - type: object - type: object - devicePlugin: - description: device plugin - properties: - devicePluginArguments: - additionalProperties: - type: string - description: |- - device plugin arguments is used to pass supported flags and their values while starting device plugin daemonset - supported flag values: {"resource_naming_strategy": {"single", "mixed"}} - type: object - devicePluginImage: - description: device plugin image - pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$ - type: string - devicePluginImagePullPolicy: - description: image pull policy for device plugin - enum: - - Always - - IfNotPresent - - Never - type: string - devicePluginTolerations: - description: tolerations for the device plugin DaemonSet - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - enableNodeLabeller: - default: true - description: enable or disable the node labeller - type: boolean - imageRegistrySecret: - description: node labeller image registry secret used to pull/push - images - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - nodeLabellerArguments: - description: |- - node labeller arguments is used to pass supported labels while starting node labeller daemonset - some flags are enabled by default as they are applicable and bare minimum for all setups and are supported in all versions of node labeller - default flags: {"vram", "cu-count", "simd-count", "device-id", "family", "product-name", "driver-version"} - supported flags: {"compute-memory-partition", "compute-partitioning-supported", "memory-partitioning-supported"} - items: - type: string - type: array - nodeLabellerImage: - description: node labeller image - pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$ - type: string - nodeLabellerImagePullPolicy: - description: image pull policy for node labeller - enum: - - Always - - IfNotPresent - - Never - type: string - nodeLabellerTolerations: - description: tolerations for the node labeller DaemonSet - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - upgradePolicy: - description: upgrade policy for device plugin and node labeller - daemons - properties: - maxUnavailable: - default: 1 - description: MaxUnavailable specifies the maximum number of - Pods that can be unavailable during the update process. Applicable - for RollingUpdate only. Default value is 1. - format: int32 - type: integer - upgradeStrategy: - description: UpgradeStrategy specifies the type of the DaemonSet - update. Valid values are "RollingUpdate" (default) or "OnDelete". - enum: - - RollingUpdate - - OnDelete - type: string - type: object - type: object - driver: - description: driver - properties: - amdgpuInstallerRepoURL: - description: |- - radeon repo URL for fetching amdgpu installer if building driver image on the fly - installer URL is https://repo.radeon.com/amdgpu-install by default - type: string - blacklist: - description: |- - blacklist amdgpu drivers on the host. Node reboot is required to apply the baclklist on the worker nodes. - Not working for OpenShift cluster. OpenShift users please use the Machine Config Operator (MCO) resource to configure amdgpu blacklist. - Example MCO resource is available at https://instinct.docs.amd.com/projects/gpu-operator/en/latest/installation/openshift-olm.html#create-blacklist-for-installing-out-of-tree-kernel-module - type: boolean - driverType: - default: container - description: |- - specify the type of driver (container/vf-passthrough/pf-passthrough) to install on the worker node. default value is container. - container: normal amdgpu-dkms driver for Bare Metal GPU nodes or guest VM. - vf-passthrough: MxGPU GIM driver on the host machine to generate VF, then mount VF to vfio-pci - pf-passthrough: directly mount PF device to vfio-pci - enum: - - container - - vf-passthrough - - pf-passthrough - type: string - enable: - default: true - description: |- - enable driver install. default value is true. - disable is for skipping driver install/uninstall for dryrun or using in-tree amdgpu kernel module - type: boolean - image: - description: |- - defines image that includes drivers and firmware blobs, don't include tag since it will be fully managed by operator - for vanilla k8s the default value is image-registry:5000/$MOD_NAMESPACE/amdgpu_kmod - for OpenShift the default value is image-registry.openshift-image-registry.svc:5000/$MOD_NAMESPACE/amdgpu_kmod - image tag will be in the format of --- - example tag is coreos-416.94-5.14.0-427.28.1.el9_4.x86_64-6.2.2 and ubuntu-22.04-5.15.0-94-generic-6.1.3 - NOTE: Updating the driver image repository is not supported. Please delete the existing DeviceConfig and create a new one with the updated image repository - pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[$a-zA-Z0-9_]+(?:[._-][$a-zA-Z0-9_]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$ - type: string - imageBuild: - description: image build configs - properties: - baseImageRegistry: - default: docker.io - description: |- - image registry to fetch base image for building driver image, default value is docker.io, the builder will search for corresponding OS base image from given registry - e.g. if your worker node is using Ubuntu 22.04, by default the base image would be docker.io/ubuntu:22.04 - Use spec.driver.imageRegistrySecret for authentication with private registries. - NOTE: this field won't apply for OpenShift since OpenShift is using its own DriverToolKit image to build driver image - type: string - baseImageRegistryTLS: - description: |- - TLS settings for fetching base image - this field will be applied to SourceImageRepo as well - properties: - insecure: - description: If true, check if the container image already - exists using plain HTTP. - type: boolean - insecureSkipTLSVerify: - description: If true, skip any TLS server certificate validation - type: boolean - type: object - sourceImageRepo: - description: |- - SourceImageRepo specifies the image repository for the driver source code (OpenShift only). - Used when spec.driver.useSourceImage is true. The operator automatically determines the image tag - based on cluster RHEL version and spec.driver.version (format: coreos--). - Default: docker.io/rocm/amdgpu-driver - Use spec.driver.imageRegistrySecret for authentication with private registries. - type: string - type: object - imageRegistrySecret: - description: secrets used for pull/push images from/to private registry - specified in driversImage - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - imageRegistryTLS: - description: driver image registry TLS setting for the container - image - properties: - insecure: - description: If true, check if the container image already exists - using plain HTTP. - type: boolean - insecureSkipTLSVerify: - description: If true, skip any TLS server certificate validation - type: boolean - type: object - imageSign: - description: |- - image signing config to sign the driver image when building driver image on the fly - image signing is required for installing driver on secure boot enabled system - properties: - certSecret: - description: |- - ImageSignCertSecret the public key used to sign kernel modules within image - necessary for secure boot enabled system - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - keySecret: - description: |- - ImageSignKeySecret the private key used to sign kernel modules within image - necessary for secure boot enabled system - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - type: object - kernelModuleConfig: - description: advanced arguments, parameters and more configs to - manage tne driver - properties: - loadArgs: - description: LoadArg are the arguments when modprobe is executed - to load the kernel module. The command will be `modprobe ${Args} - module_name`. - items: - type: string - type: array - parameters: - description: Parameters is being used for modprobe commands. - The command will be `modprobe ${Args} module_name ${Parameters}`. - items: - type: string - type: array - unloadArgs: - description: UnloadArg are the arguments when modprobe is executed - to unload the kernel module. The command will be `modprobe - -r ${Args} module_name`. - items: - type: string - type: array - type: object - tolerations: - description: tolerations for kmm module object - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - upgradePolicy: - description: policy to upgrade the drivers - properties: - enable: - description: |- - enable upgrade policy, disabled by default - If disabled, user has to manually upgrade all the nodes. - type: boolean - maxParallelUpgrades: - default: 1 - description: |- - MaxParallelUpgrades indicates how many nodes can be upgraded in parallel - 0 means no limit, all nodes will be upgraded in parallel - minimum: 0 - type: integer - maxUnavailableNodes: - anyOf: - - type: integer - - type: string - default: 25% - description: |- - MaxUnavailableNodes indicates maximum number of nodes that can be in a failed upgrade state beyond which upgrades will stop to keep cluster at a minimal healthy state - Value can be an integer (ex: 2) which would mean atmost 2 nodes can be in failed state after which new upgrades will not start. Or it can be a percentage string(ex: "50%") from which absolute number will be calculated and round up - x-kubernetes-int-or-string: true - nodeDrainPolicy: - description: Node draining policy - properties: - force: - default: false - description: Force indicates if force draining is allowed - type: boolean - gracePeriodSeconds: - default: -1 - description: GracePeriodSeconds indicates the time kubernetes - waits for a pod to shut down gracefully after receiving - a termination signal - type: integer - ignoreDaemonSets: - default: true - description: IgnoreDaemonSets indicates whether to ignore - DaemonSet-managed pods - type: boolean - ignoreNamespaces: - description: |- - IgnoreNamespaces is the list of namespaces to ignore during node drain operation. - This is useful to avoid draining pods from critical namespaces like 'kube-system', etc. - items: - type: string - type: array - timeoutSeconds: - default: 300 - description: TimeoutSecond specifies the length of time - in seconds to wait before giving up drain, zero means - infinite - minimum: 0 - type: integer - type: object - podDeletionPolicy: - description: Pod Deletion policy. If both NodeDrainPolicy and - PodDeletionPolicy config is available, NodeDrainPolicy(if - enabled) will take precedence. - properties: - force: - default: false - description: Force indicates if force deletion is allowed - type: boolean - gracePeriodSeconds: - default: -1 - description: GracePeriodSeconds indicates the time kubernetes - waits for a pod to shut down gracefully after receiving - a termination signal - type: integer - timeoutSeconds: - default: 300 - description: TimeoutSecond specifies the length of time - in seconds to wait before giving up on pod deletion, zero - means infinite - minimum: 0 - type: integer - type: object - rebootRequired: - default: true - description: reboot between driver upgrades, enabled by default, - if enabled spec.commonConfig.utilsContainer will be used to - perform reboot on worker nodes - type: boolean - type: object - useSourceImage: - description: |- - NOTE: currently only for OpenShift cluster - set to true to use source image to build driver image on the fly - otherwise use installer debian/rpm packages from radeon repo to build driver image - type: boolean - version: - description: |- - version of the drivers source code, can be used as part of image of dockerfile source image - default value for different OS is: ubuntu: 6.1.3, coreOS: 6.2.2 - type: string - vfioConfig: - description: |- - vfio config - specify the specific configs for binding PCI devices to vfio-pci kernel module, applies for driver type vf-passthrough and pf-passthrough - properties: - deviceIDs: - description: list of PCI device IDs to load into vfio-pci driver. - default is the list of AMD GPU PF/VF PCI device IDs based - on driver type vf-passthrough/pf-passthrough. - items: - type: string - type: array - type: object - type: object - metricsExporter: - description: metrics exporter - properties: - config: - description: optional configuration for metrics - properties: - name: - description: |- - Name of the configMap that defines the list of metrics - default list:[] - type: string - type: object - enable: - description: enable metrics exporter, disabled by default - type: boolean - image: - description: metrics exporter image - pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$ - type: string - imagePullPolicy: - description: image pull policy for metrics exporter - enum: - - Always - - IfNotPresent - - Never - type: string - imageRegistrySecret: - description: metrics exporter image registry secret used to pull/push - images - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - nodePort: - description: NodePort is the external port for pulling metrics from - outside the cluster, in the range 30000-32767 (assigned automatically - by default) - format: int32 - maximum: 32767 - minimum: 30000 - type: integer - podAnnotations: - additionalProperties: - type: string - description: Set PodAnnotations for metrics exporter - type: object - podResourceAPISocketPath: - default: /var/lib/kubelet/pod-resources - description: |- - Set the host path for pod-resource kubelet.socket, - vanila kubernetes path is /var/lib/kubelet/pod-resources - microk8s path is /var/snap/microk8s/common/var/lib/kubelet/pod-resources/ - path is an absolute unix path that allows a trailing slash - pattern: ^(/[^/\0]+)*(/)?$ - type: string - port: - default: 5000 - description: Port is the internal port used for in-cluster and node - access to pull metrics from the metrics-exporter (default 5000). - format: int32 - type: integer - prometheus: - description: Prometheus configuration for metrics exporter - properties: - serviceMonitor: - description: ServiceMonitor configuration for Prometheus integration - properties: - attachMetadata: - description: AttachMetadata defines if Prometheus should - attach node metadata to the target - properties: - node: - description: |- - When set to true, Prometheus attaches node metadata to the discovered - targets. - - The Prometheus service account must have the `list` and `watch` - permissions on the `Nodes` objects. - type: boolean - type: object - authorization: - description: Optional Prometheus authorization configuration - for accessing the endpoint - properties: - credentials: - description: Selects a key of a Secret in the namespace - that contains the credentials for authentication. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key - must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: - description: |- - Defines the authentication type. The value is case-insensitive. - - "Basic" is not a supported value. - - Default: "Bearer" - type: string - type: object - bearerTokenFile: - description: |- - Path to bearer token file to be used by Prometheus (e.g., service account token path) - Deprecated: Use Authorization instead. This field is kept for backward compatibility. - type: string - enable: - description: Enable or disable ServiceMonitor creation (default - false) - type: boolean - honorLabels: - default: true - description: HonorLabels chooses the metric's labels on - collisions with target labels (default true) - type: boolean - honorTimestamps: - description: HonorTimestamps controls whether the scrape - endpoints honor timestamps (default false) - type: boolean - interval: - description: 'How frequently to scrape metrics. Accepts - values with time unit suffix: "30s", "1m", "2h", "500ms"' - pattern: ^([0-9]+)(ms|s|m|h)$ - type: string - labels: - additionalProperties: - type: string - description: 'Additional labels to add to the ServiceMonitor - (default release: prometheus)' - type: object - metricRelabelings: - description: Relabeling rules applied to individual scraped - metrics - items: - description: |- - RelabelConfig allows dynamic rewriting of the label set for targets, alerts, - scraped samples and remote write samples. - - More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config - properties: - action: - default: replace - description: |- - Action to perform based on the regex matching. - - `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. - `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. - - Default: "Replace" - enum: - - replace - - Replace - - keep - - Keep - - drop - - Drop - - hashmod - - HashMod - - labelmap - - LabelMap - - labeldrop - - LabelDrop - - labelkeep - - LabelKeep - - lowercase - - Lowercase - - uppercase - - Uppercase - - keepequal - - KeepEqual - - dropequal - - DropEqual - type: string - modulus: - description: |- - Modulus to take of the hash of the source label values. - - Only applicable when the action is `HashMod`. - format: int64 - type: integer - regex: - description: Regular expression against which the - extracted value is matched. - type: string - replacement: - description: |- - Replacement value against which a Replace action is performed if the - regular expression matches. - - Regex capture groups are available. - type: string - separator: - description: Separator is the string between concatenated - SourceLabels. - type: string - sourceLabels: - description: |- - The source labels select values from existing labels. Their content is - concatenated using the configured Separator and matched against the - configured regular expression. - items: - description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ - type: string - type: array - targetLabel: - description: |- - Label to which the resulting string is written in a replacement. - - It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, - `KeepEqual` and `DropEqual` actions. - - Regex capture groups are available. - type: string - type: object - type: array - relabelings: - description: RelabelConfigs to apply to samples before ingestion - items: - description: |- - RelabelConfig allows dynamic rewriting of the label set for targets, alerts, - scraped samples and remote write samples. - - More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config - properties: - action: - default: replace - description: |- - Action to perform based on the regex matching. - - `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. - `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. - - Default: "Replace" - enum: - - replace - - Replace - - keep - - Keep - - drop - - Drop - - hashmod - - HashMod - - labelmap - - LabelMap - - labeldrop - - LabelDrop - - labelkeep - - LabelKeep - - lowercase - - Lowercase - - uppercase - - Uppercase - - keepequal - - KeepEqual - - dropequal - - DropEqual - type: string - modulus: - description: |- - Modulus to take of the hash of the source label values. - - Only applicable when the action is `HashMod`. - format: int64 - type: integer - regex: - description: Regular expression against which the - extracted value is matched. - type: string - replacement: - description: |- - Replacement value against which a Replace action is performed if the - regular expression matches. - - Regex capture groups are available. - type: string - separator: - description: Separator is the string between concatenated - SourceLabels. - type: string - sourceLabels: - description: |- - The source labels select values from existing labels. Their content is - concatenated using the configured Separator and matched against the - configured regular expression. - items: - description: |- - LabelName is a valid Prometheus label name which may only contain ASCII - letters, numbers, as well as underscores. - pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ - type: string - type: array - targetLabel: - description: |- - Label to which the resulting string is written in a replacement. - - It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, - `KeepEqual` and `DropEqual` actions. - - Regex capture groups are available. - type: string - type: object - type: array - tlsConfig: - description: TLS settings used by Prometheus to connect - to the metrics endpoint - properties: - ca: - description: Certificate authority used when verifying - server certificates. - properties: - configMap: - description: ConfigMap containing data to use for - the targets. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or - its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - secret: - description: Secret containing data to use for the - targets. - properties: - key: - description: The key of the secret to select - from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its - key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - caFile: - description: Path to the CA cert in the Prometheus container - to use for the targets. - type: string - cert: - description: Client certificate to present when doing - client-authentication. - properties: - configMap: - description: ConfigMap containing data to use for - the targets. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or - its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - secret: - description: Secret containing data to use for the - targets. - properties: - key: - description: The key of the secret to select - from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its - key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - certFile: - description: Path to the client cert file in the Prometheus - container for the targets. - type: string - insecureSkipVerify: - description: Disable target certificate validation. - type: boolean - keyFile: - description: Path to the client key file in the Prometheus - container for the targets. - type: string - keySecret: - description: Secret containing the client key file for - the targets. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key - must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - maxVersion: - description: |- - Maximum acceptable TLS version. - - It requires Prometheus >= v2.41.0. - enum: - - TLS10 - - TLS11 - - TLS12 - - TLS13 - type: string - minVersion: - description: |- - Minimum acceptable TLS version. - - It requires Prometheus >= v2.35.0. - enum: - - TLS10 - - TLS11 - - TLS12 - - TLS13 - type: string - serverName: - description: Used to verify the hostname for the targets. - type: string - type: object - type: object - type: object - rbacConfig: - description: optional kube-rbac-proxy config to provide rbac services - properties: - clientCAConfigMap: - description: 'Reference to a configmap containing the client - CA (key: ca.crt) for mTLS client validation' - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - disableHttps: - description: disable https protecting the proxy endpoint - type: boolean - enable: - description: enable kube-rbac-proxy, disabled by default - type: boolean - image: - description: kube-rbac-proxy image - pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$ - type: string - secret: - description: certificate secret to mount in kube-rbac container - for TLS, self signed certificates will be generated by default - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - staticAuthorization: - description: Optional static RBAC rules based on client certificate - Common Name (CN) - properties: - clientName: - description: Expected CN (Common Name) from client cert - (e.g., Prometheus SA identity) - type: string - enable: - description: Enables static authorization using client certificate - CN - type: boolean - type: object - type: object - resource: - default: - limits: - cpu: "2" - memory: 4G - requests: - cpu: 500m - memory: 512M - description: Set resource config for metrics exporter - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - selector: - additionalProperties: - type: string - description: Selector describes on which nodes to enable metrics - exporter - type: object - serviceAnnotations: - additionalProperties: - type: string - description: Set ServiceAnnotations for metrics exporter - type: object - serviceType: - default: ClusterIP - description: ServiceType service type for metrics, clusterIP/NodePort, - clusterIP by default - enum: - - ClusterIP - - NodePort - type: string - tolerations: - description: tolerations for metrics exporter - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - upgradePolicy: - description: upgrade policy for metrics exporter daemons - properties: - maxUnavailable: - default: 1 - description: MaxUnavailable specifies the maximum number of - Pods that can be unavailable during the update process. Applicable - for RollingUpdate only. Default value is 1. - format: int32 - type: integer - upgradeStrategy: - description: UpgradeStrategy specifies the type of the DaemonSet - update. Valid values are "RollingUpdate" (default) or "OnDelete". - enum: - - RollingUpdate - - OnDelete - type: string - type: object - type: object - remediationWorkflow: - description: remediation workflow - properties: - conditionalWorkflows: - description: Name of the ConfigMap that holds condition-to-workflow - mappings. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - enable: - description: |- - enable remediation workflows. disabled by default - enable if operator should automatically handle remediation of node incase of gpu issues - type: boolean - maxParallelWorkflows: - description: MaxParallelWorkflows specifies limit on how many remediation - workflows can be executed in parallel. 0 is the default value - and it means no limit. - type: integer - nodeDrainPolicy: - description: Node drain policy during remediation workflow execution - properties: - force: - default: false - description: Force indicates if force draining is allowed - type: boolean - gracePeriodSeconds: - default: -1 - description: GracePeriodSeconds indicates the time kubernetes - waits for a pod to shut down gracefully after receiving a - termination signal - type: integer - ignoreDaemonSets: - default: true - description: IgnoreDaemonSets indicates whether to ignore DaemonSet-managed - pods - type: boolean - ignoreNamespaces: - description: |- - IgnoreNamespaces is the list of namespaces to ignore during node drain operation. - This is useful to avoid draining pods from critical namespaces like 'kube-system', etc. - items: - type: string - type: array - timeoutSeconds: - default: 300 - description: TimeoutSecond specifies the length of time in seconds - to wait before giving up drain, zero means infinite - minimum: 0 - type: integer - type: object - nodeRemediationLabels: - additionalProperties: - type: string - description: Node Remediation labels are custom labels that we can - apply on the node to specify that the node is undergoing remediation - or needs attention by the administrator. - type: object - nodeRemediationTaints: - description: |- - Node Remediation taints are custom taints that we can apply on the node to specify that the node is undergoing remediation or needs attention by the administrator. - If user does not specify any taints, the operator will apply a taint with key "amd-gpu-unhealthy" and effect "NoSchedule" to the node under remediation. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Required. The taint key to be applied to a node. - type: string - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - required: - - effect - - key - type: object - type: array - testerImage: - description: Tester image used to run tests and verify if remediation - fixed the reported problem. - pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$ - type: string - ttlForFailedWorkflows: - default: 24 - description: Time to live for argo workflow object and its pods - for a failed workflow in hours. By default, it is set to 24 hours - type: integer - type: object - selector: - additionalProperties: - type: string - description: Selector describes on which nodes the GPU Operator should - enable the GPU device. - type: object - testRunner: - description: test runner - properties: - config: - description: config map to customize the config for test runner, - if not specified default test config will be aplied - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - enable: - description: enable test runner, disabled by default - type: boolean - image: - description: test runner image - pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$ - type: string - imagePullPolicy: - description: image pull policy for test runner - enum: - - Always - - IfNotPresent - - Never - type: string - imageRegistrySecret: - description: test runner image registry secret used to pull/push - images - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - logsLocation: - description: captures logs location and export config for test runner - logs - properties: - hostPath: - default: /var/log/amd-test-runner - description: host path to store test runner internal status - db in order to persist test running status - type: string - logsExportSecrets: - description: LogsExportSecrets is a list of secrets that contain - connectivity info to multiple cloud providers - items: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - type: array - mountPath: - default: /var/log/amd-test-runner - description: volume mount destination within test runner container - type: string - type: object - selector: - additionalProperties: - type: string - description: Selector describes on which nodes to enable test runner - type: object - tolerations: - description: tolerations for test runner - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - upgradePolicy: - description: upgrade policy for test runner daemonset - properties: - maxUnavailable: - default: 1 - description: MaxUnavailable specifies the maximum number of - Pods that can be unavailable during the update process. Applicable - for RollingUpdate only. Default value is 1. - format: int32 - type: integer - upgradeStrategy: - description: UpgradeStrategy specifies the type of the DaemonSet - update. Valid values are "RollingUpdate" (default) or "OnDelete". - enum: - - RollingUpdate - - OnDelete - type: string - type: object - type: object - type: object - status: - description: DeviceConfigStatus defines the observed state of Module. - properties: - conditions: - description: Conditions list the current status of the DeviceConfig - object - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - configManager: - description: ConfigManager contains the status of the ConfigManager - deployment - properties: - availableNumber: - description: number of the actually deployed and running pods - format: int32 - type: integer - desiredNumber: - description: number of the pods that should be deployed for daemonset - format: int32 - type: integer - nodesMatchingSelectorNumber: - description: number of nodes that are targeted by the DeviceConfig - selector - format: int32 - type: integer - type: object - devicePlugin: - description: DevicePlugin contains the status of the Device Plugin deployment - properties: - availableNumber: - description: number of the actually deployed and running pods - format: int32 - type: integer - desiredNumber: - description: number of the pods that should be deployed for daemonset - format: int32 - type: integer - nodesMatchingSelectorNumber: - description: number of nodes that are targeted by the DeviceConfig - selector - format: int32 - type: integer - type: object - driver: - description: Driver contains the status of the Drivers deployment - properties: - availableNumber: - description: number of the actually deployed and running pods - format: int32 - type: integer - desiredNumber: - description: number of the pods that should be deployed for daemonset - format: int32 - type: integer - nodesMatchingSelectorNumber: - description: number of nodes that are targeted by the DeviceConfig - selector - format: int32 - type: integer - type: object - metricsExporter: - description: MetricsExporter contains the status of the MetricsExporter - deployment - properties: - availableNumber: - description: number of the actually deployed and running pods - format: int32 - type: integer - desiredNumber: - description: number of the pods that should be deployed for daemonset - format: int32 - type: integer - nodesMatchingSelectorNumber: - description: number of nodes that are targeted by the DeviceConfig - selector - format: int32 - type: integer - type: object - nodeModuleStatus: - additionalProperties: - description: ModuleStatus contains the status of driver module installed - by operator on the node - properties: - bootId: - type: string - containerImage: - type: string - kernelVersion: - type: string - lastTransitionTime: - type: string - status: - description: UpgradeState captures the state of the upgrade process - on a node - type: string - upgradeStartTime: - type: string - type: object - description: NodeModuleStatus contains per node status of driver module - installation - type: object - observedGeneration: - description: ObservedGeneration is the latest spec generation successfully - processed by the controller - format: int64 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/helm-charts-openshift/crds/remediationworkflowstatus-crd.yaml b/helm-charts-openshift/crds/remediationworkflowstatus-crd.yaml deleted file mode 100644 index 011e3ad06..000000000 --- a/helm-charts-openshift/crds/remediationworkflowstatus-crd.yaml +++ /dev/null @@ -1,78 +0,0 @@ ---- -# Source: gpu-operator-charts/templates/remediationworkflowstatus-crd.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: remediationworkflowstatuses.amd.com - annotations: - controller-gen.kubebuilder.io/version: v0.17.0 - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - helm.sh/chart: gpu-operator-charts-v1.4.0 - app.kubernetes.io/name: gpu-operator-charts - app.kubernetes.io/instance: amd-gpu - app.kubernetes.io/version: "v1.4.0" - app.kubernetes.io/managed-by: Helm -spec: - group: amd.com - names: - kind: RemediationWorkflowStatus - listKind: RemediationWorkflowStatusList - plural: remediationworkflowstatuses - shortNames: - - rwfstatus - singular: remediationworkflowstatus - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: |- - RemediationWorkflowStatus keeps a record of recent remediation workflow runs. - We maintain this information to avoid re-running remediation workflows on nodes where a pre-defined threshold is crossed. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - status: - additionalProperties: - additionalProperties: - items: - properties: - name: - type: string - startTime: - type: string - type: object - type: array - type: object - description: |- - Status field holds remediation workflow run history for each node and node condition - Key is node name. Value is a map with key as node condition and value as list of workflow metadata(workflow name and it's start time) - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/helm-charts-openshift/templates/_helpers.tpl b/helm-charts-openshift/templates/_helpers.tpl deleted file mode 100644 index 248b12c6d..000000000 --- a/helm-charts-openshift/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "helm-charts-openshift.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "helm-charts-openshift.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "helm-charts-openshift.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "helm-charts-openshift.labels" -}} -helm.sh/chart: {{ include "helm-charts-openshift.chart" . }} -{{ include "helm-charts-openshift.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "helm-charts-openshift.selectorLabels" -}} -app.kubernetes.io/name: {{ include "helm-charts-openshift.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "helm-charts-openshift.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "helm-charts-openshift.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/helm-charts-openshift/templates/config-manager-rbac.yaml b/helm-charts-openshift/templates/config-manager-rbac.yaml deleted file mode 100644 index 0f6fc3515..000000000 --- a/helm-charts-openshift/templates/config-manager-rbac.yaml +++ /dev/null @@ -1,75 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-config-manager - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - list - - get - - update - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch - - update -- apiGroups: - - "apps" - resources: - - daemonsets - verbs: - - get - - list - - watch - - delete - - create - - update -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch - - delete - - create - - update -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-config-manager - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-openshift.fullname" . }}-config-manager' -subjects: -- kind: ServiceAccount - name: amd-gpu-operator-config-manager - namespace: '{{ .Release.Namespace }}' \ No newline at end of file diff --git a/helm-charts-openshift/templates/deployment.yaml b/helm-charts-openshift/templates/deployment.yaml deleted file mode 100644 index 69f2e1370..000000000 --- a/helm-charts-openshift/templates/deployment.yaml +++ /dev/null @@ -1,83 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-controller-manager - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - control-plane: controller-manager - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.controllerManager.replicas }} - selector: - matchLabels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - control-plane: controller-manager - {{- include "helm-charts-openshift.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - control-plane: controller-manager - {{- include "helm-charts-openshift.selectorLabels" . | nindent 8 }} - annotations: - kubectl.kubernetes.io/default-container: manager - spec: - {{- with .Values.controllerManager.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - nodeSelector: {{- toYaml .Values.controllerManager.nodeSelector | nindent 8 }} - containers: - - args: {{- toYaml .Values.controllerManager.manager.args | nindent 8 }} - env: - - name: OPERATOR_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: KUBERNETES_CLUSTER_DOMAIN - value: {{ quote .Values.kubernetesClusterDomain }} - - name: SIM_ENABLE - value: {{ quote .Values.controllerManager.env.simEnable }} - image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag - | default .Chart.AppVersion }} - imagePullPolicy: {{ .Values.controllerManager.manager.imagePullPolicy }} - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: {{- toYaml .Values.controllerManager.manager.resources | nindent 10 - }} - securityContext: {{- toYaml .Values.controllerManager.manager.containerSecurityContext - | nindent 10 }} - volumeMounts: - - mountPath: /controller_manager_config.yaml - name: manager-config - subPath: controller_manager_config.yaml - {{- if .Values.controllerManager.manager.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.controllerManager.manager.imagePullSecrets }} - {{- end}} - securityContext: - runAsNonRoot: true - serviceAccountName: {{ include "helm-charts-openshift.fullname" . }}-controller-manager - terminationGracePeriodSeconds: 10 - {{- with .Values.controllerManager.manager.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - volumes: - - configMap: - name: {{ include "helm-charts-openshift.fullname" . }}-manager-config - name: manager-config diff --git a/helm-charts-openshift/templates/event-recorder-clusterrole-rbac.yaml b/helm-charts-openshift/templates/event-recorder-clusterrole-rbac.yaml deleted file mode 100644 index ce6bf1069..000000000 --- a/helm-charts-openshift/templates/event-recorder-clusterrole-rbac.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-event-recorder-clusterrole - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch \ No newline at end of file diff --git a/helm-charts-openshift/templates/event-recorder-clusterrolebinding-rbac.yaml b/helm-charts-openshift/templates/event-recorder-clusterrolebinding-rbac.yaml deleted file mode 100644 index e2886194c..000000000 --- a/helm-charts-openshift/templates/event-recorder-clusterrolebinding-rbac.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-event-recorder-clusterrolebinding - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-openshift.fullname" . }}-event-recorder-clusterrole' -subjects: -- kind: ServiceAccount - name: '{{ include "helm-charts-openshift.fullname" . }}-controller-manager' - namespace: '{{ .Release.Namespace }}' \ No newline at end of file diff --git a/helm-charts-openshift/templates/kmm-device-plugin-rbac.yaml b/helm-charts-openshift/templates/kmm-device-plugin-rbac.yaml deleted file mode 100644 index e81a5133a..000000000 --- a/helm-charts-openshift/templates/kmm-device-plugin-rbac.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-kmm-device-plugin - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -rules: -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-kmm-device-plugin - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-openshift.fullname" . }}-kmm-device-plugin' -subjects: -- kind: ServiceAccount - name: amd-gpu-operator-kmm-device-plugin - namespace: '{{ .Release.Namespace }}' diff --git a/helm-charts-openshift/templates/kmm-module-loader-rbac.yaml b/helm-charts-openshift/templates/kmm-module-loader-rbac.yaml deleted file mode 100644 index b997dd567..000000000 --- a/helm-charts-openshift/templates/kmm-module-loader-rbac.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-kmm-module-loader - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -rules: -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-kmm-module-loader - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-openshift.fullname" . }}-kmm-module-loader' -subjects: -- kind: ServiceAccount - name: amd-gpu-operator-kmm-module-loader - namespace: '{{ .Release.Namespace }}' diff --git a/helm-charts-openshift/templates/leader-election-rbac.yaml b/helm-charts-openshift/templates/leader-election-rbac.yaml deleted file mode 100644 index 25208c9b7..000000000 --- a/helm-charts-openshift/templates/leader-election-rbac.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-leader-election-role - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-leader-election-rolebinding - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: '{{ include "helm-charts-openshift.fullname" . }}-leader-election-role' -subjects: -- kind: ServiceAccount - name: '{{ include "helm-charts-openshift.fullname" . }}-controller-manager' - namespace: '{{ .Release.Namespace }}' \ No newline at end of file diff --git a/helm-charts-openshift/templates/manager-config.yaml b/helm-charts-openshift/templates/manager-config.yaml deleted file mode 100644 index 0173a6e38..000000000 --- a/helm-charts-openshift/templates/manager-config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-manager-config - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -data: - controller_manager_config.yaml: {{ .Values.managerConfig.controllerManagerConfigYaml - | toYaml | indent 1 }} \ No newline at end of file diff --git a/helm-charts-openshift/templates/manager-rbac.yaml b/helm-charts-openshift/templates/manager-rbac.yaml deleted file mode 100644 index fe673eb6c..000000000 --- a/helm-charts-openshift/templates/manager-rbac.yaml +++ /dev/null @@ -1,219 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-manager-role - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - configmaps - - secrets - - services - verbs: - - create - - delete - - get - - list - - patch - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - patch - - watch -- apiGroups: - - "" - resources: - - nodes/finalizers - - nodes/status - verbs: - - get - - update - - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - create - - delete - - get - - list - - watch -- apiGroups: - - "" - resources: - - pods/eviction - verbs: - - create - - delete - - get - - list -- apiGroups: - - "" - resources: - - pods/finalizers - - pods/status - verbs: - - delete - - get - - list - - watch -- apiGroups: - - "" - resources: - - services/finalizers - verbs: - - create - - get - - update - - watch -- apiGroups: - - amd.com - resources: - - deviceconfigs - - remediationworkflowstatuses - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - amd.com - resources: - - deviceconfigs/finalizers - - remediationworkflowstatuses/finalizers - verbs: - - update -- apiGroups: - - amd.com - resources: - - deviceconfigs/status - - remediationworkflowstatuses/status - verbs: - - get - - patch - - update -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - delete - - get - - list - - watch -- apiGroups: - - apps - resources: - - daemonsets - - daemonsets/status - verbs: - - create - - delete - - get - - list - - patch - - watch -- apiGroups: - - apps - resources: - - daemonsets/finalizers - verbs: - - create - - get - - update - - watch -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - modules - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - modules/finalizers - - nodemodulesconfigs/finalizers - verbs: - - get - - update - - watch -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - modules/status - verbs: - - get - - patch - - update -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - nodemodulesconfigs - - nodemodulesconfigs/status - verbs: - - get - - list - - watch -- apiGroups: - - monitoring.coreos.com - resources: - - servicemonitors - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - nfd.openshift.io - resources: - - nodefeaturediscoveries - verbs: - - delete - - get - - list -- apiGroups: - - nfd.openshift.io - resources: - - nodefeaturediscoveries/finalizers - - nodefeaturediscoveries/status - verbs: - - get - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-manager-rolebinding - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-openshift.fullname" . }}-manager-role' -subjects: -- kind: ServiceAccount - name: '{{ include "helm-charts-openshift.fullname" . }}-controller-manager' - namespace: '{{ .Release.Namespace }}' \ No newline at end of file diff --git a/helm-charts-openshift/templates/metrics-exporter-rbac-proxy-rbac.yaml b/helm-charts-openshift/templates/metrics-exporter-rbac-proxy-rbac.yaml deleted file mode 100644 index 3e518c0c5..000000000 --- a/helm-charts-openshift/templates/metrics-exporter-rbac-proxy-rbac.yaml +++ /dev/null @@ -1,63 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-metrics-exporter-rbac-proxy - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -rules: -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - watch - - get - - list - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-metrics-exporter-rbac-proxy - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-openshift.fullname" . }}-metrics-exporter-rbac-proxy' -subjects: -- kind: ServiceAccount - name: amd-gpu-operator-metrics-exporter-rbac-proxy - namespace: '{{ .Release.Namespace }}' diff --git a/helm-charts-openshift/templates/metrics-exporter-rbac.yaml b/helm-charts-openshift/templates/metrics-exporter-rbac.yaml deleted file mode 100644 index cb94fd753..000000000 --- a/helm-charts-openshift/templates/metrics-exporter-rbac.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-metrics-exporter - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - watch - - get - - list - - update -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-metrics-exporter - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-openshift.fullname" . }}-metrics-exporter' -subjects: -- kind: ServiceAccount - name: amd-gpu-operator-metrics-exporter - namespace: '{{ .Release.Namespace }}' diff --git a/helm-charts-openshift/templates/node-labeller-rbac.yaml b/helm-charts-openshift/templates/node-labeller-rbac.yaml deleted file mode 100644 index bc7fd4272..000000000 --- a/helm-charts-openshift/templates/node-labeller-rbac.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-node-labeller - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - watch - - get - - list - - update -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-node-labeller - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "helm-charts-openshift.fullname" . }}-node-labeller -subjects: -- kind: ServiceAccount - name: amd-gpu-operator-node-labeller - namespace: '{{ .Release.Namespace }}' diff --git a/helm-charts-openshift/templates/nodefeaturediscovery.yaml b/helm-charts-openshift/templates/nodefeaturediscovery.yaml deleted file mode 100644 index b7a7c5291..000000000 --- a/helm-charts-openshift/templates/nodefeaturediscovery.yaml +++ /dev/null @@ -1,124 +0,0 @@ -{{- if .Values.nfd.enabled }} -apiVersion: nfd.openshift.io/v1 -kind: NodeFeatureDiscovery -metadata: - name: {{ .Release.Name }}-nfd-instance - namespace: {{ .Release.Namespace }} -spec: - #instance: "" # instance is empty by default - #labelWhiteList: "" - #extraLabelNs: - # - "example.com" - #resourceLabels: - # - "example.com/resource" - operand: - image: quay.io/openshift/origin-node-feature-discovery:4.16 - imagePullPolicy: IfNotPresent - servicePort: 12000 - workerConfig: - configData: | - core: - # labelWhiteList: - # noPublish: false - sleepInterval: 60s - # sources: [all] - # klog: - # addDirHeader: false - # alsologtostderr: false - # logBacktraceAt: - # logtostderr: true - # skipHeaders: false - # stderrthreshold: 2 - # v: 0 - # vmodule: - ## NOTE: the following options are not dynamically run-time - ## configurable and require a nfd-worker restart to take effect - ## after being changed - # logDir: - # logFile: - # logFileMaxSize: 1800 - # skipLogHeaders: false - sources: - # cpu: - # cpuid: - ## NOTE: whitelist has priority over blacklist - # attributeBlacklist: - # - "BMI1" - # - "BMI2" - # - "CLMUL" - # - "CMOV" - # - "CX16" - # - "ERMS" - # - "F16C" - # - "HTT" - # - "LZCNT" - # - "MMX" - # - "MMXEXT" - # - "NX" - # - "POPCNT" - # - "RDRAND" - # - "RDSEED" - # - "RDTSCP" - # - "SGX" - # - "SSE" - # - "SSE2" - # - "SSE3" - # - "SSE4.1" - # - "SSE4.2" - # - "SSSE3" - # attributeWhitelist: - # kernel: - # kconfigFile: "/path/to/kconfig" - # configOpts: - # - "NO_HZ" - # - "X86" - # - "DMI" - pci: - deviceClassWhitelist: - - "0200" - - "03" - - "12" - deviceLabelFields: - - "vendor" - - "device" - custom: - - name: amd-gpu - labels: - feature.node.kubernetes.io/amd-gpu: "true" - matchAny: - - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["1002"]} - device: {op: In, value: ["74a0"]} # MI300A - - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["1002"]} - device: {op: In, value: ["74a1"]} # MI300X - - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["1002"]} - device: {op: In, value: ["740f"]} # MI210 - - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["1002"]} - device: {op: In, value: ["7408"]} # MI250X - - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["1002"]} - device: {op: In, value: ["740c"]} # MI250/MI250X - - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["1002"]} - device: {op: In, value: ["738c"]} # MI100 - - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["1002"]} - device: {op: In, value: ["738e"]} # MI100 -{{- end }} \ No newline at end of file diff --git a/helm-charts-openshift/templates/post-delete-hook.yaml b/helm-charts-openshift/templates/post-delete-hook.yaml deleted file mode 100644 index 553ec78f0..000000000 --- a/helm-charts-openshift/templates/post-delete-hook.yaml +++ /dev/null @@ -1,118 +0,0 @@ -# Run helm uninstall with --no-hooks to bypass the post-delete hook -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-prune - namespace: {{ .Release.Namespace }} - labels: - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "0" - "helm.sh/hook": post-delete - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-prune - labels: - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "0" - "helm.sh/hook": post-delete - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded -rules: - - apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - delete - - get - - list ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-prune - labels: - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "1" - "helm.sh/hook": post-delete - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "helm-charts-openshift.fullname" . }}-prune -subjects: -- kind: ServiceAccount - name: {{ include "helm-charts-openshift.fullname" . }}-prune - namespace: {{ .Release.Namespace }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: delete-custom-resource-definitions - namespace: {{ .Release.Namespace }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "2" - # hook will be executed before helm uninstall - "helm.sh/hook": post-delete - # remove the resource created by the hook whether it succeeded or failed - "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded -spec: - backoffLimit: 0 # once the job finished first run, don't retry to create another pod - ttlSecondsAfterFinished: 60 # job info will be kept for 1 min then deleted - template: - spec: - serviceAccountName: {{ include "helm-charts-openshift.fullname" . }}-prune - containers: - - name: delete-custom-resource-definitions - image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag }} - command: - - /bin/sh - - -c - - | - if kubectl get crds deviceconfigs.amd.com > /dev/null 2>&1; then - kubectl delete crds deviceconfigs.amd.com - fi - {{- if .Values.nfd.enabled }} - if kubectl get crds nodefeatures.nfd.openshift.io > /dev/null 2>&1; then - kubectl delete crds nodefeatures.nfd.openshift.io - fi - if kubectl get crds nodefeaturediscoveries.nfd.openshift.io > /dev/null 2>&1; then - kubectl delete crds nodefeaturediscoveries.nfd.openshift.io - fi - if kubectl get crds nodefeaturerules.nfd.openshift.io > /dev/null 2>&1; then - kubectl delete crds nodefeaturerules.nfd.openshift.io - fi - if kubectl get crds noderesourcetopologies.topology.node.k8s.io > /dev/null 2>&1; then - kubectl delete crds noderesourcetopologies.topology.node.k8s.io - fi - {{- end }} - {{- if .Values.kmm.enabled }} - if kubectl get crds modules.kmm.sigs.x-k8s.io > /dev/null 2>&1; then - kubectl delete crds modules.kmm.sigs.x-k8s.io - fi - if kubectl get crds nodemodulesconfigs.kmm.sigs.x-k8s.io > /dev/null 2>&1; then - kubectl delete crds nodemodulesconfigs.kmm.sigs.x-k8s.io - fi - {{- end }} - {{- if .Values.controllerManager.manager.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.controllerManager.manager.imagePullSecrets }} - {{- end }} - {{- with .Values.controllerManager.manager.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.controllerManager.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - restartPolicy: Never diff --git a/helm-charts-openshift/templates/pre-delete-hook.yaml b/helm-charts-openshift/templates/pre-delete-hook.yaml deleted file mode 100644 index eddda8139..000000000 --- a/helm-charts-openshift/templates/pre-delete-hook.yaml +++ /dev/null @@ -1,146 +0,0 @@ -# Run helm uninstall with --no-hooks to bypass the pre-delete hook -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-pre-delete - namespace: {{ .Release.Namespace }} - labels: - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "0" - "helm.sh/hook": pre-delete - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-pre-delete - labels: - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "0" - "helm.sh/hook": pre-delete - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded -rules: - - apiGroups: - - amd.com - resources: - - deviceconfigs - verbs: - - get - - list - - apiGroups: - - nfd.openshift.io - resources: - - nodefeaturediscoveries - verbs: - - delete ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-pre-delete - labels: - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "1" - "helm.sh/hook": pre-delete - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "helm-charts-openshift.fullname" . }}-pre-delete -subjects: -- kind: ServiceAccount - name: {{ include "helm-charts-openshift.fullname" . }}-pre-delete - namespace: {{ .Release.Namespace }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: check-leftover-deviceconfigs - namespace: {{ .Release.Namespace }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "2" - # hook will be executed before helm uninstall - "helm.sh/hook": pre-delete - # remove the resource created by the hook whether it succeeded or failed - "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded -spec: - backoffLimit: 0 # once the job finished first run, don't retry to create another pod - ttlSecondsAfterFinished: 60 # job info will be kept for 1 min then deleted - template: - spec: - serviceAccountName: {{ include "helm-charts-openshift.fullname" . }}-pre-delete - containers: - - name: check-leftover-deviceconfigs - image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag }} - command: - - /bin/sh - - -c - - | - if kubectl get deviceconfigs -n {{ .Release.Namespace }} --no-headers | grep -q .; then - echo "DeviceConfigs resources exist. Stop uninstallation." - exit 1 - else - echo "No DeviceConfigs resources found. Proceeding with uninstallation." - exit 0 - fi - {{- if .Values.controllerManager.manager.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.controllerManager.manager.imagePullSecrets }} - {{- end}} - {{- with .Values.controllerManager.manager.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.controllerManager.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - restartPolicy: Never - ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: pre-uninstall-remove-nodefeaturediscovery - namespace: {{ .Release.Namespace }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "3" - # hook will be executed before helm uninstall - "helm.sh/hook": pre-delete - # remove the resource created by the hook whether it succeeded or failed - "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded -spec: - backoffLimit: 0 # once the job finished first run, don't retry to create another pod - ttlSecondsAfterFinished: 60 # job info will be kept for 1 min then deleted - template: - spec: - serviceAccountName: {{ include "helm-charts-openshift.fullname" . }}-pre-delete - containers: - - name: pre-uninstall-remove-nodefeaturediscovery - image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag }} - command: - - /bin/sh - - -c - - | - kubectl delete nodefeaturediscoveries --all -n {{ .Release.Namespace }} - {{- if .Values.controllerManager.manager.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.controllerManager.manager.imagePullSecrets }} - {{- end}} - {{- with .Values.controllerManager.manager.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.controllerManager.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - restartPolicy: Never diff --git a/helm-charts-openshift/templates/pre-upgrade-hook.yaml b/helm-charts-openshift/templates/pre-upgrade-hook.yaml deleted file mode 100644 index 183571749..000000000 --- a/helm-charts-openshift/templates/pre-upgrade-hook.yaml +++ /dev/null @@ -1,110 +0,0 @@ -{{- if .Values.upgradeCRD }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: upgrade-crd-hook-sa - annotations: - # hook will be executed before helm upgrade - "helm.sh/hook": pre-upgrade,pre-rollback - # don't cleanup the job on hook failure - "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "1" ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: upgrade-crd-hook-cluster-role - annotations: - # hook will be executed before helm upgrade - "helm.sh/hook": pre-upgrade,pre-rollback - # don't cleanup the job on hook failure - "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "1" -rules: - - apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - create - - get - - list - - watch - - patch - - update ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: upgrade-crd-hook-cluster-role-binding - annotations: - # hook will be executed before helm upgrade - "helm.sh/hook": pre-upgrade,pre-rollback - # don't cleanup the job on hook failure - "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "2" -subjects: - - kind: ServiceAccount - name: upgrade-crd-hook-sa - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: upgrade-crd-hook-cluster-role - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: upgrade-crd - namespace: {{ .Release.Namespace }} - labels: - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - # hook will be executed before helm upgrade - "helm.sh/hook": pre-upgrade,pre-rollback - # don't cleanup the job on hook failure - "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "3" -spec: - template: - metadata: - name: upgrade-crd - spec: - serviceAccountName: upgrade-crd-hook-sa - {{- if .Values.controllerManager.manager.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.controllerManager.manager.imagePullSecrets }} - {{- end }} - {{- with .Values.controllerManager.manager.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.controllerManager.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: upgrade-crd - image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag }} - imagePullPolicy: {{ .Values.controllerManager.manager.imagePullPolicy }} - command: - - /bin/sh - - -c - - | - kubectl apply -f /opt/helm-charts-crds-openshift/deviceconfig-crd.yaml - {{- if .Values.nfd.enabled }} - kubectl apply -f /opt/helm-charts-crds-openshift/nodefeature-crd.yaml - kubectl apply -f /opt/helm-charts-crds-openshift/nodefeaturediscovery-crd.yaml - kubectl apply -f /opt/helm-charts-crds-openshift/nodefeaturerule-crd.yaml - {{- end }} - {{- if .Values.kmm.enabled }} - kubectl apply -f /opt/helm-charts-crds-openshift/module-crd.yaml - kubectl apply -f /opt/helm-charts-crds-openshift/nodemodulesconfig-crd.yaml - {{- end }} - restartPolicy: OnFailure -{{- end }} -# Run helm upgrade with --no-hooks to bypass the pre-upgrade hook \ No newline at end of file diff --git a/helm-charts-openshift/templates/prometheus-k8s-rbac.yaml b/helm-charts-openshift/templates/prometheus-k8s-rbac.yaml deleted file mode 100644 index 74e6bf95e..000000000 --- a/helm-charts-openshift/templates/prometheus-k8s-rbac.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-prometheus-k8s - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - services - - endpoints - - pods - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-prometheus-k8s - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: '{{ include "helm-charts-openshift.fullname" . }}-prometheus-k8s' -subjects: -- kind: ServiceAccount - name: prometheus-k8s - namespace: openshift-monitoring \ No newline at end of file diff --git a/helm-charts-openshift/templates/serviceaccount.yaml b/helm-charts-openshift/templates/serviceaccount.yaml deleted file mode 100644 index 6ffc7d019..000000000 --- a/helm-charts-openshift/templates/serviceaccount.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-controller-manager - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - {{- toYaml .Values.controllerManager.serviceAccount.annotations | nindent 4 }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: amd-gpu-operator-kmm-device-plugin - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - {{- toYaml .Values.kmmDevicePlugin.serviceAccount.annotations | nindent 4 }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: amd-gpu-operator-kmm-module-loader - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - {{- toYaml .Values.kmmModuleLoader.serviceAccount.annotations | nindent 4 }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - # the node labeller service account name should be fixed, not templated - name: amd-gpu-operator-node-labeller - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - {{- toYaml .Values.nodeLabeller.serviceAccount.annotations | nindent 4 }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: amd-gpu-operator-metrics-exporter - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} - annotations: - annotations: - {{- toYaml .Values.metricsExporter.serviceAccount.annotations | nindent 4 }} diff --git a/helm-charts-openshift/templates/test-runner-rbac.yaml b/helm-charts-openshift/templates/test-runner-rbac.yaml deleted file mode 100644 index 880395387..000000000 --- a/helm-charts-openshift/templates/test-runner-rbac.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-test-runner - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - list - - get - - update - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - patch -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-test-runner - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-openshift.fullname" . }}-test-runner' -subjects: -- kind: ServiceAccount - name: amd-gpu-operator-test-runner - namespace: '{{ .Release.Namespace }}' diff --git a/helm-charts-openshift/templates/utils-container-rbac.yaml b/helm-charts-openshift/templates/utils-container-rbac.yaml deleted file mode 100644 index f4bccdb80..000000000 --- a/helm-charts-openshift/templates/utils-container-rbac.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-utils-container - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -rules: -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-openshift.fullname" . }}-utils-container - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-openshift.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-openshift.fullname" . }}-utils-container' -subjects: -- kind: ServiceAccount - name: amd-gpu-operator-utils-container - namespace: '{{ .Release.Namespace }}' diff --git a/helm-charts-openshift/values.yaml b/helm-charts-openshift/values.yaml deleted file mode 100644 index 82f247da0..000000000 --- a/helm-charts-openshift/values.yaml +++ /dev/null @@ -1,85 +0,0 @@ -nfd: - enabled: true # Set to false to disable nfd -kmm: - enabled: true # Set to false to disable kmm -installdefaultNFDRule: true # default NFD rule will detect amd gpu based on pci vendor ID -upgradeCRD: true # CRD will be patched as pre-upgrade hook when doing helm upgrade to current helm chart -controllerManager: - manager: - args: - - --config=controller_manager_config.yaml - containerSecurityContext: - allowPrivilegeEscalation: false - image: - repository: docker.io/rocm/gpu-operator - tag: dev - imagePullPolicy: Always - imagePullSecrets: "" - tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Equal" - value: "" - effect: "NoSchedule" - - key: "node-role.kubernetes.io/control-plane" - operator: "Equal" - value: "" - effect: "NoSchedule" - resources: - limits: - cpu: 1000m - memory: 1Gi - requests: - cpu: 100m - memory: 256Mi - nodeSelector: {} - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - replicas: 1 - serviceAccount: - annotations: {} - env: - simEnable: false -kmmDevicePlugin: - serviceAccount: - annotations: {} -kmmModuleLoader: - serviceAccount: - annotations: {} -kubernetesClusterDomain: cluster.local -managerConfig: - controllerManagerConfigYaml: |- - healthProbeBindAddress: :8081 - metricsBindAddress: 127.0.0.1:8080 - leaderElection: - enabled: true - resourceID: gpu.amd.com -metricsService: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: https - type: ClusterIP -nodeLabeller: - serviceAccount: - annotations: {} -metricsExporter: - serviceAccount: - annotations: {} -deviceConfig: - spec: - metricsExporter: - image: docker.io/rocm/device-metrics-exporter:latest - configManager: - image: docker.io/rocm/device-config-manager:latest - testRunner: - image: docker.io/rocm/test-runner:latest - commonConfig: - utilsContainer: - image: docker.io/rocm/gpu-operator-utils:latest diff --git a/helm-charts/Chart.lock b/helm-charts/Chart.lock deleted file mode 100644 index d9b0216ca..000000000 --- a/helm-charts/Chart.lock +++ /dev/null @@ -1,9 +0,0 @@ -dependencies: -- name: node-feature-discovery - repository: https://kubernetes-sigs.github.io/node-feature-discovery/charts - version: 0.16.1 -- name: kmm - repository: file://./charts/kmm - version: v1.0.0 -digest: sha256:f9a315dd2ce3d515ebf28c8e9a6a82158b493ca2686439ec381487761261b597 -generated: "2025-02-21T11:34:35.236291618Z" diff --git a/helm-charts/Chart.yaml b/helm-charts/Chart.yaml deleted file mode 100644 index 20b5f864b..000000000 --- a/helm-charts/Chart.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: v2 -name: gpu-operator-charts -description: AMD GPU Operator simplifies the deployment and management of AMD Instinct GPU accelerators within Kubernetes clusters. -type: application -home: https://github.com/ROCm/gpu-operator -sources: - - https://github.com/ROCm/gpu-operator -icon: https://raw.githubusercontent.com/ROCm/k8s-device-plugin/master/helm/logo.png -maintainers: - - name: Yan Sun -keywords: - - kubernetes - - cluster - - hardware - - amd - - gpu - - ai - - deep learning - - monitoring - -kubeVersion: ">= 1.29.0-0" -version: v1.2.0 -appVersion: "v1.2.0" - -dependencies: -- name: node-feature-discovery - version: v0.16.1 - repository: "https://kubernetes-sigs.github.io/node-feature-discovery/charts" - condition: node-feature-discovery.enabled -- name: kmm - version: v1.0.0 - repository: "file://./charts/kmm" - condition: kmm.enabled \ No newline at end of file diff --git a/helm-charts/README.md b/helm-charts/README.md deleted file mode 100644 index 72bb01ce3..000000000 --- a/helm-charts/README.md +++ /dev/null @@ -1,205 +0,0 @@ -# AMD GPU Operator - -:book: GPU Operator Documentation Site: https://instinct.docs.amd.com/projects/gpu-operator - -## Introduction - -AMD GPU Operator simplifies the deployment and management of AMD Instinct GPU accelerators within Kubernetes clusters. This project enables seamless configuration and operation of GPU-accelerated workloads, including machine learning, Generative AI, and other GPU-intensive applications. - -## Components - -* AMD GPU Operator Controller -* K8s Device Plugin -* K8s Node Labeller -* Device Metrics Exporter -* Device Test Runner -* Node Feature Discovery Operator -* Kernel Module Management Operator - -## Features - -* Streamlined GPU driver installation and management -* Comprehensive metrics collection and export -* Easy deployment of AMD GPU device plugin for Kubernetes -* Automated labeling of nodes with AMD GPU capabilities -* Compatibility with standard Kubernetes environments -* Efficient GPU resource allocation for containerized workloads -* GPU health monitoring and troubleshooting - -## Compatibility - -* **ROCm DKMS Compatibility**: Please refer to the [ROCM official website](https://rocm.docs.amd.com/en/latest/compatibility/compatibility-matrix.html) for the compatability matrix for ROCM driver. -* **Kubernetes**: 1.29.0+ - -## Prerequisites - -* Kubernetes v1.29.0+ -* Helm v3.2.0+ -* `kubectl` CLI tool configured to access your cluster -* [Cert Manager](https://cert-manager.io/docs/) Install it by running these commands if not already installed in the cluster: - -```bash -helm repo add jetstack https://charts.jetstack.io --force-update - -helm install cert-manager jetstack/cert-manager \ - --namespace cert-manager \ - --create-namespace \ - --version v1.15.1 \ - --set crds.enabled=true -``` - -## Quick Start - -### 1. Add the AMD Helm Repository - -```bash -helm repo add rocm https://rocm.github.io/gpu-operator -helm repo update -``` - -### 2. Install the Operator - -Basic installation: - -```bash -helm install amd-gpu-operator rocm/gpu-operator-charts \ - --namespace kube-amd-gpu \ - --create-namespace \ - --version=v1.2.0 -``` - -```{note} -Installation Options - - Skip NFD installation: `--set node-feature-discovery.enabled=false` - - Skip KMM installation: `--set kmm.enabled=false` -``` - -```{warning} - It is strongly recommended to use AMD-optimized KMM images included in the operator release. -``` - -### 3. Install Custom Resource - -After the installation of AMD GPU Operator, you need to create the `DeviceConfig` custom resource in order to trigger the operator to start to work. By preparing the `DeviceConfig` in the YAML file, you can create the resouce by running ```kubectl apply -f deviceconfigs.yaml```. For custom resource definition and more detailed information, please refer to [Custom Resource Installation Guide](https://instinct.docs.amd.com/projects/gpu-operator/en/latest/installation/kubernetes-helm.html#install-custom-resource). - -### Grafana Dashboards - -Following dashboards are provided for visualizing GPU metrics collected from device-metrics-exporter: - -* Overview Dashboard: Provides a comprehensive view of the GPU cluster. -* GPU Detail Dashboard: Offers a detailed look at individual GPUs. -* Job Detail Dashboard: Presents detailed GPU usage for specific jobs in SLURM and Kubernetes environments. -* Node Detail Dashboard: Displays detailed GPU usage at the host level. - -## Support - -For bugs and feature requests, please file an issue on our [GitHub Issues](https://github.com/ROCm/gpu-operator/issues) page. - -## License - -The AMD GPU Operator is licensed under the [Apache License 2.0](LICENSE). - -## gpu-operator-charts - -![Version: v1.2.0](https://img.shields.io/badge/Version-v1.2.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.2.0](https://img.shields.io/badge/AppVersion-v1.2.0-informational?style=flat-square) - -AMD GPU Operator simplifies the deployment and management of AMD Instinct GPU accelerators within Kubernetes clusters. - -**Homepage:** - -## Maintainers - -| Name | Email | Url | -| ---- | ------ | --- | -| Yan Sun | | | - -## Source Code - -* - -## Requirements - -Kubernetes: `>= 1.29.0-0` - -| Repository | Name | Version | -|------------|------|---------| -| file://./charts/kmm | kmm | v1.0.0 | -| https://kubernetes-sigs.github.io/node-feature-discovery/charts | node-feature-discovery | v0.16.1 | - -## Values - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| controllerManager.affinity | object | `{"nodeAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"preference":{"matchExpressions":[{"key":"node-role.kubernetes.io/control-plane","operator":"Exists"}]},"weight":1}]}}` | Deployment affinity configs for controller manager | -| controllerManager.manager.image.repository | string | `"docker.io/rocm/gpu-operator"` | AMD GPU operator controller manager image repository | -| controllerManager.manager.image.tag | string | `"v1.2.0"` | AMD GPU operator controller manager image tag | -| controllerManager.manager.imagePullPolicy | string | `"Always"` | Image pull policy for AMD GPU operator controller manager pod | -| controllerManager.manager.imagePullSecrets | string | `""` | Image pull secret name for pulling AMD GPU operator controller manager image if registry needs credential to pull image | -| controllerManager.nodeSelector | object | `{}` | Node selector for AMD GPU operator controller manager deployment | -| installdefaultNFDRule | bool | `true` | Default NFD rule will detect amd gpu based on pci vendor ID | -| kmm.enabled | bool | `true` | Set to true/false to enable/disable the installation of kernel module management (KMM) operator | -| node-feature-discovery.enabled | bool | `true` | Set to true/false to enable/disable the installation of node feature discovery (NFD) operator | -| upgradeCRD | bool | `true` | CRD will be patched as pre-upgrade/pre-rollback hook when doing helm upgrade/rollback to current helm chart | -| kmm.controller.affinity | object | `{"nodeAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"preference":{"matchExpressions":[{"key":"node-role.kubernetes.io/control-plane","operator":"Exists"}]},"weight":1}]}}` | Affinity for the KMM controller manager deployment | -| kmm.controller.manager.args[0] | string | `"--config=controller_config.yaml"` | | -| kmm.controller.manager.containerSecurityContext.allowPrivilegeEscalation | bool | `false` | | -| kmm.controller.manager.env.relatedImageBuild | string | `"gcr.io/kaniko-project/executor:v1.23.2"` | KMM kaniko builder image for building driver image within cluster | -| kmm.controller.manager.env.relatedImageBuildPullSecret | string | `""` | Image pull secret name for pulling KMM kaniko builder image if registry needs credential to pull image | -| kmm.controller.manager.env.relatedImageSign | string | `"docker.io/rocm/kernel-module-management-signimage:v1.2.0"` | KMM signer image for signing driver image's kernel module with given key pairs within cluster | -| kmm.controller.manager.env.relatedImageSignPullSecret | string | `""` | Image pull secret name for pulling KMM signer image if registry needs credential to pull image | -| kmm.controller.manager.env.relatedImageWorker | string | `"docker.io/rocm/kernel-module-management-worker:v1.2.0"` | KMM worker image for loading / unloading driver kernel module on worker nodes | -| kmm.controller.manager.env.relatedImageWorkerPullSecret | string | `""` | Image pull secret name for pulling KMM worker image if registry needs credential to pull image | -| kmm.controller.manager.image.repository | string | `"docker.io/rocm/kernel-module-management-operator"` | KMM controller manager image repository | -| kmm.controller.manager.image.tag | string | `"v1.2.0"` | KMM controller manager image tag | -| kmm.controller.manager.imagePullPolicy | string | `"Always"` | Image pull policy for KMM controller manager pod | -| kmm.controller.manager.imagePullSecrets | string | `""` | Image pull secret name for pulling KMM controller manager image if registry needs credential to pull image | -| kmm.controller.manager.resources.limits.cpu | string | `"500m"` | | -| kmm.controller.manager.resources.limits.memory | string | `"384Mi"` | | -| kmm.controller.manager.resources.requests.cpu | string | `"10m"` | | -| kmm.controller.manager.resources.requests.memory | string | `"64Mi"` | | -| kmm.controller.manager.tolerations[0].effect | string | `"NoSchedule"` | | -| kmm.controller.manager.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | -| kmm.controller.manager.tolerations[0].operator | string | `"Equal"` | | -| kmm.controller.manager.tolerations[0].value | string | `""` | | -| kmm.controller.manager.tolerations[1].effect | string | `"NoSchedule"` | | -| kmm.controller.manager.tolerations[1].key | string | `"node-role.kubernetes.io/control-plane"` | | -| kmm.controller.manager.tolerations[1].operator | string | `"Equal"` | | -| kmm.controller.manager.tolerations[1].value | string | `""` | | -| kmm.controller.nodeSelector | object | `{}` | Node selector for the KMM controller manager deployment | -| kmm.controller.replicas | int | `1` | | -| kmm.controller.serviceAccount.annotations | object | `{}` | | -| kmm.controllerMetricsService.ports[0].name | string | `"https"` | | -| kmm.controllerMetricsService.ports[0].port | int | `8443` | | -| kmm.controllerMetricsService.ports[0].protocol | string | `"TCP"` | | -| kmm.controllerMetricsService.ports[0].targetPort | string | `"https"` | | -| kmm.controllerMetricsService.type | string | `"ClusterIP"` | | -| kmm.kubernetesClusterDomain | string | `"cluster.local"` | | -| kmm.managerConfig.controllerConfigYaml | string | `"healthProbeBindAddress: :8081\nwebhookPort: 9443\nleaderElection:\n enabled: true\n resourceID: kmm.sigs.x-k8s.io\nmetrics:\n enableAuthnAuthz: true\n bindAddress: 0.0.0.0:8443\n secureServing: true\nworker:\n runAsUser: 0\n seLinuxType: spc_t\n firmwareHostPath: /var/lib/firmware"` | | -| kmm.webhookServer.affinity | object | `{"nodeAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"preference":{"matchExpressions":[{"key":"node-role.kubernetes.io/control-plane","operator":"Exists"}]},"weight":1}]}}` | KMM webhook's deployment affinity configs | -| kmm.webhookServer.nodeSelector | object | `{}` | KMM webhook's deployment node selector | -| kmm.webhookServer.replicas | int | `1` | | -| kmm.webhookServer.webhookServer.args[0] | string | `"--config=controller_config.yaml"` | | -| kmm.webhookServer.webhookServer.args[1] | string | `"--enable-module"` | | -| kmm.webhookServer.webhookServer.args[2] | string | `"--enable-namespace"` | | -| kmm.webhookServer.webhookServer.args[3] | string | `"--enable-preflightvalidation"` | | -| kmm.webhookServer.webhookServer.containerSecurityContext.allowPrivilegeEscalation | bool | `false` | | -| kmm.webhookServer.webhookServer.image.repository | string | `"docker.io/rocm/kernel-module-management-webhook-server"` | KMM webhook image repository | -| kmm.webhookServer.webhookServer.image.tag | string | `"v1.2.0"` | KMM webhook image tag | -| kmm.webhookServer.webhookServer.imagePullPolicy | string | `"Always"` | Image pull policy for KMM webhook pod | -| kmm.webhookServer.webhookServer.imagePullSecrets | string | `""` | Image pull secret name for pulling KMM webhook image if registry needs credential to pull image | -| kmm.webhookServer.webhookServer.resources.limits.cpu | string | `"500m"` | | -| kmm.webhookServer.webhookServer.resources.limits.memory | string | `"384Mi"` | | -| kmm.webhookServer.webhookServer.resources.requests.cpu | string | `"10m"` | | -| kmm.webhookServer.webhookServer.resources.requests.memory | string | `"64Mi"` | | -| kmm.webhookServer.webhookServer.tolerations[0].effect | string | `"NoSchedule"` | | -| kmm.webhookServer.webhookServer.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | -| kmm.webhookServer.webhookServer.tolerations[0].operator | string | `"Equal"` | | -| kmm.webhookServer.webhookServer.tolerations[0].value | string | `""` | | -| kmm.webhookServer.webhookServer.tolerations[1].effect | string | `"NoSchedule"` | | -| kmm.webhookServer.webhookServer.tolerations[1].key | string | `"node-role.kubernetes.io/control-plane"` | | -| kmm.webhookServer.webhookServer.tolerations[1].operator | string | `"Equal"` | | -| kmm.webhookServer.webhookServer.tolerations[1].value | string | `""` | | -| kmm.webhookService.ports[0].port | int | `443` | | -| kmm.webhookService.ports[0].protocol | string | `"TCP"` | | -| kmm.webhookService.ports[0].targetPort | int | `9443` | | -| kmm.webhookService.type | string | `"ClusterIP"` | | diff --git a/helm-charts/charts/kmm-v1.0.0.tgz b/helm-charts/charts/kmm-v1.0.0.tgz deleted file mode 100644 index a03738df303000281d24a30d7d5f559539959bf1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 30212 zcmV)cK&ZbTiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0POvHbK^FeC=Tzxr#}T=XHMBYW65@>)5*GaXUnmjj8FRFV>?N0 z&Cba{BqU)>0t^7ktx58J_Fv(_g&;+Vpe)Poj4@Rcw@Bjp;JIIJ^Zej+iiH}^ai0C# zgS5lL!^1bPU&DV74-cFFJ^cLW=--ZBfBD6m!#A%!KlArm=F#wQc=+P6H}Vs0 z`u@*wRucJe2w?U7KRW#U=<~+?fA#9stLOW_jh1kx1kW-ej$a^@!(D|x5yE03&0hmF z7$lsH>Eu7F_kS|K0NB7=`Vpp#h-;FRf~xr`XNr7OrVoU0mhrnwL1#206LKySoMD4Y z#yFD%-Xd(~UjGaY2I#wY@6at~^p>MhNwbs)L~}eLfJ)#Ojc`&yXa30@Ra44Pl9f^s zVLKBf!-}LA`i-ykXUAxgh#}<%ruRW1_&-RZ4#-E6l!}Yv+0pRT=fhXc$w)e3wMpn7 znseBd=+@xC+~7bxCp4im!q6?5qXJVQyC-^0C+rxdJh>$>Rl$?|z;rw?9S?GkTn8`_ z{TWPsc-Wlz9T&H?nP(is?j4|#xxeiy>a*v}eav`DWXnu{gvar7HQlvL6naBNnPo_l zL=c5E4S?zDAA!0o0|`1tf=sAXVvZOgsYHn&DPfA@3@L7Ug;$pW#budg*D&lc+SyrS zUNz{r*L?KZ-+MZ>Bfr;l%mKGd$Us~Z)+#LFP>P*|AO)9HaWVIUxF6!7hdb6Gc2tEY z_FaW1z%e+mYwl*jGn&j{BbW48@U&XE-jGZ9MJAb3WGngtV#C?fUd=UC$-2XAh(kI~Ng&n3>>@4z$!a7o5v zl4!Vi&##jy(Py?3HJsgmL6KobHa4mtl9wVO(!pj%bE^Em5K4;j7=3YgnEPLHk^>li z{?+Su)Pv{GC6No?9kqP+#(V{5|DLA=Ns^I7aREoJrexW18^&2f_6r0U_S$}UjDGpe z^>{KKQ%2RiS7+FMWM;3sKB@hJP$9^e2tm@bQcyOrdq~;jV#2up`5cI%Ce?n`19V5| zWU7wQk^jwgM_59RgI_gYp_1wO$3hS(D?qH8g^?%_X?z;O%K1ksmA-<4fS@4F>?9F0 znvjzu;U%-|2xg2cdn@!dS6lgxC_xi>Z9dX(7hD(!7$75&OqD9kP1lc((O0iue{TLM zxZ(-Vj?vBOrMZ1rOo+OK*W71Dm^n3c|KjpRwOISx8P73g$Bs1^W<0^!3wy+;z;@~B zRC6}J{Ws3DWAy6)2A&d}sp+NQBl0z6=}DUE%|AxRUmbpBj_i(%rkvlJx&6CaLPl^( z#JPsR;J^vPql~1-NQttd59W$poPmrslyoA89|yNz=`(v_t_Eyny_1ttO_~0`f7W^% zg_ArS!vBu_#_B^%ND>|`Yi*4mZ@m|s&BU}aI)|~Z$hT`W`;AG8U+N}kjT@{LD}VQl5MhG4kT>h z>H*j0m|QoTW|QjEd*+QkLEb31!Ha~|5Yjk0YH3xIj6Sw1<&$i;d48ZsUT9HI9{e;V zSwVyxs$zqxzz+TY=bsyK8C+Rw?BOG z$AjN~d-2OJ1C)|6Wd!YLW>3Gjg9gTJnWtPp6MtgKjNs4e$z?_FKek7)E zcwgxe^!z0nkp!2LAjxxrzVqnSk{L>}R3a>hL4qXJuVM0=w-?84dmYyGY!M6w{s z%i)U)O?@(grQI&e1xRZQf#0Y?cQnh;h@evHc}j#~@^-eZo2sAJH?J?iFdDedHIk`emrp9RJf6KxDmOG8U%%Jh0 z)&6G<;DVFXH^Um6^eMt=TF@|WWq3rgB_|4RRL9v|M!3V~Z)*udzk;NJ6M}aBw1akj z+Ub9Fc%E}+W+&TcXP%kqO-Ysy<@5k%e2jki1yPn{Wm;dz&5Kv6~au@lw7!X18WYWnH5I{#M*&jYKw8QA^5k9D|%|2sN- z^QzAOy*hk#_~kSIw~bcw|2TSaOIdo1EWUh^6NOW(uoltUky~)*(Qm&Emc!2rWD%@Y z-7H|jgy#ikgel|j(f(3_g&Od2^-4`ZJ`OaZFAxHZ7EU(v^yyc`C`$=b=(R>0ny)+d z5~aq(7WpjG3J>+0`b*^XdylxTMf%%VpO>`%SI96fT^ziunGF8lIM0I4UOnU3-&?EW z|4R3~NpGMI|DQLn7v%phj*dQm#{b)BA^yijArCweIIAp)_n-rx1m3TVI<2^W=$^M> ze6WMO4VD=yzpc%w`=u+YK4&ciVfe`RY}s@vHN@X2V4nq;LXm%nz66S0^_gu6}xd^6vcl^5pcqIpQ~h=XC=mDE_XUm2~`Mybn4y5MCNqJ_qQx z-;QIxsJsYt`?YCVs4InQGYUNhB1WoS4|~e!j4&c4I6;MoeyUXQEm3vcUliifA;FoKQl)gavg3M8In{_y)Wx- zX0=X?>|XQOX&TQFSnF|0d+&z+YQFAftk{dA!(d|08zGV>@gD??$c?T|9E&9ozx+x-Z%0Nc=R^WtOE&<6^Out)e;5bw* zbHSSkttSE)U@lw5!S{cG@}KC2Hbo7dRnCn0h~-RAqq9_tP8i3=gC7QJ0*qy-#Jc=&`M_+i!LIKbaAx27)A9qWhZ|*U{0@v;WstS}5?t5#v%#xuE|9%rP{9GT2uR&sPoqpI5J5zka^|TWRax|E~>1Y!VGvAbV_Vz%yb!0~|hBu{SReT11U* zCwAA~irp8o9Q}hN3h+|RE!Xm&(SuYx=&t<+oEglAwU`6=9sTy(?qfscbK6#{T(uMUqI{{KgB-hAD!ruH`8s@)9Xjq#KXakB*ngQrV z`P)|iO!J@Hwp*S4*VSR0iUH}={~UhNjQ@CZc=$~Jx6xYT|4yB6~= zoCp;)rsuHDI4)22s@^-HqHUalrw;^!Z5ccpgPW9s@#_kt&3g}eAQe`n+1-|GjKTAk zxpl))#6{9^3^+IIM!t6cL5&+}u%2L1Yuijen zc2I@Y)%l6XKCex;`u%UA`-4QJ_rw9K@BgdUuiiBCzkK<{>*xAUTWR%xFG@Zt!mKFQ zy!xe7Ja<>+j6nPf<*Wq{w6TxFHnwviCMO|64SYZ$KcGx=D1!tK#)9W77>a|XxW{^Y z0tN~_tarnxv5bhpgfKuZwyIDqE#>TRc=Tp?^vF2NMI0rZxtW;bIBSP^b2L*f z)cb1C0V>8b^MtZVnPJiT@e71x!h`(gh7kE3&{9_+NP<|2To6RrnA_qq#|H-!svPQ0c%GMxs=0{+p`%i9ArDeABiVtZlK~dVlq!;_ zQjh~&&;cxj8GN0mpLwl7ZCH~fQ7U0XXV>65^*A1Vxe(RxWV$j-tWH7yq_^(s{QAc0 zC(KEiL#bX}*cW#`dN(N>6Jfs7Gz9u2EG;-?%D^QcMkLEoPL=d!4Upo^Zcl+;K~8Vb zW~9v*E*Lt+Imu2ny4&tq=~IydeF#yf)r{;7UpCttMr;I;t}#d3fOFfN6^_i$tRL~i#4^)3%EO9x(;SG9J$sGHfER`j@VP~U~`HcrECu`U0i zx3nFZyB6$nG9}4vwaHDuUkZb;uu@LHE@`GN!ZJ9qfTr8moM!|xG|uqEOgtqf%8E#I zHzjK7O6o~dFcPIu0VyD3d8t7olf0$-5rH|H#bQfI(#48Aⅅ~v}#W^YSAEHU*@2G zhEW$1AmqMR5<1Z+Sf78+B@i&eR5nL^1VLD-9m!pamR=T6pwc}SyeJ{zl1r5@=Teaz zlIa7f#p!^vY`$eAx!x5^gxRXaCkWv*1t|z;mrK82-MeWSmg+`eCZV?*nz!7Am~*b$ z+l3z8QR}gg(m&{bonE;^R{m|08d}L2@eet|jO)v^c)wk{(L~nrL?favpW*yUmJ*Rc zZE)LLGR15{(jh_@V{>V#M4GnVGJePQ^;Zm)((wQ=S>N5s<%P-9=ehwlB1uT0@F-g# zysnM8$gw&`nsN_xk4xTLfvrq4Yq%mXOJx7Zfgtc?!zaoHj} zCvQk%;xIrP&TU!23OGBpY`xT-IYbeTqG3NFatYug7yFC$%zwy7DivW#y_1*w26^up z6mGk$zp|u%!peNv*tY)I(x;H?Z|T!(I%zT^)x#^zN0wyy*Jr$?@||gYPjP`5R4w)x zuRWsotW<3jBW0pJ$5OrAH#O|sFZQg*m}dIn(s76oT%bMkA4`ebpylO0Dj8sd#t2hv zFojfDXaX}QYQ2-+xe=vT54Al0MlWyK^Ntg0r}VDtMP%Rl+QL(*zmlbm0n?cR(iERd{%n3wr8bRw9-o>u%!>`Rd~Xu+tQVphgb-_$d!H^RDNEA zj7l}gaWS|h^AdUHMR;$Y@7TOSUlD=++Pl-@QZ?=Zi z7sAv%>3_;(kb9_SgyL`+Q$3WkWOw5|4uAOt@J)A-GCj6ok9 z%B?&$G(NX&w>tiJR&X})&{6Z;S&t@k;Qv>je|glb|MK}4&-i~Utrq{E=2YrG1({HY zDR0vx{orB52AiLT5YnGMbiv#@X#)CZ3D_8|vTHEvMYo}gA?yRqZMOyGHv;nS6^!2} z5Z?yj+tVO}!R%-dnyAK@W~CsmdgC{kX8wfzdjrS?22kT-@9UBG`laixD!Ek>LFbcX zoCgNZOUS0ia~?wtb*s;v7kV}OCo>ZlZ>V7S;OzYE`OWzYgc^F;h4g-srsRA)1{ENB z&l&l|Pg*kt`ltqwCSgGNz$Or|``vN)GxOH)c4U>>D~_ctBg2wYs!|v%9@Xj9`H4NM z0s8*(>}1)wRT&Ex4(+iK?-OZt`rl-5HzNXe%71TOH}$`Vub%b4TWPEGKldy2PLmCp z`Hayk9gOD3F5j%V*Fl2XO)HL^#A&=D+G~&M3!OJlw>)bFfJa3Dt77rzxcda|KZiqm(Tehx6(dCmslwx z7|DMW7J`Gf+>oXOADGt#|Hq5r zOeQ=f2cH6>)b~FTsXVYG_5Rr&7u(AF|HZ2>HM3X0|DS*P>RJD@l{Od*UVMhGL2LrH zBg_U|2ZvN$|djlq{8g3?mp>^i3qA#cXPLsEc;}l^#AH*3->3B>akfR>{ zTDgTkN6SA)%RfiUe`3+{=KeS0;Qc2IVd{#xMF@+@im1ET$S{OX#H7siU3MswFeT=2OP5E>B6L&RcZlxWmFV={nclHd zYN)a3-A2b2g3oBWcy+_3jzn)tu197-2NCovs)+@%TmKyKF|%eVanUP6JT`Ig}Npa(MHOW*Oil=0k$d zc&VrVhd;nn1xd?fX^^09kO)ApLw~f7?xr-EI=CNmfrw0S;g%Whgq$;lKOQ4GBtx|G zhv3l;%xlG%h2VNxHXIdJl2Jtj&d^TS&k&vK(IG>DKMhzjBQqlO%H3ILX?TJ~ni07? zwiW5?>ar*RL*PyuqCzF&LIcXM8{3Di1-sXo!;~lTKw)_+o%vLw-vKU)%mzRl7%pX? zNdac*0NWce@NjkTnJ5{fY%;+9Ey@OX0GffNvs)Tb?~lJL6clxY;+bZrv|+SV~d?@Fn6vZGU^SMt-v+4=jMi*GK@uU0Jv#R8yiAWHUH(n?S> ztq;2I?5rXOg&;G^OPS4mDyDXjZo+bl_F%DLHg}#c?cFxWP8)IQ9AqoR#}2g#64W_L z3yF#+M4i2_MW~jMqt4Ol3cuG*^ae^`tgN>7x&J8Lrut zrtj{R1^C@Ed`9DBIX;pXY9V`9v&EuRWJGII*8Mv&_pauB77xA!5@B>p=Ae$OD#drN zw%=S4g1UU9NFJivvM-N;JsiD2n(OX~> zewkEUR3ZXCb96?sxz8c8k4B|JX-ObCAx>`Zut=qrwhFAMO{hj!akBLW*wUvuG+>V| ztW`vMbvXFifU#I}EBY;aU#3J5jk9|U2&woIA~fO3=IBmvHi_S&IHoz_EFp!G2YjaK z9l1NW0DwpjlXCT-3h7`VZi$mk!viv??{kgrDjJFoS|vG@y*yxd9AI`AG&=opy?^O<>jMy zLIUohB!vW%*3$Yw%kGD8#yw7Tt7ykrHow^#`ti2ZrrGKkaRz*cwfdrrf<+Xw)LTWP z#@aI<4wItXx19`gl5;Wl|0Knf^~P**GC zTGl)7fo|QR~ao8FK{DzT$*-Q&Za`&OIO-g4}@+Aht z#fI7ZLljgaxTt12lqoVHvAoWp{~}tME^1vwc&Um~*=)D=nnWK4`B-oX(!IS#4MK1f zE00*W%wzQb_WtKjzYbph*WTV=4+sDLw?FOu&k+9j6mexVUl6G+HIpRZ*vMgLZC)a4@6&$ zy7UKDeaeh3)jcvFcS-rCaFwt|%hp5d=;6k?wPA*ER!p7#*(p@0AQBOVN0~L7cXW3) zXE>*cgQ$}%h_RK;Q)>X{u$aDD>qg*M~v1+Qx@E zkSo)DjonD5ToPsqkGk=lOzpukHHa)uEVvjOkz^lf8a-E2eaU?s40NPtkmEEsf&Gfa z#M25?i>3EYlAbV2vUxEh5@#B-{YHNgdZ#Xh2{X2G$Sj#gC{$;Tq2 z2?fKW7h5|9_Y~HYg+w$jGRUn2muKLF2IJdU&C|N)l8*@?sh!Fve2XJqGS<*{`nY37 z;va%fOo(x2g9S`yC^wIsY_(z?xF`t3@7SnB3kaHJUa{}};sb)|2GEYpEH%Fd78di? z$}RU{QBh2BtYiYyi19Az%LRkuiJf!9vt3!wZt!$AXI2$Og$Y6B(+zAvtX?e6jJ z#$1}sz6f4mlVfMo+tchntO7mdcMgcLFtNz`Emw3_Z`+hd?2Y2}cmiYo26g%!o+B)H z$s&9xew-3!3MLY@kKQ#V{$9(MPWProHV|P^g7Xkg2X}%h5>VDXQ$4hPc%6q(S5QM2aXS-Z z1`gnkmSzzU?$==|o_D?vyjFD`VORG<^^hsKigJRP zCV3hcxf>W9-a2C$05QldL@gF2oKa~}fGHZUsjv+R$#L4P010*?r~+@Ra<`0Ep`gZy z>yNgt?*WBQyyKfo4pE_Zp}C2iR8#95tT}a$EEne8J}*D5&XKyOy&rPS#>K=&O&Pm{ zE7cnp6lY5WyRftgVMGA=R2)&3LhTK!(ZbTWu*QI0$AN21)J<^AXPv_VUE1TKaYI~8 zTr2JAzXl-cRo@24lt~ebu!&^-xui3k5hISd>$EXqBFoV>m`}@P0{c|)o?)#1Qa-d7 zJ=xhod#0ydi3I0rb4-EVsq>FTr*mfC-8y$^42mso+t*}@XG9`NbDCk1L5SJ4StPjn zw8l_xCKT9hBP z(O{DiwJSkGfO4EnDI-q!XWoC^D!f)~m}xz=i#A~6z=F?b_)ZeD>XBYG6rbOK(rPSk zH27|q44Piu<%Sf-bE!yfd_e6zw{Qz5Gv_6`!{)XFT8+?u(Bc@KGg*qt&un3+HclUo z(^l_b6J%bG$VTuH=Y=WrLO!b3JNszo<5=qdFg2DtLv)dQ?Kb366y7D)e4x4fhKRPh zZ5Lut9~P>>LeYm%=J}2t@B}}wO@PhC*cmYPYl>?j(;_5qD=d{`1XNXhI# zrub+drYWr#ZhcKyqMew4$7{#h?T33%^)u$e52%;2bGQRCj~+R}3}pn*tUPEC!`!$# zbTVKjwg;?tpTCNNo)|mdLn*#ozt*u_iIN!Eb<9QL1y!r=Aqwt4Wi>m<)4i54YBMv) z#o4Ae*7XbOX-0o83A#A*q1i-8DwUQl(i+jFh7#|cp8OY0o=E?W&alFUH3C$(a!YLbRiKxe?HN&)S_~P-^u#hD3Gh~v zcRcOKiR=a+I(qRJJ=;_BJIZyBpw{UQ61^fgg<22$=FcAmRbC@xsF`alz3?x0qIavDJ}>{z6M}BvC$bJgAkf zrx-$I^DyUtkCcsrCvn*G=7zH-Yo zGyr}^9e20K^y2VEwfU=Gb925g&PYfx9mjhN>es)yt}D$?Nijv=T-&Q2V1bqU1~3H6 zkVehED^W(rMA4OtiuWC4RB_QS94pE<&h)EcrOE;FF8CFrgl8E^TrD5i9{1TgQaZ89 zFkgMkNaP3R=nk&VPtM+*5A*c1DZd*iJ}4y_&@K zwNm+PH;vC$Q~2B`ea~a2?&-~4MCF=UKhKQDi<`zOx=#r53i%J=Ft@-dm*yRlcsIZo zIL#vz(N7FN$2xj-vcqxG3uWo*BkOZqrnkCqV}G;uVejmMooa)mTH>*;Ep^Y9-rg(C zbIa%Q=WJkRFeuXM;5Hy1J%*dg(^9xU0q2uV95{MXvc!|toNeI@;w6TMW?(CN1J&%L zEns8Q)R{WAR)uGZ{|lTGMTD(NXX5i%OlCL5(T9StYlV~B$4N@_%ygRe%yfQ&Oy`4p z7`gjQQ!S`kkGaan*`T^7fIFX-9%yn74o{)Lig#MRV}tUIW6 zF89&@x&H9J%5_(N3Q8isJ^kW-4&S_f9q|PfBvD*I1a1{MRk&y$#udUql7a+(mbZ!w zm(S^B3I||PjgB*#C_`FIEWZH+m+%6{@}fb6ttBP}6Cg77;EA~_6aO)hcD*5Ds zR13lXVSt1(tM+vAu(To?x-*6ID=TW}IJd;q*3e8SVLlz5%{y~34x>NygN#Z&g#Qkesn-;ykhPJCzO=JRBWq8|Z**LpOBc>a zRB4aKgc6qwsx(p?5-n~?>hbRoqD`fX@Qsk;GOYPiqu;>uS_%_AiQVCF*B}rsOysk< z?fUnw8y9JEtlf@xxX>VG%U;@If4CM5itWyHW2}B@{G*fj*CuQsKvks)Eeysn#yu06 z=Mkj5Ws=dU^K(njKlG#LHj+DwI1|$zfsC8>Sj614i*Yl|J-Fy0OXaKBMkTT_?cwRO zX%9-JP3uFeP3v3CC&sV9wDm5M^FlD2Vv%f7tACam1u2LL4tYf1AB(EZ6~ZIQvr?G> z`gRdwdzc_G+2Nv4e6zEFLIl-|<9q6zHaD)sL0o}@0j$59HpeRow&Efam!TtK88MN(G$} zA<^E~M9Z2c0sqP<#Aq!Q^t^Ee74+Jc9pm)+!aaqJF}or|-CIP7cg}#*0hBT4r7+fe zUqC+LLTKI!2F4NzGSLF3Fx44MM)Xquu(9x+2Q~0|OYVxmrOv)A_ge@)qp4$lLtING zDzFSvLSQL*LP1qnom%HOMg|8K+9$HCMXHB3W{z-CWSwqW@%a*YN-oP$M&ROib`>l{ zRgyKEtMSvbf43>fZJxbV_U%gXz19fNCiQ2N`m;$LEmTuA_VTd&Mhz*s)Un+gwvNg2 ztU`-MT4~1D2Le>i2jNmN~F0YgbHSo_LF7J$<`CzIFd|Nduu_WiFR;!9vBX={{l|8P!(HP+ zRtZ0)8V|lgge5%f17muwLMmX=D!pRpTamkYgC9a<$Ht!x<{8Hm^rQeTnddj_vscl` zvtqt{3Q@aJK0o=f;jgM9hNf(*uhLCqhkdbXd*;(&Qk46)n_*6JF6R5bEB&z6w|&FZ zNk~c|2!nElo-unMzRYhxOunG1)$d?sp9M90RpWl_(-wEuSr3~xlk4vZo1qyNG7bvS zdaPJVXH@b|b8FS2N|aWuU$N4~?|Ln!aXi;vWedgSN>!A~BJ|jz;--ON(q^Zx*VnIj z=A#(1iD^%^%4*f_Rd2OwZnM_CUlon3>JB(g{{gQ>Ul!d>?kRHUeZ1m6L4UWX*?@KK zjBYGZfa|Xw=cPzqaDl}frB+KgUzz3XL91K96jQni;YO!YBPlKOjVDrrUc(){gPIez zo`v&Ovah)!PB^(WBA(yk7_DlpD8bH-^$5ucW?oAVUO_x1yVAdFhVG>>H)de#=DKG) zda0S-(Xj(#&2xNtW)Syxo=OF zb(?xJKJGatQl{=jhUnHIP*Ck-2Iogn%!AXA~>Ig$()a>^BGi*SJx zsyf3nl)%X?6zmnm&UE#(l9x6HC4_={w5FO0UQVVEaA@92GAvDu=xWb`*1oL{L!xxf za846nWk~QDm6Tfx9*o(wT#PP#Q5a)ZJk2o8(Vip(ox4k6o?IJA$>@c<{m?>(PVNih zGn>KI>9+${47mE86J}}ql)LM}YOrlzEf?TN>FS*kmh#H`XJwx!mlrbB%9wj=3+=#z zjI&9Y+|}RumHleE#-MWVjHV^d97XF~;OX^+8J13_YH&wDTW*++3dgz0(gjy_DuMcS z7VNc}d$Zg-Jckt1*8REW$mEKknC{0Nkr=rnn-*|7mwW=qBf$)~Z7_FeC;|+zxZbY& z78FSGpBNrMVhx`i(p9YFGpj)FhU7(t)y5r%ehkbjfvsO-Ak5MM&NRGV{%~p|i4ET9 z4>C&_U+7Kyr20UBf^2u>+hF5y)H;H zG}|Ws9;Mwth%hDQLv+c(N06rG&cWOT`G+%o!7+LLe1a8Wz&>W?1$ zo%<}XRUF|Nr>^Y9+D5p~4TQWq~n&S!`X3Xg_1jWoqib|$v7A61q zdBBzg+!*AnC=<;-6oL#K?`PhM(1+Nvi*7fnz1;V}KE+01QKZnd^*)cho(sVsdLKAT zD&7tbTMp5CVA-;6ur}^(MmJ;agN!6ZN-XBaC7&u6Xi-WsE;Ha|yEg+fQ-@lRgfpo` znZW5`G!p!lurOLG0s)561C{T#=1>?%$rQ5*@i|3Mj#Gf0@4R5ty$#%-5FTq$gDNP8exe+HM@q&gK zS}uP$jT#4}BG_HX1*tULT0=jawuz{5(HwdTZbXA}Ns7`k+9pt;?HG$>FD1WbU?1u* zZYmF_q#mTu%3%-T1Hu#)B*@GYHIzNal9?M-KT&h!6QX40nrw(_Wbeu{U}lEE=DAb0 zx}??xMfGjqL~=_!HMVINm-%#EXsJfB zbkcfFc|SWFv`?tUuJJql#~KG|B@M6ZF`<%sPWdymVBwNU)?a_Q=Agg=x9cIId%UnN z%TD~#qK*J_kBgSqS4{9MBLcS0Dt;R7rBIO57>eV4LYy+u%Y}^Gc?K6tlYZPe^nfzd z`3bj;8-JyLbSlRaHaDAg766G+@1cpFl9m}69%Y(q2>{8@r$Ocw58k>W5DK*Vsn%&yDxwg30Jk^I@b!?2g zYqeE}mH(V_=@VFWKOZH?c}1hcV+L_5C)0X6G~GYw0M#+;_V^=>#%0~D(6BKe%vk*@ z4+B^|XAR04-P1mzV@=388-V#ZdK#5^&T{cX>uGNkEP#!I*#tvWNDl3>sIW8B0705Q zUz*^DC3cxne2Pm{aTlRpCP3Vdy`e1CHwwrmXl!JeB!qY!sl`$8jDL+F*saR3N~T02 z$srxOR5;5>>P>a#AUSY+){GZDUcfNMZ%v@Bvg+X?!&XN(E?Ht6gsGZ`W$SCqJ1AQu zs3POp%(1b_1d0<6JU~h3hIVSXu(Gq*T)mnU`kWg*U@OSYv2VDvkeOn;p@#`&>#~jE zy7q$2T3{z%dNxuLU$R)n<3VJdX>h4{julNngGtAM$ujqK;Jmkt;U9o|vHRWg8nrW^ z_t>ytkL)Sbx5F(n;HP@d`^Ke6L24=pK{VIm5@rgebnNrtILq^@GwUOZ{s?{yL2pJ2 zoMrq@B6Y{To~*J=+TBe>y+8Ij=O$Z6H_Vz6iT*$0H) zb7dVXfFZ_9^@|t~=pCL98BsQ)n^T=%xv{?$nyIpqPH+z$0&r>gf>9JV6tD90fZune z^&-&EA!r@d>WU1}wRJ0N48E8yD`YM_*FBV&r>G z1%&N8Dh->?5X}oyod+)Yz(zefy}X}w0=}1sd<5%>nb^Ms?&+|IOSwD2p)7KE)6WJkjvSJ6!uKFi{XS}zkSHz=RGT*l@*hs?! zb}#pwiivxuiW}-K<1${?+d7bGT69guuq~_4X!aEft@Ymv!H}UWWFnHA~w$CloqCDrzAj?*8v#QegNleH>i=wFW5eM&z1hq zIRvi&P@Zuj-*W}OZ1ptE;vVBHtWl4GsSO;JO%@dhvbKBfTo(PgQHd@X;=*p>npeOM zZ3cA~hRT}VHmW@ny6>t8si>-E`MQtrB&yP<6bj_WOiTWwd)UK1Yg zgYts48Wt$Wg+_P3Xif*P!EaCxT@KL+l9Wv{QhR2tA3--ghX!%^yi-}^Y`E{E>E0pJNp#3n?@QmT;Ip+8SvqhKm+X^? z>l65)?SmjBo65e2bTMwaWdw;l!ss3@Y-sm_-!N1^tjZt7)jX|Qo zeQWF@;*qxTZ^h`6n42LLR%GAQX&q-lF&~0*p3<@PySol-r~4*@giQbuX55aRM&E6` zSL(V!g-r&4`biI6Y?OOqJ04x`h=KO_cpOx!>a2g~+M2W|rVN;C_ai<~N5Q3tt7SQ@ zB0BEL2mAo2k&n?gRV`a8!9GA;0y%woC`+=^x(OI8r}51L@ATSazWUtpQ7)~WV{;~d z_~j?IZQJ%8+qNclGO;nSZQHgcwr$%sHox84f9(s{s;j=$)pb4UC*5_<=bR=Vc*lF3 zEp8!FhSa$+*sxmmyb?qB;!xwo?>TkuMCGzwu-Z>@r?xcXkUO*NwL6hObmAV2jH8t} zwy`#%S@uin8(o-zh-8f(4=o;7rbiodU@n?4BQOCddW!#M&A2y3vFO8ZTxbUSs16ed7P6i*uSf#vT0Gks6C!HbInc8t_y{t>Q^+OyFhAI6Gkhz z`#5Lir|I#b>9C&$7RICPX*n%3u2xb!*i{gS~%@Mj!_NLaDYvazy?BIQyG0GI$WFheR8@{eL*s|J1?9&~NVYYSE9ZRf{N zOJi0AbQ$+#DG>nmtnLT*u6wm&{k3BxV}|RJbrgDA^Z4(ge+cN(KjD2eq`=J%_K@?| zZkwhYjQF(l68fbw>Ns2`>;zOx@Zhi+G;y%xxe_U`dIN^)%EQeQ+@6!*Q%t6zCSDtJ6(nqzg`+u$a%%7=9aVgMf$}ZvN>l z?2DxZQp=SrEp}yRs14#ZVt1gzAch!c3z7+)=}DTL&DyEUF5z38^Aa0~jPC%!>F1nD zAXqVf2SLn(2mJjOIbZR(nO1DE)@5;u#I`fg?09e3HKTi&dw%zx6&oA zu9#AXN2*baQOw%zs9ho@O4{Wrx_d&IndcakQ!gejm1X-eUmL={NCww%v~UX?$)yF$ zmCPpdSSi|=h%hWgNA~GXA6lTPOA)eZY842X?68f!hT-ABe;n1GiNTeivc)Xn(uXXS zY;*zlx1K?c?#TbgPd&}wJGbgXzQnzBim{jd&kj$EbOeCqT#G9Ua}aX!<6q_S(Hk6q zvLqsZ(%R;i@o)H%Ui?n|)4}(P+2M6|@WY z+%L-#V-<31^^2$-Dk20-a9mA!=eW+!LOA4T zIwdg(njkd~Mr^U{dSQUbG*eTaakGJ~S*EnUY<5!Mt?u0BFjkcgsWk6Wy%tG2s)DS` zL33sq$W?PN5~}TuFZ!X|anMgk6OUd7PSMe)qks1bG4$)oez`6n&6T~`sS&$T-lT;) z5FY7mM`(s9%AvnzwS!i@$Vc|6gF93F1XmD6E4oioo8ts3Yqhcjhb8xig8&2C4X@<% zd~|ViWL|DMj=aEYxtDay&d^d(&heZf&;->C7v!I{3K!m5EZ)NL&%id^Pipyx0qu&% zUTF8-Aj{WFeQQ&HxY3aFh>}>Ds^XKWxE47AuU87P$L(6`weZKi&FudCc zqUb6lw>)&*h2*#fP!y?-g}0AE_kUW+!4O?Ni?qmNU4HnYw9RIgZAUrG>K8LbvrUC~u-{-STg_BF(W3TeX&{rA!NI4gr@EDxl#}vL_e; z=uz*&rm&e=Nx2@#SjLfeHT9apNlmyP)|<2IW)N6- zGsry!Ch#ZNkXf2iN}1;dlUQ6ku)ftwhJKcm&{jsOb;B9Yn{m2@hAIsX)Ja?EzCPf+ z1}%_clB|0dN$#k>-E*R^~1Ka#L%1;*rwE%+Sgh= z@`5=(v7a#aHRGQHal)5|q?iLS&o78k4`8?Ez7a11H5^7&)!;0Q=9+UT&{hr@=o4NJ zWEi?`YiGS)I<_(GEOgDZxU#(xn}tYIa`&t$d%M|ua~TH4RT)Dgx4;+3!jl!o=XGp) zcw`J%Nj>$3@iUBMZTky-SYCc?^~eu34b{gY!@@h(^C;kI`+kP{E^i=fka%!qd*b+{ zU?I0fz6Qk*i3QjnwHe_P7^OZPYwU*`atxeDmE<;5=uV|dG!)ffCLqqJ(FAA9O3K|T z%Mb*+FIO=AGd{d824`+X*96&Di6=cZu_GeI8zz7_sl{ZvY-0-)5~#EH>sW(Chu0kK z`jkjEQD!!i^$!oR!Ulcn(W3(UEmDHYlV0%KbN-_WBS9{J$;ALAAFlWrzebvvXK&!JpYf2ul3LJ07 zOf5z>p>29wLj|VxL_u#!fFhSjijGuy%=Gb4*bR?2F{nn1d?Y9dqINbnlHR6J^R_?$ z{vqZced@xSLdCGZ5o_@kq+LHc(aNID+mj9#6_75gf8mv(-Ti@+-PV7EQX!f?1}|Q7 zF4wQ=t7ta)OWjrPEp#eL_)(JNnGkj*j`V^^N60q@m;pB_Rs;VA;Hd*ZJD`pMy*HM8 z6up)SOj1j4rL`1~RFb{1YF)YVD0b>c{!*PCM2{XBiS%=~m7y3pFrBhNW;v!904tI;t?zu^82!*No8`5&$)EDp-)1&)5oefr>o)f!nNP4=o{lHa~^S(>>#$v_En^yF_jcjH* z{=1pa$D^t0I=d{2veDNX`{m5uprx0fJq!XE)X13C^jQpQq7ZQEb2nP8J4nB#EIE66 z@?r4mSRWts5u-UD#r4qw@ReJBz#pN*zM3A|)Ua;(3hgJ;?_+sLmNHQSNlhV=TR0$` z#oa49YOuIPkyCcCrfgHb8?tAbTh<$#aJ=%g)LR+b=O{^;Nf6QR;sQ;+y;rw`=P^7; zkWsy>+0Xc#xqWS-^c+#GcpbDtNvsZw98~{xk~KcBa4c!YUwzM&`SHJ}69B2hEv&cnnq-jk^@h*URneC>5evpZH{h!cyIsrx}xi>&(#~AANDwW6hpO0liSnx zbACOcW_r6XY~_T}dz;cT7IQahYwT>A4LRpI7Zo9x@QB|2#K9-H#1<=?6J-|+sA$dDTT z317E@R%$*L=x%N)ZaaDPVM1vSP{zUhmEHA8dxzc5Pqrfii%WB$QH|`^yzVos63(MokF8ME&VTB>z76ltHaPLDQH7D1yq1d z{uqC@??9}80K)$V)qLgK*OYlA%(=CnYjKu?Dn;P=)^Pb6X{V74e%|2zCS@1n>| z2tyn;my$Md)hH9KV#aMa-5N-syssGc7<5l#Vt_crQbj)AsV|hVoD))31PecI0)9X= zM%3$gFDP_|Ql6MMO@QnXt1d3Pkk^>;z>f2oZT9 z>vXKiXGMbJSFII{ejP!T{v;X^3wmu=1jb~@SGloen8qhCq45D3u6_(wTaU~ced;qm zxn(Jru<9Od2EvGVRQ4kn5HiT=t&XTg{vM4J3CKziWA7pIW$Gg%e4CMrOo;-1yUGxW z`M=KufK?_EQ_H8g?-GFy;;(Y4;$s3B!+atS?{&XSQ+0B2Y1DlsJFg8t)L|Krc1Asp zT?W36Wypgc*^l>#kuF}fyydYb*@A+`rSGW6L0%?VQT=OZlo)dpIik0*5fmH${m=C% zBrsDBd0^5+o-&KvzbqXLHQ5fbEbi=PH6AEF&*6~%udR4`^^K<^VWSVG)VWt=sx4RL z8E&+q{v)~Pt@Q@*+~ED?#n%4>K)4I@O2(p~vbcPQ;$+_?V8VGHzespnjecyMTJMK! zn$H|ZN&N94ZFX2Z&j%xPV<28Vc~uhim~L z`b_>5r(l*0%U>)aGCm}HDdY@YKbblm?B?-dk&BJ^*C`(axF9Q{_RTf}o+PQ}U|K#F zsH<3>8*8(;PrZ~pz4N8Y$JsX|aMcL7GN#Kf;-s%DiE|~~NhW%;b zin@hGg+9uv9rAS5qHV0JOxZ;wpA^SdtLv#t7*VuY+~wBl-#w9>7b}9n{B;2I6`6~I zshQTfCV`D;Q)V?bJu8spJh@Ruzj14!566Rk4~z`NMYj z-s}{qAA^Q4Gu#$`e>_#Ffeg~b+M6!1L!7<`@WKY%11kOZa+}%K`jIL>1ZxbPW91cW zFwN)Q6;k1KcP0~6xc>l}i~OUIc67x3T~Lv3Ff1t<7JTOjqR29Od33_RNbHyFIO( z=HN>XY_(g-923UU^oG8j}}QBxN7?>Z-ZOv@yk%*Rze%4*WfkR?0~JqEg2;B}<*G99}h) zoW|tENRe`4OM1;|5`$uR@QFx?82UQW{y=rB^+fFpFs>d6_$1Kc@VaMfwR#EJQ&;w_ zRtTGOHht}`sxA?v4Rjc^pu2rk_L*(OjR#A;J zQ8V}Qid5_wm%CYO>6e088|hUs^V$uAJiTn%{JkTM?|MtM`3a?x>d8?+;2_51!~(`v z8;a7|6KsS3adm^}QCdry66XeUa$UQ>}LsoSP82W<$#(LbCm!N9}ey$m;!PK zfH^iyrs{Ykt-ETIUgYj);are~Pr+;ESXnVf76O8f1CEojgM6~|DNTneeBUkL@C1L3 zJ`yFay=K+&y=q%h@*8MAr8!3eYS~L#Tn^IunaD zn?{}Ab$jbWH{@vMve5s|p1SGnWjAE2`Mh;Co}O8_%CtCqr^aT4a!V!L8Lgz8kj6Dn zXE*-_5cKhEIoxT$+6I}&7uIjC&bHomG;;1zzlP#HAI*L={#Np+_mgxus4D)YEgykS z-sGg!K&|DAuz5s$4YG92&JOsq@!+IV6D-6>v3eydbr^EN8eT)To`KUQ2CR&} zJrJzkx0m5x*Xn_W;QORi6F-GpVktte^4p~}zXGZ;3>6bnqxaAjRB39JL~8%Pr2{LO zRB)JyNkLyoYsAPbrf#lRK1(tCA_q|FV10{ZQF9PWm>YSE?up$&$-kS zG%etwT3_wMm^XMoHcxu_p|*vz z-_QM4%7+V8%to`CzMt{6sSyGpgxl`RzKcu}p&9PmZqUzLD9QQXU8BkzAITzbudIz! zFkJ$J15zExZDx7!rgh_&rCRoBN_}sX#>>Ouww1NC?9!52e6^Iyr*N40D5MfXFqR2{ z2KSyZKtPE$XHRY5_VH1CS9esKqPdKW9j zV$5c)$ud@cbq?nSlc@cM52|(JOx2{q3;B)>H7l!Fd0Wgwe?)PV^|a6@I|y0aH_@V< zRWW~MtkEl=VhF+w9)yV5X#tds&W&kG=S3Qn&CC*wQPHU8c;Wa%z8*PGYcI*Wb&4lP z>v7PaqV3u-Flez=Q)&O`cPh#rjVhLS)=-vv+uqTk_zZS=20jbcKJ-7f!d?Z=>ht+4 z<9E%cL*BvVSW%TGJV)|(8l;{}@!5%IT&{A7q{99KhzoA%Qh7YHwg-vsY(OiS;-DI70x)p zJ-O2bvm;y+y3M2votO)ig|rlCL__Emm7v|%9<+a5v_Ag>uGyZiVgyW=g1g4EzoopZ zW&Bp^>`+1qCfis5PyTa17~`DtJm@`#84a36HsH|QzAanjxujW%(4-FBP&Pc71w409 z^+i5uU+kdS@-?62XW5fT*ISpjX73O*umPs~s#J1+p-;3!=+p&CBVTRh;HqyRMUr zy58PTU6He!U_$+Uo%~II$G8f@Bcd?!9Y3*&vu|h{$eLd;v*LfB>AMl*pr1xA3d7Zd zc>@c99cO{l`Hr?T8Eh*r^)D-;e;|j{1z3@ao$r2&cXGxMZ`oB&_;%v{Tl9`m#K4?B z4RveWYGTnH&V_^ag0e3w4x~!QybpdnhnECz$Hcj1j$+)r!$Jxu7RfhHM=CEfe-eqV zj9Fz_vMJ2zAK{jIe#{bSKF68T+gS=^HQ?61cF}GT6$bVevtRAmrFkgT2|Vc8!SGPWl{9J}R$n;17|6g; z*d(HG`U3yF|ND}|cHEG|J{_94sGt(vmS@@dkGp?2+op}RWYEqSWQG~KW0a(^fpF)R#Y2gI80(d;YfKId-EGaiZnVYWMfGVo&Z%`O23qt1tjCIWx!P=ZGFtv9h5VfCL3Y;eb1)Q z0E_GQ#3sMva}&9V%hV0!xr=2vi^~_wgKRKGXb(TU67{3+?Bk<0<7dhQH72(!_^&=C z*&MlSCYR_2q7$6SVhK!+kJbVh@i| z8%F*CQOB_^%)0DtG2sT?A(KQ)j=+X9p5Z1=Vn-WS@NB~$^4>uPjGCZAwN8{0OwH7> z0_>=5TL{i>t1pcUlF`nI9$`EdrC{Q6c?jLifRP+dd3#;qabp7l;dP0>@14ID(hCM9 z&J(oRm>Q(iK|!$SpHVEfNkulQ8o1_b^zne&zr`j5hdCd!z}0k5{iXhW3xkCaWGtm^J z%$z;l&s5pM%p_gNg%GGuMW9u6}W5CEx+MDYxiY5j|dn z<~-={Vxll9?hutju#pC!_O>_}44bb>g}O=7QO@9g=y&cf==;7fhB&9K`@qVlf+%qqz`|v{ACeU` z`DZm6VMr>jGqm*b##+jky@)?!&dMeuxXvQYh<(Nl?c$P6z8ai`QG~MIl){>~w3;YI zX!K;7tgKNmo5=H*GV#N^{krMP2za;bi%@R4ZpF4x2={(7Ja}0d6~9+|mO) zdPh%D%=~q5XF9;63q8ZuLPS9>Xr~XGt^W{HpN~6@nm7{Hj<>Y^`i)HyJ^U9mns{oO zhT~^T#Ft+Y_ZTy=GH*d=BqX4d-N|m)2fA}QGi57WNvezUpXZ<3OzsdjeQ#Lmv_zh! zTR*QtCAzC5p8OFd{R~uC)A||AkB})^kVjaMU3F(5Ll?N1D-!$_YXfpP$p82RYZNFa zd@T_-bc7kLXl^XoBMh^1uJcqoysG#uMe*R= z=hr7`X{X>yv<5$K+?LHjI2*_ic;5Ak?sNcmxzvR%H`ar(h=xw0&^}!QWTiO#`U@Su zo@vy|#D(PvwN`t=l~4?Auid8RnMo-3Ih){*f_MF_>Bt+9QYEy4L%I)G8MFPKe#I+B zf=z3`5hSrnKeOetZB@}pxUj`((?7dNwmTeD$FRQ7+?|6+SAH5VTuK$u#7&QRZip|j z*1n<(IA*;+)4yzmVJF)`BnKUn{yGjSFFZ0m%h1I7X%nkY#oW$&t&M;5ieCa!@T zF)e#llhLxQ+a0(2p#`WZwqol*E)qeC`wQIo_e1^_FSL+cHd9stU7#+z#-H!hTBerj zuaNzxr260xOZEox?4>JG4Qi20d?6~56YMMwKxT56p`{{YBU-|lyyqn>)#;6n!y)8rh}pcy;LOvw zb_5ot9B9~FkvqBYwtXA^oY|R;`&7TWh?PV302&}Hd z6pbDFMm>;)>wsw?JV7NXzCC`)!4>Cy#_?CK)M$aDTMV_`Dblzq>G$uIj`rL=*{Ts31i7$dGIU@JvW0^&kBlFdT3(^%IIZ&-TKFmFn5M% zz`d{g4h(v?DdmUEPd*S3*GQtW4F)9-b^(BL?dXk%`ho!X*W;vhynib0GB(PiGN0$j3FridZf z412>t`a>X73bx8{!F12WujfN!)Nl_>IYV4qy6%ULPKpZ=HDhbVO*d`O=%x0ibhVE z_!aFwWlY|>b(h?vnvy^(i|B*TU*Ea*@}%%5K$6tfr{PFWd(V!#GKCh{ioV+w#g?7)*-Zb8}*)-m!-85WtFLDZY0Tf^6 zJ$K(>oPsb6de}(;2jOv~qi z5$48{p_Dv6pfOmR%8^i01^GZ8;S5CYU>9qm1$`Mz21;CN>P#UGPXAW>)*gG0IK#lJ zT`G25YR(rbcuo>skV{#&I1!E7rqOgl;`5oOF#Sc}+9qA((klYxJN#Fq5wT zvO$tzE+xjj+l7|F&X8Gt8A)r<%u+c}8XQJZ6U?v%4 zVn!~^OG*FWJdIk|T1~wD^#O}L_nxQ+Kph9h@UhYEENANn-AaOw`oL|qM05E00(S-? zL7v4KC4=;$K@mPY@NjwmibtSy&8!_D0V974A%DvkD*$RW^0ym?Qhd-xZ`>i@kJ1+% zUw7;8@Pbnj53hUS#*uLgTM&9S8MU7%FAp&S)f^{o_>(k{H>DF5u1VU1y70R(tyK$? z(}0GEv3wQQYN8fAKWk0wHq#I;i|;~igE$m-m1eV|B68s(n%pSLq4)24lvaUMBuAj& z=q0i2et-oB&0TCbWITU)j^+XSQP)LR0XwAGw7bE28B}#$<20&kaVyKsn+w486CR?2H!+wkxF;o zYpzX=XM=N>1VxKq4VKN|g60K|aCz%lhk4l`tf&@J?kbBtt>pjI_1~Wm*fyq%x4%{O zKfKFni7dBy=L0*@SbzAFfJ6;OrT6g1JtkYBU^Sqw0jvo+8<$8$G-<%^N#A(}E_J*H z;QY2T#!UXtc9Jwf)Z?p-wWKc(qIyH}1Mx(kfEe{4&>+eMzebdyrQ}?dvra1`TWFyI ziTLfmpK>DVby@ig0Y4k zKa^Fb2?~jNyVWm}2GSrA>m_9RG+r!d-pyN?9BAjt{5#584Mxz-7kfs7hi~g-86p|# zw^8+IHj!sxz1P9ea~9_w0(+In6KSOTTLZIWRu($)%+x|atF!%ex6y zu?o5buCOZ3Kk|sx1dMY@;L|rN(A^%Ca_K6qV{)*#!e5U1{9ld7WkOE+7JrAL23Rio zBxk0e=knFnYV%QvF}n%f9V(#p)3tUbUlA)>BDvDdOm z)|y$<^yRk&nfsx6UW6QGJG=TZ<%v4{Y#-y1L7ZbA7u=dUrmx@g=+~(<4Ul zRC`EznwY+{18y>@r>L6Y-%z@2h+VZ4XxmLWr%o$GG{y9t{fl258Kfl>?5|TVCV*IL zxYD&~WQ;)MDFAoBMb^R}xAg$ErQ__7O4ZhTvte7wK{eqgBh2u4jL#D`+kV?;k8`1_ z4Ea@9WmSCB!48p{DqIQ( zU9-GNTCMh{#^0ULByLJ<1|=2!%E^MswnrFH1-Xx|YVz)XMpqq=%9o#6A0TMoj|MhE zNR-;P_>q*-U@2LJyn5Ok;3GJF#K=i!J+YZoc|3Bzvo0;K7Y)lw#>}1~rat|0$vQ}p zEP+{RrM#WW70XDHQgo#4(n8xU2uY9I$(b1Rx^P?jnHW}zS!SgM3c1}$LaM-*U`5rmiPc3OSm4lcx#KJpN`Hf!JX$5fM;=X^G674C+Q8}K;5HbX_F zwpio=1H9$5_L@pZtW4jqU=UPddI{fOIV>-X5s|3~zFtf%HRXJTr=qFicS5=%Mnd3N ze319%bww6`Za9+15AV(%tvyQP zbB)5{RV9K=>eC{8sd@gP&rcQKhFQnxhI)%>!-VB$$o zv(B0G*?R@Cdt$`%O_FC?NsPNM;?5GTPApzM`s4@`c?ibCFN6Vb)>d-Xr~p~DCth{g zYlawx6b5~U8v1W+-fkTodiw9Vdoih>$4seVQ#0yYPs*2o1+)0u`j4Wihm5Dy)cVIE z2GT03Q|yc4ck)2}OWXT%OC2pXt%Yxk|KnyvXK7u7-wi74EAtWew8n5ncuJ7a?J7qs z_emvl;HfRJV;??)dS&j;3~9e-veLM&=+H~dwcXe2S@Z32@<#p78@fjWya}ZDd7Zkf zc_&%)5R{^koTe1(f_23fsl{dZ?eT~&OI0?%-_}e`_;l7I01Z_&>Cq2%{C8E>dSl}p zJHZBZFIf$;Zb-n-x9*JAdGPSJ`XgPHbO}{$DQHdRJSZdm<>1y5{YmnH%)51R=TeT3 zA%j#&KYUXdjI_H@TrpWpL{)Wq>TpFpVq0hE3W+qmaVL~H{?;%UOa!wT25STeA5t~c z7;#j08EFmts_k@rf!*KXr_y!9Juzo>@aGDb-gH6!kI$!SDI9nnlT0mFXN|Wf@ha1a zW!QMr%Xmg+d=uiMolkUfrv0FJ*7??0ttAtsj>zT?XY}UI-2*sT&AZ?$^%34Xy)xZ6 z|25DB+eLS|_g(LdB%+9ws!bC3mT>Be2~R}Cf~Ib~tkPKsn+i0=^&!^-G?fNe${1h; z>R2^4mJ25ECRg@5ie!6awp$e^3Q0@u97SlZ^}N#@4Gr|HRH5%PIJcly0TBZ_ZvklV zfB`c`F~g1%;!ZaCNDsXA-earv=68|H2PNeqb#08j$Dk)|7tYd?y>3skYCBO> ztbFEY;>@U6Kfa;ak^Fi~tKjFZ=9qTg6WPm2^2DfZ^xx9z?* zqs`hh^bd8LJ8PKvBkdbKk{ZHShls~OcZ}#%_uW%R2Hw9HFeLX3k&`_;Nq>83U_bx} z{iiHUnf3LM>?D8r={P0Tf)kES-1;V9OL*B!Vc|%mAo(I_;f(HBamkeXD}P!r-gz5~ zG|jx>JNy%bWy~iVu+Ao|9wj_?XDAVQS0nU30%m<-9&q4|lM!*BA$7y* z2_u7_kJ_$DQT-OOk?_QejQotnp3R(m|2zK8n*Mi@H<=v2s`TehH+A+CYxCNMrJcXv z{oUq&)^a9Jz9O%t|2us7*5)c303rWpHRr$m@B4yxsH8z%xwZu^$*aAuev1Yf(MAJ7 rB-g&x(ljqTcrg9o<(|w8D_h5m_2u<1{+}Pdey`1kt^Mi&`Srg54<2+X diff --git a/helm-charts/charts/kmm/.helmignore b/helm-charts/charts/kmm/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/helm-charts/charts/kmm/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/helm-charts/charts/kmm/Chart.yaml b/helm-charts/charts/kmm/Chart.yaml deleted file mode 100644 index e5e28bebc..000000000 --- a/helm-charts/charts/kmm/Chart.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v2 -name: kmm -description: A Helm chart for deploying Kernel Module Management for AMD GPU Operator -type: application - -kubeVersion: ">= 1.18.0-0" -version: v1.0.0 -appVersion: "v20240618-v2.1.1" - diff --git a/helm-charts/charts/kmm/crds/module-crd.yaml b/helm-charts/charts/kmm/crds/module-crd.yaml deleted file mode 100644 index f099c9e7f..000000000 --- a/helm-charts/charts/kmm/crds/module-crd.yaml +++ /dev/null @@ -1,2700 +0,0 @@ ---- -# Source: kmm/templates/module-crd.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: modules.kmm.sigs.x-k8s.io - annotations: - controller-gen.kubebuilder.io/version: v0.16.1 - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - helm.sh/chart: kmm-v1.0.0 - app.kubernetes.io/name: kmm - app.kubernetes.io/instance: amd-gpu - app.kubernetes.io/version: "v20240618-v2.1.1" - app.kubernetes.io/managed-by: Helm -spec: - group: kmm.sigs.x-k8s.io - names: - kind: Module - listKind: ModuleList - plural: modules - singular: module - scope: Namespaced - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: Module describes how to load a module on different kernel versions - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: ModuleSpec describes how the KMM operator should deploy a Module - on those nodes that need it. - properties: - devicePlugin: - description: |- - DevicePlugin allows overriding some properties of the container that deploys the device plugin on the node. - Name is ignored and is set automatically by the KMM Operator. - properties: - container: - properties: - args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment variable present - in a Container. - properties: - name: - description: Name of the environment variable. Must be - a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment variable's value. - Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or - its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema the FieldPath - is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the - specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the - exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the pod's - namespace - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its - key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - image: - description: Image is the name of the container image that the - device plugin container will run. - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - volumeMounts: - description: VolumeMounts is a list of volume mounts that are - appended to the default ones. - items: - description: VolumeMount describes a mounting of a Volume - within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - required: - - image - type: object - serviceAccountName: - description: |- - ServiceAccountName is the name of the ServiceAccount to use to run this pod. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ - type: string - volumes: - items: - description: Volume represents a named volume in a pod that may - be accessed by any container in the pod. - properties: - awsElasticBlockStore: - description: |- - awsElasticBlockStore represents an AWS Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - format: int32 - type: integer - readOnly: - description: |- - readOnly value true will force the readOnly setting in VolumeMounts. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: boolean - volumeID: - description: |- - volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: string - required: - - volumeID - type: object - azureDisk: - description: azureDisk represents an Azure Data Disk mount - on the host and bind mount to the pod. - properties: - cachingMode: - description: 'cachingMode is the Host Caching mode: None, - Read Only, Read Write.' - type: string - diskName: - description: diskName is the Name of the data disk in - the blob storage - type: string - diskURI: - description: diskURI is the URI of data disk in the blob - storage - type: string - fsType: - default: ext4 - description: |- - fsType is Filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - kind: - description: 'kind expected values are Shared: multiple - blob disks per storage account Dedicated: single blob - disk per storage account Managed: azure managed data - disk (only in managed availability set). defaults to - shared' - type: string - readOnly: - default: false - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: azureFile represents an Azure File Service mount - on the host and bind mount to the pod. - properties: - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretName: - description: secretName is the name of secret that contains - Azure Storage Account Name and Key - type: string - shareName: - description: shareName is the azure share Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: cephFS represents a Ceph FS mount on the host - that shares a pod's lifetime - properties: - monitors: - description: |- - monitors is Required: Monitors is a collection of Ceph monitors - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - items: - type: string - type: array - x-kubernetes-list-type: atomic - path: - description: 'path is Optional: Used as the mounted root, - rather than the full Ceph tree, default is /' - type: string - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: boolean - secretFile: - description: |- - secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - secretRef: - description: |- - secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - user: - description: |- - user is optional: User is the rados user name, default is admin - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - required: - - monitors - type: object - cinder: - description: |- - cinder represents a cinder volume attached and mounted on kubelets host machine. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: boolean - secretRef: - description: |- - secretRef is optional: points to a secret object containing parameters used to connect - to OpenStack. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - volumeID: - description: |- - volumeID used to identify the volume in cinder. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - required: - - volumeID - type: object - configMap: - description: configMap represents a configMap that should - populate this volume - properties: - defaultMode: - description: |- - defaultMode is optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional specify whether the ConfigMap or - its keys must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - csi: - description: csi (Container Storage Interface) represents - ephemeral storage that is handled by certain external CSI - drivers (Beta feature). - properties: - driver: - description: |- - driver is the name of the CSI driver that handles this volume. - Consult with your admin for the correct name as registered in the cluster. - type: string - fsType: - description: |- - fsType to mount. Ex. "ext4", "xfs", "ntfs". - If not provided, the empty value is passed to the associated CSI driver - which will determine the default filesystem to apply. - type: string - nodePublishSecretRef: - description: |- - nodePublishSecretRef is a reference to the secret object containing - sensitive information to pass to the CSI driver to complete the CSI - NodePublishVolume and NodeUnpublishVolume calls. - This field is optional, and may be empty if no secret is required. If the - secret object contains more than one secret, all secret references are passed. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - readOnly: - description: |- - readOnly specifies a read-only configuration for the volume. - Defaults to false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: |- - volumeAttributes stores driver-specific properties that are passed to the CSI - driver. Consult your driver's documentation for supported values. - type: object - required: - - driver - type: object - downwardAPI: - description: downwardAPI represents downward API about the - pod that should populate this volume - properties: - defaultMode: - description: |- - Optional: mode bits to use on created files by default. Must be a - Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: Items is a list of downward API volume file - items: - description: DownwardAPIVolumeFile represents information - to create the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects a field of the pod: - only annotations, labels, name, namespace and - uid are supported.' - properties: - apiVersion: - description: Version of the schema the FieldPath - is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in - the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path is the relative path - name of the file to be created. Must not be absolute - or contain the ''..'' path. Must be utf-8 encoded. - The first item of the relative path must not start - with ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of - the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - emptyDir: - description: |- - emptyDir represents a temporary directory that shares a pod's lifetime. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - properties: - medium: - description: |- - medium represents what type of storage medium should back this directory. - The default is "" which means to use the node's default medium. - Must be an empty string (default) or Memory. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: |- - sizeLimit is the total amount of local storage required for this EmptyDir volume. - The size limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir would be the minimum value between - the SizeLimit specified here and the sum of memory limits of all containers in a pod. - The default is nil which means that the limit is undefined. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: |- - ephemeral represents a volume that is handled by a cluster storage driver. - The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, - and deleted when the pod is removed. - - Use this if: - a) the volume is only needed while the pod runs, - b) features of normal volumes like restoring from snapshot or capacity - tracking are needed, - c) the storage driver is specified through a storage class, and - d) the storage driver supports dynamic volume provisioning through - a PersistentVolumeClaim (see EphemeralVolumeSource for more - information on the connection between this volume type - and PersistentVolumeClaim). - - Use PersistentVolumeClaim or one of the vendor-specific - APIs for volumes that persist for longer than the lifecycle - of an individual pod. - - Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to - be used that way - see the documentation of the driver for - more information. - - A pod can use both types of ephemeral volumes and - persistent volumes at the same time. - properties: - volumeClaimTemplate: - description: |- - Will be used to create a stand-alone PVC to provision the volume. - The pod in which this EphemeralVolumeSource is embedded will be the - owner of the PVC, i.e. the PVC will be deleted together with the - pod. The name of the PVC will be `-` where - `` is the name from the `PodSpec.Volumes` array - entry. Pod validation will reject the pod if the concatenated name - is not valid for a PVC (for example, too long). - - An existing PVC with that name that is not owned by the pod - will *not* be used for the pod to avoid using an unrelated - volume by mistake. Starting the pod is then blocked until - the unrelated PVC is removed. If such a pre-created PVC is - meant to be used by the pod, the PVC has to updated with an - owner reference to the pod once the pod exists. Normally - this should not be necessary, but it may be useful when - manually reconstructing a broken cluster. - - This field is read-only and no changes will be made by Kubernetes - to the PVC after it has been created. - - Required, must not be nil. - properties: - metadata: - description: |- - May contain labels and annotations that will be copied into the PVC - when creating it. No other fields are allowed and will be rejected during - validation. - type: object - spec: - description: |- - The specification for the PersistentVolumeClaim. The entire content is - copied unchanged into the PVC that gets created from this - template. The same fields as in a PersistentVolumeClaim - are also valid here. - properties: - accessModes: - description: |- - accessModes contains the desired access modes the volume should have. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 - items: - type: string - type: array - x-kubernetes-list-type: atomic - dataSource: - description: |- - dataSource field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) - If the provisioner or an external controller can support the specified data source, - it will create a new volume based on the contents of the specified data source. - When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, - and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. - If the namespace is specified, then dataSourceRef will not be copied to dataSource. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource - being referenced - type: string - name: - description: Name is the name of resource - being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: |- - dataSourceRef specifies the object from which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a non-empty API group (non - core object) or a PersistentVolumeClaim object. - When this field is specified, volume binding will only succeed if the type of - the specified object matches some installed volume populator or dynamic - provisioner. - This field will replace the functionality of the dataSource field and as such - if both fields are non-empty, they must have the same value. For backwards - compatibility, when namespace isn't specified in dataSourceRef, - both fields (dataSource and dataSourceRef) will be set to the same - value automatically if one of them is empty and the other is non-empty. - When namespace is specified in dataSourceRef, - dataSource isn't set to the same value and must be empty. - There are three important differences between dataSource and dataSourceRef: - * While dataSource only allows two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim objects. - * While dataSource ignores disallowed values (dropping them), dataSourceRef - preserves all values, and generates an error if a disallowed value is - specified. - * While dataSource only allows local objects, dataSourceRef allows objects - in any namespaces. - (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. - (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource - being referenced - type: string - name: - description: Name is the name of resource - being referenced - type: string - namespace: - description: |- - Namespace is the namespace of resource being referenced - Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. - (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: |- - resources represents the minimum resources the volume should have. - If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements - that are lower than previous value but must still be higher than capacity recorded in the - status field of the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - selector: - description: selector is a label query over volumes - to consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of - label selector requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - description: |- - storageClassName is the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 - type: string - volumeAttributesClassName: - description: |- - volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. - If specified, the CSI driver will create or update the volume with the attributes defined - in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. - If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be - set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource - exists. - More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). - type: string - volumeMode: - description: |- - volumeMode defines what type of volume is required by the claim. - Value of Filesystem is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference - to the PersistentVolume backing this claim. - type: string - type: object - required: - - spec - type: object - type: object - fc: - description: fc represents a Fibre Channel resource that is - attached to a kubelet's host machine and then exposed to - the pod. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - lun: - description: 'lun is Optional: FC target lun number' - format: int32 - type: integer - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - targetWWNs: - description: 'targetWWNs is Optional: FC target worldwide - names (WWNs)' - items: - type: string - type: array - x-kubernetes-list-type: atomic - wwids: - description: |- - wwids Optional: FC volume world wide identifiers (wwids) - Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - flexVolume: - description: |- - flexVolume represents a generic volume resource that is - provisioned/attached using an exec based plugin. - properties: - driver: - description: driver is the name of the driver to use for - this volume. - type: string - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. - type: string - options: - additionalProperties: - type: string - description: 'options is Optional: this field holds extra - command options if any.' - type: object - readOnly: - description: |- - readOnly is Optional: defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef is Optional: secretRef is reference to the secret object containing - sensitive information to pass to the plugin scripts. This may be - empty if no secret object is specified. If the secret object - contains more than one secret, all secrets are passed to the plugin - scripts. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - required: - - driver - type: object - flocker: - description: flocker represents a Flocker volume attached - to a kubelet's host machine. This depends on the Flocker - control service being running - properties: - datasetName: - description: |- - datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker - should be considered as deprecated - type: string - datasetUUID: - description: datasetUUID is the UUID of the dataset. This - is unique identifier of a Flocker dataset - type: string - type: object - gcePersistentDisk: - description: |- - gcePersistentDisk represents a GCE Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - properties: - fsType: - description: |- - fsType is filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - format: int32 - type: integer - pdName: - description: |- - pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: boolean - required: - - pdName - type: object - gitRepo: - description: |- - gitRepo represents a git repository at a particular revision. - DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an - EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir - into the Pod's container. - properties: - directory: - description: |- - directory is the target directory name. - Must not contain or start with '..'. If '.' is supplied, the volume directory will be the - git repository. Otherwise, if specified, the volume will contain the git repository in - the subdirectory with the given name. - type: string - repository: - description: repository is the URL - type: string - revision: - description: revision is the commit hash for the specified - revision. - type: string - required: - - repository - type: object - glusterfs: - description: |- - glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. - More info: https://examples.k8s.io/volumes/glusterfs/README.md - properties: - endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: string - path: - description: |- - path is the Glusterfs volume path. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: string - readOnly: - description: |- - readOnly here will force the Glusterfs volume to be mounted with read-only permissions. - Defaults to false. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: boolean - required: - - endpoints - - path - type: object - hostPath: - description: |- - hostPath represents a pre-existing file or directory on the host - machine that is directly exposed to the container. This is generally - used for system agents or other privileged things that are allowed - to see the host machine. Most containers will NOT need this. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - properties: - path: - description: |- - path of the directory on the host. - If the path is a symlink, it will follow the link to the real path. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - type: - description: |- - type for HostPath Volume - Defaults to "" - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - required: - - path - type: object - image: - description: |- - image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. - The volume is resolved at pod startup depending on which PullPolicy value is provided: - - - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - - The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. - A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. - The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. - The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. - The volume will be mounted read-only (ro) and non-executable files (noexec). - Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). - The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. - properties: - pullPolicy: - description: |- - Policy for pulling OCI objects. Possible values are: - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - type: string - reference: - description: |- - Required: Image or artifact reference to be used. - Behaves in the same way as pod.spec.containers[*].image. - Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - type: object - iscsi: - description: |- - iscsi represents an ISCSI Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md - properties: - chapAuthDiscovery: - description: chapAuthDiscovery defines whether support - iSCSI Discovery CHAP authentication - type: boolean - chapAuthSession: - description: chapAuthSession defines whether support iSCSI - Session CHAP authentication - type: boolean - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - type: string - initiatorName: - description: |- - initiatorName is the custom iSCSI Initiator Name. - If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface - : will be created for the connection. - type: string - iqn: - description: iqn is the target iSCSI Qualified Name. - type: string - iscsiInterface: - default: default - description: |- - iscsiInterface is the interface Name that uses an iSCSI transport. - Defaults to 'default' (tcp). - type: string - lun: - description: lun represents iSCSI Target Lun number. - format: int32 - type: integer - portals: - description: |- - portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - items: - type: string - type: array - x-kubernetes-list-type: atomic - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - type: boolean - secretRef: - description: secretRef is the CHAP Secret for iSCSI target - and initiator authentication - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - targetPortal: - description: |- - targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - description: |- - name of the volume. - Must be a DNS_LABEL and unique within the pod. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - nfs: - description: |- - nfs represents an NFS mount on the host that shares a pod's lifetime - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - properties: - path: - description: |- - path that is exported by the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - readOnly: - description: |- - readOnly here will force the NFS export to be mounted with read-only permissions. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: boolean - server: - description: |- - server is the hostname or IP address of the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: |- - persistentVolumeClaimVolumeSource represents a reference to a - PersistentVolumeClaim in the same namespace. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - properties: - claimName: - description: |- - claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - type: string - readOnly: - description: |- - readOnly Will force the ReadOnly setting in VolumeMounts. - Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: photonPersistentDisk represents a PhotonController - persistent disk attached and mounted on kubelets host machine - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - pdID: - description: pdID is the ID that identifies Photon Controller - persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: portworxVolume represents a portworx volume attached - and mounted on kubelets host machine - properties: - fsType: - description: |- - fSType represents the filesystem type to mount - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - volumeID: - description: volumeID uniquely identifies a Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: projected items for all in one resources secrets, - configmaps, and downward API - properties: - defaultMode: - description: |- - defaultMode are the mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - sources: - description: |- - sources is the list of volume projections. Each entry in this list - handles one source. - items: - description: |- - Projection that may be projected along with other supported volume types. - Exactly one of these fields must be set. - properties: - clusterTrustBundle: - description: |- - ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field - of ClusterTrustBundle objects in an auto-updating file. - - Alpha, gated by the ClusterTrustBundleProjection feature gate. - - ClusterTrustBundle objects can either be selected by name, or by the - combination of signer name and a label selector. - - Kubelet performs aggressive normalization of the PEM contents written - into the pod filesystem. Esoteric PEM features such as inter-block - comments and block headers are stripped. Certificates are deduplicated. - The ordering of certificates within the file is arbitrary, and Kubelet - may change the order over time. - properties: - labelSelector: - description: |- - Select all ClusterTrustBundles that match this label selector. Only has - effect if signerName is set. Mutually-exclusive with name. If unset, - interpreted as "match nothing". If set but empty, interpreted as "match - everything". - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - name: - description: |- - Select a single ClusterTrustBundle by object name. Mutually-exclusive - with signerName and labelSelector. - type: string - optional: - description: |- - If true, don't block pod startup if the referenced ClusterTrustBundle(s) - aren't available. If using name, then the named ClusterTrustBundle is - allowed not to exist. If using signerName, then the combination of - signerName and labelSelector is allowed to match zero - ClusterTrustBundles. - type: boolean - path: - description: Relative path from the volume root - to write the bundle. - type: string - signerName: - description: |- - Select all ClusterTrustBundles that match this signer name. - Mutually-exclusive with name. The contents of all selected - ClusterTrustBundles will be unified and deduplicated. - type: string - required: - - path - type: object - configMap: - description: configMap information about the configMap - data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional specify whether the ConfigMap - or its keys must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - downwardAPI: - description: downwardAPI information about the downwardAPI - data to project - properties: - items: - description: Items is a list of DownwardAPIVolume - file - items: - description: DownwardAPIVolumeFile represents - information to create the file containing - the pod field - properties: - fieldRef: - description: 'Required: Selects a field - of the pod: only annotations, labels, - name, namespace and uid are supported.' - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path is the relative - path name of the file to be created. - Must not be absolute or contain the - ''..'' path. Must be utf-8 encoded. - The first item of the relative path - must not start with ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container name: required - for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to - select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - secret: - description: secret information about the secret - data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional field specify whether - the Secret or its key must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - serviceAccountToken: - description: serviceAccountToken is information - about the serviceAccountToken data to project - properties: - audience: - description: |- - audience is the intended audience of the token. A recipient of a token - must identify itself with an identifier specified in the audience of the - token, and otherwise should reject the token. The audience defaults to the - identifier of the apiserver. - type: string - expirationSeconds: - description: |- - expirationSeconds is the requested duration of validity of the service - account token. As the token approaches expiration, the kubelet volume - plugin will proactively rotate the service account token. The kubelet will - start trying to rotate the token if the token is older than 80 percent of - its time to live or if the token is older than 24 hours.Defaults to 1 hour - and must be at least 10 minutes. - format: int64 - type: integer - path: - description: |- - path is the path relative to the mount point of the file to project the - token into. - type: string - required: - - path - type: object - type: object - type: array - x-kubernetes-list-type: atomic - type: object - quobyte: - description: quobyte represents a Quobyte mount on the host - that shares a pod's lifetime - properties: - group: - description: |- - group to map volume access to - Default is no group - type: string - readOnly: - description: |- - readOnly here will force the Quobyte volume to be mounted with read-only permissions. - Defaults to false. - type: boolean - registry: - description: |- - registry represents a single or multiple Quobyte Registry services - specified as a string as host:port pair (multiple entries are separated with commas) - which acts as the central registry for volumes - type: string - tenant: - description: |- - tenant owning the given Quobyte volume in the Backend - Used with dynamically provisioned Quobyte volumes, value is set by the plugin - type: string - user: - description: |- - user to map volume access to - Defaults to serivceaccount user - type: string - volume: - description: volume is a string that references an already - created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: |- - rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. - More info: https://examples.k8s.io/volumes/rbd/README.md - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - type: string - image: - description: |- - image is the rados image name. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - keyring: - default: /etc/ceph/keyring - description: |- - keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - monitors: - description: |- - monitors is a collection of Ceph monitors. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - items: - type: string - type: array - x-kubernetes-list-type: atomic - pool: - default: rbd - description: |- - pool is the rados pool name. - Default is rbd. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: boolean - secretRef: - description: |- - secretRef is name of the authentication secret for RBDUser. If provided - overrides keyring. - Default is nil. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - user: - default: admin - description: |- - user is the rados user name. - Default is admin. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - required: - - image - - monitors - type: object - scaleIO: - description: scaleIO represents a ScaleIO persistent volume - attached and mounted on Kubernetes nodes. - properties: - fsType: - default: xfs - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". - Default is "xfs". - type: string - gateway: - description: gateway is the host address of the ScaleIO - API Gateway. - type: string - protectionDomain: - description: protectionDomain is the name of the ScaleIO - Protection Domain for the configured storage. - type: string - readOnly: - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef references to the secret for ScaleIO user and other - sensitive information. If this is not provided, Login operation will fail. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - sslEnabled: - description: sslEnabled Flag enable/disable SSL communication - with Gateway, default false - type: boolean - storageMode: - default: ThinProvisioned - description: |- - storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. - type: string - storagePool: - description: storagePool is the ScaleIO Storage Pool associated - with the protection domain. - type: string - system: - description: system is the name of the storage system - as configured in ScaleIO. - type: string - volumeName: - description: |- - volumeName is the name of a volume already created in the ScaleIO system - that is associated with this volume source. - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - description: |- - secret represents a secret that should populate this volume. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - properties: - defaultMode: - description: |- - defaultMode is Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values - for mode bits. Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items If unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - optional: - description: optional field specify whether the Secret - or its keys must be defined - type: boolean - secretName: - description: |- - secretName is the name of the secret in the pod's namespace to use. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - type: string - type: object - storageos: - description: storageOS represents a StorageOS volume attached - and mounted on Kubernetes nodes. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef specifies the secret to use for obtaining the StorageOS API - credentials. If not specified, default values will be attempted. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - volumeName: - description: |- - volumeName is the human-readable name of the StorageOS volume. Volume - names are only unique within a namespace. - type: string - volumeNamespace: - description: |- - volumeNamespace specifies the scope of the volume within StorageOS. If no - namespace is specified then the Pod's namespace will be used. This allows the - Kubernetes name scoping to be mirrored within StorageOS for tighter integration. - Set VolumeName to any name to override the default behaviour. - Set to "default" if you are not using namespaces within StorageOS. - Namespaces that do not pre-exist within StorageOS will be created. - type: string - type: object - vsphereVolume: - description: vsphereVolume represents a vSphere volume attached - and mounted on kubelets host machine - properties: - fsType: - description: |- - fsType is filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - storagePolicyID: - description: storagePolicyID is the storage Policy Based - Management (SPBM) profile ID associated with the StoragePolicyName. - type: string - storagePolicyName: - description: storagePolicyName is the storage Policy Based - Management (SPBM) profile name. - type: string - volumePath: - description: volumePath is the path that identifies vSphere - volume vmdk - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - required: - - container - type: object - imageRepoSecret: - description: |- - ImageRepoSecret is an optional secret that is used to pull both the module loader and the device plugin, and - to push the resulting image from the module loader build, if enabled. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - moduleLoader: - description: |- - ModuleLoader allows overriding some properties of the container that loads the kernel module on the node. - Name and image are ignored and are set automatically by the KMM Operator. - properties: - container: - description: Container holds the properties for the module loader - container that runs modprobe. - properties: - build: - description: Build contains build instructions. - properties: - baseImageRegistryTLS: - description: BaseImageRegistryTLS contains settings determining - how to access registries of the base images in the build-process' - Dockerfile. - properties: - insecure: - description: If Insecure is true, the operator will - be able to access a registry in an insecure (plain - HTTP) protocol. - type: boolean - insecureSkipTLSVerify: - description: If InsecureSkipTLSVerify, the operator - will accept any certificate provided by the registry. - type: boolean - type: object - buildArgs: - description: BuildArgs is an array of build variables that - are provided to the image building backend. - items: - description: BuildArg represents a build argument used - when building a container image. - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - dockerfileConfigMap: - description: ConfigMap that holds Dockerfile contents - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - kanikoParams: - description: KanikoParams is used to customize the building - process of the image. - properties: - tag: - description: Kaniko image tag to use when creating the - build Pod - type: string - type: object - secrets: - description: |- - Secrets is an optional list of secrets to be made available to the build system. - Those secrets should be used for private resources such as a private Github repo. - For container registries auth use module.spec.imagePullSecret instead. - items: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - type: array - selector: - additionalProperties: - type: string - description: Selector describes on which nodes will run - the building process. - type: object - required: - - dockerfileConfigMap - type: object - containerImage: - description: ContainerImage is a top-level field - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - inTreeModuleToRemove: - description: |- - Deprecated: please use InTreeModulesToRemove. - InTreeModuleToRemove specifies one in-tree kernel module that should be removed (if present) - before loading the kernel module from the ContainerImage - type: string - inTreeModulesToRemove: - description: |- - InTreeModulesToRemove specifies any number of in-tree kernel modules that should be removed (if present) - before loading the kernel module from the ContainerImage - items: - type: string - type: array - kernelMappings: - description: |- - KernelMappings is a list of kernel mappings. - When a node's labels match Selector, then the KMM Operator will look for the first mapping that matches its - kernel version, and use the corresponding container image to run the DriverContainer. - items: - description: |- - KernelMapping pairs kernel versions with a DriverContainer image. - Kernel versions can be matched literally or using a regular expression. - properties: - build: - description: Build enables in-cluster builds for this - mapping and allows overriding the Module's build settings. - properties: - baseImageRegistryTLS: - description: BaseImageRegistryTLS contains settings - determining how to access registries of the base - images in the build-process' Dockerfile. - properties: - insecure: - description: If Insecure is true, the operator - will be able to access a registry in an insecure - (plain HTTP) protocol. - type: boolean - insecureSkipTLSVerify: - description: If InsecureSkipTLSVerify, the operator - will accept any certificate provided by the - registry. - type: boolean - type: object - buildArgs: - description: BuildArgs is an array of build variables - that are provided to the image building backend. - items: - description: BuildArg represents a build argument - used when building a container image. - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - dockerfileConfigMap: - description: ConfigMap that holds Dockerfile contents - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - kanikoParams: - description: KanikoParams is used to customize the - building process of the image. - properties: - tag: - description: Kaniko image tag to use when creating - the build Pod - type: string - type: object - secrets: - description: |- - Secrets is an optional list of secrets to be made available to the build system. - Those secrets should be used for private resources such as a private Github repo. - For container registries auth use module.spec.imagePullSecret instead. - items: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - type: array - selector: - additionalProperties: - type: string - description: Selector describes on which nodes will - run the building process. - type: object - required: - - dockerfileConfigMap - type: object - containerImage: - description: ContainerImage is the name of the DriverContainer - image that should be used to deploy the module. - type: string - inTreeModuleToRemove: - description: |- - Deprecated: please use InTreeModulesToRemove. - InTreeModuleToRemove specifies one in-tree kernel module that should be removed (if present) - before loading the kernel module from the ContainerImage - type: string - inTreeModulesToRemove: - description: |- - InTreeModulesToRemove specifies any number of in-tree kernel modules that should be removed (if present) - before loading the kernel module from the ContainerImage - items: - type: string - type: array - literal: - description: Literal defines a literal target kernel version - to be matched exactly against node kernels. - type: string - regexp: - description: Regexp is a regular expression to be match - against node kernels. - type: string - registryTLS: - description: RegistryTLS set the TLS configs for accessing - the registry of the module-loader's image. - properties: - insecure: - description: If Insecure is true, the operator will - be able to access a registry in an insecure (plain - HTTP) protocol. - type: boolean - insecureSkipTLSVerify: - description: If InsecureSkipTLSVerify, the operator - will accept any certificate provided by the registry. - type: boolean - type: object - sign: - description: Sign enables in-cluster signing for this - mapping - properties: - certSecret: - description: a secret containing the public key used - to sign kernel modules for secureboot - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - filesToSign: - description: paths inside the image for the kernel - modules to sign (if ommited all kmods are signed) - items: - type: string - type: array - keySecret: - description: a secret containing the private key used - to sign kernel modules for secureboot - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - unsignedImage: - description: Image to sign, ignored if a Build is - present, required otherwise - type: string - unsignedImageRegistryTLS: - description: UnsignedImageRegistryTLS contains settings - determining how to access registries of the unsigned - image. - properties: - insecure: - description: If Insecure is true, the operator - will be able to access a registry in an insecure - (plain HTTP) protocol. - type: boolean - insecureSkipTLSVerify: - description: If InsecureSkipTLSVerify, the operator - will accept any certificate provided by the - registry. - type: boolean - type: object - required: - - certSecret - - keySecret - type: object - required: - - containerImage - type: object - minItems: 1 - type: array - modprobe: - description: Modprobe is a set of properties to customize which - module modprobe loads and with which properties. - properties: - args: - description: |- - Args is an optional list of arguments to be passed to modprobe before the name of the kernel module. - The resulting commands will be: `modprobe ${Args} module_name`. - properties: - load: - description: Load is an optional list of arguments to - be used when loading the kernel module. - items: - type: string - minItems: 1 - type: array - unload: - description: Unload is an optional list of arguments - to be used when unloading the kernel module. - items: - type: string - minItems: 1 - type: array - type: object - dirName: - default: /opt - description: |- - DirName is the root directory for modules. - It adds `-d ${DirName}` to the modprobe command-line. - type: string - firmwarePath: - description: |- - FirmwarePath is the path of the firmware(s). - The firmware(s) will be copied to the host for the kernel to find them. - type: string - moduleName: - description: |- - ModuleName is the name of the Module to be loaded. - This field can only be unset if rawArgs is set. - type: string - modulesLoadingOrder: - description: |- - ModulesLoadingOrder defines the dependency between kernel modules loading, in case - it was not created by depmod (independent kernel modules). - The list order should be: upmost module, then the module it depends on and so on. - Example: if moduleA depends on first loading moduleB, and moduleB depends on first loading moduleC - the entry should look: - ModulesLoadingOrder: - - moduleA - - moduleB - - moduleC - In order to load all 3 modules, moduleA shoud be defined in the ModuleName parameter of this struct - items: - type: string - type: array - parameters: - description: |- - Parameters is an optional list of kernel module parameters to be provided to modprobe. - They should be in the form of key=value and will be separated by spaces in the modprobe command. - The resulting loading command will be: `modprobe module_name ${Parameters}`. - items: - type: string - type: array - rawArgs: - description: |- - If RawArgs are specified, they are passed straight to the modprobe binary; all other properties in this - object are ignored. - The resulting commands will be: `modprobe ${RawArgs}`. - properties: - load: - description: Load is an optional list of arguments to - be used when loading the kernel module. - items: - type: string - minItems: 1 - type: array - unload: - description: Unload is an optional list of arguments - to be used when unloading the kernel module. - items: - type: string - minItems: 1 - type: array - type: object - type: object - registryTLS: - description: RegistryTLS set the TLS configs for accessing the - registry of the module-loader's image. - properties: - insecure: - description: If Insecure is true, the operator will be able - to access a registry in an insecure (plain HTTP) protocol. - type: boolean - insecureSkipTLSVerify: - description: If InsecureSkipTLSVerify, the operator will - accept any certificate provided by the registry. - type: boolean - type: object - sign: - description: Sign provides default kmod signing settings - properties: - certSecret: - description: a secret containing the public key used to - sign kernel modules for secureboot - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - filesToSign: - description: paths inside the image for the kernel modules - to sign (if ommited all kmods are signed) - items: - type: string - type: array - keySecret: - description: a secret containing the private key used to - sign kernel modules for secureboot - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - unsignedImage: - description: Image to sign, ignored if a Build is present, - required otherwise - type: string - unsignedImageRegistryTLS: - description: UnsignedImageRegistryTLS contains settings - determining how to access registries of the unsigned image. - properties: - insecure: - description: If Insecure is true, the operator will - be able to access a registry in an insecure (plain - HTTP) protocol. - type: boolean - insecureSkipTLSVerify: - description: If InsecureSkipTLSVerify, the operator - will accept any certificate provided by the registry. - type: boolean - type: object - required: - - certSecret - - keySecret - type: object - version: - description: |- - Version defines the current version of the kernel module being used - Used for upgrading the currently loaded kernel module to a new version - type: string - required: - - kernelMappings - - modprobe - type: object - serviceAccountName: - description: |- - ServiceAccountName is the name of the ServiceAccount to use to run this pod. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ - type: string - required: - - container - type: object - selector: - additionalProperties: - type: string - description: Selector describes on which nodes the Module should be - loaded and optionally built. - type: object - required: - - moduleLoader - - selector - type: object - status: - description: ModuleStatus defines the observed state of Module. - properties: - devicePlugin: - description: |- - DevicePlugin contains the status of the Device Plugin daemonset - if it was deployed during reconciliation - properties: - availableNumber: - description: number of the actually deployed and running pods - format: int32 - type: integer - desiredNumber: - description: number of the pods that should be deployed for daemonset - format: int32 - type: integer - nodesMatchingSelectorNumber: - description: number of nodes that are targeted by the module selector - format: int32 - type: integer - type: object - moduleLoader: - description: ModuleLoader contains the status of the ModuleLoader daemonset - properties: - availableNumber: - description: number of the actually deployed and running pods - format: int32 - type: integer - desiredNumber: - description: number of the pods that should be deployed for daemonset - format: int32 - type: integer - nodesMatchingSelectorNumber: - description: number of nodes that are targeted by the module selector - format: int32 - type: integer - type: object - required: - - moduleLoader - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/helm-charts/charts/kmm/crds/nodemodulesconfig-crd.yaml b/helm-charts/charts/kmm/crds/nodemodulesconfig-crd.yaml deleted file mode 100644 index b5af4a54e..000000000 --- a/helm-charts/charts/kmm/crds/nodemodulesconfig-crd.yaml +++ /dev/null @@ -1,367 +0,0 @@ ---- -# Source: kmm/templates/nodemodulesconfig-crd.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: nodemodulesconfigs.kmm.sigs.x-k8s.io - annotations: - controller-gen.kubebuilder.io/version: v0.16.1 - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - helm.sh/chart: kmm-v1.0.0 - app.kubernetes.io/name: kmm - app.kubernetes.io/instance: amd-gpu - app.kubernetes.io/version: "v20240618-v2.1.1" - app.kubernetes.io/managed-by: Helm -spec: - group: kmm.sigs.x-k8s.io - names: - kind: NodeModulesConfig - listKind: NodeModulesConfigList - plural: nodemodulesconfigs - shortNames: - - nmc - singular: nodemodulesconfig - scope: Cluster - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: NodeModulesConfig keeps spec and state of the KMM modules on a - node. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: |- - NodeModulesConfigSpec describes the desired state of modules on the node - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - modules: - description: |- - Modules list the spec of all the modules that need to be executed - on the node - items: - properties: - config: - properties: - containerImage: - type: string - imagePullPolicy: - default: IfNotPresent - description: PullPolicy describes a policy for if/when to - pull a container image - type: string - inTreeModuleToRemove: - type: string - inTreeModulesToRemove: - items: - type: string - type: array - insecurePull: - description: When InsecurePull is true, the container image - can be pulled without TLS. - type: boolean - kernelVersion: - type: string - modprobe: - properties: - args: - description: |- - Args is an optional list of arguments to be passed to modprobe before the name of the kernel module. - The resulting commands will be: `modprobe ${Args} module_name`. - properties: - load: - description: Load is an optional list of arguments - to be used when loading the kernel module. - items: - type: string - minItems: 1 - type: array - unload: - description: Unload is an optional list of arguments - to be used when unloading the kernel module. - items: - type: string - minItems: 1 - type: array - type: object - dirName: - default: /opt - description: |- - DirName is the root directory for modules. - It adds `-d ${DirName}` to the modprobe command-line. - type: string - firmwarePath: - description: |- - FirmwarePath is the path of the firmware(s). - The firmware(s) will be copied to the host for the kernel to find them. - type: string - moduleName: - description: |- - ModuleName is the name of the Module to be loaded. - This field can only be unset if rawArgs is set. - type: string - modulesLoadingOrder: - description: |- - ModulesLoadingOrder defines the dependency between kernel modules loading, in case - it was not created by depmod (independent kernel modules). - The list order should be: upmost module, then the module it depends on and so on. - Example: if moduleA depends on first loading moduleB, and moduleB depends on first loading moduleC - the entry should look: - ModulesLoadingOrder: - - moduleA - - moduleB - - moduleC - In order to load all 3 modules, moduleA shoud be defined in the ModuleName parameter of this struct - items: - type: string - type: array - parameters: - description: |- - Parameters is an optional list of kernel module parameters to be provided to modprobe. - They should be in the form of key=value and will be separated by spaces in the modprobe command. - The resulting loading command will be: `modprobe module_name ${Parameters}`. - items: - type: string - type: array - rawArgs: - description: |- - If RawArgs are specified, they are passed straight to the modprobe binary; all other properties in this - object are ignored. - The resulting commands will be: `modprobe ${RawArgs}`. - properties: - load: - description: Load is an optional list of arguments - to be used when loading the kernel module. - items: - type: string - minItems: 1 - type: array - unload: - description: Unload is an optional list of arguments - to be used when unloading the kernel module. - items: - type: string - minItems: 1 - type: array - type: object - type: object - required: - - containerImage - - imagePullPolicy - - insecurePull - - kernelVersion - - modprobe - type: object - imageRepoSecret: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - name: - type: string - namespace: - type: string - serviceAccountName: - type: string - required: - - config - - name - - namespace - - serviceAccountName - type: object - type: array - type: object - status: - description: |- - NodeModuleConfigStatus is the most recently observed status of the KMM modules on node. - It is populated by the system and is read-only. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - modules: - description: Modules contain observations about each Module's node state - status - items: - properties: - config: - properties: - containerImage: - type: string - imagePullPolicy: - default: IfNotPresent - description: PullPolicy describes a policy for if/when to - pull a container image - type: string - inTreeModuleToRemove: - type: string - inTreeModulesToRemove: - items: - type: string - type: array - insecurePull: - description: When InsecurePull is true, the container image - can be pulled without TLS. - type: boolean - kernelVersion: - type: string - modprobe: - properties: - args: - description: |- - Args is an optional list of arguments to be passed to modprobe before the name of the kernel module. - The resulting commands will be: `modprobe ${Args} module_name`. - properties: - load: - description: Load is an optional list of arguments - to be used when loading the kernel module. - items: - type: string - minItems: 1 - type: array - unload: - description: Unload is an optional list of arguments - to be used when unloading the kernel module. - items: - type: string - minItems: 1 - type: array - type: object - dirName: - default: /opt - description: |- - DirName is the root directory for modules. - It adds `-d ${DirName}` to the modprobe command-line. - type: string - firmwarePath: - description: |- - FirmwarePath is the path of the firmware(s). - The firmware(s) will be copied to the host for the kernel to find them. - type: string - moduleName: - description: |- - ModuleName is the name of the Module to be loaded. - This field can only be unset if rawArgs is set. - type: string - modulesLoadingOrder: - description: |- - ModulesLoadingOrder defines the dependency between kernel modules loading, in case - it was not created by depmod (independent kernel modules). - The list order should be: upmost module, then the module it depends on and so on. - Example: if moduleA depends on first loading moduleB, and moduleB depends on first loading moduleC - the entry should look: - ModulesLoadingOrder: - - moduleA - - moduleB - - moduleC - In order to load all 3 modules, moduleA shoud be defined in the ModuleName parameter of this struct - items: - type: string - type: array - parameters: - description: |- - Parameters is an optional list of kernel module parameters to be provided to modprobe. - They should be in the form of key=value and will be separated by spaces in the modprobe command. - The resulting loading command will be: `modprobe module_name ${Parameters}`. - items: - type: string - type: array - rawArgs: - description: |- - If RawArgs are specified, they are passed straight to the modprobe binary; all other properties in this - object are ignored. - The resulting commands will be: `modprobe ${RawArgs}`. - properties: - load: - description: Load is an optional list of arguments - to be used when loading the kernel module. - items: - type: string - minItems: 1 - type: array - unload: - description: Unload is an optional list of arguments - to be used when unloading the kernel module. - items: - type: string - minItems: 1 - type: array - type: object - type: object - required: - - containerImage - - imagePullPolicy - - insecurePull - - kernelVersion - - modprobe - type: object - imageRepoSecret: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - lastTransitionTime: - format: date-time - type: string - name: - type: string - namespace: - type: string - serviceAccountName: - type: string - required: - - name - - namespace - - serviceAccountName - type: object - type: array - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/helm-charts/charts/kmm/templates/_helpers.tpl b/helm-charts/charts/kmm/templates/_helpers.tpl deleted file mode 100644 index 182641509..000000000 --- a/helm-charts/charts/kmm/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "kmm.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "kmm.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "kmm.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "kmm.labels" -}} -helm.sh/chart: {{ include "kmm.chart" . }} -{{ include "kmm.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "kmm.selectorLabels" -}} -app.kubernetes.io/name: {{ include "kmm.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "kmm.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "kmm.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/helm-charts/charts/kmm/templates/controller-metrics-service.yaml b/helm-charts/charts/kmm/templates/controller-metrics-service.yaml deleted file mode 100644 index 4f17b470a..000000000 --- a/helm-charts/charts/kmm/templates/controller-metrics-service.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "kmm.fullname" . }}-controller-metrics-service - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: controller - {{- include "kmm.labels" . | nindent 4 }} -spec: - type: {{ .Values.controllerMetricsService.type }} - selector: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: controller - {{- include "kmm.selectorLabels" . | nindent 4 }} - ports: - {{- .Values.controllerMetricsService.ports | toYaml | nindent 2 }} diff --git a/helm-charts/charts/kmm/templates/deployment.yaml b/helm-charts/charts/kmm/templates/deployment.yaml deleted file mode 100644 index c7be70b4e..000000000 --- a/helm-charts/charts/kmm/templates/deployment.yaml +++ /dev/null @@ -1,203 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "kmm.fullname" . }}-controller - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: controller - {{- include "kmm.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.controller.replicas }} - selector: - matchLabels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: controller - {{- include "kmm.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: controller - {{- include "kmm.selectorLabels" . | nindent 8 }} - annotations: - kubectl.kubernetes.io/default-container: manager - spec: - {{- with .Values.controller.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - nodeSelector: {{- toYaml .Values.controller.nodeSelector | nindent 8 }} - containers: - - args: {{- toYaml .Values.controller.manager.args | nindent 8 }} - env: - - name: RELATED_IMAGE_WORKER - value: {{ quote .Values.controller.manager.env.relatedImageWorker }} - - name: OPERATOR_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: RELATED_IMAGE_BUILD - value: {{ quote .Values.controller.manager.env.relatedImageBuild }} - - name: RELATED_IMAGE_SIGN - value: {{ quote .Values.controller.manager.env.relatedImageSign }} - - name: KUBERNETES_CLUSTER_DOMAIN - value: {{ quote .Values.kubernetesClusterDomain }} - {{- if .Values.controller.manager.env.relatedImageBuildPullSecret }} - - name: RELATED_IMAGE_BUILD_PULL_SECRET - value: {{ .Values.controller.manager.env.relatedImageBuildPullSecret }} - {{- end}} - {{- if .Values.controller.manager.env.relatedImageSignPullSecret }} - - name: RELATED_IMAGE_SIGN_PULL_SECRET - value: {{ .Values.controller.manager.env.relatedImageSignPullSecret }} - {{- end}} - {{- if .Values.controller.manager.env.relatedImageWorkerPullSecret }} - - name: RELATED_IMAGE_WORKER_PULL_SECRET - value: {{ .Values.controller.manager.env.relatedImageWorkerPullSecret }} - {{- end}} - {{- if .Values.global.proxy.env | default dict}} - {{- range $key, $value := .Values.global.proxy.env }} - - name: {{ $key }} - value: {{ $value | quote }} - {{- end }} - {{- end }} - image: {{ .Values.controller.manager.image.repository }}:{{ .Values.controller.manager.image.tag - | default .Chart.AppVersion }} - imagePullPolicy: {{ .Values.controller.manager.imagePullPolicy }} - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - ports: - - containerPort: 8443 - name: metrics - protocol: TCP - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: {{- toYaml .Values.controller.manager.resources | nindent 10 }} - securityContext: {{- toYaml .Values.controller.manager.containerSecurityContext - | nindent 10 }} - volumeMounts: - - mountPath: /controller_config.yaml - name: manager-config - subPath: controller_config.yaml - {{- if .Values.controller.manager.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.controller.manager.imagePullSecrets }} - {{- end}} - securityContext: - runAsNonRoot: true - serviceAccountName: {{ include "kmm.fullname" . }}-controller - terminationGracePeriodSeconds: 10 - {{- with .Values.controller.manager.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - volumes: - - configMap: - name: {{ include "kmm.fullname" . }}-manager-config - name: manager-config ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "kmm.fullname" . }}-webhook-server - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: webhook-server - {{- include "kmm.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.webhookServer.replicas }} - selector: - matchLabels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: webhook-server - {{- include "kmm.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: webhook-server - {{- include "kmm.selectorLabels" . | nindent 8 }} - annotations: - kubectl.kubernetes.io/default-container: webhook-server - spec: - {{- with .Values.webhookServer.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - nodeSelector: {{- toYaml .Values.webhookServer.nodeSelector | nindent 8 }} - containers: - - args: {{- toYaml .Values.webhookServer.webhookServer.args | nindent 8 }} - env: - - name: KUBERNETES_CLUSTER_DOMAIN - value: {{ quote .Values.kubernetesClusterDomain }} - {{- if .Values.global.proxy.env | default dict}} - {{- range $key, $value := .Values.global.proxy.env }} - - name: {{ $key }} - value: {{ $value | quote }} - {{- end }} - {{- end }} - image: {{ .Values.webhookServer.webhookServer.image.repository }}:{{ .Values.webhookServer.webhookServer.image.tag - | default .Chart.AppVersion }} - imagePullPolicy: {{ .Values.webhookServer.webhookServer.imagePullPolicy }} - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: webhook-server - ports: - - containerPort: 9443 - name: webhook-server - protocol: TCP - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: {{- toYaml .Values.webhookServer.webhookServer.resources | nindent 10 - }} - securityContext: {{- toYaml .Values.webhookServer.webhookServer.containerSecurityContext - | nindent 10 }} - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true - - mountPath: /controller_config.yaml - name: manager-config - subPath: controller_config.yaml - {{- if .Values.webhookServer.webhookServer.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.webhookServer.webhookServer.imagePullSecrets }} - {{- end}} - securityContext: - runAsNonRoot: true - serviceAccountName: {{ include "kmm.fullname" . }}-controller - terminationGracePeriodSeconds: 10 - {{- with .Values.webhookServer.webhookServer.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - volumes: - - name: cert - secret: - defaultMode: 420 - secretName: kmm-operator-webhook-server-cert - - configMap: - name: {{ include "kmm.fullname" . }}-manager-config - name: manager-config diff --git a/helm-charts/charts/kmm/templates/event-recorder-clusterrole-rbac.yaml b/helm-charts/charts/kmm/templates/event-recorder-clusterrole-rbac.yaml deleted file mode 100644 index 6d86d628c..000000000 --- a/helm-charts/charts/kmm/templates/event-recorder-clusterrole-rbac.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "kmm.fullname" . }}-event-recorder-clusterrole - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch \ No newline at end of file diff --git a/helm-charts/charts/kmm/templates/event-recorder-clusterrolebinding-rbac.yaml b/helm-charts/charts/kmm/templates/event-recorder-clusterrolebinding-rbac.yaml deleted file mode 100644 index 21366100f..000000000 --- a/helm-charts/charts/kmm/templates/event-recorder-clusterrolebinding-rbac.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "kmm.fullname" . }}-event-recorder-clusterrolebinding - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "kmm.fullname" . }}-event-recorder-clusterrole' -subjects: -- kind: ServiceAccount - name: '{{ include "kmm.fullname" . }}-controller' - namespace: '{{ .Release.Namespace }}' \ No newline at end of file diff --git a/helm-charts/charts/kmm/templates/leader-election-rbac.yaml b/helm-charts/charts/kmm/templates/leader-election-rbac.yaml deleted file mode 100644 index d4b7df6c5..000000000 --- a/helm-charts/charts/kmm/templates/leader-election-rbac.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "kmm.fullname" . }}-leader-election-role - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "kmm.fullname" . }}-leader-election-rolebinding - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: '{{ include "kmm.fullname" . }}-leader-election-role' -subjects: -- kind: ServiceAccount - name: '{{ include "kmm.fullname" . }}-controller' - namespace: '{{ .Release.Namespace }}' \ No newline at end of file diff --git a/helm-charts/charts/kmm/templates/manager-config.yaml b/helm-charts/charts/kmm/templates/manager-config.yaml deleted file mode 100644 index 27f3a711d..000000000 --- a/helm-charts/charts/kmm/templates/manager-config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "kmm.fullname" . }}-manager-config - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -data: - controller_config.yaml: {{ .Values.managerConfig.controllerConfigYaml | toYaml - | indent 1 }} \ No newline at end of file diff --git a/helm-charts/charts/kmm/templates/manager-rbac.yaml b/helm-charts/charts/kmm/templates/manager-rbac.yaml deleted file mode 100644 index 677acd611..000000000 --- a/helm-charts/charts/kmm/templates/manager-rbac.yaml +++ /dev/null @@ -1,135 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "kmm.fullname" . }}-manager-role - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -rules: -- apiGroups: - - apps - resources: - - daemonsets - verbs: - - create - - delete - - get - - list - - patch - - watch -- apiGroups: - - cluster.open-cluster-management.io - resources: - - clusterclaims - verbs: - - create - - get - - list - - watch -- apiGroups: - - cluster.open-cluster-management.io - resourceNames: - - kernel-versions.kmm.node.kubernetes.io - resources: - - clusterclaims - verbs: - - delete - - patch - - update -- apiGroups: - - "" - resources: - - configmaps - - secrets - - serviceaccounts - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - namespaces - - nodes - verbs: - - get - - list - - patch - - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - create - - delete - - get - - list - - patch - - watch -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - modules - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - modules/status - - preflightvalidations/status - verbs: - - get - - patch - - update -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - nodemodulesconfigs - verbs: - - create - - delete - - get - - list - - patch - - watch -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - nodemodulesconfigs/status - verbs: - - patch -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - preflightvalidations - verbs: - - create - - delete - - get - - list - - patch - - update - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "kmm.fullname" . }}-manager-rolebinding - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "kmm.fullname" . }}-manager-role' -subjects: -- kind: ServiceAccount - name: '{{ include "kmm.fullname" . }}-controller' - namespace: '{{ .Release.Namespace }}' \ No newline at end of file diff --git a/helm-charts/charts/kmm/templates/metrics-reader-rbac.yaml b/helm-charts/charts/kmm/templates/metrics-reader-rbac.yaml deleted file mode 100644 index 2acb7127b..000000000 --- a/helm-charts/charts/kmm/templates/metrics-reader-rbac.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "kmm.fullname" . }}-metrics-reader - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -rules: -- nonResourceURLs: - - /metrics - verbs: - - get \ No newline at end of file diff --git a/helm-charts/charts/kmm/templates/preflightvalidation-crd.yaml b/helm-charts/charts/kmm/templates/preflightvalidation-crd.yaml deleted file mode 100644 index 6da0c1c07..000000000 --- a/helm-charts/charts/kmm/templates/preflightvalidation-crd.yaml +++ /dev/null @@ -1,243 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: preflightvalidations.kmm.sigs.x-k8s.io - annotations: - cert-manager.io/inject-ca-from: '{{ .Release.Namespace }}/{{ include "kmm.fullname" - . }}-serving-cert' - controller-gen.kubebuilder.io/version: v0.16.1 - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - name: '{{ include "kmm.fullname" . }}-webhook-service' - namespace: '{{ .Release.Namespace }}' - path: /convert - conversionReviewVersions: - - v1beta2 - - v1beta1 - group: kmm.sigs.x-k8s.io - names: - kind: PreflightValidation - listKind: PreflightValidationList - plural: preflightvalidations - shortNames: - - pfv - singular: preflightvalidation - scope: Cluster - versions: - - deprecated: true - name: v1beta1 - schema: - openAPIV3Schema: - description: PreflightValidation initiates a preflight validations for all Modules - on the current Kubernetes cluster. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: |- - PreflightValidationSpec describes the desired state of the resource, such as the kernel version - that Module CRs need to be verified against as well as the debug configuration of the logs - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - kernelVersion: - description: KernelVersion describes the kernel image that all Modules - need to be checked against. - type: string - pushBuiltImage: - description: |- - Boolean flag that determines whether images build during preflight must also - be pushed to a defined repository - type: boolean - required: - - kernelVersion - type: object - status: - description: |- - PreflightValidationStatus is the most recently observed status of the PreflightValidation. - It is populated by the system and is read-only. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - crStatuses: - additionalProperties: - properties: - lastTransitionTime: - description: |- - LastTransitionTime is the last time the CR status transitioned from one status to another. - This should be when the underlying status changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - statusReason: - description: StatusReason contains a string describing the status - source. - type: string - verificationStage: - description: |- - Current stage of the verification process: - image (image existence verification), build(build process verification) - enum: - - Image - - Build - - Sign - - Requeued - - Done - type: string - verificationStatus: - description: |- - Status of Module CR verification: true (verified), false (verification failed), - error (error during verification process), unknown (verification has not started yet) - enum: - - "True" - - "False" - type: string - required: - - lastTransitionTime - - verificationStage - - verificationStatus - type: object - description: CRStatuses contain observations about each Module's preflight - upgradability validation - type: object - type: object - required: - - spec - type: object - served: true - storage: false - subresources: - status: {} - - name: v1beta2 - schema: - openAPIV3Schema: - description: PreflightValidation initiates a preflight validations for all Modules - on the current Kubernetes cluster. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: |- - PreflightValidationSpec describes the desired state of the resource, such as the kernel version - that Module CRs need to be verified against as well as the debug configuration of the logs - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - kernelVersion: - description: KernelVersion describes the kernel image that all Modules - need to be checked against. - type: string - pushBuiltImage: - description: |- - Boolean flag that determines whether images build during preflight must also - be pushed to a defined repository - type: boolean - required: - - kernelVersion - type: object - status: - description: |- - PreflightValidationStatus is the most recently observed status of the PreflightValidation. - It is populated by the system and is read-only. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - modules: - description: Modules contain observations about each Module's preflight - upgradability validation - items: - properties: - lastTransitionTime: - description: |- - LastTransitionTime is the last time the CR status transitioned from one status to another. - This should be when the underlying status changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - name: - description: Name is the name of the Module resource. - type: string - namespace: - description: Namespace is the namespace of the Module resource. - type: string - statusReason: - description: StatusReason contains a string describing the status - source. - type: string - verificationStage: - description: |- - Current stage of the verification process: - image (image existence verification), build(build process verification) - enum: - - Image - - Build - - Sign - - Requeued - - Done - type: string - verificationStatus: - description: |- - Status of Module CR verification: true (verified), false (verification failed), - error (error during verification process), unknown (verification has not started yet) - enum: - - "True" - - "False" - type: string - required: - - lastTransitionTime - - name - - namespace - - verificationStage - - verificationStatus - type: object - type: array - x-kubernetes-list-map-keys: - - namespace - - name - x-kubernetes-list-type: map - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] \ No newline at end of file diff --git a/helm-charts/charts/kmm/templates/proxy-rbac.yaml b/helm-charts/charts/kmm/templates/proxy-rbac.yaml deleted file mode 100644 index 6cc30bba1..000000000 --- a/helm-charts/charts/kmm/templates/proxy-rbac.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "kmm.fullname" . }}-proxy-role - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "kmm.fullname" . }}-proxy-rolebinding - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "kmm.fullname" . }}-proxy-role' -subjects: -- kind: ServiceAccount - name: '{{ include "kmm.fullname" . }}-controller' - namespace: '{{ .Release.Namespace }}' \ No newline at end of file diff --git a/helm-charts/charts/kmm/templates/selfsigned-issuer.yaml b/helm-charts/charts/kmm/templates/selfsigned-issuer.yaml deleted file mode 100644 index f3c128e58..000000000 --- a/helm-charts/charts/kmm/templates/selfsigned-issuer.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: {{ include "kmm.fullname" . }}-selfsigned-issuer - labels: - {{- include "kmm.labels" . | nindent 4 }} -spec: - selfSigned: {} \ No newline at end of file diff --git a/helm-charts/charts/kmm/templates/serviceaccount.yaml b/helm-charts/charts/kmm/templates/serviceaccount.yaml deleted file mode 100644 index f581e45a6..000000000 --- a/helm-charts/charts/kmm/templates/serviceaccount.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "kmm.fullname" . }}-controller - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} - annotations: - {{- toYaml .Values.controller.serviceAccount.annotations | nindent 4 }} \ No newline at end of file diff --git a/helm-charts/charts/kmm/templates/serving-cert.yaml b/helm-charts/charts/kmm/templates/serving-cert.yaml deleted file mode 100644 index fed75c499..000000000 --- a/helm-charts/charts/kmm/templates/serving-cert.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: {{ include "kmm.fullname" . }}-serving-cert - labels: - {{- include "kmm.labels" . | nindent 4 }} -spec: - dnsNames: - - '{{ include "kmm.fullname" . }}-webhook-service.{{ .Release.Namespace }}.svc' - - '{{ include "kmm.fullname" . }}-webhook-service.{{ .Release.Namespace }}.svc.{{ - .Values.kubernetesClusterDomain }}' - issuerRef: - kind: Issuer - name: '{{ include "kmm.fullname" . }}-selfsigned-issuer' - secretName: kmm-operator-webhook-server-cert \ No newline at end of file diff --git a/helm-charts/charts/kmm/templates/validating-webhook-configuration.yaml b/helm-charts/charts/kmm/templates/validating-webhook-configuration.yaml deleted file mode 100644 index 10e02d49c..000000000 --- a/helm-charts/charts/kmm/templates/validating-webhook-configuration.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: {{ include "kmm.fullname" . }}-validating-webhook-configuration - annotations: - cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "kmm.fullname" . }}-serving-cert - labels: - {{- include "kmm.labels" . | nindent 4 }} -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: '{{ include "kmm.fullname" . }}-webhook-service' - namespace: '{{ .Release.Namespace }}' - path: /validate--v1-namespace - failurePolicy: Fail - name: namespace-deletion.kmm.sigs.k8s.io - namespaceSelector: - matchLabels: - kmm.node.k8s.io/contains-modules: "" - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - DELETE - resources: - - namespaces - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: '{{ include "kmm.fullname" . }}-webhook-service' - namespace: '{{ .Release.Namespace }}' - path: /validate-kmm-sigs-x-k8s-io-v1beta1-module - failurePolicy: Fail - name: vmodule.kb.io - rules: - - apiGroups: - - kmm.sigs.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - modules - sideEffects: None \ No newline at end of file diff --git a/helm-charts/charts/kmm/templates/webhook-service.yaml b/helm-charts/charts/kmm/templates/webhook-service.yaml deleted file mode 100644 index 90c7547f5..000000000 --- a/helm-charts/charts/kmm/templates/webhook-service.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "kmm.fullname" . }}-webhook-service - labels: - app.kubernetes.io/component: kmm - app.kubernetes.io/created-by: kernel-module-management - app.kubernetes.io/part-of: kmm - {{- include "kmm.labels" . | nindent 4 }} -spec: - type: {{ .Values.webhookService.type }} - selector: - app.kubernetes.io/component: kmm - app.kubernetes.io/part-of: kmm - control-plane: webhook-server - {{- include "kmm.selectorLabels" . | nindent 4 }} - ports: - {{- .Values.webhookService.ports | toYaml | nindent 2 }} diff --git a/helm-charts/charts/kmm/values.yaml b/helm-charts/charts/kmm/values.yaml deleted file mode 100644 index 5fd0e3868..000000000 --- a/helm-charts/charts/kmm/values.yaml +++ /dev/null @@ -1,133 +0,0 @@ -controller: - manager: - args: - - --config=controller_config.yaml - containerSecurityContext: - allowPrivilegeEscalation: false - env: - # -- KMM kaniko builder image for building driver image within cluster - relatedImageBuild: gcr.io/kaniko-project/executor:v1.23.2 - # -- KMM signer image for signing driver image's kernel module with given key pairs within cluster - relatedImageSign: docker.io/rocm/kernel-module-management-signimage:v1.2.0 - # -- KMM worker image for loading / unloading driver kernel module on worker nodes - relatedImageWorker: docker.io/rocm/kernel-module-management-worker:v1.2.0 - # -- Image pull secret name for pulling KMM kaniko builder image if registry needs credential to pull image - relatedImageBuildPullSecret: "" - # -- Image pull secret name for pulling KMM signer image if registry needs credential to pull image - relatedImageSignPullSecret: "" - # -- Image pull secret name for pulling KMM worker image if registry needs credential to pull image - relatedImageWorkerPullSecret: "" - image: - # -- KMM controller manager image repository - repository: docker.io/rocm/kernel-module-management-operator - # -- KMM controller manager image tag - tag: v1.2.0 - # -- Image pull policy for KMM controller manager pod - imagePullPolicy: Always - # -- Image pull secret name for pulling KMM controller manager image if registry needs credential to pull image - imagePullSecrets: "" - tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Equal" - value: "" - effect: "NoSchedule" - - key: "node-role.kubernetes.io/control-plane" - operator: "Equal" - value: "" - effect: "NoSchedule" - resources: - limits: - cpu: 500m - memory: 384Mi - requests: - cpu: 10m - memory: 64Mi - # -- Node selector for the KMM controller manager deployment - nodeSelector: {} - # -- Affinity for the KMM controller manager deployment - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - replicas: 1 - serviceAccount: - annotations: {} -controllerMetricsService: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: https - type: ClusterIP -kubernetesClusterDomain: cluster.local -managerConfig: - controllerConfigYaml: |- - healthProbeBindAddress: :8081 - webhookPort: 9443 - leaderElection: - enabled: true - resourceID: kmm.sigs.x-k8s.io - metrics: - enableAuthnAuthz: true - bindAddress: 0.0.0.0:8443 - secureServing: true - worker: - runAsUser: 0 - seLinuxType: spc_t - firmwareHostPath: /var/lib/firmware -webhookServer: - replicas: 1 - # -- KMM webhook's deployment node selector - nodeSelector: {} - # -- KMM webhook's deployment affinity configs - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - webhookServer: - args: - - --config=controller_config.yaml - - --enable-module - - --enable-namespace - - --enable-preflightvalidation - containerSecurityContext: - allowPrivilegeEscalation: false - image: - # -- KMM webhook image repository - repository: docker.io/rocm/kernel-module-management-webhook-server - # -- KMM webhook image tag - tag: v1.2.0 - # -- Image pull policy for KMM webhook pod - imagePullPolicy: Always - # -- Image pull secret name for pulling KMM webhook image if registry needs credential to pull image - imagePullSecrets: "" - tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Equal" - value: "" - effect: "NoSchedule" - - key: "node-role.kubernetes.io/control-plane" - operator: "Equal" - value: "" - effect: "NoSchedule" - resources: - limits: - cpu: 500m - memory: 384Mi - requests: - cpu: 10m - memory: 64Mi -webhookService: - ports: - - port: 443 - protocol: TCP - targetPort: 9443 - type: ClusterIP diff --git a/helm-charts/charts/node-feature-discovery-chart-0.16.1.tgz b/helm-charts/charts/node-feature-discovery-chart-0.16.1.tgz deleted file mode 100644 index 42296b34e30a12b3681606d205f63aa8becb7f83..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14942 zcmXxLWl$YF*S3vAacC*-#VPLYvT=8JcXxMp*NwYNaVYLyv^X1gXMer!=X>*GO=O*! zOlC5(l4FH92H_jTe+5hfL1!eT%w!@Z$06&*&2G%9&Sa{>Zl$fl&7q*K&LO8^Yi(q2 z>ZPjWz%ONHYX@=OW9Nb2nZ%Sm`;OJ2n*Y_OxJ+t2x#ySDBXu(0JfY4?5?jjnYnH|L zbB5%|E_^}=xu{0B+0VqD0Q-P(e$Ue9)VQ(!?pKHrb-`Lm(blE2rxj-Jome@}LI zuOF`F98v5z(-V=V{5gFcbzWvZL=aKO(E7QGK5>3N`fwp67Yq%YI5L5_`|Tv4iike{ z!oaKrmvEcU{4@_lzHMFoDY`qV7R~vhao?368;3Ohx@@W_DBTA_g6b7xiOJE9LK|>{oZ$ad2`?T z+(sYHhF`#`zx(-N(GDuJs+?aQe9}b*7O3xi={M>jW~|0+6&o~073raI$O&K=8XE@< zG2$X|6W!tiM&1n!IbnJ!Ar%F|20h$d1V=~UzmDK(_fLmnbhKH+P0>hUA#RM+pQDij zv5o`jAvx{7LMQoxZdDyvMTD;WR0KEND0=X65Kib)iuV!Cr`%ZB$Xw5RXqEt6<6Wqu zU6P@B{?On#H^vH?^FthE2WrJ$#5DM?6N@&S>P2junM5fMcA(1lip8QI#YeFt0==m+~~J=7lkegrcy`UJhH zmrhKPCN(q}Em|YcqlqTRL%Au31TQb|iNf)fLQR6C#%g)S%1!y*)5`M0{9&ux?KltE z)lM(p|NVaGw7>^J8uWZmmfI0JTax}@LV)NEr9#lPmpd9a#_33f^GOm`;(ODXKnR~3 z#Q0LyXX7uPtib<1?t2p0C+l|{sGv-oT@W=T8!#axqy5nN>J8Z|n#Hah%(1ti6u{}- z$hgks-AfF;AlChRID;?ovH|lcAm=M2F#@*9Qi1Xxg=~Y@M_4A2JZ8|p7ggv7Rs%FWl3uSXx7(?#A_IZp$P+`}2kZuCun~UzFxMT~3pG1B0;#_!)D{6_ z|Dr)8H?%-`9jnCKc=renib@|JD3l?S^04GiVYS?Z{j_Y!?X*@mqf!}xE15P>0oFNP z;4v~z&m&6nL?_tOu!G4odEyI~-of7NZUL!LAk%GmJTOZuS;Mf*G_s#F(2xgKhlDSU zk%|Pr{dldQ@+BAjFtmX_Z9ApoYb6!&%@)=RcI^Dgg<~eg7Ca9fFu$58ze}4`N{H6T z9h0R-tz;fD!Qg^~G^|)#VxGWdJ?jfe#x|6was0>mXfzHhoTgYkE<)Q9{o}M{Vk{Ok zvAqHWFT;%)Rd}48c|WR3;2zv>OTpkhg}jnLUBhraY~4Zr_0VrQX2u8i@1@jY2hI>D zug_4hs;GMeG55P&p-9)A+NhBZDhjCOrqH{=Xo*S8Cim{o>a8*IB!-42knxxLfw+%C z!DPA&)UPONo2NhAEHbc1Pfqq^mDKAX5!YbQp%WOEw8@Sl5o+`0MQLz~AEX&%QYoZ7 zNpKop#P4mFL?fSd4R_R$Ipkn^wWU-RiXxnyXE4fN_{)kIe_Jl8ggVvr zqb@6p`9XRNE~OphIk(!!&3`&_rM7z2Su4m0WNhm*HIXn(?WS(5v9nTVxpJUhhjxge z_ZUy?G2z9|TPz8_UKb^_AXo(EDfD*#Dpeg@pU%(?%t(%*zfXAPelQD*?NDXxJx-P= z!O>|W;UHLV12t57Vg<^T!Eklug=9jDa#ck&)MYFC+p=%8m@WiZXv@uOvix*6-kuIx zN~MjJV(Tl#3_fW&m-)hUz{J!#TL$2h0m{n5#--%)Vpi+Xcu`Of&LjI8T3=cY$wc{- zK#)S}layyeRoRkAGjR+&cC0OjroIB(EaO|ZJlu=&fs~rjAJdxM!{lT{9%y{PwBFD3 zOy<=EGH(a?B0SOA_if>Ta?;yt%&omlf^6j`I&A^K%Uwa^=Dgf{dGTBTXr;sgTKo$9 zQV))&1X`C5^}60nao{?nz7QLn*T!b`$`stK*e-)Y{V9B zjC7HP>Do@{D|$PGf3H&1Cis0dz~;Uf9+4%u8v;j(*rr9PKkzPbpbqpLr&_El{&kP> z1ZB8lti3?MPizN+hS42^8mP^h4tc1!8y&Gb14j`~8Y5YwirhkDXhOWMiOGLYVcuq1 z7083|UF>I(Z=e9}8Z%y-u^W3(MY$uq&bgQOL9VyWeV^hX8K)YJiy$oHu{Za6_?^V#BGLp((j4@`Fb{1kv?(OHht=v{t*GbhE{zg-Eql9nwuz1 zP$%sT*94z%vQl|J4i^svEMwyFD2VSX5k7F?JVxBH}(c@)+lv5h#p+G%)t#+jYjt1=ol6 z%);k9nqYkR$KZaSFCTuIz!swb|L6718yZD|p#N*p!D&Lt(B9(WWP-*>`IThE=ty}$ zPcMVO`!(_C@qaGwjrV^~6DZPVWkCAAUDU9?Ud~>w{>qQtzKFCxtBbvZ$@f*|-o@U=5Su$O;IkmtqoyC9-%GSt4*A!@2mQ)JVN zcZVRs5h<#{VL|Ld%c5bL)_Z>7**B(kvluV*KW7;MRYr#ew~ocNzY8Y&0NcmCXfLHU zV5GPqHn6F71#6ytIFa(GeDiYd0=|FMiZ@j|eI~rWp?Ru2KP#m4i~l}%UmQXBAaw%9J_6X)TuDkrER z#f}Jr(QyFUfO;`@=AbJwenz3?d9<$j3|bT_a}?vojJOGX`Oi%-vSFbB)0XOk2Zk9n z#P`47g!jA0kUHlC4ax@0=ICnZ;HESg%%NKg&1_Vy5-SlA2tTg(2Q0xKIDSO~ICDdIEF z9aPXWhb2baxJLR}a)8j^4AgJXJ^4g2iN#8DX4EmwlV+$FK24tNE5 zpX+ilDpg*iewV^B(!x3mc{L3!Vcj&Ru)vhsnuKxQ`@l==O+w(aSnf%r##BOJIXzb8Kl!y&Tqh1Rj&9O3^c1MOr)x< zOH}52KME+Rka~XvM|`DIW;Gz&vml-D2V$ugwUxmZ`}=-pRQEblT_O+Anv*|s)mZX+ zu^lIpy1+_?Ws(_c^taW1SMh}5s6-zm%w8qe;V{}k-62Yk#u-^#aHrD~dW;x>Jzh_I zx*{CIza4k^;Sg0AX~ChtS){Ya!(lGJQF-z6=dX@xcD_~P)x1?1Q@gd+j9|X+A=Vk1 z4?q9Bsm*Mp%F<^9%}Y3871d3@F~4SC%w;e}8U?^vCK}MsfA~o#gPt(#;0<~UgML7F z^#xFYKZP%*w|B7nL#_^p+6jUm>b6IeiXopv902Q&5sQ=6%?ihY1$oG{ZpZUWN@JU0Vvk> zr}3Ete{#7_R{_`6uLaDQ03AKd5g_pQ3~U zxgpCc_U3eIhkU3H`tOw%NgOjy$g)^yLE&Pfm;pl;Y94b*sZd`k`QrW%%;h>e1&t6~ zgymj1nO{w4FV3l%_P(Fpl+%g%4I6xHT+0`zC(Ho0mEBA9X-NCioP|U&pp|u^HK!3} z@{CgdMG|Ni+=B#qCK-TMnW9^M3yrCG(I9Q241S!8fEsQlpWQN7)XOwiWa+qKF&@9k*aUE^IF4p%@}fM(~_5<5MEAmpWs^ z@1p;3&YvkZs(u>o+Lg9cH1sfo0u~j7^uEyVhBh$&(J^d0P9D$qQo+BxmTmb`%Vx_S zZZTh=#vZP|_wJ}kHyz>S^y#fhIuo<-%XVd@Vhfkqp)Ar<;T^Xj5JOPe&WE-8putgE zYMy)o3sWX|{+BsVJ7I{D9}K`BVJ|YU*wd%Ke`(mp411UGo@Nez?Na%t{qBZsB~fLT z;mrBxc++RBtZTj;f5Mad+(UVaf=LU6M41c_Mm#zr*GdLGG9oDV>+Zd~H^4z>0i9bl zBB}yIl7s06D>gT=KW;dp?h6j)Vdd#oM% zQJaW~N_T+LW=BJaJNzYRjE&M>pjBMAyh)@Q!_f=s^r?z--{S(FJIq2B2gS^>va(Dt z-f5ceBDQcry{rP<*YVKjDF`puz7cj&^|^9h;fI_|w}4zok7<&qEC&aFe_Jynqb&_I z{i_iXZwHc7RsJ@@{cG`E1;Py;eS%qnv?r>vjE-Y@Vvg&RT26B-OZ5T}o%JO*8`oNJ z;=pY{cNUd(+4X1ZTw4JtsKlQ_uykDwOh}RTQgyGnp7crlRsnVizAW}jCIlQ zdr`g}*SMceL=>WdN)x$BEj!Xo0m6M_UP^PKO2_0!j_q>TJ|IUam*=_DmH2YHOK%>89j*iuPBL0M4;L7RPFI0IQ3jo4=HMB>E`& zY#0n$eQ=EU+Bb_ZlhFkoJ-sg_JFx00jvx;7-+LfF^w;h-XeI4#dlF@5g@BDK4lYLg zY<)Xd?SAdW6Vs3Rtq3ak!V9kDfEDL{GLm$6zd8X|l(ia;aJ|4f3#N}slj-wI6RzeB zRGlL8$)xOcx&*n#@#XLmNcnLDzE-cUp9#8nAA%fAMphp&S`&vW+;o;9;N#up0qSG$ z`VR4>!-rrF^$Yv>VMxnhzL{2!>-WEGL)1uyOWmZstP(VWHnY${g!I=P z7MstN1WwVDQ!dRWtIE?H?zIc8UJ%wQLc9^PbbogL9uOPm$iYvTuTg;k3-=WzdTL$@ zYUxyhXdMxW`_ zmBF0ZfiezhqX6}WO6(4jrS!iciny+XM6LmXyGU=+c!Zhni_lmhOEJIRGMEUdasXLo zs;G5wKlSPj*wvIb`K36J6(>+c$|i#5FWy#K`^d+G1o?ON_Jauf;@zT=TUgZ?D-)Lr zPF>tPy$U!^fRWUk%9zYelo-awb3|&PrW=|i2=jq1WTt5}#ueRpWPO=53#J4D6W%$T zM8qC{R9oAAe@S&PMp21i`{N|cV)5$L-CNVCa$)F z%0^dn>og72B5=|w5!e6wY#9+9uY@4YfKs`GeXyzv?T->%I zYL0&;`nsu7oWd>Ez?7Ths8(7uL2I070f^3ka;0x~?q%WgL@A@#U z$3;TxJg13Fd+c`P+VEY1@5LqoiC*Nz5YpBdG$m94hNeI7e3ijR2Pvcpwaujpy8Reg zx-eAUOl{+pyZvEKME!n;mPSx5H1(sg)zx!&sA2vhb)jeuEF#JQoFetlHi?d_48h=@ z%SItaWb)2$!hDMo^e7xl>+vlDJB@qQ-KZ!1z+J7#oaLpo%lbLC3Ar8%vbMC^fQlI@8a}JKN04 zJ5_xmbMH(%|mWpLfcZPrv|)<^7-P>NzbjEd>5fQ zx*|s4Pzi(M`P*y8Qv)O2J!EH)^SQM#PWLN)@$iJc^_JnQP$>wOQZK|)j*Rue%nwC! zz=Fh-huHG=5L0DHJ5l+zDLub2MeSz9Va+5c3iL_A4J9W5*mHT4G2=9)_rpvzza%E} zX*n0N9ve4}nzAHDMOZjubX}&+gd{=C^jYmR>*l=t_<4NRETZU(5~bBBhz_Q7i7^mw z0{U_`h3HP*D+>ft&Jbuub14bTB6BY9DJ3A)i1gdvtuY5W&(d}-m- ztG_FXQSQV+D9WgXuR5jy03SxT-YfG#S7s$N;M zCHOL~jg`lLsW}ybrK}h03XIaH{N+@uENqgg8ocw3Ip~5kmP(4R?eDRIup@ym(&G6m z-$6P`(;g#6F+rvlPEm<7g=F~JDMmWM^`=Xhy4BC^FisgEVlHK51LAr5cbIV#igO*> zbzM)(%F$3o{@9p@t=qf<4GU~DUU!j|f6|q$=?nT&NWj8T39sE@2zSW4_9dEgtRH4J z$YzJ`2xS8T2z{2fmVULkBNe?8%|zn_=kR|ky^is|3f3}WdCF8EqFIib&ouT1ZNP^2 z((Vl7O8V_FHpt}qvzyr-X}Z5#jjK)u*X0^>&;3Iaj83n+dI#urDXq-#6t$JtOldwS zQQRxItHm^g(&XEO`@w9!bLeOp)3M6ubZ8s{Z45guNkRP`vOc`r(QATA$&k-hE(+=DQE1I) zTq~~!aVH(oc)R(g-@MoW!X0`Nt56!o!B7jPTt2c6u0Bv7w$^>FUDe<%P5*Biw(jA1 z{^*mZ7Arl3oE{%t|MDv8`zKs__sJ8F+wHSnB}kDq>6%a^DYHjiCg-$ zigw6P6{AHt`U-sSYb9Frhc0bG7qt<}@|cPFTD2Ab4B^Y;T`}AM6)gq)D7E2)*G2gK zRYX>~onM0GQ8m*)mMt=y$<>L6^`uM8Ur!)H6S#0PGzF_)nPNhOU_AL8&NWZq(yb2Q zC_*9>3nl7m)YC%f^0YWeB`J5#K@4?(`s{K39Op{$bxVy09yRtfKuS(a`b=IoAY|6g z#9S2U1eT;)@*4gq_&ea3$_p0sc6olCb_ZL01-eiAhWPBSfPCcWF8u_D>?p%DghA8i zRBQU#>+7{Zv2{HDWsa2MnG^@+>2p=^?S4yAEijGK1cb&hw)f2U=PQ#W>-N{*BRi3q z;526^)M8(w&k+av)*XZFb+k2|=x+;Jd!0MO=?X?e}`3>6L?Lp7{{6AnV+#v z5}Brs;D&q z(OfLd9QLp#{P+@$yO&=pOI;dPor>y}EWpS-Z~ZCxg^{!4^U7fs>`>}^nwb8OWbPN; zoqw#$DOQ>LxAJM$!nmD2?oO~!SCc1q`xQQqZ}44^i)(5$AOW@QW+#URc*l#)+a51X z)G!xu+bm2JcFg<2GRcJ)`QI+DP(UNVx|j^vt5w zPG1H2?eY`l6&aqClk7T9-b?o@4kmHQ>__rj>YLyeXJDG;O0aFfdb90$C*p^yRVo-w z`rRIo0tQbvIDWo83~+$oBmm|hPKcWXe(XS*OD{U(@+Z*vALmpVNQcIy%;D8kX4u6+>-C@UJ`A4 zaEj>_blEPqJ#dML)eW&1`X5@Nn9M=z$f2m6QWO|SI9N+%mpFRF24X~8u4RoiBfT7q zGT|)dN}j*r?&@n=(!iQ@5OhDGQEtJ@e{+IA1FpS&J_E>sV&M87FljZKx+atjztnvN zHg1j1oQ566;h1{lSAG8Iw@K}KqY8rUcj?ep)EjeqB+vUsdy=CgxSsokklP}^KWAtw z6XFT!?cjObaxR{n{WNWd*~Q2ZNjHAJSHtv_ODDMdzh?m@pBj-L#`3Ev5D%|ApZ_hO z9c&)Z4em?2bxadR3frIjs-HX@(VqulV0ZdGVKiJ<@oM2bJU5VMT(qJ9124=f7~1$R ziJ>cGXN=b9jouwA+0kDyLT;(UF6e-GKYYGx*c~L1)h+xUzml)+RGosgxX*~^KYrAi z*gwPrHyxSqP#%2{#3d2xqu_N78F9!3UAGRhuNL0W;DH%b659|ir=BCQG5$^ab>mbS zJCUBNNB9qaD0F=_v<^dhNimE1QE=U>6_?A=k5i+j)cY{8kvpOT7TqERk@ThjkkX+Y za?p`zmmtDA&jdjpw+<4^I5Rel_W zlEXnlqxK$M>&AbhEvG1+_?PW~M90K1`omSNX&aU)2xrapC?#N|fg4kmy=vG)Km*7l zx@V$MV7;amdVP(_oLE~pr!cSxPYicLZWb6RI4$2`$+NL^CsGi3n4c|XiitS!oDYrU z_{5c^mAURh)}Kx{m7m*#=w={y8gzUa-cYyyJhns1d) z61uiP@=_;XPggvy2#Q4hYxL!K1RpMxG}E4fn!D-Xe5#Vd0E|nrvwqYV_{Pa)T?rZR1iI9uP+ z>4FGl_QJ1BtZ4~GCKJyr?@vY$Hs)rj)}CMmn(~QwdH&(~s?Y&)iUZ1u?_8`7q;s?| zx;(S$kKiBf-3V~akb6tLkc~MZ6$#UqWDyX>$m^*CBpN_nveV!&qT419Zr6n3s^z7~ z>f9FF-fNw)gqgvyMF<1O;Y;agmnqzFrnAyC((Ogp8cBFs%@{yJnIRQ^ko1*nM}`g3 z2`SM4abI9Zd4w7`#s@E-rF>+>oPKux_blA~K$~weRO`oKC$lQnmNm_HL%80Pcde1Q zpIxSFw93yvY}8ntyVPm)R{D2OIWXxDvVXn^gF-w-M1$_5i_X+?@t(2ZAt9N!o8 ztB;h<83ofee|^QOhV>~sTb9pwcTU`m_{23fKyO@{a1=?`>x5(3gWa3O|E>`MzBB)r+0Z0Avkgo(GGk7DUyPEd*$szy(yqBPF8ufMmj5Q=E%L zY21VF6xz}n%dJ!bo5k4PQ$XRvPzEl>On{V!|`&>~-rPKpQp0&L>e)yf|y-)PMZw)Q%SdtLsERNRA}bLAy^Ca&{_kJ z;FCz)W{t_O2^>W+?gtzIM&7jHs+4Ys3V&C6Y~44~@}!bgl?M1f!8Y z0C2%Q6X`9#`_2Iw&yoUi5O>g(nc5$hG?ivjqc699QR&z8*Df&C>JCRWqZ4P$Bi zvT~=t8nVd@fN(FOcfU5hYk8$IYFE9_TDQGx<-EM**S%|1%>LJE*FQ5T>$d!V_5V55 zDK@w1pT&9AYPic)Q+eN3ew}MW34%E03xMdTzvGcO_FPubnQ`^g#`F8EEH4Va6LmdM z7nmAGbL9v3eZF6ntdeFyf(BfCmq#FZ2pfq>?TJH)4`y=9pO z+gNzlZ+TpNzbSfi`r_L|p%nbN9lpp{&?lMro8~|D^bii3%s*o8aAZ6m(Y#*c`_XY- zb?#i9CBr`VzrdEH#KyCm1m&sayRIa)54FElj?$VUbyMx`9b!pcge1&cW>#T9kVkig zc_254u53RGSGQqg8}Ef9*Bv82=XS{-J*4RQTe;ngBZ7b)vD+8pZ8fmwJ`7nETzuGKW#;#eBzw^+vihy zframM*&t)#~Tdq8( z^p5yCDtm+7{OImH+RcMZjX_^}E%t+^#RTC&aOrgY$zxmnENHU5x)|E1tvrb17`1w- z!#;bnfv~a96eh%q5ju;dzj@tiPK(|x%7DFxa?uHt-(xA;dm*P0alNQ$IJY-sLWSC9 zvK4<}YW9CIVh+Z|;XMg;9C0h_N)8T$8G1yyv9N3sV{J+dFxr7YXvoyKm-n;f4(+iS z-WExhuGQlt#JA^42&_E?JYeYh*)R&ROWDO*vLa zt*Osobw+=4i)v>!hM7jM3;Bk##p@|&OI^omwuw7}3e;g?K_cK=%`l|e>an5z?_ij3 zP_2L1cvWAxUtM`!e(3q~Xc?#^%@IAT$@vAiH*Kej?5TFQfGE|P8LJx%tla6?1K)|v ztP{-SXzG+!jMmzhXapYZmvsQYQ8GNotnyh)v^_OfTPoXr|7>-$tpJ`hg!BTX)I}lum4v{(%tOGK%J7 zR-;*4QoJ+g5q4K66u+ITw5_RfriA@0>YTh_Ib;qMn>%Q>4!lp6%U>jDjSE%qb;F4PZLJIVI;Yw; zn06ySYxXGBUneOU6NR0cdDW86?4a>g%PJ+;GpcMH|Hc0QgCEZnc7bYB7Xt*k2~)lk zyMOK2Ho=^Ho^FZVVDRqr-^b6ZxKu*dq9J>=KOFpfC+?^M%GiY!=nl3!ySUfF4)A)j zUnKdEQ~9Hx33Rcib$RCBFoz2+$lw^hn?eGj%koWjPA^D^&3T%KV*&?KH-k#3cEAtL zfu&7l&T^SjhTkAW8E%Z`@N;!Pd?~0ywo!S?@+JN5hiLvO(Gv8w_ zDg#R3@7D9*sb0siKlPz3gD-NJgkELGIHCRV?|X4&ujQ_Cg02sCQj@fQb;tIAApM!6&vu>ySlzdg-90_VJc zfJ;K=ExWt7rW!3T|YlzU}G5~ zp$X*lI>?p3P70|_k6*F#YE1qU4#g^U$#KYHvGN{DX3=9Oy(r5w7Fbq~? zbdFywhqRk8ESS;`jn<=@Um{%!HKycs(-XMhJ4tz+>K@CSu`IInF6vqauWRz3+kPzD zjXb+b`RZhYIN;kdpE)#0Q{X(BP0|3-8|E!Mhr@Ejvb4B==t??92E2Pc;NF{X_FVRm@)735H>P*QCf4ahaMg<(oDS z_Gb1{_JbOHRc(>7ZnKUWx)e7d8@m0`RFo*FS|w2Cyy)2=-Fl8>Mt2%Cl8E`Vl0$H7SQ zVw0Qg81V4^$S z6#VO7s@BY$X_3#lRhzkg&A7Slym5HdsqWLlWz-i{QANFm>?(%Mk<6UjXPkdYqo>PI zmZI{DiXo`Ul^a^&rwh(SD?n>2R%q;$xjA4yYDLcvn{@gv&1Dacnk9R^$8mShTCF{C zK3%d0vONc(tO~Czw9iO@&SQ<)C^C&}g_J2N&U_}9X=fWsL-xspJPdqPQ|js0xz)#v zbp(`k0T;%J1wTMcjmC?bXxpqoIJW%DpOy3IH+6s2iv*SWvs`C9D!B!2GqN;_Wsy%; z^^4}5taQ$!j!6%?s|FQ(21p=-WQv(@6L%TfAD;cTnT}f4%Z$_5Nm=IaUH#K2C?pZB zQq>Z(*~v&5n2I4Wa$8Iy<aF17u9gOj92on?Od`LfF$3s3<9v3!!g2@29BEiRgOrputNEle z4vyrYd$Qs@o~uGUfu0m)T`O|Nv~+x#*bcpZc0ODzll)A=zP2__km=~}g~~W#?KTW_ zs~3NedZ?B4Y+I$2`&U&~S+RWc{gx7^^TTkUoP|c~pKjJ#2JRK0ms;*T5 zM+Sq)m%*q7x!V}}r&2U;yP;5m1dY4T9= z8R5LpcV_I|6fr#R#Et%?o@q0dJ2W|SgxS$$y;h5B<;o};KHR3+dH$jJ>YLM!nl~&< zpkm4IO5ahm>g>qrF*c2KGWTb1l=a%Pl++g@1L#akh(ozyDvM%t@ZM-9z3JB5dNP&Z z2(gsE{9_mT+pP47NNW|6WjBv@Ez0K$X0Y-O`{3zxcYRQWW@KB6e{Ih}R}IVIEsxnu zJsw0e*<*-m?Xax)s*5F!CyRBaI|B_IbF^OZzx?KPeKi|taz=pXwifAyh&&d-4{rWy zZQMA?4FUOZ)Fl56fQuyB!uC8OK-LHANalI;P~*lDFekp-hlyN#~p zg}lYCQw}j}iJ6Hm0lKqRq3SP1>H;fizpDQPXh-=-_E}>qxYhpfGi3zrzbRoT zfaA)7$b$ou7Z;QsE+j)<*a|r({Bl+N?4ca5P4`|+{f4h}&sllkAT_X4=nT$`_JarZ zmWb@=XF1dZIB9QcTTkp&%vc46*?rJ-7K(wRj9^BA5Ad~qbGAhaZewnH zLteG`w(I@cfU9St->glq;*hVUG}yq~)bgKJ&f@>bcUEhdtBu9A#>womKTCO~VAO`` z{N|&%ovg+exM4G&+Uwsv`Eo_~ATE908&BA;QWS3oeM_w&d6w1%9MQzQFMys{^n(18 z&!sLIf-jF-P`y9&!_So??$RsS=Enq6v8m&@rd2B=a99-`Z+ZHWvelfTt`@b}TSAv^ zGB0g{k0N5|*3odQzbOPu?P z^?bFc&(HEXF*PZ*IzsYAfLc~@q>U)F9^h(oQthKP>M+Z&Xp%-nPZ8Tat$l{ zW7dn*Q2&*QM~2vE_;+2m80)nf<{O@*Sne>zguhQPtOR2$1`d1Cu8iJapui3@&16OWX{7x}ED0 z2agMWo|abIeAdRv)NNDT%#_BqghgBMuecdoo&W2d_}5#*@NZZh$F_vEeLl#I+UZwURTEz8<$%xrz&xG?B84S8!y*JNM>AG*xohq%N1T4M zotJE}2M&H>!Y<8=OcnJfDZpYbrG3TtMzTC1+M|M(hHDgZL1C=+CeT6xDp$ z6$*U!B+_ikMHKqx3DxG&`g8XStQ+j%5b{y0WS1?^eIBUnUOAOGnj4`yxIa#L4HG|) z5GK*Gf4|92bAq~MTqh&Z3x^)|nAtIbL{q6OXfe#r4G=QLEvO=GbAC;?^q0Rl!fYpS zt^;4=oyiwMc~E|I24y$z)$-1ne_2?pDH}vlt*^AvEcl+uujDF?G71jVvSnjrBKI^d z9_mmeb)u+FegD0*7+mjdMuwc)Ch=Y=V_UR>Blg a+RSv?4%<~f!T)_F1#>+HkwNT1Li~TS@Q=U% diff --git a/helm-charts/crds/deviceconfig-crd.yaml b/helm-charts/crds/deviceconfig-crd.yaml deleted file mode 100644 index 9daf1b725..000000000 --- a/helm-charts/crds/deviceconfig-crd.yaml +++ /dev/null @@ -1,798 +0,0 @@ ---- -# Source: gpu-operator-charts/templates/deviceconfig-crd.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: deviceconfigs.amd.com - annotations: - controller-gen.kubebuilder.io/version: v0.12.0 - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - helm.sh/chart: gpu-operator-charts-v1.2.0 - app.kubernetes.io/name: gpu-operator-charts - app.kubernetes.io/instance: amd-gpu - app.kubernetes.io/version: "v1.2.0" - app.kubernetes.io/managed-by: Helm -spec: - group: amd.com - names: - kind: DeviceConfig - listKind: DeviceConfigList - plural: deviceconfigs - shortNames: - - gpue - singular: deviceconfig - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: DeviceConfig describes how to enable AMD GPU device - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: DeviceConfigSpec describes how the AMD GPU operator should - enable AMD GPU device for customer's use. - properties: - commonConfig: - description: common config - properties: - initContainerImage: - description: InitContainerImage is being used for the operands pods, - i.e. metrics exporter, test runner, device plugin and node labeller - type: string - utilsContainer: - description: UtilsContainer contains parameters to configure operator's - utils container - properties: - image: - description: Image is the image of utils container - pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$ - type: string - imagePullPolicy: - description: image pull policy for utils container - enum: - - Always - - IfNotPresent - - Never - type: string - imageRegistrySecret: - description: secret used for pull utils container image - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - type: object - type: object - devicePlugin: - description: device plugin - properties: - devicePluginImage: - description: device plugin image - pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$ - type: string - devicePluginImagePullPolicy: - description: image pull policy for device plugin - enum: - - Always - - IfNotPresent - - Never - type: string - devicePluginTolerations: - description: tolerations for the device plugin DaemonSet - items: - description: The pod this Toleration is attached to tolerates - any taint that matches the triple using the - matching operator . - properties: - effect: - description: Effect indicates the taint effect to match. Empty - means match all taint effects. When specified, allowed values - are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Key is the taint key that the toleration applies - to. Empty means match all taint keys. If the key is empty, - operator must be Exists; this combination means to match - all values and all keys. - type: string - operator: - description: Operator represents a key's relationship to the - value. Valid operators are Exists and Equal. Defaults to - Equal. Exists is equivalent to wildcard for value, so that - a pod can tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: TolerationSeconds represents the period of time - the toleration (which must be of effect NoExecute, otherwise - this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do - not evict). Zero and negative values will be treated as - 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: Value is the taint value the toleration matches - to. If the operator is Exists, the value should be empty, - otherwise just a regular string. - type: string - type: object - type: array - enableNodeLabeller: - default: true - description: enable or disable the node labeller - type: boolean - imageRegistrySecret: - description: node labeller image registry secret used to pull/push - images - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - nodeLabellerImage: - description: node labeller image - pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$ - type: string - nodeLabellerImagePullPolicy: - description: image pull policy for node labeller - enum: - - Always - - IfNotPresent - - Never - type: string - nodeLabellerTolerations: - description: tolerations for the node labeller DaemonSet - items: - description: The pod this Toleration is attached to tolerates - any taint that matches the triple using the - matching operator . - properties: - effect: - description: Effect indicates the taint effect to match. Empty - means match all taint effects. When specified, allowed values - are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Key is the taint key that the toleration applies - to. Empty means match all taint keys. If the key is empty, - operator must be Exists; this combination means to match - all values and all keys. - type: string - operator: - description: Operator represents a key's relationship to the - value. Valid operators are Exists and Equal. Defaults to - Equal. Exists is equivalent to wildcard for value, so that - a pod can tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: TolerationSeconds represents the period of time - the toleration (which must be of effect NoExecute, otherwise - this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do - not evict). Zero and negative values will be treated as - 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: Value is the taint value the toleration matches - to. If the operator is Exists, the value should be empty, - otherwise just a regular string. - type: string - type: object - type: array - upgradePolicy: - description: upgrade policy for device plugin and node labeller - daemons - properties: - maxUnavailable: - default: 1 - description: MaxUnavailable specifies the maximum number of - Pods that can be unavailable during the update process. Applicable - for RollingUpdate only. Default value is 1. - format: int32 - type: integer - upgradeStrategy: - description: UpgradeStrategy specifies the type of the DaemonSet - update. Valid values are "RollingUpdate" (default) or "OnDelete". - enum: - - RollingUpdate - - OnDelete - type: string - type: object - type: object - driver: - description: driver - properties: - amdgpuInstallerRepoURL: - description: radeon repo URL for fetching amdgpu installer if building - driver image on the fly installer URL is https://repo.radeon.com/amdgpu-install - by default - type: string - blacklist: - description: blacklist amdgpu drivers on the host - type: boolean - enable: - default: true - description: enable driver install. default value is true. disable - is for skipping driver install/uninstall for dryrun or using in-tree - amdgpu kernel module - type: boolean - image: - description: defines image that includes drivers and firmware blobs, - don't include tag since it will be fully managed by operator for - vanilla k8s the default value is image-registry:5000/$MOD_NAMESPACE/amdgpu_kmod - for OpenShift the default value is image-registry.openshift-image-registry.svc:5000/$MOD_NAMESPACE/amdgpu_kmod - image tag will be in the format of --- example tag is coreos-416.94-5.14.0-427.28.1.el9_4.x86_64-6.2.2 - and ubuntu-22.04-5.15.0-94-generic-6.1.3 - pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$ - type: string - imageRegistrySecret: - description: secrets used for pull/push images from/to private registry - specified in driversImage - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - imageRegistryTLS: - description: driver image registry TLS setting for the container - image - properties: - insecure: - description: If true, check if the container image already exists - using plain HTTP. - type: boolean - insecureSkipTLSVerify: - description: If true, skip any TLS server certificate validation - type: boolean - type: object - imageSign: - description: image signing config to sign the driver image when - building driver image on the fly image signing is required for - installing driver on secure boot enabled system - properties: - certSecret: - description: ImageSignCertSecret the public key used to sign - kernel modules within image necessary for secure boot enabled - system - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - keySecret: - description: ImageSignKeySecret the private key used to sign - kernel modules within image necessary for secure boot enabled - system - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - type: object - upgradePolicy: - description: policy to upgrade the drivers - properties: - enable: - description: enable upgrade policy, disabled by default If disabled, - user has to manually upgrade all the nodes. - type: boolean - maxParallelUpgrades: - default: 1 - description: MaxParallelUpgrades indicates how many nodes can - be upgraded in parallel 0 means no limit, all nodes will be - upgraded in parallel - minimum: 0 - type: integer - maxUnavailableNodes: - anyOf: - - type: integer - - type: string - default: 25% - description: 'MaxUnavailableNodes indicates maximum number of - nodes that can be in a failed upgrade state beyond which upgrades - will stop to keep cluster at a minimal healthy state Value - can be an integer (ex: 2) which would mean atmost 2 nodes - can be in failed state after which new upgrades will not start. - Or it can be a percentage string(ex: "50%") from which absolute - number will be calculated and round up' - x-kubernetes-int-or-string: true - nodeDrainPolicy: - description: Node draining policy - properties: - force: - default: false - description: Force indicates if force draining is allowed - type: boolean - timeoutSeconds: - default: 300 - description: TimeoutSecond specifies the length of time - in seconds to wait before giving up drain, zero means - infinite - minimum: 0 - type: integer - type: object - podDeletionPolicy: - description: Pod Deletion policy. If both NodeDrainPolicy and - PodDeletionPolicy config is available, NodeDrainPolicy(if - enabled) will take precedence. - properties: - force: - default: false - description: Force indicates if force deletion is allowed - type: boolean - timeoutSeconds: - default: 300 - description: TimeoutSecond specifies the length of time - in seconds to wait before giving up on pod deletion, zero - means infinite - minimum: 0 - type: integer - type: object - rebootRequired: - description: reboot between driver upgrades, disabled by default, - if enabled spec.commonConfig.utilsContainer will be used to - perform reboot on worker nodes - type: boolean - type: object - version: - description: 'version of the drivers source code, can be used as - part of image of dockerfile source image default value for different - OS is: ubuntu: 6.1.3, coreOS: 6.2.2' - type: string - type: object - metricsExporter: - description: metrics exporter - properties: - config: - description: optional configuration for metrics - properties: - name: - description: Name of the configMap that defines the list of - metrics default list:[] - type: string - type: object - enable: - description: enable metrics exporter, disabled by default - type: boolean - image: - description: metrics exporter image - pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$ - type: string - imagePullPolicy: - description: image pull policy for metrics exporter - enum: - - Always - - IfNotPresent - - Never - type: string - imageRegistrySecret: - description: metrics exporter image registry secret used to pull/push - images - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - nodePort: - description: NodePort is the external port for pulling metrics from - outside the cluster, in the range 30000-32767 (assigned automatically - by default) - format: int32 - maximum: 32767 - minimum: 30000 - type: integer - port: - default: 5000 - description: Port is the internal port used for in-cluster and node - access to pull metrics from the metrics-exporter (default 5000). - format: int32 - type: integer - rbacConfig: - description: optional kube-rbac-proxy config to provide rbac services - properties: - disableHttps: - description: disable https protecting the proxy endpoint - type: boolean - enable: - description: enable kube-rbac-proxy, disabled by default - type: boolean - image: - description: kube-rbac-proxy image - pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$ - type: string - secret: - description: certificate secret to mount in kube-rbac container - for TLS, self signed certificates will be generated by default - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - type: object - selector: - additionalProperties: - type: string - description: Selector describes on which nodes to enable metrics - exporter - type: object - serviceType: - default: ClusterIP - description: ServiceType service type for metrics, clusterIP/NodePort, - clusterIP by default - enum: - - ClusterIP - - NodePort - type: string - tolerations: - description: tolerations for metrics exporter - items: - description: The pod this Toleration is attached to tolerates - any taint that matches the triple using the - matching operator . - properties: - effect: - description: Effect indicates the taint effect to match. Empty - means match all taint effects. When specified, allowed values - are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Key is the taint key that the toleration applies - to. Empty means match all taint keys. If the key is empty, - operator must be Exists; this combination means to match - all values and all keys. - type: string - operator: - description: Operator represents a key's relationship to the - value. Valid operators are Exists and Equal. Defaults to - Equal. Exists is equivalent to wildcard for value, so that - a pod can tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: TolerationSeconds represents the period of time - the toleration (which must be of effect NoExecute, otherwise - this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do - not evict). Zero and negative values will be treated as - 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: Value is the taint value the toleration matches - to. If the operator is Exists, the value should be empty, - otherwise just a regular string. - type: string - type: object - type: array - upgradePolicy: - description: upgrade policy for metrics exporter daemons - properties: - maxUnavailable: - default: 1 - description: MaxUnavailable specifies the maximum number of - Pods that can be unavailable during the update process. Applicable - for RollingUpdate only. Default value is 1. - format: int32 - type: integer - upgradeStrategy: - description: UpgradeStrategy specifies the type of the DaemonSet - update. Valid values are "RollingUpdate" (default) or "OnDelete". - enum: - - RollingUpdate - - OnDelete - type: string - type: object - type: object - selector: - additionalProperties: - type: string - description: Selector describes on which nodes the GPU Operator should - enable the GPU device. - type: object - testRunner: - description: test runner - properties: - config: - description: config map to customize the config for test runner, - if not specified default test config will be aplied - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - enable: - description: enable test runner, disabled by default - type: boolean - image: - description: test runner image - pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$ - type: string - imagePullPolicy: - description: image pull policy for test runner - enum: - - Always - - IfNotPresent - - Never - type: string - imageRegistrySecret: - description: test runner image registry secret used to pull/push - images - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - x-kubernetes-map-type: atomic - logsLocation: - description: mount config for test runner logs - properties: - hostPath: - default: /var/log/amd-test-runner - description: host path to store test runner internal status - db in order to persist test running status - type: string - mountPath: - default: /var/log/amd-test-runner - description: volume mount destination within test runner container - type: string - type: object - selector: - additionalProperties: - type: string - description: Selector describes on which nodes to enable test runner - type: object - tolerations: - description: tolerations for test runner - items: - description: The pod this Toleration is attached to tolerates - any taint that matches the triple using the - matching operator . - properties: - effect: - description: Effect indicates the taint effect to match. Empty - means match all taint effects. When specified, allowed values - are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Key is the taint key that the toleration applies - to. Empty means match all taint keys. If the key is empty, - operator must be Exists; this combination means to match - all values and all keys. - type: string - operator: - description: Operator represents a key's relationship to the - value. Valid operators are Exists and Equal. Defaults to - Equal. Exists is equivalent to wildcard for value, so that - a pod can tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: TolerationSeconds represents the period of time - the toleration (which must be of effect NoExecute, otherwise - this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do - not evict). Zero and negative values will be treated as - 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: Value is the taint value the toleration matches - to. If the operator is Exists, the value should be empty, - otherwise just a regular string. - type: string - type: object - type: array - upgradePolicy: - description: upgrade policy for test runner daemonset - properties: - maxUnavailable: - default: 1 - description: MaxUnavailable specifies the maximum number of - Pods that can be unavailable during the update process. Applicable - for RollingUpdate only. Default value is 1. - format: int32 - type: integer - upgradeStrategy: - description: UpgradeStrategy specifies the type of the DaemonSet - update. Valid values are "RollingUpdate" (default) or "OnDelete". - enum: - - RollingUpdate - - OnDelete - type: string - type: object - type: object - type: object - status: - description: DeviceConfigStatus defines the observed state of Module. - properties: - conditions: - description: Conditions list the current status of the DeviceConfig - object - items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a foo's - current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating details - about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers of - specific condition types may define expected values and meanings - for this field, and whether the values are considered a guaranteed - API. The value should be a CamelCase string. This field may - not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - devicePlugin: - description: DevicePlugin contains the status of the Device Plugin deployment - properties: - availableNumber: - description: number of the actually deployed and running pods - format: int32 - type: integer - desiredNumber: - description: number of the pods that should be deployed for daemonset - format: int32 - type: integer - nodesMatchingSelectorNumber: - description: number of nodes that are targeted by the DeviceConfig - selector - format: int32 - type: integer - type: object - driver: - description: Driver contains the status of the Drivers deployment - properties: - availableNumber: - description: number of the actually deployed and running pods - format: int32 - type: integer - desiredNumber: - description: number of the pods that should be deployed for daemonset - format: int32 - type: integer - nodesMatchingSelectorNumber: - description: number of nodes that are targeted by the DeviceConfig - selector - format: int32 - type: integer - type: object - metricsExporter: - description: MetricsExporter contains the status of the MetricsExporter - deployment - properties: - availableNumber: - description: number of the actually deployed and running pods - format: int32 - type: integer - desiredNumber: - description: number of the pods that should be deployed for daemonset - format: int32 - type: integer - nodesMatchingSelectorNumber: - description: number of nodes that are targeted by the DeviceConfig - selector - format: int32 - type: integer - type: object - nodeModuleStatus: - additionalProperties: - description: ModuleStatus contains the status of driver module installed - by operator on the node - properties: - containerImage: - type: string - kernelVersion: - type: string - lastTransitionTime: - type: string - status: - description: UpgradeState captures the state of the upgrade process - on a node - type: string - upgradeStartTime: - type: string - type: object - description: NodeModuleStatus contains per node status of driver module - installation - type: object - observedGeneration: - description: ObservedGeneration is the latest spec generation successfully - processed by the controller - format: int64 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/helm-charts/templates/_helpers.tpl b/helm-charts/templates/_helpers.tpl deleted file mode 100644 index 10e24643a..000000000 --- a/helm-charts/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "helm-charts-k8s.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "helm-charts-k8s.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "helm-charts-k8s.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "helm-charts-k8s.labels" -}} -helm.sh/chart: {{ include "helm-charts-k8s.chart" . }} -{{ include "helm-charts-k8s.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "helm-charts-k8s.selectorLabels" -}} -app.kubernetes.io/name: {{ include "helm-charts-k8s.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "helm-charts-k8s.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "helm-charts-k8s.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/helm-charts/templates/deployment.yaml b/helm-charts/templates/deployment.yaml deleted file mode 100644 index 6397c246d..000000000 --- a/helm-charts/templates/deployment.yaml +++ /dev/null @@ -1,83 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-controller-manager - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - control-plane: controller-manager - {{- include "helm-charts-k8s.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.controllerManager.replicas }} - selector: - matchLabels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - control-plane: controller-manager - {{- include "helm-charts-k8s.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - control-plane: controller-manager - {{- include "helm-charts-k8s.selectorLabels" . | nindent 8 }} - annotations: - kubectl.kubernetes.io/default-container: manager - spec: - {{- with .Values.controllerManager.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - nodeSelector: {{- toYaml .Values.controllerManager.nodeSelector | nindent 8 }} - containers: - - args: {{- toYaml .Values.controllerManager.manager.args | nindent 8 }} - env: - - name: OPERATOR_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: KUBERNETES_CLUSTER_DOMAIN - value: {{ quote .Values.kubernetesClusterDomain }} - - name: SIM_ENABLE - value: {{ quote .Values.controllerManager.env.simEnable }} - image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag - | default .Chart.AppVersion }} - imagePullPolicy: {{ .Values.controllerManager.manager.imagePullPolicy }} - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: {{- toYaml .Values.controllerManager.manager.resources | nindent 10 - }} - securityContext: {{- toYaml .Values.controllerManager.manager.containerSecurityContext - | nindent 10 }} - volumeMounts: - - mountPath: /controller_manager_config.yaml - name: manager-config - subPath: controller_manager_config.yaml - {{- if .Values.controllerManager.manager.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.controllerManager.manager.imagePullSecrets }} - {{- end}} - securityContext: - runAsNonRoot: true - serviceAccountName: {{ include "helm-charts-k8s.fullname" . }}-controller-manager - terminationGracePeriodSeconds: 10 - {{- with .Values.controllerManager.manager.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - volumes: - - configMap: - name: {{ include "helm-charts-k8s.fullname" . }}-manager-config - name: manager-config diff --git a/helm-charts/templates/event-recorder-clusterrole-rbac.yaml b/helm-charts/templates/event-recorder-clusterrole-rbac.yaml deleted file mode 100644 index c2e2b41ce..000000000 --- a/helm-charts/templates/event-recorder-clusterrole-rbac.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-event-recorder-clusterrole - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch \ No newline at end of file diff --git a/helm-charts/templates/event-recorder-clusterrolebinding-rbac.yaml b/helm-charts/templates/event-recorder-clusterrolebinding-rbac.yaml deleted file mode 100644 index ffc1c0640..000000000 --- a/helm-charts/templates/event-recorder-clusterrolebinding-rbac.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-event-recorder-clusterrolebinding - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-k8s.fullname" . }}-event-recorder-clusterrole' -subjects: -- kind: ServiceAccount - name: '{{ include "helm-charts-k8s.fullname" . }}-controller-manager' - namespace: '{{ .Release.Namespace }}' \ No newline at end of file diff --git a/helm-charts/templates/leader-election-rbac.yaml b/helm-charts/templates/leader-election-rbac.yaml deleted file mode 100644 index b852c7576..000000000 --- a/helm-charts/templates/leader-election-rbac.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-leader-election-role - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-leader-election-rolebinding - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: '{{ include "helm-charts-k8s.fullname" . }}-leader-election-role' -subjects: -- kind: ServiceAccount - name: '{{ include "helm-charts-k8s.fullname" . }}-controller-manager' - namespace: '{{ .Release.Namespace }}' \ No newline at end of file diff --git a/helm-charts/templates/manager-config.yaml b/helm-charts/templates/manager-config.yaml deleted file mode 100644 index 316644687..000000000 --- a/helm-charts/templates/manager-config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-manager-config - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} -data: - controller_manager_config.yaml: {{ .Values.managerConfig.controllerManagerConfigYaml - | toYaml | indent 1 }} \ No newline at end of file diff --git a/helm-charts/templates/manager-rbac.yaml b/helm-charts/templates/manager-rbac.yaml deleted file mode 100644 index 07b5c11f7..000000000 --- a/helm-charts/templates/manager-rbac.yaml +++ /dev/null @@ -1,263 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-manager-role - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} -rules: -- apiGroups: - - amd.com - resources: - - deviceconfigs - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - amd.com - resources: - - deviceconfigs/finalizers - verbs: - - update -- apiGroups: - - amd.com - resources: - - deviceconfigs/status - verbs: - - get - - patch - - update -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - delete -- apiGroups: - - apps - resources: - - daemonsets - verbs: - - create - - delete - - get - - list - - patch - - watch -- apiGroups: - - apps - resources: - - daemonsets/finalizers - verbs: - - create - - get - - update - - watch -- apiGroups: - - apps - resources: - - daemonsets/status - verbs: - - create - - delete - - get - - list - - patch - - watch -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - delete - - get - - list - - patch - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - patch - - watch -- apiGroups: - - "" - resources: - - nodes/finalizers - verbs: - - get - - update - - watch -- apiGroups: - - "" - resources: - - nodes/status - verbs: - - get - - update - - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - create - - delete - - get - - list - - watch -- apiGroups: - - "" - resources: - - pods/eviction - verbs: - - create - - delete - - get - - list -- apiGroups: - - "" - resources: - - pods/finalizers - verbs: - - delete - - get - - list - - watch -- apiGroups: - - "" - resources: - - pods/status - verbs: - - delete - - get - - list - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - create - - delete - - get - - list - - patch - - watch -- apiGroups: - - "" - resources: - - services/finalizers - verbs: - - create - - get - - update - - watch -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - modules - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - modules/finalizers - verbs: - - get - - update - - watch -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - modules/status - verbs: - - get - - patch - - update -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - nodemodulesconfigs - verbs: - - get - - list - - watch -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - nodemodulesconfigs/finalizers - verbs: - - get - - update - - watch -- apiGroups: - - kmm.sigs.x-k8s.io - resources: - - nodemodulesconfigs/status - verbs: - - get - - list - - watch -- apiGroups: - - nfd.openshift.io - resources: - - nodefeaturediscoveries - verbs: - - delete - - get - - list -- apiGroups: - - nfd.openshift.io - resources: - - nodefeaturediscoveries/finalizers - verbs: - - get - - update -- apiGroups: - - nfd.openshift.io - resources: - - nodefeaturediscoveries/status - verbs: - - get - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-manager-rolebinding - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-k8s.fullname" . }}-manager-role' -subjects: -- kind: ServiceAccount - name: '{{ include "helm-charts-k8s.fullname" . }}-controller-manager' - namespace: '{{ .Release.Namespace }}' \ No newline at end of file diff --git a/helm-charts/templates/metrics-exporter-rbac-proxy-rbac.yaml b/helm-charts/templates/metrics-exporter-rbac-proxy-rbac.yaml deleted file mode 100644 index a16700783..000000000 --- a/helm-charts/templates/metrics-exporter-rbac-proxy-rbac.yaml +++ /dev/null @@ -1,47 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-metrics-exporter-rbac-proxy - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create -- apiGroups: - - "" - resources: - - nodes - verbs: - - watch - - get - - list - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-metrics-exporter-rbac-proxy - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-k8s.fullname" . }}-metrics-exporter-rbac-proxy' -subjects: -- kind: ServiceAccount - name: amd-gpu-operator-metrics-exporter-rbac-proxy - namespace: '{{ .Release.Namespace }}' diff --git a/helm-charts/templates/metrics-exporter-rbac.yaml b/helm-charts/templates/metrics-exporter-rbac.yaml deleted file mode 100644 index 65c59b820..000000000 --- a/helm-charts/templates/metrics-exporter-rbac.yaml +++ /dev/null @@ -1,35 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-metrics-exporter - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - watch - - get - - list - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-metrics-exporter - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-k8s.fullname" . }}-metrics-exporter' -subjects: -- kind: ServiceAccount - name: amd-gpu-operator-metrics-exporter - namespace: '{{ .Release.Namespace }}' diff --git a/helm-charts/templates/nfd-default-rule.yaml b/helm-charts/templates/nfd-default-rule.yaml deleted file mode 100644 index e2794fe39..000000000 --- a/helm-charts/templates/nfd-default-rule.yaml +++ /dev/null @@ -1,50 +0,0 @@ -{{- if .Values.installdefaultNFDRule }} -apiVersion: nfd.k8s-sigs.io/v1alpha1 -kind: NodeFeatureRule -metadata: - name: amd-gpu-label-nfd-rule - # the PCI info is from these websites: - # source1: https://admin.pci-ids.ucw.cz/read/PC/1002 - # source2: https://devicehunt.com/view/type/pci/vendor/1002 -spec: - rules: - - name: amd-gpu - labels: - feature.node.kubernetes.io/amd-gpu: "true" - matchAny: - - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["1002"]} - device: {op: In, value: ["74a0"]} # MI300A - - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["1002"]} - device: {op: In, value: ["74a1"]} # MI300X - - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["1002"]} - device: {op: In, value: ["740f"]} # MI210 - - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["1002"]} - device: {op: In, value: ["7408"]} # MI250X - - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["1002"]} - device: {op: In, value: ["740c"]} # MI250/MI250X - - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["1002"]} - device: {op: In, value: ["738c"]} # MI100 - - matchFeatures: - - feature: pci.device - matchExpressions: - vendor: {op: In, value: ["1002"]} - device: {op: In, value: ["738e"]} # MI100 -{{- end }} \ No newline at end of file diff --git a/helm-charts/templates/node-labeller-rbac.yaml b/helm-charts/templates/node-labeller-rbac.yaml deleted file mode 100644 index 7802407a0..000000000 --- a/helm-charts/templates/node-labeller-rbac.yaml +++ /dev/null @@ -1,35 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-node-labeller - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - watch - - get - - list - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-node-labeller - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-k8s.fullname" . }}-node-labeller' -subjects: -- kind: ServiceAccount - name: amd-gpu-operator-node-labeller - namespace: '{{ .Release.Namespace }}' diff --git a/helm-charts/templates/post-delete-hook.yaml b/helm-charts/templates/post-delete-hook.yaml deleted file mode 100644 index ad54a95e6..000000000 --- a/helm-charts/templates/post-delete-hook.yaml +++ /dev/null @@ -1,117 +0,0 @@ -# Run helm uninstall with --no-hooks to bypass the post-delete hook -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-prune - namespace: {{ .Release.Namespace }} - labels: - {{- include "helm-charts-k8s.labels" . | nindent 4 }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "0" - "helm.sh/hook": post-delete - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-prune - labels: - {{- include "helm-charts-k8s.labels" . | nindent 4 }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "0" - "helm.sh/hook": post-delete - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded -rules: - - apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - delete - - get - - list ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-prune - labels: - {{- include "helm-charts-k8s.labels" . | nindent 4 }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "1" - "helm.sh/hook": post-delete - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "helm-charts-k8s.fullname" . }}-prune -subjects: -- kind: ServiceAccount - name: {{ include "helm-charts-k8s.fullname" . }}-prune - namespace: {{ .Release.Namespace }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: delete-custom-resource-definitions - namespace: {{ .Release.Namespace }} - labels: - {{- include "helm-charts-k8s.labels" . | nindent 4 }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "2" - # hook will be executed after helm uninstall - "helm.sh/hook": post-delete - # remove the resource created by the hook whether it succeeded or failed - "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded -spec: - backoffLimit: 0 # once the job finished first run, don't retry to create another pod - ttlSecondsAfterFinished: 60 # job info will be kept for 1 min then deleted - template: - spec: - serviceAccountName: {{ include "helm-charts-k8s.fullname" . }}-prune - containers: - - name: delete-custom-resource-definitions - image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag }} - command: - - /bin/sh - - -c - - | - if kubectl get crds deviceconfigs.amd.com > /dev/null 2>&1; then - kubectl delete crds deviceconfigs.amd.com - fi - {{- if index .Values "node-feature-discovery" "enabled" }} - if kubectl get crds nodefeaturegroups.nfd.k8s-sigs.io > /dev/null 2>&1; then - kubectl delete crds nodefeaturegroups.nfd.k8s-sigs.io - fi - if kubectl get crds nodefeaturerules.nfd.k8s-sigs.io > /dev/null 2>&1; then - kubectl delete crds nodefeaturerules.nfd.k8s-sigs.io - fi - if kubectl get crds nodefeatures.nfd.k8s-sigs.io > /dev/null 2>&1; then - kubectl delete crds nodefeatures.nfd.k8s-sigs.io - fi - {{- end }} - {{- if .Values.kmm.enabled }} - if kubectl get crds modules.kmm.sigs.x-k8s.io > /dev/null 2>&1; then - kubectl delete crds modules.kmm.sigs.x-k8s.io - fi - if kubectl get crds nodemodulesconfigs.kmm.sigs.x-k8s.io > /dev/null 2>&1; then - kubectl delete crds nodemodulesconfigs.kmm.sigs.x-k8s.io - fi - {{- end }} - {{- if .Values.controllerManager.manager.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.controllerManager.manager.imagePullSecrets }} - {{- end }} - {{- with .Values.controllerManager.manager.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.controllerManager.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - restartPolicy: Never diff --git a/helm-charts/templates/pre-delete-hook.yaml b/helm-charts/templates/pre-delete-hook.yaml deleted file mode 100644 index 191d4d3ac..000000000 --- a/helm-charts/templates/pre-delete-hook.yaml +++ /dev/null @@ -1,101 +0,0 @@ -# Run helm uninstall with --no-hooks to bypass the pre-delete hook -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-pre-delete - namespace: {{ .Release.Namespace }} - labels: - {{- include "helm-charts-k8s.labels" . | nindent 4 }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "0" - "helm.sh/hook": pre-delete - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-pre-delete - labels: - {{- include "helm-charts-k8s.labels" . | nindent 4 }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "0" - "helm.sh/hook": pre-delete - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded -rules: - - apiGroups: - - amd.com - resources: - - deviceconfigs - verbs: - - get - - list ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-pre-delete - labels: - {{- include "helm-charts-k8s.labels" . | nindent 4 }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "1" - "helm.sh/hook": pre-delete - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "helm-charts-k8s.fullname" . }}-pre-delete -subjects: -- kind: ServiceAccount - name: {{ include "helm-charts-k8s.fullname" . }}-pre-delete - namespace: {{ .Release.Namespace }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: check-leftover-deviceconfigs - namespace: {{ .Release.Namespace }} - labels: - {{- include "helm-charts-k8s.labels" . | nindent 4 }} - annotations: - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "2" - # hook will be executed before helm uninstall - "helm.sh/hook": pre-delete - # remove the resource created by the hook whether it succeeded or failed - "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded -spec: - backoffLimit: 0 # once the job finished first run, don't retry to create another pod - ttlSecondsAfterFinished: 60 # job info will be kept for 1 min then deleted - template: - spec: - serviceAccountName: {{ include "helm-charts-k8s.fullname" . }}-pre-delete - containers: - - name: check-leftover-deviceconfigs - image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag }} - command: - - /bin/sh - - -c - - | - if kubectl get deviceconfigs -n {{ .Release.Namespace }} --no-headers | grep -q .; then - echo "DeviceConfigs resources exist. Stop uninstallation." - exit 1 - else - echo "No DeviceConfigs resources found. Proceeding with uninstallation." - exit 0 - fi - {{- if .Values.controllerManager.manager.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.controllerManager.manager.imagePullSecrets }} - {{- end}} - {{- with .Values.controllerManager.manager.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.controllerManager.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - restartPolicy: Never diff --git a/helm-charts/templates/pre-upgrade-hook.yaml b/helm-charts/templates/pre-upgrade-hook.yaml deleted file mode 100644 index d62f59757..000000000 --- a/helm-charts/templates/pre-upgrade-hook.yaml +++ /dev/null @@ -1,168 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: pre-upgrade-check - namespace: {{ .Release.Namespace }} - labels: - {{- include "helm-charts-k8s.labels" . | nindent 4 }} - annotations: - # hook will be executed before helm upgrade - "helm.sh/hook": pre-upgrade,pre-rollback - # don't cleanup the job on hook failure - "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "0" -spec: - backoffLimit: 0 # once the job finished first run, don't retry to create another pod - ttlSecondsAfterFinished: 60 # job info will be kept for 1 min then deleted - template: - spec: - serviceAccountName: {{ include "helm-charts-k8s.fullname" . }}-controller-manager - containers: - - name: pre-upgrade-check - image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag }} - command: - - /bin/sh - - -c - - | - # List all DeviceConfig CRs - deviceconfigs=$(kubectl get deviceconfigs -n {{ .Release.Namespace }} -o json) - - echo "DeviceConfigs JSON:" - echo "$deviceconfigs" | jq . - - # Check if any UpgradeState is in the blocked states - blocked_states='["Upgrade-Not-Started", "Upgrade-Started", "Install-In-Progress", "Upgrade-In-Progress"]' - if echo "$deviceconfigs" | jq --argjson blocked_states "$blocked_states" -e ' - .items[] | - .status.nodeModuleStatus // {} | - to_entries | - any(.value.status as $state | ($blocked_states | index($state)))' > /dev/null; then - echo "Upgrade blocked: Some DeviceConfigs are in a disallowed UpgradeState." - exit 1 - else - echo "All DeviceConfigs are in an allowed state. Proceeding with upgrade." - exit 0 - fi - {{- if .Values.controllerManager.manager.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.controllerManager.manager.imagePullSecrets }} - {{- end }} - {{- with .Values.controllerManager.manager.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.controllerManager.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - restartPolicy: Never -{{- if .Values.upgradeCRD }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: upgrade-crd-hook-sa - annotations: - # hook will be executed before helm upgrade - "helm.sh/hook": pre-upgrade,pre-rollback - # don't cleanup the job on hook failure - "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "1" ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: upgrade-crd-hook-cluster-role - annotations: - # hook will be executed before helm upgrade - "helm.sh/hook": pre-upgrade,pre-rollback - # don't cleanup the job on hook failure - "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "1" -rules: - - apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - create - - get - - list - - watch - - patch - - update ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: upgrade-crd-hook-cluster-role-binding - annotations: - # hook will be executed before helm upgrade - "helm.sh/hook": pre-upgrade,pre-rollback - # don't cleanup the job on hook failure - "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "2" -subjects: - - kind: ServiceAccount - name: upgrade-crd-hook-sa - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: upgrade-crd-hook-cluster-role - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: upgrade-crd - namespace: {{ .Release.Namespace }} - labels: - {{- include "helm-charts-k8s.labels" . | nindent 4 }} - annotations: - # hook will be executed before helm upgrade - "helm.sh/hook": pre-upgrade,pre-rollback - # don't cleanup the job on hook failure - "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded - # hook with lower weight value will run firstly - "helm.sh/hook-weight": "3" -spec: - template: - metadata: - name: upgrade-crd - spec: - serviceAccountName: upgrade-crd-hook-sa - {{- if .Values.controllerManager.manager.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.controllerManager.manager.imagePullSecrets }} - {{- end }} - {{- with .Values.controllerManager.manager.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.controllerManager.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: upgrade-crd - image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag }} - imagePullPolicy: {{ .Values.controllerManager.manager.imagePullPolicy }} - command: - - /bin/sh - - -c - - | - kubectl apply -f /opt/helm-charts-crds-k8s/deviceconfig-crd.yaml - {{- if index .Values "node-feature-discovery" "enabled" }} - kubectl apply -f /opt/helm-charts-crds-k8s/nfd-api-crds.yaml - {{- end }} - {{- if .Values.kmm.enabled }} - kubectl apply -f /opt/helm-charts-crds-k8s/module-crd.yaml - kubectl apply -f /opt/helm-charts-crds-k8s/nodemodulesconfig-crd.yaml - {{- end }} - restartPolicy: OnFailure -{{- end }} -# Run helm upgrade with --no-hooks to bypass the pre-upgrade hook \ No newline at end of file diff --git a/helm-charts/templates/serviceaccount.yaml b/helm-charts/templates/serviceaccount.yaml deleted file mode 100644 index 6bf5489b7..000000000 --- a/helm-charts/templates/serviceaccount.yaml +++ /dev/null @@ -1,76 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-controller-manager - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} - annotations: - {{- toYaml .Values.controllerManager.serviceAccount.annotations | nindent 4 }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: amd-gpu-operator-kmm-device-plugin - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} - annotations: - {{- toYaml .Values.kmmDevicePlugin.serviceAccount.annotations | nindent 4 }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: amd-gpu-operator-kmm-module-loader - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} - annotations: - {{- toYaml .Values.kmmModuleLoader.serviceAccount.annotations | nindent 4 }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: amd-gpu-operator-node-labeller - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} - annotations: - {{- toYaml .Values.nodeLabeller.serviceAccount.annotations | nindent 4 }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: amd-gpu-operator-metrics-exporter - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} - annotations: - {{- toYaml .Values.metricsExporter.serviceAccount.annotations | nindent 4 }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: amd-gpu-operator-metrics-exporter-rbac-proxy - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} - annotations: - {{- toYaml .Values.metricsExporter.serviceAccount.annotations | nindent 4 }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: amd-gpu-operator-test-runner - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} - annotations: - {{- toYaml .Values.testRunner.serviceAccount.annotations | nindent 4 }} diff --git a/helm-charts/templates/test-runner-rbac.yaml b/helm-charts/templates/test-runner-rbac.yaml deleted file mode 100644 index 21e7b39ec..000000000 --- a/helm-charts/templates/test-runner-rbac.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-test-runner - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - get - - list - - update -- apiGroups: - - "" - resources: - - nodes - verbs: - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "helm-charts-k8s.fullname" . }}-test-runner - labels: - app.kubernetes.io/component: amd-gpu - app.kubernetes.io/part-of: amd-gpu - {{- include "helm-charts-k8s.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "helm-charts-k8s.fullname" . }}-test-runner' -subjects: -- kind: ServiceAccount - name: amd-gpu-operator-test-runner - namespace: '{{ .Release.Namespace }}' diff --git a/helm-charts/values.yaml b/helm-charts/values.yaml deleted file mode 100644 index 0bf34ff12..000000000 --- a/helm-charts/values.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# NFD related configs -node-feature-discovery: - # -- Set to true/false to enable/disable the installation of node feature discovery (NFD) operator - enabled: true - -# KMM related configs -kmm: - # -- Set to true/false to enable/disable the installation of kernel module management (KMM) operator - enabled: true - -# -- Default NFD rule will detect amd gpu based on pci vendor ID -installdefaultNFDRule: true - -# -- CRD will be patched as pre-upgrade/pre-rollback hook when doing helm upgrade/rollback to current helm chart -upgradeCRD: true - -# AMD GPU operator controller related configs -controllerManager: - manager: - args: - - --config=controller_manager_config.yaml - containerSecurityContext: - allowPrivilegeEscalation: false - image: - # -- AMD GPU operator controller manager image repository - repository: docker.io/rocm/gpu-operator - # -- AMD GPU operator controller manager image tag - tag: v1.2.0 - # -- Image pull policy for AMD GPU operator controller manager pod - imagePullPolicy: Always - # -- Image pull secret name for pulling AMD GPU operator controller manager image if registry needs credential to pull image - imagePullSecrets: "" - tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Equal" - value: "" - effect: "NoSchedule" - - key: "node-role.kubernetes.io/control-plane" - operator: "Equal" - value: "" - effect: "NoSchedule" - resources: - limits: - cpu: 500m - memory: 384Mi - requests: - cpu: 10m - memory: 64Mi - # -- Node selector for AMD GPU operator controller manager deployment - nodeSelector: {} - # -- Deployment affinity configs for controller manager - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - replicas: 1 - serviceAccount: - annotations: {} - env: - simEnable: false -kmmDevicePlugin: - serviceAccount: - annotations: {} -kmmModuleLoader: - serviceAccount: - annotations: {} -kubernetesClusterDomain: cluster.local -managerConfig: - controllerManagerConfigYaml: |- - healthProbeBindAddress: :8081 - metricsBindAddress: 127.0.0.1:8080 - leaderElection: - enabled: true - resourceID: gpu.amd.com -metricsService: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: https - type: ClusterIP -nodeLabeller: - serviceAccount: - annotations: {} -metricsExporter: - serviceAccount: - annotations: {} -testRunner: - serviceAccount: - annotations: {} -global: - proxy: - env: {}