[Spark] Support spark-operator on multi-namespace deployments

charts/mlrun-ce/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 name: mlrun-ce
-version: 0.11.0-rc8
+version: 0.11.0-rc9
 description: MLRun Open Source Stack
 home: https://iguazio.com
 icon: https://www.iguazio.com/wp-content/uploads/2019/10/Iguazio-Logo.png

charts/mlrun-ce/admin_installation_values.yaml

@@ -40,7 +40,29 @@ minio:
   enabled: false
 
 spark-operator:
+  enabled: true
+  fullnameOverride: spark-operator
+  controller:
+    replicas: 0           # No running pods in admin
+    rbac:
+      create: true        # Creates ClusterRole (shared by all user namespaces)
+    serviceAccount:
+      create: true
+  webhook:
+    enable: true
+    replicas: 1
+  spark:
+    jobNamespaces:
+      - ""              # All namespaces (no namespaceSelector on webhook)
+    serviceAccount:
+      create: false       # No sparkapp SA in admin
+    rbac:
+      create: false
+
+spark:
   enabled: false
+  rbac:
+    enabled: false
 
 pipelines:
   enabled: false

charts/mlrun-ce/non_admin_cluster_ip_installation_values.yaml

@@ -53,7 +53,32 @@ timescaledb:
     nodePort: ""
 
 spark-operator:
-  enabled: false
+  enabled: true
+  fullnameOverride: spark-operator
+  controller:
+    replicas: 1
+    rbac:
+      create: false
+    serviceAccount:
+      create: true
+    leaderElection:
+      enable: true
+  webhook:
+    enable: false
+  spark:
+    jobNamespaces:
+      - mlrun
+    serviceAccount:
+      create: true
+      name: sparkapp
+    rbac:
+      create: true
+
+spark:
+  enabled: true
+  rbac:
+    enabled: true
+    serviceAccountName: sparkapp
 
 pipelines:
   service:

charts/mlrun-ce/non_admin_installation_values.yaml

@@ -47,7 +47,32 @@ minio:
   replicas: 1
 
 spark-operator:
-  enabled: false
+  enabled: true
+  fullnameOverride: spark-operator
+  controller:
+    replicas: 1           # Controller runs in user namespace
+    rbac:
+      create: false       # ClusterRole already exists from admin
+    serviceAccount:
+      create: true
+    leaderElection:
+      enable: true
+  webhook:
+    enable: false
+  spark:
+    jobNamespaces:
+      - mlrun             # Override with actual namespace at install time
+    serviceAccount:
+      create: true
+      name: sparkapp
+    rbac:
+      create: true        # Creates sparkapp Role + RoleBinding
+
+spark:
+  enabled: true
+  rbac:
+    enabled: true
+    serviceAccountName: sparkapp
 
 pipelines:
   service:

charts/mlrun-ce/templates/config/mlrun-spark-config.yaml

@@ -1,4 +1,4 @@
-{{- if index .Values "spark-operator" "enabled" -}}
+{{- if .Values.spark.enabled -}}
 apiVersion: v1
 kind: ConfigMap
 metadata:

charts/mlrun-ce/templates/spark-operator/spark-controller-rbac.yaml

@@ -0,0 +1,61 @@
+{{- if and (index .Values "spark-operator" "enabled") (not (index .Values "spark-operator" "controller" "rbac" "create")) -}}
+{{- /*
+  This template renders only in user multi-NS mode:
+  - spark-operator subchart is enabled (controller Deployment runs here)
+  - controller.rbac.create is false (ClusterRole already exists from admin namespace)
+
+  It creates:
+  1. RoleBinding: controller SA → shared ClusterRole (namespace-scoped access)
+  2. Role + RoleBinding: leader election leases (coordination.k8s.io)
+*/ -}}
+---
+# RoleBinding: Grant controller SA access to the shared ClusterRole (namespace-scoped)
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: spark-operator-controller
+  labels:
+    app.kubernetes.io/name: mlrun-ce
+    app.kubernetes.io/component: spark-controller-rbac
+    app.kubernetes.io/managed-by: {{ .Release.Name }}
+subjects:
+  - kind: ServiceAccount
+    name: spark-operator-controller
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: spark-operator-controller
+  apiGroup: rbac.authorization.k8s.io
+---
+# Role: Leader election leases
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: spark-operator-controller-leases
+  labels:
+    app.kubernetes.io/name: mlrun-ce
+    app.kubernetes.io/component: spark-controller-rbac
+    app.kubernetes.io/managed-by: {{ .Release.Name }}
+rules:
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["create", "get", "update"]
+---
+# RoleBinding: Grant controller SA access to leader election leases
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: spark-operator-controller-leases
+  labels:
+    app.kubernetes.io/name: mlrun-ce
+    app.kubernetes.io/component: spark-controller-rbac
+    app.kubernetes.io/managed-by: {{ .Release.Name }}
+subjects:
+  - kind: ServiceAccount
+    name: spark-operator-controller
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: Role
+  name: spark-operator-controller-leases
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}

charts/mlrun-ce/values.yaml

@@ -566,3 +566,14 @@ kafka:
     # Empty means "use the release namespace"
     # Example: "controller" if that's where you installed the operator
     operatorNamespace: ""
+
+# Spark configuration for multi-NS deployments
+# Controls CE-level spark resources (mlrun-spark-config ConfigMap)
+# In single-NS mode, both spark.enabled and spark-operator.enabled are true
+# In multi-NS admin mode, spark.enabled is false (no ConfigMap needed)
+# In multi-NS user mode, spark.enabled is true (ConfigMap needed for MLRun)
+spark:
+  enabled: true
+  rbac:
+    enabled: true
+    serviceAccountName: sparkapp