From ae52e57be215938283ee6be03eb031d93f9ecdca Mon Sep 17 00:00:00 2001 From: nxtcoder17 Date: Tue, 25 Mar 2025 08:47:12 +0530 Subject: [PATCH 01/27] chore: updates event details in reconcile filter --- toolkit/reconciler/event-predicate.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/toolkit/reconciler/event-predicate.go b/toolkit/reconciler/event-predicate.go index 7cde91d2..a8a14ffb 100644 --- a/toolkit/reconciler/event-predicate.go +++ b/toolkit/reconciler/event-predicate.go @@ -85,7 +85,7 @@ func ReconcileFilter(eventRecorder ...record.EventRecorder) predicate.Funcs { annHasChanged := false for k, v := range oldAnn { - if k != LastAppliedKey { + if k != LastAppliedKey && k != "deployment.kubernetes.io/revision" { if v != newAnn[k] { annHasChanged = true break @@ -117,11 +117,12 @@ func ReconcileFilter(eventRecorder ...record.EventRecorder) predicate.Funcs { if oldRes.Status.IsReady == nil || newRes.Status.IsReady == nil { // INFO: it means this resource is not a kloudlite resource, in that case, // it should just be always allowed, as it can be a pod or a job, that some kloudlite resource is watching over + fireEvent(newObj, ReasonStatusIsReadyChanged, "resource isReady is nil") return true } if *oldRes.Status.IsReady != *newRes.Status.IsReady { - fireEvent(newObj, ReasonStatusIsReadyChanged, fmt.Sprintf("resource isReady changed from (%v) to (%v)", newRes.Status.IsReady, oldRes.Status.IsReady)) + fireEvent(newObj, ReasonStatusIsReadyChanged, fmt.Sprintf("resource isReady changed from (%v) to (%v)", *newRes.Status.IsReady, *oldRes.Status.IsReady)) return true } From 33abde44fe0349177bc71692abff5e06201deb54 Mon Sep 17 00:00:00 2001 From: nxtcoder17 Date: Fri, 28 Mar 2025 09:32:20 +0530 Subject: [PATCH 02/27] WIP: workspaces and workmachines --- PROJECT | 22 +- apis/crds/v1/workmachine_types.go | 93 +++++++ apis/crds/v1/workspace_types.go | 82 ++++++ apis/crds/v1/zz_generated.deepcopy.go | 191 ++++++++++++++ cmd/agent-operator/main.go | 2 + cmd/main.go | 3 +- cmd/platform-operator/main.go | 2 + .../bases/crds.kloudlite.io_workmachines.yaml | 182 +++++++++++++ .../bases/crds.kloudlite.io_workspaces.yaml | 248 ++++++++++++++++++ config/crd/kustomization.yaml | 4 + config/rbac/crds_workmachine_editor_role.yaml | 27 ++ config/rbac/crds_workmachine_viewer_role.yaml | 23 ++ config/rbac/crds_workspace_editor_role.yaml | 27 ++ config/rbac/crds_workspace_viewer_role.yaml | 23 ++ config/rbac/kustomization.yaml | 9 + config/samples/crds_v1_workmachine.yaml | 9 + config/samples/crds_v1_workspace.yaml | 9 + config/samples/kustomization.yaml | 2 + go.sum | 14 - operators/wireguard/examples/device.yaml | 2 +- operators/wireguard/main.go | 4 +- .../controllers/workmachine/controller.go | 139 ++++++++++ operators/workmachine/internal/env/env.go | 25 ++ .../workmachine/internal/templates/embed.go | 23 ++ .../workmachine/internal/templates/types.go | 33 +++ .../internal/templates/workspace.yml.tpl | 229 ++++++++++++++++ operators/workmachine/main.go | 24 ++ operators/workmachine/register/register.go | 16 ++ operators/workspace/examples/sample.yml | 24 ++ .../controllers/workspace/controller.go | 207 +++++++++++++++ operators/workspace/internal/env/env.go | 25 ++ .../workspace/internal/templates/embed.go | 23 ++ .../workspace/internal/templates/types.go | 33 +++ .../internal/templates/workspace.yml.tpl | 229 ++++++++++++++++ operators/workspace/main.go | 18 ++ operators/workspace/register/register.go | 16 ++ toolkit/kubectl/yaml-client.go | 59 +++-- toolkit/reconciler/checks.go | 1 - toolkit/reconciler/event-predicate.go | 4 +- 39 files changed, 2061 insertions(+), 45 deletions(-) create mode 100644 apis/crds/v1/workmachine_types.go create mode 100644 apis/crds/v1/workspace_types.go create mode 100644 config/crd/bases/crds.kloudlite.io_workmachines.yaml create mode 100644 config/crd/bases/crds.kloudlite.io_workspaces.yaml create mode 100644 config/rbac/crds_workmachine_editor_role.yaml create mode 100644 config/rbac/crds_workmachine_viewer_role.yaml create mode 100644 config/rbac/crds_workspace_editor_role.yaml create mode 100644 config/rbac/crds_workspace_viewer_role.yaml create mode 100644 config/samples/crds_v1_workmachine.yaml create mode 100644 config/samples/crds_v1_workspace.yaml create mode 100644 operators/workmachine/internal/controllers/workmachine/controller.go create mode 100644 operators/workmachine/internal/env/env.go create mode 100644 operators/workmachine/internal/templates/embed.go create mode 100644 operators/workmachine/internal/templates/types.go create mode 100644 operators/workmachine/internal/templates/workspace.yml.tpl create mode 100644 operators/workmachine/main.go create mode 100644 operators/workmachine/register/register.go create mode 100644 operators/workspace/examples/sample.yml create mode 100644 operators/workspace/internal/controllers/workspace/controller.go create mode 100644 operators/workspace/internal/env/env.go create mode 100644 operators/workspace/internal/templates/embed.go create mode 100644 operators/workspace/internal/templates/types.go create mode 100644 operators/workspace/internal/templates/workspace.yml.tpl create mode 100644 operators/workspace/main.go create mode 100644 operators/workspace/register/register.go diff --git a/PROJECT b/PROJECT index f89a4a64..ab70d857 100644 --- a/PROJECT +++ b/PROJECT @@ -4,7 +4,7 @@ # More info: https://book.kubebuilder.io/reference/project-config.html domain: kloudlite.io layout: -- go.kubebuilder.io/v3 +- go.kubebuilder.io/v4 multigroup: true plugins: manifests.sdk.operatorframework.io/v2: {} @@ -515,8 +515,28 @@ resources: domain: kloudlite.io group: crds kind: ServiceIntercept + version: "" +- api: + crdVersion: v1 + namespaced: true group: mongodb.msvc kind: Backup path: github.com/kloudlite/operator/apis/mongodb.msvc/v1 version: v1 +- api: + crdVersion: v1 + namespaced: true + domain: kloudlite.io + group: crds + kind: Workspace + path: github.com/kloudlite/operator/api/crds/v1 + version: v1 +- api: + crdVersion: v1 + namespaced: true + domain: kloudlite.io + group: crds + kind: WorkMachine + path: github.com/kloudlite/operator/api/crds/v1 + version: v1 version: "3" diff --git a/apis/crds/v1/workmachine_types.go b/apis/crds/v1/workmachine_types.go new file mode 100644 index 00000000..18123d92 --- /dev/null +++ b/apis/crds/v1/workmachine_types.go @@ -0,0 +1,93 @@ +package v1 + +import ( + rApi "github.com/kloudlite/operator/toolkit/reconciler" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type AWSMachineConfig struct { + Region string `json:"region" graphql:"noinput"` + AvailabilityZone string `json:"availabilityZone"` + + AMI string `json:"ami"` + InstanceType string `json:"instanceType"` + + PublicSubnetID string `json:"publicSubnetId" graphql:"noinput"` + SecurityGroupID string `json:"SecurityGroupID" graphql:"noinput"` + + RootVolumeType string `json:"rootVolumeType" graphql:"noinput"` + RootVolumeSize int `json:"rootVolumeSize" graphql:"noinput"` + + ExternalVolumeType string `json:"externalVolumeType" graphql:"noinput"` + ExternalVolumeSize string `json:"externalVolumeSize"` + + IAMInstanceProfileRole *string `json:"iamInstanceProfileRole,omitempty" graphql:"noinput"` +} + +// +kubebuilder:validation:Enum=ON;OFF; +// +kubebuilder:default=ON +type WorkMachineState string + +const ( + WorkMachineStateOn WorkMachineState = "ON" + WorkMachineStateOff WorkMachineState = "OFF" +) + +// WorkMachineSpec defines the desired state of WorkMachine +type WorkMachineSpec struct { + State WorkMachineState `json:"state"` + + SSHPublicKeys []string `json:"sshPublicKeys"` + + AWSMachineConfig `json:"aws"` +} + +type WorkMachineStatus struct { + rApi.Status `json:"status,omitempty"` + MachinePulicSSHKey string `json:"machineSSHKey,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status + +// WorkMachine is the Schema for the workmachines API +type WorkMachine struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec WorkMachineSpec `json:"spec,omitempty"` + Status WorkMachineStatus `json:"status,omitempty"` +} + +func (r *WorkMachine) EnsureGVK() { + if r != nil { + r.SetGroupVersionKind(GroupVersion.WithKind("Workspace")) + } +} + +func (w *WorkMachine) GetStatus() *rApi.Status { + return &w.Status.Status +} + +func (w *WorkMachine) GetEnsuredLabels() map[string]string { + return map[string]string{ + "kloudlite.io/workmachine.name": w.Name, + } +} + +func (w *WorkMachine) GetEnsuredAnnotations() map[string]string { + return map[string]string{} +} + +// +kubebuilder:object:root=true + +// WorkMachineList contains a list of WorkMachine +type WorkMachineList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []WorkMachine `json:"items"` +} + +func init() { + SchemeBuilder.Register(&WorkMachine{}, &WorkMachineList{}) +} diff --git a/apis/crds/v1/workspace_types.go b/apis/crds/v1/workspace_types.go new file mode 100644 index 00000000..a6addaf3 --- /dev/null +++ b/apis/crds/v1/workspace_types.go @@ -0,0 +1,82 @@ +package v1 + +import ( + "github.com/kloudlite/operator/toolkit/reconciler" + rApi "github.com/kloudlite/operator/toolkit/reconciler" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +kubebuilder:validation:Enum=ON;OFF; +// +kubebuilder:default=ON +type WorkspaceState string + +const ( + WorkspaceStateOn WorkspaceState = "ON" + WorkspaceStateOff WorkspaceState = "OFF" +) + +// WorkspaceSpec defines the desired state of Workspace +type WorkspaceSpec struct { + State WorkspaceState `json:"state"` + NodeName string `json:"nodeName"` + ServiceAccountName string `json:"serviceAccountName"` + + EnableTTYD bool `json:"enableTTYD,omitempty"` + EnableJupyterNotebook bool `json:"enableJupyterNotebook,omitempty"` + EnableCodeServer bool `json:"enableCodeServer,omitempty"` + EnableVSCodeServer bool `json:"enableVSCodeServer,omitempty"` + + // +kubebuilder:default=IfNotPresent + ImagePullPolicy string `json:"imagePullPolicy"` + + Router RouterSpec `json:"router"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:JSONPath=".status.lastReconcileTime",name=Seen,type=date +// +kubebuilder:printcolumn:JSONPath=".metadata.annotations.kloudlite\\.io\\/checks",name=Checks,type=string +// +kubebuilder:printcolumn:JSONPath=".metadata.annotations.kloudlite\\.io\\/resource\\.ready",name=Ready,type=string +// +kubebuilder:printcolumn:JSONPath=".metadata.creationTimestamp",name=Age,type=date + +// Workspace is the Schema for the workspaces API +type Workspace struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec WorkspaceSpec `json:"spec,omitempty"` + Status rApi.Status `json:"status,omitempty"` +} + +func (r *Workspace) EnsureGVK() { + if r != nil { + r.SetGroupVersionKind(GroupVersion.WithKind("Workspace")) + } +} + +func (w *Workspace) GetStatus() *reconciler.Status { + return &w.Status +} + +func (w *Workspace) GetEnsuredLabels() map[string]string { + return map[string]string{ + "kloudlite.io/workspace.name": w.Name, + } +} + +func (w *Workspace) GetEnsuredAnnotations() map[string]string { + return map[string]string{} +} + +// +kubebuilder:object:root=true + +// WorkspaceList contains a list of Workspace +type WorkspaceList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Workspace `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Workspace{}, &WorkspaceList{}) +} diff --git a/apis/crds/v1/zz_generated.deepcopy.go b/apis/crds/v1/zz_generated.deepcopy.go index 45ea7b0c..e981a728 100644 --- a/apis/crds/v1/zz_generated.deepcopy.go +++ b/apis/crds/v1/zz_generated.deepcopy.go @@ -11,6 +11,26 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AWSMachineConfig) DeepCopyInto(out *AWSMachineConfig) { + *out = *in + if in.IAMInstanceProfileRole != nil { + in, out := &in.IAMInstanceProfileRole, &out.IAMInstanceProfileRole + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSMachineConfig. +func (in *AWSMachineConfig) DeepCopy() *AWSMachineConfig { + if in == nil { + return nil + } + out := new(AWSMachineConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Account) DeepCopyInto(out *Account) { *out = *in @@ -1951,3 +1971,174 @@ func (in *TcpProbe) DeepCopy() *TcpProbe { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkMachine) DeepCopyInto(out *WorkMachine) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkMachine. +func (in *WorkMachine) DeepCopy() *WorkMachine { + if in == nil { + return nil + } + out := new(WorkMachine) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *WorkMachine) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkMachineList) DeepCopyInto(out *WorkMachineList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]WorkMachine, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkMachineList. +func (in *WorkMachineList) DeepCopy() *WorkMachineList { + if in == nil { + return nil + } + out := new(WorkMachineList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *WorkMachineList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkMachineSpec) DeepCopyInto(out *WorkMachineSpec) { + *out = *in + if in.SSHPublicKeys != nil { + in, out := &in.SSHPublicKeys, &out.SSHPublicKeys + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.AWSMachineConfig.DeepCopyInto(&out.AWSMachineConfig) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkMachineSpec. +func (in *WorkMachineSpec) DeepCopy() *WorkMachineSpec { + if in == nil { + return nil + } + out := new(WorkMachineSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkMachineStatus) DeepCopyInto(out *WorkMachineStatus) { + *out = *in + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkMachineStatus. +func (in *WorkMachineStatus) DeepCopy() *WorkMachineStatus { + if in == nil { + return nil + } + out := new(WorkMachineStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Workspace) DeepCopyInto(out *Workspace) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Workspace. +func (in *Workspace) DeepCopy() *Workspace { + if in == nil { + return nil + } + out := new(Workspace) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Workspace) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkspaceList) DeepCopyInto(out *WorkspaceList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Workspace, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkspaceList. +func (in *WorkspaceList) DeepCopy() *WorkspaceList { + if in == nil { + return nil + } + out := new(WorkspaceList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *WorkspaceList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkspaceSpec) DeepCopyInto(out *WorkspaceSpec) { + *out = *in + in.Router.DeepCopyInto(&out.Router) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkspaceSpec. +func (in *WorkspaceSpec) DeepCopy() *WorkspaceSpec { + if in == nil { + return nil + } + out := new(WorkspaceSpec) + in.DeepCopyInto(out) + return out +} diff --git a/cmd/agent-operator/main.go b/cmd/agent-operator/main.go index fea5ad51..1c721406 100644 --- a/cmd/agent-operator/main.go +++ b/cmd/agent-operator/main.go @@ -15,6 +15,7 @@ import ( // routers "github.com/kloudlite/operator/operators/routers/controller" serviceIntercept "github.com/kloudlite/operator/operators/service-intercept/controller" + workspace "github.com/kloudlite/operator/operators/workspace/register" pluginHelmChart "github.com/kloudlite/plugin-helm-chart/kloudlite" pluginMongoDB "github.com/kloudlite/plugin-mongodb/kloudlite" ) @@ -46,6 +47,7 @@ func main() { networkingv1.RegisterInto(mgr) serviceIntercept.RegisterInto(mgr) + workspace.RegisterInto(mgr) pluginMongoDB.RegisterInto(mgr) pluginHelmChart.RegisterInto(mgr) diff --git a/cmd/main.go b/cmd/main.go index 1a67a122..a11f2506 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -10,9 +10,10 @@ import ( "strings" "text/template" + "github.com/urfave/cli/v2" + "github.com/kloudlite/operator/pkg/errors" "github.com/kloudlite/operator/pkg/templates" - "github.com/urfave/cli/v2" ) var ( diff --git a/cmd/platform-operator/main.go b/cmd/platform-operator/main.go index 3aa528f4..ee644d43 100644 --- a/cmd/platform-operator/main.go +++ b/cmd/platform-operator/main.go @@ -22,6 +22,7 @@ import ( // wireguard "github.com/kloudlite/operator/operators/wireguard/controller" serviceIntercept "github.com/kloudlite/operator/operators/service-intercept/controller" + workspace "github.com/kloudlite/operator/operators/workspace/register" pluginMongoDB "github.com/kloudlite/plugin-mongodb/kloudlite" ) @@ -52,6 +53,7 @@ func main() { // networkingv0.RegisterInto(mgr) // MIGRATE pluginMongoDB.RegisterInto(mgr) + workspace.RegisterInto(mgr) mgr.Start() } diff --git a/config/crd/bases/crds.kloudlite.io_workmachines.yaml b/config/crd/bases/crds.kloudlite.io_workmachines.yaml new file mode 100644 index 00000000..c7656c37 --- /dev/null +++ b/config/crd/bases/crds.kloudlite.io_workmachines.yaml @@ -0,0 +1,182 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: workmachines.crds.kloudlite.io +spec: + group: crds.kloudlite.io + names: + kind: WorkMachine + listKind: WorkMachineList + plural: workmachines + singular: workmachine + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: WorkMachine is the Schema for the workmachines API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: WorkMachineSpec defines the desired state of WorkMachine + properties: + aws: + properties: + SecurityGroupID: + type: string + ami: + type: string + availabilityZone: + type: string + externalVolumeSize: + type: string + externalVolumeType: + type: string + iamInstanceProfileRole: + type: string + instanceType: + type: string + publicSubnetId: + type: string + region: + type: string + rootVolumeSize: + type: integer + rootVolumeType: + type: string + required: + - SecurityGroupID + - ami + - availabilityZone + - externalVolumeSize + - externalVolumeType + - instanceType + - publicSubnetId + - region + - rootVolumeSize + - rootVolumeType + type: object + sshPublicKeys: + items: + type: string + type: array + state: + enum: + - "ON" + - "OFF" + type: string + required: + - aws + - sshPublicKeys + - state + type: object + status: + properties: + machineSSHKey: + type: string + status: + properties: + checkList: + items: + properties: + debug: + type: boolean + description: + type: string + hide: + type: boolean + name: + type: string + title: + type: string + required: + - name + - title + type: object + type: array + checks: + additionalProperties: + properties: + debug: + type: string + error: + type: string + generation: + format: int64 + type: integer + info: + type: string + message: + type: string + startedAt: + format: date-time + type: string + state: + type: string + status: + type: boolean + required: + - status + type: object + type: object + isReady: + type: boolean + lastReadyGeneration: + format: int64 + type: integer + lastReconcileTime: + format: date-time + type: string + resources: + items: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + type: array + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/crds.kloudlite.io_workspaces.yaml b/config/crd/bases/crds.kloudlite.io_workspaces.yaml new file mode 100644 index 00000000..9134b15d --- /dev/null +++ b/config/crd/bases/crds.kloudlite.io_workspaces.yaml @@ -0,0 +1,248 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: workspaces.crds.kloudlite.io +spec: + group: crds.kloudlite.io + names: + kind: Workspace + listKind: WorkspaceList + plural: workspaces + singular: workspace + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.lastReconcileTime + name: Seen + type: date + - jsonPath: .metadata.annotations.kloudlite\.io\/checks + name: Checks + type: string + - jsonPath: .metadata.annotations.kloudlite\.io\/resource\.ready + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: Workspace is the Schema for the workspaces API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: WorkspaceSpec defines the desired state of Workspace + properties: + enableCodeServer: + type: boolean + enableJupyterNotebook: + type: boolean + enableTTYD: + type: boolean + enableVSCodeServer: + type: boolean + imagePullPolicy: + default: IfNotPresent + type: string + nodeName: + type: string + router: + description: RouterSpec defines the desired state of Router + properties: + backendProtocol: + type: string + basicAuth: + properties: + enabled: + type: boolean + secretName: + type: string + username: + type: string + required: + - enabled + type: object + cors: + properties: + allowCredentials: + type: boolean + enabled: + default: false + type: boolean + origins: + items: + type: string + type: array + type: object + domains: + items: + type: string + type: array + https: + properties: + clusterIssuer: + type: string + enabled: + default: true + type: boolean + forceRedirect: + type: boolean + required: + - enabled + type: object + ingressClass: + type: string + maxBodySizeInMB: + type: integer + rateLimit: + properties: + connections: + type: integer + enabled: + type: boolean + rpm: + type: integer + rps: + type: integer + type: object + routes: + items: + properties: + app: + type: string + path: + description: Lambda string `json:"lambda,omitempty"` + type: string + port: + type: integer + rewrite: + default: false + type: boolean + required: + - app + - path + - port + type: object + type: array + required: + - domains + type: object + serviceAccountName: + type: string + state: + enum: + - "ON" + - "OFF" + type: string + required: + - imagePullPolicy + - nodeName + - router + - serviceAccountName + - state + type: object + status: + properties: + checkList: + items: + properties: + debug: + type: boolean + description: + type: string + hide: + type: boolean + name: + type: string + title: + type: string + required: + - name + - title + type: object + type: array + checks: + additionalProperties: + properties: + debug: + type: string + error: + type: string + generation: + format: int64 + type: integer + info: + type: string + message: + type: string + startedAt: + format: date-time + type: string + state: + type: string + status: + type: boolean + required: + - status + type: object + type: object + isReady: + type: boolean + lastReadyGeneration: + format: int64 + type: integer + lastReconcileTime: + format: date-time + type: string + resources: + items: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 82d06489..f394152e 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -58,6 +58,8 @@ resources: - bases/mongodb.msvc.kloudlite.io_standalonedatabases.yaml - bases/crds.kloudlite.io_serviceintercepts.yaml - bases/mongodb.msvc.kloudlite.io_backups.yaml +- bases/crds.kloudlite.io_workspaces.yaml +- bases/crds.kloudlite.io_workmachines.yaml # +kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -148,6 +150,8 @@ patchesStrategicMerge: #- patches/cainjection_in_serviceintercepts.yaml #- patches/cainjection_in_backups.yaml #- patches/cainjection_in_managedserviceplugins.yaml +#- path: patches/cainjection_in_crds_workspaces.yaml +#- path: patches/cainjection_in_crds_workmachines.yaml # +kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/rbac/crds_workmachine_editor_role.yaml b/config/rbac/crds_workmachine_editor_role.yaml new file mode 100644 index 00000000..10fcfb0d --- /dev/null +++ b/config/rbac/crds_workmachine_editor_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to edit workmachines. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: app + app.kubernetes.io/managed-by: kustomize + name: crds-workmachine-editor-role +rules: +- apiGroups: + - crds.kloudlite.io + resources: + - workmachines + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - crds.kloudlite.io + resources: + - workmachines/status + verbs: + - get diff --git a/config/rbac/crds_workmachine_viewer_role.yaml b/config/rbac/crds_workmachine_viewer_role.yaml new file mode 100644 index 00000000..f3d86662 --- /dev/null +++ b/config/rbac/crds_workmachine_viewer_role.yaml @@ -0,0 +1,23 @@ +# permissions for end users to view workmachines. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: app + app.kubernetes.io/managed-by: kustomize + name: crds-workmachine-viewer-role +rules: +- apiGroups: + - crds.kloudlite.io + resources: + - workmachines + verbs: + - get + - list + - watch +- apiGroups: + - crds.kloudlite.io + resources: + - workmachines/status + verbs: + - get diff --git a/config/rbac/crds_workspace_editor_role.yaml b/config/rbac/crds_workspace_editor_role.yaml new file mode 100644 index 00000000..12baea99 --- /dev/null +++ b/config/rbac/crds_workspace_editor_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to edit workspaces. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: app + app.kubernetes.io/managed-by: kustomize + name: crds-workspace-editor-role +rules: +- apiGroups: + - crds.kloudlite.io + resources: + - workspaces + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - crds.kloudlite.io + resources: + - workspaces/status + verbs: + - get diff --git a/config/rbac/crds_workspace_viewer_role.yaml b/config/rbac/crds_workspace_viewer_role.yaml new file mode 100644 index 00000000..92e422a2 --- /dev/null +++ b/config/rbac/crds_workspace_viewer_role.yaml @@ -0,0 +1,23 @@ +# permissions for end users to view workspaces. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: app + app.kubernetes.io/managed-by: kustomize + name: crds-workspace-viewer-role +rules: +- apiGroups: + - crds.kloudlite.io + resources: + - workspaces + verbs: + - get + - list + - watch +- apiGroups: + - crds.kloudlite.io + resources: + - workspaces/status + verbs: + - get diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 731832a6..285c3bc5 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -16,3 +16,12 @@ resources: - auth_proxy_role.yaml - auth_proxy_role_binding.yaml - auth_proxy_client_clusterrole.yaml +# For each CRD, "Editor" and "Viewer" roles are scaffolded by +# default, aiding admins in cluster management. Those roles are +# not used by the Project itself. You can comment the following lines +# if you do not want those helpers be installed with your Project. +- crds_workmachine_editor_role.yaml +- crds_workmachine_viewer_role.yaml +- crds_workspace_editor_role.yaml +- crds_workspace_viewer_role.yaml + diff --git a/config/samples/crds_v1_workmachine.yaml b/config/samples/crds_v1_workmachine.yaml new file mode 100644 index 00000000..0c7222d7 --- /dev/null +++ b/config/samples/crds_v1_workmachine.yaml @@ -0,0 +1,9 @@ +apiVersion: crds.kloudlite.io/v1 +kind: WorkMachine +metadata: + labels: + app.kubernetes.io/name: app + app.kubernetes.io/managed-by: kustomize + name: workmachine-sample +spec: + # TODO(user): Add fields here diff --git a/config/samples/crds_v1_workspace.yaml b/config/samples/crds_v1_workspace.yaml new file mode 100644 index 00000000..3b748115 --- /dev/null +++ b/config/samples/crds_v1_workspace.yaml @@ -0,0 +1,9 @@ +apiVersion: crds.kloudlite.io/v1 +kind: Workspace +metadata: + labels: + app.kubernetes.io/name: app + app.kubernetes.io/managed-by: kustomize + name: workspace-sample +spec: + # TODO(user): Add fields here diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index 0680afa8..fa49689a 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -110,4 +110,6 @@ resources: - crds_v1_serviceintercept.yaml - mongodb.msvc_v1_backup.yaml - crds_v1_managedserviceplugin.yaml +- crds_v1_workspace.yaml +- crds_v1_workmachine.yaml #+kubebuilder:scaffold:manifestskustomizesamples diff --git a/go.sum b/go.sum index d1e27231..f8d1eca4 100644 --- a/go.sum +++ b/go.sum @@ -277,20 +277,6 @@ github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IX github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kloudlite/plugin-helm-chart v0.0.0-20241220114210-b6d65e34990d h1:kdj4Zt4NdICz7Vm/34Q0WW+69iKO1gwyfE45yAaQ5vs= github.com/kloudlite/plugin-helm-chart v0.0.0-20241220114210-b6d65e34990d/go.mod h1:TZUQ5mREV+UjGVJhPsgmHMS67Mlf2Guit4905Pi/5sc= -github.com/kloudlite/plugin-mongodb v0.0.0-20241221132659-be42e75f149f h1:7yjk5Nh4dkw3rW0NdS7N3bAjfNFC/w9VF6LfXFqRVrQ= -github.com/kloudlite/plugin-mongodb v0.0.0-20241221132659-be42e75f149f/go.mod h1:7ZimdiRyRmVSTS0U69a2sjcvRw/ZbRk6ICwftb6CWxY= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316133014-5e99560adb76 h1:D8IceJKpSOG9LKy2VzN1xHNm4dJ+l7arGPxkj8iZxIo= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316133014-5e99560adb76/go.mod h1:dqT/Qia369uD783N8N58RZPhRPZXywV85nLGDwcq698= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316133328-275e701f2943 h1:ILrHSigArzmo2QH2SIkJEzT1CZNufzQeu1CjM/ljPKo= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316133328-275e701f2943/go.mod h1:dqT/Qia369uD783N8N58RZPhRPZXywV85nLGDwcq698= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316134709-6214224a6327 h1:YayBAW0bjYe+z1ijRpTshMbNYlhjivueuT+HtlSJhlI= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316134709-6214224a6327/go.mod h1:dqT/Qia369uD783N8N58RZPhRPZXywV85nLGDwcq698= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316140417-088f796334a9 h1:O9PJPxmuzSckAkt6uZhschc0IbDeWoTmdwnAF+QZ24U= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316140417-088f796334a9/go.mod h1:dqT/Qia369uD783N8N58RZPhRPZXywV85nLGDwcq698= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316143614-628cc21496dc h1:pqY7GdSJehGrDK5LEazzuiGvKYGnjlVK527DrlqBc3k= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316143614-628cc21496dc/go.mod h1:dqT/Qia369uD783N8N58RZPhRPZXywV85nLGDwcq698= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316173733-b67e5b812ebe h1:JVPgw0qQlAEpk9F5lZCzHM5fD8WPGtTTNeS1u/G/6yQ= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316173733-b67e5b812ebe/go.mod h1:dqT/Qia369uD783N8N58RZPhRPZXywV85nLGDwcq698= github.com/kloudlite/plugin-mongodb v0.0.0-20250316175205-312ba86d8873 h1:6GKV4bZoNzCC5JvyqxHAhjXXSpSjzAXQHklwXqz5YJQ= github.com/kloudlite/plugin-mongodb v0.0.0-20250316175205-312ba86d8873/go.mod h1:dqT/Qia369uD783N8N58RZPhRPZXywV85nLGDwcq698= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= diff --git a/operators/wireguard/examples/device.yaml b/operators/wireguard/examples/device.yaml index 26f66603..393025de 100644 --- a/operators/wireguard/examples/device.yaml +++ b/operators/wireguard/examples/device.yaml @@ -3,7 +3,7 @@ kind: Device metadata: name: device-sample spec: - serverName: server-sample + # serverName: server-sample offset: 1 ports: - port: 3000 diff --git a/operators/wireguard/main.go b/operators/wireguard/main.go index 660d844c..723dc058 100644 --- a/operators/wireguard/main.go +++ b/operators/wireguard/main.go @@ -5,7 +5,7 @@ import ( "github.com/kloudlite/operator/operator" "github.com/kloudlite/operator/operators/wireguard/internal/controllers/device" - cc "github.com/kloudlite/operator/operators/wireguard/internal/controllers/global-vpn" + // cc "github.com/kloudlite/operator/operators/wireguard/internal/controllers/global-vpn" "github.com/kloudlite/operator/operators/wireguard/internal/env" ) @@ -17,7 +17,7 @@ func main() { mgr.RegisterControllers( &device.Reconciler{Name: "Device", Env: ev}, - &cc.Reconciler{Name: "GlobalVPN", Env: ev}, + // &cc.Reconciler{Name: "GlobalVPN", Env: ev}, ) mgr.Start() diff --git a/operators/workmachine/internal/controllers/workmachine/controller.go b/operators/workmachine/internal/controllers/workmachine/controller.go new file mode 100644 index 00000000..d72d294e --- /dev/null +++ b/operators/workmachine/internal/controllers/workmachine/controller.go @@ -0,0 +1,139 @@ +package workspace + +import ( + "context" + "fmt" + + "k8s.io/client-go/tools/record" + + crdsv1 "github.com/kloudlite/operator/apis/crds/v1" + "github.com/kloudlite/operator/operators/workmachine/internal/env" + "github.com/kloudlite/operator/operators/workmachine/internal/templates" + "github.com/kloudlite/operator/pkg/constants" + "github.com/kloudlite/operator/toolkit/kubectl" + rApi "github.com/kloudlite/operator/toolkit/reconciler" + step_result "github.com/kloudlite/operator/toolkit/reconciler/step-result" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" +) + +type Reconciler struct { + client.Client + Scheme *runtime.Scheme + Env *env.Env + + YAMLClient kubectl.YAMLClient + recorder record.EventRecorder + + workspaceDeploymentTemplate []byte + templateWebhook []byte +} + +func (r *Reconciler) GetName() string { + return "workspace" +} + +const ( + CreateDeployment string = "create-deployment" + CreateService string = "create-service" + createWorkMachineJob string = "create-work-machine-job" +) + +// +kubebuilder:rbac:groups=crds.kloudlite.io,resources=apps,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=crds.kloudlite.io,resources=apps/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=crds.kloudlite.io,resources=apps/finalizers,verbs=update + +func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { + req, err := rApi.NewRequest(ctx, r.Client, request.NamespacedName, &crdsv1.WorkMachine{}) + if err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + req.PreReconcile() + defer req.PostReconcile() + + if req.Object.GetDeletionTimestamp() != nil { + if x := r.finalize(req); !x.ShouldProceed() { + return x.ReconcilerResponse() + } + return ctrl.Result{}, nil + } + + if step := req.ClearStatusIfAnnotated(); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + + if step := req.EnsureCheckList([]rApi.CheckMeta{{Name: CreateDeployment}}); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + + if step := req.RestartIfAnnotated(); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + + if step := req.EnsureLabelsAndAnnotations(); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + + if step := req.EnsureFinalizers(constants.ForegroundFinalizer, constants.CommonFinalizer); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + + if step := r.createWorkMachineCreationJob(req); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + + req.Object.Status.IsReady = true + return ctrl.Result{}, nil +} + +func (r *Reconciler) finalize(req *rApi.Request[*crdsv1.WorkMachine]) step_result.Result { + if step := req.EnsureCheckList([]rApi.CheckMeta{ + {Name: "uninstall workspace"}, + }); !step.ShouldProceed() { + return step + } + + check := rApi.NewRunningCheck("uninstall workspace", req) + + if step := req.CleanupOwnedResources(check); !step.ShouldProceed() { + return step + } + + return req.Finalize() +} + +func (r *Reconciler) createWorkMachineCreationJob(req *rApi.Request[*crdsv1.WorkMachine]) step_result.Result { + // ctx, obj := req.Context(), req.Object + check := rApi.NewRunningCheck(createWorkMachineJob, req) + + return check.Completed() +} + +func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { + r.Client = mgr.GetClient() + r.Scheme = mgr.GetScheme() + + if r.YAMLClient == nil { + return fmt.Errorf("yaml client must be set") + } + + r.recorder = mgr.GetEventRecorderFor(r.GetName()) + + var err error + r.workspaceDeploymentTemplate, err = templates.Read(templates.WorkspaceTemplate) + if err != nil { + return err + } + builder := ctrl.NewControllerManagedBy(mgr).For(&crdsv1.WorkMachine{}) + builder.WithOptions(controller.Options{MaxConcurrentReconciles: r.Env.MaxConcurrentReconciles}) + builder.Owns(&appsv1.Deployment{}) + builder.Owns(&corev1.Service{}) + builder.Owns(&crdsv1.Router{}) + builder.WithEventFilter(rApi.ReconcileFilter()) + return builder.Complete(r) +} diff --git a/operators/workmachine/internal/env/env.go b/operators/workmachine/internal/env/env.go new file mode 100644 index 00000000..ac4245c7 --- /dev/null +++ b/operators/workmachine/internal/env/env.go @@ -0,0 +1,25 @@ +package env + +import ( + "github.com/codingconcepts/env" +) + +type Env struct { + MaxConcurrentReconciles int `env:"MAX_CONCURRENT_RECONCILES"` + + WorkspaceImageInitContainer string `env:"WORKSPACE_IMAGE_INIT_CONTAINER" default:"ghcr.io/kloudlite/iac/workspace:latest"` + WorkspaceImageSSH string `env:"WORKSPACE_IMAGE_SSH" default:"ghcr.io/kloudlite/iac/workspace:latest"` + WorkspaceImageTTYD string `env:"WORKSPACE_IMAGE_TTYD" default:"ghcr.io/kloudlite/iac/ttyd:latest"` + WorkspaceImageJupyterNotebook string `env:"WORKSPACE_IMAGE_JUPYTER_NOTEBOOK" default:"ghcr.io/kloudlite/iac/jupyter:latest"` + WorkspaceImageCodeServer string `env:"WORKSPACE_IMAGE_CODE_SERVER" default:"ghcr.io/kloudlite/iac/code-server:latest"` + WorkspcaeImageVscodeServer string `env:"WORKSPCAE_IMAGE_VSCODE_SERVER" default:"ghcr.io/kloudlite/iac/vscode-server:latest"` +} + +func GetEnvOrDie() *Env { + var ev Env + if err := env.Set(&ev); err != nil { + panic(err) + } + + return &ev +} diff --git a/operators/workmachine/internal/templates/embed.go b/operators/workmachine/internal/templates/embed.go new file mode 100644 index 00000000..d94cb95f --- /dev/null +++ b/operators/workmachine/internal/templates/embed.go @@ -0,0 +1,23 @@ +package templates + +import ( + "embed" + "path/filepath" + + "github.com/kloudlite/operator/pkg/templates" +) + +//go:embed * +var templatesDir embed.FS + +type templateFile string + +const ( + WorkspaceTemplate templateFile = "./workspace.yml.tpl" +) + +func Read(t templateFile) ([]byte, error) { + return templatesDir.ReadFile(filepath.Join(string(t))) +} + +var ParseBytes = templates.ParseBytes diff --git a/operators/workmachine/internal/templates/types.go b/operators/workmachine/internal/templates/types.go new file mode 100644 index 00000000..af36ab3e --- /dev/null +++ b/operators/workmachine/internal/templates/types.go @@ -0,0 +1,33 @@ +package templates + +import ( + crdsv1 "github.com/kloudlite/operator/apis/crds/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type WorkspaceTemplateArgs struct { + Metadata metav1.ObjectMeta + NodeName string + ServiceAccountName string + + ImageInitContainer string + ImageSSH string + + EnableTTYD bool + ImageTTYD string + + EnableJupyterNotebook bool + ImageJupyterNotebook string + + EnableCodeServer bool + ImageCodeServer string + + EnableVSCodeServer bool + ImageVscodeServer string + + ImagePullPolicy string + + KloudliteDeviceFQDN string + + RouterSpec crdsv1.RouterSpec +} diff --git a/operators/workmachine/internal/templates/workspace.yml.tpl b/operators/workmachine/internal/templates/workspace.yml.tpl new file mode 100644 index 00000000..c670c0b8 --- /dev/null +++ b/operators/workmachine/internal/templates/workspace.yml.tpl @@ -0,0 +1,229 @@ +{{- with . }} +apiVersion: apps/v1 +kind: Deployment +metadata: {{.Metadata | toJson }} +spec: + selector: + matchLabels: + app: {{.Metadata.Name | squote}} + template: + metadata: + labels: + app: {{.Metadata.Name | squote}} + kloudlite.io/gateway.enabled: "false" + spec: + hostname: {{.Metadata.Name}} + nodeName: {{.NodeName}} + serviceAccount: {{.ServiceAccountName | squote}} + tolerations: + - key: "kloudlite.io/worknode" + operator: "Equal" + value: {{.NodeName |squote}} + effect: "NoExecute" + initContainers: + - name: init-home-dir + image: {{.ImageInitContainer}} + imagePullPolicy: Always + env: + - name: KL_WORKSPACE + value: {{.Metadata.Name}} + + - name: HOME + value: "/home/kl" + + - name: KL_BOX_MODE + value: "true" + + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + command: + - "bash" + - "-c" + - | + set -e + set +x + if [ ! -d "/nix/store" ]; then + curl -L https://nixos.org/nix/install | sh + mkdir -p ~/.config/nix + echo 'experimental-features = nix-command flakes' > ~/.config/nix/nix.conf + fi + + kl_bin_dir="/home/kl/.local/bin" + if [ ! -f "$kl_bin_dir/kl" ]; then + mkdir -p $kl_bin_dir + pushd $kl_bin_dir + curl https://i.jpillora.com/kloudlite/kl@v1.1.87-nightly | bash + popd + fi + + workspace_dir="/home/kl/workspaces/$(KL_WORKSPACE)" + if [ ! -d "$workspace_dir" ]; then + mkdir -p $workspace_dir + pushd $workspace_dir + export PATH=$PATH:/home/kl/.nix-profile/bin:/home/kl/.local/bin + kl init + popd + fi + + if [ -f "$workspace_dir/kl.yaml" ] || [ -f "$workspace_dir/kl.yml" ]; then + pushd $workspace_dir + PATH=$PATH:/home/kl/.nix-profile/bin:/home/kl/.local/bin /home/kl/.local/bin/kl shell -r > /env/.env + PATH=$PATH:/home/kl/.nix-profile/bin:/home/kl/.local/bin /home/kl/.local/bin/kl get env > /env/.connected_env + popd + fi + + if [ ! -f "/home/kl/.zshrc" ]; then + mkdir -p "/home/kl/.config/zsh" + cp /tmp/.zshrc /home/kl/.zshrc + cp /tmp/.aliasrc /home/kl/.config/aliasrc + fi + + if [ ! -f "/home/kl/.local/bin/starship" ]; then + curl -sS https://starship.rs/install.sh | sh -s -- -y -b /home/kl/.local/bin + fi + + if [ ! -d "/home/kl/.config/zsh/zsh-autosuggestions" ]; then + mkdir -p "/home/kl/.config/zsh" + git clone https://github.com/zsh-users/zsh-autosuggestions + fi + + if [ ! -d "/home/kl/.config/zsh/zsh-syntax-highlighting" ]; then + mkdir -p "/home/kl/.config/zsh" + git clone https://github.com/zsh-users/zsh-syntax-highlighting.git "/home/kl/.config/zsh/zsh-syntax-highlighting" + fi + + {{- /* if [ ! -d "/home/kl/.kl" ]; then */}} + {{- /* mkdir -p /home/kl/.kl */}} + {{- /* sh -c 'cat > /home/kl/.kl/kl-session.yaml < ~/.config/nix/nix.conf + fi + + kl_bin_dir="/home/kl/.local/bin" + if [ ! -f "$kl_bin_dir/kl" ]; then + mkdir -p $kl_bin_dir + pushd $kl_bin_dir + curl https://i.jpillora.com/kloudlite/kl@v1.1.87-nightly | bash + popd + fi + + workspace_dir="/home/kl/workspaces/$(KL_WORKSPACE)" + if [ ! -d "$workspace_dir" ]; then + mkdir -p $workspace_dir + pushd $workspace_dir + export PATH=$PATH:/home/kl/.nix-profile/bin:/home/kl/.local/bin + kl init + popd + fi + + if [ -f "$workspace_dir/kl.yaml" ] || [ -f "$workspace_dir/kl.yml" ]; then + pushd $workspace_dir + PATH=$PATH:/home/kl/.nix-profile/bin:/home/kl/.local/bin /home/kl/.local/bin/kl shell -r > /env/.env + PATH=$PATH:/home/kl/.nix-profile/bin:/home/kl/.local/bin /home/kl/.local/bin/kl get env > /env/.connected_env + popd + fi + + if [ ! -f "/home/kl/.zshrc" ]; then + mkdir -p "/home/kl/.config/zsh" + cp /tmp/.zshrc /home/kl/.zshrc + cp /tmp/.aliasrc /home/kl/.config/aliasrc + fi + + if [ ! -f "/home/kl/.local/bin/starship" ]; then + curl -sS https://starship.rs/install.sh | sh -s -- -y -b /home/kl/.local/bin + fi + + if [ ! -d "/home/kl/.config/zsh/zsh-autosuggestions" ]; then + mkdir -p "/home/kl/.config/zsh" + git clone https://github.com/zsh-users/zsh-autosuggestions + fi + + if [ ! -d "/home/kl/.config/zsh/zsh-syntax-highlighting" ]; then + mkdir -p "/home/kl/.config/zsh" + git clone https://github.com/zsh-users/zsh-syntax-highlighting.git "/home/kl/.config/zsh/zsh-syntax-highlighting" + fi + + {{- /* if [ ! -d "/home/kl/.kl" ]; then */}} + {{- /* mkdir -p /home/kl/.kl */}} + {{- /* sh -c 'cat > /home/kl/.kl/kl-session.yaml < Date: Wed, 2 Apr 2025 14:06:39 +0530 Subject: [PATCH 03/27] feat: workspaces and workmachines operator and CRDs --- apis/crds/v1/workmachine_types.go | 44 +++- apis/crds/v1/workspace_types.go | 1 + apis/crds/v1/zz_generated.deepcopy.go | 43 +++- cmd/agent-operator/main.go | 6 +- .../bases/crds.kloudlite.io_workmachines.yaml | 75 +++++- .../bases/crds.kloudlite.io_workspaces.yaml | 8 +- go.mod | 7 +- go.sum | 18 +- .../lifecycle-controller/controller.go | 56 +++-- .../resource-watcher/controller/register.go | 8 +- .../watch-and-update/controller.go | 8 +- .../workmachine/examples/workmachine.yml | 18 ++ .../controllers/workmachine/controller.go | 178 ++++++++++++-- operators/workmachine/internal/env/env.go | 12 +- .../workmachine/internal/templates/embed.go | 3 +- .../workmachine/internal/templates/types.go | 34 +-- .../templates/workmachine-lifecycle.yml.tpl | 117 +++++++++ .../internal/templates/workspace.yml.tpl | 229 ------------------ operators/workmachine/main.go | 21 +- operators/workmachine/register/register.go | 6 +- operators/workspace/main.go | 5 - toolkit/functions/operator-helpers.go | 1 + toolkit/go.mod | 5 + toolkit/go.sum | 28 ++- toolkit/kubectl/yaml-client.go | 8 +- toolkit/logging/logger.go | 6 +- toolkit/operator/operator.go | 43 ++-- toolkit/reconciler/checks.go | 5 + toolkit/reconciler/request.go | 16 +- 29 files changed, 626 insertions(+), 383 deletions(-) create mode 100644 operators/workmachine/examples/workmachine.yml create mode 100644 operators/workmachine/internal/templates/workmachine-lifecycle.yml.tpl delete mode 100644 operators/workmachine/internal/templates/workspace.yml.tpl diff --git a/apis/crds/v1/workmachine_types.go b/apis/crds/v1/workmachine_types.go index 18123d92..81395133 100644 --- a/apis/crds/v1/workmachine_types.go +++ b/apis/crds/v1/workmachine_types.go @@ -1,7 +1,9 @@ package v1 import ( + ct "github.com/kloudlite/operator/apis/common-types" rApi "github.com/kloudlite/operator/toolkit/reconciler" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -9,17 +11,20 @@ type AWSMachineConfig struct { Region string `json:"region" graphql:"noinput"` AvailabilityZone string `json:"availabilityZone"` - AMI string `json:"ami"` - InstanceType string `json:"instanceType"` + AMI string `json:"ami"` + InstanceType string `json:"instanceType"` + PublicSubnetID string `json:"publicSubnetID"` - PublicSubnetID string `json:"publicSubnetId" graphql:"noinput"` - SecurityGroupID string `json:"SecurityGroupID" graphql:"noinput"` + //+kubebuilder:default=50 + RootVolumeSize int `json:"rootVolumeSize" graphql:"noinput"` + //+kubebuilder:default=gp3 RootVolumeType string `json:"rootVolumeType" graphql:"noinput"` - RootVolumeSize int `json:"rootVolumeSize" graphql:"noinput"` + ExternalVolumeSize int `json:"externalVolumeSize"` + + //+kubebuilder:default=gp3 ExternalVolumeType string `json:"externalVolumeType" graphql:"noinput"` - ExternalVolumeSize string `json:"externalVolumeSize"` IAMInstanceProfileRole *string `json:"iamInstanceProfileRole,omitempty" graphql:"noinput"` } @@ -33,13 +38,27 @@ const ( WorkMachineStateOff WorkMachineState = "OFF" ) +type WorkMachineJobParams struct { + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` +} + // WorkMachineSpec defines the desired state of WorkMachine type WorkMachineSpec struct { - State WorkMachineState `json:"state"` + State WorkMachineState `json:"state"` + SSHPublicKeys []string `json:"sshPublicKeys"` + + JobParams WorkMachineJobParams `json:"jobParams"` - SSHPublicKeys []string `json:"sshPublicKeys"` + AWSMachineConfig *AWSMachineConfig `json:"aws"` +} + +func (wms *WorkMachineSpec) GetCloudProvider() ct.CloudProvider { + if wms.AWSMachineConfig != nil { + return ct.CloudProviderAWS + } - AWSMachineConfig `json:"aws"` + return ct.CloudProviderUnknown } type WorkMachineStatus struct { @@ -49,6 +68,11 @@ type WorkMachineStatus struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster +// +kubebuilder:printcolumn:JSONPath=".status.status.lastReconcileTime",name=Seen,type=date +// +kubebuilder:printcolumn:JSONPath=".metadata.annotations.kloudlite\\.io\\/operator\\.checks",name=Checks,type=string +// +kubebuilder:printcolumn:JSONPath=".metadata.annotations.kloudlite\\.io\\/operator\\.resource\\.ready",name=Ready,type=string +// +kubebuilder:printcolumn:JSONPath=".metadata.creationTimestamp",name=Age,type=date // WorkMachine is the Schema for the workmachines API type WorkMachine struct { @@ -61,7 +85,7 @@ type WorkMachine struct { func (r *WorkMachine) EnsureGVK() { if r != nil { - r.SetGroupVersionKind(GroupVersion.WithKind("Workspace")) + r.SetGroupVersionKind(GroupVersion.WithKind("WorkMachine")) } } diff --git a/apis/crds/v1/workspace_types.go b/apis/crds/v1/workspace_types.go index a6addaf3..74451347 100644 --- a/apis/crds/v1/workspace_types.go +++ b/apis/crds/v1/workspace_types.go @@ -34,6 +34,7 @@ type WorkspaceSpec struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster // +kubebuilder:printcolumn:JSONPath=".status.lastReconcileTime",name=Seen,type=date // +kubebuilder:printcolumn:JSONPath=".metadata.annotations.kloudlite\\.io\\/checks",name=Checks,type=string // +kubebuilder:printcolumn:JSONPath=".metadata.annotations.kloudlite\\.io\\/resource\\.ready",name=Ready,type=string diff --git a/apis/crds/v1/zz_generated.deepcopy.go b/apis/crds/v1/zz_generated.deepcopy.go index e981a728..3a5bee2d 100644 --- a/apis/crds/v1/zz_generated.deepcopy.go +++ b/apis/crds/v1/zz_generated.deepcopy.go @@ -1808,6 +1808,13 @@ func (in *RouterSpec) DeepCopyInto(out *RouterSpec) { *out = new(Cors) (*in).DeepCopyInto(*out) } + if in.NginxIngressAnnotations != nil { + in, out := &in.NginxIngressAnnotations, &out.NginxIngressAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouterSpec. @@ -1999,6 +2006,35 @@ func (in *WorkMachine) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkMachineJobParams) DeepCopyInto(out *WorkMachineJobParams) { + *out = *in + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]corev1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkMachineJobParams. +func (in *WorkMachineJobParams) DeepCopy() *WorkMachineJobParams { + if in == nil { + return nil + } + out := new(WorkMachineJobParams) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkMachineList) DeepCopyInto(out *WorkMachineList) { *out = *in @@ -2039,7 +2075,12 @@ func (in *WorkMachineSpec) DeepCopyInto(out *WorkMachineSpec) { *out = make([]string, len(*in)) copy(*out, *in) } - in.AWSMachineConfig.DeepCopyInto(&out.AWSMachineConfig) + in.JobParams.DeepCopyInto(&out.JobParams) + if in.AWSMachineConfig != nil { + in, out := &in.AWSMachineConfig, &out.AWSMachineConfig + *out = new(AWSMachineConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkMachineSpec. diff --git a/cmd/agent-operator/main.go b/cmd/agent-operator/main.go index 1c721406..929fe935 100644 --- a/cmd/agent-operator/main.go +++ b/cmd/agent-operator/main.go @@ -4,7 +4,7 @@ import ( "github.com/kloudlite/operator/toolkit/operator" app "github.com/kloudlite/operator/operators/app-n-lambda/controller" - helmCharts "github.com/kloudlite/operator/operators/helm-charts/controller" + // helmCharts "github.com/kloudlite/operator/operators/helm-charts/controller" lifecycle "github.com/kloudlite/operator/operators/lifecycle/controller" msvcAndMres "github.com/kloudlite/operator/operators/msvc-n-mres/controller" @@ -15,6 +15,7 @@ import ( // routers "github.com/kloudlite/operator/operators/routers/controller" serviceIntercept "github.com/kloudlite/operator/operators/service-intercept/controller" + workmachine "github.com/kloudlite/operator/operators/workmachine/register" workspace "github.com/kloudlite/operator/operators/workspace/register" pluginHelmChart "github.com/kloudlite/plugin-helm-chart/kloudlite" pluginMongoDB "github.com/kloudlite/plugin-mongodb/kloudlite" @@ -26,7 +27,7 @@ func main() { // kloudlite resources app.RegisterInto(mgr) project.RegisterInto(mgr) - helmCharts.RegisterInto(mgr) + // helmCharts.RegisterInto(mgr) // routers.RegisterInto(mgr) // kloudlite managed services @@ -48,6 +49,7 @@ func main() { serviceIntercept.RegisterInto(mgr) workspace.RegisterInto(mgr) + workmachine.RegisterInto(mgr) pluginMongoDB.RegisterInto(mgr) pluginHelmChart.RegisterInto(mgr) diff --git a/config/crd/bases/crds.kloudlite.io_workmachines.yaml b/config/crd/bases/crds.kloudlite.io_workmachines.yaml index c7656c37..3c72ba45 100644 --- a/config/crd/bases/crds.kloudlite.io_workmachines.yaml +++ b/config/crd/bases/crds.kloudlite.io_workmachines.yaml @@ -12,9 +12,22 @@ spec: listKind: WorkMachineList plural: workmachines singular: workmachine - scope: Namespaced + scope: Cluster versions: - - name: v1 + - additionalPrinterColumns: + - jsonPath: .status.status.lastReconcileTime + name: Seen + type: date + - jsonPath: .metadata.annotations.kloudlite\.io\/operator\.checks + name: Checks + type: string + - jsonPath: .metadata.annotations.kloudlite\.io\/operator\.resource\.ready + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 schema: openAPIV3Schema: description: WorkMachine is the Schema for the workmachines API @@ -41,40 +54,85 @@ spec: properties: aws: properties: - SecurityGroupID: - type: string ami: type: string availabilityZone: type: string externalVolumeSize: - type: string + type: integer externalVolumeType: + default: gp3 type: string iamInstanceProfileRole: type: string instanceType: type: string - publicSubnetId: + publicSubnetID: type: string region: type: string rootVolumeSize: + default: 50 type: integer rootVolumeType: + default: gp3 type: string required: - - SecurityGroupID - ami - availabilityZone - externalVolumeSize - externalVolumeType - instanceType - - publicSubnetId + - publicSubnetID - region - rootVolumeSize - rootVolumeType type: object + jobParams: + properties: + nodeSelector: + additionalProperties: + type: string + type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object sshPublicKeys: items: type: string @@ -86,6 +144,7 @@ spec: type: string required: - aws + - jobParams - sshPublicKeys - state type: object diff --git a/config/crd/bases/crds.kloudlite.io_workspaces.yaml b/config/crd/bases/crds.kloudlite.io_workspaces.yaml index 9134b15d..b8147b8a 100644 --- a/config/crd/bases/crds.kloudlite.io_workspaces.yaml +++ b/config/crd/bases/crds.kloudlite.io_workspaces.yaml @@ -12,7 +12,7 @@ spec: listKind: WorkspaceList plural: workspaces singular: workspace - scope: Namespaced + scope: Cluster versions: - additionalPrinterColumns: - jsonPath: .status.lastReconcileTime @@ -113,6 +113,12 @@ spec: type: string maxBodySizeInMB: type: integer + nginxIngressAnnotations: + additionalProperties: + type: string + description: NginxIngressAnnotations is additional list of annotations + on ingress resource + type: object rateLimit: properties: connections: diff --git a/go.mod b/go.mod index 0ab2dae5..75605919 100644 --- a/go.mod +++ b/go.mod @@ -21,12 +21,13 @@ require ( github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e github.com/influxdata/influxdb-client-go/v2 v2.14.0 github.com/kloudlite/operator/toolkit v0.0.0-20250316093242-493e9b587c10 - github.com/kloudlite/plugin-helm-chart v0.0.0-20241220114210-b6d65e34990d + github.com/kloudlite/plugin-helm-chart v0.0.0-20250317052100-fef043b111a2 github.com/kloudlite/plugin-mongodb v0.0.0-20250316175205-312ba86d8873 github.com/matoous/go-nanoid/v2 v2.1.0 github.com/miekg/dns v1.1.62 github.com/nats-io/nats.go v1.38.0 github.com/nxtcoder17/go-helm-client v0.0.0-20230915000026-8789cfa27bf3 + github.com/nxtcoder17/go.pkgs v0.0.0-20250401173049-502a28e591dd github.com/onsi/ginkgo/v2 v2.22.0 github.com/onsi/gomega v1.36.1 github.com/pkg/errors v0.9.1 @@ -84,6 +85,10 @@ require ( github.com/muesli/termenv v0.15.2 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/oapi-codegen/runtime v1.1.1 // indirect + github.com/rs/zerolog v1.33.0 // indirect + github.com/samber/lo v1.47.0 // indirect + github.com/samber/slog-common v0.18.1 // indirect + github.com/samber/slog-zerolog/v2 v2.7.3 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.58.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect diff --git a/go.sum b/go.sum index f8d1eca4..03ba89bd 100644 --- a/go.sum +++ b/go.sum @@ -187,6 +187,7 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo= github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -275,8 +276,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= -github.com/kloudlite/plugin-helm-chart v0.0.0-20241220114210-b6d65e34990d h1:kdj4Zt4NdICz7Vm/34Q0WW+69iKO1gwyfE45yAaQ5vs= -github.com/kloudlite/plugin-helm-chart v0.0.0-20241220114210-b6d65e34990d/go.mod h1:TZUQ5mREV+UjGVJhPsgmHMS67Mlf2Guit4905Pi/5sc= +github.com/kloudlite/plugin-helm-chart v0.0.0-20250317052100-fef043b111a2 h1:4DoLvbPEjYVBMnNpVmeR9OoI6Q9kebd6HnDpkwFts+Y= +github.com/kloudlite/plugin-helm-chart v0.0.0-20250317052100-fef043b111a2/go.mod h1:TZUQ5mREV+UjGVJhPsgmHMS67Mlf2Guit4905Pi/5sc= github.com/kloudlite/plugin-mongodb v0.0.0-20250316175205-312ba86d8873 h1:6GKV4bZoNzCC5JvyqxHAhjXXSpSjzAXQHklwXqz5YJQ= github.com/kloudlite/plugin-mongodb v0.0.0-20250316175205-312ba86d8873/go.mod h1:dqT/Qia369uD783N8N58RZPhRPZXywV85nLGDwcq698= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -306,6 +307,7 @@ github.com/matoous/go-nanoid/v2 v2.1.0/go.mod h1:KlbGNQ+FhrUNIHUxZdL63t7tl4LaPkZ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= @@ -367,6 +369,8 @@ github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/nxtcoder17/go-helm-client v0.0.0-20230915000026-8789cfa27bf3 h1:36FtxDl0ARKh5SQY3tyYD2/VCcMhZCx+01LGihpT06o= github.com/nxtcoder17/go-helm-client v0.0.0-20230915000026-8789cfa27bf3/go.mod h1:jxy7CytMOdPqv7eDdKuxaQxsdeHoSZP9tQFZt9O6nlA= +github.com/nxtcoder17/go.pkgs v0.0.0-20250401173049-502a28e591dd h1:bP80t6mgiwIVEpdfZwg/w22GzDcHBgDRVZItH5QWxis= +github.com/nxtcoder17/go.pkgs v0.0.0-20250401173049-502a28e591dd/go.mod h1:raSGHj5CMHNHZf4fCV9CWpFk0hsb2CSKFZSPd4zW8JM= github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= @@ -427,10 +431,19 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= +github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/rubenv/sql-migrate v1.7.1 h1:f/o0WgfO/GqNuVg+6801K/KW3WdDSupzSjDYODmiUq4= github.com/rubenv/sql-migrate v1.7.1/go.mod h1:Ob2Psprc0/3ggbM6wCzyYVFFuc6FyZrb2AS+ezLDFb4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= +github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= +github.com/samber/slog-common v0.18.1 h1:c0EipD/nVY9HG5shgm/XAs67mgpWDMF+MmtptdJNCkQ= +github.com/samber/slog-common v0.18.1/go.mod h1:QNZiNGKakvrfbJ2YglQXLCZauzkI9xZBjOhWFKS3IKk= +github.com/samber/slog-zerolog/v2 v2.7.3 h1:/MkPDl/tJhijN2GvB1MWwBn2FU8RiL3rQ8gpXkQm2EY= +github.com/samber/slog-zerolog/v2 v2.7.3/go.mod h1:oWU7WHof4Xp8VguiNO02r1a4VzkgoOyOZhY5CuRke60= github.com/seancfoley/bintree v1.3.1 h1:cqmmQK7Jm4aw8gna0bP+huu5leVOgHGSJBEpUx3EXGI= github.com/seancfoley/bintree v1.3.1/go.mod h1:hIUabL8OFYyFVTQ6azeajbopogQc2l5C/hiXMcemWNU= github.com/seancfoley/ipaddress-go v1.7.0 h1:vWp3SR3k+HkV3aKiNO2vEe6xbVxS0x/Ixw6hgyP238s= @@ -600,6 +613,7 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/operators/lifecycle/internal/lifecycle-controller/controller.go b/operators/lifecycle/internal/lifecycle-controller/controller.go index 064e5296..bb189e17 100644 --- a/operators/lifecycle/internal/lifecycle-controller/controller.go +++ b/operators/lifecycle/internal/lifecycle-controller/controller.go @@ -212,30 +212,48 @@ func (r *Reconciler) applyK8sJob(req *rApi.Request[*crdsv1.Lifecycle]) stepResul ctx, obj := req.Context(), req.Object check := rApi.NewRunningCheck(ApplyK8sJob, req) - if v, ok := obj.Status.Checks[ApplyK8sJob]; ok && v.Generation == obj.Generation { - switch v.State { - case rApi.CompletedState: - return check.Completed() - case rApi.ErroredState: - if obj.Spec.RetryOnFailure { - if obj.Status.LastReconcileTime != nil { - req.Logger.Info(fmt.Sprintf("time since last reconcilation: %s", time.Since(obj.Status.LastReconcileTime.Time).String())) - if time.Since(obj.Status.LastReconcileTime.Time) < obj.Spec.RetryOnFailureDelay.Duration { - return check.Completed().RequeueAfter(obj.Spec.RetryOnFailureDelay.Duration) - } + if obj.Status.Phase == crdsv1.JobPhaseFailed && obj.Status.LastReadyGeneration == obj.Generation { + if obj.Spec.RetryOnFailure { + if obj.Status.LastReconcileTime != nil { + req.Logger.Info(fmt.Sprintf("time since last reconcilation: %s", time.Since(obj.Status.LastReconcileTime.Time).String())) + if time.Since(obj.Status.LastReconcileTime.Time) < obj.Spec.RetryOnFailureDelay.Duration { + return check.Completed().RequeueAfter(obj.Spec.RetryOnFailureDelay.Duration) + } - req.Logger.Info(fmt.Sprintf("re-creating job for lifecycle %s/%s, as it failed and is past the retry delay", obj.Namespace, obj.Name)) - if err := r.Delete(ctx, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: obj.Name, Namespace: obj.Namespace}}); err != nil { - if !apiErrors.IsNotFound(err) { - return check.Failed(err) - } + req.Logger.Info(fmt.Sprintf("re-creating job for lifecycle %s/%s, as it failed and is past the retry delay", obj.Namespace, obj.Name)) + if err := r.Delete(ctx, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: obj.Name, Namespace: obj.Namespace}}); err != nil { + if !apiErrors.IsNotFound(err) { + return check.Failed(err) } } } - return check.Failed(fmt.Errorf("job failed")) } } + // if v, ok := obj.Status.Checks[ApplyK8sJob]; ok && v.Generation == obj.Generation { + // switch v.State { + // case rApi.CompletedState: + // return check.Completed() + // case rApi.ErroredState: + // if obj.Spec.RetryOnFailure { + // if obj.Status.LastReconcileTime != nil { + // req.Logger.Info(fmt.Sprintf("time since last reconcilation: %s", time.Since(obj.Status.LastReconcileTime.Time).String())) + // if time.Since(obj.Status.LastReconcileTime.Time) < obj.Spec.RetryOnFailureDelay.Duration { + // return check.Completed().RequeueAfter(obj.Spec.RetryOnFailureDelay.Duration) + // } + // + // req.Logger.Info(fmt.Sprintf("re-creating job for lifecycle %s/%s, as it failed and is past the retry delay", obj.Namespace, obj.Name)) + // if err := r.Delete(ctx, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: obj.Name, Namespace: obj.Namespace}}); err != nil { + // if !apiErrors.IsNotFound(err) { + // return check.Failed(err) + // } + // } + // } + // } + // return check.Failed(fmt.Errorf("job failed")) + // } + // } + job := &batchv1.Job{} if err := r.Get(ctx, fn.NN(obj.Namespace, obj.Name), job); err != nil { job = nil @@ -400,6 +418,8 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { return fmt.Errorf("yamlclient must be set") } + recorder := mgr.GetEventRecorderFor(r.GetName()) + var err error r.templateJobRBAC, err = templates.Read(templates.JobRBACTemplate) if err != nil { @@ -408,6 +428,6 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { builder := ctrl.NewControllerManagedBy(mgr).For(&crdsv1.Lifecycle{}).Owns(&batchv1.Job{}) builder.WithOptions(controller.Options{MaxConcurrentReconciles: r.Env.MaxConcurrentReconciles}) - builder.WithEventFilter(rApi.ReconcileFilter()) + builder.WithEventFilter(rApi.ReconcileFilter(recorder)) return builder.Complete(r) } diff --git a/operators/resource-watcher/controller/register.go b/operators/resource-watcher/controller/register.go index 8d8ae5b2..c0db04e2 100644 --- a/operators/resource-watcher/controller/register.go +++ b/operators/resource-watcher/controller/register.go @@ -8,6 +8,7 @@ import ( clustersv1 "github.com/kloudlite/operator/apis/clusters/v1" "github.com/kloudlite/operator/grpc-interfaces/grpc/messages" + "github.com/nxtcoder17/go.pkgs/log" "google.golang.org/grpc" "google.golang.org/grpc/connectivity" @@ -47,7 +48,6 @@ func RegisterInto(mgr operator.Operator) { } mgr.RegisterControllers(watchAndUpdateReconciler) - logger := mgr.Operator().Logger() ping := func(cc *grpc.ClientConn) error { ctx, cf := context.WithTimeout(context.TODO(), 500*time.Millisecond) @@ -55,10 +55,10 @@ func RegisterInto(mgr operator.Operator) { msgDispatchCli := messages.NewMessageDispatchServiceClient(cc) _, err := msgDispatchCli.Ping(ctx, &messages.Empty{}) if err != nil { - logger.Info("ping failed, client is disconnected") + log.DefaultLogger().Warn("ping failed, client is disconnected") return err } - logger.Debug("ping is successfull, client is connected") + log.DefaultLogger().Debug("ping is successfull, client is connected") return nil } @@ -116,7 +116,7 @@ func RegisterInto(mgr operator.Operator) { } go func() { - logger := logger.With("component", "grpc-client") + logger := log.DefaultLogger().With("component", "grpc-client").Slog() for { connectGrpc(logger) <-time.After(2 * time.Second) diff --git a/operators/resource-watcher/internal/controllers/watch-and-update/controller.go b/operators/resource-watcher/internal/controllers/watch-and-update/controller.go index dc52b93b..72d63775 100644 --- a/operators/resource-watcher/internal/controllers/watch-and-update/controller.go +++ b/operators/resource-watcher/internal/controllers/watch-and-update/controller.go @@ -294,7 +294,7 @@ func (r *Reconciler) dispatchEvent(ctx context.Context, logger *slog.Logger, obj return nil } - case /*NodePoolGVK.String(),*/ PersistentVolumeClaimGVK.String(), PersistentVolumeGVK.String(), VolumeAttachmentGVK.String(), IngressGVK.String(), NamespaceGVK.String(): + case /*NodePoolGVK.String(),*/ PersistentVolumeClaimGVK.String(), PersistentVolumeGVK.String(), VolumeAttachmentGVK.String(), IngressGVK.String(), NamespaceGVK.String(), WorkspaceGVK.String(), WorkMachineGVK.String(): { return r.MsgSender.DispatchInfraResourceUpdates(MessageSenderContext{mctx, logger}, t.ResourceUpdate{ Object: obj, @@ -452,6 +452,9 @@ var ( SecretGVK = newGVK("v1", "Secret") ConfigmapGVK = newGVK("v1", "ConfigMap") NamespaceGVK = newGVK("v1", "Namespace") + + WorkMachineGVK = newGVK("crds.kloudlite.io/v1", "WorkMachine") + WorkspaceGVK = newGVK("crds.kloudlite.io/v1", "Workspace") ) // SetupWithManager sets up the controllers with the Manager. @@ -494,6 +497,9 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { SecretGVK, ConfigmapGVK, NamespaceGVK, + + WorkspaceGVK, + WorkMachineGVK, } for i := range watchList { diff --git a/operators/workmachine/examples/workmachine.yml b/operators/workmachine/examples/workmachine.yml new file mode 100644 index 00000000..b45b7969 --- /dev/null +++ b/operators/workmachine/examples/workmachine.yml @@ -0,0 +1,18 @@ +apiVersion: crds.kloudlite.io/v1 +kind: WorkMachine +metadata: + name: sample +spec: + state: "ON" + sshPublicKeys: [] + jobParams: + nodeSelector: + kubernetes.io/hostname: "master-1" + aws: + region: "ap-south-1" + availabilityZone: "ap-south-1a" + publicSubnetID: "subnet-0e4f0634ba1b5be2e" + ami: "ami-05c179eced2eb9b5b" + instanceType: "t3.medium" + rootVolumeSize: 50 + externalVolumeSize: 100 diff --git a/operators/workmachine/internal/controllers/workmachine/controller.go b/operators/workmachine/internal/controllers/workmachine/controller.go index d72d294e..7474fe91 100644 --- a/operators/workmachine/internal/controllers/workmachine/controller.go +++ b/operators/workmachine/internal/controllers/workmachine/controller.go @@ -1,24 +1,29 @@ -package workspace +package workmachine import ( "context" + "encoding/json" "fmt" + "strings" "k8s.io/client-go/tools/record" + ct "github.com/kloudlite/operator/apis/common-types" crdsv1 "github.com/kloudlite/operator/apis/crds/v1" "github.com/kloudlite/operator/operators/workmachine/internal/env" "github.com/kloudlite/operator/operators/workmachine/internal/templates" - "github.com/kloudlite/operator/pkg/constants" + fn "github.com/kloudlite/operator/toolkit/functions" "github.com/kloudlite/operator/toolkit/kubectl" rApi "github.com/kloudlite/operator/toolkit/reconciler" step_result "github.com/kloudlite/operator/toolkit/reconciler/step-result" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/yaml" ) type Reconciler struct { @@ -29,20 +34,22 @@ type Reconciler struct { YAMLClient kubectl.YAMLClient recorder record.EventRecorder - workspaceDeploymentTemplate []byte - templateWebhook []byte + workmachineLifecycleTemplateSpec []byte + templateWebhook []byte } func (r *Reconciler) GetName() string { - return "workspace" + return "workmachine" } const ( - CreateDeployment string = "create-deployment" - CreateService string = "create-service" createWorkMachineJob string = "create-work-machine-job" ) +const ( + jobRefAnnotation string = "kloudlite.io/workmachine.job-ref" +) + // +kubebuilder:rbac:groups=crds.kloudlite.io,resources=apps,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=crds.kloudlite.io,resources=apps/status,verbs=get;update;patch // +kubebuilder:rbac:groups=crds.kloudlite.io,resources=apps/finalizers,verbs=update @@ -56,6 +63,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. req.PreReconcile() defer req.PostReconcile() + req.Logger.Debug("RECONCILATION starting ...") + if req.Object.GetDeletionTimestamp() != nil { if x := r.finalize(req); !x.ShouldProceed() { return x.ReconcilerResponse() @@ -67,7 +76,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. return step.ReconcilerResponse() } - if step := req.EnsureCheckList([]rApi.CheckMeta{{Name: CreateDeployment}}); !step.ShouldProceed() { + if step := req.EnsureCheckList([]rApi.CheckMeta{{Name: createWorkMachineJob}}); !step.ShouldProceed() { return step.ReconcilerResponse() } @@ -79,7 +88,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. return step.ReconcilerResponse() } - if step := req.EnsureFinalizers(constants.ForegroundFinalizer, constants.CommonFinalizer); !step.ShouldProceed() { + if step := req.EnsureFinalizers(rApi.ForegroundFinalizer, rApi.CommonFinalizer); !step.ShouldProceed() { return step.ReconcilerResponse() } @@ -93,12 +102,12 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. func (r *Reconciler) finalize(req *rApi.Request[*crdsv1.WorkMachine]) step_result.Result { if step := req.EnsureCheckList([]rApi.CheckMeta{ - {Name: "uninstall workspace"}, + {Name: "uninstall workmachine"}, }); !step.ShouldProceed() { return step } - check := rApi.NewRunningCheck("uninstall workspace", req) + check := rApi.NewRunningCheck("uninstall workmachine", req) if step := req.CleanupOwnedResources(check); !step.ShouldProceed() { return step @@ -107,10 +116,144 @@ func (r *Reconciler) finalize(req *rApi.Request[*crdsv1.WorkMachine]) step_resul return req.Finalize() } +type ClusterParams struct { + K3sServerHost string `json:"k3s_server_host"` + + K3sServerToken string `json:"k3s_server_token"` + K3sAgentToken string `json:"k3s_agent_token"` + + K3sVersion string `json:"k3s_version"` + + AwsVPCName string `json:"aws_vpc_name"` + AwsVPCId string `json:"aws_vpc_id"` + + AwsNLBDNSHost string `json:"aws_nlb_dns_host"` + + AwsSecurityGroupIDs []string `json:"aws_security_group_ids"` + AwsIAMInstanceProfileName string `json:"aws_iam_instance_profile_name"` +} + +func (r *Reconciler) parseSpecIntoTFValues(ctx context.Context, obj *crdsv1.WorkMachine) ([]byte, error) { + sp := strings.Split(r.Env.K3sParamsSecretRef, "/") + if len(sp) != 2 { + return nil, fmt.Errorf("invalid k3s params secret ref must be a valid / format") + } + + secret, err := rApi.Get(ctx, r.Client, fn.NN(sp[0], sp[1]), &corev1.Secret{}) + if err != nil { + return nil, err + } + + fmt.Printf("cluster-params.yml: \n---\n%s\n---\n", string(secret.Data["cluster-params.yml"])) + + cp := ClusterParams{} + if err := yaml.Unmarshal(secret.Data["cluster-params.yml"], &cp); err != nil { + return nil, err + } + + fmt.Printf("cluster params: %+v\n", cp) + + switch obj.Spec.GetCloudProvider() { + case ct.CloudProviderAWS: + { + return json.Marshal(map[string]any{ + "aws_region": obj.Spec.AWSMachineConfig.Region, + "trace_id": "workmachine-" + obj.Name, + "vpc_id": cp.AwsVPCId, + "name": obj.Name, + "k3s_server_host": cp.K3sServerHost, + "k3s_agent_token": cp.K3sAgentToken, + "k3s_version": cp.K3sVersion, + "ami": obj.Spec.AWSMachineConfig.AMI, + "instance_type": obj.Spec.AWSMachineConfig.InstanceType, + "instance_state": func() string { + if obj.Spec.State == crdsv1.WorkMachineStateOn { + return "running" + } + + return "stopped" + }(), + "availability_zone": obj.Spec.AWSMachineConfig.AvailabilityZone, + // "iam_instance_profile": func() string { + // if obj.Spec.AWSMachineConfig.IAMInstanceProfileRole != nil { + // return *obj.Spec.AWSMachineConfig.IAMInstanceProfileRole + // } + // return cp.AwsIAMInstanceProfileName + // }(), + "root_volume_size": obj.Spec.AWSMachineConfig.RootVolumeSize, + "root_volume_type": obj.Spec.AWSMachineConfig.RootVolumeType, + "security_group_ids": cp.AwsSecurityGroupIDs, + "subnet_id": obj.Spec.AWSMachineConfig.PublicSubnetID, + }) + } + default: + return nil, fmt.Errorf("unsupported cloud provider (%s)", obj.Spec.GetCloudProvider()) + } +} + func (r *Reconciler) createWorkMachineCreationJob(req *rApi.Request[*crdsv1.WorkMachine]) step_result.Result { - // ctx, obj := req.Context(), req.Object + ctx, obj := req.Context(), req.Object check := rApi.NewRunningCheck(createWorkMachineJob, req) + jobName := fmt.Sprintf("wm-%s", obj.Name) + + if v, ok := obj.Annotations[jobRefAnnotation]; !ok || v != jobName { + fn.MapSet(&obj.Annotations, jobRefAnnotation, jobName) + if err := r.Update(ctx, obj); err != nil { + return check.Failed(err) + } + } + + varfileJSON, err := r.parseSpecIntoTFValues(ctx, obj) + if err != nil { + return check.Failed(err) + } + + b, err := templates.ParseBytes(r.workmachineLifecycleTemplateSpec, templates.WorkMachineLifecycleVars{ + JobMetadata: metav1.ObjectMeta{ + Name: jobName, + Namespace: r.Env.IACJobsNamespace, + Labels: fn.MapFilterWithPrefix(obj.GetLabels(), "kloudlite.io/"), + Annotations: fn.FilterObservabilityAnnotations(obj.GetAnnotations()), + OwnerReferences: []metav1.OwnerReference{fn.AsOwner(obj, true)}, + }, + NodeSelector: obj.Spec.JobParams.NodeSelector, + Tolerations: obj.Spec.JobParams.Tolerations, + JobImage: r.Env.IACJobImage, + TFWorkspaceName: obj.Name, + TfWorkspaceNamespace: r.Env.TFStateSecretNamespace, + CloudProvider: obj.Spec.GetCloudProvider().String(), + ValuesJSON: string(varfileJSON), + + OutputSecretName: obj.Name + "-tf-outputs", + OutputSecretNamespace: r.Env.IACJobsNamespace, + + NodeName: obj.Name, + }) + if err != nil { + return check.Failed(err) + } + + lf := &crdsv1.Lifecycle{ObjectMeta: metav1.ObjectMeta{Name: jobName, Namespace: r.Env.IACJobsNamespace}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, lf, func() error { + lf.SetLabels(fn.MapMerge(fn.MapFilterWithPrefix(obj.GetLabels(), "kloudlite.io/"), lf.GetLabels())) + lf.SetAnnotations(fn.MapMerge(fn.MapFilterWithPrefix(obj.GetAnnotations(), "kloudlite.io/observability"), lf.GetAnnotations())) + lf.SetOwnerReferences([]metav1.OwnerReference{fn.AsOwner(obj, true)}) + return yaml.Unmarshal(b, &lf.Spec) + }); err != nil { + return check.Failed(err) + } + + if !lf.HasCompleted() { + return check.StillRunning(fmt.Errorf("waiting for lifecycle job to complete")) + } + + if lf.Status.Phase == crdsv1.JobPhaseFailed { + return check.Failed(fmt.Errorf("lifecycle job failed")) + } + + req.AddToOwnedResources(rApi.ParseResourceRef(lf)) + return check.Completed() } @@ -125,15 +268,14 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { r.recorder = mgr.GetEventRecorderFor(r.GetName()) var err error - r.workspaceDeploymentTemplate, err = templates.Read(templates.WorkspaceTemplate) + r.workmachineLifecycleTemplateSpec, err = templates.Read(templates.WorkMachineLifecycleTemplate) if err != nil { return err } + builder := ctrl.NewControllerManagedBy(mgr).For(&crdsv1.WorkMachine{}) builder.WithOptions(controller.Options{MaxConcurrentReconciles: r.Env.MaxConcurrentReconciles}) - builder.Owns(&appsv1.Deployment{}) - builder.Owns(&corev1.Service{}) - builder.Owns(&crdsv1.Router{}) - builder.WithEventFilter(rApi.ReconcileFilter()) + builder.Owns(&crdsv1.Lifecycle{}) + builder.WithEventFilter(rApi.ReconcileFilter(r.recorder)) return builder.Complete(r) } diff --git a/operators/workmachine/internal/env/env.go b/operators/workmachine/internal/env/env.go index ac4245c7..1c120632 100644 --- a/operators/workmachine/internal/env/env.go +++ b/operators/workmachine/internal/env/env.go @@ -7,12 +7,12 @@ import ( type Env struct { MaxConcurrentReconciles int `env:"MAX_CONCURRENT_RECONCILES"` - WorkspaceImageInitContainer string `env:"WORKSPACE_IMAGE_INIT_CONTAINER" default:"ghcr.io/kloudlite/iac/workspace:latest"` - WorkspaceImageSSH string `env:"WORKSPACE_IMAGE_SSH" default:"ghcr.io/kloudlite/iac/workspace:latest"` - WorkspaceImageTTYD string `env:"WORKSPACE_IMAGE_TTYD" default:"ghcr.io/kloudlite/iac/ttyd:latest"` - WorkspaceImageJupyterNotebook string `env:"WORKSPACE_IMAGE_JUPYTER_NOTEBOOK" default:"ghcr.io/kloudlite/iac/jupyter:latest"` - WorkspaceImageCodeServer string `env:"WORKSPACE_IMAGE_CODE_SERVER" default:"ghcr.io/kloudlite/iac/code-server:latest"` - WorkspcaeImageVscodeServer string `env:"WORKSPCAE_IMAGE_VSCODE_SERVER" default:"ghcr.io/kloudlite/iac/vscode-server:latest"` + K3sParamsSecretRef string `env:"K3S_PARAMS_SECRET_REF" required:"true"` + + IACJobsNamespace string `env:"IAC_JOBS_NAMESPACE" required:"true"` + IACJobImage string `env:"IAC_JOB_IMAGE" required:"true"` + + TFStateSecretNamespace string `env:"TF_STATE_SECRET_NAMESPACE" required:"true" default:"kloudlite"` } func GetEnvOrDie() *Env { diff --git a/operators/workmachine/internal/templates/embed.go b/operators/workmachine/internal/templates/embed.go index d94cb95f..a4336d51 100644 --- a/operators/workmachine/internal/templates/embed.go +++ b/operators/workmachine/internal/templates/embed.go @@ -13,7 +13,8 @@ var templatesDir embed.FS type templateFile string const ( - WorkspaceTemplate templateFile = "./workspace.yml.tpl" + WorkspaceTemplate templateFile = "./workspace.yml.tpl" + WorkMachineLifecycleTemplate templateFile = "./workmachine-lifecycle.yml.tpl" ) func Read(t templateFile) ([]byte, error) { diff --git a/operators/workmachine/internal/templates/types.go b/operators/workmachine/internal/templates/types.go index af36ab3e..34b02926 100644 --- a/operators/workmachine/internal/templates/types.go +++ b/operators/workmachine/internal/templates/types.go @@ -1,33 +1,25 @@ package templates import ( - crdsv1 "github.com/kloudlite/operator/apis/crds/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -type WorkspaceTemplateArgs struct { - Metadata metav1.ObjectMeta - NodeName string - ServiceAccountName string +type WorkMachineLifecycleVars struct { + JobMetadata metav1.ObjectMeta + NodeSelector map[string]string + Tolerations []corev1.Toleration + JobImage string - ImageInitContainer string - ImageSSH string + TFWorkspaceName string + TfWorkspaceNamespace string - EnableTTYD bool - ImageTTYD string + CloudProvider string - EnableJupyterNotebook bool - ImageJupyterNotebook string + ValuesJSON string - EnableCodeServer bool - ImageCodeServer string + OutputSecretName string + OutputSecretNamespace string - EnableVSCodeServer bool - ImageVscodeServer string - - ImagePullPolicy string - - KloudliteDeviceFQDN string - - RouterSpec crdsv1.RouterSpec + NodeName string } diff --git a/operators/workmachine/internal/templates/workmachine-lifecycle.yml.tpl b/operators/workmachine/internal/templates/workmachine-lifecycle.yml.tpl new file mode 100644 index 00000000..18d9f425 --- /dev/null +++ b/operators/workmachine/internal/templates/workmachine-lifecycle.yml.tpl @@ -0,0 +1,117 @@ +{{ with . }} +onApply: + backOffLimit: 1 + podSpec: + tolerations: &tolerations {{.Tolerations | toJson }} + nodeSelector: &node-selector {{.NodeSelector | toJson }} + + resources: + requests: + cpu: 500m + memory: 1000Mi + limits: + cpu: 500m + memory: 1000Mi + + containers: + - name: main + image: {{.JobImage}} + imagePullPolicy: Always + env: + - name: KUBE_IN_CLUSTER_CONFIG + value: "true" + + - name: KUBE_NAMESPACE + value: {{.TfWorkspaceNamespace | squote}} + command: + - bash + - -c + - |+ + set -o pipefail + set -o errexit + + eval $DECOMPRESS_CMD + + pushd "$TEMPLATES_DIR/{{.CloudProvider}}/work-machine" + + envsubst < state-backend.tf.tpl > state-backend.tf + + terraform init -reconfigure -no-color 2>&1 | tee /dev/termination-log + terraform workspace select --or-create {{.TFWorkspaceName}} + + cat > values.json <<'EOF' + {{.ValuesJSON}} + EOF + + terraform init -no-color 2>&1 | tee /dev/termination-log + terraform plan -parallelism=2 --var-file ./values.json -out=tfplan -no-color 2>&1 | tee /dev/termination-log + terraform apply -parallelism=2 -no-color tfplan 2>&1 | tee /dev/termination-log + + # terraform state pull | jq '.outputs' -r > outputs.json + cat > secret.yml < state-backend.tf + + terraform init -reconfigure -no-color 2>&1 | tee /dev/termination-log + terraform workspace select --or-create {{.TFWorkspaceName}} + + cat > values.json <<'EOF' + {{.ValuesJSON}} + EOF + + terraform init -no-color 2>&1 | tee /dev/termination-log + terraform plan -parallelism=2 --destroy --var-file ./values.json -out=tfplan -no-color 2>&1 | tee /dev/termination-log + terraform apply -parallelism=2 -no-color tfplan 2>&1 | tee /dev/termination-log + + kubectl delete secret/{{.OutputSecretName}} -n {{.OutputSecretNamespace}} --ignore-not-found=true + + kubectl delete node/{{.NodeName}} + restartPolicy: Never + +{{ end }} diff --git a/operators/workmachine/internal/templates/workspace.yml.tpl b/operators/workmachine/internal/templates/workspace.yml.tpl deleted file mode 100644 index c670c0b8..00000000 --- a/operators/workmachine/internal/templates/workspace.yml.tpl +++ /dev/null @@ -1,229 +0,0 @@ -{{- with . }} -apiVersion: apps/v1 -kind: Deployment -metadata: {{.Metadata | toJson }} -spec: - selector: - matchLabels: - app: {{.Metadata.Name | squote}} - template: - metadata: - labels: - app: {{.Metadata.Name | squote}} - kloudlite.io/gateway.enabled: "false" - spec: - hostname: {{.Metadata.Name}} - nodeName: {{.NodeName}} - serviceAccount: {{.ServiceAccountName | squote}} - tolerations: - - key: "kloudlite.io/worknode" - operator: "Equal" - value: {{.NodeName |squote}} - effect: "NoExecute" - initContainers: - - name: init-home-dir - image: {{.ImageInitContainer}} - imagePullPolicy: Always - env: - - name: KL_WORKSPACE - value: {{.Metadata.Name}} - - - name: HOME - value: "/home/kl" - - - name: KL_BOX_MODE - value: "true" - - securityContext: - runAsUser: 1000 - runAsGroup: 1000 - command: - - "bash" - - "-c" - - | - set -e - set +x - if [ ! -d "/nix/store" ]; then - curl -L https://nixos.org/nix/install | sh - mkdir -p ~/.config/nix - echo 'experimental-features = nix-command flakes' > ~/.config/nix/nix.conf - fi - - kl_bin_dir="/home/kl/.local/bin" - if [ ! -f "$kl_bin_dir/kl" ]; then - mkdir -p $kl_bin_dir - pushd $kl_bin_dir - curl https://i.jpillora.com/kloudlite/kl@v1.1.87-nightly | bash - popd - fi - - workspace_dir="/home/kl/workspaces/$(KL_WORKSPACE)" - if [ ! -d "$workspace_dir" ]; then - mkdir -p $workspace_dir - pushd $workspace_dir - export PATH=$PATH:/home/kl/.nix-profile/bin:/home/kl/.local/bin - kl init - popd - fi - - if [ -f "$workspace_dir/kl.yaml" ] || [ -f "$workspace_dir/kl.yml" ]; then - pushd $workspace_dir - PATH=$PATH:/home/kl/.nix-profile/bin:/home/kl/.local/bin /home/kl/.local/bin/kl shell -r > /env/.env - PATH=$PATH:/home/kl/.nix-profile/bin:/home/kl/.local/bin /home/kl/.local/bin/kl get env > /env/.connected_env - popd - fi - - if [ ! -f "/home/kl/.zshrc" ]; then - mkdir -p "/home/kl/.config/zsh" - cp /tmp/.zshrc /home/kl/.zshrc - cp /tmp/.aliasrc /home/kl/.config/aliasrc - fi - - if [ ! -f "/home/kl/.local/bin/starship" ]; then - curl -sS https://starship.rs/install.sh | sh -s -- -y -b /home/kl/.local/bin - fi - - if [ ! -d "/home/kl/.config/zsh/zsh-autosuggestions" ]; then - mkdir -p "/home/kl/.config/zsh" - git clone https://github.com/zsh-users/zsh-autosuggestions - fi - - if [ ! -d "/home/kl/.config/zsh/zsh-syntax-highlighting" ]; then - mkdir -p "/home/kl/.config/zsh" - git clone https://github.com/zsh-users/zsh-syntax-highlighting.git "/home/kl/.config/zsh/zsh-syntax-highlighting" - fi - - {{- /* if [ ! -d "/home/kl/.kl" ]; then */}} - {{- /* mkdir -p /home/kl/.kl */}} - {{- /* sh -c 'cat > /home/kl/.kl/kl-session.yaml <") flag.Parse() - if debugLog { - os.Setenv("LOG_DEBUG", "true") + if isDev { + debug = true } - ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts), zap.Level(zapcore.DebugLevel))) mgrConfig, mgrOptions := func() (*rest.Config, ctrl.Options) { cOpts := ctrl.Options{ @@ -126,19 +124,30 @@ func New(name string) Operator { return ctrl.GetConfigOrDie(), cOpts }() - logger := logging.New(ctrl.Log) + logger := log.New(log.Options{ + Writer: os.Stderr, + ShowTimestamp: false, + ShowCaller: true, + ShowDebugLogs: debug, + ShowLogLevel: true, + JSONFormat: false, + }) + + log.SetDefaultLogger(logger) + k8sYamlClient, err := kubectl.NewYAMLClient(mgrConfig, kubectl.YAMLClientOpts{Logger: logger}) if err != nil { - log.Fatalln(err) + logger.Fatal("failed to create YAML client", "err", err) } + logger.Debug("Starting .............") + return &operator{ startedAt: time.Now(), mgrConfig: mgrConfig, mgrOptions: mgrOptions, IsDev: isDev, k8sYamlClient: k8sYamlClient, - logger: logger, } } @@ -200,7 +209,7 @@ func (op *operator) Operator() *operator { func (op *operator) Start() { mgr, err := ctrl.NewManager(op.mgrConfig, op.mgrOptions) if err != nil { - log.Fatalln(err) + log.DefaultLogger().Fatal("failed to create new controller runtime manager", "err", err) } for i := range op.controllers { diff --git a/toolkit/reconciler/checks.go b/toolkit/reconciler/checks.go index f2c403da..080b631b 100644 --- a/toolkit/reconciler/checks.go +++ b/toolkit/reconciler/checks.go @@ -1,6 +1,8 @@ package reconciler import ( + "fmt" + "runtime" "time" fn "github.com/kloudlite/operator/toolkit/functions" @@ -57,6 +59,9 @@ type checkWrapper[T Resource] struct { func (cw *checkWrapper[T]) Failed(err error) step_result.Result { defer cw.request.LogPostCheck(cw.checkName) + _, file, line, _ := runtime.Caller(1) + cw.request.Logger.Debug("check.failed", "err", err, "caller", fmt.Sprintf("%s:%d", file, line)) + cw.Check.State = ErroredState cw.Check.Status = false if err != nil { diff --git a/toolkit/reconciler/request.go b/toolkit/reconciler/request.go index 89f16b8a..7f5e4a00 100644 --- a/toolkit/reconciler/request.go +++ b/toolkit/reconciler/request.go @@ -11,6 +11,7 @@ import ( "time" "github.com/fatih/color" + "github.com/nxtcoder17/go.pkgs/log" apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -19,10 +20,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/log" fn "github.com/kloudlite/operator/toolkit/functions" - "github.com/kloudlite/operator/toolkit/logging" stepResult "github.com/kloudlite/operator/toolkit/reconciler/step-result" ) @@ -89,14 +88,21 @@ func NewRequest[T Resource](ctx context.Context, c client.Client, nn types.Names resource.GetStatus().Checks = map[string]Check{} } - logger := log.FromContext(ctx, "NN", nn.String()) + resource.EnsureGVK() + + // for i := 1; i < 5; i++ { + // _, file, line, _ := runtime.Caller(i) + // // logger := logging.New(ctrl.Log, logging.WithCallDepth(1)).With("NN", nn.String(), "gvk", resource.GetObjectKind().GroupVersionKind().String()) + // logger := log.DefaultLogger().SkipFrames(1).With("NN", nn.String(), "gvk", resource.GetObjectKind().GroupVersionKind().String()) + // logger.Debug(fmt.Sprintf("CALLER [skip frame: %d]: %s:%d\n", i, file, line)) + // } return &Request[T]{ ctx: ctx, client: c, Object: resource, - Logger: logging.New(logger), - internalLogger: logging.New(logger, logging.WithCallDepth(3)), + Logger: log.DefaultLogger().SkipFrames(1).With("NN", nn.String(), "gvk", resource.GetObjectKind().GroupVersionKind().String()).Slog(), + internalLogger: log.DefaultLogger().SkipFrames(4).With("NN", nn.String(), "gvk", resource.GetObjectKind().GroupVersionKind().String()).Slog(), anchorName: anchorName, KV: KV{}, timerMap: map[string]time.Time{}, From eb51e194e3f4ff0a2fb6ff6150f28711c778bc2f Mon Sep 17 00:00:00 2001 From: nxtcoder17 Date: Wed, 2 Apr 2025 17:45:12 +0530 Subject: [PATCH 04/27] chore: router updates --- apis/common-types/types.go | 5 + apis/crds/v1/router_types.go | 14 ++- config/crd/bases/crds.kloudlite.io_apps.yaml | 6 + .../bases/crds.kloudlite.io_blueprints.yaml | 6 + .../crd/bases/crds.kloudlite.io_routers.yaml | 6 + .../internal/router-controller/controller.go | 106 ++++++++++-------- .../router-controller/controller_test.go | 10 +- .../internal/router-controller/helpers.go | 1 + operators/routers/internal/templates/embed.go | 11 +- .../templates/ingress-resource-v2.yml.tpl | 42 +++++++ operators/routers/internal/templates/types.go | 18 +++ 11 files changed, 166 insertions(+), 59 deletions(-) create mode 100644 operators/routers/internal/templates/ingress-resource-v2.yml.tpl create mode 100644 operators/routers/internal/templates/types.go diff --git a/apis/common-types/types.go b/apis/common-types/types.go index f4f1ac57..c396cc60 100644 --- a/apis/common-types/types.go +++ b/apis/common-types/types.go @@ -117,7 +117,12 @@ type MinMaxInt struct { // +kubebuilder:validation:Enum=aws;do;azure;gcp type CloudProvider string +func (c CloudProvider) String() string { + return string(c) +} + const ( + CloudProviderUnknown CloudProvider = "unknown" CloudProviderAWS CloudProvider = "aws" CloudProviderDigitalOcean CloudProvider = "digitalocean" CloudProviderAzure CloudProvider = "azure" diff --git a/apis/crds/v1/router_types.go b/apis/crds/v1/router_types.go index 2ae0a4ab..0a1d0d75 100644 --- a/apis/crds/v1/router_types.go +++ b/apis/crds/v1/router_types.go @@ -55,9 +55,16 @@ type RouterSpec struct { RateLimit *RateLimit `json:"rateLimit,omitempty"` MaxBodySizeInMB *int `json:"maxBodySizeInMB,omitempty"` Domains []string `json:"domains"` - Routes []Route `json:"routes,omitempty"` - BasicAuth *BasicAuth `json:"basicAuth,omitempty"` - Cors *Cors `json:"cors,omitempty"` + + // Routes is a map of [domain name] to the corresponding []Route + // Routes map[string][]Route `json:"routes"` + + Routes []Route `json:"routes,omitempty"` + BasicAuth *BasicAuth `json:"basicAuth,omitempty"` + Cors *Cors `json:"cors,omitempty"` + + // NginxIngressAnnotations is additional list of annotations on ingress resource + NginxIngressAnnotations map[string]string `json:"nginxIngressAnnotations,omitempty"` } // +kubebuilder:object:root=true @@ -98,6 +105,7 @@ func (r *Router) GetEnsuredLabels() map[string]string { func (m *Router) GetEnsuredAnnotations() map[string]string { return map[string]string{ + // "kloudlite.io/router.domains": strings.Join(fn.MapKeys(m.Spec.Routes), ","), "kloudlite.io/router.domains": strings.Join(m.Spec.Domains, ","), "kloudlite.io/router.ingress-class": m.Spec.IngressClass, } diff --git a/config/crd/bases/crds.kloudlite.io_apps.yaml b/config/crd/bases/crds.kloudlite.io_apps.yaml index e9ea3e0b..e8017c80 100644 --- a/config/crd/bases/crds.kloudlite.io_apps.yaml +++ b/config/crd/bases/crds.kloudlite.io_apps.yaml @@ -363,6 +363,12 @@ spec: type: string maxBodySizeInMB: type: integer + nginxIngressAnnotations: + additionalProperties: + type: string + description: NginxIngressAnnotations is additional list of annotations + on ingress resource + type: object rateLimit: properties: connections: diff --git a/config/crd/bases/crds.kloudlite.io_blueprints.yaml b/config/crd/bases/crds.kloudlite.io_blueprints.yaml index 4bf216b0..0dd4ed67 100644 --- a/config/crd/bases/crds.kloudlite.io_blueprints.yaml +++ b/config/crd/bases/crds.kloudlite.io_blueprints.yaml @@ -368,6 +368,12 @@ spec: type: string maxBodySizeInMB: type: integer + nginxIngressAnnotations: + additionalProperties: + type: string + description: NginxIngressAnnotations is additional list + of annotations on ingress resource + type: object rateLimit: properties: connections: diff --git a/config/crd/bases/crds.kloudlite.io_routers.yaml b/config/crd/bases/crds.kloudlite.io_routers.yaml index 4f86f74d..2c183857 100644 --- a/config/crd/bases/crds.kloudlite.io_routers.yaml +++ b/config/crd/bases/crds.kloudlite.io_routers.yaml @@ -106,6 +106,12 @@ spec: type: string maxBodySizeInMB: type: integer + nginxIngressAnnotations: + additionalProperties: + type: string + description: NginxIngressAnnotations is additional list of annotations + on ingress resource + type: object rateLimit: properties: connections: diff --git a/operators/routers/internal/router-controller/controller.go b/operators/routers/internal/router-controller/controller.go index cc2d3fb8..d0ff815e 100644 --- a/operators/routers/internal/router-controller/controller.go +++ b/operators/routers/internal/router-controller/controller.go @@ -61,18 +61,6 @@ const ( certCreatedByRouter string = "kloudlite.io/cert-created-by-router" ) -var ( - ApplyChecklist = []reconciler.CheckMeta{ - {Name: DefaultsPatched, Title: "Defaults Patched"}, - {Name: EnsuringHttpsCertsIfEnabled, Title: "Ensuring HTTPS Cert if enabled"}, - {Name: SettingUpBasicAuthIfEnabled, Title: "Setting Up Basic Auth if enabled"}, - } - - DeleteChecklist = []reconciler.CheckMeta{ - {Name: CleaningUpResources, Title: "Cleaning Up Resources"}, - } -) - // +kubebuilder:rbac:groups=crds.kloudlite.io,resources=crds,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=crds.kloudlite.io,resources=crds/status,verbs=get;update;patch // +kubebuilder:rbac:groups=crds.kloudlite.io,resources=crds/finalizers,verbs=update @@ -101,7 +89,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. return step.ReconcilerResponse() } - if step := req.EnsureCheckList(ApplyChecklist); !step.ShouldProceed() { + if step := req.EnsureCheckList([]reconciler.CheckMeta{ + {Name: DefaultsPatched, Title: "Defaults Patched"}, + {Name: EnsuringHttpsCertsIfEnabled, Title: "Ensuring HTTPS Cert if enabled"}, + {Name: SettingUpBasicAuthIfEnabled, Title: "Setting Up Basic Auth if enabled"}, + }); !step.ShouldProceed() { return step.ReconcilerResponse() } @@ -329,45 +321,61 @@ func (r *Reconciler) ensureIngresses(req *reconciler.Request[*crdsv1.Router]) st nginxIngressAnnotations := GenNginxIngressAnnotations(obj) - if len(obj.Spec.Routes) > 0 { - b, err := templates.ParseBytes( - r.templateIngress, map[string]any{ - "name": obj.Name, - "namespace": obj.Namespace, - - "owner-refs": []metav1.OwnerReference{fn.AsOwner(obj, true)}, - "labels": obj.GetLabels(), - "annotations": nginxIngressAnnotations, - - "non-wildcard-domains": nonWcDomains, - "wildcard-domains": wcDomains, - "router-domains": obj.Spec.Domains, - - "ingress-class": obj.Spec.IngressClass, - "cluster-issuer": func() string { - if obj.Spec.Https != nil && obj.Spec.Https.ClusterIssuer != "" { - return obj.Spec.Https.ClusterIssuer - } - return r.Env.DefaultClusterIssuer - }(), - - "routes": obj.Spec.Routes, - - "is-https-enabled": isHttpsEnabled(obj), - }, - ) - if err != nil { - return check.Failed(err).Err(nil) - } + if len(obj.Spec.Routes) == 0 { + return check.Completed() + } - rr, err := r.YAMLClient.ApplyYAML(ctx, b) - if err != nil { - return check.StillRunning(err) - } + // b, err := templates.ParseBytes(r.templateIngress, templates.IngressTemplateArgs{ + // Metadata: metav1.ObjectMeta{ + // Name: obj.Name, + // Namespace: obj.Namespace, + // Labels: obj.GetLabels(), + // Annotations: nginxIngressAnnotations, + // }, + // IngressClassName: obj.Spec.IngressClass, + // HttpsEnabled: isHttpsEnabled(obj), + // WildcardDomains: wcDomains, + // NonWildcardDomains: nonWcDomains, + // Routes: obj.Spec.Routes, + // }) + + b, err := templates.ParseBytes( + r.templateIngress, map[string]any{ + "name": obj.Name, + "namespace": obj.Namespace, + + "owner-refs": []metav1.OwnerReference{fn.AsOwner(obj, true)}, + "labels": obj.GetLabels(), + "annotations": nginxIngressAnnotations, + + "non-wildcard-domains": nonWcDomains, + "wildcard-domains": wcDomains, + "router-domains": obj.Spec.Routes, + + "ingress-class": obj.Spec.IngressClass, + "cluster-issuer": func() string { + if obj.Spec.Https != nil && obj.Spec.Https.ClusterIssuer != "" { + return obj.Spec.Https.ClusterIssuer + } + return r.Env.DefaultClusterIssuer + }(), + + "routes": obj.Spec.Routes, + + "is-https-enabled": isHttpsEnabled(obj), + }, + ) + if err != nil { + return check.Failed(err).Err(nil) + } - req.AddToOwnedResources(rr...) + rr, err := r.YAMLClient.ApplyYAML(ctx, b) + if err != nil { + return check.StillRunning(err) } + req.AddToOwnedResources(rr...) + return check.Completed() } @@ -380,7 +388,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { } var err error - r.templateIngress, err = templates.ReadIngressTemplate() + r.templateIngress, err = templates.Read(templates.IngressTemplate) if err != nil { return err } diff --git a/operators/routers/internal/router-controller/controller_test.go b/operators/routers/internal/router-controller/controller_test.go index 047ac4ef..c89ac7dd 100644 --- a/operators/routers/internal/router-controller/controller_test.go +++ b/operators/routers/internal/router-controller/controller_test.go @@ -29,7 +29,7 @@ func newRouter() crdsv1.Router { }, // RateLimit: crdsv1.RateLimit{}, // MaxBodySizeInMB: 0, - Domains: []string{"sample.example.com"}, + Routes: []string{"sample.example.com"}, Routes: []crdsv1.Route{ { App: "example", @@ -127,14 +127,14 @@ var _ = Describe("router controller [UPDATE] says", func() { It("adding a new domain, reflects in each of the owned k8s ingresses", func() { _, err := controllerutil.CreateOrUpdate(Suite.Context, Suite.K8sClient, &routerCr, func() error { - routerCr.Spec.Domains = append(routerCr.Spec.Domains, "dummy.example.com") + routerCr.Spec.Routes = append(routerCr.Spec.Routes, "dummy.example.com") return nil }) Expect(err).NotTo(HaveOccurred()) - dMap := make(map[string]bool, len(routerCr.Spec.Domains)) - for i := range routerCr.Spec.Domains { - dMap[routerCr.Spec.Domains[i]] = false + dMap := make(map[string]bool, len(routerCr.Spec.Routes)) + for i := range routerCr.Spec.Routes { + dMap[routerCr.Spec.Routes[i]] = false } Promise(func(g Gomega) { diff --git a/operators/routers/internal/router-controller/helpers.go b/operators/routers/internal/router-controller/helpers.go index 20131281..f84c43fc 100644 --- a/operators/routers/internal/router-controller/helpers.go +++ b/operators/routers/internal/router-controller/helpers.go @@ -41,6 +41,7 @@ func (r *Reconciler) parseAndExtractDomains(req *reconciler.Request[*crdsv1.Rout } } + // wildcardDomains, nonWildcardDomains := FilterDomains(wildcardPatterns, fn.MapKeys(obj.Spec.Routes)) wildcardDomains, nonWildcardDomains := FilterDomains(wildcardPatterns, obj.Spec.Domains) return wildcardDomains, nonWildcardDomains, nil } diff --git a/operators/routers/internal/templates/embed.go b/operators/routers/internal/templates/embed.go index f17e7d40..942d0b82 100644 --- a/operators/routers/internal/templates/embed.go +++ b/operators/routers/internal/templates/embed.go @@ -9,8 +9,15 @@ import ( //go:embed * var templatesDir embed.FS -func ReadIngressTemplate() ([]byte, error) { - return templatesDir.ReadFile("ingress-resource.yml.tpl") +type templateFile string + +const ( + // IngressTemplate templateFile = "./ingress-resource-v2.yml.tpl" + IngressTemplate templateFile = "./ingress-resource.yml.tpl" +) + +func Read(t templateFile) ([]byte, error) { + return templatesDir.ReadFile(string(t)) } var ParseBytes = templates.ParseBytes diff --git a/operators/routers/internal/templates/ingress-resource-v2.yml.tpl b/operators/routers/internal/templates/ingress-resource-v2.yml.tpl new file mode 100644 index 00000000..cd4dc57f --- /dev/null +++ b/operators/routers/internal/templates/ingress-resource-v2.yml.tpl @@ -0,0 +1,42 @@ +{{ with . }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: {{.Metadata | toJson}} +spec: + ingressClassName: {{.IngressClassName}} + {{- if .HttpsEnabled }} + tls: + {{- range $v := .NonWildcardDomains }} + - hosts: + - {{$v | squote}} + secretName: {{$v}}-tls + {{- end}} + + {{- range $v := .WildcardDomains }} + - hosts: + - {{$v | squote}} + {{- end}} + {{- end}} + + rules: + {{- range $host, $routes := .Routes }} + - host: {{$host}} + http: + paths: + {{- range $route := $routes }} + - pathType: Prefix + backend: + service: + name: {{$route.App}} + port: + number: {{$route.Port}} + + {{- if $route.Rewrite }} + path: {{$route.Path}}?(.*) + {{- else }} + {{ $x := len $route.Path }} + path: /({{if hasPrefix "/" $route.Path }}{{substr 1 $x $route.Path}}{{else}}{{$route.Path}}{{end}}.*) + {{- end}} + {{- end}} + {{- end }} +{{- end }} diff --git a/operators/routers/internal/templates/types.go b/operators/routers/internal/templates/types.go new file mode 100644 index 00000000..225824d9 --- /dev/null +++ b/operators/routers/internal/templates/types.go @@ -0,0 +1,18 @@ +package templates + +import ( + crdsv1 "github.com/kloudlite/operator/apis/crds/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type IngressTemplateArgs struct { + Metadata metav1.ObjectMeta + IngressClassName string + + HttpsEnabled bool + + WildcardDomains []string + NonWildcardDomains []string + + Routes map[string][]crdsv1.Route +} From 5aef30dfe10834194bcf9b7f4201bee1ddb8bbfc Mon Sep 17 00:00:00 2001 From: nxtcoder17 Date: Wed, 9 Apr 2025 19:40:19 +0530 Subject: [PATCH 05/27] WIP: workmachine ssh jumpserver --- apis/crds/v1/workmachine_types.go | 3 + apis/crds/v1/workspace_types.go | 3 + cmd/agent-operator/Taskfile.yml | 2 +- .../bases/crds.kloudlite.io_workmachines.yaml | 5 + .../controllers/workmachine/controller.go | 122 +++++++++++++++++- .../workmachine/internal/templates/embed.go | 1 + .../ssh-jumpserver-deployment-spec.yml.tpl | 26 ++++ .../workmachine/internal/templates/types.go | 6 + .../controllers/workspace/controller.go | 5 +- .../workspace/internal/templates/embed.go | 2 +- .../internal/templates/workspace.yml.tpl | 2 + 11 files changed, 168 insertions(+), 9 deletions(-) create mode 100644 operators/workmachine/internal/templates/ssh-jumpserver-deployment-spec.yml.tpl diff --git a/apis/crds/v1/workmachine_types.go b/apis/crds/v1/workmachine_types.go index 81395133..f2333da3 100644 --- a/apis/crds/v1/workmachine_types.go +++ b/apis/crds/v1/workmachine_types.go @@ -50,6 +50,8 @@ type WorkMachineSpec struct { JobParams WorkMachineJobParams `json:"jobParams"` + TargetNamespace string `json:"targetNamespace,omitempty"` + AWSMachineConfig *AWSMachineConfig `json:"aws"` } @@ -69,6 +71,7 @@ type WorkMachineStatus struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:scope=Cluster +// +kubebuilder:printcolumn:JSONPath=".spec.targetNamespace",name=TargetNamespace,type=string // +kubebuilder:printcolumn:JSONPath=".status.status.lastReconcileTime",name=Seen,type=date // +kubebuilder:printcolumn:JSONPath=".metadata.annotations.kloudlite\\.io\\/operator\\.checks",name=Checks,type=string // +kubebuilder:printcolumn:JSONPath=".metadata.annotations.kloudlite\\.io\\/operator\\.resource\\.ready",name=Ready,type=string diff --git a/apis/crds/v1/workspace_types.go b/apis/crds/v1/workspace_types.go index 74451347..bfd8f08b 100644 --- a/apis/crds/v1/workspace_types.go +++ b/apis/crds/v1/workspace_types.go @@ -17,6 +17,9 @@ const ( // WorkspaceSpec defines the desired state of Workspace type WorkspaceSpec struct { + // Name of work machine + WorkMachine string `json:"workMachine"` + State WorkspaceState `json:"state"` NodeName string `json:"nodeName"` ServiceAccountName string `json:"serviceAccountName"` diff --git a/cmd/agent-operator/Taskfile.yml b/cmd/agent-operator/Taskfile.yml index 625d271b..99cf8e53 100644 --- a/cmd/agent-operator/Taskfile.yml +++ b/cmd/agent-operator/Taskfile.yml @@ -13,7 +13,7 @@ tasks: dotenv: - .secrets/env cmds: - - go run . --dev + - go run . --dev --serverHost localhost:8081 build: cmds: diff --git a/config/crd/bases/crds.kloudlite.io_workmachines.yaml b/config/crd/bases/crds.kloudlite.io_workmachines.yaml index 3c72ba45..2081e64e 100644 --- a/config/crd/bases/crds.kloudlite.io_workmachines.yaml +++ b/config/crd/bases/crds.kloudlite.io_workmachines.yaml @@ -15,6 +15,9 @@ spec: scope: Cluster versions: - additionalPrinterColumns: + - jsonPath: .spec.targetNamespace + name: TargetNamespace + type: string - jsonPath: .status.status.lastReconcileTime name: Seen type: date @@ -142,6 +145,8 @@ spec: - "ON" - "OFF" type: string + targetNamespace: + type: string required: - aws - jobParams diff --git a/operators/workmachine/internal/controllers/workmachine/controller.go b/operators/workmachine/internal/controllers/workmachine/controller.go index 7474fe91..61d84902 100644 --- a/operators/workmachine/internal/controllers/workmachine/controller.go +++ b/operators/workmachine/internal/controllers/workmachine/controller.go @@ -12,10 +12,12 @@ import ( crdsv1 "github.com/kloudlite/operator/apis/crds/v1" "github.com/kloudlite/operator/operators/workmachine/internal/env" "github.com/kloudlite/operator/operators/workmachine/internal/templates" + "github.com/kloudlite/operator/pkg/constants" fn "github.com/kloudlite/operator/toolkit/functions" "github.com/kloudlite/operator/toolkit/kubectl" rApi "github.com/kloudlite/operator/toolkit/reconciler" step_result "github.com/kloudlite/operator/toolkit/reconciler/step-result" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -36,6 +38,7 @@ type Reconciler struct { workmachineLifecycleTemplateSpec []byte templateWebhook []byte + templateJumpServerDeploymentSpec []byte } func (r *Reconciler) GetName() string { @@ -43,7 +46,15 @@ func (r *Reconciler) GetName() string { } const ( - createWorkMachineJob string = "create-work-machine-job" + createWorkMachineJob string = "create-work-machine-job" + createTargetNamespace string = "create-target-namespace" + createSSHPublicKeysSecret string = "create-ssh-public-keys-secret" + createSSHJumpServerDeployment string = "create-ssh-jumpserver-deployment" +) + +const ( + sshPublicKeysSecretName string = "ssh-public-keys" + authorizedKeysSecretKey string = "authorized_keys" ) const ( @@ -76,11 +87,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. return step.ReconcilerResponse() } - if step := req.EnsureCheckList([]rApi.CheckMeta{{Name: createWorkMachineJob}}); !step.ShouldProceed() { - return step.ReconcilerResponse() - } - - if step := req.RestartIfAnnotated(); !step.ShouldProceed() { + if step := req.EnsureCheckList([]rApi.CheckMeta{ + {Name: createWorkMachineJob, Title: "Creates WorkMachine creation job"}, + {Name: createTargetNamespace, Title: "Creates a target namespace for workmachine"}, + {Name: createSSHPublicKeysSecret, Title: "store SSH public keys in a secret"}, + }); !step.ShouldProceed() { return step.ReconcilerResponse() } @@ -96,6 +107,18 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. return step.ReconcilerResponse() } + if step := r.createTargetNamespace(req); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + + if step := r.createSSHPublicKeysSecret(req); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + + if step := r.createSSHJumpServer(req); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + req.Object.Status.IsReady = true return ctrl.Result{}, nil } @@ -257,6 +280,88 @@ func (r *Reconciler) createWorkMachineCreationJob(req *rApi.Request[*crdsv1.Work return check.Completed() } +func (r *Reconciler) createTargetNamespace(req *rApi.Request[*crdsv1.WorkMachine]) step_result.Result { + ctx, obj := req.Context(), req.Object + check := rApi.NewRunningCheck(createTargetNamespace, req) + + hasUpdate := false + if obj.Spec.TargetNamespace == "" { + hasUpdate = true + obj.Spec.TargetNamespace = "wm-" + obj.Name + } + + if hasUpdate { + if err := r.Update(ctx, obj); err != nil { + return check.Failed(err) + } + + return check.StillRunning(fmt.Errorf("waiting for reconcilation")) + } + + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: obj.Spec.TargetNamespace}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, ns, func() error { + fn.MapSet(&ns.Annotations, "kloudlite.io/namespace.for", fmt.Sprintf("workmachine/%s", obj.Name)) + ns.SetOwnerReferences([]metav1.OwnerReference{fn.AsOwner(obj, true)}) + return nil + }); err != nil { + return check.Failed(err) + } + + return check.Completed() +} + +func (r *Reconciler) createSSHPublicKeysSecret(req *rApi.Request[*crdsv1.WorkMachine]) step_result.Result { + ctx, obj := req.Context(), req.Object + check := rApi.NewRunningCheck(createSSHPublicKeysSecret, req) + + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: sshPublicKeysSecretName, Namespace: obj.Spec.TargetNamespace}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, secret, func() error { + fn.MapSet(&secret.Annotations, "kloudlite.io/description", "this secret contains ssh public keys given by user in workmachine resource") + fn.MapSet(&secret.Annotations, "kloudlite.io/secret.for", fmt.Sprintf("workmachine/%s", obj.Name)) + secret.SetOwnerReferences([]metav1.OwnerReference{fn.AsOwner(obj, true)}) + + if secret.StringData == nil { + secret.StringData = make(map[string]string, 1) + } + + secret.StringData[authorizedKeysSecretKey] = strings.Join(obj.Spec.SSHPublicKeys, "\n") + return nil + }); err != nil { + return check.Failed(err) + } + + return check.Completed() +} + +func (r *Reconciler) createSSHJumpServer(req *rApi.Request[*crdsv1.WorkMachine]) step_result.Result { + ctx, obj := req.Context(), req.Object + check := rApi.NewRunningCheck(createSSHJumpServerDeployment, req) + + sshJumpServerName := "ssh-jump-server" + + b, err := templates.ParseBytes(r.templateJumpServerDeploymentSpec, templates.JumpServerDeploymentSpecTemplateArgs{ + SSHAuthorizedKeysSecretName: sshPublicKeysSecretName, + SSHAuthorizedKeysSecretKey: authorizedKeysSecretKey, + SelectorLabels: map[string]string{ + "app": "jump-server", + }, + }) + if err != nil { + return check.Failed(err) + } + + deployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: sshJumpServerName, Namespace: obj.Namespace}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { + deployment.SetOwnerReferences([]metav1.OwnerReference{fn.AsOwner(obj, true)}) + fn.MapSet(&deployment.Annotations, constants.DescriptionKey, "this deployment is a ssh jump server used to allow users to jump to different workspaces") + return yaml.Unmarshal(b, &deployment.Spec) + }); err != nil { + return check.Failed(err) + } + + return check.Completed() +} + func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { r.Client = mgr.GetClient() r.Scheme = mgr.GetScheme() @@ -273,6 +378,11 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { return err } + r.templateJumpServerDeploymentSpec, err = templates.Read(templates.JumpServerDeploymentSpec) + if err != nil { + return err + } + builder := ctrl.NewControllerManagedBy(mgr).For(&crdsv1.WorkMachine{}) builder.WithOptions(controller.Options{MaxConcurrentReconciles: r.Env.MaxConcurrentReconciles}) builder.Owns(&crdsv1.Lifecycle{}) diff --git a/operators/workmachine/internal/templates/embed.go b/operators/workmachine/internal/templates/embed.go index a4336d51..b4589382 100644 --- a/operators/workmachine/internal/templates/embed.go +++ b/operators/workmachine/internal/templates/embed.go @@ -15,6 +15,7 @@ type templateFile string const ( WorkspaceTemplate templateFile = "./workspace.yml.tpl" WorkMachineLifecycleTemplate templateFile = "./workmachine-lifecycle.yml.tpl" + JumpServerDeploymentSpec templateFile = "./ssh-jumpserver-deployment-spec.yml.tpl" ) func Read(t templateFile) ([]byte, error) { diff --git a/operators/workmachine/internal/templates/ssh-jumpserver-deployment-spec.yml.tpl b/operators/workmachine/internal/templates/ssh-jumpserver-deployment-spec.yml.tpl new file mode 100644 index 00000000..3642ced5 --- /dev/null +++ b/operators/workmachine/internal/templates/ssh-jumpserver-deployment-spec.yml.tpl @@ -0,0 +1,26 @@ +{{- with . }} +replicas: 1 +selector: + matchLabels: {{ .SelectorLabels | toJson }} +template: + metadata: + labels: {{.SelectorLabels | toJson }} + spec: + containers: + - name: sshd-server + image: ghcr.io/kloudlite/hub/ssh-server + ports: + - containerPort: 22 # Internal port used by the image + volumeMounts: + - name: ssh-secret + mountPath: /home/kl/.ssh/authorized_keys + subPath: authorized_keys + + volumes: + - name: ssh-secret + secret: + secretName: {{.SSHAuthorizedKeysSecretName}} + items: + - key: "{{.SSHAuthorizedKeysSecretKey}}" + path: "authorized_keys" +{{- end }} diff --git a/operators/workmachine/internal/templates/types.go b/operators/workmachine/internal/templates/types.go index 34b02926..8fd59180 100644 --- a/operators/workmachine/internal/templates/types.go +++ b/operators/workmachine/internal/templates/types.go @@ -23,3 +23,9 @@ type WorkMachineLifecycleVars struct { NodeName string } + +type JumpServerDeploymentSpecTemplateArgs struct { + SSHAuthorizedKeysSecretName string + SSHAuthorizedKeysSecretKey string + SelectorLabels map[string]string +} diff --git a/operators/workspace/internal/controllers/workspace/controller.go b/operators/workspace/internal/controllers/workspace/controller.go index ba41c5b0..a209a721 100644 --- a/operators/workspace/internal/controllers/workspace/controller.go +++ b/operators/workspace/internal/controllers/workspace/controller.go @@ -34,6 +34,7 @@ type Reconciler struct { recorder record.EventRecorder workspaceDeploymentTemplate []byte + jumpServerTemplate []byte templateWebhook []byte } @@ -150,7 +151,8 @@ func (r *Reconciler) createDeployment(req *rApi.Request[*crdsv1.Workspace]) step ImagePullPolicy: obj.Spec.ImagePullPolicy, KloudliteDeviceFQDN: fmt.Sprintf("%s.%s.svc.cluster.local", obj.Name, obj.Namespace), - RouterSpec: obj.Spec.Router, + + RouterSpec: obj.Spec.Router, }) if err != nil { return check.Failed(err) @@ -197,6 +199,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { if err != nil { return err } + builder := ctrl.NewControllerManagedBy(mgr).For(&crdsv1.Workspace{}) builder.WithOptions(controller.Options{MaxConcurrentReconciles: r.Env.MaxConcurrentReconciles}) builder.Owns(&appsv1.Deployment{}) diff --git a/operators/workspace/internal/templates/embed.go b/operators/workspace/internal/templates/embed.go index d94cb95f..33c17743 100644 --- a/operators/workspace/internal/templates/embed.go +++ b/operators/workspace/internal/templates/embed.go @@ -13,7 +13,7 @@ var templatesDir embed.FS type templateFile string const ( - WorkspaceTemplate templateFile = "./workspace.yml.tpl" + WorkspaceTemplate templateFile = "./workspace.yml.tpl" ) func Read(t templateFile) ([]byte, error) { diff --git a/operators/workspace/internal/templates/workspace.yml.tpl b/operators/workspace/internal/templates/workspace.yml.tpl index c670c0b8..8378d585 100644 --- a/operators/workspace/internal/templates/workspace.yml.tpl +++ b/operators/workspace/internal/templates/workspace.yml.tpl @@ -222,8 +222,10 @@ spec: port: "8080" targetPort: "8080" --- + apiVersion: crds.kloudlite.io/v1 kind: Router metadata: {{.Metadata | toJson }} spec: {{.RouterSpec | toJson }} +--- {{- end }} From 17824ddf4cee6b318cef8c36705167d6a81f6469 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 10 Apr 2025 08:20:12 +0000 Subject: [PATCH 06/27] workspace changes in progress --- .vscode/launch.json | 15 +++++ cmd/agent-operator/main.go | 28 +++----- .../controllers/workspace/controller.go | 9 +-- .../workspace/internal/templates/types.go | 3 +- .../internal/templates/workspace.yml.tpl | 66 +++++++++++++------ 5 files changed, 78 insertions(+), 43 deletions(-) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..bacc62a4 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,15 @@ +{ + "configurations": [ + { + "name": "Run Agent Operator", + "type": "go", + "request": "launch", + "mode": "debug", + "envFile": "${workspaceFolder}/cmd/agent-operator/.secrets/env", + "program": "${workspaceFolder}/cmd/agent-operator/main.go", + "args": [ + "--dev","--serverHost", "localhost:8080" + ] + } + ] +} \ No newline at end of file diff --git a/cmd/agent-operator/main.go b/cmd/agent-operator/main.go index 929fe935..87f279c0 100644 --- a/cmd/agent-operator/main.go +++ b/cmd/agent-operator/main.go @@ -3,35 +3,25 @@ package main import ( "github.com/kloudlite/operator/toolkit/operator" - app "github.com/kloudlite/operator/operators/app-n-lambda/controller" // helmCharts "github.com/kloudlite/operator/operators/helm-charts/controller" lifecycle "github.com/kloudlite/operator/operators/lifecycle/controller" - msvcAndMres "github.com/kloudlite/operator/operators/msvc-n-mres/controller" - networkingv1 "github.com/kloudlite/operator/operators/networking/register" - project "github.com/kloudlite/operator/operators/project/controller" - resourceWatcher "github.com/kloudlite/operator/operators/resource-watcher/controller" - // routers "github.com/kloudlite/operator/operators/routers/controller" - serviceIntercept "github.com/kloudlite/operator/operators/service-intercept/controller" - workmachine "github.com/kloudlite/operator/operators/workmachine/register" workspace "github.com/kloudlite/operator/operators/workspace/register" - pluginHelmChart "github.com/kloudlite/plugin-helm-chart/kloudlite" - pluginMongoDB "github.com/kloudlite/plugin-mongodb/kloudlite" ) func main() { mgr := operator.New("agent-operator") // kloudlite resources - app.RegisterInto(mgr) - project.RegisterInto(mgr) + // app.RegisterInto(mgr) + // project.RegisterInto(mgr) // helmCharts.RegisterInto(mgr) // routers.RegisterInto(mgr) // kloudlite managed services - msvcAndMres.RegisterInto(mgr) + // msvcAndMres.RegisterInto(mgr) // msvcMongo.RegisterInto(mgr) // msvcRedis.RegisterInto(mgr) @@ -41,18 +31,18 @@ func main() { lifecycle.RegisterInto(mgr) // kloudlite resource status updates - resourceWatcher.RegisterInto(mgr) + // resourceWatcher.RegisterInto(mgr) // distribution.RegisterInto(mgr) - networkingv1.RegisterInto(mgr) + // networkingv1.RegisterInto(mgr) - serviceIntercept.RegisterInto(mgr) + // serviceIntercept.RegisterInto(mgr) workspace.RegisterInto(mgr) - workmachine.RegisterInto(mgr) + // workmachine.RegisterInto(mgr) - pluginMongoDB.RegisterInto(mgr) - pluginHelmChart.RegisterInto(mgr) + // pluginMongoDB.RegisterInto(mgr) + // pluginHelmChart.RegisterInto(mgr) mgr.Start() } diff --git a/operators/workspace/internal/controllers/workspace/controller.go b/operators/workspace/internal/controllers/workspace/controller.go index a209a721..51f365e6 100644 --- a/operators/workspace/internal/controllers/workspace/controller.go +++ b/operators/workspace/internal/controllers/workspace/controller.go @@ -132,13 +132,14 @@ func (r *Reconciler) createDeployment(req *rApi.Request[*crdsv1.Workspace]) step Namespace: obj.Namespace, OwnerReferences: []metav1.OwnerReference{fn.AsOwner(obj, true)}, }, - NodeName: obj.Spec.NodeName, + + WorkMachineName: obj.Spec.WorkMachine, ServiceAccountName: obj.Spec.ServiceAccountName, ImageInitContainer: r.Env.WorkspaceImageInitContainer, ImageSSH: r.Env.WorkspaceImageSSH, - - EnableTTYD: obj.Spec.EnableTTYD, - ImageTTYD: r.Env.WorkspaceImageTTYD, + IsOn: obj.Spec.State == crdsv1.WorkspaceStateOn, + EnableTTYD: obj.Spec.EnableTTYD, + ImageTTYD: r.Env.WorkspaceImageTTYD, EnableJupyterNotebook: obj.Spec.EnableJupyterNotebook, ImageJupyterNotebook: r.Env.WorkspaceImageJupyterNotebook, diff --git a/operators/workspace/internal/templates/types.go b/operators/workspace/internal/templates/types.go index af36ab3e..afc6f9c5 100644 --- a/operators/workspace/internal/templates/types.go +++ b/operators/workspace/internal/templates/types.go @@ -7,7 +7,8 @@ import ( type WorkspaceTemplateArgs struct { Metadata metav1.ObjectMeta - NodeName string + WorkMachineName string + IsOn bool ServiceAccountName string ImageInitContainer string diff --git a/operators/workspace/internal/templates/workspace.yml.tpl b/operators/workspace/internal/templates/workspace.yml.tpl index 8378d585..0a8e3e5a 100644 --- a/operators/workspace/internal/templates/workspace.yml.tpl +++ b/operators/workspace/internal/templates/workspace.yml.tpl @@ -1,8 +1,9 @@ {{- with . }} apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: {{.Metadata | toJson }} spec: + replicas: {{ if .IsOn }}1{{ else }}0{{ end }} selector: matchLabels: app: {{.Metadata.Name | squote}} @@ -12,13 +13,15 @@ spec: app: {{.Metadata.Name | squote}} kloudlite.io/gateway.enabled: "false" spec: + securityContext: + fsGroup: 1000 hostname: {{.Metadata.Name}} - nodeName: {{.NodeName}} - serviceAccount: {{.ServiceAccountName | squote}} + nodeName: {{.WorkMachineName}} + # serviceAccount: {{.ServiceAccountName | squote}} tolerations: - - key: "kloudlite.io/worknode" + - key: "kloudlite.io/workmachine.name" operator: "Equal" - value: {{.NodeName |squote}} + value: {{.WorkMachineName |squote}} effect: "NoExecute" initContainers: - name: init-home-dir @@ -43,12 +46,24 @@ spec: - | set -e set +x + if [ ! -d "/home/kl/.ssh" ]; then + mkdir -p /home/kl/.ssh + fi + if [ -f "/home/kl/.ssh/authorized_keys" ]; then + if ! cmp -s /tmp/authorized_keys /home/kl/.ssh/authorized_keys; then + echo "authorized_keys file differs, copying new one" + cp /tmp/authorized_keys /home/kl/.ssh/authorized_keys + fi + echo "authorized_keys file is up to date" + else + echo "authorized_keys file not found, copying new one" + cp /tmp/authorized_keys /home/kl/.ssh/authorized_keys + fi if [ ! -d "/nix/store" ]; then curl -L https://nixos.org/nix/install | sh mkdir -p ~/.config/nix echo 'experimental-features = nix-command flakes' > ~/.config/nix/nix.conf fi - kl_bin_dir="/home/kl/.local/bin" if [ ! -f "$kl_bin_dir/kl" ]; then mkdir -p $kl_bin_dir @@ -82,10 +97,10 @@ spec: if [ ! -f "/home/kl/.local/bin/starship" ]; then curl -sS https://starship.rs/install.sh | sh -s -- -y -b /home/kl/.local/bin fi - + if [ ! -d "/home/kl/.config/zsh/zsh-autosuggestions" ]; then mkdir -p "/home/kl/.config/zsh" - git clone https://github.com/zsh-users/zsh-autosuggestions + git clone https://github.com/zsh-users/zsh-autosuggestions /home/kl/.config/zsh/zsh-autosuggestions fi if [ ! -d "/home/kl/.config/zsh/zsh-syntax-highlighting" ]; then @@ -100,15 +115,20 @@ spec: {{- /* team: {{.KloudliteTeam}} */}} {{- /* EOF' */}} {{- /* fi */}} - if [ ! -f "/home/kl/.local/bin/kubectl" ]; then + pushd /home/kl/.local/bin curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" chmod +x /home/kl/.local/bin/kubectl + popd fi volumeMounts: &volume-mounts - mountPath: /home/kl name: home-dir + + - mountPath: /tmp/authorized_keys + name: ssh-keys + subPath: authorized_keys - mountPath: /nix name: nix-dir @@ -191,11 +211,19 @@ spec: - name: home-dir hostPath: - path: /var/user-home/ + path: /external-volume/user-home - name: nix-dir hostPath: - path: /var/nix-dir/ + path: /external-volume/nix + + - name: ssh-keys + secret: + secretName: ssh-public-keys + defaultMode: 0400 + items: + - key: authorized_keys + path: authorized_keys --- apiVersion: v1 kind: Service @@ -204,23 +232,23 @@ spec: ports: - name: "ssh" protocol: "TCP" - port: "22" - targetPort: "22" + port: 22 + targetPort: 22 - name: "ttyd-server" protocol: "TCP" - port: "54535" - targetPort: "54535" + port: 54535 + targetPort: 54535 - name: "jupyter-server" protocol: "TCP" - port: "8888" - targetPort: "8888" + port: 8888 + targetPort: 8888 - name: "code-server" protocol: "TCP" - port: "8080" - targetPort: "8080" + port: 8080 + targetPort: 8080 --- apiVersion: crds.kloudlite.io/v1 From ea0e77bd1d8dff286b84f6e8132efbe37fc493e6 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 10 Apr 2025 15:05:00 +0000 Subject: [PATCH 07/27] workspace changes in progress --- .../controllers/workspace/controller.go | 67 +++++---- .../templates/deployments/ingress.yml.tpl | 76 ++++++++++ .../templates/deployments/service.yml.tpl | 34 +++++ .../sts.yml.tpl} | 130 +++++++++--------- .../workspace/internal/templates/embed.go | 18 ++- .../workspace/internal/templates/types.go | 10 ++ 6 files changed, 236 insertions(+), 99 deletions(-) create mode 100644 operators/workspace/internal/templates/deployments/ingress.yml.tpl create mode 100644 operators/workspace/internal/templates/deployments/service.yml.tpl rename operators/workspace/internal/templates/{workspace.yml.tpl => deployments/sts.yml.tpl} (74%) diff --git a/operators/workspace/internal/controllers/workspace/controller.go b/operators/workspace/internal/controllers/workspace/controller.go index 51f365e6..a3ea8970 100644 --- a/operators/workspace/internal/controllers/workspace/controller.go +++ b/operators/workspace/internal/controllers/workspace/controller.go @@ -18,11 +18,9 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/intstr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) type Reconciler struct { @@ -44,7 +42,7 @@ func (r *Reconciler) GetName() string { const ( CreateDeployment string = "create-deployment" - CreateService string = "create-service" + // CreateService string = "create-service" ) // +kubebuilder:rbac:groups=crds.kloudlite.io,resources=apps,verbs=get;list;watch;create;update;patch;delete @@ -87,9 +85,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. return step.ReconcilerResponse() } - if step := r.createInterceptableService(req); !step.ShouldProceed() { - return step.ReconcilerResponse() - } + // if step := r.createInterceptableService(req); !step.ShouldProceed() { + // return step.ReconcilerResponse() + // } if step := r.createDeployment(req); !step.ShouldProceed() { return step.ReconcilerResponse() @@ -99,28 +97,28 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. return ctrl.Result{}, nil } -func (r *Reconciler) createInterceptableService(req *rApi.Request[*crdsv1.Workspace]) stepResult.Result { - ctx, obj := req.Context(), req.Object - check := rApi.NewRunningCheck(CreateService, req) - - svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: obj.Name, Namespace: obj.Namespace}} - if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, svc, func() error { - svc.Spec.Ports = []corev1.ServicePort{ - { - Name: fmt.Sprintf("port-%d", 3000), - Protocol: "TCP", - Port: 3000, - TargetPort: intstr.FromInt(3000), - }, - } - return nil - }); err != nil { - return check.Failed(err) - } - - // function-body - return check.Completed() -} +// func (r *Reconciler) createInterceptableService(req *rApi.Request[*crdsv1.Workspace]) stepResult.Result { +// ctx, obj := req.Context(), req.Object +// check := rApi.NewRunningCheck(CreateService, req) + +// svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: obj.Name, Namespace: obj.Namespace}} +// if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, svc, func() error { +// svc.Spec.Ports = []corev1.ServicePort{ +// { +// Name: fmt.Sprintf("port-%d", 3000), +// Protocol: "TCP", +// Port: 3000, +// TargetPort: intstr.FromInt(3000), +// }, +// } +// return nil +// }); err != nil { +// return check.Failed(err) +// } + +// // function-body +// return check.Completed() +// } func (r *Reconciler) createDeployment(req *rApi.Request[*crdsv1.Workspace]) stepResult.Result { ctx, obj := req.Context(), req.Object @@ -132,7 +130,7 @@ func (r *Reconciler) createDeployment(req *rApi.Request[*crdsv1.Workspace]) step Namespace: obj.Namespace, OwnerReferences: []metav1.OwnerReference{fn.AsOwner(obj, true)}, }, - + KloudliteDomain: "test.khost.dev", WorkMachineName: obj.Spec.WorkMachine, ServiceAccountName: obj.Spec.ServiceAccountName, ImageInitContainer: r.Env.WorkspaceImageInitContainer, @@ -153,12 +151,21 @@ func (r *Reconciler) createDeployment(req *rApi.Request[*crdsv1.Workspace]) step ImagePullPolicy: obj.Spec.ImagePullPolicy, KloudliteDeviceFQDN: fmt.Sprintf("%s.%s.svc.cluster.local", obj.Name, obj.Namespace), + PortConfig: templates.PortConfig{ + SSHPort: 22, + TTYDPort: 56789, + NotebookPort: 56790, + CodeServerPort: 56791, + }, + RouterSpec: obj.Spec.Router, }) if err != nil { return check.Failed(err) } + fmt.Println(string(b)) + rr, err := r.YAMLClient.ApplyYAML(ctx, b) if err != nil { return check.Failed(err) @@ -196,7 +203,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { r.recorder = mgr.GetEventRecorderFor(r.GetName()) var err error - r.workspaceDeploymentTemplate, err = templates.Read(templates.WorkspaceTemplate) + r.workspaceDeploymentTemplate, err = templates.Read(templates.WorkspaceIngressTemplate, templates.WorkspaceSTSTemplate, templates.WorkspaceServiceTemplate) if err != nil { return err } diff --git a/operators/workspace/internal/templates/deployments/ingress.yml.tpl b/operators/workspace/internal/templates/deployments/ingress.yml.tpl new file mode 100644 index 00000000..138b53f1 --- /dev/null +++ b/operators/workspace/internal/templates/deployments/ingress.yml.tpl @@ -0,0 +1,76 @@ +--- +{{- with . }} + +{{ if .EnableCodeServer }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: code-server-{{.Metadata.Name}} + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/proxy-body-size: "50m" +spec: + rules: + - host: code-server.{{.Metadata.Name}}.{{.WorkMachineName}}.{{.KloudliteDomain}} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{.Metadata.Name}} + port: + number: {{.PortConfig.CodeServerPort}} +{{ end }} + +--- + +{{ if .EnableJupyterNotebook }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: nb-{{.Metadata.Name}} + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/proxy-body-size: "50m" +spec: + rules: + - host: notebook.{{.Metadata.Name}}.{{.WorkMachineName}}.{{.KloudliteDomain}} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{.Metadata.Name}} + port: + number: {{.PortConfig.NotebookPort}} +{{ end }} + +--- +{{ if .EnableTTYD }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ttyd-{{.Metadata.Name}} + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/proxy-body-size: "50m" +spec: + rules: + - host: ttyd.{{.Metadata.Name}}.{{.WorkMachineName}}.{{.KloudliteDomain}} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{.Metadata.Name}} + port: + number: {{.PortConfig.TTYDPort}} +{{ end }} + +{{- end }} diff --git a/operators/workspace/internal/templates/deployments/service.yml.tpl b/operators/workspace/internal/templates/deployments/service.yml.tpl new file mode 100644 index 00000000..a4b60aa1 --- /dev/null +++ b/operators/workspace/internal/templates/deployments/service.yml.tpl @@ -0,0 +1,34 @@ +--- +{{- with . }} +apiVersion: v1 +kind: Service +metadata: {{.Metadata | toJson }} +spec: + ports: + - name: "ssh" + protocol: "TCP" + port: {{.PortConfig.SSHPort}} + targetPort: {{.PortConfig.SSHPort}} + +{{ if .EnableTTYD }} + - name: "ttyd-server" + protocol: "TCP" + port: {{.PortConfig.TTYDPort}} + targetPort: {{.PortConfig.TTYDPort}} +{{ end }} + +{{ if .EnableJupyterNotebook }} + - name: "jupyter-server" + protocol: "TCP" + port: {{.PortConfig.NotebookPort}} + targetPort: {{.PortConfig.NotebookPort}} +{{ end }} + +{{ if .EnableCodeServer }} + - name: "code-server" + protocol: "TCP" + port: {{.PortConfig.CodeServerPort}} + targetPort: {{.PortConfig.CodeServerPort}} +{{ end }} + +{{- end }} diff --git a/operators/workspace/internal/templates/workspace.yml.tpl b/operators/workspace/internal/templates/deployments/sts.yml.tpl similarity index 74% rename from operators/workspace/internal/templates/workspace.yml.tpl rename to operators/workspace/internal/templates/deployments/sts.yml.tpl index 0a8e3e5a..c3419a34 100644 --- a/operators/workspace/internal/templates/workspace.yml.tpl +++ b/operators/workspace/internal/templates/deployments/sts.yml.tpl @@ -1,3 +1,4 @@ +--- {{- with . }} apiVersion: apps/v1 kind: StatefulSet @@ -17,7 +18,9 @@ spec: fsGroup: 1000 hostname: {{.Metadata.Name}} nodeName: {{.WorkMachineName}} + # {{- if and .ServiceAccountName (ne .ServiceAccountName "") }} # serviceAccount: {{.ServiceAccountName | squote}} + # {{- end }} tolerations: - key: "kloudlite.io/workmachine.name" operator: "Equal" @@ -59,6 +62,22 @@ spec: echo "authorized_keys file not found, copying new one" cp /tmp/authorized_keys /home/kl/.ssh/authorized_keys fi + + if [ -f "/home/kl/.ssh/id_rsa" ]; then + if ! cmp -s /tmp/id_rsa /home/kl/.ssh/id_rsa; then + echo "id_rsa file differs, copying new one" + rm /home/kl/.ssh/id_rsa* || true + cp /tmp/id_rsa /home/kl/.ssh/id_rsa + cp /tmp/id_rsa.pub /home/kl/.ssh/id_rsa.pub + fi + echo "id_rsa file is up to date" + else + echo "id_rsa file not found, copying new one" + rm /home/kl/.ssh/id_rsa* || true + cp /tmp/id_rsa /home/kl/.ssh/id_rsa + cp /tmp/id_rsa.pub /home/kl/.ssh/id_rsa.pub + fi + if [ ! -d "/nix/store" ]; then curl -L https://nixos.org/nix/install | sh mkdir -p ~/.config/nix @@ -129,6 +148,14 @@ spec: - mountPath: /tmp/authorized_keys name: ssh-keys subPath: authorized_keys + + - mountPath: /tmp/id_rsa.pub + name: ssh-keys + subPath: id_rsa.pub + + - mountPath: /tmp/id_rsa + name: ssh-keys + subPath: id_rsa - mountPath: /nix name: nix-dir @@ -160,49 +187,49 @@ spec: volumeMounts: *volume-mounts {{ if .EnableTTYD }} - - name: ttyd - image: {{.ImageTTYD}} - imagePullPolicy: {{.ImagePullPolicy}} - env: *env - ports: - - containerPort: 54535 - volumeMounts: *volume-mounts + - name: ttyd + image: {{.ImageTTYD}} + imagePullPolicy: {{.ImagePullPolicy}} + env: *env + ports: + - containerPort: 54535 + volumeMounts: *volume-mounts {{ end }} {{ if .EnableJupyterNotebook }} - - name: jupyter - image: {{.ImageJupyterNotebook}} - imagePullPolicy: {{.ImagePullPolicy}} - env: *env - ports: - - containerPort: 8888 - volumeMounts: *volume-mounts - securityContext: - runAsUser: 1000 - runAsGroup: 1000 + - name: jupyter + image: {{.ImageJupyterNotebook}} + imagePullPolicy: {{.ImagePullPolicy}} + env: *env + ports: + - containerPort: 8888 + volumeMounts: *volume-mounts + securityContext: + runAsUser: 1000 + runAsGroup: 1000 {{ end }} {{ if .EnableCodeServer }} - - name: code-server - image: {{.ImageCodeServer}} - imagePullPolicy: {{.ImagePullPolicy}} - env: *env - volumeMounts: *volume-mounts - securityContext: - runAsUser: 1000 - runAsGroup: 1000 + - name: code-server + image: {{.ImageCodeServer}} + imagePullPolicy: {{.ImagePullPolicy}} + env: *env + volumeMounts: *volume-mounts + securityContext: + runAsUser: 1000 + runAsGroup: 1000 {{ end }} {{ if .EnableVSCodeServer }} - - name: vscode-server - {{- /* image: ghcr.io/kloudlite/iac/vscode-server:latest */}} - image: {{.ImageVscodeServer}} - imagePullPolicy: {{.ImagePullPolicy}} - env: *env - volumeMounts: *volume-mounts - securityContext: - runAsUser: 1000 - runAsGroup: 1000 + - name: vscode-server + {{- /* image: ghcr.io/kloudlite/iac/vscode-server:latest */}} + image: {{.ImageVscodeServer}} + imagePullPolicy: {{.ImagePullPolicy}} + env: *env + volumeMounts: *volume-mounts + securityContext: + runAsUser: 1000 + runAsGroup: 1000 {{ end }} volumes: @@ -224,36 +251,9 @@ spec: items: - key: authorized_keys path: authorized_keys ---- -apiVersion: v1 -kind: Service -metadata: {{.Metadata | toJson }} -spec: - ports: - - name: "ssh" - protocol: "TCP" - port: 22 - targetPort: 22 - - - name: "ttyd-server" - protocol: "TCP" - port: 54535 - targetPort: 54535 - - - name: "jupyter-server" - protocol: "TCP" - port: 8888 - targetPort: 8888 + - key: id_rsa.pub + path: id_rsa.pub + - key: id_rsa + path: id_rsa - - name: "code-server" - protocol: "TCP" - port: 8080 - targetPort: 8080 ---- - -apiVersion: crds.kloudlite.io/v1 -kind: Router -metadata: {{.Metadata | toJson }} -spec: {{.RouterSpec | toJson }} ---- {{- end }} diff --git a/operators/workspace/internal/templates/embed.go b/operators/workspace/internal/templates/embed.go index 33c17743..1f031537 100644 --- a/operators/workspace/internal/templates/embed.go +++ b/operators/workspace/internal/templates/embed.go @@ -7,17 +7,27 @@ import ( "github.com/kloudlite/operator/pkg/templates" ) -//go:embed * +//go:embed deployments/* var templatesDir embed.FS type templateFile string const ( - WorkspaceTemplate templateFile = "./workspace.yml.tpl" + WorkspaceSTSTemplate templateFile = "./deployments/sts.yml.tpl" + WorkspaceIngressTemplate templateFile = "./deployments/ingress.yml.tpl" + WorkspaceServiceTemplate templateFile = "./deployments/service.yml.tpl" ) -func Read(t templateFile) ([]byte, error) { - return templatesDir.ReadFile(filepath.Join(string(t))) +func Read(tPaths ...templateFile) ([]byte, error) { + var data []byte + for _, t := range tPaths { + d, err := templatesDir.ReadFile(filepath.Join(string(t))) + if err != nil { + return nil, err + } + data = append(data, d...) + } + return data, nil } var ParseBytes = templates.ParseBytes diff --git a/operators/workspace/internal/templates/types.go b/operators/workspace/internal/templates/types.go index afc6f9c5..6aadf54f 100644 --- a/operators/workspace/internal/templates/types.go +++ b/operators/workspace/internal/templates/types.go @@ -5,6 +5,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +type PortConfig struct { + TTYDPort int32 + SSHPort int32 + NotebookPort int32 + CodeServerPort int32 +} + type WorkspaceTemplateArgs struct { Metadata metav1.ObjectMeta WorkMachineName string @@ -30,5 +37,8 @@ type WorkspaceTemplateArgs struct { KloudliteDeviceFQDN string + KloudliteDomain string + RouterSpec crdsv1.RouterSpec + PortConfig PortConfig } From 7a4723d67693cd297ac00cd68919e91241fd5f0d Mon Sep 17 00:00:00 2001 From: Karthik Th Date: Thu, 10 Apr 2025 15:13:49 +0000 Subject: [PATCH 08/27] added headless service for service intercepts --- .../internal/templates/deployments/service.yml.tpl | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/operators/workspace/internal/templates/deployments/service.yml.tpl b/operators/workspace/internal/templates/deployments/service.yml.tpl index a4b60aa1..74eca298 100644 --- a/operators/workspace/internal/templates/deployments/service.yml.tpl +++ b/operators/workspace/internal/templates/deployments/service.yml.tpl @@ -4,6 +4,8 @@ apiVersion: v1 kind: Service metadata: {{.Metadata | toJson }} spec: + selector: + app: {{.Metadata.Name | squote}} ports: - name: "ssh" protocol: "TCP" @@ -32,3 +34,15 @@ spec: {{ end }} {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{.Metadata.Name}}-headless + namespace: {{.Metadata.Namespace}} + labels: {{.Metadata.Labels | toJson }} + annotations: {{.Metadata.Annotations | toJson }} +spec: + clusterIP: None + selector: + app: {{.Metadata.Name | squote}} \ No newline at end of file From d25d47a152ff9d81afc6be787d11843827d2f37b Mon Sep 17 00:00:00 2001 From: Karthik Th Date: Thu, 10 Apr 2025 15:22:45 +0000 Subject: [PATCH 09/27] changed KloudliteDeviceFQDN to point to headless service --- .../workspace/internal/controllers/workspace/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operators/workspace/internal/controllers/workspace/controller.go b/operators/workspace/internal/controllers/workspace/controller.go index a3ea8970..817dbbb5 100644 --- a/operators/workspace/internal/controllers/workspace/controller.go +++ b/operators/workspace/internal/controllers/workspace/controller.go @@ -149,7 +149,7 @@ func (r *Reconciler) createDeployment(req *rApi.Request[*crdsv1.Workspace]) step ImageVscodeServer: r.Env.WorkspcaeImageVscodeServer, ImagePullPolicy: obj.Spec.ImagePullPolicy, - KloudliteDeviceFQDN: fmt.Sprintf("%s.%s.svc.cluster.local", obj.Name, obj.Namespace), + KloudliteDeviceFQDN: fmt.Sprintf("%s-headless.%s.svc.cluster.local", obj.Name, obj.Namespace), PortConfig: templates.PortConfig{ SSHPort: 22, From ccb12cd293aff2565aa0d60f845ebdcdea6b92ce Mon Sep 17 00:00:00 2001 From: Karthik Th Date: Thu, 10 Apr 2025 15:43:33 +0000 Subject: [PATCH 10/27] updated ingress class --- .../controllers/workspace/controller.go | 23 +++++++++++----- .../templates/deployments/ingress.yml.tpl | 26 ++++++++++++++----- .../templates/deployments/service.yml.tpl | 7 +++-- .../workspace/internal/templates/types.go | 5 ++-- 4 files changed, 43 insertions(+), 18 deletions(-) diff --git a/operators/workspace/internal/controllers/workspace/controller.go b/operators/workspace/internal/controllers/workspace/controller.go index 817dbbb5..d1443112 100644 --- a/operators/workspace/internal/controllers/workspace/controller.go +++ b/operators/workspace/internal/controllers/workspace/controller.go @@ -23,6 +23,19 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" ) +var ( + PortConfig = templates.PortConfig{ + SSHPort: 22, + TTYDPort: 56789, + NotebookPort: 56790, + CodeServerPort: 56791, + } +) + +const ( + IngressClassName = "nginx" +) + type Reconciler struct { client.Client Scheme *runtime.Scheme @@ -151,14 +164,10 @@ func (r *Reconciler) createDeployment(req *rApi.Request[*crdsv1.Workspace]) step ImagePullPolicy: obj.Spec.ImagePullPolicy, KloudliteDeviceFQDN: fmt.Sprintf("%s-headless.%s.svc.cluster.local", obj.Name, obj.Namespace), - PortConfig: templates.PortConfig{ - SSHPort: 22, - TTYDPort: 56789, - NotebookPort: 56790, - CodeServerPort: 56791, - }, + PortConfig: PortConfig, - RouterSpec: obj.Spec.Router, + RouterSpec: obj.Spec.Router, + IngressClassName: IngressClassName, }) if err != nil { return check.Failed(err) diff --git a/operators/workspace/internal/templates/deployments/ingress.yml.tpl b/operators/workspace/internal/templates/deployments/ingress.yml.tpl index 138b53f1..03c2cec8 100644 --- a/operators/workspace/internal/templates/deployments/ingress.yml.tpl +++ b/operators/workspace/internal/templates/deployments/ingress.yml.tpl @@ -1,16 +1,19 @@ --- {{- with . }} - -{{ if .EnableCodeServer }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: code-server-{{.Metadata.Name}} + namespace: {{.Metadata.Namespace}} + labels: {{.Metadata.Labels | toJson }} annotations: nginx.ingress.kubernetes.io/rewrite-target: / nginx.ingress.kubernetes.io/ssl-redirect: "true" nginx.ingress.kubernetes.io/proxy-body-size: "50m" spec: + {{- if and .IngressClassName (ne .IngressClassName "") }} + ingressClassName: {{.IngressClassName}} + {{- end }} rules: - host: code-server.{{.Metadata.Name}}.{{.WorkMachineName}}.{{.KloudliteDomain}} http: @@ -22,20 +25,25 @@ spec: name: {{.Metadata.Name}} port: number: {{.PortConfig.CodeServerPort}} -{{ end }} + --- -{{ if .EnableJupyterNotebook }} + apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: nb-{{.Metadata.Name}} + namespace: {{.Metadata.Namespace}} + labels: {{.Metadata.Labels | toJson }} annotations: nginx.ingress.kubernetes.io/rewrite-target: / nginx.ingress.kubernetes.io/ssl-redirect: "true" nginx.ingress.kubernetes.io/proxy-body-size: "50m" spec: + {{- if and .IngressClassName (ne .IngressClassName "") }} + ingressClassName: {{.IngressClassName}} + {{- end }} rules: - host: notebook.{{.Metadata.Name}}.{{.WorkMachineName}}.{{.KloudliteDomain}} http: @@ -47,19 +55,24 @@ spec: name: {{.Metadata.Name}} port: number: {{.PortConfig.NotebookPort}} -{{ end }} + --- -{{ if .EnableTTYD }} + apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: ttyd-{{.Metadata.Name}} + namespace: {{.Metadata.Namespace}} + labels: {{.Metadata.Labels | toJson }} annotations: nginx.ingress.kubernetes.io/rewrite-target: / nginx.ingress.kubernetes.io/ssl-redirect: "true" nginx.ingress.kubernetes.io/proxy-body-size: "50m" spec: + {{- if and .IngressClassName (ne .IngressClassName "") }} + ingressClassName: {{.IngressClassName}} + {{- end }} rules: - host: ttyd.{{.Metadata.Name}}.{{.WorkMachineName}}.{{.KloudliteDomain}} http: @@ -71,6 +84,5 @@ spec: name: {{.Metadata.Name}} port: number: {{.PortConfig.TTYDPort}} -{{ end }} {{- end }} diff --git a/operators/workspace/internal/templates/deployments/service.yml.tpl b/operators/workspace/internal/templates/deployments/service.yml.tpl index 74eca298..99d915db 100644 --- a/operators/workspace/internal/templates/deployments/service.yml.tpl +++ b/operators/workspace/internal/templates/deployments/service.yml.tpl @@ -33,7 +33,7 @@ spec: targetPort: {{.PortConfig.CodeServerPort}} {{ end }} -{{- end }} + --- apiVersion: v1 kind: Service @@ -42,7 +42,10 @@ metadata: namespace: {{.Metadata.Namespace}} labels: {{.Metadata.Labels | toJson }} annotations: {{.Metadata.Annotations | toJson }} + ownerReferences: {{.Metadata.OwnerReferences | toJson }} spec: clusterIP: None selector: - app: {{.Metadata.Name | squote}} \ No newline at end of file + app: {{.Metadata.Name | squote}} + +{{- end }} \ No newline at end of file diff --git a/operators/workspace/internal/templates/types.go b/operators/workspace/internal/templates/types.go index 6aadf54f..cfa6cdce 100644 --- a/operators/workspace/internal/templates/types.go +++ b/operators/workspace/internal/templates/types.go @@ -39,6 +39,7 @@ type WorkspaceTemplateArgs struct { KloudliteDomain string - RouterSpec crdsv1.RouterSpec - PortConfig PortConfig + RouterSpec crdsv1.RouterSpec + PortConfig PortConfig + IngressClassName string } From 22aceb738c7fc42dfd190933de6d07dd50583e38 Mon Sep 17 00:00:00 2001 From: Karthik Th Date: Thu, 10 Apr 2025 18:13:42 +0000 Subject: [PATCH 11/27] workspace type changes --- apis/crds/v1/workspace_types.go | 9 +- apis/crds/v1/zz_generated.deepcopy.go | 3 +- .../bases/crds.kloudlite.io_workspaces.yaml | 101 ++---------------- 3 files changed, 12 insertions(+), 101 deletions(-) diff --git a/apis/crds/v1/workspace_types.go b/apis/crds/v1/workspace_types.go index bfd8f08b..f52b9690 100644 --- a/apis/crds/v1/workspace_types.go +++ b/apis/crds/v1/workspace_types.go @@ -21,7 +21,6 @@ type WorkspaceSpec struct { WorkMachine string `json:"workMachine"` State WorkspaceState `json:"state"` - NodeName string `json:"nodeName"` ServiceAccountName string `json:"serviceAccountName"` EnableTTYD bool `json:"enableTTYD,omitempty"` @@ -32,15 +31,15 @@ type WorkspaceSpec struct { // +kubebuilder:default=IfNotPresent ImagePullPolicy string `json:"imagePullPolicy"` - Router RouterSpec `json:"router"` + // Router RouterSpec `json:"router"` } // +kubebuilder:object:root=true // +kubebuilder:subresource:status -// +kubebuilder:resource:scope=Cluster +// +kubebuilder:resource:scope=Namespaced // +kubebuilder:printcolumn:JSONPath=".status.lastReconcileTime",name=Seen,type=date -// +kubebuilder:printcolumn:JSONPath=".metadata.annotations.kloudlite\\.io\\/checks",name=Checks,type=string -// +kubebuilder:printcolumn:JSONPath=".metadata.annotations.kloudlite\\.io\\/resource\\.ready",name=Ready,type=string +// +kubebuilder:printcolumn:JSONPath=".metadata.annotations.kloudlite\\.io\\/operator\\.checks",name=Checks,type=string +// +kubebuilder:printcolumn:JSONPath=".metadata.annotations.kloudlite\\.io\\/operator\\.resource\\.ready",name=Ready,type=string // +kubebuilder:printcolumn:JSONPath=".metadata.creationTimestamp",name=Age,type=date // Workspace is the Schema for the workspaces API diff --git a/apis/crds/v1/zz_generated.deepcopy.go b/apis/crds/v1/zz_generated.deepcopy.go index 3a5bee2d..4faeaed5 100644 --- a/apis/crds/v1/zz_generated.deepcopy.go +++ b/apis/crds/v1/zz_generated.deepcopy.go @@ -2114,7 +2114,7 @@ func (in *Workspace) DeepCopyInto(out *Workspace) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) + out.Spec = in.Spec in.Status.DeepCopyInto(&out.Status) } @@ -2171,7 +2171,6 @@ func (in *WorkspaceList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkspaceSpec) DeepCopyInto(out *WorkspaceSpec) { *out = *in - in.Router.DeepCopyInto(&out.Router) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkspaceSpec. diff --git a/config/crd/bases/crds.kloudlite.io_workspaces.yaml b/config/crd/bases/crds.kloudlite.io_workspaces.yaml index b8147b8a..5b123ac8 100644 --- a/config/crd/bases/crds.kloudlite.io_workspaces.yaml +++ b/config/crd/bases/crds.kloudlite.io_workspaces.yaml @@ -12,16 +12,16 @@ spec: listKind: WorkspaceList plural: workspaces singular: workspace - scope: Cluster + scope: Namespaced versions: - additionalPrinterColumns: - jsonPath: .status.lastReconcileTime name: Seen type: date - - jsonPath: .metadata.annotations.kloudlite\.io\/checks + - jsonPath: .metadata.annotations.kloudlite\.io\/operator\.checks name: Checks type: string - - jsonPath: .metadata.annotations.kloudlite\.io\/resource\.ready + - jsonPath: .metadata.annotations.kloudlite\.io\/operator\.resource\.ready name: Ready type: string - jsonPath: .metadata.creationTimestamp @@ -63,95 +63,6 @@ spec: imagePullPolicy: default: IfNotPresent type: string - nodeName: - type: string - router: - description: RouterSpec defines the desired state of Router - properties: - backendProtocol: - type: string - basicAuth: - properties: - enabled: - type: boolean - secretName: - type: string - username: - type: string - required: - - enabled - type: object - cors: - properties: - allowCredentials: - type: boolean - enabled: - default: false - type: boolean - origins: - items: - type: string - type: array - type: object - domains: - items: - type: string - type: array - https: - properties: - clusterIssuer: - type: string - enabled: - default: true - type: boolean - forceRedirect: - type: boolean - required: - - enabled - type: object - ingressClass: - type: string - maxBodySizeInMB: - type: integer - nginxIngressAnnotations: - additionalProperties: - type: string - description: NginxIngressAnnotations is additional list of annotations - on ingress resource - type: object - rateLimit: - properties: - connections: - type: integer - enabled: - type: boolean - rpm: - type: integer - rps: - type: integer - type: object - routes: - items: - properties: - app: - type: string - path: - description: Lambda string `json:"lambda,omitempty"` - type: string - port: - type: integer - rewrite: - default: false - type: boolean - required: - - app - - path - - port - type: object - type: array - required: - - domains - type: object serviceAccountName: type: string state: @@ -159,12 +70,14 @@ spec: - "ON" - "OFF" type: string + workMachine: + description: Name of work machine + type: string required: - imagePullPolicy - - nodeName - - router - serviceAccountName - state + - workMachine type: object status: properties: From f42323964f1ea7fb372d2678df466d202c22a3b4 Mon Sep 17 00:00:00 2001 From: Karthik Th Date: Sat, 12 Apr 2025 05:20:19 +0000 Subject: [PATCH 12/27] added SessionSecretName to workmachine spec --- apis/crds/v1/workmachine_types.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apis/crds/v1/workmachine_types.go b/apis/crds/v1/workmachine_types.go index f2333da3..b6e8fd70 100644 --- a/apis/crds/v1/workmachine_types.go +++ b/apis/crds/v1/workmachine_types.go @@ -48,6 +48,8 @@ type WorkMachineSpec struct { State WorkMachineState `json:"state"` SSHPublicKeys []string `json:"sshPublicKeys"` + SessionSecretName string `json:"sessionSecretName,omitempty"` + JobParams WorkMachineJobParams `json:"jobParams"` TargetNamespace string `json:"targetNamespace,omitempty"` From 11e94fe054a99078eb0c76a201fab6c93032c7b0 Mon Sep 17 00:00:00 2001 From: Karthik Th Date: Sun, 13 Apr 2025 10:08:52 +0000 Subject: [PATCH 13/27] removed subnet id from workmachine spec. It will be same subnet as cluster --- apis/crds/v1/workmachine_types.go | 8 +++----- config/crd/bases/crds.kloudlite.io_workmachines.yaml | 2 ++ .../internal/controllers/workmachine/controller.go | 3 ++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/apis/crds/v1/workmachine_types.go b/apis/crds/v1/workmachine_types.go index b6e8fd70..963bb0ed 100644 --- a/apis/crds/v1/workmachine_types.go +++ b/apis/crds/v1/workmachine_types.go @@ -11,9 +11,9 @@ type AWSMachineConfig struct { Region string `json:"region" graphql:"noinput"` AvailabilityZone string `json:"availabilityZone"` - AMI string `json:"ami"` - InstanceType string `json:"instanceType"` - PublicSubnetID string `json:"publicSubnetID"` + AMI string `json:"ami"` + InstanceType string `json:"instanceType"` + // PublicSubnetID string `json:"publicSubnetID"` //+kubebuilder:default=50 RootVolumeSize int `json:"rootVolumeSize" graphql:"noinput"` @@ -48,8 +48,6 @@ type WorkMachineSpec struct { State WorkMachineState `json:"state"` SSHPublicKeys []string `json:"sshPublicKeys"` - SessionSecretName string `json:"sessionSecretName,omitempty"` - JobParams WorkMachineJobParams `json:"jobParams"` TargetNamespace string `json:"targetNamespace,omitempty"` diff --git a/config/crd/bases/crds.kloudlite.io_workmachines.yaml b/config/crd/bases/crds.kloudlite.io_workmachines.yaml index 2081e64e..c2109078 100644 --- a/config/crd/bases/crds.kloudlite.io_workmachines.yaml +++ b/config/crd/bases/crds.kloudlite.io_workmachines.yaml @@ -136,6 +136,8 @@ spec: type: object type: array type: object + sessionToken: + type: string sshPublicKeys: items: type: string diff --git a/operators/workmachine/internal/controllers/workmachine/controller.go b/operators/workmachine/internal/controllers/workmachine/controller.go index 61d84902..ea90c333 100644 --- a/operators/workmachine/internal/controllers/workmachine/controller.go +++ b/operators/workmachine/internal/controllers/workmachine/controller.go @@ -149,6 +149,7 @@ type ClusterParams struct { AwsVPCName string `json:"aws_vpc_name"` AwsVPCId string `json:"aws_vpc_id"` + SubnetId string `json:"subnet_id"` AwsNLBDNSHost string `json:"aws_nlb_dns_host"` @@ -206,7 +207,7 @@ func (r *Reconciler) parseSpecIntoTFValues(ctx context.Context, obj *crdsv1.Work "root_volume_size": obj.Spec.AWSMachineConfig.RootVolumeSize, "root_volume_type": obj.Spec.AWSMachineConfig.RootVolumeType, "security_group_ids": cp.AwsSecurityGroupIDs, - "subnet_id": obj.Spec.AWSMachineConfig.PublicSubnetID, + "subnet_id": cp.SubnetId, }) } default: From 81cba8e3a7e5f2746c2d88fcc3916a27ca4d75ae Mon Sep 17 00:00:00 2001 From: Karthik Th Date: Sun, 13 Apr 2025 16:13:58 +0000 Subject: [PATCH 14/27] wip --- config/crd/bases/crds.kloudlite.io_workmachines.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/config/crd/bases/crds.kloudlite.io_workmachines.yaml b/config/crd/bases/crds.kloudlite.io_workmachines.yaml index c2109078..5da42b45 100644 --- a/config/crd/bases/crds.kloudlite.io_workmachines.yaml +++ b/config/crd/bases/crds.kloudlite.io_workmachines.yaml @@ -70,8 +70,6 @@ spec: type: string instanceType: type: string - publicSubnetID: - type: string region: type: string rootVolumeSize: @@ -86,7 +84,6 @@ spec: - externalVolumeSize - externalVolumeType - instanceType - - publicSubnetID - region - rootVolumeSize - rootVolumeType @@ -136,8 +133,6 @@ spec: type: object type: array type: object - sessionToken: - type: string sshPublicKeys: items: type: string From 5b8a435298e5c545f45885357192e0a695629b37 Mon Sep 17 00:00:00 2001 From: Karthik Th Date: Sun, 13 Apr 2025 16:22:52 +0000 Subject: [PATCH 15/27] wip --- apis/crds/v1/workmachine_types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apis/crds/v1/workmachine_types.go b/apis/crds/v1/workmachine_types.go index 963bb0ed..09234f85 100644 --- a/apis/crds/v1/workmachine_types.go +++ b/apis/crds/v1/workmachine_types.go @@ -48,7 +48,7 @@ type WorkMachineSpec struct { State WorkMachineState `json:"state"` SSHPublicKeys []string `json:"sshPublicKeys"` - JobParams WorkMachineJobParams `json:"jobParams"` + JobParams WorkMachineJobParams `json:"jobParams,omitempty"` TargetNamespace string `json:"targetNamespace,omitempty"` From 42dc061d67cb734560ebbf4206594e5c0a61766f Mon Sep 17 00:00:00 2001 From: Karthik Th Date: Wed, 16 Apr 2025 17:12:11 +0530 Subject: [PATCH 16/27] wm changes --- .vscode/launch.json | 15 ++++++++ apis/crds/v1/workspace_types.go | 9 ++--- cmd/agent-operator/Taskfile.yml | 2 +- cmd/agent-operator/main.go | 27 +++++-------- .../bases/crds.kloudlite.io_workspaces.yaml | 13 ++++--- .../controllers/workmachine/controller.go | 6 +-- .../workmachine/internal/templates/embed.go | 1 + operators/workspace/examples/sample.yml | 3 +- .../controllers/workspace/controller.go | 2 +- .../workspace/internal/templates/types.go | 2 + .../internal/templates/workspace.yml.tpl | 38 +++++++++++-------- 11 files changed, 66 insertions(+), 52 deletions(-) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..bacc62a4 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,15 @@ +{ + "configurations": [ + { + "name": "Run Agent Operator", + "type": "go", + "request": "launch", + "mode": "debug", + "envFile": "${workspaceFolder}/cmd/agent-operator/.secrets/env", + "program": "${workspaceFolder}/cmd/agent-operator/main.go", + "args": [ + "--dev","--serverHost", "localhost:8080" + ] + } + ] +} \ No newline at end of file diff --git a/apis/crds/v1/workspace_types.go b/apis/crds/v1/workspace_types.go index bfd8f08b..b6de7bd7 100644 --- a/apis/crds/v1/workspace_types.go +++ b/apis/crds/v1/workspace_types.go @@ -21,8 +21,7 @@ type WorkspaceSpec struct { WorkMachine string `json:"workMachine"` State WorkspaceState `json:"state"` - NodeName string `json:"nodeName"` - ServiceAccountName string `json:"serviceAccountName"` + ServiceAccountName string `json:"serviceAccountName,omitempty"` EnableTTYD bool `json:"enableTTYD,omitempty"` EnableJupyterNotebook bool `json:"enableJupyterNotebook,omitempty"` @@ -37,10 +36,10 @@ type WorkspaceSpec struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status -// +kubebuilder:resource:scope=Cluster +// +kubebuilder:resource:scope=Namespaced // +kubebuilder:printcolumn:JSONPath=".status.lastReconcileTime",name=Seen,type=date -// +kubebuilder:printcolumn:JSONPath=".metadata.annotations.kloudlite\\.io\\/checks",name=Checks,type=string -// +kubebuilder:printcolumn:JSONPath=".metadata.annotations.kloudlite\\.io\\/resource\\.ready",name=Ready,type=string +// +kubebuilder:printcolumn:JSONPath=".metadata.annotations.kloudlite\\.io\\/operator\\.checks",name=Checks,type=string +// +kubebuilder:printcolumn:JSONPath=".metadata.annotations.kloudlite\\.io\\/operator\\.resource\\.ready",name=Ready,type=string // +kubebuilder:printcolumn:JSONPath=".metadata.creationTimestamp",name=Age,type=date // Workspace is the Schema for the workspaces API diff --git a/cmd/agent-operator/Taskfile.yml b/cmd/agent-operator/Taskfile.yml index 99cf8e53..4c947108 100644 --- a/cmd/agent-operator/Taskfile.yml +++ b/cmd/agent-operator/Taskfile.yml @@ -13,7 +13,7 @@ tasks: dotenv: - .secrets/env cmds: - - go run . --dev --serverHost localhost:8081 + - go run . --dev --serverHost localhost:8080 build: cmds: diff --git a/cmd/agent-operator/main.go b/cmd/agent-operator/main.go index 929fe935..d23cf522 100644 --- a/cmd/agent-operator/main.go +++ b/cmd/agent-operator/main.go @@ -3,35 +3,26 @@ package main import ( "github.com/kloudlite/operator/toolkit/operator" - app "github.com/kloudlite/operator/operators/app-n-lambda/controller" // helmCharts "github.com/kloudlite/operator/operators/helm-charts/controller" lifecycle "github.com/kloudlite/operator/operators/lifecycle/controller" - - msvcAndMres "github.com/kloudlite/operator/operators/msvc-n-mres/controller" - networkingv1 "github.com/kloudlite/operator/operators/networking/register" - project "github.com/kloudlite/operator/operators/project/controller" - resourceWatcher "github.com/kloudlite/operator/operators/resource-watcher/controller" + workspace "github.com/kloudlite/operator/operators/workspace/register" // routers "github.com/kloudlite/operator/operators/routers/controller" - serviceIntercept "github.com/kloudlite/operator/operators/service-intercept/controller" workmachine "github.com/kloudlite/operator/operators/workmachine/register" - workspace "github.com/kloudlite/operator/operators/workspace/register" - pluginHelmChart "github.com/kloudlite/plugin-helm-chart/kloudlite" - pluginMongoDB "github.com/kloudlite/plugin-mongodb/kloudlite" ) func main() { mgr := operator.New("agent-operator") // kloudlite resources - app.RegisterInto(mgr) - project.RegisterInto(mgr) + // app.RegisterInto(mgr) + // project.RegisterInto(mgr) // helmCharts.RegisterInto(mgr) // routers.RegisterInto(mgr) // kloudlite managed services - msvcAndMres.RegisterInto(mgr) + // msvcAndMres.RegisterInto(mgr) // msvcMongo.RegisterInto(mgr) // msvcRedis.RegisterInto(mgr) @@ -41,18 +32,18 @@ func main() { lifecycle.RegisterInto(mgr) // kloudlite resource status updates - resourceWatcher.RegisterInto(mgr) + // resourceWatcher.RegisterInto(mgr) // distribution.RegisterInto(mgr) - networkingv1.RegisterInto(mgr) + // networkingv1.RegisterInto(mgr) - serviceIntercept.RegisterInto(mgr) + // serviceIntercept.RegisterInto(mgr) workspace.RegisterInto(mgr) workmachine.RegisterInto(mgr) - pluginMongoDB.RegisterInto(mgr) - pluginHelmChart.RegisterInto(mgr) + // pluginMongoDB.RegisterInto(mgr) + // pluginHelmChart.RegisterInto(mgr) mgr.Start() } diff --git a/config/crd/bases/crds.kloudlite.io_workspaces.yaml b/config/crd/bases/crds.kloudlite.io_workspaces.yaml index b8147b8a..80a07fa7 100644 --- a/config/crd/bases/crds.kloudlite.io_workspaces.yaml +++ b/config/crd/bases/crds.kloudlite.io_workspaces.yaml @@ -12,16 +12,16 @@ spec: listKind: WorkspaceList plural: workspaces singular: workspace - scope: Cluster + scope: Namespaced versions: - additionalPrinterColumns: - jsonPath: .status.lastReconcileTime name: Seen type: date - - jsonPath: .metadata.annotations.kloudlite\.io\/checks + - jsonPath: .metadata.annotations.kloudlite\.io\/operator\.checks name: Checks type: string - - jsonPath: .metadata.annotations.kloudlite\.io\/resource\.ready + - jsonPath: .metadata.annotations.kloudlite\.io\/operator\.resource\.ready name: Ready type: string - jsonPath: .metadata.creationTimestamp @@ -63,8 +63,6 @@ spec: imagePullPolicy: default: IfNotPresent type: string - nodeName: - type: string router: description: RouterSpec defines the desired state of Router properties: @@ -159,12 +157,15 @@ spec: - "ON" - "OFF" type: string + workMachine: + description: Name of work machine + type: string required: - imagePullPolicy - - nodeName - router - serviceAccountName - state + - workMachine type: object status: properties: diff --git a/operators/workmachine/internal/controllers/workmachine/controller.go b/operators/workmachine/internal/controllers/workmachine/controller.go index 61d84902..71c057db 100644 --- a/operators/workmachine/internal/controllers/workmachine/controller.go +++ b/operators/workmachine/internal/controllers/workmachine/controller.go @@ -115,9 +115,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. return step.ReconcilerResponse() } - if step := r.createSSHJumpServer(req); !step.ShouldProceed() { - return step.ReconcilerResponse() - } + // if step := r.createSSHJumpServer(req); !step.ShouldProceed() { + // return step.ReconcilerResponse() + // } req.Object.Status.IsReady = true return ctrl.Result{}, nil diff --git a/operators/workmachine/internal/templates/embed.go b/operators/workmachine/internal/templates/embed.go index b4589382..259ad4da 100644 --- a/operators/workmachine/internal/templates/embed.go +++ b/operators/workmachine/internal/templates/embed.go @@ -16,6 +16,7 @@ const ( WorkspaceTemplate templateFile = "./workspace.yml.tpl" WorkMachineLifecycleTemplate templateFile = "./workmachine-lifecycle.yml.tpl" JumpServerDeploymentSpec templateFile = "./ssh-jumpserver-deployment-spec.yml.tpl" + AuthorizedKeysTemplate templateFile = "./ssh-jumpserver-authorized-keys.tpl" ) func Read(t templateFile) ([]byte, error) { diff --git a/operators/workspace/examples/sample.yml b/operators/workspace/examples/sample.yml index f5f22b57..76b7d235 100644 --- a/operators/workspace/examples/sample.yml +++ b/operators/workspace/examples/sample.yml @@ -2,10 +2,9 @@ apiVersion: crds.kloudlite.io/v1 kind: Workspace metadata: name: sample - namespace: default + namespace: wm-sample spec: state: "ON" - nodeName: "worker-2" serviceAccountName: "kloudlite" enableCodeServer: false enableVSCodeServer: false diff --git a/operators/workspace/internal/controllers/workspace/controller.go b/operators/workspace/internal/controllers/workspace/controller.go index a209a721..faba89a3 100644 --- a/operators/workspace/internal/controllers/workspace/controller.go +++ b/operators/workspace/internal/controllers/workspace/controller.go @@ -132,7 +132,7 @@ func (r *Reconciler) createDeployment(req *rApi.Request[*crdsv1.Workspace]) step Namespace: obj.Namespace, OwnerReferences: []metav1.OwnerReference{fn.AsOwner(obj, true)}, }, - NodeName: obj.Spec.NodeName, + WorkMachineName: obj.Spec.WorkMachine, ServiceAccountName: obj.Spec.ServiceAccountName, ImageInitContainer: r.Env.WorkspaceImageInitContainer, ImageSSH: r.Env.WorkspaceImageSSH, diff --git a/operators/workspace/internal/templates/types.go b/operators/workspace/internal/templates/types.go index af36ab3e..4ebefba8 100644 --- a/operators/workspace/internal/templates/types.go +++ b/operators/workspace/internal/templates/types.go @@ -30,4 +30,6 @@ type WorkspaceTemplateArgs struct { KloudliteDeviceFQDN string RouterSpec crdsv1.RouterSpec + + WorkMachineName string } diff --git a/operators/workspace/internal/templates/workspace.yml.tpl b/operators/workspace/internal/templates/workspace.yml.tpl index 8378d585..98071094 100644 --- a/operators/workspace/internal/templates/workspace.yml.tpl +++ b/operators/workspace/internal/templates/workspace.yml.tpl @@ -13,12 +13,12 @@ spec: kloudlite.io/gateway.enabled: "false" spec: hostname: {{.Metadata.Name}} - nodeName: {{.NodeName}} - serviceAccount: {{.ServiceAccountName | squote}} + nodeName: {{.WorkMachineName}} + # serviceAccount: {{.ServiceAccountName | squote}} tolerations: - key: "kloudlite.io/worknode" operator: "Equal" - value: {{.NodeName |squote}} + value: {{.WorkMachineName |squote}} effect: "NoExecute" initContainers: - name: init-home-dir @@ -27,13 +27,10 @@ spec: env: - name: KL_WORKSPACE value: {{.Metadata.Name}} - - name: HOME value: "/home/kl" - - name: KL_BOX_MODE value: "true" - securityContext: runAsUser: 1000 runAsGroup: 1000 @@ -43,6 +40,7 @@ spec: - | set -e set +x + sleep infinity if [ ! -d "/nix/store" ]; then curl -L https://nixos.org/nix/install | sh mkdir -p ~/.config/nix @@ -115,6 +113,10 @@ spec: - mountPath: /env name: containerenv + + - mountPath: /home/kl/.ssh/authorized_keys + name: sshkey + subPath: authorized_keys containers: - name: ssh @@ -186,16 +188,20 @@ spec: {{ end }} volumes: + - name: sshkey + secret: + secretName: ssh-public-keys + - name: containerenv emptyDir: {} - name: home-dir hostPath: - path: /var/user-home/ + path: /external-volume/user-home - name: nix-dir hostPath: - path: /var/nix-dir/ + path: /external-volume/nix --- apiVersion: v1 kind: Service @@ -204,23 +210,23 @@ spec: ports: - name: "ssh" protocol: "TCP" - port: "22" - targetPort: "22" + port: 22 + targetPort: 22 - name: "ttyd-server" protocol: "TCP" - port: "54535" - targetPort: "54535" + port: 54535 + targetPort: 54535 - name: "jupyter-server" protocol: "TCP" - port: "8888" - targetPort: "8888" + port: 8888 + targetPort: 8888 - name: "code-server" protocol: "TCP" - port: "8080" - targetPort: "8080" + port: 8080 + targetPort: 8080 --- apiVersion: crds.kloudlite.io/v1 From 370362bfb8260211ae6bee1df26d52bd81b34ada Mon Sep 17 00:00:00 2001 From: nxtcoder36 Date: Thu, 17 Apr 2025 15:40:45 +0530 Subject: [PATCH 17/27] workspace updated --- apis/crds/v1/workmachine_types.go | 4 +- cmd/agent-operator/main.go | 4 +- .../bases/crds.kloudlite.io_workmachines.yaml | 7 -- .../bases/crds.kloudlite.io_workspaces.yaml | 89 ------------------- go.mod | 1 - go.sum | 2 - .../controllers/workmachine/controller.go | 14 +-- 7 files changed, 12 insertions(+), 109 deletions(-) diff --git a/apis/crds/v1/workmachine_types.go b/apis/crds/v1/workmachine_types.go index 09234f85..ffab185b 100644 --- a/apis/crds/v1/workmachine_types.go +++ b/apis/crds/v1/workmachine_types.go @@ -8,8 +8,8 @@ import ( ) type AWSMachineConfig struct { - Region string `json:"region" graphql:"noinput"` - AvailabilityZone string `json:"availabilityZone"` + // Region string `json:"region" graphql:"noinput"` + // AvailabilityZone string `json:"availabilityZone"` AMI string `json:"ami"` InstanceType string `json:"instanceType"` diff --git a/cmd/agent-operator/main.go b/cmd/agent-operator/main.go index 87f279c0..e7215447 100644 --- a/cmd/agent-operator/main.go +++ b/cmd/agent-operator/main.go @@ -8,7 +8,7 @@ import ( // routers "github.com/kloudlite/operator/operators/routers/controller" - workspace "github.com/kloudlite/operator/operators/workspace/register" + workmachine "github.com/kloudlite/operator/operators/workmachine/register" ) func main() { @@ -38,7 +38,7 @@ func main() { // networkingv1.RegisterInto(mgr) // serviceIntercept.RegisterInto(mgr) - workspace.RegisterInto(mgr) + workmachine.RegisterInto(mgr) // workmachine.RegisterInto(mgr) // pluginMongoDB.RegisterInto(mgr) diff --git a/config/crd/bases/crds.kloudlite.io_workmachines.yaml b/config/crd/bases/crds.kloudlite.io_workmachines.yaml index 5da42b45..41a0b43e 100644 --- a/config/crd/bases/crds.kloudlite.io_workmachines.yaml +++ b/config/crd/bases/crds.kloudlite.io_workmachines.yaml @@ -59,8 +59,6 @@ spec: properties: ami: type: string - availabilityZone: - type: string externalVolumeSize: type: integer externalVolumeType: @@ -70,8 +68,6 @@ spec: type: string instanceType: type: string - region: - type: string rootVolumeSize: default: 50 type: integer @@ -80,11 +76,9 @@ spec: type: string required: - ami - - availabilityZone - externalVolumeSize - externalVolumeType - instanceType - - region - rootVolumeSize - rootVolumeType type: object @@ -146,7 +140,6 @@ spec: type: string required: - aws - - jobParams - sshPublicKeys - state type: object diff --git a/config/crd/bases/crds.kloudlite.io_workspaces.yaml b/config/crd/bases/crds.kloudlite.io_workspaces.yaml index 80a07fa7..711f160d 100644 --- a/config/crd/bases/crds.kloudlite.io_workspaces.yaml +++ b/config/crd/bases/crds.kloudlite.io_workspaces.yaml @@ -63,93 +63,6 @@ spec: imagePullPolicy: default: IfNotPresent type: string - router: - description: RouterSpec defines the desired state of Router - properties: - backendProtocol: - type: string - basicAuth: - properties: - enabled: - type: boolean - secretName: - type: string - username: - type: string - required: - - enabled - type: object - cors: - properties: - allowCredentials: - type: boolean - enabled: - default: false - type: boolean - origins: - items: - type: string - type: array - type: object - domains: - items: - type: string - type: array - https: - properties: - clusterIssuer: - type: string - enabled: - default: true - type: boolean - forceRedirect: - type: boolean - required: - - enabled - type: object - ingressClass: - type: string - maxBodySizeInMB: - type: integer - nginxIngressAnnotations: - additionalProperties: - type: string - description: NginxIngressAnnotations is additional list of annotations - on ingress resource - type: object - rateLimit: - properties: - connections: - type: integer - enabled: - type: boolean - rpm: - type: integer - rps: - type: integer - type: object - routes: - items: - properties: - app: - type: string - path: - description: Lambda string `json:"lambda,omitempty"` - type: string - port: - type: integer - rewrite: - default: false - type: boolean - required: - - app - - path - - port - type: object - type: array - required: - - domains - type: object serviceAccountName: type: string state: @@ -162,8 +75,6 @@ spec: type: string required: - imagePullPolicy - - router - - serviceAccountName - state - workMachine type: object diff --git a/go.mod b/go.mod index 75605919..d43ab8cf 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e github.com/influxdata/influxdb-client-go/v2 v2.14.0 github.com/kloudlite/operator/toolkit v0.0.0-20250316093242-493e9b587c10 - github.com/kloudlite/plugin-helm-chart v0.0.0-20250317052100-fef043b111a2 github.com/kloudlite/plugin-mongodb v0.0.0-20250316175205-312ba86d8873 github.com/matoous/go-nanoid/v2 v2.1.0 github.com/miekg/dns v1.1.62 diff --git a/go.sum b/go.sum index 03ba89bd..4ea5e481 100644 --- a/go.sum +++ b/go.sum @@ -276,8 +276,6 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= -github.com/kloudlite/plugin-helm-chart v0.0.0-20250317052100-fef043b111a2 h1:4DoLvbPEjYVBMnNpVmeR9OoI6Q9kebd6HnDpkwFts+Y= -github.com/kloudlite/plugin-helm-chart v0.0.0-20250317052100-fef043b111a2/go.mod h1:TZUQ5mREV+UjGVJhPsgmHMS67Mlf2Guit4905Pi/5sc= github.com/kloudlite/plugin-mongodb v0.0.0-20250316175205-312ba86d8873 h1:6GKV4bZoNzCC5JvyqxHAhjXXSpSjzAXQHklwXqz5YJQ= github.com/kloudlite/plugin-mongodb v0.0.0-20250316175205-312ba86d8873/go.mod h1:dqT/Qia369uD783N8N58RZPhRPZXywV85nLGDwcq698= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= diff --git a/operators/workmachine/internal/controllers/workmachine/controller.go b/operators/workmachine/internal/controllers/workmachine/controller.go index 831af07d..bba2efd6 100644 --- a/operators/workmachine/internal/controllers/workmachine/controller.go +++ b/operators/workmachine/internal/controllers/workmachine/controller.go @@ -147,9 +147,11 @@ type ClusterParams struct { K3sVersion string `json:"k3s_version"` - AwsVPCName string `json:"aws_vpc_name"` - AwsVPCId string `json:"aws_vpc_id"` - SubnetId string `json:"subnet_id"` + AwsVPCName string `json:"aws_vpc_name"` + AwsVPCId string `json:"aws_vpc_id"` + AwsPublicSubnet string `json:"aws_public_subnet"` + AwsRegion string `json:"aws_region"` + AwsAvailblityZone string `json:"aws_availblity_zone"` AwsNLBDNSHost string `json:"aws_nlb_dns_host"` @@ -181,7 +183,7 @@ func (r *Reconciler) parseSpecIntoTFValues(ctx context.Context, obj *crdsv1.Work case ct.CloudProviderAWS: { return json.Marshal(map[string]any{ - "aws_region": obj.Spec.AWSMachineConfig.Region, + "aws_region": cp.AwsRegion, "trace_id": "workmachine-" + obj.Name, "vpc_id": cp.AwsVPCId, "name": obj.Name, @@ -197,7 +199,7 @@ func (r *Reconciler) parseSpecIntoTFValues(ctx context.Context, obj *crdsv1.Work return "stopped" }(), - "availability_zone": obj.Spec.AWSMachineConfig.AvailabilityZone, + "availability_zone": cp.AwsAvailblityZone, // "iam_instance_profile": func() string { // if obj.Spec.AWSMachineConfig.IAMInstanceProfileRole != nil { // return *obj.Spec.AWSMachineConfig.IAMInstanceProfileRole @@ -207,7 +209,7 @@ func (r *Reconciler) parseSpecIntoTFValues(ctx context.Context, obj *crdsv1.Work "root_volume_size": obj.Spec.AWSMachineConfig.RootVolumeSize, "root_volume_type": obj.Spec.AWSMachineConfig.RootVolumeType, "security_group_ids": cp.AwsSecurityGroupIDs, - "subnet_id": cp.SubnetId, + "subnet_id": cp.AwsPublicSubnet, }) } default: From 79b058b419db5fc5c71ad80353754c761e443dba Mon Sep 17 00:00:00 2001 From: nxtcoder36 Date: Thu, 17 Apr 2025 16:47:23 +0530 Subject: [PATCH 18/27] workspace types updated --- apis/crds/v1/workspace_types.go | 5 +++-- config/crd/bases/crds.kloudlite.io_workspaces.yaml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/apis/crds/v1/workspace_types.go b/apis/crds/v1/workspace_types.go index c09f27a0..2d46c84d 100644 --- a/apis/crds/v1/workspace_types.go +++ b/apis/crds/v1/workspace_types.go @@ -18,8 +18,9 @@ const ( // WorkspaceSpec defines the desired state of Workspace type WorkspaceSpec struct { // Name of work machine - WorkMachine string `json:"workMachine"` + WorkMachine string `json:"workMachine" graphql:"noinput"` + // +kubebuilder:default=ON State WorkspaceState `json:"state"` ServiceAccountName string `json:"serviceAccountName,omitempty"` @@ -29,7 +30,7 @@ type WorkspaceSpec struct { EnableVSCodeServer bool `json:"enableVSCodeServer,omitempty"` // +kubebuilder:default=IfNotPresent - ImagePullPolicy string `json:"imagePullPolicy"` + ImagePullPolicy string `json:"imagePullPolicy,omitempty"` // Router RouterSpec `json:"router"` } diff --git a/config/crd/bases/crds.kloudlite.io_workspaces.yaml b/config/crd/bases/crds.kloudlite.io_workspaces.yaml index 711f160d..725fac87 100644 --- a/config/crd/bases/crds.kloudlite.io_workspaces.yaml +++ b/config/crd/bases/crds.kloudlite.io_workspaces.yaml @@ -66,6 +66,7 @@ spec: serviceAccountName: type: string state: + default: "ON" enum: - "ON" - "OFF" @@ -74,7 +75,6 @@ spec: description: Name of work machine type: string required: - - imagePullPolicy - state - workMachine type: object From 96a25ca74c4aa9bbbd45b4f52ff90fb51948739f Mon Sep 17 00:00:00 2001 From: nxtcoder36 Date: Thu, 17 Apr 2025 17:31:32 +0530 Subject: [PATCH 19/27] workmachine updated --- cmd/agent-operator/main.go | 2 ++ .../workspace/internal/controllers/workspace/controller.go | 1 + 2 files changed, 3 insertions(+) diff --git a/cmd/agent-operator/main.go b/cmd/agent-operator/main.go index e7215447..80e6aee9 100644 --- a/cmd/agent-operator/main.go +++ b/cmd/agent-operator/main.go @@ -5,6 +5,7 @@ import ( // helmCharts "github.com/kloudlite/operator/operators/helm-charts/controller" lifecycle "github.com/kloudlite/operator/operators/lifecycle/controller" + workspace "github.com/kloudlite/operator/operators/workspace/register" // routers "github.com/kloudlite/operator/operators/routers/controller" @@ -39,6 +40,7 @@ func main() { // serviceIntercept.RegisterInto(mgr) workmachine.RegisterInto(mgr) + workspace.RegisterInto(mgr) // workmachine.RegisterInto(mgr) // pluginMongoDB.RegisterInto(mgr) diff --git a/operators/workspace/internal/controllers/workspace/controller.go b/operators/workspace/internal/controllers/workspace/controller.go index 54f237c2..710b8e13 100644 --- a/operators/workspace/internal/controllers/workspace/controller.go +++ b/operators/workspace/internal/controllers/workspace/controller.go @@ -160,6 +160,7 @@ func (r *Reconciler) createDeployment(req *rApi.Request[*crdsv1.Workspace]) step EnableVSCodeServer: obj.Spec.EnableVSCodeServer, ImageVscodeServer: r.Env.WorkspcaeImageVscodeServer, + PortConfig: PortConfig, ImagePullPolicy: obj.Spec.ImagePullPolicy, KloudliteDeviceFQDN: fmt.Sprintf("%s-headless.%s.svc.cluster.local", obj.Name, obj.Namespace), From 86e6a95e99f137f75d2ecf3d82b59c8553a302b8 Mon Sep 17 00:00:00 2001 From: nxtcoder17 Date: Fri, 18 Apr 2025 11:12:13 +0530 Subject: [PATCH 20/27] feat: workmachine ssh gen --- apis/crds/v1/workmachine_types.go | 4 +- cmd/agent-operator/main.go | 37 +++++++++++-------- go.mod | 1 + go.sum | 2 + .../controllers/workmachine/controller.go | 37 ++++++++++++++++--- .../ssh-jumpserver-deployment-spec.yml.tpl | 2 +- pkg/ssh/gen_keys.go | 34 +++++++++++++++++ 7 files changed, 94 insertions(+), 23 deletions(-) create mode 100644 pkg/ssh/gen_keys.go diff --git a/apis/crds/v1/workmachine_types.go b/apis/crds/v1/workmachine_types.go index ffab185b..7060e2de 100644 --- a/apis/crds/v1/workmachine_types.go +++ b/apis/crds/v1/workmachine_types.go @@ -64,8 +64,8 @@ func (wms *WorkMachineSpec) GetCloudProvider() ct.CloudProvider { } type WorkMachineStatus struct { - rApi.Status `json:"status,omitempty"` - MachinePulicSSHKey string `json:"machineSSHKey,omitempty"` + rApi.Status `json:"status,omitempty"` + MachinePublicSSHKey string `json:"machineSSHKey,omitempty"` } // +kubebuilder:object:root=true diff --git a/cmd/agent-operator/main.go b/cmd/agent-operator/main.go index 80e6aee9..d4f3dbd3 100644 --- a/cmd/agent-operator/main.go +++ b/cmd/agent-operator/main.go @@ -3,26 +3,35 @@ package main import ( "github.com/kloudlite/operator/toolkit/operator" - // helmCharts "github.com/kloudlite/operator/operators/helm-charts/controller" + app "github.com/kloudlite/operator/operators/app-n-lambda/controller" + helmCharts "github.com/kloudlite/operator/operators/helm-charts/controller" lifecycle "github.com/kloudlite/operator/operators/lifecycle/controller" - workspace "github.com/kloudlite/operator/operators/workspace/register" - // routers "github.com/kloudlite/operator/operators/routers/controller" + msvcAndMres "github.com/kloudlite/operator/operators/msvc-n-mres/controller" + networkingv1 "github.com/kloudlite/operator/operators/networking/register" + project "github.com/kloudlite/operator/operators/project/controller" + resourceWatcher "github.com/kloudlite/operator/operators/resource-watcher/controller" + + routers "github.com/kloudlite/operator/operators/routers/controller" + serviceIntercept "github.com/kloudlite/operator/operators/service-intercept/controller" workmachine "github.com/kloudlite/operator/operators/workmachine/register" + workspace "github.com/kloudlite/operator/operators/workspace/register" + pluginHelmChart "github.com/kloudlite/plugin-helm-chart/kloudlite" + pluginMongoDB "github.com/kloudlite/plugin-mongodb/kloudlite" ) func main() { mgr := operator.New("agent-operator") // kloudlite resources - // app.RegisterInto(mgr) - // project.RegisterInto(mgr) - // helmCharts.RegisterInto(mgr) - // routers.RegisterInto(mgr) + app.RegisterInto(mgr) + project.RegisterInto(mgr) + helmCharts.RegisterInto(mgr) + routers.RegisterInto(mgr) // kloudlite managed services - // msvcAndMres.RegisterInto(mgr) + msvcAndMres.RegisterInto(mgr) // msvcMongo.RegisterInto(mgr) // msvcRedis.RegisterInto(mgr) @@ -32,19 +41,17 @@ func main() { lifecycle.RegisterInto(mgr) // kloudlite resource status updates - // resourceWatcher.RegisterInto(mgr) + resourceWatcher.RegisterInto(mgr) // distribution.RegisterInto(mgr) - // networkingv1.RegisterInto(mgr) - - // serviceIntercept.RegisterInto(mgr) + networkingv1.RegisterInto(mgr) + serviceIntercept.RegisterInto(mgr) workmachine.RegisterInto(mgr) workspace.RegisterInto(mgr) - // workmachine.RegisterInto(mgr) - // pluginMongoDB.RegisterInto(mgr) - // pluginHelmChart.RegisterInto(mgr) + pluginMongoDB.RegisterInto(mgr) + pluginHelmChart.RegisterInto(mgr) mgr.Start() } diff --git a/go.mod b/go.mod index d43ab8cf..75605919 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e github.com/influxdata/influxdb-client-go/v2 v2.14.0 github.com/kloudlite/operator/toolkit v0.0.0-20250316093242-493e9b587c10 + github.com/kloudlite/plugin-helm-chart v0.0.0-20250317052100-fef043b111a2 github.com/kloudlite/plugin-mongodb v0.0.0-20250316175205-312ba86d8873 github.com/matoous/go-nanoid/v2 v2.1.0 github.com/miekg/dns v1.1.62 diff --git a/go.sum b/go.sum index 4ea5e481..03ba89bd 100644 --- a/go.sum +++ b/go.sum @@ -276,6 +276,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/kloudlite/plugin-helm-chart v0.0.0-20250317052100-fef043b111a2 h1:4DoLvbPEjYVBMnNpVmeR9OoI6Q9kebd6HnDpkwFts+Y= +github.com/kloudlite/plugin-helm-chart v0.0.0-20250317052100-fef043b111a2/go.mod h1:TZUQ5mREV+UjGVJhPsgmHMS67Mlf2Guit4905Pi/5sc= github.com/kloudlite/plugin-mongodb v0.0.0-20250316175205-312ba86d8873 h1:6GKV4bZoNzCC5JvyqxHAhjXXSpSjzAXQHklwXqz5YJQ= github.com/kloudlite/plugin-mongodb v0.0.0-20250316175205-312ba86d8873/go.mod h1:dqT/Qia369uD783N8N58RZPhRPZXywV85nLGDwcq698= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= diff --git a/operators/workmachine/internal/controllers/workmachine/controller.go b/operators/workmachine/internal/controllers/workmachine/controller.go index bba2efd6..5e92beb6 100644 --- a/operators/workmachine/internal/controllers/workmachine/controller.go +++ b/operators/workmachine/internal/controllers/workmachine/controller.go @@ -13,6 +13,7 @@ import ( "github.com/kloudlite/operator/operators/workmachine/internal/env" "github.com/kloudlite/operator/operators/workmachine/internal/templates" "github.com/kloudlite/operator/pkg/constants" + "github.com/kloudlite/operator/pkg/ssh" fn "github.com/kloudlite/operator/toolkit/functions" "github.com/kloudlite/operator/toolkit/kubectl" rApi "github.com/kloudlite/operator/toolkit/reconciler" @@ -46,10 +47,11 @@ func (r *Reconciler) GetName() string { } const ( - createWorkMachineJob string = "create-work-machine-job" - createTargetNamespace string = "create-target-namespace" - createSSHPublicKeysSecret string = "create-ssh-public-keys-secret" - createSSHJumpServerDeployment string = "create-ssh-jumpserver-deployment" + createWorkMachineJob string = "create-work-machine-job" + createTargetNamespace string = "create-target-namespace" + createSSHPublicKeysSecret string = "create-ssh-public-keys-secret" + createMachinePublicPrivateKeyPair string = "create-machine-public-private-key-pair" + createSSHJumpServerDeployment string = "create-ssh-jumpserver-deployment" ) const ( @@ -333,6 +335,31 @@ func (r *Reconciler) createSSHPublicKeysSecret(req *rApi.Request[*crdsv1.WorkMac return check.Failed(err) } + if secret.Data["private_key"] == nil || secret.Data["public_key"] == nil { + privateKeyPEM, publicKey, err := ssh.GenerateSSHKeyPair() + if err != nil { + return check.Failed(err) + } + + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, secret, func() error { + if secret.Data == nil { + secret.Data = make(map[string][]byte, 2) + } + secret.Data["public_key"] = publicKey + secret.Data["private_key"] = privateKeyPEM + return nil + }); err != nil { + return check.Failed(err) + } + } + + // if obj.Status.MachinePublicSSHKey == "" { + // obj.Status.MachinePublicSSHKey = string(secret.Data["public_key"]) + // if err := r.Status().Update(ctx, obj); err != nil { + // return check.Failed(err) + // } + // } + return check.Completed() } @@ -353,7 +380,7 @@ func (r *Reconciler) createSSHJumpServer(req *rApi.Request[*crdsv1.WorkMachine]) return check.Failed(err) } - deployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: sshJumpServerName, Namespace: obj.Namespace}} + deployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: sshJumpServerName, Namespace: obj.Spec.TargetNamespace}} if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { deployment.SetOwnerReferences([]metav1.OwnerReference{fn.AsOwner(obj, true)}) fn.MapSet(&deployment.Annotations, constants.DescriptionKey, "this deployment is a ssh jump server used to allow users to jump to different workspaces") diff --git a/operators/workmachine/internal/templates/ssh-jumpserver-deployment-spec.yml.tpl b/operators/workmachine/internal/templates/ssh-jumpserver-deployment-spec.yml.tpl index 3642ced5..475a4eb4 100644 --- a/operators/workmachine/internal/templates/ssh-jumpserver-deployment-spec.yml.tpl +++ b/operators/workmachine/internal/templates/ssh-jumpserver-deployment-spec.yml.tpl @@ -14,7 +14,7 @@ template: volumeMounts: - name: ssh-secret mountPath: /home/kl/.ssh/authorized_keys - subPath: authorized_keys + subPath: authorized_keys volumes: - name: ssh-secret diff --git a/pkg/ssh/gen_keys.go b/pkg/ssh/gen_keys.go new file mode 100644 index 00000000..daafb13d --- /dev/null +++ b/pkg/ssh/gen_keys.go @@ -0,0 +1,34 @@ +package ssh + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + + "golang.org/x/crypto/ssh" +) + +func GenerateSSHKeyPair() (privateKeyPEM []byte, publicKey []byte, err error) { + // Generate RSA private key + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, err + } + + // Encode private key to PEM format + privDER := x509.MarshalPKCS1PrivateKey(privateKey) + privPEM := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: privDER, + }) + + // Generate public key in OpenSSH authorized_keys format + pub, err := ssh.NewPublicKey(&privateKey.PublicKey) + if err != nil { + return nil, nil, err + } + pubBytes := ssh.MarshalAuthorizedKey(pub) + + return privPEM, pubBytes, nil +} From c3c67cd7c44a67c2b99bf60f1dd8eda198be2b13 Mon Sep 17 00:00:00 2001 From: nxtcoder36 Date: Fri, 18 Apr 2025 14:19:03 +0530 Subject: [PATCH 21/27] templates file updated --- cmd/agent-operator/main.go | 36 ++++++------------- .../templates/deployments/sts.yml.tpl | 13 +++---- 2 files changed, 18 insertions(+), 31 deletions(-) diff --git a/cmd/agent-operator/main.go b/cmd/agent-operator/main.go index d4f3dbd3..80f52343 100644 --- a/cmd/agent-operator/main.go +++ b/cmd/agent-operator/main.go @@ -3,55 +3,41 @@ package main import ( "github.com/kloudlite/operator/toolkit/operator" - app "github.com/kloudlite/operator/operators/app-n-lambda/controller" - helmCharts "github.com/kloudlite/operator/operators/helm-charts/controller" - lifecycle "github.com/kloudlite/operator/operators/lifecycle/controller" - - msvcAndMres "github.com/kloudlite/operator/operators/msvc-n-mres/controller" - networkingv1 "github.com/kloudlite/operator/operators/networking/register" - project "github.com/kloudlite/operator/operators/project/controller" - resourceWatcher "github.com/kloudlite/operator/operators/resource-watcher/controller" - - routers "github.com/kloudlite/operator/operators/routers/controller" - - serviceIntercept "github.com/kloudlite/operator/operators/service-intercept/controller" workmachine "github.com/kloudlite/operator/operators/workmachine/register" workspace "github.com/kloudlite/operator/operators/workspace/register" - pluginHelmChart "github.com/kloudlite/plugin-helm-chart/kloudlite" - pluginMongoDB "github.com/kloudlite/plugin-mongodb/kloudlite" ) func main() { mgr := operator.New("agent-operator") // kloudlite resources - app.RegisterInto(mgr) - project.RegisterInto(mgr) - helmCharts.RegisterInto(mgr) - routers.RegisterInto(mgr) + // app.RegisterInto(mgr) + // project.RegisterInto(mgr) + // helmCharts.RegisterInto(mgr) + // routers.RegisterInto(mgr) // kloudlite managed services - msvcAndMres.RegisterInto(mgr) + // msvcAndMres.RegisterInto(mgr) // msvcMongo.RegisterInto(mgr) // msvcRedis.RegisterInto(mgr) // msvcMysql.RegisterInto(mgr) // msvcPostgres.RegisterInto(mgr) - lifecycle.RegisterInto(mgr) + // lifecycle.RegisterInto(mgr) // kloudlite resource status updates - resourceWatcher.RegisterInto(mgr) + // resourceWatcher.RegisterInto(mgr) // distribution.RegisterInto(mgr) - networkingv1.RegisterInto(mgr) - serviceIntercept.RegisterInto(mgr) + // networkingv1.RegisterInto(mgr) + // serviceIntercept.RegisterInto(mgr) workmachine.RegisterInto(mgr) workspace.RegisterInto(mgr) - pluginMongoDB.RegisterInto(mgr) - pluginHelmChart.RegisterInto(mgr) + // pluginMongoDB.RegisterInto(mgr) + // pluginHelmChart.RegisterInto(mgr) mgr.Start() } diff --git a/operators/workspace/internal/templates/deployments/sts.yml.tpl b/operators/workspace/internal/templates/deployments/sts.yml.tpl index 8601eac5..e386402b 100644 --- a/operators/workspace/internal/templates/deployments/sts.yml.tpl +++ b/operators/workspace/internal/templates/deployments/sts.yml.tpl @@ -49,6 +49,7 @@ spec: if [ ! -d "/home/kl/.ssh" ]; then mkdir -p /home/kl/.ssh + chown -R 1000:1000 /home/kl/.ssh fi if [ -f "/home/kl/.ssh/authorized_keys" ]; then if ! cmp -s /tmp/authorized_keys /home/kl/.ssh/authorized_keys; then @@ -64,14 +65,14 @@ spec: if [ -f "/home/kl/.ssh/id_rsa" ]; then if ! cmp -s /tmp/id_rsa /home/kl/.ssh/id_rsa; then echo "id_rsa file differs, copying new one" - rm /home/kl/.ssh/id_rsa* || true + rm -f /home/kl/.ssh/id_rsa* 2>/dev/null || true cp /tmp/id_rsa /home/kl/.ssh/id_rsa cp /tmp/id_rsa.pub /home/kl/.ssh/id_rsa.pub fi echo "id_rsa file is up to date" else echo "id_rsa file not found, copying new one" - rm /home/kl/.ssh/id_rsa* || true + rm -f /home/kl/.ssh/id_rsa* 2>/dev/null || true cp /tmp/id_rsa /home/kl/.ssh/id_rsa cp /tmp/id_rsa.pub /home/kl/.ssh/id_rsa.pub fi @@ -149,11 +150,11 @@ spec: - mountPath: /tmp/id_rsa.pub name: ssh-keys - subPath: id_rsa.pub + subPath: public_key - mountPath: /tmp/id_rsa name: ssh-keys - subPath: id_rsa + subPath: private_key - mountPath: /nix name: nix-dir @@ -162,7 +163,7 @@ spec: name: containerenv - mountPath: /home/kl/.ssh/authorized_keys - name: sshkey + name: ssh-keys subPath: authorized_keys containers: @@ -235,7 +236,7 @@ spec: {{ end }} volumes: - - name: sshkey + - name: ssh-keys secret: secretName: ssh-public-keys From 244f007ab0f77ce8f685bf57f2b558b456a73239 Mon Sep 17 00:00:00 2001 From: nxtcoder36 Date: Fri, 18 Apr 2025 14:20:31 +0530 Subject: [PATCH 22/27] small change --- cmd/agent-operator/main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/agent-operator/main.go b/cmd/agent-operator/main.go index 80f52343..38dc0848 100644 --- a/cmd/agent-operator/main.go +++ b/cmd/agent-operator/main.go @@ -3,6 +3,8 @@ package main import ( "github.com/kloudlite/operator/toolkit/operator" + lifecycle "github.com/kloudlite/operator/operators/lifecycle/controller" + workmachine "github.com/kloudlite/operator/operators/workmachine/register" workspace "github.com/kloudlite/operator/operators/workspace/register" ) @@ -24,7 +26,7 @@ func main() { // msvcMysql.RegisterInto(mgr) // msvcPostgres.RegisterInto(mgr) - // lifecycle.RegisterInto(mgr) + lifecycle.RegisterInto(mgr) // kloudlite resource status updates // resourceWatcher.RegisterInto(mgr) From 609e729e250e17c509bfb174a181cbb8e8a706bc Mon Sep 17 00:00:00 2001 From: nxtcoder36 Date: Fri, 18 Apr 2025 18:17:18 +0530 Subject: [PATCH 23/27] resouceWatcher adde --- cmd/agent-operator/main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/agent-operator/main.go b/cmd/agent-operator/main.go index 38dc0848..fddbf9d4 100644 --- a/cmd/agent-operator/main.go +++ b/cmd/agent-operator/main.go @@ -5,6 +5,8 @@ import ( lifecycle "github.com/kloudlite/operator/operators/lifecycle/controller" + resourceWatcher "github.com/kloudlite/operator/operators/resource-watcher/controller" + workmachine "github.com/kloudlite/operator/operators/workmachine/register" workspace "github.com/kloudlite/operator/operators/workspace/register" ) @@ -29,7 +31,7 @@ func main() { lifecycle.RegisterInto(mgr) // kloudlite resource status updates - // resourceWatcher.RegisterInto(mgr) + resourceWatcher.RegisterInto(mgr) // distribution.RegisterInto(mgr) From e43b8f086634e48c2a87483f695ebdaccbdc75ef Mon Sep 17 00:00:00 2001 From: nxtcoder36 Date: Mon, 21 Apr 2025 18:45:47 +0530 Subject: [PATCH 24/27] minor changes --- cmd/agent-operator/main.go | 25 +++++++++++++------ .../watch-and-update/controller.go | 2 +- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/cmd/agent-operator/main.go b/cmd/agent-operator/main.go index fddbf9d4..530f9c77 100644 --- a/cmd/agent-operator/main.go +++ b/cmd/agent-operator/main.go @@ -3,25 +3,34 @@ package main import ( "github.com/kloudlite/operator/toolkit/operator" + app "github.com/kloudlite/operator/operators/app-n-lambda/controller" + helmCharts "github.com/kloudlite/operator/operators/helm-charts/controller" lifecycle "github.com/kloudlite/operator/operators/lifecycle/controller" + msvcAndMres "github.com/kloudlite/operator/operators/msvc-n-mres/controller" + networkingv1 "github.com/kloudlite/operator/operators/networking/register" + project "github.com/kloudlite/operator/operators/project/controller" resourceWatcher "github.com/kloudlite/operator/operators/resource-watcher/controller" workmachine "github.com/kloudlite/operator/operators/workmachine/register" workspace "github.com/kloudlite/operator/operators/workspace/register" + + serviceIntercept "github.com/kloudlite/operator/operators/service-intercept/controller" + pluginHelmChart "github.com/kloudlite/plugin-helm-chart/kloudlite" + pluginMongoDB "github.com/kloudlite/plugin-mongodb/kloudlite" ) func main() { mgr := operator.New("agent-operator") // kloudlite resources - // app.RegisterInto(mgr) - // project.RegisterInto(mgr) - // helmCharts.RegisterInto(mgr) + app.RegisterInto(mgr) + project.RegisterInto(mgr) + helmCharts.RegisterInto(mgr) // routers.RegisterInto(mgr) // kloudlite managed services - // msvcAndMres.RegisterInto(mgr) + msvcAndMres.RegisterInto(mgr) // msvcMongo.RegisterInto(mgr) // msvcRedis.RegisterInto(mgr) @@ -35,13 +44,13 @@ func main() { // distribution.RegisterInto(mgr) - // networkingv1.RegisterInto(mgr) - // serviceIntercept.RegisterInto(mgr) + networkingv1.RegisterInto(mgr) + serviceIntercept.RegisterInto(mgr) workmachine.RegisterInto(mgr) workspace.RegisterInto(mgr) - // pluginMongoDB.RegisterInto(mgr) - // pluginHelmChart.RegisterInto(mgr) + pluginMongoDB.RegisterInto(mgr) + pluginHelmChart.RegisterInto(mgr) mgr.Start() } diff --git a/operators/resource-watcher/internal/controllers/watch-and-update/controller.go b/operators/resource-watcher/internal/controllers/watch-and-update/controller.go index 72d63775..e6049d1a 100644 --- a/operators/resource-watcher/internal/controllers/watch-and-update/controller.go +++ b/operators/resource-watcher/internal/controllers/watch-and-update/controller.go @@ -440,7 +440,7 @@ var ( // BuildRunGVK = newGVK("distribution.kloudlite.io/v1", "BuildRun") ClusterManagedServiceGVK = newGVK("crds.kloudlite.io/v1", "ClusterManagedService") - HelmChartGVK = newGVK("crds.kloudlite.io/v1", "HelmChart") + HelmChartGVK = newGVK("plugin-helm-chart.kloudlite.github.com/v1", "HelmChart") ProjectManageServiceGVK = newGVK("crds.kloudlite.io/v1", "ProjectManagedService") // native resources From f03add4a612eff5176fcbb76abd30302e7226dd2 Mon Sep 17 00:00:00 2001 From: nxtcoder17 Date: Tue, 20 May 2025 16:05:10 +0530 Subject: [PATCH 25/27] feat: adds job tracker functions and job status checks --- toolkit/job-helper/job-helper.go | 9 ++++ toolkit/job-helper/job-runner.go | 93 ++++++++++++++++++++++++++++++++ toolkit/reconciler/checks.go | 12 ++--- toolkit/reconciler/request.go | 4 +- 4 files changed, 110 insertions(+), 8 deletions(-) create mode 100644 toolkit/job-helper/job-runner.go diff --git a/toolkit/job-helper/job-helper.go b/toolkit/job-helper/job-helper.go index ae297ba4..938d88a5 100644 --- a/toolkit/job-helper/job-helper.go +++ b/toolkit/job-helper/job-helper.go @@ -23,6 +23,15 @@ const ( JobStatusSuccess JobStatus = "job-status-success" ) +type JobPhase string + +const ( + JobPhasePending JobPhase = "PENDING" + JobPhaseRunning JobPhase = "RUNNING" + JobPhaseFailed JobPhase = "FAILED" + JobPhaseSucceeded JobPhase = "SUCCEEDED" +) + func GetLatestPod(ctx context.Context, cli client.Client, jobNamespace string, jobName string) (*corev1.Pod, error) { var podList corev1.PodList diff --git a/toolkit/job-helper/job-runner.go b/toolkit/job-helper/job-runner.go new file mode 100644 index 00000000..89f3ec29 --- /dev/null +++ b/toolkit/job-helper/job-runner.go @@ -0,0 +1,93 @@ +package job_helper + +import ( + "context" + "errors" + + batchv1 "k8s.io/api/batch/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type JobTracker struct { + ctx context.Context + client client.Client + + job *batchv1.Job + + isTargetJob func(job *batchv1.Job) bool +} + +type JobTrackerArgs struct { + client client.Client + + JobNamespace string + JobName string + + IsTargetJob func(job *batchv1.Job) bool +} + +func NewJobTracker(ctx context.Context, kcli client.Client, args JobTrackerArgs) (*JobTracker, error) { + var job batchv1.Job + if err := kcli.Get(ctx, types.NamespacedName{Name: args.JobName, Namespace: args.JobNamespace}, &job); err != nil { + return nil, err + } + + return &JobTracker{ + ctx: ctx, + client: kcli, + job: &job, + isTargetJob: args.IsTargetJob, + }, nil +} + +func (jr *JobTracker) HasJobFinished() bool { + for _, v := range jr.job.Status.Conditions { + if v.Type == batchv1.JobComplete && v.Status == "True" { + return true + } + + if v.Type == batchv1.JobFailed && v.Status == "True" { + return true + } + + if v.Type == batchv1.JobSuspended && v.Status == "True" { + return true + } + } + + return false +} + +func (jr *JobTracker) Process(ctx context.Context) (phase JobPhase, message string, err error) { + if !jr.isTargetJob(jr.job) { + if !jr.HasJobFinished() { + return JobPhasePending, "waiting for previous jobs to finish execution", nil + } + + if err := DeleteJob(jr.ctx, jr.client, jr.job.Namespace, jr.job.Name); err != nil { + return JobPhasePending, "", errors.Join(errors.New("failed to delete job"), err) + } + + return JobPhasePending, "waiting for job to start", nil + } + + if !jr.HasJobFinished() { + return JobPhaseRunning, "waiting for running job to finish", nil + } + + // check.Message = job_manager.GetTerminationLog(ctx, r.Client, job.Namespace, job.Name) + if jr.job.Status.Succeeded > 0 { + return JobPhaseSucceeded, "", nil + } + + if jr.job.Status.Active > 0 { + return JobPhaseRunning, "waiting for job to complete", nil + } + + if jr.job.Status.Failed > 0 { + return JobPhaseFailed, "", errors.New("install or upgrade job failed") + } + + return JobPhasePending, "", nil +} diff --git a/toolkit/reconciler/checks.go b/toolkit/reconciler/checks.go index 080b631b..7ba9913d 100644 --- a/toolkit/reconciler/checks.go +++ b/toolkit/reconciler/checks.go @@ -50,13 +50,13 @@ func AreChecksEqual(c1 Check, c2 Check) bool { c1.StartedAt.Sub(c2.StartedAt.Time) == 0 } -type checkWrapper[T Resource] struct { +type CheckWrapper[T Resource] struct { checkName string request *Request[T] Check `json:",inline"` } -func (cw *checkWrapper[T]) Failed(err error) step_result.Result { +func (cw *CheckWrapper[T]) Failed(err error) step_result.Result { defer cw.request.LogPostCheck(cw.checkName) _, file, line, _ := runtime.Caller(1) @@ -82,7 +82,7 @@ func (cw *checkWrapper[T]) Failed(err error) step_result.Result { // return cw.request.updateStatus().Continue(false).Err(err) } -func (cw *checkWrapper[T]) StillRunning(err error) step_result.Result { +func (cw *CheckWrapper[T]) StillRunning(err error) step_result.Result { defer cw.request.LogPostCheck(cw.checkName) cw.Check.State = RunningState @@ -104,7 +104,7 @@ func (cw *checkWrapper[T]) StillRunning(err error) step_result.Result { // return cw.request.updateStatus().Continue(false).Err(err) } -func (cw *checkWrapper[T]) Completed() step_result.Result { +func (cw *CheckWrapper[T]) Completed() step_result.Result { defer cw.request.LogPostCheck(cw.checkName) cw.Check.State = CompletedState @@ -119,8 +119,8 @@ func (cw *checkWrapper[T]) Completed() step_result.Result { return step_result.New().Continue(true) } -func NewRunningCheck[T Resource](name string, req *Request[T]) *checkWrapper[T] { - cw := &checkWrapper[T]{ +func NewRunningCheck[T Resource](name string, req *Request[T]) *CheckWrapper[T] { + cw := &CheckWrapper[T]{ checkName: name, request: req, Check: Check{ diff --git a/toolkit/reconciler/request.go b/toolkit/reconciler/request.go index 7f5e4a00..87081935 100644 --- a/toolkit/reconciler/request.go +++ b/toolkit/reconciler/request.go @@ -448,7 +448,7 @@ func (r *Request[T]) AddToOwnedResources(refs ...ResourceRef) { r.resourceRefs = append(r.resourceRefs, refs...) } -func (r *Request[T]) CleanupOwnedResources(check *checkWrapper[T]) stepResult.Result { +func (r *Request[T]) CleanupOwnedResources(check *CheckWrapper[T]) stepResult.Result { resources := r.Object.GetStatus().Resources objects := make([]client.Object, 0, len(resources)) for i := range resources { @@ -474,7 +474,7 @@ INFO: this should only be used for very specific cases, where there is no other Like, when deleting ManagedService - all managed resources should be deleted, but since owner is already getting deleted, there is no point in their proper cleanup */ -func (r *Request[T]) ForceCleanupOwnedResources(check *checkWrapper[T]) stepResult.Result { +func (r *Request[T]) ForceCleanupOwnedResources(check *CheckWrapper[T]) stepResult.Result { ctx := r.Context() resources := r.Object.GetStatus().Resources From fc7be1032770e5275ec1075fec070b9c71fa0b8b Mon Sep 17 00:00:00 2001 From: nxtcoder17 Date: Wed, 4 Jun 2025 23:16:10 +0530 Subject: [PATCH 26/27] feat: router spec and app-controller controller updates --- apis/crds/v1/app_webhook.go | 60 ---- apis/crds/v1/router_types.go | 31 +- apis/crds/v1/zz_generated.deepcopy.go | 17 +- cmd/platform-operator/main.go | 6 +- config/crd/bases/crds.kloudlite.io_apps.yaml | 19 +- .../bases/crds.kloudlite.io_blueprints.yaml | 19 +- .../crd/bases/crds.kloudlite.io_routers.yaml | 19 +- go.mod | 19 +- go.sum | 34 +- .../internal/controllers/app/controller.go | 29 +- operators/routers/internal/env/env.go | 5 - .../internal/router-controller/controller.go | 299 ++++++++---------- .../router-controller/controller_test.go | 4 +- .../internal/router-controller/helpers.go | 38 --- operators/routers/internal/templates/embed.go | 2 +- .../templates/ingress-resource-v2.yml.tpl | 5 +- .../templates/ingress-resource.yml.tpl | 16 +- 17 files changed, 234 insertions(+), 388 deletions(-) delete mode 100644 apis/crds/v1/app_webhook.go diff --git a/apis/crds/v1/app_webhook.go b/apis/crds/v1/app_webhook.go deleted file mode 100644 index 6b3efdfd..00000000 --- a/apis/crds/v1/app_webhook.go +++ /dev/null @@ -1,60 +0,0 @@ -package v1 - -import ( - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - -// log is for logging in this package. -var applog = logf.Log.WithName("app-resource") - -func (r *App) SetupWebhookWithManager(mgr ctrl.Manager) error { - return ctrl.NewWebhookManagedBy(mgr). - For(r). - Complete() -} - -// TODO(user): EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! - -//+kubebuilder:webhook:path=/mutate-crds-kloudlite-io-v1-app,mutating=true,failurePolicy=fail,sideEffects=None,groups=crds.kloudlite.io,resources=apps,verbs=create;update,versions=v1,name=mapp.kb.io,admissionReviewVersions=v1 - -var _ webhook.Defaulter = &App{} - -// Default implements webhook.Defaulter so a webhook will be registered for the type -func (r *App) Default() { - applog.Info("default", "name", r.Name) - - // TODO(user): fill in your defaulting logic. -} - -// TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation. -//+kubebuilder:webhook:path=/validate-crds-kloudlite-io-v1-app,mutating=false,failurePolicy=fail,sideEffects=None,groups=crds.kloudlite.io,resources=apps,verbs=create;update,versions=v1,name=vapp.kb.io,admissionReviewVersions=v1 - -var _ webhook.Validator = &App{} - -// ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (r *App) ValidateCreate() (admission.Warnings, error) { - applog.Info("validate create", "name", r.Name) - - // TODO(user): fill in your validation logic upon object creation. - return nil, nil -} - -// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (r *App) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { - applog.Info("validate update", "name", r.Name) - - // TODO(user): fill in your validation logic upon object update. - return nil, nil -} - -// ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (r *App) ValidateDelete() (admission.Warnings, error) { - applog.Info("validate delete", "name", r.Name) - - // TODO(user): fill in your validation logic upon object deletion. - return nil, nil -} diff --git a/apis/crds/v1/router_types.go b/apis/crds/v1/router_types.go index 0a1d0d75..5f45845f 100644 --- a/apis/crds/v1/router_types.go +++ b/apis/crds/v1/router_types.go @@ -1,19 +1,17 @@ package v1 import ( - "strings" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/kloudlite/operator/pkg/constants" "github.com/kloudlite/operator/toolkit/reconciler" ) type Route struct { - App string `json:"app"` - // Lambda string `json:"lambda,omitempty"` - Path string `json:"path"` - Port uint16 `json:"port"` + Host string `json:"host"` + Service string `json:"service"` + Path string `json:"path"` + Port uint16 `json:"port"` + // +kubebuilder:default=false Rewrite bool `json:"rewrite,omitempty"` } @@ -50,21 +48,18 @@ type RouterSpec struct { IngressClass string `json:"ingressClass,omitempty"` BackendProtocol *string `json:"backendProtocol,omitempty"` Https *Https `json:"https,omitempty"` - // +kubebuilder:validation:Optional RateLimit *RateLimit `json:"rateLimit,omitempty"` MaxBodySizeInMB *int `json:"maxBodySizeInMB,omitempty"` - Domains []string `json:"domains"` - - // Routes is a map of [domain name] to the corresponding []Route - // Routes map[string][]Route `json:"routes"` - Routes []Route `json:"routes,omitempty"` BasicAuth *BasicAuth `json:"basicAuth,omitempty"` Cors *Cors `json:"cors,omitempty"` // NginxIngressAnnotations is additional list of annotations on ingress resource + // INFO: must be used when router does not have direct support for it NginxIngressAnnotations map[string]string `json:"nginxIngressAnnotations,omitempty"` + + Routes []Route `json:"routes,omitempty"` } // +kubebuilder:object:root=true @@ -98,17 +93,11 @@ func (r *Router) GetStatus() *reconciler.Status { } func (r *Router) GetEnsuredLabels() map[string]string { - return map[string]string{ - constants.RouterNameKey: r.Name, - } + return map[string]string{} } func (m *Router) GetEnsuredAnnotations() map[string]string { - return map[string]string{ - // "kloudlite.io/router.domains": strings.Join(fn.MapKeys(m.Spec.Routes), ","), - "kloudlite.io/router.domains": strings.Join(m.Spec.Domains, ","), - "kloudlite.io/router.ingress-class": m.Spec.IngressClass, - } + return map[string]string{} } // +kubebuilder:object:root=true diff --git a/apis/crds/v1/zz_generated.deepcopy.go b/apis/crds/v1/zz_generated.deepcopy.go index 4faeaed5..e5496ebb 100644 --- a/apis/crds/v1/zz_generated.deepcopy.go +++ b/apis/crds/v1/zz_generated.deepcopy.go @@ -8,7 +8,7 @@ import ( "github.com/kloudlite/operator/pkg/json-patch" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apimachinery/pkg/runtime" + runtime "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -1788,16 +1788,6 @@ func (in *RouterSpec) DeepCopyInto(out *RouterSpec) { *out = new(int) **out = **in } - if in.Domains != nil { - in, out := &in.Domains, &out.Domains - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Routes != nil { - in, out := &in.Routes, &out.Routes - *out = make([]Route, len(*in)) - copy(*out, *in) - } if in.BasicAuth != nil { in, out := &in.BasicAuth, &out.BasicAuth *out = new(BasicAuth) @@ -1815,6 +1805,11 @@ func (in *RouterSpec) DeepCopyInto(out *RouterSpec) { (*out)[key] = val } } + if in.Routes != nil { + in, out := &in.Routes, &out.Routes + *out = make([]Route, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouterSpec. diff --git a/cmd/platform-operator/main.go b/cmd/platform-operator/main.go index ee644d43..5fcbd254 100644 --- a/cmd/platform-operator/main.go +++ b/cmd/platform-operator/main.go @@ -23,7 +23,8 @@ import ( serviceIntercept "github.com/kloudlite/operator/operators/service-intercept/controller" workspace "github.com/kloudlite/operator/operators/workspace/register" - pluginMongoDB "github.com/kloudlite/plugin-mongodb/kloudlite" + pluginK3sCluster "github.com/kloudlite/plugin-k3s-cluster/kloudlite" + // pluginMongoDB "github.com/kloudlite/plugin-mongodb/kloudlite" ) func main() { @@ -52,7 +53,8 @@ func main() { // wireguard.RegisterInto(mgr) // MIGRATE // networkingv0.RegisterInto(mgr) // MIGRATE - pluginMongoDB.RegisterInto(mgr) + // pluginMongoDB.RegisterInto(mgr) + pluginK3sCluster.RegisterInto(mgr) workspace.RegisterInto(mgr) mgr.Start() diff --git a/config/crd/bases/crds.kloudlite.io_apps.yaml b/config/crd/bases/crds.kloudlite.io_apps.yaml index e8017c80..9cc4d5c7 100644 --- a/config/crd/bases/crds.kloudlite.io_apps.yaml +++ b/config/crd/bases/crds.kloudlite.io_apps.yaml @@ -343,10 +343,6 @@ spec: type: string type: array type: object - domains: - items: - type: string - type: array https: properties: clusterIssuer: @@ -366,8 +362,9 @@ spec: nginxIngressAnnotations: additionalProperties: type: string - description: NginxIngressAnnotations is additional list of annotations - on ingress resource + description: |- + NginxIngressAnnotations is additional list of annotations on ingress resource + INFO: must be used when router does not have direct support for it type: object rateLimit: properties: @@ -383,24 +380,24 @@ spec: routes: items: properties: - app: + host: type: string path: - description: Lambda string `json:"lambda,omitempty"` type: string port: type: integer rewrite: default: false type: boolean + service: + type: string required: - - app + - host - path - port + - service type: object type: array - required: - - domains type: object serviceAccount: type: string diff --git a/config/crd/bases/crds.kloudlite.io_blueprints.yaml b/config/crd/bases/crds.kloudlite.io_blueprints.yaml index 0dd4ed67..931df359 100644 --- a/config/crd/bases/crds.kloudlite.io_blueprints.yaml +++ b/config/crd/bases/crds.kloudlite.io_blueprints.yaml @@ -348,10 +348,6 @@ spec: type: string type: array type: object - domains: - items: - type: string - type: array https: properties: clusterIssuer: @@ -371,8 +367,9 @@ spec: nginxIngressAnnotations: additionalProperties: type: string - description: NginxIngressAnnotations is additional list - of annotations on ingress resource + description: |- + NginxIngressAnnotations is additional list of annotations on ingress resource + INFO: must be used when router does not have direct support for it type: object rateLimit: properties: @@ -388,24 +385,24 @@ spec: routes: items: properties: - app: + host: type: string path: - description: Lambda string `json:"lambda,omitempty"` type: string port: type: integer rewrite: default: false type: boolean + service: + type: string required: - - app + - host - path - port + - service type: object type: array - required: - - domains type: object serviceAccount: type: string diff --git a/config/crd/bases/crds.kloudlite.io_routers.yaml b/config/crd/bases/crds.kloudlite.io_routers.yaml index 2c183857..fe965d38 100644 --- a/config/crd/bases/crds.kloudlite.io_routers.yaml +++ b/config/crd/bases/crds.kloudlite.io_routers.yaml @@ -86,10 +86,6 @@ spec: type: string type: array type: object - domains: - items: - type: string - type: array https: properties: clusterIssuer: @@ -109,8 +105,9 @@ spec: nginxIngressAnnotations: additionalProperties: type: string - description: NginxIngressAnnotations is additional list of annotations - on ingress resource + description: |- + NginxIngressAnnotations is additional list of annotations on ingress resource + INFO: must be used when router does not have direct support for it type: object rateLimit: properties: @@ -126,24 +123,24 @@ spec: routes: items: properties: - app: + host: type: string path: - description: Lambda string `json:"lambda,omitempty"` type: string port: type: integer rewrite: default: false type: boolean + service: + type: string required: - - app + - host - path - port + - service type: object type: array - required: - - domains type: object status: properties: diff --git a/go.mod b/go.mod index 75605919..b96410fd 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/cert-manager/cert-manager v1.16.2 github.com/charmbracelet/log v0.4.0 github.com/codingconcepts/env v0.0.0-20240618133406-5b0845441187 - github.com/evanphx/json-patch/v5 v5.9.0 + github.com/evanphx/json-patch/v5 v5.9.11 github.com/fatih/color v1.18.0 github.com/go-chi/chi/v5 v5.2.0 github.com/go-redis/redis/v8 v8.11.5 @@ -20,8 +20,9 @@ require ( github.com/gobuffalo/flect v1.0.3 github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e github.com/influxdata/influxdb-client-go/v2 v2.14.0 - github.com/kloudlite/operator/toolkit v0.0.0-20250316093242-493e9b587c10 + github.com/kloudlite/operator/toolkit v0.0.0-20250323044516-4d91d0b9477a github.com/kloudlite/plugin-helm-chart v0.0.0-20250317052100-fef043b111a2 + github.com/kloudlite/plugin-k3s-cluster v0.0.0-20250420192843-f0fae9cd7d36 github.com/kloudlite/plugin-mongodb v0.0.0-20250316175205-312ba86d8873 github.com/matoous/go-nanoid/v2 v2.1.0 github.com/miekg/dns v1.1.62 @@ -46,13 +47,13 @@ require ( google.golang.org/grpc v1.69.2 google.golang.org/protobuf v1.36.0 helm.sh/helm/v3 v3.16.4 - k8s.io/api v0.32.0 - k8s.io/apiextensions-apiserver v0.32.0 - k8s.io/apimachinery v0.32.0 - k8s.io/client-go v0.32.0 + k8s.io/api v0.32.1 + k8s.io/apiextensions-apiserver v0.32.1 + k8s.io/apimachinery v0.32.1 + k8s.io/client-go v0.32.1 k8s.io/kubernetes v1.32.0 k8s.io/utils v0.0.0-20241210054802-24370beab758 - sigs.k8s.io/controller-runtime v0.19.3 + sigs.k8s.io/controller-runtime v0.20.2 sigs.k8s.io/yaml v1.4.0 ) @@ -225,9 +226,9 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20241216192217-9240e9c98484 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiserver v0.32.0 // indirect + k8s.io/apiserver v0.32.1 // indirect k8s.io/cli-runtime v0.32.0 // indirect - k8s.io/component-base v0.32.0 // indirect + k8s.io/component-base v0.32.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 // indirect k8s.io/kubectl v0.32.0 // indirect diff --git a/go.sum b/go.sum index 03ba89bd..eecda73d 100644 --- a/go.sum +++ b/go.sum @@ -130,8 +130,8 @@ github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtz github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= -github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= @@ -278,6 +278,8 @@ github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IX github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kloudlite/plugin-helm-chart v0.0.0-20250317052100-fef043b111a2 h1:4DoLvbPEjYVBMnNpVmeR9OoI6Q9kebd6HnDpkwFts+Y= github.com/kloudlite/plugin-helm-chart v0.0.0-20250317052100-fef043b111a2/go.mod h1:TZUQ5mREV+UjGVJhPsgmHMS67Mlf2Guit4905Pi/5sc= +github.com/kloudlite/plugin-k3s-cluster v0.0.0-20250420192843-f0fae9cd7d36 h1:pOkVQ9ocpQVZ3QSsck2vjzx0WWhH4l0pJDibrEQUA04= +github.com/kloudlite/plugin-k3s-cluster v0.0.0-20250420192843-f0fae9cd7d36/go.mod h1:sqCqURIr6j/SI0hIoA+22Etpcm+9U/MXQXE2iCMmUa4= github.com/kloudlite/plugin-mongodb v0.0.0-20250316175205-312ba86d8873 h1:6GKV4bZoNzCC5JvyqxHAhjXXSpSjzAXQHklwXqz5YJQ= github.com/kloudlite/plugin-mongodb v0.0.0-20250316175205-312ba86d8873/go.mod h1:dqT/Qia369uD783N8N58RZPhRPZXywV85nLGDwcq698= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -677,20 +679,20 @@ gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= helm.sh/helm/v3 v3.16.4 h1:rBn/h9MACw+QlhxQTjpl8Ifx+VTWaYsw3rguGBYBzr0= helm.sh/helm/v3 v3.16.4/go.mod h1:k8QPotUt57wWbi90w3LNmg3/MWcLPigVv+0/X4B8BzA= -k8s.io/api v0.32.0 h1:OL9JpbvAU5ny9ga2fb24X8H6xQlVp+aJMFlgtQjR9CE= -k8s.io/api v0.32.0/go.mod h1:4LEwHZEf6Q/cG96F3dqR965sYOfmPM7rq81BLgsE0p0= -k8s.io/apiextensions-apiserver v0.32.0 h1:S0Xlqt51qzzqjKPxfgX1xh4HBZE+p8KKBq+k2SWNOE0= -k8s.io/apiextensions-apiserver v0.32.0/go.mod h1:86hblMvN5yxMvZrZFX2OhIHAuFIMJIZ19bTvzkP+Fmw= -k8s.io/apimachinery v0.32.0 h1:cFSE7N3rmEEtv4ei5X6DaJPHHX0C+upp+v5lVPiEwpg= -k8s.io/apimachinery v0.32.0/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= -k8s.io/apiserver v0.32.0 h1:VJ89ZvQZ8p1sLeiWdRJpRD6oLozNZD2+qVSLi+ft5Qs= -k8s.io/apiserver v0.32.0/go.mod h1:HFh+dM1/BE/Hm4bS4nTXHVfN6Z6tFIZPi649n83b4Ag= +k8s.io/api v0.32.1 h1:f562zw9cy+GvXzXf0CKlVQ7yHJVYzLfL6JAS4kOAaOc= +k8s.io/api v0.32.1/go.mod h1:/Yi/BqkuueW1BgpoePYBRdDYfjPF5sgTr5+YqDZra5k= +k8s.io/apiextensions-apiserver v0.32.1 h1:hjkALhRUeCariC8DiVmb5jj0VjIc1N0DREP32+6UXZw= +k8s.io/apiextensions-apiserver v0.32.1/go.mod h1:sxWIGuGiYov7Io1fAS2X06NjMIk5CbRHc2StSmbaQto= +k8s.io/apimachinery v0.32.1 h1:683ENpaCBjma4CYqsmZyhEzrGz6cjn1MY/X2jB2hkZs= +k8s.io/apimachinery v0.32.1/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/apiserver v0.32.1 h1:oo0OozRos66WFq87Zc5tclUX2r0mymoVHRq8JmR7Aak= +k8s.io/apiserver v0.32.1/go.mod h1:UcB9tWjBY7aryeI5zAgzVJB/6k7E97bkr1RgqDz0jPw= k8s.io/cli-runtime v0.32.0 h1:dP+OZqs7zHPpGQMCGAhectbHU2SNCuZtIimRKTv2T1c= k8s.io/cli-runtime v0.32.0/go.mod h1:Mai8ht2+esoDRK5hr861KRy6z0zHsSTYttNVJXgP3YQ= -k8s.io/client-go v0.32.0 h1:DimtMcnN/JIKZcrSrstiwvvZvLjG0aSxy8PxN8IChp8= -k8s.io/client-go v0.32.0/go.mod h1:boDWvdM1Drk4NJj/VddSLnx59X3OPgwrOo0vGbtq9+8= -k8s.io/component-base v0.32.0 h1:d6cWHZkCiiep41ObYQS6IcgzOUQUNpywm39KVYaUqzU= -k8s.io/component-base v0.32.0/go.mod h1:JLG2W5TUxUu5uDyKiH2R/7NnxJo1HlPoRIIbVLkK5eM= +k8s.io/client-go v0.32.1 h1:otM0AxdhdBIaQh7l1Q0jQpmo7WOFIk5FFa4bg6YMdUU= +k8s.io/client-go v0.32.1/go.mod h1:aTTKZY7MdxUaJ/KiUs8D+GssR9zJZi77ZqtzcGXIiDg= +k8s.io/component-base v0.32.1 h1:/5IfJ0dHIKBWysGV0yKTFfacZ5yNV1sulPh3ilJjRZk= +k8s.io/component-base v0.32.1/go.mod h1:j1iMMHi/sqAHeG5z+O9BFNCF698a1u0186zkjMZQ28w= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 h1:hcha5B1kVACrLujCKLbr8XWMxCxzQx42DY8QKYJrDLg= @@ -703,8 +705,8 @@ k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJ k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= oras.land/oras-go v1.2.6 h1:z8cmxQXBU8yZ4mkytWqXfo6tZcamPwjsuxYU81xJ8Lk= oras.land/oras-go v1.2.6/go.mod h1:OVPc1PegSEe/K8YiLfosrlqlqTN9PUyFvOw5Y9gwrT8= -sigs.k8s.io/controller-runtime v0.19.3 h1:XO2GvC9OPftRst6xWCpTgBZO04S2cbp0Qqkj8bX1sPw= -sigs.k8s.io/controller-runtime v0.19.3/go.mod h1:j4j87DqtsThvwTv5/Tc5NFRyyF/RF0ip4+62tbTSIUM= +sigs.k8s.io/controller-runtime v0.20.2 h1:/439OZVxoEc02psi1h4QO3bHzTgu49bb347Xp4gW1pc= +sigs.k8s.io/controller-runtime v0.20.2/go.mod h1:xg2XB0K5ShQzAgsoujxuKN4LNXR2LfwwHsPj7Iaw+XY= sigs.k8s.io/gateway-api v1.2.1 h1:fZZ/+RyRb+Y5tGkwxFKuYuSRQHu9dZtbjenblleOLHM= sigs.k8s.io/gateway-api v1.2.1/go.mod h1:EpNfEXNjiYfUJypf0eZ0P5iXA9ekSGWaS1WgPaM42X0= sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= diff --git a/operators/app-n-lambda/internal/controllers/app/controller.go b/operators/app-n-lambda/internal/controllers/app/controller.go index ded3f0fe..96d56c8c 100644 --- a/operators/app-n-lambda/internal/controllers/app/controller.go +++ b/operators/app-n-lambda/internal/controllers/app/controller.go @@ -60,7 +60,7 @@ const ( AppInterceptCreated string = "app-intercept-created" - AppRouterReady string = "app-router-ready" + createAppRouter string = "create-app-router" ) var DeleteChecklist = []reconciler.CheckMeta{ @@ -101,7 +101,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. {Name: DeploymentReady, Title: "Deployment Ready", Hide: req.Object.IsInterceptEnabled()}, {Name: HPAConfigured, Title: "Horizontal pod autoscaling configured", Hide: req.Object.IsInterceptEnabled() || !req.Object.IsHPAEnabled()}, {Name: AppInterceptCreated, Title: "App Intercept Created", Hide: !req.Object.IsInterceptEnabled()}, - {Name: AppRouterReady, Title: "App Router Ready", Hide: req.Object.Spec.Router == nil}, + {Name: createAppRouter, Title: "Create App Router", Hide: req.Object.Spec.Router == nil}, }); !step.ShouldProceed() { return step.ReconcilerResponse() } @@ -138,7 +138,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. return step.ReconcilerResponse() } - if step := r.checkAppRouter(req); !step.ShouldProceed() { + if step := r.handleAppRouter(req); !step.ShouldProceed() { return step.ReconcilerResponse() } @@ -450,30 +450,21 @@ func (r *Reconciler) checkDeploymentReady(req *reconciler.Request[*crdsv1.App]) return check.Completed() } -func (r *Reconciler) checkAppRouter(req *reconciler.Request[*crdsv1.App]) stepResult.Result { +func (r *Reconciler) handleAppRouter(req *reconciler.Request[*crdsv1.App]) stepResult.Result { ctx, obj := req.Context(), req.Object - check := reconciler.NewRunningCheck(AppRouterReady, req) + check := reconciler.NewRunningCheck(createAppRouter, req) if obj.Spec.Router == nil { return check.Completed() } if len(obj.Spec.Router.Routes) == 0 { - if len(obj.Spec.Services) != 0 { - return check.Failed(fmt.Errorf("app has multiple exposed services, cannot deduce router routes automatically from services, router routes must be explicity set via .spec.router.routes")) - } - - obj.Spec.Router.Routes = append(obj.Spec.Router.Routes, crdsv1.Route{ - App: obj.Name, - Path: "/", - Port: obj.Spec.Services[0].Port, - }) - - if err := r.Update(ctx, obj); err != nil { - return check.Failed(err) - } + return check.Failed(fmt.Errorf("must specify at least 1 route")) + } - return check.StillRunning(fmt.Errorf("updating app router default routes")) + for i := range obj.Spec.Router.Routes { + // INFO: app router will only route to current app, for any such usecases Router kind must be used + obj.Spec.Router.Routes[i].Service = obj.Name } router := &crdsv1.Router{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-app-router", obj.Name), Namespace: obj.Namespace}} diff --git a/operators/routers/internal/env/env.go b/operators/routers/internal/env/env.go index cb6a7342..373fe063 100644 --- a/operators/routers/internal/env/env.go +++ b/operators/routers/internal/env/env.go @@ -6,11 +6,6 @@ import ( type Env struct { MaxConcurrentReconciles int `env:"MAX_CONCURRENT_RECONCILES" default:"5"` - - DefaultIngressClass string `env:"DEFAULT_INGRESS_CLASS" required:"true"` - DefaultClusterIssuer string `env:"DEFAULT_CLUSTER_ISSUER" required:"true"` - - CertificateNamespace string `env:"CERTIFICATE_NAMESPACE" required:"true"` } func GetEnvOrDie() *Env { diff --git a/operators/routers/internal/router-controller/controller.go b/operators/routers/internal/router-controller/controller.go index d0ff815e..9d9322c2 100644 --- a/operators/routers/internal/router-controller/controller.go +++ b/operators/routers/internal/router-controller/controller.go @@ -3,14 +3,12 @@ package router_controller import ( "context" "fmt" - "time" + "strings" certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" - certmanagermetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" "golang.org/x/crypto/bcrypt" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" - apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" @@ -54,7 +52,7 @@ const ( EnsuringHttpsCertsIfEnabled string = "ensuring-https-certs-if-enabled" SettingUpBasicAuthIfEnabled string = "setting-up-basic-auth-if-enabled" - CreatingIngressResources string = "creating-ingress-resources" + CreateIngressResource string = "create-ingress-resource" CleaningUpResources string = "cleaning-up-resourcess" @@ -90,9 +88,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. } if step := req.EnsureCheckList([]reconciler.CheckMeta{ - {Name: DefaultsPatched, Title: "Defaults Patched"}, {Name: EnsuringHttpsCertsIfEnabled, Title: "Ensuring HTTPS Cert if enabled"}, {Name: SettingUpBasicAuthIfEnabled, Title: "Setting Up Basic Auth if enabled"}, + {Name: CreateIngressResource, Title: "Creates kubernetes ingress resource"}, }); !step.ShouldProceed() { return step.ReconcilerResponse() } @@ -105,13 +103,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. return step.ReconcilerResponse() } - if step := r.patchDefaults(req); !step.ShouldProceed() { - return step.ReconcilerResponse() - } - - if step := r.EnsuringHttpsCerts(req); !step.ShouldProceed() { - return step.ReconcilerResponse() - } + // if step := r.EnsuringHttpsCerts(req); !step.ShouldProceed() { + // return step.ReconcilerResponse() + // } if step := r.reconBasicAuth(req); !step.ShouldProceed() { return step.ReconcilerResponse() @@ -125,206 +119,198 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. return ctrl.Result{}, nil } -func (r *Reconciler) patchDefaults(req *reconciler.Request[*crdsv1.Router]) stepResult.Result { +func (r *Reconciler) finalize(req *reconciler.Request[*crdsv1.Router]) stepResult.Result { + check := reconciler.NewRunningCheck("finalizing", req) + if step := req.CleanupOwnedResources(check); !step.ShouldProceed() { + return step + } + + return req.Finalize() +} + +func (r *Reconciler) findClusterIssuer(req *reconciler.Request[*crdsv1.Router]) (*certmanagerv1.ClusterIssuer, error) { ctx, obj := req.Context(), req.Object - check := reconciler.NewRunningCheck(DefaultsPatched, req) + https := obj.Spec.Https - hasUpdate := false + if https != nil && https.ClusterIssuer != "" { + var issuer certmanagerv1.ClusterIssuer + if err := r.Get(ctx, fn.NN("", https.ClusterIssuer), &issuer, &client.GetOptions{}); err != nil { + return nil, err + } - if obj.Spec.BasicAuth != nil && obj.Spec.BasicAuth.Enabled && obj.Spec.BasicAuth.SecretName == "" { - hasUpdate = true - obj.Spec.BasicAuth.SecretName = obj.Name + "-basic-auth" + return &issuer, nil } - if obj.Spec.IngressClass == "" { - hasUpdate = true - obj.Spec.IngressClass = r.Env.DefaultIngressClass + var issuerList certmanagerv1.ClusterIssuerList + if err := r.List(ctx, &issuerList, &client.ListOptions{Limit: 1}); err != nil { + return nil, nil } - if hasUpdate { - if err := r.Update(ctx, obj); err != nil { - return check.Failed(err) - } - return req.Done().RequeueAfter(1 * time.Second) + if len(issuerList.Items) != 1 { + return nil, fmt.Errorf("no cluster issuer found") } - return check.Completed() + return &issuerList.Items[0], nil } -func (r *Reconciler) finalize(req *reconciler.Request[*crdsv1.Router]) stepResult.Result { - check := reconciler.NewRunningCheck("finalizing", req) - if step := req.CleanupOwnedResources(check); !step.ShouldProceed() { - return step - } - - return req.Finalize() -} +func (r *Reconciler) findIngressClass(req *reconciler.Request[*crdsv1.Router]) (string, error) { + ctx, obj := req.Context(), req.Object -func genTLSCertName(domain string) string { - return fmt.Sprintf("%s-tls", domain) -} + if obj.Spec.IngressClass != "" { + return obj.Spec.IngressClass, nil + } -func (r *Reconciler) getRouterClusterIssuer(obj *crdsv1.Router) string { - https := obj.Spec.Https + var ingressClassList networkingv1.IngressClassList + if err := r.List(ctx, &ingressClassList, &client.ListOptions{Limit: 1}); err != nil { + return "", err + } - if https != nil && https.ClusterIssuer != "" { - return https.ClusterIssuer + if len(ingressClassList.Items) != 1 { + return "", fmt.Errorf("no/multiple ingress classes found") } - return r.Env.DefaultClusterIssuer + return ingressClassList.Items[0].Name, nil } func isHttpsEnabled(obj *crdsv1.Router) bool { return obj.Spec.Https != nil && obj.Spec.Https.Enabled } -func (r *Reconciler) EnsuringHttpsCerts(req *reconciler.Request[*crdsv1.Router]) stepResult.Result { +func (r *Reconciler) reconBasicAuth(req *reconciler.Request[*crdsv1.Router]) stepResult.Result { ctx, obj := req.Context(), req.Object - check := reconciler.NewRunningCheck(EnsuringHttpsCertsIfEnabled, req) + check := reconciler.NewRunningCheck(SettingUpBasicAuthIfEnabled, req) - if !isHttpsEnabled(obj) { + if obj.Spec.BasicAuth == nil || !obj.Spec.BasicAuth.Enabled { return check.Completed() } - _, nonWildcardDomains, err := r.parseAndExtractDomains(req) - if err != nil { - return check.Failed(err) + if len(obj.Spec.BasicAuth.Username) == 0 { + return check.Failed(fmt.Errorf(".spec.basicAuth.username must be defined when .spec.basicAuth.enabled is set to true")).Err(nil) } - for _, domain := range nonWildcardDomains { + if obj.Spec.BasicAuth.SecretName == "" { + obj.Spec.BasicAuth.SecretName = obj.Name + "-basic-auth" + if err := r.Update(ctx, obj); err != nil { + return check.Failed(err) + } + return check.StillRunning(fmt.Errorf("waiting for router reconcilation")) + } - tlsCertLabel := fmt.Sprintf("kloudlite.io/tls-cert.%s", fn.Md5([]byte(genTLSCertName(domain)))) - if v, ok := obj.Labels[tlsCertLabel]; !ok || v != "true" { - fn.MapSet(&obj.Labels, tlsCertLabel, "true") - if err := r.Update(ctx, obj); err != nil { - return check.StillRunning(err) - } + basicAuthScrt := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: obj.Spec.BasicAuth.SecretName, Namespace: obj.Namespace}, Type: "Opaque"} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, basicAuthScrt, func() error { + basicAuthScrt.SetOwnerReferences([]metav1.OwnerReference{fn.AsOwner(obj, true)}) + if _, ok := basicAuthScrt.Data["password"]; ok { + return nil } - cert, err := reconciler.Get(ctx, r.Client, fn.NN(r.Env.CertificateNamespace, genTLSCertName(domain)), &certmanagerv1.Certificate{}) + password := fn.CleanerNanoid(48) + ePass, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) if err != nil { - if !apiErrors.IsNotFound(err) { - return check.StillRunning(err) - } - cert = nil + return err } + basicAuthScrt.StringData = map[string]string{ + "auth": fmt.Sprintf("%s:%s", obj.Spec.BasicAuth.Username, ePass), + "username": obj.Spec.BasicAuth.Username, + "password": password, + } + return nil + }); err != nil { + return check.StillRunning(err) + } - if cert == nil { - cert := &certmanagerv1.Certificate{ - TypeMeta: metav1.TypeMeta{ - Kind: "Certificate", - APIVersion: certmanagerv1.SchemeGroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: genTLSCertName(domain), - Namespace: r.Env.CertificateNamespace, - Labels: map[string]string{ - certCreatedByRouter: obj.Name, - }, - }, - Spec: certmanagerv1.CertificateSpec{ - DNSNames: []string{domain}, - IssuerRef: certmanagermetav1.ObjectReference{ - Name: r.getRouterClusterIssuer(obj), - Kind: "ClusterIssuer", - Group: certmanagerv1.SchemeGroupVersion.Group, - }, - RenewBefore: &metav1.Duration{ - Duration: 15 * 24 * time.Hour, // 15 days prior - }, - SecretName: genTLSCertName(domain), - Usages: []certmanagerv1.KeyUsage{ - certmanagerv1.UsageDigitalSignature, - certmanagerv1.UsageKeyEncipherment, - }, - }, - } - if err := r.Create(ctx, cert); err != nil { - return check.StillRunning(err) + req.AddToOwnedResources(reconciler.ParseResourceRef(basicAuthScrt)) + + return check.Completed() +} + +func groupHostsByKind(issuer *certmanagerv1.ClusterIssuer, obj *crdsv1.Router) (wildcardHosts []string, nonWildcardHosts []string) { + var dnsNames []string + + for _, solver := range issuer.Spec.ACME.Solvers { + if solver.DNS01 != nil { + if solver.Selector != nil { + dnsNames = append(dnsNames, solver.Selector.DNSNames...) } } + } - if _, err := IsHttpsCertReady(cert); err != nil { - return check.StillRunning(err).NoRequeue() - // return check.StillRunning(err) - // return check.StillRunning(err).RequeueAfter(1 * time.Second) + wcFilter := map[string]struct{}{} + for _, pattern := range dnsNames { + if strings.HasPrefix(pattern, "*.") { + wcFilter[pattern[len("*."):]] = struct{}{} + continue } + wcFilter[pattern] = struct{}{} + } - certSecret, err := reconciler.Get(ctx, r.Client, fn.NN(r.Env.CertificateNamespace, genTLSCertName(domain)), &corev1.Secret{}) - if err != nil { - return check.StillRunning(err) + for _, route := range obj.Spec.Routes { + if _, ok := wcFilter[route.Host]; ok { + wildcardHosts = append(wildcardHosts, route.Host) + continue } - copyTLSSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: genTLSCertName(domain), Namespace: obj.Namespace}, Type: corev1.SecretTypeTLS} - if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, copyTLSSecret, func() error { - if copyTLSSecret.Annotations == nil { - copyTLSSecret.Annotations = make(map[string]string, 1) - } - copyTLSSecret.Annotations["kloudlite.io/secret.cloned-by"] = "router" + idx := strings.IndexByte(route.Host, '.') + if idx == -1 { + nonWildcardHosts = append(nonWildcardHosts, route.Host) + continue + } - copyTLSSecret.Data = certSecret.Data - copyTLSSecret.StringData = certSecret.StringData - return nil - }); err != nil { - return check.StillRunning(err) + if _, ok := wcFilter[route.Host[idx+1:]]; ok { + wildcardHosts = append(wildcardHosts, route.Host) + continue } + + nonWildcardHosts = append(nonWildcardHosts, route.Host) } - return check.Completed() + return wildcardHosts, nonWildcardHosts } -func (r *Reconciler) reconBasicAuth(req *reconciler.Request[*crdsv1.Router]) stepResult.Result { +func (r *Reconciler) ensureIngresses(req *reconciler.Request[*crdsv1.Router]) stepResult.Result { ctx, obj := req.Context(), req.Object - check := reconciler.NewRunningCheck(SettingUpBasicAuthIfEnabled, req) + check := reconciler.NewRunningCheck(CreateIngressResource, req) - if obj.Spec.BasicAuth != nil && obj.Spec.BasicAuth.Enabled { - if len(obj.Spec.BasicAuth.Username) == 0 { - return check.Failed(fmt.Errorf(".spec.basicAuth.username must be defined when .spec.basicAuth.enabled is set to true")).Err(nil) + if obj.Spec.IngressClass == "" { + ingClass, err := r.findIngressClass(req) + if err != nil { + return check.Failed(err) } - basicAuthScrt := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: obj.Spec.BasicAuth.SecretName, Namespace: obj.Namespace}, Type: "Opaque"} - if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, basicAuthScrt, func() error { - basicAuthScrt.SetOwnerReferences([]metav1.OwnerReference{fn.AsOwner(obj, true)}) - if _, ok := basicAuthScrt.Data["password"]; ok { - return nil - } - - password := fn.CleanerNanoid(48) - ePass, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) - if err != nil { - return err - } - basicAuthScrt.Data = map[string][]byte{ - "auth": []byte(fmt.Sprintf("%s:%s", obj.Spec.BasicAuth.Username, ePass)), - "username": []byte(obj.Spec.BasicAuth.Username), - "password": []byte(password), - } - return nil - }); err != nil { - return check.StillRunning(err) + obj.Spec.IngressClass = ingClass + if err := r.Update(ctx, obj); err != nil { + return check.Failed(err) } - req.AddToOwnedResources(reconciler.ParseResourceRef(basicAuthScrt)) + return check.StillRunning(fmt.Errorf("updating .spec.ingressClass")) } - return check.Completed() -} + if obj.Spec.Https != nil && obj.Spec.Https.ClusterIssuer == "" { + issuer, err := r.findClusterIssuer(req) + if err != nil { + return check.Failed(err) + } -func (r *Reconciler) ensureIngresses(req *reconciler.Request[*crdsv1.Router]) stepResult.Result { - ctx, obj := req.Context(), req.Object - check := reconciler.NewRunningCheck(CreatingIngressResources, req) + obj.Spec.Https.ClusterIssuer = issuer.Name + if err := r.Update(ctx, obj); err != nil { + return check.Failed(err) + } - wcDomains, nonWcDomains, err := r.parseAndExtractDomains(req) - if err != nil { - return check.Failed(err).Err(nil) + return check.StillRunning(fmt.Errorf("updating .spec.https.clusterIssuer")) } - nginxIngressAnnotations := GenNginxIngressAnnotations(obj) - if len(obj.Spec.Routes) == 0 { return check.Completed() } + issuer, err := r.findClusterIssuer(req) + if err != nil { + return check.Failed(err) + } + + wcHosts, nonWcHosts := groupHostsByKind(issuer, obj) + + nginxIngressAnnotations := GenNginxIngressAnnotations(obj) + // b, err := templates.ParseBytes(r.templateIngress, templates.IngressTemplateArgs{ // Metadata: metav1.ObjectMeta{ // Name: obj.Name, @@ -348,17 +334,9 @@ func (r *Reconciler) ensureIngresses(req *reconciler.Request[*crdsv1.Router]) st "labels": obj.GetLabels(), "annotations": nginxIngressAnnotations, - "non-wildcard-domains": nonWcDomains, - "wildcard-domains": wcDomains, - "router-domains": obj.Spec.Routes, - - "ingress-class": obj.Spec.IngressClass, - "cluster-issuer": func() string { - if obj.Spec.Https != nil && obj.Spec.Https.ClusterIssuer != "" { - return obj.Spec.Https.ClusterIssuer - } - return r.Env.DefaultClusterIssuer - }(), + "non-wildcard-domains": nonWcHosts, + "wildcard-domains": wcHosts, + "ingress-class": obj.Spec.IngressClass, "routes": obj.Spec.Routes, @@ -384,7 +362,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { r.Scheme = mgr.GetScheme() if r.YAMLClient == nil { - return fmt.Errorf("r.YAMLClient must be set") + return fmt.Errorf(".YAMLClient must be set") } var err error @@ -413,6 +391,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { return rr })) + // builder.Owns(&certmanagerv1.Certificate{}) builder.WithEventFilter(reconciler.ReconcileFilter()) diff --git a/operators/routers/internal/router-controller/controller_test.go b/operators/routers/internal/router-controller/controller_test.go index c89ac7dd..604386d3 100644 --- a/operators/routers/internal/router-controller/controller_test.go +++ b/operators/routers/internal/router-controller/controller_test.go @@ -32,7 +32,7 @@ func newRouter() crdsv1.Router { Routes: []string{"sample.example.com"}, Routes: []crdsv1.Route{ { - App: "example", + Service: "example", Path: "/", Port: 80, Rewrite: false, @@ -156,7 +156,7 @@ var _ = Describe("router controller [UPDATE] says", func() { It("adding a new route, reflects in each of the owned k8s ingresses", func() { _, err := controllerutil.CreateOrUpdate(Suite.Context, Suite.K8sClient, &routerCr, func() error { routerCr.Spec.Routes = append(routerCr.Spec.Routes, crdsv1.Route{ - App: "ginkgo-test", + Service: "ginkgo-test", Path: "/.kl/test", Port: 80, }) diff --git a/operators/routers/internal/router-controller/helpers.go b/operators/routers/internal/router-controller/helpers.go index f84c43fc..1c098741 100644 --- a/operators/routers/internal/router-controller/helpers.go +++ b/operators/routers/internal/router-controller/helpers.go @@ -6,46 +6,8 @@ import ( certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" certmanagermetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" - crdsv1 "github.com/kloudlite/operator/apis/crds/v1" - fn "github.com/kloudlite/operator/pkg/functions" - "github.com/kloudlite/operator/toolkit/reconciler" - apiErrors "k8s.io/apimachinery/pkg/api/errors" ) -func (r *Reconciler) parseAndExtractDomains(req *reconciler.Request[*crdsv1.Router]) ([]string, []string, error) { - ctx, obj := req.Context(), req.Object - - var wildcardPatterns []string - - if obj.Spec.Https != nil && obj.Spec.Https.Enabled { - issuerName := r.getRouterClusterIssuer(obj) - - if issuerName == "" { - return nil, nil, fmt.Errorf("no cluster issuer found, could not proceed, when https is enabled") - } - - clusterIssuer, err := reconciler.Get(ctx, r.Client, fn.NN("", issuerName), &certmanagerv1.ClusterIssuer{}) - if err != nil { - if !apiErrors.IsNotFound(err) { - return nil, nil, err - } - clusterIssuer = nil - } - - if clusterIssuer != nil { - for _, solver := range clusterIssuer.Spec.ACME.Solvers { - if solver.DNS01 != nil { - wildcardPatterns = solver.Selector.DNSNames - } - } - } - } - - // wildcardDomains, nonWildcardDomains := FilterDomains(wildcardPatterns, fn.MapKeys(obj.Spec.Routes)) - wildcardDomains, nonWildcardDomains := FilterDomains(wildcardPatterns, obj.Spec.Domains) - return wildcardDomains, nonWildcardDomains, nil -} - func FilterDomains(wildcardPatterns []string, domains []string) (wildcardDomains, nonWildcardDomains []string) { wildcardBases := map[string]struct{}{} for _, pattern := range wildcardPatterns { diff --git a/operators/routers/internal/templates/embed.go b/operators/routers/internal/templates/embed.go index 942d0b82..dd421b45 100644 --- a/operators/routers/internal/templates/embed.go +++ b/operators/routers/internal/templates/embed.go @@ -13,7 +13,7 @@ type templateFile string const ( // IngressTemplate templateFile = "./ingress-resource-v2.yml.tpl" - IngressTemplate templateFile = "./ingress-resource.yml.tpl" + IngressTemplate templateFile = "ingress-resource.yml.tpl" ) func Read(t templateFile) ([]byte, error) { diff --git a/operators/routers/internal/templates/ingress-resource-v2.yml.tpl b/operators/routers/internal/templates/ingress-resource-v2.yml.tpl index cd4dc57f..da6ea199 100644 --- a/operators/routers/internal/templates/ingress-resource-v2.yml.tpl +++ b/operators/routers/internal/templates/ingress-resource-v2.yml.tpl @@ -15,7 +15,7 @@ spec: {{- range $v := .WildcardDomains }} - hosts: - {{$v | squote}} - {{- end}} + {{- end }} {{- end}} rules: @@ -31,6 +31,9 @@ spec: port: number: {{$route.Port}} + path: {{ if not hasPrefix "/" $route.Path }}/{{end}}{{$route.Path}} + ({{if hasPrefix "/" $route.Path }}{{substr 1 $x $route.Path}}{{else}}{{$route.Path}}{{end}}.*) + {{- if $route.Rewrite }} path: {{$route.Path}}?(.*) {{- else }} diff --git a/operators/routers/internal/templates/ingress-resource.yml.tpl b/operators/routers/internal/templates/ingress-resource.yml.tpl index 30e4c3ae..e7099624 100644 --- a/operators/routers/internal/templates/ingress-resource.yml.tpl +++ b/operators/routers/internal/templates/ingress-resource.yml.tpl @@ -6,11 +6,9 @@ {{- $annotations := get . "annotations" | default dict }} {{- $nonWildcardDomains := get . "non-wildcard-domains" }} -{{- $routerDomains := get . "router-domains" }} {{- $wildcardDomains := get . "wildcard-domains"}} {{- $ingressClass := get . "ingress-class" }} -{{- $clusterIssuer := get . "cluster-issuer" }} {{- $routes := get . "routes" }} @@ -42,24 +40,22 @@ spec: {{- end}} rules: - {{- range $domain := $routerDomains }} - - host: {{$domain}} + {{- range $_, $route := $routes }} + - host: {{$route.Host}} http: paths: - {{- range $route := $routes }} - - pathType: Prefix + {{- /* - pathType: Prefix */}} + - pathType: ImplementationSpecific backend: service: - name: {{$route.App}} + name: {{$route.Service}} port: number: {{$route.Port}} {{- if $route.Rewrite }} path: {{$route.Path}}?(.*) {{- else }} - {{ $x := len $route.Path }} - path: /({{if hasPrefix "/" $route.Path }}{{substr 1 $x $route.Path}}{{else}}{{$route.Path}}{{end}}.*) + path: /({{substr 1 (len $route.Path) $route.Path}}.*) {{- end}} - {{- end}} {{- end }} From 5f31210de7aaa9ce710ba296c24fd531d2ee8a48 Mon Sep 17 00:00:00 2001 From: nxtcoder17 Date: Wed, 4 Jun 2025 23:52:02 +0530 Subject: [PATCH 27/27] fix: resolves PR review comments, and refactorings --- .../internal/controllers/workspace/controller.go | 16 +++++++--------- operators/workspace/internal/env/env.go | 2 +- toolkit/job-helper/job-runner.go | 7 ++++--- toolkit/reconciler/event-predicate.go | 14 +++++++------- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/operators/workspace/internal/controllers/workspace/controller.go b/operators/workspace/internal/controllers/workspace/controller.go index 710b8e13..b82ac2ec 100644 --- a/operators/workspace/internal/controllers/workspace/controller.go +++ b/operators/workspace/internal/controllers/workspace/controller.go @@ -23,14 +23,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" ) -var ( - PortConfig = templates.PortConfig{ - SSHPort: 22, - TTYDPort: 56789, - NotebookPort: 56790, - CodeServerPort: 56791, - } -) +var PortConfig = templates.PortConfig{ + SSHPort: 22, + TTYDPort: 56789, + NotebookPort: 56790, + CodeServerPort: 56791, +} const ( IngressClassName = "nginx" @@ -159,7 +157,7 @@ func (r *Reconciler) createDeployment(req *rApi.Request[*crdsv1.Workspace]) step ImageCodeServer: r.Env.WorkspaceImageCodeServer, EnableVSCodeServer: obj.Spec.EnableVSCodeServer, - ImageVscodeServer: r.Env.WorkspcaeImageVscodeServer, + ImageVscodeServer: r.Env.WorkspaceImageVscodeServer, PortConfig: PortConfig, ImagePullPolicy: obj.Spec.ImagePullPolicy, diff --git a/operators/workspace/internal/env/env.go b/operators/workspace/internal/env/env.go index ac4245c7..579086e4 100644 --- a/operators/workspace/internal/env/env.go +++ b/operators/workspace/internal/env/env.go @@ -12,7 +12,7 @@ type Env struct { WorkspaceImageTTYD string `env:"WORKSPACE_IMAGE_TTYD" default:"ghcr.io/kloudlite/iac/ttyd:latest"` WorkspaceImageJupyterNotebook string `env:"WORKSPACE_IMAGE_JUPYTER_NOTEBOOK" default:"ghcr.io/kloudlite/iac/jupyter:latest"` WorkspaceImageCodeServer string `env:"WORKSPACE_IMAGE_CODE_SERVER" default:"ghcr.io/kloudlite/iac/code-server:latest"` - WorkspcaeImageVscodeServer string `env:"WORKSPCAE_IMAGE_VSCODE_SERVER" default:"ghcr.io/kloudlite/iac/vscode-server:latest"` + WorkspaceImageVscodeServer string `env:"WORKSPACE_IMAGE_VSCODE_SERVER" default:"ghcr.io/kloudlite/iac/vscode-server:latest"` } func GetEnvOrDie() *Env { diff --git a/toolkit/job-helper/job-runner.go b/toolkit/job-helper/job-runner.go index 89f3ec29..f25afdc9 100644 --- a/toolkit/job-helper/job-runner.go +++ b/toolkit/job-helper/job-runner.go @@ -5,6 +5,7 @@ import ( "errors" batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -43,15 +44,15 @@ func NewJobTracker(ctx context.Context, kcli client.Client, args JobTrackerArgs) func (jr *JobTracker) HasJobFinished() bool { for _, v := range jr.job.Status.Conditions { - if v.Type == batchv1.JobComplete && v.Status == "True" { + if v.Type == batchv1.JobComplete && v.Status == corev1.ConditionTrue { return true } - if v.Type == batchv1.JobFailed && v.Status == "True" { + if v.Type == batchv1.JobFailed && v.Status == corev1.ConditionTrue { return true } - if v.Type == batchv1.JobSuspended && v.Status == "True" { + if v.Type == batchv1.JobSuspended && v.Status == corev1.ConditionTrue { return true } } diff --git a/toolkit/reconciler/event-predicate.go b/toolkit/reconciler/event-predicate.go index 00ac7bfe..3a2faa9e 100644 --- a/toolkit/reconciler/event-predicate.go +++ b/toolkit/reconciler/event-predicate.go @@ -76,7 +76,7 @@ func ReconcileFilter(eventRecorder ...record.EventRecorder) predicate.Funcs { } if len(oldObj.GetLabels()) != len(newObj.GetLabels()) || !reflect.DeepEqual(oldObj.GetLabels(), newObj.GetLabels()) { - fireEvent(newObj, ReasonLabelsUpdated, fmt.Sprintf("labels updated from (%+v) to (%+v)", newObj.GetLabels(), oldObj.GetLabels())) + fireEvent(newObj, ReasonLabelsUpdated, fmt.Sprintf("labels updated from (%+v) to (%+v)", oldObj.GetLabels(), newObj.GetLabels())) return true } @@ -94,18 +94,18 @@ func ReconcileFilter(eventRecorder ...record.EventRecorder) predicate.Funcs { } if len(oldAnn) != len(newAnn) || annHasChanged { - fireEvent(newObj, ReasonAnnotationsUpdated, fmt.Sprintf("annotations updated from (%+v) to (%+v)", newObj.GetAnnotations(), oldObj.GetAnnotations())) + fireEvent(newObj, ReasonAnnotationsUpdated, fmt.Sprintf("annotations updated from (%+v) to (%+v)", oldObj.GetAnnotations(), newObj.GetAnnotations())) return true } if len(oldObj.GetFinalizers()) != len(newObj.GetFinalizers()) || !reflect.DeepEqual(oldObj.GetFinalizers(), newObj.GetFinalizers()) { - fireEvent(newObj, ReasonFinalizersUpdated, fmt.Sprintf("finalizers updated from (%+v) to (%+v)", newObj.GetFinalizers(), oldObj.GetFinalizers())) + fireEvent(newObj, ReasonFinalizersUpdated, fmt.Sprintf("finalizers updated from (%+v) to (%+v)", oldObj.GetFinalizers(), newObj.GetFinalizers())) return true } if len(oldObj.GetOwnerReferences()) != len(newObj.GetOwnerReferences()) || !reflect.DeepEqual(oldObj.GetOwnerReferences(), newObj.GetOwnerReferences()) { - fireEvent(newObj, ReasonOwnerReferencesUpdated, fmt.Sprintf("owner-references updated from (%+v) to (%+v)", newObj.GetOwnerReferences(), oldObj.GetOwnerReferences())) + fireEvent(newObj, ReasonOwnerReferencesUpdated, fmt.Sprintf("owner-references updated from (%+v) to (%+v)", oldObj.GetOwnerReferences(), newObj.GetOwnerReferences())) return true } @@ -122,18 +122,18 @@ func ReconcileFilter(eventRecorder ...record.EventRecorder) predicate.Funcs { } if *oldRes.Status.IsReady != *newRes.Status.IsReady { - fireEvent(newObj, ReasonStatusIsReadyChanged, fmt.Sprintf("resource isReady changed from (%v) to (%v)", *newRes.Status.IsReady, *oldRes.Status.IsReady)) + fireEvent(newObj, ReasonStatusIsReadyChanged, fmt.Sprintf("resource isReady changed from (%v) to (%v)", *oldRes.Status.IsReady, *newRes.Status.IsReady)) return true } if len(oldRes.Status.Checks) != len(newRes.Status.Checks) { - fireEvent(newObj, ReasonStatusChecksUpdated, fmt.Sprintf("resource status.checks changed from (%+v) to (%+v)", newRes.Status.Checks, oldRes.Status.Checks)) + fireEvent(newObj, ReasonStatusChecksUpdated, fmt.Sprintf("resource status.checks changed from (%+v) to (%+v)", oldRes.Status.Checks, newRes.Status.Checks)) return true } for k, v := range oldRes.Status.Checks { if !AreChecksEqual(newRes.Status.Checks[k], v) { - fireEvent(newObj, ReasonStatusChecksUpdated, fmt.Sprintf("resource status.checks changed from (%+v) to (%+v)", newRes.Status.Checks, oldRes.Status.Checks)) + fireEvent(newObj, ReasonStatusChecksUpdated, fmt.Sprintf("resource status.checks changed from (%+v) to (%+v)", oldRes.Status.Checks, newRes.Status.Checks)) return true } }