diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..bacc62a4 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,15 @@ +{ + "configurations": [ + { + "name": "Run Agent Operator", + "type": "go", + "request": "launch", + "mode": "debug", + "envFile": "${workspaceFolder}/cmd/agent-operator/.secrets/env", + "program": "${workspaceFolder}/cmd/agent-operator/main.go", + "args": [ + "--dev","--serverHost", "localhost:8080" + ] + } + ] +} \ No newline at end of file diff --git a/PROJECT b/PROJECT index f89a4a64..ab70d857 100644 --- a/PROJECT +++ b/PROJECT @@ -4,7 +4,7 @@ # More info: https://book.kubebuilder.io/reference/project-config.html domain: kloudlite.io layout: -- go.kubebuilder.io/v3 +- go.kubebuilder.io/v4 multigroup: true plugins: manifests.sdk.operatorframework.io/v2: {} @@ -515,8 +515,28 @@ resources: domain: kloudlite.io group: crds kind: ServiceIntercept + version: "" +- api: + crdVersion: v1 + namespaced: true group: mongodb.msvc kind: Backup path: github.com/kloudlite/operator/apis/mongodb.msvc/v1 version: v1 +- api: + crdVersion: v1 + namespaced: true + domain: kloudlite.io + group: crds + kind: Workspace + path: github.com/kloudlite/operator/api/crds/v1 + version: v1 +- api: + crdVersion: v1 + namespaced: true + domain: kloudlite.io + group: crds + kind: WorkMachine + path: github.com/kloudlite/operator/api/crds/v1 + version: v1 version: "3" diff --git a/apis/common-types/types.go b/apis/common-types/types.go index f4f1ac57..c396cc60 100644 --- a/apis/common-types/types.go +++ b/apis/common-types/types.go @@ -117,7 +117,12 @@ type MinMaxInt struct { // +kubebuilder:validation:Enum=aws;do;azure;gcp type CloudProvider string +func (c CloudProvider) String() string { + return string(c) +} + const ( + CloudProviderUnknown CloudProvider = "unknown" CloudProviderAWS CloudProvider = "aws" CloudProviderDigitalOcean CloudProvider = "digitalocean" CloudProviderAzure CloudProvider = "azure" diff --git a/apis/crds/v1/app_webhook.go b/apis/crds/v1/app_webhook.go deleted file mode 100644 index 6b3efdfd..00000000 --- a/apis/crds/v1/app_webhook.go +++ /dev/null @@ -1,60 +0,0 @@ -package v1 - -import ( - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - -// log is for logging in this package. -var applog = logf.Log.WithName("app-resource") - -func (r *App) SetupWebhookWithManager(mgr ctrl.Manager) error { - return ctrl.NewWebhookManagedBy(mgr). - For(r). - Complete() -} - -// TODO(user): EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! - -//+kubebuilder:webhook:path=/mutate-crds-kloudlite-io-v1-app,mutating=true,failurePolicy=fail,sideEffects=None,groups=crds.kloudlite.io,resources=apps,verbs=create;update,versions=v1,name=mapp.kb.io,admissionReviewVersions=v1 - -var _ webhook.Defaulter = &App{} - -// Default implements webhook.Defaulter so a webhook will be registered for the type -func (r *App) Default() { - applog.Info("default", "name", r.Name) - - // TODO(user): fill in your defaulting logic. -} - -// TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation. -//+kubebuilder:webhook:path=/validate-crds-kloudlite-io-v1-app,mutating=false,failurePolicy=fail,sideEffects=None,groups=crds.kloudlite.io,resources=apps,verbs=create;update,versions=v1,name=vapp.kb.io,admissionReviewVersions=v1 - -var _ webhook.Validator = &App{} - -// ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (r *App) ValidateCreate() (admission.Warnings, error) { - applog.Info("validate create", "name", r.Name) - - // TODO(user): fill in your validation logic upon object creation. - return nil, nil -} - -// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (r *App) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { - applog.Info("validate update", "name", r.Name) - - // TODO(user): fill in your validation logic upon object update. - return nil, nil -} - -// ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (r *App) ValidateDelete() (admission.Warnings, error) { - applog.Info("validate delete", "name", r.Name) - - // TODO(user): fill in your validation logic upon object deletion. - return nil, nil -} diff --git a/apis/crds/v1/router_types.go b/apis/crds/v1/router_types.go index 2ae0a4ab..5f45845f 100644 --- a/apis/crds/v1/router_types.go +++ b/apis/crds/v1/router_types.go @@ -1,19 +1,17 @@ package v1 import ( - "strings" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/kloudlite/operator/pkg/constants" "github.com/kloudlite/operator/toolkit/reconciler" ) type Route struct { - App string `json:"app"` - // Lambda string `json:"lambda,omitempty"` - Path string `json:"path"` - Port uint16 `json:"port"` + Host string `json:"host"` + Service string `json:"service"` + Path string `json:"path"` + Port uint16 `json:"port"` + // +kubebuilder:default=false Rewrite bool `json:"rewrite,omitempty"` } @@ -50,14 +48,18 @@ type RouterSpec struct { IngressClass string `json:"ingressClass,omitempty"` BackendProtocol *string `json:"backendProtocol,omitempty"` Https *Https `json:"https,omitempty"` - // +kubebuilder:validation:Optional RateLimit *RateLimit `json:"rateLimit,omitempty"` MaxBodySizeInMB *int `json:"maxBodySizeInMB,omitempty"` - Domains []string `json:"domains"` - Routes []Route `json:"routes,omitempty"` - BasicAuth *BasicAuth `json:"basicAuth,omitempty"` - Cors *Cors `json:"cors,omitempty"` + + BasicAuth *BasicAuth `json:"basicAuth,omitempty"` + Cors *Cors `json:"cors,omitempty"` + + // NginxIngressAnnotations is additional list of annotations on ingress resource + // INFO: must be used when router does not have direct support for it + NginxIngressAnnotations map[string]string `json:"nginxIngressAnnotations,omitempty"` + + Routes []Route `json:"routes,omitempty"` } // +kubebuilder:object:root=true @@ -91,16 +93,11 @@ func (r *Router) GetStatus() *reconciler.Status { } func (r *Router) GetEnsuredLabels() map[string]string { - return map[string]string{ - constants.RouterNameKey: r.Name, - } + return map[string]string{} } func (m *Router) GetEnsuredAnnotations() map[string]string { - return map[string]string{ - "kloudlite.io/router.domains": strings.Join(m.Spec.Domains, ","), - "kloudlite.io/router.ingress-class": m.Spec.IngressClass, - } + return map[string]string{} } // +kubebuilder:object:root=true diff --git a/apis/crds/v1/workmachine_types.go b/apis/crds/v1/workmachine_types.go new file mode 100644 index 00000000..7060e2de --- /dev/null +++ b/apis/crds/v1/workmachine_types.go @@ -0,0 +1,120 @@ +package v1 + +import ( + ct "github.com/kloudlite/operator/apis/common-types" + rApi "github.com/kloudlite/operator/toolkit/reconciler" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type AWSMachineConfig struct { + // Region string `json:"region" graphql:"noinput"` + // AvailabilityZone string `json:"availabilityZone"` + + AMI string `json:"ami"` + InstanceType string `json:"instanceType"` + // PublicSubnetID string `json:"publicSubnetID"` + + //+kubebuilder:default=50 + RootVolumeSize int `json:"rootVolumeSize" graphql:"noinput"` + + //+kubebuilder:default=gp3 + RootVolumeType string `json:"rootVolumeType" graphql:"noinput"` + + ExternalVolumeSize int `json:"externalVolumeSize"` + + //+kubebuilder:default=gp3 + ExternalVolumeType string `json:"externalVolumeType" graphql:"noinput"` + + IAMInstanceProfileRole *string `json:"iamInstanceProfileRole,omitempty" graphql:"noinput"` +} + +// +kubebuilder:validation:Enum=ON;OFF; +// +kubebuilder:default=ON +type WorkMachineState string + +const ( + WorkMachineStateOn WorkMachineState = "ON" + WorkMachineStateOff WorkMachineState = "OFF" +) + +type WorkMachineJobParams struct { + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` +} + +// WorkMachineSpec defines the desired state of WorkMachine +type WorkMachineSpec struct { + State WorkMachineState `json:"state"` + SSHPublicKeys []string `json:"sshPublicKeys"` + + JobParams WorkMachineJobParams `json:"jobParams,omitempty"` + + TargetNamespace string `json:"targetNamespace,omitempty"` + + AWSMachineConfig *AWSMachineConfig `json:"aws"` +} + +func (wms *WorkMachineSpec) GetCloudProvider() ct.CloudProvider { + if wms.AWSMachineConfig != nil { + return ct.CloudProviderAWS + } + + return ct.CloudProviderUnknown +} + +type WorkMachineStatus struct { + rApi.Status `json:"status,omitempty"` + MachinePublicSSHKey string `json:"machineSSHKey,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster +// +kubebuilder:printcolumn:JSONPath=".spec.targetNamespace",name=TargetNamespace,type=string +// +kubebuilder:printcolumn:JSONPath=".status.status.lastReconcileTime",name=Seen,type=date +// +kubebuilder:printcolumn:JSONPath=".metadata.annotations.kloudlite\\.io\\/operator\\.checks",name=Checks,type=string +// +kubebuilder:printcolumn:JSONPath=".metadata.annotations.kloudlite\\.io\\/operator\\.resource\\.ready",name=Ready,type=string +// +kubebuilder:printcolumn:JSONPath=".metadata.creationTimestamp",name=Age,type=date + +// WorkMachine is the Schema for the workmachines API +type WorkMachine struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec WorkMachineSpec `json:"spec,omitempty"` + Status WorkMachineStatus `json:"status,omitempty"` +} + +func (r *WorkMachine) EnsureGVK() { + if r != nil { + r.SetGroupVersionKind(GroupVersion.WithKind("WorkMachine")) + } +} + +func (w *WorkMachine) GetStatus() *rApi.Status { + return &w.Status.Status +} + +func (w *WorkMachine) GetEnsuredLabels() map[string]string { + return map[string]string{ + "kloudlite.io/workmachine.name": w.Name, + } +} + +func (w *WorkMachine) GetEnsuredAnnotations() map[string]string { + return map[string]string{} +} + +// +kubebuilder:object:root=true + +// WorkMachineList contains a list of WorkMachine +type WorkMachineList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []WorkMachine `json:"items"` +} + +func init() { + SchemeBuilder.Register(&WorkMachine{}, &WorkMachineList{}) +} diff --git a/apis/crds/v1/workspace_types.go b/apis/crds/v1/workspace_types.go new file mode 100644 index 00000000..2d46c84d --- /dev/null +++ b/apis/crds/v1/workspace_types.go @@ -0,0 +1,86 @@ +package v1 + +import ( + "github.com/kloudlite/operator/toolkit/reconciler" + rApi "github.com/kloudlite/operator/toolkit/reconciler" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +kubebuilder:validation:Enum=ON;OFF; +// +kubebuilder:default=ON +type WorkspaceState string + +const ( + WorkspaceStateOn WorkspaceState = "ON" + WorkspaceStateOff WorkspaceState = "OFF" +) + +// WorkspaceSpec defines the desired state of Workspace +type WorkspaceSpec struct { + // Name of work machine + WorkMachine string `json:"workMachine" graphql:"noinput"` + + // +kubebuilder:default=ON + State WorkspaceState `json:"state"` + ServiceAccountName string `json:"serviceAccountName,omitempty"` + + EnableTTYD bool `json:"enableTTYD,omitempty"` + EnableJupyterNotebook bool `json:"enableJupyterNotebook,omitempty"` + EnableCodeServer bool `json:"enableCodeServer,omitempty"` + EnableVSCodeServer bool `json:"enableVSCodeServer,omitempty"` + + // +kubebuilder:default=IfNotPresent + ImagePullPolicy string `json:"imagePullPolicy,omitempty"` + + // Router RouterSpec `json:"router"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Namespaced +// +kubebuilder:printcolumn:JSONPath=".status.lastReconcileTime",name=Seen,type=date +// +kubebuilder:printcolumn:JSONPath=".metadata.annotations.kloudlite\\.io\\/operator\\.checks",name=Checks,type=string +// +kubebuilder:printcolumn:JSONPath=".metadata.annotations.kloudlite\\.io\\/operator\\.resource\\.ready",name=Ready,type=string +// +kubebuilder:printcolumn:JSONPath=".metadata.creationTimestamp",name=Age,type=date + +// Workspace is the Schema for the workspaces API +type Workspace struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec WorkspaceSpec `json:"spec,omitempty"` + Status rApi.Status `json:"status,omitempty"` +} + +func (r *Workspace) EnsureGVK() { + if r != nil { + r.SetGroupVersionKind(GroupVersion.WithKind("Workspace")) + } +} + +func (w *Workspace) GetStatus() *reconciler.Status { + return &w.Status +} + +func (w *Workspace) GetEnsuredLabels() map[string]string { + return map[string]string{ + "kloudlite.io/workspace.name": w.Name, + } +} + +func (w *Workspace) GetEnsuredAnnotations() map[string]string { + return map[string]string{} +} + +// +kubebuilder:object:root=true + +// WorkspaceList contains a list of Workspace +type WorkspaceList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Workspace `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Workspace{}, &WorkspaceList{}) +} diff --git a/apis/crds/v1/zz_generated.deepcopy.go b/apis/crds/v1/zz_generated.deepcopy.go index 45ea7b0c..e5496ebb 100644 --- a/apis/crds/v1/zz_generated.deepcopy.go +++ b/apis/crds/v1/zz_generated.deepcopy.go @@ -8,9 +8,29 @@ import ( "github.com/kloudlite/operator/pkg/json-patch" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apimachinery/pkg/runtime" + runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AWSMachineConfig) DeepCopyInto(out *AWSMachineConfig) { + *out = *in + if in.IAMInstanceProfileRole != nil { + in, out := &in.IAMInstanceProfileRole, &out.IAMInstanceProfileRole + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSMachineConfig. +func (in *AWSMachineConfig) DeepCopy() *AWSMachineConfig { + if in == nil { + return nil + } + out := new(AWSMachineConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Account) DeepCopyInto(out *Account) { *out = *in @@ -1768,16 +1788,6 @@ func (in *RouterSpec) DeepCopyInto(out *RouterSpec) { *out = new(int) **out = **in } - if in.Domains != nil { - in, out := &in.Domains, &out.Domains - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Routes != nil { - in, out := &in.Routes, &out.Routes - *out = make([]Route, len(*in)) - copy(*out, *in) - } if in.BasicAuth != nil { in, out := &in.BasicAuth, &out.BasicAuth *out = new(BasicAuth) @@ -1788,6 +1798,18 @@ func (in *RouterSpec) DeepCopyInto(out *RouterSpec) { *out = new(Cors) (*in).DeepCopyInto(*out) } + if in.NginxIngressAnnotations != nil { + in, out := &in.NginxIngressAnnotations, &out.NginxIngressAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Routes != nil { + in, out := &in.Routes, &out.Routes + *out = make([]Route, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouterSpec. @@ -1951,3 +1973,207 @@ func (in *TcpProbe) DeepCopy() *TcpProbe { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkMachine) DeepCopyInto(out *WorkMachine) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkMachine. +func (in *WorkMachine) DeepCopy() *WorkMachine { + if in == nil { + return nil + } + out := new(WorkMachine) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *WorkMachine) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkMachineJobParams) DeepCopyInto(out *WorkMachineJobParams) { + *out = *in + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]corev1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkMachineJobParams. +func (in *WorkMachineJobParams) DeepCopy() *WorkMachineJobParams { + if in == nil { + return nil + } + out := new(WorkMachineJobParams) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkMachineList) DeepCopyInto(out *WorkMachineList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]WorkMachine, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkMachineList. +func (in *WorkMachineList) DeepCopy() *WorkMachineList { + if in == nil { + return nil + } + out := new(WorkMachineList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *WorkMachineList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkMachineSpec) DeepCopyInto(out *WorkMachineSpec) { + *out = *in + if in.SSHPublicKeys != nil { + in, out := &in.SSHPublicKeys, &out.SSHPublicKeys + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.JobParams.DeepCopyInto(&out.JobParams) + if in.AWSMachineConfig != nil { + in, out := &in.AWSMachineConfig, &out.AWSMachineConfig + *out = new(AWSMachineConfig) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkMachineSpec. +func (in *WorkMachineSpec) DeepCopy() *WorkMachineSpec { + if in == nil { + return nil + } + out := new(WorkMachineSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkMachineStatus) DeepCopyInto(out *WorkMachineStatus) { + *out = *in + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkMachineStatus. +func (in *WorkMachineStatus) DeepCopy() *WorkMachineStatus { + if in == nil { + return nil + } + out := new(WorkMachineStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Workspace) DeepCopyInto(out *Workspace) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Workspace. +func (in *Workspace) DeepCopy() *Workspace { + if in == nil { + return nil + } + out := new(Workspace) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Workspace) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkspaceList) DeepCopyInto(out *WorkspaceList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Workspace, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkspaceList. +func (in *WorkspaceList) DeepCopy() *WorkspaceList { + if in == nil { + return nil + } + out := new(WorkspaceList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *WorkspaceList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkspaceSpec) DeepCopyInto(out *WorkspaceSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkspaceSpec. +func (in *WorkspaceSpec) DeepCopy() *WorkspaceSpec { + if in == nil { + return nil + } + out := new(WorkspaceSpec) + in.DeepCopyInto(out) + return out +} diff --git a/cmd/agent-operator/Taskfile.yml b/cmd/agent-operator/Taskfile.yml index 625d271b..4c947108 100644 --- a/cmd/agent-operator/Taskfile.yml +++ b/cmd/agent-operator/Taskfile.yml @@ -13,7 +13,7 @@ tasks: dotenv: - .secrets/env cmds: - - go run . --dev + - go run . --dev --serverHost localhost:8080 build: cmds: diff --git a/cmd/agent-operator/main.go b/cmd/agent-operator/main.go index fea5ad51..530f9c77 100644 --- a/cmd/agent-operator/main.go +++ b/cmd/agent-operator/main.go @@ -12,7 +12,8 @@ import ( project "github.com/kloudlite/operator/operators/project/controller" resourceWatcher "github.com/kloudlite/operator/operators/resource-watcher/controller" - // routers "github.com/kloudlite/operator/operators/routers/controller" + workmachine "github.com/kloudlite/operator/operators/workmachine/register" + workspace "github.com/kloudlite/operator/operators/workspace/register" serviceIntercept "github.com/kloudlite/operator/operators/service-intercept/controller" pluginHelmChart "github.com/kloudlite/plugin-helm-chart/kloudlite" @@ -44,8 +45,9 @@ func main() { // distribution.RegisterInto(mgr) networkingv1.RegisterInto(mgr) - serviceIntercept.RegisterInto(mgr) + workmachine.RegisterInto(mgr) + workspace.RegisterInto(mgr) pluginMongoDB.RegisterInto(mgr) pluginHelmChart.RegisterInto(mgr) diff --git a/cmd/main.go b/cmd/main.go index 1a67a122..a11f2506 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -10,9 +10,10 @@ import ( "strings" "text/template" + "github.com/urfave/cli/v2" + "github.com/kloudlite/operator/pkg/errors" "github.com/kloudlite/operator/pkg/templates" - "github.com/urfave/cli/v2" ) var ( diff --git a/cmd/platform-operator/main.go b/cmd/platform-operator/main.go index 3aa528f4..5fcbd254 100644 --- a/cmd/platform-operator/main.go +++ b/cmd/platform-operator/main.go @@ -22,7 +22,9 @@ import ( // wireguard "github.com/kloudlite/operator/operators/wireguard/controller" serviceIntercept "github.com/kloudlite/operator/operators/service-intercept/controller" - pluginMongoDB "github.com/kloudlite/plugin-mongodb/kloudlite" + workspace "github.com/kloudlite/operator/operators/workspace/register" + pluginK3sCluster "github.com/kloudlite/plugin-k3s-cluster/kloudlite" + // pluginMongoDB "github.com/kloudlite/plugin-mongodb/kloudlite" ) func main() { @@ -51,7 +53,9 @@ func main() { // wireguard.RegisterInto(mgr) // MIGRATE // networkingv0.RegisterInto(mgr) // MIGRATE - pluginMongoDB.RegisterInto(mgr) + // pluginMongoDB.RegisterInto(mgr) + pluginK3sCluster.RegisterInto(mgr) + workspace.RegisterInto(mgr) mgr.Start() } diff --git a/config/crd/bases/crds.kloudlite.io_apps.yaml b/config/crd/bases/crds.kloudlite.io_apps.yaml index e9ea3e0b..9cc4d5c7 100644 --- a/config/crd/bases/crds.kloudlite.io_apps.yaml +++ b/config/crd/bases/crds.kloudlite.io_apps.yaml @@ -343,10 +343,6 @@ spec: type: string type: array type: object - domains: - items: - type: string - type: array https: properties: clusterIssuer: @@ -363,6 +359,13 @@ spec: type: string maxBodySizeInMB: type: integer + nginxIngressAnnotations: + additionalProperties: + type: string + description: |- + NginxIngressAnnotations is additional list of annotations on ingress resource + INFO: must be used when router does not have direct support for it + type: object rateLimit: properties: connections: @@ -377,24 +380,24 @@ spec: routes: items: properties: - app: + host: type: string path: - description: Lambda string `json:"lambda,omitempty"` type: string port: type: integer rewrite: default: false type: boolean + service: + type: string required: - - app + - host - path - port + - service type: object type: array - required: - - domains type: object serviceAccount: type: string diff --git a/config/crd/bases/crds.kloudlite.io_blueprints.yaml b/config/crd/bases/crds.kloudlite.io_blueprints.yaml index 4bf216b0..931df359 100644 --- a/config/crd/bases/crds.kloudlite.io_blueprints.yaml +++ b/config/crd/bases/crds.kloudlite.io_blueprints.yaml @@ -348,10 +348,6 @@ spec: type: string type: array type: object - domains: - items: - type: string - type: array https: properties: clusterIssuer: @@ -368,6 +364,13 @@ spec: type: string maxBodySizeInMB: type: integer + nginxIngressAnnotations: + additionalProperties: + type: string + description: |- + NginxIngressAnnotations is additional list of annotations on ingress resource + INFO: must be used when router does not have direct support for it + type: object rateLimit: properties: connections: @@ -382,24 +385,24 @@ spec: routes: items: properties: - app: + host: type: string path: - description: Lambda string `json:"lambda,omitempty"` type: string port: type: integer rewrite: default: false type: boolean + service: + type: string required: - - app + - host - path - port + - service type: object type: array - required: - - domains type: object serviceAccount: type: string diff --git a/config/crd/bases/crds.kloudlite.io_routers.yaml b/config/crd/bases/crds.kloudlite.io_routers.yaml index 4f86f74d..fe965d38 100644 --- a/config/crd/bases/crds.kloudlite.io_routers.yaml +++ b/config/crd/bases/crds.kloudlite.io_routers.yaml @@ -86,10 +86,6 @@ spec: type: string type: array type: object - domains: - items: - type: string - type: array https: properties: clusterIssuer: @@ -106,6 +102,13 @@ spec: type: string maxBodySizeInMB: type: integer + nginxIngressAnnotations: + additionalProperties: + type: string + description: |- + NginxIngressAnnotations is additional list of annotations on ingress resource + INFO: must be used when router does not have direct support for it + type: object rateLimit: properties: connections: @@ -120,24 +123,24 @@ spec: routes: items: properties: - app: + host: type: string path: - description: Lambda string `json:"lambda,omitempty"` type: string port: type: integer rewrite: default: false type: boolean + service: + type: string required: - - app + - host - path - port + - service type: object type: array - required: - - domains type: object status: properties: diff --git a/config/crd/bases/crds.kloudlite.io_workmachines.yaml b/config/crd/bases/crds.kloudlite.io_workmachines.yaml new file mode 100644 index 00000000..41a0b43e --- /dev/null +++ b/config/crd/bases/crds.kloudlite.io_workmachines.yaml @@ -0,0 +1,236 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: workmachines.crds.kloudlite.io +spec: + group: crds.kloudlite.io + names: + kind: WorkMachine + listKind: WorkMachineList + plural: workmachines + singular: workmachine + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.targetNamespace + name: TargetNamespace + type: string + - jsonPath: .status.status.lastReconcileTime + name: Seen + type: date + - jsonPath: .metadata.annotations.kloudlite\.io\/operator\.checks + name: Checks + type: string + - jsonPath: .metadata.annotations.kloudlite\.io\/operator\.resource\.ready + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: WorkMachine is the Schema for the workmachines API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: WorkMachineSpec defines the desired state of WorkMachine + properties: + aws: + properties: + ami: + type: string + externalVolumeSize: + type: integer + externalVolumeType: + default: gp3 + type: string + iamInstanceProfileRole: + type: string + instanceType: + type: string + rootVolumeSize: + default: 50 + type: integer + rootVolumeType: + default: gp3 + type: string + required: + - ami + - externalVolumeSize + - externalVolumeType + - instanceType + - rootVolumeSize + - rootVolumeType + type: object + jobParams: + properties: + nodeSelector: + additionalProperties: + type: string + type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + sshPublicKeys: + items: + type: string + type: array + state: + enum: + - "ON" + - "OFF" + type: string + targetNamespace: + type: string + required: + - aws + - sshPublicKeys + - state + type: object + status: + properties: + machineSSHKey: + type: string + status: + properties: + checkList: + items: + properties: + debug: + type: boolean + description: + type: string + hide: + type: boolean + name: + type: string + title: + type: string + required: + - name + - title + type: object + type: array + checks: + additionalProperties: + properties: + debug: + type: string + error: + type: string + generation: + format: int64 + type: integer + info: + type: string + message: + type: string + startedAt: + format: date-time + type: string + state: + type: string + status: + type: boolean + required: + - status + type: object + type: object + isReady: + type: boolean + lastReadyGeneration: + format: int64 + type: integer + lastReconcileTime: + format: date-time + type: string + resources: + items: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + type: array + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/crds.kloudlite.io_workspaces.yaml b/config/crd/bases/crds.kloudlite.io_workspaces.yaml new file mode 100644 index 00000000..725fac87 --- /dev/null +++ b/config/crd/bases/crds.kloudlite.io_workspaces.yaml @@ -0,0 +1,166 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: workspaces.crds.kloudlite.io +spec: + group: crds.kloudlite.io + names: + kind: Workspace + listKind: WorkspaceList + plural: workspaces + singular: workspace + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.lastReconcileTime + name: Seen + type: date + - jsonPath: .metadata.annotations.kloudlite\.io\/operator\.checks + name: Checks + type: string + - jsonPath: .metadata.annotations.kloudlite\.io\/operator\.resource\.ready + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: Workspace is the Schema for the workspaces API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: WorkspaceSpec defines the desired state of Workspace + properties: + enableCodeServer: + type: boolean + enableJupyterNotebook: + type: boolean + enableTTYD: + type: boolean + enableVSCodeServer: + type: boolean + imagePullPolicy: + default: IfNotPresent + type: string + serviceAccountName: + type: string + state: + default: "ON" + enum: + - "ON" + - "OFF" + type: string + workMachine: + description: Name of work machine + type: string + required: + - state + - workMachine + type: object + status: + properties: + checkList: + items: + properties: + debug: + type: boolean + description: + type: string + hide: + type: boolean + name: + type: string + title: + type: string + required: + - name + - title + type: object + type: array + checks: + additionalProperties: + properties: + debug: + type: string + error: + type: string + generation: + format: int64 + type: integer + info: + type: string + message: + type: string + startedAt: + format: date-time + type: string + state: + type: string + status: + type: boolean + required: + - status + type: object + type: object + isReady: + type: boolean + lastReadyGeneration: + format: int64 + type: integer + lastReconcileTime: + format: date-time + type: string + resources: + items: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 82d06489..f394152e 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -58,6 +58,8 @@ resources: - bases/mongodb.msvc.kloudlite.io_standalonedatabases.yaml - bases/crds.kloudlite.io_serviceintercepts.yaml - bases/mongodb.msvc.kloudlite.io_backups.yaml +- bases/crds.kloudlite.io_workspaces.yaml +- bases/crds.kloudlite.io_workmachines.yaml # +kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -148,6 +150,8 @@ patchesStrategicMerge: #- patches/cainjection_in_serviceintercepts.yaml #- patches/cainjection_in_backups.yaml #- patches/cainjection_in_managedserviceplugins.yaml +#- path: patches/cainjection_in_crds_workspaces.yaml +#- path: patches/cainjection_in_crds_workmachines.yaml # +kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/rbac/crds_workmachine_editor_role.yaml b/config/rbac/crds_workmachine_editor_role.yaml new file mode 100644 index 00000000..10fcfb0d --- /dev/null +++ b/config/rbac/crds_workmachine_editor_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to edit workmachines. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: app + app.kubernetes.io/managed-by: kustomize + name: crds-workmachine-editor-role +rules: +- apiGroups: + - crds.kloudlite.io + resources: + - workmachines + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - crds.kloudlite.io + resources: + - workmachines/status + verbs: + - get diff --git a/config/rbac/crds_workmachine_viewer_role.yaml b/config/rbac/crds_workmachine_viewer_role.yaml new file mode 100644 index 00000000..f3d86662 --- /dev/null +++ b/config/rbac/crds_workmachine_viewer_role.yaml @@ -0,0 +1,23 @@ +# permissions for end users to view workmachines. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: app + app.kubernetes.io/managed-by: kustomize + name: crds-workmachine-viewer-role +rules: +- apiGroups: + - crds.kloudlite.io + resources: + - workmachines + verbs: + - get + - list + - watch +- apiGroups: + - crds.kloudlite.io + resources: + - workmachines/status + verbs: + - get diff --git a/config/rbac/crds_workspace_editor_role.yaml b/config/rbac/crds_workspace_editor_role.yaml new file mode 100644 index 00000000..12baea99 --- /dev/null +++ b/config/rbac/crds_workspace_editor_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to edit workspaces. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: app + app.kubernetes.io/managed-by: kustomize + name: crds-workspace-editor-role +rules: +- apiGroups: + - crds.kloudlite.io + resources: + - workspaces + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - crds.kloudlite.io + resources: + - workspaces/status + verbs: + - get diff --git a/config/rbac/crds_workspace_viewer_role.yaml b/config/rbac/crds_workspace_viewer_role.yaml new file mode 100644 index 00000000..92e422a2 --- /dev/null +++ b/config/rbac/crds_workspace_viewer_role.yaml @@ -0,0 +1,23 @@ +# permissions for end users to view workspaces. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: app + app.kubernetes.io/managed-by: kustomize + name: crds-workspace-viewer-role +rules: +- apiGroups: + - crds.kloudlite.io + resources: + - workspaces + verbs: + - get + - list + - watch +- apiGroups: + - crds.kloudlite.io + resources: + - workspaces/status + verbs: + - get diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 731832a6..285c3bc5 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -16,3 +16,12 @@ resources: - auth_proxy_role.yaml - auth_proxy_role_binding.yaml - auth_proxy_client_clusterrole.yaml +# For each CRD, "Editor" and "Viewer" roles are scaffolded by +# default, aiding admins in cluster management. Those roles are +# not used by the Project itself. You can comment the following lines +# if you do not want those helpers be installed with your Project. +- crds_workmachine_editor_role.yaml +- crds_workmachine_viewer_role.yaml +- crds_workspace_editor_role.yaml +- crds_workspace_viewer_role.yaml + diff --git a/config/samples/crds_v1_workmachine.yaml b/config/samples/crds_v1_workmachine.yaml new file mode 100644 index 00000000..0c7222d7 --- /dev/null +++ b/config/samples/crds_v1_workmachine.yaml @@ -0,0 +1,9 @@ +apiVersion: crds.kloudlite.io/v1 +kind: WorkMachine +metadata: + labels: + app.kubernetes.io/name: app + app.kubernetes.io/managed-by: kustomize + name: workmachine-sample +spec: + # TODO(user): Add fields here diff --git a/config/samples/crds_v1_workspace.yaml b/config/samples/crds_v1_workspace.yaml new file mode 100644 index 00000000..3b748115 --- /dev/null +++ b/config/samples/crds_v1_workspace.yaml @@ -0,0 +1,9 @@ +apiVersion: crds.kloudlite.io/v1 +kind: Workspace +metadata: + labels: + app.kubernetes.io/name: app + app.kubernetes.io/managed-by: kustomize + name: workspace-sample +spec: + # TODO(user): Add fields here diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index 0680afa8..fa49689a 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -110,4 +110,6 @@ resources: - crds_v1_serviceintercept.yaml - mongodb.msvc_v1_backup.yaml - crds_v1_managedserviceplugin.yaml +- crds_v1_workspace.yaml +- crds_v1_workmachine.yaml #+kubebuilder:scaffold:manifestskustomizesamples diff --git a/go.mod b/go.mod index 0ab2dae5..b96410fd 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/cert-manager/cert-manager v1.16.2 github.com/charmbracelet/log v0.4.0 github.com/codingconcepts/env v0.0.0-20240618133406-5b0845441187 - github.com/evanphx/json-patch/v5 v5.9.0 + github.com/evanphx/json-patch/v5 v5.9.11 github.com/fatih/color v1.18.0 github.com/go-chi/chi/v5 v5.2.0 github.com/go-redis/redis/v8 v8.11.5 @@ -20,13 +20,15 @@ require ( github.com/gobuffalo/flect v1.0.3 github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e github.com/influxdata/influxdb-client-go/v2 v2.14.0 - github.com/kloudlite/operator/toolkit v0.0.0-20250316093242-493e9b587c10 - github.com/kloudlite/plugin-helm-chart v0.0.0-20241220114210-b6d65e34990d + github.com/kloudlite/operator/toolkit v0.0.0-20250323044516-4d91d0b9477a + github.com/kloudlite/plugin-helm-chart v0.0.0-20250317052100-fef043b111a2 + github.com/kloudlite/plugin-k3s-cluster v0.0.0-20250420192843-f0fae9cd7d36 github.com/kloudlite/plugin-mongodb v0.0.0-20250316175205-312ba86d8873 github.com/matoous/go-nanoid/v2 v2.1.0 github.com/miekg/dns v1.1.62 github.com/nats-io/nats.go v1.38.0 github.com/nxtcoder17/go-helm-client v0.0.0-20230915000026-8789cfa27bf3 + github.com/nxtcoder17/go.pkgs v0.0.0-20250401173049-502a28e591dd github.com/onsi/ginkgo/v2 v2.22.0 github.com/onsi/gomega v1.36.1 github.com/pkg/errors v0.9.1 @@ -45,13 +47,13 @@ require ( google.golang.org/grpc v1.69.2 google.golang.org/protobuf v1.36.0 helm.sh/helm/v3 v3.16.4 - k8s.io/api v0.32.0 - k8s.io/apiextensions-apiserver v0.32.0 - k8s.io/apimachinery v0.32.0 - k8s.io/client-go v0.32.0 + k8s.io/api v0.32.1 + k8s.io/apiextensions-apiserver v0.32.1 + k8s.io/apimachinery v0.32.1 + k8s.io/client-go v0.32.1 k8s.io/kubernetes v1.32.0 k8s.io/utils v0.0.0-20241210054802-24370beab758 - sigs.k8s.io/controller-runtime v0.19.3 + sigs.k8s.io/controller-runtime v0.20.2 sigs.k8s.io/yaml v1.4.0 ) @@ -84,6 +86,10 @@ require ( github.com/muesli/termenv v0.15.2 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/oapi-codegen/runtime v1.1.1 // indirect + github.com/rs/zerolog v1.33.0 // indirect + github.com/samber/lo v1.47.0 // indirect + github.com/samber/slog-common v0.18.1 // indirect + github.com/samber/slog-zerolog/v2 v2.7.3 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.58.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect @@ -220,9 +226,9 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20241216192217-9240e9c98484 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiserver v0.32.0 // indirect + k8s.io/apiserver v0.32.1 // indirect k8s.io/cli-runtime v0.32.0 // indirect - k8s.io/component-base v0.32.0 // indirect + k8s.io/component-base v0.32.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 // indirect k8s.io/kubectl v0.32.0 // indirect diff --git a/go.sum b/go.sum index d1e27231..eecda73d 100644 --- a/go.sum +++ b/go.sum @@ -130,8 +130,8 @@ github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtz github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= -github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= @@ -187,6 +187,7 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo= github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -275,22 +276,10 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= -github.com/kloudlite/plugin-helm-chart v0.0.0-20241220114210-b6d65e34990d h1:kdj4Zt4NdICz7Vm/34Q0WW+69iKO1gwyfE45yAaQ5vs= -github.com/kloudlite/plugin-helm-chart v0.0.0-20241220114210-b6d65e34990d/go.mod h1:TZUQ5mREV+UjGVJhPsgmHMS67Mlf2Guit4905Pi/5sc= -github.com/kloudlite/plugin-mongodb v0.0.0-20241221132659-be42e75f149f h1:7yjk5Nh4dkw3rW0NdS7N3bAjfNFC/w9VF6LfXFqRVrQ= -github.com/kloudlite/plugin-mongodb v0.0.0-20241221132659-be42e75f149f/go.mod h1:7ZimdiRyRmVSTS0U69a2sjcvRw/ZbRk6ICwftb6CWxY= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316133014-5e99560adb76 h1:D8IceJKpSOG9LKy2VzN1xHNm4dJ+l7arGPxkj8iZxIo= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316133014-5e99560adb76/go.mod h1:dqT/Qia369uD783N8N58RZPhRPZXywV85nLGDwcq698= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316133328-275e701f2943 h1:ILrHSigArzmo2QH2SIkJEzT1CZNufzQeu1CjM/ljPKo= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316133328-275e701f2943/go.mod h1:dqT/Qia369uD783N8N58RZPhRPZXywV85nLGDwcq698= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316134709-6214224a6327 h1:YayBAW0bjYe+z1ijRpTshMbNYlhjivueuT+HtlSJhlI= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316134709-6214224a6327/go.mod h1:dqT/Qia369uD783N8N58RZPhRPZXywV85nLGDwcq698= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316140417-088f796334a9 h1:O9PJPxmuzSckAkt6uZhschc0IbDeWoTmdwnAF+QZ24U= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316140417-088f796334a9/go.mod h1:dqT/Qia369uD783N8N58RZPhRPZXywV85nLGDwcq698= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316143614-628cc21496dc h1:pqY7GdSJehGrDK5LEazzuiGvKYGnjlVK527DrlqBc3k= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316143614-628cc21496dc/go.mod h1:dqT/Qia369uD783N8N58RZPhRPZXywV85nLGDwcq698= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316173733-b67e5b812ebe h1:JVPgw0qQlAEpk9F5lZCzHM5fD8WPGtTTNeS1u/G/6yQ= -github.com/kloudlite/plugin-mongodb v0.0.0-20250316173733-b67e5b812ebe/go.mod h1:dqT/Qia369uD783N8N58RZPhRPZXywV85nLGDwcq698= +github.com/kloudlite/plugin-helm-chart v0.0.0-20250317052100-fef043b111a2 h1:4DoLvbPEjYVBMnNpVmeR9OoI6Q9kebd6HnDpkwFts+Y= +github.com/kloudlite/plugin-helm-chart v0.0.0-20250317052100-fef043b111a2/go.mod h1:TZUQ5mREV+UjGVJhPsgmHMS67Mlf2Guit4905Pi/5sc= +github.com/kloudlite/plugin-k3s-cluster v0.0.0-20250420192843-f0fae9cd7d36 h1:pOkVQ9ocpQVZ3QSsck2vjzx0WWhH4l0pJDibrEQUA04= +github.com/kloudlite/plugin-k3s-cluster v0.0.0-20250420192843-f0fae9cd7d36/go.mod h1:sqCqURIr6j/SI0hIoA+22Etpcm+9U/MXQXE2iCMmUa4= github.com/kloudlite/plugin-mongodb v0.0.0-20250316175205-312ba86d8873 h1:6GKV4bZoNzCC5JvyqxHAhjXXSpSjzAXQHklwXqz5YJQ= github.com/kloudlite/plugin-mongodb v0.0.0-20250316175205-312ba86d8873/go.mod h1:dqT/Qia369uD783N8N58RZPhRPZXywV85nLGDwcq698= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -320,6 +309,7 @@ github.com/matoous/go-nanoid/v2 v2.1.0/go.mod h1:KlbGNQ+FhrUNIHUxZdL63t7tl4LaPkZ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= @@ -381,6 +371,8 @@ github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/nxtcoder17/go-helm-client v0.0.0-20230915000026-8789cfa27bf3 h1:36FtxDl0ARKh5SQY3tyYD2/VCcMhZCx+01LGihpT06o= github.com/nxtcoder17/go-helm-client v0.0.0-20230915000026-8789cfa27bf3/go.mod h1:jxy7CytMOdPqv7eDdKuxaQxsdeHoSZP9tQFZt9O6nlA= +github.com/nxtcoder17/go.pkgs v0.0.0-20250401173049-502a28e591dd h1:bP80t6mgiwIVEpdfZwg/w22GzDcHBgDRVZItH5QWxis= +github.com/nxtcoder17/go.pkgs v0.0.0-20250401173049-502a28e591dd/go.mod h1:raSGHj5CMHNHZf4fCV9CWpFk0hsb2CSKFZSPd4zW8JM= github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= @@ -441,10 +433,19 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= +github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/rubenv/sql-migrate v1.7.1 h1:f/o0WgfO/GqNuVg+6801K/KW3WdDSupzSjDYODmiUq4= github.com/rubenv/sql-migrate v1.7.1/go.mod h1:Ob2Psprc0/3ggbM6wCzyYVFFuc6FyZrb2AS+ezLDFb4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= +github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= +github.com/samber/slog-common v0.18.1 h1:c0EipD/nVY9HG5shgm/XAs67mgpWDMF+MmtptdJNCkQ= +github.com/samber/slog-common v0.18.1/go.mod h1:QNZiNGKakvrfbJ2YglQXLCZauzkI9xZBjOhWFKS3IKk= +github.com/samber/slog-zerolog/v2 v2.7.3 h1:/MkPDl/tJhijN2GvB1MWwBn2FU8RiL3rQ8gpXkQm2EY= +github.com/samber/slog-zerolog/v2 v2.7.3/go.mod h1:oWU7WHof4Xp8VguiNO02r1a4VzkgoOyOZhY5CuRke60= github.com/seancfoley/bintree v1.3.1 h1:cqmmQK7Jm4aw8gna0bP+huu5leVOgHGSJBEpUx3EXGI= github.com/seancfoley/bintree v1.3.1/go.mod h1:hIUabL8OFYyFVTQ6azeajbopogQc2l5C/hiXMcemWNU= github.com/seancfoley/ipaddress-go v1.7.0 h1:vWp3SR3k+HkV3aKiNO2vEe6xbVxS0x/Ixw6hgyP238s= @@ -614,6 +615,7 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -677,20 +679,20 @@ gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= helm.sh/helm/v3 v3.16.4 h1:rBn/h9MACw+QlhxQTjpl8Ifx+VTWaYsw3rguGBYBzr0= helm.sh/helm/v3 v3.16.4/go.mod h1:k8QPotUt57wWbi90w3LNmg3/MWcLPigVv+0/X4B8BzA= -k8s.io/api v0.32.0 h1:OL9JpbvAU5ny9ga2fb24X8H6xQlVp+aJMFlgtQjR9CE= -k8s.io/api v0.32.0/go.mod h1:4LEwHZEf6Q/cG96F3dqR965sYOfmPM7rq81BLgsE0p0= -k8s.io/apiextensions-apiserver v0.32.0 h1:S0Xlqt51qzzqjKPxfgX1xh4HBZE+p8KKBq+k2SWNOE0= -k8s.io/apiextensions-apiserver v0.32.0/go.mod h1:86hblMvN5yxMvZrZFX2OhIHAuFIMJIZ19bTvzkP+Fmw= -k8s.io/apimachinery v0.32.0 h1:cFSE7N3rmEEtv4ei5X6DaJPHHX0C+upp+v5lVPiEwpg= -k8s.io/apimachinery v0.32.0/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= -k8s.io/apiserver v0.32.0 h1:VJ89ZvQZ8p1sLeiWdRJpRD6oLozNZD2+qVSLi+ft5Qs= -k8s.io/apiserver v0.32.0/go.mod h1:HFh+dM1/BE/Hm4bS4nTXHVfN6Z6tFIZPi649n83b4Ag= +k8s.io/api v0.32.1 h1:f562zw9cy+GvXzXf0CKlVQ7yHJVYzLfL6JAS4kOAaOc= +k8s.io/api v0.32.1/go.mod h1:/Yi/BqkuueW1BgpoePYBRdDYfjPF5sgTr5+YqDZra5k= +k8s.io/apiextensions-apiserver v0.32.1 h1:hjkALhRUeCariC8DiVmb5jj0VjIc1N0DREP32+6UXZw= +k8s.io/apiextensions-apiserver v0.32.1/go.mod h1:sxWIGuGiYov7Io1fAS2X06NjMIk5CbRHc2StSmbaQto= +k8s.io/apimachinery v0.32.1 h1:683ENpaCBjma4CYqsmZyhEzrGz6cjn1MY/X2jB2hkZs= +k8s.io/apimachinery v0.32.1/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/apiserver v0.32.1 h1:oo0OozRos66WFq87Zc5tclUX2r0mymoVHRq8JmR7Aak= +k8s.io/apiserver v0.32.1/go.mod h1:UcB9tWjBY7aryeI5zAgzVJB/6k7E97bkr1RgqDz0jPw= k8s.io/cli-runtime v0.32.0 h1:dP+OZqs7zHPpGQMCGAhectbHU2SNCuZtIimRKTv2T1c= k8s.io/cli-runtime v0.32.0/go.mod h1:Mai8ht2+esoDRK5hr861KRy6z0zHsSTYttNVJXgP3YQ= -k8s.io/client-go v0.32.0 h1:DimtMcnN/JIKZcrSrstiwvvZvLjG0aSxy8PxN8IChp8= -k8s.io/client-go v0.32.0/go.mod h1:boDWvdM1Drk4NJj/VddSLnx59X3OPgwrOo0vGbtq9+8= -k8s.io/component-base v0.32.0 h1:d6cWHZkCiiep41ObYQS6IcgzOUQUNpywm39KVYaUqzU= -k8s.io/component-base v0.32.0/go.mod h1:JLG2W5TUxUu5uDyKiH2R/7NnxJo1HlPoRIIbVLkK5eM= +k8s.io/client-go v0.32.1 h1:otM0AxdhdBIaQh7l1Q0jQpmo7WOFIk5FFa4bg6YMdUU= +k8s.io/client-go v0.32.1/go.mod h1:aTTKZY7MdxUaJ/KiUs8D+GssR9zJZi77ZqtzcGXIiDg= +k8s.io/component-base v0.32.1 h1:/5IfJ0dHIKBWysGV0yKTFfacZ5yNV1sulPh3ilJjRZk= +k8s.io/component-base v0.32.1/go.mod h1:j1iMMHi/sqAHeG5z+O9BFNCF698a1u0186zkjMZQ28w= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 h1:hcha5B1kVACrLujCKLbr8XWMxCxzQx42DY8QKYJrDLg= @@ -703,8 +705,8 @@ k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJ k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= oras.land/oras-go v1.2.6 h1:z8cmxQXBU8yZ4mkytWqXfo6tZcamPwjsuxYU81xJ8Lk= oras.land/oras-go v1.2.6/go.mod h1:OVPc1PegSEe/K8YiLfosrlqlqTN9PUyFvOw5Y9gwrT8= -sigs.k8s.io/controller-runtime v0.19.3 h1:XO2GvC9OPftRst6xWCpTgBZO04S2cbp0Qqkj8bX1sPw= -sigs.k8s.io/controller-runtime v0.19.3/go.mod h1:j4j87DqtsThvwTv5/Tc5NFRyyF/RF0ip4+62tbTSIUM= +sigs.k8s.io/controller-runtime v0.20.2 h1:/439OZVxoEc02psi1h4QO3bHzTgu49bb347Xp4gW1pc= +sigs.k8s.io/controller-runtime v0.20.2/go.mod h1:xg2XB0K5ShQzAgsoujxuKN4LNXR2LfwwHsPj7Iaw+XY= sigs.k8s.io/gateway-api v1.2.1 h1:fZZ/+RyRb+Y5tGkwxFKuYuSRQHu9dZtbjenblleOLHM= sigs.k8s.io/gateway-api v1.2.1/go.mod h1:EpNfEXNjiYfUJypf0eZ0P5iXA9ekSGWaS1WgPaM42X0= sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= diff --git a/operators/app-n-lambda/internal/controllers/app/controller.go b/operators/app-n-lambda/internal/controllers/app/controller.go index ded3f0fe..96d56c8c 100644 --- a/operators/app-n-lambda/internal/controllers/app/controller.go +++ b/operators/app-n-lambda/internal/controllers/app/controller.go @@ -60,7 +60,7 @@ const ( AppInterceptCreated string = "app-intercept-created" - AppRouterReady string = "app-router-ready" + createAppRouter string = "create-app-router" ) var DeleteChecklist = []reconciler.CheckMeta{ @@ -101,7 +101,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. {Name: DeploymentReady, Title: "Deployment Ready", Hide: req.Object.IsInterceptEnabled()}, {Name: HPAConfigured, Title: "Horizontal pod autoscaling configured", Hide: req.Object.IsInterceptEnabled() || !req.Object.IsHPAEnabled()}, {Name: AppInterceptCreated, Title: "App Intercept Created", Hide: !req.Object.IsInterceptEnabled()}, - {Name: AppRouterReady, Title: "App Router Ready", Hide: req.Object.Spec.Router == nil}, + {Name: createAppRouter, Title: "Create App Router", Hide: req.Object.Spec.Router == nil}, }); !step.ShouldProceed() { return step.ReconcilerResponse() } @@ -138,7 +138,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. return step.ReconcilerResponse() } - if step := r.checkAppRouter(req); !step.ShouldProceed() { + if step := r.handleAppRouter(req); !step.ShouldProceed() { return step.ReconcilerResponse() } @@ -450,30 +450,21 @@ func (r *Reconciler) checkDeploymentReady(req *reconciler.Request[*crdsv1.App]) return check.Completed() } -func (r *Reconciler) checkAppRouter(req *reconciler.Request[*crdsv1.App]) stepResult.Result { +func (r *Reconciler) handleAppRouter(req *reconciler.Request[*crdsv1.App]) stepResult.Result { ctx, obj := req.Context(), req.Object - check := reconciler.NewRunningCheck(AppRouterReady, req) + check := reconciler.NewRunningCheck(createAppRouter, req) if obj.Spec.Router == nil { return check.Completed() } if len(obj.Spec.Router.Routes) == 0 { - if len(obj.Spec.Services) != 0 { - return check.Failed(fmt.Errorf("app has multiple exposed services, cannot deduce router routes automatically from services, router routes must be explicity set via .spec.router.routes")) - } - - obj.Spec.Router.Routes = append(obj.Spec.Router.Routes, crdsv1.Route{ - App: obj.Name, - Path: "/", - Port: obj.Spec.Services[0].Port, - }) - - if err := r.Update(ctx, obj); err != nil { - return check.Failed(err) - } + return check.Failed(fmt.Errorf("must specify at least 1 route")) + } - return check.StillRunning(fmt.Errorf("updating app router default routes")) + for i := range obj.Spec.Router.Routes { + // INFO: app router will only route to current app, for any such usecases Router kind must be used + obj.Spec.Router.Routes[i].Service = obj.Name } router := &crdsv1.Router{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-app-router", obj.Name), Namespace: obj.Namespace}} diff --git a/operators/lifecycle/internal/lifecycle-controller/controller.go b/operators/lifecycle/internal/lifecycle-controller/controller.go index 064e5296..bb189e17 100644 --- a/operators/lifecycle/internal/lifecycle-controller/controller.go +++ b/operators/lifecycle/internal/lifecycle-controller/controller.go @@ -212,30 +212,48 @@ func (r *Reconciler) applyK8sJob(req *rApi.Request[*crdsv1.Lifecycle]) stepResul ctx, obj := req.Context(), req.Object check := rApi.NewRunningCheck(ApplyK8sJob, req) - if v, ok := obj.Status.Checks[ApplyK8sJob]; ok && v.Generation == obj.Generation { - switch v.State { - case rApi.CompletedState: - return check.Completed() - case rApi.ErroredState: - if obj.Spec.RetryOnFailure { - if obj.Status.LastReconcileTime != nil { - req.Logger.Info(fmt.Sprintf("time since last reconcilation: %s", time.Since(obj.Status.LastReconcileTime.Time).String())) - if time.Since(obj.Status.LastReconcileTime.Time) < obj.Spec.RetryOnFailureDelay.Duration { - return check.Completed().RequeueAfter(obj.Spec.RetryOnFailureDelay.Duration) - } + if obj.Status.Phase == crdsv1.JobPhaseFailed && obj.Status.LastReadyGeneration == obj.Generation { + if obj.Spec.RetryOnFailure { + if obj.Status.LastReconcileTime != nil { + req.Logger.Info(fmt.Sprintf("time since last reconcilation: %s", time.Since(obj.Status.LastReconcileTime.Time).String())) + if time.Since(obj.Status.LastReconcileTime.Time) < obj.Spec.RetryOnFailureDelay.Duration { + return check.Completed().RequeueAfter(obj.Spec.RetryOnFailureDelay.Duration) + } - req.Logger.Info(fmt.Sprintf("re-creating job for lifecycle %s/%s, as it failed and is past the retry delay", obj.Namespace, obj.Name)) - if err := r.Delete(ctx, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: obj.Name, Namespace: obj.Namespace}}); err != nil { - if !apiErrors.IsNotFound(err) { - return check.Failed(err) - } + req.Logger.Info(fmt.Sprintf("re-creating job for lifecycle %s/%s, as it failed and is past the retry delay", obj.Namespace, obj.Name)) + if err := r.Delete(ctx, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: obj.Name, Namespace: obj.Namespace}}); err != nil { + if !apiErrors.IsNotFound(err) { + return check.Failed(err) } } } - return check.Failed(fmt.Errorf("job failed")) } } + // if v, ok := obj.Status.Checks[ApplyK8sJob]; ok && v.Generation == obj.Generation { + // switch v.State { + // case rApi.CompletedState: + // return check.Completed() + // case rApi.ErroredState: + // if obj.Spec.RetryOnFailure { + // if obj.Status.LastReconcileTime != nil { + // req.Logger.Info(fmt.Sprintf("time since last reconcilation: %s", time.Since(obj.Status.LastReconcileTime.Time).String())) + // if time.Since(obj.Status.LastReconcileTime.Time) < obj.Spec.RetryOnFailureDelay.Duration { + // return check.Completed().RequeueAfter(obj.Spec.RetryOnFailureDelay.Duration) + // } + // + // req.Logger.Info(fmt.Sprintf("re-creating job for lifecycle %s/%s, as it failed and is past the retry delay", obj.Namespace, obj.Name)) + // if err := r.Delete(ctx, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: obj.Name, Namespace: obj.Namespace}}); err != nil { + // if !apiErrors.IsNotFound(err) { + // return check.Failed(err) + // } + // } + // } + // } + // return check.Failed(fmt.Errorf("job failed")) + // } + // } + job := &batchv1.Job{} if err := r.Get(ctx, fn.NN(obj.Namespace, obj.Name), job); err != nil { job = nil @@ -400,6 +418,8 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { return fmt.Errorf("yamlclient must be set") } + recorder := mgr.GetEventRecorderFor(r.GetName()) + var err error r.templateJobRBAC, err = templates.Read(templates.JobRBACTemplate) if err != nil { @@ -408,6 +428,6 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { builder := ctrl.NewControllerManagedBy(mgr).For(&crdsv1.Lifecycle{}).Owns(&batchv1.Job{}) builder.WithOptions(controller.Options{MaxConcurrentReconciles: r.Env.MaxConcurrentReconciles}) - builder.WithEventFilter(rApi.ReconcileFilter()) + builder.WithEventFilter(rApi.ReconcileFilter(recorder)) return builder.Complete(r) } diff --git a/operators/resource-watcher/controller/register.go b/operators/resource-watcher/controller/register.go index 8d8ae5b2..c0db04e2 100644 --- a/operators/resource-watcher/controller/register.go +++ b/operators/resource-watcher/controller/register.go @@ -8,6 +8,7 @@ import ( clustersv1 "github.com/kloudlite/operator/apis/clusters/v1" "github.com/kloudlite/operator/grpc-interfaces/grpc/messages" + "github.com/nxtcoder17/go.pkgs/log" "google.golang.org/grpc" "google.golang.org/grpc/connectivity" @@ -47,7 +48,6 @@ func RegisterInto(mgr operator.Operator) { } mgr.RegisterControllers(watchAndUpdateReconciler) - logger := mgr.Operator().Logger() ping := func(cc *grpc.ClientConn) error { ctx, cf := context.WithTimeout(context.TODO(), 500*time.Millisecond) @@ -55,10 +55,10 @@ func RegisterInto(mgr operator.Operator) { msgDispatchCli := messages.NewMessageDispatchServiceClient(cc) _, err := msgDispatchCli.Ping(ctx, &messages.Empty{}) if err != nil { - logger.Info("ping failed, client is disconnected") + log.DefaultLogger().Warn("ping failed, client is disconnected") return err } - logger.Debug("ping is successfull, client is connected") + log.DefaultLogger().Debug("ping is successfull, client is connected") return nil } @@ -116,7 +116,7 @@ func RegisterInto(mgr operator.Operator) { } go func() { - logger := logger.With("component", "grpc-client") + logger := log.DefaultLogger().With("component", "grpc-client").Slog() for { connectGrpc(logger) <-time.After(2 * time.Second) diff --git a/operators/resource-watcher/internal/controllers/watch-and-update/controller.go b/operators/resource-watcher/internal/controllers/watch-and-update/controller.go index dc52b93b..e6049d1a 100644 --- a/operators/resource-watcher/internal/controllers/watch-and-update/controller.go +++ b/operators/resource-watcher/internal/controllers/watch-and-update/controller.go @@ -294,7 +294,7 @@ func (r *Reconciler) dispatchEvent(ctx context.Context, logger *slog.Logger, obj return nil } - case /*NodePoolGVK.String(),*/ PersistentVolumeClaimGVK.String(), PersistentVolumeGVK.String(), VolumeAttachmentGVK.String(), IngressGVK.String(), NamespaceGVK.String(): + case /*NodePoolGVK.String(),*/ PersistentVolumeClaimGVK.String(), PersistentVolumeGVK.String(), VolumeAttachmentGVK.String(), IngressGVK.String(), NamespaceGVK.String(), WorkspaceGVK.String(), WorkMachineGVK.String(): { return r.MsgSender.DispatchInfraResourceUpdates(MessageSenderContext{mctx, logger}, t.ResourceUpdate{ Object: obj, @@ -440,7 +440,7 @@ var ( // BuildRunGVK = newGVK("distribution.kloudlite.io/v1", "BuildRun") ClusterManagedServiceGVK = newGVK("crds.kloudlite.io/v1", "ClusterManagedService") - HelmChartGVK = newGVK("crds.kloudlite.io/v1", "HelmChart") + HelmChartGVK = newGVK("plugin-helm-chart.kloudlite.github.com/v1", "HelmChart") ProjectManageServiceGVK = newGVK("crds.kloudlite.io/v1", "ProjectManagedService") // native resources @@ -452,6 +452,9 @@ var ( SecretGVK = newGVK("v1", "Secret") ConfigmapGVK = newGVK("v1", "ConfigMap") NamespaceGVK = newGVK("v1", "Namespace") + + WorkMachineGVK = newGVK("crds.kloudlite.io/v1", "WorkMachine") + WorkspaceGVK = newGVK("crds.kloudlite.io/v1", "Workspace") ) // SetupWithManager sets up the controllers with the Manager. @@ -494,6 +497,9 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { SecretGVK, ConfigmapGVK, NamespaceGVK, + + WorkspaceGVK, + WorkMachineGVK, } for i := range watchList { diff --git a/operators/routers/internal/env/env.go b/operators/routers/internal/env/env.go index cb6a7342..373fe063 100644 --- a/operators/routers/internal/env/env.go +++ b/operators/routers/internal/env/env.go @@ -6,11 +6,6 @@ import ( type Env struct { MaxConcurrentReconciles int `env:"MAX_CONCURRENT_RECONCILES" default:"5"` - - DefaultIngressClass string `env:"DEFAULT_INGRESS_CLASS" required:"true"` - DefaultClusterIssuer string `env:"DEFAULT_CLUSTER_ISSUER" required:"true"` - - CertificateNamespace string `env:"CERTIFICATE_NAMESPACE" required:"true"` } func GetEnvOrDie() *Env { diff --git a/operators/routers/internal/router-controller/controller.go b/operators/routers/internal/router-controller/controller.go index cc2d3fb8..9d9322c2 100644 --- a/operators/routers/internal/router-controller/controller.go +++ b/operators/routers/internal/router-controller/controller.go @@ -3,14 +3,12 @@ package router_controller import ( "context" "fmt" - "time" + "strings" certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" - certmanagermetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" "golang.org/x/crypto/bcrypt" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" - apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" @@ -54,25 +52,13 @@ const ( EnsuringHttpsCertsIfEnabled string = "ensuring-https-certs-if-enabled" SettingUpBasicAuthIfEnabled string = "setting-up-basic-auth-if-enabled" - CreatingIngressResources string = "creating-ingress-resources" + CreateIngressResource string = "create-ingress-resource" CleaningUpResources string = "cleaning-up-resourcess" certCreatedByRouter string = "kloudlite.io/cert-created-by-router" ) -var ( - ApplyChecklist = []reconciler.CheckMeta{ - {Name: DefaultsPatched, Title: "Defaults Patched"}, - {Name: EnsuringHttpsCertsIfEnabled, Title: "Ensuring HTTPS Cert if enabled"}, - {Name: SettingUpBasicAuthIfEnabled, Title: "Setting Up Basic Auth if enabled"}, - } - - DeleteChecklist = []reconciler.CheckMeta{ - {Name: CleaningUpResources, Title: "Cleaning Up Resources"}, - } -) - // +kubebuilder:rbac:groups=crds.kloudlite.io,resources=crds,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=crds.kloudlite.io,resources=crds/status,verbs=get;update;patch // +kubebuilder:rbac:groups=crds.kloudlite.io,resources=crds/finalizers,verbs=update @@ -101,7 +87,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. return step.ReconcilerResponse() } - if step := req.EnsureCheckList(ApplyChecklist); !step.ShouldProceed() { + if step := req.EnsureCheckList([]reconciler.CheckMeta{ + {Name: EnsuringHttpsCertsIfEnabled, Title: "Ensuring HTTPS Cert if enabled"}, + {Name: SettingUpBasicAuthIfEnabled, Title: "Setting Up Basic Auth if enabled"}, + {Name: CreateIngressResource, Title: "Creates kubernetes ingress resource"}, + }); !step.ShouldProceed() { return step.ReconcilerResponse() } @@ -113,13 +103,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. return step.ReconcilerResponse() } - if step := r.patchDefaults(req); !step.ShouldProceed() { - return step.ReconcilerResponse() - } - - if step := r.EnsuringHttpsCerts(req); !step.ShouldProceed() { - return step.ReconcilerResponse() - } + // if step := r.EnsuringHttpsCerts(req); !step.ShouldProceed() { + // return step.ReconcilerResponse() + // } if step := r.reconBasicAuth(req); !step.ShouldProceed() { return step.ReconcilerResponse() @@ -133,241 +119,241 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. return ctrl.Result{}, nil } -func (r *Reconciler) patchDefaults(req *reconciler.Request[*crdsv1.Router]) stepResult.Result { +func (r *Reconciler) finalize(req *reconciler.Request[*crdsv1.Router]) stepResult.Result { + check := reconciler.NewRunningCheck("finalizing", req) + if step := req.CleanupOwnedResources(check); !step.ShouldProceed() { + return step + } + + return req.Finalize() +} + +func (r *Reconciler) findClusterIssuer(req *reconciler.Request[*crdsv1.Router]) (*certmanagerv1.ClusterIssuer, error) { ctx, obj := req.Context(), req.Object - check := reconciler.NewRunningCheck(DefaultsPatched, req) + https := obj.Spec.Https - hasUpdate := false + if https != nil && https.ClusterIssuer != "" { + var issuer certmanagerv1.ClusterIssuer + if err := r.Get(ctx, fn.NN("", https.ClusterIssuer), &issuer, &client.GetOptions{}); err != nil { + return nil, err + } - if obj.Spec.BasicAuth != nil && obj.Spec.BasicAuth.Enabled && obj.Spec.BasicAuth.SecretName == "" { - hasUpdate = true - obj.Spec.BasicAuth.SecretName = obj.Name + "-basic-auth" + return &issuer, nil } - if obj.Spec.IngressClass == "" { - hasUpdate = true - obj.Spec.IngressClass = r.Env.DefaultIngressClass + var issuerList certmanagerv1.ClusterIssuerList + if err := r.List(ctx, &issuerList, &client.ListOptions{Limit: 1}); err != nil { + return nil, nil } - if hasUpdate { - if err := r.Update(ctx, obj); err != nil { - return check.Failed(err) - } - return req.Done().RequeueAfter(1 * time.Second) + if len(issuerList.Items) != 1 { + return nil, fmt.Errorf("no cluster issuer found") } - return check.Completed() + return &issuerList.Items[0], nil } -func (r *Reconciler) finalize(req *reconciler.Request[*crdsv1.Router]) stepResult.Result { - check := reconciler.NewRunningCheck("finalizing", req) - if step := req.CleanupOwnedResources(check); !step.ShouldProceed() { - return step - } - - return req.Finalize() -} +func (r *Reconciler) findIngressClass(req *reconciler.Request[*crdsv1.Router]) (string, error) { + ctx, obj := req.Context(), req.Object -func genTLSCertName(domain string) string { - return fmt.Sprintf("%s-tls", domain) -} + if obj.Spec.IngressClass != "" { + return obj.Spec.IngressClass, nil + } -func (r *Reconciler) getRouterClusterIssuer(obj *crdsv1.Router) string { - https := obj.Spec.Https + var ingressClassList networkingv1.IngressClassList + if err := r.List(ctx, &ingressClassList, &client.ListOptions{Limit: 1}); err != nil { + return "", err + } - if https != nil && https.ClusterIssuer != "" { - return https.ClusterIssuer + if len(ingressClassList.Items) != 1 { + return "", fmt.Errorf("no/multiple ingress classes found") } - return r.Env.DefaultClusterIssuer + return ingressClassList.Items[0].Name, nil } func isHttpsEnabled(obj *crdsv1.Router) bool { return obj.Spec.Https != nil && obj.Spec.Https.Enabled } -func (r *Reconciler) EnsuringHttpsCerts(req *reconciler.Request[*crdsv1.Router]) stepResult.Result { +func (r *Reconciler) reconBasicAuth(req *reconciler.Request[*crdsv1.Router]) stepResult.Result { ctx, obj := req.Context(), req.Object - check := reconciler.NewRunningCheck(EnsuringHttpsCertsIfEnabled, req) + check := reconciler.NewRunningCheck(SettingUpBasicAuthIfEnabled, req) - if !isHttpsEnabled(obj) { + if obj.Spec.BasicAuth == nil || !obj.Spec.BasicAuth.Enabled { return check.Completed() } - _, nonWildcardDomains, err := r.parseAndExtractDomains(req) - if err != nil { - return check.Failed(err) + if len(obj.Spec.BasicAuth.Username) == 0 { + return check.Failed(fmt.Errorf(".spec.basicAuth.username must be defined when .spec.basicAuth.enabled is set to true")).Err(nil) } - for _, domain := range nonWildcardDomains { + if obj.Spec.BasicAuth.SecretName == "" { + obj.Spec.BasicAuth.SecretName = obj.Name + "-basic-auth" + if err := r.Update(ctx, obj); err != nil { + return check.Failed(err) + } + return check.StillRunning(fmt.Errorf("waiting for router reconcilation")) + } - tlsCertLabel := fmt.Sprintf("kloudlite.io/tls-cert.%s", fn.Md5([]byte(genTLSCertName(domain)))) - if v, ok := obj.Labels[tlsCertLabel]; !ok || v != "true" { - fn.MapSet(&obj.Labels, tlsCertLabel, "true") - if err := r.Update(ctx, obj); err != nil { - return check.StillRunning(err) - } + basicAuthScrt := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: obj.Spec.BasicAuth.SecretName, Namespace: obj.Namespace}, Type: "Opaque"} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, basicAuthScrt, func() error { + basicAuthScrt.SetOwnerReferences([]metav1.OwnerReference{fn.AsOwner(obj, true)}) + if _, ok := basicAuthScrt.Data["password"]; ok { + return nil } - cert, err := reconciler.Get(ctx, r.Client, fn.NN(r.Env.CertificateNamespace, genTLSCertName(domain)), &certmanagerv1.Certificate{}) + password := fn.CleanerNanoid(48) + ePass, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) if err != nil { - if !apiErrors.IsNotFound(err) { - return check.StillRunning(err) - } - cert = nil + return err + } + basicAuthScrt.StringData = map[string]string{ + "auth": fmt.Sprintf("%s:%s", obj.Spec.BasicAuth.Username, ePass), + "username": obj.Spec.BasicAuth.Username, + "password": password, } + return nil + }); err != nil { + return check.StillRunning(err) + } - if cert == nil { - cert := &certmanagerv1.Certificate{ - TypeMeta: metav1.TypeMeta{ - Kind: "Certificate", - APIVersion: certmanagerv1.SchemeGroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: genTLSCertName(domain), - Namespace: r.Env.CertificateNamespace, - Labels: map[string]string{ - certCreatedByRouter: obj.Name, - }, - }, - Spec: certmanagerv1.CertificateSpec{ - DNSNames: []string{domain}, - IssuerRef: certmanagermetav1.ObjectReference{ - Name: r.getRouterClusterIssuer(obj), - Kind: "ClusterIssuer", - Group: certmanagerv1.SchemeGroupVersion.Group, - }, - RenewBefore: &metav1.Duration{ - Duration: 15 * 24 * time.Hour, // 15 days prior - }, - SecretName: genTLSCertName(domain), - Usages: []certmanagerv1.KeyUsage{ - certmanagerv1.UsageDigitalSignature, - certmanagerv1.UsageKeyEncipherment, - }, - }, - } - if err := r.Create(ctx, cert); err != nil { - return check.StillRunning(err) + req.AddToOwnedResources(reconciler.ParseResourceRef(basicAuthScrt)) + + return check.Completed() +} + +func groupHostsByKind(issuer *certmanagerv1.ClusterIssuer, obj *crdsv1.Router) (wildcardHosts []string, nonWildcardHosts []string) { + var dnsNames []string + + for _, solver := range issuer.Spec.ACME.Solvers { + if solver.DNS01 != nil { + if solver.Selector != nil { + dnsNames = append(dnsNames, solver.Selector.DNSNames...) } } + } - if _, err := IsHttpsCertReady(cert); err != nil { - return check.StillRunning(err).NoRequeue() - // return check.StillRunning(err) - // return check.StillRunning(err).RequeueAfter(1 * time.Second) + wcFilter := map[string]struct{}{} + for _, pattern := range dnsNames { + if strings.HasPrefix(pattern, "*.") { + wcFilter[pattern[len("*."):]] = struct{}{} + continue } + wcFilter[pattern] = struct{}{} + } - certSecret, err := reconciler.Get(ctx, r.Client, fn.NN(r.Env.CertificateNamespace, genTLSCertName(domain)), &corev1.Secret{}) - if err != nil { - return check.StillRunning(err) + for _, route := range obj.Spec.Routes { + if _, ok := wcFilter[route.Host]; ok { + wildcardHosts = append(wildcardHosts, route.Host) + continue } - copyTLSSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: genTLSCertName(domain), Namespace: obj.Namespace}, Type: corev1.SecretTypeTLS} - if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, copyTLSSecret, func() error { - if copyTLSSecret.Annotations == nil { - copyTLSSecret.Annotations = make(map[string]string, 1) - } - copyTLSSecret.Annotations["kloudlite.io/secret.cloned-by"] = "router" + idx := strings.IndexByte(route.Host, '.') + if idx == -1 { + nonWildcardHosts = append(nonWildcardHosts, route.Host) + continue + } - copyTLSSecret.Data = certSecret.Data - copyTLSSecret.StringData = certSecret.StringData - return nil - }); err != nil { - return check.StillRunning(err) + if _, ok := wcFilter[route.Host[idx+1:]]; ok { + wildcardHosts = append(wildcardHosts, route.Host) + continue } + + nonWildcardHosts = append(nonWildcardHosts, route.Host) } - return check.Completed() + return wildcardHosts, nonWildcardHosts } -func (r *Reconciler) reconBasicAuth(req *reconciler.Request[*crdsv1.Router]) stepResult.Result { +func (r *Reconciler) ensureIngresses(req *reconciler.Request[*crdsv1.Router]) stepResult.Result { ctx, obj := req.Context(), req.Object - check := reconciler.NewRunningCheck(SettingUpBasicAuthIfEnabled, req) + check := reconciler.NewRunningCheck(CreateIngressResource, req) - if obj.Spec.BasicAuth != nil && obj.Spec.BasicAuth.Enabled { - if len(obj.Spec.BasicAuth.Username) == 0 { - return check.Failed(fmt.Errorf(".spec.basicAuth.username must be defined when .spec.basicAuth.enabled is set to true")).Err(nil) + if obj.Spec.IngressClass == "" { + ingClass, err := r.findIngressClass(req) + if err != nil { + return check.Failed(err) } - basicAuthScrt := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: obj.Spec.BasicAuth.SecretName, Namespace: obj.Namespace}, Type: "Opaque"} - if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, basicAuthScrt, func() error { - basicAuthScrt.SetOwnerReferences([]metav1.OwnerReference{fn.AsOwner(obj, true)}) - if _, ok := basicAuthScrt.Data["password"]; ok { - return nil - } - - password := fn.CleanerNanoid(48) - ePass, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) - if err != nil { - return err - } - basicAuthScrt.Data = map[string][]byte{ - "auth": []byte(fmt.Sprintf("%s:%s", obj.Spec.BasicAuth.Username, ePass)), - "username": []byte(obj.Spec.BasicAuth.Username), - "password": []byte(password), - } - return nil - }); err != nil { - return check.StillRunning(err) + obj.Spec.IngressClass = ingClass + if err := r.Update(ctx, obj); err != nil { + return check.Failed(err) } - req.AddToOwnedResources(reconciler.ParseResourceRef(basicAuthScrt)) + return check.StillRunning(fmt.Errorf("updating .spec.ingressClass")) } - return check.Completed() -} + if obj.Spec.Https != nil && obj.Spec.Https.ClusterIssuer == "" { + issuer, err := r.findClusterIssuer(req) + if err != nil { + return check.Failed(err) + } -func (r *Reconciler) ensureIngresses(req *reconciler.Request[*crdsv1.Router]) stepResult.Result { - ctx, obj := req.Context(), req.Object - check := reconciler.NewRunningCheck(CreatingIngressResources, req) + obj.Spec.Https.ClusterIssuer = issuer.Name + if err := r.Update(ctx, obj); err != nil { + return check.Failed(err) + } + + return check.StillRunning(fmt.Errorf("updating .spec.https.clusterIssuer")) + } + + if len(obj.Spec.Routes) == 0 { + return check.Completed() + } - wcDomains, nonWcDomains, err := r.parseAndExtractDomains(req) + issuer, err := r.findClusterIssuer(req) if err != nil { - return check.Failed(err).Err(nil) + return check.Failed(err) } - nginxIngressAnnotations := GenNginxIngressAnnotations(obj) + wcHosts, nonWcHosts := groupHostsByKind(issuer, obj) - if len(obj.Spec.Routes) > 0 { - b, err := templates.ParseBytes( - r.templateIngress, map[string]any{ - "name": obj.Name, - "namespace": obj.Namespace, - - "owner-refs": []metav1.OwnerReference{fn.AsOwner(obj, true)}, - "labels": obj.GetLabels(), - "annotations": nginxIngressAnnotations, - - "non-wildcard-domains": nonWcDomains, - "wildcard-domains": wcDomains, - "router-domains": obj.Spec.Domains, - - "ingress-class": obj.Spec.IngressClass, - "cluster-issuer": func() string { - if obj.Spec.Https != nil && obj.Spec.Https.ClusterIssuer != "" { - return obj.Spec.Https.ClusterIssuer - } - return r.Env.DefaultClusterIssuer - }(), - - "routes": obj.Spec.Routes, - - "is-https-enabled": isHttpsEnabled(obj), - }, - ) - if err != nil { - return check.Failed(err).Err(nil) - } + nginxIngressAnnotations := GenNginxIngressAnnotations(obj) - rr, err := r.YAMLClient.ApplyYAML(ctx, b) - if err != nil { - return check.StillRunning(err) - } + // b, err := templates.ParseBytes(r.templateIngress, templates.IngressTemplateArgs{ + // Metadata: metav1.ObjectMeta{ + // Name: obj.Name, + // Namespace: obj.Namespace, + // Labels: obj.GetLabels(), + // Annotations: nginxIngressAnnotations, + // }, + // IngressClassName: obj.Spec.IngressClass, + // HttpsEnabled: isHttpsEnabled(obj), + // WildcardDomains: wcDomains, + // NonWildcardDomains: nonWcDomains, + // Routes: obj.Spec.Routes, + // }) + + b, err := templates.ParseBytes( + r.templateIngress, map[string]any{ + "name": obj.Name, + "namespace": obj.Namespace, + + "owner-refs": []metav1.OwnerReference{fn.AsOwner(obj, true)}, + "labels": obj.GetLabels(), + "annotations": nginxIngressAnnotations, + + "non-wildcard-domains": nonWcHosts, + "wildcard-domains": wcHosts, + "ingress-class": obj.Spec.IngressClass, + + "routes": obj.Spec.Routes, + + "is-https-enabled": isHttpsEnabled(obj), + }, + ) + if err != nil { + return check.Failed(err).Err(nil) + } - req.AddToOwnedResources(rr...) + rr, err := r.YAMLClient.ApplyYAML(ctx, b) + if err != nil { + return check.StillRunning(err) } + req.AddToOwnedResources(rr...) + return check.Completed() } @@ -376,11 +362,11 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { r.Scheme = mgr.GetScheme() if r.YAMLClient == nil { - return fmt.Errorf("r.YAMLClient must be set") + return fmt.Errorf(".YAMLClient must be set") } var err error - r.templateIngress, err = templates.ReadIngressTemplate() + r.templateIngress, err = templates.Read(templates.IngressTemplate) if err != nil { return err } @@ -405,6 +391,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { return rr })) + // builder.Owns(&certmanagerv1.Certificate{}) builder.WithEventFilter(reconciler.ReconcileFilter()) diff --git a/operators/routers/internal/router-controller/controller_test.go b/operators/routers/internal/router-controller/controller_test.go index 047ac4ef..604386d3 100644 --- a/operators/routers/internal/router-controller/controller_test.go +++ b/operators/routers/internal/router-controller/controller_test.go @@ -29,10 +29,10 @@ func newRouter() crdsv1.Router { }, // RateLimit: crdsv1.RateLimit{}, // MaxBodySizeInMB: 0, - Domains: []string{"sample.example.com"}, + Routes: []string{"sample.example.com"}, Routes: []crdsv1.Route{ { - App: "example", + Service: "example", Path: "/", Port: 80, Rewrite: false, @@ -127,14 +127,14 @@ var _ = Describe("router controller [UPDATE] says", func() { It("adding a new domain, reflects in each of the owned k8s ingresses", func() { _, err := controllerutil.CreateOrUpdate(Suite.Context, Suite.K8sClient, &routerCr, func() error { - routerCr.Spec.Domains = append(routerCr.Spec.Domains, "dummy.example.com") + routerCr.Spec.Routes = append(routerCr.Spec.Routes, "dummy.example.com") return nil }) Expect(err).NotTo(HaveOccurred()) - dMap := make(map[string]bool, len(routerCr.Spec.Domains)) - for i := range routerCr.Spec.Domains { - dMap[routerCr.Spec.Domains[i]] = false + dMap := make(map[string]bool, len(routerCr.Spec.Routes)) + for i := range routerCr.Spec.Routes { + dMap[routerCr.Spec.Routes[i]] = false } Promise(func(g Gomega) { @@ -156,7 +156,7 @@ var _ = Describe("router controller [UPDATE] says", func() { It("adding a new route, reflects in each of the owned k8s ingresses", func() { _, err := controllerutil.CreateOrUpdate(Suite.Context, Suite.K8sClient, &routerCr, func() error { routerCr.Spec.Routes = append(routerCr.Spec.Routes, crdsv1.Route{ - App: "ginkgo-test", + Service: "ginkgo-test", Path: "/.kl/test", Port: 80, }) diff --git a/operators/routers/internal/router-controller/helpers.go b/operators/routers/internal/router-controller/helpers.go index 20131281..1c098741 100644 --- a/operators/routers/internal/router-controller/helpers.go +++ b/operators/routers/internal/router-controller/helpers.go @@ -6,45 +6,8 @@ import ( certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" certmanagermetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" - crdsv1 "github.com/kloudlite/operator/apis/crds/v1" - fn "github.com/kloudlite/operator/pkg/functions" - "github.com/kloudlite/operator/toolkit/reconciler" - apiErrors "k8s.io/apimachinery/pkg/api/errors" ) -func (r *Reconciler) parseAndExtractDomains(req *reconciler.Request[*crdsv1.Router]) ([]string, []string, error) { - ctx, obj := req.Context(), req.Object - - var wildcardPatterns []string - - if obj.Spec.Https != nil && obj.Spec.Https.Enabled { - issuerName := r.getRouterClusterIssuer(obj) - - if issuerName == "" { - return nil, nil, fmt.Errorf("no cluster issuer found, could not proceed, when https is enabled") - } - - clusterIssuer, err := reconciler.Get(ctx, r.Client, fn.NN("", issuerName), &certmanagerv1.ClusterIssuer{}) - if err != nil { - if !apiErrors.IsNotFound(err) { - return nil, nil, err - } - clusterIssuer = nil - } - - if clusterIssuer != nil { - for _, solver := range clusterIssuer.Spec.ACME.Solvers { - if solver.DNS01 != nil { - wildcardPatterns = solver.Selector.DNSNames - } - } - } - } - - wildcardDomains, nonWildcardDomains := FilterDomains(wildcardPatterns, obj.Spec.Domains) - return wildcardDomains, nonWildcardDomains, nil -} - func FilterDomains(wildcardPatterns []string, domains []string) (wildcardDomains, nonWildcardDomains []string) { wildcardBases := map[string]struct{}{} for _, pattern := range wildcardPatterns { diff --git a/operators/routers/internal/templates/embed.go b/operators/routers/internal/templates/embed.go index f17e7d40..dd421b45 100644 --- a/operators/routers/internal/templates/embed.go +++ b/operators/routers/internal/templates/embed.go @@ -9,8 +9,15 @@ import ( //go:embed * var templatesDir embed.FS -func ReadIngressTemplate() ([]byte, error) { - return templatesDir.ReadFile("ingress-resource.yml.tpl") +type templateFile string + +const ( + // IngressTemplate templateFile = "./ingress-resource-v2.yml.tpl" + IngressTemplate templateFile = "ingress-resource.yml.tpl" +) + +func Read(t templateFile) ([]byte, error) { + return templatesDir.ReadFile(string(t)) } var ParseBytes = templates.ParseBytes diff --git a/operators/routers/internal/templates/ingress-resource-v2.yml.tpl b/operators/routers/internal/templates/ingress-resource-v2.yml.tpl new file mode 100644 index 00000000..da6ea199 --- /dev/null +++ b/operators/routers/internal/templates/ingress-resource-v2.yml.tpl @@ -0,0 +1,45 @@ +{{ with . }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: {{.Metadata | toJson}} +spec: + ingressClassName: {{.IngressClassName}} + {{- if .HttpsEnabled }} + tls: + {{- range $v := .NonWildcardDomains }} + - hosts: + - {{$v | squote}} + secretName: {{$v}}-tls + {{- end}} + + {{- range $v := .WildcardDomains }} + - hosts: + - {{$v | squote}} + {{- end }} + {{- end}} + + rules: + {{- range $host, $routes := .Routes }} + - host: {{$host}} + http: + paths: + {{- range $route := $routes }} + - pathType: Prefix + backend: + service: + name: {{$route.App}} + port: + number: {{$route.Port}} + + path: {{ if not hasPrefix "/" $route.Path }}/{{end}}{{$route.Path}} + ({{if hasPrefix "/" $route.Path }}{{substr 1 $x $route.Path}}{{else}}{{$route.Path}}{{end}}.*) + + {{- if $route.Rewrite }} + path: {{$route.Path}}?(.*) + {{- else }} + {{ $x := len $route.Path }} + path: /({{if hasPrefix "/" $route.Path }}{{substr 1 $x $route.Path}}{{else}}{{$route.Path}}{{end}}.*) + {{- end}} + {{- end}} + {{- end }} +{{- end }} diff --git a/operators/routers/internal/templates/ingress-resource.yml.tpl b/operators/routers/internal/templates/ingress-resource.yml.tpl index 30e4c3ae..e7099624 100644 --- a/operators/routers/internal/templates/ingress-resource.yml.tpl +++ b/operators/routers/internal/templates/ingress-resource.yml.tpl @@ -6,11 +6,9 @@ {{- $annotations := get . "annotations" | default dict }} {{- $nonWildcardDomains := get . "non-wildcard-domains" }} -{{- $routerDomains := get . "router-domains" }} {{- $wildcardDomains := get . "wildcard-domains"}} {{- $ingressClass := get . "ingress-class" }} -{{- $clusterIssuer := get . "cluster-issuer" }} {{- $routes := get . "routes" }} @@ -42,24 +40,22 @@ spec: {{- end}} rules: - {{- range $domain := $routerDomains }} - - host: {{$domain}} + {{- range $_, $route := $routes }} + - host: {{$route.Host}} http: paths: - {{- range $route := $routes }} - - pathType: Prefix + {{- /* - pathType: Prefix */}} + - pathType: ImplementationSpecific backend: service: - name: {{$route.App}} + name: {{$route.Service}} port: number: {{$route.Port}} {{- if $route.Rewrite }} path: {{$route.Path}}?(.*) {{- else }} - {{ $x := len $route.Path }} - path: /({{if hasPrefix "/" $route.Path }}{{substr 1 $x $route.Path}}{{else}}{{$route.Path}}{{end}}.*) + path: /({{substr 1 (len $route.Path) $route.Path}}.*) {{- end}} - {{- end}} {{- end }} diff --git a/operators/routers/internal/templates/types.go b/operators/routers/internal/templates/types.go new file mode 100644 index 00000000..225824d9 --- /dev/null +++ b/operators/routers/internal/templates/types.go @@ -0,0 +1,18 @@ +package templates + +import ( + crdsv1 "github.com/kloudlite/operator/apis/crds/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type IngressTemplateArgs struct { + Metadata metav1.ObjectMeta + IngressClassName string + + HttpsEnabled bool + + WildcardDomains []string + NonWildcardDomains []string + + Routes map[string][]crdsv1.Route +} diff --git a/operators/wireguard/examples/device.yaml b/operators/wireguard/examples/device.yaml index 26f66603..393025de 100644 --- a/operators/wireguard/examples/device.yaml +++ b/operators/wireguard/examples/device.yaml @@ -3,7 +3,7 @@ kind: Device metadata: name: device-sample spec: - serverName: server-sample + # serverName: server-sample offset: 1 ports: - port: 3000 diff --git a/operators/wireguard/main.go b/operators/wireguard/main.go index 660d844c..723dc058 100644 --- a/operators/wireguard/main.go +++ b/operators/wireguard/main.go @@ -5,7 +5,7 @@ import ( "github.com/kloudlite/operator/operator" "github.com/kloudlite/operator/operators/wireguard/internal/controllers/device" - cc "github.com/kloudlite/operator/operators/wireguard/internal/controllers/global-vpn" + // cc "github.com/kloudlite/operator/operators/wireguard/internal/controllers/global-vpn" "github.com/kloudlite/operator/operators/wireguard/internal/env" ) @@ -17,7 +17,7 @@ func main() { mgr.RegisterControllers( &device.Reconciler{Name: "Device", Env: ev}, - &cc.Reconciler{Name: "GlobalVPN", Env: ev}, + // &cc.Reconciler{Name: "GlobalVPN", Env: ev}, ) mgr.Start() diff --git a/operators/workmachine/examples/workmachine.yml b/operators/workmachine/examples/workmachine.yml new file mode 100644 index 00000000..b45b7969 --- /dev/null +++ b/operators/workmachine/examples/workmachine.yml @@ -0,0 +1,18 @@ +apiVersion: crds.kloudlite.io/v1 +kind: WorkMachine +metadata: + name: sample +spec: + state: "ON" + sshPublicKeys: [] + jobParams: + nodeSelector: + kubernetes.io/hostname: "master-1" + aws: + region: "ap-south-1" + availabilityZone: "ap-south-1a" + publicSubnetID: "subnet-0e4f0634ba1b5be2e" + ami: "ami-05c179eced2eb9b5b" + instanceType: "t3.medium" + rootVolumeSize: 50 + externalVolumeSize: 100 diff --git a/operators/workmachine/internal/controllers/workmachine/controller.go b/operators/workmachine/internal/controllers/workmachine/controller.go new file mode 100644 index 00000000..5e92beb6 --- /dev/null +++ b/operators/workmachine/internal/controllers/workmachine/controller.go @@ -0,0 +1,421 @@ +package workmachine + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "k8s.io/client-go/tools/record" + + ct "github.com/kloudlite/operator/apis/common-types" + crdsv1 "github.com/kloudlite/operator/apis/crds/v1" + "github.com/kloudlite/operator/operators/workmachine/internal/env" + "github.com/kloudlite/operator/operators/workmachine/internal/templates" + "github.com/kloudlite/operator/pkg/constants" + "github.com/kloudlite/operator/pkg/ssh" + fn "github.com/kloudlite/operator/toolkit/functions" + "github.com/kloudlite/operator/toolkit/kubectl" + rApi "github.com/kloudlite/operator/toolkit/reconciler" + step_result "github.com/kloudlite/operator/toolkit/reconciler/step-result" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/yaml" +) + +type Reconciler struct { + client.Client + Scheme *runtime.Scheme + Env *env.Env + + YAMLClient kubectl.YAMLClient + recorder record.EventRecorder + + workmachineLifecycleTemplateSpec []byte + templateWebhook []byte + templateJumpServerDeploymentSpec []byte +} + +func (r *Reconciler) GetName() string { + return "workmachine" +} + +const ( + createWorkMachineJob string = "create-work-machine-job" + createTargetNamespace string = "create-target-namespace" + createSSHPublicKeysSecret string = "create-ssh-public-keys-secret" + createMachinePublicPrivateKeyPair string = "create-machine-public-private-key-pair" + createSSHJumpServerDeployment string = "create-ssh-jumpserver-deployment" +) + +const ( + sshPublicKeysSecretName string = "ssh-public-keys" + authorizedKeysSecretKey string = "authorized_keys" +) + +const ( + jobRefAnnotation string = "kloudlite.io/workmachine.job-ref" +) + +// +kubebuilder:rbac:groups=crds.kloudlite.io,resources=apps,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=crds.kloudlite.io,resources=apps/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=crds.kloudlite.io,resources=apps/finalizers,verbs=update + +func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { + req, err := rApi.NewRequest(ctx, r.Client, request.NamespacedName, &crdsv1.WorkMachine{}) + if err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + req.PreReconcile() + defer req.PostReconcile() + + req.Logger.Debug("RECONCILATION starting ...") + + if req.Object.GetDeletionTimestamp() != nil { + if x := r.finalize(req); !x.ShouldProceed() { + return x.ReconcilerResponse() + } + return ctrl.Result{}, nil + } + + if step := req.ClearStatusIfAnnotated(); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + + if step := req.EnsureCheckList([]rApi.CheckMeta{ + {Name: createWorkMachineJob, Title: "Creates WorkMachine creation job"}, + {Name: createTargetNamespace, Title: "Creates a target namespace for workmachine"}, + {Name: createSSHPublicKeysSecret, Title: "store SSH public keys in a secret"}, + }); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + + if step := req.EnsureLabelsAndAnnotations(); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + + if step := req.EnsureFinalizers(rApi.ForegroundFinalizer, rApi.CommonFinalizer); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + + if step := r.createWorkMachineCreationJob(req); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + + if step := r.createTargetNamespace(req); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + + if step := r.createSSHPublicKeysSecret(req); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + + // if step := r.createSSHJumpServer(req); !step.ShouldProceed() { + // return step.ReconcilerResponse() + // } + + req.Object.Status.IsReady = true + return ctrl.Result{}, nil +} + +func (r *Reconciler) finalize(req *rApi.Request[*crdsv1.WorkMachine]) step_result.Result { + if step := req.EnsureCheckList([]rApi.CheckMeta{ + {Name: "uninstall workmachine"}, + }); !step.ShouldProceed() { + return step + } + + check := rApi.NewRunningCheck("uninstall workmachine", req) + + if step := req.CleanupOwnedResources(check); !step.ShouldProceed() { + return step + } + + return req.Finalize() +} + +type ClusterParams struct { + K3sServerHost string `json:"k3s_server_host"` + + K3sServerToken string `json:"k3s_server_token"` + K3sAgentToken string `json:"k3s_agent_token"` + + K3sVersion string `json:"k3s_version"` + + AwsVPCName string `json:"aws_vpc_name"` + AwsVPCId string `json:"aws_vpc_id"` + AwsPublicSubnet string `json:"aws_public_subnet"` + AwsRegion string `json:"aws_region"` + AwsAvailblityZone string `json:"aws_availblity_zone"` + + AwsNLBDNSHost string `json:"aws_nlb_dns_host"` + + AwsSecurityGroupIDs []string `json:"aws_security_group_ids"` + AwsIAMInstanceProfileName string `json:"aws_iam_instance_profile_name"` +} + +func (r *Reconciler) parseSpecIntoTFValues(ctx context.Context, obj *crdsv1.WorkMachine) ([]byte, error) { + sp := strings.Split(r.Env.K3sParamsSecretRef, "/") + if len(sp) != 2 { + return nil, fmt.Errorf("invalid k3s params secret ref must be a valid / format") + } + + secret, err := rApi.Get(ctx, r.Client, fn.NN(sp[0], sp[1]), &corev1.Secret{}) + if err != nil { + return nil, err + } + + fmt.Printf("cluster-params.yml: \n---\n%s\n---\n", string(secret.Data["cluster-params.yml"])) + + cp := ClusterParams{} + if err := yaml.Unmarshal(secret.Data["cluster-params.yml"], &cp); err != nil { + return nil, err + } + + fmt.Printf("cluster params: %+v\n", cp) + + switch obj.Spec.GetCloudProvider() { + case ct.CloudProviderAWS: + { + return json.Marshal(map[string]any{ + "aws_region": cp.AwsRegion, + "trace_id": "workmachine-" + obj.Name, + "vpc_id": cp.AwsVPCId, + "name": obj.Name, + "k3s_server_host": cp.K3sServerHost, + "k3s_agent_token": cp.K3sAgentToken, + "k3s_version": cp.K3sVersion, + "ami": obj.Spec.AWSMachineConfig.AMI, + "instance_type": obj.Spec.AWSMachineConfig.InstanceType, + "instance_state": func() string { + if obj.Spec.State == crdsv1.WorkMachineStateOn { + return "running" + } + + return "stopped" + }(), + "availability_zone": cp.AwsAvailblityZone, + // "iam_instance_profile": func() string { + // if obj.Spec.AWSMachineConfig.IAMInstanceProfileRole != nil { + // return *obj.Spec.AWSMachineConfig.IAMInstanceProfileRole + // } + // return cp.AwsIAMInstanceProfileName + // }(), + "root_volume_size": obj.Spec.AWSMachineConfig.RootVolumeSize, + "root_volume_type": obj.Spec.AWSMachineConfig.RootVolumeType, + "security_group_ids": cp.AwsSecurityGroupIDs, + "subnet_id": cp.AwsPublicSubnet, + }) + } + default: + return nil, fmt.Errorf("unsupported cloud provider (%s)", obj.Spec.GetCloudProvider()) + } +} + +func (r *Reconciler) createWorkMachineCreationJob(req *rApi.Request[*crdsv1.WorkMachine]) step_result.Result { + ctx, obj := req.Context(), req.Object + check := rApi.NewRunningCheck(createWorkMachineJob, req) + + jobName := fmt.Sprintf("wm-%s", obj.Name) + + if v, ok := obj.Annotations[jobRefAnnotation]; !ok || v != jobName { + fn.MapSet(&obj.Annotations, jobRefAnnotation, jobName) + if err := r.Update(ctx, obj); err != nil { + return check.Failed(err) + } + } + + varfileJSON, err := r.parseSpecIntoTFValues(ctx, obj) + if err != nil { + return check.Failed(err) + } + + b, err := templates.ParseBytes(r.workmachineLifecycleTemplateSpec, templates.WorkMachineLifecycleVars{ + JobMetadata: metav1.ObjectMeta{ + Name: jobName, + Namespace: r.Env.IACJobsNamespace, + Labels: fn.MapFilterWithPrefix(obj.GetLabels(), "kloudlite.io/"), + Annotations: fn.FilterObservabilityAnnotations(obj.GetAnnotations()), + OwnerReferences: []metav1.OwnerReference{fn.AsOwner(obj, true)}, + }, + NodeSelector: obj.Spec.JobParams.NodeSelector, + Tolerations: obj.Spec.JobParams.Tolerations, + JobImage: r.Env.IACJobImage, + TFWorkspaceName: obj.Name, + TfWorkspaceNamespace: r.Env.TFStateSecretNamespace, + CloudProvider: obj.Spec.GetCloudProvider().String(), + ValuesJSON: string(varfileJSON), + + OutputSecretName: obj.Name + "-tf-outputs", + OutputSecretNamespace: r.Env.IACJobsNamespace, + + NodeName: obj.Name, + }) + if err != nil { + return check.Failed(err) + } + + lf := &crdsv1.Lifecycle{ObjectMeta: metav1.ObjectMeta{Name: jobName, Namespace: r.Env.IACJobsNamespace}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, lf, func() error { + lf.SetLabels(fn.MapMerge(fn.MapFilterWithPrefix(obj.GetLabels(), "kloudlite.io/"), lf.GetLabels())) + lf.SetAnnotations(fn.MapMerge(fn.MapFilterWithPrefix(obj.GetAnnotations(), "kloudlite.io/observability"), lf.GetAnnotations())) + lf.SetOwnerReferences([]metav1.OwnerReference{fn.AsOwner(obj, true)}) + return yaml.Unmarshal(b, &lf.Spec) + }); err != nil { + return check.Failed(err) + } + + if !lf.HasCompleted() { + return check.StillRunning(fmt.Errorf("waiting for lifecycle job to complete")) + } + + if lf.Status.Phase == crdsv1.JobPhaseFailed { + return check.Failed(fmt.Errorf("lifecycle job failed")) + } + + req.AddToOwnedResources(rApi.ParseResourceRef(lf)) + + return check.Completed() +} + +func (r *Reconciler) createTargetNamespace(req *rApi.Request[*crdsv1.WorkMachine]) step_result.Result { + ctx, obj := req.Context(), req.Object + check := rApi.NewRunningCheck(createTargetNamespace, req) + + hasUpdate := false + if obj.Spec.TargetNamespace == "" { + hasUpdate = true + obj.Spec.TargetNamespace = "wm-" + obj.Name + } + + if hasUpdate { + if err := r.Update(ctx, obj); err != nil { + return check.Failed(err) + } + + return check.StillRunning(fmt.Errorf("waiting for reconcilation")) + } + + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: obj.Spec.TargetNamespace}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, ns, func() error { + fn.MapSet(&ns.Annotations, "kloudlite.io/namespace.for", fmt.Sprintf("workmachine/%s", obj.Name)) + ns.SetOwnerReferences([]metav1.OwnerReference{fn.AsOwner(obj, true)}) + return nil + }); err != nil { + return check.Failed(err) + } + + return check.Completed() +} + +func (r *Reconciler) createSSHPublicKeysSecret(req *rApi.Request[*crdsv1.WorkMachine]) step_result.Result { + ctx, obj := req.Context(), req.Object + check := rApi.NewRunningCheck(createSSHPublicKeysSecret, req) + + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: sshPublicKeysSecretName, Namespace: obj.Spec.TargetNamespace}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, secret, func() error { + fn.MapSet(&secret.Annotations, "kloudlite.io/description", "this secret contains ssh public keys given by user in workmachine resource") + fn.MapSet(&secret.Annotations, "kloudlite.io/secret.for", fmt.Sprintf("workmachine/%s", obj.Name)) + secret.SetOwnerReferences([]metav1.OwnerReference{fn.AsOwner(obj, true)}) + + if secret.StringData == nil { + secret.StringData = make(map[string]string, 1) + } + + secret.StringData[authorizedKeysSecretKey] = strings.Join(obj.Spec.SSHPublicKeys, "\n") + return nil + }); err != nil { + return check.Failed(err) + } + + if secret.Data["private_key"] == nil || secret.Data["public_key"] == nil { + privateKeyPEM, publicKey, err := ssh.GenerateSSHKeyPair() + if err != nil { + return check.Failed(err) + } + + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, secret, func() error { + if secret.Data == nil { + secret.Data = make(map[string][]byte, 2) + } + secret.Data["public_key"] = publicKey + secret.Data["private_key"] = privateKeyPEM + return nil + }); err != nil { + return check.Failed(err) + } + } + + // if obj.Status.MachinePublicSSHKey == "" { + // obj.Status.MachinePublicSSHKey = string(secret.Data["public_key"]) + // if err := r.Status().Update(ctx, obj); err != nil { + // return check.Failed(err) + // } + // } + + return check.Completed() +} + +func (r *Reconciler) createSSHJumpServer(req *rApi.Request[*crdsv1.WorkMachine]) step_result.Result { + ctx, obj := req.Context(), req.Object + check := rApi.NewRunningCheck(createSSHJumpServerDeployment, req) + + sshJumpServerName := "ssh-jump-server" + + b, err := templates.ParseBytes(r.templateJumpServerDeploymentSpec, templates.JumpServerDeploymentSpecTemplateArgs{ + SSHAuthorizedKeysSecretName: sshPublicKeysSecretName, + SSHAuthorizedKeysSecretKey: authorizedKeysSecretKey, + SelectorLabels: map[string]string{ + "app": "jump-server", + }, + }) + if err != nil { + return check.Failed(err) + } + + deployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: sshJumpServerName, Namespace: obj.Spec.TargetNamespace}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { + deployment.SetOwnerReferences([]metav1.OwnerReference{fn.AsOwner(obj, true)}) + fn.MapSet(&deployment.Annotations, constants.DescriptionKey, "this deployment is a ssh jump server used to allow users to jump to different workspaces") + return yaml.Unmarshal(b, &deployment.Spec) + }); err != nil { + return check.Failed(err) + } + + return check.Completed() +} + +func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { + r.Client = mgr.GetClient() + r.Scheme = mgr.GetScheme() + + if r.YAMLClient == nil { + return fmt.Errorf("yaml client must be set") + } + + r.recorder = mgr.GetEventRecorderFor(r.GetName()) + + var err error + r.workmachineLifecycleTemplateSpec, err = templates.Read(templates.WorkMachineLifecycleTemplate) + if err != nil { + return err + } + + r.templateJumpServerDeploymentSpec, err = templates.Read(templates.JumpServerDeploymentSpec) + if err != nil { + return err + } + + builder := ctrl.NewControllerManagedBy(mgr).For(&crdsv1.WorkMachine{}) + builder.WithOptions(controller.Options{MaxConcurrentReconciles: r.Env.MaxConcurrentReconciles}) + builder.Owns(&crdsv1.Lifecycle{}) + builder.WithEventFilter(rApi.ReconcileFilter(r.recorder)) + return builder.Complete(r) +} diff --git a/operators/workmachine/internal/env/env.go b/operators/workmachine/internal/env/env.go new file mode 100644 index 00000000..1c120632 --- /dev/null +++ b/operators/workmachine/internal/env/env.go @@ -0,0 +1,25 @@ +package env + +import ( + "github.com/codingconcepts/env" +) + +type Env struct { + MaxConcurrentReconciles int `env:"MAX_CONCURRENT_RECONCILES"` + + K3sParamsSecretRef string `env:"K3S_PARAMS_SECRET_REF" required:"true"` + + IACJobsNamespace string `env:"IAC_JOBS_NAMESPACE" required:"true"` + IACJobImage string `env:"IAC_JOB_IMAGE" required:"true"` + + TFStateSecretNamespace string `env:"TF_STATE_SECRET_NAMESPACE" required:"true" default:"kloudlite"` +} + +func GetEnvOrDie() *Env { + var ev Env + if err := env.Set(&ev); err != nil { + panic(err) + } + + return &ev +} diff --git a/operators/workmachine/internal/templates/embed.go b/operators/workmachine/internal/templates/embed.go new file mode 100644 index 00000000..259ad4da --- /dev/null +++ b/operators/workmachine/internal/templates/embed.go @@ -0,0 +1,26 @@ +package templates + +import ( + "embed" + "path/filepath" + + "github.com/kloudlite/operator/pkg/templates" +) + +//go:embed * +var templatesDir embed.FS + +type templateFile string + +const ( + WorkspaceTemplate templateFile = "./workspace.yml.tpl" + WorkMachineLifecycleTemplate templateFile = "./workmachine-lifecycle.yml.tpl" + JumpServerDeploymentSpec templateFile = "./ssh-jumpserver-deployment-spec.yml.tpl" + AuthorizedKeysTemplate templateFile = "./ssh-jumpserver-authorized-keys.tpl" +) + +func Read(t templateFile) ([]byte, error) { + return templatesDir.ReadFile(filepath.Join(string(t))) +} + +var ParseBytes = templates.ParseBytes diff --git a/operators/workmachine/internal/templates/ssh-jumpserver-deployment-spec.yml.tpl b/operators/workmachine/internal/templates/ssh-jumpserver-deployment-spec.yml.tpl new file mode 100644 index 00000000..475a4eb4 --- /dev/null +++ b/operators/workmachine/internal/templates/ssh-jumpserver-deployment-spec.yml.tpl @@ -0,0 +1,26 @@ +{{- with . }} +replicas: 1 +selector: + matchLabels: {{ .SelectorLabels | toJson }} +template: + metadata: + labels: {{.SelectorLabels | toJson }} + spec: + containers: + - name: sshd-server + image: ghcr.io/kloudlite/hub/ssh-server + ports: + - containerPort: 22 # Internal port used by the image + volumeMounts: + - name: ssh-secret + mountPath: /home/kl/.ssh/authorized_keys + subPath: authorized_keys + + volumes: + - name: ssh-secret + secret: + secretName: {{.SSHAuthorizedKeysSecretName}} + items: + - key: "{{.SSHAuthorizedKeysSecretKey}}" + path: "authorized_keys" +{{- end }} diff --git a/operators/workmachine/internal/templates/types.go b/operators/workmachine/internal/templates/types.go new file mode 100644 index 00000000..8fd59180 --- /dev/null +++ b/operators/workmachine/internal/templates/types.go @@ -0,0 +1,31 @@ +package templates + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type WorkMachineLifecycleVars struct { + JobMetadata metav1.ObjectMeta + NodeSelector map[string]string + Tolerations []corev1.Toleration + JobImage string + + TFWorkspaceName string + TfWorkspaceNamespace string + + CloudProvider string + + ValuesJSON string + + OutputSecretName string + OutputSecretNamespace string + + NodeName string +} + +type JumpServerDeploymentSpecTemplateArgs struct { + SSHAuthorizedKeysSecretName string + SSHAuthorizedKeysSecretKey string + SelectorLabels map[string]string +} diff --git a/operators/workmachine/internal/templates/workmachine-lifecycle.yml.tpl b/operators/workmachine/internal/templates/workmachine-lifecycle.yml.tpl new file mode 100644 index 00000000..18d9f425 --- /dev/null +++ b/operators/workmachine/internal/templates/workmachine-lifecycle.yml.tpl @@ -0,0 +1,117 @@ +{{ with . }} +onApply: + backOffLimit: 1 + podSpec: + tolerations: &tolerations {{.Tolerations | toJson }} + nodeSelector: &node-selector {{.NodeSelector | toJson }} + + resources: + requests: + cpu: 500m + memory: 1000Mi + limits: + cpu: 500m + memory: 1000Mi + + containers: + - name: main + image: {{.JobImage}} + imagePullPolicy: Always + env: + - name: KUBE_IN_CLUSTER_CONFIG + value: "true" + + - name: KUBE_NAMESPACE + value: {{.TfWorkspaceNamespace | squote}} + command: + - bash + - -c + - |+ + set -o pipefail + set -o errexit + + eval $DECOMPRESS_CMD + + pushd "$TEMPLATES_DIR/{{.CloudProvider}}/work-machine" + + envsubst < state-backend.tf.tpl > state-backend.tf + + terraform init -reconfigure -no-color 2>&1 | tee /dev/termination-log + terraform workspace select --or-create {{.TFWorkspaceName}} + + cat > values.json <<'EOF' + {{.ValuesJSON}} + EOF + + terraform init -no-color 2>&1 | tee /dev/termination-log + terraform plan -parallelism=2 --var-file ./values.json -out=tfplan -no-color 2>&1 | tee /dev/termination-log + terraform apply -parallelism=2 -no-color tfplan 2>&1 | tee /dev/termination-log + + # terraform state pull | jq '.outputs' -r > outputs.json + cat > secret.yml < state-backend.tf + + terraform init -reconfigure -no-color 2>&1 | tee /dev/termination-log + terraform workspace select --or-create {{.TFWorkspaceName}} + + cat > values.json <<'EOF' + {{.ValuesJSON}} + EOF + + terraform init -no-color 2>&1 | tee /dev/termination-log + terraform plan -parallelism=2 --destroy --var-file ./values.json -out=tfplan -no-color 2>&1 | tee /dev/termination-log + terraform apply -parallelism=2 -no-color tfplan 2>&1 | tee /dev/termination-log + + kubectl delete secret/{{.OutputSecretName}} -n {{.OutputSecretNamespace}} --ignore-not-found=true + + kubectl delete node/{{.NodeName}} + restartPolicy: Never + +{{ end }} diff --git a/operators/workmachine/main.go b/operators/workmachine/main.go new file mode 100644 index 00000000..637dbd15 --- /dev/null +++ b/operators/workmachine/main.go @@ -0,0 +1,15 @@ +package main + +import ( + crdsv1 "github.com/kloudlite/operator/apis/crds/v1" + "github.com/kloudlite/operator/operators/workmachine/register" + "github.com/kloudlite/operator/toolkit/operator" +) + +func main() { + mgr := operator.New("workmachine") + mgr.AddToSchemes(crdsv1.AddToScheme) + + register.RegisterInto(mgr) + mgr.Start() +} diff --git a/operators/workmachine/register/register.go b/operators/workmachine/register/register.go new file mode 100644 index 00000000..2b2f8921 --- /dev/null +++ b/operators/workmachine/register/register.go @@ -0,0 +1,16 @@ +package register + +import ( + crdsv1 "github.com/kloudlite/operator/apis/crds/v1" + "github.com/kloudlite/operator/operators/workmachine/internal/controllers/workmachine" + "github.com/kloudlite/operator/operators/workmachine/internal/env" + "github.com/kloudlite/operator/toolkit/operator" +) + +func RegisterInto(mgr operator.Operator) { + ev := env.GetEnvOrDie() + mgr.AddToSchemes(crdsv1.AddToScheme) + mgr.RegisterControllers( + &workmachine.Reconciler{Env: ev, YAMLClient: mgr.Operator().KubeYAMLClient()}, + ) +} diff --git a/operators/workspace/examples/sample.yml b/operators/workspace/examples/sample.yml new file mode 100644 index 00000000..76b7d235 --- /dev/null +++ b/operators/workspace/examples/sample.yml @@ -0,0 +1,23 @@ +apiVersion: crds.kloudlite.io/v1 +kind: Workspace +metadata: + name: sample + namespace: wm-sample +spec: + state: "ON" + serviceAccountName: "kloudlite" + enableCodeServer: false + enableVSCodeServer: false + enableTTYD: false + enableJupyterNotebook: false + imagePullPolicy: IfNotPresent + router: + ingressClass: "nginx" + domains: + - "sample.demo.kloudlite.io" + https: + enabled: true + routes: + - app: sample + path: "/" + port: 3000 diff --git a/operators/workspace/internal/controllers/workspace/controller.go b/operators/workspace/internal/controllers/workspace/controller.go new file mode 100644 index 00000000..b82ac2ec --- /dev/null +++ b/operators/workspace/internal/controllers/workspace/controller.go @@ -0,0 +1,221 @@ +package workspace + +import ( + "context" + "fmt" + + "k8s.io/client-go/tools/record" + + crdsv1 "github.com/kloudlite/operator/apis/crds/v1" + "github.com/kloudlite/operator/operators/workspace/internal/env" + "github.com/kloudlite/operator/operators/workspace/internal/templates" + "github.com/kloudlite/operator/pkg/constants" + fn "github.com/kloudlite/operator/toolkit/functions" + "github.com/kloudlite/operator/toolkit/kubectl" + rApi "github.com/kloudlite/operator/toolkit/reconciler" + stepResult "github.com/kloudlite/operator/toolkit/reconciler/step-result" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" +) + +var PortConfig = templates.PortConfig{ + SSHPort: 22, + TTYDPort: 56789, + NotebookPort: 56790, + CodeServerPort: 56791, +} + +const ( + IngressClassName = "nginx" +) + +type Reconciler struct { + client.Client + Scheme *runtime.Scheme + Env *env.Env + + YAMLClient kubectl.YAMLClient + recorder record.EventRecorder + + workspaceDeploymentTemplate []byte + jumpServerTemplate []byte + templateWebhook []byte +} + +func (r *Reconciler) GetName() string { + return "workspace" +} + +const ( + CreateDeployment string = "create-deployment" + // CreateService string = "create-service" +) + +// +kubebuilder:rbac:groups=crds.kloudlite.io,resources=apps,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=crds.kloudlite.io,resources=apps/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=crds.kloudlite.io,resources=apps/finalizers,verbs=update + +func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { + req, err := rApi.NewRequest(ctx, r.Client, request.NamespacedName, &crdsv1.Workspace{}) + if err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + req.PreReconcile() + defer req.PostReconcile() + + if req.Object.GetDeletionTimestamp() != nil { + if x := r.finalize(req); !x.ShouldProceed() { + return x.ReconcilerResponse() + } + return ctrl.Result{}, nil + } + + if step := req.ClearStatusIfAnnotated(); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + + if step := req.EnsureCheckList([]rApi.CheckMeta{{Name: CreateDeployment}}); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + + if step := req.RestartIfAnnotated(); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + + if step := req.EnsureLabelsAndAnnotations(); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + + if step := req.EnsureFinalizers(constants.ForegroundFinalizer, constants.CommonFinalizer); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + + // if step := r.createInterceptableService(req); !step.ShouldProceed() { + // return step.ReconcilerResponse() + // } + + if step := r.createDeployment(req); !step.ShouldProceed() { + return step.ReconcilerResponse() + } + + req.Object.Status.IsReady = true + return ctrl.Result{}, nil +} + +// func (r *Reconciler) createInterceptableService(req *rApi.Request[*crdsv1.Workspace]) stepResult.Result { +// ctx, obj := req.Context(), req.Object +// check := rApi.NewRunningCheck(CreateService, req) + +// svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: obj.Name, Namespace: obj.Namespace}} +// if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, svc, func() error { +// svc.Spec.Ports = []corev1.ServicePort{ +// { +// Name: fmt.Sprintf("port-%d", 3000), +// Protocol: "TCP", +// Port: 3000, +// TargetPort: intstr.FromInt(3000), +// }, +// } +// return nil +// }); err != nil { +// return check.Failed(err) +// } + +// // function-body +// return check.Completed() +// } + +func (r *Reconciler) createDeployment(req *rApi.Request[*crdsv1.Workspace]) stepResult.Result { + ctx, obj := req.Context(), req.Object + check := rApi.NewRunningCheck(CreateDeployment, req) + + b, err := templates.ParseBytes(r.workspaceDeploymentTemplate, templates.WorkspaceTemplateArgs{ + Metadata: metav1.ObjectMeta{ + Name: obj.Name, + Namespace: obj.Namespace, + OwnerReferences: []metav1.OwnerReference{fn.AsOwner(obj, true)}, + }, + KloudliteDomain: "test.khost.dev", + WorkMachineName: obj.Spec.WorkMachine, + ServiceAccountName: obj.Spec.ServiceAccountName, + ImageInitContainer: r.Env.WorkspaceImageInitContainer, + ImageSSH: r.Env.WorkspaceImageSSH, + IsOn: obj.Spec.State == crdsv1.WorkspaceStateOn, + EnableTTYD: obj.Spec.EnableTTYD, + ImageTTYD: r.Env.WorkspaceImageTTYD, + + EnableJupyterNotebook: obj.Spec.EnableJupyterNotebook, + ImageJupyterNotebook: r.Env.WorkspaceImageJupyterNotebook, + + EnableCodeServer: obj.Spec.EnableCodeServer, + ImageCodeServer: r.Env.WorkspaceImageCodeServer, + + EnableVSCodeServer: obj.Spec.EnableVSCodeServer, + ImageVscodeServer: r.Env.WorkspaceImageVscodeServer, + PortConfig: PortConfig, + + ImagePullPolicy: obj.Spec.ImagePullPolicy, + KloudliteDeviceFQDN: fmt.Sprintf("%s-headless.%s.svc.cluster.local", obj.Name, obj.Namespace), + }) + if err != nil { + return check.Failed(err) + } + + fmt.Println(string(b)) + + rr, err := r.YAMLClient.ApplyYAML(ctx, b) + if err != nil { + return check.Failed(err) + } + + req.AddToOwnedResources(rr...) + + return check.Completed() +} + +func (r *Reconciler) finalize(req *rApi.Request[*crdsv1.Workspace]) stepResult.Result { + if step := req.EnsureCheckList([]rApi.CheckMeta{ + {Name: "uninstall workspace"}, + }); !step.ShouldProceed() { + return step + } + + check := rApi.NewRunningCheck("uninstall workspace", req) + + if step := req.CleanupOwnedResources(check); !step.ShouldProceed() { + return step + } + + return req.Finalize() +} + +func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { + r.Client = mgr.GetClient() + r.Scheme = mgr.GetScheme() + + if r.YAMLClient == nil { + return fmt.Errorf("yaml client must be set") + } + + r.recorder = mgr.GetEventRecorderFor(r.GetName()) + + var err error + r.workspaceDeploymentTemplate, err = templates.Read(templates.WorkspaceIngressTemplate, templates.WorkspaceSTSTemplate, templates.WorkspaceServiceTemplate) + if err != nil { + return err + } + + builder := ctrl.NewControllerManagedBy(mgr).For(&crdsv1.Workspace{}) + builder.WithOptions(controller.Options{MaxConcurrentReconciles: r.Env.MaxConcurrentReconciles}) + builder.Owns(&appsv1.Deployment{}) + builder.Owns(&corev1.Service{}) + builder.Owns(&crdsv1.Router{}) + builder.WithEventFilter(rApi.ReconcileFilter()) + return builder.Complete(r) +} diff --git a/operators/workspace/internal/env/env.go b/operators/workspace/internal/env/env.go new file mode 100644 index 00000000..579086e4 --- /dev/null +++ b/operators/workspace/internal/env/env.go @@ -0,0 +1,25 @@ +package env + +import ( + "github.com/codingconcepts/env" +) + +type Env struct { + MaxConcurrentReconciles int `env:"MAX_CONCURRENT_RECONCILES"` + + WorkspaceImageInitContainer string `env:"WORKSPACE_IMAGE_INIT_CONTAINER" default:"ghcr.io/kloudlite/iac/workspace:latest"` + WorkspaceImageSSH string `env:"WORKSPACE_IMAGE_SSH" default:"ghcr.io/kloudlite/iac/workspace:latest"` + WorkspaceImageTTYD string `env:"WORKSPACE_IMAGE_TTYD" default:"ghcr.io/kloudlite/iac/ttyd:latest"` + WorkspaceImageJupyterNotebook string `env:"WORKSPACE_IMAGE_JUPYTER_NOTEBOOK" default:"ghcr.io/kloudlite/iac/jupyter:latest"` + WorkspaceImageCodeServer string `env:"WORKSPACE_IMAGE_CODE_SERVER" default:"ghcr.io/kloudlite/iac/code-server:latest"` + WorkspaceImageVscodeServer string `env:"WORKSPACE_IMAGE_VSCODE_SERVER" default:"ghcr.io/kloudlite/iac/vscode-server:latest"` +} + +func GetEnvOrDie() *Env { + var ev Env + if err := env.Set(&ev); err != nil { + panic(err) + } + + return &ev +} diff --git a/operators/workspace/internal/templates/deployments/ingress.yml.tpl b/operators/workspace/internal/templates/deployments/ingress.yml.tpl new file mode 100644 index 00000000..03c2cec8 --- /dev/null +++ b/operators/workspace/internal/templates/deployments/ingress.yml.tpl @@ -0,0 +1,88 @@ +--- +{{- with . }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: code-server-{{.Metadata.Name}} + namespace: {{.Metadata.Namespace}} + labels: {{.Metadata.Labels | toJson }} + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/proxy-body-size: "50m" +spec: + {{- if and .IngressClassName (ne .IngressClassName "") }} + ingressClassName: {{.IngressClassName}} + {{- end }} + rules: + - host: code-server.{{.Metadata.Name}}.{{.WorkMachineName}}.{{.KloudliteDomain}} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{.Metadata.Name}} + port: + number: {{.PortConfig.CodeServerPort}} + + +--- + + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: nb-{{.Metadata.Name}} + namespace: {{.Metadata.Namespace}} + labels: {{.Metadata.Labels | toJson }} + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/proxy-body-size: "50m" +spec: + {{- if and .IngressClassName (ne .IngressClassName "") }} + ingressClassName: {{.IngressClassName}} + {{- end }} + rules: + - host: notebook.{{.Metadata.Name}}.{{.WorkMachineName}}.{{.KloudliteDomain}} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{.Metadata.Name}} + port: + number: {{.PortConfig.NotebookPort}} + + +--- + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ttyd-{{.Metadata.Name}} + namespace: {{.Metadata.Namespace}} + labels: {{.Metadata.Labels | toJson }} + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/proxy-body-size: "50m" +spec: + {{- if and .IngressClassName (ne .IngressClassName "") }} + ingressClassName: {{.IngressClassName}} + {{- end }} + rules: + - host: ttyd.{{.Metadata.Name}}.{{.WorkMachineName}}.{{.KloudliteDomain}} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{.Metadata.Name}} + port: + number: {{.PortConfig.TTYDPort}} + +{{- end }} diff --git a/operators/workspace/internal/templates/deployments/service.yml.tpl b/operators/workspace/internal/templates/deployments/service.yml.tpl new file mode 100644 index 00000000..99d915db --- /dev/null +++ b/operators/workspace/internal/templates/deployments/service.yml.tpl @@ -0,0 +1,51 @@ +--- +{{- with . }} +apiVersion: v1 +kind: Service +metadata: {{.Metadata | toJson }} +spec: + selector: + app: {{.Metadata.Name | squote}} + ports: + - name: "ssh" + protocol: "TCP" + port: {{.PortConfig.SSHPort}} + targetPort: {{.PortConfig.SSHPort}} + +{{ if .EnableTTYD }} + - name: "ttyd-server" + protocol: "TCP" + port: {{.PortConfig.TTYDPort}} + targetPort: {{.PortConfig.TTYDPort}} +{{ end }} + +{{ if .EnableJupyterNotebook }} + - name: "jupyter-server" + protocol: "TCP" + port: {{.PortConfig.NotebookPort}} + targetPort: {{.PortConfig.NotebookPort}} +{{ end }} + +{{ if .EnableCodeServer }} + - name: "code-server" + protocol: "TCP" + port: {{.PortConfig.CodeServerPort}} + targetPort: {{.PortConfig.CodeServerPort}} +{{ end }} + + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{.Metadata.Name}}-headless + namespace: {{.Metadata.Namespace}} + labels: {{.Metadata.Labels | toJson }} + annotations: {{.Metadata.Annotations | toJson }} + ownerReferences: {{.Metadata.OwnerReferences | toJson }} +spec: + clusterIP: None + selector: + app: {{.Metadata.Name | squote}} + +{{- end }} \ No newline at end of file diff --git a/operators/workspace/internal/templates/deployments/sts.yml.tpl b/operators/workspace/internal/templates/deployments/sts.yml.tpl new file mode 100644 index 00000000..e386402b --- /dev/null +++ b/operators/workspace/internal/templates/deployments/sts.yml.tpl @@ -0,0 +1,253 @@ +--- +{{- with . }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: {{.Metadata | toJson }} +spec: + replicas: {{ if .IsOn }}1{{ else }}0{{ end }} + selector: + matchLabels: + app: {{.Metadata.Name | squote}} + template: + metadata: + labels: + app: {{.Metadata.Name | squote}} + kloudlite.io/gateway.enabled: "false" + spec: + securityContext: + fsGroup: 1000 + hostname: {{.Metadata.Name}} + nodeName: {{.WorkMachineName}} + # {{- if and .ServiceAccountName (ne .ServiceAccountName "") }} + # serviceAccount: {{.ServiceAccountName | squote}} + # {{- end }} + tolerations: + - key: "kloudlite.io/workmachine.name" + operator: "Equal" + value: {{.WorkMachineName |squote}} + effect: "NoExecute" + initContainers: + - name: init-home-dir + image: {{.ImageInitContainer}} + imagePullPolicy: Always + env: + - name: KL_WORKSPACE + value: {{.Metadata.Name}} + - name: HOME + value: "/home/kl" + - name: KL_BOX_MODE + value: "true" + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + command: + - "bash" + - "-c" + - | + set -e + set +x + + if [ ! -d "/home/kl/.ssh" ]; then + mkdir -p /home/kl/.ssh + chown -R 1000:1000 /home/kl/.ssh + fi + if [ -f "/home/kl/.ssh/authorized_keys" ]; then + if ! cmp -s /tmp/authorized_keys /home/kl/.ssh/authorized_keys; then + echo "authorized_keys file differs, copying new one" + cp /tmp/authorized_keys /home/kl/.ssh/authorized_keys + fi + echo "authorized_keys file is up to date" + else + echo "authorized_keys file not found, copying new one" + cp /tmp/authorized_keys /home/kl/.ssh/authorized_keys + fi + + if [ -f "/home/kl/.ssh/id_rsa" ]; then + if ! cmp -s /tmp/id_rsa /home/kl/.ssh/id_rsa; then + echo "id_rsa file differs, copying new one" + rm -f /home/kl/.ssh/id_rsa* 2>/dev/null || true + cp /tmp/id_rsa /home/kl/.ssh/id_rsa + cp /tmp/id_rsa.pub /home/kl/.ssh/id_rsa.pub + fi + echo "id_rsa file is up to date" + else + echo "id_rsa file not found, copying new one" + rm -f /home/kl/.ssh/id_rsa* 2>/dev/null || true + cp /tmp/id_rsa /home/kl/.ssh/id_rsa + cp /tmp/id_rsa.pub /home/kl/.ssh/id_rsa.pub + fi + + if [ ! -d "/nix/store" ]; then + curl -L https://nixos.org/nix/install | sh + mkdir -p ~/.config/nix + echo 'experimental-features = nix-command flakes' > ~/.config/nix/nix.conf + fi + kl_bin_dir="/home/kl/.local/bin" + if [ ! -f "$kl_bin_dir/kl" ]; then + mkdir -p $kl_bin_dir + pushd $kl_bin_dir + curl https://i.jpillora.com/kloudlite/kl@v1.1.87-nightly | bash + popd + fi + + workspace_dir="/home/kl/workspaces/$(KL_WORKSPACE)" + if [ ! -d "$workspace_dir" ]; then + mkdir -p $workspace_dir + pushd $workspace_dir + export PATH=$PATH:/home/kl/.nix-profile/bin:/home/kl/.local/bin + kl init + popd + fi + + if [ -f "$workspace_dir/kl.yaml" ] || [ -f "$workspace_dir/kl.yml" ]; then + pushd $workspace_dir + PATH=$PATH:/home/kl/.nix-profile/bin:/home/kl/.local/bin /home/kl/.local/bin/kl shell -r > /env/.env + PATH=$PATH:/home/kl/.nix-profile/bin:/home/kl/.local/bin /home/kl/.local/bin/kl get env > /env/.connected_env + popd + fi + + if [ ! -f "/home/kl/.zshrc" ]; then + mkdir -p "/home/kl/.config/zsh" + cp /tmp/.zshrc /home/kl/.zshrc + cp /tmp/.aliasrc /home/kl/.config/aliasrc + fi + + if [ ! -f "/home/kl/.local/bin/starship" ]; then + curl -sS https://starship.rs/install.sh | sh -s -- -y -b /home/kl/.local/bin + fi + + if [ ! -d "/home/kl/.config/zsh/zsh-autosuggestions" ]; then + mkdir -p "/home/kl/.config/zsh" + git clone https://github.com/zsh-users/zsh-autosuggestions /home/kl/.config/zsh/zsh-autosuggestions + fi + + if [ ! -d "/home/kl/.config/zsh/zsh-syntax-highlighting" ]; then + mkdir -p "/home/kl/.config/zsh" + git clone https://github.com/zsh-users/zsh-syntax-highlighting.git "/home/kl/.config/zsh/zsh-syntax-highlighting" + fi + + {{- /* if [ ! -d "/home/kl/.kl" ]; then */}} + {{- /* mkdir -p /home/kl/.kl */}} + {{- /* sh -c 'cat > /home/kl/.kl/kl-session.yaml < 0 { + return JobPhaseSucceeded, "", nil + } + + if jr.job.Status.Active > 0 { + return JobPhaseRunning, "waiting for job to complete", nil + } + + if jr.job.Status.Failed > 0 { + return JobPhaseFailed, "", errors.New("install or upgrade job failed") + } + + return JobPhasePending, "", nil +} diff --git a/toolkit/kubectl/yaml-client.go b/toolkit/kubectl/yaml-client.go index 663b8abd..96444984 100644 --- a/toolkit/kubectl/yaml-client.go +++ b/toolkit/kubectl/yaml-client.go @@ -6,10 +6,10 @@ import ( "encoding/json" "fmt" "io" - "log/slog" "time" "github.com/kloudlite/operator/toolkit/errors" + "github.com/nxtcoder17/go.pkgs/log" rApi "github.com/kloudlite/operator/toolkit/reconciler" "sigs.k8s.io/controller-runtime/pkg/client" @@ -45,7 +45,7 @@ type yamlClient struct { k8sClient *kubernetes.Clientset dynamicClient dynamic.Interface mapper meta.RESTMapper - logger *slog.Logger + logger log.Logger } func (yc *yamlClient) Client() *kubernetes.Clientset { @@ -125,7 +125,11 @@ func (yc *yamlClient) ApplyYAML(ctx context.Context, yamls ...[]byte) ([]rApi.Re // Check if the resource exists cobj, err := resourceClient.Get(ctx, obj.GetName(), metav1.GetOptions{}) - if err != nil && apiErrors.IsNotFound(err) { + if err != nil { + if !apiErrors.IsNotFound(err) { + return nil, err + } + // If not exists, create it obj.SetAnnotations(ann) obj.SetLabels(labels) @@ -135,40 +139,49 @@ func (yc *yamlClient) ApplyYAML(ctx context.Context, yamls ...[]byte) ([]rApi.Re } logger.Info("created resource") + cobj = &obj + } + + if err != nil && apiErrors.IsNotFound(err) { continue } - if cobj != nil { - prevLastApplied, ok := cobj.GetAnnotations()[rApi.LastAppliedKey] - if ok { - if prevLastApplied == ann[rApi.LastAppliedKey] { - logger.Info("No changes for resource") - continue - } + if cobj == nil { + // INFO: it should not happen, but just for sanity check + return resources, nil + } - var prevAppliedObj unstructured.Unstructured - if err := json.Unmarshal([]byte(prevLastApplied), &prevAppliedObj); err != nil { - return nil, err - } + prevLastApplied, ok := cobj.GetAnnotations()[rApi.LastAppliedKey] + if ok { + logger.Debug("prev last applied", "value", prevLastApplied) + logger.Debug("new last applied", "value", ann[rApi.LastAppliedKey]) + if prevLastApplied == ann[rApi.LastAppliedKey] { + logger.Info("No changes for resource") + continue + } - prevAnn := prevAppliedObj.GetAnnotations() + var prevAppliedObj unstructured.Unstructured + if err := json.Unmarshal([]byte(prevLastApplied), &prevAppliedObj); err != nil { + return nil, err + } - for k, v := range cobj.GetAnnotations() { - if !fn.MapHasKey(ann, k) && !fn.MapHasKey(prevAnn, k) { - ann[k] = v - } + prevAnn := prevAppliedObj.GetAnnotations() + + for k, v := range cobj.GetAnnotations() { + if !fn.MapHasKey(ann, k) && !fn.MapHasKey(prevAnn, k) { + ann[k] = v } + } - prevLabels := prevAppliedObj.GetLabels() + prevLabels := prevAppliedObj.GetLabels() - for k, v := range cobj.GetLabels() { - if !fn.MapHasKey(labels, k) && !fn.MapHasKey(prevLabels, k) { - labels[k] = v - } + for k, v := range cobj.GetLabels() { + if !fn.MapHasKey(labels, k) && !fn.MapHasKey(prevLabels, k) { + labels[k] = v } } - obj.Object["metadata"] = cobj.Object["metadata"] } + obj.Object["metadata"] = cobj.Object["metadata"] obj.SetAnnotations(ann) obj.SetLabels(labels) @@ -413,7 +426,7 @@ func (yc *yamlClient) RolloutRestart(ctx context.Context, kind Restartable, name } type YAMLClientOpts struct { - Logger *slog.Logger + Logger log.Logger } func NewYAMLClient(config *rest.Config, opts YAMLClientOpts) (YAMLClient, error) { @@ -435,7 +448,7 @@ func NewYAMLClient(config *rest.Config, opts YAMLClientOpts) (YAMLClient, error) mapper := restmapper.NewDiscoveryRESTMapper(gr) if opts.Logger == nil { - opts.Logger = slog.Default() + opts.Logger = log.DefaultLogger() } return &yamlClient{ diff --git a/toolkit/logging/logger.go b/toolkit/logging/logger.go index 7ffa5f90..a2ae4499 100644 --- a/toolkit/logging/logger.go +++ b/toolkit/logging/logger.go @@ -22,7 +22,7 @@ func WithCallDepth(depth int) Opt { } } -func New(logger logr.Logger, opts ...Opt) *slog.Logger { +func Slog(logger logr.Logger, opts ...Opt) *slog.Logger { options := defaultOptions() for i := range opts { opts[i](options) @@ -34,3 +34,7 @@ func New(logger logr.Logger, opts ...Opt) *slog.Logger { return slog.New(logr.ToSlogHandler(logger)) } + +func New(logger logr.Logger, opts ...Opt) *slog.Logger { + return Slog(logger, opts...) +} diff --git a/toolkit/operator/operator.go b/toolkit/operator/operator.go index 56fd0e03..c57fc447 100644 --- a/toolkit/operator/operator.go +++ b/toolkit/operator/operator.go @@ -3,13 +3,12 @@ package operator import ( "flag" "fmt" - "log" - "log/slog" "os" "time" + "github.com/nxtcoder17/go.pkgs/log" + "github.com/kloudlite/operator/toolkit/kubectl" - "github.com/kloudlite/operator/toolkit/logging" reconciler "github.com/kloudlite/operator/toolkit/reconciler" "go.uber.org/zap/zapcore" "k8s.io/apimachinery/pkg/runtime" @@ -54,17 +53,12 @@ type operator struct { Scheme *runtime.Scheme k8sYamlClient kubectl.YAMLClient - logger *slog.Logger } func (op *operator) KubeYAMLClient() kubectl.YAMLClient { return op.k8sYamlClient } -func (op *operator) Logger() *slog.Logger { - return op.logger -} - func New(name string) Operator { printBuildInfo() @@ -72,7 +66,7 @@ func New(name string) Operator { var enableLeaderElection bool var probeAddr string var isDev bool - var debugLog bool + var debug bool var devServerHost string flag.StringVar(&metricsAddr, "metrics-bind-address", ":12345", "The address the metric endpoint binds to.") @@ -83,11 +77,15 @@ func New(name string) Operator { "Enabling this will ensure there is only one active controllers manager.", ) + flag.BoolVar(&debug, "debug", false, "--debug") + opts := zap.Options{ Development: true, EncoderConfigOptions: []zap.EncoderConfigOption{ func(ec *zapcore.EncoderConfig) { // ec.EncodeLevel = zapcore.CapitalColorLevelEncoder + ec.CallerKey = "CALLER" + ec.EncodeCaller = zapcore.ShortCallerEncoder ec.TimeKey = "" }, }, @@ -97,15 +95,15 @@ func New(name string) Operator { rest.SetDefaultWarningHandler(rest.NoWarnings{}) flag.BoolVar(&isDev, "dev", false, "--dev") - flag.BoolVar(&debugLog, "debug-log", false, "--debug-log") + flag.StringVar(&devServerHost, "serverHost", "localhost:8080", "--serverHost ") flag.Parse() - if debugLog { - os.Setenv("LOG_DEBUG", "true") + if isDev { + debug = true } - ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts), zap.Level(zapcore.DebugLevel))) mgrConfig, mgrOptions := func() (*rest.Config, ctrl.Options) { cOpts := ctrl.Options{ @@ -126,19 +124,30 @@ func New(name string) Operator { return ctrl.GetConfigOrDie(), cOpts }() - logger := logging.New(ctrl.Log) + logger := log.New(log.Options{ + Writer: os.Stderr, + ShowTimestamp: false, + ShowCaller: true, + ShowDebugLogs: debug, + ShowLogLevel: true, + JSONFormat: false, + }) + + log.SetDefaultLogger(logger) + k8sYamlClient, err := kubectl.NewYAMLClient(mgrConfig, kubectl.YAMLClientOpts{Logger: logger}) if err != nil { - log.Fatalln(err) + logger.Fatal("failed to create YAML client", "err", err) } + logger.Debug("Starting .............") + return &operator{ startedAt: time.Now(), mgrConfig: mgrConfig, mgrOptions: mgrOptions, IsDev: isDev, k8sYamlClient: k8sYamlClient, - logger: logger, } } @@ -200,7 +209,7 @@ func (op *operator) Operator() *operator { func (op *operator) Start() { mgr, err := ctrl.NewManager(op.mgrConfig, op.mgrOptions) if err != nil { - log.Fatalln(err) + log.DefaultLogger().Fatal("failed to create new controller runtime manager", "err", err) } for i := range op.controllers { diff --git a/toolkit/reconciler/checks.go b/toolkit/reconciler/checks.go index 8132659c..7ba9913d 100644 --- a/toolkit/reconciler/checks.go +++ b/toolkit/reconciler/checks.go @@ -1,6 +1,8 @@ package reconciler import ( + "fmt" + "runtime" "time" fn "github.com/kloudlite/operator/toolkit/functions" @@ -48,15 +50,18 @@ func AreChecksEqual(c1 Check, c2 Check) bool { c1.StartedAt.Sub(c2.StartedAt.Time) == 0 } -type checkWrapper[T Resource] struct { +type CheckWrapper[T Resource] struct { checkName string request *Request[T] Check `json:",inline"` } -func (cw *checkWrapper[T]) Failed(err error) step_result.Result { +func (cw *CheckWrapper[T]) Failed(err error) step_result.Result { defer cw.request.LogPostCheck(cw.checkName) + _, file, line, _ := runtime.Caller(1) + cw.request.Logger.Debug("check.failed", "err", err, "caller", fmt.Sprintf("%s:%d", file, line)) + cw.Check.State = ErroredState cw.Check.Status = false if err != nil { @@ -77,7 +82,7 @@ func (cw *checkWrapper[T]) Failed(err error) step_result.Result { // return cw.request.updateStatus().Continue(false).Err(err) } -func (cw *checkWrapper[T]) StillRunning(err error) step_result.Result { +func (cw *CheckWrapper[T]) StillRunning(err error) step_result.Result { defer cw.request.LogPostCheck(cw.checkName) cw.Check.State = RunningState @@ -99,7 +104,7 @@ func (cw *checkWrapper[T]) StillRunning(err error) step_result.Result { // return cw.request.updateStatus().Continue(false).Err(err) } -func (cw *checkWrapper[T]) Completed() step_result.Result { +func (cw *CheckWrapper[T]) Completed() step_result.Result { defer cw.request.LogPostCheck(cw.checkName) cw.Check.State = CompletedState @@ -112,11 +117,10 @@ func (cw *checkWrapper[T]) Completed() step_result.Result { return step_result.New().Err(err) } return step_result.New().Continue(true) - // return cw.request.updateStatus().Continue(true) } -func NewRunningCheck[T Resource](name string, req *Request[T]) *checkWrapper[T] { - cw := &checkWrapper[T]{ +func NewRunningCheck[T Resource](name string, req *Request[T]) *CheckWrapper[T] { + cw := &CheckWrapper[T]{ checkName: name, request: req, Check: Check{ diff --git a/toolkit/reconciler/event-predicate.go b/toolkit/reconciler/event-predicate.go index 7cde91d2..3a2faa9e 100644 --- a/toolkit/reconciler/event-predicate.go +++ b/toolkit/reconciler/event-predicate.go @@ -76,7 +76,7 @@ func ReconcileFilter(eventRecorder ...record.EventRecorder) predicate.Funcs { } if len(oldObj.GetLabels()) != len(newObj.GetLabels()) || !reflect.DeepEqual(oldObj.GetLabels(), newObj.GetLabels()) { - fireEvent(newObj, ReasonLabelsUpdated, fmt.Sprintf("labels updated from (%+v) to (%+v)", newObj.GetLabels(), oldObj.GetLabels())) + fireEvent(newObj, ReasonLabelsUpdated, fmt.Sprintf("labels updated from (%+v) to (%+v)", oldObj.GetLabels(), newObj.GetLabels())) return true } @@ -85,7 +85,7 @@ func ReconcileFilter(eventRecorder ...record.EventRecorder) predicate.Funcs { annHasChanged := false for k, v := range oldAnn { - if k != LastAppliedKey { + if k != LastAppliedKey && k != "deployment.kubernetes.io/revision" { if v != newAnn[k] { annHasChanged = true break @@ -94,18 +94,18 @@ func ReconcileFilter(eventRecorder ...record.EventRecorder) predicate.Funcs { } if len(oldAnn) != len(newAnn) || annHasChanged { - fireEvent(newObj, ReasonAnnotationsUpdated, fmt.Sprintf("annotations updated from (%+v) to (%+v)", newObj.GetAnnotations(), oldObj.GetAnnotations())) + fireEvent(newObj, ReasonAnnotationsUpdated, fmt.Sprintf("annotations updated from (%+v) to (%+v)", oldObj.GetAnnotations(), newObj.GetAnnotations())) return true } if len(oldObj.GetFinalizers()) != len(newObj.GetFinalizers()) || !reflect.DeepEqual(oldObj.GetFinalizers(), newObj.GetFinalizers()) { - fireEvent(newObj, ReasonFinalizersUpdated, fmt.Sprintf("finalizers updated from (%+v) to (%+v)", newObj.GetFinalizers(), oldObj.GetFinalizers())) + fireEvent(newObj, ReasonFinalizersUpdated, fmt.Sprintf("finalizers updated from (%+v) to (%+v)", oldObj.GetFinalizers(), newObj.GetFinalizers())) return true } if len(oldObj.GetOwnerReferences()) != len(newObj.GetOwnerReferences()) || !reflect.DeepEqual(oldObj.GetOwnerReferences(), newObj.GetOwnerReferences()) { - fireEvent(newObj, ReasonOwnerReferencesUpdated, fmt.Sprintf("owner-references updated from (%+v) to (%+v)", newObj.GetOwnerReferences(), oldObj.GetOwnerReferences())) + fireEvent(newObj, ReasonOwnerReferencesUpdated, fmt.Sprintf("owner-references updated from (%+v) to (%+v)", oldObj.GetOwnerReferences(), newObj.GetOwnerReferences())) return true } @@ -115,24 +115,25 @@ func ReconcileFilter(eventRecorder ...record.EventRecorder) predicate.Funcs { } if oldRes.Status.IsReady == nil || newRes.Status.IsReady == nil { - // INFO: it means this resource is not a kloudlite resource, in that case, + // INFO: it means this resource is not a kloudlite resource, in that case, // it should just be always allowed, as it can be a pod or a job, that some kloudlite resource is watching over + // fireEvent(newObj, ReasonStatusIsReadyChanged, "resource isReady is nil") return true } if *oldRes.Status.IsReady != *newRes.Status.IsReady { - fireEvent(newObj, ReasonStatusIsReadyChanged, fmt.Sprintf("resource isReady changed from (%v) to (%v)", newRes.Status.IsReady, oldRes.Status.IsReady)) + fireEvent(newObj, ReasonStatusIsReadyChanged, fmt.Sprintf("resource isReady changed from (%v) to (%v)", *oldRes.Status.IsReady, *newRes.Status.IsReady)) return true } if len(oldRes.Status.Checks) != len(newRes.Status.Checks) { - fireEvent(newObj, ReasonStatusChecksUpdated, fmt.Sprintf("resource status.checks changed from (%+v) to (%+v)", newRes.Status.Checks, oldRes.Status.Checks)) + fireEvent(newObj, ReasonStatusChecksUpdated, fmt.Sprintf("resource status.checks changed from (%+v) to (%+v)", oldRes.Status.Checks, newRes.Status.Checks)) return true } for k, v := range oldRes.Status.Checks { if !AreChecksEqual(newRes.Status.Checks[k], v) { - fireEvent(newObj, ReasonStatusChecksUpdated, fmt.Sprintf("resource status.checks changed from (%+v) to (%+v)", newRes.Status.Checks, oldRes.Status.Checks)) + fireEvent(newObj, ReasonStatusChecksUpdated, fmt.Sprintf("resource status.checks changed from (%+v) to (%+v)", oldRes.Status.Checks, newRes.Status.Checks)) return true } } diff --git a/toolkit/reconciler/request.go b/toolkit/reconciler/request.go index 89f16b8a..87081935 100644 --- a/toolkit/reconciler/request.go +++ b/toolkit/reconciler/request.go @@ -11,6 +11,7 @@ import ( "time" "github.com/fatih/color" + "github.com/nxtcoder17/go.pkgs/log" apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -19,10 +20,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/log" fn "github.com/kloudlite/operator/toolkit/functions" - "github.com/kloudlite/operator/toolkit/logging" stepResult "github.com/kloudlite/operator/toolkit/reconciler/step-result" ) @@ -89,14 +88,21 @@ func NewRequest[T Resource](ctx context.Context, c client.Client, nn types.Names resource.GetStatus().Checks = map[string]Check{} } - logger := log.FromContext(ctx, "NN", nn.String()) + resource.EnsureGVK() + + // for i := 1; i < 5; i++ { + // _, file, line, _ := runtime.Caller(i) + // // logger := logging.New(ctrl.Log, logging.WithCallDepth(1)).With("NN", nn.String(), "gvk", resource.GetObjectKind().GroupVersionKind().String()) + // logger := log.DefaultLogger().SkipFrames(1).With("NN", nn.String(), "gvk", resource.GetObjectKind().GroupVersionKind().String()) + // logger.Debug(fmt.Sprintf("CALLER [skip frame: %d]: %s:%d\n", i, file, line)) + // } return &Request[T]{ ctx: ctx, client: c, Object: resource, - Logger: logging.New(logger), - internalLogger: logging.New(logger, logging.WithCallDepth(3)), + Logger: log.DefaultLogger().SkipFrames(1).With("NN", nn.String(), "gvk", resource.GetObjectKind().GroupVersionKind().String()).Slog(), + internalLogger: log.DefaultLogger().SkipFrames(4).With("NN", nn.String(), "gvk", resource.GetObjectKind().GroupVersionKind().String()).Slog(), anchorName: anchorName, KV: KV{}, timerMap: map[string]time.Time{}, @@ -442,7 +448,7 @@ func (r *Request[T]) AddToOwnedResources(refs ...ResourceRef) { r.resourceRefs = append(r.resourceRefs, refs...) } -func (r *Request[T]) CleanupOwnedResources(check *checkWrapper[T]) stepResult.Result { +func (r *Request[T]) CleanupOwnedResources(check *CheckWrapper[T]) stepResult.Result { resources := r.Object.GetStatus().Resources objects := make([]client.Object, 0, len(resources)) for i := range resources { @@ -468,7 +474,7 @@ INFO: this should only be used for very specific cases, where there is no other Like, when deleting ManagedService - all managed resources should be deleted, but since owner is already getting deleted, there is no point in their proper cleanup */ -func (r *Request[T]) ForceCleanupOwnedResources(check *checkWrapper[T]) stepResult.Result { +func (r *Request[T]) ForceCleanupOwnedResources(check *CheckWrapper[T]) stepResult.Result { ctx := r.Context() resources := r.Object.GetStatus().Resources