diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml index 9278e97..82ee622 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test.yaml @@ -23,6 +23,8 @@ jobs: - name: Set up chart-testing uses: helm/chart-testing-action@v2.7.0 + with: + yamale_version: "6.0.0" - name: Run chart-testing (list-changed) id: list-changed diff --git a/charts/currents/ci/lint-values.yaml b/charts/currents/ci/lint-values.yaml index 2a25423..497df16 100644 --- a/charts/currents/ci/lint-values.yaml +++ b/charts/currents/ci/lint-values.yaml @@ -2,7 +2,7 @@ currents: mongoConnection: secretName: mongodb-currents-currents-user key: connectionString.standardSrv - elastic: - admin: - secretName: elasticsearch-es-elastic-user - secretKey: elastic + clickhouse: + user: + secretName: clickhouse-currents-pass + secretPasswordKey: password diff --git a/charts/currents/templates/_common.tpl b/charts/currents/templates/_common.tpl index 2550290..cbb3fcf 100644 --- a/charts/currents/templates/_common.tpl +++ b/charts/currents/templates/_common.tpl @@ -103,19 +103,25 @@ Create the name of the service account to use name: {{ .Values.currents.mongoConnection.secretName }} key: {{ .Values.currents.mongoConnection.key }} {{- end }} -- name: ELASTIC_URI - value: {{ printf "%s://%s:%d" (.Values.currents.elastic.tls.enabled | ternary "https" "http") (tpl .Values.currents.elastic.host .) (.Values.currents.elastic.port | int) }} -{{- if .Values.currents.elastic.apiUser.secretName }} -- name: ELASTIC_API_ID +- name: CLICKHOUSE_URL + value: {{ printf "%s://%s:%d" (.Values.currents.clickhouse.tls.enabled | ternary "https" "http") (tpl .Values.currents.clickhouse.host .) (.Values.currents.clickhouse.port | int) }} +{{- if and .Values.currents.clickhouse.user.secretName .Values.currents.clickhouse.user.secretPasswordKey }} +- name: CLICKHOUSE_PASSWORD valueFrom: secretKeyRef: - name: {{ .Values.currents.elastic.apiUser.secretName }} - key: {{ .Values.currents.elastic.apiUser.idKey }} -- name: ELASTIC_API_KEY + name: {{ .Values.currents.clickhouse.user.secretName }} + key: {{ .Values.currents.clickhouse.user.secretPasswordKey }} + {{- if and .Values.currents.clickhouse.user.username }} +- name: CLICKHOUSE_USERNAME + value: {{ .Values.currents.clickhouse.user.username }} + {{- end }} +{{- end }} +{{- if and .Values.currents.clickhouse.user.secretName .Values.currents.clickhouse.user.secretAccessTokenKey }} +- name: CLICKHOUSE_ACCESS_TOKEN valueFrom: secretKeyRef: - name: {{ .Values.currents.elastic.apiUser.secretName }} - key: {{ .Values.currents.elastic.apiUser.secretKey }} + name: {{ .Values.currents.clickhouse.user.secretName }} + key: {{ .Values.currents.clickhouse.user.secretAccessTokenKey }} {{- end }} - name: S3_BUCKET value: {{ .Values.currents.objectStorage.bucket }} @@ -171,21 +177,6 @@ Create the name of the service account to use value: {{ include "currents.url" (dict "context" . "input" .Values.currents.domains.recordApiHost) }} {{- end -}} -{{- define "currents.elasticDataStreamsEnv" -}} -{{- if .Values.currents.elastic.datastreams.tests }} -- name: ELASTIC_DATASTREAM_TESTS - value: {{ .Values.currents.elastic.datastreams.tests }} -{{- end }} -{{- if .Values.currents.elastic.datastreams.runs }} -- name: ELASTIC_DATASTREAM_RUNS - value: {{ .Values.currents.elastic.datastreams.runs }} -{{- end }} -{{- if .Values.currents.elastic.datastreams.instances }} -- name: ELASTIC_DATASTREAM_INSTANCES - value: {{ .Values.currents.elastic.datastreams.instances }} -{{- end }} -{{- end -}} - {{- define "currents.emailSMTPEnv" -}} - name: EMAIL_TRANSPORTER value: smtp diff --git a/charts/currents/templates/changestreams/deployment.yaml b/charts/currents/templates/changestreams/deployment.yaml index 57a10f5..5409e10 100644 --- a/charts/currents/templates/changestreams/deployment.yaml +++ b/charts/currents/templates/changestreams/deployment.yaml @@ -47,7 +47,6 @@ spec: value: "onprem" {{- include "currents.connectionConfigEnv" . | nindent 12 }} {{- include "currents.URLConfigEnv" . | nindent 12 }} - {{- include "currents.elasticDataStreamsEnv" . | nindent 12 }} {{- with (concat .Values.global.env .Values.changestreams.env) }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/charts/currents/templates/scheduler/deployment.yaml b/charts/currents/templates/scheduler/deployment.yaml index 385bc25..e82906f 100644 --- a/charts/currents/templates/scheduler/deployment.yaml +++ b/charts/currents/templates/scheduler/deployment.yaml @@ -48,13 +48,6 @@ spec: - name: RUN_STARTUP_TASKS value: "true" {{- include "currents.connectionConfigEnv" . | nindent 12 }} - - name: ELASTIC_USERNAME - value: {{ .Values.currents.elastic.admin.username | quote }} - - name: ELASTIC_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.currents.elastic.admin.secretName }} - key: {{ .Values.currents.elastic.admin.secretKey }} {{- with (concat .Values.global.env .Values.scheduler.env) }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/charts/currents/templates/server/deployment.yaml b/charts/currents/templates/server/deployment.yaml index 3ca7f02..dd00534 100644 --- a/charts/currents/templates/server/deployment.yaml +++ b/charts/currents/templates/server/deployment.yaml @@ -62,7 +62,6 @@ spec: value: {{ tpl .Values.currents.rootUser.email . }} {{- include "currents.connectionConfigEnv" . | nindent 12 }} {{- include "currents.URLConfigEnv" . | nindent 12 }} - {{- include "currents.elasticDataStreamsEnv" . | nindent 12 }} {{- include "currents.emailSMTPEnv" . | nindent 12 }} {{- if .Values.currents.apiJwtToken.secretName }} - name: JWT_SECRET diff --git a/charts/currents/templates/webhooks/deployment.yaml b/charts/currents/templates/webhooks/deployment.yaml index 3cbe8d4..222775d 100644 --- a/charts/currents/templates/webhooks/deployment.yaml +++ b/charts/currents/templates/webhooks/deployment.yaml @@ -47,7 +47,6 @@ spec: value: "onprem" {{- include "currents.connectionConfigEnv" . | nindent 12 }} {{- include "currents.URLConfigEnv" . | nindent 12 }} - {{- include "currents.elasticDataStreamsEnv" . | nindent 12 }} {{- with (concat .Values.global.env .Values.webhooks.env) }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/charts/currents/templates/writer/deployment.yaml b/charts/currents/templates/writer/deployment.yaml index e574269..f8ed3d5 100644 --- a/charts/currents/templates/writer/deployment.yaml +++ b/charts/currents/templates/writer/deployment.yaml @@ -47,7 +47,6 @@ spec: value: "onprem" {{- include "currents.connectionConfigEnv" . | nindent 12 }} {{- include "currents.URLConfigEnv" . | nindent 12 }} - {{- include "currents.elasticDataStreamsEnv" . | nindent 12 }} {{- include "currents.emailSMTPEnv" . | nindent 12 }} {{- with (concat .Values.global.env .Values.writer.env) }} {{- toYaml . | nindent 12 }} diff --git a/charts/currents/values.yaml b/charts/currents/values.yaml index e43a0fa..b460e75 100644 --- a/charts/currents/values.yaml +++ b/charts/currents/values.yaml @@ -64,43 +64,29 @@ currents: # -- (tpl) set the redis hostname to talk to # @default -- `{{ .Release.Name }}-redis-master` host: "{{ .Release.Name }}-redis-master" - elastic: - datastreams: - # -- The elasticsearch index to use for instances - instances: currents_dev_instances - # -- The elasticsearch index to use for tests - tests: currents_dev_tests - # -- The elasticsearch index to use for runs - runs: currents_dev_runs - admin: - # -- The elasticsearch admin username (used to manage the indexes) + clickhouse: + user: + # -- The ClickHouse username to use # @section -- Frequently Used - username: elastic - # -- The k8s secret to use for the admin password + username: currents + # -- The k8s secret to use for the ClickHouse password # @section -- Required secretName: "" - # -- The k8s secret key to use for the admin password + # -- The k8s secret key to use to access the ClickHouse password # @section -- Required - secretKey: "" - apiUser: - # -- The k8s secret to use for the elasticsearch api key + secretPasswordKey: "" + # -- The k8s secret key to use to access the ClickHouse access token # @section -- Required - secretName: "" - # -- The k8s secret key to use for the elasticsearch api ID - # @section -- Frequently Used - idKey: apiId - # -- The k8s secret key to use for the elasticsearch api key - # @section -- Frequently Used - secretKey: apiKey + secretAccessTokenKey: "" tls: - # -- Whether to use TLS for the elasticsearch connection + # -- Whether to use TLS for the ClickHouse connection # @section -- Frequently Used enabled: true - # -- The elasticsearch host to use + # -- The ClickHouse host to use # @section -- Required host: "" - # -- The elasticsearch port to use - port: 9200 + # -- The ClickHouse port to use + port: 8123 objectStorage: # -- The object storage endpoint to use # @section -- Required diff --git a/docs/configuration.md b/docs/configuration.md index 76084d8..6522ec9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -22,10 +22,10 @@ The following table lists the configurable parameters of the `currents` chart an | currents.email.smtp.secretName | string | `""` | K8s secret to use for the SMTP username/password | | currents.apiJwtToken.secretName | string | `""` | The K8s secret to use for the JWT token | | currents.apiInternalToken.secretName | string | `""` | The K8s secret to use for the internal API token | -| currents.elastic.admin.secretName | string | `""` | The k8s secret to use for the admin password | -| currents.elastic.admin.secretKey | string | `""` | The k8s secret key to use for the admin password | -| currents.elastic.apiUser.secretName | string | `""` | The k8s secret to use for the elasticsearch api key | -| currents.elastic.host | string | `""` | The elasticsearch host to use | +| currents.clickhouse.user.secretName | string | `""` | The k8s secret to use for the ClickHouse password | +| currents.clickhouse.user.secretPasswordKey | string | `""` | The k8s secret key to use to access the ClickHouse password | +| currents.clickhouse.user.secretAccessTokenKey | string | `""` | The k8s secret key to use to access the ClickHouse access token | +| currents.clickhouse.host | string | `""` | The ClickHouse host to use | | currents.objectStorage.endpoint | string | `""` | The object storage endpoint to use | | currents.objectStorage.secretName | string | `""` | The K8s secret to use for the object storage access key | | currents.objectStorage.bucket | string | `""` | The object storage bucket to use | @@ -44,10 +44,8 @@ The following table lists the configurable parameters of the `currents` chart an | currents.email.smtp.secretPasswordKey | string | `"password"` | The K8s secret key to use for the SMTP password | | currents.apiJwtToken.key | string | `"token"` | The K8s secret key to use for the JWT token | | currents.apiInternalToken.key | string | `"token"` | The K8s secret key to use for the internal API token | -| currents.elastic.admin.username | string | `"elastic"` | The elasticsearch admin username (used to manage the indexes) | -| currents.elastic.apiUser.idKey | string | `"apiId"` | The k8s secret key to use for the elasticsearch api ID | -| currents.elastic.apiUser.secretKey | string | `"apiKey"` | The k8s secret key to use for the elasticsearch api key | -| currents.elastic.tls.enabled | bool | `true` | Whether to use TLS for the elasticsearch connection | +| currents.clickhouse.user.username | string | `"currents"` | The ClickHouse username to use | +| currents.clickhouse.tls.enabled | bool | `true` | Whether to use TLS for the ClickHouse connection | | currents.objectStorage.secretIdKey | string | `"keyId"` | The K8s secret key to use for the object storage access key ID | | currents.objectStorage.secretAccessKey | string | `"keySecret"` | The K8s secret key to use for the object storage secret access key | | global.imagePullSecrets | list | `[]` | Reference to one or more secrets to be used when pulling images. [Pull an Image from a Private Registry](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/). | @@ -80,10 +78,7 @@ The following table lists the configurable parameters of the `currents` chart an | currents.ingress.enabled | bool | `false` | Whether to enable the both default ingresses (server, and director) | | currents.apiJwtToken.expiry | string | `"1d"` | How often to expire session tokens signed by the JWT token | | currents.redis.host | tpl | `{{ .Release.Name }}-redis-master` | set the redis hostname to talk to | -| currents.elastic.datastreams.instances | string | `"currents_dev_instances"` | The elasticsearch index to use for instances | -| currents.elastic.datastreams.tests | string | `"currents_dev_tests"` | The elasticsearch index to use for tests | -| currents.elastic.datastreams.runs | string | `"currents_dev_runs"` | The elasticsearch index to use for runs | -| currents.elastic.port | int | `9200` | The elasticsearch port to use | +| currents.clickhouse.port | int | `8123` | The ClickHouse port to use | | currents.objectStorage.internalEndpoint | string | `""` | The object storage internal endpoint to use (for internal communication) | | currents.objectStorage.region | string | `""` | The region to use for the object storage | | currents.objectStorage.pathStyle | bool | `false` | Whether to use path style access for the object storage | diff --git a/docs/developer-guide/README.md b/docs/developer-guide/README.md index d62578d..086d0f3 100644 --- a/docs/developer-guide/README.md +++ b/docs/developer-guide/README.md @@ -33,8 +33,7 @@ helm upgrade --install ingress-nginx ingress-nginx \ MongoDB ```sh -helm repo add mongodb https://mongodb.github.io/helm-charts -helm install community-operator mongodb/community-operator +helm install community-operator community-operator --repo https://mongodb.github.io/helm-charts ``` Edit the password in `samples/mongodb-community-replicaset.yml` @@ -46,42 +45,20 @@ kubectl apply -f samples/mongodb-community-replicaset.yml
-Elasticsearch +Clickhouse -Advanced options avail at: -(docs: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-install-helm.html ) - - -```sh -helm repo add elastic https://helm.elastic.co -helm install elastic-operator-crds elastic/eck-operator-crds -helm install elastic-operator elastic/eck-operator \ - --set=installCRDs=false \ - --set=managedNamespaces='{currents}' \ - --set=createClusterScopedResources=false \ - --set=webhook.enabled=false \ - --set=config.validateStorageClass=false -``` - -Install sample es cluster (docs: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-deploy-elasticsearch.html ) +Create a secrets for user passwords ```sh -kubectl apply -f samples/elasticsearch.yml +kubectl create secret generic clickhouse-default-pass --from-literal=password=$(head -c 512 /dev/urandom | LC_ALL=C tr -cd 'a-zA-Z0-9' | head -c 32) +kubectl create secret generic clickhouse-currents-pass --from-literal=password=$(head -c 512 /dev/urandom | LC_ALL=C tr -cd 'a-zA-Z0-9' | head -c 32) ``` -Wait for es to be available, then generate an api key by: - ```sh -PASSWORD=$(kubectl get secret elasticsearch-es-elastic-user -o go-template='{{.data.elastic | base64decode}}') -kubectl exec elasticsearch-es-default-0 -- curl -u "elastic:$PASSWORD" -X POST -H "Content-Type: application/json" -d "{ \"name\": \"currents-key\" }" "http://elasticsearch-es-http:9200/_security/api_key" > es-api.key.json +helm install clickhouse clickhouse --repo https://helm.altinity.com \ + --set=clickhouse.defaultUser.password_secret_name=clickhouse-default-pass \ + --set-json='clickhouse.users=[{"name":"currents","password_secret_name":"clickhouse-currents-pass"}]' ``` - -Create a new secret with the api info from the key we just created (requires jq installed locally) - -```sh -kubectl create secret generic currents-es-api-key --from-literal=apiId=$(jq -r .id es-api.key.json) --from-literal=apiKey=$(jq -r .api_key es-api.key.json) -``` -
@@ -90,8 +67,8 @@ kubectl create secret generic currents-es-api-key --from-literal=apiId=$(jq -r . Add the minio operator ```sh -helm repo add minio https://operator.min.io/ -helm install minio-operator minio/operator \ +helm install minio-operator operator \ + --repo https://operator.min.io/ \ --set operator.env\[0\].name=WATCHED_NAMESPACE \ --set operator.env\[0\].value=currents \ --set operator.replicaCount=1 @@ -112,7 +89,7 @@ kubectl create secret generic currents-minio-user --from-literal=CONSOLE_ACCESS_ Create a minio tenant instance ```sh -helm install tenant minio/tenant -f samples/minio-tenant-helm-config.yaml +helm install tenant tenant --repo https://operator.min.io/ -f samples/minio-tenant-helm-config.yaml ``` Create an ingress for minio diff --git a/docs/eks/dependencies.md b/docs/eks/dependencies.md index 235f267..c781665 100644 --- a/docs/eks/dependencies.md +++ b/docs/eks/dependencies.md @@ -183,9 +183,8 @@ Creates a single Pod instance of Minio with 10Gb of storage. ```yaml tenant: name: currents-minio - configuration: - name: currents-minio-env-configuration configSecret: + name: currents-minio-env-configuration existingSecret: true accessKey: null secretKey: null diff --git a/samples/local/chart-config.yaml b/samples/local/chart-config.yaml index b9e4cd1..966ae7b 100644 --- a/samples/local/chart-config.yaml +++ b/samples/local/chart-config.yaml @@ -27,15 +27,13 @@ currents: mongoConnection: secretName: mongodb-currents-currents-user key: connectionString.standardSrv - elastic: + clickhouse: + user: + secretName: clickhouse-currents-pass + secretPasswordKey: password tls: enabled: false - host: elasticsearch-es-http - admin: - secretName: elasticsearch-es-elastic-user - secretKey: elastic - apiUser: - secretName: currents-es-api-key + host: clickhouse-clickhouse objectStorage: endpoint: http://minio.localhost internalEndpoint: https://minio diff --git a/samples/minio-tenant-helm-config.yaml b/samples/minio-tenant-helm-config.yaml index f5c8c25..4350798 100644 --- a/samples/minio-tenant-helm-config.yaml +++ b/samples/minio-tenant-helm-config.yaml @@ -1,8 +1,7 @@ tenant: name: currents-minio - configuration: - name: currents-minio-env-configuration configSecret: + name: currents-minio-env-configuration existingSecret: true accessKey: null secretKey: null