[CSR-3235] feat: switch from elasticsearch to clickhouse

.github/workflows/lint-test.yaml

@@ -23,6 +23,8 @@ jobs:
 
       - name: Set up chart-testing
         uses: helm/chart-testing-action@v2.7.0
+        with:
+          yamale_version: "6.0.0"
 
       - name: Run chart-testing (list-changed)
         id: list-changed

charts/currents/ci/lint-values.yaml

@@ -2,7 +2,7 @@ currents:
   mongoConnection:
     secretName: mongodb-currents-currents-user
     key: connectionString.standardSrv
-  elastic:
-    admin:
-      secretName: elasticsearch-es-elastic-user
-      secretKey: elastic
+  clickhouse:
+    user:
+      secretName: clickhouse-currents-pass
+      secretPasswordKey: password

charts/currents/templates/_common.tpl

@@ -103,19 +103,25 @@ Create the name of the service account to use
       name: {{ .Values.currents.mongoConnection.secretName }}
       key: {{ .Values.currents.mongoConnection.key }}
 {{- end }}
-- name: ELASTIC_URI
-  value: {{ printf "%s://%s:%d" (.Values.currents.elastic.tls.enabled | ternary "https" "http") (tpl .Values.currents.elastic.host .) (.Values.currents.elastic.port | int) }}
-{{- if .Values.currents.elastic.apiUser.secretName }}
-- name: ELASTIC_API_ID
+- name: CLICKHOUSE_URL
+  value: {{ printf "%s://%s:%d" (.Values.currents.clickhouse.tls.enabled | ternary "https" "http") (tpl .Values.currents.clickhouse.host .) (.Values.currents.clickhouse.port | int) }}
+{{- if and .Values.currents.clickhouse.user.secretName .Values.currents.clickhouse.user.secretPasswordKey }}
+- name: CLICKHOUSE_PASSWORD
   valueFrom:
     secretKeyRef:
-      name: {{ .Values.currents.elastic.apiUser.secretName }}
-      key: {{ .Values.currents.elastic.apiUser.idKey }}
-- name: ELASTIC_API_KEY
+      name: {{ .Values.currents.clickhouse.user.secretName }}
+      key: {{ .Values.currents.clickhouse.user.secretPasswordKey }}
+  {{- if and .Values.currents.clickhouse.user.username }}
+- name: CLICKHOUSE_USERNAME
+  value: {{ .Values.currents.clickhouse.user.username }}
+  {{- end }}
+{{- end }}
+{{- if and .Values.currents.clickhouse.user.secretName .Values.currents.clickhouse.user.secretAccessTokenKey }}
+- name: CLICKHOUSE_ACCESS_TOKEN
   valueFrom:
     secretKeyRef:
-      name: {{ .Values.currents.elastic.apiUser.secretName }}
-      key: {{ .Values.currents.elastic.apiUser.secretKey }}
+      name: {{ .Values.currents.clickhouse.user.secretName }}
+      key: {{ .Values.currents.clickhouse.user.secretAccessTokenKey }}
 {{- end }}
 - name: S3_BUCKET
   value: {{ .Values.currents.objectStorage.bucket }}
@@ -171,21 +177,6 @@ Create the name of the service account to use
   value: {{ include "currents.url" (dict "context" . "input" .Values.currents.domains.recordApiHost) }}
 {{- end -}}
 
-{{- define "currents.elasticDataStreamsEnv" -}}
-{{- if .Values.currents.elastic.datastreams.tests }}
-- name: ELASTIC_DATASTREAM_TESTS
-  value: {{ .Values.currents.elastic.datastreams.tests }}
-{{- end }}
-{{- if .Values.currents.elastic.datastreams.runs }}
-- name: ELASTIC_DATASTREAM_RUNS
-  value: {{ .Values.currents.elastic.datastreams.runs }}
-{{- end }}
-{{- if .Values.currents.elastic.datastreams.instances }}
-- name: ELASTIC_DATASTREAM_INSTANCES
-  value: {{ .Values.currents.elastic.datastreams.instances }}
-{{- end }}
-{{- end -}}
-
 {{- define "currents.emailSMTPEnv" -}}
 - name: EMAIL_TRANSPORTER
   value: smtp

charts/currents/templates/changestreams/deployment.yaml

@@ -47,7 +47,6 @@ spec:
               value: "onprem"
             {{- include "currents.connectionConfigEnv" . | nindent 12 }}
             {{- include "currents.URLConfigEnv" . | nindent 12 }}
-            {{- include "currents.elasticDataStreamsEnv" . | nindent 12 }}
             {{- with (concat .Values.global.env .Values.changestreams.env) }}
               {{- toYaml . | nindent 12 }}
             {{- end }}

charts/currents/templates/scheduler/deployment.yaml

@@ -48,13 +48,6 @@ spec:
             - name: RUN_STARTUP_TASKS
               value: "true"
             {{- include "currents.connectionConfigEnv" . | nindent 12 }}
-            - name: ELASTIC_USERNAME
-              value: {{ .Values.currents.elastic.admin.username | quote }}
-            - name: ELASTIC_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: {{ .Values.currents.elastic.admin.secretName }}
-                  key: {{ .Values.currents.elastic.admin.secretKey }}
             {{- with (concat .Values.global.env .Values.scheduler.env) }}
               {{- toYaml . | nindent 12 }}
             {{- end }}

charts/currents/templates/server/deployment.yaml

@@ -62,7 +62,6 @@ spec:
               value: {{ tpl .Values.currents.rootUser.email . }}
           {{- include "currents.connectionConfigEnv" . | nindent 12 }}
           {{- include "currents.URLConfigEnv" . | nindent 12 }}
-          {{- include "currents.elasticDataStreamsEnv" . | nindent 12 }}
           {{- include "currents.emailSMTPEnv" . | nindent 12 }}
           {{- if .Values.currents.apiJwtToken.secretName }}
             - name: JWT_SECRET

charts/currents/templates/webhooks/deployment.yaml

@@ -47,7 +47,6 @@ spec:
               value: "onprem"
             {{- include "currents.connectionConfigEnv" . | nindent 12 }}
             {{- include "currents.URLConfigEnv" . | nindent 12 }}
-            {{- include "currents.elasticDataStreamsEnv" . | nindent 12 }}
             {{- with (concat .Values.global.env .Values.webhooks.env) }}
               {{- toYaml . | nindent 12 }}
             {{- end }}

charts/currents/templates/writer/deployment.yaml

@@ -47,7 +47,6 @@ spec:
               value: "onprem"
             {{- include "currents.connectionConfigEnv" . | nindent 12 }}
             {{- include "currents.URLConfigEnv" . | nindent 12 }}
-            {{- include "currents.elasticDataStreamsEnv" . | nindent 12 }}
             {{- include "currents.emailSMTPEnv" . | nindent 12 }}
             {{- with (concat .Values.global.env .Values.writer.env) }}
               {{- toYaml . | nindent 12 }}

charts/currents/values.yaml

@@ -64,43 +64,29 @@ currents:
     # -- (tpl) set the redis hostname to talk to
     # @default -- `{{ .Release.Name }}-redis-master`
     host: "{{ .Release.Name }}-redis-master"
-  elastic:
-    datastreams:
-      # -- The elasticsearch index to use for instances
-      instances: currents_dev_instances
-      # -- The elasticsearch index to use for tests
-      tests: currents_dev_tests
-      # -- The elasticsearch index to use for runs
-      runs: currents_dev_runs
-    admin:
-      # -- The elasticsearch admin username (used to manage the indexes)
+  clickhouse:
+    user:
+      # -- The ClickHouse username to use
       # @section -- Frequently Used
-      username: elastic
-      # -- The k8s secret to use for the admin password
+      username: currents
+      # -- The k8s secret to use for the ClickHouse password
       # @section -- Required
       secretName: ""
-      # -- The k8s secret key to use for the admin password
+      # -- The k8s secret key to use to access the ClickHouse password
       # @section -- Required
-      secretKey: ""
-    apiUser:
-      # -- The k8s secret to use for the elasticsearch api key
+      secretPasswordKey: ""
+      # -- The k8s secret key to use to access the ClickHouse access token
       # @section -- Required
-      secretName: ""
-      # -- The k8s secret key to use for the elasticsearch api ID
-      # @section -- Frequently Used
-      idKey: apiId
-      # -- The k8s secret key to use for the elasticsearch api key
-      # @section -- Frequently Used
-      secretKey: apiKey
+      secretAccessTokenKey: ""
     tls:
-      # -- Whether to use TLS for the elasticsearch connection
+      # -- Whether to use TLS for the ClickHouse connection
       # @section -- Frequently Used
       enabled: true
-    # -- The elasticsearch host to use
+    # -- The ClickHouse host to use
     # @section -- Required
     host: ""
-    # -- The elasticsearch port to use
-    port: 9200
+    # -- The ClickHouse port to use
+    port: 8123
   objectStorage:
     # -- The object storage endpoint to use
     # @section -- Required

docs/configuration.md

@@ -22,10 +22,10 @@ The following table lists the configurable parameters of the `currents` chart an
 | currents.email.smtp.secretName | string | `""` | K8s secret to use for the SMTP username/password |
 | currents.apiJwtToken.secretName | string | `""` | The K8s secret to use for the JWT token |
 | currents.apiInternalToken.secretName | string | `""` | The K8s secret to use for the internal API token |
-| currents.elastic.admin.secretName | string | `""` | The k8s secret to use for the admin password |
-| currents.elastic.admin.secretKey | string | `""` | The k8s secret key to use for the admin password |
-| currents.elastic.apiUser.secretName | string | `""` | The k8s secret to use for the elasticsearch api key |
-| currents.elastic.host | string | `""` | The elasticsearch host to use |
+| currents.clickhouse.user.secretName | string | `""` | The k8s secret to use for the ClickHouse password |
+| currents.clickhouse.user.secretPasswordKey | string | `""` | The k8s secret key to use to access the ClickHouse password |
+| currents.clickhouse.user.secretAccessTokenKey | string | `""` | The k8s secret key to use to access the ClickHouse access token |
+| currents.clickhouse.host | string | `""` | The ClickHouse host to use |
 | currents.objectStorage.endpoint | string | `""` | The object storage endpoint to use |
 | currents.objectStorage.secretName | string | `""` | The K8s secret to use for the object storage access key |
 | currents.objectStorage.bucket | string | `""` | The object storage bucket to use |
@@ -44,10 +44,8 @@ The following table lists the configurable parameters of the `currents` chart an
 | currents.email.smtp.secretPasswordKey | string | `"password"` | The K8s secret key to use for the SMTP password |
 | currents.apiJwtToken.key | string | `"token"` | The K8s secret key to use for the JWT token |
 | currents.apiInternalToken.key | string | `"token"` | The K8s secret key to use for the internal API token |
-| currents.elastic.admin.username | string | `"elastic"` | The elasticsearch admin username (used to manage the indexes) |
-| currents.elastic.apiUser.idKey | string | `"apiId"` | The k8s secret key to use for the elasticsearch api ID |
-| currents.elastic.apiUser.secretKey | string | `"apiKey"` | The k8s secret key to use for the elasticsearch api key |
-| currents.elastic.tls.enabled | bool | `true` | Whether to use TLS for the elasticsearch connection |
+| currents.clickhouse.user.username | string | `"currents"` | The ClickHouse username to use |
+| currents.clickhouse.tls.enabled | bool | `true` | Whether to use TLS for the ClickHouse connection |
 | currents.objectStorage.secretIdKey | string | `"keyId"` | The K8s secret key to use for the object storage access key ID |
 | currents.objectStorage.secretAccessKey | string | `"keySecret"` | The K8s secret key to use for the object storage secret access key |
 | global.imagePullSecrets | list | `[]` | Reference to one or more secrets to be used when pulling images. [Pull an Image from a Private Registry](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/). |
@@ -80,10 +78,7 @@ The following table lists the configurable parameters of the `currents` chart an
 | currents.ingress.enabled | bool | `false` | Whether to enable the both default ingresses (server, and director) |
 | currents.apiJwtToken.expiry | string | `"1d"` | How often to expire session tokens signed by the JWT token |
 | currents.redis.host | tpl | `{{ .Release.Name }}-redis-master` | set the redis hostname to talk to |
-| currents.elastic.datastreams.instances | string | `"currents_dev_instances"` | The elasticsearch index to use for instances |
-| currents.elastic.datastreams.tests | string | `"currents_dev_tests"` | The elasticsearch index to use for tests |
-| currents.elastic.datastreams.runs | string | `"currents_dev_runs"` | The elasticsearch index to use for runs |
-| currents.elastic.port | int | `9200` | The elasticsearch port to use |
+| currents.clickhouse.port | int | `8123` | The ClickHouse port to use |
 | currents.objectStorage.internalEndpoint | string | `""` | The object storage internal endpoint to use (for internal communication) |
 | currents.objectStorage.region | string | `""` | The region to use for the object storage |
 | currents.objectStorage.pathStyle | bool | `false` | Whether to use path style access for the object storage |

docs/developer-guide/README.md

@@ -33,8 +33,7 @@ helm upgrade --install ingress-nginx ingress-nginx \
 <summary>MongoDB</summary>
 
 ```sh
-helm repo add mongodb https://mongodb.github.io/helm-charts
-helm install community-operator mongodb/community-operator
+helm install community-operator community-operator --repo https://mongodb.github.io/helm-charts
 ```
 
 Edit the password in `samples/mongodb-community-replicaset.yml`
@@ -46,42 +45,20 @@ kubectl apply -f samples/mongodb-community-replicaset.yml
 </details>
 
 <details>
-<summary>Elasticsearch</summary>
+<summary>Clickhouse</summary>
 
-Advanced options avail at:
-(docs: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-install-helm.html )
-
-
-```sh
-helm repo add elastic https://helm.elastic.co
-helm install elastic-operator-crds elastic/eck-operator-crds
-helm install elastic-operator elastic/eck-operator  \
-  --set=installCRDs=false \
-  --set=managedNamespaces='{currents}' \
-  --set=createClusterScopedResources=false \
-  --set=webhook.enabled=false \
-  --set=config.validateStorageClass=false
-```
-
-Install sample es cluster (docs: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-deploy-elasticsearch.html )
+Create a secrets for user passwords
 
 ```sh
-kubectl apply -f samples/elasticsearch.yml
+kubectl create secret generic clickhouse-default-pass --from-literal=password=$(head -c 512 /dev/urandom | LC_ALL=C tr -cd 'a-zA-Z0-9' | head -c 32)
+kubectl create secret generic clickhouse-currents-pass --from-literal=password=$(head -c 512 /dev/urandom | LC_ALL=C tr -cd 'a-zA-Z0-9' | head -c 32)
 ```
 
-Wait for es to be available, then generate an api key by:
-
 ```sh
-PASSWORD=$(kubectl get secret elasticsearch-es-elastic-user -o go-template='{{.data.elastic | base64decode}}')
-kubectl exec elasticsearch-es-default-0 -- curl -u "elastic:$PASSWORD" -X POST -H "Content-Type: application/json" -d "{ \"name\": \"currents-key\" }"  "http://elasticsearch-es-http:9200/_security/api_key" > es-api.key.json
+helm install clickhouse clickhouse --repo https://helm.altinity.com \
+    --set=clickhouse.defaultUser.password_secret_name=clickhouse-default-pass \
+    --set-json='clickhouse.users=[{"name":"currents","password_secret_name":"clickhouse-currents-pass"}]'
 ```
-
-Create a new secret with the api info from the key we just created (requires jq installed locally)
-
-```sh
-kubectl create secret generic currents-es-api-key --from-literal=apiId=$(jq -r .id es-api.key.json) --from-literal=apiKey=$(jq -r .api_key es-api.key.json)
-```
-
 </details>
 
 <details>
@@ -90,8 +67,8 @@ kubectl create secret generic currents-es-api-key --from-literal=apiId=$(jq -r .
 Add the minio operator
 
 ```sh
-helm repo add minio https://operator.min.io/
-helm install minio-operator minio/operator \
+helm install minio-operator operator \
+  --repo https://operator.min.io/ \
   --set operator.env\[0\].name=WATCHED_NAMESPACE \
   --set operator.env\[0\].value=currents \
   --set operator.replicaCount=1
@@ -112,7 +89,7 @@ kubectl create secret generic currents-minio-user --from-literal=CONSOLE_ACCESS_
 Create a minio tenant instance
 
 ```sh
-helm install tenant minio/tenant -f samples/minio-tenant-helm-config.yaml
+helm install tenant tenant --repo https://operator.min.io/ -f samples/minio-tenant-helm-config.yaml
 ```
 
 Create an ingress for minio

docs/eks/dependencies.md

@@ -183,9 +183,8 @@ Creates a single Pod instance of Minio with 10Gb of storage.
    ```yaml
    tenant:
      name: currents-minio
-     configuration:
-       name: currents-minio-env-configuration
      configSecret:
+       name: currents-minio-env-configuration
        existingSecret: true
        accessKey: null
        secretKey: null

samples/local/chart-config.yaml

@@ -27,15 +27,13 @@ currents:
   mongoConnection:
     secretName: mongodb-currents-currents-user
     key: connectionString.standardSrv
-  elastic:
+  clickhouse:
+    user:
+      secretName: clickhouse-currents-pass
+      secretPasswordKey: password
     tls:
       enabled: false
-    host: elasticsearch-es-http
-    admin:
-      secretName: elasticsearch-es-elastic-user 
-      secretKey: elastic
-    apiUser:
-      secretName: currents-es-api-key
+    host: clickhouse-clickhouse
   objectStorage:
     endpoint: http://minio.localhost
     internalEndpoint: https://minio

samples/minio-tenant-helm-config.yaml

@@ -1,8 +1,7 @@
 tenant:
   name: currents-minio
-  configuration:
-    name: currents-minio-env-configuration
   configSecret:
+    name: currents-minio-env-configuration
     existingSecret: true
     accessKey: null
     secretKey: null