Add OpenSearch 2.x/3.x support via Ductile 0.6.0

.github/workflows/pr.yml

@@ -218,12 +218,14 @@ jobs:
     strategy:
       matrix:
         this-split: ${{fromJson(needs.setup.outputs.matrix)}}
+        engine: [es, os]
     env:
       CTIA_TEST_SUITE: ${{ matrix.this-split.test_suite }}
       CTIA_THIS_SPLIT: ${{ matrix.this-split.this_split }}
       CTIA_NSPLITS: ${{ matrix.this-split.total_splits }}
       CTIA_CI_PROFILES: ${{ matrix.this-split.ci_profiles }}
       JAVA_VERSION: ${{ matrix.this-split.java_version }}
+      CTIA_TEST_ENGINES: ${{ matrix.engine }}
     steps:
       - uses: actions/checkout@v4
       - name: Binary Cache
@@ -310,14 +312,14 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           retention-days: 1
-          name: test-timing-${{matrix.this-split.test_suite}}-${{matrix.this-split.java_version}}-${{matrix.this-split.this_split}}
+          name: test-timing-${{matrix.engine}}-${{matrix.this-split.test_suite}}-${{matrix.this-split.java_version}}-${{matrix.this-split.this_split}}
           path: target/test-results/*.edn
       - name: Upload docker compose
         if: ${{ always() }}
         uses: actions/upload-artifact@v4
         with:
           retention-days: 10
-          name: docker-compose-${{matrix.this-split.test_suite}}-${{matrix.this-split.java_version}}-${{matrix.this-split.this_split}}.log
+          name: docker-compose-${{matrix.engine}}-${{matrix.this-split.test_suite}}-${{matrix.this-split.java_version}}-${{matrix.this-split.this_split}}.log
           path: ${{env.LOG_PATH}}/docker-compose.log
   # fan-in tests so there's a single job we can add to protected branches.
   # otherwise, we'll have add all (range ${CTIA_NSPLITS}) jobs, and keep

containers/dev/docker-compose.yml

@@ -16,6 +16,28 @@ services:
     ports:
       - "9207:9200"
       - "9307:9300"
+  opensearch:
+    image: opensearchproject/opensearch:2.19.0
+    environment:
+      - cluster.name=opensearch2
+      - discovery.type=single-node
+      - plugins.security.disabled=true
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=Ductile123!
+      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
+    ports:
+      - "9202:9200"
+      - "9302:9300"
+  opensearch3:
+    image: opensearchproject/opensearch:3.1.0
+    environment:
+      - cluster.name=opensearch3
+      - discovery.type=single-node
+      - plugins.security.disabled=true
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=Ductile123!
+      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
+    ports:
+      - "9203:9200"
+      - "9303:9300"
   zookeeper:
     image: confluentinc/cp-zookeeper:7.2.0
     hostname: "zookeeper"

dependabot/dependency-tree.txt

@@ -58,7 +58,7 @@ ctia:ctia:jar:1.1.1-SNAPSHOT
 |  +- com.andrewmcveigh:cljs-time:jar:0.5.2:compile
 |  \- threatgrid:metrics-clojure-riemann:jar:2.10.1:compile
 |     \- io.riemann:metrics3-riemann-reporter:jar:0.4.6:compile
-+- threatgrid:ductile:jar:0.5.0:compile
++- threatgrid:ductile:jar:0.6.0:compile
 +- com.arohner:uri:jar:0.1.2:compile
 |  \- pathetic:pathetic:jar:0.5.0:compile
 |     \- com.cemerick:clojurescript.test:jar:0.0.4:compile

dependabot/pom.xml

@@ -551,7 +551,7 @@
     <dependency>
       <groupId>threatgrid</groupId>
       <artifactId>ductile</artifactId>
-      <version>0.5.0</version>
+      <version>0.6.0</version>
       <exclusions>
         <exclusion>
           <artifactId>slf4j-nop</artifactId>

project.clj

@@ -94,7 +94,7 @@
                   :exclusions [com.cognitect/transit-java]] ;; ring-middleware-format takes precedence
                  [instaparse "1.4.10"] ;; com.gfredericks/test.chuck > threatgrid/ctim
                  [threatgrid/clj-momo "0.4.1"]
-                 [threatgrid/ductile "0.5.0"]
+                 [threatgrid/ductile "0.6.0"]
 
                  [com.arohner/uri "0.1.2"]
 

resources/ctia-default.properties

@@ -84,6 +84,18 @@ ctia.events.log=false
 ctia.store.es.default.host=127.0.0.1
 ctia.store.es.default.port=9207
 ctia.store.es.default.version=7
+# Engine defaults to elasticsearch if not specified
+#ctia.store.es.default.engine=elasticsearch
+
+# OpenSearch Configuration (requires ductile 0.6.0+)
+# OpenSearch 2.x: use port 9202, version 2, engine opensearch, auth admin/admin
+# OpenSearch 3.x: use port 9203, version 3, engine opensearch, auth admin/admin
+#ctia.store.es.default.port=9202
+#ctia.store.es.default.version=2
+#ctia.store.es.default.engine=opensearch
+#ctia.store.es.default.auth.params.user=admin
+#ctia.store.es.default.auth.params.pwd=admin
+
 #ctia.store.es.default.host=localhost
 #ctia.store.es.default.port=9207
 #ctia.store.es.default.protocol=https

src/ctia/properties.clj

@@ -40,6 +40,7 @@
    (str prefix store ".default_operator") (s/enum "OR" "AND")
    (str prefix store ".timeout") s/Num
    (str prefix store ".version") s/Num
+   (str prefix store ".engine") s/Str
    (str prefix store ".allow_partial_search_results") s/Bool
    (str prefix store ".update-mappings")  s/Bool
    (str prefix store ".update-settings")  s/Bool

src/ctia/stores/es/init.clj

@@ -8,6 +8,7 @@
    [ductile.conn :refer [connect]]
    [ductile.document :as document]
    [ductile.index :as index]
+   [ductile.lifecycle :as lifecycle]
    [schema-tools.core :as st]
    [schema.core :as s]))
 
@@ -45,14 +46,19 @@
        {:rollover rollover}}}}))
 
 (s/defn mk-index-ilm-config
-  [{:keys [index props config] :as store-config}]
+  [{:keys [index props config conn] :as store-config}]
   (let [{:keys [mappings settings]} config
         write-alias (:write-index props)
         policy (mk-policy props)
         lifecycle {:name index
                    :rollover_alias write-alias}
-        settings-ilm (assoc-in settings [:index :lifecycle] lifecycle)
-        base-config {:settings settings-ilm
+        ;; Only add ILM lifecycle settings for Elasticsearch, not OpenSearch
+        ;; OpenSearch uses ISM which doesn't support these settings in templates
+        is-elasticsearch? (= :elasticsearch (:engine conn :elasticsearch))
+        settings-with-lifecycle (if is-elasticsearch?
+                                  (assoc-in settings [:index :lifecycle] lifecycle)
+                                  settings)
+        base-config {:settings settings-with-lifecycle
                      :mappings mappings
                      :aliases {index {}}}
         template {:index_patterns (str index "*")
@@ -107,7 +113,7 @@
     (log/infof "found legacy template for %s Deleting it." index)
     (index/delete-template! conn index))
   (log/info "Creating policy: " index)
-  (index/create-policy! conn index (:policy config))
+  (lifecycle/create-policy! conn index (:policy config))
   (log/info "Creating index template: " index)
   (index/create-index-template! conn index (:template config))
   (log/infof "Updated index template: %s" index))
@@ -262,14 +268,31 @@
                      (select-keys config [:mappings :settings :aliases])))
       conn-state))
 
+(def valid-engines
+  "Valid search engine types supported by CTIA"
+  #{:elasticsearch :opensearch})
+
 (s/defn get-store-properties :- StoreProperties
   "Lookup the merged store properties map"
   [store-kw :- s/Keyword
    get-in-config]
-  (merge
-   {:entity store-kw}
-   (get-in-config [:ctia :store :es :default] {})
-   (get-in-config [:ctia :store :es store-kw] {})))
+  (let [props (merge
+               {:entity store-kw}
+               (get-in-config [:ctia :store :es :default] {})
+               (get-in-config [:ctia :store :es store-kw] {}))
+        ;; Convert :engine from string to keyword if present
+        ;; Properties system reads it as string but ductile expects keyword
+        props-with-engine (cond-> props
+                            (:engine props) (update :engine keyword))]
+    ;; Validate engine if specified
+    (when-let [engine (:engine props-with-engine)]
+      (when-not (valid-engines engine)
+        (throw (ex-info (str "Invalid search engine: " engine
+                             ". Valid engines are: " valid-engines)
+                        {:engine engine
+                         :valid-engines valid-engines
+                         :store store-kw}))))
+    props-with-engine))
 
 (s/defn ^:private make-factory
   "Return a store instance factory. Most of the ES stores are

src/ctia/stores/es/store.clj

@@ -6,6 +6,7 @@
    [ctia.stores.es.crud :as crud]
    [ductile.conn :as es-conn]
    [ductile.index :as es-index]
+   [ductile.lifecycle :as es-lifecycle]
    [ductile.pagination :refer [default-limit]]
    [ductile.schemas :refer [ESConn]]
    [schema.core :as s]))
@@ -14,7 +15,7 @@
   (when conn
     (es-index/delete-template! conn (str index "*"))
     (es-index/delete-index-template! conn (str index "*"))
-    (es-index/delete-policy! conn (str index "*"))
+    (es-lifecycle/delete-policy! conn (str index "*"))
     (es-index/delete! conn (str index "*"))))
 
 (s/defn close-connections!

test/ctia/properties_test.clj

@@ -19,18 +19,19 @@
             "ctia.store.es.malware.rollover.max_age" s/Str
             "ctia.store.es.malware.aliased"  s/Bool
             "ctia.store.es.malware.default_operator" (s/enum "OR" "AND")
-            "ctia.store.es.malware.allow_partial_search_results" s/Bool
+            "ctia.store.es.malware.timeout" s/Num
             "ctia.store.es.malware.version" s/Num
+            "ctia.store.es.malware.engine" s/Str
+            "ctia.store.es.malware.allow_partial_search_results" s/Bool
             "ctia.store.es.malware.update-mappings" s/Bool
             "ctia.store.es.malware.update-settings" s/Bool
             "ctia.store.es.malware.refresh-mappings" s/Bool
             "ctia.store.es.malware.migrate-to-ilm" s/Bool
             "ctia.store.es.malware.default-sort" s/Str
-            "ctia.store.es.malware.timeout" s/Num
             "ctia.store.es.malware.auth.type" sut/AuthParamsType
             "ctia.store.es.malware.auth.params.id" s/Str
-            "ctia.store.es.malware.auth.params.api-key" s/Str
             "ctia.store.es.malware.auth.params.headers.authorization" s/Str
+            "ctia.store.es.malware.auth.params.api-key" s/Str
             "ctia.store.es.malware.auth.params.user" s/Str
             "ctia.store.es.malware.auth.params.pwd" s/Str}
            (sut/es-store-impl-properties "ctia.store.es." "malware")))
@@ -48,14 +49,15 @@
             "prefix.sighting.rollover.max_age" s/Str
             "prefix.sighting.aliased"  s/Bool
             "prefix.sighting.default_operator" (s/enum "OR" "AND")
-            "prefix.sighting.allow_partial_search_results" s/Bool
+            "prefix.sighting.timeout" s/Num
             "prefix.sighting.version" s/Num
+            "prefix.sighting.engine" s/Str
+            "prefix.sighting.allow_partial_search_results" s/Bool
             "prefix.sighting.update-mappings" s/Bool
             "prefix.sighting.update-settings" s/Bool
             "prefix.sighting.refresh-mappings" s/Bool
             "prefix.sighting.migrate-to-ilm" s/Bool
             "prefix.sighting.default-sort" s/Str
-            "prefix.sighting.timeout" s/Num
             "prefix.sighting.auth.type" sut/AuthParamsType
             "prefix.sighting.auth.params.id" s/Str
             "prefix.sighting.auth.params.api-key" s/Str