OCSF Schema Support

.github/workflows/build.yml

@@ -14,17 +14,18 @@ on:
 jobs:
   setup:
     runs-on: ubuntu-20.04
+    timeout-minutes: 10
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Install clojure tools
         uses: DeLaGuardo/setup-clojure@3.7
         with:
           lein: 2.9.8
       - name: Cache project dependencies
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.m2/repository
@@ -40,16 +41,17 @@ jobs:
           set -x
           case "${GITHUB_EVENT_NAME}" in
             #scheduled)
-            #  echo '::set-output name=matrix::{"jdk":["8","11","17","21"],"cmd":["test"]}}'
+            #  echo '::set-output name=matrix::{"jdk":["11","17","21"],"cmd":["test"]}}'
             # ;;
             *)
-              echo '::set-output name=matrix::{"jdk":["8","11","17","21"],"cmd":["test"]}}'
+              echo '::set-output name=matrix::{"jdk":["11","17","21"],"cmd":["test"]}}'
               ;;
           esac
   lint:
     runs-on: ubuntu-20.04
+    timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install Java
         uses: actions/setup-java@v3
@@ -70,9 +72,10 @@ jobs:
     strategy:
       matrix: ${{fromJson(needs.setup.outputs.matrix)}}
     runs-on: ubuntu-20.04
+    timeout-minutes: 10
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Cache project dependencies
         uses: actions/cache@v3
         with:
@@ -98,6 +101,7 @@ jobs:
           CMD: ${{ matrix.cmd }}
   all-pr-checks:
     runs-on: ubuntu-20.04
+    timeout-minutes: 10
     needs: [test, lint]
     steps:
       - run: echo "All tests pass!"

.gitignore

@@ -12,3 +12,4 @@ pom.xml.asc
 /.clj-kondo/.cache
 /.lsp/.cache
 /.cpcache/
+/tmp

README.md

@@ -5,9 +5,143 @@ to define data types, to annotate them with domain specific
 information, and to generate artifacts such as schemas, documentation,
 validators, etc.
 
+## OCSF Schemas
+
+[flanders.ocsf](src/flanders/ocsf.cljc) creates flanders schemas from OCSF schemas.
+
+Use `flanders.ocsf/->flanders` to translate an OCSF schema to flanders. The OCSF schemas
+are in an internal format returned by urls like https://schema.ocsf.io/api/objects/cve.
+
+You can export OCSF schemas in bulk at https://schema.ocsf.io/export/schema.
+This repository depends on [ocsf-schema-export](https://github.com/threatgrid/ocsf-schema-export)
+which provides bulk OCSF schemas exports for each major OCSF version.
+
+Please add the following library to your classpath (already a dev dep in flanders):
+
+```clojure
+[io.github.threatgrid/ocsf-schema-export "1.0.0-SNAPSHOT"]
+```
+
+These files are now available on the classpath:
+
+```
+threatgrid/ocsf-1.0.0-export.json
+threatgrid/ocsf-1.1.0-export.json
+threatgrid/ocsf-1.2.0-export.json
+threatgrid/ocsf-1.3.0-export.json
+```
+
+Once you choose your version, they can be converted in bulk to flanders using `flanders.ocsf/parse-exported-schemas`.
+
+```clojure
+(require '[flanders.ocsf :as ocsf]
+         '[cheshire.core :as json]
+         '[clojure.java.io :as io])
+
+(def ocsf-1-3-0-export-json
+  (-> "threatgrid/ocsf-1.3.0-export.json" io/resource slurp json/decode))
+
+(def ocsf-1-3-0-schemas
+  (ocsf/parse-exported-schemas ocsf-1-3-0-export-json))
+```
+
+The result `ocsf-1-3-0-schemas` will have the vals of the `"objects"` and `"classes"` maps to converted to flanders schemas
+(and also the `"base-event"` field).
+
+```clojure
+(-> ocsf-1-3-0-schemas
+    (select-keys ["base_event" "objects" "classes"])
+    (update "base_event" class)
+    (update "objects" update-vals class)
+    (update "classes" update-vals class)
+    prn)
+;=> {"base_event" flanders.types.MapType,
+;    "objects"
+;    {"kill_chain_phase" flanders.types.MapType,
+;     "sub_technique" flanders.types.MapType,
+;     "table" flanders.types.MapType,
+;     ...},
+;    "classes"
+;    {"win/registry_key_query" flanders.types.MapType,
+;     "datastore_activity" flanders.types.MapType,
+;     "event_log" flanders.types.MapType,
+;     ...}}
+```
+
+From there, you can convert them to other formats such as malli or schema:
+
+```clojure
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Plumatic Schema
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(require '[schema.core :as s]
+         'flanders.schema)
+
+(s/defschema OCSFAuthorizationSchema
+  (flanders.schema/->schema (get-in ocsf-1-3-0-schemas ["objects" "authorization"])))
+
+(s/explain OCSFAuthorizationSchema)
+;=> {(optional-key :decision) Str, (optional-key :policy) {Any Any}}
+
+(meta OCSFAuthorizationSchema)
+;=> {:json-schema {:example {:decision "string", :policy {"anything" "anything"}},
+;    :description "The Authorization Result object provides details about the authorization outcome and associated policies related to activity."},
+;    :name OCSFAuthorizationSchema, :ns user}
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; malli
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(require '[malli.core :as m]
+         'flanders.malli)
+
+(def OCSFAuthorizationMalli
+  (flanders.malli/->malli (get-in ocsf-1-3-0-schemas ["objects" "authorization"])))
+
+(m/form OCSFAuthorizationMalli)
+;=> [:map
+;    {:closed true,
+;     :json-schema/example
+;     {:decision "string", :policy {"anything" "anything"}},
+;     :json-schema/description
+;     "The Authorization Result object provides details about the authorization outcome and associated policies related to activity."}
+;    [:decision
+;     {:json-schema/example "string",
+;      :optional true,
+;      :json-schema/description
+;      "Authorization Result/outcome, e.g. allowed, denied."}
+;     [:string {:json-schema/example "string"}]]
+;    [:policy
+;     {:json-schema/example {"anything" "anything"},
+;      :optional true,
+;      :json-schema/description
+;      "Details about the Identity/Access management policies that are applicable."}
+;     [:map
+;      {:closed true, :json-schema/example {"anything" "anything"}}
+;      [:malli.core/default
+;       {:json-schema/example "anything"}
+;       [:map-of :any [:any {:json-schema/example "anything"}]]]]]]
+```
+
+
+You can also just convert the schemas you need directly from the OCSF schema instead
+of via the bulk export:
+
+```clojure
+;; schema
+(s/defschema OCSFAuthorization
+  (flanders.schema/->schema (ocsf/->flanders (get-in ocsf-1-3-0-export ["objects" "authorization"]))))
+
+;; malli
+(def OCSFAuthorization
+  (flanders.malli/->malli (ocsf/->flanders (get-in ocsf-1-3-0-export ["objects" "authorization"]))))
+```
+
 ## License
 
-Copyright © 2016-2023 Cisco Systems
+Copyright © 2016-2025 Cisco Systems
 
 Distributed under the Eclipse Public License either version 1.0 or (at
 your option) any later version.

deps.edn

@@ -1,13 +1,19 @@
 {:paths ["src"]
- :deps {metosin/ring-swagger {:mvn/version "0.26.2"}
-        metosin/schema-tools {:mvn/version "0.13.1"}
-        org.clojure/clojure {:mvn/version "1.10.1"}
+ :deps {org.clojure/clojure {:mvn/version "1.12.0"}
         org.clojure/core.match {:mvn/version "1.0.0"}
-        prismatic/schema {:mvn/version "1.4.1"}}
- :aliases {:test {:extra-paths ["test"]
+        prismatic/schema {:mvn/version "1.2.0"}
+        metosin/ring-swagger {:mvn/version "1.0.0"}
+        metosin/schema-tools {:mvn/version "0.12.3"}
+        io.github.threatgrid/ocsf-schema-export {:mvn/version "1.0.0-SNAPSHOT"}
+        org.clojure/math.combinatorics {:mvn/version "0.3.0"}}
+ :aliases {:test {:extra-paths ["test" "test-resources"]
                   :extra-deps {;; test runner
                                io.github.cognitect-labs/test-runner {:git/tag "v0.5.1" :git/sha "dfb30dd"}
                                ;; dev deps
-                               org.clojure/test.check {:mvn/version "1.1.1"}}
+                               org.clojure/test.check {:mvn/version "1.1.1"}
+                               cheshire/cheshire {:mvn/version "5.13.0"}
+                               clj-http/clj-http {:mvn/version "3.13.0"}
+                               babashka/process {:mvn/version "0.5.22"}
+                               metosin/malli {:mvn/version "0.17.0"}}
                   :main-opts ["-m" "cognitect.test-runner"]
                   :exec-fn cognitect.test-runner.api/test}}}

project.clj

@@ -4,13 +4,12 @@
   :license {:name "Eclipse Public License"
             :url "http://www.eclipse.org/legal/epl-v10.html"}
   :pedantic? :abort
-  :dependencies [[org.clojure/clojure "1.11.3"]
+  :dependencies [[org.clojure/clojure "1.12.0"]
                  [org.clojure/core.match "1.0.0"]
-                 [cheshire "5.9.0"]
-
                  [prismatic/schema "1.2.0"]
                  [metosin/ring-swagger "1.0.0"]
-                 [metosin/schema-tools "0.12.3"]]
+                 [metosin/schema-tools "0.12.3"]
+                 [org.clojure/math.combinatorics "0.3.0"]]
   :global-vars {*warn-on-reflection* true}
   :release-tasks [["clean"]
                   ["vcs" "assert-committed"]
@@ -21,7 +20,13 @@
                   ["change" "version" "leiningen.release/bump-version"]
                   ["vcs" "commit"]
                   ["vcs" "push"]]
-
+  :resource-paths ["resources"]
   :profiles {:dev
-             {:dependencies [[org.clojure/test.check "1.1.1"]
-                             [metosin/malli "0.13.0"]]}})
+             {:resource-paths ["test-resources"]
+              :dependencies [[org.clojure/test.check "1.1.1"]
+                             [babashka/process "0.5.22"]
+                             [cheshire "5.13.0"]
+                             [clj-http "3.13.0"]
+                             [potemkin "0.4.7"]
+                             [metosin/malli "0.17.0"]
+                             [io.github.threatgrid/ocsf-schema-export "1.0.0-SNAPSHOT"]]}})

src/flanders/malli.clj

@@ -79,7 +79,10 @@
   MapType
   (->malli' [{:keys [entries key?] :as dll} opts]
     (let [f #(->malli' % opts)
-          s (-> (into [:merge] (map (fn [e] [:map (f e)])) entries)
+          s (-> (case (count entries)
+                  ;workaround https://github.com/metosin/malli/pull/1147
+                  0 [:merge [:map]]
+                  (into [:merge] (map (fn [e] (m/schema [:map (f e)] opts))) entries))
                 (m/schema opts)
                 m/deref ;; eliminate :merge
                 (mu/update-properties assoc :closed true)

src/flanders/ocsf.cljc

@@ -0,0 +1,76 @@
+(ns flanders.ocsf
+  (:require [flanders.core :as f]
+            #?(:clj flanders.types
+               :cljs [flanders.types :refer [IntegerType NumberType StringType]]))
+  #?(:clj (:import [flanders.types IntegerType NumberType StringType])))
+
+;; :caption => title
+;; all maps are closed
+;; :observable => seems to be a class id
+(defn parse-attribute [[k {:strs [description requirement enum type is_array caption]}] _opts]
+  (let [f (case type
+            "string_t" (f/str)
+            "integer_t" (f/int)
+            "long_t" (f/int)
+            "float_t" (f/num)
+            "uuid_t" (f/str)
+            "boolean_t" (f/bool)
+            "port_t" (f/int)
+            "file_hash_t" (f/str)
+            "file_name_t" (f/str)
+            "process_name_t" (f/str)
+            "username_t" (f/str)
+            "timestamp_t" (f/int)
+            "user" (f/str)
+            "account" (f/str)
+            "actor" (f/str)
+            "affected_code" (f/str)
+            "url_t" (f/str)
+            "datetime_t" (f/str)
+            "object_t" (f/map [(f/entry f/any f/any :required? false)])
+            "hostname_t" (f/str)
+            "ip_t" (f/str)
+            "mac_t" (f/str)
+            "subnet_t" (f/str)
+            "email_t" (f/str)
+            ("json_t" nil) f/any)
+        f (if enum
+            (cond
+              (instance? IntegerType f) (assoc f :values (mapv #?(:clj Long/parseLong :cljs parse-long) (keys enum)))
+              (instance? NumberType f) (assoc f :values (mapv #?(:clj Double/parseDouble :cljs parse-double) (keys enum)))
+              (instance? StringType f) (assoc f :values (mapv str (keys enum)))
+              :else (throw (ex-info (str "enum on " (type f)) {})))
+            f)
+        f (cond-> f
+            is_array f/seq-of)]
+    (f/entry (keyword k) f
+             :description (or description caption)
+             :required? (case requirement
+                          "required" true
+                          ("recommended" "optional" nil) false))))
+
+(defn- normalize-attributes [attributes]
+  (into (sorted-map) (if (map? attributes) [attributes] attributes)))
+
+(defn ->flanders
+  "Converts parsed OCSF schemas to Flanders."
+  ([v] (->flanders v nil))
+  ([{:strs [attributes description]} opts]
+   (-> (f/map (mapv #(parse-attribute % opts) (normalize-attributes attributes)))
+       (assoc :description description))))
+
+(defn parse-exported-schemas
+  "Takes the result of https://schema.ocsf.io/export/schema parsed as edn
+  and updates the base_event, objects and classes schemas to flanders.
+
+  Flanders includes a dependency on https://github.com/frenchy64/ocsf-schema-export
+  which provides OCSF schemas on the classpath.
+
+  Example:
+  (parse-exported-schemas (-> \"threatgrid/ocsf-1.3.0-export.json\" io/resource slurp json/decode))"
+  ([export] (parse-exported-schemas export nil))
+  ([export opts]
+   (-> export
+       (update "base_event" ->flanders opts)
+       (update "objects" update-vals #(->flanders % opts))
+       (update "classes" update-vals #(->flanders % opts)))))

src/flanders/schema.cljc

@@ -10,7 +10,7 @@
                      SequenceOfType SetOfType SignatureType StringType]])
    #?(:clj [ring.swagger.json-schema :as rs])
    [flanders.predicates :as fp]
-   [flanders.example :as example]
+   #?(:clj [flanders.example :as example])
    [flanders.protocols :as prots]
    [schema-tools.core :as st]
    [schema.core :as s])
@@ -42,7 +42,7 @@
 (def get-schema
   (memoize ->schema))
 
-(defn- describe [schema dll]
+(defn- describe [schema #?(:cljs _dll :clj dll)]
   #?(:cljs schema
      :clj (rs/field
             schema