From 99ae3c1bff61765e2016d50b4faa494875f3a595 Mon Sep 17 00:00:00 2001 From: Mikhail Kuzmin Date: Sun, 30 Mar 2025 20:20:00 +0400 Subject: [PATCH] memoize will-be-memoized refactor test fix clear idea test fix .clear test refactoring fix test add profiler idea 2 next save fix --- src/darkleaf/di/core.clj | 188 +++++++++++------- test/darkleaf/di/add_side_dependency_test.clj | 15 +- test/darkleaf/di/dependencies_test.clj | 13 +- test/darkleaf/di/memoize_test.clj | 145 ++++++++++++++ test/darkleaf/di/tutorial/x_inspect_test.clj | 97 +++++---- test/darkleaf/di/tutorial/x_log_test.clj | 8 +- .../di/tutorial/z_memoize_test.clj.back | 54 +++++ 7 files changed, 399 insertions(+), 121 deletions(-) create mode 100644 test/darkleaf/di/memoize_test.clj create mode 100644 test/darkleaf/di/tutorial/z_memoize_test.clj.back diff --git a/src/darkleaf/di/core.clj b/src/darkleaf/di/core.clj index c3259005..9a4b942a 100644 --- a/src/darkleaf/di/core.clj +++ b/src/darkleaf/di/core.clj @@ -22,7 +22,8 @@ (clojure.lang IDeref IFn Var Indexed ILookup) (java.io FileNotFoundException Writer) (java.lang AutoCloseable) - (java.util.function Function))) + (java.util.function Function) + (java.util.concurrent ConcurrentHashMap))) (set! *warn-on-reflection* true) @@ -181,11 +182,11 @@ (map? mw) (apply-map registry mw) (instance? Function mw) (.apply ^Function mw registry) :else (throw (IllegalArgumentException. "Wrong middleware kind"))) - (with-meta {::idx (-> registry meta ::idx inc)}))) + (with-meta {::idx (-> registry meta (::idx 0) inc)}))) -(defn- apply-middlewares [registry middlewares init-idx] +(defn- apply-middlewares [registry middlewares] (reduce apply-middleware - (-> registry (with-meta {::idx init-idx})) + registry (flatten middlewares))) (declare var->factory) @@ -227,68 +228,29 @@ (let [factory (cond (vector? key) (->> key (map ref) template) (map? key) (-> key (update-vals ref) template) - :else (-> key ref))] + :else (-> key ref)) + factory (reify + p/Factory + p/FactoryDescription + (dependencies [_] + (concat (seq (p/dependencies factory)) + (seq {::side-dependency :optional}))) + (build [_ deps] + (p/build factory deps)) + (demolish [_ obj] + (p/demolish factory obj)) + (description [_] + (p/description factory)))] {::implicit-root factory})) -(defn start - "Starts a system of dependent objects. - - key is a name of the system root. - Use symbols for var names, keywords for abstract dependencies, - and strings for environments variables. - - key is looked up in a registry. - By default registry uses Clojure namespaces and system env - to resolve symbols and strings, respectively. - - You can extend it with registry middlewares. - Each middleware can be one of the following form: - - - a function `registry -> key -> Factory` - - a map of key and `p/Factory` instance - - nil, as no-op middleware - - a sequence of the previous forms - - Middlewares also allows you to instrument built objects. - It's useful for logging, schema validation, AOP, etc. - See `update-key`. +(def ^:private initial-registry + (-> undefined-registry with-env with-ns)) - ```clojure - (di/start `root - {:my-abstraction implemntation - `some-key replacement - \"LOG_LEVEL\" \"info\"} - [dev-middlwares test-middlewares] - (if dev-routes? - (di/update-key `route-data conj `dev-route-data) - (di/instrument `log)) - ``` - - Returns a container contains started root of the system. - The container implements `AutoCloseable`, `IDeref`, `IFn`, `Indexed` and `ILookup`. - - Use `with-open` in tests to stop the system reliably. - - You can pass a vector as the key argument to start many keys: - - ```clojure - (with-open [root (di/start [`handler `helper])] - (let [[handler helper] root] - ...)) - ``` - - See the tests for use cases. - See `update-key`." - ^AutoCloseable [key & middlewares] - (let [base-mws [with-env - with-ns - (implicit-root key)] - init-idx (- (count base-mws)) - middlewares (concat base-mws middlewares) - registry (apply-middlewares undefined-registry middlewares init-idx) - ctx {:registry registry - :*stop-list (atom '())} - obj (try-build ctx ::implicit-root)] +(defn- start* ^AutoCloseable [key middlewares] + (let [registry (apply-middlewares initial-registry middlewares) + ctx {:registry registry + :*stop-list (atom '())} + obj (try-build ctx key)] ^{:type ::root ::print obj} (reify @@ -363,6 +325,58 @@ (applyTo [_ args] (.applyTo ^IFn obj args))))) +(defn start + "Starts a system of dependent objects. + + key is a name of the system root. + Use symbols for var names, keywords for abstract dependencies, + and strings for environments variables. + + key is looked up in a registry. + By default registry uses Clojure namespaces and system env + to resolve symbols and strings, respectively. + + You can extend it with registry middlewares. + Each middleware can be one of the following form: + + - a function `registry -> key -> Factory` + - a map of key and `p/Factory` instance + - nil, as no-op middleware + - a sequence of the previous forms + + Middlewares also allows you to instrument built objects. + It's useful for logging, schema validation, AOP, etc. + See `update-key`. + + ```clojure + (di/start `root + {:my-abstraction implemntation + `some-key replacement + \"LOG_LEVEL\" \"info\"} + [dev-middlwares test-middlewares] + (if dev-routes? + (di/update-key `route-data conj `dev-route-data) + (di/instrument `log)) + ``` + + Returns a container contains started root of the system. + The container implements `AutoCloseable`, `IDeref`, `IFn`, `Indexed` and `ILookup`. + + Use `with-open` in tests to stop the system reliably. + + You can pass a vector as the key argument to start many keys: + + ```clojure + (with-open [root (di/start [`handler `helper])] + (let [[handler helper] root] + ...)) + ``` + + See the tests for use cases. + See `update-key`." + ^AutoCloseable [key & middlewares] + (start* ::implicit-root [middlewares (implicit-root key)])) + (defn stop "Stops the root of a system" [^AutoCloseable root] @@ -611,7 +625,7 @@ (fn [registry] (fn [key] (let [factory (registry key)] - (if (= ::implicit-root key) + (if (= ::side-dependency key) (reify p/Factory (dependencies [_] @@ -978,7 +992,47 @@ {:key `bar}] ```" [key & middlewares] - (with-open [components (start key - middlewares - with-inspect)] + (with-open [components (start* ::implicit-root + [middlewares + (implicit-root key) + with-inspect])] @components)) + + +(defn ->memoize + "Returns a statefull middleware that memoizes all registry build accesses. + + To stop all memoized components use `(di/stop mem)`." + ^AutoCloseable [& middlewares] + (let [registry (apply-middlewares initial-registry middlewares) + factories (ConcurrentHashMap.) + objs (ConcurrentHashMap.) + *stop-list (atom '())] + (reify + AutoCloseable + (close [_] + (.clear factories) + (.clear objs) + (try-stop-started {:*stop-list *stop-list})) + Function + (apply [_ previous-registry] + (when-not (identical? previous-registry initial-registry) + (throw (ex-info "memoize should be first" {}))) + (fn [key] + (let [factory #_(registry key) (.computeIfAbsent factories key registry)] + (reify + p/Factory + (dependencies [_] + (p/dependencies factory)) + (build [_ deps] + (.computeIfAbsent objs [factory deps] + (fn [_] + (let [obj (p/build factory deps)] + (swap! *stop-list conj #(p/demolish factory obj)) + obj)))) + (demolish [_ obj]) + + p/FactoryDescription + (description [_] + (assoc (p/description factory) + ::memoize {:will-be-memoized true}))))))))) diff --git a/test/darkleaf/di/add_side_dependency_test.clj b/test/darkleaf/di/add_side_dependency_test.clj index d594f938..8af6f4f8 100644 --- a/test/darkleaf/di/add_side_dependency_test.clj +++ b/test/darkleaf/di/add_side_dependency_test.clj @@ -77,8 +77,9 @@ (t/deftest bug-array-map->hash-map (let [log (atom []) - after-build! (fn [{:keys [key]}] - (swap! log conj key))] + after-build! (fn [{:keys [key object]}] + (when (some? object) + (swap! log conj key)))] (with-open [root (di/start ::root {::root :ok} (di/add-side-dependency `d1) @@ -91,15 +92,16 @@ (di/add-side-dependency `d8) (di/add-side-dependency `extra-d9) (di/log :after-build! after-build!))] - (t/is (= [::root `d1 `d2 `d3 `d4 `d5 `d6 `d7 `d8 `extra-d9 ::di/implicit-root] + (t/is (= [::root `d1 `d2 `d3 `d4 `d5 `d6 `d7 `d8 `extra-d9] @log)) (t/is (= :ok @root))))) (t/deftest bug-array-map->hash-map-2 (let [log (atom []) - after-build! (fn [{:keys [key]}] - (swap! log conj key))] + after-build! (fn [{:keys [key object]}] + (when (some? object) + (swap! log conj key)))] (with-open [root (di/start ::root {::root :ok} (di/add-side-dependency `extra-d9) @@ -114,8 +116,7 @@ (di/add-side-dependency `extra-d10) (di/log :after-build! after-build!))] (t/is (= [::root - `extra-d9 `d1 `d2 `d3 `d4 `d5 `d6 `d7 `d8 `extra-d10 - ::di/implicit-root] + `extra-d9 `d1 `d2 `d3 `d4 `d5 `d6 `d7 `d8 `extra-d10] @log)) (t/is (= :ok @root))))) diff --git a/test/darkleaf/di/dependencies_test.clj b/test/darkleaf/di/dependencies_test.clj index b4d5d5d0..d6207b94 100644 --- a/test/darkleaf/di/dependencies_test.clj +++ b/test/darkleaf/di/dependencies_test.clj @@ -33,19 +33,18 @@ (t/deftest order-test (let [log (atom []) - after-build! (fn [{:keys [key]}] - (swap! log conj [key :built])) - after-demolish! (fn [{:keys [key]}] - (swap! log conj [key :stopped]))] + after-build! (fn [{:keys [key object]}] + (when (some? object) + (swap! log conj [key :built]))) + after-demolish! (fn [{:keys [key object]}] + (when (some? object) + (swap! log conj [key :stopped])))] (with-open [root (di/start `root (di/log :after-build! after-build! :after-demolish! after-demolish!))]) (t/is (= [[`c :built] [`a :built] [`b :built] [`root :built] - [::di/implicit-root :built] - - [::di/implicit-root :stopped] [`root :stopped] [`b :stopped] [`a :stopped] diff --git a/test/darkleaf/di/memoize_test.clj b/test/darkleaf/di/memoize_test.clj new file mode 100644 index 00000000..519a4ad0 --- /dev/null +++ b/test/darkleaf/di/memoize_test.clj @@ -0,0 +1,145 @@ +(ns darkleaf.di.memoize-test + (:require + [clojure.test :as t] + [darkleaf.di.core :as di])) + +(set! *warn-on-reflection* true) + +(t/deftest memoize-test + (let [a 'a + identity* (memoize identity)] + (identity* a) + (t/is (not (identical? a (identity 'a)))) + (t/is (identical? a (identity* 'a))))) + +(defn- some+identical? [a b] + (and (some? a) + (some? b) + (identical? a b))) + +(defn- some+not-identical? [a b] + (and (some? a) + (some? b) + (not (identical? a b)))) + + +(defn a + {::di/kind :component} + [{_ ::param}] + (Object.)) + + +(t/deftest changed-not-identical-test + (with-open [mem (di/->memoize {::param (Object.)}) + first (di/start `a mem) + second (di/start `a mem {::param (Object.)})] + (t/is (some+not-identical? @first @second)))) + + +(t/deftest changed-equal-and-identical-test + (with-open [mem (di/->memoize {::param :equal-and-identical}) + first (di/start `a mem) + second (di/start `a mem {::param :equal-and-identical})] + (t/is (some+identical? @first @second)))) + + +(t/deftest changed-equal-but-not-identical-test + (with-open [mem (di/->memoize {::param 'equal-but-not-identical}) + first (di/start `a mem) + second (di/start `a mem {::param 'equal-but-not-identical})] + (t/is (some+identical? @first @second)))) + + +(t/deftest changed-equal-but-different-test + (with-open [mem (di/->memoize {::param []}) + first (di/start `a mem) + second (di/start `a mem {::param '()})] + (t/is (some+identical? @first @second)))) + +(t/deftest start-stop-order-test + (let [log (atom []) + log-mw (fn [key-predicate] + (di/log :after-build! + #(when (and (some? (:object %)) + (key-predicate (:key %))) + (swap! log conj [:start (:key %)])) + :after-demolish! + #(when (and (some? (:object %)) + (key-predicate (:key %))) + (swap! log conj [:stop (:key %)])))) + mem (di/->memoize {::param :param} (log-mw any?))] + (-> (di/start `a mem) + (di/stop)) + (t/is (= [[:start ::param] + [:start `a]] + @log)) + (swap! log empty) + + (-> (di/start `a mem) + (di/stop)) + (t/is (= [] @log)) + + (-> (di/start `a mem + {::param :new-param} + (log-mw #{::param})) + (di/stop)) + (t/is (= [[:start ::param] + [:start `a] + [:stop ::param]] + @log)) + (swap! log empty) + + (di/stop mem) + (t/is (= [[:stop `a] + [:stop `a] + [:stop ::param]] + @log)) + (swap! log empty) + + (-> (di/start `a mem) + (di/stop)) + (t/is (= [[:start ::param] + [:start `a]] + @log)))) + + +;; todo: thrown-with-msg? +(t/deftest should-be-first-test + (with-open [mem (di/->memoize)] + (t/is (thrown? RuntimeException + (di/start `a {::param 42} mem))))) + + +(comment + + (require '[clj-async-profiler.core :as prof]) + (prof/serve-ui 8080) + + + (prof/profile {} + (dotimes [_ 1000000] + (di/start `a {::param 42}))) + + + + (let [mem (di/->memoize {::param 42})] + (prof/profile {} + (dotimes [_ 1000000] + (di/start `a mem)))) + + + (prof/generate-diffgraph 1 2 {}) + + + + (time + (dotimes [_ 10000] + (di/start `a {::param 42}))) + + (let [mem (di/->memoize {::param 42})] + (time + (dotimes [_ 10000] + (di/start `a mem)))) + + + ,,,) diff --git a/test/darkleaf/di/tutorial/x_inspect_test.clj b/test/darkleaf/di/tutorial/x_inspect_test.clj index c95141b6..cd9f029a 100644 --- a/test/darkleaf/di/tutorial/x_inspect_test.clj +++ b/test/darkleaf/di/tutorial/x_inspect_test.clj @@ -7,16 +7,22 @@ (defn implicit-root [key] {:key ::di/implicit-root - :dependencies {key :required} + :dependencies (seq {key :required + ::di/side-dependency :optional}) :description {::di/kind :ref :key key :type :required}}) +(defn side-dependency [] + {:key ::di/side-dependency + :description {::di/kind :undefined}}) + (t/deftest no-description-test (t/is (= [(implicit-root `foo) {:key `foo - #_"NOTE: no description as it is not implemented"}] + #_"NOTE: no description as it is not implemented"} + (side-dependency)] (di/inspect `foo {`foo (reify p/Factory (dependencies [_]) @@ -28,7 +34,8 @@ (t/deftest env-test (t/is (= [(implicit-root "FOO") {:key "FOO" - :description {::di/kind :env}}] + :description {::di/kind :env}} + (side-dependency)] (di/inspect "FOO")))) @@ -36,7 +43,8 @@ (t/is (= [(implicit-root "FOO") {:key "FOO" :description {::di/kind :trivial - :object "value"}}] + :object "value"}} + (side-dependency)] (di/inspect "FOO" {"FOO" "value"})))) @@ -47,7 +55,8 @@ {:key `variable :description {::di/kind :trivial :object :obj - ::di/variable #'variable}}] + ::di/variable #'variable}} + (side-dependency)] (di/inspect `variable)))) @@ -62,7 +71,8 @@ (t/is (= [(implicit-root `variable+factory) {:key `variable+factory :description {#_"NOTE: no description as it is not implemented" - ::di/variable #'variable+factory}}] + ::di/variable #'variable+factory}} + (side-dependency)] (di/inspect `variable+factory)))) @@ -80,7 +90,8 @@ (t/is (= [(implicit-root `variable+description) {:key `variable+description :description {::di/kind ::variable+description - ::di/variable #'variable+description}}] + ::di/variable #'variable+description}} + (side-dependency)] (di/inspect `variable+description)))) @@ -92,7 +103,8 @@ {:key `variable+template :description {::di/kind :template :template [42] - ::di/variable #'variable+template}}] + ::di/variable #'variable+template}} + (side-dependency)] (di/inspect `variable+template)))) @@ -105,7 +117,8 @@ (t/is (= [(implicit-root `component-0-arity) {:key `component-0-arity :description {::di/kind :component - ::di/variable #'component-0-arity}}] + ::di/variable #'component-0-arity}} + (side-dependency)] (di/inspect `component-0-arity)))) @@ -118,7 +131,8 @@ (t/is (= [(implicit-root `component-1-arity) {:key `component-1-arity :description {::di/kind :component - ::di/variable #'component-1-arity}}] + ::di/variable #'component-1-arity}} + (side-dependency)] (di/inspect `component-1-arity)))) @@ -131,7 +145,8 @@ (t/is (= [(implicit-root `service-0-arity) {:key `service-0-arity :description {::di/kind :service - ::di/variable #'service-0-arity}}] + ::di/variable #'service-0-arity}} + (side-dependency)] (di/inspect `service-0-arity)))) @@ -144,7 +159,8 @@ (t/is (= [(implicit-root `service-n-arity) {:key `service-n-arity :description {::di/kind :service - ::di/variable #'service-n-arity}}] + ::di/variable #'service-n-arity}} + (side-dependency)] (di/inspect `service-n-arity)))) @@ -156,7 +172,8 @@ (t/is (= [(implicit-root `multimethod-service) {:key `multimethod-service :description {::di/kind :service - ::di/variable #'multimethod-service}}] + ::di/variable #'multimethod-service}} + (side-dependency)] (di/inspect `multimethod-service)))) @@ -168,7 +185,8 @@ :key `bar :type :required}} {:key `bar - :description {::di/kind :undefined}}] + :description {::di/kind :undefined}} + (side-dependency)] (di/inspect `foo {`foo (di/ref `bar)})))) @@ -179,7 +197,8 @@ :description {::di/kind :template :template [42 (di/ref `bar)]}} {:key `bar - :description {::di/kind :undefined}}] + :description {::di/kind :undefined}} + (side-dependency)] (di/inspect `foo {`foo (di/template [42 (di/ref `bar)])})))) @@ -192,7 +211,8 @@ :f str :args ["arg"]}} {:key `bar - :description {::di/kind :undefined}}] + :description {::di/kind :undefined}} + (side-dependency)] (di/inspect `foo {`foo (di/derive `bar str "arg")})))) @@ -200,7 +220,8 @@ (t/is (= [(implicit-root `foo) {:key `foo :description {::di/kind :trivial - :object nil}}] + :object nil}} + (side-dependency)] (di/inspect `foo {`foo nil})))) @@ -208,7 +229,8 @@ (t/is (= [(implicit-root `foo) {:key `foo :description {::di/kind :trivial - :object str}}] + :object str}} + (side-dependency)] (di/inspect `foo {`foo str})))) @@ -238,7 +260,8 @@ :description {::di/kind :trivial :object "arg" ::di/update-key {:target `a - :role :arg}}}] + :role :arg}}} + (side-dependency)] (di/inspect `a {`a :obj} (di/update-key `a str "arg"))))) @@ -246,16 +269,19 @@ (t/deftest add-side-dependency-test (t/is (= [{:key ::di/implicit-root - :dependencies (seq {`a :required - `side-dep-1 :required - `side-dep-2 :required}) - :description {::di/kind :ref - :key `a - :type :required - ::di/side-dependencies [`side-dep-1 `side-dep-2]}} + :dependencies (seq {`a :required + ::di/side-dependency :optional}) + :description {::di/kind :ref + :key `a + :type :required}} {:key `a :description {::di/kind :trivial :object :obj}} + {:key ::di/side-dependency + :dependencies (seq {`side-dep-1 :required + `side-dep-2 :required}) + :description {::di/kind :undefined + ::di/side-dependencies [`side-dep-1 `side-dep-2]}} {:key `side-dep-1 :description {::di/kind :trivial :object :side-dep}} @@ -289,7 +315,8 @@ {:key `x-ns-publics-test/ok-test :description {::di/kind :trivial :object x-ns-publics-test/ok-test - ::di/variable #'x-ns-publics-test/ok-test}}] + ::di/variable #'x-ns-publics-test/ok-test}} + (side-dependency)] (di/inspect :ns-publics/darkleaf.di.tutorial.x-ns-publics-test (di/ns-publics))))) @@ -303,25 +330,23 @@ :cmap {:env.long parse-long}}} {:key "PORT" :description {::di/kind :trivial - :object "8080"}}] + :object "8080"}} + (side-dependency)] (di/inspect :env.long/PORT (di/env-parsing :env.long parse-long) {"PORT" "8080"})))) (t/deftest log-test - (t/is (= [{:key ::di/implicit-root - :dependencies {`foo :required} - :description {::di/kind :ref - :key `foo - :type :required - ::di/log {:will-be-logged true - #_#_:opts nil}}} + (t/is (= [(implicit-root `foo) {:key `foo :description {::di/kind :trivial :object :obj ::di/log {:will-be-logged true - #_#_:opts nil}}}] + #_#_:opts nil}}} + {:key ::di/side-dependency + :description {::di/kind :undefined + ::di/log {:will-be-logged true}}}] (di/inspect `foo {`foo :obj} (di/log))))) diff --git a/test/darkleaf/di/tutorial/x_log_test.clj b/test/darkleaf/di/tutorial/x_log_test.clj index 122277b9..efdd36f2 100644 --- a/test/darkleaf/di/tutorial/x_log_test.clj +++ b/test/darkleaf/di/tutorial/x_log_test.clj @@ -19,17 +19,17 @@ (t/deftest log (let [logs (atom []) after-build! (fn [{:keys [key object]}] - (swap! logs conj [:built key (pr-str object)])) + (when (some? object) + (swap! logs conj [:built key (pr-str object)]))) after-demolish! (fn [{:keys [key object]}] - (swap! logs conj [:demolished key (pr-str object)]))] + (when (some? object) + (swap! logs conj [:demolished key (pr-str object)])))] (with-open [root (di/start `c (di/log :after-build! after-build! :after-demolish! after-demolish!))]) (t/is (= [[:built `a ":a"] [:built `b "#darkleaf.di.core/service #'darkleaf.di.tutorial.x-log-test/b"] [:built `c ":c"] - [:built ::di/implicit-root ":c"] - [:demolished ::di/implicit-root ":c"] [:demolished `c ":c"] [:demolished `b "#darkleaf.di.core/service #'darkleaf.di.tutorial.x-log-test/b"] diff --git a/test/darkleaf/di/tutorial/z_memoize_test.clj.back b/test/darkleaf/di/tutorial/z_memoize_test.clj.back new file mode 100644 index 00000000..cd84556b --- /dev/null +++ b/test/darkleaf/di/tutorial/z_memoize_test.clj.back @@ -0,0 +1,54 @@ +(ns darkleaf.di.tutorial.z-memoize-test ;;todo: name + (:require + [clojure.test :as t] + [darkleaf.di.core :as di])) + +(defn connection + {::di/stop #(reset! % :stopped)} + [{url "CONNECTION_URL"}] + (atom :started)) + +(defn migrations + "A long running side effect" + {::di/kind :component} + [{connection `connection}] + #_ + (when (= :stopped @connection) + (throw (IllegalStateException. "Connection is not started"))) + (random-uuid)) + +(defn root + {::di/kind :component} + [{migrations `migrations}] + {:migrations migrations}) + +;; todo: check registry placement + +#_ +(t/deftest ok + (let [[cache global-system :as root] + (di/start [::di/cache `root] + {"CONNECTION_URL" "1"} + (di/collect-cache))] + + (with-open [local (di/start `root + (di/use-cache cache))] + (t/is (identical? global-system @local))) + + (with-open [local (di/start `root + (di/use-cache cache) + {"CONNECTION_URL" "1"})] + (t/is (identical? global-system @local))) + + (with-open [local (di/start `root + ;; `use-cache` should be the first registry + (di/use-cache cache) + {"CONNECTION_URL" "2"})] + (t/is (not (identical? global-system @local)))) + + + (di/stop root) + + (t/is (thrown? IllegalStateException + (di/start `root + (di/use-cache cache))))))