WIP: single-commit release flow

.github/workflows/build.yml

@@ -64,3 +64,38 @@ jobs:
     needs: test
     steps:
       - run: echo "All tests pass!"
+  deploy:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    needs: all-pr-checks
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+          eval $(/home/linuxbrew/.linuxbrew/bin/brew shellenv)
+          brew install clojure
+          clojure -T:build build :release true :source-date-epoch 1749762342
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ctim-jar
+          path: target/*.jar
+          if-no-files-found: error
+          compression-level: 0
+  attest:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    needs: deploy
+    permissions:
+      #id-token: write
+      contents: read
+      #attestations: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download jar
+        uses: actions/download-artifact@v4
+        with:
+          name: ctim-jar
+          path: ./target
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v2
+        with:
+          subject-path: 'PATH/TO/ARTIFACT'

.gitignore

@@ -17,3 +17,4 @@ pom.xml.asc
 .lsp/
 .dir-locals-2.el
 .clj-kondo/
+.cpcache

README.md

@@ -29,24 +29,12 @@ lein doc
 
 ## Releases
 
-```clojure
-# snapshot release
-lein deploy
-
-# for releases, set project.clj version to x.y.z-SNAPSHOT
-# this command then releases as x.y.z and bumps to x.y.(z+1)-SNAPSHOT
-# aliased as ./script/release.sh
-lein release :patch
-
-# if release fails partway through, use these commands to recover
-git tag --delete x.y.z
-# you might have a redundant commit "Version x.y.z", undo with:
-git reset --hard SHA_BEFORE_FAILED_RELEASE
-```
+To create a new release, call `./script/prep-release`. Push the result to a feature branch
+and merge the release to master. The commit message must include `[ctim-release]`.
 
 ## License
 
-Copyright © 2016-2024 Cisco Systems
+Copyright © 2016-2025 Cisco Systems
 
 Distributed under the Eclipse Public License either version 1.0 or (at
 your option) any later version.

build.clj

@@ -0,0 +1,178 @@
+(ns build
+  (:require [clojure.edn :as edn]
+            [clojure.set :as set]
+            [clojure.string :as str]
+            [clojure.tools.build.api :as b]
+            [clojure.pprint :as pp]
+            [clojure.java.io :as io]
+            [clojure.walk :as walk]
+            [clojure.java.shell :as sh]
+            clojure.tools.build.util.zip
+            [clj-commons.digest :as digest])
+  (:import
+    [java.io File InputStream OutputStream]
+    [java.nio.file Files LinkOption]
+    [java.nio.file.attribute BasicFileAttributes FileTime]
+    [java.util.zip ZipFile ZipInputStream ZipOutputStream ZipEntry]
+    [java.util.jar Manifest Attributes$Name]
+    [java.util.regex Pattern]))
+
+(defn artifact-version [params]
+  (let [{:keys [major minor schema release dev] :as m} (-> (io/file "resources/ctim/version.edn") slurp edn/read-string)]
+    (str major "."
+         minor "."
+         schema "."
+         (if (:release params) release (str dev "-SNAPSHOT")))))
+
+(def lib 'threatgrid/ctim)
+(def class-dir "target/classes")
+(def basis (delay (b/create-basis {:project "deps.edn"})))
+
+(defn clean [params]
+  (b/delete {:path "target"}))
+
+#_
+(defn- set-modification-times [{:keys [target source-date-epoch] :as params}]
+  (when source-date-epoch
+    (let [ms-epoch (* 1000 source-date-epoch)]
+      (run! #(File/.setLastModified % ms-epoch)
+            (file-seq (io/file target)))))
+  nil)
+
+#_
+(defn strip-nondeterminism [{:keys [jar-file] :as params}]
+  (set-modification-times (assoc params :target jar-file))
+  ;; sets last modification time of MANIFEST.MF
+  (let [{:keys [exit out err]} (sh/sh "strip-nondeterminism" jar-file)]
+    (when-not (zero? exit)
+      (println out)
+      (println err)
+      (throw (ex-info "strip-nondeterminism failed" {})))
+    params))
+
+(defn jar [{:keys [version] :as params}]
+  {:pre [version]}
+  (clean params)
+  (b/write-pom {:class-dir class-dir
+                :lib lib
+                :version version
+                :basis basis
+                :scm {:url "https://github.com/threatgrid/ctim"
+                      :connection "scm:git:git://github.com/threatgrid/ctim.git"
+                      :developerConnection "scm:git:git@github.com:threatgrid/ctim.git"
+                      :tag (if (str/ends-with? version "-SNAPSHOT")
+                             (b/git-process {:git-args "rev-parse HEAD"})
+                             ;; we commit the checksum of the jar to the repo ahead of time to ensure it is reproducible.
+                             ;; so it cannot contain the release SHA.
+                             ;; TODO include command to reproduce jar
+                             ;; TODO push sha to maven-metadata.xml
+                             version)}
+                ;;TODO copy from basis
+                :src-dirs ["src"]})
+  (b/copy-dir {;;TODO copy from basis
+               :src-dirs ["src" "doc"]
+               :target-dir class-dir})
+  (let [jar-file (format "target/%s-%s.jar" (name lib) version)]
+    (b/jar {:class-dir class-dir
+            :jar-file jar-file
+            :manifest {"Build-Jdk-Spec" "8"}})
+    (-> params
+        (assoc :jar-file jar-file)
+        #_
+        strip-nondeterminism)))
+
+(defn tag-release [{:keys [version] :as params}]
+  {:pre [version]}
+  (b/git-process {:git-args (format "tag %s" version)})
+  params)
+
+(defn infer-version [{:keys [release] :as params}]
+  (-> params
+      (update :version #(or % (artifact-version params)))))
+
+(defn build
+  "If current commit starts with [ctim-release], installs release version.
+  Otherwise snapshot version.
+
+  :release true/false to force release/snapshot
+  :version <string> to override artifact version
+  :source-date-epoch <seconds-from-epoch> to set SOURCE_DATE_EPOCH"
+  [{:keys [source-date-epoch print-params] :as params}]
+  (if (or (not source-date-epoch)
+          (= (some-> (System/getenv "SOURCE_DATE_EPOCH") parse-long)
+             source-date-epoch))
+    (-> params
+        (update :release #(if (boolean? %)
+                            %
+                            (if-some [msg (not-empty (b/git-process {:git-args "show-branch --no-name HEAD"}))]
+                              (str/starts-with? msg "[ctim-release]")
+                              (throw (ex-info "Could not determine current commit. Use clojure -T:build build :release true/false." {})))))
+        infer-version
+        jar
+        (cond->
+          print-params (doto prn)))
+    ;;FIXME support passing SOURCE_DATE_EPOCH to tools.build via parameter
+    (do (assert (set/subset? (-> params keys set) #{:release :version :source-date-epoch})
+                (set/difference #{:release :version :source-date-epoch} (-> params keys set)))
+        (println "Launching subprocess to set SOURCE_DATE_EPOCH...")
+        (let [{:keys [out exit err]} (apply sh/sh
+                                            (cond-> ["clojure" "-T:build" "build" ":print-params" "true" ":source-date-epoch" (str source-date-epoch)]
+                                              (boolean? (:release params)) (conj ":release" (str (:release params)))
+                                              (:version params) (conj ":version" (pr-str (:version params)))
+                                              true (conj :env (assoc (into {} (System/getenv)) "SOURCE_DATE_EPOCH" (str source-date-epoch)))))]
+          (some-> out str/trim not-empty print)
+          (binding [*out* *err*] (some-> err str/trim not-empty print))
+          (assert (zero? exit) exit)
+          (into params (edn/read-string out))))))
+
+(defn- serialize [m]
+  (binding [*print-namespace-maps* false
+            *print-level* nil
+            *print-length* nil]
+    (with-out-str
+      (pp/pprint
+        (walk/postwalk
+          (fn [m]
+            (cond
+              (map? m) (into (sorted-map) m)
+              (set? m) (into (sorted-set) m)
+              :else m))
+          m)))))
+
+(defn checksums [file]
+  {;; clojars uses these
+   :md5 (digest/md5 file)
+   :sha-1 (digest/sha-1 file)
+   ;; something stronger
+   :sha3-512 (digest/sha3-512 file)})
+
+(defn update-reproducible-releases [{:keys [version jar-file source-date-epoch] :as params}]
+  (let [cs (checksums (io/file jar-file))
+        prev-releases (-> "reproducible-releases.edn"
+                           io/file
+                           slurp
+                           edn/read-string)
+        new-releases (assoc prev-releases version {:git-tag version
+                                                   :source-date-epoch source-date-epoch
+                                                   :reproduction {:commands (str "clojure -T:build build :release true :source-date-epoch " source-date-epoch)
+                                                                  :artifact->checksums {jar-file cs}}})]
+    (spit "reproducible-releases.edn" (serialize new-releases))))
+
+;;FIXME still not reproducible, even with SOURCE_DATE_EPOCH
+(defn schedule-release [params]
+  (let [source-date-epoch (or (:source-date-epoch params)
+                              (some-> (System/getenv "SOURCE_DATE_EPOCH") parse-long)
+                              (.getEpochSecond (java.time.Instant/now)))
+        {:keys [version] :as params} (-> params
+                                         (assoc :release true
+                                                :source-date-epoch source-date-epoch)
+                                         build
+                                         update-reproducible-releases)]
+    ;; TODO gen docs to next stable version. keep them until the next stable version
+    #_#_
+    (println (b/git-process {:git-args "add ."}))
+    (println (b/git-process {:git-args (format "commit -m '[ctim-release] %s'" version)}))
+    params))
+
+(defn perform-release [params]
+  (tag-release params))

deploy.clj

@@ -0,0 +1,40 @@
+(ns deploy
+  (:require [clojure.tools.build.api :as b]
+            [clojure.java.io :as io]
+            [deps-deploy.deps-deploy :as deploy])
+  (:import (java.util.jar JarFile)))
+
+(defn- tag-release [{:keys [version] :as params}]
+  {:pre [version]}
+  (b/git-process {:git-args (format "tag %s" version)})
+  (b/git-process {:git-args (format "push tag %s" version)})
+  params)
+
+(defn- deploy* [{:keys [jar-file pom-file] :as params}]
+  (deploy/deploy
+    {:installer :remote
+     :sign-releases? false
+     :artifact (str jar-file)
+     :pom-file pom-file})
+  params)
+
+(defn extract-pom [{:keys [jar-file] :as params}]
+  (assert jar-file "Must supply :jar-file")
+  (let [pom-file "pom.xml"
+        jf (JarFile. (io/file (str jar-file)))]
+    (spit pom-file
+          (slurp (.getInputStream jf (.getEntry jf "META-INF/maven/threatgrid/ctim/pom.xml"))))
+    (assoc params :pom-file pom-file)))
+
+(defn- infer-version [{:keys []}])
+
+(defn deploy-snapshot [params]
+  (-> params
+      extract-pom
+      deploy*))
+
+(defn deploy-release [params]
+  (-> params
+      extract-pom
+      tag-release
+      deploy*))

deps.edn

@@ -0,0 +1,36 @@
+{:paths ["src" "doc" "resources"]
+ :deps {org.clojure/clojure {:mvn/version "1.11.3"}
+        prismatic/schema {:mvn/version "1.2.0"}
+        com.google.protobuf/protobuf-java {:mvn/version "3.7.1"} ;clj-momo > org.clojure/clojurescript
+        threatgrid/clj-momo {:mvn/version "0.3.5"
+                             :exclusions [;flanders > threatgrid/clj-momo
+                                          metosin/schema-tools]}
+        org.mozilla/rhino {:mvn/version "1.7.7.1"} ;threatgrid/flanders > kovacnica/clojure.network.ip
+        threatgrid/flanders {:mvn/version "1.0.2"}
+        metosin/ring-swagger {:mvn/version "1.0.0"}
+        org.clojure/test.check {:mvn/version "1.1.1"}
+        com.gfredericks/test.chuck {:mvn/version "0.2.13"}
+        prismatic/schema-generators {:mvn/version "0.1.3"}
+        kovacnica/clojure.network.ip {:mvn/version "0.1.3"}}
+ :aliases {:dev {:extra-paths ["dev"]
+                 :extra-deps {;https://clojure.atlassian.net/browse/CLJS-3047
+                              com.google.errorprone/error_prone_annotations {:mvn/version "2.1.3"}
+                              ;;https://clojure.atlassian.net/browse/CLJS-3047
+                              com.google.code.findbugs/jsr305 {:mvn/version "3.0.2"}
+                              org.clojure/clojurescript {:mvn/version "1.10.597"}}}
+           :build {:replace-deps {io.github.clojure/tools.build {:git/url "https://github.com/frenchy64/tools.build.git"
+                                                                 :git/sha "d71a9140cff5a1efecf8793fac29b2b7d3f77b45"}
+                                  org.clj-commons/digest {:git/url "https://github.com/clj-commons/digest.git"
+                                                          :git/tag "Release-1.4.100"
+                                                          :git/sha "bec1e0e6b887bdb408674f0025357cc49b02b434"}}
+                   :ns-default build}
+           ;; Generate an example: clojure -M:gen actor
+           :gen {:main-opts ["run" "-m" "ctim.generate"]}
+           :deploy {:replace-deps {io.github.clojure/tools.build {:git/url "https://github.com/frenchy64/tools.build.git"
+                                                                  :git/sha "d71a9140cff5a1efecf8793fac29b2b7d3f77b45"}
+                                   slipset/deps-deploy {:git/url "https://github.com/slipset/deps-deploy.git"
+                                                        :git/sha "07022b92d768590ab25b9ceb619ef17d2922da9a"}}
+                    :ns-default deploy}
+           :print-version {:main-opts ["-m" "ctim.print-version"]}
+           :prep-release {:main-opts ["-m" "ctim.prep-release"]}
+           :doc {:main-opts ["-m" "ctim.document"]}}}

dev/ctim/prep_release.clj

@@ -0,0 +1,32 @@
+(ns ctim.prep-release
+  (:require [ctim.version :refer [-ctim-version]]
+            [clojure.java.shell :as sh]
+            [clojure.string :as str]))
+
+(defn- sh [& args]
+  (let [{:keys [exit out err]} (apply sh/sh args)]
+    (some-> out str/trim not-empty println)
+    (some-> err str/trim not-empty println)
+    (assert (zero? exit))))
+
+(defn assert-clean []
+  (let [{:keys [out exit]} (sh/sh "git" "status" "--short")]
+    (assert (zero? exit))
+    (when (seq out)
+      (println out)
+      (println "Working directory is not clean, please commit changes first.")
+      (System/exit 1))))
+
+(defn -main []
+  (assert-clean)
+  (loop [prev-version nil
+         version (-ctim-version)]
+    (if (= prev-version version)
+      (do (sh "./script/doc")
+          (sh "git" "add" ".")
+          (sh "git" "commit" "--allow-empty" "-m" (format "[ctim-release] %s" version))
+          (System/exit 0))
+      (do (println (str "Press enter to prepare release `" (-ctim-version) "`. To customize, update `resources/ctim/version.edn`, then press enter."))
+          (read-line)
+          (recur version
+                 (-ctim-version))))))

dev/ctim/print_version.clj

@@ -0,0 +1,5 @@
+(ns ctim.print-version
+  (:require [ctim.version :refer [-ctim-version]]))
+
+(defn -main []
+  (print (-ctim-version)))