diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9308b7f2..d25b44f3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,3 +64,38 @@ jobs: needs: test steps: - run: echo "All tests pass!" + deploy: + runs-on: ubuntu-22.04 + timeout-minutes: 10 + needs: all-pr-checks + steps: + - uses: actions/checkout@v4 + - run: | + eval $(/home/linuxbrew/.linuxbrew/bin/brew shellenv) + brew install clojure + clojure -T:build build :release true :source-date-epoch 1749762342 + - uses: actions/upload-artifact@v4 + with: + name: ctim-jar + path: target/*.jar + if-no-files-found: error + compression-level: 0 + attest: + runs-on: ubuntu-22.04 + timeout-minutes: 10 + needs: deploy + permissions: + #id-token: write + contents: read + #attestations: write + steps: + - uses: actions/checkout@v4 + - name: Download jar + uses: actions/download-artifact@v4 + with: + name: ctim-jar + path: ./target + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v2 + with: + subject-path: 'PATH/TO/ARTIFACT' diff --git a/.gitignore b/.gitignore index 3d75c442..8056779a 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ pom.xml.asc .lsp/ .dir-locals-2.el .clj-kondo/ +.cpcache diff --git a/README.md b/README.md index 4e2eaa21..526d8fff 100644 --- a/README.md +++ b/README.md @@ -29,24 +29,12 @@ lein doc ## Releases -```clojure -# snapshot release -lein deploy - -# for releases, set project.clj version to x.y.z-SNAPSHOT -# this command then releases as x.y.z and bumps to x.y.(z+1)-SNAPSHOT -# aliased as ./script/release.sh -lein release :patch - -# if release fails partway through, use these commands to recover -git tag --delete x.y.z -# you might have a redundant commit "Version x.y.z", undo with: -git reset --hard SHA_BEFORE_FAILED_RELEASE -``` +To create a new release, call `./script/prep-release`. Push the result to a feature branch +and merge the release to master. The commit message must include `[ctim-release]`. ## License -Copyright © 2016-2024 Cisco Systems +Copyright © 2016-2025 Cisco Systems Distributed under the Eclipse Public License either version 1.0 or (at your option) any later version. diff --git a/build.clj b/build.clj new file mode 100644 index 00000000..72bdc9a9 --- /dev/null +++ b/build.clj @@ -0,0 +1,178 @@ +(ns build + (:require [clojure.edn :as edn] + [clojure.set :as set] + [clojure.string :as str] + [clojure.tools.build.api :as b] + [clojure.pprint :as pp] + [clojure.java.io :as io] + [clojure.walk :as walk] + [clojure.java.shell :as sh] + clojure.tools.build.util.zip + [clj-commons.digest :as digest]) + (:import + [java.io File InputStream OutputStream] + [java.nio.file Files LinkOption] + [java.nio.file.attribute BasicFileAttributes FileTime] + [java.util.zip ZipFile ZipInputStream ZipOutputStream ZipEntry] + [java.util.jar Manifest Attributes$Name] + [java.util.regex Pattern])) + +(defn artifact-version [params] + (let [{:keys [major minor schema release dev] :as m} (-> (io/file "resources/ctim/version.edn") slurp edn/read-string)] + (str major "." + minor "." + schema "." + (if (:release params) release (str dev "-SNAPSHOT"))))) + +(def lib 'threatgrid/ctim) +(def class-dir "target/classes") +(def basis (delay (b/create-basis {:project "deps.edn"}))) + +(defn clean [params] + (b/delete {:path "target"})) + +#_ +(defn- set-modification-times [{:keys [target source-date-epoch] :as params}] + (when source-date-epoch + (let [ms-epoch (* 1000 source-date-epoch)] + (run! #(File/.setLastModified % ms-epoch) + (file-seq (io/file target))))) + nil) + +#_ +(defn strip-nondeterminism [{:keys [jar-file] :as params}] + (set-modification-times (assoc params :target jar-file)) + ;; sets last modification time of MANIFEST.MF + (let [{:keys [exit out err]} (sh/sh "strip-nondeterminism" jar-file)] + (when-not (zero? exit) + (println out) + (println err) + (throw (ex-info "strip-nondeterminism failed" {}))) + params)) + +(defn jar [{:keys [version] :as params}] + {:pre [version]} + (clean params) + (b/write-pom {:class-dir class-dir + :lib lib + :version version + :basis basis + :scm {:url "https://github.com/threatgrid/ctim" + :connection "scm:git:git://github.com/threatgrid/ctim.git" + :developerConnection "scm:git:git@github.com:threatgrid/ctim.git" + :tag (if (str/ends-with? version "-SNAPSHOT") + (b/git-process {:git-args "rev-parse HEAD"}) + ;; we commit the checksum of the jar to the repo ahead of time to ensure it is reproducible. + ;; so it cannot contain the release SHA. + ;; TODO include command to reproduce jar + ;; TODO push sha to maven-metadata.xml + version)} + ;;TODO copy from basis + :src-dirs ["src"]}) + (b/copy-dir {;;TODO copy from basis + :src-dirs ["src" "doc"] + :target-dir class-dir}) + (let [jar-file (format "target/%s-%s.jar" (name lib) version)] + (b/jar {:class-dir class-dir + :jar-file jar-file + :manifest {"Build-Jdk-Spec" "8"}}) + (-> params + (assoc :jar-file jar-file) + #_ + strip-nondeterminism))) + +(defn tag-release [{:keys [version] :as params}] + {:pre [version]} + (b/git-process {:git-args (format "tag %s" version)}) + params) + +(defn infer-version [{:keys [release] :as params}] + (-> params + (update :version #(or % (artifact-version params))))) + +(defn build + "If current commit starts with [ctim-release], installs release version. + Otherwise snapshot version. + + :release true/false to force release/snapshot + :version to override artifact version + :source-date-epoch to set SOURCE_DATE_EPOCH" + [{:keys [source-date-epoch print-params] :as params}] + (if (or (not source-date-epoch) + (= (some-> (System/getenv "SOURCE_DATE_EPOCH") parse-long) + source-date-epoch)) + (-> params + (update :release #(if (boolean? %) + % + (if-some [msg (not-empty (b/git-process {:git-args "show-branch --no-name HEAD"}))] + (str/starts-with? msg "[ctim-release]") + (throw (ex-info "Could not determine current commit. Use clojure -T:build build :release true/false." {}))))) + infer-version + jar + (cond-> + print-params (doto prn))) + ;;FIXME support passing SOURCE_DATE_EPOCH to tools.build via parameter + (do (assert (set/subset? (-> params keys set) #{:release :version :source-date-epoch}) + (set/difference #{:release :version :source-date-epoch} (-> params keys set))) + (println "Launching subprocess to set SOURCE_DATE_EPOCH...") + (let [{:keys [out exit err]} (apply sh/sh + (cond-> ["clojure" "-T:build" "build" ":print-params" "true" ":source-date-epoch" (str source-date-epoch)] + (boolean? (:release params)) (conj ":release" (str (:release params))) + (:version params) (conj ":version" (pr-str (:version params))) + true (conj :env (assoc (into {} (System/getenv)) "SOURCE_DATE_EPOCH" (str source-date-epoch)))))] + (some-> out str/trim not-empty print) + (binding [*out* *err*] (some-> err str/trim not-empty print)) + (assert (zero? exit) exit) + (into params (edn/read-string out)))))) + +(defn- serialize [m] + (binding [*print-namespace-maps* false + *print-level* nil + *print-length* nil] + (with-out-str + (pp/pprint + (walk/postwalk + (fn [m] + (cond + (map? m) (into (sorted-map) m) + (set? m) (into (sorted-set) m) + :else m)) + m))))) + +(defn checksums [file] + {;; clojars uses these + :md5 (digest/md5 file) + :sha-1 (digest/sha-1 file) + ;; something stronger + :sha3-512 (digest/sha3-512 file)}) + +(defn update-reproducible-releases [{:keys [version jar-file source-date-epoch] :as params}] + (let [cs (checksums (io/file jar-file)) + prev-releases (-> "reproducible-releases.edn" + io/file + slurp + edn/read-string) + new-releases (assoc prev-releases version {:git-tag version + :source-date-epoch source-date-epoch + :reproduction {:commands (str "clojure -T:build build :release true :source-date-epoch " source-date-epoch) + :artifact->checksums {jar-file cs}}})] + (spit "reproducible-releases.edn" (serialize new-releases)))) + +;;FIXME still not reproducible, even with SOURCE_DATE_EPOCH +(defn schedule-release [params] + (let [source-date-epoch (or (:source-date-epoch params) + (some-> (System/getenv "SOURCE_DATE_EPOCH") parse-long) + (.getEpochSecond (java.time.Instant/now))) + {:keys [version] :as params} (-> params + (assoc :release true + :source-date-epoch source-date-epoch) + build + update-reproducible-releases)] + ;; TODO gen docs to next stable version. keep them until the next stable version + #_#_ + (println (b/git-process {:git-args "add ."})) + (println (b/git-process {:git-args (format "commit -m '[ctim-release] %s'" version)})) + params)) + +(defn perform-release [params] + (tag-release params)) diff --git a/deploy.clj b/deploy.clj new file mode 100644 index 00000000..aecfcf29 --- /dev/null +++ b/deploy.clj @@ -0,0 +1,40 @@ +(ns deploy + (:require [clojure.tools.build.api :as b] + [clojure.java.io :as io] + [deps-deploy.deps-deploy :as deploy]) + (:import (java.util.jar JarFile))) + +(defn- tag-release [{:keys [version] :as params}] + {:pre [version]} + (b/git-process {:git-args (format "tag %s" version)}) + (b/git-process {:git-args (format "push tag %s" version)}) + params) + +(defn- deploy* [{:keys [jar-file pom-file] :as params}] + (deploy/deploy + {:installer :remote + :sign-releases? false + :artifact (str jar-file) + :pom-file pom-file}) + params) + +(defn extract-pom [{:keys [jar-file] :as params}] + (assert jar-file "Must supply :jar-file") + (let [pom-file "pom.xml" + jf (JarFile. (io/file (str jar-file)))] + (spit pom-file + (slurp (.getInputStream jf (.getEntry jf "META-INF/maven/threatgrid/ctim/pom.xml")))) + (assoc params :pom-file pom-file))) + +(defn- infer-version [{:keys []}]) + +(defn deploy-snapshot [params] + (-> params + extract-pom + deploy*)) + +(defn deploy-release [params] + (-> params + extract-pom + tag-release + deploy*)) diff --git a/deps.edn b/deps.edn new file mode 100644 index 00000000..1331d4ad --- /dev/null +++ b/deps.edn @@ -0,0 +1,36 @@ +{:paths ["src" "doc" "resources"] + :deps {org.clojure/clojure {:mvn/version "1.11.3"} + prismatic/schema {:mvn/version "1.2.0"} + com.google.protobuf/protobuf-java {:mvn/version "3.7.1"} ;clj-momo > org.clojure/clojurescript + threatgrid/clj-momo {:mvn/version "0.3.5" + :exclusions [;flanders > threatgrid/clj-momo + metosin/schema-tools]} + org.mozilla/rhino {:mvn/version "1.7.7.1"} ;threatgrid/flanders > kovacnica/clojure.network.ip + threatgrid/flanders {:mvn/version "1.0.2"} + metosin/ring-swagger {:mvn/version "1.0.0"} + org.clojure/test.check {:mvn/version "1.1.1"} + com.gfredericks/test.chuck {:mvn/version "0.2.13"} + prismatic/schema-generators {:mvn/version "0.1.3"} + kovacnica/clojure.network.ip {:mvn/version "0.1.3"}} + :aliases {:dev {:extra-paths ["dev"] + :extra-deps {;https://clojure.atlassian.net/browse/CLJS-3047 + com.google.errorprone/error_prone_annotations {:mvn/version "2.1.3"} + ;;https://clojure.atlassian.net/browse/CLJS-3047 + com.google.code.findbugs/jsr305 {:mvn/version "3.0.2"} + org.clojure/clojurescript {:mvn/version "1.10.597"}}} + :build {:replace-deps {io.github.clojure/tools.build {:git/url "https://github.com/frenchy64/tools.build.git" + :git/sha "d71a9140cff5a1efecf8793fac29b2b7d3f77b45"} + org.clj-commons/digest {:git/url "https://github.com/clj-commons/digest.git" + :git/tag "Release-1.4.100" + :git/sha "bec1e0e6b887bdb408674f0025357cc49b02b434"}} + :ns-default build} + ;; Generate an example: clojure -M:gen actor + :gen {:main-opts ["run" "-m" "ctim.generate"]} + :deploy {:replace-deps {io.github.clojure/tools.build {:git/url "https://github.com/frenchy64/tools.build.git" + :git/sha "d71a9140cff5a1efecf8793fac29b2b7d3f77b45"} + slipset/deps-deploy {:git/url "https://github.com/slipset/deps-deploy.git" + :git/sha "07022b92d768590ab25b9ceb619ef17d2922da9a"}} + :ns-default deploy} + :print-version {:main-opts ["-m" "ctim.print-version"]} + :prep-release {:main-opts ["-m" "ctim.prep-release"]} + :doc {:main-opts ["-m" "ctim.document"]}}} diff --git a/dev/ctim/prep_release.clj b/dev/ctim/prep_release.clj new file mode 100644 index 00000000..b3be22a3 --- /dev/null +++ b/dev/ctim/prep_release.clj @@ -0,0 +1,32 @@ +(ns ctim.prep-release + (:require [ctim.version :refer [-ctim-version]] + [clojure.java.shell :as sh] + [clojure.string :as str])) + +(defn- sh [& args] + (let [{:keys [exit out err]} (apply sh/sh args)] + (some-> out str/trim not-empty println) + (some-> err str/trim not-empty println) + (assert (zero? exit)))) + +(defn assert-clean [] + (let [{:keys [out exit]} (sh/sh "git" "status" "--short")] + (assert (zero? exit)) + (when (seq out) + (println out) + (println "Working directory is not clean, please commit changes first.") + (System/exit 1)))) + +(defn -main [] + (assert-clean) + (loop [prev-version nil + version (-ctim-version)] + (if (= prev-version version) + (do (sh "./script/doc") + (sh "git" "add" ".") + (sh "git" "commit" "--allow-empty" "-m" (format "[ctim-release] %s" version)) + (System/exit 0)) + (do (println (str "Press enter to prepare release `" (-ctim-version) "`. To customize, update `resources/ctim/version.edn`, then press enter.")) + (read-line) + (recur version + (-ctim-version)))))) diff --git a/dev/ctim/print_version.clj b/dev/ctim/print_version.clj new file mode 100644 index 00000000..ec23bfaa --- /dev/null +++ b/dev/ctim/print_version.clj @@ -0,0 +1,5 @@ +(ns ctim.print-version + (:require [ctim.version :refer [-ctim-version]])) + +(defn -main [] + (print (-ctim-version))) diff --git a/project.clj b/project.clj index e7644df9..866610f6 100644 --- a/project.clj +++ b/project.clj @@ -1,8 +1,8 @@ (defproject threatgrid/ctim "1.3.27-SNAPSHOT" :description "Cisco Threat Intelligence Model" - :url "http://github.com/threatgrid/ctim" + :url "https://github.com/threatgrid/ctim" :license {:name "Eclipse Public License" - :url "http://www.eclipse.org/legal/epl-v10.html"} + :url "https://www.eclipse.org/legal/epl-v10.html"} :pedantic? :abort :dependencies [[org.clojure/clojure "1.11.3"] [prismatic/schema "1.2.0"] @@ -18,19 +18,11 @@ [prismatic/schema-generators "0.1.3"] [kovacnica/clojure.network.ip "0.1.3"]] - :uberjar-name "ctim.jar" - :resource-paths ["doc"] + :resource-paths ["resources" "doc"] :plugins [[lein-cljsbuild "1.1.7"] [com.google.guava/guava "20.0"] ;resolve internal conflict in `lein-doo` - [lein-doo "0.1.11" :exclusions [org.clojure/clojure]] - ;;uncomment for lein-git-down - #_[reifyhealth/lein-git-down "0.3.5"]] - - ;;uncomment for lein-git-down - ;:middleware [lein-git-down.plugin/inject-properties] - ;:repositories [["public-github" {:url "git://github.com"}] - ; ["private-github" {:url "git://github.com" :protocol :ssh}]] + [lein-doo "0.1.11" :exclusions [org.clojure/clojure]]] :release-tasks [["clean"] ["doc"] @@ -70,7 +62,7 @@ :pretty-print true}}}} :test-selectors {:no-gen #(not (:gen %))} :global-vars {*warn-on-reflection* true} - :profiles {:provided + :profiles {:dev {:dependencies [;https://clojure.atlassian.net/browse/CLJS-3047 [com.google.errorprone/error_prone_annotations "2.1.3"] ;;https://clojure.atlassian.net/browse/CLJS-3047 diff --git a/release-checksums.edn b/release-checksums.edn new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/release-checksums.edn @@ -0,0 +1 @@ +{} diff --git a/reproducible-releases.edn b/reproducible-releases.edn new file mode 100644 index 00000000..a7a7c328 --- /dev/null +++ b/reproducible-releases.edn @@ -0,0 +1,12 @@ +{"1.3.27.1" + {:git-tag "1.3.27.1", + :reproduction + {:artifact->checksums + {"target/ctim-1.3.27.1.jar" + {:md5 "09f9c55ce09cd4a96aadd9e190aee3f2", + :sha-1 "f1fe7619ec4f75575f6125322881f026519d2a96", + :sha3-512 + "932b095acf82d488721e69efeb63bbdd4bb7637edc5a3fe3856707d33d7b15bbf8b09735106593e534d495934c46b7b705361f305be61c16a9463328033f539b"}}, + :commands + "clojure -T:build build :release true :source-date-epoch 1749762342"}, + :source-date-epoch 1749762342}} diff --git a/resources/ctim/version.edn b/resources/ctim/version.edn new file mode 100644 index 00000000..adfdf27e --- /dev/null +++ b/resources/ctim/version.edn @@ -0,0 +1,4 @@ +;; ctim-schema-version: => major.minor.schema +;; maven release version => major.minor.schema.release +;; maven snapshot version => major.minor.schema.dev-SNAPSHOT +{:major 1 :minor 3 :schema 27 :release 1 :dev 2} diff --git a/script/doc b/script/doc new file mode 100755 index 00000000..93f8989f --- /dev/null +++ b/script/doc @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Generate docs for the latest stable ctim-schema-version. + +clojure -M:doc diff --git a/script/prep-release b/script/prep-release new file mode 100755 index 00000000..e962e535 --- /dev/null +++ b/script/prep-release @@ -0,0 +1,3 @@ +#!/bin/bash + +clojure -M:dev:prep-release diff --git a/src/ctim/schemas/common.cljc b/src/ctim/schemas/common.cljc index 54880e34..829e7b88 100644 --- a/src/ctim/schemas/common.cljc +++ b/src/ctim/schemas/common.cljc @@ -1,5 +1,7 @@ (ns ctim.schemas.common (:refer-clojure :exclude [ref uri?]) + #?(:cljs + (:require-macros [ctim.version :refer [ctim-version]])) (:require [clj-momo.lib.clj-time.coerce :refer [to-long]] [clojure.set :refer [map-invert]] #?(:clj [clojure.spec.alpha :as cs] @@ -16,16 +18,18 @@ :cljs [flanders.core :as f :refer-macros [def-map-type def-enum-type def-eq]]) + #?(:clj [ctim.version :refer [ctim-version]]) [flanders.navigation :as fn] [flanders.predicates :as fp] [clojure.string :as str])) -(def ctim-schema-version "1.3.27") +;; do not edit -- use ./script/update-version +(def ctim-schema-version "1.3.26") (def-eq CTIMSchemaVersion ctim-schema-version) (cs/def ::ctim-schema-version - #(re-matches #"\w+.\w+\.\w+" %)) + #(re-matches #"\d+\.\d+\.\d+" %)) (def SchemaVersion (f/str diff --git a/src/ctim/version.clj b/src/ctim/version.clj new file mode 100644 index 00000000..a7963f3b --- /dev/null +++ b/src/ctim/version.clj @@ -0,0 +1,14 @@ +(ns ctim.version + (:require [clojure.edn :as edn] + [clojure.java.io :as io] + [clojure.string :as str])) + +(defn -ctim-schema-version [] + (let [{:keys [major minor schema]} (-> (io/resource "ctim/version.edn") slurp edn/read-string)] + (str major "." minor "." schema))) + +(defmacro ctim-version + "This is a macro to support cljs compile-time inlining of the version string + from the JVM resource." + [] + (-ctim-schema-version))