From fb226528831f252663c3cf6adb53e96fcc396b73 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Thu, 19 Dec 2024 10:32:47 -0500
Subject: [PATCH 01/51] Rebased.

---
 CMake/FindEnzyme.cmake                    | 186 ++++++++++++++++++++++
 CMakeLists.txt                            |   8 +-
 Examples/CMakeLists.txt                   |   4 +
 Examples/Enzyme/CMakeLists.txt            |   2 +
 Examples/Enzyme/Library/CMakeLists.txt    |   4 +
 Examples/Enzyme/Library/library.cpp       |   3 +
 Examples/Enzyme/Library/library.hpp       |   2 +
 Examples/Enzyme/Library/main.cpp          |  10 ++
 Examples/Enzyme/Standalone/CMakeLists.txt |   4 +
 Examples/Enzyme/Standalone/main.cpp       |  17 ++
 10 files changed, 239 insertions(+), 1 deletion(-)
 create mode 100644 CMake/FindEnzyme.cmake
 create mode 100644 Examples/Enzyme/CMakeLists.txt
 create mode 100644 Examples/Enzyme/Library/CMakeLists.txt
 create mode 100644 Examples/Enzyme/Library/library.cpp
 create mode 100644 Examples/Enzyme/Library/library.hpp
 create mode 100644 Examples/Enzyme/Library/main.cpp
 create mode 100644 Examples/Enzyme/Standalone/CMakeLists.txt
 create mode 100644 Examples/Enzyme/Standalone/main.cpp

diff --git a/CMake/FindEnzyme.cmake b/CMake/FindEnzyme.cmake
new file mode 100644
index 000000000..8974ed7bb
--- /dev/null
+++ b/CMake/FindEnzyme.cmake
@@ -0,0 +1,186 @@
+# 
+# Copyright (c) 2017, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+# Written by Slaven Peles <peles2@llnl.gov>.
+# LLNL-CODE-718378.
+# All rights reserved.
+# 
+# This file is part of GridKit™. For details, see github.com/LLNL/GridKit 
+# Please also read the LICENSE file. 
+# 
+# Redistribution and use in source and binary forms, with or without 
+# modification, are permitted provided that the following conditions are met:
+# - Redistributions of source code must retain the above copyright notice, 
+#   this list of conditions and the disclaimer below.
+# - Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the disclaimer (as noted below) in the 
+#   documentation and/or other materials provided with the distribution.
+# - Neither the name of the LLNS/LLNL nor the names of its contributors may 
+#   be used to endorse or promote products derived from this software without 
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL
+# SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISINGIN ANY 
+# WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+# THE POSSIBILITY OF SUCH DAMAGE.
+# 
+# Lawrence Livermore National Laboratory is operated by Lawrence Livermore 
+# National Security, LLC, for the U.S. Department of Energy, National 
+# Nuclear Security Administration under Contract DE-AC52-07NA27344.
+# 
+# This document was prepared as an account of work sponsored by an agency 
+# of the United States government. Neither the United States government nor 
+# Lawrence Livermore National Security, LLC, nor any of their employees 
+# makes any warranty, expressed or implied, or assumes any legal liability 
+# or responsibility for the accuracy, completeness, or usefulness of any 
+# information, apparatus, product, or process disclosed, or represents that 
+# its use would not infringe privately owned rights. Reference herein to 
+# any specific commercial product, process, or service by trade name, 
+# trademark, manufacturer, or otherwise does not necessarily constitute or 
+# imply its endorsement, recommendation, or favoring by the United States 
+# government or Lawrence Livermore National Security, LLC. The views and 
+# opinions of authors expressed herein do not necessarily state or reflect 
+# those of the United States government or Lawrence Livermore National 
+# Security, LLC, and shall not be used for advertising or product 
+# endorsement purposes. 
+# 
+
+#[[
+
+Finds Enzyme Clang plugin
+
+User may set:
+- ENZYME_DIR
+
+Author(s):
+- Asher Mancinelli <ashermancinelli@gmail.com>
+
+]]
+
+if(NOT ${CMAKE_C_COMPILER_ID} STREQUAL "Clang" OR NOT ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
+  message(FATAL_ERROR "Enzyme may only be enabled if building with Clang")
+endif()
+
+find_library(ENZYME_LLVM_PLUGIN_LIBRARY
+  NAMES
+  LLVMEnzyme-15.so
+  LLVMEnzyme-14.so
+  LLVMEnzyme-13.so
+  LLVMEnzyme-12.so
+  LLVMEnzyme-11.so
+  LLVMEnzyme-10.so
+  LLVMEnzyme-9.so
+  PATHS
+  ${ENZYME_DIR} $ENV{ENZYME_DIR}
+  ENV LD_LIBRARY_PATH ENV DYLD_LIBRARY_PATH
+  PATH_SUFFIXES
+  lib64 lib)
+
+find_library(ENZYME_CLANG_PLUGIN_LIBRARY
+  NAMES
+  ClangEnzyme-15.so
+  ClangEnzyme-14.so
+  ClangEnzyme-13.so
+  ClangEnzyme-12.so
+  ClangEnzyme-11.so
+  ClangEnzyme-10.so
+  ClangEnzyme-9.so
+  PATHS
+  ${ENZYME_DIR} $ENV{ENZYME_DIR}
+  ENV LD_LIBRARY_PATH ENV DYLD_LIBRARY_PATH
+  PATH_SUFFIXES
+  lib64 lib)
+
+find_program(GRIDKIT_LLVM_LINK llvm-link)
+find_program(GRIDKIT_OPT opt)
+
+if("${GRIDKIT_LLVM_LINK}" STREQUAL "GRIDKIT_LLVM_LINK-NOTFOUND")
+  message(FATAL_ERROR "Could not find llvm-link! Will not be able to build Enzyme targets.")
+endif()
+
+if("${GRIDKIT_OPT}" STREQUAL "GRIDKIT_OPT-NOTFOUND")
+  message(FATAL_ERROR "Could not find opt! Will not be able to build Enzyme targets.")
+endif()
+
+message(STATUS "${ENZYME_CLANG_PLUGIN_LIBRARY};${ENZYME_LLVM_PLUGIN_LIBRARY}")
+if((NOT ${ENZYME_LLVM_PLUGIN_LIBRARY}) OR (NOT ${ENZYME_CLANG_PLUGIN_LIBRARY}))
+  set(ENZYME_CLANG_PLUGIN_LIBRARY CACHE FILEPATH "Path to Enzyme Clang plugin library")
+  set(ENZYME_LLVM_PLUGIN_LIBRARY CACHE FILEPATH "Path to Enzyme LLVM plugin library")
+  message(STATUS "Found Enzyme clang plugin: ${ENZYME_CLANG_PLUGIN_LIBRARY}")
+  message(STATUS "Found Enzyme LLVM plugin: ${ENZYME_LLVM_PLUGIN_LIBRARY}")
+  get_filename_component(ENZYME_LIBRARY_DIR ${ENZYME_CLANG_PLUGIN_LIBRARY} DIRECTORY CACHE "Enzyme library directory")
+else()
+  message(FATAL_ERROR "Enzyme library not found!"
+    " Set ENZYME_DIR to Enzyme's installation prefix.")
+endif()
+
+macro(enzyme_add_executable)
+  set(options)
+  set(oneValueArgs NAME)
+  set(multiValueArgs SOURCES LINK_LIBRARIES)
+  cmake_parse_arguments(enzyme_add_executable "${options}" "${oneValueArgs}"
+    "${multiValueArgs}" ${ARGN})
+
+  set(PHASE2 "${CMAKE_CURRENT_BINARY_DIR}/${enzyme_add_executable_NAME}.bc")
+  set(PHASE3 "${CMAKE_CURRENT_BINARY_DIR}/${enzyme_add_executable_NAME}_enzyme.ll")
+  set(PHASE4 "${CMAKE_CURRENT_BINARY_DIR}/${enzyme_add_executable_NAME}_opt.ll")
+  set(PHASE5 "${CMAKE_CURRENT_BINARY_DIR}/${enzyme_add_executable_NAME}")
+
+  set(OBJS "")
+
+  foreach(SRC ${enzyme_add_executable_SOURCES})
+    set(PHASE0 "${CMAKE_CURRENT_SOURCE_DIR}/${SRC}")
+    set(PHASE1 "${CMAKE_CURRENT_BINARY_DIR}/${enzyme_add_executable_NAME}_${SRC}_compile.o")
+    add_custom_command(
+      DEPENDS ${PHASE0}
+      OUTPUT ${PHASE1}
+      COMMAND ${CMAKE_CXX_COMPILER} -flto -c ${PHASE0} -O2 -fno-vectorize -ffast-math -fno-unroll-loops -o ${PHASE1}
+      COMMENT "Compiling ${SRC} to object file for target ${enzyme_add_executable_NAME}"
+      )
+    set(OBJS "${OBJS} ${PHASE1}")
+  endforeach()
+
+  cmake_language(EVAL CODE "
+  add_custom_command(
+    DEPENDS ${OBJS}
+    OUTPUT ${PHASE2}
+    COMMAND ${GRIDKIT_LLVM_LINK} ${OBJS} -o ${PHASE2}
+    COMMENT \"Linking object files to LLVM bytecode for target ${enzyme_add_executable_NAME}\"
+    )
+  ")
+
+  add_custom_command(
+    DEPENDS ${PHASE2}
+    OUTPUT ${PHASE3}
+    COMMAND ${GRIDKIT_OPT} ${PHASE2} --enable-new-pm=0 -load=${ENZYME_LLVM_PLUGIN_LIBRARY} -enzyme -o ${PHASE3} -S
+    COMMENT "Running Enzyme opt pass on target ${enzyme_add_executable_NAME}"
+    )
+
+  add_custom_command(
+    DEPENDS ${PHASE3}
+    OUTPUT ${PHASE4}
+    COMMAND ${GRIDKIT_OPT} ${PHASE3} -O2 -o ${PHASE4} -S
+    COMMENT "Running remaining opt passes on target ${enzyme_add_executable_NAME}"
+    )
+
+  add_custom_command(
+    DEPENDS ${PHASE4}
+    OUTPUT ${PHASE5}
+    COMMAND ${CMAKE_CXX_COMPILER} ${PHASE4} -o ${PHASE5}
+    )
+
+  add_custom_target(
+    ${enzyme_add_executable_NAME} ALL
+    DEPENDS ${PHASE5}
+    )
+endmacro()
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 95d46006d..a111e525f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -76,7 +76,7 @@ set(PACKAGE_VERSION_PATCH "7")
 set(PACKAGE_VERSION "${PACKAGE_VERSION_MAJOR}.${PACKAGE_VERSION_MINOR}.${PACKAGE_VERSION_PATCH}")
 
 # Ipopt support is disabled by default
-option(GRIDKIT_ENABLE_IPOPT "Enable Ipopt support" ON)
+option(GRIDKIT_ENABLE_IPOPT "Enable Ipopt support" OFF)
 
 # SUNDIALS support is enabled by default
 option(GRIDKIT_ENABLE_SUNDIALS "Enable SUNDIALS support" ON)
@@ -84,6 +84,8 @@ option(GRIDKIT_ENABLE_SUNDIALS "Enable SUNDIALS support" ON)
 # Enable KLU
 option(GRIDKIT_ENABLE_SUNDIALS_SPARSE "Enable SUNDIALS sparse linear solvers" ON)
 
+# Enzyme support is disabled by default
+option(GRIDKIT_ENABLE_ENZYME "Enable automatic differentiation with Enzyme" OFF)
 
 set(CMAKE_MACOSX_RPATH 1)
 
@@ -133,6 +135,10 @@ if(GRIDKIT_ENABLE_SUNDIALS_SPARSE)
   include(FindSuiteSparse)
 endif()
 
+if(${GRIDKIT_ENABLE_ENZYME})
+  include(FindEnzyme)
+endif()
+
 # Macro that adds libraries
 include(GridkitAddLibrary)
 
diff --git a/Examples/CMakeLists.txt b/Examples/CMakeLists.txt
index 67e0da154..5b75c689b 100644
--- a/Examples/CMakeLists.txt
+++ b/Examples/CMakeLists.txt
@@ -75,3 +75,7 @@ if(TARGET SUNDIALS::idas)
     add_subdirectory(ParameterEstimation)
   endif()
 endif()
+
+if(GRIDKIT_ENABLE_ENZYME)
+  add_subdirectory(Enzyme)
+endif()
diff --git a/Examples/Enzyme/CMakeLists.txt b/Examples/Enzyme/CMakeLists.txt
new file mode 100644
index 000000000..794ec850c
--- /dev/null
+++ b/Examples/Enzyme/CMakeLists.txt
@@ -0,0 +1,2 @@
+add_subdirectory(Standalone)
+add_subdirectory(Library)
diff --git a/Examples/Enzyme/Library/CMakeLists.txt b/Examples/Enzyme/Library/CMakeLists.txt
new file mode 100644
index 000000000..afc6d1e76
--- /dev/null
+++ b/Examples/Enzyme/Library/CMakeLists.txt
@@ -0,0 +1,4 @@
+enzyme_add_executable(
+  NAME LibraryEnzymeCheck
+  SOURCES main.cpp library.cpp
+  )
diff --git a/Examples/Enzyme/Library/library.cpp b/Examples/Enzyme/Library/library.cpp
new file mode 100644
index 000000000..42e2436d5
--- /dev/null
+++ b/Examples/Enzyme/Library/library.cpp
@@ -0,0 +1,3 @@
+int square(int x) {
+  return x * x;
+}
diff --git a/Examples/Enzyme/Library/library.hpp b/Examples/Enzyme/Library/library.hpp
new file mode 100644
index 000000000..6874174a1
--- /dev/null
+++ b/Examples/Enzyme/Library/library.hpp
@@ -0,0 +1,2 @@
+#pragma once
+int square(int x);
diff --git a/Examples/Enzyme/Library/main.cpp b/Examples/Enzyme/Library/main.cpp
new file mode 100644
index 000000000..1fd429bbf
--- /dev/null
+++ b/Examples/Enzyme/Library/main.cpp
@@ -0,0 +1,10 @@
+#include "library.hpp"
+
+int __enzyme_autodiff(int (*)(int), ...);
+
+int main() {
+  int fail = 0;
+  if (__enzyme_autodiff(square, 5) != 25)
+    fail++;
+  return fail;
+}
diff --git a/Examples/Enzyme/Standalone/CMakeLists.txt b/Examples/Enzyme/Standalone/CMakeLists.txt
new file mode 100644
index 000000000..ba099289c
--- /dev/null
+++ b/Examples/Enzyme/Standalone/CMakeLists.txt
@@ -0,0 +1,4 @@
+enzyme_add_executable(
+  NAME StandaloneEnzymeCheck
+  SOURCES main.cpp
+  )
diff --git a/Examples/Enzyme/Standalone/main.cpp b/Examples/Enzyme/Standalone/main.cpp
new file mode 100644
index 000000000..5b3a2348c
--- /dev/null
+++ b/Examples/Enzyme/Standalone/main.cpp
@@ -0,0 +1,17 @@
+#include <iostream>
+
+int foo(int x) {
+  return x * x;
+}
+
+int __enzyme_autodiff(int(*)(int), ...);
+int dfoo(int x) {
+  return __enzyme_autodiff(foo, x);
+}
+
+int main() {
+  int fail = 0;
+  if (dfoo(5) != 25)
+    fail++;
+  return fail;
+}

From acaaba232f3d99ac957745ba12c5db51482fac98 Mon Sep 17 00:00:00 2001
From: Asher Mancinelli <ashermancinelli@gmail.com>
Date: Thu, 3 Mar 2022 08:05:19 +0600
Subject: [PATCH 02/51] Use better naming conventions

---
 Examples/Enzyme/Standalone/main.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Examples/Enzyme/Standalone/main.cpp b/Examples/Enzyme/Standalone/main.cpp
index 5b3a2348c..5ee9a6cda 100644
--- a/Examples/Enzyme/Standalone/main.cpp
+++ b/Examples/Enzyme/Standalone/main.cpp
@@ -1,17 +1,17 @@
 #include <iostream>
 
-int foo(int x) {
+int square(int x) {
   return x * x;
 }
 
 int __enzyme_autodiff(int(*)(int), ...);
-int dfoo(int x) {
-  return __enzyme_autodiff(foo, x);
+int dsquare(int x) {
+  return __enzyme_autodiff(square, x);
 }
 
 int main() {
   int fail = 0;
-  if (dfoo(5) != 25)
+  if (dsquare(5) != 25)
     fail++;
   return fail;
 }

From 31fea4d57f8ea61ebd9adb907d14d496c97403f2 Mon Sep 17 00:00:00 2001
From: Slaven Peles <peless@ornl.gov>
Date: Thu, 26 May 2022 12:50:07 -0400
Subject: [PATCH 03/51] More verbose tests for enzyme support.

---
 Examples/Enzyme/Library/CMakeLists.txt    |  8 ++++----
 Examples/Enzyme/Library/main.cpp          | 17 +++++++++++++++--
 Examples/Enzyme/Standalone/CMakeLists.txt |  2 ++
 Examples/Enzyme/Standalone/main.cpp       |  9 +++++++--
 4 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/Examples/Enzyme/Library/CMakeLists.txt b/Examples/Enzyme/Library/CMakeLists.txt
index afc6d1e76..dfd665ca0 100644
--- a/Examples/Enzyme/Library/CMakeLists.txt
+++ b/Examples/Enzyme/Library/CMakeLists.txt
@@ -1,4 +1,4 @@
-enzyme_add_executable(
-  NAME LibraryEnzymeCheck
-  SOURCES main.cpp library.cpp
-  )
+enzyme_add_executable(NAME libcheck SOURCES main.cpp library.cpp)
+#install(TARGETS ${CMAKE_CURRENT_BINARY_DIR}/libcheck DESTINATION bin)
+
+add_test(NAME "libcheck" COMMAND ${CMAKE_CURRENT_BINARY_DIR}/libcheck)
diff --git a/Examples/Enzyme/Library/main.cpp b/Examples/Enzyme/Library/main.cpp
index 1fd429bbf..e47ee535e 100644
--- a/Examples/Enzyme/Library/main.cpp
+++ b/Examples/Enzyme/Library/main.cpp
@@ -1,10 +1,23 @@
+#include <iostream>
 #include "library.hpp"
 
 int __enzyme_autodiff(int (*)(int), ...);
 
 int main() {
   int fail = 0;
-  if (__enzyme_autodiff(square, 5) != 25)
-    fail++;
+  int sq = square(5);
+  int dersq = __enzyme_autodiff(square, 5);
+
+  std::cout << "x = 5, x^2 = " << sq << ", d(x^2)/dx = " << dersq << "\n"; 
+  if (__enzyme_autodiff(square, 5) != 10)
+  {
+    --fail;
+    // std::cout << "Incorrect result\n";
+  }
+  if(fail < 0)
+    std::cout << "Result incorrect\n\n";
+  else
+    std::cout << "Bingo!!\n\n";
+  std:: cout << fail << "\n";
   return fail;
 }
diff --git a/Examples/Enzyme/Standalone/CMakeLists.txt b/Examples/Enzyme/Standalone/CMakeLists.txt
index ba099289c..269bf1183 100644
--- a/Examples/Enzyme/Standalone/CMakeLists.txt
+++ b/Examples/Enzyme/Standalone/CMakeLists.txt
@@ -2,3 +2,5 @@ enzyme_add_executable(
   NAME StandaloneEnzymeCheck
   SOURCES main.cpp
   )
+
+  add_test(NAME "standalonecheck" COMMAND ${CMAKE_CURRENT_BINARY_DIR}/StandaloneEnzymeCheck)
diff --git a/Examples/Enzyme/Standalone/main.cpp b/Examples/Enzyme/Standalone/main.cpp
index 5ee9a6cda..a57401b7b 100644
--- a/Examples/Enzyme/Standalone/main.cpp
+++ b/Examples/Enzyme/Standalone/main.cpp
@@ -9,9 +9,14 @@ int dsquare(int x) {
   return __enzyme_autodiff(square, x);
 }
 
-int main() {
+int sq  = square(5);
+int dsq = dsquare(5);
+
+int main()
+{
   int fail = 0;
-  if (dsquare(5) != 25)
+  std::cout << "x = 5, x^2 = " << sq << ", d(x^2)/dx = " << __enzyme_autodiff(square, 5) << "\n"; 
+  if (dsq != 10)
     fail++;
   return fail;
 }

From d5e410c725ca6912052f9b669da870f322b59bd5 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Tue, 29 Oct 2024 10:55:40 -0400
Subject: [PATCH 04/51] Extend list of LLVM/ClangEnzime libraries.

---
 CMake/FindEnzyme.cmake | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/CMake/FindEnzyme.cmake b/CMake/FindEnzyme.cmake
index 8974ed7bb..feda79ca4 100644
--- a/CMake/FindEnzyme.cmake
+++ b/CMake/FindEnzyme.cmake
@@ -73,6 +73,10 @@ endif()
 
 find_library(ENZYME_LLVM_PLUGIN_LIBRARY
   NAMES
+  LLVMEnzyme-19.so
+  LLVMEnzyme-18.so
+  LLVMEnzyme-17.so
+  LLVMEnzyme-16.so
   LLVMEnzyme-15.so
   LLVMEnzyme-14.so
   LLVMEnzyme-13.so
@@ -88,6 +92,10 @@ find_library(ENZYME_LLVM_PLUGIN_LIBRARY
 
 find_library(ENZYME_CLANG_PLUGIN_LIBRARY
   NAMES
+  ClangEnzyme-19.so
+  ClangEnzyme-18.so
+  ClangEnzyme-17.so
+  ClangEnzyme-16.so
   ClangEnzyme-15.so
   ClangEnzyme-14.so
   ClangEnzyme-13.so

From 0b10c4723a071754c0f41e595864650e7960fd79 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Tue, 29 Oct 2024 11:10:05 -0400
Subject: [PATCH 05/51] int --> double in Enzyme examples.

---
 Examples/Enzyme/Library/library.cpp |  2 +-
 Examples/Enzyme/Library/library.hpp |  2 +-
 Examples/Enzyme/Library/main.cpp    | 21 +++++++++------------
 Examples/Enzyme/Standalone/main.cpp | 19 ++++++++++++-------
 4 files changed, 23 insertions(+), 21 deletions(-)

diff --git a/Examples/Enzyme/Library/library.cpp b/Examples/Enzyme/Library/library.cpp
index 42e2436d5..6ff34da7a 100644
--- a/Examples/Enzyme/Library/library.cpp
+++ b/Examples/Enzyme/Library/library.cpp
@@ -1,3 +1,3 @@
-int square(int x) {
+double square(double x) {
   return x * x;
 }
diff --git a/Examples/Enzyme/Library/library.hpp b/Examples/Enzyme/Library/library.hpp
index 6874174a1..b498a295b 100644
--- a/Examples/Enzyme/Library/library.hpp
+++ b/Examples/Enzyme/Library/library.hpp
@@ -1,2 +1,2 @@
 #pragma once
-int square(int x);
+double square(double x);
diff --git a/Examples/Enzyme/Library/main.cpp b/Examples/Enzyme/Library/main.cpp
index e47ee535e..e0aef4ecd 100644
--- a/Examples/Enzyme/Library/main.cpp
+++ b/Examples/Enzyme/Library/main.cpp
@@ -1,23 +1,20 @@
 #include <iostream>
+#include <limits>
 #include "library.hpp"
 
-int __enzyme_autodiff(int (*)(int), ...);
+double __enzyme_autodiff(double (*)(double), ...);
 
 int main() {
   int fail = 0;
-  int sq = square(5);
-  int dersq = __enzyme_autodiff(square, 5);
+  double sq = square(5.0);
+  double dsq = __enzyme_autodiff(square, 5.0);
 
-  std::cout << "x = 5, x^2 = " << sq << ", d(x^2)/dx = " << dersq << "\n"; 
-  if (__enzyme_autodiff(square, 5) != 10)
+  std::cout << "x = 5, x^2 = " << sq << ", d(x^2)/dx = " << dsq << "\n"; 
+  if (std::abs(dsq - 10.0) > std::numeric_limits<double>::epsilon())
   {
-    --fail;
-    // std::cout << "Incorrect result\n";
+    fail++;
+    std::cout << "Result incorrect\n";
   }
-  if(fail < 0)
-    std::cout << "Result incorrect\n\n";
-  else
-    std::cout << "Bingo!!\n\n";
-  std:: cout << fail << "\n";
+  std::cout << "Status: " << fail << "\n";
   return fail;
 }
diff --git a/Examples/Enzyme/Standalone/main.cpp b/Examples/Enzyme/Standalone/main.cpp
index a57401b7b..c9b81ef05 100644
--- a/Examples/Enzyme/Standalone/main.cpp
+++ b/Examples/Enzyme/Standalone/main.cpp
@@ -1,22 +1,27 @@
 #include <iostream>
+#include <limits>
 
-int square(int x) {
+double square(double x) {
   return x * x;
 }
 
-int __enzyme_autodiff(int(*)(int), ...);
-int dsquare(int x) {
+double __enzyme_autodiff(double(*)(double), ...);
+double dsquare(double x) {
   return __enzyme_autodiff(square, x);
 }
 
-int sq  = square(5);
-int dsq = dsquare(5);
+double sq  = square(5.0);
+double dsq = dsquare(5.0);
 
 int main()
 {
   int fail = 0;
-  std::cout << "x = 5, x^2 = " << sq << ", d(x^2)/dx = " << __enzyme_autodiff(square, 5) << "\n"; 
-  if (dsq != 10)
+  std::cout << "x = 5, x^2 = " << sq << ", d(x^2)/dx = " << dsq << "\n"; 
+  if (std::abs(dsq - 10.0) > std::numeric_limits<double>::epsilon())
+  {
     fail++;
+    std::cout << "Result incorrect\n";
+  }
+  std::cout << "Status: " << fail << "\n";
   return fail;
 }

From fd75087bb45f7ebeb6310bd2577b53c6b7ffbd24 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Tue, 29 Oct 2024 13:41:16 -0400
Subject: [PATCH 06/51] Minor name changes for Enzyme tests.

---
 Examples/Enzyme/Library/CMakeLists.txt    | 6 +++---
 Examples/Enzyme/Standalone/CMakeLists.txt | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Examples/Enzyme/Library/CMakeLists.txt b/Examples/Enzyme/Library/CMakeLists.txt
index dfd665ca0..78566c983 100644
--- a/Examples/Enzyme/Library/CMakeLists.txt
+++ b/Examples/Enzyme/Library/CMakeLists.txt
@@ -1,4 +1,4 @@
-enzyme_add_executable(NAME libcheck SOURCES main.cpp library.cpp)
-#install(TARGETS ${CMAKE_CURRENT_BINARY_DIR}/libcheck DESTINATION bin)
+enzyme_add_executable(NAME EnzymeLibCheck SOURCES main.cpp library.cpp)
+#install(TARGETS ${CMAKE_CURRENT_BINARY_DIR}/EnzymeLibCheck DESTINATION bin)
 
-add_test(NAME "libcheck" COMMAND ${CMAKE_CURRENT_BINARY_DIR}/libcheck)
+add_test(NAME "EnzymeLibCheck" COMMAND ${CMAKE_CURRENT_BINARY_DIR}/EnzymeLibCheck)
diff --git a/Examples/Enzyme/Standalone/CMakeLists.txt b/Examples/Enzyme/Standalone/CMakeLists.txt
index 269bf1183..919d755e0 100644
--- a/Examples/Enzyme/Standalone/CMakeLists.txt
+++ b/Examples/Enzyme/Standalone/CMakeLists.txt
@@ -1,6 +1,6 @@
 enzyme_add_executable(
-  NAME StandaloneEnzymeCheck
+  NAME EnzymeStandaloneCheck
   SOURCES main.cpp
   )
 
-  add_test(NAME "standalonecheck" COMMAND ${CMAKE_CURRENT_BINARY_DIR}/StandaloneEnzymeCheck)
+add_test(NAME "EnzymeStandaloneCheck" COMMAND ${CMAKE_CURRENT_BINARY_DIR}/EnzymeStandaloneCheck)

From ff0352181f79835aeeae796ce72457f8b5234781 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Thu, 19 Dec 2024 12:49:36 -0500
Subject: [PATCH 07/51] Changed --enable-new-pm to directly use the new format.

---
 CMake/FindEnzyme.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMake/FindEnzyme.cmake b/CMake/FindEnzyme.cmake
index feda79ca4..3840a526f 100644
--- a/CMake/FindEnzyme.cmake
+++ b/CMake/FindEnzyme.cmake
@@ -170,7 +170,7 @@ macro(enzyme_add_executable)
   add_custom_command(
     DEPENDS ${PHASE2}
     OUTPUT ${PHASE3}
-    COMMAND ${GRIDKIT_OPT} ${PHASE2} --enable-new-pm=0 -load=${ENZYME_LLVM_PLUGIN_LIBRARY} -enzyme -o ${PHASE3} -S
+    COMMAND ${GRIDKIT_OPT} ${PHASE2} -load-pass-plugin=${ENZYME_LLVM_PLUGIN_LIBRARY} -passes=enzyme -o ${PHASE3} -S
     COMMENT "Running Enzyme opt pass on target ${enzyme_add_executable_NAME}"
     )
 

From 6c25725b6600a76c53a15db77fcb0afd566fa1f5 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Thu, 19 Dec 2024 13:01:42 -0500
Subject: [PATCH 08/51] gridkit +enzyme in CI.

---
 .github/workflows/spack_default_build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 55dc44a24..4c9552d12 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -96,7 +96,7 @@ jobs:
       matrix:
         # Minimal Build(s) - GHCR mirror speeds these up a lot!
         spack_spec:
-          - gridkit@develop
+          - gridkit@develop +enzyme
 
     name: Build gridkit with Spack
     steps:

From d5d6f83abf4e4580ed956fdecb3acf23aef261b2 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Thu, 19 Dec 2024 13:02:59 -0500
Subject: [PATCH 09/51] Update local Spack.

---
 .gitmodules             | 2 +-
 Buildsystem/spack/spack | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitmodules b/.gitmodules
index 66dd18b30..8980bd16c 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,4 +1,4 @@
 [submodule "Buildsystem/spack/spack"]
 	path = Buildsystem/spack/spack
 	url = https://github.com/nkoukpaizan/spack.git
-	branch = Gridkit-package-dev
+	branch = Gridkit-package-dev+enzyme
diff --git a/Buildsystem/spack/spack b/Buildsystem/spack/spack
index bb4ca5ad4..e7d45f0f8 160000
--- a/Buildsystem/spack/spack
+++ b/Buildsystem/spack/spack
@@ -1 +1 @@
-Subproject commit bb4ca5ad45c437b504e9542577010f4318414836
+Subproject commit e7d45f0f89eedf19ac485ede49c024820983e3c4

From c00a160f4ea2255fac82156077af3e598201bce4 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Thu, 19 Dec 2024 13:21:12 -0500
Subject: [PATCH 10/51] Request llvm@19 in CI.

---
 .github/workflows/spack_default_build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 4c9552d12..4d5383b61 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -96,7 +96,7 @@ jobs:
       matrix:
         # Minimal Build(s) - GHCR mirror speeds these up a lot!
         spack_spec:
-          - gridkit@develop +enzyme
+          - gridkit@develop +enzyme ^ llvm@19
 
     name: Build gridkit with Spack
     steps:

From db2f15801bcc709c7b943f1e84b1357d4d9449a9 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Thu, 26 Dec 2024 09:57:18 -0500
Subject: [PATCH 11/51] Try using another build cache for LLVM.

---
 .github/workflows/spack_default_build.yaml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 4d5383b61..22099736f 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -100,6 +100,11 @@ jobs:
 
     name: Build gridkit with Spack
     steps:
+      - name: Install LLVM and Clang
+        uses: KyleMayes/install-llvm-action@v2
+        with:
+          version: "19.1"
+
       - name: Checkout
         uses: actions/checkout@v4
         with:

From 6824032e97dc25c309d335952c3f05b9d8654fdb Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Thu, 26 Dec 2024 09:59:02 -0500
Subject: [PATCH 12/51] Add timeout for Github actions.

---
 .github/workflows/spack_default_build.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 22099736f..8887fe4df 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -91,6 +91,7 @@ jobs:
     permissions:
       packages: write
       contents: read
+    timeout-minutes: 30
 
     strategy:
       matrix:

From 443c18c4781ceaa900d19cd9e0d54a45368ed234 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Thu, 26 Dec 2024 10:25:15 -0500
Subject: [PATCH 13/51] Try LLVM 18.

---
 .github/workflows/spack_default_build.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 8887fe4df..81da62f92 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -97,14 +97,14 @@ jobs:
       matrix:
         # Minimal Build(s) - GHCR mirror speeds these up a lot!
         spack_spec:
-          - gridkit@develop +enzyme ^ llvm@19
+          - gridkit@develop +enzyme ^ llvm@18
 
     name: Build gridkit with Spack
     steps:
       - name: Install LLVM and Clang
         uses: KyleMayes/install-llvm-action@v2
         with:
-          version: "19.1"
+          version: "18.1"
 
       - name: Checkout
         uses: actions/checkout@v4

From 11df3ec06c9967345502284850cce5f926450268 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Thu, 26 Dec 2024 11:00:47 -0500
Subject: [PATCH 14/51] Explicitly list LLVM as external Spack package with
 specified path.

---
 .github/workflows/spack_default_build.yaml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 81da62f92..e8b3c4078 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -131,6 +131,11 @@ jobs:
             mirrors:
               local-buildcache: oci://${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
               spack: https://binaries.spack.io/develop
+            packages:
+              llvm:
+                externals:
+                - spec llvm@18
+                  prefix: $PWD/llvm
           EOF
 
       - name: Configure GHCR mirror

From 8460d352777c7893298b8305bc50a7c4bb5d5f25 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Thu, 26 Dec 2024 11:02:58 -0500
Subject: [PATCH 15/51] Bug fix 'spec:'

---
 .github/workflows/spack_default_build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index e8b3c4078..bfc72259f 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -134,7 +134,7 @@ jobs:
             packages:
               llvm:
                 externals:
-                - spec llvm@18
+                  - spec: llvm@18
                   prefix: $PWD/llvm
           EOF
 

From 1e8930d51cc159fe664d90d1598d7308a682f686 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Thu, 26 Dec 2024 11:08:38 -0500
Subject: [PATCH 16/51] Bug fix in spack.yaml indents.

---
 .github/workflows/spack_default_build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index bfc72259f..6bb931cec 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -134,7 +134,7 @@ jobs:
             packages:
               llvm:
                 externals:
-                  - spec: llvm@18
+                - spec: llvm@18
                   prefix: $PWD/llvm
           EOF
 

From de1ce732155dcc11d4feb490a6c24a09b0db6be5 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Thu, 26 Dec 2024 11:20:50 -0500
Subject: [PATCH 17/51] Check LLVM install.

---
 .github/workflows/spack_default_build.yaml | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 6bb931cec..82a63cd14 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -101,11 +101,6 @@ jobs:
 
     name: Build gridkit with Spack
     steps:
-      - name: Install LLVM and Clang
-        uses: KyleMayes/install-llvm-action@v2
-        with:
-          version: "18.1"
-
       - name: Checkout
         uses: actions/checkout@v4
         with:
@@ -113,6 +108,20 @@ jobs:
           # Also need to change build script to use spack from base image
           submodules: true
 
+      - name: Install LLVM and Clang
+        uses: KyleMayes/install-llvm-action@v2
+        with:
+          version: "18.1"
+
+      - name: Check LLVM install
+        run: ls $PWD/llvm
+
+      - name: Check LLVM install bin
+        run: ls $PWD/llvm/bin
+
+      - name: Check LLVM install lib
+        run: ls $PWD/llvm/lib
+
       - name: Setup Spack
         run: echo "$PWD/Buildsystem/spack/spack/bin" >> "$GITHUB_PATH"
 

From 4a150f3c4ae9b575f3ca4994a03bd5ca06f61eda Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Thu, 26 Dec 2024 13:10:36 -0500
Subject: [PATCH 18/51] Go back to trying to build LLVM from scratch. Timeout
 at 500 minutes.

---
 .github/workflows/spack_default_build.yaml | 26 +++++-----------------
 1 file changed, 5 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 82a63cd14..a8ce717d5 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -91,13 +91,13 @@ jobs:
     permissions:
       packages: write
       contents: read
-    timeout-minutes: 30
+    timeout-minutes: 500
 
     strategy:
       matrix:
         # Minimal Build(s) - GHCR mirror speeds these up a lot!
         spack_spec:
-          - gridkit@develop +enzyme ^ llvm@18
+          - gridkit@develop +enzyme ^ llvm@19
 
     name: Build gridkit with Spack
     steps:
@@ -108,23 +108,12 @@ jobs:
           # Also need to change build script to use spack from base image
           submodules: true
 
-      - name: Install LLVM and Clang
-        uses: KyleMayes/install-llvm-action@v2
-        with:
-          version: "18.1"
-
-      - name: Check LLVM install
-        run: ls $PWD/llvm
-
-      - name: Check LLVM install bin
-        run: ls $PWD/llvm/bin
-
-      - name: Check LLVM install lib
-        run: ls $PWD/llvm/lib
-
       - name: Setup Spack
         run: echo "$PWD/Buildsystem/spack/spack/bin" >> "$GITHUB_PATH"
 
+      - name: Clean Spack
+        run: spack -e . clean -a
+
       - name: Create heredoc spack.yaml
         run: |
           cat << EOF > spack.yaml
@@ -140,11 +129,6 @@ jobs:
             mirrors:
               local-buildcache: oci://${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
               spack: https://binaries.spack.io/develop
-            packages:
-              llvm:
-                externals:
-                - spec: llvm@18
-                  prefix: $PWD/llvm
           EOF
 
       - name: Configure GHCR mirror

From 5c7ef6328ab202cd03314108a7eb681530d0b119 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Thu, 26 Dec 2024 13:15:23 -0500
Subject: [PATCH 19/51] Remove spack clean.

---
 .github/workflows/spack_default_build.yaml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index a8ce717d5..0bbc01089 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -111,9 +111,6 @@ jobs:
       - name: Setup Spack
         run: echo "$PWD/Buildsystem/spack/spack/bin" >> "$GITHUB_PATH"
 
-      - name: Clean Spack
-        run: spack -e . clean -a
-
       - name: Create heredoc spack.yaml
         run: |
           cat << EOF > spack.yaml

From 83c6f9db5e70db104d3953415c27f8af1eb6b834 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Thu, 26 Dec 2024 13:25:28 -0500
Subject: [PATCH 20/51] Request verbose output for spack install.

---
 .github/workflows/spack_default_build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 0bbc01089..a269d8ada 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -144,7 +144,7 @@ jobs:
         run: spack -e . concretize
 
       - name: Install dependencies
-        run: spack -e . install --no-check-signature --only dependencies
+        run: spack -e . install -v --no-check-signature --only dependencies
 
       - name: Install package
         run: spack -e . install --keep-stage --no-check-signature --no-cache --fresh

From 90dc06475fe5e8e30149eca9822147dee6c1aed7 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Thu, 26 Dec 2024 17:04:22 -0500
Subject: [PATCH 21/51] Check storage on CI platform.

---
 .github/workflows/spack_default_build.yaml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index a269d8ada..602a6942d 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -91,7 +91,7 @@ jobs:
     permissions:
       packages: write
       contents: read
-    timeout-minutes: 500
+    timeout-minutes: 10
 
     strategy:
       matrix:
@@ -101,6 +101,12 @@ jobs:
 
     name: Build gridkit with Spack
     steps:
+      - name: Check storage locations
+        run: df -h
+
+      - name: Check Spack storage
+        run: du -h -max-depth=1 /opt/spack
+
       - name: Checkout
         uses: actions/checkout@v4
         with:
@@ -144,7 +150,7 @@ jobs:
         run: spack -e . concretize
 
       - name: Install dependencies
-        run: spack -e . install -v --no-check-signature --only dependencies
+        run: spack -e . install -j16 --no-check-signature --only dependencies
 
       - name: Install package
         run: spack -e . install --keep-stage --no-check-signature --no-cache --fresh

From 1c381b85c5b181fd33702414760f905d2363e4c1 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Thu, 26 Dec 2024 17:06:09 -0500
Subject: [PATCH 22/51] Bug fix in storage-checking commands.

---
 .github/workflows/spack_default_build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 602a6942d..c096875f4 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -105,7 +105,7 @@ jobs:
         run: df -h
 
       - name: Check Spack storage
-        run: du -h -max-depth=1 /opt/spack
+        run: du -h --max-depth=1 /opt/spack
 
       - name: Checkout
         uses: actions/checkout@v4

From 425502b6409f1101e66a4e6d72667edc656eaec1 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Thu, 26 Dec 2024 17:11:50 -0500
Subject: [PATCH 23/51] Check storage after Spack setup.

---
 .github/workflows/spack_default_build.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index c096875f4..87dbbf53e 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -101,12 +101,6 @@ jobs:
 
     name: Build gridkit with Spack
     steps:
-      - name: Check storage locations
-        run: df -h
-
-      - name: Check Spack storage
-        run: du -h --max-depth=1 /opt/spack
-
       - name: Checkout
         uses: actions/checkout@v4
         with:
@@ -140,6 +134,12 @@ jobs:
       - name: Trust keys
         run: spack -e . buildcache keys --install --trust
 
+      - name: Check storage locations
+        run: df -h
+
+      - name: Check Spack storage
+        run: du -h --max-depth=1 /opt/spack
+
       - name: Find external packages
         run: spack -e . external find --all
 

From 2e05f1772f428098dd517d6e3714553a7dcf903d Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Thu, 26 Dec 2024 17:17:52 -0500
Subject: [PATCH 24/51] Check storage in /.

---
 .github/workflows/spack_default_build.yaml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 87dbbf53e..15508e3e0 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -135,10 +135,7 @@ jobs:
         run: spack -e . buildcache keys --install --trust
 
       - name: Check storage locations
-        run: df -h
-
-      - name: Check Spack storage
-        run: du -h --max-depth=1 /opt/spack
+        run: du -h --max-depth=1 /
 
       - name: Find external packages
         run: spack -e . external find --all

From 94962dcdb3f2d5b0bd3139fa039cd2d2eaa1ef61 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Thu, 26 Dec 2024 17:22:31 -0500
Subject: [PATCH 25/51] Check storage in /opt/.

---
 .github/workflows/spack_default_build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 15508e3e0..ad7deac74 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -135,7 +135,7 @@ jobs:
         run: spack -e . buildcache keys --install --trust
 
       - name: Check storage locations
-        run: du -h --max-depth=1 /
+        run: du -h --max-depth=1 /opt/
 
       - name: Find external packages
         run: spack -e . external find --all

From ab8b9e92952c20e8e8931ac0ed78b225ce5f1d27 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Thu, 26 Dec 2024 17:29:05 -0500
Subject: [PATCH 26/51] Remove storage-checking step.

---
 .github/workflows/spack_default_build.yaml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index ad7deac74..94252fc79 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -134,9 +134,6 @@ jobs:
       - name: Trust keys
         run: spack -e . buildcache keys --install --trust
 
-      - name: Check storage locations
-        run: du -h --max-depth=1 /opt/
-
       - name: Find external packages
         run: spack -e . external find --all
 
@@ -147,7 +144,7 @@ jobs:
         run: spack -e . concretize
 
       - name: Install dependencies
-        run: spack -e . install -j16 --no-check-signature --only dependencies
+        run: spack -e . install --no-check-signature --only dependencies
 
       - name: Install package
         run: spack -e . install --keep-stage --no-check-signature --no-cache --fresh

From 5f84f6c0cfc9eaeb68919f501fa25fd4a80d3d28 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Fri, 27 Dec 2024 13:44:30 -0500
Subject: [PATCH 27/51] Try apt-get LLVM@18.

---
 .github/workflows/spack_default_build.yaml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 94252fc79..d4c8df87f 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -7,7 +7,7 @@ env:
   # Our repo name contains upper case characters, so we can't use ${{ github.repository }}
   IMAGE_NAME: gridkit
   USERNAME: gridkit-bot
-  BASE_VERSION: ubuntu-22.04-fortran-v0.1.0
+  BASE_VERSION: ubuntu-22.04-fortran-v0.2.0
 
 # Until we remove the need to clone submodules to build, this should on be in PRs
 on: [pull_request]
@@ -66,6 +66,9 @@ jobs:
                 gcc \
                 libstdc++6 \
               && rm -rf /var/lib/apt/lists/*
+          RUN apt-get install -y --no-install-recommends \
+                llvm-18 \
+              && rm -rf /var/lib/apt/lists/*
           EOF
 
       - name: Extract metadata (tags, labels) for Docker
@@ -97,7 +100,7 @@ jobs:
       matrix:
         # Minimal Build(s) - GHCR mirror speeds these up a lot!
         spack_spec:
-          - gridkit@develop +enzyme ^ llvm@19
+          - gridkit@develop +enzyme ^ llvm@18
 
     name: Build gridkit with Spack
     steps:

From b5b682ff57440882f3284dd04875138ecf715f0d Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Fri, 27 Dec 2024 13:56:02 -0500
Subject: [PATCH 28/51] Try wget on llvm.sh script (should build latest stable
 version).

---
 .github/workflows/spack_default_build.yaml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index d4c8df87f..88f49c5a9 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -66,9 +66,7 @@ jobs:
                 gcc \
                 libstdc++6 \
               && rm -rf /var/lib/apt/lists/*
-          RUN apt-get install -y --no-install-recommends \
-                llvm-18 \
-              && rm -rf /var/lib/apt/lists/*
+          RUN bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
           EOF
 
       - name: Extract metadata (tags, labels) for Docker
@@ -100,7 +98,7 @@ jobs:
       matrix:
         # Minimal Build(s) - GHCR mirror speeds these up a lot!
         spack_spec:
-          - gridkit@develop +enzyme ^ llvm@18
+          - gridkit@develop +enzyme
 
     name: Build gridkit with Spack
     steps:

From d210ddf3faa9c53254ea4a86ef78807d284070ad Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Fri, 27 Dec 2024 14:03:55 -0500
Subject: [PATCH 29/51] Revert "Try wget on llvm.sh script (should build latest
 stable version)."

This reverts commit b5b682ff57440882f3284dd04875138ecf715f0d.
---
 .github/workflows/spack_default_build.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 88f49c5a9..d4c8df87f 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -66,7 +66,9 @@ jobs:
                 gcc \
                 libstdc++6 \
               && rm -rf /var/lib/apt/lists/*
-          RUN bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
+          RUN apt-get install -y --no-install-recommends \
+                llvm-18 \
+              && rm -rf /var/lib/apt/lists/*
           EOF
 
       - name: Extract metadata (tags, labels) for Docker
@@ -98,7 +100,7 @@ jobs:
       matrix:
         # Minimal Build(s) - GHCR mirror speeds these up a lot!
         spack_spec:
-          - gridkit@develop +enzyme
+          - gridkit@develop +enzyme ^ llvm@18
 
     name: Build gridkit with Spack
     steps:

From 1b1ca43a750a5cc23ce8eb4a05adee9ac32597bb Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Fri, 27 Dec 2024 14:15:00 -0500
Subject: [PATCH 30/51] Add apt-add-repository for llvm.

---
 .github/workflows/spack_default_build.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index d4c8df87f..a43316fa0 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -66,7 +66,8 @@ jobs:
                 gcc \
                 libstdc++6 \
               && rm -rf /var/lib/apt/lists/*
-          RUN apt-get install -y --no-install-recommends \
+          RUN apt-add-repository -y "deb http://apt.llvm.org/`lsb_release -cs`/ llvm-toolchain-`lsb_release -cs`-$LLVM_VERSION main" && \
+              apt-get install -y --no-install-recommends \
                 llvm-18 \
               && rm -rf /var/lib/apt/lists/*
           EOF

From 7e4bb57cd96084d8ac14f1cfbfb2ff0fcc739a9a Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Fri, 27 Dec 2024 14:21:45 -0500
Subject: [PATCH 31/51] Bug fix: -->18.

---
 .github/workflows/spack_default_build.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index a43316fa0..82cf12910 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -66,9 +66,11 @@ jobs:
                 gcc \
                 libstdc++6 \
               && rm -rf /var/lib/apt/lists/*
-          RUN apt-add-repository -y "deb http://apt.llvm.org/`lsb_release -cs`/ llvm-toolchain-`lsb_release -cs`-$LLVM_VERSION main" && \
+          RUN apt-add-repository -y "deb http://apt.llvm.org/`lsb_release -cs`/ llvm-toolchain-`lsb_release -cs`-18 main" && \
+              apt-get update &&
               apt-get install -y --no-install-recommends \
                 llvm-18 \
+                clang-18 \
               && rm -rf /var/lib/apt/lists/*
           EOF
 

From d899618a4dafbe46e4b280d2bd6d8ffd32659d1e Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Fri, 27 Dec 2024 14:31:18 -0500
Subject: [PATCH 32/51] wget the llvm snapshot

---
 .github/workflows/spack_default_build.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 82cf12910..f8c7b63ec 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -66,8 +66,7 @@ jobs:
                 gcc \
                 libstdc++6 \
               && rm -rf /var/lib/apt/lists/*
-          RUN apt-add-repository -y "deb http://apt.llvm.org/`lsb_release -cs`/ llvm-toolchain-`lsb_release -cs`-18 main" && \
-              apt-get update &&
+          RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - && \
               apt-get install -y --no-install-recommends \
                 llvm-18 \
                 clang-18 \

From d2e4f6dd4db4225adaa2d2006f580eb4841687d4 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Fri, 27 Dec 2024 14:35:40 -0500
Subject: [PATCH 33/51] wget->curl

---
 .github/workflows/spack_default_build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index f8c7b63ec..f9416add1 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -66,7 +66,7 @@ jobs:
                 gcc \
                 libstdc++6 \
               && rm -rf /var/lib/apt/lists/*
-          RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - && \
+          RUN curl -fsSL - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - && \
               apt-get install -y --no-install-recommends \
                 llvm-18 \
                 clang-18 \

From 115f7c74f83a6f296f2f9629e9a41bbdee6040a1 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Fri, 27 Dec 2024 14:40:09 -0500
Subject: [PATCH 34/51] Need to install wget or curl before using it.

---
 .github/workflows/spack_default_build.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index f9416add1..52df9aada 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -66,7 +66,8 @@ jobs:
                 gcc \
                 libstdc++6 \
               && rm -rf /var/lib/apt/lists/*
-          RUN curl -fsSL - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - && \
+          RUN apt-get install -y --no-install-recommends wget && \
+              wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
               apt-get install -y --no-install-recommends \
                 llvm-18 \
                 clang-18 \

From c4405ffd72724a114d4ea5907cdc274468b9c4df Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Fri, 27 Dec 2024 14:44:47 -0500
Subject: [PATCH 35/51] 'apt-get update' before 'apt-get install wget'.

---
 .github/workflows/spack_default_build.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 52df9aada..e956897b9 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -66,7 +66,8 @@ jobs:
                 gcc \
                 libstdc++6 \
               && rm -rf /var/lib/apt/lists/*
-          RUN apt-get install -y --no-install-recommends wget && \
+          RUN apt-get update && \
+              apt-get install -y --no-install-recommends wget && \
               wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
               apt-get install -y --no-install-recommends \
                 llvm-18 \

From 00aaf82d641cc2210b17a4e5876bef978f72ce78 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Fri, 27 Dec 2024 14:52:35 -0500
Subject: [PATCH 36/51] Improved apt-get commands for LLVM build based on
 Enzyme Dockerfile.

---
 .github/workflows/spack_default_build.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index e956897b9..ade07042e 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -49,10 +49,12 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       # Need to build custom base image with gfortran
+      # For LLVM build setup, see https://github.com/EnzymeAD/enzyme-dev-docker/blob/main/Dockerfile
       - name: Create Dockerfile heredoc
         if: ${{ env.IMAGE_EXISTS == 'false' }}
         run: |
           cat << EOF > Dockerfile
+          ARG LLVM_VERSION=18
           FROM ubuntu:22.04
           RUN apt-get update && \
               apt-get install -y --no-install-recommends \
@@ -69,6 +71,8 @@ jobs:
           RUN apt-get update && \
               apt-get install -y --no-install-recommends wget && \
               wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+              apt-add-repository -y "deb http://apt.llvm.org/`lsb_release -cs`/ llvm-toolchain-`lsb_release -cs`-$LLVM_VERSION main" \
+              apt-get update && \
               apt-get install -y --no-install-recommends \
                 llvm-18 \
                 clang-18 \

From 7e06efb5502ab0b0d2d0bc5e557a441b5972b2ba Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Fri, 27 Dec 2024 14:57:36 -0500
Subject: [PATCH 37/51] Explicit request of LLVM 18.

---
 .github/workflows/spack_default_build.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index ade07042e..e2461a210 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -54,7 +54,6 @@ jobs:
         if: ${{ env.IMAGE_EXISTS == 'false' }}
         run: |
           cat << EOF > Dockerfile
-          ARG LLVM_VERSION=18
           FROM ubuntu:22.04
           RUN apt-get update && \
               apt-get install -y --no-install-recommends \
@@ -71,7 +70,7 @@ jobs:
           RUN apt-get update && \
               apt-get install -y --no-install-recommends wget && \
               wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
-              apt-add-repository -y "deb http://apt.llvm.org/`lsb_release -cs`/ llvm-toolchain-`lsb_release -cs`-$LLVM_VERSION main" \
+              apt-add-repository -y "deb http://apt.llvm.org/`lsb_release -cs`/ llvm-toolchain-`lsb_release -cs`-18 main" \
               apt-get update && \
               apt-get install -y --no-install-recommends \
                 llvm-18 \

From 4af323da0e6ee4ddaebfc2d8b1f3f7c7837947d6 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Fri, 27 Dec 2024 15:39:31 -0500
Subject: [PATCH 38/51] Build extra LLVM components in base image.

---
 .github/workflows/spack_default_build.yaml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index e2461a210..10aa6fcbe 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -7,7 +7,7 @@ env:
   # Our repo name contains upper case characters, so we can't use ${{ github.repository }}
   IMAGE_NAME: gridkit
   USERNAME: gridkit-bot
-  BASE_VERSION: ubuntu-22.04-fortran-v0.2.0
+  BASE_VERSION: ubuntu-22.04-fortran-v0.2.1
 
 # Until we remove the need to clone submodules to build, this should on be in PRs
 on: [pull_request]
@@ -74,7 +74,10 @@ jobs:
               apt-get update && \
               apt-get install -y --no-install-recommends \
                 llvm-18 \
-                clang-18 \
+                llvm-18-dev \
+                clang-18 \ 
+                libclang-18-dev \
+                libomp-18-dev \
               && rm -rf /var/lib/apt/lists/*
           EOF
 
@@ -107,7 +110,7 @@ jobs:
       matrix:
         # Minimal Build(s) - GHCR mirror speeds these up a lot!
         spack_spec:
-          - gridkit@develop +enzyme ^ llvm@18
+          - gridkit@develop +enzyme
 
     name: Build gridkit with Spack
     steps:

From 40b4ce74e07f0fa9dcff523db500ab52a6a2ae37 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Mon, 30 Dec 2024 14:18:52 -0500
Subject: [PATCH 39/51] Look for LLVM in usr/lib.

---
 .github/workflows/spack_default_build.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 10aa6fcbe..12adeb946 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -147,6 +147,9 @@ jobs:
       - name: Trust keys
         run: spack -e . buildcache keys --install --trust
 
+      - name: Check llvm
+        run: ls -ltrh /usr/lib/llvm*
+
       - name: Find external packages
         run: spack -e . external find --all
 

From 1b7d7fb021da6cec34eacc710f5e70f11bad0167 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Mon, 30 Dec 2024 14:26:38 -0500
Subject: [PATCH 40/51] Look for LLVM in /usr/bin.

---
 .github/workflows/spack_default_build.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 12adeb946..80bd463ec 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -147,8 +147,8 @@ jobs:
       - name: Trust keys
         run: spack -e . buildcache keys --install --trust
 
-      - name: Check llvm
-        run: ls -ltrh /usr/lib/llvm*
+      - name: Check usr/bin
+        run: ls -ltrh /usr/bin/
 
       - name: Find external packages
         run: spack -e . external find --all

From cd104b7bb0574e4490ae9a4ea7a3cef32dab733d Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Mon, 30 Dec 2024 14:37:39 -0500
Subject: [PATCH 41/51] Try ubuntu-22.04-->ubuntu-24.04. The latter should have
 llvm-18 available.

---
 .github/workflows/spack_default_build.yaml | 24 ++++------------------
 1 file changed, 4 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 80bd463ec..0b54c3f20 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -7,7 +7,7 @@ env:
   # Our repo name contains upper case characters, so we can't use ${{ github.repository }}
   IMAGE_NAME: gridkit
   USERNAME: gridkit-bot
-  BASE_VERSION: ubuntu-22.04-fortran-v0.2.1
+  BASE_VERSION: ubuntu-24.04-fortran-v0.2.2
 
 # Until we remove the need to clone submodules to build, this should on be in PRs
 on: [pull_request]
@@ -15,7 +15,7 @@ on: [pull_request]
 jobs:
   base_image_build:
     name: Build base image
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     permissions:
       contents: read
       packages: write
@@ -49,12 +49,11 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       # Need to build custom base image with gfortran
-      # For LLVM build setup, see https://github.com/EnzymeAD/enzyme-dev-docker/blob/main/Dockerfile
       - name: Create Dockerfile heredoc
         if: ${{ env.IMAGE_EXISTS == 'false' }}
         run: |
           cat << EOF > Dockerfile
-          FROM ubuntu:22.04
+          FROM ubuntu:24.04
           RUN apt-get update && \
               apt-get install -y --no-install-recommends \
                 software-properties-common \
@@ -67,18 +66,6 @@ jobs:
                 gcc \
                 libstdc++6 \
               && rm -rf /var/lib/apt/lists/*
-          RUN apt-get update && \
-              apt-get install -y --no-install-recommends wget && \
-              wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
-              apt-add-repository -y "deb http://apt.llvm.org/`lsb_release -cs`/ llvm-toolchain-`lsb_release -cs`-18 main" \
-              apt-get update && \
-              apt-get install -y --no-install-recommends \
-                llvm-18 \
-                llvm-18-dev \
-                clang-18 \ 
-                libclang-18-dev \
-                libomp-18-dev \
-              && rm -rf /var/lib/apt/lists/*
           EOF
 
       - name: Extract metadata (tags, labels) for Docker
@@ -100,7 +87,7 @@ jobs:
 
   gridkit_spack_builds:
     needs: base_image_build
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     permissions:
       packages: write
       contents: read
@@ -147,9 +134,6 @@ jobs:
       - name: Trust keys
         run: spack -e . buildcache keys --install --trust
 
-      - name: Check usr/bin
-        run: ls -ltrh /usr/bin/
-
       - name: Find external packages
         run: spack -e . external find --all
 

From e61f96394b44215cd66dbee6bc4917210d781117 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Mon, 30 Dec 2024 14:51:11 -0500
Subject: [PATCH 42/51] Check /usr/lib/cmake/

---
 .github/workflows/spack_default_build.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 0b54c3f20..035762638 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -128,6 +128,9 @@ jobs:
               spack: https://binaries.spack.io/develop
           EOF
 
+      - name: Check /usr/lib/cmake/
+        run: ls -ltrh /usr/lib/cmake/
+
       - name: Configure GHCR mirror
         run: spack -e . mirror set --oci-username ${{ env.USERNAME }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache
 

From 24aec42210dc092ef33a2f4b2049904c16bdd0cd Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Mon, 30 Dec 2024 14:57:32 -0500
Subject: [PATCH 43/51] External Spack package for llvm explicitly set to
 /usr/lib/llvm-18.

---
 .github/workflows/spack_default_build.yaml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 035762638..1e16461c5 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -126,11 +126,13 @@ jobs:
             mirrors:
               local-buildcache: oci://${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
               spack: https://binaries.spack.io/develop
+            packages:
+              llvm:
+                externals:
+                - spec: llvm@18
+                  prefix: /usr/lib/llvm-18
           EOF
 
-      - name: Check /usr/lib/cmake/
-        run: ls -ltrh /usr/lib/cmake/
-
       - name: Configure GHCR mirror
         run: spack -e . mirror set --oci-username ${{ env.USERNAME }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache
 

From ef8d8ef8ee7f8e90050b121d8babe4e68c60e7bd Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Mon, 30 Dec 2024 15:08:37 -0500
Subject: [PATCH 44/51] Remove llvm from 'spack external find', so that it
 tries to use the path explicitly set in spack.yaml.

---
 .github/workflows/spack_default_build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index 1e16461c5..f97eea90b 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -140,7 +140,7 @@ jobs:
         run: spack -e . buildcache keys --install --trust
 
       - name: Find external packages
-        run: spack -e . external find --all
+        run: spack -e . external find --all --exclude llvm
 
       - name: Spack develop gridkit
         run: spack -e . develop --path=$(pwd) gridkit@develop

From 15759208f20211e646538eeece066873f836599f Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Mon, 30 Dec 2024 15:24:12 -0500
Subject: [PATCH 45/51] Update local Spack submodule.

---
 Buildsystem/spack/spack | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Buildsystem/spack/spack b/Buildsystem/spack/spack
index e7d45f0f8..aa927fad0 160000
--- a/Buildsystem/spack/spack
+++ b/Buildsystem/spack/spack
@@ -1 +1 @@
-Subproject commit e7d45f0f89eedf19ac485ede49c024820983e3c4
+Subproject commit aa927fad059b6f3c372e0033f302aa68a5c1b3c3

From 9b3af35472e377ea63e2465c074cce7cd017a76b Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Mon, 30 Dec 2024 15:24:50 -0500
Subject: [PATCH 46/51] Increase timeout to 30min.

---
 .github/workflows/spack_default_build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/spack_default_build.yaml b/.github/workflows/spack_default_build.yaml
index f97eea90b..e4f596c2f 100644
--- a/.github/workflows/spack_default_build.yaml
+++ b/.github/workflows/spack_default_build.yaml
@@ -91,7 +91,7 @@ jobs:
     permissions:
       packages: write
       contents: read
-    timeout-minutes: 10
+    timeout-minutes: 30
 
     strategy:
       matrix:

From f1e402f6e439bb6b9636b1f0b35d2561d1819464 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Mon, 30 Dec 2024 15:42:49 -0500
Subject: [PATCH 47/51] Update local Spack submodule.

---
 Buildsystem/spack/spack | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Buildsystem/spack/spack b/Buildsystem/spack/spack
index aa927fad0..0c7917e9f 160000
--- a/Buildsystem/spack/spack
+++ b/Buildsystem/spack/spack
@@ -1 +1 @@
-Subproject commit aa927fad059b6f3c372e0033f302aa68a5c1b3c3
+Subproject commit 0c7917e9f0fbae36181b91b8641d06cbc02c0acf

From bc7dd510770472dd80a6dc0bbb881f82246b1b2d Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Mon, 30 Dec 2024 16:45:03 -0500
Subject: [PATCH 48/51] Update local Spack submodule.

---
 Buildsystem/spack/spack | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Buildsystem/spack/spack b/Buildsystem/spack/spack
index 0c7917e9f..5dd0fd33c 160000
--- a/Buildsystem/spack/spack
+++ b/Buildsystem/spack/spack
@@ -1 +1 @@
-Subproject commit 0c7917e9f0fbae36181b91b8641d06cbc02c0acf
+Subproject commit 5dd0fd33c5b7dc1b58fff15732a2b6e5a9989934

From c3d7c0dbc56a9ef573c50014123601e1bcdb2d4d Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Tue, 7 Jan 2025 10:57:14 -0500
Subject: [PATCH 49/51] Update local Spack submodule.

---
 Buildsystem/spack/spack | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Buildsystem/spack/spack b/Buildsystem/spack/spack
index 5dd0fd33c..6ff72a5b1 160000
--- a/Buildsystem/spack/spack
+++ b/Buildsystem/spack/spack
@@ -1 +1 @@
-Subproject commit 5dd0fd33c5b7dc1b58fff15732a2b6e5a9989934
+Subproject commit 6ff72a5b1af5925c3ac36d90e4e13f5f53d4b725

From db4d6c1bae8fb18d4d643575c3483c8654f4be62 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Tue, 7 Jan 2025 11:15:51 -0500
Subject: [PATCH 50/51] Update README with Enzyme dependency.

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 74acceff3..33f53555c 100644
--- a/README.md
+++ b/README.md
@@ -17,6 +17,8 @@ You should have all of the following installed before installing GridKit™
 	- [SUNDIALS](https://github.com/LLNL/sundials) >= 7.0.0
 	- [Suitesparse](https://github.com/DrTimothyAldenDavis/SuiteSparse) >= 5.x (optional)
 	- [Ipopt](https://github.com/coin-or/Ipopt) >= 3.x (optional)
+    - [Enzyme](https://github.com/EnzymeAD/Enzyme) >=0.0.131 (optional)
+        - [LLVM](https://github.com/llvm/llvm-project) >= 18.x 
 - [CMake](https://cmake.org/) >= 3.12
 - C++ 17 compliant compiler
 

From 20470b6d9980cfa18ef54239b3b7431da53cafd4 Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <koukpaizannk@ornl.gov>
Date: Mon, 13 Jan 2025 10:54:58 -0500
Subject: [PATCH 51/51] Changed Enzyme add_custom_target name to fix build
 issues with Ninja.

---
 CMake/FindEnzyme.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMake/FindEnzyme.cmake b/CMake/FindEnzyme.cmake
index 3840a526f..21cec75f1 100644
--- a/CMake/FindEnzyme.cmake
+++ b/CMake/FindEnzyme.cmake
@@ -188,7 +188,7 @@ macro(enzyme_add_executable)
     )
 
   add_custom_target(
-    ${enzyme_add_executable_NAME} ALL
+    "${enzyme_add_executable_NAME}_target" ALL
     DEPENDS ${PHASE5}
     )
 endmacro()