diff --git a/cmake/common.cmake b/cmake/common.cmake index 2de6dc758f..f1e21913f5 100755 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -1143,6 +1143,11 @@ option(NSC_DEBUG_EDIF_LINE_BIT "Add \"-fspv-debug=line\" to NSC Debug CLI" OFF) option(NSC_DEBUG_EDIF_TOOL_BIT "Add \"-fspv-debug=tool\" to NSC Debug CLI" ON) option(NSC_DEBUG_EDIF_NON_SEMANTIC_BIT "Add \"-fspv-debug=vulkan-with-source\" to NSC Debug CLI" OFF) option(NSC_USE_DEPFILE "Generate depfiles for NSC custom commands" ON) +option(NBL_NSC_DISABLE_CUSTOM_COMMANDS "Disable NSC custom commands" OFF) +option(NBL_NSC_VERBOSE "Enable NSC verbose logging to .log" ON) +option(NSC_SHADER_CACHE "Enable NSC shader cache" ON) +option(NSC_PREPROCESS_CACHE "Enable NSC preprocess cache" ON) +set(NSC_CACHE_DIR "" CACHE PATH "Optional root directory for NSC cache files (shader/preprocess)") function(NBL_CREATE_NSC_COMPILE_RULES) set(COMMENT "this code has been autogenerated with Nabla CMake NBL_CREATE_HLSL_COMPILE_RULES utility") @@ -1152,18 +1157,13 @@ function(NBL_CREATE_NSC_COMPILE_RULES) // -> @COMMENT@! #ifndef _PERMUTATION_CAPS_AUTO_GEN_GLOBALS_INCLUDED_ #define _PERMUTATION_CAPS_AUTO_GEN_GLOBALS_INCLUDED_ -#ifdef __HLSL_VERSION #include struct DeviceConfigCaps { @CAPS_EVAL@ }; - -#include "@TARGET_INPUT@" - -#endif // __HLSL_VERSION -#endif // _PERMUTATION_CAPS_AUTO_GEN_GLOBALS_INCLUDED_ // <- @COMMENT@! +#endif // _PERMUTATION_CAPS_AUTO_GEN_GLOBALS_INCLUDED_ ]=]) @@ -1219,9 +1219,14 @@ struct DeviceConfigCaps set(REQUIRED_SINGLE_ARGS TARGET BINARY_DIR OUTPUT_VAR INPUTS INCLUDE NAMESPACE MOUNT_POINT_DEFINE) set(OPTIONAL_SINGLE_ARGS GLOB_DIR) - cmake_parse_arguments(IMPL "DISCARD_DEFAULT_GLOB" "${REQUIRED_SINGLE_ARGS};${OPTIONAL_SINGLE_ARGS};LINK_TO" "COMMON_OPTIONS;DEPENDS" ${ARGV}) + cmake_parse_arguments(IMPL "DISCARD_DEFAULT_GLOB;DISABLE_CUSTOM_COMMANDS" "${REQUIRED_SINGLE_ARGS};${OPTIONAL_SINGLE_ARGS};LINK_TO" "COMMON_OPTIONS;DEPENDS" ${ARGV}) NBL_PARSE_REQUIRED(IMPL ${REQUIRED_SINGLE_ARGS}) + set(_NBL_DISABLE_CUSTOM_COMMANDS FALSE) + if(NBL_NSC_DISABLE_CUSTOM_COMMANDS OR IMPL_DISABLE_CUSTOM_COMMANDS) + set(_NBL_DISABLE_CUSTOM_COMMANDS TRUE) + endif() + set(IMPL_HLSL_GLOB "") if(NOT IMPL_DISCARD_DEFAULT_GLOB) set(GLOB_ROOT "${CMAKE_CURRENT_SOURCE_DIR}") @@ -1250,7 +1255,17 @@ struct DeviceConfigCaps if(NOT HEADER_RULE_GENERATED) set(INCLUDE_DIR "$/${IMPL_TARGET}/.cmake/include/$") set(INCLUDE_FILE "${INCLUDE_DIR}/$") - set(INCLUDE_CONTENT $) + set(NBL_HEADER_GUARD_RAW "${IMPL_TARGET}_${IMPL_NAMESPACE}_SPIRV_KEYS_HPP_INCLUDED") + string(SHA1 NBL_HEADER_GUARD_HASH "${NBL_HEADER_GUARD_RAW}") + string(TOUPPER "${NBL_HEADER_GUARD_HASH}" NBL_HEADER_GUARD_HASH_UPPER) + set(NBL_HEADER_GUARD "SPIRV_KEYS_${NBL_HEADER_GUARD_HASH_UPPER}_HPP_INCLUDED") + set(INCLUDE_CONTENT_TEMPLATE [=[ +#ifndef @NBL_HEADER_GUARD@ +#define @NBL_HEADER_GUARD@ +$ +#endif +]=]) + string(CONFIGURE "${INCLUDE_CONTENT_TEMPLATE}" INCLUDE_CONTENT @ONLY) file(GENERATE OUTPUT ${INCLUDE_FILE} CONTENT $ @@ -1268,7 +1283,9 @@ struct DeviceConfigCaps set_target_properties(${IMPL_TARGET} PROPERTIES NBL_HEADER_GENERATED_RULE ON) set(HEADER_ITEM_VIEW [=[ -#include "nabla.h" +#include +#include +#include "nbl/core/string/SpirvKeyHelpers.h" ]=]) set_property(TARGET ${IMPL_TARGET} APPEND_STRING PROPERTY NBL_HEADER_CONTENT "${HEADER_ITEM_VIEW}") @@ -1283,13 +1300,18 @@ struct DeviceConfigCaps if(NOT NS_IMPL_KEYS_PROPERTY_DEFINED) set(HEADER_ITEM_VIEW [=[ namespace @IMPL_NAMESPACE@ { - template - inline const nbl::core::string get_spirv_key(const nbl::video::SPhysicalDeviceLimits& limits, const nbl::video::SPhysicalDeviceFeatures& features); + template + requires ((... && !std::is_pointer_v>)) + inline constexpr typename nbl::core::detail::StringLiteralBufferType::type get_spirv_key(const Args&... args) + { + return nbl::core::detail::SpirvKeyBuilder::build(args...); + } - template - inline const nbl::core::string get_spirv_key(const nbl::video::ILogicalDevice* device) + template + inline std::string get_spirv_key(const Device* device, const Args&... args) { - return get_spirv_key(device->getPhysicalDevice()->getLimits(), device->getEnabledFeatures()); + const auto key = nbl::core::detail::SpirvKeyBuilder::build_from_device(device, args...); + return std::string(key.view()); } } @@ -1368,67 +1390,331 @@ namespace @IMPL_NAMESPACE@ { ) endfunction() + macro(NBL_NSC_RESOLVE_CAP_KIND _CAP_KIND_RAW _CAP_STRUCT _CAP_NAME _OUT_KIND) + set(_CAP_KIND_RAW "${_CAP_KIND_RAW}") + set(_CAP_STRUCT "${_CAP_STRUCT}") + + if(_CAP_KIND_RAW STREQUAL "custom") + if(_CAP_STRUCT STREQUAL "") + ERROR_WHILE_PARSING_ITEM( + "CAPS entry with kind \"custom\" requires \"struct\".\n" + ) + endif() + set(${_OUT_KIND} "${_CAP_STRUCT}") + else() + set(${_OUT_KIND} "${_CAP_KIND_RAW}") + endif() + + if(NOT "${${_OUT_KIND}}" MATCHES "^[A-Za-z_][A-Za-z0-9_]*$") + ERROR_WHILE_PARSING_ITEM( + "Invalid CAP kind \"${${_OUT_KIND}}\" for ${_CAP_NAME}\n" + "CAP kinds must be valid C/C++ identifiers." + ) + endif() + endmacro() + + macro(NBL_REQUIRE_PYTHON) + if(NOT Python3_EXECUTABLE) + find_package(Python3 COMPONENTS Interpreter REQUIRED) + endif() + endmacro() + + macro(NBL_NORMALIZE_FLOAT_LITERAL _CAP_NAME _VALUE _MANTISSA_DIGITS _TYPE_LABEL _OUT_VAR) + NBL_REQUIRE_PYTHON() + set(_NBL_RAW "${_VALUE}") + if(_TYPE_LABEL STREQUAL "float") + if("${_NBL_RAW}" MATCHES "^[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[fF]$") + string(REGEX REPLACE "[fF]$" "" _NBL_RAW "${_NBL_RAW}") + endif() + elseif(_TYPE_LABEL STREQUAL "double") + if("${_NBL_RAW}" MATCHES "^[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[dD]$") + string(REGEX REPLACE "[dD]$" "" _NBL_RAW "${_NBL_RAW}") + endif() + endif() + + set(_NBL_CANON_DONE FALSE) + if("${_NBL_RAW}" MATCHES "^[+-]?[0-9]\\.([0-9]+)e([+-][0-9]+)$") + set(_NBL_MANTISSA "${CMAKE_MATCH_1}") + set(_NBL_EXPONENT "${CMAKE_MATCH_2}") + string(LENGTH "${_NBL_MANTISSA}" _NBL_MANTISSA_LEN) + string(LENGTH "${_NBL_EXPONENT}" _NBL_EXPONENT_LEN) + math(EXPR _NBL_EXPONENT_DIGITS "${_NBL_EXPONENT_LEN} - 1") + if(_NBL_MANTISSA_LEN EQUAL ${_MANTISSA_DIGITS} AND _NBL_EXPONENT_DIGITS GREATER_EQUAL 2 AND _NBL_EXPONENT_DIGITS LESS_EQUAL 3) + string(TOLOWER "${_NBL_RAW}" _NBL_CANON) + set(_NBL_CANON_DONE TRUE) + endif() + endif() + + if(NOT _NBL_CANON_DONE) + set(_NBL_PY_SCRIPT [=[ +import sys,math,struct +t=sys.argv[1] +s=sys.argv[2] +if t=="float" and s[-1:] in ("f","F"): + s=s[:-1] +if t=="double" and s[-1:] in ("d","D"): + s=s[:-1] +try: + x=float(s) +except Exception: + sys.exit(2) +if t=="float": + x=struct.unpack("!f",struct.pack("!f",x))[0] +if not math.isfinite(x): + sys.exit(2) +p=8 if t=="float" else 16 +sign="-" if x<0 else "" +x=abs(x) +if x==0.0: + sys.stdout.write(sign+"0."+"0"*p+"e+00") + sys.exit(0) +m=x +e=0 +while m>=10.0: + m/=10.0 + e+=1 +while m<1.0: + m*=10.0 + e-=1 +digits=[0]*(p+1) +digits[0]=int(m) +frac=m-digits[0] +for i in range(1,p+1): + frac*=10.0 + d=int(frac) + if d>9: + d=9 + digits[i]=d + frac-=d +frac*=10.0 +rd=int(frac) +if rd>9: + rd=9 +rem=frac-rd +ru = rd>5 or (rd==5 and (rem>0 or (digits[p]%2))) +if ru: + i=p + while i>=0 and digits[i]==9: + digits[i]=0 + i-=1 + if i>=0: + digits[i]+=1 + else: + digits[0]=1 + for j in range(1,p+1): + digits[j]=0 + e+=1 +es="-" if e<0 else "+" +if e<0: + e=-e +ew=3 if e>=100 else 2 +sys.stdout.write(sign+str(digits[0])+"."+("".join(str(d) for d in digits[1:]))+"e"+es+str(e).zfill(ew)) +]=]) + execute_process( + COMMAND "${Python3_EXECUTABLE}" -c "${_NBL_PY_SCRIPT}" "${_TYPE_LABEL}" "${_NBL_RAW}" + RESULT_VARIABLE _NBL_FMT_RESULT + OUTPUT_VARIABLE _NBL_CANON + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(NOT _NBL_FMT_RESULT EQUAL 0) + ERROR_WHILE_PARSING_ITEM( + "Invalid CAP value \"${_VALUE}\" for ${_CAP_NAME}\n" + "${_TYPE_LABEL} values must be numbers or numeric strings." + ) + endif() + endif() + set(${_OUT_VAR} "${_NBL_CANON}") + endmacro() + + macro(NBL_HASH_SPIRV_KEY _VALUE _OUT_VAR) + NBL_REQUIRE_PYTHON() + set(_NBL_PY_HASH [=[ +import sys +s=sys.argv[1] +h=14695981039346656037 +for b in s.encode("utf-8"): + h^=b + h=(h*1099511628211)&0xFFFFFFFFFFFFFFFF +sys.stdout.write(str(h)) +]=]) + execute_process( + COMMAND "${Python3_EXECUTABLE}" -c "${_NBL_PY_HASH}" "${_VALUE}" + RESULT_VARIABLE _NBL_HASH_RESULT + OUTPUT_VARIABLE _NBL_HASH_OUT + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(NOT _NBL_HASH_RESULT EQUAL 0) + message(FATAL_ERROR "Failed to hash SPIR-V key \"${_VALUE}\"") + endif() + set(${_OUT_VAR} "${_NBL_HASH_OUT}") + endmacro() + set(CAP_NAMES "") set(CAP_TYPES "") set(CAP_KINDS "") + set(CAP_VALUES_INDEX 0) if(HAS_CAPS) math(EXPR LAST_CAP "${CAPS_LENGTH} - 1") foreach(CAP_IDX RANGE 0 ${LAST_CAP}) - string(JSON CAP_KIND ERROR_VARIABLE CAP_TYPE_ERROR GET "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} kind) - string(JSON CAP_NAME GET "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} name) - string(JSON CAP_TYPE GET "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} type) + string(JSON MEMBERS_TYPE ERROR_VARIABLE MEMBERS_ERROR TYPE "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} members) + if(MEMBERS_TYPE STREQUAL "ARRAY") + string(JSON CAP_KIND_RAW ERROR_VARIABLE CAP_KIND_ERROR GET "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} kind) + if(CAP_KIND_ERROR) + set(CAP_KIND_RAW limits) + endif() + + string(JSON CAP_STRUCT ERROR_VARIABLE CAP_STRUCT_ERROR GET "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} struct) + if(CAP_STRUCT_ERROR) + set(CAP_STRUCT "") + endif() - # -> TODO: improve validation, input should be string - if(CAP_TYPE_ERROR) - set(CAP_KIND limits) # I assume its limit by default (or when invalid value present, currently) + NBL_NSC_RESOLVE_CAP_KIND("${CAP_KIND_RAW}" "${CAP_STRUCT}" "member group" CAP_KIND) + + string(JSON MEMBERS_LENGTH LENGTH "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} members) + if(MEMBERS_LENGTH GREATER 0) + math(EXPR LAST_MEMBER "${MEMBERS_LENGTH} - 1") + foreach(MEMBER_IDX RANGE 0 ${LAST_MEMBER}) + string(JSON CAP_NAME GET "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} members ${MEMBER_IDX} name) + string(JSON CAP_TYPE GET "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} members ${MEMBER_IDX} type) + + if(NOT CAP_TYPE MATCHES "^(bool|uint16_t|uint32_t|uint64_t|int16_t|int32_t|int64_t|float|double)$") + ERROR_WHILE_PARSING_ITEM( + "Invalid CAP type \"${CAP_TYPE}\" for ${CAP_NAME}\n" + "Allowed types are: bool, uint16_t, uint32_t, uint64_t, int16_t, int32_t, int64_t, float, double" + ) + endif() + + string(JSON CAP_VALUES_LENGTH LENGTH "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} members ${MEMBER_IDX} values) + + set(VALUES "") + math(EXPR LAST_VAL "${CAP_VALUES_LENGTH} - 1") + foreach(VAL_IDX RANGE 0 ${LAST_VAL}) + string(JSON VALUE GET "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} members ${MEMBER_IDX} values ${VAL_IDX}) + string(JSON VAL_TYPE TYPE "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} members ${MEMBER_IDX} values ${VAL_IDX}) + + if(CAP_TYPE STREQUAL "float") + if(NOT (VAL_TYPE STREQUAL "STRING" OR VAL_TYPE STREQUAL "NUMBER")) + ERROR_WHILE_PARSING_ITEM( + "Invalid CAP value \"${VALUE}\" for ${CAP_NAME}\n" + "Float values must be numbers or numeric strings." + ) + endif() + NBL_NORMALIZE_FLOAT_LITERAL("${CAP_NAME}" "${VALUE}" 8 "float" VALUE) + elseif(CAP_TYPE STREQUAL "double") + if(NOT (VAL_TYPE STREQUAL "STRING" OR VAL_TYPE STREQUAL "NUMBER")) + ERROR_WHILE_PARSING_ITEM( + "Invalid CAP value \"${VALUE}\" for ${CAP_NAME}\n" + "Double values must be numbers or numeric strings." + ) + endif() + NBL_NORMALIZE_FLOAT_LITERAL("${CAP_NAME}" "${VALUE}" 16 "double" VALUE) + elseif(NOT VAL_TYPE STREQUAL "NUMBER") + ERROR_WHILE_PARSING_ITEM( + "Invalid CAP value \"${VALUE}\" for CAP \"${CAP_NAME}\" of type ${CAP_TYPE}\n" + "Use numbers for uint*_t and 0/1 for bools." + ) + elseif(NOT VAL_TYPE STREQUAL "NUMBER") + ERROR_WHILE_PARSING_ITEM( + "Invalid CAP value \"${VALUE}\" for CAP \"${CAP_NAME}\" of type ${CAP_TYPE}\n" + "Use numbers for uint*_t and 0/1 for bools." + ) + endif() + + if(CAP_TYPE STREQUAL "bool") + if(NOT VALUE MATCHES "^[01]$") + ERROR_WHILE_PARSING_ITEM( + "Invalid bool value \"${VALUE}\" for ${CAP_NAME}\n" + "Boolean CAPs can only have values 0 or 1." + ) + endif() + endif() + list(APPEND VALUES "${VALUE}") + endforeach() + + set(CAP_VALUES_${CAP_VALUES_INDEX} "${VALUES}") + list(APPEND CAP_NAMES "${CAP_NAME}") + list(APPEND CAP_TYPES "${CAP_TYPE}") + list(APPEND CAP_KINDS "${CAP_KIND}") + math(EXPR CAP_VALUES_INDEX "${CAP_VALUES_INDEX} + 1") + endforeach() + endif() else() - if(NOT CAP_KIND MATCHES "^(limits|features)$") + if(NOT MEMBERS_ERROR) ERROR_WHILE_PARSING_ITEM( - "Invalid CAP kind \"${CAP_KIND}\" for ${CAP_NAME}\n" - "Allowed kinds are: limits, features" + "CAPS.members must be an array when provided." ) endif() - endif() - # <- - if(NOT CAP_TYPE MATCHES "^(bool|uint16_t|uint32_t|uint64_t)$") - ERROR_WHILE_PARSING_ITEM( - "Invalid CAP type \"${CAP_TYPE}\" for ${CAP_NAME}\n" - "Allowed types are: bool, uint16_t, uint32_t, uint64_t" - ) - endif() + string(JSON CAP_KIND_RAW ERROR_VARIABLE CAP_KIND_ERROR GET "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} kind) + string(JSON CAP_NAME GET "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} name) + string(JSON CAP_TYPE GET "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} type) - string(JSON CAP_VALUES_LENGTH LENGTH "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} values) + if(CAP_KIND_ERROR) + set(CAP_KIND_RAW limits) # I assume its limit by default (or when invalid value present, currently) + endif() - set(VALUES "") - math(EXPR LAST_VAL "${CAP_VALUES_LENGTH} - 1") - foreach(VAL_IDX RANGE 0 ${LAST_VAL}) - string(JSON VALUE GET "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} values ${VAL_IDX}) - string(JSON VAL_TYPE TYPE "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} values ${VAL_IDX}) + string(JSON CAP_STRUCT ERROR_VARIABLE CAP_STRUCT_ERROR GET "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} struct) + if(CAP_STRUCT_ERROR) + set(CAP_STRUCT "") + endif() + + NBL_NSC_RESOLVE_CAP_KIND("${CAP_KIND_RAW}" "${CAP_STRUCT}" "${CAP_NAME}" CAP_KIND) - if(NOT VAL_TYPE STREQUAL "NUMBER") + if(NOT CAP_TYPE MATCHES "^(bool|uint16_t|uint32_t|uint64_t|int16_t|int32_t|int64_t|float|double)$") ERROR_WHILE_PARSING_ITEM( - "Invalid CAP value \"${VALUE}\" for CAP \"${CAP_NAME}\" of type ${CAP_TYPE}\n" - "Use numbers for uint*_t and 0/1 for bools." + "Invalid CAP type \"${CAP_TYPE}\" for ${CAP_NAME}\n" + "Allowed types are: bool, uint16_t, uint32_t, uint64_t, int16_t, int32_t, int64_t, float, double" ) endif() - if(CAP_TYPE STREQUAL "bool") - if(NOT VALUE MATCHES "^[01]$") + string(JSON CAP_VALUES_LENGTH LENGTH "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} values) + + set(VALUES "") + math(EXPR LAST_VAL "${CAP_VALUES_LENGTH} - 1") + foreach(VAL_IDX RANGE 0 ${LAST_VAL}) + string(JSON VALUE GET "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} values ${VAL_IDX}) + string(JSON VAL_TYPE TYPE "${IMPL_INPUTS}" ${INDEX} CAPS ${CAP_IDX} values ${VAL_IDX}) + + if(CAP_TYPE STREQUAL "float") + if(NOT (VAL_TYPE STREQUAL "STRING" OR VAL_TYPE STREQUAL "NUMBER")) + ERROR_WHILE_PARSING_ITEM( + "Invalid CAP value \"${VALUE}\" for ${CAP_NAME}\n" + "Float values must be numbers or numeric strings." + ) + endif() + NBL_NORMALIZE_FLOAT_LITERAL("${CAP_NAME}" "${VALUE}" 8 "float" VALUE) + elseif(CAP_TYPE STREQUAL "double") + if(NOT (VAL_TYPE STREQUAL "STRING" OR VAL_TYPE STREQUAL "NUMBER")) + ERROR_WHILE_PARSING_ITEM( + "Invalid CAP value \"${VALUE}\" for ${CAP_NAME}\n" + "Double values must be numbers or numeric strings." + ) + endif() + NBL_NORMALIZE_FLOAT_LITERAL("${CAP_NAME}" "${VALUE}" 16 "double" VALUE) + elseif(NOT VAL_TYPE STREQUAL "NUMBER") ERROR_WHILE_PARSING_ITEM( - "Invalid bool value \"${VALUE}\" for ${CAP_NAME}\n" - "Boolean CAPs can only have values 0 or 1." + "Invalid CAP value \"${VALUE}\" for CAP \"${CAP_NAME}\" of type ${CAP_TYPE}\n" + "Use numbers for uint*_t and 0/1 for bools." ) endif() - endif() - list(APPEND VALUES "${VALUE}") - endforeach() + if(CAP_TYPE STREQUAL "bool") + if(NOT VALUE MATCHES "^[01]$") + ERROR_WHILE_PARSING_ITEM( + "Invalid bool value \"${VALUE}\" for ${CAP_NAME}\n" + "Boolean CAPs can only have values 0 or 1." + ) + endif() + endif() + list(APPEND VALUES "${VALUE}") + endforeach() - set(CAP_VALUES_${CAP_IDX} "${VALUES}") - list(APPEND CAP_NAMES "${CAP_NAME}") - list(APPEND CAP_TYPES "${CAP_TYPE}") - list(APPEND CAP_KINDS "${CAP_KIND}") + set(CAP_VALUES_${CAP_VALUES_INDEX} "${VALUES}") + list(APPEND CAP_NAMES "${CAP_NAME}") + list(APPEND CAP_TYPES "${CAP_TYPE}") + list(APPEND CAP_KINDS "${CAP_KIND}") + math(EXPR CAP_VALUES_INDEX "${CAP_VALUES_INDEX} + 1") + endif() endforeach() endif() @@ -1453,64 +1739,441 @@ namespace @IMPL_NAMESPACE@ { set_property(TARGET ${IMPL_TARGET} APPEND PROPERTY NBL_CANONICAL_IDENTIFIERS "${NEW_CANONICAL_IDENTIFIER}") - set(HEADER_ITEM_VIEW [=[ -namespace @IMPL_NAMESPACE@ { - template<> - inline const nbl::core::string get_spirv_key - (const nbl::video::SPhysicalDeviceLimits& limits, const nbl::video::SPhysicalDeviceFeatures& features) - { - nbl::core::string retval = "@BASE_KEY@"; -@RETVAL_EVAL@ - retval += ".spv"; - return "$/" + retval; - } -} + if(NUM_CAPS GREATER 0) + set(KIND_ORDER "") + foreach(_NBL_KIND IN LISTS CAP_KINDS) + list(FIND KIND_ORDER "${_NBL_KIND}" _NBL_KIND_INDEX) + if(_NBL_KIND_INDEX EQUAL -1) + list(APPEND KIND_ORDER "${_NBL_KIND}") + endif() + endforeach() + + set(ORDERED_KINDS "${KIND_ORDER}") + + foreach(_NBL_KIND IN LISTS ORDERED_KINDS) + unset(_NBL_KIND_INDICES_${_NBL_KIND}) + endforeach() + + math(EXPR LAST_CAP "${NUM_CAPS} - 1") + foreach(i RANGE 0 ${LAST_CAP}) + list(GET CAP_KINDS ${i} _NBL_KIND) + set(_NBL_ORIG_CAP_VALUES_${i} "${CAP_VALUES_${i}}") + list(APPEND _NBL_KIND_INDICES_${_NBL_KIND} ${i}) + endforeach() + + set(_NBL_ORDERED_INDICES "") + foreach(_NBL_KIND IN LISTS ORDERED_KINDS) + if(DEFINED _NBL_KIND_INDICES_${_NBL_KIND}) + list(APPEND _NBL_ORDERED_INDICES ${_NBL_KIND_INDICES_${_NBL_KIND}}) + endif() + endforeach() + + set(_NBL_ORDERED_CAP_NAMES "") + set(_NBL_ORDERED_CAP_TYPES "") + set(_NBL_ORDERED_CAP_KINDS "") + set(_NBL_ORDERED_VALUES_INDEX 0) + foreach(_NBL_INDEX IN LISTS _NBL_ORDERED_INDICES) + list(GET CAP_NAMES ${_NBL_INDEX} _NBL_CAP_NAME) + list(GET CAP_TYPES ${_NBL_INDEX} _NBL_CAP_TYPE) + list(GET CAP_KINDS ${_NBL_INDEX} _NBL_CAP_KIND) + set(_NBL_CAP_VALUES "${_NBL_ORIG_CAP_VALUES_${_NBL_INDEX}}") + list(APPEND _NBL_ORDERED_CAP_NAMES "${_NBL_CAP_NAME}") + list(APPEND _NBL_ORDERED_CAP_TYPES "${_NBL_CAP_TYPE}") + list(APPEND _NBL_ORDERED_CAP_KINDS "${_NBL_CAP_KIND}") + set(CAP_VALUES_${_NBL_ORDERED_VALUES_INDEX} "${_NBL_CAP_VALUES}") + math(EXPR _NBL_ORDERED_VALUES_INDEX "${_NBL_ORDERED_VALUES_INDEX} + 1") + endforeach() + + set(CAP_NAMES "${_NBL_ORDERED_CAP_NAMES}") + set(CAP_TYPES "${_NBL_ORDERED_CAP_TYPES}") + set(CAP_KINDS "${_NBL_ORDERED_CAP_KINDS}") + list(LENGTH CAP_NAMES NUM_CAPS) + else() + set(ORDERED_KINDS "") + endif() + + list(LENGTH ORDERED_KINDS ORDERED_KIND_COUNT) + set(NON_DEVICE_KINDS "") + set(HAS_LIMITS FALSE) + set(HAS_FEATURES FALSE) + foreach(_NBL_KIND IN LISTS ORDERED_KINDS) + if(_NBL_KIND STREQUAL "limits") + set(HAS_LIMITS TRUE) + elseif(_NBL_KIND STREQUAL "features") + set(HAS_FEATURES TRUE) + else() + list(APPEND NON_DEVICE_KINDS "${_NBL_KIND}") + endif() + endforeach() + list(LENGTH NON_DEVICE_KINDS NON_DEVICE_COUNT) + + string(MAKE_C_IDENTIFIER "${BASE_KEY}" BASE_KEY_IDENT) + string(MD5 BASE_KEY_HASH "${BASE_KEY}") + string(SUBSTRING "${BASE_KEY_HASH}" 0 8 BASE_KEY_HASH8) + set(KIND_PREFIX "${BASE_KEY_IDENT}_${BASE_KEY_HASH8}") + + set(MATCH_KINDS "") + foreach(_NBL_KIND IN LISTS ORDERED_KINDS) + list(APPEND MATCH_KINDS "${_NBL_KIND}") + endforeach() + + foreach(_NBL_KIND IN LISTS MATCH_KINDS) + set(_NBL_KIND_MEMBERS_${_NBL_KIND} "") + set(_NBL_KIND_TYPES_${_NBL_KIND} "") + endforeach() + + if(NUM_CAPS GREATER 0) + math(EXPR _NBL_LAST_CAP "${NUM_CAPS} - 1") + foreach(i RANGE ${_NBL_LAST_CAP}) + list(GET CAP_KINDS ${i} _NBL_KIND) + list(GET CAP_NAMES ${i} _NBL_CAP) + list(GET CAP_TYPES ${i} _NBL_TYPE) + list(FIND _NBL_KIND_MEMBERS_${_NBL_KIND} "${_NBL_CAP}" _NBL_MEMBER_INDEX) + if(_NBL_MEMBER_INDEX EQUAL -1) + list(APPEND _NBL_KIND_MEMBERS_${_NBL_KIND} "${_NBL_CAP}") + list(APPEND _NBL_KIND_TYPES_${_NBL_KIND} "${_NBL_TYPE}") + endif() + endforeach() + endif() -]=]) - unset(RETVAL_EVAL) list(LENGTH CAP_NAMES CAP_COUNT) + + set(RETVAL_FMT "${BASE_KEY}") + set(RETVAL_ARGS "") + set(CX_CAPACITY 0) + string(LENGTH "${BASE_KEY}" CX_BASE_LEN) + math(EXPR CX_CAPACITY "${CX_BASE_LEN} + 4 + 24") if(CAP_COUNT GREATER 0) math(EXPR LAST_CAP "${CAP_COUNT} - 1") + set(PREV_KIND "") foreach(i RANGE ${LAST_CAP}) list(GET CAP_NAMES ${i} CAP) list(GET CAP_KINDS ${i} KIND) + list(GET CAP_TYPES ${i} TYPE) + if(NOT KIND STREQUAL PREV_KIND) + string(APPEND RETVAL_FMT "__${KIND}") + string(LENGTH "${KIND}" KIND_LEN) + math(EXPR CX_CAPACITY "${CX_CAPACITY} + 2 + ${KIND_LEN}") + set(PREV_KIND "${KIND}") + endif() + string(APPEND RETVAL_FMT ".${CAP}_%s") + list(APPEND RETVAL_ARGS "nbl_spirv_${KIND}.${CAP}") + string(LENGTH "${CAP}" CAP_LEN) + math(EXPR CX_CAPACITY "${CX_CAPACITY} + 2 + ${CAP_LEN}") + if(TYPE STREQUAL "bool") + set(DIGITS 1) + elseif(TYPE STREQUAL "uint16_t") + set(DIGITS 5) + elseif(TYPE STREQUAL "uint32_t") + set(DIGITS 10) + elseif(TYPE STREQUAL "int16_t") + set(DIGITS 6) + elseif(TYPE STREQUAL "int32_t") + set(DIGITS 11) + elseif(TYPE STREQUAL "int64_t") + set(DIGITS 20) + elseif(TYPE STREQUAL "uint64_t") + set(DIGITS 20) + elseif(TYPE STREQUAL "float") + set(DIGITS 16) + elseif(TYPE STREQUAL "double") + set(DIGITS 24) + else() + set(DIGITS 20) + endif() + math(EXPR CX_CAPACITY "${CX_CAPACITY} + ${DIGITS}") + endforeach() + endif() + string(APPEND RETVAL_FMT ".spv") + if(RETVAL_ARGS) + string(JOIN ", " RETVAL_ARGS_JOINED ${RETVAL_ARGS}) + set(RETVAL_ARGS_STR ", ${RETVAL_ARGS_JOINED}") + else() + set(RETVAL_ARGS_STR "") + endif() + string(CONFIGURE [=[ + typename StringLiteralBufferType::type nbl_spirv_full = {}; + nbl::core::detail::append_printf_s(nbl_spirv_full@RETVAL_ARGS_STR@); + retval.append("$/"); + nbl::core::detail::put(retval, nbl::core::FNV1a_64(nbl_spirv_full.view())); + retval.append(".spv"); +]=] RETVAL_EVAL_CONSTEXPR @ONLY) + + set(SPIRV_CUSTOM_TRAITS "") + foreach(_NBL_KIND IN LISTS MATCH_KINDS) + set(_NBL_MEMBER_LINES "") + list(LENGTH _NBL_KIND_MEMBERS_${_NBL_KIND} _NBL_MEMBER_COUNT) + set(KIND_TRAIT "${KIND_PREFIX}_${_NBL_KIND}") + if(_NBL_MEMBER_COUNT GREATER 0) + math(EXPR _NBL_MEMBER_LAST "${_NBL_MEMBER_COUNT} - 1") + foreach(_NBL_MEMBER_INDEX RANGE ${_NBL_MEMBER_LAST}) + list(GET _NBL_KIND_MEMBERS_${_NBL_KIND} ${_NBL_MEMBER_INDEX} _NBL_MEMBER_NAME) + list(GET _NBL_KIND_TYPES_${_NBL_KIND} ${_NBL_MEMBER_INDEX} _NBL_MEMBER_TYPE) + set(MEMBER_NAME "${_NBL_MEMBER_NAME}") + set(MEMBER_TYPE "${_NBL_MEMBER_TYPE}") + string(CONFIGURE [=[ + requires std::is_same_v, @MEMBER_TYPE@>; +]=] _NBL_MEMBER_LINE @ONLY) + string(APPEND _NBL_MEMBER_LINES "${_NBL_MEMBER_LINE}") + endforeach() + set(KIND "${KIND_TRAIT}") + set(MEMBER_LINES "${_NBL_MEMBER_LINES}") string(CONFIGURE [=[ - retval += ".@CAP@_" + std::to_string(@KIND@.@CAP@); -]=] RETVALUE_VIEW @ONLY) - string(APPEND RETVAL_EVAL "${RETVALUE_VIEW}") + template + struct SpirvPerm_@KIND@ + { + static constexpr bool value = requires(const T& v) + { +@MEMBER_LINES@ }; + }; + +]=] _NBL_CUSTOM_TRAIT @ONLY) + else() + set(KIND "${KIND_TRAIT}") + string(CONFIGURE [=[ + template + struct SpirvPerm_@KIND@ + { + static constexpr bool value = false; + }; + +]=] _NBL_CUSTOM_TRAIT @ONLY) + endif() + string(APPEND SPIRV_CUSTOM_TRAITS "${_NBL_CUSTOM_TRAIT}") + endforeach() + + set(SPIRV_BUILD_REQUIRES "") + if(ORDERED_KIND_COUNT EQUAL 0) + set(SPIRV_BUILD_REQUIRES "requires (sizeof...(Args) == 0)") + else() + set(_NBL_REQS "") + set(_NBL_KIND_INDEX 0) + foreach(_NBL_KIND IN LISTS ORDERED_KINDS) + set(KIND_TRAIT "${KIND_PREFIX}_${_NBL_KIND}") + list(APPEND _NBL_REQS "SpirvPerm_${KIND_TRAIT}>>>::value") + math(EXPR _NBL_KIND_INDEX "${_NBL_KIND_INDEX} + 1") endforeach() + string(JOIN " && " _NBL_REQS_JOINED ${_NBL_REQS}) + set(SPIRV_BUILD_REQUIRES "requires (sizeof...(Args) == ${ORDERED_KIND_COUNT} && ${_NBL_REQS_JOINED})") endif() - + + set(SPIRV_ARG_DECLS "") + set(_NBL_KIND_INDEX 0) + foreach(_NBL_KIND IN LISTS ORDERED_KINDS) + string(APPEND SPIRV_ARG_DECLS "\t\tconst auto& nbl_spirv_${_NBL_KIND} = std::get<${_NBL_KIND_INDEX}>(std::forward_as_tuple(args...));\n") + math(EXPR _NBL_KIND_INDEX "${_NBL_KIND_INDEX} + 1") + endforeach() + + set(SPIRV_BUILD_FROM_DEVICE_REQUIRES "") + set(_NBL_DEVICE_REQS "") + if(HAS_LIMITS) + list(APPEND _NBL_DEVICE_REQS "nbl::core::detail::spirv_device_has_limits") + endif() + if(HAS_FEATURES) + list(APPEND _NBL_DEVICE_REQS "nbl::core::detail::spirv_device_has_features") + endif() + if(NON_DEVICE_COUNT EQUAL 0) + list(APPEND _NBL_DEVICE_REQS "sizeof...(Args) == 0") + else() + list(APPEND _NBL_DEVICE_REQS "sizeof...(Args) == ${NON_DEVICE_COUNT}") + set(_NBL_REQS "") + set(_NBL_KIND_INDEX 0) + foreach(_NBL_KIND IN LISTS NON_DEVICE_KINDS) + set(KIND_TRAIT "${KIND_PREFIX}_${_NBL_KIND}") + list(APPEND _NBL_REQS "SpirvPerm_${KIND_TRAIT}>>>::value") + math(EXPR _NBL_KIND_INDEX "${_NBL_KIND_INDEX} + 1") + endforeach() + if(_NBL_REQS) + string(JOIN " && " _NBL_REQS_JOINED ${_NBL_REQS}) + list(APPEND _NBL_DEVICE_REQS "${_NBL_REQS_JOINED}") + endif() + endif() + string(JOIN " && " SPIRV_DEVICE_REQUIRES_EXPR ${_NBL_DEVICE_REQS}) + set(SPIRV_BUILD_FROM_DEVICE_REQUIRES "requires (${SPIRV_DEVICE_REQUIRES_EXPR})") + + set(SPIRV_BUILD_FROM_DEVICE_ARGS "") + set(_NBL_ARG_INDEX 0) + foreach(_NBL_KIND IN LISTS ORDERED_KINDS) + if(_NBL_KIND STREQUAL "limits") + list(APPEND SPIRV_BUILD_FROM_DEVICE_ARGS "nbl::core::detail::spirv_device_get_limits(device)") + elseif(_NBL_KIND STREQUAL "features") + list(APPEND SPIRV_BUILD_FROM_DEVICE_ARGS "nbl::core::detail::spirv_device_get_features(device)") + else() + list(APPEND SPIRV_BUILD_FROM_DEVICE_ARGS "std::get<${_NBL_ARG_INDEX}>(std::forward_as_tuple(args...))") + math(EXPR _NBL_ARG_INDEX "${_NBL_ARG_INDEX} + 1") + endif() + endforeach() + if(SPIRV_BUILD_FROM_DEVICE_ARGS) + string(JOIN ", " SPIRV_BUILD_FROM_DEVICE_ARGS_JOINED ${SPIRV_BUILD_FROM_DEVICE_ARGS}) + else() + set(SPIRV_BUILD_FROM_DEVICE_ARGS_JOINED "") + endif() + + set(SPIRV_TRIVIAL_ASSERTS "") + + set(HEADER_ITEM_VIEW [=[ +namespace nbl::core::detail { + template<> + struct StringLiteralBufferType + { + using type = StringLiteralBuffer<@CX_CAPACITY@ + 1>; + }; + + template<> + struct SpirvKeyBuilder + { +@SPIRV_CUSTOM_TRAITS@ template + @SPIRV_BUILD_REQUIRES@ + static constexpr typename StringLiteralBufferType::type build(const Args&... args) + { +@SPIRV_ARG_DECLS@@SPIRV_TRIVIAL_ASSERTS@ typename StringLiteralBufferType::type retval = {}; +@RETVAL_EVAL_CONSTEXPR@ + return retval; + } + + template + @SPIRV_BUILD_FROM_DEVICE_REQUIRES@ + static constexpr typename StringLiteralBufferType::type build_from_device(const Device* device, const Args&... args) + { + return build(@SPIRV_BUILD_FROM_DEVICE_ARGS_JOINED@); + } + }; +} + +]=]) string(CONFIGURE "${HEADER_ITEM_VIEW}" HEADER_ITEM_EVAL @ONLY) set_property(TARGET ${IMPL_TARGET} APPEND_STRING PROPERTY NBL_HEADER_CONTENT "${HEADER_ITEM_EVAL}") - - function(GENERATE_KEYS PREFIX CAP_INDEX CAPS_EVAL_PART) + + function(GENERATE_KEYS PREFIX CAP_INDEX) + set(CAPS_VALUES_PART "${ARGN}") if(NUM_CAPS EQUAL 0 OR CAP_INDEX EQUAL ${NUM_CAPS}) - # generate .config file set(FINAL_KEY "${BASE_KEY}${PREFIX}.spv") # always add ext even if its already there to make sure asset loader always is able to load as IShader - set(CONFIG_FILE_TARGET_OUTPUT "${IMPL_BINARY_DIR}/${FINAL_KEY}") - set(CONFIG_FILE "${CONFIG_FILE_TARGET_OUTPUT}.config") - set(CAPS_EVAL "${CAPS_EVAL_PART}") + NBL_HASH_SPIRV_KEY("${FINAL_KEY}" FINAL_KEY_HASH) + set(HASHED_KEY "${FINAL_KEY_HASH}.spv") + set(CONFIG_FILE_TARGET_OUTPUT "${IMPL_BINARY_DIR}/${FINAL_KEY_HASH}") + set(CONFIG_FILE "${CONFIG_FILE_TARGET_OUTPUT}.in.hlsl") + set(CAPS_EVAL "") + if(NUM_CAPS GREATER 0) + set(CAPS_EVAL_LIMITS "") + set(CAPS_EVAL_FEATURES "") + set(_NBL_CUSTOM_KIND_LIST "") + foreach(_NBL_KIND IN LISTS ORDERED_KINDS) + if(NOT _NBL_KIND STREQUAL "limits" AND NOT _NBL_KIND STREQUAL "features") + list(APPEND _NBL_CUSTOM_KIND_LIST "${_NBL_KIND}") + set(_NBL_CUSTOM_LINES_${_NBL_KIND} "") + endif() + endforeach() + + math(EXPR _NBL_LAST_CAP "${NUM_CAPS} - 1") + foreach(i RANGE 0 ${_NBL_LAST_CAP}) + list(GET CAP_NAMES ${i} _NBL_CAP_NAME) + list(GET CAP_TYPES ${i} _NBL_CAP_TYPE) + list(GET CAP_KINDS ${i} _NBL_CAP_KIND) + list(GET CAPS_VALUES_PART ${i} _NBL_CAP_VALUE) + set(MEMBER_NAME "${_NBL_CAP_NAME}") + set(MEMBER_TYPE "${_NBL_CAP_TYPE}") + set(MEMBER_VALUE "${_NBL_CAP_VALUE}") + if(MEMBER_TYPE STREQUAL "double") + set(MEMBER_VALUE "${_NBL_CAP_VALUE}L") + endif() + string(CONFIGURE [=[ +NBL_CONSTEXPR_STATIC_INLINE @MEMBER_TYPE@ @MEMBER_NAME@ = (@MEMBER_TYPE@) @MEMBER_VALUE@; +]=] _NBL_MEMBER_LINE @ONLY) + if(_NBL_CAP_KIND STREQUAL "limits") + string(APPEND CAPS_EVAL_LIMITS " ${_NBL_MEMBER_LINE}") + elseif(_NBL_CAP_KIND STREQUAL "features") + string(APPEND CAPS_EVAL_FEATURES " ${_NBL_MEMBER_LINE}") + else() + set(_NBL_CUSTOM_LINE_VAR "_NBL_CUSTOM_LINES_${_NBL_CAP_KIND}") + set(${_NBL_CUSTOM_LINE_VAR} "${${_NBL_CUSTOM_LINE_VAR}} ${_NBL_MEMBER_LINE}") + endif() + endforeach() + + if(CAPS_EVAL_LIMITS) + string(APPEND CAPS_EVAL " // limits\n") + string(APPEND CAPS_EVAL "${CAPS_EVAL_LIMITS}") + endif() + if(CAPS_EVAL_FEATURES) + string(APPEND CAPS_EVAL " // features\n") + string(APPEND CAPS_EVAL "${CAPS_EVAL_FEATURES}") + endif() + + set(_NBL_HAS_CUSTOM FALSE) + foreach(_NBL_KIND IN LISTS _NBL_CUSTOM_KIND_LIST) + if(_NBL_CUSTOM_LINES_${_NBL_KIND}) + set(_NBL_HAS_CUSTOM TRUE) + endif() + endforeach() + + if(_NBL_HAS_CUSTOM) + string(APPEND CAPS_EVAL " // custom structs\n") + foreach(_NBL_KIND IN LISTS ORDERED_KINDS) + if(NOT _NBL_KIND STREQUAL "limits" AND NOT _NBL_KIND STREQUAL "features") + if(_NBL_CUSTOM_LINES_${_NBL_KIND}) + set(NBL_KIND_NAME "${_NBL_KIND}") + set(MEMBER_LINES "${_NBL_CUSTOM_LINES_${_NBL_KIND}}") + string(CONFIGURE [=[ + struct @NBL_KIND_NAME@ + { +@MEMBER_LINES@ }; +]=] _NBL_KIND_STRUCT @ONLY) + string(APPEND CAPS_EVAL "${_NBL_KIND_STRUCT}") + endif() + endif() + endforeach() + endif() + endif() + if(CAPS_EVAL STREQUAL "") + set(CAPS_EVAL " // no caps\n") + endif() string(CONFIGURE "${DEVICE_CONFIG_VIEW}" CONFIG_CONTENT @ONLY) file(WRITE "${CONFIG_FILE}" "${CONFIG_CONTENT}") + list(APPEND DEPENDS_ON "${TARGET_INPUT}" "${CONFIG_FILE}") # generate keys and commands for compiling shaders - set(FINAL_KEY_REL_PATH "$/${FINAL_KEY}") + set(FINAL_KEY_REL_PATH "$/${HASHED_KEY}") set(TARGET_OUTPUT "${IMPL_BINARY_DIR}/${FINAL_KEY_REL_PATH}") - set(DEPFILE_PATH "${TARGET_OUTPUT}.d") + set(DEPFILE_PATH "${TARGET_OUTPUT}.dep") set(NBL_NSC_LOG_PATH "${TARGET_OUTPUT}.log") + set(NBL_NSC_PREPROCESSED_PATH "${TARGET_OUTPUT}.pre.hlsl") + if(NSC_CACHE_DIR) + get_filename_component(NBL_NSC_CACHE_ROOT "${NSC_CACHE_DIR}" ABSOLUTE BASE_DIR "${CMAKE_BINARY_DIR}") + file(RELATIVE_PATH NBL_NSC_CACHE_REL "${IMPL_BINARY_DIR}" "${TARGET_OUTPUT}") + set(NBL_NSC_CACHE_PATH "${NBL_NSC_CACHE_ROOT}/${NBL_NSC_CACHE_REL}.ppcache") + set(NBL_NSC_PREPROCESS_CACHE_PATH "${NBL_NSC_CACHE_ROOT}/${NBL_NSC_CACHE_REL}.ppcache.pre") + else() + set(NBL_NSC_CACHE_PATH "${TARGET_OUTPUT}.ppcache") + set(NBL_NSC_PREPROCESS_CACHE_PATH "${TARGET_OUTPUT}.ppcache.pre") + endif() set(NBL_NSC_DEPFILE_ARGS "") if(NSC_USE_DEPFILE) set(NBL_NSC_DEPFILE_ARGS -MD -MF "${DEPFILE_PATH}") endif() + set(NBL_NSC_CACHE_ARGS "") + if(NSC_SHADER_CACHE) + list(APPEND NBL_NSC_CACHE_ARGS -shader-cache) + if(NSC_CACHE_DIR) + list(APPEND NBL_NSC_CACHE_ARGS -shader-cache-file "${NBL_NSC_CACHE_PATH}") + endif() + endif() + if(NSC_PREPROCESS_CACHE) + list(APPEND NBL_NSC_CACHE_ARGS -preprocess-cache) + if(NSC_CACHE_DIR) + list(APPEND NBL_NSC_CACHE_ARGS -preprocess-cache-file "${NBL_NSC_PREPROCESS_CACHE_PATH}") + endif() + endif() + set(NBL_NSC_COMPILE_COMMAND "$" -Fc "${TARGET_OUTPUT}" ${COMPILE_OPTIONS} ${REQUIRED_OPTIONS} ${IMPL_COMMON_OPTIONS} ${NBL_NSC_DEPFILE_ARGS} - "${CONFIG_FILE}" + $<$:-verbose> + ${NBL_NSC_CACHE_ARGS} + -FI "${CONFIG_FILE}" + "${TARGET_INPUT}" ) get_filename_component(NBL_NSC_INPUT_NAME "${TARGET_INPUT}" NAME) @@ -1519,50 +2182,135 @@ namespace @IMPL_NAMESPACE@ { if(NSC_USE_DEPFILE) list(APPEND NBL_NSC_BYPRODUCTS "${DEPFILE_PATH}") endif() + if(NSC_SHADER_CACHE) + list(APPEND NBL_NSC_BYPRODUCTS "${NBL_NSC_CACHE_PATH}") + endif() + if(NSC_PREPROCESS_CACHE) + list(APPEND NBL_NSC_BYPRODUCTS "${NBL_NSC_PREPROCESS_CACHE_PATH}") + list(APPEND NBL_NSC_BYPRODUCTS "${NBL_NSC_PREPROCESSED_PATH}") + endif() set(NBL_NSC_CUSTOM_COMMAND_ARGS OUTPUT "${TARGET_OUTPUT}" BYPRODUCTS ${NBL_NSC_BYPRODUCTS} COMMAND ${NBL_NSC_COMPILE_COMMAND} DEPENDS ${DEPENDS_ON} - COMMENT "${NBL_NSC_CONFIG_NAME} (${NBL_NSC_INPUT_NAME})" + COMMENT "${NBL_NSC_INPUT_NAME} (${NBL_NSC_CONFIG_NAME})" VERBATIM COMMAND_EXPAND_LISTS ) if(NSC_USE_DEPFILE) list(APPEND NBL_NSC_CUSTOM_COMMAND_ARGS DEPFILE "${DEPFILE_PATH}") endif() - add_custom_command(${NBL_NSC_CUSTOM_COMMAND_ARGS}) - set(NBL_NSC_OUT_FILES "${TARGET_OUTPUT}" "${NBL_NSC_LOG_PATH}") - if(NSC_USE_DEPFILE) - list(APPEND NBL_NSC_OUT_FILES "${DEPFILE_PATH}") + if(NOT _NBL_DISABLE_CUSTOM_COMMANDS) + add_custom_command(${NBL_NSC_CUSTOM_COMMAND_ARGS}) + endif() + set(NBL_NSC_OUT_FILES "") + if(NOT _NBL_DISABLE_CUSTOM_COMMANDS) + set(NBL_NSC_OUT_FILES "${TARGET_OUTPUT}" "${NBL_NSC_LOG_PATH}") + if(NSC_USE_DEPFILE) + list(APPEND NBL_NSC_OUT_FILES "${DEPFILE_PATH}") + endif() + if(NSC_SHADER_CACHE) + list(APPEND NBL_NSC_OUT_FILES "${NBL_NSC_CACHE_PATH}") + endif() + if(NSC_PREPROCESS_CACHE) + list(APPEND NBL_NSC_OUT_FILES "${NBL_NSC_PREPROCESS_CACHE_PATH}") + list(APPEND NBL_NSC_OUT_FILES "${NBL_NSC_PREPROCESSED_PATH}") + endif() + set_source_files_properties(${NBL_NSC_OUT_FILES} PROPERTIES GENERATED TRUE) endif() - set_source_files_properties(${NBL_NSC_OUT_FILES} PROPERTIES GENERATED TRUE) - - set(HEADER_ONLY_LIKE "${CONFIG_FILE}" "${TARGET_INPUT}" ${NBL_NSC_OUT_FILES}) - target_sources(${IMPL_TARGET} PRIVATE ${HEADER_ONLY_LIKE}) + set(HEADER_ONLY_LIKE "${TARGET_INPUT}") + if(NBL_NSC_OUT_FILES AND NOT CMAKE_CONFIGURATION_TYPES) + list(APPEND HEADER_ONLY_LIKE ${NBL_NSC_OUT_FILES}) + endif() + target_sources(${IMPL_TARGET} PRIVATE ${HEADER_ONLY_LIKE} "${CONFIG_FILE}") set_source_files_properties(${HEADER_ONLY_LIKE} PROPERTIES HEADER_FILE_ONLY ON VS_TOOL_OVERRIDE None ) - if(CMAKE_CONFIGURATION_TYPES) - foreach(_CFG IN LISTS CMAKE_CONFIGURATION_TYPES) - set(TARGET_OUTPUT_IDE "${IMPL_BINARY_DIR}/${_CFG}/${FINAL_KEY}") + set_source_files_properties("${CONFIG_FILE}" PROPERTIES + GENERATED TRUE + VS_TOOL_OVERRIDE None + ) + if(NOT _NBL_DISABLE_CUSTOM_COMMANDS) + if(CMAKE_CONFIGURATION_TYPES) + foreach(_CFG IN LISTS CMAKE_CONFIGURATION_TYPES) + set(TARGET_OUTPUT_IDE "${IMPL_BINARY_DIR}/${_CFG}/${HASHED_KEY}") + set(TARGET_OUTPUT_IDE_PREPROCESSED "${TARGET_OUTPUT_IDE}.pre.hlsl") + if(NSC_CACHE_DIR) + file(RELATIVE_PATH TARGET_OUTPUT_IDE_REL "${IMPL_BINARY_DIR}" "${TARGET_OUTPUT_IDE}") + set(TARGET_OUTPUT_IDE_CACHE "${NBL_NSC_CACHE_ROOT}/${TARGET_OUTPUT_IDE_REL}.ppcache") + set(TARGET_OUTPUT_IDE_PRECACHE "${NBL_NSC_CACHE_ROOT}/${TARGET_OUTPUT_IDE_REL}.ppcache.pre") + else() + set(TARGET_OUTPUT_IDE_CACHE "${TARGET_OUTPUT_IDE}.ppcache") + set(TARGET_OUTPUT_IDE_PRECACHE "${TARGET_OUTPUT_IDE}.ppcache.pre") + endif() + set(NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE}" "${TARGET_OUTPUT_IDE}.log") + if(NSC_USE_DEPFILE) + list(APPEND NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE}.dep") + endif() + if(NSC_SHADER_CACHE) + list(APPEND NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE_CACHE}") + endif() + if(NSC_PREPROCESS_CACHE) + list(APPEND NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE_PRECACHE}") + list(APPEND NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE_PREPROCESSED}") + endif() + target_sources(${IMPL_TARGET} PRIVATE ${NBL_NSC_OUT_FILES_IDE}) + set_source_files_properties(${NBL_NSC_OUT_FILES_IDE} PROPERTIES + HEADER_FILE_ONLY ON + VS_TOOL_OVERRIDE None + GENERATED TRUE + ) + if(NSC_SHADER_CACHE) + set_source_files_properties("${TARGET_OUTPUT_IDE_CACHE}" PROPERTIES HEADER_FILE_ONLY OFF) + endif() + if(NSC_PREPROCESS_CACHE) + set_source_files_properties("${TARGET_OUTPUT_IDE_PRECACHE}" PROPERTIES HEADER_FILE_ONLY OFF) + set_source_files_properties("${TARGET_OUTPUT_IDE_PREPROCESSED}" PROPERTIES HEADER_FILE_ONLY ON) + endif() + source_group("${OUT}/${_CFG}" FILES ${NBL_NSC_OUT_FILES_IDE}) + endforeach() + else() + set(TARGET_OUTPUT_IDE "${IMPL_BINARY_DIR}/${HASHED_KEY}") + set(TARGET_OUTPUT_IDE_PREPROCESSED "${TARGET_OUTPUT_IDE}.pre.hlsl") + if(NSC_CACHE_DIR) + file(RELATIVE_PATH TARGET_OUTPUT_IDE_REL "${IMPL_BINARY_DIR}" "${TARGET_OUTPUT_IDE}") + set(TARGET_OUTPUT_IDE_CACHE "${NBL_NSC_CACHE_ROOT}/${TARGET_OUTPUT_IDE_REL}.ppcache") + set(TARGET_OUTPUT_IDE_PRECACHE "${NBL_NSC_CACHE_ROOT}/${TARGET_OUTPUT_IDE_REL}.ppcache.pre") + else() + set(TARGET_OUTPUT_IDE_CACHE "${TARGET_OUTPUT_IDE}.ppcache") + set(TARGET_OUTPUT_IDE_PRECACHE "${TARGET_OUTPUT_IDE}.ppcache.pre") + endif() set(NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE}" "${TARGET_OUTPUT_IDE}.log") if(NSC_USE_DEPFILE) - list(APPEND NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE}.d") + list(APPEND NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE}.dep") endif() - source_group("${OUT}/${_CFG}" FILES ${NBL_NSC_OUT_FILES_IDE}) - endforeach() - else() - set(TARGET_OUTPUT_IDE "${IMPL_BINARY_DIR}/${FINAL_KEY}") - set(NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE}" "${TARGET_OUTPUT_IDE}.log") - if(NSC_USE_DEPFILE) - list(APPEND NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE}.d") + if(NSC_SHADER_CACHE) + list(APPEND NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE_CACHE}") + endif() + if(NSC_PREPROCESS_CACHE) + list(APPEND NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE_PRECACHE}") + list(APPEND NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE_PREPROCESSED}") + endif() + target_sources(${IMPL_TARGET} PRIVATE ${NBL_NSC_OUT_FILES_IDE}) + set_source_files_properties(${NBL_NSC_OUT_FILES_IDE} PROPERTIES + HEADER_FILE_ONLY ON + VS_TOOL_OVERRIDE None + GENERATED TRUE + ) + if(NSC_SHADER_CACHE) + set_source_files_properties("${TARGET_OUTPUT_IDE_CACHE}" PROPERTIES HEADER_FILE_ONLY OFF) + endif() + if(NSC_PREPROCESS_CACHE) + set_source_files_properties("${TARGET_OUTPUT_IDE_PRECACHE}" PROPERTIES HEADER_FILE_ONLY OFF) + set_source_files_properties("${TARGET_OUTPUT_IDE_PREPROCESSED}" PROPERTIES HEADER_FILE_ONLY ON) + endif() + source_group("${OUT}" FILES ${NBL_NSC_OUT_FILES_IDE}) endif() - source_group("${OUT}" FILES ${NBL_NSC_OUT_FILES_IDE}) endif() set_source_files_properties("${TARGET_OUTPUT}" PROPERTIES @@ -1582,15 +2330,27 @@ namespace @IMPL_NAMESPACE@ { set(VAR_NAME "CAP_VALUES_${CAP_INDEX}") set(VALUES "${${VAR_NAME}}") + set(KEY_PREFIX ".") + if(CAP_INDEX EQUAL 0) + set(KEY_PREFIX "__${CURRENT_KIND}.") + else() + math(EXPR PREV_INDEX "${CAP_INDEX} - 1") + list(GET CAP_KINDS ${PREV_INDEX} PREV_KIND) + if(NOT CURRENT_KIND STREQUAL PREV_KIND) + set(KEY_PREFIX "__${CURRENT_KIND}.") + endif() + endif() foreach(V IN LISTS VALUES) - set(NEW_PREFIX "${PREFIX}.${CURRENT_CAP}_${V}") - set(NEW_EVAL "${CAPS_EVAL_PART}NBL_CONSTEXPR_STATIC_INLINE ${CURRENT_TYPE} ${CURRENT_CAP} = (${CURRENT_TYPE}) ${V}; // got permuted\n") + set(NEW_PREFIX "${PREFIX}${KEY_PREFIX}${CURRENT_CAP}_${V}") + set(NEW_VALUES "${CAPS_VALUES_PART}") + list(APPEND NEW_VALUES "${V}") math(EXPR NEXT_INDEX "${CAP_INDEX} + 1") - GENERATE_KEYS("${NEW_PREFIX}" "${NEXT_INDEX}" "${NEW_EVAL}") + GENERATE_KEYS("${NEW_PREFIX}" "${NEXT_INDEX}" ${NEW_VALUES}) endforeach() endfunction() - GENERATE_KEYS("" 0 "") + GENERATE_KEYS("" 0) + endforeach() unset(KEYS) @@ -1605,7 +2365,8 @@ namespace @IMPL_NAMESPACE@ { list(APPEND KEYS ${ACCESS_KEY}) endforeach() - source_group("${IN}" FILES ${CONFIGS} ${INPUTS}) + source_group("${IN}/autogen" FILES ${CONFIGS}) + source_group("${IN}" FILES ${INPUTS}) if(IMPL_HLSL_GLOB) target_sources(${IMPL_TARGET} PRIVATE ${IMPL_HLSL_GLOB}) set_source_files_properties(${IMPL_HLSL_GLOB} PROPERTIES diff --git a/docs/nsc-prebuilds.md b/docs/nsc-prebuilds.md index 400aff5eb7..38c0e48716 100644 --- a/docs/nsc-prebuilds.md +++ b/docs/nsc-prebuilds.md @@ -43,14 +43,24 @@ For each registered input it generates: - One `.spv` output **per CMake configuration** (`Debug/`, `Release/`, `RelWithDebInfo/`). - If you use `CAPS`, it generates a **cartesian product** of permutations and emits a `.spv` for each. - A generated header (you choose the path via `INCLUDE`) containing: - - a primary template `get_spirv_key(limits, features)` and `get_spirv_key(device)` +- a primary template `get_spirv_key(...args)` and `get_spirv_key(device, ...args)` +- `get_spirv_key` returns a small owning buffer; use `.view()` or implicit `std::string_view` to consume it +- arguments must follow the **kind order** as it appears in `CAPS` (first appearance), validated structurally by required member names/types for each kind (including `limits`/`features`, no strong typing) + - `get_spirv_key(device, ...)` expects only **non-device** kinds in that same order; `limits`/`features` are injected from the device + - note: an order-agnostic API would require enforcing unique member sets across kinds to guarantee unambiguous matching; we keep a conventional order instead to stay flexible without extra constraints - explicit specializations for each registered base `KEY` - the returned key already includes the build config prefix (compiled into the header). -Keys are strings that match the output layout: +Keys are hashed to keep filenames short and stable across long permutation strings. The **full key string** is built as: ``` -/(._)(._)....spv +(__._)(._)....spv +``` + +Then `FNV-1a 64-bit` is computed from that full key (no `` prefix), and the **final output key** is: + +``` +/.spv ``` ## The JSON "INPUTS" format @@ -96,6 +106,38 @@ By default `NBL_CREATE_NSC_COMPILE_RULES` also collects `*.hlsl` files for IDE v - `GLOB_DIR` (optional): root directory for the default `*.hlsl` scan. - `DISCARD_DEFAULT_GLOB` (flag): disables the default scan and IDE grouping. +## Cache layers (SPIR-V + preprocess) + +There are two independent caches: + +- `NSC_SHADER_CACHE` (default `ON`) -> SPIR-V cache (`.spv.ppcache`) for full compilation results. +- `NSC_PREPROCESS_CACHE` (default `ON`) -> preprocessor prefix cache (`.spv.ppcache.pre`) to avoid repeating Boost.Wave include work when only the main shader changes. +- Both caches are used only for compilation (not `-P` preprocess-only runs). +- When preprocess cache is enabled and used, NSC also writes a combined preprocessed view (`.spv.pre.hlsl`) next to the outputs. + - This file is the exact input fed to DXC on the preprocess-cache path, so it's ready to paste into Godbolt for repros (use the same flags/includes). + +With `-verbose`, `.log` shows: + +- `Cache: ` and `Cache hit!/miss! ...` for SPIR-V cache. +- `Preprocess cache: ` and `Preprocess cache hit!/miss! ...` for the prefix cache. +- Timing lines (performance): + - `Shader cache lookup took: ...` + - `Preprocess cache lookup took: ...` + - `Total cache probe took: ...` + - `Preprocess took: ...` (only on compile path) + - `Compile took: ...` (only on compile path) + - `Total build time: ...` (preprocess + compile) + - `Total took: ...` (overall tool runtime) + +You can redirect both caches into a shared directory with: + +- `NSC_CACHE_DIR` (path). The cache files keep the same relative layout as `BINARY_DIR` (including `/`), but live under the given root. This is handy for CI or persistent cache volumes. + +The preprocess cache key is based on the **prefix** of the input file (leading directives/comments plus forced includes), and cache validity is checked against include dependency hashes. That means: + +- edits to the shader body still hit (fast path) +- changes to prefix directives, forced-includes, or included headers cause a cold run + ## Minimal usage (no permutations) Example pattern (as in `examples_tests/27_MPMCScheduler/CMakeLists.txt`): @@ -133,11 +175,12 @@ Then include the generated header and use the key to load the SPIR-V: ```cpp #include "nbl/this_example/builtin/build/spirv/keys.hpp" // ... -auto key = nbl::this_example::builtin::build::get_spirv_key<"shader">(device); -auto bundle = assetMgr->getAsset(key.c_str(), loadParams); +auto keyBuf = nbl::this_example::builtin::build::get_spirv_key<"shader">(device); +std::string_view key = keyBuf; +auto bundle = assetMgr->getAsset(key.data(), loadParams); ``` -`OUTPUT_VAR` (here: `KEYS`) is assigned the list of **all** produced access keys (all configurations + all permutations). This list is intended to be fed into `NBL_CREATE_RESOURCE_ARCHIVE(BUILTINS ${KEYS})`. +`OUTPUT_VAR` (here: `KEYS`) is assigned the list of **all** produced access keys (all configurations + all permutations). These are already hashed (e.g. `Debug/123456789.spv`) and are intended to be fed into `NBL_CREATE_RESOURCE_ARCHIVE(BUILTINS ${KEYS})`. ## Permutations via `CAPS` @@ -145,17 +188,74 @@ auto bundle = assetMgr->getAsset(key.c_str(), loadParams); Each `CAPS` entry looks like: -- `kind` (string, optional): `"limits"` or `"features"` (defaults to `"limits"` if omitted/invalid). +- `kind` (string, optional): `"limits"`, `"features"`, or `"custom"` (defaults to `"limits"` if omitted/invalid). +- `struct` (string, required for `kind="custom"`): name of the custom permutation struct (valid C/C++ identifier). If you use `limits` or `features` here, do not also use the built-in `limits`/`features` kinds in the same rule. - `name` (string, required): identifier used in both generated HLSL config and C++ key (must be a valid C/C++ identifier). -- `type` (string, required): `bool`, `uint16_t`, `uint32_t`, `uint64_t`. +- `type` (string, required): `bool`, `uint16_t`, `uint32_t`, `uint64_t`, `int16_t`, `int32_t`, `int64_t`, `float`, `double`. - `values` (array of numbers, required): the values you want to prebuild. - for `bool`, values must be `0` or `1`. + - for signed integer types, negative values are allowed. + - for `float`/`double`, you can provide **numbers or numeric strings** (e.g. `-1`, `-1.0`, `1e-3`, or `-1.f` for floats). Values are **normalized** to canonical scientific notation (1 digit before the decimal, 8 digits after for `float` or 16 for `double`, signed exponent with 2 or 3 digits). The normalized text becomes part of the key. + +At build time, NSC compiles each combination of values (cartesian product). At runtime, `get_spirv_key` appends suffixes using the structs you pass in for `limits`/`features` (duck-typed by required members) and any custom kinds. Each group starts with `__limits`, `__features`, or `__`, followed by `.member_` entries. Group order follows the **first appearance of each kind in `CAPS`** (and this same order is the required argument order for `get_spirv_key`); groups with no members are omitted. + +Each generated `.config` file defines a `DeviceConfigCaps` struct for HLSL. It includes: +- flat members for `limits`/`features` (backwards compatibility with older shaders) +- nested structs for custom kinds only, e.g. `DeviceConfigCaps::userA` + +Example shape: + +```hlsl +struct DeviceConfigCaps +{ + NBL_CONSTEXPR_STATIC_INLINE uint32_t maxImageDimension2D = 16384u; + NBL_CONSTEXPR_STATIC_INLINE bool shaderCullDistance = true; + + struct userA + { + NBL_CONSTEXPR_STATIC_INLINE uint32_t mode = 0u; + NBL_CONSTEXPR_STATIC_INLINE uint32_t quality = 1u; + }; +}; +``` + +For more complex usage and regression-style checks (constexpr vs runtime, hashing, mixed payloads), see `examples_tests/73_SpirvKeysTest`. -At build time, NSC compiles each combination of values (cartesian product). At runtime, `get_spirv_key` appends suffixes using the `limits`/`features` you pass in. +### Grouping caps by kind (optional) + +To avoid repeating the same `kind`, you can group caps with `members`: + +```cmake +set(JSON [=[ +[ + { + "INPUT": "app_resources/shader.hlsl", + "KEY": "shader", + "COMPILE_OPTIONS": ["-T", "lib_6_8"], + "CAPS": [ + { + "kind": "custom", + "struct": "userA", + "members": [ + { "name": "mode", "type": "uint32_t", "values": [0, 1] }, + { "name": "quality", "type": "uint32_t", "values": [1, 2, 4] } + ] + }, + { + "kind": "features", + "members": [ + { "name": "shaderFloat64", "type": "bool", "values": [0, 1] } + ] + } + ] + } +] +]=]) +``` ### Example: mixing `limits` and `features` -This example permutes over one device limit and one device feature (order matters: the suffix order matches the `CAPS` array order): +This example permutes over one device limit and one device feature. Suffix order follows the `CAPS` order (`__limits` then `__features` here), and member order within each group follows the `CAPS` order for that group: ```cmake set(JSON [=[ @@ -190,6 +290,74 @@ NBL_CREATE_NSC_COMPILE_RULES( ) ``` +## Custom permutation structs + +If you need permutations based on data outside of device `limits`/`features`, define a custom struct in C++ and use `kind: "custom"` with `struct` set to the parameter name. At runtime you can pass any struct type that exposes the required members with matching types; **argument order follows the `CAPS` kind order**. Using custom names `limits` or `features` is allowed, but you cannot mix them with the built-in `limits`/`features` kinds in the same rule. + +Example: + +```cmake +set(JSON [=[ +[ + { + "INPUT": "app_resources/fft.hlsl", + "KEY": "fft", + "COMPILE_OPTIONS": ["-T", "cs_6_8"], + "CAPS": [ + { + "kind": "custom", + "struct": "fftConfig", + "name": "passCount", + "type": "uint32_t", + "values": [4, 8] + } + ] + } +] +]=]) + +NBL_CREATE_NSC_COMPILE_RULES( + # ... + OUTPUT_VAR KEYS + INPUTS ${JSON} +) +``` + +Runtime usage: + +```cpp +nbl::this_example::FFTConfig cfg = {}; +cfg.passCount = 4; +auto key = nbl::this_example::builtin::build::get_spirv_key<"fft">(device, cfg); +``` + +Constexpr usage with extra structs (order must match `CAPS` kind order, first appearance): + +```cpp +struct MyLimits { uint32_t maxImageDimension2D; }; +struct MyFeatures { bool shaderCullDistance; }; +struct UserA { uint32_t mode; uint32_t quality; }; +struct UserB { bool useAlternatePath; bool useFastPath; }; + +constexpr UserA userA = { 0u, 1u }; +constexpr UserB userB = { false, true }; +constexpr MyLimits limits = { 16384u }; +constexpr MyFeatures features = { true }; + +static constexpr auto keyBuf = + nbl::this_example::builtin::build::get_spirv_key<"shader_cd">(userA, userB, limits, features); +static constexpr std::string_view keyView = keyBuf; + +``` + +## Common pitfalls + +- Argument order must follow the **first appearance of each kind in `CAPS`**; this is an intentional convention to keep the API flexible. +- `get_spirv_key` returns a buffer; prefer `std::string_view key = buf;` or `buf.view()` to consume it. +- Do not store a `std::string_view` from a temporary buffer; keep the buffer alive. +- `float`/`double` CAP values are normalized to canonical scientific notation (1 digit before the decimal, 8 or 16 digits after, signed exponent); values passed to `get_spirv_key` must match one of the CAP values exactly. +- `constexpr` key generation works with `float`/`double` members when the values match the CAP list. + This produces `3 * 2 = 6` permutations per build configuration, and `KEYS` contains all of them (for example): ``` diff --git a/examples_tests b/examples_tests index 671d1f16b0..b6ee9c8971 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 671d1f16b0837a70c3016c2472864528f35db0bc +Subproject commit b6ee9c897111dd6dea8479edaed249e5459a9f29 diff --git a/include/nbl/asset/utils/CHLSLCompiler.h b/include/nbl/asset/utils/CHLSLCompiler.h index 92a1dca394..b093ff98ed 100644 --- a/include/nbl/asset/utils/CHLSLCompiler.h +++ b/include/nbl/asset/utils/CHLSLCompiler.h @@ -51,7 +51,7 @@ class NBL_API2 CHLSLCompiler final : public IShaderCompiler //} std::string preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions, std::vector* dependencies = nullptr) const override; - std::string preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions, std::vector& dxc_compile_flags_override, std::vector* dependencies = nullptr) const; + std::string preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions, std::vector& dxc_compile_flags_override, std::vector* dependencies = nullptr, std::vector* macro_defs = nullptr) const; void insertIntoStart(std::string& code, std::ostringstream&& ins) const override; @@ -118,6 +118,9 @@ class NBL_API2 CHLSLCompiler final : public IShaderCompiler return std::span(RequiredArguments); } + protected: + bool preprocessPrefixForCache(std::string_view code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions, CPreprocessCache::SEntry& outEntry) const override; + protected: // This can't be a unique_ptr due to it being an undefined type // when Nabla is used as a lib @@ -155,4 +158,4 @@ class NBL_API2 CHLSLCompiler final : public IShaderCompiler #endif -#endif \ No newline at end of file +#endif diff --git a/include/nbl/asset/utils/IShaderCompiler.h b/include/nbl/asset/utils/IShaderCompiler.h index 9fd4eee833..168e9e6632 100644 --- a/include/nbl/asset/utils/IShaderCompiler.h +++ b/include/nbl/asset/utils/IShaderCompiler.h @@ -33,9 +33,11 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted { system::path absolutePath = {}; std::string contents = {}; - core::blake3_hash_t hash = {}; // TODO: we're not yet using IFile::getPrecomputedHash(), so for builtins we can maybe use that in the future - // Could be used in the future for early rejection of cache hit - //nbl::system::IFileBase::time_point_t lastWriteTime = {}; + core::blake3_hash_t hash = {}; + uint64_t fileSize = 0; + int64_t lastWriteTime = 0; + bool hasFileInfo = false; + bool hasHash = false; explicit inline operator bool() const {return !absolutePath.empty();} }; @@ -118,6 +120,54 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted std::string_view definition; }; + // Forward declaration for dependency access. + struct CCache; + + struct SPreprocessingDependency + { + public: + // Perf note: hashing while preprocessor lexing is likely to be slower than just hashing the whole array like this + inline SPreprocessingDependency(const system::path& _requestingSourceDir, const std::string_view& _identifier, bool _standardInclude, core::blake3_hash_t _hash, + const system::path& _absolutePath = {}, uint64_t _fileSize = 0, int64_t _lastWriteTime = 0, bool _hasFileInfo = false) : + requestingSourceDir(_requestingSourceDir), identifier(_identifier), standardInclude(_standardInclude), hash(_hash), + absolutePath(_absolutePath), fileSize(_fileSize), lastWriteTime(_lastWriteTime), hasFileInfo(_hasFileInfo) + {} + + inline SPreprocessingDependency(SPreprocessingDependency&) = default; + inline SPreprocessingDependency& operator=(SPreprocessingDependency&) = delete; + inline SPreprocessingDependency(SPreprocessingDependency&&) = default; + inline SPreprocessingDependency& operator=(SPreprocessingDependency&&) = default; + + // Needed for json vector serialization. Making it private and declaring from_json(_, SEntry&) as friend didn't work + inline SPreprocessingDependency() {} + + inline const system::path& getRequestingSourceDir() const { return requestingSourceDir; } + inline std::string_view getIdentifier() const { return identifier; } + inline bool isStandardInclude() const { return standardInclude; } + inline const core::blake3_hash_t& getHash() const { return hash; } + inline const system::path& getAbsolutePath() const { return absolutePath; } + inline uint64_t getFileSize() const { return fileSize; } + inline int64_t getLastWriteTime() const { return lastWriteTime; } + inline bool getHasFileInfo() const { return hasFileInfo; } + + private: + friend void to_json(nlohmann::json& j, const SPreprocessingDependency& dependency); + friend void from_json(const nlohmann::json& j, SPreprocessingDependency& dependency); + friend class CCache; + + // path or identifier + system::path requestingSourceDir = ""; + std::string identifier = ""; + // hash of the contents - used to check against a found_t + core::blake3_hash_t hash = {}; + // If true, then `getIncludeStandard` was used to find, otherwise `getIncludeRelative` + bool standardInclude = false; + system::path absolutePath = {}; + uint64_t fileSize = 0; + int64_t lastWriteTime = 0; + bool hasFileInfo = false; + }; + // using E_SPIRV_VERSION = nbl::hlsl::SpirvVersion; @@ -136,6 +186,9 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted system::logger_opt_ptr logger = nullptr; const CIncludeFinder* includeFinder = nullptr; std::span extraDefines = {}; + std::span forceIncludes = {}; + std::string_view codeForCache = {}; + bool applyForceIncludes = true; E_SPIRV_VERSION targetSpirvVersion = E_SPIRV_VERSION::ESV_1_6; bool depfile = false; system::path depfilePath = {}; @@ -153,7 +206,6 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted }; // Forward declaration for SCompilerOptions use - struct CCache; /* @stage shaderStage, can be ESS_ALL_OR_LIBRARY to make multi-entrypoint shaders @targetSpirvVersion spirv version @@ -185,15 +237,20 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted SPreprocessorOptions preprocessorOptions = {}; CCache* readCache = nullptr; CCache* writeCache = nullptr; + bool* cacheHit = nullptr; + const std::vector* dependencyOverrides = nullptr; }; + static std::string applyForceIncludes(std::string_view code, std::span forceIncludes); + static bool probeShaderCache(const CCache* cache, std::string_view code, const SCompilerOptions& options, const CIncludeFinder* finder); + class CCache final : public IReferenceCounted { friend class IShaderCompiler; public: // Used to check compatibility of Caches before reading - constexpr static inline std::string_view VERSION = "1.1.0"; + constexpr static inline std::string_view VERSION = "1.2.4"; static auto const SHADER_BUFFER_SIZE_BYTES = sizeof(uint64_t) / sizeof(uint8_t); // It's obviously 8 @@ -201,40 +258,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted { friend class CCache; - struct SPreprocessingDependency - { - public: - // Perf note: hashing while preprocessor lexing is likely to be slower than just hashing the whole array like this - inline SPreprocessingDependency(const system::path& _requestingSourceDir, const std::string_view& _identifier, bool _standardInclude, core::blake3_hash_t _hash) : - requestingSourceDir(_requestingSourceDir), identifier(_identifier), standardInclude(_standardInclude), hash(_hash) - {} - - inline SPreprocessingDependency(SPreprocessingDependency&) = default; - inline SPreprocessingDependency& operator=(SPreprocessingDependency&) = delete; - inline SPreprocessingDependency(SPreprocessingDependency&&) = default; - inline SPreprocessingDependency& operator=(SPreprocessingDependency&&) = default; - - // Needed for json vector serialization. Making it private and declaring from_json(_, SEntry&) as friend didn't work - inline SPreprocessingDependency() {} - - inline const system::path& getRequestingSourceDir() const { return requestingSourceDir; } - inline std::string_view getIdentifier() const { return identifier; } - inline bool isStandardInclude() const { return standardInclude; } - - private: - friend void to_json(nlohmann::json& j, const SEntry::SPreprocessingDependency& dependency); - friend void from_json(const nlohmann::json& j, SEntry::SPreprocessingDependency& dependency); - friend class CCache; - - // path or identifier - system::path requestingSourceDir = ""; - std::string identifier = ""; - // hash of the contents - used to check against a found_t - core::blake3_hash_t hash = {}; - // If true, then `getIncludeStandard` was used to find, otherwise `getIncludeRelative` - bool standardInclude = false; - - }; + using SPreprocessingDependency = IShaderCompiler::SPreprocessingDependency; struct SCompilerArgs; // Forward declaration for SPreprocessorArgs's friend declaration @@ -251,6 +275,8 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted if (definesIt->identifier != otherDefinesIt->identifier || definesIt->definition != otherDefinesIt->definition) return false; + if (forceIncludes != other.forceIncludes) return false; + return true; } @@ -275,11 +301,15 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted for (auto define : options.extraDefines) extraDefines.emplace_back(std::string(define.identifier), std::string(define.definition)); + for (const auto& inc : options.forceIncludes) + forceIncludes.emplace_back(inc); + // Sort them so equality and hashing are well defined std::sort(extraDefines.begin(), extraDefines.end(), [](const SMacroDefinition& lhs, const SMacroDefinition& rhs) {return lhs.identifier < rhs.identifier; }); }; std::string sourceIdentifier; std::vector extraDefines; + std::vector forceIncludes; }; // TODO: SPreprocessorArgs could just be folded into `SCompilerArgs` to have less classes and decompressShader struct SCompilerArgs final @@ -351,6 +381,8 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted hashable.insert(hashable.end(), defines.identifier.begin(), defines.identifier.end()); hashable.insert(hashable.end(), defines.definition.begin(), defines.definition.end()); } + for (const auto& inc : compilerArgs.preprocessorArgs.forceIncludes) + hashable.insert(hashable.end(), inc.begin(), inc.end()); // Insert rest of stuff from this struct. We're going to treat stage, targetSpirvVersion and debugInfoFlags.value as byte arrays for simplicity hashable.insert(hashable.end(), reinterpret_cast(&compilerArgs.stage), reinterpret_cast(&compilerArgs.stage) + sizeof(compilerArgs.stage)); @@ -400,6 +432,8 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted inline void insert(SEntry&& entry) { + if (auto found = m_container.find(entry); found != m_container.end()) + m_container.erase(found); m_container.insert(std::move(entry)); } @@ -420,6 +454,9 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted } NBL_API2 core::smart_refctd_ptr find(const SEntry& mainFile, const CIncludeFinder* finder) const; + NBL_API2 bool contains(const SEntry& mainFile, const CIncludeFinder* finder) const; + NBL_API2 bool findEntryForCode(std::string_view code, const SCompilerOptions& options, const CIncludeFinder* finder, SEntry& outEntry) const; + NBL_API2 core::smart_refctd_ptr decompressEntry(const SEntry& entry) const; inline CCache() {} @@ -453,6 +490,79 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted NBL_API2 EntrySet::const_iterator find_impl(const SEntry& mainFile, const CIncludeFinder* finder) const; }; + class CPreprocessCache final : public IReferenceCounted + { + public: + constexpr static inline std::string_view VERSION = "2.2"; + + struct SEntry + { + core::blake3_hash_t prefixHash = {}; + std::string preprocessedPrefix; + std::vector macroDefs; + std::vector dxcFlags; + uint32_t pragmaStage = static_cast(IShader::E_SHADER_STAGE::ESS_UNKNOWN); + CCache::SEntry::dependency_container_t dependencies; + }; + + enum class ELoadStatus : uint8_t + { + Missing, + Invalid, + Loaded + }; + + enum class EProbeStatus : uint8_t + { + Hit, + NoPrefix, + Missing, + Invalid, + PrefixChanged, + DependenciesChanged, + EntryInvalid + }; + + struct SProbeResult + { + std::string codeStorage; + std::string_view prefix = {}; + std::string_view body = {}; + core::blake3_hash_t prefixHash = {}; + EProbeStatus status = EProbeStatus::EntryInvalid; + bool hasPrefix = false; + bool cacheHit = false; + }; + + inline bool hasEntry() const { return m_hasEntry; } + inline const SEntry& getEntry() const { return m_entry; } + inline void setEntry(SEntry&& entry) { m_entry = std::move(entry); m_hasEntry = true; } + + NBL_API2 core::smart_refctd_ptr serialize() const; + NBL_API2 static core::smart_refctd_ptr deserialize(const std::span serializedCache); + NBL_API2 static core::smart_refctd_ptr loadFromFile(const system::path& path, ELoadStatus& status); + NBL_API2 static bool writeToFile(const system::path& path, const CPreprocessCache& cache); + NBL_API2 static SProbeResult probe(std::string_view code, const CPreprocessCache* cache, ELoadStatus loadStatus, const SPreprocessorOptions& preprocessOptions); + NBL_API2 static const char* getProbeReason(EProbeStatus status); + NBL_API2 bool validateDependencies(const CIncludeFinder* finder) const; + NBL_API2 std::string buildCombinedCode(std::string_view body, std::string_view sourceIdentifier) const; + + private: + bool m_hasEntry = false; + SEntry m_entry; + }; + + struct SPreprocessCacheResult + { + bool ok = true; + bool cacheUsed = false; + bool cacheHit = false; + bool cacheUpdated = false; + CPreprocessCache::EProbeStatus status = CPreprocessCache::EProbeStatus::EntryInvalid; + IShader::E_SHADER_STAGE stage = IShader::E_SHADER_STAGE::ESS_UNKNOWN; + std::string code; + }; + struct DepfileWriteParams { system::ISystem* system = nullptr; @@ -465,6 +575,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted static bool writeDepfile(const DepfileWriteParams& params, const CCache::SEntry::dependency_container_t& dependencies, const CIncludeFinder* includeFinder = nullptr, system::logger_opt_ptr logger = nullptr); core::smart_refctd_ptr compileToSPIRV(const std::string_view code, const SCompilerOptions& options) const; + SPreprocessCacheResult preprocessWithCache(std::string_view code, IShader::E_SHADER_STAGE stage, const SPreprocessorOptions& preprocessOptions, CPreprocessCache& cache, CPreprocessCache::ELoadStatus loadStatus, std::string_view sourceIdentifier) const; inline core::smart_refctd_ptr compileToSPIRV(const char* code, const SCompilerOptions& options) const { @@ -593,6 +704,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted virtual void insertIntoStart(std::string& code, std::ostringstream&& ins) const = 0; virtual core::smart_refctd_ptr compileToSPIRV_impl(const std::string_view code, const SCompilerOptions& options, std::vector* dependencies) const = 0; + virtual bool preprocessPrefixForCache(std::string_view code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions, CPreprocessCache::SEntry& outEntry) const; core::smart_refctd_ptr m_system; diff --git a/include/nbl/core/hash/fnv1a64.h b/include/nbl/core/hash/fnv1a64.h new file mode 100644 index 0000000000..96f5315fbb --- /dev/null +++ b/include/nbl/core/hash/fnv1a64.h @@ -0,0 +1,28 @@ +// Copyright (C) 2026 - DevSH Graphics Programming Sp. z O.O. +// This file is part of the "Nabla Engine". +// For conditions of distribution and use, see copyright notice in nabla.h +#ifndef _NBL_CORE_FNV1A64_H_INCLUDED_ +#define _NBL_CORE_FNV1A64_H_INCLUDED_ + +#include +#include +#include + +namespace nbl::core +{ + +// FNV-1a 64-bit hash. +constexpr uint64_t FNV1a_64(std::string_view sv) +{ + uint64_t h = 14695981039346656037ull; + for (unsigned char c : sv) + { + h ^= c; + h *= 1099511628211ull; + } + return h; +} + +} + +#endif // _NBL_CORE_FNV1A64_H_INCLUDED_ diff --git a/include/nbl/core/string/SpirvKeyHelpers.h b/include/nbl/core/string/SpirvKeyHelpers.h new file mode 100644 index 0000000000..c9f3150c2d --- /dev/null +++ b/include/nbl/core/string/SpirvKeyHelpers.h @@ -0,0 +1,55 @@ +#ifndef _NBL_CORE_STRING_SPIRV_KEY_HELPERS_H_INCLUDED_ +#define _NBL_CORE_STRING_SPIRV_KEY_HELPERS_H_INCLUDED_ + +#include +#include +#include +#include + +#include "nbl/core/string/StringLiteral.h" + +namespace nbl::core::detail +{ + +template +struct SpirvKeyBuilderMissing : std::false_type {}; + +template +struct SpirvKeyBuilder +{ + template + static constexpr void build(const Args&...) + { + static_assert(SpirvKeyBuilderMissing::value, "Unknown SPIR-V key"); + } +}; + +template +concept spirv_device_has_limits = requires(const Device* device) +{ + device->getPhysicalDevice()->getLimits(); +}; + +template +concept spirv_device_has_features = requires(const Device* device) +{ + device->getEnabledFeatures(); +}; + +template +constexpr decltype(auto) spirv_device_get_limits(const Device* device) +{ + static_assert(spirv_device_has_limits, "Device does not provide getLimits"); + return device->getPhysicalDevice()->getLimits(); +} + +template +constexpr decltype(auto) spirv_device_get_features(const Device* device) +{ + static_assert(spirv_device_has_features, "Device does not provide getEnabledFeatures"); + return device->getEnabledFeatures(); +} + +} + +#endif diff --git a/include/nbl/core/string/StringLiteral.h b/include/nbl/core/string/StringLiteral.h index ebbed673f6..d48ebce7a5 100644 --- a/include/nbl/core/string/StringLiteral.h +++ b/include/nbl/core/string/StringLiteral.h @@ -5,6 +5,15 @@ #define _NBL_CORE_STRING_LITERAL_H_INCLUDED_ #include +#include +#include +#include +#include +#include +#include +#include + +#include "nbl/core/hash/fnv1a64.h" namespace nbl::core { @@ -25,4 +34,289 @@ struct StringLiteral // for compatibility's sake #define NBL_CORE_UNIQUE_STRING_LITERAL_TYPE(STRING_LITERAL) nbl::core::StringLiteral(STRING_LITERAL) +namespace nbl::core::detail +{ + +template +struct StringLiteralBufferType +{ + using type = void; +}; + +template +class StringLiteralBuffer +{ +public: + constexpr void append(char c) + { + if (!ensure_capacity(1)) + return; + b[n++] = c; + } + constexpr void append(std::string_view sv) + { + if (!ensure_capacity(sv.size())) + return; + for (char c : sv) + b[n++] = c; + } + constexpr void append(const char* s) + { + for (; *s; ++s) + append(*s); + } + + constexpr std::string_view view() const { return { b.data(), n }; } + constexpr operator std::string_view() const { return view(); } + constexpr const char* data() const { return b.data(); } + constexpr size_t size() const { return n; } + +private: + constexpr bool ensure_capacity(size_t add) + { + if (n + add <= Cap) + return true; + if (std::is_constant_evaluated()) + throw "overflow"; + assert(false && "StringLiteralBuffer overflow"); + return false; + } + + std::array b{}; + size_t n = 0; +}; + +template +constexpr std::string_view to_string_view(const StringLiteralBuffer& v) +{ + return v.view(); +} + +template +constexpr void append_uint_padded(Out& o, unsigned value, int width) +{ + char buf[16]; + int len = 0; + do + { + buf[len++] = static_cast('0' + (value % 10u)); + value /= 10u; + } while (value); + while (len < width) + buf[len++] = '0'; + for (int i = len - 1; i >= 0; --i) + o.append(buf[i]); +} + +template +constexpr void append_float_scientific(Out& o, T v) +{ + using Limits = std::numeric_limits; + constexpr int precision = Limits::max_digits10 - 1; + if (v != v) + { + assert(false && "StringLiteralBuffer float format failed"); + return; + } + if constexpr (Limits::has_infinity) + { + if (v == Limits::infinity() || v == -Limits::infinity()) + { + assert(false && "StringLiteralBuffer float format failed"); + return; + } + } + if (v < T(0)) + { + o.append('-'); + v = -v; + } + if (v == T(0)) + { + o.append('0'); + o.append('.'); + for (int i = 0; i < precision; ++i) + o.append('0'); + o.append('e'); + o.append('+'); + append_uint_padded(o, 0u, 2); + return; + } + + long double m = static_cast(v); + int exp10 = 0; + while (m >= 10.0L) + { + m /= 10.0L; + ++exp10; + } + while (m < 1.0L) + { + m *= 10.0L; + --exp10; + } + + std::array digits{}; + digits[0] = static_cast(m); + long double frac = m - static_cast(digits[0]); + for (int i = 1; i <= precision; ++i) + { + frac *= 10.0L; + int d = static_cast(frac); + if (d > 9) + d = 9; + digits[i] = d; + frac -= static_cast(d); + } + + frac *= 10.0L; + int round_digit = static_cast(frac); + if (round_digit > 9) + round_digit = 9; + long double remainder = frac - static_cast(round_digit); + bool round_up = false; + if (round_digit > 5) + round_up = true; + else if (round_digit == 5) + { + if (remainder > 0.0L) + round_up = true; + else + round_up = (digits[precision] % 2) != 0; + } + + if (round_up) + { + int i = precision; + for (; i >= 0; --i) + { + if (digits[i] < 9) + { + digits[i]++; + break; + } + digits[i] = 0; + } + if (i < 0) + { + digits[0] = 1; + for (int j = 1; j <= precision; ++j) + digits[j] = 0; + ++exp10; + } + } + + o.append(static_cast('0' + digits[0])); + o.append('.'); + for (int i = 1; i <= precision; ++i) + o.append(static_cast('0' + digits[i])); + o.append('e'); + if (exp10 < 0) + { + o.append('-'); + exp10 = -exp10; + } + else + { + o.append('+'); + } + const int exp_width = (exp10 >= 100) ? 3 : 2; + append_uint_padded(o, static_cast(exp10), exp_width); +} + +template +constexpr void put(Out& o, const T& v) +{ + using U = std::remove_cvref_t; + + if constexpr (std::is_same_v) + { + o.append(v ? '1' : '0'); + } + else if constexpr (std::is_integral_v) + { + using UU = std::make_unsigned_t; + UU x{}; + + if constexpr (std::is_signed_v) + { + if (v < 0) + { + o.append('-'); + x = UU(-(v + 1)) + 1; + } + else + { + x = UU(v); + } + } + else + { + x = UU(v); + } + + char tmp[3 + sizeof(U) * 8]; + size_t k = 0; + do { + tmp[k++] = char('0' + (x % 10)); + x /= 10; + } while (x); + while (k) + o.append(tmp[--k]); + } + else if constexpr (std::is_same_v || std::is_same_v) + { + append_float_scientific(o, v); + } + else if constexpr (std::is_floating_point_v) + { + static_assert(!sizeof(U), "Unsupported %s argument type"); + } + else if constexpr (std::is_convertible_v) + { + o.append(std::string_view(v)); + } + else if constexpr (std::is_same_v || std::is_same_v) + { + o.append((const char*)v); + } + else + { + static_assert(!sizeof(U), "Unsupported %s argument type"); + } +} + +template +constexpr void append_printf_s(Out& out, const Args&... args) +{ + auto tup = std::forward_as_tuple(args...); + size_t ai = 0; + + for (size_t i = 0; Fmt.value[i]; ++i) + { + if (Fmt.value[i] != '%') + { + out.append(Fmt.value[i]); + continue; + } + + char c = Fmt.value[++i]; + if (c == '%') + { + out.append('%'); + continue; + } + if (c == 's') + { + std::apply([&](auto const&... xs) { + size_t k = 0; + (((k++ == ai) ? (put(out, xs), 0) : 0), ...); + }, tup); + ++ai; + } + } +} + +} + #endif // _NBL_CORE_STRING_LITERAL_H_INCLUDED_ diff --git a/include/nbl/system/CFileArchive.h b/include/nbl/system/CFileArchive.h index 818bd8f6ba..35cb2e9413 100644 --- a/include/nbl/system/CFileArchive.h +++ b/include/nbl/system/CFileArchive.h @@ -10,6 +10,8 @@ #include "nbl/system/CFileView.h" #include "nbl/system/IFileViewAllocator.h" +#include + #ifdef _NBL_PLATFORM_ANDROID_ #include "nbl/system/CFileViewAPKAllocator.h" #endif @@ -22,13 +24,21 @@ template class CInnerArchiveFile : public CFileView { std::atomic_flag* alive; + std::optional m_precomputedHash; public: template - CInnerArchiveFile(std::atomic_flag* _flag, Args&&... args) : CFileView(std::forward(args)...), alive(_flag) + CInnerArchiveFile(std::atomic_flag* _flag, std::optional precomputedHash, Args&&... args) + : CFileView(std::forward(args)...), alive(_flag), m_precomputedHash(std::move(precomputedHash)) { } ~CInnerArchiveFile() = default; + // Non-empty return means the file came from an archive that embeds a precomputed hash. + std::optional getPrecomputedHash() const override + { + return m_precomputedHash; + } + static void* operator new(size_t size) noexcept { assert(false); @@ -144,6 +154,7 @@ class CFileArchive : public IFileArchive // coast is clear, do placement new new (file, &m_fileFlags[found->ID]) CInnerArchiveFile( m_fileFlags+found->ID, + std::move(fileBuffer.precomputedHash), getDefaultAbsolutePath()/found->pathRelativeToArchive, flags, fileBuffer.initialModified, @@ -162,6 +173,7 @@ class CFileArchive : public IFileArchive void* buffer; size_t size; void* allocatorState; + std::optional precomputedHash = {}; // TODO: Implement this !!! IFileBase::time_point_t initialModified = std::chrono::utc_clock::now(); }; diff --git a/include/nbl/system/IFileBase.h b/include/nbl/system/IFileBase.h index c9ceb13a04..ae336e24cd 100644 --- a/include/nbl/system/IFileBase.h +++ b/include/nbl/system/IFileBase.h @@ -41,6 +41,7 @@ class IFileBase : public core::IReferenceCounted //! Optional, if not present this means that the hash was not already precomputed for you. // Equivalent to calling `xxHash256(getMappedPointer(),getSize(),&retval.x)` // Only really available for built-in resources or some other files that had to be read in their entirety at some point. + // Non-empty return means the file comes from an archive that embeds a precomputed hash. virtual inline std::optional getPrecomputedHash() const {return {};} //! diff --git a/src/nbl/asset/utils/CGLSLCompiler.cpp b/src/nbl/asset/utils/CGLSLCompiler.cpp index a593a11597..7f9763f5c4 100644 --- a/src/nbl/asset/utils/CGLSLCompiler.cpp +++ b/src/nbl/asset/utils/CGLSLCompiler.cpp @@ -136,6 +136,9 @@ CGLSLCompiler::CGLSLCompiler(core::smart_refctd_ptr&& system) std::string CGLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions, std::vector* dependencies) const { + if (preprocessOptions.applyForceIncludes && !preprocessOptions.forceIncludes.empty()) + code = IShaderCompiler::applyForceIncludes(code, preprocessOptions.forceIncludes); + if (!preprocessOptions.extraDefines.empty()) { std::ostringstream insertion; diff --git a/src/nbl/asset/utils/CHLSLCompiler.cpp b/src/nbl/asset/utils/CHLSLCompiler.cpp index 1020fa9446..4132fe6fe6 100644 --- a/src/nbl/asset/utils/CHLSLCompiler.cpp +++ b/src/nbl/asset/utils/CHLSLCompiler.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -363,7 +364,7 @@ namespace nbl::wave extern nbl::core::string preprocess(std::string& code, const IShaderCompiler::SPreprocessorOptions& preprocessOptions, bool withCaching, std::function post); } -std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions, std::vector& dxc_compile_flags_override, std::vector* dependencies) const +std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions, std::vector& dxc_compile_flags_override, std::vector* dependencies, std::vector* macro_defs) const { const bool depfileEnabled = preprocessOptions.depfile; if (depfileEnabled) @@ -375,6 +376,9 @@ std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADE } } + if (preprocessOptions.applyForceIncludes && !preprocessOptions.forceIncludes.empty()) + code = IShaderCompiler::applyForceIncludes(code, preprocessOptions.forceIncludes); + std::vector localDependencies; auto* dependenciesOut = dependencies; if (depfileEnabled && !dependenciesOut) @@ -395,7 +399,7 @@ std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADE // preprocess core::string resolvedString = nbl::wave::preprocess(code, preprocessOptions, bool(dependenciesOut), - [&dxc_compile_flags_override, &stage, &dependenciesOut](nbl::wave::context& context) -> void + [&dxc_compile_flags_override, &stage, &dependenciesOut, macro_defs](nbl::wave::context& context) -> void { if (context.get_hooks().m_dxc_compile_flags_override.size() != 0) dxc_compile_flags_override = context.get_hooks().m_dxc_compile_flags_override; @@ -406,6 +410,8 @@ std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADE if (dependenciesOut) *dependenciesOut = std::move(context.get_dependencies()); + if (macro_defs) + context.dump_macro_definitions(*macro_defs); } ); @@ -444,7 +450,24 @@ std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADE std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions, std::vector* dependencies) const { std::vector extra_dxc_compile_flags = {}; - return preprocessShader(std::move(code), stage, preprocessOptions, extra_dxc_compile_flags, dependencies); + return preprocessShader(std::move(code), stage, preprocessOptions, extra_dxc_compile_flags, dependencies, nullptr); +} + +bool CHLSLCompiler::preprocessPrefixForCache(std::string_view code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions, CPreprocessCache::SEntry& outEntry) const +{ + outEntry = {}; + std::vector deps; + std::vector dxcFlags; + std::vector macroDefs; + auto text = preprocessShader(std::string(code), stage, preprocessOptions, dxcFlags, &deps, ¯oDefs); + if (text.empty()) + return false; + outEntry.preprocessedPrefix = std::move(text); + outEntry.dependencies = std::move(deps); + outEntry.dxcFlags = std::move(dxcFlags); + outEntry.macroDefs = std::move(macroDefs); + outEntry.pragmaStage = static_cast(stage); + return true; } core::smart_refctd_ptr CHLSLCompiler::compileToSPIRV_impl(const std::string_view code, const IShaderCompiler::SCompilerOptions& options, std::vector* dependencies) const @@ -459,7 +482,11 @@ core::smart_refctd_ptr CHLSLCompiler::compileToSPIRV_impl(const std::st std::vector dxc_compile_flags = {}; IShader::E_SHADER_STAGE stage = options.stage; + using clock_t = std::chrono::high_resolution_clock; + const auto preprocessStart = clock_t::now(); auto newCode = preprocessShader(std::string(code), stage, hlslOptions.preprocessorOptions, dxc_compile_flags, dependencies); + const auto preprocessEnd = clock_t::now(); + logger.log("Preprocess took: %lld ms.", system::ILogger::ELL_PERFORMANCE, static_cast(std::chrono::duration_cast(preprocessEnd - preprocessStart).count())); if (newCode.empty()) return nullptr; // Suffix is the shader model version @@ -543,6 +570,7 @@ core::smart_refctd_ptr CHLSLCompiler::compileToSPIRV_impl(const std::st for (size_t i = 0; i < argc; i++) argsArray[i] = arguments[i].c_str(); + const auto compileStart = clock_t::now(); auto compileResult = dxcCompile( this, m_dxcCompilerTypes, @@ -551,6 +579,9 @@ core::smart_refctd_ptr CHLSLCompiler::compileToSPIRV_impl(const std::st argc, hlslOptions ); + const auto compileEnd = clock_t::now(); + logger.log("Compile took: %lld ms.", system::ILogger::ELL_PERFORMANCE, static_cast(std::chrono::duration_cast(compileEnd - compileStart).count())); + logger.log("Total build time: %lld ms.", system::ILogger::ELL_PERFORMANCE, static_cast(std::chrono::duration_cast(compileEnd - preprocessStart).count())); if (argsArray) delete[] argsArray; diff --git a/src/nbl/asset/utils/CWaveStringResolver.cpp b/src/nbl/asset/utils/CWaveStringResolver.cpp index a2165972e5..456aadb4ac 100644 --- a/src/nbl/asset/utils/CWaveStringResolver.cpp +++ b/src/nbl/asset/utils/CWaveStringResolver.cpp @@ -70,14 +70,24 @@ namespace nbl::wave stream << i->get_value(); resolvedString = stream.str(); } - catch (boost::wave::preprocess_exception& e) + catch (const boost::wave::cpp_exception& e) { - preprocessOptions.logger.log("%s exception caught. %s [%s:%d:%d]",system::ILogger::ELL_ERROR,e.what(),e.description(),e.file_name(),e.line_no(),e.column_no()); + preprocessOptions.logger.log("%s exception caught. %s [%s:%d:%d]", system::ILogger::ELL_ERROR, e.what(), e.description(), e.file_name(), e.line_no(), e.column_no()); + return {}; + } + catch (const boost::wave::cpplexer::lexing_exception& e) + { + preprocessOptions.logger.log("%s exception caught. %s [%s:%d:%d]", system::ILogger::ELL_ERROR, e.what(), e.description(), e.file_name(), e.line_no(), e.column_no()); + return {}; + } + catch (const std::exception& e) + { + preprocessOptions.logger.log("Exception caught. %s", system::ILogger::ELL_ERROR, e.what()); return {}; } catch (...) { - preprocessOptions.logger.log("Unknown exception caught!",system::ILogger::ELL_ERROR); + preprocessOptions.logger.log("Unknown exception caught!", system::ILogger::ELL_ERROR); return {}; } @@ -85,4 +95,4 @@ namespace nbl::wave return resolvedString; } -} \ No newline at end of file +} diff --git a/src/nbl/asset/utils/IShaderCompiler.cpp b/src/nbl/asset/utils/IShaderCompiler.cpp index a6cd95b441..f21436d9ea 100644 --- a/src/nbl/asset/utils/IShaderCompiler.cpp +++ b/src/nbl/asset/utils/IShaderCompiler.cpp @@ -4,12 +4,15 @@ #include "nbl/asset/utils/IShaderCompiler.h" #include "nbl/asset/utils/shadercUtils.h" #include "nbl/asset/utils/shaderCompiler_serialization.h" - +#include "nbl/core/hash/blake.h" #include #include #include #include #include +#include +#include +#include #include #include @@ -17,6 +20,87 @@ using namespace nbl; using namespace nbl::asset; +namespace +{ + void splitPrefix(std::string_view code, std::string_view& prefix, std::string_view& body) + { + size_t pos = 0; + size_t prefixEnd = 0; + bool inContinuation = false; + bool inBlockComment = false; + + while (pos < code.size()) + { + const size_t lineStart = pos; + size_t lineEnd = code.find('\n', pos); + if (lineEnd == std::string_view::npos) + lineEnd = code.size(); + + std::string_view line = code.substr(lineStart, lineEnd - lineStart); + if (!line.empty() && line.back() == '\r') + line.remove_suffix(1); + + bool directiveLine = false; + if (inContinuation || inBlockComment) + { + directiveLine = true; + } + else + { + size_t i = 0; + if (line.size() >= 3 && static_cast(line[0]) == 0xEF && + static_cast(line[1]) == 0xBB && static_cast(line[2]) == 0xBF) + i = 3; + while (i < line.size() && (line[i] == ' ' || line[i] == '\t')) + ++i; + if (i == line.size()) + { + directiveLine = true; + } + else if (line[i] == '#') + { + directiveLine = true; + } + else if (line[i] == '/' && i + 1 < line.size() && line[i + 1] == '/') + { + directiveLine = true; + } + else if (line[i] == '/' && i + 1 < line.size() && line[i + 1] == '*') + { + directiveLine = true; + if (line.find("*/", i + 2) == std::string_view::npos) + inBlockComment = true; + } + } + + if (!directiveLine) + break; + + prefixEnd = lineEnd < code.size() ? lineEnd + 1 : lineEnd; + + if (inBlockComment && line.find("*/") != std::string_view::npos) + inBlockComment = false; + + bool continuation = false; + if (!line.empty()) + { + size_t j = line.size(); + while (j > 0 && (line[j - 1] == ' ' || line[j - 1] == '\t')) + --j; + if (j > 0 && line[j - 1] == '\\') + continuation = true; + } + inContinuation = continuation; + if (lineEnd == code.size()) + break; + pos = lineEnd + 1; + } + + prefix = code.substr(0, prefixEnd); + body = code.substr(prefixEnd); + } +} + IShaderCompiler::IShaderCompiler(core::smart_refctd_ptr&& system) : m_system(std::move(system)) { @@ -233,6 +317,7 @@ core::smart_refctd_ptr nbl::asset::IShaderCompiler::compileToSPIRV(cons { const bool depfileEnabled = options.preprocessorOptions.depfile; const bool supportsDependencies = options.getCodeContentType() == IShader::E_CONTENT_TYPE::ECT_HLSL; + const auto* dependencyOverrides = options.dependencyOverrides; auto writeDepfileFromDependencies = [&](const CCache::SEntry::dependency_container_t& dependencies) -> bool { @@ -255,15 +340,21 @@ core::smart_refctd_ptr nbl::asset::IShaderCompiler::compileToSPIRV(cons return IShaderCompiler::writeDepfile(params, dependencies, options.preprocessorOptions.includeFinder, options.preprocessorOptions.logger); }; + const std::string_view cacheCode = options.preprocessorOptions.codeForCache.empty() ? code : options.preprocessorOptions.codeForCache; CCache::SEntry entry; if (options.readCache || options.writeCache) - entry = CCache::SEntry(code, options); + entry = CCache::SEntry(cacheCode, options); + + if (options.cacheHit) + *options.cacheHit = false; if (options.readCache) { auto found = options.readCache->find_impl(entry, options.preprocessorOptions.includeFinder); if (found != options.readCache->m_container.end()) { + if (options.cacheHit) + *options.cacheHit = true; if (options.writeCache) { CCache::SEntry writeEntry = *found; @@ -278,10 +369,13 @@ core::smart_refctd_ptr nbl::asset::IShaderCompiler::compileToSPIRV(cons CCache::SEntry::dependency_container_t depfileDependencies; CCache::SEntry::dependency_container_t* dependenciesPtr = nullptr; - if (options.writeCache) - dependenciesPtr = &entry.dependencies; - else if (depfileEnabled && supportsDependencies) - dependenciesPtr = &depfileDependencies; + if (!dependencyOverrides) + { + if (options.writeCache) + dependenciesPtr = &entry.dependencies; + else if (depfileEnabled && supportsDependencies) + dependenciesPtr = &depfileDependencies; + } auto retVal = compileToSPIRV_impl(code, options, dependenciesPtr); if (retVal) @@ -290,9 +384,17 @@ core::smart_refctd_ptr nbl::asset::IShaderCompiler::compileToSPIRV(cons const_cast(backingBuffer)->setContentHash(backingBuffer->computeContentHash()); } + if (retVal && options.writeCache && dependencyOverrides) + { + entry.dependencies.clear(); + entry.dependencies.reserve(dependencyOverrides->size()); + for (const auto& dep : *dependencyOverrides) + entry.dependencies.emplace_back(dep.getRequestingSourceDir(), dep.getIdentifier(), dep.isStandardInclude(), dep.getHash(), dep.getAbsolutePath(), dep.getFileSize(), dep.getLastWriteTime(), dep.getHasFileInfo()); + } + if (retVal && depfileEnabled && supportsDependencies) { - const auto* deps = options.writeCache ? &entry.dependencies : &depfileDependencies; + const auto* deps = dependencyOverrides ? dependencyOverrides : (options.writeCache ? &entry.dependencies : &depfileDependencies); if (!writeDepfileFromDependencies(*deps)) return nullptr; } @@ -377,7 +479,17 @@ auto IShaderCompiler::CFileSystemIncludeLoader::getInclude(const system::path& s const bool success = bool(succ); assert(success); - return { f->getFileName(),std::move(contents) }; + found_t ret = {}; + ret.absolutePath = f->getFileName(); + ret.contents = std::move(contents); + ret.fileSize = size; + if (auto precomputed = f->getPrecomputedHash()) + { + static_assert(sizeof(ret.hash.data) == sizeof(*precomputed)); + std::memcpy(ret.hash.data, &(*precomputed), sizeof(ret.hash.data)); + ret.hasHash = true; + } + return ret; } IShaderCompiler::CIncludeFinder::CIncludeFinder(core::smart_refctd_ptr&& system) @@ -399,10 +511,26 @@ auto IShaderCompiler::CIncludeFinder::getIncludeStandard(const system::path& req retVal = std::move(contents); else retVal = m_defaultFileSystemLoader->getInclude(requestingSourceDir.string(), includeName); + if (retVal.fileSize == 0 && !retVal.contents.empty()) + retVal.fileSize = retVal.contents.size(); + if (!retVal.absolutePath.empty()) + { + std::error_code ec; + const auto fileTime = std::filesystem::last_write_time(retVal.absolutePath, ec); + if (!ec) + { + retVal.lastWriteTime = fileTime.time_since_epoch().count(); + retVal.hasFileInfo = true; + } + } - core::blake3_hasher hasher; - hasher.update(reinterpret_cast(retVal.contents.data()), retVal.contents.size() * (sizeof(char) / sizeof(uint8_t))); - retVal.hash = static_cast(hasher); + if (!retVal.hasHash) + { + core::blake3_hasher hasher; + hasher.update(reinterpret_cast(retVal.contents.data()), retVal.contents.size() * (sizeof(char) / sizeof(uint8_t))); + retVal.hash = static_cast(hasher); + retVal.hasHash = true; + } return retVal; } @@ -416,9 +544,26 @@ auto IShaderCompiler::CIncludeFinder::getIncludeRelative(const system::path& req retVal = std::move(contents); else retVal = std::move(trySearchPaths(includeName)); - core::blake3_hasher hasher; - hasher.update(reinterpret_cast(retVal.contents.data()), retVal.contents.size() * (sizeof(char) / sizeof(uint8_t))); - retVal.hash = static_cast(hasher); + if (retVal.fileSize == 0 && !retVal.contents.empty()) + retVal.fileSize = retVal.contents.size(); + if (!retVal.absolutePath.empty()) + { + std::error_code ec; + const auto fileTime = std::filesystem::last_write_time(retVal.absolutePath, ec); + if (!ec) + { + retVal.lastWriteTime = fileTime.time_since_epoch().count(); + retVal.hasFileInfo = true; + } + } + + if (!retVal.hasHash) + { + core::blake3_hasher hasher; + hasher.update(reinterpret_cast(retVal.contents.data()), retVal.contents.size() * (sizeof(char) / sizeof(uint8_t))); + retVal.hash = static_cast(hasher); + retVal.hasHash = true; + } return retVal; } @@ -426,7 +571,16 @@ void IShaderCompiler::CIncludeFinder::addSearchPath(const std::string& searchPat { if (!loader) return; - m_loaders.emplace_back(LoaderSearchPath{ loader, searchPath }); + if (searchPath.empty()) + { + m_loaders.emplace_back(LoaderSearchPath{ loader, searchPath }); + return; + } + const auto insertPos = std::find_if(m_loaders.begin(), m_loaders.end(), [](const LoaderSearchPath& entry) + { + return entry.searchPath.empty(); + }); + m_loaders.insert(insertPos, LoaderSearchPath{ loader, searchPath }); } void IShaderCompiler::CIncludeFinder::addGenerator(const core::smart_refctd_ptr& generatorToAdd) @@ -513,23 +667,125 @@ core::smart_refctd_ptr IShaderCompiler::CCache::find(const SEntr return found->decompressShader(); } +bool IShaderCompiler::CCache::contains(const SEntry& mainFile, const IShaderCompiler::CIncludeFinder* finder) const +{ + return find_impl(mainFile, finder) != m_container.end(); +} + +bool IShaderCompiler::CCache::findEntryForCode(std::string_view code, const SCompilerOptions& options, const IShaderCompiler::CIncludeFinder* finder, SEntry& outEntry) const +{ + const std::string_view cacheCode = options.preprocessorOptions.codeForCache.empty() ? code : options.preprocessorOptions.codeForCache; + const CCache::SEntry entry(cacheCode, options); + const auto found = find_impl(entry, finder); + if (found == m_container.end()) + return false; + outEntry = SEntry(*found); + return true; +} + +core::smart_refctd_ptr IShaderCompiler::CCache::decompressEntry(const SEntry& entry) const +{ + return entry.decompressShader(); +} + IShaderCompiler::CCache::EntrySet::const_iterator IShaderCompiler::CCache::find_impl(const SEntry& mainFile, const IShaderCompiler::CIncludeFinder* finder) const { auto found = m_container.find(mainFile); // go through all dependencies if (found!=m_container.end()) { + std::unordered_map fileStatus; + std::unordered_map logicalStatus; for (const auto& dependency : found->dependencies) { - IIncludeLoader::found_t header; - if (dependency.standardInclude) - header = finder->getIncludeStandard(dependency.requestingSourceDir, dependency.identifier); + if (dependency.getHasFileInfo() && !dependency.getAbsolutePath().empty()) + { + if (auto it = fileStatus.find(dependency.getAbsolutePath()); it != fileStatus.end()) + { + if (!it->second) + return m_container.end(); + continue; + } + } else - header = finder->getIncludeRelative(dependency.requestingSourceDir, dependency.identifier); + { + std::string key; + key.reserve(dependency.getIdentifier().size() + dependency.getRequestingSourceDir().string().size() + 4); + key.append(dependency.getRequestingSourceDir().string()); + key.push_back('|'); + key.append(dependency.getIdentifier()); + key.push_back('|'); + key.push_back(dependency.isStandardInclude() ? '1' : '0'); + if (auto it = logicalStatus.find(key); it != logicalStatus.end()) + { + if (!it->second) + return m_container.end(); + continue; + } + } + + bool valid = false; + if (dependency.getHasFileInfo() && !dependency.getAbsolutePath().empty()) + { + std::error_code ec; + std::filesystem::directory_entry entry(dependency.getAbsolutePath(), ec); + if (!ec) + { + const auto time = entry.last_write_time(ec); + if (!ec) + { + const auto ticks = time.time_since_epoch().count(); + if (dependency.getLastWriteTime() == ticks) + { + const auto size = entry.file_size(ec); + if (!ec && size == dependency.getFileSize()) + valid = true; + } + } + } + } - if (header.hash != dependency.hash) + if (!valid) { - return m_container.end(); + if (!finder) + return m_container.end(); + IIncludeLoader::found_t header; + if (dependency.standardInclude) + header = finder->getIncludeStandard(dependency.requestingSourceDir, dependency.identifier); + else + header = finder->getIncludeRelative(dependency.requestingSourceDir, dependency.identifier); + + if (header.hash != dependency.hash) + { + if (dependency.getHasFileInfo() && !dependency.getAbsolutePath().empty()) + fileStatus.emplace(dependency.getAbsolutePath(), false); + else + { + std::string key; + key.reserve(dependency.getIdentifier().size() + dependency.getRequestingSourceDir().string().size() + 4); + key.append(dependency.getRequestingSourceDir().string()); + key.push_back('|'); + key.append(dependency.getIdentifier()); + key.push_back('|'); + key.push_back(dependency.isStandardInclude() ? '1' : '0'); + logicalStatus.emplace(std::move(key), false); + } + return m_container.end(); + } + } + + if (dependency.getHasFileInfo() && !dependency.getAbsolutePath().empty()) + fileStatus.emplace(dependency.getAbsolutePath(), true); + else + { + std::string key; + key.reserve(dependency.getIdentifier().size() + dependency.getRequestingSourceDir().string().size() + 4); + key.append(dependency.getRequestingSourceDir().string()); + key.push_back('|'); + key.append(dependency.getIdentifier()); + key.push_back('|'); + key.push_back(dependency.isStandardInclude() ? '1' : '0'); + logicalStatus.emplace(std::move(key), true); } } } @@ -636,6 +892,586 @@ core::smart_refctd_ptr IShaderCompiler::CCache::deseria return retVal; } +static std::string normalizeLinePath(std::string_view path) +{ + std::string out(path); + std::replace(out.begin(), out.end(), '\\', '/'); + return out; +} + +std::string IShaderCompiler::applyForceIncludes(std::string_view code, std::span forceIncludes) +{ + if (forceIncludes.empty()) + return std::string(code); + + size_t reserveSize = code.size(); + for (const auto& inc : forceIncludes) + reserveSize += inc.size() + 16; + + std::string out; + out.reserve(reserveSize); + for (const auto& inc : forceIncludes) + { + const auto incPath = std::filesystem::path(inc).generic_string(); + out.append("#include \""); + out.append(incPath); + out.append("\"\n"); + } + out.append(code); + return out; +} + +bool IShaderCompiler::probeShaderCache(const CCache* cache, std::string_view code, const SCompilerOptions& options, const CIncludeFinder* finder) +{ + if (!cache) + return false; + const std::string_view cacheCode = options.preprocessorOptions.codeForCache.empty() ? code : options.preprocessorOptions.codeForCache; + const CCache::SEntry entry(cacheCode, options); + return cache->contains(entry, finder); +} + +bool IShaderCompiler::preprocessPrefixForCache(std::string_view code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions, CPreprocessCache::SEntry& outEntry) const +{ + outEntry = {}; + std::vector deps; + auto text = preprocessShader(std::string(code), stage, preprocessOptions, &deps); + if (text.empty()) + return false; + outEntry.preprocessedPrefix = std::move(text); + outEntry.dependencies = std::move(deps); + outEntry.pragmaStage = static_cast(stage); + return true; +} + +IShaderCompiler::CPreprocessCache::SProbeResult IShaderCompiler::CPreprocessCache::probe(std::string_view code, const CPreprocessCache* cache, ELoadStatus loadStatus, const SPreprocessorOptions& preprocessOptions) +{ + SProbeResult result = {}; + const CIncludeFinder* finder = preprocessOptions.includeFinder; + std::string_view codeToSplit = code; + if (preprocessOptions.applyForceIncludes && !preprocessOptions.forceIncludes.empty()) + { + result.codeStorage = applyForceIncludes(code, preprocessOptions.forceIncludes); + codeToSplit = result.codeStorage; + } + splitPrefix(codeToSplit, result.prefix, result.body); + result.hasPrefix = !result.prefix.empty(); + if (!result.hasPrefix) + { + result.status = EProbeStatus::NoPrefix; + result.cacheHit = false; + return result; + } + + { + core::blake3_hasher hasher; + hasher.update(result.prefix.data(), result.prefix.size()); + result.prefixHash = static_cast(hasher); + } + const bool hasEntry = cache && cache->hasEntry(); + if (!hasEntry) + { + result.cacheHit = false; + if (loadStatus == ELoadStatus::Missing) + result.status = EProbeStatus::Missing; + else if (loadStatus == ELoadStatus::Invalid) + result.status = EProbeStatus::Invalid; + else + result.status = EProbeStatus::EntryInvalid; + return result; + } + + const bool prefixMatch = cache->getEntry().prefixHash == result.prefixHash; + const bool depsValid = cache->validateDependencies(finder); + if (prefixMatch && depsValid) + { + result.cacheHit = true; + result.status = EProbeStatus::Hit; + return result; + } + + result.cacheHit = false; + if (!prefixMatch) + result.status = EProbeStatus::PrefixChanged; + else if (!depsValid) + result.status = EProbeStatus::DependenciesChanged; + else + result.status = EProbeStatus::EntryInvalid; + + return result; +} + +const char* IShaderCompiler::CPreprocessCache::getProbeReason(EProbeStatus status) +{ + switch (status) + { + case EProbeStatus::Missing: + return "cache file missing; first build, cleaned, output moved, or out of date"; + case EProbeStatus::Invalid: + return "cache file invalid or version mismatch"; + case EProbeStatus::PrefixChanged: + return "prefix changed; cache invalidated"; + case EProbeStatus::DependenciesChanged: + return "dependencies changed; cache invalidated"; + case EProbeStatus::EntryInvalid: + return "cache entry invalid"; + case EProbeStatus::NoPrefix: + return "no prefix"; + case EProbeStatus::Hit: + return "hit"; + default: + return "unknown"; + } +} + +IShaderCompiler::SPreprocessCacheResult IShaderCompiler::preprocessWithCache(std::string_view code, IShader::E_SHADER_STAGE stage, const SPreprocessorOptions& preprocessOptions, CPreprocessCache& cache, CPreprocessCache::ELoadStatus loadStatus, std::string_view sourceIdentifier) const +{ + SPreprocessCacheResult result = {}; + result.stage = stage; + + const auto probe = CPreprocessCache::probe(code, &cache, loadStatus, preprocessOptions); + result.status = probe.status; + if (!probe.hasPrefix) + return result; + + if (probe.cacheHit) + { + result.cacheHit = true; + result.cacheUsed = true; + } + else + { + CPreprocessCache::SEntry entry; + IShader::E_SHADER_STAGE prefixStage = stage; + SPreprocessorOptions preCacheOpt = preprocessOptions; + preCacheOpt.depfile = false; + if (!preprocessPrefixForCache(probe.prefix, prefixStage, preCacheOpt, entry)) + { + result.ok = false; + return result; + } + entry.prefixHash = probe.prefixHash; + entry.pragmaStage = static_cast(prefixStage); + cache.setEntry(std::move(entry)); + result.cacheUsed = true; + result.cacheUpdated = true; + } + + if (!cache.hasEntry()) + { + result.ok = false; + return result; + } + + result.code = cache.buildCombinedCode(probe.body, sourceIdentifier); + if (result.code.empty()) + { + result.ok = false; + return result; + } + + const auto& entry = cache.getEntry(); + if (entry.pragmaStage != static_cast(IShader::E_SHADER_STAGE::ESS_UNKNOWN)) + result.stage = static_cast(entry.pragmaStage); + + return result; +} + +core::smart_refctd_ptr IShaderCompiler::CPreprocessCache::serialize() const +{ + if (!m_hasEntry) + return nullptr; + + auto write_bytes = [](std::vector& out, const void* data, size_t size) + { + const auto* ptr = reinterpret_cast(data); + out.insert(out.end(), ptr, ptr + size); + }; + auto write_u32 = [&write_bytes](std::vector& out, uint32_t value) + { + write_bytes(out, &value, sizeof(value)); + }; + auto write_string = [&write_u32, &write_bytes](std::vector& out, std::string_view value) + { + write_u32(out, static_cast(value.size())); + if (!value.empty()) + write_bytes(out, value.data(), value.size()); + }; + + std::vector out; + out.reserve(m_entry.preprocessedPrefix.size() + 256); + const uint32_t magic = 0x50435250u; + write_u32(out, magic); + write_string(out, VERSION); + write_bytes(out, &m_entry.prefixHash, sizeof(m_entry.prefixHash)); + write_u32(out, m_entry.pragmaStage); + write_string(out, m_entry.preprocessedPrefix); + + write_u32(out, static_cast(m_entry.macroDefs.size())); + for (const auto& macro : m_entry.macroDefs) + write_string(out, macro); + + write_u32(out, static_cast(m_entry.dxcFlags.size())); + for (const auto& flag : m_entry.dxcFlags) + write_string(out, flag); + + write_u32(out, static_cast(m_entry.dependencies.size())); + for (const auto& dep : m_entry.dependencies) + { + const auto dir = dep.getRequestingSourceDir().generic_string(); + write_string(out, dir); + write_string(out, dep.getIdentifier()); + const auto abs = dep.getAbsolutePath().generic_string(); + write_string(out, abs); + const uint8_t standardInclude = dep.isStandardInclude() ? 1u : 0u; + write_bytes(out, &standardInclude, sizeof(standardInclude)); + write_bytes(out, dep.getHash().data, sizeof(dep.getHash().data)); + const uint64_t fileSize = dep.getFileSize(); + write_bytes(out, &fileSize, sizeof(fileSize)); + const int64_t lastWriteTime = dep.getLastWriteTime(); + write_bytes(out, &lastWriteTime, sizeof(lastWriteTime)); + const uint8_t hasFileInfo = dep.getHasFileInfo() ? 1u : 0u; + write_bytes(out, &hasFileInfo, sizeof(hasFileInfo)); + } + + auto buffer = ICPUBuffer::create({ out.size() }); + if (!buffer) + return nullptr; + std::memcpy(buffer->getPointer(), out.data(), out.size()); + return buffer; +} + +core::smart_refctd_ptr IShaderCompiler::CPreprocessCache::deserialize(const std::span serializedCache) +{ + if (serializedCache.empty()) + return nullptr; + + auto read_bytes = [](const std::span data, size_t& offset, void* dst, size_t size) -> bool + { + if (offset + size > data.size()) + return false; + std::memcpy(dst, data.data() + offset, size); + offset += size; + return true; + }; + auto read_u32 = [&read_bytes](const std::span data, size_t& offset, uint32_t& out) -> bool + { + return read_bytes(data, offset, &out, sizeof(out)); + }; + auto read_string = [&read_u32, &read_bytes](const std::span data, size_t& offset, std::string& out) -> bool + { + uint32_t size = 0; + if (!read_u32(data, offset, size)) + return false; + if (offset + size > data.size()) + return false; + out.assign(reinterpret_cast(data.data() + offset), size); + offset += size; + return true; + }; + + size_t offset = 0; + uint32_t magic = 0; + if (!read_u32(serializedCache, offset, magic)) + return nullptr; + if (magic != 0x50435250u) + return nullptr; + + std::string version; + if (!read_string(serializedCache, offset, version)) + return nullptr; + if (version != VERSION) + return nullptr; + + auto retVal = core::make_smart_refctd_ptr(); + auto& entry = retVal->m_entry; + if (!read_bytes(serializedCache, offset, &entry.prefixHash, sizeof(entry.prefixHash))) + return nullptr; + if (!read_u32(serializedCache, offset, entry.pragmaStage)) + return nullptr; + if (!read_string(serializedCache, offset, entry.preprocessedPrefix)) + return nullptr; + + uint32_t macroCount = 0; + if (!read_u32(serializedCache, offset, macroCount)) + return nullptr; + entry.macroDefs.clear(); + entry.macroDefs.reserve(macroCount); + for (uint32_t i = 0; i < macroCount; ++i) + { + std::string macro; + if (!read_string(serializedCache, offset, macro)) + return nullptr; + entry.macroDefs.emplace_back(std::move(macro)); + } + + uint32_t flagCount = 0; + if (!read_u32(serializedCache, offset, flagCount)) + return nullptr; + entry.dxcFlags.clear(); + entry.dxcFlags.reserve(flagCount); + for (uint32_t i = 0; i < flagCount; ++i) + { + std::string flag; + if (!read_string(serializedCache, offset, flag)) + return nullptr; + entry.dxcFlags.emplace_back(std::move(flag)); + } + + uint32_t depCount = 0; + if (!read_u32(serializedCache, offset, depCount)) + return nullptr; + entry.dependencies.clear(); + entry.dependencies.reserve(depCount); + for (uint32_t i = 0; i < depCount; ++i) + { + std::string dir; + std::string identifier; + if (!read_string(serializedCache, offset, dir)) + return nullptr; + if (!read_string(serializedCache, offset, identifier)) + return nullptr; + std::string absolutePath; + if (!read_string(serializedCache, offset, absolutePath)) + return nullptr; + uint8_t standardInclude = 0; + if (!read_bytes(serializedCache, offset, &standardInclude, sizeof(standardInclude))) + return nullptr; + core::blake3_hash_t hash = {}; + if (!read_bytes(serializedCache, offset, hash.data, sizeof(hash.data))) + return nullptr; + uint64_t fileSize = 0; + if (!read_bytes(serializedCache, offset, &fileSize, sizeof(fileSize))) + return nullptr; + int64_t lastWriteTime = 0; + if (!read_bytes(serializedCache, offset, &lastWriteTime, sizeof(lastWriteTime))) + return nullptr; + uint8_t hasFileInfo = 0; + if (!read_bytes(serializedCache, offset, &hasFileInfo, sizeof(hasFileInfo))) + return nullptr; + entry.dependencies.emplace_back(system::path(dir), identifier, standardInclude != 0, hash, system::path(absolutePath), fileSize, lastWriteTime, hasFileInfo != 0); + } + + retVal->m_hasEntry = true; + return retVal; +} + +core::smart_refctd_ptr IShaderCompiler::CPreprocessCache::loadFromFile(const system::path& path, ELoadStatus& status) +{ + status = ELoadStatus::Missing; + if (!std::filesystem::exists(path)) + return nullptr; + + std::ifstream in(path, std::ios::binary); + if (!in) + { + status = ELoadStatus::Invalid; + return nullptr; + } + + in.seekg(0, std::ios::end); + const auto size = static_cast(in.tellg()); + in.seekg(0, std::ios::beg); + if (!size) + { + status = ELoadStatus::Invalid; + return nullptr; + } + + std::vector data(size); + if (!in.read(reinterpret_cast(data.data()), data.size())) + { + status = ELoadStatus::Invalid; + return nullptr; + } + + auto cache = deserialize(std::span(data.data(), data.size())); + if (!cache) + { + status = ELoadStatus::Invalid; + return nullptr; + } + + status = ELoadStatus::Loaded; + return cache; +} + +bool IShaderCompiler::CPreprocessCache::writeToFile(const system::path& path, const CPreprocessCache& cache) +{ + auto buffer = cache.serialize(); + if (!buffer) + return false; + + const auto parent = path.parent_path(); + if (!parent.empty() && !std::filesystem::exists(parent)) + std::filesystem::create_directories(parent); + + std::ofstream out(path, std::ios::binary | std::ios::trunc); + if (!out) + return false; + + out.write(reinterpret_cast(buffer->getPointer()), buffer->getSize()); + return bool(out); +} + +bool IShaderCompiler::CPreprocessCache::validateDependencies(const CIncludeFinder* finder) const +{ + if (!m_hasEntry || !finder) + return false; + + std::unordered_map fileStatus; + std::unordered_map logicalStatus; + for (const auto& dep : m_entry.dependencies) + { + if (dep.getHasFileInfo() && !dep.getAbsolutePath().empty()) + { + if (auto it = fileStatus.find(dep.getAbsolutePath()); it != fileStatus.end()) + { + if (!it->second) + return false; + continue; + } + } + else + { + std::string key; + key.reserve(dep.getIdentifier().size() + dep.getRequestingSourceDir().string().size() + 4); + key.append(dep.getRequestingSourceDir().string()); + key.push_back('|'); + key.append(dep.getIdentifier()); + key.push_back('|'); + key.push_back(dep.isStandardInclude() ? '1' : '0'); + if (auto it = logicalStatus.find(key); it != logicalStatus.end()) + { + if (!it->second) + return false; + continue; + } + } + + bool valid = false; + if (dep.getHasFileInfo() && !dep.getAbsolutePath().empty()) + { + std::error_code ec; + std::filesystem::directory_entry entry(dep.getAbsolutePath(), ec); + if (!ec) + { + const auto time = entry.last_write_time(ec); + if (!ec) + { + const auto ticks = time.time_since_epoch().count(); + if (dep.getLastWriteTime() == ticks) + { + const auto size = entry.file_size(ec); + if (!ec && size == dep.getFileSize()) + valid = true; + } + } + } + } + + if (!valid) + { + IIncludeLoader::found_t header; + if (dep.isStandardInclude()) + header = finder->getIncludeStandard(dep.getRequestingSourceDir(), std::string(dep.getIdentifier())); + else + header = finder->getIncludeRelative(dep.getRequestingSourceDir(), std::string(dep.getIdentifier())); + + if (!header || header.hash != dep.getHash()) + { + if (dep.getHasFileInfo() && !dep.getAbsolutePath().empty()) + fileStatus.emplace(dep.getAbsolutePath(), false); + else + { + std::string key; + key.reserve(dep.getIdentifier().size() + dep.getRequestingSourceDir().string().size() + 4); + key.append(dep.getRequestingSourceDir().string()); + key.push_back('|'); + key.append(dep.getIdentifier()); + key.push_back('|'); + key.push_back(dep.isStandardInclude() ? '1' : '0'); + logicalStatus.emplace(std::move(key), false); + } + return false; + } + } + + if (dep.getHasFileInfo() && !dep.getAbsolutePath().empty()) + fileStatus.emplace(dep.getAbsolutePath(), true); + else + { + std::string key; + key.reserve(dep.getIdentifier().size() + dep.getRequestingSourceDir().string().size() + 4); + key.append(dep.getRequestingSourceDir().string()); + key.push_back('|'); + key.append(dep.getIdentifier()); + key.push_back('|'); + key.push_back(dep.isStandardInclude() ? '1' : '0'); + logicalStatus.emplace(std::move(key), true); + } + } + return true; +} + +std::string IShaderCompiler::CPreprocessCache::buildCombinedCode(std::string_view body, std::string_view sourceIdentifier) const +{ + if (!m_hasEntry) + return std::string(body); + + std::string out; + size_t reserve = m_entry.preprocessedPrefix.size() + body.size(); + for (const auto& m : m_entry.macroDefs) + reserve += m.size() + 16; + for (const auto& f : m_entry.dxcFlags) + reserve += f.size() + 1; + reserve += 64; + out.reserve(reserve); + + if (!m_entry.dxcFlags.empty()) + { + out.append("#pragma dxc_compile_flags "); + for (size_t i = 0; i < m_entry.dxcFlags.size(); ++i) + { + if (i) + out.push_back(' '); + out.append(m_entry.dxcFlags[i]); + } + out.push_back('\n'); + } + + if (!m_entry.preprocessedPrefix.empty()) + { + out.append(m_entry.preprocessedPrefix); + if (out.back() != '\n') + out.push_back('\n'); + } + + for (const auto& macro : m_entry.macroDefs) + { + const auto eq = macro.find('='); + std::string_view name = eq == std::string::npos ? std::string_view(macro) : std::string_view(macro).substr(0, eq); + std::string_view def = eq == std::string::npos ? std::string_view() : std::string_view(macro).substr(eq + 1); + out.append("#define "); + out.append(name); + if (!def.empty()) + { + out.push_back(' '); + out.append(def); + } + out.push_back('\n'); + } + + if (!sourceIdentifier.empty()) + { + out.append("#line 1 \""); + out.append(normalizeLinePath(sourceIdentifier)); + out.append("\"\n"); + } + + out.append(body); + return out; +} + static void* SzAlloc(ISzAllocPtr p, size_t size) { p = p; return _NBL_ALIGNED_MALLOC(size, _NBL_SIMD_ALIGNMENT); } static void SzFree(ISzAllocPtr p, void* address) { p = p; _NBL_ALIGNED_FREE(address); } diff --git a/src/nbl/asset/utils/shaderCompiler_serialization.h b/src/nbl/asset/utils/shaderCompiler_serialization.h index 6ad33a2ff5..3f0c882a72 100644 --- a/src/nbl/asset/utils/shaderCompiler_serialization.h +++ b/src/nbl/asset/utils/shaderCompiler_serialization.h @@ -35,6 +35,7 @@ inline void to_json(json& j, const SEntry::SPreprocessorArgs& preprocArgs) j = json{ { "sourceIdentifier", preprocArgs.sourceIdentifier }, { "extraDefines", preprocArgs.extraDefines}, + { "forceIncludes", preprocArgs.forceIncludes}, }; } @@ -42,6 +43,7 @@ inline void from_json(const json& j, SEntry::SPreprocessorArgs& preprocArgs) { j.at("sourceIdentifier").get_to(preprocArgs.sourceIdentifier); j.at("extraDefines").get_to(preprocArgs.extraDefines); + j.at("forceIncludes").get_to(preprocArgs.forceIncludes); } // Optimizer pass has its own method for easier vector serialization @@ -118,6 +120,10 @@ inline void to_json(json& j, const SEntry::SPreprocessingDependency& dependency) { "identifier", dependency.identifier }, { "hash", dependency.hash.data }, { "standardInclude", dependency.standardInclude }, + { "absolutePath", dependency.absolutePath }, + { "fileSize", dependency.fileSize }, + { "lastWriteTime", dependency.lastWriteTime }, + { "hasFileInfo", dependency.hasFileInfo }, }; } @@ -127,6 +133,14 @@ inline void from_json(const json& j, SEntry::SPreprocessingDependency& dependenc j.at("identifier").get_to(dependency.identifier); j.at("hash").get_to(dependency.hash.data); j.at("standardInclude").get_to(dependency.standardInclude); + if (j.contains("absolutePath")) + j.at("absolutePath").get_to(dependency.absolutePath); + if (j.contains("fileSize")) + j.at("fileSize").get_to(dependency.fileSize); + if (j.contains("lastWriteTime")) + j.at("lastWriteTime").get_to(dependency.lastWriteTime); + if (j.contains("hasFileInfo")) + j.at("hasFileInfo").get_to(dependency.hasFileInfo); } // We serialize shader creation parameters into a json, along with indexing info into the .bin buffer where the cache is serialized @@ -193,4 +207,4 @@ inline void from_json(const json& j, SEntry& entry) } } -#endif \ No newline at end of file +#endif diff --git a/src/nbl/asset/utils/waveContext.h b/src/nbl/asset/utils/waveContext.h index f6c0014e39..1958be6109 100644 --- a/src/nbl/asset/utils/waveContext.h +++ b/src/nbl/asset/utils/waveContext.h @@ -9,6 +9,8 @@ #include #include +#include + #include "nbl/asset/utils/IShaderCompiler.h" namespace nbl::wave @@ -280,6 +282,70 @@ class context : private boost::noncopyable { macros.reset_macromap(); macros.init_predefined_macros(); } + void dump_macro_definitions(std::vector& out) const + { + out.clear(); + std::vector names; + names.reserve(std::distance(macro_names_begin(), macro_names_end())); + for (auto it = macro_names_begin(); it != macro_names_end(); ++it) + names.emplace_back(util::to_string(*it)); + std::sort(names.begin(), names.end()); + for (const auto& name : names) + { + bool has_params = false; + bool is_predefined = false; + position_type pos; + std::vector parameters; + token_sequence_type definition; + if (!get_macro_definition(name, has_params, is_predefined, pos, parameters, definition)) + continue; + if (is_predefined) + continue; + if (name.size() >= 2 && name[0] == '_' && name[1] == '_') + continue; + + std::string params_str; + if (has_params) + { + bool first_param = true; + for (const auto& tok : parameters) + { + auto tok_str = util::to_string(tok.get_value()); + if (tok_str == ",") + continue; + if (!first_param) + params_str.append(", "); + params_str.append(tok_str); + first_param = false; + } + } + + std::string def_str; + std::string prev_tok; + for (const auto& tok : definition) + { + auto tok_str = util::to_string(tok.get_value()); + if (!def_str.empty()) + { + if (!(prev_tok == "__VA_OPT__" && tok_str == "(")) + def_str.push_back(' '); + } + def_str.append(tok_str); + prev_tok = std::move(tok_str); + } + + std::string full = name; + if (has_params) + { + full.push_back('('); + full.append(params_str); + full.push_back(')'); + } + full.push_back('='); + full.append(def_str); + out.push_back(std::move(full)); + } + } // Iterate over names of defined macros typedef boost::wave::util::macromap macromap_type; @@ -527,7 +593,7 @@ template<> inline bool boost::wave::impl::pp_iterator_functor inline bool boost::wave::impl::pp_iterator_functor @@ -129,7 +130,7 @@ class {NBL_BR_API} CArchive final : public nbl::system::CFileArchive file_buffer_t getFileBuffer(const nbl::system::IFileArchive::SFileList::found_t& found) override {{ auto resource = get_resource_runtime(found->pathRelativeToArchive.string()); - return {{const_cast(resource.contents),resource.size,nullptr}}; + return {{const_cast(resource.contents),resource.size,nullptr,nbl::hlsl::uint64_t4{{resource.xx256Hash[0],resource.xx256Hash[1],resource.xx256Hash[2],resource.xx256Hash[3]}}}}; }} }}; }} @@ -143,4 +144,4 @@ class {NBL_BR_API} CArchive final : public nbl::system::CFileArchive if __name__ == "__main__": args: argparse.Namespace = parser.parse_args() - execute(args) \ No newline at end of file + execute(args) diff --git a/src/nbl/system/ISystem.cpp b/src/nbl/system/ISystem.cpp index 6b25471f8d..ca092efd9a 100644 --- a/src/nbl/system/ISystem.cpp +++ b/src/nbl/system/ISystem.cpp @@ -337,6 +337,8 @@ void ISystem::unmountBuiltins() { }; removeByKey("nbl"); + removeByKey("nbl/builtin"); + removeByKey("nbl/video"); removeByKey("spirv"); removeByKey("boost"); } @@ -394,4 +396,4 @@ bool ISystem::isDebuggerAttached() return false; } -#endif \ No newline at end of file +#endif diff --git a/tools/nsc/main.cpp b/tools/nsc/main.cpp index 5ab01d72e5..a5275acad8 100644 --- a/tools/nsc/main.cpp +++ b/tools/nsc/main.cpp @@ -9,11 +9,15 @@ #include #include #include -#include +#include #include #include +#include +#include #include #include "nbl/asset/metadata/CHLSLMetadata.h" +#include "nbl/asset/utils/shaderCompiler_serialization.h" +#include "nbl/core/hash/fnv1a64.h" #include "nlohmann/json.hpp" using json = nlohmann::json; @@ -105,6 +109,9 @@ class ShaderLogger final : public IThreadsafeLogger if (!m_file) return; + std::error_code ec; + std::filesystem::resize_file(m_logPath, 0, ec); + m_fileLogger = make_smart_refctd_ptr(smart_refctd_ptr(m_file), true, m_fileMask); } @@ -163,6 +170,7 @@ class ShaderCompiler final : public IApplicationFramework { const auto rawArgs = std::vector(argv.begin(), argv.end()); const auto expandedArgs = expandJoinedArgs(rawArgs); + m_logger = make_smart_refctd_ptr(bitflag(ILogger::ELL_ALL)); argparse::ArgumentParser program("nsc"); program.add_argument("--dump-build-info").default_value(false).implicit_value(true); @@ -178,6 +186,10 @@ class ShaderCompiler final : public IApplicationFramework program.add_argument("-nolog").default_value(false).implicit_value(true); program.add_argument("-quiet").default_value(false).implicit_value(true); program.add_argument("-verbose").default_value(false).implicit_value(true); + program.add_argument("-shader-cache").default_value(false).implicit_value(true); + program.add_argument("-shader-cache-file").default_value(std::string{}); + program.add_argument("-preprocess-cache").default_value(false).implicit_value(true); + program.add_argument("-preprocess-cache-file").default_value(std::string{}); std::vector unknownArgs; try @@ -186,36 +198,46 @@ class ShaderCompiler final : public IApplicationFramework } catch (const std::runtime_error& err) { - std::cerr << err.what() << std::endl << program; + std::ostringstream usage; + usage << program; + if (m_logger) + m_logger->log("%s\n%s", ILogger::ELL_ERROR, err.what(), usage.str().c_str()); return false; } - if (program.get("--dump-build-info")) - { - dumpBuildInfo(program); - std::exit(0); - } - if (!isAPILoaded()) { - std::cerr << "Could not load Nabla API, terminating!"; + if (m_logger) + m_logger->log("Could not load Nabla API, terminating!", ILogger::ELL_ERROR); return false; } m_system = system ? std::move(system) : IApplicationFramework::createSystem(); if (!m_system) + { + if (m_logger) + m_logger->log("Failed to create system.", ILogger::ELL_ERROR); return false; + } + + if (program.get("--dump-build-info")) + { + dumpBuildInfo(program); + std::exit(0); + } if (rawArgs.size() < 2) { - std::cerr << "Insufficient arguments.\n"; + if (m_logger) + m_logger->log("Insufficient arguments.", ILogger::ELL_ERROR); return false; } const std::string fileToCompile = rawArgs.back(); if (!m_system->exists(fileToCompile, IFileBase::ECF_READ)) { - std::cerr << "Input shader file does not exist: " << fileToCompile << "\n"; + if (m_logger) + m_logger->log("Input shader file does not exist: %s", ILogger::ELL_ERROR, fileToCompile.c_str()); return false; } @@ -226,24 +248,40 @@ class ShaderCompiler final : public IApplicationFramework if (hasFc == hasFo) { if (hasFc) - std::cerr << "Invalid arguments. Passed both -Fo and -Fc.\n"; + { + if (m_logger) + m_logger->log("Invalid arguments. Passed both -Fo and -Fc.", ILogger::ELL_ERROR); + } else - std::cerr << "Missing arguments. Expecting `-Fc {filename}` or `-Fo {filename}`.\n"; + { + if (m_logger) + m_logger->log("Missing arguments. Expecting `-Fc {filename}` or `-Fo {filename}`.", ILogger::ELL_ERROR); + } return false; } const std::string outputFilepath = hasFc ? program.get("-Fc") : program.get("-Fo"); if (outputFilepath.empty()) { - std::cerr << "Invalid output file path.\n"; + if (m_logger) + m_logger->log("Invalid output file path.", ILogger::ELL_ERROR); return false; } const bool quiet = program.get("-quiet"); const bool verbose = program.get("-verbose"); + bool shaderCacheEnabled = program.get("-shader-cache"); + const std::string shaderCachePathOverride = program.is_used("-shader-cache-file") ? program.get("-shader-cache-file") : std::string{}; + if (!shaderCachePathOverride.empty()) + shaderCacheEnabled = true; + bool preprocessCacheEnabled = program.get("-preprocess-cache"); + const std::string preprocessCachePathOverride = program.is_used("-preprocess-cache-file") ? program.get("-preprocess-cache-file") : std::string{}; + if (!preprocessCachePathOverride.empty()) + preprocessCacheEnabled = true; if (quiet && verbose) { - std::cerr << "Invalid arguments. Passed both -quiet and -verbose.\n"; + if (m_logger) + m_logger->log("Invalid arguments. Passed both -quiet and -verbose.", ILogger::ELL_ERROR); return false; } @@ -251,7 +289,8 @@ class ShaderCompiler final : public IApplicationFramework const std::string logPathOverride = program.is_used("-log") ? program.get("-log") : std::string{}; if (noLog && !logPathOverride.empty()) { - std::cerr << "Invalid arguments. Passed both -nolog and -log.\n"; + if (m_logger) + m_logger->log("Invalid arguments. Passed both -nolog and -log.", ILogger::ELL_ERROR); return false; } @@ -260,10 +299,44 @@ class ShaderCompiler final : public IApplicationFramework const auto consoleMask = bitflag(ILogger::ELL_WARNING) | ILogger::ELL_ERROR; m_logger = make_smart_refctd_ptr(m_system, logPath, fileMask, consoleMask, noLog); + const auto configName = std::filesystem::path(outputFilepath).parent_path().filename().string(); + const auto configLabel = configName.empty() ? "Unknown" : configName; m_arguments = std::move(unknownArgs); if (!m_arguments.empty() && m_arguments.back() == fileToCompile) m_arguments.pop_back(); + if (!m_arguments.empty()) + { + std::vector filteredArgs; + for (size_t i = 0; i < m_arguments.size(); ++i) + { + const auto& arg = m_arguments[i]; + if (arg == "-FI" || arg == "-include" || arg == "/FI") + { + if (i + 1 >= m_arguments.size()) + { + if (m_logger) + m_logger->log("Missing argument for %s.", ILogger::ELL_ERROR, arg.c_str()); + return false; + } + m_force_includes.push_back(m_arguments[i + 1]); + ++i; + continue; + } + if ((arg.rfind("-FI", 0) == 0 || arg.rfind("/FI", 0) == 0) && arg.size() > 3) + { + m_force_includes.push_back(arg.substr(3)); + continue; + } + if (arg.rfind("-include", 0) == 0 && arg.size() > 8) + { + m_force_includes.push_back(arg.substr(8)); + continue; + } + filteredArgs.push_back(arg); + } + m_arguments = std::move(filteredArgs); + } bool noNblBuiltins = program.get("-no-nbl-builtins"); if (noNblBuiltins) @@ -278,9 +351,19 @@ class ShaderCompiler final : public IApplicationFramework if (program.is_used("-MF")) dep.path = program.get("-MF"); if (dep.enabled && dep.path.empty()) - dep.path = outputFilepath + ".d"; - if (dep.enabled) - m_logger->log("Dependency file will be saved to %s", ILogger::ELL_INFO, dep.path.c_str()); + dep.path = outputFilepath + ".dep"; + + ShaderCacheConfig shaderCache; + shaderCache.enabled = shaderCacheEnabled && !preprocessOnly; + shaderCache.verbose = verbose; + if (shaderCache.enabled) + shaderCache.path = shaderCachePathOverride.empty() ? makeCachePath(outputFilepath) : std::filesystem::path(shaderCachePathOverride); + + PreprocessCacheConfig preCache; + preCache.enabled = preprocessCacheEnabled && !preprocessOnly; + preCache.verbose = verbose; + if (preCache.enabled) + preCache.path = preprocessCachePathOverride.empty() ? makePreprocessCachePath(outputFilepath) : std::filesystem::path(preprocessCachePathOverride); #ifndef NBL_EMBED_BUILTIN_RESOURCES if (!noNblBuiltins) @@ -303,10 +386,37 @@ class ShaderCompiler final : public IApplicationFramework m_include_search_paths.emplace_back(m_arguments[i + 1]); } + if (verbose) + { + auto join = [](const std::vector& items) + { + std::string out; + for (const auto& item : items) + { + if (!out.empty()) + out.push_back(' '); + out.append(item); + } + return out; + }; + m_logger->log("Verbose logging enabled.", ILogger::ELL_DEBUG); + m_logger->log("Variant: %s", ILogger::ELL_DEBUG, configLabel.c_str()); + if (!rawArgs.empty()) + m_logger->log("Compiler: %s", ILogger::ELL_DEBUG, rawArgs.front().c_str()); + m_logger->log("Command line: %s", ILogger::ELL_DEBUG, join(rawArgs).c_str()); + m_logger->log("Input: %s", ILogger::ELL_DEBUG, fileToCompile.c_str()); + m_logger->log("Output: %s", ILogger::ELL_DEBUG, outputFilepath.c_str()); + if (dep.enabled) + m_logger->log("Depfile: %s", ILogger::ELL_DEBUG, dep.path.c_str()); + if (shaderCache.enabled) + m_logger->log("Shader Cache: %s", ILogger::ELL_DEBUG, shaderCache.path.string().c_str()); + if (preCache.enabled) + m_logger->log("Preprocess cache: %s", ILogger::ELL_DEBUG, preCache.path.string().c_str()); + } + const char* const action = preprocessOnly ? "Preprocessing" : "Compiling"; const char* const outType = preprocessOnly ? "Preprocessed" : "Compiled"; - m_logger->log("%s %s", ILogger::ELL_INFO, action, fileToCompile.c_str()); - m_logger->log("%s shader code will be saved to %s", ILogger::ELL_INFO, outType, outputFilepath.c_str()); + m_logger->log("%s the input file.", ILogger::ELL_INFO, action); auto [shader, shaderStage] = open_shader_file(fileToCompile); if (!shader || shader->getContentType() != IShader::E_CONTENT_TYPE::ECT_HLSL) @@ -316,7 +426,8 @@ class ShaderCompiler final : public IApplicationFramework } const auto start = std::chrono::high_resolution_clock::now(); - const auto job = runShaderJob(shader.get(), shaderStage, fileToCompile, dep, preprocessOnly); + const std::string preprocessedOutputPath = outputFilepath + ".pre.hlsl"; + const auto job = runShaderJob(shader.get(), shaderStage, fileToCompile, dep, shaderCache, preCache, preprocessOnly, preprocessedOutputPath, verbose); const auto end = std::chrono::high_resolution_clock::now(); const char* const op = preprocessOnly ? "preprocessing" : "compilation"; @@ -326,9 +437,14 @@ class ShaderCompiler final : public IApplicationFramework return false; } - const auto took = std::to_string(std::chrono::duration_cast(end - start).count()); m_logger->log("Shader %s successful.", ILogger::ELL_INFO, op); - m_logger->log("Took %s ms.", ILogger::ELL_PERFORMANCE, took.c_str()); + if (dep.enabled) + { + const bool depWritten = m_system->exists(dep.path, IFileBase::ECF_READ); + if (!depWritten) + m_logger->log("Dependency file missing at %s", ILogger::ELL_WARNING, dep.path.c_str()); + m_logger->log(depWritten ? "Depfile written successfully." : "Depfile write failed.", depWritten ? ILogger::ELL_INFO : ILogger::ELL_WARNING); + } const auto outParent = std::filesystem::path(outputFilepath).parent_path(); if (!outParent.empty() && !std::filesystem::exists(outParent)) @@ -340,30 +456,14 @@ class ShaderCompiler final : public IApplicationFramework } } - std::fstream out(outputFilepath, std::ios::out | std::ios::binary); - if (!out.is_open()) - { - m_logger->log("Failed to open output file: %s", ILogger::ELL_ERROR, outputFilepath.c_str()); - return false; - } - - out.write(job.view.data(), job.view.size()); - if (out.fail()) - { - m_logger->log("Failed to write to output file: %s", ILogger::ELL_ERROR, outputFilepath.c_str()); - out.close(); - return false; - } - - out.close(); - if (out.fail()) + if (!writeBinaryFile(m_system.get(), std::filesystem::path(outputFilepath), job.view.data(), job.view.size())) { - m_logger->log("Failed to close output file: %s", ILogger::ELL_ERROR, outputFilepath.c_str()); + m_logger->log("Failed to write output file: %s", ILogger::ELL_ERROR, outputFilepath.c_str()); return false; } - if (dep.enabled) - m_logger->log("Dependency file written to %s", ILogger::ELL_INFO, dep.path.c_str()); + const auto took = std::to_string(std::chrono::duration_cast(end - start).count()); + m_logger->log("Total took: %s ms.", ILogger::ELL_PERFORMANCE, took.c_str()); return true; } @@ -378,6 +478,27 @@ class ShaderCompiler final : public IApplicationFramework std::string path; }; + struct ShaderCacheConfig + { + bool enabled = false; + bool verbose = false; + std::filesystem::path path; + }; + + struct PreprocessCacheConfig + { + bool enabled = false; + bool verbose = false; + std::filesystem::path path; + }; + + enum class CacheLoadStatus : uint8_t + { + Missing, + Invalid, + Loaded + }; + struct RunResult { bool ok = false; @@ -386,6 +507,110 @@ class ShaderCompiler final : public IApplicationFramework std::string_view view; }; + static std::filesystem::path makeCachePath(std::filesystem::path outputPath) + { + outputPath += ".ppcache"; + return outputPath; + } + + static std::filesystem::path makePreprocessCachePath(std::filesystem::path outputPath) + { + outputPath += ".ppcache.pre"; + return outputPath; + } + + static smart_refctd_ptr loadShaderCache(system::ISystem* system, const std::filesystem::path& path, CacheLoadStatus& status) + { + status = CacheLoadStatus::Missing; + if (!system) + { + status = CacheLoadStatus::Invalid; + return nullptr; + } + + if (!system->exists(path, IFileBase::ECF_READ)) + return nullptr; + + ISystem::future_t> future; + system->createFile(future, path, IFileBase::ECF_READ); + if (!future.wait()) + { + status = CacheLoadStatus::Invalid; + return nullptr; + } + + smart_refctd_ptr file; + if (auto lock = future.acquire(); lock) + lock.move_into(file); + if (!file) + { + status = CacheLoadStatus::Invalid; + return nullptr; + } + + const size_t size = file->getSize(); + if (!size) + { + status = CacheLoadStatus::Invalid; + return nullptr; + } + + std::vector data(size); + IFile::success_t succ; + file->read(succ, data.data(), 0, size); + if (!succ || succ.getBytesProcessed(true) != size) + { + status = CacheLoadStatus::Invalid; + return nullptr; + } + + auto cache = IShaderCompiler::CCache::deserialize(std::span(data.data(), data.size())); + if (!cache) + { + status = CacheLoadStatus::Invalid; + return nullptr; + } + + status = CacheLoadStatus::Loaded; + return cache; + } + + static bool writeBinaryFile(system::ISystem* system, const std::filesystem::path& path, const void* data, size_t size) + { + if (!system) + return false; + + const auto parent = path.parent_path(); + if (!parent.empty() && !std::filesystem::exists(parent)) + std::filesystem::create_directories(parent); + + system->deleteFile(path); + + ISystem::future_t> future; + system->createFile(future, path, bitflag(IFileBase::ECF_WRITE) | IFileBase::ECF_SHARE_READ_WRITE | IFileBase::ECF_SHARE_DELETE); + if (!future.wait()) + return false; + + smart_refctd_ptr file; + if (auto lock = future.acquire(); lock) + lock.move_into(file); + if (!file) + return false; + + IFile::success_t succ; + file->write(succ, data, 0, size); + return succ.getBytesProcessed(true) == size; + } + + static bool writeShaderCache(system::ISystem* system, const std::filesystem::path& path, const IShaderCompiler::CCache& cache) + { + auto buffer = cache.serialize(); + if (!buffer) + return false; + return writeBinaryFile(system, path, buffer->getPointer(), buffer->getSize()); + } + + static std::vector expandJoinedArgs(const std::vector& args) { std::vector out; @@ -414,7 +639,7 @@ class ShaderCompiler final : public IApplicationFramework return out; } - static void dumpBuildInfo(const argparse::ArgumentParser& program) + void dumpBuildInfo(const argparse::ArgumentParser& program) { json j; auto& modules = j["modules"]; @@ -451,60 +676,310 @@ class ShaderCompiler final : public IApplicationFramework oPath = filePath; } - std::ofstream outFile(oPath); - if (!outFile.is_open()) + if (!m_system) { - std::printf("Failed to open \"%s\" for writing\n", oPath.string().c_str()); + if (m_logger) + m_logger->log("Failed to create system for writing \"%s\"", ILogger::ELL_ERROR, oPath.string().c_str()); std::exit(-1); } - outFile << pretty; - std::printf("Saved \"%s\"\n", oPath.string().c_str()); + if (!writeBinaryFile(m_system.get(), oPath, pretty.data(), pretty.size())) + { + if (m_logger) + m_logger->log("Failed to write \"%s\"", ILogger::ELL_ERROR, oPath.string().c_str()); + std::exit(-1); + } + + if (m_logger) + m_logger->log("Saved \"%s\"", ILogger::ELL_INFO, oPath.string().c_str()); } - RunResult runShaderJob(const IShader* shader, hlsl::ShaderStage shaderStage, std::string_view sourceIdentifier, const DepfileConfig& dep, const bool preprocessOnly) + RunResult runShaderJob(const IShader* shader, hlsl::ShaderStage shaderStage, std::string_view sourceIdentifier, const DepfileConfig& dep, const ShaderCacheConfig& shaderCache, const PreprocessCacheConfig& preCache, const bool preprocessOnly, std::string_view preprocessedOutputPath, const bool verbose) { RunResult r; - auto hlslcompiler = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); + auto makeIncludeFinder = [&]() + { + auto finder = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); + auto loader = finder->getDefaultFileSystemLoader(); + for (const auto& p : m_include_search_paths) + finder->addSearchPath(p, loader); + return finder; + }; - auto includeFinder = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); - auto includeLoader = includeFinder->getDefaultFileSystemLoader(); - for (const auto& p : m_include_search_paths) - includeFinder->addSearchPath(p, includeLoader); + const char* codePtr = (const char*)shader->getContent()->getPointer(); + std::string_view code(codePtr, std::strlen(codePtr)); + CHLSLCompiler::SPreprocessorOptions preOpt = {}; + preOpt.sourceIdentifier = sourceIdentifier; + preOpt.logger = m_logger.get(); + preOpt.forceIncludes = std::span(m_force_includes); + preOpt.depfile = dep.enabled; + preOpt.depfilePath = dep.path; + preOpt.codeForCache = code; - if (preprocessOnly) + CHLSLCompiler::SOptions opt = {}; + opt.stage = static_cast(shaderStage); + opt.preprocessorOptions = preOpt; + opt.debugInfoFlags = bitflag(IShaderCompiler::E_DEBUG_INFO_FLAGS::EDIF_TOOL_BIT); + opt.dxcOptions = std::span(m_arguments); + + auto writeTextFile = [&](std::string_view path, std::string_view contents) -> bool + { + if (path.empty()) + return false; + return writeBinaryFile(m_system.get(), std::filesystem::path(std::string(path)), contents.data(), contents.size()); + }; + + const bool useShaderCache = shaderCache.enabled && !preprocessOnly; + const bool usePreCache = preCache.enabled && !preprocessOnly; + + struct ShaderCacheProbeResult + { + CacheLoadStatus status = CacheLoadStatus::Missing; + bool hit = false; + bool entryReady = false; + smart_refctd_ptr cacheObj; + IShaderCompiler::CCache::SEntry entry; + std::chrono::nanoseconds duration = {}; + }; + + struct PreprocessCacheProbeResult + { + bool skipped = false; + bool ok = false; + IShaderCompiler::SPreprocessCacheResult result = {}; + IShaderCompiler::CPreprocessCache::ELoadStatus loadStatus = IShaderCompiler::CPreprocessCache::ELoadStatus::Missing; + smart_refctd_ptr cacheObj; + std::chrono::nanoseconds duration = {}; + }; + + ShaderCacheProbeResult shaderProbe; + PreprocessCacheProbeResult preProbe; + using clock_t = std::chrono::high_resolution_clock; + const auto probeStart = clock_t::now(); + + if (useShaderCache) + { + const auto start = clock_t::now(); + auto finder = makeIncludeFinder(); + shaderProbe.cacheObj = loadShaderCache(m_system.get(), shaderCache.path, shaderProbe.status); + if (!shaderProbe.cacheObj) + shaderProbe.cacheObj = make_smart_refctd_ptr(); + if (shaderProbe.status == CacheLoadStatus::Loaded) + { + shaderProbe.hit = shaderProbe.cacheObj->findEntryForCode(code, opt, finder.get(), shaderProbe.entry); + shaderProbe.entryReady = shaderProbe.hit; + } + shaderProbe.duration = clock_t::now() - start; + } + + if (usePreCache) + { + if (useShaderCache && shaderProbe.hit) + { + preProbe.skipped = true; + preProbe.ok = true; + preProbe.duration = {}; + } + else + { + const auto start = clock_t::now(); + auto finder = makeIncludeFinder(); + preProbe.cacheObj = IShaderCompiler::CPreprocessCache::loadFromFile(preCache.path, preProbe.loadStatus); + if (!preProbe.cacheObj) + preProbe.cacheObj = make_smart_refctd_ptr(); + + auto localCompiler = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); + CHLSLCompiler::SPreprocessorOptions preOptThread = preOpt; + preOptThread.includeFinder = finder.get(); + IShader::E_SHADER_STAGE stageOverrideThread = static_cast(shaderStage); + preProbe.result = localCompiler->preprocessWithCache(code, stageOverrideThread, preOptThread, *preProbe.cacheObj, preProbe.loadStatus, sourceIdentifier); + preProbe.ok = preProbe.result.ok; + preProbe.duration = clock_t::now() - start; + } + } + + const auto probeEnd = clock_t::now(); + + std::string preprocessedCode; + bool preprocessedReady = false; + std::string_view codeToCompile = code; + smart_refctd_ptr preCacheObj; + IShader::E_SHADER_STAGE stageOverride = static_cast(shaderStage); + auto cacheMissReason = [](CacheLoadStatus status) -> const char* + { + if (status == CacheLoadStatus::Missing) + return "cache file missing; first build, cleaned, output moved, or out of date"; + if (status == CacheLoadStatus::Invalid) + return "cache file invalid or version mismatch"; + return "input/deps/options changed; cache invalidated"; + }; + + auto toMs = [](const std::chrono::nanoseconds duration) -> long long + { + return std::chrono::duration_cast(duration).count(); + }; + + auto writeDepfileFromDependencies = [&](const IShaderCompiler::CCache::SEntry::dependency_container_t& dependencies) -> bool + { + if (!dep.enabled) + return true; + if (preOpt.depfilePath.empty()) + { + m_logger->log("Depfile path is empty.", ILogger::ELL_ERROR); + return false; + } + IShaderCompiler::DepfileWriteParams params = {}; + const std::string depfilePathString = preOpt.depfilePath.generic_string(); + params.depfilePath = depfilePathString; + params.sourceIdentifier = preOpt.sourceIdentifier; + if (!params.sourceIdentifier.empty()) + params.workingDirectory = std::filesystem::path(std::string(params.sourceIdentifier)).parent_path(); + params.system = m_system.get(); + return IShaderCompiler::writeDepfile(params, dependencies, nullptr, preOpt.logger); + }; + + if (verbose && (useShaderCache || usePreCache)) + { + if (useShaderCache) + m_logger->log("Shader cache lookup took: %lld ms.", ILogger::ELL_PERFORMANCE, static_cast(toMs(shaderProbe.duration))); + if (usePreCache) + m_logger->log("Preprocess cache lookup took: %lld ms.", ILogger::ELL_PERFORMANCE, static_cast(toMs(preProbe.duration))); + m_logger->log("Total cache probe took: %lld ms.", ILogger::ELL_PERFORMANCE, static_cast(toMs(std::chrono::duration_cast(probeEnd - probeStart)))); + } + + smart_refctd_ptr cacheObj = shaderProbe.cacheObj; + CacheLoadStatus cacheStatus = shaderProbe.status; + const bool shaderCacheHitExpected = shaderProbe.hit; + + if (usePreCache && preCache.verbose && useShaderCache) + { + if (shaderCacheHitExpected) + m_logger->log("Cache hit! Preprocess cache skipped.", ILogger::ELL_DEBUG); + else + m_logger->log("Cache miss! Cold run (%s). Checking preprocess cache.", ILogger::ELL_DEBUG, cacheMissReason(cacheStatus)); + } + + if (usePreCache && !shaderCacheHitExpected) + { + if (!preProbe.ok) + return r; + if (preCache.verbose) + { + if (preProbe.result.cacheHit) + m_logger->log("Preprocess cache hit!", ILogger::ELL_DEBUG); + else + m_logger->log("Preprocess cache miss! Cold run (%s).", ILogger::ELL_DEBUG, IShaderCompiler::CPreprocessCache::getProbeReason(preProbe.result.status)); + } + if (preProbe.result.cacheUsed) + { + preprocessedCode = std::move(preProbe.result.code); + preprocessedReady = true; + stageOverride = preProbe.result.stage; + preCacheObj = preProbe.cacheObj; + if (!preprocessedOutputPath.empty() && !writeTextFile(preprocessedOutputPath, preprocessedCode)) + return r; + } + } + else if (usePreCache && preCache.verbose) { - CHLSLCompiler::SPreprocessorOptions opt = {}; - opt.sourceIdentifier = sourceIdentifier; - opt.logger = m_logger.get(); - opt.includeFinder = includeFinder.get(); - opt.depfile = dep.enabled; - opt.depfilePath = dep.path; + if (preProbe.skipped) + { + m_logger->log("Preprocess cache lookup skipped (shader cache hit).", ILogger::ELL_DEBUG); + } + else if (preProbe.ok) + { + if (preProbe.result.cacheHit) + m_logger->log("Preprocess cache hit (ignored, shader cache hit).", ILogger::ELL_DEBUG); + else + m_logger->log("Preprocess cache miss! Cold run (%s). (ignored, shader cache hit).", ILogger::ELL_DEBUG, IShaderCompiler::CPreprocessCache::getProbeReason(preProbe.result.status)); + } + else + { + m_logger->log("Preprocess cache failed (ignored, shader cache hit).", ILogger::ELL_DEBUG); + } + } - const char* codePtr = (const char*)shader->getContent()->getPointer(); - std::string_view code(codePtr, std::strlen(codePtr)); + if (usePreCache && preProbe.result.cacheUpdated && preProbe.cacheObj) + IShaderCompiler::CPreprocessCache::writeToFile(preCache.path, *preProbe.cacheObj); - r.text = hlslcompiler->preprocessShader(std::string(code), shaderStage, opt, nullptr); + if (useShaderCache && shaderProbe.hit && shaderProbe.entryReady) + { + if (verbose) + m_logger->log("Shader cache hit: using cached SPIR-V.", ILogger::ELL_DEBUG); + r.compiled = cacheObj->decompressEntry(shaderProbe.entry); + r.ok = bool(r.compiled); + if (!r.ok) + return r; + r.view = { (const char*)r.compiled->getContent()->getPointer(), r.compiled->getContent()->getSize() }; + + if (!writeDepfileFromDependencies(shaderProbe.entry.dependencies)) + return r; + + return r; + } + + auto hlslcompiler = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); + + if (preprocessOnly) + { + const auto preprocessStart = std::chrono::high_resolution_clock::now(); + auto finder = makeIncludeFinder(); + preOpt.includeFinder = finder.get(); + r.text = hlslcompiler->preprocessShader(std::string(code), shaderStage, preOpt, nullptr); r.ok = !r.text.empty(); r.view = r.text; + const auto preprocessEnd = std::chrono::high_resolution_clock::now(); + if (verbose) + { + const auto duration = std::chrono::duration_cast(preprocessEnd - preprocessStart).count(); + m_logger->log("Preprocess took: %lld ms.", ILogger::ELL_PERFORMANCE, static_cast(duration)); + } return r; } - CHLSLCompiler::SOptions opt = {}; - opt.stage = shaderStage; - opt.preprocessorOptions.sourceIdentifier = sourceIdentifier; - opt.preprocessorOptions.logger = m_logger.get(); - opt.preprocessorOptions.includeFinder = includeFinder.get(); - opt.preprocessorOptions.depfile = dep.enabled; - opt.preprocessorOptions.depfilePath = dep.path; - opt.debugInfoFlags = bitflag(IShaderCompiler::E_DEBUG_INFO_FLAGS::EDIF_TOOL_BIT); - opt.dxcOptions = std::span(m_arguments); + opt.stage = stageOverride; + + bool cacheHit = false; + if (shaderCache.enabled && cacheObj) + { + opt.readCache = cacheObj.get(); + opt.writeCache = cacheObj.get(); + opt.cacheHit = &cacheHit; + } + + if (preprocessedReady) + { + opt.preprocessorOptions.applyForceIncludes = false; + if (preCacheObj && preCacheObj->hasEntry()) + opt.dependencyOverrides = &preCacheObj->getEntry().dependencies; + codeToCompile = preprocessedCode; + } - r.compiled = hlslcompiler->compileToSPIRV((const char*)shader->getContent()->getPointer(), opt); + auto compileFinder = makeIncludeFinder(); + opt.preprocessorOptions.includeFinder = compileFinder.get(); + r.compiled = hlslcompiler->compileToSPIRV(codeToCompile, opt); r.ok = bool(r.compiled); if (r.ok) r.view = { (const char*)r.compiled->getContent()->getPointer(), r.compiled->getContent()->getSize() }; + if (shaderCache.enabled && cacheObj) + { + const bool logShaderCache = verbose && !usePreCache; + if (logShaderCache) + { + if (cacheHit) + { + m_logger->log("Cache hit!", ILogger::ELL_DEBUG); + } + else + { + m_logger->log("Cache miss! Cold run (%s).", ILogger::ELL_DEBUG, cacheMissReason(cacheStatus)); + } + } + if (!writeShaderCache(m_system.get(), shaderCache.path, *cacheObj)) + m_logger->log("Failed to write shader cache: %s", ILogger::ELL_WARNING, shaderCache.path.string().c_str()); + } + return r; } @@ -547,7 +1022,7 @@ class ShaderCompiler final : public IApplicationFramework smart_refctd_ptr m_system; smart_refctd_ptr m_logger; - std::vector m_arguments, m_include_search_paths; + std::vector m_arguments, m_include_search_paths, m_force_includes; smart_refctd_ptr m_assetMgr; };