From 65d34b1e9a6ec35c4521262332d02fb38c813842 Mon Sep 17 00:00:00 2001 From: Rebecca Wallander Date: Sat, 18 Oct 2025 22:04:15 +0200 Subject: [PATCH 1/6] Add flux implementation --- CMakeLists.txt | 3 +- docs/spec/blocks/datastream_payload.adoc | 77 ++ docs/spec/blocks/flux.adoc | 67 +- docs/spec/spec.adoc | 4 + docs/spec/version_history.adoc | 2 + include/aaruformat/context.h | 10 + include/aaruformat/decls.h | 12 + include/aaruformat/enums.h | 8 +- include/aaruformat/errors.h | 1 + include/aaruformat/structs.h | 1 + include/aaruformat/structs/flux.h | 289 ++++++ include/internal.h | 2 + src/blocks/flux.c | 1111 ++++++++++++++++++++++ src/close.c | 674 +++++++++++++ src/create.c | 1 + src/open.c | 4 + templates/aaruformat.bt | 4 +- templates/aaruformat.hexpat | 53 +- tool/convert.c | 140 +++ tool/info.c | 4 + 20 files changed, 2446 insertions(+), 21 deletions(-) create mode 100644 docs/spec/blocks/datastream_payload.adoc create mode 100644 include/aaruformat/structs/flux.h create mode 100644 src/blocks/flux.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 61dabcb..d671a4e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -234,7 +234,8 @@ add_library(aaruformat SHARED src/metadata.c src/dump.c include/aaruformat/structs/tape.h - src/blocks/tape.c) + src/blocks/tape.c + src/blocks/flux.c) # Set up include directories for the target target_include_directories(aaruformat diff --git a/docs/spec/blocks/datastream_payload.adoc b/docs/spec/blocks/datastream_payload.adoc new file mode 100644 index 0000000..c8767c1 --- /dev/null +++ b/docs/spec/blocks/datastream_payload.adoc @@ -0,0 +1,77 @@ +=== Data Stream Payload Block (`DSPL`) + +This block contains a generic data stream payload. Can be used for data streams such as bitstreams or flux data. +Each `DataStreamPayloadBlock` stores a compressed or uncompressed stream of binary data of variable length. +The block may be compressed using LZMA compression if compression is enabled for the image. + +For flux captures, the payload data consists of: +* Data buffer: Raw flux transition timing data +* Index buffer: Index structure mapping logical positions to offsets within the data buffer + +These two buffers are concatenated: `[data_buffer][index_buffer]` before compression and storage. +For other data types, the payload layout is specific to the data type. + +==== Structure Definition + +[source,c] +#define DATA_STREAM_PAYLOAD_MAGIC 0x4C505344 +/** Data stream payload block header */ +typedef struct DataStreamPayloadHeader +{ + uint32_t identifier; ///< Block identifier, must be BlockType::DataStreamPayloadBlock (0x4C505344). + uint16_t compression; ///< Compression type (0 = None, 1 = Lzma). + uint32_t cmpLength; ///< Compressed length in bytes (includes LZMA properties if compressed). + uint32_t length; ///< Uncompressed length in bytes. + uint64_t cmpCrc64; ///< CRC64-ECMA checksum of the compressed payload (or same as crc64 if uncompressed). + uint64_t crc64; ///< CRC64-ECMA checksum of the uncompressed payload. +} DataStreamPayloadHeader; + +==== Field Descriptions + +[cols="2,2,2,6",options="header"] +|=== +|Type +|Size +|Name +|Description + +|uint32_t +|4 bytes +|identifier +|The data stream payload block identifier, always `DSPL` (`0x4C505344`) + +|uint16_t +|2 bytes +|compression +|Compression type: 0 = None, 1 = Lzma + +|uint32_t +|4 bytes +|cmpLength +|Compressed length in bytes (includes LZMA properties if compression = Lzma) + +|uint32_t +|4 bytes +|length +|Uncompressed length in bytes (for flux captures: total of data buffer + index buffer) + +|uint64_t +|8 bytes +|cmpCrc64 +|CRC64-ECMA checksum of the compressed payload data + +|uint64_t +|8 bytes +|crc64 +|CRC64-ECMA checksum of the uncompressed payload data (for flux captures: data + index buffers) +|=== + +==== Payload Data + +The payload data immediately follows the `DataStreamPayloadHeader`. If compression is Lzma, the first 5 bytes contain LZMA properties, followed by the compressed data. If compression is None, the payload data is stored uncompressed. + +For flux captures, the uncompressed payload consists of: +* Data buffer: `length - indexOffset` bytes of flux transition timing data +* Index buffer: `indexOffset` bytes of index structure data + +The `indexOffset` value from the corresponding `FluxEntry` indicates where the index buffer starts within the payload. For other data types, the payload layout is specific to the data type. \ No newline at end of file diff --git a/docs/spec/blocks/flux.adoc b/docs/spec/blocks/flux.adoc index 7bcdaff..8d0aab3 100644 --- a/docs/spec/blocks/flux.adoc +++ b/docs/spec/blocks/flux.adoc @@ -1,6 +1,6 @@ === Flux Data Block (`FLUX`) -This block lists all known flux captures. +This block contains metadata for all flux captures in the image. Certain hardware devices, such as Kryoflux, Pauline, and Applesauce, read magnetic media at the flux transition level. Flux transition reads are digital representations of the analog properties of the media, and cannot be reliably interpreted on a sector-by-sector basis without further processing. @@ -14,13 +14,24 @@ Flux data is represented as an array of `uint8_t` bytes. Each byte stores the tick count since the last flux transition. If no transition is detected within a byte's range, the value `0xFF` is used, and counting resumes in the next byte with ticks accumulated. -Flux data is stored in `DataBlocks` of the flux data type, referenced from a deduplication table of the same type. -Only one flux-type deduplication table is allowed per image, and it must have exactly one level. +The flux capture system uses two block types: +* `FluxDataBlock` (`FLUX` / `0x58554C46`): Contains metadata entries that describe all flux captures in the image +* `DataStreamPayloadBlock` (`DSPL` / `0x4C505344`): Contains the actual flux data payload (data and index buffers) for individual captures. This is a generic block type that can also be used for other data streams. + +Each flux capture has one entry in the `FluxDataBlock` and one corresponding `DataStreamPayloadBlock` containing its data. +The `FluxEntry` structure in the data block contains a `payloadOffset` field that points to the file offset where the corresponding `DataStreamPayloadBlock` is stored. ==== Structure Definition [source,c] -/* Undefined */ +#define FLUX_DATA_MAGIC 0x58554C46 +/** Flux data block header */ +typedef struct FluxHeader +{ + uint32_t identifier; ///< Block identifier, must be BlockType::FluxDataBlock (0x58554C46). + uint16_t entries; ///< Number of FluxEntry records following this header. + uint64_t crc64; ///< CRC64-ECMA checksum of the FluxEntry array (header excluded). +} FluxHeader; ==== Field Descriptions @@ -34,23 +45,34 @@ Only one flux-type deduplication table is allowed per image, and it must have ex |uint32_t |4 bytes |identifier -|The flux data block identifier, always `FLUX` +|The flux data block identifier, always `FLUX` (`0x58554C46`) |uint16_t |2 bytes |entries -|The number of entries following this header +|The number of flux entry records following this header |uint64_t |8 bytes |crc64 -|The CRC64-ECMA checksum of the data following this header +|The CRC64-ECMA checksum of the FluxEntry array following this header |=== ==== Flux entries [source,c] -/* Undefined */ +/** Flux capture entry, describes one flux capture in the image */ +typedef struct FluxEntry +{ + uint32_t head; ///< Head number the capture corresponds to. + uint16_t track; ///< Track number the capture corresponds to. + uint8_t subtrack; ///< Subtrack number the capture corresponds to. + uint32_t captureIndex; ///< Capture index, allows multiple captures for the same location. + uint64_t indexResolution; ///< Resolution in picoseconds for the index stream. + uint64_t dataResolution; ///< Resolution in picoseconds for the data stream. + uint64_t indexOffset; ///< Byte offset within the payload where the index buffer starts. + uint64_t payloadOffset; ///< File offset where the DataStreamPayloadBlock for this capture is stored. +} FluxEntry; ==== Field Descriptions @@ -64,25 +86,40 @@ Only one flux-type deduplication table is allowed per image, and it must have ex |uint32_t |4 bytes |head -|Head the data corresponds to. +|Head number the flux capture corresponds to. |uint16_t |2 bytes |track -|Track the data corresponds to. +|Track number the flux capture corresponds to. |uint8_t |1 byte |subtrack -|Substep of a track that the data corresponds to. +|Subtrack number the flux capture corresponds to. + +|uint32_t +|4 bytes +|captureIndex +|Capture index, allows multiple captures for the same head/track/subtrack combination. + +|uint64_t +|8 bytes +|indexResolution +|Resolution in picoseconds at which the index stream was sampled. + +|uint64_t +|8 bytes +|dataResolution +|Resolution in picoseconds at which the data stream was sampled. |uint64_t |8 bytes -|resolution -|Number of picoseconds at which the sampling was performed. +|indexOffset +|Byte offset within the payload block where the index buffer starts (equals data_length). |uint64_t |8 bytes -|tableEntry -|Entry number in the deduplication table where the data corresponding to this flux entry is stored +|payloadOffset +|File offset where the DataStreamPayloadBlock containing this capture's data is stored. |=== \ No newline at end of file diff --git a/docs/spec/spec.adoc b/docs/spec/spec.adoc index d1c1177..d89850a 100644 --- a/docs/spec/spec.adoc +++ b/docs/spec/spec.adoc @@ -118,6 +118,10 @@ include::blocks/flux.adoc[] <<< +include::blocks/datastream_payload.adoc[] + +<<< + include::blocks/bitstream.adoc[] <<< diff --git a/docs/spec/version_history.adoc b/docs/spec/version_history.adoc index 8701ecb..8a6f1bb 100644 --- a/docs/spec/version_history.adoc +++ b/docs/spec/version_history.adoc @@ -98,4 +98,6 @@ Clarify deduplication table sector status. Add encrypted and decrypted sector status. Add Aaru Metadata JSON. + +Rework the flux workflow. |=== diff --git a/include/aaruformat/context.h b/include/aaruformat/context.h index f97dc07..db099bd 100644 --- a/include/aaruformat/context.h +++ b/include/aaruformat/context.h @@ -21,6 +21,7 @@ #include "blake3.h" #include "crc64.h" +#include "structs/flux.h" #include "hash_map.h" #include "lru.h" #include "md5.h" @@ -30,6 +31,8 @@ #include "structs.h" #include "utarray.h" +typedef struct FluxCaptureMapEntry FluxCaptureMapEntry; + /** \file aaruformat/context.h * \brief Central runtime context structures for libaaruformat (image state, caches, checksum buffers). * @@ -304,6 +307,12 @@ typedef struct aaruformat_context TapePartitionHashEntry *tape_partitions; ///< Hash table root for tape partitions bool is_tape; ///< True if the image is a tape image + /* Flux data structures */ + FluxHeader flux_data_header; ///< Flux data header (if present). + FluxEntry *flux_entries; ///< Array of flux entries (flux_data_header.entries elements). + UT_array *flux_captures; ///< Pending flux capture payloads (write path). + FluxCaptureMapEntry *flux_map; ///< Hash map for flux capture lookup by head/track/subtrack/capture index. + /* Dirty flags (controls write behavior in close.c) */ bool dirty_secondary_ddt; ///< True if secondary DDT tables should be written during close bool dirty_primary_ddt; ///< True if primary DDT table should be written during close @@ -327,6 +336,7 @@ typedef struct aaruformat_context bool dirty_dumphw_block; ///< True if dump hardware block should be written during close bool dirty_cicm_block; ///< True if CICM metadata block should be written during close bool dirty_json_block; ///< True if JSON metadata block should be written during close + bool dirty_flux_block; ///< True if flux block should be written during close bool dirty_index_block; ///< True if index block should be written during close } aaruformat_context; diff --git a/include/aaruformat/decls.h b/include/aaruformat/decls.h index 51eb86e..0c0abfa 100644 --- a/include/aaruformat/decls.h +++ b/include/aaruformat/decls.h @@ -21,6 +21,7 @@ #include "aaru.h" #include "crc64.h" +#include "enums.h" #include "md5.h" #include "sha1.h" #include "sha256.h" @@ -88,6 +89,17 @@ AARU_EXPORT uint64_t AARU_CALL aaruf_crc64_data(const uint8_t *data, uint32_t AARU_EXPORT int32_t AARU_CALL aaruf_get_tracks(const void *context, uint8_t *buffer, size_t *length); AARU_EXPORT int32_t AARU_CALL aaruf_set_tracks(void *context, TrackEntry *tracks, const int count); +AARU_EXPORT int32_t AARU_CALL aaruf_get_flux_captures(void *context, uint8_t *buffer, size_t *length); +AARU_EXPORT int32_t AARU_CALL aaruf_read_flux_capture(void *context, uint32_t head, uint16_t track, uint8_t subtrack, + uint32_t capture_index, uint8_t *index_data, + uint32_t *index_length, uint8_t *data_data, + uint32_t *data_length); +AARU_EXPORT int32_t AARU_CALL aaruf_write_flux_capture(void *context, uint32_t head, uint16_t track, uint8_t subtrack, + uint32_t capture_index, uint64_t data_resolution, + uint64_t index_resolution, const uint8_t *data, + uint32_t data_length, const uint8_t *index, + uint32_t index_length); +AARU_EXPORT int32_t AARU_CALL aaruf_clear_flux_captures(void *context); AARU_EXPORT int32_t AARU_CALL aaruf_read_sector(void *context, uint64_t sector_address, bool negative, uint8_t *data, uint32_t *length, uint8_t *sector_status); diff --git a/include/aaruformat/enums.h b/include/aaruformat/enums.h index e439145..edf5876 100644 --- a/include/aaruformat/enums.h +++ b/include/aaruformat/enums.h @@ -130,7 +130,9 @@ typedef enum DvdSectorEdc = 85, ///< DVD Error Detection Code (EDC) DvdSectorEccPi = 86, ///< DVD Error Correction Code (ECC) Parity of Inner Code (PI) DvdEccBlockPo = 87, ///< DVD Error Correction Code (ECC) Parity of Outer Code (PO) - DvdPfi2ndLayer = 88 ///< DVD Physical Format Information for the second layer + DvdPfi2ndLayer = 88, ///< DVD Physical Format Information for the second layer + FluxData = 89, ///< Flux data. + BitstreamData = 90 ///< Bitstream data. } DataType; /** @@ -158,7 +160,9 @@ typedef enum DumpHardwareBlock = 0x2A504D44, ///< Block containing an array of hardware used to create the image. TapeFileBlock = 0x454C4654, ///< Block containing list of files for a tape image. TapePartitionBlock = 0x54425054, ///< Block containing list of partitions for a tape image. - AaruMetadataJsonBlock = 0x444D534A ///< Block containing JSON version of Aaru Metadata + AaruMetadataJsonBlock = 0x444D534A, ///< Block containing JSON version of Aaru Metadata + FluxDataBlock = 0x58554C46, ///< Block containing flux data metadata. + DataStreamPayloadBlock = 0x4C505344 ///< Block containing compressed data stream payload (e.g., flux data, bitstreams). } BlockType; /** diff --git a/include/aaruformat/errors.h b/include/aaruformat/errors.h index bf9d6a0..04ca8c9 100644 --- a/include/aaruformat/errors.h +++ b/include/aaruformat/errors.h @@ -68,6 +68,7 @@ #define AARUF_ERROR_TAPE_PARTITION_NOT_FOUND (-29) ///< Requested tape partition not present in image. #define AARUF_ERROR_METADATA_NOT_PRESENT (-30) ///< Requested metadata not present in image. #define AARUF_ERROR_INVALID_SECTOR_LENGTH (-31) ///< Sector length is too big. +#define AARUF_ERROR_FLUX_DATA_NOT_FOUND (-32) ///< Requested flux data not present in image. /** @} */ /** \name Non-fatal sector status codes (non-negative) diff --git a/include/aaruformat/structs.h b/include/aaruformat/structs.h index b302790..2af1786 100644 --- a/include/aaruformat/structs.h +++ b/include/aaruformat/structs.h @@ -39,6 +39,7 @@ #include "structs/optical.h" #include "structs/options.h" #include "structs/tape.h" +#include "structs/flux.h" #endif // LIBAARUFORMAT_STRUCTS_H diff --git a/include/aaruformat/structs/flux.h b/include/aaruformat/structs/flux.h new file mode 100644 index 0000000..61befd9 --- /dev/null +++ b/include/aaruformat/structs/flux.h @@ -0,0 +1,289 @@ +/* + * This file is part of the Aaru Data Preservation Suite. + * Copyright (c) 2019-2025 Natalia Portillo. + * + * This library is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +/** + * @file flux.h + * @brief Data structures for flux transition capture support in Aaru disk images. + * + * This header defines structures used to represent flux transition data captured + * from magnetic media. Flux transitions are the raw analog signals read from + * magnetic storage devices (such as floppy disks, hard drives, and tape) before + * they are interpreted into digital data. + * + * **Flux Capture Overview:** + * Certain hardware devices, such as Kryoflux, Pauline, and Applesauce, read + * magnetic media at the flux transition level rather than at the sector level. + * This provides a more complete representation of the analog properties of the + * media, allowing for advanced recovery techniques and preservation of media + * characteristics that would be lost in sector-level imaging. + * + * **Block Structure:** + * Flux data is stored in two block types: + * - FluxDataBlock: Contains metadata entries describing all flux captures in + * the image. Each entry includes location information (head, track, subtrack), + * capture index, resolution data, and file offsets to the payload blocks. + * - DataStreamPayloadBlock: Contains the actual flux data and index buffers for a + * single capture. Payload blocks may be compressed using LZMA compression. + * + * **Data Organization:** + * Each flux capture includes two data streams: + * - Data buffer: Raw flux transition timing data, represented as an array of + * uint8_t bytes where each byte stores the tick count since the last flux + * transition. The value 0xFF indicates no transition within the byte range. + * - Index buffer: Index structure mapping logical positions to offsets within + * the data buffer, enabling efficient access to specific regions of the flux + * data. + * + * **Capture Identification:** + * Flux captures are uniquely identified by a tuple of (head, track, subtrack, + * captureIndex). This allows multiple captures for the same physical location, + * which is useful for capturing multiple revolutions of a floppy disk track or + * different read attempts. + * + * **Resolution:** + * Each capture specifies two resolution values: + * - dataResolution: The sampling resolution in picoseconds for the data stream + * - indexResolution: The sampling resolution in picoseconds for the index stream + * + * These resolutions determine the precision of the timing measurements and are + * hardware-dependent. + * + * **Storage Efficiency:** + * Payload blocks support LZMA compression to reduce storage requirements. The + * compression is transparent to the API user - aaruf_read_flux_capture() handles + * decompression automatically. + * + * **Use Cases:** + * - Preservation of magnetic media at the lowest level possible + * - Recovery of data from damaged or degraded media + * - Analysis of media characteristics and timing variations + * - Emulation and accurate reproduction of original media behavior + * - Forensic imaging where complete bit-level accuracy is required + * + * @note All structures in this file use packed alignment (#pragma pack(push, 1)) + * to ensure consistent on-disk layout across different compilers and platforms. + * + * @see BlockType for the identifier constants used in FluxHeader and DataStreamPayloadHeader + * @see aaruf_get_flux_captures() to retrieve metadata for all captures + * @see aaruf_read_flux_capture() to read actual flux data + * @see aaruf_write_flux_capture() to add flux captures during write mode + */ + +#ifndef LIBAARUFORMAT_FLUX_H +#define LIBAARUFORMAT_FLUX_H + +#include + +#pragma pack(push, 1) + +/** + * @struct FluxHeader + * @brief Header structure for a FluxDataBlock containing flux capture metadata. + * + * This structure is the header of a FluxDataBlock, which lists all flux captures + * in the image. The block contains this header followed by an array of FluxEntry + * structures, one for each flux capture. + * + * The header includes a CRC64 checksum computed over the FluxEntry array to ensure + * data integrity. The identifier field must match BlockType::FluxDataBlock + * (0x58554C46, "FLUX" in ASCII). + * + * @note Only one FluxDataBlock is allowed per image. + * @note The entries field is limited to UINT16_MAX (65535) captures per image. + */ +typedef struct FluxHeader +{ + uint32_t identifier; ///< Block identifier, must be BlockType::FluxDataBlock (0x58554C46, "FLUX"). + uint16_t entries; ///< Number of FluxEntry records following this header. Maximum value: 65535. + uint64_t crc64; ///< CRC64-ECMA checksum of the FluxEntry array (header excluded). +} FluxHeader; + +/** + * @struct FluxEntry + * @brief Metadata entry describing a single flux capture in the FluxDataBlock. + * + * This structure describes one flux capture, including its location on the media, + * capture index, sampling resolutions, and file offsets to the payload data. Each + * FluxEntry corresponds to one DataStreamPayloadBlock containing the actual flux data. + * + * **Location Identification:** + * The head, track, and subtrack fields identify the physical location on the media + * where the capture was taken. The captureIndex allows multiple captures for the + * same location (e.g., multiple revolutions of a floppy disk track). + * + * **Payload Access:** + * The payloadOffset field points to the file offset where the corresponding + * DataStreamPayloadBlock is stored. The indexOffset field indicates where the index + * buffer starts within the payload (the payload is stored as [data_buffer][index_buffer] + * concatenated). + * + * **Resolution:** + * Both indexResolution and dataResolution are specified in picoseconds, indicating + * the precision of the timing measurements. These values are hardware-dependent and + * determine how accurately the flux transitions are represented. + * + * @note The combination of (head, track, subtrack, captureIndex) uniquely identifies + * a flux capture within an image. + * @note The indexOffset equals the data_length, as the index buffer immediately + * follows the data buffer in the payload. + */ +typedef struct FluxEntry +{ + uint32_t head; ///< Head number the flux capture corresponds to. Typically 0 or 1 for double-sided media. + uint16_t track; ///< Track number the flux capture corresponds to. Track numbering is format-dependent. + uint8_t subtrack; ///< Subtrack number, allowing sub-stepping within a track. Used for fine positioning. + uint32_t captureIndex; ///< Capture index, allowing multiple captures for the same location (e.g., multiple revolutions). + uint64_t indexResolution; ///< Resolution in picoseconds at which the index stream was sampled. + uint64_t dataResolution; ///< Resolution in picoseconds at which the data stream was sampled. + uint64_t indexOffset; ///< Byte offset within the payload where the index buffer starts (equals data_length). + uint64_t payloadOffset; ///< File offset where the DataStreamPayloadBlock containing this capture's data is stored. +} FluxEntry; + +/** + * @struct FluxCaptureMeta + * @brief Metadata structure returned by aaruf_get_flux_captures(). + * + * This structure contains the public metadata for a flux capture, excluding + * internal file offsets. It is used when retrieving metadata for all captures + * in an image without needing to access the actual flux data. + * + * The structure contains the same location and resolution information as FluxEntry, + * but omits the indexOffset and payloadOffset fields which are implementation + * details not needed by API users. + * + * @see aaruf_get_flux_captures() to retrieve metadata for all captures + * @see FluxEntry for the complete structure including file offsets + */ +typedef struct FluxCaptureMeta +{ + uint32_t head; ///< Head number the flux capture corresponds to. + uint16_t track; ///< Track number the flux capture corresponds to. + uint8_t subtrack; ///< Subtrack number the flux capture corresponds to. + uint32_t captureIndex; ///< Capture index, allowing multiple captures for the same location. + uint64_t indexResolution; ///< Resolution in picoseconds at which the index stream was sampled. + uint64_t dataResolution; ///< Resolution in picoseconds at which the data stream was sampled. +} FluxCaptureMeta; + +/** + * @struct FluxCaptureMapEntry + * @brief Internal hash table entry for flux capture lookup. + * + * This structure is used internally by the library to provide O(1) lookup of + * flux captures by their identifier tuple. It maps a FluxCaptureKey to an index + * in the flux_entries array. + * + * @note This structure is opaque to API users and is only used internally. + * @internal + */ +typedef struct FluxCaptureMapEntry FluxCaptureMapEntry; + +/** + * @struct DataStreamPayloadHeader + * @brief Header structure for a DataStreamPayloadBlock containing data stream payload. + * + * This structure is the header of a DataStreamPayloadBlock, which contains a generic + * compressed data stream payload. The payload data immediately follows this header and + * may be compressed using LZMA compression. Currently used for flux capture data, but + * can be used for other data streams such as bitstreams or PNG data. + * + * **Compression:** + * The compression field indicates whether the payload is compressed: + * - 0 (None): Payload is stored uncompressed + * - 1 (Lzma): Payload is compressed using LZMA, with LZMA properties stored + * in the first 5 bytes of the compressed data + * + * **Checksums:** + * Two CRC64 checksums are stored: + * - cmpCrc64: Checksum of the compressed payload (or same as crc64 if uncompressed) + * - crc64: Checksum of the uncompressed payload + * + * Both checksums are validated when reading the payload to ensure data integrity. + * + * **Payload Layout:** + * The uncompressed payload is an arbitrary stream of binary data. For flux captures, + * this consists of concatenated data and index buffers: [data_buffer][index_buffer], + * with the indexOffset from the corresponding FluxEntry indicating where the index + * buffer starts. For other data types, the layout is specific to the data type. + * + * @note The identifier field must match BlockType::DataStreamPayloadBlock (0x4C505344, "DSPL"). + * @note If compression is Lzma, cmpLength includes the 5-byte LZMA properties header. + */ +typedef struct DataStreamPayloadHeader +{ + uint32_t identifier; ///< Block identifier, must be BlockType::DataStreamPayloadBlock (0x4C505344, "DSPL"). + uint16_t compression; ///< Compression type: 0 = None, 1 = Lzma. + uint32_t cmpLength; ///< Compressed length in bytes (includes LZMA properties if compression = Lzma). + uint32_t length; ///< Uncompressed length in bytes. + uint64_t cmpCrc64; ///< CRC64-ECMA checksum of the compressed payload (or same as crc64 if uncompressed). + uint64_t crc64; ///< CRC64-ECMA checksum of the uncompressed payload. +} DataStreamPayloadHeader; + +/** + * @struct FluxCaptureRecord + * @brief Internal structure for storing flux capture data during write mode. + * + * This structure is used internally by the library to store flux capture data + * in memory before it is written to the image. It combines the FluxEntry metadata + * with pointers to the actual data and index buffers. + * + * The structure is stored in a utarray (ctx->flux_captures) during write mode, + * and the buffers are freed automatically when the record is removed from the + * array or when the array is freed. + * + * @note This structure is used internally and is not part of the public API. + * @note The data_buffer and index_buffer are owned by the utarray and are freed + * automatically via the flux_capture_record_dtor() destructor. + * @internal + */ +typedef struct FluxCaptureRecord +{ + FluxEntry entry; ///< Flux entry metadata describing this capture. + uint8_t *data_buffer; ///< Pointer to the flux data buffer. Owned by the utarray, freed automatically. + uint32_t data_length; ///< Length of the data buffer in bytes. + uint8_t *index_buffer; ///< Pointer to the flux index buffer. Owned by the utarray, freed automatically. + uint32_t index_length; ///< Length of the index buffer in bytes. +} FluxCaptureRecord; + +/** + * @struct FluxCaptureKey + * @brief Key structure for flux capture lookup map. + * + * This structure uniquely identifies a flux capture within an image. It is used + * as the key in the internal hash table (flux_map) that provides O(1) lookup + * of flux captures by their identifier tuple. + * + * The structure matches the first four fields of FluxEntry (head, track, subtrack, + * captureIndex). + * + * @note This structure is used internally for efficient lookup and is not part of + * the public API. + * @note The combination of (head, track, subtrack, captureIndex) must be unique + * within an image. + * @internal + */ +typedef struct FluxCaptureKey +{ + uint32_t head; ///< Head number identifying the capture location. + uint16_t track; ///< Track number identifying the capture location. + uint8_t subtrack; ///< Subtrack number identifying the capture location. + uint32_t captureIndex; ///< Capture index, allowing multiple captures for the same location. +} FluxCaptureKey; + +#pragma pack(pop) + +#endif // LIBAARUFORMAT_FLUX_H \ No newline at end of file diff --git a/include/internal.h b/include/internal.h index f5fc117..6fb1829 100644 --- a/include/internal.h +++ b/include/internal.h @@ -39,6 +39,8 @@ void process_dumphw_block(aaruformat_context *ctx, const IndexEntry *entry) void process_checksum_block(aaruformat_context *ctx, const IndexEntry *entry); void process_tape_files_block(aaruformat_context *ctx, const IndexEntry *entry); void process_tape_partitions_block(aaruformat_context *ctx, const IndexEntry *entry); +void process_flux_data_block(aaruformat_context *ctx, const IndexEntry *entry); +int32_t flux_map_rebuild_from_entries(aaruformat_context *ctx); int32_t decode_ddt_entry_v1(aaruformat_context *ctx, uint64_t sector_address, uint64_t *offset, uint64_t *block_offset, uint8_t *sector_status); int32_t decode_ddt_entry_v2(aaruformat_context *ctx, uint64_t sector_address, bool negative, uint64_t *offset, diff --git a/src/blocks/flux.c b/src/blocks/flux.c new file mode 100644 index 0000000..4a88c4e --- /dev/null +++ b/src/blocks/flux.c @@ -0,0 +1,1111 @@ +/* + * This file is part of the Aaru Data Preservation Suite. + * Copyright (c) 2019-2025 Natalia Portillo. + * + * This library is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#include +#include +#include +#include +#include +#include "aaruformat/context.h" +#include "aaruformat/errors.h" +#include "aaruformat/structs/index.h" +#include "consts.h" +#include "decls.h" +#include "log.h" +#include "utarray.h" +#include "uthash.h" + +struct FluxCaptureMapEntry +{ + FluxCaptureKey key; + uint32_t index; + UT_hash_handle hh; +}; + +/** + * @brief Destructor callback for FluxCaptureRecord elements in a utarray. + * + * This function is used by the utarray library to clean up dynamically allocated + * memory when a FluxCaptureRecord is removed from or when the array is freed. + * It safely frees both the data_buffer and index_buffer fields if they are non-NULL, + * and then nullifies the pointers to prevent double-free errors. + * + * @param element Pointer to the FluxCaptureRecord element to destroy. May be NULL. + * + * @note This function is registered with utarray via FLUX_CAPTURE_RECORD_ICD and + * is called automatically by utarray_free() and utarray_erase() operations. + * @internal + */ +static void flux_capture_record_dtor(void *element) +{ + if(element == NULL) return; + + FluxCaptureRecord *record = element; + free(record->data_buffer); + free(record->index_buffer); + record->data_buffer = NULL; + record->index_buffer = NULL; +} + +static const UT_icd FLUX_CAPTURE_RECORD_ICD = {sizeof(FluxCaptureRecord), NULL, NULL, flux_capture_record_dtor}; + +/** + * @brief Clear and deallocate the flux capture lookup map. + * + * This function iterates through all entries in the UTHASH-based flux capture + * lookup map, removes each entry from the hash table, frees its memory, and + * sets the map pointer to NULL. This is used during cleanup operations and when + * rebuilding the map from scratch. + * + * The function is safe to call even if the map is already NULL or empty. + * + * @param ctx Pointer to the aaruformat context containing the flux_map to clear. + * Must not be NULL. + * + * @note This function does not free the flux_entries array or flux_captures utarray; + * it only clears the hash table used for O(1) lookup of flux captures. + * @internal + */ +static void flux_map_clear(aaruformat_context *ctx) +{ + if(ctx->flux_map == NULL) return; + + FluxCaptureMapEntry *entry, *tmp; + HASH_ITER(hh, ctx->flux_map, entry, tmp) + { + HASH_DEL(ctx->flux_map, entry); + free(entry); + } + + ctx->flux_map = NULL; +} + +/** + * @brief Add or update a flux capture entry in the lookup map. + * + * This function adds a new entry to the UTHASH-based flux capture lookup map, + * or updates an existing entry if a matching key is already present. The map + * enables O(1) lookup of flux captures by their (head, track, subtrack, captureIndex) + * identifier tuple. + * + * If an entry with the same key already exists, its index is updated to the new value. + * If no entry exists, a new FluxCaptureMapEntry is allocated and added to the map. + * + * @param ctx Pointer to the aaruformat context containing the flux_map. Must not be NULL. + * @param key Pointer to the FluxCaptureKey identifying the flux capture (head, track, + * subtrack, captureIndex). Must not be NULL. + * @param index The array index in ctx->flux_entries that corresponds to this flux capture. + * + * @return 0 on success, -1 on memory allocation failure. + * + * @note On allocation failure, the function returns -1 and the map remains unchanged. + * @note This function is used internally during map rebuilding and when adding new + * flux captures in write mode. + * @internal + */ +static int flux_map_add(aaruformat_context *ctx, const FluxCaptureKey *key, uint32_t index) +{ + FluxCaptureMapEntry *entry = NULL; + HASH_FIND(hh, ctx->flux_map, key, sizeof(FluxCaptureKey), entry); + + if(entry == NULL) + { + entry = malloc(sizeof(FluxCaptureMapEntry)); + if(entry == NULL) return -1; + entry->key = *key; + HASH_ADD(hh, ctx->flux_map, key, sizeof(FluxCaptureKey), entry); + } + + entry->index = index; + return 0; +} + +/** + * @brief Rebuild the flux capture lookup map from the flux_entries array. + * + * This function clears any existing flux capture lookup map and rebuilds it from + * the ctx->flux_entries array. The map provides O(1) lookup of flux captures by + * their (head, track, subtrack, captureIndex) identifier tuple, which is used by + * aaruf_read_flux_capture() and other flux access functions. + * + * The function iterates through all entries in ctx->flux_entries and adds each one + * to the hash table, mapping its identifier to its array index. If any entry fails + * to be added (due to memory allocation failure), the entire map is cleared and an + * error is returned. + * + * @param ctx Pointer to the aaruformat context containing flux_entries and flux_map. + * Must not be NULL. + * + * @return AARUF_STATUS_OK on success, or AARUF_ERROR_NOT_ENOUGH_MEMORY if memory + * allocation fails during map construction. + * + * @retval AARUF_STATUS_OK The map was successfully rebuilt (or no entries to process). + * @retval AARUF_ERROR_NOT_ENOUGH_MEMORY Memory allocation failed; the map is cleared. + * + * @note If ctx->flux_entries is NULL or ctx->flux_data_header.entries is 0, the + * function clears the map and returns AARUF_STATUS_OK (no-op). + * @note On failure, the map is cleared to ensure a consistent state. + * @note This function is called automatically by process_flux_data_block() after + * successfully reading flux entries from the image. + * + * @see flux_map_clear() for clearing the map + * @see flux_map_add() for adding individual entries + * @see process_flux_data_block() for when this is called during image opening + */ +int32_t flux_map_rebuild_from_entries(aaruformat_context *ctx) +{ + flux_map_clear(ctx); + + if(ctx->flux_entries == NULL || ctx->flux_data_header.entries == 0) return AARUF_STATUS_OK; + + for(uint32_t i = 0; i < ctx->flux_data_header.entries; i++) + { + const FluxEntry *entry = &ctx->flux_entries[i]; + FluxCaptureKey key = {entry->head, entry->track, entry->subtrack, entry->captureIndex}; + + if(flux_map_add(ctx, &key, i) != 0) + { + FATAL("Could not add flux capture to lookup map"); + flux_map_clear(ctx); + return AARUF_ERROR_NOT_ENOUGH_MEMORY; + } + } + + return AARUF_STATUS_OK; +} + +/** + * @brief Parse and integrate a Flux Data block from the image stream into the context. + * + * This function seeks to the byte offset specified by the supplied @p entry, reads a + * FluxHeader followed by the declared number of FluxEntry records, validates the block + * through its CRC64, rebuilds the flux capture lookup map, and populates multiple fields + * in the provided @p ctx: + * + * - ctx->flux_data_header (identifier, entries, crc64) + * - ctx->flux_entries (raw array with ALL flux entries in on-disk order) + * - ctx->flux_capture_map (hash table mapping (head, track, subtrack, captureIndex) to entry indices) + * - ctx->imageInfo.ImageSize (incremented by sizeof(FluxEntry) * entries) + * + * Before reading new data, the function performs cleanup of any existing flux-related state: + * - Frees and clears ctx->flux_captures if it exists (write-mode remnants) + * - Frees and clears ctx->flux_entries if it exists (from previous calls or partial reads) + * - Clears ctx->flux_capture_map (via flux_map_clear()) + * - Zeros ctx->flux_data_header + * + * After successfully reading and validating the flux entries, the function rebuilds the + * in-memory flux capture lookup map by calling flux_map_rebuild_from_entries(). This map + * enables O(1) lookup of flux captures by their identifiers (head, track, subtrack, captureIndex), + * which is used by aaruf_read_flux_capture() and other flux access functions. + * + * The function is intended for internal library use during image opening / indexing and is NOT + * part of the stable public API (no versioning guarantees). Callers outside the library should use + * the higher-level image open helpers that trigger this parsing implicitly. + * + * **Processing Flow:** + * 1. **Validation:** Check ctx and ctx->imageStream are valid + * 2. **Cleanup:** Free existing flux_captures, flux_entries, and clear flux_capture_map + * 3. **Seek:** Seek to entry->offset in the image stream + * 4. **Read Header:** Read FluxHeader structure + * 5. **Validate Identifier:** Verify identifier == FluxDataBlock + * 6. **Allocate Entries:** Allocate memory for flux_entries array + * 7. **Read Entries:** Read all FluxEntry structures from the stream + * 8. **Validate CRC64:** Compute and verify CRC64 checksum over entries + * 9. **Rebuild Map:** Call flux_map_rebuild_from_entries() to build lookup map + * 10. **Update ImageSize:** Increment ctx->image_info.ImageSize + * + * **Error & early-return behavior (no exception mechanism, all via logging + early return):** + * - NULL @p ctx or NULL ctx->imageStream: Logs FATAL and returns immediately; context left untouched. + * - Seek failure: FATAL + return; context left untouched (cleanup already performed). + * - FluxHeader read short: flux_data_header zeroed, TRACE logged, return. + * - Identifier mismatch: flux_data_header zeroed, TRACE logged, return. + * - Allocation failure for flux_entries: flux_data_header zeroed, FATAL logged, return. + * - Short read of FluxEntry array: flux_data_header zeroed, allocated flux_entries freed, FATAL logged, return. + * - CRC mismatch: TRACE logged and return; (NOTE: at this point flux_entries remain allocated and + * flux_data_header retains the just-read values, but the lookup map is not built, so entries + * are not accessible via lookup functions. Caller may wish to discard them or trigger re-read.) + * - Map rebuild failure: flux_entries freed, flux_data_header zeroed, FATAL logged, return. + * + * **Memory management:** + * - Frees any pre-existing ctx->flux_captures (utarray) before processing + * - Frees any pre-existing ctx->flux_entries before allocating new ones + * - Allocates ctx->flux_entries with malloc() sized to entries * sizeof(FluxEntry) + * - On certain failure paths (short reads, map rebuild failure) allocated memory is freed + * - On CRC mismatch, allocated memory is kept but map is not built (entries not accessible via lookup) + * - The function is idempotent in terms of memory: it cleans up before processing, so repeated + * calls will not leak memory. However, the function is expected to be called exactly once + * per context lifetime during image opening. + * + * **Flux Capture Lookup Map:** + * After successfully reading and validating flux entries, the function rebuilds the in-memory + * hash table (ctx->flux_capture_map) that maps flux capture identifiers to array indices. This + * enables efficient lookup of flux captures by their (head, track, subtrack, captureIndex) tuple. + * The map is built using UTHASH and is used by aaruf_read_flux_capture() and other flux access + * functions. If map rebuild fails, the function cleans up all allocated resources and returns. + * + * **Thread safety:** + * - Not thread-safe: mutates shared state in @p ctx without synchronization. + * - Must not be called concurrently with readers/writers referencing the same context. + * + * **Preconditions (@pre):** + * - @p ctx != NULL + * - @p ctx->imageStream is a valid FILE* opened for reading at least up to the block region. + * - @p entry != NULL and entry->offset points to the start of a well-formed Flux Data block. + * + * **Postconditions (@post) on success (CRC valid and map rebuilt):** + * - ctx->flux_data_header.identifier == FluxDataBlock + * - ctx->flux_data_header.entries > 0 implies ctx->flux_entries != NULL + * - ctx->flux_capture_map is populated with entries mapping identifiers to indices + * - ctx->imageInfo.ImageSize incremented by flux data size + * - ctx->flux_captures == NULL (cleared if it existed) + * + * **Limitations / Caveats:** + * - No explicit status code: callers infer success by inspecting ctx->flux_data_header.entries and + * presence of ctx->flux_entries after invocation. + * - In case of CRC mismatch, flux_entries are retained but flux_capture_map is not built, so + * entries are not accessible via lookup functions. Caller may wish to discard them or trigger re-read. + * - The function is idempotent in terms of memory (cleans up before processing), but is expected + * to be called exactly once per context lifetime during image opening. + * + * **Logging strategy:** + * - FATAL used for unrecoverable structural or resource errors (seek failure, allocation failure, + * map rebuild failure). + * - TRACE used for informational / soft failures (e.g., CRC mismatch, identifier mismatch, short read). + * + * @param ctx Mutable pointer to an aaruformatContext receiving parsed flux metadata. + * @param entry Pointer to the index entry describing this Flux Data block (offset required; size not + * strictly used beyond informational logging and sequential reading). + * + * @return void This function does not return a status code; errors are reported via logging side effects. + * + * @warning Absence of a returned status requires defensive post-call validation by the caller. + * @warning CRC mismatch leaves possibly invalid data in ctx->flux_entries, and the lookup map + * is not built, making entries inaccessible via lookup functions. + * @warning Map rebuild failure results in complete cleanup of flux-related state, leaving the + * context without flux data even if the entries were successfully read and validated. + * + * @see flux_map_rebuild_from_entries() for the map rebuilding logic + * @see flux_map_clear() for the map clearing logic + * @see aaruf_read_flux_capture() for using the lookup map to access flux captures + */ +void process_flux_data_block(aaruformat_context *ctx, const IndexEntry *entry) +{ + int pos = 0; + size_t read_bytes = 0; + uint64_t crc64 = 0; + + // Check if the context and image stream are valid + if(ctx == NULL || ctx->imageStream == NULL) + { + FATAL("Invalid context or image stream."); + return; + } + + if(ctx->flux_captures != NULL) + { + utarray_free(ctx->flux_captures); + ctx->flux_captures = NULL; + } + + if(ctx->flux_entries != NULL) + { + free(ctx->flux_entries); + ctx->flux_entries = NULL; + } + + flux_map_clear(ctx); + + memset(&ctx->flux_data_header, 0, sizeof(FluxHeader)); + + // Seek to block + pos = fseek(ctx->imageStream, entry->offset, SEEK_SET); + if(pos < 0 || ftell(ctx->imageStream) != entry->offset) + { + FATAL("Could not seek to %" PRIu64 " as indicated by index entry...\n", entry->offset); + return; + } + + // Even if those two checks shall have been done before + read_bytes = fread(&ctx->flux_data_header, 1, sizeof(FluxHeader), ctx->imageStream); + + if(read_bytes != sizeof(FluxHeader)) + { + memset(&ctx->flux_data_header, 0, sizeof(FluxHeader)); + TRACE("Could not read flux data header, continuing...\n"); + return; + } + + if(ctx->flux_data_header.identifier != FluxDataBlock) + { + memset(&ctx->flux_data_header, 0, sizeof(FluxHeader)); + TRACE("Incorrect identifier for flux data block at position %" PRIu64 "\n", entry->offset); + return; + } + + ctx->image_info.ImageSize += sizeof(FluxEntry) * ctx->flux_data_header.entries; + + ctx->flux_entries = (FluxEntry *)malloc(sizeof(FluxEntry) * ctx->flux_data_header.entries); + + if(ctx->flux_entries == NULL) + { + memset(&ctx->flux_data_header, 0, sizeof(FluxHeader)); + FATAL("Could not allocate memory for flux data block, continuing...\n"); + return; + } + + read_bytes = fread(ctx->flux_entries, sizeof(FluxEntry), ctx->flux_data_header.entries, ctx->imageStream); + + if(read_bytes != ctx->flux_data_header.entries) + { + memset(&ctx->flux_data_header, 0, sizeof(FluxHeader)); + free(ctx->flux_entries); + ctx->flux_entries = NULL; + FATAL("Could not read flux data block, continuing...\n"); + return; + } + + crc64 = aaruf_crc64_data((const uint8_t *)ctx->flux_entries, ctx->flux_data_header.entries * sizeof(FluxEntry)); + + if(crc64 != ctx->flux_data_header.crc64) + { + TRACE("Incorrect CRC found: 0x%" PRIx64 " found, expected 0x%" PRIx64 ", continuing...\n", crc64, + ctx->flux_data_header.crc64); + return; + } + + if(flux_map_rebuild_from_entries(ctx) != AARUF_STATUS_OK) + { + free(ctx->flux_entries); + ctx->flux_entries = NULL; + memset(&ctx->flux_data_header, 0, sizeof(FluxHeader)); + return; + } + + TRACE("Found %d flux entries at position %" PRIu64 ".\n", ctx->flux_data_header.entries, entry->offset); +} + +/** + * @brief Retrieve metadata for all flux captures in the image. + * + * This function retrieves metadata for all flux captures stored in the AaruFormat image. + * The metadata includes head, track, subtrack, captureIndex, indexResolution, and + * dataResolution for each capture, but does not include the actual flux data or index + * buffers (use aaruf_read_flux_capture() to retrieve those). + * + * The function can be called with a NULL buffer to determine the required buffer size. + * In this case, the required length is written to *length and AARUF_ERROR_BUFFER_TOO_SMALL + * is returned. + * + * @param context Pointer to an initialized aaruformat context opened for reading. + * Must not be NULL. + * @param buffer Pointer to a buffer to receive the FluxCaptureMeta array. May be NULL + * to query the required size. + * @param length Pointer to the size of the buffer (in bytes). On input, must contain + * the buffer size. On output, contains the required size (if buffer is too + * small) or the actual size written. Must not be NULL. + * + * @return AARUF_STATUS_OK on success, or an error code on failure. + * + * @retval AARUF_STATUS_OK Metadata successfully retrieved and written to buffer. + * @retval AARUF_ERROR_NOT_AARUFORMAT Invalid context or context magic mismatch. + * @retval AARUF_ERROR_FLUX_DATA_NOT_FOUND Image contains no flux captures. + * @retval AARUF_ERROR_BUFFER_TOO_SMALL Buffer is NULL or too small; *length contains + * the required size. + * + * @note The buffer must be large enough to hold ctx->flux_data_header.entries * + * sizeof(FluxCaptureMeta) bytes. + * @note The returned metadata array is in the same order as the flux entries in the + * FluxDataBlock (on-disk order). + * @note This function only returns metadata; use aaruf_read_flux_capture() to retrieve + * the actual flux data and index buffers. + * + * @see aaruf_read_flux_capture() to retrieve actual flux data for a specific capture + * @see FluxCaptureMeta for the structure of each metadata entry + */ +AARU_EXPORT int32_t AARU_CALL aaruf_get_flux_captures(void *context, uint8_t *buffer, size_t *length) +{ + TRACE("Entering aaruf_get_flux_captures(%p, %p, %zu)", context, buffer, (length ? *length : 0)); + + // Check context is correct AaruFormat context + if(context == NULL) + { + FATAL("Invalid context"); + return AARUF_ERROR_NOT_AARUFORMAT; + } + + aaruformat_context *ctx = context; + + // Not a libaaruformat context + if(ctx->magic != AARU_MAGIC) + { + FATAL("Invalid context"); + return AARUF_ERROR_NOT_AARUFORMAT; + } + + if(ctx->flux_data_header.entries == 0 || ctx->flux_entries == NULL) + { + FATAL("Image contains no flux captures"); + TRACE("Exiting aaruf_get_flux_captures() = AARUF_ERROR_FLUX_DATA_NOT_FOUND"); + return AARUF_ERROR_FLUX_DATA_NOT_FOUND; + } + + size_t required_length = ctx->flux_data_header.entries * sizeof(FluxCaptureMeta); + + if(buffer == NULL || length == NULL || *length < required_length) + { + if(length) *length = required_length; + TRACE("Buffer too small for flux captures, required %zu bytes", required_length); + + TRACE("Exiting aaruf_get_flux_captures() = AARUF_ERROR_BUFFER_TOO_SMALL"); + return AARUF_ERROR_BUFFER_TOO_SMALL; + } + + FluxCaptureMeta *out_entries = (FluxCaptureMeta *)buffer; + for(uint16_t i = 0; i < ctx->flux_data_header.entries; i++) + { + const FluxEntry *entry = &ctx->flux_entries[i]; + out_entries[i].head = entry->head; + out_entries[i].track = entry->track; + out_entries[i].subtrack = entry->subtrack; + out_entries[i].captureIndex = entry->captureIndex; + out_entries[i].indexResolution = entry->indexResolution; + out_entries[i].dataResolution = entry->dataResolution; + } + + *length = required_length; + + TRACE("Exiting aaruf_get_flux_captures(%p, %p, %zu) = AARUF_STATUS_OK", context, buffer, *length); + return AARUF_STATUS_OK; +} + +/** + * @brief Add a flux capture to the image during write mode. + * + * This function adds a new flux capture to the image being written. The capture + * includes both data and index buffers, along with metadata specifying the head, + * track, subtrack, capture index, and resolution information for both streams. + * + * The function copies the provided data and index buffers into internal storage + * (utarray) and creates a corresponding FluxEntry in the flux_entries array. The + * actual payload data is written to the image later during image finalization + * (in write_flux_blocks() and write_flux_capture_payload()). + * + * The flux capture lookup map is updated to enable efficient retrieval of the + * capture by its identifier tuple. + * + * @param context Pointer to an initialized aaruformat context opened for writing. + * Must not be NULL. + * @param head Head number the flux capture corresponds to. + * @param track Track number the flux capture corresponds to. + * @param subtrack Subtrack number the flux capture corresponds to. + * @param capture_index Capture index, allowing multiple captures for the same location. + * @param data_resolution Resolution in picoseconds for the data stream. + * @param index_resolution Resolution in picoseconds for the index stream. + * @param data Pointer to the flux data buffer. May be NULL if data_length is 0. + * @param data_length Length of the data buffer in bytes. + * @param index Pointer to the flux index buffer. May be NULL if index_length is 0. + * @param index_length Length of the index buffer in bytes. + * + * @return AARUF_STATUS_OK on success, or an error code on failure. + * + * @retval AARUF_STATUS_OK Flux capture successfully added to the context. + * @retval AARUF_ERROR_NOT_AARUFORMAT Invalid context or context magic mismatch. + * @retval AARUF_READ_ONLY Context is not in write mode. + * @retval AARUF_ERROR_INCORRECT_DATA_SIZE Invalid buffer pointers or combined size exceeds UINT32_MAX. + * @retval AARUF_ERROR_NOT_ENOUGH_MEMORY Memory allocation failed for buffers or map entry. + * + * @note The function copies the data and index buffers; the caller retains ownership + * of the original buffers and may free them after this call. + * @note The combined size of data_length + index_length must not exceed UINT32_MAX. + * @note The maximum number of flux captures is limited to UINT16_MAX (65535). + * @note The payloadOffset field in the FluxEntry is set to 0 initially and is + * populated later when the payload block is written during image finalization. + * @note If adding the capture to the lookup map fails, the function performs cleanup + * and restores the previous state. + * + * @see aaruf_read_flux_capture() to read flux captures from an image + * @see aaruf_clear_flux_captures() to remove all flux captures + * @see write_flux_blocks() for when payload blocks are written + */ +AARU_EXPORT int32_t AARU_CALL aaruf_write_flux_capture(void *context, uint32_t head, uint16_t track, uint8_t subtrack, + uint32_t capture_index, uint64_t data_resolution, + uint64_t index_resolution, const uint8_t *data, + uint32_t data_length, const uint8_t *index, + uint32_t index_length) +{ + TRACE("Entering aaruf_add_flux_capture(%p, %u, %u, %u, %u, %" PRIu64 ", %" PRIu64 ", %p, %u, %p, %u)", context, + head, track, subtrack, capture_index, data_resolution, index_resolution, data, data_length, index, + index_length); + + if(context == NULL) + { + FATAL("Invalid context"); + return AARUF_ERROR_NOT_AARUFORMAT; + } + + aaruformat_context *ctx = context; + + if(ctx->magic != AARU_MAGIC) + { + FATAL("Invalid context"); + return AARUF_ERROR_NOT_AARUFORMAT; + } + + if(!ctx->is_writing) + { + FATAL("Flux captures can only be added when writing"); + return AARUF_READ_ONLY; + } + + if((index_length != 0 && index == NULL) || (data_length != 0 && data == NULL)) + { + FATAL("Invalid flux capture buffers"); + return AARUF_ERROR_INCORRECT_DATA_SIZE; + } + + if((uint64_t)data_length + index_length > UINT32_MAX) + { + FATAL("Flux capture too large (%" PRIu64 " bytes)", (uint64_t)data_length + index_length); + return AARUF_ERROR_INCORRECT_DATA_SIZE; + } + + if(ctx->flux_captures == NULL) + { + utarray_new(ctx->flux_captures, &FLUX_CAPTURE_RECORD_ICD); + if(ctx->flux_captures == NULL) + { + FATAL("Could not allocate flux capture storage"); + return AARUF_ERROR_NOT_ENOUGH_MEMORY; + } + } + + size_t existing_captures = utarray_len(ctx->flux_captures); + if(existing_captures >= UINT16_MAX) + { + FATAL("Flux capture limit exceeded (%zu >= %u)", existing_captures, UINT16_MAX); + return AARUF_ERROR_INCORRECT_DATA_SIZE; + } + + uint8_t *data_copy = NULL; + uint8_t *index_copy = NULL; + + if(data_length != 0) + { + data_copy = malloc(data_length); + if(data_copy == NULL) + { + FATAL("Could not allocate %u bytes for flux data", data_length); + return AARUF_ERROR_NOT_ENOUGH_MEMORY; + } + memcpy(data_copy, data, data_length); + } + + if(index_length != 0) + { + index_copy = malloc(index_length); + if(index_copy == NULL) + { + free(data_copy); + FATAL("Could not allocate %u bytes for flux index", index_length); + return AARUF_ERROR_NOT_ENOUGH_MEMORY; + } + memcpy(index_copy, index, index_length); + } + + FluxCaptureRecord record = {0}; + record.entry.head = head; + record.entry.track = track; + record.entry.subtrack = subtrack; + record.entry.captureIndex = capture_index; + record.entry.dataResolution = data_resolution; + record.entry.indexResolution = index_resolution; + record.entry.indexOffset = data_length; + record.entry.payloadOffset = 0; + record.data_buffer = data_copy; + record.data_length = data_length; + record.index_buffer = index_copy; + record.index_length = index_length; + + FluxEntry *new_entries = realloc(ctx->flux_entries, (existing_captures + 1) * sizeof(FluxEntry)); + if(new_entries == NULL) + { + free(data_copy); + free(index_copy); + FATAL("Could not grow flux entry array"); + return AARUF_ERROR_NOT_ENOUGH_MEMORY; + } + + ctx->flux_entries = new_entries; + utarray_push_back(ctx->flux_captures, &record); + + ctx->flux_entries[existing_captures] = record.entry; + ctx->flux_data_header.identifier = FluxDataBlock; + ctx->flux_data_header.entries = (uint16_t)(existing_captures + 1); + ctx->flux_data_header.crc64 = + aaruf_crc64_data((const uint8_t *)ctx->flux_entries, ctx->flux_data_header.entries * sizeof(FluxEntry)); + ctx->dirty_flux_block = true; + + FluxCaptureKey key = {head, track, subtrack, capture_index}; + if(flux_map_add(ctx, &key, (uint32_t)existing_captures) != 0) + { + FATAL("Could not add flux capture to lookup map"); + + size_t len = utarray_len(ctx->flux_captures); + if(len > 0) utarray_erase(ctx->flux_captures, len - 1, 1); + + if(existing_captures == 0) + { + free(ctx->flux_entries); + ctx->flux_entries = NULL; + } + else + { + FluxEntry *shrunk = realloc(ctx->flux_entries, existing_captures * sizeof(FluxEntry)); + if(shrunk != NULL) ctx->flux_entries = shrunk; + } + + ctx->flux_data_header.entries = (uint16_t)existing_captures; + ctx->flux_data_header.crc64 = existing_captures == 0 ? 0 + : aaruf_crc64_data((const uint8_t *)ctx->flux_entries, + existing_captures * sizeof(FluxEntry)); + return AARUF_ERROR_NOT_ENOUGH_MEMORY; + } + + TRACE("Exiting aaruf_add_flux_capture() = AARUF_STATUS_OK (captures=%u)", ctx->flux_data_header.entries); + return AARUF_STATUS_OK; +} + +/** + * @brief Clear all flux captures from the context. + * + * This function removes all flux captures from the context, freeing all associated + * memory including the flux_captures utarray, flux_entries array, and the flux + * capture lookup map. The flux_data_header is zeroed. + * + * This function is useful for resetting the flux capture state, particularly in + * write mode when starting a new image or when discarding previously added captures. + * + * @param context Pointer to an initialized aaruformat context. Must not be NULL. + * + * @return AARUF_STATUS_OK on success, or an error code on failure. + * + * @retval AARUF_STATUS_OK All flux captures successfully cleared. + * @retval AARUF_ERROR_NOT_AARUFORMAT Invalid context or context magic mismatch. + * + * @note This function is safe to call even if no flux captures are present (no-op). + * @note After this call, ctx->flux_captures, ctx->flux_entries, and ctx->flux_map + * are all NULL/cleared, and ctx->flux_data_header is zeroed. + * @note The function does not affect the image file itself; it only clears in-memory + * state. To remove flux data from an image file, the image must be rewritten. + * + * @see aaruf_write_flux_capture() to add flux captures + * @see aaruf_get_flux_captures() to retrieve flux capture metadata + */ +AARU_EXPORT int32_t AARU_CALL aaruf_clear_flux_captures(void *context) +{ + TRACE("Entering aaruf_clear_flux_captures(%p)", context); + + if(context == NULL) + { + FATAL("Invalid context"); + return AARUF_ERROR_NOT_AARUFORMAT; + } + + aaruformat_context *ctx = context; + + if(ctx->magic != AARU_MAGIC) + { + FATAL("Invalid context"); + return AARUF_ERROR_NOT_AARUFORMAT; + } + + if(ctx->flux_captures != NULL) + { + utarray_free(ctx->flux_captures); + ctx->flux_captures = NULL; + } + + flux_map_clear(ctx); + + free(ctx->flux_entries); + ctx->flux_entries = NULL; + + memset(&ctx->flux_data_header, 0, sizeof(FluxHeader)); + + TRACE("Exiting aaruf_clear_flux_captures() = AARUF_STATUS_OK"); + return AARUF_STATUS_OK; +} + +/** + * @brief Read a specific flux capture's data and index buffers from the image. + * + * This function retrieves the actual flux data and index buffers for a specific + * flux capture identified by its head, track, subtrack, and capture_index. The + * function locates the corresponding FluxEntry in the flux_entries array (using + * the lookup map for efficiency), seeks to the DataStreamPayloadBlock at the specified + * payloadOffset, reads and decompresses the payload, validates CRC64 checksums, + * and extracts the data and index buffers. + * + * The function supports both uncompressed and LZMA-compressed payload blocks. + * CRC64 validation is performed on both the compressed and uncompressed data. + * + * The function can be called with NULL data buffers to determine the required + * buffer sizes. In this case, the required lengths are written to *data_length + * and *index_length, and AARUF_ERROR_BUFFER_TOO_SMALL is returned. + * + * @param context Pointer to an initialized aaruformat context opened for reading. + * Must not be NULL. + * @param head Head number of the flux capture to retrieve. + * @param track Track number of the flux capture to retrieve. + * @param subtrack Subtrack number of the flux capture to retrieve. + * @param capture_index Capture index of the flux capture to retrieve. + * @param index_data Pointer to a buffer to receive the index buffer. May be NULL + * to query the required size. + * @param index_length Pointer to the size of the index buffer (in bytes). On input, + * must contain the buffer size. On output, contains the required + * size (if buffer is too small) or the actual size written. + * Must not be NULL. + * @param data_data Pointer to a buffer to receive the data buffer. May be NULL + * to query the required size. + * @param data_length Pointer to the size of the data buffer (in bytes). On input, + * must contain the buffer size. On output, contains the required + * size (if buffer is too small) or the actual size written. + * Must not be NULL. + * + * @return AARUF_STATUS_OK on success, or an error code on failure. + * + * @retval AARUF_STATUS_OK Flux capture data successfully retrieved and written to buffers. + * @retval AARUF_ERROR_NOT_AARUFORMAT Invalid context, context magic mismatch, or invalid image stream. + * @retval AARUF_ERROR_FLUX_DATA_NOT_FOUND No flux captures in image or specified capture not found. + * @retval AARUF_ERROR_BUFFER_TOO_SMALL One or both buffers are NULL or too small; + * *data_length and *index_length contain required sizes. + * @retval AARUF_ERROR_CANNOT_READ_BLOCK Failed to seek to payload or read payload header. + * @retval AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK LZMA decompression failed or invalid compression format. + * @retval AARUF_ERROR_INVALID_BLOCK_CRC CRC64 checksum mismatch (compressed or uncompressed). + * @retval AARUF_ERROR_INCORRECT_DATA_SIZE Payload section length exceeds 32-bit limits. + * @retval AARUF_ERROR_NOT_ENOUGH_MEMORY Memory allocation failed for decompression buffers. + * @retval AARUF_ERROR_UNSUPPORTED_COMPRESSION Unsupported compression type in payload header. + * + * @note The function first attempts to locate the flux entry using the lookup map + * (O(1) lookup). If the map is not available or the entry is not found, it falls + * back to a linear search through flux_entries (O(n)). + * @note The payload data is stored as [data_buffer][index_buffer] concatenated, with + * indexOffset indicating where the index buffer starts. + * @note Both buffers must be large enough to hold the respective data. The function + * validates buffer sizes before copying data. + * @note The caller is responsible for freeing the returned buffers if they were + * allocated by the caller. + * @note CRC64 validation is performed on both compressed and uncompressed data to + * ensure data integrity. + * + * @see aaruf_get_flux_captures() to retrieve metadata for all flux captures + * @see aaruf_write_flux_capture() to add flux captures during write mode + * @see DataStreamPayloadHeader for the structure of payload blocks + */ +AARU_EXPORT int32_t AARU_CALL aaruf_read_flux_capture(void *context, uint32_t head, uint16_t track, uint8_t subtrack, + uint32_t capture_index, uint8_t *index_data, + uint32_t *index_length, uint8_t *data_data, uint32_t *data_length) +{ + TRACE("Entering aaruf_read_flux_capture(%p, %u, %u, %u, %u, %p, %p, %p, %p)", context, head, track, subtrack, + capture_index, index_data, index_length, data_data, data_length); + + if(context == NULL) + { + FATAL("Invalid context"); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_NOT_AARUFORMAT"); + return AARUF_ERROR_NOT_AARUFORMAT; + } + + const aaruformat_context *ctx = context; + + if(ctx->magic != AARU_MAGIC) + { + FATAL("Invalid context"); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_NOT_AARUFORMAT"); + return AARUF_ERROR_NOT_AARUFORMAT; + } + + if(ctx->flux_data_header.entries == 0 || ctx->flux_entries == NULL) + { + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_FLUX_DATA_NOT_FOUND"); + return AARUF_ERROR_FLUX_DATA_NOT_FOUND; + } + + if(index_length == NULL || data_length == NULL) + { + FATAL("index_length or data_length pointers are NULL"); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_BUFFER_TOO_SMALL\n"); + return AARUF_ERROR_BUFFER_TOO_SMALL; + } + + const FluxEntry *flux_entry = NULL; + FluxCaptureKey key = {head, track, subtrack, capture_index}; + + // Find the flux entry in the map. + if(ctx->flux_map != NULL) + { + FluxCaptureMapEntry *map_entry = NULL; + HASH_FIND(hh, ctx->flux_map, &key, sizeof(FluxCaptureKey), map_entry); + if(map_entry != NULL && map_entry->index < ctx->flux_data_header.entries) + flux_entry = &ctx->flux_entries[map_entry->index]; + } + + // If the flux entry is not found in the map, search the entries array for a match. + if(flux_entry == NULL) + { + for(uint32_t i = 0; i < ctx->flux_data_header.entries; i++) + { + const FluxEntry *candidate = &ctx->flux_entries[i]; + if(candidate->head == head && candidate->track == track && candidate->subtrack == subtrack && + candidate->captureIndex == capture_index) + { + flux_entry = candidate; + break; + } + } + } + + if(flux_entry == NULL) + { + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_FLUX_DATA_NOT_FOUND\n"); + return AARUF_ERROR_FLUX_DATA_NOT_FOUND; + } + + TRACE("Requested flux capture: head=%u track=%u subtrack=%u captureIndex=%u payloadOffset=%" PRIu64 "\n", + flux_entry->head, flux_entry->track, flux_entry->subtrack, flux_entry->captureIndex, + flux_entry->payloadOffset); + + if(ctx->imageStream == NULL) + { + FATAL("Invalid image stream"); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_NOT_AARUFORMAT\n"); + return AARUF_ERROR_NOT_AARUFORMAT; + } + + if(fseek(ctx->imageStream, flux_entry->payloadOffset, SEEK_SET) < 0) + { + FATAL("Could not seek to flux payload at offset %" PRIu64, flux_entry->payloadOffset); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_CANNOT_READ_BLOCK\n"); + return AARUF_ERROR_CANNOT_READ_BLOCK; + } + + long file_position = ftell(ctx->imageStream); + if(file_position < 0 || (uint64_t)file_position != flux_entry->payloadOffset) + { + FATAL("Invalid flux payload position (expected %" PRIu64 ", got %ld)", flux_entry->payloadOffset, + file_position); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_CANNOT_READ_BLOCK\n"); + return AARUF_ERROR_CANNOT_READ_BLOCK; + } + + DataStreamPayloadHeader payload_header; + size_t read_bytes = fread(&payload_header, 1, sizeof(DataStreamPayloadHeader), ctx->imageStream); + if(read_bytes != sizeof(DataStreamPayloadHeader)) + { + FATAL("Could not read flux payload header at offset %" PRIu64, flux_entry->payloadOffset); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_CANNOT_READ_BLOCK\n"); + return AARUF_ERROR_CANNOT_READ_BLOCK; + } + + if(payload_header.identifier != DataStreamPayloadBlock) + { + FATAL("Incorrect identifier 0x%08" PRIx32 " for flux payload at offset %" PRIu64, payload_header.identifier, + flux_entry->payloadOffset); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_CANNOT_READ_BLOCK\n"); + return AARUF_ERROR_CANNOT_READ_BLOCK; + } + + const CompressionType compression = (CompressionType)payload_header.compression; + uint8_t *cmp_buffer = NULL; + uint8_t *payload = NULL; + size_t cmp_length = payload_header.cmpLength; + size_t raw_length = payload_header.length; + + if(compression == None) + { + if(cmp_length != raw_length) + { + FATAL("Flux payload lengths mismatch for uncompressed block (cmp=%u, raw=%u)", payload_header.cmpLength, + payload_header.length); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK\n"); + return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK; + } + + if(cmp_length != 0) + { + cmp_buffer = (uint8_t *)malloc(cmp_length); + if(cmp_buffer == NULL) + { + FATAL("Could not allocate %zu bytes for flux payload", cmp_length); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_NOT_ENOUGH_MEMORY\n"); + return AARUF_ERROR_NOT_ENOUGH_MEMORY; + } + + read_bytes = fread(cmp_buffer, 1, cmp_length, ctx->imageStream); + if(read_bytes != cmp_length) + { + FATAL("Could not read %zu bytes of flux payload", cmp_length); + free(cmp_buffer); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_CANNOT_READ_BLOCK\n"); + return AARUF_ERROR_CANNOT_READ_BLOCK; + } + + payload = cmp_buffer; + } + } + else if(compression == Lzma) + { + if(cmp_length <= LZMA_PROPERTIES_LENGTH) + { + FATAL("Flux payload compressed length %u too small for LZMA", payload_header.cmpLength); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK\n"); + return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK; + } + + cmp_buffer = (uint8_t *)malloc(cmp_length); + if(cmp_buffer == NULL) + { + FATAL("Could not allocate %zu bytes for flux payload", cmp_length); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_NOT_ENOUGH_MEMORY\n"); + return AARUF_ERROR_NOT_ENOUGH_MEMORY; + } + + read_bytes = fread(cmp_buffer, 1, cmp_length, ctx->imageStream); + if(read_bytes != cmp_length) + { + FATAL("Could not read %zu bytes of flux payload", cmp_length); + free(cmp_buffer); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_CANNOT_READ_BLOCK\n"); + return AARUF_ERROR_CANNOT_READ_BLOCK; + } + + if(raw_length != 0) + { + payload = (uint8_t *)malloc(raw_length); + if(payload == NULL) + { + FATAL("Could not allocate %zu bytes for decompressed flux payload", raw_length); + free(cmp_buffer); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_NOT_ENOUGH_MEMORY\n"); + return AARUF_ERROR_NOT_ENOUGH_MEMORY; + } + + size_t cmp_stream_len = cmp_length - LZMA_PROPERTIES_LENGTH; + size_t dst_len = raw_length; + size_t src_len = cmp_stream_len; + const uint8_t *cmp_props = cmp_buffer; + const uint8_t *cmp_stream = cmp_buffer + LZMA_PROPERTIES_LENGTH; + int32_t error_no = + aaruf_lzma_decode_buffer(payload, &dst_len, cmp_stream, &src_len, cmp_props, LZMA_PROPERTIES_LENGTH); + if(error_no != 0 || dst_len != raw_length) + { + FATAL("LZMA decompression failed for flux payload (err=%d, dst=%zu/%zu)", error_no, dst_len, + raw_length); + free(payload); + free(cmp_buffer); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK\n"); + return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK; + } + } + } + else + { + FATAL("Unsupported flux payload compression type %u", payload_header.compression); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_UNSUPPORTED_COMPRESSION\n"); + return AARUF_ERROR_UNSUPPORTED_COMPRESSION; + } + + uint64_t cmp_crc = 0; + if(cmp_length != 0 && cmp_buffer != NULL) cmp_crc = aaruf_crc64_data(cmp_buffer, cmp_length); + if(cmp_length == 0) cmp_crc = 0; + + if(cmp_crc != payload_header.cmpCrc64) + { + FATAL("Flux payload compressed CRC mismatch (expected 0x%" PRIx64 ", got 0x%" PRIx64 ")", + payload_header.cmpCrc64, cmp_crc); + if(payload != NULL && payload != cmp_buffer) free(payload); + if(cmp_buffer != NULL) free(cmp_buffer); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_INVALID_BLOCK_CRC\n"); + return AARUF_ERROR_INVALID_BLOCK_CRC; + } + + uint64_t raw_crc = 0; + if(raw_length != 0 && payload != NULL) raw_crc = aaruf_crc64_data(payload, raw_length); + if(raw_length == 0) raw_crc = 0; + + if(raw_crc != payload_header.crc64) + { + FATAL("Flux payload raw CRC mismatch (expected 0x%" PRIx64 ", got 0x%" PRIx64 ")", payload_header.crc64, + raw_crc); + if(payload != NULL && payload != cmp_buffer) free(payload); + if(cmp_buffer != NULL) free(cmp_buffer); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_INVALID_BLOCK_CRC\n"); + return AARUF_ERROR_INVALID_BLOCK_CRC; + } + + if(flux_entry->indexOffset > raw_length) + { + FATAL("Flux index offset %" PRIu64 " beyond payload length %zu", flux_entry->indexOffset, raw_length); + if(payload != NULL && payload != cmp_buffer) free(payload); + if(cmp_buffer != NULL) free(cmp_buffer); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_INVALID_BLOCK_CRC\n"); + return AARUF_ERROR_INVALID_BLOCK_CRC; + } + + uint64_t data_length_required64 = flux_entry->indexOffset; + uint64_t index_length_required64 = raw_length - flux_entry->indexOffset; + + if(data_length_required64 > UINT32_MAX || index_length_required64 > UINT32_MAX) + { + FATAL("Flux payload section length exceeds 32-bit limits (data=%" PRIu64 ", index=%" PRIu64 ")", + data_length_required64, index_length_required64); + if(payload != NULL && payload != cmp_buffer) free(payload); + if(cmp_buffer != NULL) free(cmp_buffer); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_INCORRECT_DATA_SIZE\n"); + return AARUF_ERROR_INCORRECT_DATA_SIZE; + } + + uint32_t data_required = (uint32_t)data_length_required64; + uint32_t index_required = (uint32_t)index_length_required64; + + uint32_t data_capacity = *data_length; + uint32_t index_capacity = *index_length; + + *data_length = data_required; + *index_length = index_required; + + if(data_data == NULL || index_data == NULL || data_capacity < data_required || index_capacity < index_required) + { + TRACE("Returning required flux capture sizes (data=%u, index=%u)\n", data_required, index_required); + if(payload != NULL && payload != cmp_buffer) free(payload); + if(cmp_buffer != NULL) free(cmp_buffer); + return AARUF_ERROR_BUFFER_TOO_SMALL; + } + + uint8_t *index_ptr = payload ? payload + data_length_required64 : NULL; + uint8_t *data_ptr = payload; + + if(data_required != 0 && data_ptr != NULL) memcpy(data_data, data_ptr, data_required); + if(index_required != 0 && index_ptr != NULL) memcpy(index_data, index_ptr, index_required); + + if(payload != NULL && payload != cmp_buffer) free(payload); + if(cmp_buffer != NULL) free(cmp_buffer); + + TRACE("Exiting aaruf_read_flux_capture() = AARUF_STATUS_OK\n"); + return AARUF_STATUS_OK; +} diff --git a/src/close.c b/src/close.c index 26526f6..119ed1c 100644 --- a/src/close.c +++ b/src/close.c @@ -4185,6 +4185,674 @@ static void write_aaru_json_block(aaruformat_context *ctx) } } +/** + * @brief Serialize a single flux capture payload block to the image file. + * + * This helper function writes a DataStreamPayloadBlock containing the raw flux data and index + * buffers for a single flux capture. Flux captures represent analog signal transitions + * recorded from floppy disk drives or other magnetic media, enabling bit-level analysis + * and preservation of timing information that cannot be reconstructed from sector data alone. + * + * The function concatenates the data buffer and index buffer into a single raw payload, + * optionally compresses it using LZMA if compression is enabled, calculates CRC64 checksums + * for both raw and compressed data, writes the DataStreamPayloadHeader followed by the payload + * data, and adds an IndexEntry to enable fast location during subsequent reads. + * + * **Block Structure:** + * The serialized block consists of: + * ``` + * +----------------------------------+ + * | DataStreamPayloadHeader (32 B) | <- identifier, compression, lengths, CRCs + * +----------------------------------+ + * | LZMA Properties (5 B) | <- Only if compression == Lzma + * +----------------------------------+ + * | Compressed/Uncompressed | <- Flux data + index buffers + * | Payload Data (variable) | + * +----------------------------------+ + * ``` + * + * **Processing Flow:** + * 1. **Buffer Concatenation:** Combine data_buffer and index_buffer into a single raw_buffer + * 2. **CRC64 Calculation:** Compute CRC64-ECMA over the raw concatenated buffer + * 3. **Compression Attempt:** If compression enabled, attempt LZMA compression + * - If compression is effective (reduces size), use compressed data + * - If compression is ineffective or fails, fall back to uncompressed + * 4. **Alignment:** Seek to EOF and align to block boundary (blockAlignmentShift) + * 5. **Header Construction:** Build DataStreamPayloadHeader with compression type, lengths, CRCs + * 6. **Write Operations:** Write header, LZMA properties (if compressed), then payload data + * 7. **Indexing:** Add IndexEntry with blockType=DataStreamPayloadBlock, dataType=FluxData + * 8. **Entry Population:** Populate the output FluxEntry with metadata and payload offset + * 9. **Cleanup:** Free temporary compression buffers + * + * **Data and Index Buffers:** + * - data_buffer: Contains the raw flux transition timing data (typically in nanoseconds or + * arbitrary time units based on dataResolution) + * - index_buffer: Contains an index structure that maps logical positions to offsets within + * the data buffer, enabling efficient random access to specific flux transitions + * - The two buffers are concatenated: [data_buffer][index_buffer] before compression/writing + * - indexOffset in the FluxEntry records where the index starts (equals data_length) + * + * **Compression Strategy:** + * - Compression is attempted only if ctx->compression_enabled is true + * - LZMA compression with preset level 9 is used + * - If compressed size >= raw size, compression is disabled and raw data is written + * - LZMA properties (5 bytes) are prepended to compressed data + * - Both raw and compressed data have separate CRC64 checksums for integrity verification + * + * **Alignment Strategy:** + * Before writing, the file position is: + * 1. Moved to EOF using fseek(SEEK_END) + * 2. Aligned forward to next boundary: (position + alignment_mask) & ~alignment_mask + * 3. Where alignment_mask = (1 << blockAlignmentShift) - 1 + * This ensures the flux payload block starts on a properly aligned offset for efficient + * I/O and compliance with the Aaru format specification. + * + * **CRC64 Integrity Protection:** + * Two CRC64-ECMA checksums are computed and stored: + * - crc64: Checksum over the raw concatenated buffer (data + index) + * - cmpCrc64: Checksum over the compressed data (or same as crc64 if uncompressed) + * These checksums allow verification of flux payload integrity when reading the image, + * detecting corruption in either the raw or compressed representation. + * + * **Index Entry:** + * On successful write, an IndexEntry is appended to ctx->indexEntries with: + * - blockType = DataStreamPayloadBlock (identifies this as data stream payload) + * - dataType = FluxData (identifies the payload content type) + * - offset = file position where DataStreamPayloadHeader was written + * + * This index entry enables aaruf_read_flux_capture() to quickly locate the payload + * during subsequent reads without scanning the entire file. + * + * **FluxEntry Population:** + * The output @p entry parameter is populated with: + * - head, track, subtrack, captureIndex: Identifiers from the record + * - dataResolution, indexResolution: Timing resolution metadata + * - indexOffset: Offset within payload where index buffer starts (equals data_length) + * - payloadOffset: File offset where this payload block was written + * + * This metadata is later written to the FluxDataBlock to enable efficient lookup + * of flux captures by their identifiers. + * + * **Error Handling:** + * The function returns error codes on failure: + * - AARUF_ERROR_INCORRECT_DATA_SIZE: Raw length exceeds 32-bit limit + * - AARUF_ERROR_NOT_ENOUGH_MEMORY: Memory allocation failure + * - AARUF_ERROR_CANNOT_WRITE_BLOCK_HEADER: Failed writing header + * - AARUF_ERROR_CANNOT_WRITE_BLOCK_DATA: Failed writing payload data + * + * On error, all allocated buffers are freed before returning. The function does not + * modify ctx state on failure, allowing the caller to retry or handle the error gracefully. + * + * **Memory Management:** + * - Allocates raw_buffer to hold concatenated data + index buffers + * - Allocates temporary cmp_stream buffer for LZMA compression attempt + * - Allocates compressed_buffer if compression is effective + * - All buffers are freed before function returns, even on error + * - Source buffers in @p record are not modified or freed (managed by caller) + * + * **Thread Safety:** + * This function is NOT thread-safe. It modifies shared ctx state (imageStream file + * position, indexEntries array) and must only be called during single-threaded + * finalization (within write_flux_blocks). + * + * **Use Cases:** + * - Preserving analog flux transitions from floppy disk drives + * - Enabling bit-level analysis and timing reconstruction + * - Supporting forensic analysis of magnetic media + * - Preserving drive-specific timing characteristics + * - Enabling advanced data recovery techniques + * + * **Relationship to Other Functions:** + * - Called by write_flux_blocks() for each flux capture in ctx->flux_captures + * - FluxEntry populated here is later written to FluxDataBlock by write_flux_blocks() + * - aaruf_read_flux_capture() reads these payload blocks during image reading + * - aaruf_write_flux_capture() adds records to ctx->flux_captures during image creation + * + * @param ctx Pointer to an initialized aaruformatContext in write mode. Must not be NULL. + * ctx->imageStream must be open and writable. ctx->indexEntries must be + * initialized (utarray) to accept new index entries. + * @param record Pointer to a FluxCaptureRecord containing the data and index buffers to + * serialize. The record's entry field will be updated with the final + * FluxEntry metadata including payloadOffset. + * @param entry Output parameter that will be populated with the complete FluxEntry + * metadata including the payloadOffset where this block was written. + * + * @return Returns one of the following status codes: + * @retval AARUF_STATUS_OK (0) Successfully wrote the flux payload block. The @p entry + * parameter has been populated with complete metadata. + * @retval AARUF_ERROR_INCORRECT_DATA_SIZE (-5) The combined length of data and index + * buffers exceeds UINT32_MAX (4GB). This should not occur in practice but is + * checked for safety. + * @retval AARUF_ERROR_NOT_ENOUGH_MEMORY (-9) Memory allocation failed for raw buffer, + * compression buffer, or compressed output buffer. + * @retval AARUF_ERROR_CANNOT_WRITE_BLOCK_HEADER (-22) Failed to write the + * DataStreamPayloadHeader to the image stream. + * @retval AARUF_ERROR_CANNOT_WRITE_BLOCK_DATA (-23) Failed to write the payload data + * to the image stream. + * + * @note The function does not validate that data_length and index_length match the + * actual buffer sizes. The caller is responsible for ensuring these values are + * correct. + * + * @note Compression effectiveness is evaluated by comparing compressed size to raw size. + * If compression doesn't reduce size, the raw data is written instead. This + * prevents wasting space on incompressible flux data. + * + * @note The payloadOffset stored in the FluxEntry enables direct seeking to the payload + * during reads without requiring a full index scan. This is critical for efficient + * random access to flux captures. + * + * @note LZMA properties are written immediately after the header when compression is + * enabled. The properties are included in cmpLength but not in the separate + * LZMA_PROPERTIES_LENGTH field, so readers must account for this when decompressing. + * + * @warning The function assumes data_buffer and index_buffer are valid for the specified + * lengths. Buffer overruns may occur if the lengths are incorrect. The caller + * must ensure these buffers are properly sized and valid. + * + * @warning Memory allocation failures result in immediate return with an error code. + * No partial writes occur on allocation failure, ensuring image consistency. + * + * @warning Write failures (fwrite returning != 1) result in error codes being returned. + * The caller (write_flux_blocks) is responsible for handling these errors and + * cleaning up any partially written state. + * + * @see DataStreamPayloadHeader for the block header structure definition + * @see FluxEntry for the metadata entry structure + * @see FluxCaptureRecord for the input record structure + * @see write_flux_blocks() for the caller that orchestrates writing all flux captures + * @see aaruf_read_flux_capture() for reading these payload blocks + * + * @internal + */ +static int32_t write_flux_capture_payload(aaruformat_context *ctx, FluxCaptureRecord *record, FluxEntry *entry) +{ + uint64_t data_length = record->data_length; + uint64_t index_length = record->index_length; + uint64_t raw_length = data_length + index_length; + + if(raw_length > UINT32_MAX) + { + FATAL("Flux capture raw length exceeds 32-bit limit (%" PRIu64 ")", raw_length); + return AARUF_ERROR_INCORRECT_DATA_SIZE; + } + + uint8_t *raw_buffer = NULL; + if(raw_length != 0) + { + raw_buffer = malloc((size_t)raw_length); + if(raw_buffer == NULL) + { + FATAL("Could not allocate %" PRIu64 " bytes for flux serialization", raw_length); + return AARUF_ERROR_NOT_ENOUGH_MEMORY; + } + + if(data_length != 0 && record->data_buffer != NULL) memcpy(raw_buffer, record->data_buffer, data_length); + if(index_length != 0 && record->index_buffer != NULL) + memcpy(raw_buffer + data_length, record->index_buffer, index_length); + } + + uint64_t raw_crc = raw_length != 0 && raw_buffer != NULL ? aaruf_crc64_data(raw_buffer, (size_t)raw_length) : 0; + + CompressionType compression = ctx->compression_enabled ? Lzma : None; + + uint8_t *compressed_buffer = NULL; + uint32_t cmp_length = 0; + uint64_t cmp_crc = 0; + + if(compression == Lzma) + { + size_t cmp_capacity = raw_length ? (size_t)raw_length * 2 + 65536 : LZMA_PROPERTIES_LENGTH + 16; + if(cmp_capacity < raw_length + LZMA_PROPERTIES_LENGTH) cmp_capacity = raw_length + LZMA_PROPERTIES_LENGTH; + + uint8_t *cmp_stream = malloc(cmp_capacity); + if(cmp_stream == NULL) + { + free(raw_buffer); + FATAL("Could not allocate %zu bytes for LZMA flux compression", cmp_capacity); + return AARUF_ERROR_NOT_ENOUGH_MEMORY; + } + + size_t dst_size = cmp_capacity; + uint8_t lzma_props[LZMA_PROPERTIES_LENGTH] = {0}; + size_t props_size = LZMA_PROPERTIES_LENGTH; + int32_t error_no = aaruf_lzma_encode_buffer(cmp_stream, &dst_size, raw_buffer, (size_t)raw_length, lzma_props, + &props_size, 9, ctx->lzma_dict_size, 4, 0, 2, 273, 8); + + if(error_no != 0 || props_size != LZMA_PROPERTIES_LENGTH || dst_size >= raw_length) + { + TRACE("Flux capture compression fell back to uncompressed (err=%d, dst=%zu, raw=%" PRIu64 ")", error_no, + dst_size, raw_length); + compression = None; + free(cmp_stream); + } + else + { + cmp_length = (uint32_t)(dst_size + LZMA_PROPERTIES_LENGTH); + compressed_buffer = malloc(cmp_length); + if(compressed_buffer == NULL) + { + free(cmp_stream); + free(raw_buffer); + FATAL("Could not allocate %u bytes for flux compressed payload", cmp_length); + return AARUF_ERROR_NOT_ENOUGH_MEMORY; + } + + memcpy(compressed_buffer, lzma_props, LZMA_PROPERTIES_LENGTH); + memcpy(compressed_buffer + LZMA_PROPERTIES_LENGTH, cmp_stream, dst_size); + cmp_crc = aaruf_crc64_data(compressed_buffer, cmp_length); + free(cmp_stream); + free(raw_buffer); + raw_buffer = NULL; + } + } + + if(compression == None) + { + cmp_length = (uint32_t)raw_length; + cmp_crc = raw_crc; + compressed_buffer = raw_buffer; + raw_buffer = NULL; + } + + // Align stream position to block boundary + fseek(ctx->imageStream, 0, SEEK_END); + long payload_position = ftell(ctx->imageStream); + const uint64_t alignment_mask = (1ULL << ctx->user_data_ddt_header.blockAlignmentShift) - 1; + if(payload_position & alignment_mask) + { + const uint64_t aligned_position = payload_position + alignment_mask & ~alignment_mask; + fseek(ctx->imageStream, aligned_position, SEEK_SET); + payload_position = aligned_position; + } + + DataStreamPayloadHeader payload_header = {0}; + payload_header.identifier = DataStreamPayloadBlock; + payload_header.compression = (uint16_t)compression; + payload_header.cmpLength = cmp_length; + payload_header.length = (uint32_t)raw_length; + payload_header.cmpCrc64 = cmp_crc; + payload_header.crc64 = raw_crc; + + if(fwrite(&payload_header, sizeof(DataStreamPayloadHeader), 1, ctx->imageStream) != 1) + { + free(compressed_buffer); + free(raw_buffer); + FATAL("Could not write flux payload header"); + return AARUF_ERROR_CANNOT_WRITE_BLOCK_HEADER; + } + + if(cmp_length != 0 && compressed_buffer != NULL) + { + if(fwrite(compressed_buffer, cmp_length, 1, ctx->imageStream) != 1) + { + free(compressed_buffer); + free(raw_buffer); + FATAL("Could not write flux payload data"); + return AARUF_ERROR_CANNOT_WRITE_BLOCK_DATA; + } + } + + IndexEntry payload_entry; + payload_entry.blockType = DataStreamPayloadBlock; + payload_entry.dataType = FluxData; + payload_entry.offset = payload_position; + utarray_push_back(ctx->index_entries, &payload_entry); + + entry->head = record->entry.head; + entry->track = record->entry.track; + entry->subtrack = record->entry.subtrack; + entry->captureIndex = record->entry.captureIndex; + entry->dataResolution = record->entry.dataResolution; + entry->indexResolution = record->entry.indexResolution; + entry->indexOffset = record->data_length; + entry->payloadOffset = payload_position; + + record->entry = *entry; + + free(compressed_buffer); + free(raw_buffer); + + return AARUF_STATUS_OK; +} + +/** + * @brief Serialize all accumulated flux capture blocks to the image file. + * + * This function writes the complete flux capture data structure to the Aaru image file. + * Flux captures represent analog signal transitions recorded from floppy disk drives or + * other magnetic media, preserving timing information that enables bit-level analysis + * and advanced data recovery techniques that cannot be performed using sector data alone. + * + * The function processes all flux captures that have been enqueued in ctx->flux_captures + * during image creation. For each capture, it writes a DataStreamPayloadBlock containing the + * raw flux data and index buffers. After all payload blocks are written, it writes a + * FluxDataBlock containing metadata entries that enable efficient lookup and access to + * the flux captures. Finally, it rebuilds the in-memory flux capture lookup map for + * fast access during the remainder of the close operation. + * + * The flux capture system supports multiple captures per disk (identified by head, + * track, subtrack, and captureIndex), enabling preservation of multiple read attempts, + * different drive characteristics, or verification passes. + * + * **Block Structure:** + * The serialized flux capture structure consists of: + * ``` + * +------------------------------+ + * | DataStreamPayloadBlock 0 | <- First capture's data + index + * +------------------------------+ + * | DataStreamPayloadBlock 1 | <- Second capture's data + index + * +------------------------------+ + * | ... | + * +------------------------------+ + * | DataStreamPayloadBlock (n-1) | <- Last capture's data + index + * +------------------------------+ + * | FluxDataBlock | <- Metadata block with all entries + * | + FluxHeader (16 B) | identifier, entries count, crc64 + * | + FluxEntry 0 (32 B) | head, track, subtrack, offsets, etc. + * | + FluxEntry 1 (32 B) | + * | + ... | + * | + FluxEntry (n-1) (32 B) | + * +------------------------------+ + * ``` + * + * **Processing Flow:** + * 1. **Validation:** Check context validity and presence of flux captures + * 2. **Entry Count Validation:** Verify capture count doesn't exceed UINT16_MAX + * 3. **Entry Array Allocation:** Allocate temporary array for FluxEntry structures + * 4. **Payload Writing:** For each capture, call write_flux_capture_payload() to write + * the payload block and populate the corresponding FluxEntry + * 5. **Header Construction:** Build FluxHeader with entry count and CRC64 over entries + * 6. **Alignment:** Seek to EOF and align to block boundary (blockAlignmentShift) + * 7. **Metadata Write:** Write FluxHeader followed by FluxEntry array + * 8. **Indexing:** Add IndexEntry for FluxDataBlock to enable fast location during reads + * 9. **Context Update:** Update ctx->flux_entries and ctx->flux_data_header + * 10. **Cleanup:** Free ctx->flux_captures array (no longer needed) + * 11. **Map Rebuild:** Rebuild flux capture lookup map from entries for fast access + * + * **Flux Capture Organization:** + * - Each flux capture is identified by: head, track, subtrack, captureIndex + * - Multiple captures can exist for the same track (e.g., different read attempts) + * - The captureIndex allows distinguishing between multiple captures of the same location + * - Entries are written in the order they appear in ctx->flux_captures (insertion order) + * + * **CRC64 Integrity Protection:** + * A CRC64-ECMA checksum is computed over the complete array of FluxEntry structures + * using aaruf_crc64_data(). This checksum is stored in the FluxHeader and verified + * during image opening by process_flux_data_block() to detect corruption in the flux + * metadata. The checksum covers only the entry data, not the header itself. + * + * **Alignment Strategy:** + * Before writing the FluxDataBlock, the file position is: + * 1. Moved to EOF using fseek(SEEK_END) + * 2. Aligned forward to next boundary: (position + alignment_mask) & ~alignment_mask + * 3. Where alignment_mask = (1 << blockAlignmentShift) - 1 + * This ensures the flux data block starts on a properly aligned offset for efficient + * I/O and compliance with the Aaru format specification. Individual payload blocks + * are also aligned by write_flux_capture_payload(). + * + * **Write Sequence:** + * The function performs a multi-stage write operation: + * 1. Write all DataStreamPayloadBlock entries (one per capture, via write_flux_capture_payload) + * 2. Write FluxHeader (sizeof(FluxHeader) = 16 bytes) + * 3. Write FluxEntry array (capture_count * sizeof(FluxEntry) bytes) + * + * All writes must succeed for the index entry to be added. If any write fails, the + * function returns an error code and the caller (aaruf_close) handles cleanup. + * + * **Indexing:** + * On successful write of the FluxDataBlock, an IndexEntry is appended to ctx->indexEntries: + * - blockType = FluxDataBlock (identifies this as flux capture metadata) + * - dataType = 0 (flux data blocks have no subtype) + * - offset = file position where FluxHeader was written + * + * Individual DataStreamPayloadBlock entries are indexed by write_flux_capture_payload() with + * blockType=DataStreamPayloadBlock and dataType=FluxData. + * + * This index entry enables process_flux_data_block() to quickly locate the flux metadata + * during subsequent image opens without scanning the entire file. + * + * **Flux Capture Lookup Map:** + * After writing, the function calls flux_map_rebuild_from_entries() to rebuild the + * in-memory hash table that maps (head, track, subtrack, captureIndex) tuples to + * FluxEntry array indices. This map enables O(1) lookup of flux captures by their + * identifiers, which is used by aaruf_read_flux_capture() and other flux access functions. + * + * The map is stored in ctx->flux_capture_map and uses UTHASH for efficient hash table + * operations. The map is cleared and rebuilt from scratch each time this function is + * called, ensuring consistency with the newly written entries. + * + * **Error Handling:** + * The function returns error codes on failure: + * - AARUF_ERROR_NOT_AARUFORMAT: Invalid context or image stream + * - AARUF_ERROR_INCORRECT_DATA_SIZE: Capture count exceeds UINT16_MAX + * - AARUF_ERROR_NOT_ENOUGH_MEMORY: Failed to allocate entry array + * - Error codes propagated from write_flux_capture_payload() on payload write failure + * - Error codes propagated from flux_map_rebuild_from_entries() on map rebuild failure + * + * On error, allocated memory is freed and ctx->flux_entries is restored to its previous + * value (if any), ensuring the context remains in a consistent state. + * + * **Memory Management:** + * - Allocates temporary entries array sized to hold all FluxEntry structures + * - Previous ctx->flux_entries (if any) are preserved and restored on error + * - On success, previous entries are freed and replaced with new entries + * - ctx->flux_captures array is freed after successful write (no longer needed) + * - Individual payload buffers are managed by write_flux_capture_payload() + * + * **No-op Conditions:** + * If ctx->flux_captures is NULL or empty (utarray_len == 0), the function returns + * AARUF_STATUS_OK immediately without writing anything. This allows images without + * flux captures to be created normally. + * + * **Thread Safety:** + * This function is NOT thread-safe. It modifies shared ctx state (imageStream file + * position, indexEntries array, flux_entries, flux_data_header, flux_capture_map) and + * must only be called during single-threaded finalization (within aaruf_close). + * + * **Use Cases:** + * - Preserving analog flux transitions from floppy disk drives for forensic analysis + * - Enabling bit-level timing analysis and reconstruction + * - Supporting advanced data recovery from damaged or degraded media + * - Preserving drive-specific characteristics and read variations + * - Enabling research into magnetic media encoding and decoding + * - Supporting preservation of non-standard or copy-protected disk formats + * + * **Relationship to Other Functions:** + * - Flux captures are added via aaruf_write_flux_capture() during image creation + * - Captures are stored in ctx->flux_captures array until image close + * - This function serializes all captures to disk during aaruf_close() + * - process_flux_data_block() reads and reconstructs the entries during aaruf_open() + * - aaruf_read_flux_capture() uses the lookup map to access specific captures + * - flux_map_rebuild_from_entries() rebuilds the lookup map from entries + * + * **Format Considerations:** + * - The FluxDataBlock must be written after all DataStreamPayloadBlock entries + * - The entry count in FluxHeader must match the number of payload blocks written + * - Each FluxEntry's payloadOffset must point to a valid DataStreamPayloadBlock + * - The CRC64 in FluxHeader enables verification of entry array integrity + * - Multiple captures per track are supported via captureIndex differentiation + * + * @param ctx Pointer to an initialized aaruformatContext in write mode. Must not be NULL. + * ctx->flux_captures should contain the array of flux captures to serialize + * (may be NULL or empty if no flux captures were added). ctx->imageStream + * must be open and writable. ctx->indexEntries must be initialized (utarray) + * to accept new index entries. + * + * @return Returns one of the following status codes: + * @retval AARUF_STATUS_OK (0) Successfully wrote all flux capture blocks. This occurs when: + * - The context is valid and imageStream is open + * - All payload blocks were written successfully + * - The FluxDataBlock header and entries were written successfully + * - The flux capture lookup map was rebuilt successfully + * - Or when no flux captures were present (no-op success) + * + * @retval AARUF_ERROR_NOT_AARUFORMAT (-1) The context is invalid or imageStream is NULL. + * This indicates a programming error or corrupted context state. + * + * @retval AARUF_ERROR_INCORRECT_DATA_SIZE (-5) The number of flux captures exceeds + * UINT16_MAX (65535). This is a practical limit to keep the header size reasonable. + * + * @retval AARUF_ERROR_NOT_ENOUGH_MEMORY (-9) Memory allocation failed for the FluxEntry + * array. This can occur when there are extremely many flux captures or system + * memory is exhausted. + * + * @retval AARUF_ERROR_CANNOT_WRITE_BLOCK_HEADER (-22) Failed to write the FluxHeader. + * This can occur due to disk full, I/O errors, or stream corruption. + * + * @retval AARUF_ERROR_CANNOT_WRITE_BLOCK_DATA (-23) Failed to write the FluxEntry array. + * This can occur due to disk full, I/O errors, or stream corruption. + * + * @retval Propagated from write_flux_capture_payload() if any + * payload block write fails, or from flux_map_rebuild_from_entries() if map + * rebuild fails. + * + * @note The function preserves ctx->flux_entries if it already exists, freeing the + * previous array only after successful write of the new entries. This ensures + * that read operations can continue to use the old entries if the write fails. + * + * @note The flux capture lookup map is rebuilt even if it already exists, ensuring + * consistency with the newly written entries. The previous map is cleared + * by flux_map_rebuild_from_entries() before rebuilding. + * + * @note Entry ordering in the FluxDataBlock matches the order in ctx->flux_captures, + * which is typically insertion order. Applications reading flux captures should + * not rely on any specific ordering beyond what is guaranteed by the identifiers. + * + * @note The function does not validate that payloadOffset values in FluxEntry structures + * point to valid DataStreamPayloadBlock locations. This validation is performed during + * reading by process_flux_data_block() and aaruf_read_flux_capture(). + * + * @warning If write_flux_capture_payload() fails for any capture, this function + * immediately returns the error code without attempting to write remaining + * captures. Partial flux capture data may be present in the image, but the + * FluxDataBlock will not be written, making the flux data inaccessible. + * + * @warning Memory allocation failure for the entries array results in immediate return + * with an error code. No flux data is written in this case, even if some + * payload blocks were already written (though this should not occur as payload + * writing happens before entry array allocation). + * + * @warning The function modifies ctx->flux_entries, ctx->flux_data_header, and + * ctx->flux_capture_map. These modifications occur even if a later step fails, + * so the caller must handle cleanup if needed. However, the function does + * restore ctx->flux_entries to its previous value on error. + * + * @see FluxHeader for the metadata block header structure definition + * @see FluxEntry for individual flux capture entry structure definition + * @see DataStreamPayloadHeader for payload block header structure definition + * @see write_flux_capture_payload() for writing individual payload blocks + * @see flux_map_rebuild_from_entries() for rebuilding the lookup map + * @see process_flux_data_block() for reading flux metadata during image opening + * @see aaruf_write_flux_capture() for adding flux captures during image creation + * @see aaruf_read_flux_capture() for reading flux captures from opened images + * + * @internal + */ +static int32_t write_flux_blocks(aaruformat_context *ctx) +{ + TRACE("Entering write_flux_blocks(%p)", ctx); + + if(ctx == NULL || ctx->imageStream == NULL) + { + FATAL("Invalid context when writing flux blocks"); + return AARUF_ERROR_NOT_AARUFORMAT; + } + + if(ctx->flux_captures == NULL || utarray_len(ctx->flux_captures) == 0) + { + TRACE("No flux captures enqueued, skipping flux block serialization"); + return AARUF_STATUS_OK; + } + + size_t capture_count = utarray_len(ctx->flux_captures); + if(capture_count > UINT16_MAX) + { + FATAL("Flux capture count exceeds header capacity (%zu > %u)", capture_count, UINT16_MAX); + return AARUF_ERROR_INCORRECT_DATA_SIZE; + } + + FluxEntry *previous_entries = ctx->flux_entries; + ctx->flux_entries = NULL; + + FluxEntry *entries = malloc(capture_count * sizeof(FluxEntry)); + if(entries == NULL) + { + ctx->flux_entries = previous_entries; + FATAL("Could not allocate %zu bytes for flux entries", capture_count * sizeof(FluxEntry)); + return AARUF_ERROR_NOT_ENOUGH_MEMORY; + } + + size_t idx = 0; + for(FluxCaptureRecord *record = (FluxCaptureRecord *)utarray_front(ctx->flux_captures); record != NULL; + record = (FluxCaptureRecord *)utarray_next(ctx->flux_captures, record), ++idx) + { + int32_t res = write_flux_capture_payload(ctx, record, &entries[idx]); + if(res != AARUF_STATUS_OK) + { + free(entries); + ctx->flux_entries = previous_entries; + return res; + } + } + + FluxHeader header = {0}; + header.identifier = FluxDataBlock; + header.entries = (uint16_t)capture_count; + header.crc64 = + capture_count == 0 ? 0 : aaruf_crc64_data((const uint8_t *)entries, capture_count * sizeof(FluxEntry)); + + // Align stream position to block boundary + fseek(ctx->imageStream, 0, SEEK_END); + long metadata_position = ftell(ctx->imageStream); + const uint64_t alignment_mask = (1ULL << ctx->user_data_ddt_header.blockAlignmentShift) - 1; + if(metadata_position & alignment_mask) + { + const uint64_t aligned_position = metadata_position + alignment_mask & ~alignment_mask; + fseek(ctx->imageStream, aligned_position, SEEK_SET); + metadata_position = aligned_position; + } + + if(fwrite(&header, sizeof(FluxHeader), 1, ctx->imageStream) != 1) + { + free(entries); + ctx->flux_entries = previous_entries; + FATAL("Could not write flux metadata header"); + return AARUF_ERROR_CANNOT_WRITE_BLOCK_HEADER; + } + + if(capture_count != 0) + { + size_t written_entries = fwrite(entries, sizeof(FluxEntry), capture_count, ctx->imageStream); + if(written_entries != capture_count) + { + free(entries); + ctx->flux_entries = previous_entries; + FATAL("Could not write %zu flux entries (wrote %zu)", capture_count, written_entries); + return AARUF_ERROR_CANNOT_WRITE_BLOCK_DATA; + } + } + + IndexEntry metadata_entry; + metadata_entry.blockType = FluxDataBlock; + metadata_entry.dataType = 0; + metadata_entry.offset = metadata_position; + utarray_push_back(ctx->index_entries, &metadata_entry); + + if(previous_entries != NULL) free(previous_entries); + ctx->flux_entries = entries; + ctx->flux_data_header = header; + + utarray_free(ctx->flux_captures); + ctx->flux_captures = NULL; + + int32_t map_result = flux_map_rebuild_from_entries(ctx); + if(map_result != AARUF_STATUS_OK) return map_result; + + TRACE("Wrote %zu flux captures", capture_count); + return AARUF_STATUS_OK; +} + /** * @brief Serialize the accumulated index entries at the end of the image and back-patch the header. * @@ -4427,6 +5095,12 @@ AARU_EXPORT int AARU_CALL aaruf_close(void *context) // Write tracks block if(ctx->dirty_tracks_block) write_tracks_block(ctx); + // Write flux capture blocks + if(ctx->dirty_flux_block) { + res = write_flux_blocks(ctx); + if(res != AARUF_STATUS_OK) return res; + } + // Write MODE 2 subheader data block if(ctx->dirty_mode2_subheaders_block) write_mode2_subheaders_block(ctx); diff --git a/src/create.c b/src/create.c index cac155c..e8dddb6 100644 --- a/src/create.c +++ b/src/create.c @@ -587,6 +587,7 @@ AARU_EXPORT void AARU_CALL *aaruf_create(const char *filepath, const uint32_t me ctx->dirty_dumphw_block = true; ctx->dirty_cicm_block = true; ctx->dirty_json_block = true; + ctx->dirty_flux_block = true; ctx->dirty_index_block = true; TRACE("Exiting aaruf_create() = %p", ctx); diff --git a/src/open.c b/src/open.c index 86b2661..2d1976f 100644 --- a/src/open.c +++ b/src/open.c @@ -545,6 +545,10 @@ AARU_EXPORT void AARU_CALL *aaruf_open(const char *filepath, const bool resume_m case TapePartitionBlock: process_tape_partitions_block(ctx, entry); + break; + case FluxDataBlock: + process_flux_data_block(ctx, entry); + break; default: TRACE("Unhandled block type %4.4s with data type %d is indexed to be at %" PRIu64 "", diff --git a/templates/aaruformat.bt b/templates/aaruformat.bt index 1c58d6d..6f216a4 100644 --- a/templates/aaruformat.bt +++ b/templates/aaruformat.bt @@ -701,7 +701,9 @@ enum DataType DvdSectorEdc = 85, DvdSectorEccPi = 86, DvdEccBlockPo = 87, - DvdPfi2ndLayer = 88 + DvdPfi2ndLayer = 88, + FluxData = 89, + BitstreamData = 90 }; enum BlockType diff --git a/templates/aaruformat.hexpat b/templates/aaruformat.hexpat index cf8f8fd..f59fe65 100644 --- a/templates/aaruformat.hexpat +++ b/templates/aaruformat.hexpat @@ -727,8 +727,13 @@ enum DataType : u16 DecryptedDVDDiscKey = 80, DVD_CPI_MAI = 81, DecryptedDVDTitleKey = 82, - FluxData = 83, - BitstreamData = 84 + DvdSectorId = 83, + DvdSectorIed = 84, + DvdSectorEdc = 85, + DvdSectorEccPi = 86, + DvdEccBlockPo = 87, + FluxData = 88, + BitstreamData = 89 }; enum BlockType : u32 @@ -753,6 +758,7 @@ enum BlockType : u32 TwinSectorTable = 0x42545754, CompactDiscIndexesBlock = 0x58494443, FluxDataBlock = 0x58554C46, + DataStreamPayloadBlock = 0x4C505344, BitstreamDataBlock = 0x53544942, TrackLayoutBlock = 0x594C4B54, JsonMetadataBlock = 0x444D534A @@ -1130,6 +1136,47 @@ struct CompactDiscIndexesBlock CompactDiscIndexEntry indexes[entries]; }; +struct FluxEntry +{ + u32 head; + u16 track; + u8 subtrack; + u32 captureIndex; + u64 indexResolution; + u64 dataResolution; + u64 indexOffset; + u64 payloadOffset; +}; + +struct FluxHeader +{ + BlockType identifier; + u16 entries; + u64 crc64; +}; + +struct FluxDataBlock +{ + FluxHeader header; + FluxEntry entries[header.entries]; +}; + +struct DataStreamPayloadHeader +{ + BlockType identifier; + u16 compression; + u32 cmpLength; + u32 length; + u64 cmpCrc64; + u64 crc64; +}; + +struct DataStreamPayloadBlock +{ + DataStreamPayloadHeader header; + u8 payload[header.cmpLength]; +}; + using Index; struct IndexEntry @@ -1153,6 +1200,8 @@ struct IndexEntry (BlockType::TapePartitionBlock): TapePartitionHeader *tapePartitions : u64 [[inline]]; (BlockType::CompactDiscIndexesBlock): CompactDiscIndexesBlock *compactDiscIndexes : u64 [[inline]]; (BlockType::Index3): Index *index : u64 [[inline]]; + (BlockType::FluxDataBlock): FluxDataBlock *fluxData : u64 [[inline]]; + (BlockType::DataStreamPayloadBlock): DataStreamPayloadBlock *dataStreamPayload : u64 [[inline]]; (_): BlockType *offset : u64 [[inline]]; } }; diff --git a/tool/convert.c b/tool/convert.c index bccf386..a79546e 100644 --- a/tool/convert.c +++ b/tool/convert.c @@ -1286,6 +1286,146 @@ int convert(const char *input_path, const char *output_path, bool use_long) print_info("Time elapsed:", elapsed_str); snprintf(buffer, sizeof(buffer), "%s/sec", speed_str); print_info("Average speed:", buffer); + + // Copy flux captures from input to output + size_t flux_captures_length = 0; + res = aaruf_get_flux_captures(input_ctx, NULL, &flux_captures_length); + if(res == AARUF_ERROR_BUFFER_TOO_SMALL) + { + uint8_t *flux_captures = malloc(flux_captures_length); + if(flux_captures == NULL) + { + printf("\nError allocating memory for flux captures buffer.\n"); + free(sector_data); + aaruf_close(input_ctx); + aaruf_close(output_ctx); + return AARUF_ERROR_NOT_ENOUGH_MEMORY; + } + + res = aaruf_get_flux_captures(input_ctx, flux_captures, &flux_captures_length); + if(res == AARUF_STATUS_OK) + { + size_t capture_count = flux_captures_length / sizeof(FluxCaptureMeta); + FluxCaptureMeta *captures = (FluxCaptureMeta *)flux_captures; + uint8_t *index_data = NULL; + uint8_t *data_data = NULL; + uint32_t index_length = 0; + uint32_t data_length = 0; + + printf("Copying %zu flux captures...\n", capture_count); + + for(size_t i = 0; i < capture_count; i++) + { + // First call to get required buffer sizes + res = aaruf_read_flux_capture(input_ctx, captures[i].head, captures[i].track, captures[i].subtrack, + captures[i].captureIndex, NULL, &index_length, NULL, &data_length); + + if(res == AARUF_ERROR_BUFFER_TOO_SMALL) + { + // Allocate buffers + if(index_length > 0) + { + index_data = malloc(index_length); + if(index_data == NULL) + { + printf("Error allocating memory for flux index buffer.\n"); + free(flux_captures); + free(sector_data); + aaruf_close(input_ctx); + aaruf_close(output_ctx); + return AARUF_ERROR_NOT_ENOUGH_MEMORY; + } + } + + if(data_length > 0) + { + data_data = malloc(data_length); + if(data_data == NULL) + { + printf("Error allocating memory for flux data buffer.\n"); + free(index_data); + free(flux_captures); + free(sector_data); + aaruf_close(input_ctx); + aaruf_close(output_ctx); + return AARUF_ERROR_NOT_ENOUGH_MEMORY; + } + } + + // Read flux capture data + res = aaruf_read_flux_capture(input_ctx, captures[i].head, captures[i].track, captures[i].subtrack, + captures[i].captureIndex, index_data, &index_length, data_data, + &data_length); + + if(res == AARUF_STATUS_OK) + { + // Write flux capture to output + res = aaruf_write_flux_capture(output_ctx, captures[i].head, captures[i].track, + captures[i].subtrack, captures[i].captureIndex, + captures[i].dataResolution, captures[i].indexResolution, data_data, + data_length, index_data, index_length); + + if(res != AARUF_STATUS_OK) + { + printf("Error %d when writing flux capture (head=%u, track=%u, subtrack=%u, index=%u) to output image.\n", + res, captures[i].head, captures[i].track, captures[i].subtrack, + captures[i].captureIndex); + free(index_data); + free(data_data); + free(flux_captures); + free(sector_data); + aaruf_close(input_ctx); + aaruf_close(output_ctx); + return res; + } + } + else + { + printf("Error %d when reading flux capture (head=%u, track=%u, subtrack=%u, index=%u) from input image.\n", + res, captures[i].head, captures[i].track, captures[i].subtrack, captures[i].captureIndex); + free(index_data); + free(data_data); + free(flux_captures); + free(sector_data); + aaruf_close(input_ctx); + aaruf_close(output_ctx); + return res; + } + + free(index_data); + free(data_data); + index_data = NULL; + data_data = NULL; + index_length = 0; + data_length = 0; + } + else if(res != AARUF_ERROR_FLUX_DATA_NOT_FOUND) + { + printf("Error %d when reading flux capture (head=%u, track=%u, subtrack=%u, index=%u) from input image.\n", + res, captures[i].head, captures[i].track, captures[i].subtrack, captures[i].captureIndex); + free(flux_captures); + free(sector_data); + aaruf_close(input_ctx); + aaruf_close(output_ctx); + return res; + } + } + + printf("Successfully copied %zu flux captures.\n", capture_count); + } + else if(res != AARUF_ERROR_FLUX_DATA_NOT_FOUND) + { + printf("\nError %d when getting flux captures from input image.\n", res); + } + + free(flux_captures); + } + else if(res != AARUF_ERROR_FLUX_DATA_NOT_FOUND) + { + printf("\nError %d when getting flux captures from input image.\n", res); + } + + printf("\n"); // Clean up free(sector_data); diff --git a/tool/info.c b/tool/info.c index d9a7dbb..2de8c57 100644 --- a/tool/info.c +++ b/tool/info.c @@ -855,6 +855,10 @@ int info(const char *path) draw_box_bottom(COLOR_HEADER); } + if(ctx->flux_data_header.entries > 0) { + printf("Image contains %d flux captures.\n", ctx->flux_data_header.entries); + } + // Checksums Section bool hasChecksums = ctx->checksums.hasMd5 || ctx->checksums.hasSha1 || ctx->checksums.hasSha256 || ctx->checksums.hasSpamSum; From d644f81fe370f5b174f8ee8789b8f36efa579384 Mon Sep 17 00:00:00 2001 From: Rebecca Wallander Date: Sun, 21 Dec 2025 17:07:16 +0100 Subject: [PATCH 2/6] Address some Sonar issues --- src/blocks/flux.c | 542 ++++++++++++++++++++++++++++------------------ src/close.c | 22 +- tool/convert.c | 12 +- 3 files changed, 353 insertions(+), 223 deletions(-) diff --git a/src/blocks/flux.c b/src/blocks/flux.c index 4a88c4e..7a7bfec 100644 --- a/src/blocks/flux.c +++ b/src/blocks/flux.c @@ -85,7 +85,8 @@ static void flux_map_clear(aaruformat_context *ctx) { if(ctx->flux_map == NULL) return; - FluxCaptureMapEntry *entry, *tmp; + FluxCaptureMapEntry *entry; + FluxCaptureMapEntry *tmp; HASH_ITER(hh, ctx->flux_map, entry, tmp) { HASH_DEL(ctx->flux_map, entry); @@ -448,7 +449,7 @@ AARU_EXPORT int32_t AARU_CALL aaruf_get_flux_captures(void *context, uint8_t *bu return AARUF_ERROR_NOT_AARUFORMAT; } - aaruformat_context *ctx = context; + const aaruformat_context *ctx = context; // Not a libaaruformat context if(ctx->magic != AARU_MAGIC) @@ -466,11 +467,17 @@ AARU_EXPORT int32_t AARU_CALL aaruf_get_flux_captures(void *context, uint8_t *bu size_t required_length = ctx->flux_data_header.entries * sizeof(FluxCaptureMeta); - if(buffer == NULL || length == NULL || *length < required_length) + if(length == NULL) { - if(length) *length = required_length; TRACE("Buffer too small for flux captures, required %zu bytes", required_length); + TRACE("Exiting aaruf_get_flux_captures() = AARUF_ERROR_BUFFER_TOO_SMALL"); + return AARUF_ERROR_BUFFER_TOO_SMALL; + } + if(buffer == NULL || *length < required_length) + { + *length = required_length; + TRACE("Buffer too small for flux captures, required %zu bytes", required_length); TRACE("Exiting aaruf_get_flux_captures() = AARUF_ERROR_BUFFER_TOO_SMALL"); return AARUF_ERROR_BUFFER_TOO_SMALL; } @@ -751,6 +758,307 @@ AARU_EXPORT int32_t AARU_CALL aaruf_clear_flux_captures(void *context) return AARUF_STATUS_OK; } +/** + * @brief Find a flux entry by its identifier key. + * + * @param ctx Pointer to the aaruformat context containing flux data. Must not be NULL. + * @param key Pointer to the FluxCaptureKey identifying the flux capture. Must not be NULL. + * @return Pointer to the matching FluxEntry, or NULL if not found. + * @internal + */ +static const FluxEntry *find_flux_entry_by_key(const aaruformat_context *ctx, const FluxCaptureKey *key) +{ + // Try lookup map first (O(1)) + if(ctx->flux_map != NULL) + { + FluxCaptureMapEntry *map_entry = NULL; + HASH_FIND(hh, ctx->flux_map, key, sizeof(FluxCaptureKey), map_entry); + if(map_entry != NULL && map_entry->index < ctx->flux_data_header.entries) + return &ctx->flux_entries[map_entry->index]; + } + + // Fall back to linear search (O(n)) + for(uint32_t i = 0; i < ctx->flux_data_header.entries; i++) + { + const FluxEntry *entry = &ctx->flux_entries[i]; + if(entry->head == key->head && entry->track == key->track && entry->subtrack == key->subtrack && + entry->captureIndex == key->captureIndex) + return entry; + } + + return NULL; +} + +/** + * @brief Read and validate a flux payload block header from the image stream. + * + * @param ctx Pointer to the aaruformat context. Must not be NULL. + * @param payload_offset File offset where the payload block starts. + * @param header Output parameter for the read header. Must not be NULL. + * @return AARUF_STATUS_OK on success, or an error code on failure. + * @internal + */ +static int32_t read_flux_payload_header(const aaruformat_context *ctx, uint64_t payload_offset, + DataStreamPayloadHeader *header) +{ + if(fseek(ctx->imageStream, payload_offset, SEEK_SET) < 0) + { + FATAL("Could not seek to flux payload at offset %" PRIu64, payload_offset); + TRACE("Exiting read_flux_payload_header() = AARUF_ERROR_CANNOT_READ_BLOCK\n"); + return AARUF_ERROR_CANNOT_READ_BLOCK; + } + + long file_position = ftell(ctx->imageStream); + if(file_position < 0 || (uint64_t)file_position != payload_offset) + { + FATAL("Invalid flux payload position (expected %" PRIu64 ", got %ld)", payload_offset, file_position); + TRACE("Exiting read_flux_payload_header() = AARUF_ERROR_CANNOT_READ_BLOCK\n"); + return AARUF_ERROR_CANNOT_READ_BLOCK; + } + + size_t read_bytes = fread(header, 1, sizeof(DataStreamPayloadHeader), ctx->imageStream); + if(read_bytes != sizeof(DataStreamPayloadHeader)) + { + FATAL("Could not read flux payload header at offset %" PRIu64, payload_offset); + TRACE("Exiting read_flux_payload_header() = AARUF_ERROR_CANNOT_READ_BLOCK\n"); + return AARUF_ERROR_CANNOT_READ_BLOCK; + } + + if(header->identifier != DataStreamPayloadBlock) + { + FATAL("Incorrect identifier 0x%08" PRIx32 " for flux payload at offset %" PRIu64, header->identifier, + payload_offset); + TRACE("Exiting read_flux_payload_header() = AARUF_ERROR_CANNOT_READ_BLOCK\n"); + return AARUF_ERROR_CANNOT_READ_BLOCK; + } + + return AARUF_STATUS_OK; +} + +/** + * @brief Read an uncompressed flux payload from the image stream. + * + * @param ctx Pointer to the aaruformat context. Must not be NULL. + * @param cmp_length Compressed length (should equal raw_length for uncompressed). + * @param raw_length Uncompressed length. + * @param cmp_buffer Output parameter for compressed buffer pointer (caller must free). Can be NULL if allocation fails. + * @param payload Output parameter for payload buffer pointer (same as cmp_buffer for uncompressed). + * @return AARUF_STATUS_OK on success, or an error code on failure. + * @internal + */ +static int32_t read_uncompressed_payload(const aaruformat_context *ctx, size_t cmp_length, size_t raw_length, + uint8_t **cmp_buffer, uint8_t **payload) +{ + if(cmp_length != raw_length) + { + FATAL("Flux payload lengths mismatch for uncompressed block (cmp=%zu, raw=%zu)", cmp_length, raw_length); + TRACE("Exiting read_uncompressed_payload() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK\n"); + return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK; + } + + if(cmp_length == 0) + { + *cmp_buffer = NULL; + *payload = NULL; + return AARUF_STATUS_OK; + } + + *cmp_buffer = (uint8_t *)malloc(cmp_length); + if(*cmp_buffer == NULL) + { + FATAL("Could not allocate %zu bytes for flux payload", cmp_length); + TRACE("Exiting read_uncompressed_payload() = AARUF_ERROR_NOT_ENOUGH_MEMORY\n"); + return AARUF_ERROR_NOT_ENOUGH_MEMORY; + } + + size_t read_bytes = fread(*cmp_buffer, 1, cmp_length, ctx->imageStream); + if(read_bytes != cmp_length) + { + FATAL("Could not read %zu bytes of flux payload", cmp_length); + free(*cmp_buffer); + *cmp_buffer = NULL; + TRACE("Exiting read_uncompressed_payload() = AARUF_ERROR_CANNOT_READ_BLOCK\n"); + return AARUF_ERROR_CANNOT_READ_BLOCK; + } + + *payload = *cmp_buffer; + return AARUF_STATUS_OK; +} + +/** + * @brief Read and decompress an LZMA-compressed flux payload from the image stream. + * + * @param ctx Pointer to the aaruformat context. Must not be NULL. + * @param cmp_length Compressed length including LZMA properties. + * @param raw_length Expected uncompressed length. + * @param cmp_buffer Output parameter for compressed buffer pointer (caller must free). Can be NULL if allocation fails. + * @param payload Output parameter for decompressed payload buffer pointer (caller must free). Can be NULL if allocation fails. + * @return AARUF_STATUS_OK on success, or an error code on failure. + * @internal + */ +static int32_t read_lzma_compressed_payload(const aaruformat_context *ctx, size_t cmp_length, size_t raw_length, + uint8_t **cmp_buffer, uint8_t **payload) +{ + if(cmp_length <= LZMA_PROPERTIES_LENGTH) + { + FATAL("Flux payload compressed length %zu too small for LZMA", cmp_length); + TRACE("Exiting read_lzma_compressed_payload() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK\n"); + return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK; + } + + *cmp_buffer = (uint8_t *)malloc(cmp_length); + if(*cmp_buffer == NULL) + { + FATAL("Could not allocate %zu bytes for flux payload", cmp_length); + TRACE("Exiting read_lzma_compressed_payload() = AARUF_ERROR_NOT_ENOUGH_MEMORY\n"); + return AARUF_ERROR_NOT_ENOUGH_MEMORY; + } + + size_t read_bytes = fread(*cmp_buffer, 1, cmp_length, ctx->imageStream); + if(read_bytes != cmp_length) + { + FATAL("Could not read %zu bytes of flux payload", cmp_length); + free(*cmp_buffer); + *cmp_buffer = NULL; + TRACE("Exiting read_lzma_compressed_payload() = AARUF_ERROR_CANNOT_READ_BLOCK\n"); + return AARUF_ERROR_CANNOT_READ_BLOCK; + } + + if(raw_length == 0) + { + *payload = NULL; + return AARUF_STATUS_OK; + } + + *payload = (uint8_t *)malloc(raw_length); + if(*payload == NULL) + { + FATAL("Could not allocate %zu bytes for decompressed flux payload", raw_length); + free(*cmp_buffer); + *cmp_buffer = NULL; + TRACE("Exiting read_lzma_compressed_payload() = AARUF_ERROR_NOT_ENOUGH_MEMORY\n"); + return AARUF_ERROR_NOT_ENOUGH_MEMORY; + } + + size_t cmp_stream_len = cmp_length - LZMA_PROPERTIES_LENGTH; + size_t dst_len = raw_length; + size_t src_len = cmp_stream_len; + const uint8_t *cmp_props = *cmp_buffer; + const uint8_t *cmp_stream = *cmp_buffer + LZMA_PROPERTIES_LENGTH; + int32_t error_no = + aaruf_lzma_decode_buffer(*payload, &dst_len, cmp_stream, &src_len, cmp_props, LZMA_PROPERTIES_LENGTH); + if(error_no != 0 || dst_len != raw_length) + { + FATAL("LZMA decompression failed for flux payload (err=%d, dst=%zu/%zu)", error_no, dst_len, raw_length); + free(*payload); + free(*cmp_buffer); + *payload = NULL; + *cmp_buffer = NULL; + TRACE("Exiting read_lzma_compressed_payload() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK\n"); + return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK; + } + + return AARUF_STATUS_OK; +} + +/** + * @brief Validate CRC64 checksums for a flux payload block. + * + * @param cmp_buffer Compressed buffer to validate. Can be NULL if cmp_length is 0. + * @param cmp_length Compressed length. + * @param payload Uncompressed payload buffer to validate. Can be NULL if raw_length is 0. + * @param raw_length Uncompressed length. + * @param expected_cmp_crc Expected CRC64 of compressed data. + * @param expected_raw_crc Expected CRC64 of uncompressed data. + * @return AARUF_STATUS_OK on success, or AARUF_ERROR_INVALID_BLOCK_CRC on mismatch. + * @internal + */ +static int32_t validate_flux_payload_crcs(const uint8_t *cmp_buffer, size_t cmp_length, const uint8_t *payload, + size_t raw_length, uint64_t expected_cmp_crc, uint64_t expected_raw_crc) +{ + uint64_t cmp_crc = 0; + if(cmp_length != 0 && cmp_buffer != NULL) cmp_crc = aaruf_crc64_data(cmp_buffer, cmp_length); + + if(cmp_crc != expected_cmp_crc) + { + FATAL("Flux payload compressed CRC mismatch (expected 0x%" PRIx64 ", got 0x%" PRIx64 ")", expected_cmp_crc, + cmp_crc); + TRACE("Exiting validate_flux_payload_crcs() = AARUF_ERROR_INVALID_BLOCK_CRC\n"); + return AARUF_ERROR_INVALID_BLOCK_CRC; + } + + uint64_t raw_crc = 0; + if(raw_length != 0 && payload != NULL) raw_crc = aaruf_crc64_data(payload, raw_length); + + if(raw_crc != expected_raw_crc) + { + FATAL("Flux payload raw CRC mismatch (expected 0x%" PRIx64 ", got 0x%" PRIx64 ")", expected_raw_crc, raw_crc); + TRACE("Exiting validate_flux_payload_crcs() = AARUF_ERROR_INVALID_BLOCK_CRC\n"); + return AARUF_ERROR_INVALID_BLOCK_CRC; + } + + return AARUF_STATUS_OK; +} + +/** + * @brief Extract data and index buffers from decompressed payload and copy to output buffers. + * + * @param flux_entry Pointer to the flux entry containing index offset information. Must not be NULL. + * @param payload Pointer to the decompressed payload buffer. Can be NULL if raw_length is 0. + * @param raw_length Length of the decompressed payload. + * @param data_data Output buffer for data portion. Can be NULL for size query. + * @param data_length Input/output parameter for data buffer size/required size. Must not be NULL. + * @param index_data Output buffer for index portion. Can be NULL for size query. + * @param index_length Input/output parameter for index buffer size/required size. Must not be NULL. + * @return AARUF_STATUS_OK on success, or an error code on failure. + * @internal + */ +static int32_t extract_flux_data_buffers(const FluxEntry *flux_entry, const uint8_t *payload, size_t raw_length, + uint8_t *data_data, uint32_t *data_length, uint8_t *index_data, + uint32_t *index_length) +{ + if(flux_entry->indexOffset > raw_length) + { + FATAL("Flux index offset %" PRIu64 " beyond payload length %zu", flux_entry->indexOffset, raw_length); + TRACE("Exiting extract_flux_data_buffers() = AARUF_ERROR_INVALID_BLOCK_CRC\n"); + return AARUF_ERROR_INVALID_BLOCK_CRC; + } + + uint64_t data_length_required64 = flux_entry->indexOffset; + uint64_t index_length_required64 = raw_length - flux_entry->indexOffset; + + if(data_length_required64 > UINT32_MAX || index_length_required64 > UINT32_MAX) + { + FATAL("Flux payload section length exceeds 32-bit limits (data=%" PRIu64 ", index=%" PRIu64 ")", + data_length_required64, index_length_required64); + TRACE("Exiting extract_flux_data_buffers() = AARUF_ERROR_INCORRECT_DATA_SIZE\n"); + return AARUF_ERROR_INCORRECT_DATA_SIZE; + } + + uint32_t data_required = (uint32_t)data_length_required64; + uint32_t index_required = (uint32_t)index_length_required64; + + uint32_t data_capacity = *data_length; + uint32_t index_capacity = *index_length; + + *data_length = data_required; + *index_length = index_required; + + if(data_data == NULL || index_data == NULL || data_capacity < data_required || index_capacity < index_required) + { + TRACE("Returning required flux capture sizes (data=%u, index=%u)\n", data_required, index_required); + return AARUF_ERROR_BUFFER_TOO_SMALL; + } + + const uint8_t *index_ptr = payload ? payload + data_length_required64 : NULL; + const uint8_t *data_ptr = payload; + + if(data_required != 0 && data_ptr != NULL) memcpy(data_data, data_ptr, data_required); + if(index_required != 0 && index_ptr != NULL) memcpy(index_data, index_ptr, index_required); + + return AARUF_STATUS_OK; +} + /** * @brief Read a specific flux capture's data and index buffers from the image. * @@ -853,33 +1161,15 @@ AARU_EXPORT int32_t AARU_CALL aaruf_read_flux_capture(void *context, uint32_t he return AARUF_ERROR_BUFFER_TOO_SMALL; } - const FluxEntry *flux_entry = NULL; - FluxCaptureKey key = {head, track, subtrack, capture_index}; - - // Find the flux entry in the map. - if(ctx->flux_map != NULL) - { - FluxCaptureMapEntry *map_entry = NULL; - HASH_FIND(hh, ctx->flux_map, &key, sizeof(FluxCaptureKey), map_entry); - if(map_entry != NULL && map_entry->index < ctx->flux_data_header.entries) - flux_entry = &ctx->flux_entries[map_entry->index]; - } - - // If the flux entry is not found in the map, search the entries array for a match. - if(flux_entry == NULL) + if(ctx->imageStream == NULL) { - for(uint32_t i = 0; i < ctx->flux_data_header.entries; i++) - { - const FluxEntry *candidate = &ctx->flux_entries[i]; - if(candidate->head == head && candidate->track == track && candidate->subtrack == subtrack && - candidate->captureIndex == capture_index) - { - flux_entry = candidate; - break; - } - } + FATAL("Invalid image stream"); + TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_NOT_AARUFORMAT\n"); + return AARUF_ERROR_NOT_AARUFORMAT; } + FluxCaptureKey key = {head, track, subtrack, capture_index}; + const FluxEntry *flux_entry = find_flux_entry_by_key(ctx, &key); if(flux_entry == NULL) { TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_FLUX_DATA_NOT_FOUND\n"); @@ -890,44 +1180,12 @@ AARU_EXPORT int32_t AARU_CALL aaruf_read_flux_capture(void *context, uint32_t he flux_entry->head, flux_entry->track, flux_entry->subtrack, flux_entry->captureIndex, flux_entry->payloadOffset); - if(ctx->imageStream == NULL) - { - FATAL("Invalid image stream"); - TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_NOT_AARUFORMAT\n"); - return AARUF_ERROR_NOT_AARUFORMAT; - } - - if(fseek(ctx->imageStream, flux_entry->payloadOffset, SEEK_SET) < 0) - { - FATAL("Could not seek to flux payload at offset %" PRIu64, flux_entry->payloadOffset); - TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_CANNOT_READ_BLOCK\n"); - return AARUF_ERROR_CANNOT_READ_BLOCK; - } - - long file_position = ftell(ctx->imageStream); - if(file_position < 0 || (uint64_t)file_position != flux_entry->payloadOffset) - { - FATAL("Invalid flux payload position (expected %" PRIu64 ", got %ld)", flux_entry->payloadOffset, - file_position); - TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_CANNOT_READ_BLOCK\n"); - return AARUF_ERROR_CANNOT_READ_BLOCK; - } - DataStreamPayloadHeader payload_header; - size_t read_bytes = fread(&payload_header, 1, sizeof(DataStreamPayloadHeader), ctx->imageStream); - if(read_bytes != sizeof(DataStreamPayloadHeader)) - { - FATAL("Could not read flux payload header at offset %" PRIu64, flux_entry->payloadOffset); - TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_CANNOT_READ_BLOCK\n"); - return AARUF_ERROR_CANNOT_READ_BLOCK; - } - - if(payload_header.identifier != DataStreamPayloadBlock) + int32_t res = read_flux_payload_header(ctx, flux_entry->payloadOffset, &payload_header); + if(res != AARUF_STATUS_OK) { - FATAL("Incorrect identifier 0x%08" PRIx32 " for flux payload at offset %" PRIu64, payload_header.identifier, - flux_entry->payloadOffset); - TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_CANNOT_READ_BLOCK\n"); - return AARUF_ERROR_CANNOT_READ_BLOCK; + TRACE("Exiting aaruf_read_flux_capture() = %d\n", res); + return res; } const CompressionType compression = (CompressionType)payload_header.compression; @@ -938,90 +1196,11 @@ AARU_EXPORT int32_t AARU_CALL aaruf_read_flux_capture(void *context, uint32_t he if(compression == None) { - if(cmp_length != raw_length) - { - FATAL("Flux payload lengths mismatch for uncompressed block (cmp=%u, raw=%u)", payload_header.cmpLength, - payload_header.length); - TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK\n"); - return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK; - } - - if(cmp_length != 0) - { - cmp_buffer = (uint8_t *)malloc(cmp_length); - if(cmp_buffer == NULL) - { - FATAL("Could not allocate %zu bytes for flux payload", cmp_length); - TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_NOT_ENOUGH_MEMORY\n"); - return AARUF_ERROR_NOT_ENOUGH_MEMORY; - } - - read_bytes = fread(cmp_buffer, 1, cmp_length, ctx->imageStream); - if(read_bytes != cmp_length) - { - FATAL("Could not read %zu bytes of flux payload", cmp_length); - free(cmp_buffer); - TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_CANNOT_READ_BLOCK\n"); - return AARUF_ERROR_CANNOT_READ_BLOCK; - } - - payload = cmp_buffer; - } + res = read_uncompressed_payload(ctx, cmp_length, raw_length, &cmp_buffer, &payload); } else if(compression == Lzma) { - if(cmp_length <= LZMA_PROPERTIES_LENGTH) - { - FATAL("Flux payload compressed length %u too small for LZMA", payload_header.cmpLength); - TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK\n"); - return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK; - } - - cmp_buffer = (uint8_t *)malloc(cmp_length); - if(cmp_buffer == NULL) - { - FATAL("Could not allocate %zu bytes for flux payload", cmp_length); - TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_NOT_ENOUGH_MEMORY\n"); - return AARUF_ERROR_NOT_ENOUGH_MEMORY; - } - - read_bytes = fread(cmp_buffer, 1, cmp_length, ctx->imageStream); - if(read_bytes != cmp_length) - { - FATAL("Could not read %zu bytes of flux payload", cmp_length); - free(cmp_buffer); - TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_CANNOT_READ_BLOCK\n"); - return AARUF_ERROR_CANNOT_READ_BLOCK; - } - - if(raw_length != 0) - { - payload = (uint8_t *)malloc(raw_length); - if(payload == NULL) - { - FATAL("Could not allocate %zu bytes for decompressed flux payload", raw_length); - free(cmp_buffer); - TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_NOT_ENOUGH_MEMORY\n"); - return AARUF_ERROR_NOT_ENOUGH_MEMORY; - } - - size_t cmp_stream_len = cmp_length - LZMA_PROPERTIES_LENGTH; - size_t dst_len = raw_length; - size_t src_len = cmp_stream_len; - const uint8_t *cmp_props = cmp_buffer; - const uint8_t *cmp_stream = cmp_buffer + LZMA_PROPERTIES_LENGTH; - int32_t error_no = - aaruf_lzma_decode_buffer(payload, &dst_len, cmp_stream, &src_len, cmp_props, LZMA_PROPERTIES_LENGTH); - if(error_no != 0 || dst_len != raw_length) - { - FATAL("LZMA decompression failed for flux payload (err=%d, dst=%zu/%zu)", error_no, dst_len, - raw_length); - free(payload); - free(cmp_buffer); - TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK\n"); - return AARUF_ERROR_CANNOT_DECOMPRESS_BLOCK; - } - } + res = read_lzma_compressed_payload(ctx, cmp_length, raw_length, &cmp_buffer, &payload); } else { @@ -1030,79 +1209,32 @@ AARU_EXPORT int32_t AARU_CALL aaruf_read_flux_capture(void *context, uint32_t he return AARUF_ERROR_UNSUPPORTED_COMPRESSION; } - uint64_t cmp_crc = 0; - if(cmp_length != 0 && cmp_buffer != NULL) cmp_crc = aaruf_crc64_data(cmp_buffer, cmp_length); - if(cmp_length == 0) cmp_crc = 0; - - if(cmp_crc != payload_header.cmpCrc64) + if(res != AARUF_STATUS_OK) { - FATAL("Flux payload compressed CRC mismatch (expected 0x%" PRIx64 ", got 0x%" PRIx64 ")", - payload_header.cmpCrc64, cmp_crc); - if(payload != NULL && payload != cmp_buffer) free(payload); - if(cmp_buffer != NULL) free(cmp_buffer); - TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_INVALID_BLOCK_CRC\n"); - return AARUF_ERROR_INVALID_BLOCK_CRC; + TRACE("Exiting aaruf_read_flux_capture() = %d\n", res); + return res; } - uint64_t raw_crc = 0; - if(raw_length != 0 && payload != NULL) raw_crc = aaruf_crc64_data(payload, raw_length); - if(raw_length == 0) raw_crc = 0; - - if(raw_crc != payload_header.crc64) + res = validate_flux_payload_crcs(cmp_buffer, cmp_length, payload, raw_length, payload_header.cmpCrc64, + payload_header.crc64); + if(res != AARUF_STATUS_OK) { - FATAL("Flux payload raw CRC mismatch (expected 0x%" PRIx64 ", got 0x%" PRIx64 ")", payload_header.crc64, - raw_crc); if(payload != NULL && payload != cmp_buffer) free(payload); if(cmp_buffer != NULL) free(cmp_buffer); - TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_INVALID_BLOCK_CRC\n"); - return AARUF_ERROR_INVALID_BLOCK_CRC; + TRACE("Exiting aaruf_read_flux_capture() = %d\n", res); + return res; } - if(flux_entry->indexOffset > raw_length) + res = extract_flux_data_buffers(flux_entry, payload, raw_length, data_data, data_length, index_data, + index_length); + if(res != AARUF_STATUS_OK) { - FATAL("Flux index offset %" PRIu64 " beyond payload length %zu", flux_entry->indexOffset, raw_length); if(payload != NULL && payload != cmp_buffer) free(payload); if(cmp_buffer != NULL) free(cmp_buffer); - TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_INVALID_BLOCK_CRC\n"); - return AARUF_ERROR_INVALID_BLOCK_CRC; + TRACE("Exiting aaruf_read_flux_capture() = %d\n", res); + return res; } - uint64_t data_length_required64 = flux_entry->indexOffset; - uint64_t index_length_required64 = raw_length - flux_entry->indexOffset; - - if(data_length_required64 > UINT32_MAX || index_length_required64 > UINT32_MAX) - { - FATAL("Flux payload section length exceeds 32-bit limits (data=%" PRIu64 ", index=%" PRIu64 ")", - data_length_required64, index_length_required64); - if(payload != NULL && payload != cmp_buffer) free(payload); - if(cmp_buffer != NULL) free(cmp_buffer); - TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_INCORRECT_DATA_SIZE\n"); - return AARUF_ERROR_INCORRECT_DATA_SIZE; - } - - uint32_t data_required = (uint32_t)data_length_required64; - uint32_t index_required = (uint32_t)index_length_required64; - - uint32_t data_capacity = *data_length; - uint32_t index_capacity = *index_length; - - *data_length = data_required; - *index_length = index_required; - - if(data_data == NULL || index_data == NULL || data_capacity < data_required || index_capacity < index_required) - { - TRACE("Returning required flux capture sizes (data=%u, index=%u)\n", data_required, index_required); - if(payload != NULL && payload != cmp_buffer) free(payload); - if(cmp_buffer != NULL) free(cmp_buffer); - return AARUF_ERROR_BUFFER_TOO_SMALL; - } - - uint8_t *index_ptr = payload ? payload + data_length_required64 : NULL; - uint8_t *data_ptr = payload; - - if(data_required != 0 && data_ptr != NULL) memcpy(data_data, data_ptr, data_required); - if(index_required != 0 && index_ptr != NULL) memcpy(index_data, index_ptr, index_required); - if(payload != NULL && payload != cmp_buffer) free(payload); if(cmp_buffer != NULL) free(cmp_buffer); diff --git a/src/close.c b/src/close.c index 119ed1c..958662a 100644 --- a/src/close.c +++ b/src/close.c @@ -4380,7 +4380,7 @@ static int32_t write_flux_capture_payload(aaruformat_context *ctx, FluxCaptureRe uint8_t *raw_buffer = NULL; if(raw_length != 0) { - raw_buffer = malloc((size_t)raw_length); + raw_buffer = malloc(raw_length); if(raw_buffer == NULL) { FATAL("Could not allocate %" PRIu64 " bytes for flux serialization", raw_length); @@ -4392,7 +4392,7 @@ static int32_t write_flux_capture_payload(aaruformat_context *ctx, FluxCaptureRe memcpy(raw_buffer + data_length, record->index_buffer, index_length); } - uint64_t raw_crc = raw_length != 0 && raw_buffer != NULL ? aaruf_crc64_data(raw_buffer, (size_t)raw_length) : 0; + uint64_t raw_crc = raw_length != 0 && raw_buffer != NULL ? aaruf_crc64_data(raw_buffer, raw_length) : 0; CompressionType compression = ctx->compression_enabled ? Lzma : None; @@ -4402,7 +4402,7 @@ static int32_t write_flux_capture_payload(aaruformat_context *ctx, FluxCaptureRe if(compression == Lzma) { - size_t cmp_capacity = raw_length ? (size_t)raw_length * 2 + 65536 : LZMA_PROPERTIES_LENGTH + 16; + size_t cmp_capacity = raw_length ? raw_length * 2 + 65536 : LZMA_PROPERTIES_LENGTH + 16; if(cmp_capacity < raw_length + LZMA_PROPERTIES_LENGTH) cmp_capacity = raw_length + LZMA_PROPERTIES_LENGTH; uint8_t *cmp_stream = malloc(cmp_capacity); @@ -4416,7 +4416,7 @@ static int32_t write_flux_capture_payload(aaruformat_context *ctx, FluxCaptureRe size_t dst_size = cmp_capacity; uint8_t lzma_props[LZMA_PROPERTIES_LENGTH] = {0}; size_t props_size = LZMA_PROPERTIES_LENGTH; - int32_t error_no = aaruf_lzma_encode_buffer(cmp_stream, &dst_size, raw_buffer, (size_t)raw_length, lzma_props, + int32_t error_no = aaruf_lzma_encode_buffer(cmp_stream, &dst_size, raw_buffer, raw_length, lzma_props, &props_size, 9, ctx->lzma_dict_size, 4, 0, 2, 273, 8); if(error_no != 0 || props_size != LZMA_PROPERTIES_LENGTH || dst_size >= raw_length) @@ -4482,15 +4482,13 @@ static int32_t write_flux_capture_payload(aaruformat_context *ctx, FluxCaptureRe return AARUF_ERROR_CANNOT_WRITE_BLOCK_HEADER; } - if(cmp_length != 0 && compressed_buffer != NULL) + if(cmp_length != 0 && compressed_buffer != NULL && + fwrite(compressed_buffer, cmp_length, 1, ctx->imageStream) != 1) { - if(fwrite(compressed_buffer, cmp_length, 1, ctx->imageStream) != 1) - { - free(compressed_buffer); - free(raw_buffer); - FATAL("Could not write flux payload data"); - return AARUF_ERROR_CANNOT_WRITE_BLOCK_DATA; - } + free(compressed_buffer); + free(raw_buffer); + FATAL("Could not write flux payload data"); + return AARUF_ERROR_CANNOT_WRITE_BLOCK_DATA; } IndexEntry payload_entry; diff --git a/tool/convert.c b/tool/convert.c index a79546e..f941205 100644 --- a/tool/convert.c +++ b/tool/convert.c @@ -1305,12 +1305,12 @@ int convert(const char *input_path, const char *output_path, bool use_long) res = aaruf_get_flux_captures(input_ctx, flux_captures, &flux_captures_length); if(res == AARUF_STATUS_OK) { - size_t capture_count = flux_captures_length / sizeof(FluxCaptureMeta); - FluxCaptureMeta *captures = (FluxCaptureMeta *)flux_captures; - uint8_t *index_data = NULL; - uint8_t *data_data = NULL; - uint32_t index_length = 0; - uint32_t data_length = 0; + size_t capture_count = flux_captures_length / sizeof(FluxCaptureMeta); + const FluxCaptureMeta *captures = (FluxCaptureMeta *)flux_captures; + uint8_t *index_data = NULL; + uint8_t *data_data = NULL; + uint32_t index_length = 0; + uint32_t data_length = 0; printf("Copying %zu flux captures...\n", capture_count); From 588b35472563332b25f856715e9c7e8498f71d49 Mon Sep 17 00:00:00 2001 From: Rebecca Wallander Date: Thu, 1 Jan 2026 00:25:44 +0100 Subject: [PATCH 3/6] Print more verbose info for flux images with tool --- tool/info.c | 108 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/tool/info.c b/tool/info.c index 2de8c57..9324c76 100644 --- a/tool/info.c +++ b/tool/info.c @@ -857,6 +857,114 @@ int info(const char *path) if(ctx->flux_data_header.entries > 0) { printf("Image contains %d flux captures.\n", ctx->flux_data_header.entries); + + // Get flux capture metadata + size_t flux_captures_length = 0; + int32_t res = aaruf_get_flux_captures(ctx, NULL, &flux_captures_length); + if(res == AARUF_ERROR_BUFFER_TOO_SMALL) + { + uint8_t *flux_captures = malloc(flux_captures_length); + if(flux_captures != NULL) + { + res = aaruf_get_flux_captures(ctx, flux_captures, &flux_captures_length); + if(res == AARUF_STATUS_OK) + { + size_t capture_count = flux_captures_length / sizeof(FluxCaptureMeta); + const FluxCaptureMeta *captures = (const FluxCaptureMeta *)flux_captures; + + printf("Flux capture details:\n"); + + // Calculate statistics + uint64_t min_index_res = UINT64_MAX; + uint64_t max_index_res = 0; + uint64_t min_data_res = UINT64_MAX; + uint64_t max_data_res = 0; + + // Track unique heads and tracks + uint16_t *seen_tracks = calloc(capture_count, sizeof(uint16_t)); + uint32_t *seen_heads = calloc(capture_count, sizeof(uint32_t)); + uint32_t track_count = 0; + uint32_t head_count = 0; + + for(size_t i = 0; i < capture_count; i++) + { + // Update resolution statistics + if(captures[i].indexResolution < min_index_res) min_index_res = captures[i].indexResolution; + if(captures[i].indexResolution > max_index_res) max_index_res = captures[i].indexResolution; + if(captures[i].dataResolution < min_data_res) min_data_res = captures[i].dataResolution; + if(captures[i].dataResolution > max_data_res) max_data_res = captures[i].dataResolution; + + // Track unique tracks + bool track_seen = false; + for(uint32_t j = 0; j < track_count; j++) + { + if(seen_tracks[j] == captures[i].track) + { + track_seen = true; + break; + } + } + if(!track_seen) + { + seen_tracks[track_count++] = captures[i].track; + } + + // Track unique heads + bool head_seen = false; + for(uint32_t j = 0; j < head_count; j++) + { + if(seen_heads[j] == captures[i].head) + { + head_seen = true; + break; + } + } + if(!head_seen) + { + seen_heads[head_count++] = captures[i].head; + } + } + + free(seen_tracks); + free(seen_heads); + + // Display statistics + printf("\tStatistics:\n"); + printf("\t\tUnique heads: %u\n", head_count); + printf("\t\tUnique tracks: %u\n", track_count); + if(min_index_res != UINT64_MAX) + { + printf("\t\tIndex resolution: %llu - %llu picoseconds (%.3f - %.3f nanoseconds)\n", + (unsigned long long)min_index_res, (unsigned long long)max_index_res, + min_index_res / 1000.0, max_index_res / 1000.0); + } + if(min_data_res != UINT64_MAX) + { + printf("\t\tData resolution: %llu - %llu picoseconds (%.3f - %.3f nanoseconds)\n", + (unsigned long long)min_data_res, (unsigned long long)max_data_res, + min_data_res / 1000.0, max_data_res / 1000.0); + } + + // Display individual captures + printf("\tCaptures:\n"); + for(size_t i = 0; i < capture_count; i++) + { + printf("\t\tCapture %zu:\n", i + 1); + printf("\t\t\tHead: %u\n", captures[i].head); + printf("\t\t\tTrack: %u\n", captures[i].track); + printf("\t\t\tSubtrack: %u\n", captures[i].subtrack); + printf("\t\t\tCapture index: %u\n", captures[i].captureIndex); + printf("\t\t\tIndex resolution: %llu picoseconds (%.3f nanoseconds)\n", + (unsigned long long)captures[i].indexResolution, + captures[i].indexResolution / 1000.0); + printf("\t\t\tData resolution: %llu picoseconds (%.3f nanoseconds)\n", + (unsigned long long)captures[i].dataResolution, + captures[i].dataResolution / 1000.0); + } + } + free(flux_captures); + } + } } // Checksums Section From 86b6680e3eebf17386fe977e0b339031caa4336f Mon Sep 17 00:00:00 2001 From: Rebecca Wallander Date: Thu, 1 Jan 2026 00:27:32 +0100 Subject: [PATCH 4/6] Add data type to datastream block --- docs/spec/blocks/datastream_payload.adoc | 6 ++++++ include/aaruformat/structs/flux.h | 7 +++++++ src/blocks/flux.c | 8 ++++++++ src/close.c | 1 + templates/aaruformat.hexpat | 1 + 5 files changed, 23 insertions(+) diff --git a/docs/spec/blocks/datastream_payload.adoc b/docs/spec/blocks/datastream_payload.adoc index c8767c1..2cc138c 100644 --- a/docs/spec/blocks/datastream_payload.adoc +++ b/docs/spec/blocks/datastream_payload.adoc @@ -19,6 +19,7 @@ For other data types, the payload layout is specific to the data type. typedef struct DataStreamPayloadHeader { uint32_t identifier; ///< Block identifier, must be BlockType::DataStreamPayloadBlock (0x4C505344). + uint16_t dataType; ///< Data type classification (value from DataType), e.g., FluxData or BitstreamData. uint16_t compression; ///< Compression type (0 = None, 1 = Lzma). uint32_t cmpLength; ///< Compressed length in bytes (includes LZMA properties if compressed). uint32_t length; ///< Uncompressed length in bytes. @@ -40,6 +41,11 @@ typedef struct DataStreamPayloadHeader |identifier |The data stream payload block identifier, always `DSPL` (`0x4C505344`) +|uint16_t +|2 bytes +|dataType +|The data type contained in this block (value from DataType), e.g., FluxData or BitstreamData. See Annex B. + |uint16_t |2 bytes |compression diff --git a/include/aaruformat/structs/flux.h b/include/aaruformat/structs/flux.h index 61befd9..f8f8f08 100644 --- a/include/aaruformat/structs/flux.h +++ b/include/aaruformat/structs/flux.h @@ -201,6 +201,12 @@ typedef struct FluxCaptureMapEntry FluxCaptureMapEntry; * may be compressed using LZMA compression. Currently used for flux capture data, but * can be used for other data streams such as bitstreams or PNG data. * + * **Data Type:** + * The dataType field identifies the type of data stored in the payload. Common values include: + * - FluxData: Flux capture data (data and index buffers) + * - BitstreamData: Bitstream data + * This field enables the block to be self-describing and allows validation of the payload content. + * * **Compression:** * The compression field indicates whether the payload is compressed: * - 0 (None): Payload is stored uncompressed @@ -226,6 +232,7 @@ typedef struct FluxCaptureMapEntry FluxCaptureMapEntry; typedef struct DataStreamPayloadHeader { uint32_t identifier; ///< Block identifier, must be BlockType::DataStreamPayloadBlock (0x4C505344, "DSPL"). + uint16_t dataType; ///< Data type classification (value from \ref DataType), e.g., FluxData or BitstreamData. uint16_t compression; ///< Compression type: 0 = None, 1 = Lzma. uint32_t cmpLength; ///< Compressed length in bytes (includes LZMA properties if compression = Lzma). uint32_t length; ///< Uncompressed length in bytes. diff --git a/src/blocks/flux.c b/src/blocks/flux.c index 7a7bfec..23470db 100644 --- a/src/blocks/flux.c +++ b/src/blocks/flux.c @@ -832,6 +832,14 @@ static int32_t read_flux_payload_header(const aaruformat_context *ctx, uint64_t return AARUF_ERROR_CANNOT_READ_BLOCK; } + if(header->dataType != FluxData) + { + FATAL("Incorrect data type %u for flux payload at offset %" PRIu64 " (expected FluxData)", header->dataType, + payload_offset); + TRACE("Exiting read_flux_payload_header() = AARUF_ERROR_CANNOT_READ_BLOCK\n"); + return AARUF_ERROR_CANNOT_READ_BLOCK; + } + return AARUF_STATUS_OK; } diff --git a/src/close.c b/src/close.c index 958662a..5d15f30 100644 --- a/src/close.c +++ b/src/close.c @@ -4468,6 +4468,7 @@ static int32_t write_flux_capture_payload(aaruformat_context *ctx, FluxCaptureRe DataStreamPayloadHeader payload_header = {0}; payload_header.identifier = DataStreamPayloadBlock; + payload_header.dataType = FluxData; payload_header.compression = (uint16_t)compression; payload_header.cmpLength = cmp_length; payload_header.length = (uint32_t)raw_length; diff --git a/templates/aaruformat.hexpat b/templates/aaruformat.hexpat index f59fe65..adffe1d 100644 --- a/templates/aaruformat.hexpat +++ b/templates/aaruformat.hexpat @@ -1164,6 +1164,7 @@ struct FluxDataBlock struct DataStreamPayloadHeader { BlockType identifier; + u16 dataType; u16 compression; u32 cmpLength; u32 length; From c412030a8bd65d9891e246c921e57378ebef8079 Mon Sep 17 00:00:00 2001 From: Rebecca Wallander Date: Thu, 1 Jan 2026 12:51:26 +0100 Subject: [PATCH 5/6] Use block alignment offset instead of absolute offset --- docs/spec/blocks/flux.adoc | 16 +++++++++++----- include/aaruformat/structs/flux.h | 24 +++++++++++++++++------- src/blocks/flux.c | 16 ++++++++++++---- src/close.c | 20 ++++++++++++-------- templates/aaruformat.hexpat | 1 + 5 files changed, 53 insertions(+), 24 deletions(-) diff --git a/docs/spec/blocks/flux.adoc b/docs/spec/blocks/flux.adoc index 8d0aab3..de9bfdf 100644 --- a/docs/spec/blocks/flux.adoc +++ b/docs/spec/blocks/flux.adoc @@ -19,7 +19,7 @@ The flux capture system uses two block types: * `DataStreamPayloadBlock` (`DSPL` / `0x4C505344`): Contains the actual flux data payload (data and index buffers) for individual captures. This is a generic block type that can also be used for other data streams. Each flux capture has one entry in the `FluxDataBlock` and one corresponding `DataStreamPayloadBlock` containing its data. -The `FluxEntry` structure in the data block contains a `payloadOffset` field that points to the file offset where the corresponding `DataStreamPayloadBlock` is stored. +The `FluxEntry` structure in the data block contains a `payloadOffset` field that points to the file offset where the corresponding `DataStreamPayloadBlock` is stored. The offset is stored divided by the block alignment (as indicated by `blockAlignmentShift` in the `FluxHeader`), consistent with DDT table offset storage. ==== Structure Definition @@ -28,9 +28,10 @@ The `FluxEntry` structure in the data block contains a `payloadOffset` field tha /** Flux data block header */ typedef struct FluxHeader { - uint32_t identifier; ///< Block identifier, must be BlockType::FluxDataBlock (0x58554C46). - uint16_t entries; ///< Number of FluxEntry records following this header. - uint64_t crc64; ///< CRC64-ECMA checksum of the FluxEntry array (header excluded). + uint32_t identifier; ///< Block identifier, must be BlockType::FluxDataBlock (0x58554C46). + uint16_t entries; ///< Number of FluxEntry records following this header. + uint8_t blockAlignmentShift; ///< Block alignment shift: 2^blockAlignmentShift = block alignment boundary in bytes. + uint64_t crc64; ///< CRC64-ECMA checksum of the FluxEntry array (header excluded). } FluxHeader; ==== Field Descriptions @@ -52,6 +53,11 @@ typedef struct FluxHeader |entries |The number of flux entry records following this header +|uint8_t +|1 byte +|blockAlignmentShift +|Block alignment shift: 2^blockAlignmentShift = block alignment boundary in bytes. Used to decode payloadOffset values in FluxEntry structures, which are stored divided by this alignment. Stored in the header to make the block self-contained, similar to DDT headers. + |uint64_t |8 bytes |crc64 @@ -121,5 +127,5 @@ typedef struct FluxEntry |uint64_t |8 bytes |payloadOffset -|File offset where the DataStreamPayloadBlock containing this capture's data is stored. +|Block-aligned file offset where the DataStreamPayloadBlock containing this capture's data is stored, divided by (1 << blockAlignmentShift). To get the absolute file offset, multiply by (1 << blockAlignmentShift) using the blockAlignmentShift value from FluxHeader. This storage method is consistent with DDT table offset storage. |=== \ No newline at end of file diff --git a/include/aaruformat/structs/flux.h b/include/aaruformat/structs/flux.h index f8f8f08..dff2fa7 100644 --- a/include/aaruformat/structs/flux.h +++ b/include/aaruformat/structs/flux.h @@ -103,14 +103,22 @@ * data integrity. The identifier field must match BlockType::FluxDataBlock * (0x58554C46, "FLUX" in ASCII). * + * The blockAlignmentShift field stores the block alignment shift value used to + * decode the payloadOffset values in FluxEntry structures. This makes the block + * self-contained, similar to DDT headers, allowing correct decoding of offsets + * without requiring access to the main image header. + * * @note Only one FluxDataBlock is allowed per image. * @note The entries field is limited to UINT16_MAX (65535) captures per image. + * @note The blockAlignmentShift field is stored in the header to support reliable + * decoding of payloadOffset values, which are stored divided by (1 << blockAlignmentShift). */ typedef struct FluxHeader { - uint32_t identifier; ///< Block identifier, must be BlockType::FluxDataBlock (0x58554C46, "FLUX"). - uint16_t entries; ///< Number of FluxEntry records following this header. Maximum value: 65535. - uint64_t crc64; ///< CRC64-ECMA checksum of the FluxEntry array (header excluded). + uint32_t identifier; ///< Block identifier, must be BlockType::FluxDataBlock (0x58554C46, "FLUX"). + uint16_t entries; ///< Number of FluxEntry records following this header. Maximum value: 65535. + uint8_t blockAlignmentShift; ///< Block alignment shift: 2^blockAlignmentShift = block alignment boundary in bytes. + uint64_t crc64; ///< CRC64-ECMA checksum of the FluxEntry array (header excluded). } FluxHeader; /** @@ -128,9 +136,11 @@ typedef struct FluxHeader * * **Payload Access:** * The payloadOffset field points to the file offset where the corresponding - * DataStreamPayloadBlock is stored. The indexOffset field indicates where the index - * buffer starts within the payload (the payload is stored as [data_buffer][index_buffer] - * concatenated). + * DataStreamPayloadBlock is stored. The offset is stored divided by the block alignment + * (blockAlignmentShift from FluxHeader), consistent with DDT table offset storage. + * To convert to an absolute file offset, multiply by (1 << blockAlignmentShift). + * The indexOffset field indicates where the index buffer starts within the payload + * (the payload is stored as [data_buffer][index_buffer] concatenated). * * **Resolution:** * Both indexResolution and dataResolution are specified in picoseconds, indicating @@ -151,7 +161,7 @@ typedef struct FluxEntry uint64_t indexResolution; ///< Resolution in picoseconds at which the index stream was sampled. uint64_t dataResolution; ///< Resolution in picoseconds at which the data stream was sampled. uint64_t indexOffset; ///< Byte offset within the payload where the index buffer starts (equals data_length). - uint64_t payloadOffset; ///< File offset where the DataStreamPayloadBlock containing this capture's data is stored. + uint64_t payloadOffset; ///< Block-aligned file offset where the DataStreamPayloadBlock containing this capture's data is stored, divided by (1 << blockAlignmentShift). To get the absolute offset, multiply by (1 << blockAlignmentShift) from FluxHeader. } FluxEntry; /** diff --git a/src/blocks/flux.c b/src/blocks/flux.c index 23470db..c808efc 100644 --- a/src/blocks/flux.c +++ b/src/blocks/flux.c @@ -1073,9 +1073,10 @@ static int32_t extract_flux_data_buffers(const FluxEntry *flux_entry, const uint * This function retrieves the actual flux data and index buffers for a specific * flux capture identified by its head, track, subtrack, and capture_index. The * function locates the corresponding FluxEntry in the flux_entries array (using - * the lookup map for efficiency), seeks to the DataStreamPayloadBlock at the specified - * payloadOffset, reads and decompresses the payload, validates CRC64 checksums, - * and extracts the data and index buffers. + * the lookup map for efficiency), converts the payloadOffset (stored divided by block alignment) + * to an absolute file offset using blockAlignmentShift from FluxHeader, seeks to the + * DataStreamPayloadBlock at the specified offset, reads and decompresses the payload, + * validates CRC64 checksums, and extracts the data and index buffers. * * The function supports both uncompressed and LZMA-compressed payload blocks. * CRC64 validation is performed on both the compressed and uncompressed data. @@ -1188,8 +1189,15 @@ AARU_EXPORT int32_t AARU_CALL aaruf_read_flux_capture(void *context, uint32_t he flux_entry->head, flux_entry->track, flux_entry->subtrack, flux_entry->captureIndex, flux_entry->payloadOffset); + // Get block alignment shift from FluxHeader + uint8_t block_alignment_shift = ctx->flux_data_header.blockAlignmentShift; + + // Convert payloadOffset from block-aligned units to absolute file offset + // payloadOffset is stored divided by (1 << blockAlignmentShift), consistent with DDT + uint64_t absolute_payload_offset = flux_entry->payloadOffset << block_alignment_shift; + DataStreamPayloadHeader payload_header; - int32_t res = read_flux_payload_header(ctx, flux_entry->payloadOffset, &payload_header); + int32_t res = read_flux_payload_header(ctx, absolute_payload_offset, &payload_header); if(res != AARUF_STATUS_OK) { TRACE("Exiting aaruf_read_flux_capture() = %d\n", res); diff --git a/src/close.c b/src/close.c index 5d15f30..9b6165c 100644 --- a/src/close.c +++ b/src/close.c @@ -4268,7 +4268,8 @@ static void write_aaru_json_block(aaruformat_context *ctx) * - head, track, subtrack, captureIndex: Identifiers from the record * - dataResolution, indexResolution: Timing resolution metadata * - indexOffset: Offset within payload where index buffer starts (equals data_length) - * - payloadOffset: File offset where this payload block was written + * - payloadOffset: Block-aligned file offset divided by (1 << blockAlignmentShift), + * consistent with DDT offset storage. Multiply by (1 << blockAlignmentShift) to get absolute offset. * * This metadata is later written to the FluxDataBlock to enable efficient lookup * of flux captures by their identifiers. @@ -4338,9 +4339,11 @@ static void write_aaru_json_block(aaruformat_context *ctx) * If compression doesn't reduce size, the raw data is written instead. This * prevents wasting space on incompressible flux data. * - * @note The payloadOffset stored in the FluxEntry enables direct seeking to the payload - * during reads without requiring a full index scan. This is critical for efficient - * random access to flux captures. + * @note The payloadOffset stored in the FluxEntry is divided by block alignment + * (blockAlignmentShift), consistent with DDT table offset storage. It enables direct + * seeking to the payload during reads without requiring a full index scan. This is + * critical for efficient random access to flux captures. The blockAlignmentShift + * value is stored in the FluxHeader to allow correct decoding. * * @note LZMA properties are written immediately after the header when compression is * enabled. The properties are included in cmpLength but not in the separate @@ -4505,7 +4508,7 @@ static int32_t write_flux_capture_payload(aaruformat_context *ctx, FluxCaptureRe entry->dataResolution = record->entry.dataResolution; entry->indexResolution = record->entry.indexResolution; entry->indexOffset = record->data_length; - entry->payloadOffset = payload_position; + entry->payloadOffset = payload_position >> ctx->user_data_ddt_header.blockAlignmentShift; record->entry = *entry; @@ -4668,7 +4671,7 @@ static int32_t write_flux_capture_payload(aaruformat_context *ctx, FluxCaptureRe * **Format Considerations:** * - The FluxDataBlock must be written after all DataStreamPayloadBlock entries * - The entry count in FluxHeader must match the number of payload blocks written - * - Each FluxEntry's payloadOffset must point to a valid DataStreamPayloadBlock + * - Each FluxEntry's payloadOffset must point to a valid DataStreamPayloadBlock (stored divided by block alignment) * - The CRC64 in FluxHeader enables verification of entry array integrity * - Multiple captures per track are supported via captureIndex differentiation * @@ -4796,8 +4799,9 @@ static int32_t write_flux_blocks(aaruformat_context *ctx) } FluxHeader header = {0}; - header.identifier = FluxDataBlock; - header.entries = (uint16_t)capture_count; + header.identifier = FluxDataBlock; + header.entries = (uint16_t)capture_count; + header.blockAlignmentShift = ctx->user_data_ddt_header.blockAlignmentShift; header.crc64 = capture_count == 0 ? 0 : aaruf_crc64_data((const uint8_t *)entries, capture_count * sizeof(FluxEntry)); diff --git a/templates/aaruformat.hexpat b/templates/aaruformat.hexpat index adffe1d..2e9a73d 100644 --- a/templates/aaruformat.hexpat +++ b/templates/aaruformat.hexpat @@ -1152,6 +1152,7 @@ struct FluxHeader { BlockType identifier; u16 entries; + u8 blockAlignmentShift; u64 crc64; }; From 869376604c78a5cf81011834b1910a79ad350e67 Mon Sep 17 00:00:00 2001 From: Rebecca Wallander Date: Thu, 1 Jan 2026 13:14:28 +0100 Subject: [PATCH 6/6] Use lazy loading of flux captures --- src/blocks/flux.c | 82 ++++++++++++++++++++++++++++++++++++++++++++--- src/open.c | 5 +-- 2 files changed, 80 insertions(+), 7 deletions(-) diff --git a/src/blocks/flux.c b/src/blocks/flux.c index c808efc..3f74ee6 100644 --- a/src/blocks/flux.c +++ b/src/blocks/flux.c @@ -214,9 +214,9 @@ int32_t flux_map_rebuild_from_entries(aaruformat_context *ctx) * enables O(1) lookup of flux captures by their identifiers (head, track, subtrack, captureIndex), * which is used by aaruf_read_flux_capture() and other flux access functions. * - * The function is intended for internal library use during image opening / indexing and is NOT - * part of the stable public API (no versioning guarantees). Callers outside the library should use - * the higher-level image open helpers that trigger this parsing implicitly. + * The function is intended for internal library use and is NOT part of the stable public API + * (no versioning guarantees). It is called on-demand (lazy loading) when flux data is first + * accessed, rather than during image opening, to avoid unnecessary I/O if flux data is never used. * * **Processing Flow:** * 1. **Validation:** Check ctx and ctx->imageStream are valid @@ -400,6 +400,60 @@ void process_flux_data_block(aaruformat_context *ctx, const IndexEntry *entry) TRACE("Found %d flux entries at position %" PRIu64 ".\n", ctx->flux_data_header.entries, entry->offset); } +/** + * @brief Lazy load flux data block if not already loaded. + * + * This helper function checks if flux entries are loaded, and if not, finds the + * FluxDataBlock in the index and loads it. This enables lazy loading of flux + * entries only when they are actually needed. + * + * @param ctx Pointer to the aaruformat context. Must not be NULL. + * @return AARUF_STATUS_OK if flux entries are loaded (or no flux data exists), + * or an error code on failure. + * @internal + */ +static int32_t ensure_flux_entries_loaded(aaruformat_context *ctx) +{ + // If already loaded, nothing to do + if(ctx->flux_entries != NULL || ctx->flux_data_header.entries > 0) + { + return AARUF_STATUS_OK; + } + + // Find FluxDataBlock in index + if(ctx->index_entries == NULL) + { + return AARUF_ERROR_FLUX_DATA_NOT_FOUND; + } + + IndexEntry *entry = NULL; + for(unsigned int i = 0; i < utarray_len(ctx->index_entries); i++) + { + entry = (IndexEntry *)utarray_eltptr(ctx->index_entries, i); + if(entry && entry->blockType == FluxDataBlock) + { + break; + } + entry = NULL; + } + + if(entry == NULL) + { + return AARUF_ERROR_FLUX_DATA_NOT_FOUND; + } + + // Load the flux data block + process_flux_data_block(ctx, entry); + + // Check if loading was successful + if(ctx->flux_entries == NULL || ctx->flux_data_header.entries == 0) + { + return AARUF_ERROR_FLUX_DATA_NOT_FOUND; + } + + return AARUF_STATUS_OK; +} + /** * @brief Retrieve metadata for all flux captures in the image. * @@ -434,6 +488,8 @@ void process_flux_data_block(aaruformat_context *ctx, const IndexEntry *entry) * FluxDataBlock (on-disk order). * @note This function only returns metadata; use aaruf_read_flux_capture() to retrieve * the actual flux data and index buffers. + * @note Flux entries are loaded on-demand (lazy loading) the first time this function + * or aaruf_read_flux_capture() is called, avoiding unnecessary I/O during image opening. * * @see aaruf_read_flux_capture() to retrieve actual flux data for a specific capture * @see FluxCaptureMeta for the structure of each metadata entry @@ -449,7 +505,7 @@ AARU_EXPORT int32_t AARU_CALL aaruf_get_flux_captures(void *context, uint8_t *bu return AARUF_ERROR_NOT_AARUFORMAT; } - const aaruformat_context *ctx = context; + aaruformat_context *ctx = (aaruformat_context *)context; // Not a libaaruformat context if(ctx->magic != AARU_MAGIC) @@ -458,6 +514,14 @@ AARU_EXPORT int32_t AARU_CALL aaruf_get_flux_captures(void *context, uint8_t *bu return AARUF_ERROR_NOT_AARUFORMAT; } + // Lazy load flux entries if not already loaded + int32_t res = ensure_flux_entries_loaded(ctx); + if(res != AARUF_STATUS_OK) + { + TRACE("Exiting aaruf_get_flux_captures() = %d", res); + return res; + } + if(ctx->flux_data_header.entries == 0 || ctx->flux_entries == NULL) { FATAL("Image contains no flux captures"); @@ -1148,7 +1212,7 @@ AARU_EXPORT int32_t AARU_CALL aaruf_read_flux_capture(void *context, uint32_t he return AARUF_ERROR_NOT_AARUFORMAT; } - const aaruformat_context *ctx = context; + aaruformat_context *ctx = (aaruformat_context *)context; if(ctx->magic != AARU_MAGIC) { @@ -1157,6 +1221,14 @@ AARU_EXPORT int32_t AARU_CALL aaruf_read_flux_capture(void *context, uint32_t he return AARUF_ERROR_NOT_AARUFORMAT; } + // Lazy load flux entries if not already loaded + int32_t load_res = ensure_flux_entries_loaded(ctx); + if(load_res != AARUF_STATUS_OK) + { + TRACE("Exiting aaruf_read_flux_capture() = %d", load_res); + return load_res; + } + if(ctx->flux_data_header.entries == 0 || ctx->flux_entries == NULL) { TRACE("Exiting aaruf_read_flux_capture() = AARUF_ERROR_FLUX_DATA_NOT_FOUND"); diff --git a/src/open.c b/src/open.c index 2d1976f..0c7e3f2 100644 --- a/src/open.c +++ b/src/open.c @@ -547,8 +547,9 @@ AARU_EXPORT void AARU_CALL *aaruf_open(const char *filepath, const bool resume_m break; case FluxDataBlock: - process_flux_data_block(ctx, entry); - + // Store the FluxDataBlock offset for lazy loading + // Don't read flux entries during open - load them on-demand when actually needed + // This avoids unnecessary I/O if flux data is never accessed break; default: TRACE("Unhandled block type %4.4s with data type %d is indexed to be at %" PRIu64 "",