Add flux implementation

CMakeLists.txt

@@ -234,7 +234,8 @@ add_library(aaruformat SHARED
             src/metadata.c
             src/dump.c
             include/aaruformat/structs/tape.h
-            src/blocks/tape.c)
+            src/blocks/tape.c
+            src/blocks/flux.c)
 
 # Set up include directories for the target
 target_include_directories(aaruformat

docs/spec/blocks/datastream_payload.adoc

@@ -0,0 +1,83 @@
+=== Data Stream Payload Block (`DSPL`)
+
+This block contains a generic data stream payload. Can be used for data streams such as bitstreams or flux data.
+Each `DataStreamPayloadBlock` stores a compressed or uncompressed stream of binary data of variable length.
+The block may be compressed using LZMA compression if compression is enabled for the image.
+
+For flux captures, the payload data consists of:
+* Data buffer: Raw flux transition timing data
+* Index buffer: Index structure mapping logical positions to offsets within the data buffer
+
+These two buffers are concatenated: `[data_buffer][index_buffer]` before compression and storage.
+For other data types, the payload layout is specific to the data type.
+
+==== Structure Definition
+
+[source,c]
+#define DATA_STREAM_PAYLOAD_MAGIC 0x4C505344
+/** Data stream payload block header */
+typedef struct DataStreamPayloadHeader
+{
+    uint32_t identifier;  ///< Block identifier, must be BlockType::DataStreamPayloadBlock (0x4C505344).
+    uint16_t dataType;    ///< Data type classification (value from DataType), e.g., FluxData or BitstreamData.
+    uint16_t compression; ///< Compression type (0 = None, 1 = Lzma).
+    uint32_t cmpLength;   ///< Compressed length in bytes (includes LZMA properties if compressed).
+    uint32_t length;      ///< Uncompressed length in bytes.
+    uint64_t cmpCrc64;    ///< CRC64-ECMA checksum of the compressed payload (or same as crc64 if uncompressed).
+    uint64_t crc64;       ///< CRC64-ECMA checksum of the uncompressed payload.
+} DataStreamPayloadHeader;
+
+==== Field Descriptions
+
+[cols="2,2,2,6",options="header"]
+|===
+|Type
+|Size
+|Name
+|Description
+
+|uint32_t
+|4 bytes
+|identifier
+|The data stream payload block identifier, always `DSPL` (`0x4C505344`)
+
+|uint16_t
+|2 bytes
+|dataType
+|The data type contained in this block (value from DataType), e.g., FluxData or BitstreamData. See Annex B.
+
+|uint16_t
+|2 bytes
+|compression
+|Compression type: 0 = None, 1 = Lzma
+
+|uint32_t
+|4 bytes
+|cmpLength
+|Compressed length in bytes (includes LZMA properties if compression = Lzma)
+
+|uint32_t
+|4 bytes
+|length
+|Uncompressed length in bytes (for flux captures: total of data buffer + index buffer)
+
+|uint64_t
+|8 bytes
+|cmpCrc64
+|CRC64-ECMA checksum of the compressed payload data
+
+|uint64_t
+|8 bytes
+|crc64
+|CRC64-ECMA checksum of the uncompressed payload data (for flux captures: data + index buffers)
+|===
+
+==== Payload Data
+
+The payload data immediately follows the `DataStreamPayloadHeader`. If compression is Lzma, the first 5 bytes contain LZMA properties, followed by the compressed data. If compression is None, the payload data is stored uncompressed.
+
+For flux captures, the uncompressed payload consists of:
+* Data buffer: `length - indexOffset` bytes of flux transition timing data
+* Index buffer: `indexOffset` bytes of index structure data
+
+The `indexOffset` value from the corresponding `FluxEntry` indicates where the index buffer starts within the payload. For other data types, the payload layout is specific to the data type.

docs/spec/blocks/flux.adoc

@@ -1,6 +1,6 @@
 === Flux Data Block (`FLUX`)
 
-This block lists all known flux captures.
+This block contains metadata for all flux captures in the image.
 Certain hardware devices, such as Kryoflux, Pauline, and Applesauce, read magnetic media at the flux transition level.
 
 Flux transition reads are digital representations of the analog properties of the media, and cannot be reliably interpreted on a sector-by-sector basis without further processing.
@@ -14,13 +14,25 @@ Flux data is represented as an array of `uint8_t` bytes.
 Each byte stores the tick count since the last flux transition.
 If no transition is detected within a byte's range, the value `0xFF` is used, and counting resumes in the next byte with ticks accumulated.
 
-Flux data is stored in `DataBlocks` of the flux data type, referenced from a deduplication table of the same type.
-Only one flux-type deduplication table is allowed per image, and it must have exactly one level.
+The flux capture system uses two block types:
+* `FluxDataBlock` (`FLUX` / `0x58554C46`): Contains metadata entries that describe all flux captures in the image
+* `DataStreamPayloadBlock` (`DSPL` / `0x4C505344`): Contains the actual flux data payload (data and index buffers) for individual captures. This is a generic block type that can also be used for other data streams.
+
+Each flux capture has one entry in the `FluxDataBlock` and one corresponding `DataStreamPayloadBlock` containing its data.
+The `FluxEntry` structure in the data block contains a `payloadOffset` field that points to the file offset where the corresponding `DataStreamPayloadBlock` is stored. The offset is stored divided by the block alignment (as indicated by `blockAlignmentShift` in the `FluxHeader`), consistent with DDT table offset storage.
 
 ==== Structure Definition
 
 [source,c]
-/* Undefined */
+#define FLUX_DATA_MAGIC 0x58554C46
+/** Flux data block header */
+typedef struct FluxHeader
+{
+    uint32_t identifier;          ///< Block identifier, must be BlockType::FluxDataBlock (0x58554C46).
+    uint16_t entries;             ///< Number of FluxEntry records following this header.
+    uint8_t  blockAlignmentShift; ///< Block alignment shift: 2^blockAlignmentShift = block alignment boundary in bytes.
+    uint64_t crc64;               ///< CRC64-ECMA checksum of the FluxEntry array (header excluded).
+} FluxHeader;
 
 ==== Field Descriptions
 
@@ -34,23 +46,39 @@ Only one flux-type deduplication table is allowed per image, and it must have ex
 |uint32_t
 |4 bytes
 |identifier
-|The flux data block identifier, always `FLUX`
+|The flux data block identifier, always `FLUX` (`0x58554C46`)
 
 |uint16_t
 |2 bytes
 |entries
-|The number of entries following this header
+|The number of flux entry records following this header
+
+|uint8_t
+|1 byte
+|blockAlignmentShift
+|Block alignment shift: 2^blockAlignmentShift = block alignment boundary in bytes. Used to decode payloadOffset values in FluxEntry structures, which are stored divided by this alignment. Stored in the header to make the block self-contained, similar to DDT headers.
 
 |uint64_t
 |8 bytes
 |crc64
-|The CRC64-ECMA checksum of the data following this header
+|The CRC64-ECMA checksum of the FluxEntry array following this header
 |===
 
 ==== Flux entries
 
 [source,c]
-/* Undefined */
+/** Flux capture entry, describes one flux capture in the image */
+typedef struct FluxEntry
+{
+    uint32_t head;          ///< Head number the capture corresponds to.
+    uint16_t track;          ///< Track number the capture corresponds to.
+    uint8_t  subtrack;      ///< Subtrack number the capture corresponds to.
+    uint32_t captureIndex;  ///< Capture index, allows multiple captures for the same location.
+    uint64_t indexResolution; ///< Resolution in picoseconds for the index stream.
+    uint64_t dataResolution;  ///< Resolution in picoseconds for the data stream.
+    uint64_t indexOffset;      ///< Byte offset within the payload where the index buffer starts.
+    uint64_t payloadOffset;   ///< File offset where the DataStreamPayloadBlock for this capture is stored.
+} FluxEntry;
 
 ==== Field Descriptions
 
@@ -64,25 +92,40 @@ Only one flux-type deduplication table is allowed per image, and it must have ex
 |uint32_t
 |4 bytes
 |head
-|Head the data corresponds to.
+|Head number the flux capture corresponds to.
 
 |uint16_t
 |2 bytes
 |track
-|Track the data corresponds to.
+|Track number the flux capture corresponds to.
 
 |uint8_t
 |1 byte
 |subtrack
-|Substep of a track that the data corresponds to.
+|Subtrack number the flux capture corresponds to.
+
+|uint32_t
+|4 bytes
+|captureIndex
+|Capture index, allows multiple captures for the same head/track/subtrack combination.
+
+|uint64_t
+|8 bytes
+|indexResolution
+|Resolution in picoseconds at which the index stream was sampled.
+
+|uint64_t
+|8 bytes
+|dataResolution
+|Resolution in picoseconds at which the data stream was sampled.
 
 |uint64_t
 |8 bytes
-|resolution
-|Number of picoseconds at which the sampling was performed.
+|indexOffset
+|Byte offset within the payload block where the index buffer starts (equals data_length).
 
 |uint64_t
 |8 bytes
-|tableEntry
-|Entry number in the deduplication table where the data corresponding to this flux entry is stored
+|payloadOffset
+|Block-aligned file offset where the DataStreamPayloadBlock containing this capture's data is stored, divided by (1 << blockAlignmentShift). To get the absolute file offset, multiply by (1 << blockAlignmentShift) using the blockAlignmentShift value from FluxHeader. This storage method is consistent with DDT table offset storage.
 |===

docs/spec/spec.adoc

@@ -118,6 +118,10 @@ include::blocks/flux.adoc[]
 
 <<<
 
+include::blocks/datastream_payload.adoc[]
+
+<<<
+
 include::blocks/bitstream.adoc[]
 
 <<<

docs/spec/version_history.adoc

@@ -98,4 +98,6 @@ Clarify deduplication table sector status.
 Add encrypted and decrypted sector status.
 
 Add Aaru Metadata JSON.
+
+Rework the flux workflow.
 |===

include/aaruformat/context.h

@@ -21,6 +21,7 @@
 
 #include "blake3.h"
 #include "crc64.h"
+#include "structs/flux.h"
 #include "hash_map.h"
 #include "lru.h"
 #include "md5.h"
@@ -30,6 +31,8 @@
 #include "structs.h"
 #include "utarray.h"
 
+typedef struct FluxCaptureMapEntry FluxCaptureMapEntry;
+
 /** \file aaruformat/context.h
  *  \brief Central runtime context structures for libaaruformat (image state, caches, checksum buffers).
  *
@@ -304,6 +307,12 @@ typedef struct aaruformat_context
     TapePartitionHashEntry *tape_partitions;  ///< Hash table root for tape partitions
     bool                    is_tape;          ///< True if the image is a tape image
 
+    /* Flux data structures */
+    FluxHeader  flux_data_header;  ///< Flux data header (if present).
+    FluxEntry  *flux_entries;      ///< Array of flux entries (flux_data_header.entries elements).
+    UT_array   *flux_captures;     ///< Pending flux capture payloads (write path).
+    FluxCaptureMapEntry *flux_map; ///< Hash map for flux capture lookup by head/track/subtrack/capture index.
+
     /* Dirty flags (controls write behavior in close.c) */
     bool dirty_secondary_ddt;                  ///< True if secondary DDT tables should be written during close
     bool dirty_primary_ddt;                    ///< True if primary DDT table should be written during close
@@ -327,6 +336,7 @@ typedef struct aaruformat_context
     bool dirty_dumphw_block;                   ///< True if dump hardware block should be written during close
     bool dirty_cicm_block;                     ///< True if CICM metadata block should be written during close
     bool dirty_json_block;                     ///< True if JSON metadata block should be written during close
+    bool dirty_flux_block;                     ///< True if flux block should be written during close
     bool dirty_index_block;                    ///< True if index block should be written during close
 } aaruformat_context;
 

include/aaruformat/decls.h

@@ -21,6 +21,7 @@
 
 #include "aaru.h"
 #include "crc64.h"
+#include "enums.h"
 #include "md5.h"
 #include "sha1.h"
 #include "sha256.h"
@@ -88,6 +89,17 @@ AARU_EXPORT uint64_t AARU_CALL   aaruf_crc64_data(const uint8_t *data, uint32_t
 
 AARU_EXPORT int32_t AARU_CALL aaruf_get_tracks(const void *context, uint8_t *buffer, size_t *length);
 AARU_EXPORT int32_t AARU_CALL aaruf_set_tracks(void *context, TrackEntry *tracks, const int count);
+AARU_EXPORT int32_t AARU_CALL aaruf_get_flux_captures(void *context, uint8_t *buffer, size_t *length);
+AARU_EXPORT int32_t AARU_CALL aaruf_read_flux_capture(void *context, uint32_t head, uint16_t track, uint8_t subtrack,
+                                                      uint32_t capture_index, uint8_t *index_data,
+                                                      uint32_t *index_length, uint8_t *data_data,
+                                                      uint32_t *data_length);
+AARU_EXPORT int32_t AARU_CALL aaruf_write_flux_capture(void *context, uint32_t head, uint16_t track, uint8_t subtrack,
+                                                     uint32_t capture_index, uint64_t data_resolution,
+                                                     uint64_t index_resolution, const uint8_t *data,
+                                                     uint32_t data_length, const uint8_t *index,
+                                                     uint32_t index_length);
+AARU_EXPORT int32_t AARU_CALL aaruf_clear_flux_captures(void *context);
 
 AARU_EXPORT int32_t AARU_CALL aaruf_read_sector(void *context, uint64_t sector_address, bool negative, uint8_t *data,
                                                 uint32_t *length, uint8_t *sector_status);

include/aaruformat/enums.h

@@ -130,7 +130,9 @@ typedef enum
     DvdSectorEdc                     = 85,  ///< DVD Error Detection Code (EDC)
     DvdSectorEccPi                   = 86,  ///< DVD Error Correction Code (ECC) Parity of Inner Code (PI)
     DvdEccBlockPo                    = 87,  ///< DVD Error Correction Code (ECC) Parity of Outer Code (PO)
-    DvdPfi2ndLayer                   = 88   ///< DVD Physical Format Information for the second layer
+    DvdPfi2ndLayer                   = 88,  ///< DVD Physical Format Information for the second layer
+    FluxData                         = 89,  ///< Flux data.
+    BitstreamData                    = 90   ///< Bitstream data.
 } DataType;
 
 /**
@@ -158,7 +160,9 @@ typedef enum
     DumpHardwareBlock            = 0x2A504D44,  ///< Block containing an array of hardware used to create the image.
     TapeFileBlock                = 0x454C4654,  ///< Block containing list of files for a tape image.
     TapePartitionBlock           = 0x54425054,  ///< Block containing list of partitions for a tape image.
-    AaruMetadataJsonBlock        = 0x444D534A   ///< Block containing JSON version of Aaru Metadata
+    AaruMetadataJsonBlock        = 0x444D534A,  ///< Block containing JSON version of Aaru Metadata
+    FluxDataBlock                = 0x58554C46,  ///< Block containing flux data metadata.
+    DataStreamPayloadBlock       = 0x4C505344   ///< Block containing compressed data stream payload (e.g., flux data, bitstreams).
 } BlockType;
 
 /**

include/aaruformat/errors.h

@@ -68,6 +68,7 @@
 #define AARUF_ERROR_TAPE_PARTITION_NOT_FOUND  (-29)  ///< Requested tape partition not present in image.
 #define AARUF_ERROR_METADATA_NOT_PRESENT      (-30)  ///< Requested metadata not present in image.
 #define AARUF_ERROR_INVALID_SECTOR_LENGTH     (-31)  ///< Sector length is too big.
+#define AARUF_ERROR_FLUX_DATA_NOT_FOUND       (-32)  ///< Requested flux data not present in image.
 /** @} */
 
 /** \name Non-fatal sector status codes (non-negative)

include/aaruformat/structs.h

@@ -39,6 +39,7 @@
 #include "structs/optical.h"
 #include "structs/options.h"
 #include "structs/tape.h"
+#include "structs/flux.h"
 
 #endif  // LIBAARUFORMAT_STRUCTS_H