fzm_format.h

Go to the documentation of this file.

#pragma once
 
#include <cstdint>
#include <cstring>
#include <stdexcept>
#include <string>
 
namespace fz {
 
constexpr uint32_t FZM_MAGIC = 0x464D5A32;
 
constexpr uint8_t  FZM_VERSION_MAJOR = 3;
constexpr uint8_t  FZM_VERSION_MINOR = 1;
constexpr uint16_t FZM_VERSION = (static_cast<uint16_t>(FZM_VERSION_MAJOR) << 8)
                                | static_cast<uint16_t>(FZM_VERSION_MINOR);
 
constexpr size_t FZM_LEGACY_HEADER_CORE_SIZE = 72;
 
constexpr uint16_t FZM_FLAG_HAS_DATA_CHECKSUM   = 0x0001u;  
constexpr uint16_t FZM_FLAG_HAS_HEADER_CHECKSUM = 0x0002u;  
 
constexpr uint8_t fzmVersionMajor(uint16_t v) {
    return (v <= 0xFF) ? static_cast<uint8_t>(v) : static_cast<uint8_t>(v >> 8);
}
constexpr uint8_t fzmVersionMinor(uint16_t v) {
    return (v <= 0xFF) ? 0u : static_cast<uint8_t>(v & 0xFF);
}
 
constexpr size_t FZM_MAX_BUFFERS      = 32;   
constexpr size_t FZM_MAX_NAME_LEN     = 64;   
constexpr size_t FZM_STAGE_CONFIG_SIZE = 128; 
constexpr size_t FZM_MAX_SOURCES      = 4;    
 
constexpr uint16_t FZM_STAGE_FLAG_PRIMARY_SOURCE = 0x1;
 
// ─────────────────────────────────────────────────────────────────────────────
 
enum class StageType : uint16_t {
    UNKNOWN    = 0,
    LORENZO_QUANT = 1,
    DIFFERENCE = 2,   
    SCALE      = 3,   
    PASSTHROUGH= 4,   
    RLE        = 5,   
    HUFFMAN    = 6,   
    BITPACK    = 7,   
    SPLIT      = 10,  
    MERGE      = 11,  
    LORENZO    = 12,  
    QUANTIZER  = 14,
    ZIGZAG     = 15, 
    NEGABINARY = 16,  
    BITSHUFFLE = 17, 
    RZE        = 18,  
    ANS        = 20,   
    ADM        = 19,   
    G_INTERP   = 22,   
    BITPLANE_RZE = 23, 
    ADAPTIVE_BITPACK = 24, 
    TILED_LORENZO = 25, 
    RRE        = 26,   
    RARE       = 27,   
    RAZE       = 28,   
    CLOG       = 29,   
    HCLOG      = 30,   
    TUPL       = 31,   
    GPULZ      = 32,   
    LOG_TRANSFORM = 33, 
    ADAPTIVE_LORENZO = 34, 
    ROIBIN_SPLIT = 35, 
    SZX        = 36,   
    SZP        = 37,   
    // 38 (TEE, SPERR-branch-only) never shipped on main -- bindExternalInput()
    // replaced the fan-out/duplicate-copy stage it would have been for before
    // this branch merged. Not reused -- a future stage type reading an old
    // archive that used it should fail loudly, not silently collide.
    CDF97      = 39,   
    SPECK2D    = 40,   
    CDF97_OUTLIER_CORRECT = 41, 
};
 
enum class DataType : uint8_t {
    UINT8   = 0,
    UINT16  = 1,
    UINT32  = 2,
    UINT64  = 3,
    INT8    = 4,
    INT16   = 5,
    INT32   = 6,
    INT64   = 7,
    FLOAT32 = 8,
    FLOAT64 = 9,
    UNKNOWN = 0xFF,  
};
 
// ─────────────────────────────────────────────────────────────────────────────
 
constexpr size_t FZM_MAX_STAGE_INPUTS  = 8;
constexpr size_t FZM_MAX_STAGE_OUTPUTS = 8;
 
struct FZMStageInfo {
    StageType stage_type;     
    uint16_t  stage_version;  
    uint8_t   num_inputs;     
    uint8_t   num_outputs;    
    uint16_t  stage_flags;    
 
    uint16_t input_buffer_ids[FZM_MAX_STAGE_INPUTS];   
    uint16_t output_buffer_ids[FZM_MAX_STAGE_OUTPUTS]; 
 
    uint8_t  stage_config[FZM_STAGE_CONFIG_SIZE]; 
    uint32_t config_size;  
 
    uint8_t reserved2[84]; 
    // Total: 2+2+1+1+2+16+16+128+4+84 = 256 bytes
 
    FZMStageInfo() {
        // Zero the WHOLE object, not just the named arrays. This struct is
        // written to disk verbatim, and the compiler's implicit tail padding
        // (fields end at 254, sizeof is 256) is not covered by the member-wise
        // initialization below — so an .fzm archive carried a few bytes of stack
        // garbage and was not byte-reproducible across runs. See FZMBufferEntry.
        std::memset(this, 0, sizeof(*this));
        stage_type   = StageType::UNKNOWN;
        stage_version = 0;
        num_inputs   = 0;
        num_outputs  = 0;
        stage_flags  = 0;
        memset(input_buffer_ids,  0xFF, sizeof(input_buffer_ids));
        memset(output_buffer_ids, 0xFF, sizeof(output_buffer_ids));
        memset(stage_config, 0, FZM_STAGE_CONFIG_SIZE);
        config_size = 0;
        memset(reserved2, 0, 84);
    }
};
static_assert(sizeof(FZMStageInfo) == 256, "FZMStageInfo must be 256 bytes");
 
struct FZMBufferEntry {
    StageType stage_type;         
    uint16_t  stage_version;      
    DataType  data_type;          
    uint8_t   producer_output_idx;
    uint16_t  dag_buffer_id;      
    char      name[FZM_MAX_NAME_LEN]; 
 
    uint64_t  data_size;          
    uint64_t  allocated_size;     
    uint64_t  uncompressed_size;  
    uint64_t  byte_offset;        
 
    uint8_t  stage_config[FZM_STAGE_CONFIG_SIZE]; 
    uint32_t config_size; 
 
    uint8_t reserved2[14]; 
 
    FZMBufferEntry() {
        // Zero the WHOLE object before setting fields. The named members end at
        // byte 250 but sizeof(FZMBufferEntry) is 256 (8-byte alignment), and
        // those 6 padding bytes are written straight to disk with the rest of
        // the struct. Left uninitialized they made compression non-deterministic:
        // two runs over CESM-2D/CLDHGH through gpu_zstd_lossless.toml produced
        // archives differing in exactly 12 bytes — 2 padding bytes in each of the
        // 4 buffer entries, plus the 4-byte header_checksum computed over them.
        // Round-trip was unaffected (the padding is never read back), which is
        // why it survived; but it breaks byte-for-byte reproducibility, and
        // benchkit records compressed_sha256 per row and compares baselines
        // across machines on exactly that.
        std::memset(this, 0, sizeof(*this));
        stage_type         = StageType::UNKNOWN;
        stage_version      = 0;
        data_type          = DataType::UINT8;
        producer_output_idx = 0;
        dag_buffer_id      = 0xFFFF;
        memset(name, 0, FZM_MAX_NAME_LEN);
        data_size          = 0;
        allocated_size     = 0;
        uncompressed_size  = 0;
        byte_offset        = 0;
        memset(stage_config, 0, FZM_STAGE_CONFIG_SIZE);
        config_size        = 0;
        memset(reserved2, 0, 14);
    }
};
static_assert(sizeof(FZMBufferEntry) == 256, "FZMBufferEntry must be 256 bytes");
 
struct FZMHeaderCore {
    uint32_t magic;             
    uint16_t version;           
    uint16_t num_buffers;       
 
    uint64_t uncompressed_size; 
    uint64_t compressed_size;   
    uint64_t header_size;       
 
    uint32_t num_stages;        
    uint16_t num_sources;       
    uint16_t flags;             
 
    uint64_t source_uncompressed_sizes[FZM_MAX_SOURCES]; 
 
    uint32_t data_checksum;   
    uint32_t header_checksum; 
 
    FZMHeaderCore() {
        magic              = FZM_MAGIC;
        version            = FZM_VERSION;
        num_buffers        = 0;
        uncompressed_size  = 0;
        compressed_size    = 0;
        header_size        = sizeof(FZMHeaderCore);
        num_stages         = 0;
        num_sources        = 0;
        flags              = 0;
        memset(source_uncompressed_sizes, 0, sizeof(source_uncompressed_sizes));
        data_checksum      = 0;
        header_checksum    = 0;
    }
 
    uint64_t computeHeaderSize() const {
        return sizeof(FZMHeaderCore)
             + num_stages  * sizeof(FZMStageInfo)
             + num_buffers * sizeof(FZMBufferEntry);
    }
};
static_assert(sizeof(FZMHeaderCore) == 80, "FZMHeaderCore must be 80 bytes");
 
// ─────────────────────────────────────────────────────────────────────────────
// Helper functions
// ─────────────────────────────────────────────────────────────────────────────
 
inline size_t getDataTypeSize(DataType type) {
    switch (type) {
        case DataType::UINT8:  case DataType::INT8:                        return 1;
        case DataType::UINT16: case DataType::INT16:                       return 2;
        case DataType::UINT32: case DataType::INT32: case DataType::FLOAT32: return 4;
        case DataType::UINT64: case DataType::INT64: case DataType::FLOAT64: return 8;
        default: throw std::runtime_error("Unknown data type");
    }
}
 
inline std::string dataTypeToString(DataType type) {
    switch (type) {
        case DataType::UINT8:   return "uint8";
        case DataType::UINT16:  return "uint16";
        case DataType::UINT32:  return "uint32";
        case DataType::UINT64:  return "uint64";
        case DataType::INT8:    return "int8";
        case DataType::INT16:   return "int16";
        case DataType::INT32:   return "int32";
        case DataType::INT64:   return "int64";
        case DataType::FLOAT32: return "float32";
        case DataType::FLOAT64: return "float64";
        default:                return "unknown";
    }
}
 
inline std::string stageTypeToString(StageType type) {
    switch (type) {
        case StageType::LORENZO_QUANT:  return "LorenzoQuant";
        case StageType::DIFFERENCE:  return "Difference";
        case StageType::SCALE:       return "Scale";
        case StageType::PASSTHROUGH: return "PassThrough";
        case StageType::RLE:         return "RLE";
        case StageType::HUFFMAN:     return "Huffman";
        case StageType::BITPACK:     return "BitPack";
        case StageType::SPLIT:       return "Split";
        case StageType::MERGE:       return "Merge";
        case StageType::ROIBIN_SPLIT: return "ROIBinSplit";
        case StageType::SZX: return "SZx";
        case StageType::SZP: return "SZp";
        case StageType::QUANTIZER:   return "Quantizer";
        case StageType::ZIGZAG:      return "Zigzag";
        case StageType::NEGABINARY:  return "Negabinary";
        case StageType::BITSHUFFLE:  return "Bitshuffle";
        case StageType::RZE:         return "RZE";
        case StageType::RRE:         return "RRE";
        case StageType::RARE:        return "RARE";
        case StageType::RAZE:        return "RAZE";
        case StageType::CLOG:        return "CLOG";
        case StageType::HCLOG:       return "HCLOG";
        case StageType::TUPL:        return "TUPL";
        case StageType::LORENZO:     return "Lorenzo";
        case StageType::ANS:  return "ANS";
        case StageType::ADM:  return "ADM";
        case StageType::G_INTERP:    return "GInterp";
        case StageType::BITPLANE_RZE: return "BitplaneRZE";
        case StageType::ADAPTIVE_BITPACK: return "AdaptiveBitpack";
        case StageType::TILED_LORENZO: return "TiledLorenzo";
        case StageType::GPULZ:  return "GPULZ";
        case StageType::LOG_TRANSFORM: return "LogTransform";
        case StageType::ADAPTIVE_LORENZO: return "AdaptiveLorenzo";
        case StageType::CDF97: return "CDF97";
        case StageType::SPECK2D: return "SPECK2D";
        case StageType::CDF97_OUTLIER_CORRECT: return "Cdf97OutlierCorrect";
        default:                     return "Unknown";
    }
}
 
} // namespace fz