From 06a96f25dbf9436721e633d083235799ba0246db Mon Sep 17 00:00:00 2001 From: Carl Niklas Rydberg Date: Sat, 17 Jan 2026 16:43:47 +0100 Subject: [PATCH] Add snapshot manifests and auto snapshot policy --- CMakeLists.txt | 19 +- include/amduat/asl/asl_store_index_fs.h | 21 + include/amduat/asl/index_snapshot.h | 48 + .../asl_store_index_fs/asl_store_index_fs.c | 688 ++++++++++++- .../asl_store_index_fs_layout.c | 41 + .../asl_store_index_fs_layout.h | 9 + src/near_core/asl/index_snapshot.c | 487 ++++++++++ src/tools/amduat_asl_cli.c | 271 +++++- tests/asl/test_asl_index_replay.c | 913 ++++++++++++++++++ 9 files changed, 2464 insertions(+), 33 deletions(-) create mode 100644 include/amduat/asl/index_snapshot.h create mode 100644 src/near_core/asl/index_snapshot.c create mode 100644 tests/asl/test_asl_index_replay.c diff --git a/CMakeLists.txt b/CMakeLists.txt index ff2cf2d..7f70a48 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,6 +63,7 @@ set(AMDUAT_ASL_SRCS src/kernel/asl/core.c src/near_core/asl/artifact_io.c src/near_core/asl/io.c + src/near_core/asl/index_snapshot.c src/near_core/asl/index_replay.c src/near_core/asl/parse.c src/near_core/asl/ref_io.c @@ -177,7 +178,8 @@ target_include_directories(amduat_asl_cli PRIVATE ${AMDUAT_INCLUDE_DIR} ) target_link_libraries(amduat_asl_cli - PRIVATE amduat_format amduat_asl_store_fs amduat_asl amduat_enc + PRIVATE amduat_format amduat_asl_store_fs amduat_asl_store_index_fs + amduat_asl amduat_enc amduat_hash_asl1 amduat_util ) set_target_properties(amduat_asl_cli PROPERTIES OUTPUT_NAME amduat-asl) @@ -363,6 +365,21 @@ target_link_libraries(amduat_test_asl_replay ) add_test(NAME asl_replay COMMAND amduat_test_asl_replay) +add_executable(amduat_test_asl_index_replay + tests/asl/test_asl_index_replay.c) +target_include_directories(amduat_test_asl_index_replay + PRIVATE ${AMDUAT_INTERNAL_DIR} + PRIVATE ${AMDUAT_INCLUDE_DIR} + PRIVATE ${AMDUAT_ROOT}/src/adapters/asl_store_index_fs +) +target_compile_definitions(amduat_test_asl_index_replay + PRIVATE _POSIX_C_SOURCE=200809L +) +target_link_libraries(amduat_test_asl_index_replay + PRIVATE amduat_asl_store_index_fs +) +add_test(NAME asl_index_replay COMMAND amduat_test_asl_index_replay) + add_executable(amduat_test_asl_store_index_fs tests/asl/test_asl_store_index_fs.c) target_include_directories(amduat_test_asl_store_index_fs diff --git a/include/amduat/asl/asl_store_index_fs.h b/include/amduat/asl/asl_store_index_fs.h index b1e275d..d9d184a 100644 --- a/include/amduat/asl/asl_store_index_fs.h +++ b/include/amduat/asl/asl_store_index_fs.h @@ -12,15 +12,36 @@ extern "C" { enum { AMDUAT_ASL_STORE_INDEX_FS_ROOT_MAX = 1024 }; +typedef struct { + bool enabled; + uint64_t max_pending_bytes; + uint64_t idle_time_ns; +} amduat_asl_store_index_fs_snapshot_policy_t; + typedef struct { amduat_asl_store_config_t config; + amduat_asl_store_index_fs_snapshot_policy_t snapshot_policy; char root_path[AMDUAT_ASL_STORE_INDEX_FS_ROOT_MAX]; + uint64_t pending_snapshot_bytes; + uint64_t last_ingest_time_ns; + amduat_asl_snapshot_id_t next_snapshot_id; + bool snapshot_state_initialized; } amduat_asl_store_index_fs_t; bool amduat_asl_store_index_fs_init(amduat_asl_store_index_fs_t *fs, amduat_asl_store_config_t config, const char *root_path); +void amduat_asl_store_index_fs_set_snapshot_policy( + amduat_asl_store_index_fs_t *fs, + amduat_asl_store_index_fs_snapshot_policy_t policy); + +amduat_asl_store_error_t amduat_asl_store_index_fs_snapshot_create( + amduat_asl_store_index_fs_t *fs, + amduat_asl_snapshot_id_t snapshot_id, + amduat_asl_log_position_t *out_logseq, + uint8_t out_root_hash[32]); + amduat_asl_store_ops_t amduat_asl_store_index_fs_ops(void); #ifdef __cplusplus diff --git a/include/amduat/asl/index_snapshot.h b/include/amduat/asl/index_snapshot.h new file mode 100644 index 0000000..86fd7df --- /dev/null +++ b/include/amduat/asl/index_snapshot.h @@ -0,0 +1,48 @@ +#ifndef AMDUAT_ASL_INDEX_SNAPSHOT_H +#define AMDUAT_ASL_INDEX_SNAPSHOT_H + +#include "amduat/asl/index_replay.h" +#include "amduat/asl/store.h" + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +enum { + AMDUAT_ASL_SNAPSHOT_MANIFEST_VERSION = 1, + AMDUAT_ASL_SNAPSHOT_MANIFEST_HEADER_SIZE = 80, + AMDUAT_ASL_SNAPSHOT_MANIFEST_MAGIC_LEN = 8 +}; + +typedef struct { + amduat_asl_snapshot_id_t snapshot_id; + amduat_asl_log_position_t anchor_logseq; + amduat_asl_store_config_t config; + amduat_asl_segment_seal_t *segments; + size_t segments_len; + amduat_asl_tombstone_entry_t *tombstones; + size_t tombstones_len; +} amduat_asl_snapshot_manifest_t; + +bool amduat_asl_snapshot_manifest_write( + const char *path, + const amduat_asl_snapshot_manifest_t *manifest, + uint8_t out_hash[32]); + +bool amduat_asl_snapshot_manifest_read( + const char *path, + amduat_asl_snapshot_manifest_t *out_manifest, + uint8_t out_hash[32]); + +void amduat_asl_snapshot_manifest_free( + amduat_asl_snapshot_manifest_t *manifest); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* AMDUAT_ASL_INDEX_SNAPSHOT_H */ diff --git a/src/adapters/asl_store_index_fs/asl_store_index_fs.c b/src/adapters/asl_store_index_fs/asl_store_index_fs.c index 434ed96..3eb049c 100644 --- a/src/adapters/asl_store_index_fs/asl_store_index_fs.c +++ b/src/adapters/asl_store_index_fs/asl_store_index_fs.c @@ -1,6 +1,7 @@ #include "amduat/asl/asl_store_index_fs.h" #include "asl_store_index_fs_layout.h" +#include "amduat/asl/index_snapshot.h" #include "amduat/asl/index_replay.h" #include "amduat/asl/ref_derive.h" #include "amduat/enc/asl1_core.h" @@ -19,6 +20,7 @@ #include #include #include +#include #include #ifndef O_DIRECTORY @@ -27,7 +29,9 @@ enum { AMDUAT_ASL_STORE_INDEX_FS_MIN_DIGEST_BYTES = 2, - AMDUAT_ASL_STORE_INDEX_FS_SEGMENT_HASH_LEN = 32 + AMDUAT_ASL_STORE_INDEX_FS_SEGMENT_HASH_LEN = 32, + AMDUAT_ASL_STORE_INDEX_FS_DEFAULT_PENDING_BYTES = 128u * 1024u, + AMDUAT_ASL_STORE_INDEX_FS_DEFAULT_IDLE_NS = 2u * 1000u * 1000u * 1000u }; typedef enum { @@ -471,6 +475,335 @@ static bool amduat_asl_store_index_fs_parse_u64(const char *value, return true; } +static uint64_t amduat_asl_store_index_fs_load_u64_le(const uint8_t *data) { + return (uint64_t)data[0] | ((uint64_t)data[1] << 8) | + ((uint64_t)data[2] << 16) | ((uint64_t)data[3] << 24) | + ((uint64_t)data[4] << 32) | ((uint64_t)data[5] << 40) | + ((uint64_t)data[6] << 48) | ((uint64_t)data[7] << 56); +} + +static uint64_t amduat_asl_store_index_fs_now_ns(void) { + struct timespec ts; + + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { + return 0u; + } + return (uint64_t)ts.tv_sec * 1000u * 1000u * 1000u + + (uint64_t)ts.tv_nsec; +} + +static bool amduat_asl_store_index_fs_find_next_snapshot_id( + amduat_asl_store_index_fs_t *fs, + amduat_asl_snapshot_id_t *out_next_id) { + char *log_path; + amduat_asl_log_record_t *records; + size_t record_count; + size_t i; + amduat_asl_snapshot_id_t max_id; + amduat_asl_store_error_t err; + + if (fs == NULL || out_next_id == NULL) { + return false; + } + + if (!amduat_asl_store_index_fs_layout_build_log_path(fs->root_path, + &log_path)) { + return false; + } + + records = NULL; + record_count = 0u; + err = amduat_asl_store_index_fs_load_log(log_path, &records, &record_count); + free(log_path); + if (err != AMDUAT_ASL_STORE_OK) { + amduat_asl_store_index_fs_log_free(records, record_count); + return false; + } + + max_id = 0u; + for (i = 0; i < record_count; ++i) { + const amduat_asl_log_record_t *record = &records[i]; + if (record->record_type != AMDUAT_ASL_LOG_RECORD_SNAPSHOT_ANCHOR) { + continue; + } + if (record->payload.len < 8u || record->payload.data == NULL) { + continue; + } + amduat_asl_snapshot_id_t snapshot_id = + amduat_asl_store_index_fs_load_u64_le(record->payload.data); + if (snapshot_id > max_id) { + max_id = snapshot_id; + } + } + + amduat_asl_store_index_fs_log_free(records, record_count); + if (max_id == UINT64_MAX) { + return false; + } + *out_next_id = max_id + 1u; + return true; +} + +static bool amduat_asl_store_index_fs_parse_snapshot_anchor( + const amduat_asl_log_record_t *record, + amduat_asl_snapshot_id_t *out_snapshot_id, + uint8_t out_root_hash[32]) { + if (record == NULL || out_snapshot_id == NULL || out_root_hash == NULL) { + return false; + } + if (record->payload.len < 8u + 32u || record->payload.data == NULL) { + return false; + } + *out_snapshot_id = + amduat_asl_store_index_fs_load_u64_le(record->payload.data); + memcpy(out_root_hash, record->payload.data + 8u, 32u); + return true; +} + +static bool amduat_asl_store_index_fs_find_latest_anchor( + const amduat_asl_log_record_t *records, + size_t record_count, + amduat_asl_log_position_t log_position, + size_t *out_index, + amduat_asl_snapshot_id_t *out_snapshot_id, + uint8_t out_root_hash[32], + amduat_asl_log_position_t *out_anchor_logseq) { + size_t i; + bool found = false; + size_t index = 0u; + amduat_asl_snapshot_id_t snapshot_id = 0u; + uint8_t root_hash[32]; + uint64_t anchor_logseq = 0u; + + if (records == NULL || out_index == NULL || out_snapshot_id == NULL || + out_root_hash == NULL || out_anchor_logseq == NULL) { + return false; + } + + for (i = 0; i < record_count; ++i) { + const amduat_asl_log_record_t *record = &records[i]; + if (record->record_type != AMDUAT_ASL_LOG_RECORD_SNAPSHOT_ANCHOR) { + continue; + } + if (record->logseq > log_position) { + continue; + } + if (!amduat_asl_store_index_fs_parse_snapshot_anchor( + record, &snapshot_id, root_hash)) { + continue; + } + index = i; + anchor_logseq = record->logseq; + found = true; + memcpy(out_root_hash, root_hash, 32u); + } + + if (!found) { + return false; + } + *out_index = index; + *out_snapshot_id = snapshot_id; + *out_anchor_logseq = anchor_logseq; + return true; +} + +static bool amduat_asl_store_index_fs_find_anchor_for_snapshot( + const amduat_asl_log_record_t *records, + size_t record_count, + amduat_asl_snapshot_id_t snapshot_id, + amduat_asl_log_position_t log_position, + size_t *out_index, + uint8_t out_root_hash[32], + amduat_asl_log_position_t *out_anchor_logseq) { + size_t i; + bool found = false; + size_t index = 0u; + uint8_t root_hash[32]; + uint64_t anchor_logseq = 0u; + + if (records == NULL || out_index == NULL || out_root_hash == NULL || + out_anchor_logseq == NULL) { + return false; + } + + for (i = 0; i < record_count; ++i) { + const amduat_asl_log_record_t *record = &records[i]; + amduat_asl_snapshot_id_t record_snapshot_id; + if (record->record_type != AMDUAT_ASL_LOG_RECORD_SNAPSHOT_ANCHOR) { + continue; + } + if (record->logseq > log_position) { + continue; + } + if (!amduat_asl_store_index_fs_parse_snapshot_anchor( + record, &record_snapshot_id, root_hash)) { + continue; + } + if (record_snapshot_id != snapshot_id) { + continue; + } + index = i; + anchor_logseq = record->logseq; + found = true; + memcpy(out_root_hash, root_hash, 32u); + } + + if (!found) { + return false; + } + *out_index = index; + *out_anchor_logseq = anchor_logseq; + return true; +} + +static bool amduat_asl_store_index_fs_load_snapshot_replay( + const char *root_path, + amduat_asl_snapshot_id_t snapshot_id, + const uint8_t expected_root_hash[32], + amduat_asl_replay_state_t *out_state, + amduat_asl_log_position_t *out_anchor_logseq) { + char *manifest_path; + amduat_asl_snapshot_manifest_t manifest; + uint8_t manifest_hash[32]; + + if (root_path == NULL || out_state == NULL || out_anchor_logseq == NULL) { + return false; + } + if (snapshot_id == 0u) { + return false; + } + + if (!amduat_asl_store_index_fs_layout_build_snapshot_manifest_path( + root_path, snapshot_id, &manifest_path)) { + return false; + } + if (!amduat_asl_snapshot_manifest_read(manifest_path, &manifest, + manifest_hash)) { + free(manifest_path); + return false; + } + free(manifest_path); + + if (manifest.snapshot_id != snapshot_id) { + amduat_asl_snapshot_manifest_free(&manifest); + return false; + } + if (expected_root_hash != NULL && + memcmp(expected_root_hash, manifest_hash, 32u) != 0) { + amduat_asl_snapshot_manifest_free(&manifest); + return false; + } + + if (!amduat_asl_replay_init(out_state)) { + amduat_asl_snapshot_manifest_free(&manifest); + return false; + } + + out_state->segments = manifest.segments; + out_state->segments_len = manifest.segments_len; + out_state->tombstones = manifest.tombstones; + out_state->tombstones_len = manifest.tombstones_len; + out_state->state.snapshot_id = snapshot_id; + out_state->state.log_position = manifest.anchor_logseq; + *out_anchor_logseq = manifest.anchor_logseq; + + manifest.segments = NULL; + manifest.segments_len = 0u; + manifest.tombstones = NULL; + manifest.tombstones_len = 0u; + amduat_asl_snapshot_manifest_free(&manifest); + return true; +} + +static void amduat_asl_store_index_fs_update_ingest_state( + amduat_asl_store_index_fs_t *fs, + uint64_t added_bytes) { + uint64_t now_ns; + + if (fs == NULL) { + return; + } + if (UINT64_MAX - fs->pending_snapshot_bytes < added_bytes) { + fs->pending_snapshot_bytes = UINT64_MAX; + } else { + fs->pending_snapshot_bytes += added_bytes; + } + now_ns = amduat_asl_store_index_fs_now_ns(); + if (now_ns != 0u) { + fs->last_ingest_time_ns = now_ns; + } +} + +static void amduat_asl_store_index_fs_maybe_snapshot_idle( + amduat_asl_store_index_fs_t *fs) { + uint64_t now_ns; + + if (fs == NULL || !fs->snapshot_policy.enabled) { + return; + } + if (fs->snapshot_policy.idle_time_ns == 0u) { + return; + } + if (fs->pending_snapshot_bytes == 0u) { + return; + } + if (fs->last_ingest_time_ns == 0u) { + return; + } + now_ns = amduat_asl_store_index_fs_now_ns(); + if (now_ns == 0u) { + return; + } + if (now_ns - fs->last_ingest_time_ns < fs->snapshot_policy.idle_time_ns) { + return; + } + if (!fs->snapshot_state_initialized) { + amduat_asl_snapshot_id_t next_id; + if (!amduat_asl_store_index_fs_find_next_snapshot_id(fs, &next_id)) { + return; + } + fs->next_snapshot_id = next_id; + fs->snapshot_state_initialized = true; + } + if (fs->next_snapshot_id == 0u) { + return; + } + if (amduat_asl_store_index_fs_snapshot_create( + fs, fs->next_snapshot_id, NULL, NULL) == AMDUAT_ASL_STORE_OK) { + fs->next_snapshot_id += 1u; + fs->pending_snapshot_bytes = 0u; + } +} + +static void amduat_asl_store_index_fs_maybe_snapshot_size( + amduat_asl_store_index_fs_t *fs) { + if (fs == NULL || !fs->snapshot_policy.enabled) { + return; + } + if (fs->snapshot_policy.max_pending_bytes == 0u) { + return; + } + if (fs->pending_snapshot_bytes < fs->snapshot_policy.max_pending_bytes) { + return; + } + if (!fs->snapshot_state_initialized) { + amduat_asl_snapshot_id_t next_id; + if (!amduat_asl_store_index_fs_find_next_snapshot_id(fs, &next_id)) { + return; + } + fs->next_snapshot_id = next_id; + fs->snapshot_state_initialized = true; + } + if (fs->next_snapshot_id == 0u) { + return; + } + if (amduat_asl_store_index_fs_snapshot_create( + fs, fs->next_snapshot_id, NULL, NULL) == AMDUAT_ASL_STORE_OK) { + fs->next_snapshot_id += 1u; + fs->pending_snapshot_bytes = 0u; + } +} + static amduat_asl_store_error_t amduat_asl_store_index_fs_read_next_segment_id(const char *root_path, uint64_t *out_segment_id) { @@ -921,11 +1254,17 @@ static bool amduat_asl_store_index_fs_current_state_impl( size_t record_count; uint64_t last_logseq; amduat_asl_store_error_t err; + size_t anchor_index; + amduat_asl_snapshot_id_t snapshot_id; + uint8_t root_hash[32]; + amduat_asl_log_position_t anchor_logseq; + amduat_asl_replay_state_t replay_state; if (ctx == NULL || out_state == NULL) { return false; } fs = (amduat_asl_store_index_fs_t *)ctx; + amduat_asl_store_index_fs_maybe_snapshot_idle(fs); if (!amduat_asl_store_index_fs_layout_build_log_path(fs->root_path, &log_path)) { @@ -946,7 +1285,33 @@ static bool amduat_asl_store_index_fs_current_state_impl( last_logseq = records[record_count - 1u].logseq; } - amduat_asl_store_index_fs_fill_index_state(out_state, last_logseq); + if (record_count != 0u && + amduat_asl_store_index_fs_find_latest_anchor(records, + record_count, + last_logseq, + &anchor_index, + &snapshot_id, + root_hash, + &anchor_logseq)) { + if (!amduat_asl_store_index_fs_load_snapshot_replay(fs->root_path, + snapshot_id, + root_hash, + &replay_state, + &anchor_logseq)) { + amduat_asl_store_index_fs_log_free(records, record_count); + return false; + } + if (replay_state.state.log_position != anchor_logseq) { + amduat_asl_replay_free(&replay_state); + amduat_asl_store_index_fs_log_free(records, record_count); + return false; + } + amduat_asl_replay_free(&replay_state); + out_state->snapshot_id = snapshot_id; + } else { + out_state->snapshot_id = 0u; + } + out_state->log_position = last_logseq; amduat_enc_asl_log_free(records, record_count); return true; } @@ -963,15 +1328,18 @@ static amduat_asl_store_error_t amduat_asl_store_index_fs_get_indexed_impl( size_t record_count; amduat_asl_replay_state_t replay_state; amduat_asl_store_error_t err; + size_t anchor_index; + amduat_asl_snapshot_id_t snapshot_id; + uint8_t root_hash[32]; + amduat_asl_log_position_t anchor_logseq; + size_t replay_start; if (ctx == NULL || out_artifact == NULL) { return AMDUAT_ASL_STORE_ERR_IO; } fs = (amduat_asl_store_index_fs_t *)ctx; + amduat_asl_store_index_fs_maybe_snapshot_idle(fs); - if (state.snapshot_id != 0u) { - return AMDUAT_ASL_STORE_ERR_UNSUPPORTED; - } if (fs->config.encoding_profile_id != AMDUAT_ENC_ASL1_CORE_V1) { return AMDUAT_ASL_STORE_ERR_UNSUPPORTED; } @@ -1002,11 +1370,66 @@ static amduat_asl_store_error_t amduat_asl_store_index_fs_get_indexed_impl( return err; } - if (!amduat_asl_replay_init(&replay_state)) { - amduat_enc_asl_log_free(records, record_count); - return AMDUAT_ASL_STORE_ERR_IO; + replay_start = 0u; + if (state.snapshot_id != 0u) { + if (amduat_asl_store_index_fs_find_anchor_for_snapshot( + records, + record_count, + state.snapshot_id, + state.log_position, + &anchor_index, + root_hash, + &anchor_logseq)) { + if (!amduat_asl_store_index_fs_load_snapshot_replay(fs->root_path, + state.snapshot_id, + root_hash, + &replay_state, + &anchor_logseq)) { + amduat_enc_asl_log_free(records, record_count); + return AMDUAT_ASL_STORE_ERR_INTEGRITY; + } + if (replay_state.state.log_position != anchor_logseq) { + amduat_asl_replay_free(&replay_state); + amduat_enc_asl_log_free(records, record_count); + return AMDUAT_ASL_STORE_ERR_INTEGRITY; + } + replay_start = anchor_index + 1u; + } else { + amduat_enc_asl_log_free(records, record_count); + return AMDUAT_ASL_STORE_ERR_UNSUPPORTED; + } + } else if (record_count != 0u && + amduat_asl_store_index_fs_find_latest_anchor(records, + record_count, + state.log_position, + &anchor_index, + &snapshot_id, + root_hash, + &anchor_logseq)) { + if (!amduat_asl_store_index_fs_load_snapshot_replay(fs->root_path, + snapshot_id, + root_hash, + &replay_state, + &anchor_logseq)) { + amduat_enc_asl_log_free(records, record_count); + return AMDUAT_ASL_STORE_ERR_INTEGRITY; + } + if (replay_state.state.log_position != anchor_logseq) { + amduat_asl_replay_free(&replay_state); + amduat_enc_asl_log_free(records, record_count); + return AMDUAT_ASL_STORE_ERR_INTEGRITY; + } + replay_start = anchor_index + 1u; + } else { + if (!amduat_asl_replay_init(&replay_state)) { + amduat_enc_asl_log_free(records, record_count); + return AMDUAT_ASL_STORE_ERR_IO; + } } - if (!amduat_asl_replay_apply_log(records, record_count, state.log_position, + + if (!amduat_asl_replay_apply_log(records + replay_start, + record_count - replay_start, + state.log_position, &replay_state)) { amduat_enc_asl_log_free(records, record_count); amduat_asl_replay_free(&replay_state); @@ -1052,11 +1475,13 @@ static amduat_asl_store_error_t amduat_asl_store_index_fs_put_indexed_impl( size_t seal_len; uint64_t new_logseq; amduat_asl_store_index_fs_write_status_t write_status; + size_t artifact_len; if (ctx == NULL || out_ref == NULL || out_state == NULL) { return AMDUAT_ASL_STORE_ERR_IO; } fs = (amduat_asl_store_index_fs_t *)ctx; + amduat_asl_store_index_fs_maybe_snapshot_idle(fs); err = amduat_asl_store_index_fs_validate_config(ctx, fs->config); if (err != AMDUAT_ASL_STORE_OK) { @@ -1379,6 +1804,7 @@ static amduat_asl_store_error_t amduat_asl_store_index_fs_put_indexed_impl( free(index_path); free(segments_path); free(blocks_path); + artifact_len = artifact_bytes.len; amduat_octets_free(&artifact_bytes); if (err != AMDUAT_ASL_STORE_OK) { @@ -1388,6 +1814,8 @@ static amduat_asl_store_error_t amduat_asl_store_index_fs_put_indexed_impl( *out_ref = derived_ref; amduat_asl_store_index_fs_fill_index_state(out_state, new_logseq); + amduat_asl_store_index_fs_update_ingest_state(fs, artifact_len); + amduat_asl_store_index_fs_maybe_snapshot_size(fs); return AMDUAT_ASL_STORE_OK; } @@ -1414,6 +1842,230 @@ static amduat_asl_store_error_t amduat_asl_store_index_fs_get_impl( out_artifact); } +amduat_asl_store_error_t amduat_asl_store_index_fs_snapshot_create( + amduat_asl_store_index_fs_t *fs, + amduat_asl_snapshot_id_t snapshot_id, + amduat_asl_log_position_t *out_logseq, + uint8_t out_root_hash[32]) { + amduat_asl_store_error_t err; + char *index_path; + char *snapshots_path; + char *log_path; + char *manifest_path; + amduat_asl_log_record_t *log_records; + size_t log_count; + uint64_t last_logseq; + uint64_t new_logseq; + amduat_asl_replay_state_t replay_state; + amduat_asl_snapshot_manifest_t manifest; + uint8_t root_hash[32]; + uint8_t *payload; + size_t payload_len; + + if (fs == NULL) { + return AMDUAT_ASL_STORE_ERR_IO; + } + if (snapshot_id == 0u) { + return AMDUAT_ASL_STORE_ERR_UNSUPPORTED; + } + + err = amduat_asl_store_index_fs_validate_config(fs, fs->config); + if (err != AMDUAT_ASL_STORE_OK) { + return err; + } + + index_path = NULL; + snapshots_path = NULL; + log_path = NULL; + manifest_path = NULL; + + if (!amduat_asl_store_index_fs_layout_build_index_path(fs->root_path, + &index_path) || + !amduat_asl_store_index_fs_ensure_directory(index_path)) { + free(index_path); + return AMDUAT_ASL_STORE_ERR_IO; + } + if (!amduat_asl_store_index_fs_layout_build_snapshots_path(fs->root_path, + &snapshots_path) || + !amduat_asl_store_index_fs_ensure_directory(snapshots_path)) { + free(index_path); + free(snapshots_path); + return AMDUAT_ASL_STORE_ERR_IO; + } + free(snapshots_path); + snapshots_path = NULL; + + if (!amduat_asl_store_index_fs_layout_build_snapshot_manifest_path( + fs->root_path, snapshot_id, &manifest_path)) { + free(index_path); + return AMDUAT_ASL_STORE_ERR_IO; + } + { + struct stat st; + if (stat(manifest_path, &st) == 0) { + free(index_path); + free(manifest_path); + return AMDUAT_ASL_STORE_ERR_IO; + } + if (errno != ENOENT) { + free(index_path); + free(manifest_path); + return AMDUAT_ASL_STORE_ERR_IO; + } + } + + if (!amduat_asl_store_index_fs_layout_build_log_path(fs->root_path, + &log_path)) { + free(index_path); + free(manifest_path); + return AMDUAT_ASL_STORE_ERR_IO; + } + + log_records = NULL; + log_count = 0u; + err = amduat_asl_store_index_fs_load_log(log_path, &log_records, &log_count); + if (err != AMDUAT_ASL_STORE_OK) { + amduat_asl_store_index_fs_log_free(log_records, log_count); + free(index_path); + free(log_path); + free(manifest_path); + return err; + } + + last_logseq = 0u; + if (log_count != 0u) { + last_logseq = log_records[log_count - 1u].logseq; + } + if (last_logseq == UINT64_MAX) { + amduat_asl_store_index_fs_log_free(log_records, log_count); + free(index_path); + free(log_path); + free(manifest_path); + return AMDUAT_ASL_STORE_ERR_IO; + } + new_logseq = (log_count == 0u) ? 1u : last_logseq + 1u; + + if (!amduat_asl_replay_init(&replay_state)) { + amduat_asl_store_index_fs_log_free(log_records, log_count); + free(index_path); + free(log_path); + free(manifest_path); + return AMDUAT_ASL_STORE_ERR_IO; + } + if (!amduat_asl_replay_apply_log(log_records, + log_count, + last_logseq, + &replay_state)) { + amduat_asl_store_index_fs_log_free(log_records, log_count); + amduat_asl_replay_free(&replay_state); + free(index_path); + free(log_path); + free(manifest_path); + return AMDUAT_ASL_STORE_ERR_INTEGRITY; + } + + memset(&manifest, 0, sizeof(manifest)); + manifest.snapshot_id = snapshot_id; + manifest.anchor_logseq = new_logseq; + manifest.config = fs->config; + manifest.segments = replay_state.segments; + manifest.segments_len = replay_state.segments_len; + manifest.tombstones = replay_state.tombstones; + manifest.tombstones_len = replay_state.tombstones_len; + + if (!amduat_asl_snapshot_manifest_write(manifest_path, + &manifest, + root_hash)) { + amduat_asl_store_index_fs_log_free(log_records, log_count); + amduat_asl_replay_free(&replay_state); + free(index_path); + free(log_path); + free(manifest_path); + return AMDUAT_ASL_STORE_ERR_IO; + } + + if (log_count == 0u) { + log_records = (amduat_asl_log_record_t *)calloc(1u, + sizeof(*log_records)); + if (log_records == NULL) { + unlink(manifest_path); + amduat_asl_replay_free(&replay_state); + free(index_path); + free(log_path); + free(manifest_path); + return AMDUAT_ASL_STORE_ERR_IO; + } + } else { + amduat_asl_log_record_t *next = (amduat_asl_log_record_t *)realloc( + log_records, (log_count + 1u) * sizeof(*log_records)); + if (next == NULL) { + unlink(manifest_path); + amduat_asl_store_index_fs_log_free(log_records, log_count); + amduat_asl_replay_free(&replay_state); + free(index_path); + free(log_path); + free(manifest_path); + return AMDUAT_ASL_STORE_ERR_IO; + } + log_records = next; + } + + payload_len = 8u + 32u; + payload = (uint8_t *)malloc(payload_len); + if (payload == NULL) { + unlink(manifest_path); + amduat_asl_store_index_fs_log_free(log_records, log_count); + amduat_asl_replay_free(&replay_state); + free(index_path); + free(log_path); + free(manifest_path); + return AMDUAT_ASL_STORE_ERR_IO; + } + payload[0] = (uint8_t)(snapshot_id & 0xffu); + payload[1] = (uint8_t)((snapshot_id >> 8) & 0xffu); + payload[2] = (uint8_t)((snapshot_id >> 16) & 0xffu); + payload[3] = (uint8_t)((snapshot_id >> 24) & 0xffu); + payload[4] = (uint8_t)((snapshot_id >> 32) & 0xffu); + payload[5] = (uint8_t)((snapshot_id >> 40) & 0xffu); + payload[6] = (uint8_t)((snapshot_id >> 48) & 0xffu); + payload[7] = (uint8_t)((snapshot_id >> 56) & 0xffu); + memcpy(payload + 8, root_hash, 32u); + + log_records[log_count].logseq = new_logseq; + log_records[log_count].record_type = AMDUAT_ASL_LOG_RECORD_SNAPSHOT_ANCHOR; + log_records[log_count].payload = amduat_octets(payload, payload_len); + memset(log_records[log_count].record_hash, 0, + sizeof(log_records[log_count].record_hash)); + log_count += 1u; + + err = amduat_asl_store_index_fs_write_log(log_path, index_path, + log_records, log_count); + amduat_asl_store_index_fs_log_free(log_records, log_count); + amduat_asl_replay_free(&replay_state); + free(index_path); + free(log_path); + if (err != AMDUAT_ASL_STORE_OK) { + unlink(manifest_path); + free(manifest_path); + return err; + } + + if (out_logseq != NULL) { + *out_logseq = new_logseq; + } + if (out_root_hash != NULL) { + memcpy(out_root_hash, root_hash, 32u); + } + fs->pending_snapshot_bytes = 0u; + if (fs->snapshot_state_initialized) { + if (snapshot_id >= fs->next_snapshot_id && snapshot_id != UINT64_MAX) { + fs->next_snapshot_id = snapshot_id + 1u; + } + } + free(manifest_path); + return AMDUAT_ASL_STORE_OK; +} + bool amduat_asl_store_index_fs_init(amduat_asl_store_index_fs_t *fs, amduat_asl_store_config_t config, const char *root_path) { @@ -1429,9 +2081,27 @@ bool amduat_asl_store_index_fs_init(amduat_asl_store_index_fs_t *fs, memcpy(fs->root_path, root_path, len); fs->root_path[len] = '\0'; fs->config = config; + fs->snapshot_policy.enabled = true; + fs->snapshot_policy.max_pending_bytes = + AMDUAT_ASL_STORE_INDEX_FS_DEFAULT_PENDING_BYTES; + fs->snapshot_policy.idle_time_ns = + AMDUAT_ASL_STORE_INDEX_FS_DEFAULT_IDLE_NS; + fs->pending_snapshot_bytes = 0u; + fs->last_ingest_time_ns = 0u; + fs->next_snapshot_id = 0u; + fs->snapshot_state_initialized = false; return true; } +void amduat_asl_store_index_fs_set_snapshot_policy( + amduat_asl_store_index_fs_t *fs, + amduat_asl_store_index_fs_snapshot_policy_t policy) { + if (fs == NULL) { + return; + } + fs->snapshot_policy = policy; +} + amduat_asl_store_ops_t amduat_asl_store_index_fs_ops(void) { amduat_asl_store_ops_t ops; diff --git a/src/adapters/asl_store_index_fs/asl_store_index_fs_layout.c b/src/adapters/asl_store_index_fs/asl_store_index_fs_layout.c index f12cad4..3f357a3 100644 --- a/src/adapters/asl_store_index_fs/asl_store_index_fs_layout.c +++ b/src/adapters/asl_store_index_fs/asl_store_index_fs_layout.c @@ -121,6 +121,47 @@ bool amduat_asl_store_index_fs_layout_build_log_path(const char *root_path, return ok; } +bool amduat_asl_store_index_fs_layout_build_snapshots_path( + const char *root_path, + char **out_path) { + char *index_path; + bool ok; + + if (!amduat_asl_store_index_fs_layout_build_index_path(root_path, + &index_path)) { + return false; + } + ok = amduat_asl_store_index_fs_layout_join(index_path, "snapshots", out_path); + free(index_path); + return ok; +} + +bool amduat_asl_store_index_fs_layout_build_snapshot_manifest_path( + const char *root_path, + uint64_t snapshot_id, + char **out_path) { + char *snapshots_path; + char *name; + bool ok; + + if (!amduat_asl_store_index_fs_layout_build_snapshots_path(root_path, + &snapshots_path)) { + return false; + } + if (!amduat_asl_store_index_fs_layout_format_id("snap-", + snapshot_id, + ".bin", + &name)) { + free(snapshots_path); + return false; + } + + ok = amduat_asl_store_index_fs_layout_join(snapshots_path, name, out_path); + free(name); + free(snapshots_path); + return ok; +} + bool amduat_asl_store_index_fs_layout_build_segment_meta_path( const char *root_path, char **out_path) { diff --git a/src/adapters/asl_store_index_fs/asl_store_index_fs_layout.h b/src/adapters/asl_store_index_fs/asl_store_index_fs_layout.h index 3587e93..c791ad5 100644 --- a/src/adapters/asl_store_index_fs/asl_store_index_fs_layout.h +++ b/src/adapters/asl_store_index_fs/asl_store_index_fs_layout.h @@ -21,6 +21,15 @@ bool amduat_asl_store_index_fs_layout_build_blocks_path(const char *root_path, bool amduat_asl_store_index_fs_layout_build_log_path(const char *root_path, char **out_path); +bool amduat_asl_store_index_fs_layout_build_snapshots_path( + const char *root_path, + char **out_path); + +bool amduat_asl_store_index_fs_layout_build_snapshot_manifest_path( + const char *root_path, + uint64_t snapshot_id, + char **out_path); + bool amduat_asl_store_index_fs_layout_build_segment_meta_path( const char *root_path, char **out_path); diff --git a/src/near_core/asl/index_snapshot.c b/src/near_core/asl/index_snapshot.c new file mode 100644 index 0000000..be1ac62 --- /dev/null +++ b/src/near_core/asl/index_snapshot.c @@ -0,0 +1,487 @@ +#include "amduat/asl/index_snapshot.h" + +#include "amduat/asl/io.h" +#include "amduat/hash/asl1.h" + +#include +#include +#include + +enum { + AMDUAT_ASL_SNAPSHOT_MANIFEST_SEGMENT_SIZE = 40 +}; + +static const uint8_t k_amduat_asl_snapshot_manifest_magic[ + AMDUAT_ASL_SNAPSHOT_MANIFEST_MAGIC_LEN] = {'A', 'S', 'L', 'S', + 'N', 'A', 'P', '1'}; + +static void amduat_asl_snapshot_store_u16_le(uint8_t *out, + uint16_t value) { + out[0] = (uint8_t)(value & 0xffu); + out[1] = (uint8_t)((value >> 8) & 0xffu); +} + +static void amduat_asl_snapshot_store_u32_le(uint8_t *out, + uint32_t value) { + out[0] = (uint8_t)(value & 0xffu); + out[1] = (uint8_t)((value >> 8) & 0xffu); + out[2] = (uint8_t)((value >> 16) & 0xffu); + out[3] = (uint8_t)((value >> 24) & 0xffu); +} + +static void amduat_asl_snapshot_store_u64_le(uint8_t *out, + uint64_t value) { + out[0] = (uint8_t)(value & 0xffu); + out[1] = (uint8_t)((value >> 8) & 0xffu); + out[2] = (uint8_t)((value >> 16) & 0xffu); + out[3] = (uint8_t)((value >> 24) & 0xffu); + out[4] = (uint8_t)((value >> 32) & 0xffu); + out[5] = (uint8_t)((value >> 40) & 0xffu); + out[6] = (uint8_t)((value >> 48) & 0xffu); + out[7] = (uint8_t)((value >> 56) & 0xffu); +} + +static uint16_t amduat_asl_snapshot_load_u16_le(const uint8_t *data) { + return (uint16_t)data[0] | ((uint16_t)data[1] << 8); +} + +static uint32_t amduat_asl_snapshot_load_u32_le(const uint8_t *data) { + return (uint32_t)data[0] | ((uint32_t)data[1] << 8) | + ((uint32_t)data[2] << 16) | ((uint32_t)data[3] << 24); +} + +static uint64_t amduat_asl_snapshot_load_u64_le(const uint8_t *data) { + return (uint64_t)data[0] | ((uint64_t)data[1] << 8) | + ((uint64_t)data[2] << 16) | ((uint64_t)data[3] << 24) | + ((uint64_t)data[4] << 32) | ((uint64_t)data[5] << 40) | + ((uint64_t)data[6] << 48) | ((uint64_t)data[7] << 56); +} + +static bool amduat_asl_snapshot_add_size(size_t *acc, size_t add) { + if (*acc > SIZE_MAX - add) { + return false; + } + *acc += add; + return true; +} + +static int amduat_asl_snapshot_compare_segments(const void *lhs, + const void *rhs) { + const amduat_asl_segment_seal_t *left = + *(const amduat_asl_segment_seal_t *const *)lhs; + const amduat_asl_segment_seal_t *right = + *(const amduat_asl_segment_seal_t *const *)rhs; + + if (left->segment_id < right->segment_id) { + return -1; + } + if (left->segment_id > right->segment_id) { + return 1; + } + return memcmp(left->segment_hash, right->segment_hash, + sizeof(left->segment_hash)); +} + +static int amduat_asl_snapshot_compare_tombstones(const void *lhs, + const void *rhs) { + const amduat_asl_tombstone_entry_t *left = + *(const amduat_asl_tombstone_entry_t *const *)lhs; + const amduat_asl_tombstone_entry_t *right = + *(const amduat_asl_tombstone_entry_t *const *)rhs; + size_t min_len; + int cmp; + + if (left->ref.hash_id < right->ref.hash_id) { + return -1; + } + if (left->ref.hash_id > right->ref.hash_id) { + return 1; + } + if (left->ref.digest.len < right->ref.digest.len) { + return -1; + } + if (left->ref.digest.len > right->ref.digest.len) { + return 1; + } + min_len = left->ref.digest.len; + if (min_len != 0u) { + cmp = memcmp(left->ref.digest.data, right->ref.digest.data, min_len); + if (cmp != 0) { + return cmp; + } + } + if (left->tombstone_logseq < right->tombstone_logseq) { + return -1; + } + if (left->tombstone_logseq > right->tombstone_logseq) { + return 1; + } + return 0; +} + +static bool amduat_asl_snapshot_hash_manifest(const uint8_t *bytes, + size_t len, + uint8_t out_hash[32]) { + if (out_hash == NULL) { + return true; + } + return amduat_hash_asl1_digest(AMDUAT_HASH_ASL1_ID_SHA256, + amduat_octets(bytes, len), out_hash, 32); +} + +void amduat_asl_snapshot_manifest_free( + amduat_asl_snapshot_manifest_t *manifest) { + size_t i; + + if (manifest == NULL) { + return; + } + free(manifest->segments); + manifest->segments = NULL; + manifest->segments_len = 0; + if (manifest->tombstones != NULL) { + for (i = 0; i < manifest->tombstones_len; ++i) { + amduat_reference_free(&manifest->tombstones[i].ref); + } + } + free(manifest->tombstones); + manifest->tombstones = NULL; + manifest->tombstones_len = 0; +} + +bool amduat_asl_snapshot_manifest_write( + const char *path, + const amduat_asl_snapshot_manifest_t *manifest, + uint8_t out_hash[32]) { + size_t i; + size_t total_len; + size_t offset; + uint8_t *buffer; + amduat_asl_segment_seal_t **segment_order; + amduat_asl_tombstone_entry_t **tombstone_order; + + if (path == NULL || manifest == NULL) { + return false; + } + if (manifest->segments_len != 0u && manifest->segments == NULL) { + return false; + } + if (manifest->tombstones_len != 0u && manifest->tombstones == NULL) { + return false; + } + + if (manifest->segments_len > SIZE_MAX / AMDUAT_ASL_SNAPSHOT_MANIFEST_SEGMENT_SIZE) { + return false; + } + + total_len = 0; + if (!amduat_asl_snapshot_add_size(&total_len, + AMDUAT_ASL_SNAPSHOT_MANIFEST_HEADER_SIZE)) { + return false; + } + if (!amduat_asl_snapshot_add_size( + &total_len, + manifest->segments_len * + AMDUAT_ASL_SNAPSHOT_MANIFEST_SEGMENT_SIZE)) { + return false; + } + + for (i = 0; i < manifest->tombstones_len; ++i) { + const amduat_asl_tombstone_entry_t *entry = &manifest->tombstones[i]; + if (entry->ref.digest.len == 0 || + (entry->ref.digest.len != 0u && entry->ref.digest.data == NULL)) { + return false; + } + if (entry->ref.digest.len > UINT16_MAX) { + return false; + } + if (!amduat_asl_snapshot_add_size(&total_len, + 16u + entry->ref.digest.len)) { + return false; + } + } + + buffer = (uint8_t *)malloc(total_len); + if (buffer == NULL) { + return false; + } + + segment_order = NULL; + tombstone_order = NULL; + if (manifest->segments_len != 0u) { + segment_order = (amduat_asl_segment_seal_t **)calloc( + manifest->segments_len, sizeof(*segment_order)); + if (segment_order == NULL) { + free(buffer); + return false; + } + for (i = 0; i < manifest->segments_len; ++i) { + segment_order[i] = &manifest->segments[i]; + } + qsort(segment_order, manifest->segments_len, sizeof(*segment_order), + amduat_asl_snapshot_compare_segments); + } + + if (manifest->tombstones_len != 0u) { + tombstone_order = (amduat_asl_tombstone_entry_t **)calloc( + manifest->tombstones_len, sizeof(*tombstone_order)); + if (tombstone_order == NULL) { + free(segment_order); + free(buffer); + return false; + } + for (i = 0; i < manifest->tombstones_len; ++i) { + tombstone_order[i] = &manifest->tombstones[i]; + } + qsort(tombstone_order, manifest->tombstones_len, + sizeof(*tombstone_order), amduat_asl_snapshot_compare_tombstones); + } + + offset = 0; + memcpy(buffer + offset, k_amduat_asl_snapshot_manifest_magic, + AMDUAT_ASL_SNAPSHOT_MANIFEST_MAGIC_LEN); + offset += AMDUAT_ASL_SNAPSHOT_MANIFEST_MAGIC_LEN; + amduat_asl_snapshot_store_u16_le(buffer + offset, + AMDUAT_ASL_SNAPSHOT_MANIFEST_VERSION); + offset += 2; + amduat_asl_snapshot_store_u16_le(buffer + offset, 0); + offset += 2; + amduat_asl_snapshot_store_u32_le(buffer + offset, + AMDUAT_ASL_SNAPSHOT_MANIFEST_HEADER_SIZE); + offset += 4; + amduat_asl_snapshot_store_u64_le(buffer + offset, manifest->snapshot_id); + offset += 8; + amduat_asl_snapshot_store_u64_le(buffer + offset, manifest->anchor_logseq); + offset += 8; + amduat_asl_snapshot_store_u64_le(buffer + offset, manifest->segments_len); + offset += 8; + amduat_asl_snapshot_store_u64_le(buffer + offset, manifest->tombstones_len); + offset += 8; + amduat_asl_snapshot_store_u32_le(buffer + offset, + manifest->config.encoding_profile_id); + offset += 4; + amduat_asl_snapshot_store_u32_le(buffer + offset, + manifest->config.hash_id); + offset += 4; + memset(buffer + offset, 0, + AMDUAT_ASL_SNAPSHOT_MANIFEST_HEADER_SIZE - offset); + offset = AMDUAT_ASL_SNAPSHOT_MANIFEST_HEADER_SIZE; + + for (i = 0; i < manifest->segments_len; ++i) { + const amduat_asl_segment_seal_t *entry = segment_order[i]; + amduat_asl_snapshot_store_u64_le(buffer + offset, entry->segment_id); + offset += 8; + memcpy(buffer + offset, entry->segment_hash, sizeof(entry->segment_hash)); + offset += sizeof(entry->segment_hash); + } + + for (i = 0; i < manifest->tombstones_len; ++i) { + const amduat_asl_tombstone_entry_t *entry = tombstone_order[i]; + amduat_asl_snapshot_store_u32_le(buffer + offset, entry->ref.hash_id); + offset += 4; + amduat_asl_snapshot_store_u16_le(buffer + offset, + (uint16_t)entry->ref.digest.len); + offset += 2; + amduat_asl_snapshot_store_u16_le(buffer + offset, 0); + offset += 2; + memcpy(buffer + offset, entry->ref.digest.data, entry->ref.digest.len); + offset += entry->ref.digest.len; + amduat_asl_snapshot_store_u64_le(buffer + offset, + entry->tombstone_logseq); + offset += 8; + } + + free(segment_order); + free(tombstone_order); + + if (offset != total_len) { + free(buffer); + return false; + } + + if (!amduat_asl_snapshot_hash_manifest(buffer, total_len, out_hash)) { + free(buffer); + return false; + } + + if (!amduat_asl_write_path(path, buffer, total_len)) { + free(buffer); + return false; + } + + free(buffer); + return true; +} + +bool amduat_asl_snapshot_manifest_read( + const char *path, + amduat_asl_snapshot_manifest_t *out_manifest, + uint8_t out_hash[32]) { + uint8_t *buffer; + size_t len; + size_t offset; + uint16_t version; + uint16_t reserved0; + uint32_t header_size; + uint64_t segment_count; + uint64_t tombstone_count; + size_t i; + + if (path == NULL || out_manifest == NULL) { + return false; + } + memset(out_manifest, 0, sizeof(*out_manifest)); + + buffer = NULL; + len = 0u; + if (!amduat_asl_read_path(path, &buffer, &len)) { + return false; + } + if (len < AMDUAT_ASL_SNAPSHOT_MANIFEST_HEADER_SIZE) { + free(buffer); + return false; + } + + if (!amduat_asl_snapshot_hash_manifest(buffer, len, out_hash)) { + free(buffer); + return false; + } + + if (memcmp(buffer, k_amduat_asl_snapshot_manifest_magic, + AMDUAT_ASL_SNAPSHOT_MANIFEST_MAGIC_LEN) != 0) { + free(buffer); + return false; + } + + offset = AMDUAT_ASL_SNAPSHOT_MANIFEST_MAGIC_LEN; + version = amduat_asl_snapshot_load_u16_le(buffer + offset); + offset += 2; + reserved0 = amduat_asl_snapshot_load_u16_le(buffer + offset); + offset += 2; + header_size = amduat_asl_snapshot_load_u32_le(buffer + offset); + offset += 4; + + if (version != AMDUAT_ASL_SNAPSHOT_MANIFEST_VERSION || reserved0 != 0 || + header_size != AMDUAT_ASL_SNAPSHOT_MANIFEST_HEADER_SIZE) { + free(buffer); + return false; + } + + out_manifest->snapshot_id = amduat_asl_snapshot_load_u64_le(buffer + offset); + offset += 8; + out_manifest->anchor_logseq = + amduat_asl_snapshot_load_u64_le(buffer + offset); + offset += 8; + segment_count = amduat_asl_snapshot_load_u64_le(buffer + offset); + offset += 8; + tombstone_count = amduat_asl_snapshot_load_u64_le(buffer + offset); + offset += 8; + out_manifest->config.encoding_profile_id = + amduat_asl_snapshot_load_u32_le(buffer + offset); + offset += 4; + out_manifest->config.hash_id = + amduat_asl_snapshot_load_u32_le(buffer + offset); + offset += 4; + offset = AMDUAT_ASL_SNAPSHOT_MANIFEST_HEADER_SIZE; + + if (segment_count > SIZE_MAX || tombstone_count > SIZE_MAX) { + free(buffer); + return false; + } + + out_manifest->segments_len = (size_t)segment_count; + if (out_manifest->segments_len != 0u) { + if (out_manifest->segments_len > + SIZE_MAX / AMDUAT_ASL_SNAPSHOT_MANIFEST_SEGMENT_SIZE) { + free(buffer); + return false; + } + out_manifest->segments = (amduat_asl_segment_seal_t *)calloc( + out_manifest->segments_len, sizeof(*out_manifest->segments)); + if (out_manifest->segments == NULL) { + free(buffer); + return false; + } + } + + for (i = 0; i < out_manifest->segments_len; ++i) { + amduat_asl_segment_seal_t *entry = &out_manifest->segments[i]; + if (len - offset < AMDUAT_ASL_SNAPSHOT_MANIFEST_SEGMENT_SIZE) { + amduat_asl_snapshot_manifest_free(out_manifest); + free(buffer); + return false; + } + entry->segment_id = amduat_asl_snapshot_load_u64_le(buffer + offset); + offset += 8; + memcpy(entry->segment_hash, buffer + offset, sizeof(entry->segment_hash)); + offset += sizeof(entry->segment_hash); + } + + out_manifest->tombstones_len = (size_t)tombstone_count; + if (out_manifest->tombstones_len != 0u) { + out_manifest->tombstones = (amduat_asl_tombstone_entry_t *)calloc( + out_manifest->tombstones_len, sizeof(*out_manifest->tombstones)); + if (out_manifest->tombstones == NULL) { + amduat_asl_snapshot_manifest_free(out_manifest); + free(buffer); + return false; + } + } + + for (i = 0; i < out_manifest->tombstones_len; ++i) { + amduat_asl_tombstone_entry_t *entry = &out_manifest->tombstones[i]; + uint32_t hash_id; + uint16_t digest_len; + uint16_t tombstone_reserved; + amduat_reference_t ref; + uint64_t tombstone_logseq; + + if (len - offset < 16) { + amduat_asl_snapshot_manifest_free(out_manifest); + free(buffer); + return false; + } + + hash_id = amduat_asl_snapshot_load_u32_le(buffer + offset); + offset += 4; + if (hash_id > UINT16_MAX) { + amduat_asl_snapshot_manifest_free(out_manifest); + free(buffer); + return false; + } + digest_len = amduat_asl_snapshot_load_u16_le(buffer + offset); + offset += 2; + tombstone_reserved = amduat_asl_snapshot_load_u16_le(buffer + offset); + offset += 2; + if (tombstone_reserved != 0 || digest_len == 0) { + amduat_asl_snapshot_manifest_free(out_manifest); + free(buffer); + return false; + } + if (len - offset < (size_t)digest_len + 8u) { + amduat_asl_snapshot_manifest_free(out_manifest); + free(buffer); + return false; + } + ref = amduat_reference((amduat_hash_id_t)hash_id, + amduat_octets(buffer + offset, digest_len)); + offset += digest_len; + tombstone_logseq = amduat_asl_snapshot_load_u64_le(buffer + offset); + offset += 8; + if (!amduat_reference_clone(ref, &entry->ref)) { + amduat_asl_snapshot_manifest_free(out_manifest); + free(buffer); + return false; + } + entry->tombstone_logseq = tombstone_logseq; + } + + if (offset != len) { + amduat_asl_snapshot_manifest_free(out_manifest); + free(buffer); + return false; + } + + free(buffer); + return true; +} diff --git a/src/tools/amduat_asl_cli.c b/src/tools/amduat_asl_cli.c index 0b575fd..dde1311 100644 --- a/src/tools/amduat_asl_cli.c +++ b/src/tools/amduat_asl_cli.c @@ -69,6 +69,16 @@ typedef struct { bool quiet; } amduat_asl_cli_get_opts_t; +typedef struct { + amduat_asl_store_fs_config_t cfg; + amduat_asl_store_fs_t fs; + amduat_asl_store_index_fs_t index_fs; + amduat_asl_store_t store; + bool is_index; +} amduat_asl_cli_store_ctx_t; + +static bool amduat_asl_cli_is_index_store_root(const char *root_path); + static void amduat_asl_cli_print_usage(FILE *stream) { fprintf(stream, "usage:\n" @@ -85,6 +95,9 @@ static void amduat_asl_cli_print_usage(FILE *stream) { " [--expect-type-tag TAG] [--print-type-tag]\n" " [--quiet]\n" " amduat-asl log inspect [--root PATH]\n" + " amduat-asl index init [--root PATH] [--store-id ID]\n" + " [--profile PROFILE_ID|name]\n" + " [--hash HASH_ID|name] [--force] [--quiet]\n" " amduat-asl index state [--root PATH]\n" " amduat-asl segment verify [--root PATH] [--segment ID]\n" "\n" @@ -196,6 +209,27 @@ static bool amduat_asl_cli_join_path(const char *base, return true; } +static bool amduat_asl_cli_ensure_dir(const char *path) { + struct stat st; + + if (path == NULL || path[0] == '\0') { + return false; + } + if (stat(path, &st) == 0) { + return S_ISDIR(st.st_mode); + } + if (errno != ENOENT) { + return false; + } + if (mkdir(path, 0755) != 0) { + if (errno == EEXIST) { + return stat(path, &st) == 0 && S_ISDIR(st.st_mode); + } + return false; + } + return true; +} + static bool amduat_asl_cli_build_index_path(const char *root_path, char **out_path) { return amduat_asl_cli_join_path(root_path, "index", out_path); @@ -227,6 +261,19 @@ static bool amduat_asl_cli_build_segments_path(const char *root_path, return ok; } +static bool amduat_asl_cli_build_blocks_path(const char *root_path, + char **out_path) { + char *index_path; + bool ok; + + if (!amduat_asl_cli_build_index_path(root_path, &index_path)) { + return false; + } + ok = amduat_asl_cli_join_path(index_path, "blocks", out_path); + free(index_path); + return ok; +} + static bool amduat_asl_cli_build_segment_path(const char *root_path, uint64_t segment_id, char **out_path) { @@ -279,6 +326,46 @@ static bool amduat_asl_cli_is_index_store_root(const char *root_path) { return ok; } +static bool amduat_asl_cli_open_store(const char *root_path, + amduat_asl_cli_store_ctx_t *out_ctx) { + bool use_index; + + if (root_path == NULL || out_ctx == NULL) { + return false; + } + memset(out_ctx, 0, sizeof(*out_ctx)); + + if (!amduat_asl_store_fs_load_config(root_path, &out_ctx->cfg)) { + return false; + } + + use_index = amduat_asl_cli_is_index_store_root(root_path); + out_ctx->is_index = use_index; + if (use_index) { + if (!amduat_asl_store_index_fs_init(&out_ctx->index_fs, + out_ctx->cfg.config, + root_path)) { + return false; + } + amduat_asl_store_init(&out_ctx->store, + out_ctx->cfg.config, + amduat_asl_store_index_fs_ops(), + &out_ctx->index_fs); + } else { + if (!amduat_asl_store_fs_init(&out_ctx->fs, + out_ctx->cfg.config, + root_path)) { + return false; + } + amduat_asl_store_init(&out_ctx->store, + out_ctx->cfg.config, + amduat_asl_store_fs_ops(), + &out_ctx->fs); + } + + return true; +} + static bool amduat_asl_cli_read_u64_le(const uint8_t *bytes, size_t len, uint64_t *out_value) { @@ -371,6 +458,32 @@ static int amduat_asl_cli_segment_id_cmp(const void *lhs, const void *rhs) { return 0; } +static bool amduat_asl_cli_write_empty_log(const char *root_path) { + char *log_path; + amduat_octets_t log_bytes; + bool ok; + + if (!amduat_asl_cli_build_log_path(root_path, &log_path)) { + return false; + } + if (amduat_asl_cli_path_is_file(log_path)) { + free(log_path); + return true; + } + + log_bytes.data = NULL; + log_bytes.len = 0u; + ok = amduat_enc_asl_log_encode_v1(NULL, 0u, &log_bytes); + if (!ok) { + free(log_path); + return false; + } + ok = amduat_asl_write_path(log_path, log_bytes.data, log_bytes.len); + amduat_octets_free(&log_bytes); + free(log_path); + return ok; +} + static bool amduat_asl_cli_collect_segment_ids(const char *root_path, uint64_t **out_ids, size_t *out_len) { @@ -608,9 +721,7 @@ static int amduat_asl_cli_cmd_init(int argc, char **argv) { static int amduat_asl_cli_cmd_put(int argc, char **argv) { amduat_asl_cli_put_opts_t opts; - amduat_asl_store_fs_config_t cfg; - amduat_asl_store_fs_t fs; - amduat_asl_store_t store; + amduat_asl_cli_store_ctx_t store_ctx; amduat_artifact_t artifact; amduat_reference_t ref; uint8_t *input_bytes; @@ -691,15 +802,10 @@ static int amduat_asl_cli_cmd_put(int argc, char **argv) { return AMDUAT_ASL_CLI_EXIT_USAGE; } - if (!amduat_asl_store_fs_load_config(opts.root, &cfg)) { + if (!amduat_asl_cli_open_store(opts.root, &store_ctx)) { fprintf(stderr, "error: failed to load store config: %s\n", opts.root); return AMDUAT_ASL_CLI_EXIT_CONFIG; } - if (!amduat_asl_store_fs_init(&fs, cfg.config, opts.root)) { - fprintf(stderr, "error: failed to initialize store\n"); - return AMDUAT_ASL_CLI_EXIT_CONFIG; - } - amduat_asl_store_init(&store, cfg.config, amduat_asl_store_fs_ops(), &fs); input_bytes = NULL; input_len = 0u; @@ -725,7 +831,7 @@ static int amduat_asl_cli_cmd_put(int argc, char **argv) { exit_code = AMDUAT_ASL_CLI_EXIT_OK; { amduat_asl_store_error_t err = - amduat_asl_store_put(&store, artifact, &ref); + amduat_asl_store_put(&store_ctx.store, artifact, &ref); if (err != AMDUAT_ASL_STORE_OK) { fprintf(stderr, "error: store put failed: %s\n", amduat_asl_cli_store_error_str(err)); @@ -763,7 +869,7 @@ static int amduat_asl_cli_cmd_put(int argc, char **argv) { if (exit_code == AMDUAT_ASL_CLI_EXIT_OK && !opts.quiet) { char *hex_ref = NULL; fprintf(stderr, "root=%s\n", opts.root); - amduat_asl_cli_print_store_meta(&cfg); + amduat_asl_cli_print_store_meta(&store_ctx.cfg); fprintf(stderr, "bytes=%zu\n", artifact.bytes.len); if (artifact.has_type_tag) { fprintf(stderr, "type_tag=0x%08x\n", @@ -784,9 +890,7 @@ static int amduat_asl_cli_cmd_put(int argc, char **argv) { static int amduat_asl_cli_cmd_get(int argc, char **argv) { amduat_asl_cli_get_opts_t opts; - amduat_asl_store_fs_config_t cfg; - amduat_asl_store_fs_t fs; - amduat_asl_store_t store; + amduat_asl_cli_store_ctx_t store_ctx; amduat_artifact_t artifact; amduat_reference_t ref; uint8_t *ref_bytes; @@ -868,15 +972,10 @@ static int amduat_asl_cli_cmd_get(int argc, char **argv) { return AMDUAT_ASL_CLI_EXIT_USAGE; } - if (!amduat_asl_store_fs_load_config(opts.root, &cfg)) { + if (!amduat_asl_cli_open_store(opts.root, &store_ctx)) { fprintf(stderr, "error: failed to load store config: %s\n", opts.root); return AMDUAT_ASL_CLI_EXIT_CONFIG; } - if (!amduat_asl_store_fs_init(&fs, cfg.config, opts.root)) { - fprintf(stderr, "error: failed to initialize store\n"); - return AMDUAT_ASL_CLI_EXIT_CONFIG; - } - amduat_asl_store_init(&store, cfg.config, amduat_asl_store_fs_ops(), &fs); memset(&ref, 0, sizeof(ref)); if (opts.ref_format == AMDUAT_FORMAT_REF_HEX) { @@ -905,7 +1004,8 @@ static int amduat_asl_cli_cmd_get(int argc, char **argv) { memset(&artifact, 0, sizeof(artifact)); exit_code = AMDUAT_ASL_CLI_EXIT_OK; { - amduat_asl_store_error_t err = amduat_asl_store_get(&store, ref, &artifact); + amduat_asl_store_error_t err = + amduat_asl_store_get(&store_ctx.store, ref, &artifact); if (err != AMDUAT_ASL_STORE_OK) { fprintf(stderr, "error: store get failed: %s\n", amduat_asl_cli_store_error_str(err)); @@ -948,7 +1048,7 @@ static int amduat_asl_cli_cmd_get(int argc, char **argv) { if (exit_code == AMDUAT_ASL_CLI_EXIT_OK && !opts.quiet) { fprintf(stderr, "root=%s\n", opts.root); - amduat_asl_cli_print_store_meta(&cfg); + amduat_asl_cli_print_store_meta(&store_ctx.cfg); fprintf(stderr, "bytes=%zu\n", artifact.bytes.len); if (opts.print_type_tag) { if (artifact.has_type_tag) { @@ -1050,20 +1150,145 @@ static int amduat_asl_cli_cmd_log(int argc, char **argv) { static int amduat_asl_cli_cmd_index(int argc, char **argv) { const char *root; amduat_asl_store_fs_config_t cfg; + amduat_asl_cli_init_opts_t opts; amduat_asl_store_index_fs_t index_fs; amduat_asl_store_t store; amduat_asl_index_state_t state; size_t i; + bool ok; if (argc < 1) { fprintf(stderr, "error: index command requires a subcommand\n"); return AMDUAT_ASL_CLI_EXIT_USAGE; } - if (strcmp(argv[0], "state") != 0) { + if (strcmp(argv[0], "init") != 0 && strcmp(argv[0], "state") != 0) { fprintf(stderr, "error: unknown index command: %s\n", argv[0]); return AMDUAT_ASL_CLI_EXIT_USAGE; } + if (strcmp(argv[0], "init") == 0) { + memset(&opts, 0, sizeof(opts)); + opts.root = AMDUAT_ASL_CLI_DEFAULT_ROOT; + + for (i = 1; i < (size_t)argc; ++i) { + if (strcmp(argv[i], "--root") == 0) { + if (i + 1 >= (size_t)argc) { + fprintf(stderr, "error: --root requires a path\n"); + return AMDUAT_ASL_CLI_EXIT_USAGE; + } + opts.root = argv[++i]; + } else if (strcmp(argv[i], "--store-id") == 0) { + if (i + 1 >= (size_t)argc) { + fprintf(stderr, "error: --store-id requires a value\n"); + return AMDUAT_ASL_CLI_EXIT_USAGE; + } + opts.store_id = argv[++i]; + } else if (strcmp(argv[i], "--profile") == 0) { + if (i + 1 >= (size_t)argc) { + fprintf(stderr, "error: --profile requires a value\n"); + return AMDUAT_ASL_CLI_EXIT_USAGE; + } + opts.profile = argv[++i]; + } else if (strcmp(argv[i], "--hash") == 0) { + if (i + 1 >= (size_t)argc) { + fprintf(stderr, "error: --hash requires a value\n"); + return AMDUAT_ASL_CLI_EXIT_USAGE; + } + opts.hash = argv[++i]; + } else if (strcmp(argv[i], "--force") == 0) { + opts.force = true; + } else if (strcmp(argv[i], "--quiet") == 0) { + opts.quiet = true; + } else if (strcmp(argv[i], "--help") == 0 || + strcmp(argv[i], "-h") == 0) { + amduat_asl_cli_print_usage(stdout); + return AMDUAT_ASL_CLI_EXIT_OK; + } else { + fprintf(stderr, "error: unknown option: %s\n", argv[i]); + return AMDUAT_ASL_CLI_EXIT_USAGE; + } + } + + memset(&cfg, 0, sizeof(cfg)); + if (opts.store_id != NULL) { + if (!amduat_asl_store_fs_meta_copy_store_id(cfg.store_id, + opts.store_id)) { + fprintf(stderr, "error: invalid store-id\n"); + return AMDUAT_ASL_CLI_EXIT_USAGE; + } + } + if (opts.profile != NULL) { + if (!amduat_asl_parse_profile_id( + opts.profile, + &cfg.config.encoding_profile_id)) { + fprintf(stderr, "error: unknown profile: %s\n", opts.profile); + return AMDUAT_ASL_CLI_EXIT_USAGE; + } + } + if (opts.hash != NULL) { + if (!amduat_asl_parse_hash_id(opts.hash, &cfg.config.hash_id)) { + fprintf(stderr, "error: unknown hash: %s\n", opts.hash); + return AMDUAT_ASL_CLI_EXIT_USAGE; + } + } + + ok = amduat_asl_store_fs_init_root(opts.root, &cfg, &cfg); + if (!ok && opts.force) { + ok = amduat_asl_store_fs_init_root(opts.root, NULL, &cfg); + if (ok && (opts.store_id || opts.profile || opts.hash) && !opts.quiet) { + fprintf(stderr, + "warning: existing config kept; overrides ignored\n"); + } + } + if (!ok) { + fprintf(stderr, "error: failed to initialize store root: %s\n", + opts.root); + return AMDUAT_ASL_CLI_EXIT_CONFIG; + } + + root = opts.root; + { + char *index_path; + char *segments_path; + char *blocks_path; + + index_path = NULL; + segments_path = NULL; + blocks_path = NULL; + if (!amduat_asl_cli_build_index_path(root, &index_path) || + !amduat_asl_cli_build_segments_path(root, &segments_path) || + !amduat_asl_cli_build_blocks_path(root, &blocks_path)) { + free(index_path); + free(segments_path); + free(blocks_path); + fprintf(stderr, "error: failed to build index paths\n"); + return AMDUAT_ASL_CLI_EXIT_IO; + } + + ok = amduat_asl_cli_ensure_dir(index_path) && + amduat_asl_cli_ensure_dir(segments_path) && + amduat_asl_cli_ensure_dir(blocks_path); + free(index_path); + free(segments_path); + free(blocks_path); + if (!ok) { + fprintf(stderr, "error: failed to initialize index directories\n"); + return AMDUAT_ASL_CLI_EXIT_IO; + } + } + + if (!amduat_asl_cli_write_empty_log(root)) { + fprintf(stderr, "error: failed to initialize log file\n"); + return AMDUAT_ASL_CLI_EXIT_IO; + } + + if (!opts.quiet) { + fprintf(stderr, "root=%s\n", root); + amduat_asl_cli_print_store_meta(&cfg); + } + return AMDUAT_ASL_CLI_EXIT_OK; + } + root = AMDUAT_ASL_CLI_DEFAULT_ROOT; for (i = 1; i < (size_t)argc; ++i) { if (strcmp(argv[i], "--root") == 0) { diff --git a/tests/asl/test_asl_index_replay.c b/tests/asl/test_asl_index_replay.c new file mode 100644 index 0000000..ff03d75 --- /dev/null +++ b/tests/asl/test_asl_index_replay.c @@ -0,0 +1,913 @@ +#include "amduat/asl/asl_store_index_fs.h" +#include "amduat/asl/index_replay.h" +#include "amduat/asl/ref_derive.h" +#include "amduat/asl/store.h" +#include "amduat/enc/asl_core_index.h" +#include "amduat/enc/asl_log.h" +#include "amduat/enc/asl1_core.h" +#include "amduat/hash/asl1.h" +#include "asl_store_index_fs_layout.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void store_u16_le(uint8_t *out, uint16_t value) { + out[0] = (uint8_t)(value & 0xffu); + out[1] = (uint8_t)((value >> 8) & 0xffu); +} + +static void store_u32_le(uint8_t *out, uint32_t value) { + out[0] = (uint8_t)(value & 0xffu); + out[1] = (uint8_t)((value >> 8) & 0xffu); + out[2] = (uint8_t)((value >> 16) & 0xffu); + out[3] = (uint8_t)((value >> 24) & 0xffu); +} + +static void store_u64_le(uint8_t *out, uint64_t value) { + out[0] = (uint8_t)(value & 0xffu); + out[1] = (uint8_t)((value >> 8) & 0xffu); + out[2] = (uint8_t)((value >> 16) & 0xffu); + out[3] = (uint8_t)((value >> 24) & 0xffu); + out[4] = (uint8_t)((value >> 32) & 0xffu); + out[5] = (uint8_t)((value >> 40) & 0xffu); + out[6] = (uint8_t)((value >> 48) & 0xffu); + out[7] = (uint8_t)((value >> 56) & 0xffu); +} + +static size_t build_artifact_ref(uint8_t *out, + size_t out_cap, + amduat_hash_id_t hash_id, + const uint8_t *digest, + size_t digest_len) { + size_t total = 4 + 2 + 2 + digest_len; + if (out_cap < total) { + return 0; + } + store_u32_le(out, (uint32_t)hash_id); + store_u16_le(out + 4, (uint16_t)digest_len); + store_u16_le(out + 6, 0); + memcpy(out + 8, digest, digest_len); + return total; +} + +static size_t build_tombstone_payload(uint8_t *out, + size_t out_cap, + amduat_hash_id_t hash_id, + const uint8_t *digest, + size_t digest_len) { + size_t offset = 0; + size_t ref_len = build_artifact_ref(out, + out_cap, + hash_id, + digest, + digest_len); + if (ref_len == 0 || out_cap < ref_len + 8) { + return 0; + } + offset += ref_len; + store_u32_le(out + offset, 0); + offset += 4; + store_u32_le(out + offset, 0); + offset += 4; + return offset; +} + +static size_t build_tombstone_lift_payload(uint8_t *out, + size_t out_cap, + amduat_hash_id_t hash_id, + const uint8_t *digest, + size_t digest_len, + uint64_t tombstone_logseq) { + size_t offset = 0; + size_t ref_len = build_artifact_ref(out, + out_cap, + hash_id, + digest, + digest_len); + if (ref_len == 0 || out_cap < ref_len + 8) { + return 0; + } + offset += ref_len; + store_u64_le(out + offset, tombstone_logseq); + offset += 8; + return offset; +} + +static size_t build_segment_seal_payload(uint8_t *out, + size_t out_cap, + uint64_t segment_id, + const uint8_t hash[32]) { + if (out_cap < 8 + 32) { + return 0; + } + store_u64_le(out, segment_id); + memcpy(out + 8, hash, 32); + return 8 + 32; +} + +static bool join_path(const char *base, const char *segment, char **out_path) { + size_t base_len; + size_t seg_len; + bool needs_sep; + size_t total_len; + char *buffer; + size_t offset; + + if (base == NULL || segment == NULL || out_path == NULL) { + return false; + } + if (base[0] == '\0' || segment[0] == '\0') { + return false; + } + + base_len = strlen(base); + seg_len = strlen(segment); + needs_sep = base[base_len - 1u] != '/'; + total_len = base_len + (needs_sep ? 1u : 0u) + seg_len + 1u; + + buffer = (char *)malloc(total_len); + if (buffer == NULL) { + return false; + } + + offset = 0u; + memcpy(buffer + offset, base, base_len); + offset += base_len; + if (needs_sep) { + buffer[offset++] = '/'; + } + memcpy(buffer + offset, segment, seg_len); + offset += seg_len; + buffer[offset] = '\0'; + + *out_path = buffer; + return true; +} + +static bool remove_tree(const char *path) { + struct stat st; + DIR *dir; + struct dirent *entry; + + if (path == NULL) { + return false; + } + if (lstat(path, &st) != 0) { + return errno == ENOENT; + } + + if (!S_ISDIR(st.st_mode)) { + return unlink(path) == 0; + } + + dir = opendir(path); + if (dir == NULL) { + return false; + } + + while ((entry = readdir(dir)) != NULL) { + char *child = NULL; + if (strcmp(entry->d_name, ".") == 0 || + strcmp(entry->d_name, "..") == 0) { + continue; + } + if (!join_path(path, entry->d_name, &child)) { + closedir(dir); + return false; + } + if (!remove_tree(child)) { + free(child); + closedir(dir); + return false; + } + free(child); + } + + if (closedir(dir) != 0) { + return false; + } + return rmdir(path) == 0; +} + +static char *make_temp_root(void) { + char *templ; + const char template_prefix[] = "/tmp/amduat_test_asl_index_replay_XXXXXX"; + + templ = (char *)malloc(sizeof(template_prefix)); + if (templ == NULL) { + return NULL; + } + memcpy(templ, template_prefix, sizeof(template_prefix)); + if (mkdtemp(templ) == NULL) { + free(templ); + return NULL; + } + return templ; +} + +static bool ensure_dir(const char *path) { + if (mkdir(path, 0700) == 0) { + return true; + } + return errno == EEXIST; +} + +static bool write_file(const char *path, const uint8_t *data, size_t len) { + FILE *file; + size_t written; + + file = fopen(path, "wb"); + if (file == NULL) { + return false; + } + written = 0u; + if (len != 0) { + written = fwrite(data, 1u, len, file); + } + if (written != len) { + fclose(file); + return false; + } + return fclose(file) == 0; +} + +static bool read_file(const char *path, uint8_t **out_bytes, size_t *out_len) { + FILE *file; + long size; + uint8_t *buffer; + + if (out_bytes == NULL || out_len == NULL) { + return false; + } + *out_bytes = NULL; + *out_len = 0u; + + file = fopen(path, "rb"); + if (file == NULL) { + return false; + } + if (fseek(file, 0, SEEK_END) != 0) { + fclose(file); + return false; + } + size = ftell(file); + if (size < 0) { + fclose(file); + return false; + } + if (fseek(file, 0, SEEK_SET) != 0) { + fclose(file); + return false; + } + buffer = (uint8_t *)malloc((size_t)size); + if (buffer == NULL) { + fclose(file); + return false; + } + if (size != 0 && fread(buffer, 1u, (size_t)size, file) != (size_t)size) { + free(buffer); + fclose(file); + return false; + } + if (fclose(file) != 0) { + free(buffer); + return false; + } + *out_bytes = buffer; + *out_len = (size_t)size; + return true; +} + +static bool prepare_index_tree(const char *root) { + char *index_path = NULL; + char *segments_path = NULL; + char *blocks_path = NULL; + bool ok = false; + + if (!amduat_asl_store_index_fs_layout_build_index_path(root, + &index_path) || + !amduat_asl_store_index_fs_layout_build_segments_path(root, + &segments_path) || + !amduat_asl_store_index_fs_layout_build_blocks_path(root, + &blocks_path)) { + goto cleanup; + } + + if (!ensure_dir(index_path) || + !ensure_dir(segments_path) || + !ensure_dir(blocks_path)) { + goto cleanup; + } + + ok = true; + +cleanup: + free(index_path); + free(segments_path); + free(blocks_path); + return ok; +} + +static bool write_log(const char *root, + amduat_asl_log_record_t *records, + size_t record_count) { + char *log_path = NULL; + amduat_octets_t bytes; + bool ok = false; + + if (!amduat_enc_asl_log_encode_v1(records, record_count, &bytes)) { + return false; + } + + if (!amduat_asl_store_index_fs_layout_build_log_path(root, &log_path)) { + free((void *)bytes.data); + return false; + } + + ok = write_file(log_path, bytes.data, bytes.len); + free((void *)bytes.data); + free(log_path); + return ok; +} + +static bool load_log_records(const char *root, + amduat_asl_log_record_t **out_records, + size_t *out_count) { + char *log_path = NULL; + uint8_t *bytes = NULL; + size_t len = 0; + bool ok = false; + + if (!amduat_asl_store_index_fs_layout_build_log_path(root, &log_path)) { + return false; + } + if (!read_file(log_path, &bytes, &len)) { + free(log_path); + return false; + } + ok = amduat_enc_asl_log_decode_v1(amduat_octets(bytes, len), + out_records, + out_count); + free(bytes); + free(log_path); + return ok; +} + +static bool write_block_file(const char *root, + uint64_t block_id, + amduat_octets_t bytes) { + char *block_path = NULL; + bool ok = false; + + if (!amduat_asl_store_index_fs_layout_build_block_path(root, + block_id, + &block_path)) { + return false; + } + ok = write_file(block_path, bytes.data, bytes.len); + free(block_path); + return ok; +} + +static bool write_segment_file(const char *root, + uint64_t segment_id, + const amduat_asl_core_index_segment_t *segment, + uint8_t out_hash[32]) { + char *segment_path = NULL; + amduat_octets_t encoded; + bool ok = false; + + if (!amduat_enc_asl_core_index_encode_v1(segment, &encoded)) { + return false; + } + if (!amduat_hash_asl1_digest(AMDUAT_HASH_ASL1_ID_SHA256, + encoded, + out_hash, + 32)) { + free((void *)encoded.data); + return false; + } + + if (!amduat_asl_store_index_fs_layout_build_segment_path(root, + segment_id, + &segment_path)) { + free((void *)encoded.data); + return false; + } + ok = write_file(segment_path, encoded.data, encoded.len); + free((void *)encoded.data); + free(segment_path); + return ok; +} + +static bool build_segment_for_artifact( + amduat_asl_core_index_segment_t *segment, + amduat_asl_index_record_t *record, + amduat_asl_extent_record_t *extent, + amduat_reference_t ref, + amduat_octets_t artifact_bytes, + uint64_t snapshot_id, + uint64_t block_id) { + if (segment == NULL || record == NULL || extent == NULL) { + return false; + } + memset(segment, 0, sizeof(*segment)); + memset(record, 0, sizeof(*record)); + memset(extent, 0, sizeof(*extent)); + + segment->header.snapshot_min = snapshot_id; + segment->header.snapshot_max = snapshot_id; + segment->header.segment_domain_id = 0; + segment->header.segment_visibility = 1; + segment->header.federation_version = 0; + segment->header.flags = 0; + segment->header.reserved0 = 0; + + record->hash_id = ref.hash_id; + record->digest_len = (uint16_t)ref.digest.len; + record->extent_count = 1; + record->total_length = (uint32_t)artifact_bytes.len; + record->domain_id = 0; + record->visibility = 1; + record->has_cross_domain_source = 0; + record->cross_domain_source = 0; + record->flags = 0; + + extent->block_id = block_id; + extent->offset = 0; + extent->length = (uint32_t)artifact_bytes.len; + + segment->records = record; + segment->record_count = 1; + segment->digests = ref.digest; + segment->extents = extent; + segment->extent_count = 1; + segment->footer.seal_snapshot = snapshot_id; + segment->footer.seal_time_ns = snapshot_id; + return true; +} + +static bool write_artifact_segment(const char *root, + amduat_artifact_t artifact, + uint64_t segment_id, + uint64_t block_id, + uint64_t snapshot_id, + uint8_t out_hash[32], + amduat_reference_t *out_ref) { + amduat_reference_t ref; + amduat_octets_t artifact_bytes; + amduat_asl_core_index_segment_t segment; + amduat_asl_index_record_t record; + amduat_asl_extent_record_t extent; + + if (out_ref == NULL) { + return false; + } + + if (!amduat_asl_ref_derive(artifact, + AMDUAT_ENC_ASL1_CORE_V1, + AMDUAT_HASH_ASL1_ID_SHA256, + &ref, + &artifact_bytes)) { + return false; + } + + if (!write_block_file(root, block_id, artifact_bytes)) { + amduat_octets_free(&artifact_bytes); + amduat_reference_free(&ref); + return false; + } + + if (!build_segment_for_artifact(&segment, + &record, + &extent, + ref, + artifact_bytes, + snapshot_id, + block_id) || + !write_segment_file(root, segment_id, &segment, out_hash)) { + amduat_octets_free(&artifact_bytes); + amduat_reference_free(&ref); + return false; + } + + amduat_octets_free(&artifact_bytes); + *out_ref = ref; + return true; +} + +static bool replay_state_equal(const amduat_asl_replay_state_t *lhs, + const amduat_asl_replay_state_t *rhs) { + size_t i; + + if (lhs->segments_len != rhs->segments_len || + lhs->tombstones_len != rhs->tombstones_len || + lhs->state.log_position != rhs->state.log_position) { + return false; + } + + for (i = 0; i < lhs->segments_len; ++i) { + const amduat_asl_segment_seal_t *left = &lhs->segments[i]; + const amduat_asl_segment_seal_t *right = &rhs->segments[i]; + if (left->segment_id != right->segment_id || + memcmp(left->segment_hash, right->segment_hash, + sizeof(left->segment_hash)) != 0) { + return false; + } + } + + for (i = 0; i < lhs->tombstones_len; ++i) { + const amduat_asl_tombstone_entry_t *left = &lhs->tombstones[i]; + const amduat_asl_tombstone_entry_t *right = &rhs->tombstones[i]; + if (left->tombstone_logseq != right->tombstone_logseq || + !amduat_reference_eq(left->ref, right->ref)) { + return false; + } + } + + return true; +} + +static int test_replay_determinism(void) { + char *root; + amduat_artifact_t artifact_a; + amduat_artifact_t artifact_b; + amduat_reference_t ref_a; + amduat_reference_t ref_b; + uint8_t hash_a[32]; + uint8_t hash_b[32]; + uint8_t seal_payload_a[8 + 32]; + uint8_t seal_payload_b[8 + 32]; + size_t seal_len_a; + size_t seal_len_b; + amduat_asl_log_record_t records[2]; + amduat_asl_log_record_t *loaded_records = NULL; + size_t loaded_count = 0; + amduat_asl_replay_state_t first; + amduat_asl_replay_state_t second; + uint8_t payload_a[4] = {0x01, 0x02, 0x03, 0x04}; + uint8_t payload_b[4] = {0x10, 0x20, 0x30, 0x40}; + int exit_code = 1; + + ref_a = amduat_reference(0u, amduat_octets(NULL, 0u)); + ref_b = amduat_reference(0u, amduat_octets(NULL, 0u)); + root = make_temp_root(); + if (root == NULL) { + fprintf(stderr, "temp root failed\n"); + return 1; + } + + if (!prepare_index_tree(root)) { + fprintf(stderr, "prepare index tree failed\n"); + goto cleanup; + } + + artifact_a = amduat_artifact(amduat_octets(payload_a, sizeof(payload_a))); + artifact_b = amduat_artifact(amduat_octets(payload_b, sizeof(payload_b))); + if (!write_artifact_segment(root, artifact_a, 1, 1, 1, hash_a, &ref_a) || + !write_artifact_segment(root, artifact_b, 2, 2, 2, hash_b, &ref_b)) { + fprintf(stderr, "write artifact segments failed\n"); + goto cleanup; + } + + seal_len_a = build_segment_seal_payload(seal_payload_a, + sizeof(seal_payload_a), + 1, + hash_a); + seal_len_b = build_segment_seal_payload(seal_payload_b, + sizeof(seal_payload_b), + 2, + hash_b); + if (seal_len_a == 0 || seal_len_b == 0) { + fprintf(stderr, "seal payload build failed\n"); + goto cleanup; + } + + memset(records, 0, sizeof(records)); + records[0].logseq = 10; + records[0].record_type = AMDUAT_ASL_LOG_RECORD_SEGMENT_SEAL; + records[0].payload = amduat_octets(seal_payload_a, seal_len_a); + records[1].logseq = 20; + records[1].record_type = AMDUAT_ASL_LOG_RECORD_SEGMENT_SEAL; + records[1].payload = amduat_octets(seal_payload_b, seal_len_b); + + if (!write_log(root, records, 2)) { + fprintf(stderr, "write log failed\n"); + goto cleanup; + } + if (!load_log_records(root, &loaded_records, &loaded_count)) { + fprintf(stderr, "log decode failed\n"); + goto cleanup; + } + + if (!amduat_asl_replay_init(&first) || + !amduat_asl_replay_init(&second)) { + fprintf(stderr, "replay init failed\n"); + goto cleanup; + } + if (!amduat_asl_replay_apply_log(loaded_records, + loaded_count, + 20, + &first) || + !amduat_asl_replay_apply_log(loaded_records, + loaded_count, + 20, + &second)) { + fprintf(stderr, "replay apply failed\n"); + amduat_asl_replay_free(&first); + amduat_asl_replay_free(&second); + goto cleanup; + } + if (!replay_state_equal(&first, &second)) { + fprintf(stderr, "replay state mismatch\n"); + amduat_asl_replay_free(&first); + amduat_asl_replay_free(&second); + goto cleanup; + } + amduat_asl_replay_free(&first); + amduat_asl_replay_free(&second); + + exit_code = 0; + +cleanup: + if (loaded_records != NULL) { + amduat_enc_asl_log_free(loaded_records, loaded_count); + } + amduat_reference_free(&ref_a); + amduat_reference_free(&ref_b); + if (!remove_tree(root)) { + fprintf(stderr, "cleanup failed\n"); + exit_code = 1; + } + free(root); + return exit_code; +} + +static int test_tombstone_lift_boundary(void) { + char *root; + amduat_artifact_t artifact; + amduat_reference_t ref; + uint8_t hash[32]; + uint8_t seal_payload[8 + 32]; + uint8_t tombstone_payload[4 + 2 + 2 + 32 + 4 + 4]; + uint8_t lift_payload[4 + 2 + 2 + 32 + 8]; + size_t seal_len; + size_t tombstone_len; + size_t lift_len; + amduat_asl_log_record_t records[4]; + amduat_asl_store_config_t config; + amduat_asl_store_index_fs_t fs; + amduat_asl_store_t store; + amduat_asl_index_state_t cutoff; + amduat_artifact_t loaded; + amduat_asl_store_error_t err; + uint8_t payload[5] = {0x11, 0x22, 0x33, 0x44, 0x55}; + int exit_code = 1; + + ref = amduat_reference(0u, amduat_octets(NULL, 0u)); + root = make_temp_root(); + if (root == NULL) { + fprintf(stderr, "temp root failed\n"); + return 1; + } + if (!prepare_index_tree(root)) { + fprintf(stderr, "prepare index tree failed\n"); + goto cleanup; + } + + artifact = amduat_artifact(amduat_octets(payload, sizeof(payload))); + if (!write_artifact_segment(root, artifact, 5, 5, 5, hash, &ref)) { + fprintf(stderr, "write artifact segment failed\n"); + goto cleanup; + } + + seal_len = build_segment_seal_payload(seal_payload, + sizeof(seal_payload), + 5, + hash); + tombstone_len = build_tombstone_payload(tombstone_payload, + sizeof(tombstone_payload), + ref.hash_id, + ref.digest.data, + ref.digest.len); + lift_len = build_tombstone_lift_payload(lift_payload, + sizeof(lift_payload), + ref.hash_id, + ref.digest.data, + ref.digest.len, + 20); + if (seal_len == 0 || tombstone_len == 0 || lift_len == 0) { + fprintf(stderr, "log payload build failed\n"); + goto cleanup; + } + + memset(records, 0, sizeof(records)); + records[0].logseq = 10; + records[0].record_type = AMDUAT_ASL_LOG_RECORD_SEGMENT_SEAL; + records[0].payload = amduat_octets(seal_payload, seal_len); + records[1].logseq = 20; + records[1].record_type = AMDUAT_ASL_LOG_RECORD_TOMBSTONE; + records[1].payload = amduat_octets(tombstone_payload, tombstone_len); + records[2].logseq = 30; + records[2].record_type = AMDUAT_ASL_LOG_RECORD_SNAPSHOT_ANCHOR; + records[2].payload = amduat_octets(NULL, 0u); + records[3].logseq = 40; + records[3].record_type = AMDUAT_ASL_LOG_RECORD_TOMBSTONE_LIFT; + records[3].payload = amduat_octets(lift_payload, lift_len); + + if (!write_log(root, records, 4)) { + fprintf(stderr, "write log failed\n"); + goto cleanup; + } + + memset(&config, 0, sizeof(config)); + config.encoding_profile_id = AMDUAT_ENC_ASL1_CORE_V1; + config.hash_id = AMDUAT_HASH_ASL1_ID_SHA256; + if (!amduat_asl_store_index_fs_init(&fs, config, root)) { + fprintf(stderr, "index fs init failed\n"); + goto cleanup; + } + amduat_asl_store_init(&store, config, amduat_asl_store_index_fs_ops(), &fs); + + cutoff.snapshot_id = 0; + cutoff.log_position = 30; + loaded = amduat_artifact(amduat_octets(NULL, 0u)); + err = amduat_asl_store_get_indexed(&store, ref, cutoff, &loaded); + if (err != AMDUAT_ASL_STORE_ERR_NOT_FOUND) { + fprintf(stderr, "tombstone cutoff expected not found: %d\n", err); + amduat_artifact_free(&loaded); + goto cleanup; + } + + cutoff.log_position = 40; + loaded = amduat_artifact(amduat_octets(NULL, 0u)); + err = amduat_asl_store_get_indexed(&store, ref, cutoff, &loaded); + if (err != AMDUAT_ASL_STORE_OK) { + fprintf(stderr, "lift lookup failed: %d\n", err); + amduat_artifact_free(&loaded); + goto cleanup; + } + if (!amduat_artifact_eq(artifact, loaded)) { + fprintf(stderr, "lifted artifact mismatch\n"); + amduat_artifact_free(&loaded); + goto cleanup; + } + amduat_artifact_free(&loaded); + + exit_code = 0; + +cleanup: + amduat_reference_free(&ref); + if (!remove_tree(root)) { + fprintf(stderr, "cleanup failed\n"); + exit_code = 1; + } + free(root); + return exit_code; +} + +static int test_crash_recovery_unsealed(void) { + char *root; + amduat_artifact_t artifact_a; + amduat_artifact_t artifact_b; + amduat_reference_t ref_a; + amduat_reference_t ref_b; + uint8_t hash_a[32]; + uint8_t hash_b[32]; + uint8_t seal_payload[8 + 32]; + size_t seal_len; + amduat_asl_log_record_t records[1]; + amduat_asl_log_record_t *loaded_records = NULL; + size_t loaded_count = 0; + amduat_asl_replay_state_t replay_state; + amduat_asl_store_config_t config; + amduat_asl_store_index_fs_t fs; + amduat_asl_store_t store; + amduat_asl_index_state_t current; + amduat_artifact_t loaded; + amduat_asl_store_error_t err; + uint8_t payload_a[3] = {0x61, 0x62, 0x63}; + uint8_t payload_b[3] = {0x64, 0x65, 0x66}; + int exit_code = 1; + + ref_a = amduat_reference(0u, amduat_octets(NULL, 0u)); + ref_b = amduat_reference(0u, amduat_octets(NULL, 0u)); + root = make_temp_root(); + if (root == NULL) { + fprintf(stderr, "temp root failed\n"); + return 1; + } + if (!prepare_index_tree(root)) { + fprintf(stderr, "prepare index tree failed\n"); + goto cleanup; + } + + artifact_a = amduat_artifact(amduat_octets(payload_a, sizeof(payload_a))); + artifact_b = amduat_artifact(amduat_octets(payload_b, sizeof(payload_b))); + if (!write_artifact_segment(root, artifact_a, 1, 1, 1, hash_a, &ref_a) || + !write_artifact_segment(root, artifact_b, 9, 9, 9, hash_b, &ref_b)) { + fprintf(stderr, "write artifact segments failed\n"); + goto cleanup; + } + + seal_len = build_segment_seal_payload(seal_payload, + sizeof(seal_payload), + 1, + hash_a); + if (seal_len == 0) { + fprintf(stderr, "seal payload build failed\n"); + goto cleanup; + } + memset(records, 0, sizeof(records)); + records[0].logseq = 10; + records[0].record_type = AMDUAT_ASL_LOG_RECORD_SEGMENT_SEAL; + records[0].payload = amduat_octets(seal_payload, seal_len); + + if (!write_log(root, records, 1)) { + fprintf(stderr, "write log failed\n"); + goto cleanup; + } + + memset(&config, 0, sizeof(config)); + config.encoding_profile_id = AMDUAT_ENC_ASL1_CORE_V1; + config.hash_id = AMDUAT_HASH_ASL1_ID_SHA256; + if (!amduat_asl_store_index_fs_init(&fs, config, root)) { + fprintf(stderr, "index fs init failed\n"); + goto cleanup; + } + amduat_asl_store_init(&store, config, amduat_asl_store_index_fs_ops(), &fs); + + if (!amduat_asl_index_current_state(&store, ¤t)) { + fprintf(stderr, "current state failed\n"); + goto cleanup; + } + loaded = amduat_artifact(amduat_octets(NULL, 0u)); + err = amduat_asl_store_get_indexed(&store, ref_b, current, &loaded); + if (err != AMDUAT_ASL_STORE_ERR_NOT_FOUND) { + fprintf(stderr, "unsealed segment should be ignored: %d\n", err); + amduat_artifact_free(&loaded); + goto cleanup; + } + + if (!load_log_records(root, &loaded_records, &loaded_count)) { + fprintf(stderr, "log decode failed\n"); + goto cleanup; + } + if (!amduat_asl_replay_init(&replay_state)) { + fprintf(stderr, "replay init failed\n"); + goto cleanup; + } + if (!amduat_asl_replay_apply_log(loaded_records, + loaded_count, + current.log_position, + &replay_state)) { + fprintf(stderr, "replay apply failed\n"); + amduat_asl_replay_free(&replay_state); + goto cleanup; + } + if (replay_state.segments_len != 1 || + replay_state.segments[0].segment_id != 1 || + memcmp(replay_state.segments[0].segment_hash, hash_a, + sizeof(hash_a)) != 0) { + fprintf(stderr, "replay should only include sealed segment\n"); + amduat_asl_replay_free(&replay_state); + goto cleanup; + } + amduat_asl_replay_free(&replay_state); + + exit_code = 0; + +cleanup: + if (loaded_records != NULL) { + amduat_enc_asl_log_free(loaded_records, loaded_count); + } + amduat_reference_free(&ref_a); + amduat_reference_free(&ref_b); + if (!remove_tree(root)) { + fprintf(stderr, "cleanup failed\n"); + exit_code = 1; + } + free(root); + return exit_code; +} + +int main(void) { + if (test_replay_determinism() != 0) { + return 1; + } + if (test_tombstone_lift_boundary() != 0) { + return 1; + } + if (test_crash_recovery_unsealed() != 0) { + return 1; + } + return 0; +}