From 745cf89eb724606209680c4119105c4880439fc6 Mon Sep 17 00:00:00 2001 From: Carl Niklas Rydberg Date: Sun, 18 Jan 2026 06:20:48 +0100 Subject: [PATCH] Add ASL index accel routing key --- CMakeLists.txt | 11 ++ include/amduat/asl/index_accel.h | 33 ++++ .../asl_store_index_fs/asl_store_index_fs.c | 34 +--- src/near_core/asl/index_accel.c | 158 ++++++++++++++++++ tests/asl/test_asl_index_accel.c | 154 +++++++++++++++++ tests/asl/test_asl_store_index_fs.c | 28 +--- 6 files changed, 362 insertions(+), 56 deletions(-) create mode 100644 include/amduat/asl/index_accel.h create mode 100644 src/near_core/asl/index_accel.c create mode 100644 tests/asl/test_asl_index_accel.c diff --git a/CMakeLists.txt b/CMakeLists.txt index d10eb0a..64c6b38 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,6 +63,7 @@ set(AMDUAT_ASL_SRCS src/kernel/asl/core.c src/near_core/asl/artifact_io.c src/near_core/asl/io.c + src/near_core/asl/index_accel.c src/near_core/asl/index_bloom.c src/near_core/asl/index_snapshot.c src/near_core/asl/index_replay.c @@ -420,6 +421,16 @@ set_tests_properties(asl_store_index_fs PROPERTIES ENVIRONMENT "AMDUAT_ASL_PERF_COUNT=100;AMDUAT_ASL_STRESS_SECS=20" ) +add_executable(amduat_test_asl_index_accel tests/asl/test_asl_index_accel.c) +target_include_directories(amduat_test_asl_index_accel + PRIVATE ${AMDUAT_INTERNAL_DIR} + PRIVATE ${AMDUAT_INCLUDE_DIR} +) +target_link_libraries(amduat_test_asl_index_accel + PRIVATE amduat_asl amduat_hash_asl1 amduat_util +) +add_test(NAME asl_index_accel COMMAND amduat_test_asl_index_accel) + add_executable(amduat_test_pel_program_dag_exec tests/pel/test_pel_program_dag_exec.c) target_include_directories(amduat_test_pel_program_dag_exec diff --git a/include/amduat/asl/index_accel.h b/include/amduat/asl/index_accel.h new file mode 100644 index 0000000..8130354 --- /dev/null +++ b/include/amduat/asl/index_accel.h @@ -0,0 +1,33 @@ +#ifndef AMDUAT_ASL_INDEX_ACCEL_H +#define AMDUAT_ASL_INDEX_ACCEL_H + +#include "amduat/asl/core.h" + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +bool amduat_asl_index_accel_routing_key_from_ref( + amduat_reference_t ref, + bool has_type_tag, + amduat_type_tag_t type_tag, + amduat_octets_t *out_key); + +bool amduat_asl_index_accel_routing_key_hash(amduat_octets_t key, + uint64_t *out_hash); + +uint16_t amduat_asl_index_accel_shard_for_ref( + amduat_reference_t ref, + bool has_type_tag, + amduat_type_tag_t type_tag, + uint16_t shard_count); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* AMDUAT_ASL_INDEX_ACCEL_H */ diff --git a/src/adapters/asl_store_index_fs/asl_store_index_fs.c b/src/adapters/asl_store_index_fs/asl_store_index_fs.c index a6220aa..73644d8 100644 --- a/src/adapters/asl_store_index_fs/asl_store_index_fs.c +++ b/src/adapters/asl_store_index_fs/asl_store_index_fs.c @@ -1,6 +1,7 @@ #include "amduat/asl/asl_store_index_fs.h" #include "asl_store_index_fs_layout.h" +#include "amduat/asl/index_accel.h" #include "amduat/asl/index_bloom.h" #include "amduat/asl/index_snapshot.h" #include "amduat/asl/index_replay.h" @@ -2832,40 +2833,11 @@ static bool amduat_asl_store_index_fs_make_segment_id( return true; } -static uint64_t amduat_asl_store_index_fs_fnv1a64(const uint8_t *data, - size_t len, - uint64_t seed) { - size_t i; - uint64_t hash = seed; - - for (i = 0; i < len; ++i) { - hash ^= data[i]; - hash *= 1099511628211ull; - } - return hash; -} - static uint16_t amduat_asl_store_index_fs_ref_shard( amduat_reference_t ref, uint16_t shard_count) { - uint64_t hash; - uint8_t hash_id_bytes[2]; - - if (shard_count == 0u) { - return 0u; - } - - hash_id_bytes[0] = (uint8_t)(ref.hash_id & 0xffu); - hash_id_bytes[1] = (uint8_t)((ref.hash_id >> 8) & 0xffu); - hash = amduat_asl_store_index_fs_fnv1a64(hash_id_bytes, - sizeof(hash_id_bytes), - 14695981039346656037ull); - if (ref.digest.len != 0u && ref.digest.data != NULL) { - hash = amduat_asl_store_index_fs_fnv1a64(ref.digest.data, - ref.digest.len, - hash); - } - return (uint16_t)(hash % shard_count); + return amduat_asl_index_accel_shard_for_ref( + ref, false, amduat_type_tag(0u), shard_count); } static bool amduat_asl_store_index_fs_find_next_snapshot_id( diff --git a/src/near_core/asl/index_accel.c b/src/near_core/asl/index_accel.c new file mode 100644 index 0000000..6da45f8 --- /dev/null +++ b/src/near_core/asl/index_accel.c @@ -0,0 +1,158 @@ +#include "amduat/asl/index_accel.h" + +#include +#include + +enum { + AMDUAT_ASL_INDEX_ACCEL_ROUTING_KEY_HEADER_SIZE = 4, + AMDUAT_ASL_INDEX_ACCEL_ROUTING_KEY_TAG_SIZE = 5 +}; + +static uint64_t amduat_asl_index_accel_fnv1a64_update(uint64_t hash, + const uint8_t *data, + size_t len) { + size_t i; + + if (data == NULL || len == 0u) { + return hash; + } + for (i = 0u; i < len; ++i) { + hash ^= data[i]; + hash *= 1099511628211ull; + } + return hash; +} + +static void amduat_asl_index_accel_store_u16_le(uint8_t *out, + uint16_t value) { + out[0] = (uint8_t)(value & 0xffu); + out[1] = (uint8_t)((value >> 8) & 0xffu); +} + +static void amduat_asl_index_accel_store_u32_le(uint8_t *out, + uint32_t value) { + out[0] = (uint8_t)(value & 0xffu); + out[1] = (uint8_t)((value >> 8) & 0xffu); + out[2] = (uint8_t)((value >> 16) & 0xffu); + out[3] = (uint8_t)((value >> 24) & 0xffu); +} + +static bool amduat_asl_index_accel_hash_ref( + amduat_reference_t ref, + bool has_type_tag, + amduat_type_tag_t type_tag, + uint64_t *out_hash) { + uint8_t header[AMDUAT_ASL_INDEX_ACCEL_ROUTING_KEY_HEADER_SIZE]; + uint8_t tag_buf[AMDUAT_ASL_INDEX_ACCEL_ROUTING_KEY_TAG_SIZE]; + uint16_t digest_len; + uint64_t hash; + + if (out_hash == NULL) { + return false; + } + if (ref.digest.len != 0u && ref.digest.data == NULL) { + return false; + } + if (ref.digest.len > UINT16_MAX) { + return false; + } + + digest_len = (uint16_t)ref.digest.len; + amduat_asl_index_accel_store_u16_le(header, ref.hash_id); + amduat_asl_index_accel_store_u16_le(header + 2u, digest_len); + + tag_buf[0] = has_type_tag ? 1u : 0u; + amduat_asl_index_accel_store_u32_le( + tag_buf + 1u, has_type_tag ? type_tag.tag_id : 0u); + + hash = 14695981039346656037ull; + hash = amduat_asl_index_accel_fnv1a64_update(hash, header, sizeof(header)); + hash = amduat_asl_index_accel_fnv1a64_update( + hash, ref.digest.data, ref.digest.len); + hash = amduat_asl_index_accel_fnv1a64_update(hash, tag_buf, sizeof(tag_buf)); + *out_hash = hash; + return true; +} + +bool amduat_asl_index_accel_routing_key_from_ref( + amduat_reference_t ref, + bool has_type_tag, + amduat_type_tag_t type_tag, + amduat_octets_t *out_key) { + size_t total_len; + uint8_t *buffer; + uint8_t tag_flag; + uint16_t digest_len; + + if (out_key == NULL) { + return false; + } + *out_key = amduat_octets(NULL, 0u); + + if (ref.digest.len != 0u && ref.digest.data == NULL) { + return false; + } + if (ref.digest.len > UINT16_MAX) { + return false; + } + + digest_len = (uint16_t)ref.digest.len; + total_len = AMDUAT_ASL_INDEX_ACCEL_ROUTING_KEY_HEADER_SIZE + + ref.digest.len + AMDUAT_ASL_INDEX_ACCEL_ROUTING_KEY_TAG_SIZE; + + buffer = (uint8_t *)malloc(total_len); + if (buffer == NULL) { + return false; + } + + amduat_asl_index_accel_store_u16_le(buffer, ref.hash_id); + amduat_asl_index_accel_store_u16_le(buffer + 2u, digest_len); + if (ref.digest.len != 0u) { + memcpy(buffer + AMDUAT_ASL_INDEX_ACCEL_ROUTING_KEY_HEADER_SIZE, + ref.digest.data, + ref.digest.len); + } + tag_flag = has_type_tag ? 1u : 0u; + buffer[AMDUAT_ASL_INDEX_ACCEL_ROUTING_KEY_HEADER_SIZE + ref.digest.len] = + tag_flag; + amduat_asl_index_accel_store_u32_le( + buffer + AMDUAT_ASL_INDEX_ACCEL_ROUTING_KEY_HEADER_SIZE + ref.digest.len + + 1u, + has_type_tag ? type_tag.tag_id : 0u); + + *out_key = amduat_octets(buffer, total_len); + return true; +} + +bool amduat_asl_index_accel_routing_key_hash(amduat_octets_t key, + uint64_t *out_hash) { + uint64_t hash; + + if (out_hash == NULL) { + return false; + } + if (key.len != 0u && key.data == NULL) { + return false; + } + + hash = 14695981039346656037ull; + hash = amduat_asl_index_accel_fnv1a64_update(hash, key.data, key.len); + *out_hash = hash; + return true; +} + +uint16_t amduat_asl_index_accel_shard_for_ref( + amduat_reference_t ref, + bool has_type_tag, + amduat_type_tag_t type_tag, + uint16_t shard_count) { + uint64_t hash = 0u; + + if (shard_count == 0u) { + return 0u; + } + if (!amduat_asl_index_accel_hash_ref(ref, has_type_tag, type_tag, &hash)) { + return 0u; + } + return (uint16_t)(hash % shard_count); +} diff --git a/tests/asl/test_asl_index_accel.c b/tests/asl/test_asl_index_accel.c new file mode 100644 index 0000000..6b91044 --- /dev/null +++ b/tests/asl/test_asl_index_accel.c @@ -0,0 +1,154 @@ +#include "amduat/asl/index_accel.h" +#include "amduat/asl/index_bloom.h" + +#include +#include +#include +#include +#include + +static int test_routing_key_layout(void) { + uint8_t digest_bytes[2] = {0xaa, 0xbb}; + amduat_reference_t ref = + amduat_reference(0x1234u, amduat_octets(digest_bytes, sizeof(digest_bytes))); + amduat_octets_t key; + int exit_code = 1; + + if (!amduat_asl_index_accel_routing_key_from_ref( + ref, true, amduat_type_tag(0x11223344u), &key)) { + fprintf(stderr, "routing key build failed\n"); + return exit_code; + } + + if (key.len != 2u + 2u + sizeof(digest_bytes) + 1u + 4u) { + fprintf(stderr, "routing key length mismatch\n"); + goto cleanup; + } + if (key.data[0] != 0x34u || key.data[1] != 0x12u || + key.data[2] != 0x02u || key.data[3] != 0x00u) { + fprintf(stderr, "routing key header mismatch\n"); + goto cleanup; + } + if (key.data[4] != 0xaau || key.data[5] != 0xbbu) { + fprintf(stderr, "routing key digest mismatch\n"); + goto cleanup; + } + if (key.data[6] != 0x01u) { + fprintf(stderr, "routing key type tag flag mismatch\n"); + goto cleanup; + } + if (key.data[7] != 0x44u || key.data[8] != 0x33u || + key.data[9] != 0x22u || key.data[10] != 0x11u) { + fprintf(stderr, "routing key type tag bytes mismatch\n"); + goto cleanup; + } + + exit_code = 0; + +cleanup: + amduat_octets_free(&key); + return exit_code; +} + +static int test_routing_key_absence(void) { + uint8_t digest_bytes[1] = {0x7f}; + amduat_reference_t ref = + amduat_reference(0x0001u, amduat_octets(digest_bytes, sizeof(digest_bytes))); + amduat_octets_t key; + int exit_code = 1; + + if (!amduat_asl_index_accel_routing_key_from_ref( + ref, false, amduat_type_tag(0xdeadbeefu), &key)) { + fprintf(stderr, "routing key build failed\n"); + return exit_code; + } + + if (key.data[4] != 0x7fu) { + fprintf(stderr, "routing key digest mismatch\n"); + goto cleanup; + } + if (key.data[5] != 0x00u) { + fprintf(stderr, "routing key absence flag mismatch\n"); + goto cleanup; + } + if (key.data[6] != 0x00u || key.data[7] != 0x00u || + key.data[8] != 0x00u || key.data[9] != 0x00u) { + fprintf(stderr, "routing key absence tag mismatch\n"); + goto cleanup; + } + + exit_code = 0; + +cleanup: + amduat_octets_free(&key); + return exit_code; +} + +static int test_shard_determinism(void) { + uint8_t digest_bytes[3] = {0x01, 0x02, 0x03}; + amduat_reference_t ref = + amduat_reference(0x00f0u, amduat_octets(digest_bytes, sizeof(digest_bytes))); + uint16_t shard_a; + uint16_t shard_b; + + shard_a = amduat_asl_index_accel_shard_for_ref( + ref, false, amduat_type_tag(0u), 8u); + shard_b = amduat_asl_index_accel_shard_for_ref( + ref, false, amduat_type_tag(0u), 8u); + + if (shard_a != shard_b) { + fprintf(stderr, "shard selection mismatch\n"); + return 1; + } + if (shard_a >= 8u) { + fprintf(stderr, "shard out of range\n"); + return 1; + } + return 0; +} + +static int test_bloom_advisory(void) { + uint8_t digest_bytes[4] = {0x10, 0x20, 0x30, 0x40}; + amduat_octets_t bloom; + amduat_octets_t empty = amduat_octets(NULL, 0u); + int exit_code = 1; + + if (!amduat_asl_index_bloom_init(&bloom)) { + fprintf(stderr, "bloom init failed\n"); + return exit_code; + } + if (!amduat_asl_index_bloom_add( + bloom, 0x0001u, amduat_octets(digest_bytes, sizeof(digest_bytes)))) { + fprintf(stderr, "bloom add failed\n"); + goto cleanup; + } + if (!amduat_asl_index_bloom_maybe_contains( + bloom, 0x0001u, amduat_octets(digest_bytes, sizeof(digest_bytes)))) { + fprintf(stderr, "bloom false negative\n"); + goto cleanup; + } + if (!amduat_asl_index_bloom_maybe_contains( + empty, 0x0002u, amduat_octets(NULL, 0u))) { + fprintf(stderr, "empty bloom should be advisory\n"); + goto cleanup; + } + + exit_code = 0; + +cleanup: + amduat_octets_free(&bloom); + return exit_code; +} + +int main(void) { + if (test_routing_key_layout() != 0) { + return 1; + } + if (test_routing_key_absence() != 0) { + return 1; + } + if (test_shard_determinism() != 0) { + return 1; + } + return test_bloom_advisory(); +} diff --git a/tests/asl/test_asl_store_index_fs.c b/tests/asl/test_asl_store_index_fs.c index 48d4d74..3367822 100644 --- a/tests/asl/test_asl_store_index_fs.c +++ b/tests/asl/test_asl_store_index_fs.c @@ -1,4 +1,5 @@ #include "amduat/asl/asl_store_index_fs.h" +#include "amduat/asl/index_accel.h" #include "amduat/asl/index_bloom.h" #include "amduat/asl/ref_text.h" #include "amduat/asl/ref_derive.h" @@ -240,17 +241,6 @@ static bool build_shard_segment_path(const char *root, return true; } -static uint64_t fnv1a64(const uint8_t *data, size_t len, uint64_t seed) { - size_t i; - uint64_t hash = seed; - - for (i = 0; i < len; ++i) { - hash ^= data[i]; - hash *= 1099511628211ull; - } - return hash; -} - static uint64_t load_u64_le(const uint8_t *data) { return (uint64_t)data[0] | ((uint64_t)data[1] << 8) | ((uint64_t)data[2] << 16) | ((uint64_t)data[3] << 24) | @@ -259,20 +249,8 @@ static uint64_t load_u64_le(const uint8_t *data) { } static uint16_t ref_shard(amduat_reference_t ref, uint16_t shard_count) { - uint64_t hash; - uint8_t hash_id_bytes[2]; - - if (shard_count == 0u) { - return 0u; - } - hash_id_bytes[0] = (uint8_t)(ref.hash_id & 0xffu); - hash_id_bytes[1] = (uint8_t)((ref.hash_id >> 8) & 0xffu); - hash = fnv1a64(hash_id_bytes, sizeof(hash_id_bytes), - 14695981039346656037ull); - if (ref.digest.len != 0u && ref.digest.data != NULL) { - hash = fnv1a64(ref.digest.data, ref.digest.len, hash); - } - return (uint16_t)(hash % shard_count); + return amduat_asl_index_accel_shard_for_ref( + ref, false, amduat_type_tag(0u), shard_count); } static bool parse_size_env(const char *value, size_t *out) {