From 070265085f9604a28c67ce82287d194600b00932 Mon Sep 17 00:00:00 2001 From: Carl Niklas Rydberg Date: Sun, 21 Dec 2025 20:36:19 +0100 Subject: [PATCH] =?UTF-8?q?Implemented=20order=E2=80=91key=20pagination=20?= =?UTF-8?q?for=20scan=5Fedges=20in=20the=20in=E2=80=91memory=20TGK=20store?= =?UTF-8?q?.=20The=20cursor=20is=20now=20the=20canonical=20ReferenceBytes?= =?UTF-8?q?=20of=20the=20last=20edge=20in=20the=20page,=20and=20results=20?= =?UTF-8?q?honor=20the=20same=20(hash=5Fid,=20digest)=20ordering.=20Defaul?= =?UTF-8?q?t=20page=20size=20is=20256=20edges.=20Invalid=20or=20unsupporte?= =?UTF-8?q?d=20tokens=20now=20fail=20the=20call.=20Changes=20are=20in=20tg?= =?UTF-8?q?k=5Fstore=5Fmem.c=20Added=20a=20pagination=20test=20that=20walk?= =?UTF-8?q?s=20multiple=20scan=5Fedges=20pages=20and=20validates=20full=20?= =?UTF-8?q?coverage=20+=20ordering?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/adapters/tgk_store_mem/tgk_store_mem.c | 103 +++++++++- tests/tgk/test_tgk_store_mem.c | 221 +++++++++++++++++++++ 2 files changed, 320 insertions(+), 4 deletions(-) diff --git a/src/adapters/tgk_store_mem/tgk_store_mem.c b/src/adapters/tgk_store_mem/tgk_store_mem.c index e355394..d07125c 100644 --- a/src/adapters/tgk_store_mem/tgk_store_mem.c +++ b/src/adapters/tgk_store_mem/tgk_store_mem.c @@ -1,10 +1,15 @@ #include "amduat/tgk/tgk_store_mem.h" +#include "amduat/enc/asl1_core_codec.h" #include "amduat/enc/tgk1_edge.h" #include #include +enum { AMDUAT_TGK_STORE_MEM_SCAN_PAGE_SIZE = 256u }; + +static void amduat_tgk_store_mem_reference_free(amduat_reference_t *ref); + static bool amduat_tgk_store_mem_id_space_valid( amduat_tgk_id_space_config_t id_space) { size_t i; @@ -140,6 +145,27 @@ static bool amduat_tgk_store_mem_node_in_list( return false; } +static bool amduat_tgk_store_mem_decode_page_token( + const amduat_tgk_store_mem_t *mem, + amduat_octets_t token, + amduat_reference_t *out_ref) { + if (out_ref == NULL) { + return false; + } + *out_ref = amduat_reference(0, amduat_octets(NULL, 0)); + if (token.len == 0 || token.data == NULL) { + return false; + } + if (!amduat_enc_asl1_core_decode_reference_v1(token, out_ref)) { + return false; + } + if (!amduat_tgk_store_mem_hash_id_supported(mem, out_ref->hash_id)) { + amduat_tgk_store_mem_reference_free(out_ref); + return false; + } + return true; +} + static int amduat_tgk_store_mem_octets_cmp(amduat_octets_t a, amduat_octets_t b) { size_t min_len; @@ -593,9 +619,13 @@ static bool amduat_tgk_store_mem_scan_edges( bool has_page_token, amduat_tgk_graph_scan_result_t *out_scan) { amduat_tgk_store_mem_t *mem = (amduat_tgk_store_mem_t *)ctx; + amduat_reference_t start_ref; + bool have_start = false; + size_t selected[AMDUAT_TGK_STORE_MEM_SCAN_PAGE_SIZE]; + size_t selected_len = 0; + bool have_more = false; + size_t i; - (void)page_token; - (void)has_page_token; if (out_scan == NULL) { return false; } @@ -605,8 +635,73 @@ static bool amduat_tgk_store_mem_scan_edges( out_scan->next_page_token.len = 0; out_scan->has_next_page = false; - return amduat_tgk_store_mem_build_edge_list(mem, type_filter, NULL, false, - false, &out_scan->edges); + if (mem == NULL) { + return false; + } + if (has_page_token) { + if (!amduat_tgk_store_mem_decode_page_token(mem, page_token, + &start_ref)) { + return false; + } + have_start = true; + } + + for (i = 0; i < mem->edges_len; ++i) { + const amduat_tgk_graph_edge_view_t *edge = &mem->edges[i]; + if (!amduat_tgk_store_mem_type_filter_match(type_filter, edge->body.type)) { + continue; + } + if (have_start && + amduat_tgk_store_mem_ref_cmp(edge->edge_ref, start_ref) <= 0) { + continue; + } + if (selected_len < AMDUAT_TGK_STORE_MEM_SCAN_PAGE_SIZE) { + selected[selected_len++] = i; + continue; + } + have_more = true; + break; + } + + if (selected_len != 0) { + size_t j; + out_scan->edges.edges = (amduat_tgk_graph_edge_view_t *)calloc( + selected_len, sizeof(*out_scan->edges.edges)); + if (out_scan->edges.edges == NULL) { + goto cleanup; + } + for (j = 0; j < selected_len; ++j) { + const amduat_tgk_graph_edge_view_t *edge = &mem->edges[selected[j]]; + if (!amduat_tgk_store_mem_edge_view_clone(edge, + &out_scan->edges.edges[j])) { + out_scan->edges.len = j; + amduat_tgk_graph_edge_view_list_free(&out_scan->edges); + goto cleanup; + } + } + out_scan->edges.len = selected_len; + } + + if (have_more) { + amduat_reference_t last_ref = mem->edges[selected[selected_len - 1]].edge_ref; + if (!amduat_enc_asl1_core_encode_reference_v1( + last_ref, &out_scan->next_page_token)) { + amduat_tgk_graph_edge_view_list_free(&out_scan->edges); + goto cleanup; + } + out_scan->has_next_page = true; + } + + if (have_start) { + amduat_tgk_store_mem_reference_free(&start_ref); + } + return true; + +cleanup: + if (have_start) { + amduat_tgk_store_mem_reference_free(&start_ref); + } + return false; } static bool amduat_tgk_store_mem_node_list_add( diff --git a/tests/tgk/test_tgk_store_mem.c b/tests/tgk/test_tgk_store_mem.c index 424acdc..14b864c 100644 --- a/tests/tgk/test_tgk_store_mem.c +++ b/tests/tgk/test_tgk_store_mem.c @@ -554,6 +554,226 @@ static int test_duplicate_edge_ref_conflict(void) { return 0; } +static int ref_cmp(const void *a, const void *b) { + const amduat_reference_t *ref_a = (const amduat_reference_t *)a; + const amduat_reference_t *ref_b = (const amduat_reference_t *)b; + size_t min_len; + int cmp; + + if (ref_a->hash_id < ref_b->hash_id) { + return -1; + } + if (ref_a->hash_id > ref_b->hash_id) { + return 1; + } + + min_len = ref_a->digest.len < ref_b->digest.len ? ref_a->digest.len + : ref_b->digest.len; + if (min_len != 0 && ref_a->digest.data != NULL && + ref_b->digest.data != NULL) { + cmp = memcmp(ref_a->digest.data, ref_b->digest.data, min_len); + if (cmp != 0) { + return cmp; + } + } else if (min_len != 0) { + return (ref_a->digest.data == NULL) ? -1 : 1; + } + + if (ref_a->digest.len < ref_b->digest.len) { + return -1; + } + if (ref_a->digest.len > ref_b->digest.len) { + return 1; + } + return 0; +} + +static amduat_reference_t make_index_ref(uint16_t index, uint8_t *storage) { + memset(storage, 0, 32); + storage[0] = (uint8_t)((index >> 8) & 0xffu); + storage[1] = (uint8_t)(index & 0xffu); + return amduat_reference(AMDUAT_HASH_ASL1_ID_SHA256, + amduat_octets(storage, 32)); +} + +static int test_scan_edges_pagination(void) { + const size_t edge_count = 300; + amduat_tgk_store_mem_t mem; + amduat_tgk_store_t store; + amduat_tgk_store_config_t config; + amduat_tgk_identity_domain_t domains[1]; + uint32_t edge_tags[1]; + amduat_tgk_edge_type_id_t edge_types[1]; + amduat_asl_encoding_profile_id_t encodings[1]; + amduat_tgk_store_mem_artifact_t *artifacts = NULL; + amduat_octets_t *edge_bytes = NULL; + amduat_reference_t *expected = NULL; + uint8_t *digests = NULL; + amduat_reference_t node_a; + amduat_reference_t node_b; + amduat_reference_t payload; + amduat_tgk_edge_body_t edge; + amduat_reference_t from_refs[1]; + amduat_reference_t to_refs[1]; + uint8_t digest_a[32]; + uint8_t digest_b[32]; + uint8_t digest_payload[32]; + amduat_octets_t page_token; + bool has_page_token = false; + size_t seen = 0; + size_t pages = 0; + size_t i; + int exit_code = 1; + + memset(&mem, 0, sizeof(mem)); + memset(&config, 0, sizeof(config)); + memset(&edge, 0, sizeof(edge)); + page_token = amduat_octets(NULL, 0); + + artifacts = (amduat_tgk_store_mem_artifact_t *)calloc( + edge_count, sizeof(*artifacts)); + edge_bytes = (amduat_octets_t *)calloc(edge_count, sizeof(*edge_bytes)); + expected = (amduat_reference_t *)calloc(edge_count, sizeof(*expected)); + digests = (uint8_t *)calloc(edge_count, 32); + if (artifacts == NULL || edge_bytes == NULL || + expected == NULL || digests == NULL) { + fprintf(stderr, "pagination alloc failed\n"); + goto cleanup; + } + + domains[0].encoding_profile = AMDUAT_ENC_ASL1_CORE_V1; + domains[0].hash_id = AMDUAT_HASH_ASL1_ID_SHA256; + edge_tags[0] = TYPE_TAG_TGK1_EDGE_V1; + edge_types[0] = 0x10; + encodings[0] = TGK1_EDGE_ENC_V1; + + config.id_space.domains = domains; + config.id_space.domains_len = 1; + config.tgk_profiles.edge_tags = edge_tags; + config.tgk_profiles.edge_tags_len = 1; + config.tgk_profiles.edge_types = edge_types; + config.tgk_profiles.edge_types_len = 1; + config.tgk_profiles.encodings = encodings; + config.tgk_profiles.encodings_len = 1; + + node_a = make_ref(0xa1, digest_a); + node_b = make_ref(0xb1, digest_b); + payload = make_ref(0xe1, digest_payload); + + edge.type = 0x10; + from_refs[0] = node_a; + edge.from = from_refs; + edge.from_len = 1; + to_refs[0] = node_b; + edge.to = to_refs; + edge.to_len = 1; + edge.payload = payload; + + for (i = 0; i < edge_count; ++i) { + expected[i] = make_index_ref((uint16_t)i, digests + (i * 32)); + if (!amduat_enc_tgk1_edge_encode_v1(&edge, &edge_bytes[i])) { + fprintf(stderr, "pagination encode failed\n"); + goto cleanup; + } + artifacts[i].ref = expected[i]; + artifacts[i].artifact = + amduat_artifact_with_type(edge_bytes[i], + amduat_type_tag(TYPE_TAG_TGK1_EDGE_V1)); + } + + if (!amduat_tgk_store_mem_init(&mem, config, artifacts, edge_count)) { + fprintf(stderr, "pagination init failed\n"); + goto cleanup; + } + amduat_tgk_store_init(&store, config, amduat_tgk_store_mem_ops(), &mem); + + qsort(expected, edge_count, sizeof(*expected), ref_cmp); + + for (;;) { + amduat_tgk_graph_scan_result_t scan; + bool has_next; + amduat_octets_t next_token; + + if (!amduat_tgk_store_scan_edges(&store, + (amduat_tgk_edge_type_filter_t){0}, + page_token, has_page_token, &scan)) { + fprintf(stderr, "pagination scan failed\n"); + goto cleanup_store; + } + pages++; + if (seen + scan.edges.len > edge_count) { + fprintf(stderr, "pagination overflow\n"); + amduat_tgk_graph_scan_result_free(&scan); + goto cleanup_store; + } + for (i = 0; i < scan.edges.len; ++i) { + if (!amduat_reference_eq(scan.edges.edges[i].edge_ref, + expected[seen + i])) { + fprintf(stderr, "pagination order mismatch\n"); + amduat_tgk_graph_scan_result_free(&scan); + goto cleanup_store; + } + } + seen += scan.edges.len; + has_next = scan.has_next_page; + next_token = scan.next_page_token; + + if (has_next) { + uint8_t *token_copy; + if (next_token.len == 0 || next_token.data == NULL) { + fprintf(stderr, "pagination empty token\n"); + amduat_tgk_graph_scan_result_free(&scan); + goto cleanup_store; + } + token_copy = (uint8_t *)malloc(next_token.len); + if (token_copy == NULL) { + fprintf(stderr, "pagination token alloc failed\n"); + amduat_tgk_graph_scan_result_free(&scan); + goto cleanup_store; + } + memcpy(token_copy, next_token.data, next_token.len); + free((void *)page_token.data); + page_token = amduat_octets(token_copy, next_token.len); + has_page_token = true; + } else { + free((void *)page_token.data); + page_token = amduat_octets(NULL, 0); + has_page_token = false; + } + + amduat_tgk_graph_scan_result_free(&scan); + if (!has_next) { + break; + } + } + + if (pages < 2) { + fprintf(stderr, "pagination did not paginate\n"); + goto cleanup_store; + } + if (seen != edge_count) { + fprintf(stderr, "pagination count mismatch\n"); + goto cleanup_store; + } + + exit_code = 0; + +cleanup_store: + amduat_tgk_store_mem_free(&mem); +cleanup: + free((void *)page_token.data); + if (edge_bytes != NULL) { + for (i = 0; i < edge_count; ++i) { + free((void *)edge_bytes[i].data); + } + } + free(edge_bytes); + free(artifacts); + free(expected); + free(digests); + return exit_code; +} + static int test_resolve_edge_unsupported(const test_env_t *env) { amduat_reference_t ref; amduat_tgk_edge_body_t body; @@ -768,6 +988,7 @@ int main(void) { test_init_rejects_duplicate_hash_id() != 0 || test_duplicate_edge_ref_same_artifact() != 0 || test_duplicate_edge_ref_conflict() != 0 || + test_scan_edges_pagination() != 0 || test_type_filter(&env) != 0 || test_ordering(&env) != 0 || test_adjacency(&env) != 0 ||