From 5bcf06783b1c0371589465afa37ab3fe3cd43ab2 Mon Sep 17 00:00:00 2001 From: Carl Niklas Rydberg Date: Sat, 21 Feb 2026 18:22:41 +0100 Subject: [PATCH] Add substrate contract wit oversteps into app building. --- CMakeLists.txt | 37 ++ docs/SUBSTRATE_CONTRACT_V0_1.md | 167 +++++++ include/amduat/asl/log_cursor.h | 30 ++ include/amduat/asl/projection_checkpoint.h | 54 ++ include/amduat/util/string.h | 17 + .../asl_store_minio/asl_store_minio.c | 11 +- src/internal/string.c | 20 + src/near_core/asl/log_cursor.c | 101 ++++ src/near_core/asl/projection_checkpoint.c | 472 ++++++++++++++++++ src/tools/amduat_conformance_run.c | 7 +- src/tools/amduat_scriptlab_adapter.c | 402 +++++++++++++++ .../asl_store/asl_store_conformance.c | 7 +- .../tgk_store/tgk_store_conformance.c | 10 +- tests/substrate/test_scriptlab_adapter.c | 442 ++++++++++++++++ 14 files changed, 1768 insertions(+), 9 deletions(-) create mode 100644 docs/SUBSTRATE_CONTRACT_V0_1.md create mode 100644 include/amduat/asl/log_cursor.h create mode 100644 include/amduat/asl/projection_checkpoint.h create mode 100644 include/amduat/util/string.h create mode 100644 src/internal/string.c create mode 100644 src/near_core/asl/log_cursor.c create mode 100644 src/near_core/asl/projection_checkpoint.c create mode 100644 src/tools/amduat_scriptlab_adapter.c create mode 100644 tests/substrate/test_scriptlab_adapter.c diff --git a/CMakeLists.txt b/CMakeLists.txt index ff87da7..f757c3e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -58,6 +58,7 @@ set(AMDUAT_UTIL_SRCS src/internal/endian.c src/internal/hex.c src/internal/log.c + src/internal/string.c ) set(AMDUAT_ASL_SRCS @@ -70,7 +71,9 @@ set(AMDUAT_ASL_SRCS src/near_core/asl/index_bloom.c src/near_core/asl/index_snapshot.c src/near_core/asl/index_replay.c + src/near_core/asl/log_cursor.c src/near_core/asl/parse.c + src/near_core/asl/projection_checkpoint.c src/near_core/asl/ref_io.c src/near_core/asl/store.c src/near_core/asl/ref_text.c @@ -313,6 +316,23 @@ target_link_libraries(amduat_pel_build ) set_target_properties(amduat_pel_build PROPERTIES OUTPUT_NAME amduat-pel-build) +add_executable(amduat_scriptlab_adapter src/tools/amduat_scriptlab_adapter.c) +target_include_directories(amduat_scriptlab_adapter + PRIVATE ${AMDUAT_INTERNAL_DIR} + PRIVATE ${AMDUAT_INCLUDE_DIR} +) +target_compile_definitions(amduat_scriptlab_adapter + PRIVATE _POSIX_C_SOURCE=200809L +) +target_link_libraries(amduat_scriptlab_adapter + PRIVATE amduat_tgk_store_asl_index_fs amduat_asl_store_index_fs + amduat_asl_collection amduat_asl_record amduat_asl_derivation_index_fs + amduat_pel amduat_tgk amduat_asl amduat_enc amduat_hash_asl1 + amduat_util pthread +) +set_target_properties(amduat_scriptlab_adapter + PROPERTIES OUTPUT_NAME amduat-scriptlab-adapter) + add_executable(amduat_conformance_run src/tools/amduat_conformance_run.c tests/conformance/asl_store/asl_store_conformance.c @@ -732,3 +752,20 @@ target_link_libraries(amduat_test_fed_view PRIVATE amduat_fed ) add_test(NAME fed_view COMMAND amduat_test_fed_view) + +add_executable(amduat_test_scriptlab_adapter + tests/substrate/test_scriptlab_adapter.c) +target_include_directories(amduat_test_scriptlab_adapter + PRIVATE ${AMDUAT_INTERNAL_DIR} + PRIVATE ${AMDUAT_INCLUDE_DIR} +) +target_compile_definitions(amduat_test_scriptlab_adapter + PRIVATE _POSIX_C_SOURCE=200809L +) +target_link_libraries(amduat_test_scriptlab_adapter + PRIVATE amduat_tgk_store_asl_index_fs amduat_asl_store_index_fs + amduat_asl_collection amduat_asl_record amduat_asl_derivation_index_fs + amduat_pel amduat_tgk amduat_asl amduat_enc amduat_hash_asl1 + amduat_util pthread +) +add_test(NAME scriptlab_adapter COMMAND amduat_test_scriptlab_adapter) diff --git a/docs/SUBSTRATE_CONTRACT_V0_1.md b/docs/SUBSTRATE_CONTRACT_V0_1.md new file mode 100644 index 0000000..f2fe71b --- /dev/null +++ b/docs/SUBSTRATE_CONTRACT_V0_1.md @@ -0,0 +1,167 @@ +# Substrate Contract v0.1 + +Status: Draft (repo-grounded) +Scope: Defines the substrate surfaces and invariants implemented in this repo for v0.1. + +This contract is grounded in the current Amduat code and specs. Each requirement +below lists file references that define the behavior. If a line is marked **NEW**, +it is introduced in this change and implemented in code (referenced below). + +--- + +## 1. Artifact Model (ASL/1-CORE) + +1. **Artifact** = bytes + optional type tag; identity is content-addressed and immutable. + Sources: `tier1/asl-1-core.md`, `include/amduat/asl/core.h`. + +2. **Reference** = `(hash_id, digest)`; equality is hash-id + digest equality. + Sources: `tier1/asl-1-core.md`, `include/amduat/asl/core.h`. + +3. **Reference derivation** is based on canonical encoding + hash family; the API + exposes this via `amduat_asl_ref_derive`. + Sources: `tier1/asl-1-core.md`, `include/amduat/asl/ref_derive.h`. + +4. **Hash family** in use for the stack is HASH/ASL1 (e.g., SHA-256). + Sources: `tier1/hash-asl1.md`, `include/amduat/hash/asl1.h`. + +--- + +## 2. Record Artifacts (ASL records) + +1. ASL records are artifacts with type tag `TYPE_TAG_ASL_RECORD_1` and a + payload encoding defined by `amduat_asl_record_encode_v1`/`decode_v1`. + Sources: `include/amduat/asl/record.h`, `src/core/asl_record.c`. + +2. **Record schema is required and ASCII**; payload may be empty but must be + well-formed per the encoder rules. + Sources: `src/core/asl_record.c`. + +3. Records are stored and retrieved via `amduat_asl_record_store_put/get`, which + wrap ASL store `put/get` over the encoded record bytes. + Sources: `include/amduat/asl/record.h`, `src/core/asl_record.c`, `include/amduat/asl/store.h`. + +--- + +## 3. Authoritative CAS + Index/Log (ASL/1-STORE) + +1. The authoritative content-addressed store surface is `amduat_asl_store_t` with + `put/get` and indexed variants `put_indexed/get_indexed`. + Sources: `include/amduat/asl/store.h`. + +2. Index/log state is represented as `amduat_asl_index_state_t` with + `snapshot_id` and `log_position`. + Sources: `include/amduat/asl/store.h`, `docs/index-log-api-sketch.md`. + +3. Index/log record types are encoded as ASL log records, including artifact + publish/unpublish, snapshot anchors, and tombstones. + Sources: `include/amduat/enc/asl_log.h`, `tier1/asl-log-1.md`. + +4. The filesystem index/log adapter implements the indexed store surface and + emits log records with monotonically increasing `logseq` values. + Sources: `include/amduat/asl/asl_store_index_fs.h`, `src/adapters/asl_store_index_fs/asl_store_index_fs.c`. + +--- + +## 4. Derivation / Provenance Index + +1. Derivation records bind an output artifact ref to a program ref, input refs, + output index, and optional params/exec-profile. + Sources: `include/amduat/asl/asl_derivation_index_fs.h`. + +2. Deterministic derivation SIDs are computed from program + inputs (+ optional + params/profile) via `amduat_pel_derivation_sid_compute`. + Sources: `include/amduat/pel/derivation_sid.h`, `src/core/derivation_sid.c`. + +3. The filesystem derivation index stores and lists derivation records by + artifact reference. + Sources: `include/amduat/asl/asl_derivation_index_fs.h`, + `src/adapters/asl_derivation_index_fs/asl_derivation_index_fs.c`. + +--- + +## 5. TGK Projection (Edges as Artifacts) + +1. TGK edges are artifacts encoded via `ENC/TGK1-EDGE/1` with type tag + `TYPE_TAG_TGK1_EDGE_V1` and edge bodies containing `from`, `to`, and `payload` refs. + Sources: `include/amduat/enc/tgk1_edge.h`, `include/amduat/tgk/core.h`. + +2. The TGK store surface provides graph queries (edges_from/to/scan/neighbors) + over projected edges. + Sources: `include/amduat/tgk/store.h`, `tier1/tgk-store-1.md`. + +3. `amduat_tgk_store_asl_index_fs` implements a TGK projection over the ASL + index/log store. + Sources: `include/amduat/tgk/tgk_store_asl_index_fs.h`, + `src/adapters/tgk_store_asl_index_fs/tgk_store_asl_index_fs.c`. + +--- + +## 6. Deterministic Transforms (PEL) + +1. PEL surface execution results are stored as artifacts with type tag + `TYPE_TAG_PEL1_RESULT_1`, encoding `PEL_ENC_EXECUTION_RESULT_V1`. + Sources: `include/amduat/enc/pel1_result.h`, `include/amduat/pel/surf.h`. + +2. Trace DAGs and Program DAGs are encoded as artifacts with their respective + type tags and encoding profiles. + Sources: `include/amduat/enc/pel_trace_dag.h`, `include/amduat/enc/pel_program_dag.h`. + +3. A minimal job record for deterministic transforms is the tuple: + `(program_ref, input_refs, params_ref?, exec_profile?, result_ref, trace_ref?)`. + Sources: `include/amduat/pel/surf.h`, `include/amduat/enc/pel1_result.h`, + `include/amduat/pel/derivation_sid.h`. + +--- + +## 7. Projection Hooks + Checkpoints (for external projectors) + +1. **Projection feed cursor** is the ASL log position (`amduat_asl_log_position_t`) + from `amduat_asl_index_state_t` and log records are read via the ASL log API. + Sources: `include/amduat/asl/store.h`, `include/amduat/enc/asl_log.h`. + +2. **NEW:** `amduat_asl_log_read_since` provides a cursor-based view of log + records (records with `logseq` > cursor), enabling deterministic replay from + a stable log position. + Sources: `include/amduat/asl/log_cursor.h`, `src/near_core/asl/log_cursor.c`. + +3. **NEW:** Projection checkpoints are encoded via + `amduat_asl_projection_checkpoint_encode_v1/decode_v1` and stored as ASL + record artifacts. The v0.1 schema is derived from `docs/projection-checkpoint-schema.md`. + Sources: `include/amduat/asl/projection_checkpoint.h`, + `src/near_core/asl/projection_checkpoint.c`, `docs/projection-checkpoint-schema.md`, + `include/amduat/asl/record.h`. + +4. **NEW (v0.1 convention):** Checkpoint record schema string is + `projection/checkpoint` and the record payload is the v1 checkpoint encoding. + Sources: `src/tools/amduat_scriptlab_adapter.c`, `tests/substrate/test_scriptlab_adapter.c`, + `include/amduat/asl/record.h`. + +--- + +## 8. Non-Authoritative Projections + +1. External projections (e.g., Elasticsearch, IPFS, JanusGraph) are derived and + must remain observationally equivalent to the ASL/TGK substrate. + Sources: `docs/amduat-oss-backend-proposal.md`, `docs/indexing-requirements-and-projections.md`. + +--- + +## 9. Definition of Done (v0.1) + +1. CAS, index/log store, and derivation index are available via the C surfaces + and filesystem adapters. + Sources: `include/amduat/asl/store.h`, `include/amduat/asl/asl_store_index_fs.h`, + `include/amduat/asl/asl_derivation_index_fs.h`. + +2. TGK projection is available over ASL index/log store. + Sources: `include/amduat/tgk/tgk_store_asl_index_fs.h`. + +3. PEL execution artifacts are produced and encoded as artifacts and can be + linked via derivation SIDs. + Sources: `include/amduat/pel/surf.h`, `include/amduat/enc/pel1_result.h`, + `include/amduat/pel/derivation_sid.h`. + +4. **NEW:** Projection feed cursor + checkpoint artifacts are implemented and + validated by the scriptlab adapter flow. + Sources: `include/amduat/asl/log_cursor.h`, `include/amduat/asl/projection_checkpoint.h`, + `tests/substrate/test_scriptlab_adapter.c`. diff --git a/include/amduat/asl/log_cursor.h b/include/amduat/asl/log_cursor.h new file mode 100644 index 0000000..3304105 --- /dev/null +++ b/include/amduat/asl/log_cursor.h @@ -0,0 +1,30 @@ +#ifndef AMDUAT_ASL_LOG_CURSOR_H +#define AMDUAT_ASL_LOG_CURSOR_H + +#include "amduat/asl/store.h" +#include "amduat/enc/asl_log.h" + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Read log records with logseq > after_logseq. + * Caller owns out_records; free with amduat_enc_asl_log_free. + */ +bool amduat_asl_log_read_since(amduat_asl_store_t *store, + amduat_asl_log_position_t after_logseq, + size_t limit, + amduat_asl_log_record_t **out_records, + size_t *out_count, + amduat_asl_log_position_t *out_next_logseq, + bool *out_end); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* AMDUAT_ASL_LOG_CURSOR_H */ diff --git a/include/amduat/asl/projection_checkpoint.h b/include/amduat/asl/projection_checkpoint.h new file mode 100644 index 0000000..b7b3fb9 --- /dev/null +++ b/include/amduat/asl/projection_checkpoint.h @@ -0,0 +1,54 @@ +#ifndef AMDUAT_ASL_PROJECTION_CHECKPOINT_H +#define AMDUAT_ASL_PROJECTION_CHECKPOINT_H + +#include "amduat/asl/store.h" + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +enum { + AMDUAT_ASL_PROJECTION_CHECKPOINT_VERSION_V1 = 1 +}; + +typedef struct { + uint32_t checkpoint_version; + amduat_octets_t projection_name; + uint32_t projection_schema_version; + bool has_snapshot_id; + amduat_asl_snapshot_id_t snapshot_id; + amduat_asl_log_position_t log_position; + uint64_t created_at; + amduat_octets_t builder_id; + bool has_artifact_count; + uint64_t artifact_count; + bool has_edge_count; + uint64_t edge_count; + bool has_error_count; + uint64_t error_count; + bool has_lag_millis; + uint64_t lag_millis; + bool has_note; + amduat_octets_t note; +} amduat_asl_projection_checkpoint_v1_t; + +bool amduat_asl_projection_checkpoint_encode_v1( + const amduat_asl_projection_checkpoint_v1_t *checkpoint, + amduat_octets_t *out_bytes); + +bool amduat_asl_projection_checkpoint_decode_v1( + amduat_octets_t bytes, + amduat_asl_projection_checkpoint_v1_t *out_checkpoint); + +void amduat_asl_projection_checkpoint_free( + amduat_asl_projection_checkpoint_v1_t *checkpoint); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* AMDUAT_ASL_PROJECTION_CHECKPOINT_H */ diff --git a/include/amduat/util/string.h b/include/amduat/util/string.h new file mode 100644 index 0000000..b706b1f --- /dev/null +++ b/include/amduat/util/string.h @@ -0,0 +1,17 @@ +#ifndef AMDUAT_UTIL_STRING_H +#define AMDUAT_UTIL_STRING_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* strdup replacement for strict C11 builds. */ +char *amduat_strdup(const char *src); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* AMDUAT_UTIL_STRING_H */ diff --git a/src/adapters/asl_store_minio/asl_store_minio.c b/src/adapters/asl_store_minio/asl_store_minio.c index 189780f..a147b56 100644 --- a/src/adapters/asl_store_minio/asl_store_minio.c +++ b/src/adapters/asl_store_minio/asl_store_minio.c @@ -48,6 +48,15 @@ typedef struct { size_t body_len; } amduat_http_response_t; +static void amduat_asl_store_minio_sleep_ms(uint32_t millis) { + struct timespec ts; + + ts.tv_sec = (time_t)(millis / 1000u); + ts.tv_nsec = (long)((millis % 1000u) * 1000000u); + while (nanosleep(&ts, &ts) == -1 && errno == EINTR) { + } +} + static bool amduat_asl_store_minio_copy_string(char *dst, size_t dst_len, const char *src) { @@ -920,7 +929,7 @@ static bool amduat_asl_store_minio_http_request( if (attempt >= minio->retry_max || last_err != AMDUAT_MINIO_ERR_TRANSIENT) { break; } - usleep(200000u); + amduat_asl_store_minio_sleep_ms(200u); } free(auth_header); diff --git a/src/internal/string.c b/src/internal/string.c new file mode 100644 index 0000000..f83d4b6 --- /dev/null +++ b/src/internal/string.c @@ -0,0 +1,20 @@ +#include "amduat/util/string.h" + +#include +#include + +char *amduat_strdup(const char *src) { + char *out; + size_t len; + + if (src == NULL) { + return NULL; + } + len = strlen(src) + 1u; + out = (char *)malloc(len); + if (out == NULL) { + return NULL; + } + memcpy(out, src, len); + return out; +} diff --git a/src/near_core/asl/log_cursor.c b/src/near_core/asl/log_cursor.c new file mode 100644 index 0000000..5180f73 --- /dev/null +++ b/src/near_core/asl/log_cursor.c @@ -0,0 +1,101 @@ +#include "amduat/asl/log_cursor.h" + +#include "amduat/enc/asl_log.h" + +#include +#include + +static bool amduat_asl_log_cursor_copy_record( + const amduat_asl_log_record_t *src, + amduat_asl_log_record_t *out) { + if (src == NULL || out == NULL) { + return false; + } + *out = *src; + if (src->payload.len == 0u) { + out->payload = amduat_octets(NULL, 0u); + return true; + } + if (src->payload.data == NULL) { + return false; + } + { + uint8_t *buf = (uint8_t *)malloc(src->payload.len); + if (buf == NULL) { + return false; + } + memcpy(buf, src->payload.data, src->payload.len); + out->payload = amduat_octets(buf, src->payload.len); + } + return true; +} + +bool amduat_asl_log_read_since(amduat_asl_store_t *store, + amduat_asl_log_position_t after_logseq, + size_t limit, + amduat_asl_log_record_t **out_records, + size_t *out_count, + amduat_asl_log_position_t *out_next_logseq, + bool *out_end) { + amduat_asl_log_record_t *records = NULL; + size_t record_count = 0u; + amduat_asl_store_error_t err; + size_t start = 0u; + size_t available; + size_t take; + + if (out_records == NULL || out_count == NULL || out_next_logseq == NULL || + out_end == NULL) { + return false; + } + *out_records = NULL; + *out_count = 0u; + *out_next_logseq = after_logseq; + *out_end = true; + + if (store == NULL) { + return false; + } + + err = amduat_asl_log_scan(store, &records, &record_count); + if (err != AMDUAT_ASL_STORE_OK) { + return false; + } + + while (start < record_count && records[start].logseq <= after_logseq) { + start++; + } + + available = record_count - start; + take = (limit == 0u || limit > available) ? available : limit; + if (take == 0u) { + *out_end = (start >= record_count); + amduat_enc_asl_log_free(records, record_count); + return true; + } + + *out_records = (amduat_asl_log_record_t *)calloc(take, sizeof(**out_records)); + if (*out_records == NULL) { + amduat_enc_asl_log_free(records, record_count); + return false; + } + + for (size_t i = 0u; i < take; ++i) { + if (!amduat_asl_log_cursor_copy_record(&records[start + i], + &(*out_records)[i])) { + amduat_enc_asl_log_free(*out_records, i); + free(*out_records); + *out_records = NULL; + *out_count = 0u; + amduat_enc_asl_log_free(records, record_count); + return false; + } + } + + *out_count = take; + *out_next_logseq = (*out_records)[take - 1u].logseq; + *out_end = (start + take >= record_count); + + amduat_enc_asl_log_free(records, record_count); + return true; +} diff --git a/src/near_core/asl/projection_checkpoint.c b/src/near_core/asl/projection_checkpoint.c new file mode 100644 index 0000000..e98853e --- /dev/null +++ b/src/near_core/asl/projection_checkpoint.c @@ -0,0 +1,472 @@ +#include "amduat/asl/projection_checkpoint.h" + +#include +#include +#include + +enum { + AMDUAT_ASL_PROJECTION_CHECKPOINT_MAGIC_LEN = 8, + AMDUAT_ASL_PROJECTION_CHECKPOINT_VERSION = 1, + AMDUAT_ASL_PROJECTION_CHECKPOINT_FLAG_SNAPSHOT_ID = 1u << 0u, + AMDUAT_ASL_PROJECTION_CHECKPOINT_FLAG_ARTIFACT_COUNT = 1u << 1u, + AMDUAT_ASL_PROJECTION_CHECKPOINT_FLAG_EDGE_COUNT = 1u << 2u, + AMDUAT_ASL_PROJECTION_CHECKPOINT_FLAG_ERROR_COUNT = 1u << 3u, + AMDUAT_ASL_PROJECTION_CHECKPOINT_FLAG_LAG_MILLIS = 1u << 4u, + AMDUAT_ASL_PROJECTION_CHECKPOINT_FLAG_NOTE = 1u << 5u +}; + +static const uint8_t k_amduat_asl_projection_checkpoint_magic[ + AMDUAT_ASL_PROJECTION_CHECKPOINT_MAGIC_LEN] = { + 'A', 'S', 'L', 'C', 'K', 'P', '1', '\0' +}; + +static void amduat_asl_checkpoint_store_u32_le(uint8_t *out, uint32_t value) { + out[0] = (uint8_t)(value & 0xffu); + out[1] = (uint8_t)((value >> 8) & 0xffu); + out[2] = (uint8_t)((value >> 16) & 0xffu); + out[3] = (uint8_t)((value >> 24) & 0xffu); +} + +static void amduat_asl_checkpoint_store_u64_le(uint8_t *out, uint64_t value) { + out[0] = (uint8_t)(value & 0xffu); + out[1] = (uint8_t)((value >> 8) & 0xffu); + out[2] = (uint8_t)((value >> 16) & 0xffu); + out[3] = (uint8_t)((value >> 24) & 0xffu); + out[4] = (uint8_t)((value >> 32) & 0xffu); + out[5] = (uint8_t)((value >> 40) & 0xffu); + out[6] = (uint8_t)((value >> 48) & 0xffu); + out[7] = (uint8_t)((value >> 56) & 0xffu); +} + +static bool amduat_asl_checkpoint_read_u32_le(const uint8_t *data, + size_t len, + size_t *offset, + uint32_t *out) { + if (len - *offset < 4u) { + return false; + } + *out = (uint32_t)data[*offset] | + ((uint32_t)data[*offset + 1u] << 8) | + ((uint32_t)data[*offset + 2u] << 16) | + ((uint32_t)data[*offset + 3u] << 24); + *offset += 4u; + return true; +} + +static bool amduat_asl_checkpoint_read_u64_le(const uint8_t *data, + size_t len, + size_t *offset, + uint64_t *out) { + if (len - *offset < 8u) { + return false; + } + *out = (uint64_t)data[*offset] | + ((uint64_t)data[*offset + 1u] << 8) | + ((uint64_t)data[*offset + 2u] << 16) | + ((uint64_t)data[*offset + 3u] << 24) | + ((uint64_t)data[*offset + 4u] << 32) | + ((uint64_t)data[*offset + 5u] << 40) | + ((uint64_t)data[*offset + 6u] << 48) | + ((uint64_t)data[*offset + 7u] << 56); + *offset += 8u; + return true; +} + +static bool amduat_asl_checkpoint_add_size(size_t *acc, size_t add) { + if (*acc > SIZE_MAX - add) { + return false; + } + *acc += add; + return true; +} + +static bool amduat_asl_checkpoint_copy_octets(amduat_octets_t src, + amduat_octets_t *out) { + if (out == NULL) { + return false; + } + *out = amduat_octets(NULL, 0u); + if (src.len == 0u) { + return true; + } + if (src.data == NULL) { + return false; + } + { + uint8_t *buf = (uint8_t *)malloc(src.len); + if (buf == NULL) { + return false; + } + memcpy(buf, src.data, src.len); + *out = amduat_octets(buf, src.len); + } + return true; +} + +bool amduat_asl_projection_checkpoint_encode_v1( + const amduat_asl_projection_checkpoint_v1_t *checkpoint, + amduat_octets_t *out_bytes) { + uint8_t *buffer; + size_t total_len = 0u; + size_t offset = 0u; + uint32_t flags = 0u; + + if (checkpoint == NULL || out_bytes == NULL) { + return false; + } + out_bytes->data = NULL; + out_bytes->len = 0u; + + if (checkpoint->checkpoint_version != AMDUAT_ASL_PROJECTION_CHECKPOINT_VERSION) { + return false; + } + if (checkpoint->projection_name.len == 0u || + checkpoint->projection_name.data == NULL) { + return false; + } + if (checkpoint->builder_id.len == 0u || + checkpoint->builder_id.data == NULL) { + return false; + } + if (checkpoint->projection_name.len > UINT32_MAX || + checkpoint->builder_id.len > UINT32_MAX || + checkpoint->note.len > UINT32_MAX) { + return false; + } + + if (checkpoint->has_snapshot_id) { + flags |= AMDUAT_ASL_PROJECTION_CHECKPOINT_FLAG_SNAPSHOT_ID; + } + if (checkpoint->has_artifact_count) { + flags |= AMDUAT_ASL_PROJECTION_CHECKPOINT_FLAG_ARTIFACT_COUNT; + } + if (checkpoint->has_edge_count) { + flags |= AMDUAT_ASL_PROJECTION_CHECKPOINT_FLAG_EDGE_COUNT; + } + if (checkpoint->has_error_count) { + flags |= AMDUAT_ASL_PROJECTION_CHECKPOINT_FLAG_ERROR_COUNT; + } + if (checkpoint->has_lag_millis) { + flags |= AMDUAT_ASL_PROJECTION_CHECKPOINT_FLAG_LAG_MILLIS; + } + if (checkpoint->has_note) { + flags |= AMDUAT_ASL_PROJECTION_CHECKPOINT_FLAG_NOTE; + } + + if (!amduat_asl_checkpoint_add_size( + &total_len, + AMDUAT_ASL_PROJECTION_CHECKPOINT_MAGIC_LEN + 4u + 4u)) { + return false; + } + if (!amduat_asl_checkpoint_add_size(&total_len, + 4u + checkpoint->projection_name.len)) { + return false; + } + if (!amduat_asl_checkpoint_add_size(&total_len, 4u)) { + return false; + } + if (checkpoint->has_snapshot_id) { + if (!amduat_asl_checkpoint_add_size(&total_len, 8u)) { + return false; + } + } + if (!amduat_asl_checkpoint_add_size(&total_len, 8u + 8u)) { + return false; + } + if (!amduat_asl_checkpoint_add_size(&total_len, + 4u + checkpoint->builder_id.len)) { + return false; + } + if (checkpoint->has_artifact_count && + !amduat_asl_checkpoint_add_size(&total_len, 8u)) { + return false; + } + if (checkpoint->has_edge_count && + !amduat_asl_checkpoint_add_size(&total_len, 8u)) { + return false; + } + if (checkpoint->has_error_count && + !amduat_asl_checkpoint_add_size(&total_len, 8u)) { + return false; + } + if (checkpoint->has_lag_millis && + !amduat_asl_checkpoint_add_size(&total_len, 8u)) { + return false; + } + if (checkpoint->has_note && + !amduat_asl_checkpoint_add_size(&total_len, 4u + checkpoint->note.len)) { + return false; + } + + buffer = (uint8_t *)malloc(total_len); + if (buffer == NULL) { + return false; + } + + memcpy(buffer + offset, k_amduat_asl_projection_checkpoint_magic, + AMDUAT_ASL_PROJECTION_CHECKPOINT_MAGIC_LEN); + offset += AMDUAT_ASL_PROJECTION_CHECKPOINT_MAGIC_LEN; + amduat_asl_checkpoint_store_u32_le(buffer + offset, + AMDUAT_ASL_PROJECTION_CHECKPOINT_VERSION); + offset += 4u; + amduat_asl_checkpoint_store_u32_le(buffer + offset, flags); + offset += 4u; + + amduat_asl_checkpoint_store_u32_le(buffer + offset, + (uint32_t)checkpoint->projection_name.len); + offset += 4u; + memcpy(buffer + offset, + checkpoint->projection_name.data, + checkpoint->projection_name.len); + offset += checkpoint->projection_name.len; + + amduat_asl_checkpoint_store_u32_le(buffer + offset, + checkpoint->projection_schema_version); + offset += 4u; + + if (checkpoint->has_snapshot_id) { + amduat_asl_checkpoint_store_u64_le(buffer + offset, + checkpoint->snapshot_id); + offset += 8u; + } + + amduat_asl_checkpoint_store_u64_le(buffer + offset, + checkpoint->log_position); + offset += 8u; + amduat_asl_checkpoint_store_u64_le(buffer + offset, + checkpoint->created_at); + offset += 8u; + + amduat_asl_checkpoint_store_u32_le(buffer + offset, + (uint32_t)checkpoint->builder_id.len); + offset += 4u; + memcpy(buffer + offset, checkpoint->builder_id.data, + checkpoint->builder_id.len); + offset += checkpoint->builder_id.len; + + if (checkpoint->has_artifact_count) { + amduat_asl_checkpoint_store_u64_le(buffer + offset, + checkpoint->artifact_count); + offset += 8u; + } + if (checkpoint->has_edge_count) { + amduat_asl_checkpoint_store_u64_le(buffer + offset, + checkpoint->edge_count); + offset += 8u; + } + if (checkpoint->has_error_count) { + amduat_asl_checkpoint_store_u64_le(buffer + offset, + checkpoint->error_count); + offset += 8u; + } + if (checkpoint->has_lag_millis) { + amduat_asl_checkpoint_store_u64_le(buffer + offset, + checkpoint->lag_millis); + offset += 8u; + } + if (checkpoint->has_note) { + amduat_asl_checkpoint_store_u32_le(buffer + offset, + (uint32_t)checkpoint->note.len); + offset += 4u; + if (checkpoint->note.len != 0u) { + memcpy(buffer + offset, checkpoint->note.data, checkpoint->note.len); + offset += checkpoint->note.len; + } + } + + if (offset != total_len) { + free(buffer); + return false; + } + + *out_bytes = amduat_octets(buffer, total_len); + return true; +} + +bool amduat_asl_projection_checkpoint_decode_v1( + amduat_octets_t bytes, + amduat_asl_projection_checkpoint_v1_t *out_checkpoint) { + size_t offset = 0u; + uint32_t version; + uint32_t flags; + uint32_t name_len; + uint32_t builder_len; + uint32_t note_len; + amduat_asl_projection_checkpoint_v1_t checkpoint; + + if (out_checkpoint == NULL) { + return false; + } + memset(&checkpoint, 0, sizeof(checkpoint)); + checkpoint.projection_name = amduat_octets(NULL, 0u); + checkpoint.builder_id = amduat_octets(NULL, 0u); + checkpoint.note = amduat_octets(NULL, 0u); + + if (bytes.data == NULL || bytes.len < AMDUAT_ASL_PROJECTION_CHECKPOINT_MAGIC_LEN + 8u) { + return false; + } + if (memcmp(bytes.data, + k_amduat_asl_projection_checkpoint_magic, + AMDUAT_ASL_PROJECTION_CHECKPOINT_MAGIC_LEN) != 0) { + return false; + } + offset += AMDUAT_ASL_PROJECTION_CHECKPOINT_MAGIC_LEN; + + if (!amduat_asl_checkpoint_read_u32_le(bytes.data, bytes.len, &offset, &version) || + version != AMDUAT_ASL_PROJECTION_CHECKPOINT_VERSION) { + return false; + } + checkpoint.checkpoint_version = version; + + if (!amduat_asl_checkpoint_read_u32_le(bytes.data, bytes.len, &offset, &flags)) { + return false; + } + + if (!amduat_asl_checkpoint_read_u32_le(bytes.data, bytes.len, &offset, &name_len)) { + return false; + } + if (bytes.len - offset < name_len) { + return false; + } + if (name_len != 0u) { + checkpoint.projection_name = amduat_octets(NULL, 0u); + checkpoint.projection_name.len = name_len; + if (!amduat_asl_checkpoint_copy_octets( + amduat_octets(bytes.data + offset, name_len), + &checkpoint.projection_name)) { + return false; + } + } + offset += name_len; + + if (!amduat_asl_checkpoint_read_u32_le(bytes.data, bytes.len, &offset, + &checkpoint.projection_schema_version)) { + amduat_asl_projection_checkpoint_free(&checkpoint); + return false; + } + + checkpoint.has_snapshot_id = + (flags & AMDUAT_ASL_PROJECTION_CHECKPOINT_FLAG_SNAPSHOT_ID) != 0u; + if (checkpoint.has_snapshot_id) { + if (!amduat_asl_checkpoint_read_u64_le(bytes.data, bytes.len, &offset, + &checkpoint.snapshot_id)) { + amduat_asl_projection_checkpoint_free(&checkpoint); + return false; + } + } + + if (!amduat_asl_checkpoint_read_u64_le(bytes.data, bytes.len, &offset, + &checkpoint.log_position) || + !amduat_asl_checkpoint_read_u64_le(bytes.data, bytes.len, &offset, + &checkpoint.created_at)) { + amduat_asl_projection_checkpoint_free(&checkpoint); + return false; + } + + if (!amduat_asl_checkpoint_read_u32_le(bytes.data, bytes.len, &offset, + &builder_len)) { + amduat_asl_projection_checkpoint_free(&checkpoint); + return false; + } + if (bytes.len - offset < builder_len) { + amduat_asl_projection_checkpoint_free(&checkpoint); + return false; + } + if (builder_len != 0u) { + checkpoint.builder_id = amduat_octets(NULL, 0u); + checkpoint.builder_id.len = builder_len; + if (!amduat_asl_checkpoint_copy_octets( + amduat_octets(bytes.data + offset, builder_len), + &checkpoint.builder_id)) { + amduat_asl_projection_checkpoint_free(&checkpoint); + return false; + } + } + offset += builder_len; + + checkpoint.has_artifact_count = + (flags & AMDUAT_ASL_PROJECTION_CHECKPOINT_FLAG_ARTIFACT_COUNT) != 0u; + if (checkpoint.has_artifact_count) { + if (!amduat_asl_checkpoint_read_u64_le(bytes.data, bytes.len, &offset, + &checkpoint.artifact_count)) { + amduat_asl_projection_checkpoint_free(&checkpoint); + return false; + } + } + + checkpoint.has_edge_count = + (flags & AMDUAT_ASL_PROJECTION_CHECKPOINT_FLAG_EDGE_COUNT) != 0u; + if (checkpoint.has_edge_count) { + if (!amduat_asl_checkpoint_read_u64_le(bytes.data, bytes.len, &offset, + &checkpoint.edge_count)) { + amduat_asl_projection_checkpoint_free(&checkpoint); + return false; + } + } + + checkpoint.has_error_count = + (flags & AMDUAT_ASL_PROJECTION_CHECKPOINT_FLAG_ERROR_COUNT) != 0u; + if (checkpoint.has_error_count) { + if (!amduat_asl_checkpoint_read_u64_le(bytes.data, bytes.len, &offset, + &checkpoint.error_count)) { + amduat_asl_projection_checkpoint_free(&checkpoint); + return false; + } + } + + checkpoint.has_lag_millis = + (flags & AMDUAT_ASL_PROJECTION_CHECKPOINT_FLAG_LAG_MILLIS) != 0u; + if (checkpoint.has_lag_millis) { + if (!amduat_asl_checkpoint_read_u64_le(bytes.data, bytes.len, &offset, + &checkpoint.lag_millis)) { + amduat_asl_projection_checkpoint_free(&checkpoint); + return false; + } + } + + checkpoint.has_note = + (flags & AMDUAT_ASL_PROJECTION_CHECKPOINT_FLAG_NOTE) != 0u; + if (checkpoint.has_note) { + if (!amduat_asl_checkpoint_read_u32_le(bytes.data, bytes.len, &offset, + ¬e_len)) { + amduat_asl_projection_checkpoint_free(&checkpoint); + return false; + } + if (bytes.len - offset < note_len) { + amduat_asl_projection_checkpoint_free(&checkpoint); + return false; + } + if (note_len != 0u) { + checkpoint.note = amduat_octets(NULL, 0u); + checkpoint.note.len = note_len; + if (!amduat_asl_checkpoint_copy_octets( + amduat_octets(bytes.data + offset, note_len), + &checkpoint.note)) { + amduat_asl_projection_checkpoint_free(&checkpoint); + return false; + } + } + offset += note_len; + } + + if (offset != bytes.len) { + amduat_asl_projection_checkpoint_free(&checkpoint); + return false; + } + + *out_checkpoint = checkpoint; + return true; +} + +void amduat_asl_projection_checkpoint_free( + amduat_asl_projection_checkpoint_v1_t *checkpoint) { + if (checkpoint == NULL) { + return; + } + amduat_octets_free(&checkpoint->projection_name); + amduat_octets_free(&checkpoint->builder_id); + amduat_octets_free(&checkpoint->note); + checkpoint->projection_name = amduat_octets(NULL, 0u); + checkpoint->builder_id = amduat_octets(NULL, 0u); + checkpoint->note = amduat_octets(NULL, 0u); +} diff --git a/src/tools/amduat_conformance_run.c b/src/tools/amduat_conformance_run.c index 24e0bf8..997c2ae 100644 --- a/src/tools/amduat_conformance_run.c +++ b/src/tools/amduat_conformance_run.c @@ -1,5 +1,6 @@ #include "asl_store_conformance.h" #include "tgk_store_conformance.h" +#include "amduat/util/string.h" #include #include @@ -38,7 +39,7 @@ static bool append_backend(char **list, const char *backend) { } if (*list == NULL) { - *list = strdup(backend); + *list = amduat_strdup(backend); return *list != NULL; } @@ -124,7 +125,7 @@ int main(int argc, char **argv) { const char *value = argv[++i]; if (strcmp(value, "all") == 0) { free(backend_selection); - backend_selection = strdup("all"); + backend_selection = amduat_strdup("all"); if (backend_selection == NULL) { fprintf(stderr, "out of memory\n"); return 1; @@ -158,7 +159,7 @@ int main(int argc, char **argv) { fixtures = fallback_fixtures_root(fixtures); if (backend_selection == NULL) { - backend_selection = strdup("all"); + backend_selection = amduat_strdup("all"); if (backend_selection == NULL) { fprintf(stderr, "out of memory\n"); return 1; diff --git a/src/tools/amduat_scriptlab_adapter.c b/src/tools/amduat_scriptlab_adapter.c new file mode 100644 index 0000000..203ab7c --- /dev/null +++ b/src/tools/amduat_scriptlab_adapter.c @@ -0,0 +1,402 @@ +#include "amduat/asl/asl_derivation_index_fs.h" +#include "amduat/asl/asl_store_index_fs.h" +#include "amduat/asl/collection.h" +#include "amduat/asl/log_cursor.h" +#include "amduat/asl/projection_checkpoint.h" +#include "amduat/asl/record.h" +#include "amduat/asl/ref_text.h" +#include "amduat/asl/store.h" +#include "amduat/enc/asl1_core.h" +#include "amduat/enc/tgk1_edge.h" +#include "amduat/hash/asl1.h" +#include "amduat/pel/derivation_sid.h" +#include "amduat/tgk/store.h" +#include "amduat/tgk/tgk_store_asl_index_fs.h" +#include "amduat/util/string.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static char *make_temp_root(void) { + char *templ; + const char template_prefix[] = "/tmp/amduat_scriptlab_adapter_XXXXXX"; + + templ = (char *)malloc(sizeof(template_prefix)); + if (templ == NULL) { + return NULL; + } + memcpy(templ, template_prefix, sizeof(template_prefix)); + if (mkdtemp(templ) == NULL) { + free(templ); + return NULL; + } + return templ; +} + +static void print_ref(const char *label, amduat_reference_t ref) { + char *hex = NULL; + if (!amduat_asl_ref_encode_hex(ref, &hex)) { + fprintf(stderr, "%s: \n", label); + return; + } + printf("%s=%s\n", label, hex); + free(hex); +} + +static bool ref_list_contains(amduat_reference_t needle, + const amduat_reference_t *list, + size_t len) { + for (size_t i = 0u; i < len; ++i) { + if (amduat_reference_eq(needle, list[i])) { + return true; + } + } + return false; +} + +int main(int argc, char **argv) { + const char *root_arg = NULL; + char *root = NULL; + amduat_asl_store_config_t config; + amduat_asl_store_index_fs_t asl_fs; + amduat_asl_store_t asl_store; + amduat_asl_collection_store_t collection_store; + amduat_asl_derivation_index_fs_t deriv_index; + amduat_reference_t node_a; + amduat_reference_t node_b; + amduat_reference_t edge_payload_ref; + amduat_reference_t edge_ref; + amduat_reference_t snapshot_ref; + amduat_reference_t program_ref; + amduat_reference_t checkpoint_ref; + amduat_octets_t edge_bytes = amduat_octets(NULL, 0u); + amduat_octets_t checkpoint_bytes = amduat_octets(NULL, 0u); + amduat_octets_t sid = amduat_octets(NULL, 0u); + amduat_asl_log_record_t *log_records = NULL; + size_t log_count = 0u; + amduat_asl_log_position_t next_logseq = 0u; + bool log_end = true; + amduat_asl_index_state_t index_state; + uint64_t collection_offset = 0u; + bool swapped = false; + int exit_code = 1; + + for (int i = 1; i < argc; ++i) { + if (strcmp(argv[i], "--root") == 0) { + if (i + 1 >= argc) { + fprintf(stderr, "--root requires a path\n"); + return 2; + } + root_arg = argv[++i]; + } else if (strcmp(argv[i], "--help") == 0 || + strcmp(argv[i], "-h") == 0) { + fprintf(stdout, "usage: amduat-scriptlab-adapter [--root PATH]\n"); + return 0; + } else { + fprintf(stderr, "unknown argument: %s\n", argv[i]); + return 2; + } + } + + if (root_arg != NULL) { + root = amduat_strdup(root_arg); + } else { + root = make_temp_root(); + } + if (root == NULL) { + fprintf(stderr, "failed to create root\n"); + return 1; + } + + memset(&config, 0, sizeof(config)); + config.encoding_profile_id = AMDUAT_ENC_ASL1_CORE_V1; + config.hash_id = AMDUAT_HASH_ASL1_ID_SHA256; + + if (!amduat_asl_store_index_fs_init(&asl_fs, config, root)) { + fprintf(stderr, "index fs init failed: %s\n", root); + goto cleanup_root; + } + amduat_asl_store_init(&asl_store, config, amduat_asl_store_index_fs_ops(), &asl_fs); + + if (!amduat_asl_collection_store_init(&collection_store, root, &asl_store)) { + fprintf(stderr, "collection store init failed\n"); + goto cleanup_root; + } + if (!amduat_asl_derivation_index_fs_init(&deriv_index, root)) { + fprintf(stderr, "derivation index init failed\n"); + goto cleanup_root; + } + + if (amduat_asl_record_store_put( + &asl_store, + amduat_octets("scriptlab/node", strlen("scriptlab/node")), + amduat_octets("A", 1u), + &node_a) != AMDUAT_ASL_STORE_OK) { + fprintf(stderr, "node A store failed\n"); + goto cleanup_root; + } + if (amduat_asl_record_store_put( + &asl_store, + amduat_octets("scriptlab/node", strlen("scriptlab/node")), + amduat_octets("B", 1u), + &node_b) != AMDUAT_ASL_STORE_OK) { + fprintf(stderr, "node B store failed\n"); + goto cleanup_nodes; + } + + if (amduat_asl_record_store_put( + &asl_store, + amduat_octets("tgk/edge_payload", strlen("tgk/edge_payload")), + amduat_octets("relates", strlen("relates")), + &edge_payload_ref) != AMDUAT_ASL_STORE_OK) { + fprintf(stderr, "edge payload store failed\n"); + goto cleanup_nodes; + } + + { + amduat_tgk_edge_body_t edge; + amduat_reference_t from_refs[1] = {node_a}; + amduat_reference_t to_refs[1] = {node_b}; + + memset(&edge, 0, sizeof(edge)); + edge.type = 1u; + edge.from = from_refs; + edge.from_len = 1u; + edge.to = to_refs; + edge.to_len = 1u; + edge.payload = edge_payload_ref; + + if (!amduat_enc_tgk1_edge_encode_v1(&edge, &edge_bytes)) { + fprintf(stderr, "edge encode failed\n"); + goto cleanup_nodes; + } + } + + { + amduat_artifact_t edge_artifact = amduat_artifact_with_type( + edge_bytes, amduat_type_tag(AMDUAT_TYPE_TAG_TGK1_EDGE_V1)); + if (amduat_asl_store_put(&asl_store, edge_artifact, &edge_ref) != AMDUAT_ASL_STORE_OK) { + fprintf(stderr, "edge store failed\n"); + goto cleanup_edge_bytes; + } + } + amduat_octets_free(&edge_bytes); + + if (amduat_asl_collection_append(&collection_store, + "tgk_edges", + edge_ref, + 0u, + amduat_octets(NULL, 0u), + &collection_offset) != AMDUAT_ASL_COLLECTION_OK) { + fprintf(stderr, "collection append failed\n"); + goto cleanup_edge_ref; + } + + if (amduat_asl_collection_snapshot(&collection_store, + "tgk_edges", + UINT64_MAX, + &snapshot_ref, + &swapped) != AMDUAT_ASL_COLLECTION_OK) { + fprintf(stderr, "collection snapshot failed\n"); + goto cleanup_edge_ref; + } + + if (amduat_asl_record_store_put( + &asl_store, + amduat_octets("scriptlab/transform", strlen("scriptlab/transform")), + amduat_octets("tgk_snapshot_v1", strlen("tgk_snapshot_v1")), + &program_ref) != AMDUAT_ASL_STORE_OK) { + fprintf(stderr, "program record store failed\n"); + goto cleanup_snapshot_ref; + } + + { + amduat_pel_derivation_sid_input_t sid_input; + amduat_asl_derivation_record_t record; + amduat_octets_t exec_profile = + amduat_octets("scriptlab/adapter", strlen("scriptlab/adapter")); + amduat_reference_t inputs[1] = {edge_ref}; + + memset(&sid_input, 0, sizeof(sid_input)); + sid_input.program_ref = program_ref; + sid_input.input_refs = inputs; + sid_input.input_refs_len = 1u; + sid_input.has_params_ref = false; + sid_input.has_exec_profile = true; + sid_input.exec_profile = exec_profile; + + if (!amduat_pel_derivation_sid_compute(&sid_input, &sid)) { + fprintf(stderr, "sid compute failed\n"); + goto cleanup_program_ref; + } + + memset(&record, 0, sizeof(record)); + record.sid = sid; + record.program_ref = program_ref; + record.output_index = 0u; + record.input_refs = inputs; + record.input_refs_len = 1u; + record.has_params_ref = false; + record.has_exec_profile = true; + record.exec_profile = exec_profile; + + if (amduat_asl_derivation_index_fs_add(&deriv_index, + snapshot_ref, + &record) != AMDUAT_ASL_STORE_OK) { + fprintf(stderr, "derivation index add failed\n"); + goto cleanup_sid; + } + } + + if (!amduat_asl_index_current_state(&asl_store, &index_state)) { + fprintf(stderr, "index state read failed\n"); + goto cleanup_sid; + } + + if (!amduat_asl_log_read_since(&asl_store, + 0u, + 0u, + &log_records, + &log_count, + &next_logseq, + &log_end)) { + fprintf(stderr, "log read since failed\n"); + goto cleanup_sid; + } + + { + amduat_asl_projection_checkpoint_v1_t checkpoint; + memset(&checkpoint, 0, sizeof(checkpoint)); + checkpoint.checkpoint_version = AMDUAT_ASL_PROJECTION_CHECKPOINT_VERSION_V1; + checkpoint.projection_name = + amduat_octets("scriptlab", strlen("scriptlab")); + checkpoint.projection_schema_version = 1u; + checkpoint.has_snapshot_id = true; + checkpoint.snapshot_id = index_state.snapshot_id; + checkpoint.log_position = next_logseq; + checkpoint.created_at = 0u; + checkpoint.builder_id = + amduat_octets("scriptlab-adapter", strlen("scriptlab-adapter")); + checkpoint.has_artifact_count = true; + checkpoint.artifact_count = log_count; + + if (!amduat_asl_projection_checkpoint_encode_v1(&checkpoint, + &checkpoint_bytes)) { + fprintf(stderr, "checkpoint encode failed\n"); + goto cleanup_log_records; + } + + if (amduat_asl_record_store_put( + &asl_store, + amduat_octets("projection/checkpoint", + strlen("projection/checkpoint")), + checkpoint_bytes, + &checkpoint_ref) != AMDUAT_ASL_STORE_OK) { + fprintf(stderr, "checkpoint store failed\n"); + goto cleanup_checkpoint_bytes; + } + } + amduat_octets_free(&checkpoint_bytes); + + { + amduat_tgk_identity_domain_t domains[1]; + uint32_t edge_tags[1]; + amduat_tgk_edge_type_id_t edge_types[1]; + amduat_asl_encoding_profile_id_t encodings[1]; + amduat_tgk_store_config_t tgk_config; + amduat_tgk_store_asl_index_fs_t tgk_fs; + amduat_tgk_store_t tgk_store; + amduat_tgk_edge_type_filter_t filter; + amduat_tgk_graph_edge_view_list_t edges; + bool found = false; + + memset(&tgk_config, 0, sizeof(tgk_config)); + domains[0].encoding_profile = AMDUAT_ENC_ASL1_CORE_V1; + domains[0].hash_id = AMDUAT_HASH_ASL1_ID_SHA256; + tgk_config.id_space.domains = domains; + tgk_config.id_space.domains_len = 1u; + tgk_config.artifact_scope.description = amduat_octets(NULL, 0u); + edge_tags[0] = AMDUAT_TYPE_TAG_TGK1_EDGE_V1; + edge_types[0] = 1u; + encodings[0] = TGK1_EDGE_ENC_V1; + tgk_config.tgk_profiles.edge_tags = edge_tags; + tgk_config.tgk_profiles.edge_tags_len = 1u; + tgk_config.tgk_profiles.edge_types = edge_types; + tgk_config.tgk_profiles.edge_types_len = 1u; + tgk_config.tgk_profiles.encodings = encodings; + tgk_config.tgk_profiles.encodings_len = 1u; + + if (!amduat_tgk_store_asl_index_fs_init(&tgk_fs, tgk_config, &asl_fs)) { + fprintf(stderr, "tgk store init failed\n"); + goto cleanup_checkpoint_ref; + } + amduat_tgk_store_init(&tgk_store, + tgk_config, + amduat_tgk_store_asl_index_fs_ops(), + &tgk_fs); + + filter.types = edge_types; + filter.types_len = 1u; + + if (!amduat_tgk_store_edges_from(&tgk_store, node_a, filter, &edges)) { + fprintf(stderr, "tgk edges_from failed\n"); + goto cleanup_checkpoint_ref; + } + + for (size_t i = 0u; i < edges.len; ++i) { + amduat_tgk_edge_body_t *body = &edges.edges[i].body; + if (ref_list_contains(node_b, body->to, body->to_len)) { + found = true; + break; + } + } + amduat_tgk_graph_edge_view_list_free(&edges); + + if (!found) { + fprintf(stderr, "tgk traversal missing expected edge\n"); + goto cleanup_checkpoint_ref; + } + } + + print_ref("node_a", node_a); + print_ref("node_b", node_b); + print_ref("edge", edge_ref); + print_ref("snapshot", snapshot_ref); + print_ref("checkpoint", checkpoint_ref); + + exit_code = 0; + +cleanup_checkpoint_ref: + amduat_reference_free(&checkpoint_ref); +cleanup_log_records: + amduat_enc_asl_log_free(log_records, log_count); +cleanup_checkpoint_bytes: + amduat_octets_free(&checkpoint_bytes); +cleanup_sid: + amduat_octets_free(&sid); +cleanup_program_ref: + amduat_reference_free(&program_ref); +cleanup_snapshot_ref: + amduat_reference_free(&snapshot_ref); +cleanup_edge_ref: + amduat_reference_free(&edge_ref); +cleanup_edge_bytes: + amduat_octets_free(&edge_bytes); +cleanup_nodes: + amduat_reference_free(&edge_payload_ref); + amduat_reference_free(&node_b); + amduat_reference_free(&node_a); +cleanup_root: + if (root_arg == NULL) { + fprintf(stderr, "root=%s\n", root); + } + free(root); + return exit_code; +} diff --git a/tests/conformance/asl_store/asl_store_conformance.c b/tests/conformance/asl_store/asl_store_conformance.c index 90d69f6..e234ee6 100644 --- a/tests/conformance/asl_store/asl_store_conformance.c +++ b/tests/conformance/asl_store/asl_store_conformance.c @@ -11,6 +11,7 @@ #include "amduat/enc/asl_log.h" #include "amduat/hash/asl1.h" #include "amduat/asl/index_replay.h" +#include "amduat/util/string.h" #include #include @@ -109,7 +110,7 @@ static bool conformance_append_list(char **list, const char *entry) { } if (*list == NULL) { - *list = strdup(entry); + *list = amduat_strdup(entry); return *list != NULL; } @@ -182,7 +183,7 @@ static bool conformance_remove_tree(const char *path) { if (path == NULL) { return false; } - if (lstat(path, &st) != 0) { + if (stat(path, &st) != 0) { return errno == ENOENT; } @@ -227,7 +228,7 @@ static bool conformance_ensure_dir(const char *path) { return false; } - temp = strdup(path); + temp = amduat_strdup(path); if (temp == NULL) { return false; } diff --git a/tests/conformance/tgk_store/tgk_store_conformance.c b/tests/conformance/tgk_store/tgk_store_conformance.c index 5070747..9bf349c 100644 --- a/tests/conformance/tgk_store/tgk_store_conformance.c +++ b/tests/conformance/tgk_store/tgk_store_conformance.c @@ -13,6 +13,7 @@ #include "amduat/tgk/tgk_store_asl_index_fs.h" #include "amduat/tgk/tgk_store_fs.h" #include "amduat/tgk/tgk_store_mem.h" +#include "amduat/util/string.h" #include #include @@ -98,6 +99,11 @@ typedef struct { amduat_tgk_store_t store; } amduat_tgk_asl_index_ctx_t; +static bool conformance_collect_node(amduat_reference_t node, + amduat_reference_t **nodes, + size_t *nodes_len, + size_t *nodes_cap); + static bool conformance_join_path(const char *base, const char *segment, char **out_path) { @@ -147,7 +153,7 @@ static bool conformance_remove_tree(const char *path) { if (path == NULL) { return false; } - if (lstat(path, &st) != 0) { + if (stat(path, &st) != 0) { return errno == ENOENT; } @@ -192,7 +198,7 @@ static bool conformance_ensure_dir(const char *path) { return false; } - temp = strdup(path); + temp = amduat_strdup(path); if (temp == NULL) { return false; } diff --git a/tests/substrate/test_scriptlab_adapter.c b/tests/substrate/test_scriptlab_adapter.c new file mode 100644 index 0000000..f00a2e5 --- /dev/null +++ b/tests/substrate/test_scriptlab_adapter.c @@ -0,0 +1,442 @@ +#include "amduat/asl/asl_derivation_index_fs.h" +#include "amduat/asl/asl_store_index_fs.h" +#include "amduat/asl/collection.h" +#include "amduat/asl/log_cursor.h" +#include "amduat/asl/projection_checkpoint.h" +#include "amduat/asl/record.h" +#include "amduat/asl/store.h" +#include "amduat/enc/asl1_core.h" +#include "amduat/enc/tgk1_edge.h" +#include "amduat/hash/asl1.h" +#include "amduat/pel/derivation_sid.h" +#include "amduat/tgk/store.h" +#include "amduat/tgk/tgk_store_asl_index_fs.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static char *make_temp_root(void) { + char *templ; + const char template_prefix[] = "/tmp/amduat_test_scriptlab_adapter_XXXXXX"; + + templ = (char *)malloc(sizeof(template_prefix)); + if (templ == NULL) { + return NULL; + } + memcpy(templ, template_prefix, sizeof(template_prefix)); + if (mkdtemp(templ) == NULL) { + free(templ); + return NULL; + } + return templ; +} + +static bool remove_tree(const char *path) { + struct stat st; + DIR *dir; + struct dirent *entry; + + if (path == NULL) { + return false; + } + if (lstat(path, &st) != 0) { + return errno == ENOENT; + } + if (!S_ISDIR(st.st_mode)) { + return unlink(path) == 0; + } + + dir = opendir(path); + if (dir == NULL) { + return false; + } + while ((entry = readdir(dir)) != NULL) { + char child[2048]; + if (strcmp(entry->d_name, ".") == 0 || + strcmp(entry->d_name, "..") == 0) { + continue; + } + snprintf(child, sizeof(child), "%s/%s", path, entry->d_name); + if (!remove_tree(child)) { + closedir(dir); + return false; + } + } + closedir(dir); + return rmdir(path) == 0; +} + +static bool ref_list_contains(amduat_reference_t needle, + const amduat_reference_t *list, + size_t len) { + for (size_t i = 0u; i < len; ++i) { + if (amduat_reference_eq(needle, list[i])) { + return true; + } + } + return false; +} + +static int test_scriptlab_adapter_flow(void) { + amduat_asl_store_config_t config; + amduat_asl_store_index_fs_t asl_fs; + amduat_asl_store_t asl_store; + amduat_asl_collection_store_t collection_store; + amduat_asl_derivation_index_fs_t deriv_index; + amduat_reference_t node_a; + amduat_reference_t node_b; + amduat_reference_t edge_payload_ref; + amduat_reference_t edge_ref; + amduat_reference_t snapshot_ref; + amduat_reference_t program_ref; + amduat_reference_t checkpoint_ref; + amduat_octets_t edge_bytes = amduat_octets(NULL, 0u); + amduat_octets_t checkpoint_bytes = amduat_octets(NULL, 0u); + amduat_octets_t sid = amduat_octets(NULL, 0u); + amduat_asl_log_record_t *log_records = NULL; + size_t log_count = 0u; + amduat_asl_log_position_t next_logseq = 0u; + bool log_end = true; + amduat_asl_index_state_t index_state; + uint64_t collection_offset = 0u; + bool swapped = false; + char *root; + int exit_code = 1; + + root = make_temp_root(); + if (root == NULL) { + fprintf(stderr, "temp root failed\n"); + return 1; + } + + memset(&config, 0, sizeof(config)); + config.encoding_profile_id = AMDUAT_ENC_ASL1_CORE_V1; + config.hash_id = AMDUAT_HASH_ASL1_ID_SHA256; + + if (!amduat_asl_store_index_fs_init(&asl_fs, config, root)) { + fprintf(stderr, "index fs init failed\n"); + goto cleanup_root; + } + amduat_asl_store_init(&asl_store, config, amduat_asl_store_index_fs_ops(), &asl_fs); + + if (!amduat_asl_collection_store_init(&collection_store, root, &asl_store)) { + fprintf(stderr, "collection store init failed\n"); + goto cleanup_root; + } + if (!amduat_asl_derivation_index_fs_init(&deriv_index, root)) { + fprintf(stderr, "derivation index init failed\n"); + goto cleanup_root; + } + + if (amduat_asl_record_store_put( + &asl_store, + amduat_octets("scriptlab/node", strlen("scriptlab/node")), + amduat_octets("A", 1u), + &node_a) != AMDUAT_ASL_STORE_OK) { + fprintf(stderr, "node A store failed\n"); + goto cleanup_root; + } + if (amduat_asl_record_store_put( + &asl_store, + amduat_octets("scriptlab/node", strlen("scriptlab/node")), + amduat_octets("B", 1u), + &node_b) != AMDUAT_ASL_STORE_OK) { + fprintf(stderr, "node B store failed\n"); + goto cleanup_nodes; + } + + if (amduat_asl_record_store_put( + &asl_store, + amduat_octets("tgk/edge_payload", strlen("tgk/edge_payload")), + amduat_octets("relates", strlen("relates")), + &edge_payload_ref) != AMDUAT_ASL_STORE_OK) { + fprintf(stderr, "edge payload store failed\n"); + goto cleanup_nodes; + } + + { + amduat_tgk_edge_body_t edge; + amduat_reference_t from_refs[1] = {node_a}; + amduat_reference_t to_refs[1] = {node_b}; + + memset(&edge, 0, sizeof(edge)); + edge.type = 1u; + edge.from = from_refs; + edge.from_len = 1u; + edge.to = to_refs; + edge.to_len = 1u; + edge.payload = edge_payload_ref; + + if (!amduat_enc_tgk1_edge_encode_v1(&edge, &edge_bytes)) { + fprintf(stderr, "edge encode failed\n"); + goto cleanup_nodes; + } + } + + { + amduat_artifact_t edge_artifact = amduat_artifact_with_type( + edge_bytes, amduat_type_tag(AMDUAT_TYPE_TAG_TGK1_EDGE_V1)); + if (amduat_asl_store_put(&asl_store, edge_artifact, &edge_ref) != AMDUAT_ASL_STORE_OK) { + fprintf(stderr, "edge store failed\n"); + goto cleanup_edge_bytes; + } + } + amduat_octets_free(&edge_bytes); + + if (amduat_asl_collection_append(&collection_store, + "tgk_edges", + edge_ref, + 0u, + amduat_octets(NULL, 0u), + &collection_offset) != AMDUAT_ASL_COLLECTION_OK) { + fprintf(stderr, "collection append failed\n"); + goto cleanup_edge_ref; + } + + if (amduat_asl_collection_snapshot(&collection_store, + "tgk_edges", + UINT64_MAX, + &snapshot_ref, + &swapped) != AMDUAT_ASL_COLLECTION_OK) { + fprintf(stderr, "collection snapshot failed\n"); + goto cleanup_edge_ref; + } + + { + amduat_asl_record_t snapshot_record; + if (amduat_asl_record_store_get(&asl_store, snapshot_ref, &snapshot_record) != + AMDUAT_ASL_STORE_OK) { + fprintf(stderr, "snapshot record fetch failed\n"); + goto cleanup_snapshot_ref; + } + { + amduat_asl_collection_snapshot_payload_t payload; + if (!amduat_asl_collection_snapshot_payload_decode_v1( + snapshot_record.payload, &payload)) { + fprintf(stderr, "snapshot payload decode failed\n"); + amduat_asl_record_free(&snapshot_record); + goto cleanup_snapshot_ref; + } + if (!ref_list_contains(edge_ref, payload.refs, payload.refs_len)) { + fprintf(stderr, "snapshot missing edge ref\n"); + amduat_asl_collection_snapshot_payload_free(&payload); + amduat_asl_record_free(&snapshot_record); + goto cleanup_snapshot_ref; + } + amduat_asl_collection_snapshot_payload_free(&payload); + } + amduat_asl_record_free(&snapshot_record); + } + + if (amduat_asl_record_store_put( + &asl_store, + amduat_octets("scriptlab/transform", strlen("scriptlab/transform")), + amduat_octets("tgk_snapshot_v1", strlen("tgk_snapshot_v1")), + &program_ref) != AMDUAT_ASL_STORE_OK) { + fprintf(stderr, "program record store failed\n"); + goto cleanup_snapshot_ref; + } + + { + amduat_pel_derivation_sid_input_t sid_input; + amduat_asl_derivation_record_t record; + amduat_octets_t exec_profile = + amduat_octets("scriptlab/adapter", strlen("scriptlab/adapter")); + amduat_reference_t inputs[1] = {edge_ref}; + + memset(&sid_input, 0, sizeof(sid_input)); + sid_input.program_ref = program_ref; + sid_input.input_refs = inputs; + sid_input.input_refs_len = 1u; + sid_input.has_params_ref = false; + sid_input.has_exec_profile = true; + sid_input.exec_profile = exec_profile; + + if (!amduat_pel_derivation_sid_compute(&sid_input, &sid)) { + fprintf(stderr, "sid compute failed\n"); + goto cleanup_program_ref; + } + + memset(&record, 0, sizeof(record)); + record.sid = sid; + record.program_ref = program_ref; + record.output_index = 0u; + record.input_refs = inputs; + record.input_refs_len = 1u; + record.has_params_ref = false; + record.has_exec_profile = true; + record.exec_profile = exec_profile; + + if (amduat_asl_derivation_index_fs_add(&deriv_index, + snapshot_ref, + &record) != AMDUAT_ASL_STORE_OK) { + fprintf(stderr, "derivation index add failed\n"); + goto cleanup_sid; + } + } + + if (!amduat_asl_index_current_state(&asl_store, &index_state)) { + fprintf(stderr, "index state read failed\n"); + goto cleanup_sid; + } + + if (!amduat_asl_log_read_since(&asl_store, + 0u, + 0u, + &log_records, + &log_count, + &next_logseq, + &log_end)) { + fprintf(stderr, "log read since failed\n"); + goto cleanup_sid; + } + + { + amduat_asl_projection_checkpoint_v1_t checkpoint; + amduat_asl_projection_checkpoint_v1_t decoded; + + memset(&checkpoint, 0, sizeof(checkpoint)); + checkpoint.checkpoint_version = AMDUAT_ASL_PROJECTION_CHECKPOINT_VERSION_V1; + checkpoint.projection_name = + amduat_octets("scriptlab", strlen("scriptlab")); + checkpoint.projection_schema_version = 1u; + checkpoint.has_snapshot_id = true; + checkpoint.snapshot_id = index_state.snapshot_id; + checkpoint.log_position = next_logseq; + checkpoint.created_at = 0u; + checkpoint.builder_id = + amduat_octets("scriptlab-adapter", strlen("scriptlab-adapter")); + checkpoint.has_artifact_count = true; + checkpoint.artifact_count = log_count; + + if (!amduat_asl_projection_checkpoint_encode_v1(&checkpoint, + &checkpoint_bytes)) { + fprintf(stderr, "checkpoint encode failed\n"); + goto cleanup_log_records; + } + + if (amduat_asl_record_store_put( + &asl_store, + amduat_octets("projection/checkpoint", + strlen("projection/checkpoint")), + checkpoint_bytes, + &checkpoint_ref) != AMDUAT_ASL_STORE_OK) { + fprintf(stderr, "checkpoint store failed\n"); + goto cleanup_checkpoint_bytes; + } + + if (!amduat_asl_projection_checkpoint_decode_v1(checkpoint_bytes, + &decoded)) { + fprintf(stderr, "checkpoint decode failed\n"); + goto cleanup_checkpoint_ref; + } + if (decoded.log_position != next_logseq) { + fprintf(stderr, "checkpoint log position mismatch\n"); + amduat_asl_projection_checkpoint_free(&decoded); + goto cleanup_checkpoint_ref; + } + amduat_asl_projection_checkpoint_free(&decoded); + } + + { + amduat_tgk_identity_domain_t domains[1]; + uint32_t edge_tags[1]; + amduat_tgk_edge_type_id_t edge_types[1]; + amduat_asl_encoding_profile_id_t encodings[1]; + amduat_tgk_store_config_t tgk_config; + amduat_tgk_store_asl_index_fs_t tgk_fs; + amduat_tgk_store_t tgk_store; + amduat_tgk_edge_type_filter_t filter; + amduat_tgk_graph_edge_view_list_t edges; + bool found = false; + + memset(&tgk_config, 0, sizeof(tgk_config)); + domains[0].encoding_profile = AMDUAT_ENC_ASL1_CORE_V1; + domains[0].hash_id = AMDUAT_HASH_ASL1_ID_SHA256; + tgk_config.id_space.domains = domains; + tgk_config.id_space.domains_len = 1u; + tgk_config.artifact_scope.description = amduat_octets(NULL, 0u); + edge_tags[0] = AMDUAT_TYPE_TAG_TGK1_EDGE_V1; + edge_types[0] = 1u; + encodings[0] = TGK1_EDGE_ENC_V1; + tgk_config.tgk_profiles.edge_tags = edge_tags; + tgk_config.tgk_profiles.edge_tags_len = 1u; + tgk_config.tgk_profiles.edge_types = edge_types; + tgk_config.tgk_profiles.edge_types_len = 1u; + tgk_config.tgk_profiles.encodings = encodings; + tgk_config.tgk_profiles.encodings_len = 1u; + + if (!amduat_tgk_store_asl_index_fs_init(&tgk_fs, tgk_config, &asl_fs)) { + fprintf(stderr, "tgk store init failed\n"); + goto cleanup_checkpoint_ref; + } + amduat_tgk_store_init(&tgk_store, + tgk_config, + amduat_tgk_store_asl_index_fs_ops(), + &tgk_fs); + + filter.types = edge_types; + filter.types_len = 1u; + + if (!amduat_tgk_store_edges_from(&tgk_store, node_a, filter, &edges)) { + fprintf(stderr, "tgk edges_from failed\n"); + goto cleanup_checkpoint_ref; + } + + for (size_t i = 0u; i < edges.len; ++i) { + amduat_tgk_edge_body_t *body = &edges.edges[i].body; + if (ref_list_contains(node_b, body->to, body->to_len)) { + found = true; + break; + } + } + amduat_tgk_graph_edge_view_list_free(&edges); + + if (!found) { + fprintf(stderr, "tgk traversal missing expected edge\n"); + goto cleanup_checkpoint_ref; + } + } + + exit_code = 0; + +cleanup_checkpoint_ref: + amduat_reference_free(&checkpoint_ref); +cleanup_log_records: + amduat_enc_asl_log_free(log_records, log_count); +cleanup_checkpoint_bytes: + amduat_octets_free(&checkpoint_bytes); +cleanup_sid: + amduat_octets_free(&sid); +cleanup_program_ref: + amduat_reference_free(&program_ref); +cleanup_snapshot_ref: + amduat_reference_free(&snapshot_ref); +cleanup_edge_ref: + amduat_reference_free(&edge_ref); +cleanup_edge_bytes: + amduat_octets_free(&edge_bytes); +cleanup_nodes: + amduat_reference_free(&edge_payload_ref); + amduat_reference_free(&node_b); + amduat_reference_free(&node_a); +cleanup_root: + if (!remove_tree(root)) { + fprintf(stderr, "cleanup failed\n"); + exit_code = 1; + } + free(root); + return exit_code; +} + +int main(void) { + return test_scriptlab_adapter_flow(); +}