diff --git a/CMakeLists.txt b/CMakeLists.txt index 02e104c..560949f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,7 +26,7 @@ target_link_libraries(amduat_federation set(amduatd_sources src/amduatd.c src/amduatd_http.c src/amduatd_caps.c src/amduatd_space.c src/amduatd_concepts.c - src/amduatd_store.c) + src/amduatd_store.c src/amduatd_derivation_index.c) if(AMDUATD_ENABLE_UI) list(APPEND amduatd_sources src/amduatd_ui.c) endif() @@ -44,8 +44,9 @@ target_compile_definitions(amduatd target_link_libraries(amduatd PRIVATE amduat_tgk amduat_pel amduat_format amduat_asl_store_fs - amduat_asl_store_index_fs amduat_asl amduat_enc amduat_hash_asl1 - amduat_util amduat_federation + amduat_asl_store_index_fs amduat_asl_derivation_index_fs + amduat_asl amduat_enc amduat_hash_asl1 amduat_util + amduat_federation ) add_executable(amduat_pel_gc @@ -82,3 +83,20 @@ target_link_libraries(amduatd_test_store_backend ) add_test(NAME amduatd_store_backend COMMAND amduatd_test_store_backend) + +add_executable(amduatd_test_derivation_index + tests/test_amduatd_derivation_index.c + src/amduatd_derivation_index.c +) + +target_include_directories(amduatd_test_derivation_index + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/vendor/amduat/include +) + +target_link_libraries(amduatd_test_derivation_index + PRIVATE amduat_asl_derivation_index_fs amduat_pel amduat_asl amduat_util +) + +add_test(NAME amduatd_derivation_index COMMAND amduatd_test_derivation_index) diff --git a/README.md b/README.md index 61bda31..95f7b89 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,18 @@ Run the daemon with the index-backed store: Note: `/v1/fed/records` requires the index backend. +Run the daemon with derivation indexing enabled: + +```sh +./build/amduatd --root .amduat-asl --sock amduatd.sock --enable-derivation-index +``` + +To fail `/v1/pel/run` if the derivation index write fails: + +```sh +./build/amduatd --root .amduat-asl --sock amduatd.sock --derivation-index-strict +``` + Dev loop (build + restart): ```sh @@ -108,6 +120,9 @@ curl --unix-socket amduatd.sock -X POST http://localhost/v1/pel/run \ -d '{"program_ref":"","input_refs":[""],"params_ref":""}' ``` +When derivation indexing is enabled, successful PEL runs record derivations under +`/index/derivations/by_artifact/` keyed by output refs (plus result/trace/receipt refs). + Define a PEL/PROGRAM-DAG/1 program (store-backed): ```sh diff --git a/src/amduatd.c b/src/amduatd.c index 030498e..5ebc015 100644 --- a/src/amduatd.c +++ b/src/amduatd.c @@ -39,6 +39,7 @@ #include "amduatd_caps.h" #include "amduatd_space.h" #include "amduatd_store.h" +#include "amduatd_derivation_index.h" #include #include @@ -1367,6 +1368,7 @@ static bool amduatd_handle_post_pel_run(int fd, const amduat_asl_store_fs_config_t *cfg, const amduatd_concepts_t *concepts, const amduatd_cfg_t *dcfg, + const char *root_path, const amduatd_http_req_t *req) { uint8_t *body = NULL; const char *p = NULL; @@ -2393,6 +2395,30 @@ static bool amduatd_handle_post_pel_run(int fd, receipt_emitted = true; } + if (dcfg != NULL && dcfg->derivation_index_enabled) { + amduat_asl_store_error_t idx_err = AMDUAT_ASL_STORE_OK; + if (!amduatd_derivation_index_pel_run(root_path, + true, + program_ref, + input_refs, + input_refs_len, + has_params_ref, + params_ref, + &run_result, + receipt_emitted, + receipt_ref, + &idx_err)) { + if (dcfg->derivation_index_strict) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", + "derivation index failed"); + goto pel_run_cleanup; + } + amduat_log(AMDUAT_LOG_WARN, + "pel run derivation index failed: %d", + (int)idx_err); + } + } + { amduatd_strbuf_t resp; char *result_hex = NULL; @@ -3950,7 +3976,8 @@ static bool amduatd_handle_conn(int fd, goto conn_cleanup; } if (strcmp(req.method, "POST") == 0 && strcmp(no_query, "/v1/pel/run") == 0) { - ok = amduatd_handle_post_pel_run(fd, store, cfg, concepts, dcfg, &req); + ok = amduatd_handle_post_pel_run(fd, store, cfg, concepts, dcfg, + root_path, &req); goto conn_cleanup; } if (strcmp(req.method, "POST") == 0 && @@ -4035,6 +4062,8 @@ static void amduatd_print_usage(FILE *stream) { " [--store-backend fs|index]\n" " [--allow-uid UID] [--allow-user NAME]\n" " [--enable-cap-reads]\n" + " [--enable-derivation-index]\n" + " [--derivation-index-strict]\n" "\n" "defaults:\n" " --root %s\n" @@ -4154,6 +4183,11 @@ int main(int argc, char **argv) { } } else if (strcmp(argv[i], "--enable-cap-reads") == 0) { enable_cap_reads = true; + } else if (strcmp(argv[i], "--enable-derivation-index") == 0) { + dcfg.derivation_index_enabled = true; + } else if (strcmp(argv[i], "--derivation-index-strict") == 0) { + dcfg.derivation_index_enabled = true; + dcfg.derivation_index_strict = true; } else if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-h") == 0) { amduatd_print_usage(stdout); return 0; @@ -4175,6 +4209,10 @@ int main(int argc, char **argv) { amduat_log(AMDUAT_LOG_INFO, "store backend=%s", amduatd_store_backend_name(store_backend)); + amduat_log(AMDUAT_LOG_INFO, + "derivation index=%s strict=%s", + dcfg.derivation_index_enabled ? "on" : "off", + dcfg.derivation_index_strict ? "on" : "off"); if (!amduatd_caps_init(&caps, &dcfg, root)) { amduat_log(AMDUAT_LOG_WARN, "capabilities unavailable"); } diff --git a/src/amduatd_caps.h b/src/amduatd_caps.h index 688ae4f..8f27729 100644 --- a/src/amduatd_caps.h +++ b/src/amduatd_caps.h @@ -15,6 +15,8 @@ extern "C" { typedef struct amduatd_cfg_t { amduatd_space_t space; uint64_t edges_refresh_ms; + bool derivation_index_enabled; + bool derivation_index_strict; } amduatd_cfg_t; typedef enum { diff --git a/src/amduatd_derivation_index.c b/src/amduatd_derivation_index.c new file mode 100644 index 0000000..4091688 --- /dev/null +++ b/src/amduatd_derivation_index.c @@ -0,0 +1,197 @@ +#include "amduatd_derivation_index.h" + +#include "amduat/asl/asl_derivation_index_fs.h" +#include "amduat/pel/core.h" +#include "amduat/pel/derivation_sid.h" +#include + +/* Reserve output_index values for non-output artifacts (result/trace/receipt). */ +enum { + AMDUATD_DERIVATION_OUTPUT_INDEX_RESULT = UINT32_MAX, + AMDUATD_DERIVATION_OUTPUT_INDEX_TRACE = UINT32_MAX - 1u, + AMDUATD_DERIVATION_OUTPUT_INDEX_RECEIPT = UINT32_MAX - 2u +}; + +static bool amduatd_derivation_index_add_record( + amduat_asl_derivation_index_fs_t *index, + amduat_octets_t sid, + amduat_reference_t program_ref, + const amduat_reference_t *input_refs, + size_t input_refs_len, + bool has_params_ref, + amduat_reference_t params_ref, + amduat_reference_t artifact_ref, + uint32_t output_index, + amduat_asl_store_error_t *out_err) { + amduat_asl_derivation_record_t record; + amduat_octets_t record_sid = amduat_octets(NULL, 0u); + amduat_asl_store_error_t err; + + if (!amduat_octets_clone(sid, &record_sid)) { + if (out_err != NULL) { + *out_err = AMDUAT_ASL_STORE_ERR_INTEGRITY; + } + return false; + } + + memset(&record, 0, sizeof(record)); + record.sid = record_sid; + record.program_ref = program_ref; + record.output_index = output_index; + record.input_refs = (amduat_reference_t *)input_refs; + record.input_refs_len = input_refs_len; + record.has_params_ref = has_params_ref; + if (has_params_ref) { + record.params_ref = params_ref; + } + record.has_exec_profile = false; + record.exec_profile = amduat_octets(NULL, 0u); + + err = amduat_asl_derivation_index_fs_add(index, artifact_ref, &record); + amduat_octets_free(&record.sid); + if (err != AMDUAT_ASL_STORE_OK) { + if (out_err != NULL) { + *out_err = err; + } + return false; + } + + return true; +} + +bool amduatd_derivation_index_pel_run(const char *root_path, + bool enabled, + amduat_reference_t program_ref, + const amduat_reference_t *input_refs, + size_t input_refs_len, + bool has_params_ref, + amduat_reference_t params_ref, + const amduat_pel_run_result_t *run_result, + bool has_receipt_ref, + amduat_reference_t receipt_ref, + amduat_asl_store_error_t *out_err) { + amduat_asl_derivation_index_fs_t index; + amduat_pel_derivation_sid_input_t sid_input; + amduat_octets_t sid = amduat_octets(NULL, 0u); + size_t i; + + if (out_err != NULL) { + *out_err = AMDUAT_ASL_STORE_OK; + } + + if (!enabled) { + return true; + } + if (root_path == NULL || root_path[0] == '\0' || run_result == NULL) { + if (out_err != NULL) { + *out_err = AMDUAT_ASL_STORE_ERR_INTEGRITY; + } + return false; + } + if (!run_result->has_result_value) { + return true; + } + if (run_result->result_value.has_store_failure) { + return true; + } + if (run_result->result_value.core_result.status != AMDUAT_PEL_EXEC_STATUS_OK) { + return true; + } + if (run_result->output_refs_len > UINT32_MAX) { + if (out_err != NULL) { + *out_err = AMDUAT_ASL_STORE_ERR_INTEGRITY; + } + return false; + } + if (!amduat_asl_derivation_index_fs_init(&index, root_path)) { + if (out_err != NULL) { + *out_err = AMDUAT_ASL_STORE_ERR_IO; + } + return false; + } + + memset(&sid_input, 0, sizeof(sid_input)); + sid_input.program_ref = program_ref; + sid_input.input_refs = input_refs; + sid_input.input_refs_len = input_refs_len; + sid_input.has_params_ref = has_params_ref; + if (has_params_ref) { + sid_input.params_ref = params_ref; + } + sid_input.has_exec_profile = false; + sid_input.exec_profile = amduat_octets(NULL, 0u); + + if (!amduat_pel_derivation_sid_compute(&sid_input, &sid)) { + if (out_err != NULL) { + *out_err = AMDUAT_ASL_STORE_ERR_INTEGRITY; + } + return false; + } + + for (i = 0u; i < run_result->output_refs_len; ++i) { + if (!amduatd_derivation_index_add_record(&index, + sid, + program_ref, + input_refs, + input_refs_len, + has_params_ref, + params_ref, + run_result->output_refs[i], + (uint32_t)i, + out_err)) { + amduat_octets_free(&sid); + return false; + } + } + + if (!amduatd_derivation_index_add_record(&index, + sid, + program_ref, + input_refs, + input_refs_len, + has_params_ref, + params_ref, + run_result->result_ref, + AMDUATD_DERIVATION_OUTPUT_INDEX_RESULT, + out_err)) { + amduat_octets_free(&sid); + return false; + } + + if (run_result->result_value.has_trace_ref) { + if (!amduatd_derivation_index_add_record( + &index, + sid, + program_ref, + input_refs, + input_refs_len, + has_params_ref, + params_ref, + run_result->result_value.trace_ref, + AMDUATD_DERIVATION_OUTPUT_INDEX_TRACE, + out_err)) { + amduat_octets_free(&sid); + return false; + } + } + + if (has_receipt_ref) { + if (!amduatd_derivation_index_add_record( + &index, + sid, + program_ref, + input_refs, + input_refs_len, + has_params_ref, + params_ref, + receipt_ref, + AMDUATD_DERIVATION_OUTPUT_INDEX_RECEIPT, + out_err)) { + amduat_octets_free(&sid); + return false; + } + } + + amduat_octets_free(&sid); + return true; +} diff --git a/src/amduatd_derivation_index.h b/src/amduatd_derivation_index.h new file mode 100644 index 0000000..42876cb --- /dev/null +++ b/src/amduatd_derivation_index.h @@ -0,0 +1,32 @@ +#ifndef AMDUATD_DERIVATION_INDEX_H +#define AMDUATD_DERIVATION_INDEX_H + +#include "amduat/asl/core.h" +#include "amduat/asl/store.h" +#include "amduat/pel/run.h" + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +bool amduatd_derivation_index_pel_run(const char *root_path, + bool enabled, + amduat_reference_t program_ref, + const amduat_reference_t *input_refs, + size_t input_refs_len, + bool has_params_ref, + amduat_reference_t params_ref, + const amduat_pel_run_result_t *run_result, + bool has_receipt_ref, + amduat_reference_t receipt_ref, + amduat_asl_store_error_t *out_err); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* AMDUATD_DERIVATION_INDEX_H */ diff --git a/tests/test_amduatd_derivation_index.c b/tests/test_amduatd_derivation_index.c new file mode 100644 index 0000000..d986e61 --- /dev/null +++ b/tests/test_amduatd_derivation_index.c @@ -0,0 +1,199 @@ +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 200809L +#endif + +#include "amduatd_derivation_index.h" + +#include "amduat/asl/asl_derivation_index_fs.h" +#include "amduat/hash/asl1.h" +#include "amduat/pel/derivation_sid.h" + +#include +#include +#include + +static char *amduatd_test_make_temp_dir(void) { + char tmpl[] = "/tmp/amduatd-deriv-XXXXXX"; + char *dir = mkdtemp(tmpl); + size_t len; + char *copy; + if (dir == NULL) { + perror("mkdtemp"); + return NULL; + } + len = strlen(dir); + copy = (char *)malloc(len + 1u); + if (copy == NULL) { + fprintf(stderr, "failed to allocate temp dir copy\n"); + return NULL; + } + memcpy(copy, dir, len + 1u); + return copy; +} + +static bool amduatd_test_make_ref(uint8_t seed, amduat_reference_t *out_ref) { + uint8_t *digest; + size_t i; + + if (out_ref == NULL) { + return false; + } + digest = (uint8_t *)malloc(32u); + if (digest == NULL) { + return false; + } + for (i = 0u; i < 32u; ++i) { + digest[i] = (uint8_t)(seed + i); + } + *out_ref = amduat_reference(AMDUAT_HASH_ASL1_ID_SHA256, + amduat_octets(digest, 32u)); + return true; +} + +int main(void) { + char *root = amduatd_test_make_temp_dir(); + amduat_asl_derivation_index_fs_t index; + amduat_reference_t program_ref; + amduat_reference_t input_ref; + amduat_reference_t output_ref; + amduat_reference_t result_ref; + amduat_reference_t params_ref; + amduat_reference_t input_refs[1]; + amduat_reference_t output_refs[1]; + amduat_pel_run_result_t run_result; + amduat_asl_derivation_record_t *records = NULL; + size_t record_count = 0u; + amduat_asl_store_error_t err; + + if (root == NULL) { + return 1; + } + + if (!amduatd_test_make_ref(1u, &program_ref) || + !amduatd_test_make_ref(2u, &input_ref) || + !amduatd_test_make_ref(3u, &output_ref) || + !amduatd_test_make_ref(4u, &result_ref)) { + fprintf(stderr, "failed to build refs\n"); + free(root); + return 1; + } + + input_refs[0] = input_ref; + output_refs[0] = output_ref; + memset(¶ms_ref, 0, sizeof(params_ref)); + memset(&run_result, 0, sizeof(run_result)); + run_result.has_result_value = true; + run_result.result_value.core_result.status = AMDUAT_PEL_EXEC_STATUS_OK; + run_result.result_value.has_store_failure = false; + run_result.output_refs = output_refs; + run_result.output_refs_len = 1u; + run_result.result_ref = result_ref; + + err = AMDUAT_ASL_STORE_OK; + if (!amduatd_derivation_index_pel_run(root, + false, + program_ref, + input_refs, + 1u, + false, + params_ref, + &run_result, + false, + params_ref, + &err)) { + fprintf(stderr, "unexpected derivation index failure when disabled\n"); + free(root); + return 1; + } + + if (!amduat_asl_derivation_index_fs_init(&index, root)) { + fprintf(stderr, "failed to init derivation index\n"); + free(root); + return 1; + } + + err = amduat_asl_derivation_index_fs_list(&index, output_ref, + &records, &record_count); + if (err != AMDUAT_ASL_STORE_ERR_NOT_FOUND) { + fprintf(stderr, "expected no records when disabled, got %d\n", (int)err); + free(root); + return 1; + } + + err = AMDUAT_ASL_STORE_OK; + if (!amduatd_derivation_index_pel_run(root, + true, + program_ref, + input_refs, + 1u, + false, + params_ref, + &run_result, + false, + params_ref, + &err)) { + fprintf(stderr, "expected derivation index to succeed\n"); + free(root); + return 1; + } + + err = amduat_asl_derivation_index_fs_list(&index, output_ref, + &records, &record_count); + if (err != AMDUAT_ASL_STORE_OK || record_count != 1u) { + fprintf(stderr, "expected one derivation record, got %d count=%zu\n", + (int)err, record_count); + free(root); + return 1; + } + + { + amduat_pel_derivation_sid_input_t sid_input; + amduat_octets_t sid = amduat_octets(NULL, 0u); + + memset(&sid_input, 0, sizeof(sid_input)); + sid_input.program_ref = program_ref; + sid_input.input_refs = input_refs; + sid_input.input_refs_len = 1u; + sid_input.has_params_ref = false; + sid_input.has_exec_profile = false; + sid_input.exec_profile = amduat_octets(NULL, 0u); + + if (!amduat_pel_derivation_sid_compute(&sid_input, &sid)) { + fprintf(stderr, "failed to compute expected sid\n"); + amduat_asl_derivation_records_free(records, record_count); + free(root); + return 1; + } + + if (!amduat_octets_eq(records[0].sid, sid)) { + fprintf(stderr, "sid mismatch\n"); + amduat_octets_free(&sid); + amduat_asl_derivation_records_free(records, record_count); + free(root); + return 1; + } + if (!amduat_reference_eq(records[0].program_ref, program_ref)) { + fprintf(stderr, "program_ref mismatch\n"); + amduat_octets_free(&sid); + amduat_asl_derivation_records_free(records, record_count); + free(root); + return 1; + } + if (records[0].output_index != 0u) { + fprintf(stderr, "output_index mismatch\n"); + amduat_octets_free(&sid); + amduat_asl_derivation_records_free(records, record_count); + free(root); + return 1; + } + amduat_octets_free(&sid); + } + + amduat_asl_derivation_records_free(records, record_count); + amduat_reference_free(&program_ref); + amduat_reference_free(&input_ref); + amduat_reference_free(&output_ref); + amduat_reference_free(&result_ref); + free(root); + return 0; +}