amduatd: optionally index pel derivations

CMakeLists.txt

@@ -26,7 +26,7 @@ target_link_libraries(amduat_federation
 
 set(amduatd_sources src/amduatd.c src/amduatd_http.c src/amduatd_caps.c
                     src/amduatd_space.c src/amduatd_concepts.c
-                    src/amduatd_store.c)
+                    src/amduatd_store.c src/amduatd_derivation_index.c)
 if(AMDUATD_ENABLE_UI)
   list(APPEND amduatd_sources src/amduatd_ui.c)
 endif()
@@ -44,8 +44,9 @@ target_compile_definitions(amduatd
 
 target_link_libraries(amduatd
   PRIVATE amduat_tgk amduat_pel amduat_format amduat_asl_store_fs
-          amduat_asl_store_index_fs amduat_asl amduat_enc amduat_hash_asl1
-          amduat_util amduat_federation
+          amduat_asl_store_index_fs amduat_asl_derivation_index_fs
+          amduat_asl amduat_enc amduat_hash_asl1 amduat_util
+          amduat_federation
 )
 
 add_executable(amduat_pel_gc
@@ -82,3 +83,20 @@ target_link_libraries(amduatd_test_store_backend
 )
 
 add_test(NAME amduatd_store_backend COMMAND amduatd_test_store_backend)
+
+add_executable(amduatd_test_derivation_index
+  tests/test_amduatd_derivation_index.c
+  src/amduatd_derivation_index.c
+)
+
+target_include_directories(amduatd_test_derivation_index
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/vendor/amduat/include
+)
+
+target_link_libraries(amduatd_test_derivation_index
+  PRIVATE amduat_asl_derivation_index_fs amduat_pel amduat_asl amduat_util
+)
+
+add_test(NAME amduatd_derivation_index COMMAND amduatd_test_derivation_index)

README.md

@@ -47,6 +47,18 @@ Run the daemon with the index-backed store:
 
 Note: `/v1/fed/records` requires the index backend.
 
+Run the daemon with derivation indexing enabled:
+
+```sh
+./build/amduatd --root .amduat-asl --sock amduatd.sock --enable-derivation-index
+```
+
+To fail `/v1/pel/run` if the derivation index write fails:
+
+```sh
+./build/amduatd --root .amduat-asl --sock amduatd.sock --derivation-index-strict
+```
+
 Dev loop (build + restart):
 
 ```sh
@@ -108,6 +120,9 @@ curl --unix-socket amduatd.sock -X POST http://localhost/v1/pel/run \
   -d '{"program_ref":"<program_ref>","input_refs":["<input_ref_0>"],"params_ref":"<params_ref>"}'
 ```
 
+When derivation indexing is enabled, successful PEL runs record derivations under
+`<root>/index/derivations/by_artifact/` keyed by output refs (plus result/trace/receipt refs).
+
 Define a PEL/PROGRAM-DAG/1 program (store-backed):
 
 ```sh

src/amduatd.c

@@ -39,6 +39,7 @@
 #include "amduatd_caps.h"
 #include "amduatd_space.h"
 #include "amduatd_store.h"
+#include "amduatd_derivation_index.h"
 
 #include <errno.h>
 #include <signal.h>
@@ -1367,6 +1368,7 @@ static bool amduatd_handle_post_pel_run(int fd,
                                         const amduat_asl_store_fs_config_t *cfg,
                                         const amduatd_concepts_t *concepts,
                                         const amduatd_cfg_t *dcfg,
+                                        const char *root_path,
                                         const amduatd_http_req_t *req) {
   uint8_t *body = NULL;
   const char *p = NULL;
@@ -2393,6 +2395,30 @@ static bool amduatd_handle_post_pel_run(int fd,
     receipt_emitted = true;
   }
 
+  if (dcfg != NULL && dcfg->derivation_index_enabled) {
+    amduat_asl_store_error_t idx_err = AMDUAT_ASL_STORE_OK;
+    if (!amduatd_derivation_index_pel_run(root_path,
+                                          true,
+                                          program_ref,
+                                          input_refs,
+                                          input_refs_len,
+                                          has_params_ref,
+                                          params_ref,
+                                          &run_result,
+                                          receipt_emitted,
+                                          receipt_ref,
+                                          &idx_err)) {
+      if (dcfg->derivation_index_strict) {
+        ok = amduatd_send_json_error(fd, 500, "Internal Server Error",
+                                     "derivation index failed");
+        goto pel_run_cleanup;
+      }
+      amduat_log(AMDUAT_LOG_WARN,
+                 "pel run derivation index failed: %d",
+                 (int)idx_err);
+    }
+  }
+
   {
     amduatd_strbuf_t resp;
     char *result_hex = NULL;
@@ -3950,7 +3976,8 @@ static bool amduatd_handle_conn(int fd,
     goto conn_cleanup;
   }
   if (strcmp(req.method, "POST") == 0 && strcmp(no_query, "/v1/pel/run") == 0) {
-    ok = amduatd_handle_post_pel_run(fd, store, cfg, concepts, dcfg, &req);
+    ok = amduatd_handle_post_pel_run(fd, store, cfg, concepts, dcfg,
+                                     root_path, &req);
     goto conn_cleanup;
   }
   if (strcmp(req.method, "POST") == 0 &&
@@ -4035,6 +4062,8 @@ static void amduatd_print_usage(FILE *stream) {
           "         [--store-backend fs|index]\n"
           "         [--allow-uid UID] [--allow-user NAME]\n"
           "         [--enable-cap-reads]\n"
+          "         [--enable-derivation-index]\n"
+          "         [--derivation-index-strict]\n"
           "\n"
           "defaults:\n"
           "  --root %s\n"
@@ -4154,6 +4183,11 @@ int main(int argc, char **argv) {
       }
     } else if (strcmp(argv[i], "--enable-cap-reads") == 0) {
       enable_cap_reads = true;
+    } else if (strcmp(argv[i], "--enable-derivation-index") == 0) {
+      dcfg.derivation_index_enabled = true;
+    } else if (strcmp(argv[i], "--derivation-index-strict") == 0) {
+      dcfg.derivation_index_enabled = true;
+      dcfg.derivation_index_strict = true;
     } else if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-h") == 0) {
       amduatd_print_usage(stdout);
       return 0;
@@ -4175,6 +4209,10 @@ int main(int argc, char **argv) {
   amduat_log(AMDUAT_LOG_INFO,
              "store backend=%s",
              amduatd_store_backend_name(store_backend));
+  amduat_log(AMDUAT_LOG_INFO,
+             "derivation index=%s strict=%s",
+             dcfg.derivation_index_enabled ? "on" : "off",
+             dcfg.derivation_index_strict ? "on" : "off");
   if (!amduatd_caps_init(&caps, &dcfg, root)) {
     amduat_log(AMDUAT_LOG_WARN, "capabilities unavailable");
   }

src/amduatd_caps.h

@@ -15,6 +15,8 @@ extern "C" {
 typedef struct amduatd_cfg_t {
   amduatd_space_t space;
   uint64_t edges_refresh_ms;
+  bool derivation_index_enabled;
+  bool derivation_index_strict;
 } amduatd_cfg_t;
 
 typedef enum {

src/amduatd_derivation_index.c

@@ -0,0 +1,197 @@
+#include "amduatd_derivation_index.h"
+
+#include "amduat/asl/asl_derivation_index_fs.h"
+#include "amduat/pel/core.h"
+#include "amduat/pel/derivation_sid.h"
+#include <string.h>
+
+/* Reserve output_index values for non-output artifacts (result/trace/receipt). */
+enum {
+  AMDUATD_DERIVATION_OUTPUT_INDEX_RESULT = UINT32_MAX,
+  AMDUATD_DERIVATION_OUTPUT_INDEX_TRACE = UINT32_MAX - 1u,
+  AMDUATD_DERIVATION_OUTPUT_INDEX_RECEIPT = UINT32_MAX - 2u
+};
+
+static bool amduatd_derivation_index_add_record(
+    amduat_asl_derivation_index_fs_t *index,
+    amduat_octets_t sid,
+    amduat_reference_t program_ref,
+    const amduat_reference_t *input_refs,
+    size_t input_refs_len,
+    bool has_params_ref,
+    amduat_reference_t params_ref,
+    amduat_reference_t artifact_ref,
+    uint32_t output_index,
+    amduat_asl_store_error_t *out_err) {
+  amduat_asl_derivation_record_t record;
+  amduat_octets_t record_sid = amduat_octets(NULL, 0u);
+  amduat_asl_store_error_t err;
+
+  if (!amduat_octets_clone(sid, &record_sid)) {
+    if (out_err != NULL) {
+      *out_err = AMDUAT_ASL_STORE_ERR_INTEGRITY;
+    }
+    return false;
+  }
+
+  memset(&record, 0, sizeof(record));
+  record.sid = record_sid;
+  record.program_ref = program_ref;
+  record.output_index = output_index;
+  record.input_refs = (amduat_reference_t *)input_refs;
+  record.input_refs_len = input_refs_len;
+  record.has_params_ref = has_params_ref;
+  if (has_params_ref) {
+    record.params_ref = params_ref;
+  }
+  record.has_exec_profile = false;
+  record.exec_profile = amduat_octets(NULL, 0u);
+
+  err = amduat_asl_derivation_index_fs_add(index, artifact_ref, &record);
+  amduat_octets_free(&record.sid);
+  if (err != AMDUAT_ASL_STORE_OK) {
+    if (out_err != NULL) {
+      *out_err = err;
+    }
+    return false;
+  }
+
+  return true;
+}
+
+bool amduatd_derivation_index_pel_run(const char *root_path,
+                                      bool enabled,
+                                      amduat_reference_t program_ref,
+                                      const amduat_reference_t *input_refs,
+                                      size_t input_refs_len,
+                                      bool has_params_ref,
+                                      amduat_reference_t params_ref,
+                                      const amduat_pel_run_result_t *run_result,
+                                      bool has_receipt_ref,
+                                      amduat_reference_t receipt_ref,
+                                      amduat_asl_store_error_t *out_err) {
+  amduat_asl_derivation_index_fs_t index;
+  amduat_pel_derivation_sid_input_t sid_input;
+  amduat_octets_t sid = amduat_octets(NULL, 0u);
+  size_t i;
+
+  if (out_err != NULL) {
+    *out_err = AMDUAT_ASL_STORE_OK;
+  }
+
+  if (!enabled) {
+    return true;
+  }
+  if (root_path == NULL || root_path[0] == '\0' || run_result == NULL) {
+    if (out_err != NULL) {
+      *out_err = AMDUAT_ASL_STORE_ERR_INTEGRITY;
+    }
+    return false;
+  }
+  if (!run_result->has_result_value) {
+    return true;
+  }
+  if (run_result->result_value.has_store_failure) {
+    return true;
+  }
+  if (run_result->result_value.core_result.status != AMDUAT_PEL_EXEC_STATUS_OK) {
+    return true;
+  }
+  if (run_result->output_refs_len > UINT32_MAX) {
+    if (out_err != NULL) {
+      *out_err = AMDUAT_ASL_STORE_ERR_INTEGRITY;
+    }
+    return false;
+  }
+  if (!amduat_asl_derivation_index_fs_init(&index, root_path)) {
+    if (out_err != NULL) {
+      *out_err = AMDUAT_ASL_STORE_ERR_IO;
+    }
+    return false;
+  }
+
+  memset(&sid_input, 0, sizeof(sid_input));
+  sid_input.program_ref = program_ref;
+  sid_input.input_refs = input_refs;
+  sid_input.input_refs_len = input_refs_len;
+  sid_input.has_params_ref = has_params_ref;
+  if (has_params_ref) {
+    sid_input.params_ref = params_ref;
+  }
+  sid_input.has_exec_profile = false;
+  sid_input.exec_profile = amduat_octets(NULL, 0u);
+
+  if (!amduat_pel_derivation_sid_compute(&sid_input, &sid)) {
+    if (out_err != NULL) {
+      *out_err = AMDUAT_ASL_STORE_ERR_INTEGRITY;
+    }
+    return false;
+  }
+
+  for (i = 0u; i < run_result->output_refs_len; ++i) {
+    if (!amduatd_derivation_index_add_record(&index,
+                                             sid,
+                                             program_ref,
+                                             input_refs,
+                                             input_refs_len,
+                                             has_params_ref,
+                                             params_ref,
+                                             run_result->output_refs[i],
+                                             (uint32_t)i,
+                                             out_err)) {
+      amduat_octets_free(&sid);
+      return false;
+    }
+  }
+
+  if (!amduatd_derivation_index_add_record(&index,
+                                           sid,
+                                           program_ref,
+                                           input_refs,
+                                           input_refs_len,
+                                           has_params_ref,
+                                           params_ref,
+                                           run_result->result_ref,
+                                           AMDUATD_DERIVATION_OUTPUT_INDEX_RESULT,
+                                           out_err)) {
+    amduat_octets_free(&sid);
+    return false;
+  }
+
+  if (run_result->result_value.has_trace_ref) {
+    if (!amduatd_derivation_index_add_record(
+            &index,
+            sid,
+            program_ref,
+            input_refs,
+            input_refs_len,
+            has_params_ref,
+            params_ref,
+            run_result->result_value.trace_ref,
+            AMDUATD_DERIVATION_OUTPUT_INDEX_TRACE,
+            out_err)) {
+      amduat_octets_free(&sid);
+      return false;
+    }
+  }
+
+  if (has_receipt_ref) {
+    if (!amduatd_derivation_index_add_record(
+            &index,
+            sid,
+            program_ref,
+            input_refs,
+            input_refs_len,
+            has_params_ref,
+            params_ref,
+            receipt_ref,
+            AMDUATD_DERIVATION_OUTPUT_INDEX_RECEIPT,
+            out_err)) {
+      amduat_octets_free(&sid);
+      return false;
+    }
+  }
+
+  amduat_octets_free(&sid);
+  return true;
+}

src/amduatd_derivation_index.h

@@ -0,0 +1,32 @@
+#ifndef AMDUATD_DERIVATION_INDEX_H
+#define AMDUATD_DERIVATION_INDEX_H
+
+#include "amduat/asl/core.h"
+#include "amduat/asl/store.h"
+#include "amduat/pel/run.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+bool amduatd_derivation_index_pel_run(const char *root_path,
+                                      bool enabled,
+                                      amduat_reference_t program_ref,
+                                      const amduat_reference_t *input_refs,
+                                      size_t input_refs_len,
+                                      bool has_params_ref,
+                                      amduat_reference_t params_ref,
+                                      const amduat_pel_run_result_t *run_result,
+                                      bool has_receipt_ref,
+                                      amduat_reference_t receipt_ref,
+                                      amduat_asl_store_error_t *out_err);
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif /* AMDUATD_DERIVATION_INDEX_H */

tests/test_amduatd_derivation_index.c

@@ -0,0 +1,199 @@
+#ifndef _POSIX_C_SOURCE
+#define _POSIX_C_SOURCE 200809L
+#endif
+
+#include "amduatd_derivation_index.h"
+
+#include "amduat/asl/asl_derivation_index_fs.h"
+#include "amduat/hash/asl1.h"
+#include "amduat/pel/derivation_sid.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static char *amduatd_test_make_temp_dir(void) {
+  char tmpl[] = "/tmp/amduatd-deriv-XXXXXX";
+  char *dir = mkdtemp(tmpl);
+  size_t len;
+  char *copy;
+  if (dir == NULL) {
+    perror("mkdtemp");
+    return NULL;
+  }
+  len = strlen(dir);
+  copy = (char *)malloc(len + 1u);
+  if (copy == NULL) {
+    fprintf(stderr, "failed to allocate temp dir copy\n");
+    return NULL;
+  }
+  memcpy(copy, dir, len + 1u);
+  return copy;
+}
+
+static bool amduatd_test_make_ref(uint8_t seed, amduat_reference_t *out_ref) {
+  uint8_t *digest;
+  size_t i;
+
+  if (out_ref == NULL) {
+    return false;
+  }
+  digest = (uint8_t *)malloc(32u);
+  if (digest == NULL) {
+    return false;
+  }
+  for (i = 0u; i < 32u; ++i) {
+    digest[i] = (uint8_t)(seed + i);
+  }
+  *out_ref = amduat_reference(AMDUAT_HASH_ASL1_ID_SHA256,
+                              amduat_octets(digest, 32u));
+  return true;
+}
+
+int main(void) {
+  char *root = amduatd_test_make_temp_dir();
+  amduat_asl_derivation_index_fs_t index;
+  amduat_reference_t program_ref;
+  amduat_reference_t input_ref;
+  amduat_reference_t output_ref;
+  amduat_reference_t result_ref;
+  amduat_reference_t params_ref;
+  amduat_reference_t input_refs[1];
+  amduat_reference_t output_refs[1];
+  amduat_pel_run_result_t run_result;
+  amduat_asl_derivation_record_t *records = NULL;
+  size_t record_count = 0u;
+  amduat_asl_store_error_t err;
+
+  if (root == NULL) {
+    return 1;
+  }
+
+  if (!amduatd_test_make_ref(1u, &program_ref) ||
+      !amduatd_test_make_ref(2u, &input_ref) ||
+      !amduatd_test_make_ref(3u, &output_ref) ||
+      !amduatd_test_make_ref(4u, &result_ref)) {
+    fprintf(stderr, "failed to build refs\n");
+    free(root);
+    return 1;
+  }
+
+  input_refs[0] = input_ref;
+  output_refs[0] = output_ref;
+  memset(&params_ref, 0, sizeof(params_ref));
+  memset(&run_result, 0, sizeof(run_result));
+  run_result.has_result_value = true;
+  run_result.result_value.core_result.status = AMDUAT_PEL_EXEC_STATUS_OK;
+  run_result.result_value.has_store_failure = false;
+  run_result.output_refs = output_refs;
+  run_result.output_refs_len = 1u;
+  run_result.result_ref = result_ref;
+
+  err = AMDUAT_ASL_STORE_OK;
+  if (!amduatd_derivation_index_pel_run(root,
+                                        false,
+                                        program_ref,
+                                        input_refs,
+                                        1u,
+                                        false,
+                                        params_ref,
+                                        &run_result,
+                                        false,
+                                        params_ref,
+                                        &err)) {
+    fprintf(stderr, "unexpected derivation index failure when disabled\n");
+    free(root);
+    return 1;
+  }
+
+  if (!amduat_asl_derivation_index_fs_init(&index, root)) {
+    fprintf(stderr, "failed to init derivation index\n");
+    free(root);
+    return 1;
+  }
+
+  err = amduat_asl_derivation_index_fs_list(&index, output_ref,
+                                            &records, &record_count);
+  if (err != AMDUAT_ASL_STORE_ERR_NOT_FOUND) {
+    fprintf(stderr, "expected no records when disabled, got %d\n", (int)err);
+    free(root);
+    return 1;
+  }
+
+  err = AMDUAT_ASL_STORE_OK;
+  if (!amduatd_derivation_index_pel_run(root,
+                                        true,
+                                        program_ref,
+                                        input_refs,
+                                        1u,
+                                        false,
+                                        params_ref,
+                                        &run_result,
+                                        false,
+                                        params_ref,
+                                        &err)) {
+    fprintf(stderr, "expected derivation index to succeed\n");
+    free(root);
+    return 1;
+  }
+
+  err = amduat_asl_derivation_index_fs_list(&index, output_ref,
+                                            &records, &record_count);
+  if (err != AMDUAT_ASL_STORE_OK || record_count != 1u) {
+    fprintf(stderr, "expected one derivation record, got %d count=%zu\n",
+            (int)err, record_count);
+    free(root);
+    return 1;
+  }
+
+  {
+    amduat_pel_derivation_sid_input_t sid_input;
+    amduat_octets_t sid = amduat_octets(NULL, 0u);
+
+    memset(&sid_input, 0, sizeof(sid_input));
+    sid_input.program_ref = program_ref;
+    sid_input.input_refs = input_refs;
+    sid_input.input_refs_len = 1u;
+    sid_input.has_params_ref = false;
+    sid_input.has_exec_profile = false;
+    sid_input.exec_profile = amduat_octets(NULL, 0u);
+
+    if (!amduat_pel_derivation_sid_compute(&sid_input, &sid)) {
+      fprintf(stderr, "failed to compute expected sid\n");
+      amduat_asl_derivation_records_free(records, record_count);
+      free(root);
+      return 1;
+    }
+
+    if (!amduat_octets_eq(records[0].sid, sid)) {
+      fprintf(stderr, "sid mismatch\n");
+      amduat_octets_free(&sid);
+      amduat_asl_derivation_records_free(records, record_count);
+      free(root);
+      return 1;
+    }
+    if (!amduat_reference_eq(records[0].program_ref, program_ref)) {
+      fprintf(stderr, "program_ref mismatch\n");
+      amduat_octets_free(&sid);
+      amduat_asl_derivation_records_free(records, record_count);
+      free(root);
+      return 1;
+    }
+    if (records[0].output_index != 0u) {
+      fprintf(stderr, "output_index mismatch\n");
+      amduat_octets_free(&sid);
+      amduat_asl_derivation_records_free(records, record_count);
+      free(root);
+      return 1;
+    }
+    amduat_octets_free(&sid);
+  }
+
+  amduat_asl_derivation_records_free(records, record_count);
+  amduat_reference_free(&program_ref);
+  amduat_reference_free(&input_ref);
+  amduat_reference_free(&output_ref);
+  amduat_reference_free(&result_ref);
+  free(root);
+  return 0;
+}