Patched overflow checks in the encoder and added regression tests.

Changes:

Added early bounds checks for nodes_len/roots_len against UINT32_MAX, plus safe guards for order allocation and roots_len * 8 overflow. pel_program_dag.c
New tests for oversized counts: test_pel_program_dag_encode.c
Wired the new test into CMake: CMakeLists.txt

CMakeLists.txt

@@ -297,6 +297,17 @@ target_link_libraries(amduat_test_pel_program_dag_exec
 )
 add_test(NAME pel_program_dag_exec COMMAND amduat_test_pel_program_dag_exec)
 
+add_executable(amduat_test_pel_program_dag_encode
+  tests/pel/test_pel_program_dag_encode.c)
+target_include_directories(amduat_test_pel_program_dag_encode
+  PRIVATE ${AMDUAT_INTERNAL_DIR}
+  PRIVATE ${AMDUAT_INCLUDE_DIR}
+)
+target_link_libraries(amduat_test_pel_program_dag_encode
+  PRIVATE amduat_enc
+)
+add_test(NAME pel_program_dag_encode COMMAND amduat_test_pel_program_dag_encode)
+
 add_executable(amduat_test_pel_program_dag_desc
   tests/pel/test_pel_program_dag_desc.c)
 target_include_directories(amduat_test_pel_program_dag_desc

src/near_core/enc/pel_program_dag.c

@@ -295,6 +295,12 @@ bool amduat_enc_pel_program_dag_encode_v1(
   if (program->roots_len > 0 && program->roots == NULL) {
     return false;
   }
+  if (program->nodes_len > UINT32_MAX || program->roots_len > UINT32_MAX) {
+    return false;
+  }
+  if (program->nodes_len > SIZE_MAX / sizeof(*order)) {
+    return false;
+  }
 
   order = NULL;
   if (program->nodes_len > 0) {
@@ -312,10 +318,6 @@ bool amduat_enc_pel_program_dag_encode_v1(
     free(order);
     return false;
   }
-  if (program->nodes_len > UINT32_MAX || program->roots_len > UINT32_MAX) {
-    free(order);
-    return false;
-  }
   if (!amduat_add_size(&total_len, 4)) {
     free(order);
     return false;
@@ -403,6 +405,10 @@ bool amduat_enc_pel_program_dag_encode_v1(
     free(order);
     return false;
   }
+  if (program->roots_len > SIZE_MAX / 8) {
+    free(order);
+    return false;
+  }
   if (!amduat_add_size(&total_len, program->roots_len * 8)) {
     free(order);
     return false;

tests/pel/test_pel_program_dag_encode.c

@@ -0,0 +1,60 @@
+#include "amduat/enc/pel_program_dag.h"
+
+#include <limits.h>
+#include <stdio.h>
+#include <string.h>
+
+static int test_encode_nodes_count_overflow(void) {
+  amduat_pel_program_t program;
+  amduat_octets_t encoded = amduat_octets(NULL, 0);
+
+  if (SIZE_MAX <= UINT32_MAX) {
+    return 0;
+  }
+
+  memset(&program, 0, sizeof(program));
+  program.nodes_len = (size_t)UINT32_MAX + 1u;
+  program.roots_len = 0;
+
+  if (amduat_enc_pel_program_dag_encode_v1(&program, &encoded)) {
+    fprintf(stderr, "expected encode failure for nodes_len overflow\n");
+    amduat_octets_free(&encoded);
+    return 1;
+  }
+
+  return 0;
+}
+
+static int test_encode_roots_len_overflow(void) {
+  amduat_pel_program_t program;
+  amduat_octets_t encoded = amduat_octets(NULL, 0);
+  size_t roots_len;
+
+  if (SIZE_MAX / 8 == 0) {
+    return 0;
+  }
+
+  roots_len = (SIZE_MAX / 8) + 1u;
+
+  memset(&program, 0, sizeof(program));
+  program.nodes_len = 0;
+  program.roots_len = roots_len;
+
+  if (amduat_enc_pel_program_dag_encode_v1(&program, &encoded)) {
+    fprintf(stderr, "expected encode failure for roots_len overflow\n");
+    amduat_octets_free(&encoded);
+    return 1;
+  }
+
+  return 0;
+}
+
+int main(void) {
+  if (test_encode_nodes_count_overflow() != 0) {
+    return 1;
+  }
+  if (test_encode_roots_len_overflow() != 0) {
+    return 1;
+  }
+  return 0;
+}

tests/pel/test_pel_program_dag_exec.c

@@ -1,4 +1,6 @@
 #include "pel_program_dag_internal.h"
+#include "amduat/hash/asl1.h"
+#include "amduat/pel/run.h"
 #include <stdbool.h>
 #include <stdint.h>
 #include <stdio.h>
@@ -297,6 +299,60 @@ static int test_invalid_input_index(void) {
   return 0;
 }
 
+static int test_exec_scheme_mismatch(void) {
+  uint8_t digest[32];
+  amduat_octets_t digest_bytes;
+  amduat_reference_t scheme_ref;
+  amduat_artifact_t program_artifact;
+  amduat_artifact_t *outputs = NULL;
+  size_t outputs_len = 0;
+  amduat_pel_execution_result_value_t result;
+  int exit_code = 1;
+
+  memset(digest, 0x5au, sizeof(digest));
+  digest_bytes = amduat_octets(NULL, 0);
+  if (!amduat_octets_clone(amduat_octets(digest, sizeof(digest)),
+                           &digest_bytes)) {
+    fprintf(stderr, "digest alloc failed\n");
+    return exit_code;
+  }
+  scheme_ref = amduat_reference(AMDUAT_HASH_ASL1_ID_SHA256, digest_bytes);
+  program_artifact = amduat_artifact(amduat_octets(NULL, 0));
+  memset(&result, 0, sizeof(result));
+
+  if (!amduat_pel_exec_program_artifact_with_scheme(scheme_ref,
+                                                    program_artifact,
+                                                    NULL,
+                                                    0,
+                                                    NULL,
+                                                    &outputs,
+                                                    &outputs_len,
+                                                    &result)) {
+    fprintf(stderr, "exec failed\n");
+    goto cleanup;
+  }
+
+  if (result.status != AMDUAT_PEL_EXEC_STATUS_SCHEME_UNSUPPORTED ||
+      result.summary.kind != AMDUAT_PEL_EXEC_ERROR_SCHEME ||
+      result.summary.status_code != 1) {
+    fprintf(stderr, "unexpected scheme mismatch status\n");
+    goto cleanup;
+  }
+
+  if (outputs_len != 0 || outputs != NULL) {
+    fprintf(stderr, "unexpected outputs\n");
+    goto cleanup;
+  }
+
+  exit_code = 0;
+
+cleanup:
+  amduat_pel_program_dag_free_outputs(outputs, outputs_len);
+  amduat_pel_execution_result_free(&result);
+  amduat_reference_free(&scheme_ref);
+  return exit_code;
+}
+
 int main(void) {
   if (test_valid_program() != 0) {
     return 1;
@@ -310,5 +366,8 @@ int main(void) {
   if (test_invalid_input_index() != 0) {
     return 1;
   }
+  if (test_exec_scheme_mismatch() != 0) {
+    return 1;
+  }
   return 0;
 }