From 1f4b8ff5bc199333fe5f51eee1ead218585beaeb Mon Sep 17 00:00:00 2001 From: Carl Niklas Rydberg Date: Sat, 21 Feb 2026 17:03:56 +0100 Subject: [PATCH] Add OSS backend proposal specifics and otiline conformance tests --- CMakeLists.txt | 62 ++ docs/adapter-error-taxonomy.md | 98 ++ docs/amduat-oss-backend-proposal.md | 351 +++++++ docs/conformance-harness-spec.md | 165 ++++ docs/decision-checklist.md | 101 ++ docs/es-indexing-plan.md | 120 +++ docs/implementation-plan.md | 460 +++++++++ docs/indexing-requirements-and-projections.md | 145 +++ docs/oss-backend-adapter-boundary.md | 274 ++++++ docs/phase0-1-execution-checklist.md | 160 +++ docs/primitive-vs-pluggable-ops.md | 253 +++++ docs/projection-checkpoint-schema.md | 81 ++ docs/tgk-ordering-enforcement.md | 76 ++ src/tools/amduat_conformance_run.c | 203 ++++ .../asl_store/asl_store_conformance.c | 908 ++++++++++++++++++ .../asl_store/asl_store_conformance.h | 26 + .../asl_store/test_asl_store_conformance.c | 87 ++ tests/conformance/conformance_result.h | 19 + .../tgk_store/test_tgk_store_conformance.c | 87 ++ .../tgk_store/tgk_store_conformance.c | 628 ++++++++++++ .../tgk_store/tgk_store_conformance.h | 26 + .../conformance/asl_put_get/artifacts/a0.bin | 0 .../conformance/asl_put_get/artifacts/a1.bin | 1 + .../conformance/asl_put_get/artifacts/a2.bin | Bin 0 -> 256 bytes .../conformance/asl_put_get/manifest.json | 8 + .../pel_trace_tgk_test_vectors_v1.json | 118 +++ tier1/hash-asl1.md | 69 +- 27 files changed, 4523 insertions(+), 3 deletions(-) create mode 100644 docs/adapter-error-taxonomy.md create mode 100644 docs/amduat-oss-backend-proposal.md create mode 100644 docs/conformance-harness-spec.md create mode 100644 docs/decision-checklist.md create mode 100644 docs/es-indexing-plan.md create mode 100644 docs/implementation-plan.md create mode 100644 docs/indexing-requirements-and-projections.md create mode 100644 docs/oss-backend-adapter-boundary.md create mode 100644 docs/phase0-1-execution-checklist.md create mode 100644 docs/primitive-vs-pluggable-ops.md create mode 100644 docs/projection-checkpoint-schema.md create mode 100644 docs/tgk-ordering-enforcement.md create mode 100644 src/tools/amduat_conformance_run.c create mode 100644 tests/conformance/asl_store/asl_store_conformance.c create mode 100644 tests/conformance/asl_store/asl_store_conformance.h create mode 100644 tests/conformance/asl_store/test_asl_store_conformance.c create mode 100644 tests/conformance/conformance_result.h create mode 100644 tests/conformance/tgk_store/test_tgk_store_conformance.c create mode 100644 tests/conformance/tgk_store/tgk_store_conformance.c create mode 100644 tests/conformance/tgk_store/tgk_store_conformance.h create mode 100644 tests/fixtures/conformance/asl_put_get/artifacts/a0.bin create mode 100644 tests/fixtures/conformance/asl_put_get/artifacts/a1.bin create mode 100644 tests/fixtures/conformance/asl_put_get/artifacts/a2.bin create mode 100644 tests/fixtures/conformance/asl_put_get/manifest.json create mode 100644 tests/vectors/pel_trace_tgk_test_vectors_v1.json diff --git a/CMakeLists.txt b/CMakeLists.txt index 2d82535..f6753b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -304,6 +304,25 @@ target_link_libraries(amduat_pel_build ) set_target_properties(amduat_pel_build PROPERTIES OUTPUT_NAME amduat-pel-build) +add_executable(amduat_conformance_run + src/tools/amduat_conformance_run.c + tests/conformance/asl_store/asl_store_conformance.c + tests/conformance/tgk_store/tgk_store_conformance.c +) +target_include_directories(amduat_conformance_run + PRIVATE ${AMDUAT_INTERNAL_DIR} + PRIVATE ${AMDUAT_INCLUDE_DIR} + PRIVATE ${AMDUAT_ROOT} + PRIVATE ${AMDUAT_ROOT}/tests/conformance/asl_store + PRIVATE ${AMDUAT_ROOT}/tests/conformance/tgk_store +) +target_link_libraries(amduat_conformance_run + PRIVATE amduat_asl_store_fs amduat_asl_store_index_fs + amduat_tgk_store_mem amduat_tgk_store_fs amduat_tgk_store_asl_index_fs + amduat_tgk amduat_asl amduat_enc amduat_hash_asl1 amduat_util + amduat_format +) + enable_testing() add_executable(amduat_test_pel_program_dag tests/enc/test_pel_program_dag.c) @@ -564,6 +583,49 @@ target_link_libraries(amduat_test_asl_derivation_index_fs add_test(NAME asl_derivation_index_fs COMMAND amduat_test_asl_derivation_index_fs) +add_executable(amduat_test_asl_store_conformance + tests/conformance/asl_store/test_asl_store_conformance.c + tests/conformance/asl_store/asl_store_conformance.c +) +target_include_directories(amduat_test_asl_store_conformance + PRIVATE ${AMDUAT_INTERNAL_DIR} + PRIVATE ${AMDUAT_INCLUDE_DIR} + PRIVATE ${AMDUAT_ROOT} + PRIVATE ${AMDUAT_ROOT}/tests/conformance/asl_store +) +target_compile_definitions(amduat_test_asl_store_conformance + PRIVATE _POSIX_C_SOURCE=200809L +) +target_link_libraries(amduat_test_asl_store_conformance + PRIVATE amduat_asl_store_fs amduat_asl_store_index_fs + amduat_asl amduat_enc amduat_hash_asl1 amduat_util +) +add_test(NAME asl_store_conformance + COMMAND amduat_test_asl_store_conformance +) + +add_executable(amduat_test_tgk_store_conformance + tests/conformance/tgk_store/test_tgk_store_conformance.c + tests/conformance/tgk_store/tgk_store_conformance.c +) +target_include_directories(amduat_test_tgk_store_conformance + PRIVATE ${AMDUAT_INTERNAL_DIR} + PRIVATE ${AMDUAT_INCLUDE_DIR} + PRIVATE ${AMDUAT_ROOT} + PRIVATE ${AMDUAT_ROOT}/tests/conformance/tgk_store +) +target_compile_definitions(amduat_test_tgk_store_conformance + PRIVATE _POSIX_C_SOURCE=200809L +) +target_link_libraries(amduat_test_tgk_store_conformance + PRIVATE amduat_tgk_store_mem amduat_tgk_store_fs amduat_tgk_store_asl_index_fs + amduat_asl_store_fs amduat_asl_store_index_fs amduat_tgk amduat_asl + amduat_enc amduat_hash_asl1 amduat_util amduat_format +) +add_test(NAME tgk_store_conformance + COMMAND amduat_test_tgk_store_conformance +) + add_executable(amduat_test_pel_program_dag_exec tests/pel/test_pel_program_dag_exec.c) target_include_directories(amduat_test_pel_program_dag_exec diff --git a/docs/adapter-error-taxonomy.md b/docs/adapter-error-taxonomy.md new file mode 100644 index 0000000..338e582 --- /dev/null +++ b/docs/adapter-error-taxonomy.md @@ -0,0 +1,98 @@ +# Adapter Error Taxonomy + +Status: Draft +Scope: Canonical error mapping for ASL and TGK adapters and derived projections, used by conformance tests to prevent drift. + +--- + +# 1. Purpose + +This taxonomy defines a canonical internal error set. Adapters MUST map backend-specific errors into these categories so that conformance tests remain stable across backends. + +--- + +# 2. Canonical Error Set + +## 2.1 ASL Store Errors + +Canonical enum (from `include/amduat/asl/store.h`): +1. `AMDUAT_ASL_STORE_ERR_INTEGRITY` +2. `AMDUAT_ASL_STORE_ERR_UNSUPPORTED` +3. `AMDUAT_ASL_STORE_ERR_NOT_FOUND` +4. `AMDUAT_ASL_STORE_ERR_IO` + +## 2.2 TGK Graph Store Errors + +Canonical enum (from `include/amduat/tgk/store.h`): +1. `GS_ERR_NOT_EDGE` +2. `GS_ERR_ARTIFACT_ERROR` +3. `GS_ERR_UNSUPPORTED` +4. `GS_ERR_INTEGRITY` + +## 2.3 Projection Errors (Derived Indexers) + +Canonical categories: +1. NotFound +2. PermissionDenied +3. TransientIO +4. InvalidDocument +5. InternalError + +Projection errors must never be surfaced as authoritative ASL or TGK errors. + +--- + +# 3. Mapping Rules + +## 3.1 ASL/1-STORE Mapping + +1. `ERR_NOT_FOUND` → `AMDUAT_ASL_STORE_ERR_NOT_FOUND` +2. `ERR_INTEGRITY` → `AMDUAT_ASL_STORE_ERR_INTEGRITY` +3. `ERR_UNSUPPORTED` → `AMDUAT_ASL_STORE_ERR_UNSUPPORTED` +4. Backend I/O or transport failure → `AMDUAT_ASL_STORE_ERR_IO` + +Notes: +1. `ASL/1-STORE` does not standardize I/O or auth errors; those must map to `ERR_IO` for conformance tests. +2. A `put` collision on identical reference with different bytes MUST map to `ERR_INTEGRITY`. + +## 3.2 TGK/STORE/1 Mapping + +1. Reference resolves to non-edge artifact → `GS_ERR_NOT_EDGE` +2. Artifact-layer integrity failure during resolve → `GS_ERR_ARTIFACT_ERROR` +3. Unsupported profile/encoding → `GS_ERR_UNSUPPORTED` +4. Detected graph-level corruption → `GS_ERR_INTEGRITY` + +Notes: +1. `GS_ERR_ARTIFACT_ERROR` is required for artifact-level conflicts; do not use `GS_ERR_INTEGRITY`. +2. Backend-specific errors must be normalized to the canonical TGK error set. + +## 3.3 Projection Adapter Mapping + +1. Index missing doc → NotFound +2. Authorization failure → PermissionDenied +3. Timeout, transport, or retryable failure → TransientIO +4. Schema or decode failure → InvalidDocument +5. Unknown failure → InternalError + +Projection errors are operational only and must not be treated as ASL/TGK semantic failures. + +--- + +# 4. Test Requirements + +Conformance tests MUST include: +1. ASL: `get` on missing ref → `ERR_NOT_FOUND`. +2. ASL: `put` collision → `ERR_INTEGRITY`. +3. ASL: unsupported config → `ERR_UNSUPPORTED`. +4. ASL: I/O failure path → `ERR_IO`. +5. TGK: `resolve_edge` on non-edge → `GS_ERR_NOT_EDGE`. +6. TGK: artifact integrity conflict → `GS_ERR_ARTIFACT_ERROR`. +7. TGK: unsupported encoding/profile → `GS_ERR_UNSUPPORTED`. + +--- + +# 5. Non-Goals + +1. Backend-specific error codes. +2. Retry policy definitions. +3. Human-readable error messaging conventions. diff --git a/docs/amduat-oss-backend-proposal.md b/docs/amduat-oss-backend-proposal.md new file mode 100644 index 0000000..8c9fb9a --- /dev/null +++ b/docs/amduat-oss-backend-proposal.md @@ -0,0 +1,351 @@ +# Amduat Open-Source Backend Proposal + +## Contract-First Backend Integration Architecture + +**Status:** Proposal +**Scope:** Define how the OSS stack (MinIO, JanusGraph, Cassandra, Elasticsearch, Spark, Nessie, IPFS, Ollama) can be deployed as a backend implementation of Amduat while preserving semantic invariants. + +--- + +# Related Documents + +* `docs/implementation-plan.md` +* `docs/phase0-1-execution-checklist.md` +* `docs/decision-checklist.md` +* `docs/conformance-harness-spec.md` +* `docs/adapter-error-taxonomy.md` +* `docs/tgk-ordering-enforcement.md` +* `docs/indexing-requirements-and-projections.md` +* `docs/es-indexing-plan.md` +* `docs/projection-checkpoint-schema.md` +* `docs/oss-backend-adapter-boundary.md` + +--- + +# 1. Executive Summary + +This document proposes integrating a scalable open-source infrastructure stack as a backend implementation of Amduat. + +The proposal preserves Amduat’s defining principle: + +> The semantic core is invariant. +> Backends are interchangeable implementations of store/query surfaces. + +The OSS stack provides scalability, distribution, analytics, and optional acceleration layers without altering the deterministic semantics defined by the Amduat core specifications. + +--- + +# 2. Architectural Position + +## 2.1 Semantic Core + +Core semantics are defined by: + +* `ASL/1-CORE` +* `PEL/1-CORE` +* `TGK/1-CORE` + +Canonical encodings are provided by: + +* `ENC/ASL1-CORE` (and related encoding profiles) + +Identity derivation is defined by: + +* `ASL/1-CORE` +* `HASH/ASL1` + +These define: + +* Artifact identity and immutability +* Core execution semantics +* Core graph semantics +* Deterministic projection rules + +These are deployment-invariant. + +## 2.2 Store / Query Surfaces + +Operational interfaces are defined by: + +* `ASL/1-STORE` +* `TGK/STORE/1` + +Optional provenance semantics: + +* `TGK/PROV/1` + +Storage and indexing layers are pluggable implementations of these surfaces. +They do not redefine core semantics. + +--- + +# 3. No Semantic Drift Rule + +To preserve Amduat’s portability and determinism, all backend implementations must adhere to the following: + +1. **No Backend-Specific Semantics** + + * No interpretation of artifacts beyond canonical encoding. + * No implicit schema expansion. + * No hidden type coercion. + +2. **No Semantic Elevation of Projections** + + * Search indexes (e.g., Elasticsearch) + * Distribution caches (e.g., IPFS) + * Analytics views (e.g., Spark) + * AI-generated outputs + + …are derived projections unless explicitly promoted by governance. + +3. **Deterministic Reproducibility** + + * The logical TGK graph must be reproducible from ASL state. + * Snapshot exports must reproduce identical graph projections. + +4. **Surface Conformance Only** + + * The OSS stack may optimize storage and indexing. + * It may not redefine core behavior. + +5. **Observational Equivalence** + + * Any backend-specific optimization must be observationally equivalent to the core semantics and query results defined by the specifications. + +This rule guarantees backend substitution without semantic drift. + +--- + +# 4. ASL/1-STORE Implementation (Artifact Layer) + +## 4.1 Contract + +Must implement: + +* `put(Artifact) -> Reference` +* `get(Reference) -> Artifact | NotFound` + +Identity derivation follows: + +1. Canonical encoding (`ENC/ASL1-CORE`) +2. Identity derivation rules (`ASL/1-CORE`) +3. Hash function (`HASH/ASL1`) + +In abstract form: + +``` +Reference = HASH( CANONICAL_ENCODE(Artifact) ) +``` + +Reference derivation is deterministic and caller-independent. + +## 4.2 Immutability + +* Artifacts are immutable. +* Rewriting the same Artifact yields the same Reference. +* Mutation semantics do not exist at this layer. + +## 4.3 Authority Rule + +The authoritative store in any deployment is the backend implementation that conforms to `ASL/1-STORE`. + +`ASL/1-STORE` is a specification; authority resides in whichever concrete backend implements it. + +Caches and projections derive from this authoritative ASL implementation unless explicitly promoted by governance. + +## 4.4 Proposed Backend + +Primary implementation: + +* MinIO-backed object storage + +Optional: + +* IPFS as distribution/cache layer + +--- + +# 5. TGK/STORE/1 Implementation (Graph Query Surface) + +## 5.1 Projection Model + +The TGK graph is a logical projection derived from: + +* Artifacts stored via ASL +* Core graph semantics defined in `TGK/1-CORE` + +It is not an independently authoritative database. + +Graph state must be reproducible from ASL state. + +## 5.2 Required Operations + +Backends must implement the required query operations and error model defined in `TGK/STORE/1`. + +This proposal references the specification directly to avoid semantic drift through re-enumeration. + +## 5.3 Read-Only Nature + +* `GraphStoreInstance` is logically read-only. +* Edges arise from writing EdgeArtifacts to ASL. +* Deletion, compaction, or tombstone policies are backend or profile-level implementation concerns and are not part of `TGK/STORE/1`. + +## 5.4 Proposed Backend + +* JanusGraph as query engine +* Cassandra as storage backend +* Optional Elasticsearch as search projection + +JanusGraph functions as an implementation detail of the `TGK/STORE/1` surface. + +--- + +# 6. TGK/PROV/1 (Optional Provenance Layer) + +`TGK/PROV/1` defines pure functions over a `ProvenanceGraph`. + +* It does not require a particular store. +* It is evaluated against `TGK/1-CORE` graph semantics. +* Implementations may use `TGK/STORE/1` queries to realize provenance functions. + +Provenance is layered above store semantics and does not alter core behavior. + +Implementation options: + +* Native traversal realization +* Snapshot-based traversal in analytics engines +* Hybrid realization + +--- + +# 7. Projection & Acceleration Layers + +Unless explicitly promoted by governance: + +* Elasticsearch +* IPFS +* Lakehouse datasets (MinIO + Nessie) +* Spark analytics outputs +* AI-derived artifacts + +…are derived projections. + +Authoritative truth resides in: + +* The deployed `ASL/1-STORE` implementation +* Deterministic TGK projection derived from ASL state + +Projections may be eventually consistent. +Audit-grade claims must reference reproducible ASL state or snapshot identifiers. + +--- + +# 8. Capability Tiers + +Capability tiers define operational envelope, not semantic differences. + +--- + +## Tier 0 — Embedded / Edge + +Implements: + +* `ASL/1-STORE` +* `TGK/STORE/1` +* Optional minimal `TGK/PROV/1` + +Characteristics: + +* Single-node +* Deterministic local projection +* Strong local consistency +* Optional sync/export + +Use cases: + +* Personal knowledge systems +* Offline assistant systems + +--- + +## Tier 1 — Platform / Scale-Out + +Implements: + +* `ASL/1-STORE` +* `TGK/STORE/1` +* Optional `TGK/PROV/1` + +Typical infrastructure: + +* Distributed object store (e.g., MinIO) +* Distributed graph backend (e.g., JanusGraph over Cassandra) +* Optional derived search index + +Characteristics: + +* Multi-node +* Horizontal scaling +* Operational consistency +* Deterministic core preserved + +Use cases: + +* Community platforms +* Shared knowledge systems + +--- + +## Tier 2 — Enterprise / Audit-Grade + +Includes Tier 1 plus: + +* Snapshot export capability +* Lakehouse projection +* Versioned snapshots (e.g., Nessie) +* Governance workflows +* Promotion pipelines + +Characteristics: + +* Snapshot-based reproducibility +* Audit traceability +* Multi-environment promotion + +Use cases: + +* Enterprise backbone systems +* Regulated environments + +--- + +# 9. Conformance Requirements + +Each backend must: + +* Pass `ASL/1-STORE` conformance tests +* Pass `TGK/STORE/1` conformance and deterministic ordering tests +* Demonstrate projection reproducibility from ASL state +* Preserve encoding stability across versions + +Tier 2 must additionally: + +* Validate snapshot equivalence +* Demonstrate replay determinism from ASL state + +Conformance is versioned alongside spec revisions. + +--- + +# 10. Design Outcome + +This architecture ensures: + +* One invariant semantic kernel +* Backend substitution without semantic drift +* Scaling through infrastructure choice +* Deterministic reproducibility +* Clear separation between authority and projection layers +* Portability from embedded edge systems to enterprise-scale deployments + +The OSS stack is therefore a backend implementation of Amduat — not a semantic authority. diff --git a/docs/conformance-harness-spec.md b/docs/conformance-harness-spec.md new file mode 100644 index 0000000..bf4923e --- /dev/null +++ b/docs/conformance-harness-spec.md @@ -0,0 +1,165 @@ +# Conformance Harness Specification + +Status: Draft +Scope: Define the Phase 0 conformance harness goals, backend matrix, fixture format, and pass/fail gates for ASL and TGK store surfaces. + +--- + +# 1. Purpose + +The harness ensures that all backend adapters implement existing store semantics without drift. + +It MUST: +1. Run the same test vectors across multiple backends. +2. Enforce deterministic ordering rules. +3. Validate replay equivalence. +4. Provide a single command entry point for CI. + +--- + +# 2. Targets and Scope + +## 2.1 Surfaces Under Test + +1. ASL/1-STORE +2. TGK/STORE/1 + +Optional extensions: +1. ASL/STORE-INDEX/1 (index/log store) +2. TGK/PROV/1 (if provenance surface is exposed) + +## 2.2 Conformance Mode + +All tests are: +1. Deterministic. +2. Backend-agnostic. +3. Semantics-first (no timing assumptions). + +--- + +# 3. Backend Matrix + +The harness MUST be able to run each test suite against: + +## 3.1 ASL Backends + +1. `asl_store_fs` +2. `asl_store_index_fs` +3. Future: `asl_store_minio` + +## 3.2 TGK Backends + +1. `tgk_store_mem` +2. `tgk_store_fs` +3. `tgk_store_asl_index_fs` +4. Future: `tgk_store_janus` + +--- + +# 4. Canonical Fixture Format + +Fixtures are the source of truth for all conformance tests. + +## 4.1 Requirements + +1. Versioned fixture schema. +2. Stable ordering of artifacts and edges. +3. Explicit references encoded as canonical `ReferenceBytes` hex. +4. Deterministic content generation (no randomness at runtime). + +## 4.2 Minimal Fields + +For ASL fixtures: +1. `artifact_bytes` (hex or base64, fixed encoding). +2. `type_tag` (optional). +3. `expected_reference` (canonical Reference hex). + +For TGK fixtures: +1. `edge_artifact_reference` (canonical Reference hex). +2. `edge_body` (decoded representation). +3. `edge_type_id` +4. `from_refs` and `to_refs` (ordered lists, canonical hex). + +--- + +# 5. Test Suite Requirements + +## 5.1 ASL/1-STORE Suite + +1. Put/get round-trip. +2. Idempotent `put`. +3. `get` on missing ref yields `ERR_NOT_FOUND`. +4. Collision on same ref yields `ERR_INTEGRITY`. +5. Unsupported config yields `ERR_UNSUPPORTED`. +6. Optional I/O fault path yields `ERR_IO`. + +## 5.2 TGK/STORE/1 Suite + +1. `resolve_edge` for valid edge returns `EdgeBody`. +2. `resolve_edge` for non-edge returns `GS_ERR_NOT_EDGE`. +3. `resolve_edge` for artifact integrity failure yields `GS_ERR_ARTIFACT_ERROR`. +4. `edges_from`, `edges_to`, `edges_incident` return identical lists across backends. +5. Deterministic ordering per `TGK/STORE/1` canonical `(hash_id, digest)` rule. +6. Optional `neighbors` returns canonical node order. +7. Optional `scan_edges` enumerates all edges exactly once. + +## 5.3 Replay Equivalence + +1. Build a fixture ASL log/index state. +2. Replay into a fresh store. +3. Assert identical references and TGK results. + +--- + +# 6. Ordering and Equivalence Rules + +## 6.1 Edge Ordering + +Edge lists MUST be sorted in ascending lexicographic order of: +`hash_id_bytes || digest_bytes`, with `hash_id` encoded as big-endian `u16`. + +## 6.2 Node Ordering + +Neighbor lists MUST be sorted in ascending lexicographic order of: +`hash_id_bytes || digest_bytes`, with `hash_id` encoded as big-endian `u16`. + +## 6.3 Cross-Backend Equivalence + +For a fixed snapshot and `GraphStoreConfig`, all backends MUST return: +1. Identical edge sets. +2. Identical ordered lists for adjacency queries. +3. Identical results for `resolve_edge`. + +--- + +# 7. Harness Structure + +Suggested structure: +1. `tests/conformance/asl_store/*.c` +2. `tests/conformance/tgk_store/*.c` +3. `tests/fixtures/*.json` +4. `tools/amduat_conformance_run` + +The harness runner MUST: +1. Accept backend selector flags. +2. Accept fixture set selection. +3. Emit a machine-readable summary. + +--- + +# 8. CI Gate + +An adapter cannot be merged unless: +1. It passes its conformance suite. +2. Cross-backend equivalence is demonstrated. +3. Replay equivalence passes on at least one backend pair. + +--- + +# 9. Out of Scope + +1. Performance benchmarking. +2. Load testing. +3. Backend-specific tuning. + +Those are separate gates after conformance is proven. diff --git a/docs/decision-checklist.md b/docs/decision-checklist.md new file mode 100644 index 0000000..63e42f9 --- /dev/null +++ b/docs/decision-checklist.md @@ -0,0 +1,101 @@ +# Decision Checklist (Gating Artifact) + +This file records decisions that must be resolved before moving forward with +backend integration. It is the gating artifact for Phase 0/1. + +--- + +## Phase 0 — Conformance Harness + Determinism + +### Conformance Scope + +* Tier 0 MUST-pass specs (backend/store focus): +* `ASL/1-CORE` +* `ASL/1-STORE` +* `ENC/ASL1-CORE` +* `HASH/ASL1` +* `TGK/1-CORE` +* `ENC/TGK1-EDGE/1` +* `TGK/STORE/1` +* Tier 1 MUST-pass specs (index/log + optional provenance): +* `ASL/LOG/1` +* `ENC/ASL-LOG/1` +* `ASL/1-CORE-INDEX` +* `ENC/ASL-CORE-INDEX/1` +* `ASL/STORE-INDEX/1` +* `ASL/INDEX-ACCEL/1` +* `ASL/FEDERATION/1` and `ASL/FEDERATION-REPLAY/1` (if federation is in scope) +* `TGK/PROV/1` (if provenance operators are surfaced) + +Notes: +* The legacy filesystem store (`asl_store_fs`) is not a Tier 1 conformance target once index/log is used. +* PEL specs are implemented and tested in this repo, but they are not required for the Phase 0/1 backend gate unless the surface is shipped as part of the backend. + +### TGK/STORE Equivalence Definition + +* Exact definition of “same result” across backends: +* For a fixed `GraphStoreConfig` and snapshot where `(Artifacts, config.tgk_profiles)` are equal, all `TGK/STORE/1` operations must return identical results per `TGK/STORE-DET/1`. +* `resolve_edge(ref)` returns the same `EdgeBody` or the same error code. +* `edges_from`, `edges_to`, `edges_incident` return identical `GraphEdgeView` lists (same elements in the same order). +* If implemented, `neighbors` returns identical `Node` lists (same elements in the same order). +* If implemented, `scan_edges` enumerates the same edges with identical per-page ordering; `PageToken` bytes may differ but must represent the same position in the canonical ordering. +* Ordering requirements (explicitly enumerated): +* Edge ordering is canonical and global: ascending lexicographic order of `order_key(edge_ref) = hash_id_bytes || digest_bytes`, with `hash_id` as big-endian `u16` (`TGK/STORE-EDGE-ORDER/1`). +* Neighbor ordering is canonical: ascending lexicographic order of `(hash_id, digest)` with the same `u16` big-endian encoding (`TGK/STORE-NEIGHBORS/1`). +* Pagination/scan stability rules: +* `scan_edges` must enumerate each matching edge exactly once and preserve the canonical per-page ordering. +* Snapshot stability across pages is optional; if not stable, the implementation must still preserve per-call ordering and enumeration semantics for the snapshot used by that call. + +### Error Model Mapping Policy + +* Canonical internal error taxonomy: +* ASL store errors: `AMDUAT_ASL_STORE_ERR_INTEGRITY`, `AMDUAT_ASL_STORE_ERR_UNSUPPORTED`, `AMDUAT_ASL_STORE_ERR_NOT_FOUND`, `AMDUAT_ASL_STORE_ERR_IO`. +* TGK store errors: `GS_ERR_NOT_EDGE`, `GS_ERR_ARTIFACT_ERROR`, `GS_ERR_UNSUPPORTED`, `GS_ERR_INTEGRITY`. +* Adapter-specific error mapping rules: +* `ASL/1-STORE` specifies `ERR_NOT_FOUND`, `ERR_INTEGRITY`, `ERR_UNSUPPORTED`. I/O/auth failures are out of scope for the spec; in this repo they map to `AMDUAT_ASL_STORE_ERR_IO`. +* `TGK/STORE/1` requires artifact-layer conflicts during `resolve_edge` to surface as `GS_ERR_ARTIFACT_ERROR` (not `GS_ERR_INTEGRITY`). +* `GS_ERR_NOT_EDGE` is returned when a reference resolves to an Artifact that is not a TGK edge under the active profile set. +* Required error tests: +* ASL: `get` on missing ref → `ERR_NOT_FOUND`. +* ASL: `put` collision (same ref, different artifact) → `ERR_INTEGRITY`. +* ASL: unsupported config or hash/profile mismatch → `ERR_UNSUPPORTED`. +* ASL: adapter I/O faults → `ERR_IO` (non-normative but required for harness). +* TGK: `resolve_edge` non-edge artifact → `GS_ERR_NOT_EDGE`. +* TGK: `resolve_edge` artifact-layer integrity failure → `GS_ERR_ARTIFACT_ERROR`. +* TGK: unsupported profile/encoding → `GS_ERR_UNSUPPORTED`. + +--- + +## Phase 1 — ASL MinIO Adapter + +### Object Key Layout + +* Reference → bucket/key mapping: TBD (not specified by current specs/tests). +* Prefix sharding / directory fanout: TBD (not specified by current specs/tests). + +### Metadata Policy + +* Encoding/profile version tracking: TBD (not specified by current specs/tests). +* Metadata location (object metadata/tags/in-band): TBD (not specified by current specs/tests). + +### Multipart Policy + +* Multipart threshold: TBD (not specified by current specs/tests). +* Fidelity guarantees and validation: TBD (not specified by current specs/tests). + +### Consistency Assumptions + +* Read-after-write expectations: TBD (not specified by current specs/tests). +* Deployment modes covered by tests: TBD (not specified by current specs/tests). + +### Security / Credentials + +* Credential source and rotation policy: TBD (not specified by current specs/tests). +* Permission boundaries: TBD (not specified by current specs/tests). + +--- + +## Status + +* Phase 0 decisions: DRAFTED (spec-derived; requires owner confirmation) +* Phase 1 decisions: NOT STARTED diff --git a/docs/es-indexing-plan.md b/docs/es-indexing-plan.md new file mode 100644 index 0000000..692aa9d --- /dev/null +++ b/docs/es-indexing-plan.md @@ -0,0 +1,120 @@ +# Elasticsearch Indexing Plan + +Status: Draft +Scope: Minimal phased ES deployment aligned with indexing contract. + +--- + +# Phase A — Artifact Metadata Projection (Minimal) + +## Goal + +Accelerate search and metadata filtering only. + +## Index Name + +`amduat_artifacts_v1` + +## Document Structure + +1. `_id`: canonical `ReferenceBytes` hex. +2. `reference`: canonical hex. +3. `projection_schema_version`: integer. +4. `kind`: keyword. +5. `enc_profile`: keyword. +6. `labels`: keyword array. +7. `created_at`: date, only if intrinsic to artifact. +8. `text`: text, optional, deterministic extractor. +9. `extractor_version`: keyword, if `text` present. + +## Data Source + +1. ASL authoritative log/index. +2. Deterministic decoding only. + +## Query Patterns Accelerated + +1. Text search. +2. Filter by kind. +3. Filter by labels. +4. Narrow candidate set. + +## Verification Rule + +All returned References MUST be confirmed via ASL/1-STORE before returning artifact data. + +--- + +# Phase B — Edge Projection (Optional) + +## Goal + +Accelerate candidate neighbor discovery. + +## Index Name + +`amduat_edges_v1` + +## Document Structure + +1. `_id`: edge Reference hex. +2. `edge_reference`: canonical hex. +3. `src_ref`: canonical hex. +4. `dst_ref`: canonical hex. +5. `edge_label`: keyword. +6. `hash_id`: keyword or numeric. +7. `digest`: keyword. +8. `projection_schema_version`: integer. + +## Data Source + +1. EdgeArtifacts only. +2. TGK profile rules. +3. No inferred edges. + +## Verification Rule + +Returned edges MUST be validated via TGK/STORE/1 authoritative query, and canonical ordering MUST be applied before client response. + +--- + +# Phase C — Enterprise Projection Hardening + +## Additions + +1. Projection checkpoint artifacts stored in ASL. +2. Schema version recorded in checkpoint. +3. Full rebuild test automation. +4. Lag metrics published. +5. Projection audit job comparing snapshot vs ES. + +--- + +# Operational Guarantees + +1. ES may be disabled without breaking semantics. +2. Projection rebuild must be deterministic. +3. Schema changes require replay. +4. Checkpoint artifact defines replay boundary. + +--- + +# Observability Requirements + +Expose: +1. ASL head log position. +2. Indexer checkpoint position. +3. Computed lag. +4. Projection error counts. +5. Rebuild duration. +6. Verification fallback rate. + +--- + +# Final Guardrail + +Elasticsearch accelerates discovery. + +Authoritative semantics remain in ASL/1-STORE and TGK/STORE/1. + +Ordering, identity, and correctness are never delegated to ES. diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md new file mode 100644 index 0000000..a590205 --- /dev/null +++ b/docs/implementation-plan.md @@ -0,0 +1,460 @@ +# Amduat Open-Source Backend + +# Implementation Plan Proposal + +**Status:** Architecture & Implementation Baseline v1 +**Purpose:** Define phased implementation of the OSS backend stack as a spec-compliant implementation of Amduat store/query surfaces. + +--- + +# 1. Goal and Scope + +## 1.1 Purpose + +This plan defines how to implement the open-source stack (MinIO, JanusGraph, Cassandra, Elasticsearch, Spark, Nessie, IPFS, Ollama) as a backend implementation of: + +* `ASL/1-STORE` +* `TGK/STORE/1` +* Optional `TGK/PROV/1` + +The plan ensures: + +* Strict semantic conformance +* Deterministic reproducibility +* Backend substitutability +* Observational equivalence + +## 1.2 Out of Scope + +* Modifying core specifications +* Introducing new kernel ops +* Elevating projections to semantic authority +* Redefining identity or encoding rules + +## 1.3 Target Tiers + +This plan covers: + +* Tier 0 — Embedded / Edge +* Tier 1 — Platform / Scale-Out +* Tier 2 — Enterprise / Audit-Grade + +--- + +# 2. Semantic Invariants + +All phases must preserve: + +## 2.1 No Semantic Drift Rule + +* No backend-specific interpretation of artifacts +* No implicit schema expansion +* No hidden type coercion +* No projection elevation without governance +* All optimizations must be observationally equivalent to spec-defined behavior + +## 2.2 Non-Negotiable Properties + +* Canonical encoding (`ENC/ASL1-CORE`) +* Deterministic identity (`ASL/1-CORE` + `HASH/ASL1`) +* Deterministic TGK projection (`TGK/1-CORE`) +* Store/query surface conformance +* Replay equivalence + +--- + +# 3. Decision Checklist (Must Resolve Upfront) + +These decisions must be explicitly documented before Phase 1 begins. + +--- + +## 3.1 Conformance Tests + +* Exact test suites for: + + * `ASL/1-STORE` + * `TGK/STORE/1` +* Deterministic ordering checks +* Replay equivalence tests +* Acceptance criteria per tier + +--- + +## 3.2 Backend Mapping + +### ASL Mapping + +* Artifact storage layout in MinIO +* Namespace strategy +* Object key derivation rules +* Versioning policy (if enabled) + +### TGK Mapping + +* Vertex/edge mapping strategy +* Deterministic ordering enforcement +* Cassandra schema design +* Index strategy in Elasticsearch (if used) + +--- + +## 3.3 Consistency Model + +Per tier: + +* Write guarantees +* Read-after-write expectations +* Projection lag tolerance +* Eventual consistency boundaries +* Snapshot isolation model + +--- + +## 3.4 Snapshot & Export Strategy + +* Snapshot format (artifact set, graph export, columnar dataset) +* Snapshot identity model +* Export cadence (manual, periodic, event-triggered) +* Replay and equivalence validation rules + +--- + +## 3.5 Governance & Projection Promotion + +* Rules for elevating a projection to authoritative +* Audit requirements +* Approval workflow + +--- + +## 3.6 Failure & Recovery + +* Partial-write handling +* Corruption detection +* Divergence reconciliation +* Reprojection strategy +* Disaster recovery model + +--- + +## 3.7 Pluggable Ops Registry + +* Storage location +* Versioning strategy +* Discovery mechanism +* Trust policy + +--- + +## 3.8 Execution Artifact Schemas + +Define required fields for: + +* ExecutionIntent +* ExecutionReceipt +* ExecutionOutput +* ExecutionObservation + +Define canonical encoding and validation rules. + +--- + +## 3.9 Security & Isolation + +* Executor trust boundaries +* Secret management +* Artifact integrity validation +* Network isolation policies + +--- + +## 3.10 Performance Targets + +Define SLOs per tier: + +* Ingest throughput +* Query latency +* Snapshot generation time +* Recovery time objectives + +--- + +# 4. Architecture Mapping + +--- + +## 4.1 ASL/1-STORE Implementation + +### Responsibilities + +* Canonical encoding validation +* Identity derivation +* Artifact immutability +* Retrieval guarantees + +### Concrete Mapping + +* MinIO object store +* Object key = Reference +* Metadata includes encoding profile version +* Optional replication policy + +--- + +## 4.2 TGK/STORE/1 Implementation + +### Responsibilities + +* Deterministic graph projection +* Required query operations +* Error model conformance +* Deterministic ordering guarantees + +### Concrete Mapping + +* JanusGraph for query layer +* Cassandra for durable storage +* Explicit ordering enforcement in query results +* Elasticsearch as optional derived projection + +--- + +## 4.3 Optional Projections + +### Elasticsearch + +* Full-text search +* Attribute filtering + +### IPFS + +* Artifact distribution and caching + +### Lakehouse (MinIO + Nessie + Spark) + +* Snapshot export +* Batch provenance analysis +* Audit workflows + +All remain derived projections. + +--- + +## 4.4 Provenance Strategy + +`TGK/PROV/1` may be realized via: + +* Native graph traversal +* Snapshot-based analytics +* Hybrid strategy + +Must remain observationally equivalent. + +--- + +# 5. Interfaces and Contracts + +--- + +## 5.1 ASL Interface + +* `put(Artifact) -> Reference` +* `get(Reference) -> Artifact` + +Error model strictly per spec. + +--- + +## 5.2 TGK Interface + +* Required query operations per `TGK/STORE/1` +* Deterministic ordering rules +* Explicit error semantics + +--- + +## 5.3 Serialization + +* Canonical encoding at artifact boundary +* No alternative encoding permitted +* Versioned encoding profile tracking + +--- + +# 6. Execution Model (Pluggable Ops) + +--- + +## 6.1 Registry Structure + +* External registry +* Versioned definitions +* Immutable op identifiers +* Policy-scoped availability + +--- + +## 6.2 Executor Lifecycle + +* Intent creation +* Dispatch +* Receipt capture +* Output capture +* Optional observation + +--- + +## 6.3 Artifact Workflow + +1. Write ExecutionIntent +2. Executor produces ExecutionReceipt +3. Executor produces ExecutionOutput +4. Optional ExecutionObservation +5. Link artifacts via EdgeArtifacts + +All artifacts stored in ASL. + +--- + +## 6.4 TGK Linking + +* Intent → Receipt +* Receipt → Output +* Output → Observation + +Graph projection ensures traceability. + +--- + +# 7. Implementation Phases + +--- + +## Phase 0 — Conformance Baseline + +* Build ASL/TGK conformance harness +* Define canonical test datasets +* Establish replay checks + +--- + +## Phase 1 — ASL Store + +* MinIO integration +* Canonical encoding enforcement +* Hash-based identity derivation +* Conformance validation + +--- + +## Phase 2 — TGK Projection + +* Graph projection pipeline +* JanusGraph integration +* Deterministic ordering enforcement +* Conformance validation + +--- + +## Phase 3 — Projections & Indexing + +* Elasticsearch integration +* IPFS optional layer +* Monitoring of projection lag + +--- + +## Phase 4 — Snapshot & Governance + +* Snapshot export format +* Nessie integration +* Replay equivalence testing +* Governance workflows + +--- + +## Phase 5 — Pluggable Ops & Executors + +* Registry implementation +* Execution artifact schemas +* Executor runtime +* Traceability validation + +--- + +# 8. Testing and Verification + +Each phase must include: + +* Deterministic replay checks +* Ordering validation tests +* Snapshot equivalence tests +* Cross-backend equivalence tests + +No phase advances without passing conformance. + +--- + +# 9. Operational Concerns + +--- + +## 9.1 Deployment per Tier + +Tier 0: + +* Single-node embedded + +Tier 1: + +* Janus + Cassandra + ES +* MinIO +* Optional IPFS + +Tier 2: + +* Add Spark +* Add Nessie +* Governance workflows + +--- + +## 9.2 Observability + +* Artifact write metrics +* Graph projection metrics +* Projection lag monitoring +* Snapshot health indicators + +--- + +## 9.3 Backup & Restore + +* MinIO replication or erasure coding +* Cassandra repair strategy +* Snapshot archival policy + +--- + +# 10. Open Questions and Risks + +* Canonical test dataset size +* Cassandra compaction strategy +* Projection drift detection +* Snapshot storage growth +* Executor trust hardening + +Each risk must have a mitigation strategy before Tier 2 rollout. + +--- + +# Design Outcome + +This implementation plan: + +* Preserves core invariants +* Enables infrastructure scaling +* Prevents semantic leakage +* Structures phased, verifiable progress +* Makes the OSS stack a compliant backend implementation diff --git a/docs/indexing-requirements-and-projections.md b/docs/indexing-requirements-and-projections.md new file mode 100644 index 0000000..64ac310 --- /dev/null +++ b/docs/indexing-requirements-and-projections.md @@ -0,0 +1,145 @@ +# Indexing Requirements and Projections + +Status: Draft +Scope: Define indexing contract, authority boundaries, and projection invariants for derived search/index systems (e.g., Elasticsearch). + +--- + +# API Usage Rule (Normative) + +1. Elasticsearch returns candidate References only. +2. Authoritative confirmation MUST occur via ASL/1-STORE and/or TGK/STORE/1. +3. Any ordering visible to clients MUST be computed using the canonical `(hash_id, digest)` ordering defined in TGK/STORE/1. +4. Elasticsearch MUST NOT be treated as a source of semantic truth. + +--- + +# 1. Decision Checklist (Must Resolve Before Implementation) + +1. Which artifact types are searchable. +2. Which fields are eligible for text extraction. +3. Whether `created_at` is intrinsic or ingestion-derived. +4. Checkpoint source of truth (ASL log position required). +5. Verification policy (which APIs require authoritative confirmation). +6. Edge indexing included in Phase A or deferred. +7. Redaction policy per TypeTag. +8. Projection schema versioning strategy. + +No indexing implementation proceeds without resolving these. + +--- + +# 2. Authority Boundary + +## 2.1 Authoritative Sources + +1. Artifact identity and content: ASL/1-STORE. +2. Graph semantics and ordering: TGK/STORE/1. +3. Canonical encoding: ENC/ASL1-CORE. +4. Identity derivation: ASL/1-CORE and HASH/ASL1. + +Elasticsearch is a derived projection only. + +## 2.2 No Semantic Drift Rule + +1. No backend-specific interpretation of artifacts. +2. No inferred edges. +3. No implicit ordering. +4. No hidden type coercion. +5. All optimizations must be observationally equivalent to authoritative results. + +--- + +# 3. Reference Encoding Rule + +All `reference` fields indexed in ES MUST use canonical `ReferenceBytes` hex encoding as defined by ENC/ASL1-CORE. + +This guarantees: +1. Unambiguous equality. +2. Stable identity across backends. +3. No ambiguity between `(hash_id, digest)` representations. + +No alternative encodings are permitted. + +--- + +# 4. Ordering Rule + +Elasticsearch ordering MUST NOT be treated as authoritative. + +Any client-visible ordering MUST be computed via the canonical `(hash_id, digest)` ordering defined in TGK/STORE/1. + +If ES provides candidate edges, authoritative logic must re-order them. + +--- + +# 5. Projection Contract + +## 5.1 Replayability + +Index state MUST be fully replayable from ASL log/index state, EdgeArtifacts, and deterministic decoding rules. + +## 5.2 Observational Equivalence + +Given identical ASL state, ES projection rebuilt from scratch MUST produce identical documents within a schema version. + +## 5.3 Schema Versioning + +Each projection index MUST have: +1. An explicit schema version, e.g., `projection_schema_version`. +2. The version recorded in a checkpoint artifact. +3. A versioned rebuild strategy. + +Schema changes require full replay. + +--- + +# 6. Lag Metric Definition + +Lag is formally defined as: + +``` +Lag = ASL_head_log_position - indexer_checkpoint_log_position +``` + +Where: +1. `ASL_head_log_position` is authoritative. +2. `indexer_checkpoint_log_position` is recorded in a projection checkpoint artifact. + +Lag MUST be observable via metrics. + +--- + +# 7. Decoding and Extraction Rules + +1. Artifact decoding MUST use only the declared encoding profile. +2. No heuristic decoding permitted. +3. Text extraction MUST be deterministic and versioned. +4. Extractor version MUST be stored in index documents. + +--- + +# 8. Edge Indexing Guardrail + +If edges are indexed: +1. Edge documents MUST be derived only from EdgeArtifacts. +2. No inferred adjacency allowed. +3. Edge semantics MUST respect TGK profile rules. +4. Ordering MUST be applied by the authoritative layer. + +--- + +# 9. Security and Redaction + +1. Per-TypeTag redaction policy is mandatory. +2. No secrets indexed unless explicitly approved. +3. Redaction rules MUST be declared before indexing begins. + +--- + +# 10. Conformance Gates + +1. System must function with ES disabled. +2. Full rebuild equivalence test required. +3. Ordering correctness test required. +4. Cross-backend equivalence test required. diff --git a/docs/oss-backend-adapter-boundary.md b/docs/oss-backend-adapter-boundary.md new file mode 100644 index 0000000..d85bda3 --- /dev/null +++ b/docs/oss-backend-adapter-boundary.md @@ -0,0 +1,274 @@ +# OSS Backend Adapter Boundary + +Status: Draft +Purpose: Define architectural boundaries, responsibilities, and conformance requirements for implementing the OSS backend stack (MinIO, JanusGraph, Cassandra, Elasticsearch, etc.) as spec-compliant adapters. + +--- + +# 1. Architectural Position + +The OSS backend is an implementation of store/query surfaces, not a semantic extension. + +It implements: +1. ASL/1-STORE. +2. TGK/STORE/1. +3. Optional: TGK/PROV/1. +4. Optional: derived projections (ES, lakehouse). + +It does not redefine: +1. Core semantics (ASL/1-CORE, PEL/1-CORE, TGK/1-CORE). +2. Canonical encoding (ENC/ASL1-CORE). +3. Identity derivation (ASL/1-CORE and HASH/ASL1). + +--- + +# 2. Authority and Responsibility Model + +## 2.1 Authoritative Components + +Authoritative state resides in: +1. Deployed ASL/1-STORE implementation. +2. Deterministic TGK projection derived from ASL state. + +Adapters must preserve this. + +## 2.2 Derived Components + +The following are derived projections: +1. Elasticsearch. +2. IPFS. +3. Lakehouse snapshots. +4. AI outputs. + +Derived components must: +1. Be replayable. +2. Be observationally equivalent. +3. Never redefine semantics. + +--- + +# 3. Adapter Modules + +This document defines four adapter classes. + +## 3.1 ASL Store Adapter (MinIO or S3) + +### Purpose + +Implements ASL/1-STORE. + +### Responsibilities + +1. Accept `put(Artifact) -> Reference`. +2. Enforce canonical encoding validation. +3. Persist artifact bytes under canonical identity. +4. Implement `get(Reference) -> Artifact | NotFound`. +5. Map backend errors to canonical error model. + +### Non-Responsibilities + +1. Computing identity (handled by core logic). +2. Interpreting artifact contents. +3. Modifying encoding. +4. Adding metadata not declared by spec. + +### Determinism Requirements + +1. Same Artifact → same Reference. +2. Same Reference → identical bytes returned. +3. No backend-specific mutation. + +### Configuration Surface + +1. Endpoint. +2. Credentials. +3. Bucket name. +4. Retry policy. +5. Multipart threshold. +6. Timeout policy. + +--- + +## 3.2 TGK Store Adapter (JanusGraph plus Cassandra) + +### Purpose + +Implements TGK/STORE/1. + +### Responsibilities + +1. Provide required query operations defined in TGK/STORE/1. +2. Enforce canonical `(hash_id, digest)` ordering. +3. Maintain deterministic projection behavior. +4. Map errors to spec-defined error model. + +### Non-Responsibilities + +1. Creating semantic edges (done via EdgeArtifacts). +2. Redefining graph semantics. +3. Implicitly inferring relationships. +4. Providing authoritative ordering outside spec. + +### Determinism Requirements + +1. Same ASL state → same TGK answers. +2. Same queries → same ordered results. +3. Ordering must not depend on Janus internal traversal order. + +### Ordering Enforcement Point + +Ordering MUST be applied: +1. Explicitly in the adapter layer, or +2. In a canonical result transformation layer. + +Never rely on backend default sort semantics. + +--- + +## 3.3 Projection Adapter (Elasticsearch) + +### Purpose + +Accelerate search and filtering. + +### Responsibilities + +1. Project canonical artifact metadata. +2. Optionally project EdgeArtifacts. +3. Maintain replayable checkpoint. +4. Expose lag metric. +5. Enforce schema versioning. + +### Non-Responsibilities + +1. Authoritative artifact reads. +2. Authoritative graph queries. +3. Deterministic ordering. + +### Verification Requirement + +Any ES-derived candidate results must be: +1. Confirmed via ASL/1-STORE. +2. Ordered via TGK/STORE/1 logic before returning to clients. + +--- + +## 3.4 Snapshot or Lakehouse Adapter (Future) + +### Purpose + +Batch export and audit workflows. + +### Responsibilities + +1. Snapshot export from authoritative state. +2. Versioned dataset generation. +3. Replay equivalence validation. + +### Non-Responsibilities + +1. Live semantic authority. +2. Mutation of ASL state. + +--- + +# 4. Adapter Error Taxonomy + +All adapters must map backend-specific errors into a canonical internal taxonomy: +1. NotFound. +2. InvalidArtifact. +3. PermissionDenied. +4. TransientIO. +5. InternalError. + +No backend-specific error types may leak across the adapter boundary. + +--- + +# 5. Conformance Gates + +No adapter may be merged or promoted unless: + +## 5.1 ASL Adapter + +1. Passes ASL/1-STORE conformance suite. +2. Passes replay equivalence test. +3. Passes cross-backend equivalence test. + +## 5.2 TGK Adapter + +1. Passes TGK/STORE/1 required query suite. +2. Passes deterministic ordering test. +3. Passes cross-backend equivalence test. + +## 5.3 Projection Adapter + +1. System passes integration tests with projection disabled. +2. Rebuild equivalence test passes. +3. Lag metric exposed and validated. + +--- + +# 6. Observability Contract + +All adapters must expose: +1. Health endpoint. +2. Error metrics. +3. Operation latency metrics. +4. Determinism assertion failures. +5. Projection lag, if applicable. + +--- + +# 7. Security Boundary + +Adapters must: +1. Not embed secrets in artifacts. +2. Respect per-TypeTag redaction policies before indexing. +3. Enforce explicit credential configuration. +4. Avoid implicit trust of backend defaults. + +--- + +# 8. Interaction with Conformance Harness + +Adapter design is subordinate to the Phase 0 harness. + +Implementation order: +1. Conformance harness operational. +2. Adapter scaffold. +3. Adapter passes harness. +4. Integration and load testing. + +No adapter bypasses this sequence. + +--- + +# 9. Design Outcome + +This boundary ensures: +1. Thin adapters. +2. Zero semantic drift. +3. Deterministic reproducibility. +4. Backend substitutability. +5. Operational scalability without semantic mutation. + +The OSS backend becomes: + +An infrastructure implementation of store/query surfaces, not an evolution of the semantic core. + +--- + +# Where You Are Now (Strategic Position) + +You now have: +1. Semantic core locked. +2. Execution boundary defined. +3. Indexing contract defined. +4. Backend boundary defined. +5. Implementation phases defined. + +You are ready to: +1. Design adapter scaffolds. +2. Build Phase 0 conformance harness. +3. Begin MinIO adapter implementation safely. diff --git a/docs/phase0-1-execution-checklist.md b/docs/phase0-1-execution-checklist.md new file mode 100644 index 0000000..7e7a5fb --- /dev/null +++ b/docs/phase0-1-execution-checklist.md @@ -0,0 +1,160 @@ +# Phase 0/1 Execution Checklist + +## OSS Backend Enablement (MinIO-first) + +### 0. Purpose + +This checklist gates work so we do not start infrastructure integration until we can prove: + +* spec conformance holds across multiple backends, +* deterministic ordering/replay invariants are testable, +* error-model mappings are pinned down. + +--- + +## Phase 0 — Conformance Harness + Determinism Gates + +### 0.1 Deliverables + +1. **Multi-backend conformance runner** (same test vectors, multiple backends) +2. **Golden dataset generator** for ASL + TGK fixtures +3. **Deterministic equivalence suite**: + +* ordering checks +* replay equivalence +* cross-backend observational equivalence (surface behavior) + +### 0.2 Required decisions (turn into `docs/decision-checklist.md`) + +* **Conformance test scope**: which spec assertions are MUST-pass for Tier 0 vs Tier 1 +* **Equivalence definition**: what counts as “same result” for TGK/STORE operations (including ordering) +* **Error model mapping policy**: canonical internal error types vs adapter-specific errors + +### 0.3 Conformance harness: what to build + +**A) ASL/1-STORE conformance suite** + +* Put/get round-trip for a set of canonical Artifacts +* Idempotent `put(Artifact)` returns same `Reference` +* `get(Reference)` exact artifact fidelity +* NotFound behavior and error codes +* Corruption detection behavior (if defined/expected by your layers) +* Optional: latency-independent invariants (no timing assumptions) + +**B) TGK/STORE/1 conformance suite** + +* Required operations per spec and error model +* Deterministic ordering assertions (as spec defines) +* “Same ASL state → same TGK answers” invariants +* Cross-backend equivalence: mem vs fs vs asl_index_fs must match for the same fixture set + +**C) Replay determinism** + +* Build a fixture log/index state +* Replay into a fresh store +* Assert equivalent outcomes (references returned, query results, ordering) + +### 0.4 Acceptance criteria (Phase 0 gate) + +* ✅ Same fixture set passes on at least **two existing backends**: + +* `asl_store_fs` and `asl_store_index_fs` +* `tgk_store_mem` and `tgk_store_fs` (or `tgk_store_asl_index_fs`) + +* ✅ Deterministic ordering checks are explicit and enforced +* ✅ A “backend contract test” can be run by CI with a single command + +### 0.5 Where this touches the repo (suggested structure) + +* `tests/conformance/asl_store/*.c` +* `tests/conformance/tgk_store/*.c` +* `tests/fixtures/*.bin|*.json|*.asl` (whatever your canonical fixture format is) +* `tools/amduat_conformance_run` (small runner, like your other tools) + +--- + +## Phase 1 — ASL MinIO Adapter (Strict ASL/1-STORE) + +### 1.1 Deliverables + +1. `src/adapters/asl_store_s3_minio/asl_store_s3_minio.c` (name as you prefer) +2. Config + credentials plumbing for MinIO endpoint +3. Full Phase 0 conformance suite passing against MinIO +4. CI job (optional initially) that spins MinIO and runs conformance tests + +### 1.2 Required decisions (Phase 1-specific) + +* **Object key layout**: how `Reference` maps to bucket/key (e.g., prefix sharding) +* **Metadata policy**: do we store encoding/profile version as object metadata, tags, or in-band? +* **Multipart handling**: threshold + exact fidelity guarantees +* **Read-after-write expectation**: what you assume from MinIO deployment mode; what you test for + +### 1.3 Adapter contract requirements (what must not change) + +* `put(Artifact) -> Reference` computed by core rules; adapter must not accept a caller-provided reference +* `get(Reference) -> Artifact | NotFound` must return exact bytes for canonical decode +* Error mapping must be stable (auth errors, NotFound, transient IO) + +### 1.4 Failure modes to explicitly test (Phase 1 gate) + +* Bucket missing / permission denied +* NotFound for unknown reference +* Partial upload interruption (must not produce a “phantom success”) +* Corrupt object payload (if detectable at decode step, ensure it surfaces as decode error) + +### 1.5 Acceptance criteria (Phase 1 gate) + +* ✅ ASL/1-STORE conformance suite passes on MinIO +* ✅ Cross-backend equivalence: MinIO results identical to index/fs for the same fixture set +* ✅ Determinism: repeated `put` gives same reference, independent of environment + +--- + +# Missing interfaces and tests to write (concrete list) + +## A) Missing test harness capability (Phase 0) + +* A single abstraction that can run the same tests against: + +* `asl_store_fs` +* `asl_store_index_fs` +* future `asl_store_minio` + +* Same for TGK store adapters +* A “golden fixture set” format (even if it’s just embedded byte arrays initially) + +## B) Deterministic ordering assertions (Phase 0) + +The repo has lots of tests, but not: + +* a named suite that asserts **ordering** exactly as TGK/STORE defines +* cross-backend “same query, same ordering” checks + +## C) Cross-backend equivalence harness (Phase 0) + +You need an explicit mechanism that: + +* seeds the same ASL artifact set into multiple ASL backends +* builds the same TGK projection inputs +* runs the same TGK queries and diffs results + +## D) Adapter error normalization (Phase 0/1) + +Define a small internal error taxonomy used by conformance tests so adapters can map: + +* NotFound +* InvalidArtifact / decode error +* PermissionDenied +* TransientNetwork +* Internal/Unknown + +Without this, MinIO will “pass” but be impossible to reason about operationally. + +--- + +# What I would do next (lowest-risk sequence) + +1. **Add `docs/decision-checklist.md`** and fill only Phase 0/1 decisions (don’t boil the ocean). +2. Implement **conformance runner** that can already test FS vs index/log backends (prove the harness works). +3. Add deterministic ordering + replay equivalence tests. +4. Only then implement `asl_store_minio`, using the harness as your guardrail. diff --git a/docs/primitive-vs-pluggable-ops.md b/docs/primitive-vs-pluggable-ops.md new file mode 100644 index 0000000..3b94d5d --- /dev/null +++ b/docs/primitive-vs-pluggable-ops.md @@ -0,0 +1,253 @@ +# Primitive Ops vs Pluggable Ops + +This section defines the execution boundary between deterministic kernel semantics and extensible domain execution. + +It preserves: + +* Kernel determinism +* Store-neutral traceability +* Backend substitutability +* Observational equivalence + +--- + +# 1. Terminology + +## 1.1 Kernel / Primitive Op + +A **Kernel Op** (Primitive Op) is an operation defined in: + +`OPREG/PEL1-KERNEL` + +It is part of the `PEL/1-CORE` contract. + +Kernel ops define fixed, deterministic semantics and are considered part of Amduat’s invariant semantic core. + +--- + +## 1.2 Pluggable Op + +A **Pluggable Op** is an operation defined outside `OPREG/PEL1-KERNEL`. + +It is not part of `PEL/1-CORE`. + +Its execution semantics: + +* Are versioned independently +* May be effectful +* Are governed by profile or deployment policy +* Must emit execution artifacts for traceability + +Pluggable ops extend behavior without extending the kernel. + +--- + +# 2. Kernel / Primitive Ops + +## 2.1 Properties + +Kernel ops: + +* Are defined in `OPREG/PEL1-KERNEL` +* Are pure and deterministic +* Have no side effects +* Depend only on explicit inputs +* Produce canonical artifacts +* Have stable semantics governed by spec revision + +Kernel ops are portable across all backend tiers. + +They cannot: + +* Access external systems +* Read hidden state +* Depend on environmental configuration +* Produce nondeterministic outputs + +--- + +## 2.2 Examples + +* `bytes.concat` +* `bytes.slice` +* `hash.asl1_256` + +These operations: + +* Operate solely on canonical encoded inputs +* Produce deterministic outputs +* Are observationally equivalent across implementations + +--- + +# 3. Pluggable Ops + +Pluggable ops enable domain-specific execution without extending the kernel. + +They may: + +* Invoke LLMs +* Execute SSH commands +* Configure infrastructure +* Perform device operations +* Call external APIs + +They are not part of the core semantic contract. + +--- + +## 3.1 Properties + +Pluggable ops: + +* Are defined in external registries +* May be effectful +* May interact with external systems +* May be nondeterministic +* Must produce execution artifacts +* Must remain observationally separable from kernel semantics + +They extend behavior while preserving traceability. + +--- + +# 4. Execution Artifact Model + +Every pluggable op MUST emit execution artifacts to ASL. + +These artifacts preserve determinism at the semantic layer, even if the effect itself is nondeterministic. + +## 4.1 Required Artifacts + +At minimum: + +### 1. ExecutionIntent + +Declares: + +* Target +* Parameters +* Configuration +* Version of pluggable op + +Represents the desired action. + +--- + +### 2. ExecutionReceipt + +Confirms: + +* Action dispatch +* Acceptance by executor +* Execution attempt metadata + +Represents the attempt. + +--- + +### 3. ExecutionOutput + +Captures: + +* Stdout/stderr +* Returned data +* Produced artifacts +* Exit codes + +Represents the immediate result. + +--- + +### 4. ExecutionObservation (Recommended) + +Captures: + +* Observed external state +* Post-condition checks +* Environment validation + +Represents the verified outcome. + +--- + +All execution artifacts: + +* Are canonical ASL artifacts +* Are immutable +* Derive identity via `ENC/ASL1-CORE` + `ASL/1-CORE` + `HASH/ASL1` +* Participate in TGK projection + +--- + +# 5. Traceability Requirements + +All pluggable ops MUST be traceable in TGK by linking: + +* `ExecutionIntent → ExecutionReceipt` +* `ExecutionReceipt → ExecutionOutput` +* Optional: `ExecutionOutput → ExecutionObservation` + +These links are created via EdgeArtifacts written to ASL. + +This ensures: + +* Deterministic graph projection +* Provenance recoverability +* Replay safety +* Auditability + +Even if external execution is nondeterministic, the trace is deterministic. + +--- + +# 6. Governance Boundary + +The boundary between kernel and pluggable ops is strict. + +## 6.1 Kernel + +* Small +* Deterministic +* Slow-moving +* Versioned with PEL core +* Subject to formal governance + +## 6.2 Pluggable + +* Extensible +* Fast-moving +* Policy-scoped +* Profile-scoped +* May be deployment-specific + +No pluggable op may redefine kernel semantics. + +--- + +# 7. Interaction with Backend Plug-in Model + +This execution model aligns with the No Semantic Drift Rule: + +* Kernel ops are invariant across all backends. +* Pluggable ops must emit artifacts. +* Backends must treat execution artifacts as ordinary ASL artifacts. +* TGK projection must remain deterministic. + +Backend-specific optimizations must remain observationally equivalent to the artifact graph defined by execution artifacts. + +--- + +# 8. Design Outcome + +This separation: + +* Preserves deterministic core semantics +* Enables real-world execution +* Prevents semantic leakage into infrastructure +* Maintains backend substitutability +* Ensures enterprise-grade auditability +* Supports edge and embedded deployments + +Primitive ops define what Amduat *is*. +Pluggable ops define what Amduat *does*. diff --git a/docs/projection-checkpoint-schema.md b/docs/projection-checkpoint-schema.md new file mode 100644 index 0000000..44cc8c6 --- /dev/null +++ b/docs/projection-checkpoint-schema.md @@ -0,0 +1,81 @@ +# Projection Checkpoint Schema + +Status: Draft +Scope: Schema for projection checkpoint artifacts used by derived indexers (e.g., Elasticsearch) to bind schema versions, replay state, and lag metrics to authoritative ASL state. + +--- + +# 1. Purpose + +Projection checkpoints make derived indexes replayable, auditable, and bounded. They tie a projection schema version to a specific ASL log position and snapshot identity so that rebuilds are deterministic and lag can be measured. + +--- + +# 2. Canonical Checkpoint Artifact + +Checkpoint artifacts are ASL artifacts whose bytes encode the following fields. The encoding is deterministic and versioned. + +## 2.1 Minimal Fields + +1. `checkpoint_version` (u32) +2. `projection_name` (string) +3. `projection_schema_version` (u32) +4. `asl_snapshot_id` (u64, optional if not available) +5. `asl_log_position` (u64, required) +6. `created_at` (u64, epoch milliseconds) +7. `builder_id` (string, identifying the indexer instance) + +## 2.2 Optional Fields + +1. `artifact_count` (u64) +2. `edge_count` (u64) +3. `error_count` (u64) +4. `lag_millis` (u64, derived) +5. `note` (string) + +--- + +# 3. Identity and Storage + +1. Checkpoints are stored as ASL artifacts using ASL/1-STORE semantics. +2. `projection_name` plus `projection_schema_version` must uniquely identify a projection format. +3. The checkpoint artifact reference is the authoritative identifier for the projection state. + +--- + +# 4. Lag Metric Binding + +Lag is defined as: + +``` +Lag = ASL_head_log_position - checkpoint.asl_log_position +``` + +The ASL head position is authoritative. The checkpoint log position is the last fully processed position. + +--- + +# 5. Replay Rules + +1. Rebuild begins at `checkpoint.asl_log_position + 1`. +2. Any schema change requires full replay from the earliest available log position. +3. Checkpoints must be monotonic in `asl_log_position` for a given projection. + +--- + +# 6. Validation Requirements + +When loading a checkpoint, the indexer MUST verify: + +1. `projection_schema_version` matches the current build. +2. `asl_log_position` is not ahead of the authoritative ASL head. +3. `checkpoint_version` is supported. +4. If present, `asl_snapshot_id` matches the expected snapshot boundary. + +--- + +# 7. Operational Guidance + +1. Checkpoints should be written after a successful batch commit. +2. Checkpoints should be immutable; new checkpoints are separate artifacts. +3. Retain checkpoints for audit and rollback. diff --git a/docs/tgk-ordering-enforcement.md b/docs/tgk-ordering-enforcement.md new file mode 100644 index 0000000..27ed82e --- /dev/null +++ b/docs/tgk-ordering-enforcement.md @@ -0,0 +1,76 @@ +# TGK Ordering Enforcement + +Status: Draft +Scope: Canonical ordering rules for TGK/STORE/1 and where to enforce them in adapters and query pipelines. + +--- + +# 1. Purpose + +TGK/STORE/1 defines a canonical deterministic ordering for all edge and neighbor query results. This document records the rule and the enforcement points so implementations do not drift. + +--- + +# 2. Canonical Ordering Rules + +## 2.1 Edge Ordering + +All adjacency and scan results MUST be ordered by: + +1. For each edge `(edge_ref, body)`, form: + + `order_key(edge_ref) = hash_id_bytes || digest_bytes` + +2. `hash_id_bytes` is `edge_ref.hash_id` encoded as big-endian `u16`. +3. `digest_bytes` is the raw `edge_ref.digest`. +4. Sort edges in ascending lexicographic order of `order_key(edge_ref)`. + +This is the normative ordering defined by TGK/STORE/1 and is equivalent to sorting by canonical `ReferenceBytes`. + +## 2.2 Neighbor Ordering + +If `neighbors` is implemented, neighbor results MUST be ordered by: + +1. For each node `n`, form: + + `node_order_key(n) = hash_id_bytes || digest_bytes` + +2. `hash_id_bytes` is `n.hash_id` encoded as big-endian `u16`. +3. `digest_bytes` is the raw `n.digest`. +4. Sort neighbors in ascending lexicographic order of `node_order_key(n)`. + +--- + +# 3. Enforcement Points + +Ordering MUST be enforced by TGK/STORE/1 logic, not by backend defaults. + +Acceptable enforcement points: + +1. Adapter layer (preferred): apply ordering before returning `GraphEdgeView` lists. +2. Canonical result transformation layer: a shared sorting function used by all adapters. + +Disallowed enforcement: + +1. Relying on JanusGraph traversal order. +2. Relying on Elasticsearch sort order. +3. Any backend-specific implicit ordering. + +--- + +# 4. Test Requirements + +Conformance tests MUST verify: + +1. Adjacency results are ordered per the canonical `(hash_id, digest)` rule. +2. `scan_edges` results follow the same ordering per page. +3. `neighbors` results use the canonical node ordering if the operation is exposed. +4. Cross-backend equivalence: the same snapshot yields identical ordered lists. + +--- + +# 5. Implementation Notes + +1. Sorting uses the canonical `ReferenceBytes` ordering for `edge_ref` and `node` references. +2. Ordering is snapshot-relative; it must be stable for a fixed snapshot. +3. If ES is used for candidate retrieval, the authoritative layer must re-apply ordering. diff --git a/src/tools/amduat_conformance_run.c b/src/tools/amduat_conformance_run.c new file mode 100644 index 0000000..0e11d90 --- /dev/null +++ b/src/tools/amduat_conformance_run.c @@ -0,0 +1,203 @@ +#include "asl_store_conformance.h" +#include "tgk_store_conformance.h" + +#include +#include +#include +#include +#include + +static void usage(FILE *out) { + fprintf(out, + "Usage: amduat_conformance_run [options]\n" + "\n" + "Options:\n" + " --suite asl|tgk|all Suite to run (default: all)\n" + " --backend Backend to run (repeatable)\n" + " --workdir Workdir for backend state\n" + " --fixtures Fixtures root (default: tests/fixtures/conformance)\n" + " --help Show this help\n"); +} + +static bool env_truthy(const char *value) { + if (value == NULL) { + return false; + } + return strcmp(value, "1") == 0 || strcmp(value, "true") == 0 || + strcmp(value, "yes") == 0; +} + +static bool append_backend(char **list, const char *backend) { + size_t list_len; + size_t backend_len; + size_t next_len; + char *next; + + if (list == NULL || backend == NULL) { + return false; + } + + if (*list == NULL) { + *list = strdup(backend); + return *list != NULL; + } + + list_len = strlen(*list); + backend_len = strlen(backend); + next_len = list_len + 1u + backend_len + 1u; + + next = (char *)realloc(*list, next_len); + if (next == NULL) { + return false; + } + + next[list_len] = ','; + memcpy(next + list_len + 1u, backend, backend_len); + next[next_len - 1u] = '\0'; + *list = next; + return true; +} + +static int print_results(const char *suite, + const amduat_conformance_backend_result_t *results, + size_t results_len) { + size_t i; + int rc = 0; + + if (results_len == 0) { + fprintf(stderr, "%s: no backends selected\n", suite); + return 1; + } + + for (i = 0; i < results_len; ++i) { + const char *status = results[i].status == 0 ? "PASS" : "FAIL"; + printf("%s backend=%s %s\n", suite, results[i].backend, status); + if (results[i].status != 0) { + rc = 1; + } + } + + return rc; +} + +static const char *fallback_fixtures_root(const char *path) { + struct stat st; + + if (path == NULL) { + return NULL; + } + if (stat(path, &st) == 0) { + return path; + } + if (strcmp(path, "tests/fixtures/conformance") == 0) { + if (stat("../tests/fixtures/conformance", &st) == 0) { + return "../tests/fixtures/conformance"; + } + } + return path; +} + +int main(int argc, char **argv) { + const char *suite = "all"; + const char *fixtures = "tests/fixtures/conformance"; + const char *workdir = "/tmp/amduat_conformance"; + char *backend_selection = NULL; + bool include_optional = env_truthy(getenv("AMDUAT_CONFORMANCE_INCLUDE_OPTIONAL")); + int i; + int rc = 0; + + for (i = 1; i < argc; ++i) { + if (strcmp(argv[i], "--help") == 0) { + usage(stdout); + free(backend_selection); + return 0; + } + if (strcmp(argv[i], "--suite") == 0 && i + 1 < argc) { + suite = argv[++i]; + continue; + } + if (strcmp(argv[i], "--backend") == 0 && i + 1 < argc) { + const char *value = argv[++i]; + if (strcmp(value, "all") == 0) { + free(backend_selection); + backend_selection = strdup("all"); + if (backend_selection == NULL) { + fprintf(stderr, "out of memory\n"); + return 1; + } + } else { + if (backend_selection == NULL || strcmp(backend_selection, "all") != 0) { + if (!append_backend(&backend_selection, value)) { + fprintf(stderr, "out of memory\n"); + free(backend_selection); + return 1; + } + } + } + continue; + } + if (strcmp(argv[i], "--workdir") == 0 && i + 1 < argc) { + workdir = argv[++i]; + continue; + } + if (strcmp(argv[i], "--fixtures") == 0 && i + 1 < argc) { + fixtures = argv[++i]; + continue; + } + + fprintf(stderr, "unknown argument: %s\n", argv[i]); + usage(stderr); + free(backend_selection); + return 1; + } + + fixtures = fallback_fixtures_root(fixtures); + + if (backend_selection == NULL) { + backend_selection = strdup("all"); + if (backend_selection == NULL) { + fprintf(stderr, "out of memory\n"); + return 1; + } + } + + if (strcmp(suite, "asl") == 0 || strcmp(suite, "all") == 0) { + amduat_conformance_backend_result_t *results = NULL; + size_t results_len = 0u; + int suite_rc = amduat_asl_store_conformance_run( + backend_selection, fixtures, workdir, include_optional, true, + &results, &results_len); + if (suite_rc != 0) { + rc = 1; + } + if (print_results("asl", results, results_len) != 0) { + rc = 1; + } + free(results); + } + + if (strcmp(suite, "tgk") == 0 || strcmp(suite, "all") == 0) { + amduat_conformance_backend_result_t *results = NULL; + size_t results_len = 0u; + int suite_rc = amduat_tgk_store_conformance_run( + backend_selection, fixtures, workdir, include_optional, true, + &results, &results_len); + if (suite_rc != 0) { + rc = 1; + } + if (print_results("tgk", results, results_len) != 0) { + rc = 1; + } + free(results); + } + + if (strcmp(suite, "asl") != 0 && strcmp(suite, "tgk") != 0 && + strcmp(suite, "all") != 0) { + fprintf(stderr, "invalid suite: %s\n", suite); + usage(stderr); + rc = 1; + } + + free(backend_selection); + return rc; +} diff --git a/tests/conformance/asl_store/asl_store_conformance.c b/tests/conformance/asl_store/asl_store_conformance.c new file mode 100644 index 0000000..da4abef --- /dev/null +++ b/tests/conformance/asl_store/asl_store_conformance.c @@ -0,0 +1,908 @@ +#include "asl_store_conformance.h" + +#include "amduat/asl/asl_store_fs.h" +#include "amduat/asl/asl_store_index_fs.h" +#include "amduat/asl/core.h" +#include "amduat/asl/store.h" +#include "amduat/enc/asl1_core.h" +#include "amduat/enc/asl1_core_codec.h" +#include "amduat/hash/asl1.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct { + char *id; + char *artifact; + char *notes; +} amduat_conformance_fixture_t; + +typedef struct { + const char *name; + bool optional; + bool (*init)(const char *root, amduat_asl_store_t *out_store, void **out_ctx); + void (*cleanup)(void *ctx); +} amduat_asl_conformance_backend_t; + +typedef struct { + amduat_asl_store_fs_t fs; +} amduat_asl_fs_ctx_t; + +typedef struct { + amduat_asl_store_index_fs_t fs; +} amduat_asl_index_fs_ctx_t; + +static bool conformance_join_path(const char *base, + const char *segment, + char **out_path) { + size_t base_len; + size_t seg_len; + bool needs_sep; + size_t total_len; + char *buffer; + size_t offset; + + if (base == NULL || segment == NULL || out_path == NULL) { + return false; + } + if (base[0] == '\0' || segment[0] == '\0') { + return false; + } + + base_len = strlen(base); + seg_len = strlen(segment); + needs_sep = base[base_len - 1u] != '/'; + total_len = base_len + (needs_sep ? 1u : 0u) + seg_len + 1u; + + buffer = (char *)malloc(total_len); + if (buffer == NULL) { + return false; + } + + offset = 0u; + memcpy(buffer + offset, base, base_len); + offset += base_len; + if (needs_sep) { + buffer[offset++] = '/'; + } + memcpy(buffer + offset, segment, seg_len); + offset += seg_len; + buffer[offset] = '\0'; + + *out_path = buffer; + return true; +} + +static bool conformance_remove_tree(const char *path) { + struct stat st; + DIR *dir; + struct dirent *entry; + + if (path == NULL) { + return false; + } + if (lstat(path, &st) != 0) { + return errno == ENOENT; + } + + if (!S_ISDIR(st.st_mode)) { + return unlink(path) == 0; + } + + dir = opendir(path); + if (dir == NULL) { + return false; + } + + while ((entry = readdir(dir)) != NULL) { + char *child = NULL; + if (strcmp(entry->d_name, ".") == 0 || + strcmp(entry->d_name, "..") == 0) { + continue; + } + if (!conformance_join_path(path, entry->d_name, &child)) { + closedir(dir); + return false; + } + if (!conformance_remove_tree(child)) { + free(child); + closedir(dir); + return false; + } + free(child); + } + + if (closedir(dir) != 0) { + return false; + } + return rmdir(path) == 0; +} + +static bool conformance_ensure_dir(const char *path) { + char *cursor; + char *temp; + + if (path == NULL || path[0] == '\0') { + return false; + } + + temp = strdup(path); + if (temp == NULL) { + return false; + } + + cursor = temp; + if (cursor[0] == '/') { + cursor++; + } + + while (true) { + char saved; + char *next = strchr(cursor, '/'); + if (next == NULL) { + break; + } + saved = *next; + *next = '\0'; + if (temp[0] != '\0') { + if (mkdir(temp, 0700) != 0 && errno != EEXIST) { + free(temp); + return false; + } + } + *next = saved; + cursor = next + 1; + } + + if (mkdir(temp, 0700) != 0 && errno != EEXIST) { + free(temp); + return false; + } + free(temp); + return true; +} + +static bool conformance_prepare_workdir(const char *base, + const char *backend, + const char *suite, + char **out_path) { + char *backend_root = NULL; + char *suite_root = NULL; + + if (base == NULL || backend == NULL || suite == NULL || out_path == NULL) { + return false; + } + + if (!conformance_join_path(base, backend, &backend_root)) { + return false; + } + if (!conformance_join_path(backend_root, suite, &suite_root)) { + free(backend_root); + return false; + } + + if (!conformance_ensure_dir(backend_root)) { + free(backend_root); + free(suite_root); + return false; + } + if (!conformance_remove_tree(suite_root)) { + free(backend_root); + free(suite_root); + return false; + } + if (!conformance_ensure_dir(suite_root)) { + free(backend_root); + free(suite_root); + return false; + } + + free(backend_root); + *out_path = suite_root; + return true; +} + +static bool conformance_read_file(const char *path, + uint8_t **out_bytes, + size_t *out_len) { + FILE *fp; + long size; + uint8_t *buffer; + size_t read_len; + + if (path == NULL || out_bytes == NULL || out_len == NULL) { + return false; + } + *out_bytes = NULL; + *out_len = 0u; + + fp = fopen(path, "rb"); + if (fp == NULL) { + return false; + } + if (fseek(fp, 0, SEEK_END) != 0) { + fclose(fp); + return false; + } + size = ftell(fp); + if (size < 0) { + fclose(fp); + return false; + } + if (fseek(fp, 0, SEEK_SET) != 0) { + fclose(fp); + return false; + } + + buffer = (uint8_t *)malloc((size_t)size + 1u); + if (buffer == NULL) { + fclose(fp); + return false; + } + read_len = fread(buffer, 1u, (size_t)size, fp); + fclose(fp); + if (read_len != (size_t)size) { + free(buffer); + return false; + } + buffer[size] = '\0'; + + *out_bytes = buffer; + *out_len = (size_t)size; + return true; +} + +static void conformance_fixture_free(amduat_conformance_fixture_t *fixture) { + if (fixture == NULL) { + return; + } + free(fixture->id); + free(fixture->artifact); + free(fixture->notes); + fixture->id = NULL; + fixture->artifact = NULL; + fixture->notes = NULL; +} + +static void conformance_fixture_list_free(amduat_conformance_fixture_t *list, + size_t len) { + size_t i; + + if (list == NULL) { + return; + } + for (i = 0; i < len; ++i) { + conformance_fixture_free(&list[i]); + } + free(list); +} + +static void conformance_skip_ws(const char **cursor) { + while (*cursor != NULL && **cursor != '\0' && isspace((unsigned char)**cursor)) { + (*cursor)++; + } +} + +static bool conformance_parse_string(const char **cursor, char **out_text) { + const char *start; + const char *end; + size_t len; + char *buffer; + + if (cursor == NULL || out_text == NULL) { + return false; + } + conformance_skip_ws(cursor); + if (**cursor != '"') { + return false; + } + (*cursor)++; + start = *cursor; + end = strchr(start, '"'); + if (end == NULL) { + return false; + } + len = (size_t)(end - start); + buffer = (char *)malloc(len + 1u); + if (buffer == NULL) { + return false; + } + memcpy(buffer, start, len); + buffer[len] = '\0'; + *cursor = end + 1; + *out_text = buffer; + return true; +} + +static bool conformance_parse_manifest(const char *manifest_path, + amduat_conformance_fixture_t **out_list, + size_t *out_len) { + uint8_t *bytes = NULL; + size_t len = 0u; + const char *cursor; + const char *fixtures_key; + const char *array_start; + amduat_conformance_fixture_t *list = NULL; + size_t list_len = 0u; + size_t list_cap = 0u; + + if (out_list == NULL || out_len == NULL || manifest_path == NULL) { + return false; + } + *out_list = NULL; + *out_len = 0u; + + if (!conformance_read_file(manifest_path, &bytes, &len)) { + return false; + } + + cursor = (const char *)bytes; + fixtures_key = strstr(cursor, "\"fixtures\""); + if (fixtures_key == NULL) { + free(bytes); + return false; + } + array_start = strchr(fixtures_key, '['); + if (array_start == NULL) { + free(bytes); + return false; + } + cursor = array_start + 1; + + while (true) { + amduat_conformance_fixture_t fixture; + char *key = NULL; + char *value = NULL; + bool done = false; + + memset(&fixture, 0, sizeof(fixture)); + + conformance_skip_ws(&cursor); + if (*cursor == '\0' || *cursor == ']') { + break; + } + if (*cursor != '{') { + conformance_fixture_free(&fixture); + break; + } + cursor++; + + while (!done) { + conformance_skip_ws(&cursor); + if (*cursor == '}') { + cursor++; + done = true; + break; + } + + if (!conformance_parse_string(&cursor, &key)) { + conformance_fixture_free(&fixture); + free(bytes); + conformance_fixture_list_free(list, list_len); + return false; + } + conformance_skip_ws(&cursor); + if (*cursor != ':') { + free(key); + conformance_fixture_free(&fixture); + free(bytes); + conformance_fixture_list_free(list, list_len); + return false; + } + cursor++; + + if (!conformance_parse_string(&cursor, &value)) { + free(key); + conformance_fixture_free(&fixture); + free(bytes); + conformance_fixture_list_free(list, list_len); + return false; + } + + if (strcmp(key, "id") == 0) { + fixture.id = value; + } else if (strcmp(key, "artifact") == 0) { + fixture.artifact = value; + } else if (strcmp(key, "notes") == 0) { + fixture.notes = value; + } else { + free(value); + } + free(key); + key = NULL; + value = NULL; + + conformance_skip_ws(&cursor); + if (*cursor == ',') { + cursor++; + continue; + } + if (*cursor == '}') { + cursor++; + done = true; + break; + } + } + + if (fixture.id == NULL || fixture.artifact == NULL) { + conformance_fixture_free(&fixture); + free(bytes); + conformance_fixture_list_free(list, list_len); + return false; + } + + if (list_len == list_cap) { + size_t next_cap = list_cap == 0u ? 4u : list_cap * 2u; + amduat_conformance_fixture_t *next = + (amduat_conformance_fixture_t *)realloc(list, + next_cap * sizeof(*list)); + if (next == NULL) { + conformance_fixture_free(&fixture); + free(bytes); + conformance_fixture_list_free(list, list_len); + return false; + } + list = next; + list_cap = next_cap; + } + + list[list_len++] = fixture; + + conformance_skip_ws(&cursor); + if (*cursor == ',') { + cursor++; + continue; + } + if (*cursor == ']') { + break; + } + } + + free(bytes); + *out_list = list; + *out_len = list_len; + return list_len > 0u; +} + +static bool conformance_backend_selected(const char *selection, + const char *name) { + const char *cursor; + size_t name_len; + + if (selection == NULL || selection[0] == '\0') { + return true; + } + if (strcmp(selection, "all") == 0) { + return true; + } + + name_len = strlen(name); + cursor = selection; + while (*cursor != '\0') { + const char *start = cursor; + const char *end = strchr(start, ','); + size_t len = end == NULL ? strlen(start) : (size_t)(end - start); + + while (len > 0u && isspace((unsigned char)start[0])) { + start++; + len--; + } + while (len > 0u && isspace((unsigned char)start[len - 1u])) { + len--; + } + + if (len == name_len && strncmp(start, name, len) == 0) { + return true; + } + + if (end == NULL) { + break; + } + cursor = end + 1; + } + return false; +} + +static bool conformance_build_artifact(amduat_octets_t payload, + amduat_artifact_t *out_artifact) { + amduat_artifact_t raw; + amduat_octets_t encoded = amduat_octets(NULL, 0); + amduat_artifact_t decoded; + + if (out_artifact == NULL) { + return false; + } + + raw = amduat_artifact(payload); + if (!amduat_enc_asl1_core_encode_artifact_v1(raw, &encoded)) { + return false; + } + if (!amduat_enc_asl1_core_decode_artifact_v1(encoded, &decoded)) { + amduat_octets_free(&encoded); + return false; + } + amduat_octets_free(&encoded); + + *out_artifact = decoded; + return true; +} + +static bool conformance_asl_is_not_found(amduat_asl_store_error_t err) { + return err == AMDUAT_ASL_STORE_ERR_NOT_FOUND; +} + +static int conformance_test_put_get(amduat_asl_store_t *store, + const amduat_conformance_fixture_t *fixtures, + size_t fixtures_len, + const char *fixture_root) { + size_t i; + + for (i = 0; i < fixtures_len; ++i) { + char *artifact_path = NULL; + uint8_t *payload_bytes = NULL; + size_t payload_len = 0u; + amduat_artifact_t artifact; + amduat_artifact_t fetched; + amduat_reference_t ref; + amduat_asl_store_error_t err; + + memset(&artifact, 0, sizeof(artifact)); + memset(&fetched, 0, sizeof(fetched)); + memset(&ref, 0, sizeof(ref)); + + if (!conformance_join_path(fixture_root, fixtures[i].artifact, + &artifact_path)) { + fprintf(stderr, "fixture path join failed\n"); + return 1; + } + if (!conformance_read_file(artifact_path, &payload_bytes, &payload_len)) { + fprintf(stderr, "fixture read failed: %s\n", artifact_path); + free(artifact_path); + return 1; + } + free(artifact_path); + + if (!conformance_build_artifact(amduat_octets(payload_bytes, payload_len), + &artifact)) { + fprintf(stderr, "artifact encode/decode failed\n"); + free(payload_bytes); + return 1; + } + free(payload_bytes); + + err = amduat_asl_store_put(store, artifact, &ref); + if (err != AMDUAT_ASL_STORE_OK) { + fprintf(stderr, "put failed (%d) for fixture %s\n", err, fixtures[i].id); + amduat_artifact_free(&artifact); + return 1; + } + + err = amduat_asl_store_get(store, ref, &fetched); + if (err != AMDUAT_ASL_STORE_OK) { + fprintf(stderr, "get failed (%d) for fixture %s\n", err, fixtures[i].id); + amduat_reference_free(&ref); + amduat_artifact_free(&artifact); + return 1; + } + + if (!amduat_artifact_eq(artifact, fetched)) { + fprintf(stderr, "payload mismatch for fixture %s\n", fixtures[i].id); + amduat_reference_free(&ref); + amduat_artifact_free(&artifact); + amduat_artifact_free(&fetched); + return 1; + } + + amduat_reference_free(&ref); + amduat_artifact_free(&artifact); + amduat_artifact_free(&fetched); + } + + return 0; +} + +static int conformance_test_put_idempotent( + amduat_asl_store_t *store, + const amduat_conformance_fixture_t *fixture, + const char *fixture_root) { + char *artifact_path = NULL; + uint8_t *payload_bytes = NULL; + size_t payload_len = 0u; + amduat_artifact_t artifact; + amduat_reference_t ref_a; + amduat_reference_t ref_b; + amduat_asl_store_error_t err; + + memset(&artifact, 0, sizeof(artifact)); + memset(&ref_a, 0, sizeof(ref_a)); + memset(&ref_b, 0, sizeof(ref_b)); + + if (!conformance_join_path(fixture_root, fixture->artifact, &artifact_path)) { + fprintf(stderr, "fixture path join failed\n"); + return 1; + } + if (!conformance_read_file(artifact_path, &payload_bytes, &payload_len)) { + fprintf(stderr, "fixture read failed: %s\n", artifact_path); + free(artifact_path); + return 1; + } + free(artifact_path); + + if (!conformance_build_artifact(amduat_octets(payload_bytes, payload_len), + &artifact)) { + fprintf(stderr, "artifact encode/decode failed\n"); + free(payload_bytes); + return 1; + } + free(payload_bytes); + + err = amduat_asl_store_put(store, artifact, &ref_a); + if (err != AMDUAT_ASL_STORE_OK) { + fprintf(stderr, "put failed (%d)\n", err); + amduat_artifact_free(&artifact); + return 1; + } + + err = amduat_asl_store_put(store, artifact, &ref_b); + if (err != AMDUAT_ASL_STORE_OK) { + fprintf(stderr, "put (second) failed (%d)\n", err); + amduat_reference_free(&ref_a); + amduat_artifact_free(&artifact); + return 1; + } + + if (!amduat_reference_eq(ref_a, ref_b)) { + fprintf(stderr, "idempotent put returned different refs\n"); + amduat_reference_free(&ref_a); + amduat_reference_free(&ref_b); + amduat_artifact_free(&artifact); + return 1; + } + + amduat_reference_free(&ref_a); + amduat_reference_free(&ref_b); + amduat_artifact_free(&artifact); + return 0; +} + +static int conformance_test_get_not_found(amduat_asl_store_t *store) { + uint8_t digest[32]; + amduat_reference_t missing; + amduat_artifact_t out_artifact; + amduat_asl_store_error_t err; + + memset(digest, 0x7a, sizeof(digest)); + missing = amduat_reference(AMDUAT_HASH_ASL1_ID_SHA256, + amduat_octets(digest, sizeof(digest))); + memset(&out_artifact, 0, sizeof(out_artifact)); + + err = amduat_asl_store_get(store, missing, &out_artifact); + if (!conformance_asl_is_not_found(err)) { + fprintf(stderr, "expected NOT_FOUND, got %d\n", err); + if (err == AMDUAT_ASL_STORE_OK) { + amduat_artifact_free(&out_artifact); + } + return 1; + } + + return 0; +} + +static bool conformance_init_asl_store_fs(const char *root, + amduat_asl_store_t *out_store, + void **out_ctx) { + amduat_asl_fs_ctx_t *ctx; + amduat_asl_store_config_t config; + + if (out_store == NULL || out_ctx == NULL) { + return false; + } + + ctx = (amduat_asl_fs_ctx_t *)calloc(1, sizeof(*ctx)); + if (ctx == NULL) { + return false; + } + + memset(&config, 0, sizeof(config)); + config.encoding_profile_id = AMDUAT_ENC_ASL1_CORE_V1; + config.hash_id = AMDUAT_HASH_ASL1_ID_SHA256; + + if (!amduat_asl_store_fs_init(&ctx->fs, config, root)) { + free(ctx); + return false; + } + + amduat_asl_store_init(out_store, config, amduat_asl_store_fs_ops(), &ctx->fs); + *out_ctx = ctx; + return true; +} + +static bool conformance_init_asl_store_index_fs( + const char *root, + amduat_asl_store_t *out_store, + void **out_ctx) { + amduat_asl_index_fs_ctx_t *ctx; + amduat_asl_store_config_t config; + + if (out_store == NULL || out_ctx == NULL) { + return false; + } + + ctx = (amduat_asl_index_fs_ctx_t *)calloc(1, sizeof(*ctx)); + if (ctx == NULL) { + return false; + } + + memset(&config, 0, sizeof(config)); + config.encoding_profile_id = AMDUAT_ENC_ASL1_CORE_V1; + config.hash_id = AMDUAT_HASH_ASL1_ID_SHA256; + + if (!amduat_asl_store_index_fs_init(&ctx->fs, config, root)) { + free(ctx); + return false; + } + + amduat_asl_store_init(out_store, config, amduat_asl_store_index_fs_ops(), + &ctx->fs); + *out_ctx = ctx; + return true; +} + +static void conformance_free_ctx(void *ctx) { + free(ctx); +} + +static int conformance_run_backend( + const amduat_asl_conformance_backend_t *backend, + const char *workdir, + const amduat_conformance_fixture_t *fixtures, + size_t fixtures_len, + const char *fixture_root) { + amduat_asl_store_t store; + void *ctx = NULL; + int rc = 1; + + memset(&store, 0, sizeof(store)); + + if (!backend->init(workdir, &store, &ctx)) { + fprintf(stderr, "backend init failed: %s\n", backend->name); + return 1; + } + + if (conformance_test_put_get(&store, fixtures, fixtures_len, fixture_root) != 0 || + conformance_test_put_idempotent(&store, &fixtures[0], fixture_root) != 0 || + conformance_test_get_not_found(&store) != 0) { + rc = 1; + } else { + rc = 0; + } + + if (backend->cleanup != NULL) { + backend->cleanup(ctx); + } + return rc; +} + +int amduat_asl_store_conformance_run( + const char *backend_selection, + const char *fixtures_root, + const char *workdir, + bool include_optional, + bool verbose, + amduat_conformance_backend_result_t **out_results, + size_t *out_len) { + const amduat_asl_conformance_backend_t backends[] = { + {"asl_store_fs", false, conformance_init_asl_store_fs, conformance_free_ctx}, + {"asl_store_index_fs", false, conformance_init_asl_store_index_fs, + conformance_free_ctx}, + }; + char *fixture_dir = NULL; + char *manifest_path = NULL; + char *artifact_dir = NULL; + amduat_conformance_fixture_t *fixtures = NULL; + size_t fixtures_len = 0u; + size_t i; + size_t results_len = 0u; + amduat_conformance_backend_result_t *results = NULL; + + if (out_results == NULL || out_len == NULL) { + return 1; + } + *out_results = NULL; + *out_len = 0u; + + if (fixtures_root == NULL || workdir == NULL) { + return 1; + } + + if (!conformance_join_path(fixtures_root, "asl_put_get", &fixture_dir)) { + return 1; + } + if (!conformance_join_path(fixture_dir, "manifest.json", &manifest_path)) { + free(fixture_dir); + return 1; + } + if (!conformance_join_path(fixture_dir, "artifacts", &artifact_dir)) { + free(fixture_dir); + free(manifest_path); + return 1; + } + + if (!conformance_parse_manifest(manifest_path, &fixtures, &fixtures_len)) { + fprintf(stderr, "failed to parse manifest: %s\n", manifest_path); + free(fixture_dir); + free(manifest_path); + free(artifact_dir); + return 1; + } + + for (i = 0; i < sizeof(backends) / sizeof(backends[0]); ++i) { + const amduat_asl_conformance_backend_t *backend = &backends[i]; + char *backend_workdir = NULL; + + if (!conformance_backend_selected(backend_selection, backend->name)) { + continue; + } + if (backend->optional && !include_optional) { + continue; + } + + if (!conformance_prepare_workdir(workdir, backend->name, "asl", + &backend_workdir)) { + fprintf(stderr, "workdir prep failed for %s\n", backend->name); + conformance_fixture_list_free(fixtures, fixtures_len); + free(fixture_dir); + free(manifest_path); + free(artifact_dir); + free(results); + return 1; + } + + results = (amduat_conformance_backend_result_t *)realloc( + results, (results_len + 1u) * sizeof(*results)); + if (results == NULL) { + free(backend_workdir); + conformance_fixture_list_free(fixtures, fixtures_len); + free(fixture_dir); + free(manifest_path); + free(artifact_dir); + return 1; + } + + results[results_len].backend = backend->name; + results[results_len].status = + conformance_run_backend(backend, backend_workdir, fixtures, + fixtures_len, artifact_dir); + if (verbose) { + fprintf(stdout, "asl %s %s\n", backend->name, + results[results_len].status == 0 ? "PASS" : "FAIL"); + } + results_len++; + free(backend_workdir); + } + + conformance_fixture_list_free(fixtures, fixtures_len); + free(fixture_dir); + free(manifest_path); + free(artifact_dir); + + *out_results = results; + *out_len = results_len; + + for (i = 0; i < results_len; ++i) { + if (results[i].status != 0) { + return 1; + } + } + + return 0; +} diff --git a/tests/conformance/asl_store/asl_store_conformance.h b/tests/conformance/asl_store/asl_store_conformance.h new file mode 100644 index 0000000..1224acf --- /dev/null +++ b/tests/conformance/asl_store/asl_store_conformance.h @@ -0,0 +1,26 @@ +#ifndef AMDUAT_TESTS_CONFORMANCE_ASL_STORE_CONFORMANCE_H +#define AMDUAT_TESTS_CONFORMANCE_ASL_STORE_CONFORMANCE_H + +#include "tests/conformance/conformance_result.h" + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +int amduat_asl_store_conformance_run( + const char *backend_selection, + const char *fixtures_root, + const char *workdir, + bool include_optional, + bool verbose, + amduat_conformance_backend_result_t **out_results, + size_t *out_len); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* AMDUAT_TESTS_CONFORMANCE_ASL_STORE_CONFORMANCE_H */ diff --git a/tests/conformance/asl_store/test_asl_store_conformance.c b/tests/conformance/asl_store/test_asl_store_conformance.c new file mode 100644 index 0000000..7e0e71b --- /dev/null +++ b/tests/conformance/asl_store/test_asl_store_conformance.c @@ -0,0 +1,87 @@ +#include "asl_store_conformance.h" + +#include +#include +#include +#include +#include +#include + +static bool env_truthy(const char *value) { + if (value == NULL) { + return false; + } + return strcmp(value, "1") == 0 || strcmp(value, "true") == 0 || + strcmp(value, "yes") == 0; +} + +static char *make_temp_root(void) { + char *templ; + const char template_prefix[] = "/tmp/amduat_conformance_asl_XXXXXX"; + + templ = (char *)malloc(sizeof(template_prefix)); + if (templ == NULL) { + return NULL; + } + memcpy(templ, template_prefix, sizeof(template_prefix)); + if (mkdtemp(templ) == NULL) { + free(templ); + return NULL; + } + return templ; +} + +static const char *default_fixtures_root(void) { + struct stat st; + + if (stat("tests/fixtures/conformance", &st) == 0) { + return "tests/fixtures/conformance"; + } + if (stat("../tests/fixtures/conformance", &st) == 0) { + return "../tests/fixtures/conformance"; + } + return "tests/fixtures/conformance"; +} + +int main(void) { + const char *backend_env = getenv("AMDUAT_CONFORMANCE_ASL_BACKENDS"); + const char *fixtures_env = getenv("AMDUAT_CONFORMANCE_FIXTURES"); + const char *workdir_env = getenv("AMDUAT_CONFORMANCE_WORKDIR"); + const char *optional_env = getenv("AMDUAT_CONFORMANCE_INCLUDE_OPTIONAL"); + const char *backend_selection = backend_env != NULL ? backend_env : "all"; + const char *fixtures_root = + fixtures_env != NULL ? fixtures_env : default_fixtures_root(); + char *owned_workdir = NULL; + const char *workdir = workdir_env; + bool include_optional = env_truthy(optional_env); + amduat_conformance_backend_result_t *results = NULL; + size_t results_len = 0u; + size_t i; + int rc; + + if (workdir == NULL) { + owned_workdir = make_temp_root(); + if (owned_workdir == NULL) { + fprintf(stderr, "failed to create temp workdir\n"); + return 1; + } + workdir = owned_workdir; + } + + rc = amduat_asl_store_conformance_run(backend_selection, fixtures_root, + workdir, include_optional, false, + &results, &results_len); + if (rc != 0) { + fprintf(stderr, "asl store conformance failed\n"); + } + + for (i = 0; i < results_len; ++i) { + if (results[i].status != 0) { + rc = 1; + } + } + + free(results); + free(owned_workdir); + return rc; +} diff --git a/tests/conformance/conformance_result.h b/tests/conformance/conformance_result.h new file mode 100644 index 0000000..45a23f0 --- /dev/null +++ b/tests/conformance/conformance_result.h @@ -0,0 +1,19 @@ +#ifndef AMDUAT_TESTS_CONFORMANCE_RESULT_H +#define AMDUAT_TESTS_CONFORMANCE_RESULT_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + const char *backend; + int status; +} amduat_conformance_backend_result_t; + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* AMDUAT_TESTS_CONFORMANCE_RESULT_H */ diff --git a/tests/conformance/tgk_store/test_tgk_store_conformance.c b/tests/conformance/tgk_store/test_tgk_store_conformance.c new file mode 100644 index 0000000..705954a --- /dev/null +++ b/tests/conformance/tgk_store/test_tgk_store_conformance.c @@ -0,0 +1,87 @@ +#include "tgk_store_conformance.h" + +#include +#include +#include +#include +#include +#include + +static bool env_truthy(const char *value) { + if (value == NULL) { + return false; + } + return strcmp(value, "1") == 0 || strcmp(value, "true") == 0 || + strcmp(value, "yes") == 0; +} + +static char *make_temp_root(void) { + char *templ; + const char template_prefix[] = "/tmp/amduat_conformance_tgk_XXXXXX"; + + templ = (char *)malloc(sizeof(template_prefix)); + if (templ == NULL) { + return NULL; + } + memcpy(templ, template_prefix, sizeof(template_prefix)); + if (mkdtemp(templ) == NULL) { + free(templ); + return NULL; + } + return templ; +} + +static const char *default_fixtures_root(void) { + struct stat st; + + if (stat("tests/fixtures/conformance", &st) == 0) { + return "tests/fixtures/conformance"; + } + if (stat("../tests/fixtures/conformance", &st) == 0) { + return "../tests/fixtures/conformance"; + } + return "tests/fixtures/conformance"; +} + +int main(void) { + const char *backend_env = getenv("AMDUAT_CONFORMANCE_TGK_BACKENDS"); + const char *fixtures_env = getenv("AMDUAT_CONFORMANCE_FIXTURES"); + const char *workdir_env = getenv("AMDUAT_CONFORMANCE_WORKDIR"); + const char *optional_env = getenv("AMDUAT_CONFORMANCE_INCLUDE_OPTIONAL"); + const char *backend_selection = backend_env != NULL ? backend_env : "all"; + const char *fixtures_root = + fixtures_env != NULL ? fixtures_env : default_fixtures_root(); + char *owned_workdir = NULL; + const char *workdir = workdir_env; + bool include_optional = env_truthy(optional_env); + amduat_conformance_backend_result_t *results = NULL; + size_t results_len = 0u; + size_t i; + int rc; + + if (workdir == NULL) { + owned_workdir = make_temp_root(); + if (owned_workdir == NULL) { + fprintf(stderr, "failed to create temp workdir\n"); + return 1; + } + workdir = owned_workdir; + } + + rc = amduat_tgk_store_conformance_run(backend_selection, fixtures_root, + workdir, include_optional, false, + &results, &results_len); + if (rc != 0) { + fprintf(stderr, "tgk store conformance failed\n"); + } + + for (i = 0; i < results_len; ++i) { + if (results[i].status != 0) { + rc = 1; + } + } + + free(results); + free(owned_workdir); + return rc; +} diff --git a/tests/conformance/tgk_store/tgk_store_conformance.c b/tests/conformance/tgk_store/tgk_store_conformance.c new file mode 100644 index 0000000..be25f2d --- /dev/null +++ b/tests/conformance/tgk_store/tgk_store_conformance.c @@ -0,0 +1,628 @@ +#include "tgk_store_conformance.h" + +#include "amduat/asl/asl_store_fs.h" +#include "amduat/asl/asl_store_index_fs.h" +#include "amduat/asl/store.h" +#include "amduat/enc/asl1_core.h" +#include "amduat/enc/tgk1_edge.h" +#include "amduat/format/ref.h" +#include "amduat/hash/asl1.h" +#include "amduat/tgk/store.h" +#include "amduat/tgk/tgk_store_asl_index_fs.h" +#include "amduat/tgk/tgk_store_fs.h" +#include "amduat/tgk/tgk_store_mem.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct { + const char *name; + bool optional; + bool (*init)(const char *root, amduat_tgk_store_t *out_store, void **out_ctx); + void (*cleanup)(void *ctx); +} amduat_tgk_conformance_backend_t; + +typedef struct { + amduat_tgk_store_config_t config; + amduat_tgk_identity_domain_t domains[1]; + uint32_t edge_tags[1]; + amduat_tgk_edge_type_id_t edge_types[1]; + amduat_asl_encoding_profile_id_t encodings[1]; + amduat_tgk_store_mem_t mem; +} amduat_tgk_mem_ctx_t; + +typedef struct { + amduat_tgk_store_config_t config; + amduat_tgk_identity_domain_t domains[1]; + uint32_t edge_tags[1]; + amduat_tgk_edge_type_id_t edge_types[1]; + amduat_asl_encoding_profile_id_t encodings[1]; + amduat_asl_store_fs_t asl_fs; + amduat_asl_store_t asl_store; + amduat_tgk_store_fs_t fs; +} amduat_tgk_fs_ctx_t; + +typedef struct { + amduat_tgk_store_config_t config; + amduat_tgk_identity_domain_t domains[1]; + uint32_t edge_tags[1]; + amduat_tgk_edge_type_id_t edge_types[1]; + amduat_asl_encoding_profile_id_t encodings[1]; + amduat_asl_store_index_fs_t asl_fs; + amduat_tgk_store_asl_index_fs_t tgk; +} amduat_tgk_asl_index_ctx_t; + +static bool conformance_join_path(const char *base, + const char *segment, + char **out_path) { + size_t base_len; + size_t seg_len; + bool needs_sep; + size_t total_len; + char *buffer; + size_t offset; + + if (base == NULL || segment == NULL || out_path == NULL) { + return false; + } + if (base[0] == '\0' || segment[0] == '\0') { + return false; + } + + base_len = strlen(base); + seg_len = strlen(segment); + needs_sep = base[base_len - 1u] != '/'; + total_len = base_len + (needs_sep ? 1u : 0u) + seg_len + 1u; + + buffer = (char *)malloc(total_len); + if (buffer == NULL) { + return false; + } + + offset = 0u; + memcpy(buffer + offset, base, base_len); + offset += base_len; + if (needs_sep) { + buffer[offset++] = '/'; + } + memcpy(buffer + offset, segment, seg_len); + offset += seg_len; + buffer[offset] = '\0'; + + *out_path = buffer; + return true; +} + +static bool conformance_remove_tree(const char *path) { + struct stat st; + DIR *dir; + struct dirent *entry; + + if (path == NULL) { + return false; + } + if (lstat(path, &st) != 0) { + return errno == ENOENT; + } + + if (!S_ISDIR(st.st_mode)) { + return unlink(path) == 0; + } + + dir = opendir(path); + if (dir == NULL) { + return false; + } + + while ((entry = readdir(dir)) != NULL) { + char *child = NULL; + if (strcmp(entry->d_name, ".") == 0 || + strcmp(entry->d_name, "..") == 0) { + continue; + } + if (!conformance_join_path(path, entry->d_name, &child)) { + closedir(dir); + return false; + } + if (!conformance_remove_tree(child)) { + free(child); + closedir(dir); + return false; + } + free(child); + } + + if (closedir(dir) != 0) { + return false; + } + return rmdir(path) == 0; +} + +static bool conformance_ensure_dir(const char *path) { + char *cursor; + char *temp; + + if (path == NULL || path[0] == '\0') { + return false; + } + + temp = strdup(path); + if (temp == NULL) { + return false; + } + + cursor = temp; + if (cursor[0] == '/') { + cursor++; + } + + while (true) { + char saved; + char *next = strchr(cursor, '/'); + if (next == NULL) { + break; + } + saved = *next; + *next = '\0'; + if (temp[0] != '\0') { + if (mkdir(temp, 0700) != 0 && errno != EEXIST) { + free(temp); + return false; + } + } + *next = saved; + cursor = next + 1; + } + + if (mkdir(temp, 0700) != 0 && errno != EEXIST) { + free(temp); + return false; + } + free(temp); + return true; +} + +static bool conformance_prepare_workdir(const char *base, + const char *backend, + const char *suite, + char **out_path) { + char *backend_root = NULL; + char *suite_root = NULL; + + if (base == NULL || backend == NULL || suite == NULL || out_path == NULL) { + return false; + } + + if (!conformance_join_path(base, backend, &backend_root)) { + return false; + } + if (!conformance_join_path(backend_root, suite, &suite_root)) { + free(backend_root); + return false; + } + + if (!conformance_ensure_dir(backend_root)) { + free(backend_root); + free(suite_root); + return false; + } + if (!conformance_remove_tree(suite_root)) { + free(backend_root); + free(suite_root); + return false; + } + if (!conformance_ensure_dir(suite_root)) { + free(backend_root); + free(suite_root); + return false; + } + + free(backend_root); + *out_path = suite_root; + return true; +} + +static bool conformance_backend_selected(const char *selection, + const char *name) { + const char *cursor; + size_t name_len; + + if (selection == NULL || selection[0] == '\0') { + return true; + } + if (strcmp(selection, "all") == 0) { + return true; + } + + name_len = strlen(name); + cursor = selection; + while (*cursor != '\0') { + const char *start = cursor; + const char *end = strchr(start, ','); + size_t len = end == NULL ? strlen(start) : (size_t)(end - start); + + while (len > 0u && isspace((unsigned char)start[0])) { + start++; + len--; + } + while (len > 0u && isspace((unsigned char)start[len - 1u])) { + len--; + } + + if (len == name_len && strncmp(start, name, len) == 0) { + return true; + } + + if (end == NULL) { + break; + } + cursor = end + 1; + } + return false; +} + +static void conformance_init_tgk_config(amduat_tgk_store_config_t *config, + amduat_tgk_identity_domain_t *domains, + uint32_t *edge_tags, + amduat_tgk_edge_type_id_t *edge_types, + amduat_asl_encoding_profile_id_t *encodings) { + memset(config, 0, sizeof(*config)); + domains[0].encoding_profile = AMDUAT_ENC_ASL1_CORE_V1; + domains[0].hash_id = AMDUAT_HASH_ASL1_ID_SHA256; + edge_tags[0] = TYPE_TAG_TGK1_EDGE_V1; + edge_types[0] = 0x10; + encodings[0] = TGK1_EDGE_ENC_V1; + + config->id_space.domains = domains; + config->id_space.domains_len = 1; + config->artifact_scope.description = amduat_octets(NULL, 0); + config->tgk_profiles.edge_tags = edge_tags; + config->tgk_profiles.edge_tags_len = 1; + config->tgk_profiles.edge_types = edge_types; + config->tgk_profiles.edge_types_len = 1; + config->tgk_profiles.encodings = encodings; + config->tgk_profiles.encodings_len = 1; +} + +static bool conformance_init_tgk_store_mem(const char *root, + amduat_tgk_store_t *out_store, + void **out_ctx) { + amduat_tgk_mem_ctx_t *ctx; + + (void)root; + if (out_store == NULL || out_ctx == NULL) { + return false; + } + + ctx = (amduat_tgk_mem_ctx_t *)calloc(1, sizeof(*ctx)); + if (ctx == NULL) { + return false; + } + + conformance_init_tgk_config(&ctx->config, ctx->domains, ctx->edge_tags, + ctx->edge_types, ctx->encodings); + if (!amduat_tgk_store_mem_init(&ctx->mem, ctx->config, NULL, 0u)) { + free(ctx); + return false; + } + + amduat_tgk_store_init(out_store, ctx->config, amduat_tgk_store_mem_ops(), + &ctx->mem); + *out_ctx = ctx; + return true; +} + +static bool conformance_write_empty_manifest(const char *path) { + FILE *stream = fopen(path, "wb"); + if (stream == NULL) { + return false; + } + return fclose(stream) == 0; +} + +static bool conformance_init_tgk_store_fs(const char *root, + amduat_tgk_store_t *out_store, + void **out_ctx) { + amduat_tgk_fs_ctx_t *ctx; + amduat_asl_store_config_t asl_config; + char *asl_root = NULL; + char *manifest_path = NULL; + + if (out_store == NULL || out_ctx == NULL) { + return false; + } + + ctx = (amduat_tgk_fs_ctx_t *)calloc(1, sizeof(*ctx)); + if (ctx == NULL) { + return false; + } + + conformance_init_tgk_config(&ctx->config, ctx->domains, ctx->edge_tags, + ctx->edge_types, ctx->encodings); + + if (!conformance_join_path(root, "asl", &asl_root) || + !conformance_join_path(root, "manifest.txt", &manifest_path)) { + free(asl_root); + free(manifest_path); + free(ctx); + return false; + } + + if (!conformance_ensure_dir(asl_root)) { + free(asl_root); + free(manifest_path); + free(ctx); + return false; + } + + if (!conformance_write_empty_manifest(manifest_path)) { + free(asl_root); + free(manifest_path); + free(ctx); + return false; + } + + memset(&asl_config, 0, sizeof(asl_config)); + asl_config.encoding_profile_id = AMDUAT_ENC_ASL1_CORE_V1; + asl_config.hash_id = AMDUAT_HASH_ASL1_ID_SHA256; + if (!amduat_asl_store_fs_init(&ctx->asl_fs, asl_config, asl_root)) { + free(asl_root); + free(manifest_path); + free(ctx); + return false; + } + amduat_asl_store_init(&ctx->asl_store, asl_config, + amduat_asl_store_fs_ops(), &ctx->asl_fs); + + if (!amduat_tgk_store_fs_init(&ctx->fs, ctx->config, manifest_path, + AMDUAT_FORMAT_REF_HEX, &ctx->asl_store)) { + free(asl_root); + free(manifest_path); + free(ctx); + return false; + } + + amduat_tgk_store_init(out_store, ctx->config, amduat_tgk_store_fs_ops(), + &ctx->fs); + *out_ctx = ctx; + + free(asl_root); + free(manifest_path); + return true; +} + +static bool conformance_init_tgk_store_asl_index_fs( + const char *root, + amduat_tgk_store_t *out_store, + void **out_ctx) { + amduat_tgk_asl_index_ctx_t *ctx; + amduat_asl_store_config_t asl_config; + char *asl_root = NULL; + + if (out_store == NULL || out_ctx == NULL) { + return false; + } + + ctx = (amduat_tgk_asl_index_ctx_t *)calloc(1, sizeof(*ctx)); + if (ctx == NULL) { + return false; + } + + conformance_init_tgk_config(&ctx->config, ctx->domains, ctx->edge_tags, + ctx->edge_types, ctx->encodings); + + if (!conformance_join_path(root, "asl_index", &asl_root)) { + free(ctx); + return false; + } + if (!conformance_ensure_dir(asl_root)) { + free(asl_root); + free(ctx); + return false; + } + + memset(&asl_config, 0, sizeof(asl_config)); + asl_config.encoding_profile_id = AMDUAT_ENC_ASL1_CORE_V1; + asl_config.hash_id = AMDUAT_HASH_ASL1_ID_SHA256; + if (!amduat_asl_store_index_fs_init(&ctx->asl_fs, asl_config, asl_root)) { + free(asl_root); + free(ctx); + return false; + } + + if (!amduat_tgk_store_asl_index_fs_init(&ctx->tgk, ctx->config, &ctx->asl_fs)) { + free(asl_root); + free(ctx); + return false; + } + + amduat_tgk_store_init(out_store, ctx->config, + amduat_tgk_store_asl_index_fs_ops(), &ctx->tgk); + *out_ctx = ctx; + + free(asl_root); + return true; +} + +static void conformance_free_tgk_mem(void *ctx) { + amduat_tgk_mem_ctx_t *mem = (amduat_tgk_mem_ctx_t *)ctx; + if (mem == NULL) { + return; + } + amduat_tgk_store_mem_free(&mem->mem); + free(mem); +} + +static void conformance_free_tgk_fs(void *ctx) { + amduat_tgk_fs_ctx_t *fs = (amduat_tgk_fs_ctx_t *)ctx; + if (fs == NULL) { + return; + } + amduat_tgk_store_fs_free(&fs->fs); + free(fs); +} + +static void conformance_free_tgk_asl_index(void *ctx) { + amduat_tgk_asl_index_ctx_t *tgk = (amduat_tgk_asl_index_ctx_t *)ctx; + if (tgk == NULL) { + return; + } + amduat_tgk_store_asl_index_fs_free(&tgk->tgk); + free(tgk); +} + +static int conformance_test_empty_graph(amduat_tgk_store_t *store) { + uint8_t digest[32]; + amduat_reference_t node; + amduat_tgk_edge_type_filter_t filter; + amduat_tgk_graph_edge_view_list_t edges; + amduat_tgk_graph_scan_result_t scan; + + memset(digest, 0x42, sizeof(digest)); + node = amduat_reference(AMDUAT_HASH_ASL1_ID_SHA256, + amduat_octets(digest, sizeof(digest))); + + filter.types = NULL; + filter.types_len = 0u; + + memset(&edges, 0, sizeof(edges)); + if (!amduat_tgk_store_edges_from(store, node, filter, &edges)) { + fprintf(stderr, "edges_from failed\n"); + return 1; + } + if (edges.len != 0u) { + fprintf(stderr, "edges_from expected empty list\n"); + amduat_tgk_graph_edge_view_list_free(&edges); + return 1; + } + amduat_tgk_graph_edge_view_list_free(&edges); + + memset(&scan, 0, sizeof(scan)); + if (!amduat_tgk_store_scan_edges(store, filter, amduat_octets(NULL, 0), + false, &scan)) { + fprintf(stderr, "scan_edges failed\n"); + return 1; + } + if (scan.edges.len != 0u || scan.has_next_page) { + fprintf(stderr, "scan_edges expected empty list\n"); + amduat_tgk_graph_scan_result_free(&scan); + return 1; + } + amduat_tgk_graph_scan_result_free(&scan); + + /* TODO: add ordering/adjacency checks per docs/tgk-ordering-enforcement.md */ + + return 0; +} + +static int conformance_run_backend( + const amduat_tgk_conformance_backend_t *backend, + const char *workdir) { + amduat_tgk_store_t store; + void *ctx = NULL; + int rc; + + memset(&store, 0, sizeof(store)); + + if (!backend->init(workdir, &store, &ctx)) { + fprintf(stderr, "backend init failed: %s\n", backend->name); + return 1; + } + + rc = conformance_test_empty_graph(&store); + + if (backend->cleanup != NULL) { + backend->cleanup(ctx); + } + + return rc; +} + +int amduat_tgk_store_conformance_run( + const char *backend_selection, + const char *fixtures_root, + const char *workdir, + bool include_optional, + bool verbose, + amduat_conformance_backend_result_t **out_results, + size_t *out_len) { + const amduat_tgk_conformance_backend_t backends[] = { + {"tgk_store_mem", false, conformance_init_tgk_store_mem, + conformance_free_tgk_mem}, + {"tgk_store_fs", false, conformance_init_tgk_store_fs, + conformance_free_tgk_fs}, + {"tgk_store_asl_index_fs", true, conformance_init_tgk_store_asl_index_fs, + conformance_free_tgk_asl_index}, + }; + size_t i; + size_t results_len = 0u; + amduat_conformance_backend_result_t *results = NULL; + + (void)fixtures_root; + + if (out_results == NULL || out_len == NULL) { + return 1; + } + *out_results = NULL; + *out_len = 0u; + + if (workdir == NULL) { + return 1; + } + + bool selection_all =\n+ (backend_selection == NULL || backend_selection[0] == '\\0' ||\n+ strcmp(backend_selection, \"all\") == 0);\n+\n+ for (i = 0; i < sizeof(backends) / sizeof(backends[0]); ++i) { + const amduat_tgk_conformance_backend_t *backend = &backends[i]; + char *backend_workdir = NULL; + + if (!conformance_backend_selected(backend_selection, backend->name)) { + continue; + } + if (backend->optional && selection_all && !include_optional) { + continue; + } + + if (!conformance_prepare_workdir(workdir, backend->name, "tgk", + &backend_workdir)) { + fprintf(stderr, "workdir prep failed for %s\n", backend->name); + free(results); + return 1; + } + + results = (amduat_conformance_backend_result_t *)realloc( + results, (results_len + 1u) * sizeof(*results)); + if (results == NULL) { + free(backend_workdir); + return 1; + } + + results[results_len].backend = backend->name; + results[results_len].status = conformance_run_backend(backend, + backend_workdir); + if (verbose) { + fprintf(stdout, "tgk %s %s\n", backend->name, + results[results_len].status == 0 ? "PASS" : "FAIL"); + } + results_len++; + free(backend_workdir); + } + + *out_results = results; + *out_len = results_len; + + for (i = 0; i < results_len; ++i) { + if (results[i].status != 0) { + return 1; + } + } + + return 0; +} diff --git a/tests/conformance/tgk_store/tgk_store_conformance.h b/tests/conformance/tgk_store/tgk_store_conformance.h new file mode 100644 index 0000000..7db67ac --- /dev/null +++ b/tests/conformance/tgk_store/tgk_store_conformance.h @@ -0,0 +1,26 @@ +#ifndef AMDUAT_TESTS_CONFORMANCE_TGK_STORE_CONFORMANCE_H +#define AMDUAT_TESTS_CONFORMANCE_TGK_STORE_CONFORMANCE_H + +#include "tests/conformance/conformance_result.h" + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +int amduat_tgk_store_conformance_run( + const char *backend_selection, + const char *fixtures_root, + const char *workdir, + bool include_optional, + bool verbose, + amduat_conformance_backend_result_t **out_results, + size_t *out_len); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* AMDUAT_TESTS_CONFORMANCE_TGK_STORE_CONFORMANCE_H */ diff --git a/tests/fixtures/conformance/asl_put_get/artifacts/a0.bin b/tests/fixtures/conformance/asl_put_get/artifacts/a0.bin new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/conformance/asl_put_get/artifacts/a1.bin b/tests/fixtures/conformance/asl_put_get/artifacts/a1.bin new file mode 100644 index 0000000..9e04b85 --- /dev/null +++ b/tests/fixtures/conformance/asl_put_get/artifacts/a1.bin @@ -0,0 +1 @@ +hello amduat \ No newline at end of file diff --git a/tests/fixtures/conformance/asl_put_get/artifacts/a2.bin b/tests/fixtures/conformance/asl_put_get/artifacts/a2.bin new file mode 100644 index 0000000000000000000000000000000000000000..c86626638e0bc8cf47ca49bb1525b40e9737ee64 GIT binary patch literal 256 zcmV+b0ssC00RjUA1qKHQ2?`4g4Gs?w5fT#=6&4p585$cL9UdPbAtECrB_<~*DJm;0 zEiNxGF)}kWH8wXmIXXK$Jw87`K|(`BMMg(RNlHshO-@fxQBqS>RaRG6Sz23MU0z>c zVPa!sWoBn+X=-b1ZEkOHadLBXb#`}nd3t+%eSUv{fr5jCg@%WSiHeJijgF6yk&=^? zm6n&7nVOrNot~edp`xRtrKYE-sj922t*)=Iv9hzYwYImoxw^Z&y}rM|!NSAD#m2|T z$;!*j&Cbuz(bCh@)z;V8+1lIO-QM5e;o{@u<>u$;>FVq3?e6dJ@$&QZ_4fDp`TG0( G{r>;0_J4r@ literal 0 HcmV?d00001 diff --git a/tests/fixtures/conformance/asl_put_get/manifest.json b/tests/fixtures/conformance/asl_put_get/manifest.json new file mode 100644 index 0000000..6760b6c --- /dev/null +++ b/tests/fixtures/conformance/asl_put_get/manifest.json @@ -0,0 +1,8 @@ +{ + "schema": "amduat.conformance.fixtures.v0", + "fixtures": [ + {"id": "empty", "artifact": "a0.bin", "notes": "empty payload"}, + {"id": "hello", "artifact": "a1.bin", "notes": "ascii payload"}, + {"id": "binary", "artifact": "a2.bin", "notes": "0..255"} + ] +} diff --git a/tests/vectors/pel_trace_tgk_test_vectors_v1.json b/tests/vectors/pel_trace_tgk_test_vectors_v1.json new file mode 100644 index 0000000..e984c75 --- /dev/null +++ b/tests/vectors/pel_trace_tgk_test_vectors_v1.json @@ -0,0 +1,118 @@ +{ + "hash_id": "0x0001", + "artifacts": [ + { + "name": "scheme_desc", + "case": null, + "type_tag": "0x00000100", + "artifact_bytes_hex": "0100000100000000000000001f00010000001150454c2f50524f4752414d2d4441472f310000010101010000", + "ref": { + "hash_id": "0x0001", + "digest_hex": "c50fb2a734a5cc233c3875b70a7d96eaad374f000029771d8bef1af2cd6384dd" + } + }, + { + "name": "program", + "case": null, + "type_tag": "0x00000101", + "artifact_bytes_hex": "01000001010000000000000036000100000001000000000000001070656c2e62797465732e706172616d73000000010000000000000000000000010000000000000000", + "ref": { + "hash_id": "0x0001", + "digest_hex": "f59617255b09759dfff00952babc61ee9b4163cf4715a181539c5294fa4aa63d" + } + }, + { + "name": "result_case_a", + "case": "A", + "type_tag": "0x00000103", + "artifact_bytes_hex": "010000010300000000000000d70001000000220001c50fb2a734a5cc233c3875b70a7d96eaad374f000029771d8bef1af2cd6384dd000000220001f59617255b09759dfff00952babc61ee9b4163cf4715a181539c5294fa4aa63d00000000000000010000002200013e7077fd2f66d689e0cee6a7cf5b37bf2dca7c979af356d0a31cbc5c85605c7d010000002200013e7077fd2f66d689e0cee6a7cf5b37bf2dca7c979af356d0a31cbc5c85605c7d0000000100000000220001c50fb2a734a5cc233c3875b70a7d96eaad374f000029771d8bef1af2cd6384dd000000000000000000", + "ref": { + "hash_id": "0x0001", + "digest_hex": "9aec50c18bd4e079da3acf54bc17598a404c08395044f11a462f9326b3ececc0" + } + }, + { + "name": "result_case_b", + "case": "B", + "type_tag": "0x00000103", + "artifact_bytes_hex": "0100000103000000000000008b0001000000220001c50fb2a734a5cc233c3875b70a7d96eaad374f000029771d8bef1af2cd6384dd000000220001f59617255b09759dfff00952babc61ee9b4163cf4715a181539c5294fa4aa63d0000000000000000000000000103000000220001c50fb2a734a5cc233c3875b70a7d96eaad374f000029771d8bef1af2cd6384dd030000000300000000", + "ref": { + "hash_id": "0x0001", + "digest_hex": "97c4e17c93a1539794040d0bb0cd79a0bd173ab38a530e9503435029bc6e57a9" + } + }, + { + "name": "trace_case_a_with_exec_result_ref", + "case": "A", + "type_tag": "0x00000102", + "artifact_bytes_hex": "010000010200000000000000f90001000000220001c50fb2a734a5cc233c3875b70a7d96eaad374f000029771d8bef1af2cd6384dd000000220001f59617255b09759dfff00952babc61ee9b4163cf4715a181539c5294fa4aa63d000000000000010000002200019aec50c18bd4e079da3acf54bc17598a404c08395044f11a462f9326b3ececc000000000010000002200013e7077fd2f66d689e0cee6a7cf5b37bf2dca7c979af356d0a31cbc5c85605c7d00000001000000000000001070656c2e62797465732e706172616d73000000010000000000000000010000002200013e7077fd2f66d689e0cee6a7cf5b37bf2dca7c979af356d0a31cbc5c85605c7d00000000", + "ref": { + "hash_id": "0x0001", + "digest_hex": "b303ac68081673e7785e4765d895efed475a5d1c1b4f2a675b42df40ef06913c" + } + }, + { + "name": "trace_case_b_with_exec_result_ref", + "case": "B", + "type_tag": "0x00000102", + "artifact_bytes_hex": "010000010200000000000000840001000000220001c50fb2a734a5cc233c3875b70a7d96eaad374f000029771d8bef1af2cd6384dd000000220001f59617255b09759dfff00952babc61ee9b4163cf4715a181539c5294fa4aa63d0303000000030100000022000197c4e17c93a1539794040d0bb0cd79a0bd173ab38a530e9503435029bc6e57a9000000000000000000", + "ref": { + "hash_id": "0x0001", + "digest_hex": "23eb6d3f723a18d265c9e3f049b033623b219d0ff828e227ece1d557f223e4f8" + } + }, + { + "name": "edge_case_a_payload_result_ref", + "case": "A", + "type_tag": "0x00000201", + "artifact_bytes_hex": "010000020100000000000000a600010000001000000001000000220001f59617255b09759dfff00952babc61ee9b4163cf4715a181539c5294fa4aa63d000000020000002200013e7077fd2f66d689e0cee6a7cf5b37bf2dca7c979af356d0a31cbc5c85605c7d0000002200019aec50c18bd4e079da3acf54bc17598a404c08395044f11a462f9326b3ececc00000002200019aec50c18bd4e079da3acf54bc17598a404c08395044f11a462f9326b3ececc0", + "ref": { + "hash_id": "0x0001", + "digest_hex": "21de6c0c1fb9112a485558ed5596e36594ecc16b7869f1ef4d4dd3cd74c8a6c4" + } + }, + { + "name": "edge_case_b_payload_result_ref", + "case": "B", + "type_tag": "0x00000201", + "artifact_bytes_hex": "0100000201000000000000008000010000001000000001000000220001f59617255b09759dfff00952babc61ee9b4163cf4715a181539c5294fa4aa63d0000000100000022000197c4e17c93a1539794040d0bb0cd79a0bd173ab38a530e9503435029bc6e57a900000022000197c4e17c93a1539794040d0bb0cd79a0bd173ab38a530e9503435029bc6e57a9", + "ref": { + "hash_id": "0x0001", + "digest_hex": "3aa41e2c2c5ad9884a00fb047a0728ce163e3be5493d55c1d4261be3b116f317" + } + } + ], + "links": { + "A": { + "program_ref": "0x0001:f59617255b09759dfff00952babc61ee9b4163cf4715a181539c5294fa4aa63d", + "execution_result_ref": "0x0001:9aec50c18bd4e079da3acf54bc17598a404c08395044f11a462f9326b3ececc0", + "trace_ref": "0x0001:b303ac68081673e7785e4765d895efed475a5d1c1b4f2a675b42df40ef06913c", + "edge_ref": "0x0001:21de6c0c1fb9112a485558ed5596e36594ecc16b7869f1ef4d4dd3cd74c8a6c4", + "expected_edge_body": { + "from": [ + "0x0001:f59617255b09759dfff00952babc61ee9b4163cf4715a181539c5294fa4aa63d" + ], + "to": [ + "0x0001:3e7077fd2f66d689e0cee6a7cf5b37bf2dca7c979af356d0a31cbc5c85605c7d", + "0x0001:9aec50c18bd4e079da3acf54bc17598a404c08395044f11a462f9326b3ececc0" + ], + "payload": "0x0001:9aec50c18bd4e079da3acf54bc17598a404c08395044f11a462f9326b3ececc0" + } + }, + "B": { + "program_ref": "0x0001:f59617255b09759dfff00952babc61ee9b4163cf4715a181539c5294fa4aa63d", + "execution_result_ref": "0x0001:97c4e17c93a1539794040d0bb0cd79a0bd173ab38a530e9503435029bc6e57a9", + "trace_ref": "0x0001:23eb6d3f723a18d265c9e3f049b033623b219d0ff828e227ece1d557f223e4f8", + "edge_ref": "0x0001:3aa41e2c2c5ad9884a00fb047a0728ce163e3be5493d55c1d4261be3b116f317", + "expected_edge_body": { + "from": [ + "0x0001:f59617255b09759dfff00952babc61ee9b4163cf4715a181539c5294fa4aa63d" + ], + "to": [ + "0x0001:97c4e17c93a1539794040d0bb0cd79a0bd173ab38a530e9503435029bc6e57a9" + ], + "payload": "0x0001:97c4e17c93a1539794040d0bb0cd79a0bd173ab38a530e9503435029bc6e57a9" + } + } + } +} diff --git a/tier1/hash-asl1.md b/tier1/hash-asl1.md index 654a0cc..fffc82b 100644 --- a/tier1/hash-asl1.md +++ b/tier1/hash-asl1.md @@ -2,9 +2,9 @@ Status: Approved Owner: Niklas Rydberg -Version: 0.2.4 +Version: 0.2.5 SoT: Yes -Last Updated: 2025-11-16 +Last Updated: 2026-02-20 Linked Phase Pack: N/A Tags: [deterministic, registry] @@ -364,9 +364,71 @@ Higher layers can use `"ASL1"`’s crypto agility by: --- +## 6.4 Identity Policy (Normative — Amduat 2.0 Baseline) + +This section defines the baseline identity policy for Amduat 2.0. + +### 6.4.1 Baseline Identity Algorithm + +For Amduat 2.0 baseline conformance: + +> All identity-critical `Reference` values MUST use +> `hash_id = 0x0001` (HASH-ASL1-256), +> unless a higher-level profile explicitly specifies otherwise. + +Identity-critical contexts include, but are not limited to: + +* Artifact addressing in `ASL/1-STORE` +* Provenance edges in `TGK/1-CORE` +* Execution traces in `PEL/1` +* Any content-addressed storage or federation surface +* Any equality or integrity comparison of Artifacts + +Unless explicitly overridden by a profile: + +* `HASH-ASL1-256` is the canonical algorithm. +* No other `HashId` is required for baseline conformance. + +### 6.4.2 Multi-Hash Extension (Optional Profiles) + +Higher-level profiles MAY define multi-hash policies. + +Such profiles MUST: + +1. Explicitly enumerate the additional `HashId` values used. +2. Define how multiple References for the same Artifact are represented. +3. Define equality semantics (e.g., which hash governs canonical identity). +4. Define migration and downgrade handling rules. + +Unless a profile explicitly requires multi-hash: + +* Implementations MUST NOT assume that multiple hash algorithms are present. +* Baseline-conformant systems MUST interoperate using `0x0001` only. + +### 6.4.3 Forward Compatibility + +The presence of a `HashId` field in `Reference` enables future algorithm agility. + +However: + +* The meaning of existing `HashId` assignments MUST remain immutable. +* Migration to additional algorithms MUST NOT alter the digest produced by existing assignments for identical input bytes. +* Historical References using `0x0001` remain valid indefinitely. + +### 6.4.4 Federation Invariant (Baseline) + +For Amduat 2.0 baseline systems: + +> Two nodes that possess identical canonical ArtifactBytes +> MUST derive identical `Reference { hash_id=0x0001, digest }` values. + +This invariant defines baseline interoperability. + +--- + ## 7. Conformance -An implementation is **HASH/ASL1–conformant** (v0.2.4) if: +An implementation is **HASH/ASL1–conformant** (v0.2.5) if: 1. **Correct HASH-ASL1-256 implementation** @@ -487,4 +549,5 @@ This `Reference` can be used consistently across `ASL/1-STORE`, `TGK/1-CORE`, `P ## Document History +* **0.2.5 (2026-02-20):** Added baseline identity policy (§6.4) and clarified multi-hash profile requirements. * **0.2.4 (2025-11-16):** Registered as Tier-1 spec and aligned to the Amduat 2.0 substrate baseline.