diff --git a/CMakeLists.txt b/CMakeLists.txt index 90ef5d9..02e104c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,7 +25,8 @@ target_link_libraries(amduat_federation ) set(amduatd_sources src/amduatd.c src/amduatd_http.c src/amduatd_caps.c - src/amduatd_space.c src/amduatd_concepts.c) + src/amduatd_space.c src/amduatd_concepts.c + src/amduatd_store.c) if(AMDUATD_ENABLE_UI) list(APPEND amduatd_sources src/amduatd_ui.c) endif() @@ -42,8 +43,9 @@ target_compile_definitions(amduatd ) target_link_libraries(amduatd - PRIVATE amduat_tgk amduat_pel amduat_format amduat_asl_store_fs amduat_asl - amduat_enc amduat_hash_asl1 amduat_util amduat_federation + PRIVATE amduat_tgk amduat_pel amduat_format amduat_asl_store_fs + amduat_asl_store_index_fs amduat_asl amduat_enc amduat_hash_asl1 + amduat_util amduat_federation ) add_executable(amduat_pel_gc @@ -61,3 +63,22 @@ target_link_libraries(amduat_pel_gc PRIVATE amduat_asl_store_fs amduat_asl_record amduat_asl amduat_enc amduat_hash_asl1 amduat_pel amduat_util ) + +enable_testing() + +add_executable(amduatd_test_store_backend + tests/test_amduatd_store_backend.c + src/amduatd_store.c +) + +target_include_directories(amduatd_test_store_backend + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/vendor/amduat/include +) + +target_link_libraries(amduatd_test_store_backend + PRIVATE amduat_asl_store_fs amduat_asl_store_index_fs +) + +add_test(NAME amduatd_store_backend COMMAND amduatd_test_store_backend) diff --git a/README.md b/README.md index d4d8c86..61bda31 100644 --- a/README.md +++ b/README.md @@ -33,12 +33,20 @@ Initialize a store: ./vendor/amduat/build/amduat-asl init --root .amduat-asl ``` -Run the daemon: +Run the daemon (fs backend default): ```sh ./build/amduatd --root .amduat-asl --sock amduatd.sock ``` +Run the daemon with the index-backed store: + +```sh +./build/amduatd --root .amduat-asl --sock amduatd.sock --store-backend index +``` + +Note: `/v1/fed/records` requires the index backend. + Dev loop (build + restart): ```sh diff --git a/docs/state_report.md b/docs/state_report.md new file mode 100644 index 0000000..d0143c5 --- /dev/null +++ b/docs/state_report.md @@ -0,0 +1,139 @@ +# Repo State Report + +## 1) Build + test entrypoints +- Tests: `ctest --test-dir build` (top-level build includes vendor tests via `add_subdirectory(vendor/amduat)` in `CMakeLists.txt`); core-only tests: `ctest --test-dir vendor/amduat/build` (binaries present in `vendor/amduat/build`). +- Main daemon: `./build/amduatd --root .amduat-asl --sock amduatd.sock` (`README.md`, `src/amduatd.c`). +- Dev loop: `./scripts/dev-restart.sh` (build + restart in `scripts/dev-restart.sh`). +- Core CLIs/tools: `./vendor/amduat/build/amduat-asl ...` (store/init/log/index commands in `vendor/amduat/src/tools/amduat_asl_cli.c`), `./vendor/amduat/build/amduat-pel ...` (PEL tooling in `vendor/amduat/src/tools/amduat_pel_cli.c`), `./build/amduat-pel gc --root .amduat-asl` (GC tool in `src/amduat_pel_gc.c`). +- Languages/toolchains: C11 + CMake (`CMakeLists.txt`, `vendor/amduat/CMakeLists.txt`), shell scripts (`scripts/*.sh`), embedded HTML/JS/CSS in C strings (`src/amduatd_ui.c`). +- CI config: none found (no `.github/workflows/*`, `.gitlab-ci.yml`, etc.). + +## 2) Top-level architecture map (as implemented) +- Daemon runtime + HTTP surface: Paths `src/amduatd.c`, `src/amduatd_http.c`, `src/amduatd_http.h`, `src/amduatd_ui.c`; main types `amduatd_cfg_t`, `amduatd_http_req_t`, `amduatd_http_resp_t`; entrypoints `main`, `amduatd_handle_conn`, `amduatd_http_read_req`; wiring: initializes store fs config, concepts, caps, federation coord, then serves HTTP over Unix socket with a `select()` loop (`src/amduatd.c`). +- Capability tokens: Paths `src/amduatd_caps.c`, `src/amduatd_caps.h`; main types `amduatd_caps_t`, `amduatd_caps_token_t`; entrypoints `amduatd_caps_init`, `amduatd_caps_handle`, `amduatd_caps_check`; wiring: per-request token validation with optional space/pointer scoping (`src/amduatd_caps.c`). +- Concepts/relations + edge graph index: Paths `src/amduatd_concepts.c`, `src/amduatd_concepts.h`; main types `amduatd_concepts_t`, `amduatd_edge_index_state_t`, `amduatd_edge_list_t`; entrypoints `amduatd_concepts_init`, `amduatd_concepts_refresh_edges`, `amduatd_handle_get_relations`, `amduatd_handle_get_concepts`; wiring: edge records stored in a collection log, plus derived edge graph + index pointer state (`src/amduatd_concepts.c`). +- Space scoping: Paths `src/amduatd_space.c`, `src/amduatd_space.h`; main types `amduatd_space_t`; entrypoints `amduatd_space_init`, `amduatd_space_scope_name`, `amduatd_space_unscoped_name`; wiring: scopes pointer/collection names with prefix `space//` when enabled (`src/amduatd_space.c`). +- Federation coordinator: Paths `federation/coord.c`, `federation/coord.h`; main types `amduat_fed_coord_t`, `amduat_fed_coord_cfg_t`; entrypoints `amduat_fed_coord_open`, `amduat_fed_coord_tick`, `amduat_fed_coord_get_status`; wiring: daemon ticks coordinator on interval (`src/amduatd.c`). +- Federation transport: Paths `federation/transport_unix.c`, `federation/transport_stub.c`; main types `amduat_fed_transport_t`; entrypoints `amduat_fed_transport_unix_ops`, `amduat_fed_transport_stub_ops`; wiring: daemon currently uses stub transport (`src/amduatd.c`). +- CAS store (filesystem): Paths `vendor/amduat/src/adapters/asl_store_fs/*`, `vendor/amduat/include/amduat/asl/store.h`; main types `amduat_asl_store_fs_t`, `amduat_asl_store_t`; entrypoints `amduat_asl_store_fs_init`, `amduat_asl_store_put`, `amduat_asl_store_get`; wiring: used by daemon and tools as the backing store. +- Pointer store: Paths `vendor/amduat/src/adapters/asl_pointer_fs/asl_pointer_fs.c`; main types `amduat_asl_pointer_store_t`; entrypoints `amduat_asl_pointer_get`, `amduat_asl_pointer_cas`; wiring: used by log store, collection store, concept index, and caps checks. +- Log store: Paths `vendor/amduat/src/core/asl_log_store.c`, `vendor/amduat/include/amduat/asl/log_store.h`; main types `amduat_asl_log_store_t`, `amduat_asl_log_entry_t`; entrypoints `amduat_asl_log_append`, `amduat_asl_log_read`; wiring: collection store writes log chunks in CAS and advances pointer heads. +- Collection store: Paths `vendor/amduat/src/core/asl_collection.c`, `vendor/amduat/include/amduat/asl/collection.h`; main types `amduat_asl_collection_store_t`; entrypoints `amduat_asl_collection_append`, `amduat_asl_collection_snapshot`, `amduat_asl_collection_read`; wiring: concept edges are stored as records appended to a collection log. +- ASL index store: Paths `vendor/amduat/src/adapters/asl_store_index_fs/*`; main types `amduat_asl_store_index_fs_t`, `amduat_asl_index_state_t`; entrypoints `amduat_asl_store_index_fs_put_indexed`, `amduat_asl_store_index_fs_log_scan`, `amduat_asl_store_index_fs_gc`; wiring: available in core but not wired into amduatd (amduatd uses store_fs ops). +- PEL execution + materialization cache: Paths `vendor/amduat/src/pel_stack/surf/surf.c`, `vendor/amduat/src/adapters/asl_materialization_cache_fs/asl_materialization_cache_fs.c`; main types `amduat_pel_program_t`, `amduat_pel_run_result_t`; entrypoints `amduat_pel_surf_run_with_result`, `amduat_asl_materialization_cache_fs_get`, `amduat_asl_materialization_cache_fs_put`; wiring: used by `/v1/pel/run` and by the collection view implementation. +- Derivation index (core-only): Paths `vendor/amduat/src/adapters/asl_derivation_index_fs/asl_derivation_index_fs.c`, `vendor/amduat/include/amduat/asl/asl_derivation_index_fs.h`; main types `amduat_asl_derivation_index_fs_t`, `amduat_asl_derivation_record_t`; entrypoints `amduat_asl_derivation_index_fs_add`, `amduat_asl_derivation_index_fs_list`; wiring: used by CLI tools (`vendor/amduat/src/tools/amduat_asl_cli.c`, `vendor/amduat/src/tools/amduat_pel_cli.c`), not by amduatd. + +## 3) “Space” concept: definition + storage layout +- Definition: “space” is a daemon-level scoping prefix for pointer/collection names, not a separate store; it rewrites names into `space//...` when enabled (`src/amduatd_space.c`, `amduatd_space_scope_name`, `amduatd_space_unscoped_name`). +- Storage layout: scoped names are stored in the pointer store under `root/pointers/space//.../head` per `amduat_asl_pointer_build_head_path` (`vendor/amduat/src/adapters/asl_pointer_fs/asl_pointer_fs.c`). No dedicated per-space directory outside pointer name paths is created by amduatd. +- Identification: `space_id` is a pointer-name-safe token without `/`, validated by `amduatd_space_space_id_is_valid` (calls `amduat_asl_pointer_name_is_valid`) (`src/amduatd_space.c`). +- Multi-space support: only one space can be active per daemon process via `--space` (`src/amduatd.c`, `amduatd_space_init`); code does not show per-request space selection. + +## 4) Source of truth: pointers + logs (actual) +- Canonical “head/root” pointer for a space: there is no single global space root pointer. The main space-scoped heads used by amduatd are the edge index head (`daemon/edges/index/head` scoped by `amduatd_space_edges_index_head_name`) and collection snapshot/log heads for the edge collection (see below) (`src/amduatd_space.c`, `src/amduatd_concepts.c`). +- Pointer storage location + format: + - Location: `root/pointers//head` using path segments from the pointer name (`amduat_asl_pointer_build_head_path` in `vendor/amduat/src/adapters/asl_pointer_fs/asl_pointer_fs.c`). + - Format: `ASLPTR1` magic with version and flags, then name/ref/prev fields encoded via `amduat_enc_asl1_core_encode_reference_v1` (`amduat_asl_pointer_read_head` and `amduat_asl_pointer_write_head` in `vendor/amduat/src/adapters/asl_pointer_fs/asl_pointer_fs.c`). + - Read/write: `amduat_asl_pointer_get` and `amduat_asl_pointer_cas` (`vendor/amduat/src/adapters/asl_pointer_fs/asl_pointer_fs.c`). +- Collection heads used by amduatd concepts: + - Snapshot head pointer: `collection//head` built by `amduatd_build_collection_head_name` (`src/amduatd_concepts.c`) and also by `amduat_asl_collection_build_head_name` (`vendor/amduat/src/core/asl_collection.c`). + - Log head pointer: `log/collection//log/head` built by `amduatd_build_collection_log_head_name` (`src/amduatd_concepts.c`) and `amduat_asl_log_build_pointer_name` (`vendor/amduat/src/core/asl_log_store.c`). +- Log entry schema (ASL index log): `amduat_asl_log_record_t {logseq, record_type, payload, record_hash}` with record types defined in `vendor/amduat/include/amduat/enc/asl_log.h`. +- Log append behavior: + - Collection log: `amduat_asl_log_append` writes CAS log chunks (`ASL_LOG_CHUNK_1`) and advances the log head pointer via CAS (`vendor/amduat/src/core/asl_log_store.c`). + - Index log: `amduat_asl_store_index_fs_append_log_record` appends to `index/log.asl` (`vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs.c`). +- Integrity mechanisms: + - Collection log chunks form a chain via `prev_ref` and are rooted at the log head pointer (`amduat_asl_log_append`, `amduat_asl_log_chunk_t` in `vendor/amduat/src/core/asl_log_store.c`). + - Index log uses a hash chain where `record_hash = SHA256(prev_hash || logseq || type || payload_len || payload)` (`amduat_asl_store_index_fs_log_hash_record` in `vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs.c`). +- Log traversal: + - Collection log: `amduat_asl_log_read` walks the head pointer and `prev_ref` chain to gather chunks (`vendor/amduat/src/core/asl_log_store.c`). + - Index log: replay happens via `amduat_asl_store_index_fs_build_replay_state` using `amduat_asl_replay_apply_log` (`vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs.c`, `vendor/amduat/include/amduat/asl/index_replay.h`). +- Other persistent state that can diverge from logs/pointers: + - Edge index state and edge graph artifacts (`tgk/edge_index_state` record + `TGK_EDGE_GRAPH_1` artifact) are separate derived state from the edge collection log (`src/amduatd_concepts.c`). + - Legacy `.amduatd.edges` file (used only for migration) can diverge until migrated (`src/amduatd_concepts.c`). + - Materialization cache in `index/materializations` can diverge from CAS and is treated as a cache (`vendor/amduat/src/adapters/asl_materialization_cache_fs/asl_materialization_cache_fs.c`). + +## 5) CAS (content-addressed store) +- Hash algorithm: SHA-256 is the only defined ASL1 hash id and default (`AMDUAT_HASH_ASL1_ID_SHA256` in `vendor/amduat/include/amduat/hash/asl1.h`, default config in `vendor/amduat/src/adapters/asl_store_fs/asl_store_fs_meta.c`). +- Object types stored: arbitrary artifacts with optional type tags (`amduat_artifact_t` in `vendor/amduat/include/amduat/asl/core.h`); records are stored via `amduat_asl_record_store_put` (schema + payload) in `vendor/amduat/src/core/asl_record.c` and used by amduatd for concept edges (`src/amduatd_concepts.c`). +- Address format: refs are `{hash_id, digest}`; on disk, objects are stored under hex-digest filenames derived from the raw digest (`amduat_asl_store_fs_layout_build_paths` in `vendor/amduat/src/adapters/asl_store_fs/asl_store_fs_layout.c`). +- Disk layout and APIs: + - Layout: `root/objects/////` (`amduat_asl_store_fs_layout_build_paths` in `vendor/amduat/src/adapters/asl_store_fs/asl_store_fs_layout.c`). + - APIs: `amduat_asl_store_put/get` (generic) and `amduat_asl_store_fs_ops` (filesystem implementation) (`vendor/amduat/include/amduat/asl/store.h`, `vendor/amduat/src/adapters/asl_store_fs/asl_store_fs.c`). +- Garbage collection: + - Exists as a standalone tool: `amduat-pel gc` uses `amduat_asl_gc_fs_run` to mark from pointer/log/collection roots and optionally delete artifacts (`src/amduat_pel_gc.c`, `src/asl_gc_fs.c`). +- Pinning concept: no explicit pin API found; reachability is derived from pointers/logs/snapshots (GC uses those roots) (`src/asl_gc_fs.c`). + +## 6) Deterministic derivations (current reality) +- Derivation-related types exist in core: `amduat_pel_derivation_sid_input_t`, `amduat_pel_derivation_sid_compute` (`vendor/amduat/include/amduat/pel/derivation_sid.h`, `vendor/amduat/src/core/derivation_sid.c`). +- Execution model: PEL DAGs execute in-process (no external sandbox) via `amduat_pel_program_dag_exec_trace` and `amduat_pel_surf_run_with_result` (`vendor/amduat/src/pel_stack/surf/surf.c`). +- Inputs: referenced as CAS refs from the store (`amduat_pel_surf_run_with_result` loads program/input/params artifacts by ref in `vendor/amduat/src/pel_stack/surf/surf.c`). +- Outputs: stored back into CAS (`amduat_asl_store_put` in `vendor/amduat/src/pel_stack/surf/surf.c`); results/traces/receipts are separate artifacts (`amduat_enc_pel1_result`, `amduat_enc_pel_trace_dag` etc. used in `src/amduatd.c`). +- Provenance/audit: optional FER1 receipt data is accepted/serialized in `/v1/pel/run` (`src/amduatd.c`), but no daemon-side derivation index is written. +- Derivation index persistence: exists in core (`vendor/amduat/src/adapters/asl_derivation_index_fs/asl_derivation_index_fs.c`) and CLI tools (`vendor/amduat/src/tools/amduat_asl_cli.c`, `vendor/amduat/src/tools/amduat_pel_cli.c`), but amduatd does not write derivation records (no references in `src/`). + +## 7) ASL index: what it is and what it depends on +- Storage location: `root/index/` with `log.asl`, `segments/`, `blocks/`, and `snapshots/` (layout in `vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs_layout.c`). +- Derived vs authoritative: the index log (`index/log.asl`) and segment files are the authoritative index state; higher-level state can be rebuilt by replaying the log plus snapshots (`amduat_asl_store_index_fs_build_replay_state` + `amduat_asl_replay_apply_log` in `vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs.c`). +- Build/update path: `amduat_asl_store_index_fs_put_indexed` appends log records, writes segments/blocks, and may create snapshots (`vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs.c`). +- Queries relying on it: `amduat_asl_log_scan`, tombstone operations, and `amduat_asl_index_current_state` are implemented by the index-backed store ops (`vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs.c`, `vendor/amduat/include/amduat/asl/store.h`). +- What breaks if deleted: if `index/` (including `log.asl`) is removed, index-backed stores cannot answer log/state/tombstone queries; recovery requires a log to replay, which lives under `index/` itself (`vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs.c`). + +## 8) Daemon / runtime model (if any) +- Daemon process: `amduatd` in `src/amduatd.c` binds a Unix domain socket, listens, and handles one connection per `accept()` in a single-threaded loop using `select()`. +- Single-space or multi-space: single-space per daemon process via `--space` (`src/amduatd.c`, `src/amduatd_space.c`). +- Config mechanism: CLI flags `--root`, `--sock`, `--space`, `--migrate-unscoped-edges`, `--edges-refresh-ms`, `--allow-uid`, `--allow-user`, `--enable-cap-reads` (`src/amduatd.c`). +- IPC/RPC/HTTP APIs: HTTP over Unix socket, routes in `src/amduatd.c` and handlers in `src/amduatd_concepts.c` and `src/amduatd_caps.c`. +- Background workers: federation tick every 1s (`AMDUATD_FED_TICK_MS`) and optional edge refresh on `--edges-refresh-ms` interval (`src/amduatd.c`). + +## 9) Projections / query layer +- Projection concept in this repo: the edge graph and edge index state are projections derived from the edge collection log (`AMDUATD_EDGE_INDEX_SCHEMA`, `amduatd_concepts_write_edge_index_state`, `amduatd_concepts_refresh_edges_internal` in `src/amduatd_concepts.c`). +- Generation + storage: edge graph stored as `TGK_EDGE_GRAPH_1` artifact plus pointer to `tgk/edge_index_state` record (`src/amduatd_concepts.c`). +- Derived from logs/CAS: refresh reads collection log entries via `amduat_asl_log_read` and rebuilds the edge list/graph (`src/amduatd_concepts.c`). +- Query APIs: HTTP endpoints for concepts/relations/resolve in `src/amduatd_concepts.c` and routing in `src/amduatd.c`; collection view is generated by a PEL program over collection snapshot + log (`amduatd_collection_view` in `src/amduatd_concepts.c`). + +## 10) Federation / collaboration +- Networking code: `federation/transport_unix.c` builds HTTP requests over Unix sockets for `/v1/fed/records` and `/v1/fed/artifacts` (see `amduat_fed_transport_unix_ops`). +- Federation coordinator: `federation/coord.c` maintains registry state and a cached view (`amduat_fed_coord_t`). +- Daemon behavior: federation is wired but uses the stub transport (`amduat_fed_transport_stub_ops` in `src/amduatd.c`), so no remote sync by default. +- Remote refs/replication/merge/signatures: not implemented in daemon beyond read-only federation endpoints; no CRDT/merge logic found in this repo (coordinator logic delegates to core types in `vendor/amduat/include/amduat/fed/*`). + +## 11) Invariants (explicit + implicit) +- Pointer names are path-safe and forbid `..` segments; used throughout for space IDs and pointer names (`amduat_asl_pointer_name_is_valid` in `vendor/amduat/src/adapters/asl_pointer_fs/asl_pointer_fs.c`). +- Collection log append is append-only with CAS+pointer CAS; chunks point to previous chunk to form a chain (`amduat_asl_log_append`, `amduat_asl_log_chunk_t` in `vendor/amduat/src/core/asl_log_store.c`). +- Log chunk entries must be consistent about timestamp/actor presence across the batch (`amduat_asl_log_entries_consistent` in `vendor/amduat/src/core/asl_log_store.c`). +- Collection snapshot heads and log heads are stable pointer names derived from collection name (`amduat_asl_collection_build_head_name` and `amduat_asl_collection_build_log_name` in `vendor/amduat/src/core/asl_collection.c`). +- Index log hash chain uses `prev_hash` + record fields to compute the next hash (`amduat_asl_store_index_fs_log_hash_record` in `vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs.c`). +- Edge index state is written via pointer CAS, implying the index head should only advance if the expected previous ref matches (`amduatd_concepts_write_edge_index_state` in `src/amduatd_concepts.c`). + +## 12) Immediate risks of “parallel mechanisms” +- Edge index state vs edge collection log: the derived edge graph + `tgk/edge_index_state` can diverge from the log if refresh fails; there is a deterministic rebuild path by replaying the collection log (`amduatd_concepts_refresh_edges_internal` in `src/amduatd_concepts.c`). +- Legacy `.amduatd.edges` vs collection log: migration reads `.amduatd.edges` and writes into the collection log, so stale files can diverge until migrated (`amduatd_concepts_migrate_edges` in `src/amduatd_concepts.c`). +- Materialization cache vs CAS: cache entries are validated against CAS and are treated as a performance-only layer; missing or stale cache forces recompute (`amduat_asl_materialization_cache_fs_get` usage in `vendor/amduat/src/pel_stack/surf/surf.c`). +- ASL index files vs index log: segment/summary/snapshot files are derived from `index/log.asl`; if any are deleted, `amduat_asl_store_index_fs_build_replay_state` can rebuild from the log, but if `index/log.asl` is deleted the rebuild story is gone (`vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs.c`). +- Federation coordinator cached view vs store log: coordinator caches `last_view` in memory and refreshes on tick; divergence is possible across restarts or if tick stops (`federation/coord.c`, `src/amduatd.c`). + +## 13) Recommendation inputs (no decision yet) +- Current capability summary: + - Local Unix-socket HTTP daemon over a single ASL store root with artifact CRUD, concepts/relations, and PEL execution (`src/amduatd.c`, `src/amduatd_concepts.c`). + - CAS store on disk with pointer and log primitives; collection logs and snapshot pointers are wired (`vendor/amduat/src/adapters/asl_store_fs`, `vendor/amduat/src/core/asl_log_store.c`, `vendor/amduat/src/core/asl_collection.c`). + - Edge graph projection maintained as derived state from collection logs (`src/amduatd_concepts.c`). + - Federation coordinator scaffolding and endpoints, but using stub transport by default (`federation/coord.c`, `federation/transport_stub.c`, `src/amduatd.c`). + - Core tooling for index/derivation/GC exists in vendor and repo tools (`vendor/amduat/src/tools/*`, `src/amduat_pel_gc.c`). +- Top 5 unknowns/blockers for the next step (grounded in code): + - Whether amduatd should switch to `amduat_asl_store_index_fs_ops` to enable `amduat_asl_log_scan` and tombstones; current store fs ops do not implement log scan (`vendor/amduat/src/adapters/asl_store_fs/asl_store_fs.c`, `vendor/amduat/src/near_core/asl/store.c`). + - The intended authoritative store for federation records: amduatd’s `/v1/fed/records` relies on `amduat_asl_log_scan`, which is unsupported by store_fs (`src/amduatd.c`, `vendor/amduat/src/near_core/asl/store.c`). + - How/when to persist derivation index records from daemon PEL runs (present in core but unused in `src/`). + - Whether the “space” scope should be exposed as a first-class selector in the API (only CLI flag currently applies globally). + - The intended registry flow for federation (coordinator expects registry refs, but no daemon config path for them besides code defaults in `src/amduatd.c`). +- Top 5 next commit candidates grounded in repo reality (no new architecture): + - Wire amduatd to an index-backed store (use `amduat_asl_store_index_fs_ops`) so federation record scanning and index state endpoints are meaningful. + - Add a daemon flag/config to load a federation registry ref and domain id into `amduat_fed_coord_cfg_t` (values are currently hardcoded in `src/amduatd.c`). + - Persist derivation index records for `/v1/pel/run` outputs using existing core `amduat_asl_derivation_index_fs_*` APIs. + - Expose index state or health endpoints that surface `amduat_asl_index_current_state` (available in core API, already used in `/v1/fed/records`). + - Formalize edge index rebuild tooling (e.g., CLI switch or maintenance endpoint) using existing `amduatd_concepts_refresh_edges` and pointer state. +- Method used: + - Searched for key terms with `rg -n "cas|content address|hash|log|append|pointer|head|space|derivation|projection|ASL|index|daemon|server|coordinator|sync|replication"`. + - Read key sources: `src/amduatd.c`, `src/amduatd_concepts.c`, `src/amduatd_space.c`, `vendor/amduat/src/adapters/asl_store_fs/asl_store_fs_layout.c`, `vendor/amduat/src/adapters/asl_pointer_fs/asl_pointer_fs.c`, `vendor/amduat/src/core/asl_log_store.c`, `vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs_layout.c`. + - Ran CLIs: `./build/amduatd --help`, `./vendor/amduat/build/amduat-asl --help`, `./vendor/amduat/build/amduat-asl index state --root .amduat-asl`, `./vendor/amduat/build/amduat-asl log inspect --root .amduat-asl`. + - Tests were not executed; enumerated `ctest` commands above. diff --git a/src/amduatd.c b/src/amduatd.c index 6b79ad2..030498e 100644 --- a/src/amduatd.c +++ b/src/amduatd.c @@ -38,6 +38,7 @@ #include "amduatd_ui.h" #include "amduatd_caps.h" #include "amduatd_space.h" +#include "amduatd_store.h" #include #include @@ -1016,14 +1017,25 @@ static bool amduatd_handle_get_fed_records(int fd, "invalid limit\n", false); } } + if (store->ops.log_scan == NULL || store->ops.current_state == NULL) { + return amduatd_http_send_text(fd, 501, "Not Implemented", + "requires index backend\n", false); + } if (!amduat_asl_index_current_state(store, &state)) { return amduatd_http_send_text(fd, 500, "Internal Server Error", "store error\n", false); } - if (amduat_asl_log_scan(store, &records, &record_count) != - AMDUAT_ASL_STORE_OK) { - return amduatd_http_send_text(fd, 500, "Internal Server Error", - "log scan error\n", false); + { + amduat_asl_store_error_t scan_err; + scan_err = amduat_asl_log_scan(store, &records, &record_count); + if (scan_err == AMDUAT_ASL_STORE_ERR_UNSUPPORTED) { + return amduatd_http_send_text(fd, 501, "Not Implemented", + "requires index backend\n", false); + } + if (scan_err != AMDUAT_ASL_STORE_OK) { + return amduatd_http_send_text(fd, 500, "Internal Server Error", + "log scan error\n", false); + } } memset(&b, 0, sizeof(b)); @@ -4020,6 +4032,7 @@ static void amduatd_print_usage(FILE *stream) { " amduatd [--root PATH] [--sock PATH]\n" " [--space SPACE_ID] [--migrate-unscoped-edges]\n" " [--edges-refresh-ms MS]\n" + " [--store-backend fs|index]\n" " [--allow-uid UID] [--allow-user NAME]\n" " [--enable-cap-reads]\n" "\n" @@ -4036,10 +4049,11 @@ int main(int argc, char **argv) { const char *space_id = NULL; bool migrate_unscoped_edges = false; bool enable_cap_reads = false; + amduatd_store_backend_t store_backend = AMDUATD_STORE_BACKEND_FS; amduatd_cfg_t dcfg; amduatd_caps_t caps; amduat_asl_store_fs_config_t cfg; - amduat_asl_store_fs_t fs; + amduatd_store_ctx_t store_ctx; amduat_asl_store_t store; amduat_reference_t api_contract_ref; amduat_reference_t ui_ref; @@ -4098,6 +4112,15 @@ int main(int argc, char **argv) { return 2; } dcfg.edges_refresh_ms = (uint64_t)refresh_val; + } else if (strcmp(argv[i], "--store-backend") == 0) { + if (i + 1 >= argc) { + fprintf(stderr, "error: --store-backend requires a value\n"); + return 2; + } + if (!amduatd_store_backend_parse(argv[++i], &store_backend)) { + fprintf(stderr, "error: invalid --store-backend\n"); + return 2; + } } else if (strcmp(argv[i], "--allow-uid") == 0) { char *endp = NULL; unsigned long uid_val; @@ -4145,18 +4168,13 @@ int main(int argc, char **argv) { return 2; } - memset(&cfg, 0, sizeof(cfg)); - if (!amduat_asl_store_fs_load_config(root, &cfg)) { - fprintf(stderr, "error: failed to load store config: %s\n", root); - return 8; - } - - memset(&fs, 0, sizeof(fs)); - if (!amduat_asl_store_fs_init(&fs, cfg.config, root)) { + if (!amduatd_store_init(&store, &cfg, &store_ctx, root, store_backend)) { fprintf(stderr, "error: failed to initialize store: %s\n", root); return 8; } - amduat_asl_store_init(&store, cfg.config, amduat_asl_store_fs_ops(), &fs); + amduat_log(AMDUAT_LOG_INFO, + "store backend=%s", + amduatd_store_backend_name(store_backend)); if (!amduatd_caps_init(&caps, &dcfg, root)) { amduat_log(AMDUAT_LOG_WARN, "capabilities unavailable"); } diff --git a/src/amduatd_store.c b/src/amduatd_store.c new file mode 100644 index 0000000..5ef3818 --- /dev/null +++ b/src/amduatd_store.c @@ -0,0 +1,76 @@ +#include "amduatd_store.h" + +#include "amduat/asl/asl_store_fs_meta.h" + +#include + +bool amduatd_store_backend_parse(const char *value, + amduatd_store_backend_t *out_backend) { + if (value == NULL || out_backend == NULL) { + return false; + } + if (strcmp(value, "fs") == 0) { + *out_backend = AMDUATD_STORE_BACKEND_FS; + return true; + } + if (strcmp(value, "index") == 0) { + *out_backend = AMDUATD_STORE_BACKEND_INDEX; + return true; + } + return false; +} + +const char *amduatd_store_backend_name(amduatd_store_backend_t backend) { + switch (backend) { + case AMDUATD_STORE_BACKEND_FS: + return "fs"; + case AMDUATD_STORE_BACKEND_INDEX: + return "index"; + default: + return "unknown"; + } +} + +bool amduatd_store_init(amduat_asl_store_t *store, + amduat_asl_store_fs_config_t *cfg, + amduatd_store_ctx_t *ctx, + const char *root_path, + amduatd_store_backend_t backend) { + if (store == NULL || cfg == NULL || ctx == NULL || root_path == NULL) { + return false; + } + + memset(store, 0, sizeof(*store)); + memset(ctx, 0, sizeof(*ctx)); + memset(cfg, 0, sizeof(*cfg)); + + if (!amduat_asl_store_fs_load_config(root_path, cfg)) { + return false; + } + + if (backend == AMDUATD_STORE_BACKEND_FS) { + if (!amduat_asl_store_fs_init(&ctx->fs, cfg->config, root_path)) { + return false; + } + amduat_asl_store_init(store, + cfg->config, + amduat_asl_store_fs_ops(), + &ctx->fs); + return true; + } + + if (backend == AMDUATD_STORE_BACKEND_INDEX) { + if (!amduat_asl_store_index_fs_init(&ctx->index_fs, + cfg->config, + root_path)) { + return false; + } + amduat_asl_store_init(store, + cfg->config, + amduat_asl_store_index_fs_ops(), + &ctx->index_fs); + return true; + } + + return false; +} diff --git a/src/amduatd_store.h b/src/amduatd_store.h new file mode 100644 index 0000000..574810b --- /dev/null +++ b/src/amduatd_store.h @@ -0,0 +1,40 @@ +#ifndef AMDUATD_STORE_H +#define AMDUATD_STORE_H + +#include "amduat/asl/asl_store_fs.h" +#include "amduat/asl/asl_store_fs_meta.h" +#include "amduat/asl/asl_store_index_fs.h" +#include "amduat/asl/store.h" + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + AMDUATD_STORE_BACKEND_FS = 0, + AMDUATD_STORE_BACKEND_INDEX = 1 +} amduatd_store_backend_t; + +typedef struct { + amduat_asl_store_fs_t fs; + amduat_asl_store_index_fs_t index_fs; +} amduatd_store_ctx_t; + +bool amduatd_store_backend_parse(const char *value, + amduatd_store_backend_t *out_backend); + +const char *amduatd_store_backend_name(amduatd_store_backend_t backend); + +bool amduatd_store_init(amduat_asl_store_t *store, + amduat_asl_store_fs_config_t *cfg, + amduatd_store_ctx_t *ctx, + const char *root_path, + amduatd_store_backend_t backend); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* AMDUATD_STORE_H */ diff --git a/tests/test_amduatd_store_backend.c b/tests/test_amduatd_store_backend.c new file mode 100644 index 0000000..3b05aaf --- /dev/null +++ b/tests/test_amduatd_store_backend.c @@ -0,0 +1,109 @@ +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 200809L +#endif + +#include "amduatd_store.h" + +#include "amduat/asl/asl_store_fs_meta.h" +#include "amduat/enc/asl_log.h" + +#include +#include +#include + +static char *amduatd_test_make_temp_dir(void) { + char tmpl[] = "/tmp/amduatd-store-XXXXXX"; + char *dir = mkdtemp(tmpl); + size_t len; + char *copy; + if (dir == NULL) { + perror("mkdtemp"); + return NULL; + } + len = strlen(dir); + copy = (char *)malloc(len + 1u); + if (copy == NULL) { + fprintf(stderr, "failed to allocate temp dir copy\n"); + return NULL; + } + memcpy(copy, dir, len + 1u); + return copy; +} + +int main(void) { + char *root = amduatd_test_make_temp_dir(); + amduat_asl_store_fs_config_t cfg; + amduatd_store_ctx_t store_ctx; + amduat_asl_store_t store; + amduat_asl_log_record_t *records = NULL; + size_t record_count = 0; + amduat_asl_store_error_t scan_err; + + if (root == NULL) { + return 1; + } + + memset(&cfg, 0, sizeof(cfg)); + if (!amduat_asl_store_fs_init_root(root, NULL, &cfg)) { + fprintf(stderr, "failed to init store root\n"); + free(root); + return 1; + } + + memset(&store_ctx, 0, sizeof(store_ctx)); + memset(&store, 0, sizeof(store)); + if (!amduatd_store_init(&store, + &cfg, + &store_ctx, + root, + AMDUATD_STORE_BACKEND_FS)) { + fprintf(stderr, "failed to init fs backend\n"); + free(root); + return 1; + } + + if (store.ops.log_scan != NULL) { + fprintf(stderr, "fs backend unexpectedly supports log_scan\n"); + free(root); + return 1; + } + + scan_err = amduat_asl_log_scan(&store, &records, &record_count); + if (scan_err != AMDUAT_ASL_STORE_ERR_UNSUPPORTED) { + fprintf(stderr, "fs backend log_scan returned %d\n", (int)scan_err); + free(root); + return 1; + } + + memset(&store_ctx, 0, sizeof(store_ctx)); + memset(&store, 0, sizeof(store)); + if (!amduatd_store_init(&store, + &cfg, + &store_ctx, + root, + AMDUATD_STORE_BACKEND_INDEX)) { + fprintf(stderr, "failed to init index backend\n"); + free(root); + return 1; + } + + if (store.ops.log_scan == NULL) { + fprintf(stderr, "index backend missing log_scan\n"); + free(root); + return 1; + } + + scan_err = amduat_asl_log_scan(&store, &records, &record_count); + if (scan_err != AMDUAT_ASL_STORE_OK) { + fprintf(stderr, "index backend log_scan returned %d\n", (int)scan_err); + free(root); + return 1; + } + + if (records != NULL) { + amduat_enc_asl_log_free(records, record_count); + } + + free(root); + return 0; +}