Add selectable fs/index store backend for amduatd
This commit is contained in:
Carl Niklas Rydberg
parent
a299b6c463
commit
ebdd37cfcd
|
|
@ -25,7 +25,8 @@ target_link_libraries(amduat_federation
|
|||
)
|
||||
|
||||
set(amduatd_sources src/amduatd.c src/amduatd_http.c src/amduatd_caps.c
|
||||
src/amduatd_space.c src/amduatd_concepts.c)
|
||||
src/amduatd_space.c src/amduatd_concepts.c
|
||||
src/amduatd_store.c)
|
||||
if(AMDUATD_ENABLE_UI)
|
||||
list(APPEND amduatd_sources src/amduatd_ui.c)
|
||||
endif()
|
||||
|
|
@ -42,8 +43,9 @@ target_compile_definitions(amduatd
|
|||
)
|
||||
|
||||
target_link_libraries(amduatd
|
||||
PRIVATE amduat_tgk amduat_pel amduat_format amduat_asl_store_fs amduat_asl
|
||||
amduat_enc amduat_hash_asl1 amduat_util amduat_federation
|
||||
PRIVATE amduat_tgk amduat_pel amduat_format amduat_asl_store_fs
|
||||
amduat_asl_store_index_fs amduat_asl amduat_enc amduat_hash_asl1
|
||||
amduat_util amduat_federation
|
||||
)
|
||||
|
||||
add_executable(amduat_pel_gc
|
||||
|
|
@ -61,3 +63,22 @@ target_link_libraries(amduat_pel_gc
|
|||
PRIVATE amduat_asl_store_fs amduat_asl_record amduat_asl amduat_enc
|
||||
amduat_hash_asl1 amduat_pel amduat_util
|
||||
)
|
||||
|
||||
enable_testing()
|
||||
|
||||
add_executable(amduatd_test_store_backend
|
||||
tests/test_amduatd_store_backend.c
|
||||
src/amduatd_store.c
|
||||
)
|
||||
|
||||
target_include_directories(amduatd_test_store_backend
|
||||
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
|
||||
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src
|
||||
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/vendor/amduat/include
|
||||
)
|
||||
|
||||
target_link_libraries(amduatd_test_store_backend
|
||||
PRIVATE amduat_asl_store_fs amduat_asl_store_index_fs
|
||||
)
|
||||
|
||||
add_test(NAME amduatd_store_backend COMMAND amduatd_test_store_backend)
|
||||
|
|
|
|||
10
README.md
10
README.md
|
|
@ -33,12 +33,20 @@ Initialize a store:
|
|||
./vendor/amduat/build/amduat-asl init --root .amduat-asl
|
||||
```
|
||||
|
||||
Run the daemon:
|
||||
Run the daemon (fs backend default):
|
||||
|
||||
```sh
|
||||
./build/amduatd --root .amduat-asl --sock amduatd.sock
|
||||
```
|
||||
|
||||
Run the daemon with the index-backed store:
|
||||
|
||||
```sh
|
||||
./build/amduatd --root .amduat-asl --sock amduatd.sock --store-backend index
|
||||
```
|
||||
|
||||
Note: `/v1/fed/records` requires the index backend.
|
||||
|
||||
Dev loop (build + restart):
|
||||
|
||||
```sh
|
||||
|
|
|
|||
139
docs/state_report.md
Normal file
139
docs/state_report.md
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
# Repo State Report
|
||||
|
||||
## 1) Build + test entrypoints
|
||||
- Tests: `ctest --test-dir build` (top-level build includes vendor tests via `add_subdirectory(vendor/amduat)` in `CMakeLists.txt`); core-only tests: `ctest --test-dir vendor/amduat/build` (binaries present in `vendor/amduat/build`).
|
||||
- Main daemon: `./build/amduatd --root .amduat-asl --sock amduatd.sock` (`README.md`, `src/amduatd.c`).
|
||||
- Dev loop: `./scripts/dev-restart.sh` (build + restart in `scripts/dev-restart.sh`).
|
||||
- Core CLIs/tools: `./vendor/amduat/build/amduat-asl ...` (store/init/log/index commands in `vendor/amduat/src/tools/amduat_asl_cli.c`), `./vendor/amduat/build/amduat-pel ...` (PEL tooling in `vendor/amduat/src/tools/amduat_pel_cli.c`), `./build/amduat-pel gc --root .amduat-asl` (GC tool in `src/amduat_pel_gc.c`).
|
||||
- Languages/toolchains: C11 + CMake (`CMakeLists.txt`, `vendor/amduat/CMakeLists.txt`), shell scripts (`scripts/*.sh`), embedded HTML/JS/CSS in C strings (`src/amduatd_ui.c`).
|
||||
- CI config: none found (no `.github/workflows/*`, `.gitlab-ci.yml`, etc.).
|
||||
|
||||
## 2) Top-level architecture map (as implemented)
|
||||
- Daemon runtime + HTTP surface: Paths `src/amduatd.c`, `src/amduatd_http.c`, `src/amduatd_http.h`, `src/amduatd_ui.c`; main types `amduatd_cfg_t`, `amduatd_http_req_t`, `amduatd_http_resp_t`; entrypoints `main`, `amduatd_handle_conn`, `amduatd_http_read_req`; wiring: initializes store fs config, concepts, caps, federation coord, then serves HTTP over Unix socket with a `select()` loop (`src/amduatd.c`).
|
||||
- Capability tokens: Paths `src/amduatd_caps.c`, `src/amduatd_caps.h`; main types `amduatd_caps_t`, `amduatd_caps_token_t`; entrypoints `amduatd_caps_init`, `amduatd_caps_handle`, `amduatd_caps_check`; wiring: per-request token validation with optional space/pointer scoping (`src/amduatd_caps.c`).
|
||||
- Concepts/relations + edge graph index: Paths `src/amduatd_concepts.c`, `src/amduatd_concepts.h`; main types `amduatd_concepts_t`, `amduatd_edge_index_state_t`, `amduatd_edge_list_t`; entrypoints `amduatd_concepts_init`, `amduatd_concepts_refresh_edges`, `amduatd_handle_get_relations`, `amduatd_handle_get_concepts`; wiring: edge records stored in a collection log, plus derived edge graph + index pointer state (`src/amduatd_concepts.c`).
|
||||
- Space scoping: Paths `src/amduatd_space.c`, `src/amduatd_space.h`; main types `amduatd_space_t`; entrypoints `amduatd_space_init`, `amduatd_space_scope_name`, `amduatd_space_unscoped_name`; wiring: scopes pointer/collection names with prefix `space/<space_id>/` when enabled (`src/amduatd_space.c`).
|
||||
- Federation coordinator: Paths `federation/coord.c`, `federation/coord.h`; main types `amduat_fed_coord_t`, `amduat_fed_coord_cfg_t`; entrypoints `amduat_fed_coord_open`, `amduat_fed_coord_tick`, `amduat_fed_coord_get_status`; wiring: daemon ticks coordinator on interval (`src/amduatd.c`).
|
||||
- Federation transport: Paths `federation/transport_unix.c`, `federation/transport_stub.c`; main types `amduat_fed_transport_t`; entrypoints `amduat_fed_transport_unix_ops`, `amduat_fed_transport_stub_ops`; wiring: daemon currently uses stub transport (`src/amduatd.c`).
|
||||
- CAS store (filesystem): Paths `vendor/amduat/src/adapters/asl_store_fs/*`, `vendor/amduat/include/amduat/asl/store.h`; main types `amduat_asl_store_fs_t`, `amduat_asl_store_t`; entrypoints `amduat_asl_store_fs_init`, `amduat_asl_store_put`, `amduat_asl_store_get`; wiring: used by daemon and tools as the backing store.
|
||||
- Pointer store: Paths `vendor/amduat/src/adapters/asl_pointer_fs/asl_pointer_fs.c`; main types `amduat_asl_pointer_store_t`; entrypoints `amduat_asl_pointer_get`, `amduat_asl_pointer_cas`; wiring: used by log store, collection store, concept index, and caps checks.
|
||||
- Log store: Paths `vendor/amduat/src/core/asl_log_store.c`, `vendor/amduat/include/amduat/asl/log_store.h`; main types `amduat_asl_log_store_t`, `amduat_asl_log_entry_t`; entrypoints `amduat_asl_log_append`, `amduat_asl_log_read`; wiring: collection store writes log chunks in CAS and advances pointer heads.
|
||||
- Collection store: Paths `vendor/amduat/src/core/asl_collection.c`, `vendor/amduat/include/amduat/asl/collection.h`; main types `amduat_asl_collection_store_t`; entrypoints `amduat_asl_collection_append`, `amduat_asl_collection_snapshot`, `amduat_asl_collection_read`; wiring: concept edges are stored as records appended to a collection log.
|
||||
- ASL index store: Paths `vendor/amduat/src/adapters/asl_store_index_fs/*`; main types `amduat_asl_store_index_fs_t`, `amduat_asl_index_state_t`; entrypoints `amduat_asl_store_index_fs_put_indexed`, `amduat_asl_store_index_fs_log_scan`, `amduat_asl_store_index_fs_gc`; wiring: available in core but not wired into amduatd (amduatd uses store_fs ops).
|
||||
- PEL execution + materialization cache: Paths `vendor/amduat/src/pel_stack/surf/surf.c`, `vendor/amduat/src/adapters/asl_materialization_cache_fs/asl_materialization_cache_fs.c`; main types `amduat_pel_program_t`, `amduat_pel_run_result_t`; entrypoints `amduat_pel_surf_run_with_result`, `amduat_asl_materialization_cache_fs_get`, `amduat_asl_materialization_cache_fs_put`; wiring: used by `/v1/pel/run` and by the collection view implementation.
|
||||
- Derivation index (core-only): Paths `vendor/amduat/src/adapters/asl_derivation_index_fs/asl_derivation_index_fs.c`, `vendor/amduat/include/amduat/asl/asl_derivation_index_fs.h`; main types `amduat_asl_derivation_index_fs_t`, `amduat_asl_derivation_record_t`; entrypoints `amduat_asl_derivation_index_fs_add`, `amduat_asl_derivation_index_fs_list`; wiring: used by CLI tools (`vendor/amduat/src/tools/amduat_asl_cli.c`, `vendor/amduat/src/tools/amduat_pel_cli.c`), not by amduatd.
|
||||
|
||||
## 3) “Space” concept: definition + storage layout
|
||||
- Definition: “space” is a daemon-level scoping prefix for pointer/collection names, not a separate store; it rewrites names into `space/<space_id>/...` when enabled (`src/amduatd_space.c`, `amduatd_space_scope_name`, `amduatd_space_unscoped_name`).
|
||||
- Storage layout: scoped names are stored in the pointer store under `root/pointers/space/<space_id>/.../head` per `amduat_asl_pointer_build_head_path` (`vendor/amduat/src/adapters/asl_pointer_fs/asl_pointer_fs.c`). No dedicated per-space directory outside pointer name paths is created by amduatd.
|
||||
- Identification: `space_id` is a pointer-name-safe token without `/`, validated by `amduatd_space_space_id_is_valid` (calls `amduat_asl_pointer_name_is_valid`) (`src/amduatd_space.c`).
|
||||
- Multi-space support: only one space can be active per daemon process via `--space` (`src/amduatd.c`, `amduatd_space_init`); code does not show per-request space selection.
|
||||
|
||||
## 4) Source of truth: pointers + logs (actual)
|
||||
- Canonical “head/root” pointer for a space: there is no single global space root pointer. The main space-scoped heads used by amduatd are the edge index head (`daemon/edges/index/head` scoped by `amduatd_space_edges_index_head_name`) and collection snapshot/log heads for the edge collection (see below) (`src/amduatd_space.c`, `src/amduatd_concepts.c`).
|
||||
- Pointer storage location + format:
|
||||
- Location: `root/pointers/<name>/head` using path segments from the pointer name (`amduat_asl_pointer_build_head_path` in `vendor/amduat/src/adapters/asl_pointer_fs/asl_pointer_fs.c`).
|
||||
- Format: `ASLPTR1` magic with version and flags, then name/ref/prev fields encoded via `amduat_enc_asl1_core_encode_reference_v1` (`amduat_asl_pointer_read_head` and `amduat_asl_pointer_write_head` in `vendor/amduat/src/adapters/asl_pointer_fs/asl_pointer_fs.c`).
|
||||
- Read/write: `amduat_asl_pointer_get` and `amduat_asl_pointer_cas` (`vendor/amduat/src/adapters/asl_pointer_fs/asl_pointer_fs.c`).
|
||||
- Collection heads used by amduatd concepts:
|
||||
- Snapshot head pointer: `collection/<name>/head` built by `amduatd_build_collection_head_name` (`src/amduatd_concepts.c`) and also by `amduat_asl_collection_build_head_name` (`vendor/amduat/src/core/asl_collection.c`).
|
||||
- Log head pointer: `log/collection/<name>/log/head` built by `amduatd_build_collection_log_head_name` (`src/amduatd_concepts.c`) and `amduat_asl_log_build_pointer_name` (`vendor/amduat/src/core/asl_log_store.c`).
|
||||
- Log entry schema (ASL index log): `amduat_asl_log_record_t {logseq, record_type, payload, record_hash}` with record types defined in `vendor/amduat/include/amduat/enc/asl_log.h`.
|
||||
- Log append behavior:
|
||||
- Collection log: `amduat_asl_log_append` writes CAS log chunks (`ASL_LOG_CHUNK_1`) and advances the log head pointer via CAS (`vendor/amduat/src/core/asl_log_store.c`).
|
||||
- Index log: `amduat_asl_store_index_fs_append_log_record` appends to `index/log.asl` (`vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs.c`).
|
||||
- Integrity mechanisms:
|
||||
- Collection log chunks form a chain via `prev_ref` and are rooted at the log head pointer (`amduat_asl_log_append`, `amduat_asl_log_chunk_t` in `vendor/amduat/src/core/asl_log_store.c`).
|
||||
- Index log uses a hash chain where `record_hash = SHA256(prev_hash || logseq || type || payload_len || payload)` (`amduat_asl_store_index_fs_log_hash_record` in `vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs.c`).
|
||||
- Log traversal:
|
||||
- Collection log: `amduat_asl_log_read` walks the head pointer and `prev_ref` chain to gather chunks (`vendor/amduat/src/core/asl_log_store.c`).
|
||||
- Index log: replay happens via `amduat_asl_store_index_fs_build_replay_state` using `amduat_asl_replay_apply_log` (`vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs.c`, `vendor/amduat/include/amduat/asl/index_replay.h`).
|
||||
- Other persistent state that can diverge from logs/pointers:
|
||||
- Edge index state and edge graph artifacts (`tgk/edge_index_state` record + `TGK_EDGE_GRAPH_1` artifact) are separate derived state from the edge collection log (`src/amduatd_concepts.c`).
|
||||
- Legacy `.amduatd.edges` file (used only for migration) can diverge until migrated (`src/amduatd_concepts.c`).
|
||||
- Materialization cache in `index/materializations` can diverge from CAS and is treated as a cache (`vendor/amduat/src/adapters/asl_materialization_cache_fs/asl_materialization_cache_fs.c`).
|
||||
|
||||
## 5) CAS (content-addressed store)
|
||||
- Hash algorithm: SHA-256 is the only defined ASL1 hash id and default (`AMDUAT_HASH_ASL1_ID_SHA256` in `vendor/amduat/include/amduat/hash/asl1.h`, default config in `vendor/amduat/src/adapters/asl_store_fs/asl_store_fs_meta.c`).
|
||||
- Object types stored: arbitrary artifacts with optional type tags (`amduat_artifact_t` in `vendor/amduat/include/amduat/asl/core.h`); records are stored via `amduat_asl_record_store_put` (schema + payload) in `vendor/amduat/src/core/asl_record.c` and used by amduatd for concept edges (`src/amduatd_concepts.c`).
|
||||
- Address format: refs are `{hash_id, digest}`; on disk, objects are stored under hex-digest filenames derived from the raw digest (`amduat_asl_store_fs_layout_build_paths` in `vendor/amduat/src/adapters/asl_store_fs/asl_store_fs_layout.c`).
|
||||
- Disk layout and APIs:
|
||||
- Layout: `root/objects/<profile_hex>/<hash_hex>/<byte0>/<byte1>/<digest_hex>` (`amduat_asl_store_fs_layout_build_paths` in `vendor/amduat/src/adapters/asl_store_fs/asl_store_fs_layout.c`).
|
||||
- APIs: `amduat_asl_store_put/get` (generic) and `amduat_asl_store_fs_ops` (filesystem implementation) (`vendor/amduat/include/amduat/asl/store.h`, `vendor/amduat/src/adapters/asl_store_fs/asl_store_fs.c`).
|
||||
- Garbage collection:
|
||||
- Exists as a standalone tool: `amduat-pel gc` uses `amduat_asl_gc_fs_run` to mark from pointer/log/collection roots and optionally delete artifacts (`src/amduat_pel_gc.c`, `src/asl_gc_fs.c`).
|
||||
- Pinning concept: no explicit pin API found; reachability is derived from pointers/logs/snapshots (GC uses those roots) (`src/asl_gc_fs.c`).
|
||||
|
||||
## 6) Deterministic derivations (current reality)
|
||||
- Derivation-related types exist in core: `amduat_pel_derivation_sid_input_t`, `amduat_pel_derivation_sid_compute` (`vendor/amduat/include/amduat/pel/derivation_sid.h`, `vendor/amduat/src/core/derivation_sid.c`).
|
||||
- Execution model: PEL DAGs execute in-process (no external sandbox) via `amduat_pel_program_dag_exec_trace` and `amduat_pel_surf_run_with_result` (`vendor/amduat/src/pel_stack/surf/surf.c`).
|
||||
- Inputs: referenced as CAS refs from the store (`amduat_pel_surf_run_with_result` loads program/input/params artifacts by ref in `vendor/amduat/src/pel_stack/surf/surf.c`).
|
||||
- Outputs: stored back into CAS (`amduat_asl_store_put` in `vendor/amduat/src/pel_stack/surf/surf.c`); results/traces/receipts are separate artifacts (`amduat_enc_pel1_result`, `amduat_enc_pel_trace_dag` etc. used in `src/amduatd.c`).
|
||||
- Provenance/audit: optional FER1 receipt data is accepted/serialized in `/v1/pel/run` (`src/amduatd.c`), but no daemon-side derivation index is written.
|
||||
- Derivation index persistence: exists in core (`vendor/amduat/src/adapters/asl_derivation_index_fs/asl_derivation_index_fs.c`) and CLI tools (`vendor/amduat/src/tools/amduat_asl_cli.c`, `vendor/amduat/src/tools/amduat_pel_cli.c`), but amduatd does not write derivation records (no references in `src/`).
|
||||
|
||||
## 7) ASL index: what it is and what it depends on
|
||||
- Storage location: `root/index/` with `log.asl`, `segments/`, `blocks/`, and `snapshots/` (layout in `vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs_layout.c`).
|
||||
- Derived vs authoritative: the index log (`index/log.asl`) and segment files are the authoritative index state; higher-level state can be rebuilt by replaying the log plus snapshots (`amduat_asl_store_index_fs_build_replay_state` + `amduat_asl_replay_apply_log` in `vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs.c`).
|
||||
- Build/update path: `amduat_asl_store_index_fs_put_indexed` appends log records, writes segments/blocks, and may create snapshots (`vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs.c`).
|
||||
- Queries relying on it: `amduat_asl_log_scan`, tombstone operations, and `amduat_asl_index_current_state` are implemented by the index-backed store ops (`vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs.c`, `vendor/amduat/include/amduat/asl/store.h`).
|
||||
- What breaks if deleted: if `index/` (including `log.asl`) is removed, index-backed stores cannot answer log/state/tombstone queries; recovery requires a log to replay, which lives under `index/` itself (`vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs.c`).
|
||||
|
||||
## 8) Daemon / runtime model (if any)
|
||||
- Daemon process: `amduatd` in `src/amduatd.c` binds a Unix domain socket, listens, and handles one connection per `accept()` in a single-threaded loop using `select()`.
|
||||
- Single-space or multi-space: single-space per daemon process via `--space` (`src/amduatd.c`, `src/amduatd_space.c`).
|
||||
- Config mechanism: CLI flags `--root`, `--sock`, `--space`, `--migrate-unscoped-edges`, `--edges-refresh-ms`, `--allow-uid`, `--allow-user`, `--enable-cap-reads` (`src/amduatd.c`).
|
||||
- IPC/RPC/HTTP APIs: HTTP over Unix socket, routes in `src/amduatd.c` and handlers in `src/amduatd_concepts.c` and `src/amduatd_caps.c`.
|
||||
- Background workers: federation tick every 1s (`AMDUATD_FED_TICK_MS`) and optional edge refresh on `--edges-refresh-ms` interval (`src/amduatd.c`).
|
||||
|
||||
## 9) Projections / query layer
|
||||
- Projection concept in this repo: the edge graph and edge index state are projections derived from the edge collection log (`AMDUATD_EDGE_INDEX_SCHEMA`, `amduatd_concepts_write_edge_index_state`, `amduatd_concepts_refresh_edges_internal` in `src/amduatd_concepts.c`).
|
||||
- Generation + storage: edge graph stored as `TGK_EDGE_GRAPH_1` artifact plus pointer to `tgk/edge_index_state` record (`src/amduatd_concepts.c`).
|
||||
- Derived from logs/CAS: refresh reads collection log entries via `amduat_asl_log_read` and rebuilds the edge list/graph (`src/amduatd_concepts.c`).
|
||||
- Query APIs: HTTP endpoints for concepts/relations/resolve in `src/amduatd_concepts.c` and routing in `src/amduatd.c`; collection view is generated by a PEL program over collection snapshot + log (`amduatd_collection_view` in `src/amduatd_concepts.c`).
|
||||
|
||||
## 10) Federation / collaboration
|
||||
- Networking code: `federation/transport_unix.c` builds HTTP requests over Unix sockets for `/v1/fed/records` and `/v1/fed/artifacts` (see `amduat_fed_transport_unix_ops`).
|
||||
- Federation coordinator: `federation/coord.c` maintains registry state and a cached view (`amduat_fed_coord_t`).
|
||||
- Daemon behavior: federation is wired but uses the stub transport (`amduat_fed_transport_stub_ops` in `src/amduatd.c`), so no remote sync by default.
|
||||
- Remote refs/replication/merge/signatures: not implemented in daemon beyond read-only federation endpoints; no CRDT/merge logic found in this repo (coordinator logic delegates to core types in `vendor/amduat/include/amduat/fed/*`).
|
||||
|
||||
## 11) Invariants (explicit + implicit)
|
||||
- Pointer names are path-safe and forbid `..` segments; used throughout for space IDs and pointer names (`amduat_asl_pointer_name_is_valid` in `vendor/amduat/src/adapters/asl_pointer_fs/asl_pointer_fs.c`).
|
||||
- Collection log append is append-only with CAS+pointer CAS; chunks point to previous chunk to form a chain (`amduat_asl_log_append`, `amduat_asl_log_chunk_t` in `vendor/amduat/src/core/asl_log_store.c`).
|
||||
- Log chunk entries must be consistent about timestamp/actor presence across the batch (`amduat_asl_log_entries_consistent` in `vendor/amduat/src/core/asl_log_store.c`).
|
||||
- Collection snapshot heads and log heads are stable pointer names derived from collection name (`amduat_asl_collection_build_head_name` and `amduat_asl_collection_build_log_name` in `vendor/amduat/src/core/asl_collection.c`).
|
||||
- Index log hash chain uses `prev_hash` + record fields to compute the next hash (`amduat_asl_store_index_fs_log_hash_record` in `vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs.c`).
|
||||
- Edge index state is written via pointer CAS, implying the index head should only advance if the expected previous ref matches (`amduatd_concepts_write_edge_index_state` in `src/amduatd_concepts.c`).
|
||||
|
||||
## 12) Immediate risks of “parallel mechanisms”
|
||||
- Edge index state vs edge collection log: the derived edge graph + `tgk/edge_index_state` can diverge from the log if refresh fails; there is a deterministic rebuild path by replaying the collection log (`amduatd_concepts_refresh_edges_internal` in `src/amduatd_concepts.c`).
|
||||
- Legacy `.amduatd.edges` vs collection log: migration reads `.amduatd.edges` and writes into the collection log, so stale files can diverge until migrated (`amduatd_concepts_migrate_edges` in `src/amduatd_concepts.c`).
|
||||
- Materialization cache vs CAS: cache entries are validated against CAS and are treated as a performance-only layer; missing or stale cache forces recompute (`amduat_asl_materialization_cache_fs_get` usage in `vendor/amduat/src/pel_stack/surf/surf.c`).
|
||||
- ASL index files vs index log: segment/summary/snapshot files are derived from `index/log.asl`; if any are deleted, `amduat_asl_store_index_fs_build_replay_state` can rebuild from the log, but if `index/log.asl` is deleted the rebuild story is gone (`vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs.c`).
|
||||
- Federation coordinator cached view vs store log: coordinator caches `last_view` in memory and refreshes on tick; divergence is possible across restarts or if tick stops (`federation/coord.c`, `src/amduatd.c`).
|
||||
|
||||
## 13) Recommendation inputs (no decision yet)
|
||||
- Current capability summary:
|
||||
- Local Unix-socket HTTP daemon over a single ASL store root with artifact CRUD, concepts/relations, and PEL execution (`src/amduatd.c`, `src/amduatd_concepts.c`).
|
||||
- CAS store on disk with pointer and log primitives; collection logs and snapshot pointers are wired (`vendor/amduat/src/adapters/asl_store_fs`, `vendor/amduat/src/core/asl_log_store.c`, `vendor/amduat/src/core/asl_collection.c`).
|
||||
- Edge graph projection maintained as derived state from collection logs (`src/amduatd_concepts.c`).
|
||||
- Federation coordinator scaffolding and endpoints, but using stub transport by default (`federation/coord.c`, `federation/transport_stub.c`, `src/amduatd.c`).
|
||||
- Core tooling for index/derivation/GC exists in vendor and repo tools (`vendor/amduat/src/tools/*`, `src/amduat_pel_gc.c`).
|
||||
- Top 5 unknowns/blockers for the next step (grounded in code):
|
||||
- Whether amduatd should switch to `amduat_asl_store_index_fs_ops` to enable `amduat_asl_log_scan` and tombstones; current store fs ops do not implement log scan (`vendor/amduat/src/adapters/asl_store_fs/asl_store_fs.c`, `vendor/amduat/src/near_core/asl/store.c`).
|
||||
- The intended authoritative store for federation records: amduatd’s `/v1/fed/records` relies on `amduat_asl_log_scan`, which is unsupported by store_fs (`src/amduatd.c`, `vendor/amduat/src/near_core/asl/store.c`).
|
||||
- How/when to persist derivation index records from daemon PEL runs (present in core but unused in `src/`).
|
||||
- Whether the “space” scope should be exposed as a first-class selector in the API (only CLI flag currently applies globally).
|
||||
- The intended registry flow for federation (coordinator expects registry refs, but no daemon config path for them besides code defaults in `src/amduatd.c`).
|
||||
- Top 5 next commit candidates grounded in repo reality (no new architecture):
|
||||
- Wire amduatd to an index-backed store (use `amduat_asl_store_index_fs_ops`) so federation record scanning and index state endpoints are meaningful.
|
||||
- Add a daemon flag/config to load a federation registry ref and domain id into `amduat_fed_coord_cfg_t` (values are currently hardcoded in `src/amduatd.c`).
|
||||
- Persist derivation index records for `/v1/pel/run` outputs using existing core `amduat_asl_derivation_index_fs_*` APIs.
|
||||
- Expose index state or health endpoints that surface `amduat_asl_index_current_state` (available in core API, already used in `/v1/fed/records`).
|
||||
- Formalize edge index rebuild tooling (e.g., CLI switch or maintenance endpoint) using existing `amduatd_concepts_refresh_edges` and pointer state.
|
||||
- Method used:
|
||||
- Searched for key terms with `rg -n "cas|content address|hash|log|append|pointer|head|space|derivation|projection|ASL|index|daemon|server|coordinator|sync|replication"`.
|
||||
- Read key sources: `src/amduatd.c`, `src/amduatd_concepts.c`, `src/amduatd_space.c`, `vendor/amduat/src/adapters/asl_store_fs/asl_store_fs_layout.c`, `vendor/amduat/src/adapters/asl_pointer_fs/asl_pointer_fs.c`, `vendor/amduat/src/core/asl_log_store.c`, `vendor/amduat/src/adapters/asl_store_index_fs/asl_store_index_fs_layout.c`.
|
||||
- Ran CLIs: `./build/amduatd --help`, `./vendor/amduat/build/amduat-asl --help`, `./vendor/amduat/build/amduat-asl index state --root .amduat-asl`, `./vendor/amduat/build/amduat-asl log inspect --root .amduat-asl`.
|
||||
- Tests were not executed; enumerated `ctest` commands above.
|
||||
|
|
@ -38,6 +38,7 @@
|
|||
#include "amduatd_ui.h"
|
||||
#include "amduatd_caps.h"
|
||||
#include "amduatd_space.h"
|
||||
#include "amduatd_store.h"
|
||||
|
||||
#include <errno.h>
|
||||
#include <signal.h>
|
||||
|
|
@ -1016,14 +1017,25 @@ static bool amduatd_handle_get_fed_records(int fd,
|
|||
"invalid limit\n", false);
|
||||
}
|
||||
}
|
||||
if (store->ops.log_scan == NULL || store->ops.current_state == NULL) {
|
||||
return amduatd_http_send_text(fd, 501, "Not Implemented",
|
||||
"requires index backend\n", false);
|
||||
}
|
||||
if (!amduat_asl_index_current_state(store, &state)) {
|
||||
return amduatd_http_send_text(fd, 500, "Internal Server Error",
|
||||
"store error\n", false);
|
||||
}
|
||||
if (amduat_asl_log_scan(store, &records, &record_count) !=
|
||||
AMDUAT_ASL_STORE_OK) {
|
||||
return amduatd_http_send_text(fd, 500, "Internal Server Error",
|
||||
"log scan error\n", false);
|
||||
{
|
||||
amduat_asl_store_error_t scan_err;
|
||||
scan_err = amduat_asl_log_scan(store, &records, &record_count);
|
||||
if (scan_err == AMDUAT_ASL_STORE_ERR_UNSUPPORTED) {
|
||||
return amduatd_http_send_text(fd, 501, "Not Implemented",
|
||||
"requires index backend\n", false);
|
||||
}
|
||||
if (scan_err != AMDUAT_ASL_STORE_OK) {
|
||||
return amduatd_http_send_text(fd, 500, "Internal Server Error",
|
||||
"log scan error\n", false);
|
||||
}
|
||||
}
|
||||
|
||||
memset(&b, 0, sizeof(b));
|
||||
|
|
@ -4020,6 +4032,7 @@ static void amduatd_print_usage(FILE *stream) {
|
|||
" amduatd [--root PATH] [--sock PATH]\n"
|
||||
" [--space SPACE_ID] [--migrate-unscoped-edges]\n"
|
||||
" [--edges-refresh-ms MS]\n"
|
||||
" [--store-backend fs|index]\n"
|
||||
" [--allow-uid UID] [--allow-user NAME]\n"
|
||||
" [--enable-cap-reads]\n"
|
||||
"\n"
|
||||
|
|
@ -4036,10 +4049,11 @@ int main(int argc, char **argv) {
|
|||
const char *space_id = NULL;
|
||||
bool migrate_unscoped_edges = false;
|
||||
bool enable_cap_reads = false;
|
||||
amduatd_store_backend_t store_backend = AMDUATD_STORE_BACKEND_FS;
|
||||
amduatd_cfg_t dcfg;
|
||||
amduatd_caps_t caps;
|
||||
amduat_asl_store_fs_config_t cfg;
|
||||
amduat_asl_store_fs_t fs;
|
||||
amduatd_store_ctx_t store_ctx;
|
||||
amduat_asl_store_t store;
|
||||
amduat_reference_t api_contract_ref;
|
||||
amduat_reference_t ui_ref;
|
||||
|
|
@ -4098,6 +4112,15 @@ int main(int argc, char **argv) {
|
|||
return 2;
|
||||
}
|
||||
dcfg.edges_refresh_ms = (uint64_t)refresh_val;
|
||||
} else if (strcmp(argv[i], "--store-backend") == 0) {
|
||||
if (i + 1 >= argc) {
|
||||
fprintf(stderr, "error: --store-backend requires a value\n");
|
||||
return 2;
|
||||
}
|
||||
if (!amduatd_store_backend_parse(argv[++i], &store_backend)) {
|
||||
fprintf(stderr, "error: invalid --store-backend\n");
|
||||
return 2;
|
||||
}
|
||||
} else if (strcmp(argv[i], "--allow-uid") == 0) {
|
||||
char *endp = NULL;
|
||||
unsigned long uid_val;
|
||||
|
|
@ -4145,18 +4168,13 @@ int main(int argc, char **argv) {
|
|||
return 2;
|
||||
}
|
||||
|
||||
memset(&cfg, 0, sizeof(cfg));
|
||||
if (!amduat_asl_store_fs_load_config(root, &cfg)) {
|
||||
fprintf(stderr, "error: failed to load store config: %s\n", root);
|
||||
return 8;
|
||||
}
|
||||
|
||||
memset(&fs, 0, sizeof(fs));
|
||||
if (!amduat_asl_store_fs_init(&fs, cfg.config, root)) {
|
||||
if (!amduatd_store_init(&store, &cfg, &store_ctx, root, store_backend)) {
|
||||
fprintf(stderr, "error: failed to initialize store: %s\n", root);
|
||||
return 8;
|
||||
}
|
||||
amduat_asl_store_init(&store, cfg.config, amduat_asl_store_fs_ops(), &fs);
|
||||
amduat_log(AMDUAT_LOG_INFO,
|
||||
"store backend=%s",
|
||||
amduatd_store_backend_name(store_backend));
|
||||
if (!amduatd_caps_init(&caps, &dcfg, root)) {
|
||||
amduat_log(AMDUAT_LOG_WARN, "capabilities unavailable");
|
||||
}
|
||||
|
|
|
|||
76
src/amduatd_store.c
Normal file
76
src/amduatd_store.c
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
#include "amduatd_store.h"
|
||||
|
||||
#include "amduat/asl/asl_store_fs_meta.h"
|
||||
|
||||
#include <string.h>
|
||||
|
||||
bool amduatd_store_backend_parse(const char *value,
|
||||
amduatd_store_backend_t *out_backend) {
|
||||
if (value == NULL || out_backend == NULL) {
|
||||
return false;
|
||||
}
|
||||
if (strcmp(value, "fs") == 0) {
|
||||
*out_backend = AMDUATD_STORE_BACKEND_FS;
|
||||
return true;
|
||||
}
|
||||
if (strcmp(value, "index") == 0) {
|
||||
*out_backend = AMDUATD_STORE_BACKEND_INDEX;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
const char *amduatd_store_backend_name(amduatd_store_backend_t backend) {
|
||||
switch (backend) {
|
||||
case AMDUATD_STORE_BACKEND_FS:
|
||||
return "fs";
|
||||
case AMDUATD_STORE_BACKEND_INDEX:
|
||||
return "index";
|
||||
default:
|
||||
return "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
bool amduatd_store_init(amduat_asl_store_t *store,
|
||||
amduat_asl_store_fs_config_t *cfg,
|
||||
amduatd_store_ctx_t *ctx,
|
||||
const char *root_path,
|
||||
amduatd_store_backend_t backend) {
|
||||
if (store == NULL || cfg == NULL || ctx == NULL || root_path == NULL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
memset(store, 0, sizeof(*store));
|
||||
memset(ctx, 0, sizeof(*ctx));
|
||||
memset(cfg, 0, sizeof(*cfg));
|
||||
|
||||
if (!amduat_asl_store_fs_load_config(root_path, cfg)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (backend == AMDUATD_STORE_BACKEND_FS) {
|
||||
if (!amduat_asl_store_fs_init(&ctx->fs, cfg->config, root_path)) {
|
||||
return false;
|
||||
}
|
||||
amduat_asl_store_init(store,
|
||||
cfg->config,
|
||||
amduat_asl_store_fs_ops(),
|
||||
&ctx->fs);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (backend == AMDUATD_STORE_BACKEND_INDEX) {
|
||||
if (!amduat_asl_store_index_fs_init(&ctx->index_fs,
|
||||
cfg->config,
|
||||
root_path)) {
|
||||
return false;
|
||||
}
|
||||
amduat_asl_store_init(store,
|
||||
cfg->config,
|
||||
amduat_asl_store_index_fs_ops(),
|
||||
&ctx->index_fs);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
40
src/amduatd_store.h
Normal file
40
src/amduatd_store.h
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
#ifndef AMDUATD_STORE_H
|
||||
#define AMDUATD_STORE_H
|
||||
|
||||
#include "amduat/asl/asl_store_fs.h"
|
||||
#include "amduat/asl/asl_store_fs_meta.h"
|
||||
#include "amduat/asl/asl_store_index_fs.h"
|
||||
#include "amduat/asl/store.h"
|
||||
|
||||
#include <stdbool.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef enum {
|
||||
AMDUATD_STORE_BACKEND_FS = 0,
|
||||
AMDUATD_STORE_BACKEND_INDEX = 1
|
||||
} amduatd_store_backend_t;
|
||||
|
||||
typedef struct {
|
||||
amduat_asl_store_fs_t fs;
|
||||
amduat_asl_store_index_fs_t index_fs;
|
||||
} amduatd_store_ctx_t;
|
||||
|
||||
bool amduatd_store_backend_parse(const char *value,
|
||||
amduatd_store_backend_t *out_backend);
|
||||
|
||||
const char *amduatd_store_backend_name(amduatd_store_backend_t backend);
|
||||
|
||||
bool amduatd_store_init(amduat_asl_store_t *store,
|
||||
amduat_asl_store_fs_config_t *cfg,
|
||||
amduatd_store_ctx_t *ctx,
|
||||
const char *root_path,
|
||||
amduatd_store_backend_t backend);
|
||||
|
||||
#ifdef __cplusplus
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
||||
#endif /* AMDUATD_STORE_H */
|
||||
109
tests/test_amduatd_store_backend.c
Normal file
109
tests/test_amduatd_store_backend.c
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
#ifndef _POSIX_C_SOURCE
|
||||
#define _POSIX_C_SOURCE 200809L
|
||||
#endif
|
||||
|
||||
#include "amduatd_store.h"
|
||||
|
||||
#include "amduat/asl/asl_store_fs_meta.h"
|
||||
#include "amduat/enc/asl_log.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
static char *amduatd_test_make_temp_dir(void) {
|
||||
char tmpl[] = "/tmp/amduatd-store-XXXXXX";
|
||||
char *dir = mkdtemp(tmpl);
|
||||
size_t len;
|
||||
char *copy;
|
||||
if (dir == NULL) {
|
||||
perror("mkdtemp");
|
||||
return NULL;
|
||||
}
|
||||
len = strlen(dir);
|
||||
copy = (char *)malloc(len + 1u);
|
||||
if (copy == NULL) {
|
||||
fprintf(stderr, "failed to allocate temp dir copy\n");
|
||||
return NULL;
|
||||
}
|
||||
memcpy(copy, dir, len + 1u);
|
||||
return copy;
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
char *root = amduatd_test_make_temp_dir();
|
||||
amduat_asl_store_fs_config_t cfg;
|
||||
amduatd_store_ctx_t store_ctx;
|
||||
amduat_asl_store_t store;
|
||||
amduat_asl_log_record_t *records = NULL;
|
||||
size_t record_count = 0;
|
||||
amduat_asl_store_error_t scan_err;
|
||||
|
||||
if (root == NULL) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
memset(&cfg, 0, sizeof(cfg));
|
||||
if (!amduat_asl_store_fs_init_root(root, NULL, &cfg)) {
|
||||
fprintf(stderr, "failed to init store root\n");
|
||||
free(root);
|
||||
return 1;
|
||||
}
|
||||
|
||||
memset(&store_ctx, 0, sizeof(store_ctx));
|
||||
memset(&store, 0, sizeof(store));
|
||||
if (!amduatd_store_init(&store,
|
||||
&cfg,
|
||||
&store_ctx,
|
||||
root,
|
||||
AMDUATD_STORE_BACKEND_FS)) {
|
||||
fprintf(stderr, "failed to init fs backend\n");
|
||||
free(root);
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (store.ops.log_scan != NULL) {
|
||||
fprintf(stderr, "fs backend unexpectedly supports log_scan\n");
|
||||
free(root);
|
||||
return 1;
|
||||
}
|
||||
|
||||
scan_err = amduat_asl_log_scan(&store, &records, &record_count);
|
||||
if (scan_err != AMDUAT_ASL_STORE_ERR_UNSUPPORTED) {
|
||||
fprintf(stderr, "fs backend log_scan returned %d\n", (int)scan_err);
|
||||
free(root);
|
||||
return 1;
|
||||
}
|
||||
|
||||
memset(&store_ctx, 0, sizeof(store_ctx));
|
||||
memset(&store, 0, sizeof(store));
|
||||
if (!amduatd_store_init(&store,
|
||||
&cfg,
|
||||
&store_ctx,
|
||||
root,
|
||||
AMDUATD_STORE_BACKEND_INDEX)) {
|
||||
fprintf(stderr, "failed to init index backend\n");
|
||||
free(root);
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (store.ops.log_scan == NULL) {
|
||||
fprintf(stderr, "index backend missing log_scan\n");
|
||||
free(root);
|
||||
return 1;
|
||||
}
|
||||
|
||||
scan_err = amduat_asl_log_scan(&store, &records, &record_count);
|
||||
if (scan_err != AMDUAT_ASL_STORE_OK) {
|
||||
fprintf(stderr, "index backend log_scan returned %d\n", (int)scan_err);
|
||||
free(root);
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (records != NULL) {
|
||||
amduat_enc_asl_log_free(records, record_count);
|
||||
}
|
||||
|
||||
free(root);
|
||||
return 0;
|
||||
}
|
||||
Loading…
Reference in a new issue