From b8c0a6e6d0dea9d6adcfe0c1360564f1badd457e Mon Sep 17 00:00:00 2001 From: Carl Niklas Rydberg Date: Sat, 7 Feb 2026 19:46:59 +0100 Subject: [PATCH] Implement v2 graph API surface and contract/test coverage --- CMakeLists.txt | 10 + README.md | 101 + docs/amduatd-api-v2-design.md | 196 + docs/v2-app-developer-guide.md | 237 + registry/README.md | 1 + registry/amduatd-api-contract.v2.json | 820 ++ scripts/graph_client_helpers.sh | 186 + scripts/test_graph_contract.sh | 379 + scripts/test_graph_queries.sh | 778 ++ src/amduatd.c | 2236 +++++ src/amduatd_concepts.c | 12181 +++++++++++++++++++++++- src/amduatd_concepts.h | 44 + 12 files changed, 17157 insertions(+), 12 deletions(-) create mode 100644 docs/amduatd-api-v2-design.md create mode 100644 docs/v2-app-developer-guide.md create mode 100644 registry/amduatd-api-contract.v2.json create mode 100755 scripts/graph_client_helpers.sh create mode 100755 scripts/test_graph_contract.sh create mode 100755 scripts/test_graph_queries.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 333ac31..d48b805 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -327,6 +327,16 @@ add_test(NAME amduatd_fed_ingest ) set_tests_properties(amduatd_fed_ingest PROPERTIES SKIP_RETURN_CODE 77) +add_test(NAME amduatd_graph_queries + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/scripts/test_graph_queries.sh +) +set_tests_properties(amduatd_graph_queries PROPERTIES SKIP_RETURN_CODE 77) + +add_test(NAME amduatd_graph_contract + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/scripts/test_graph_contract.sh +) +set_tests_properties(amduatd_graph_contract PROPERTIES SKIP_RETURN_CODE 77) + add_executable(amduatd_test_space_doctor tests/test_amduatd_space_doctor.c src/amduatd_space_doctor.c diff --git a/README.md b/README.md index a7b8ff4..9c5e0c1 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,13 @@ `amduat-api` builds `amduatd`, a minimal HTTP server over a Unix domain socket that exposes Amduat substrate operations for a **single ASL store root**. +## App Developer Handoff + +For a compact, implementation-focused guide for external app teams, use: + +- `docs/v2-app-developer-guide.md` +- `registry/amduatd-api-contract.v2.json` (machine-readable contract) + ## Build ```sh @@ -444,6 +451,57 @@ curl --unix-socket amduatd.sock -X POST http://localhost/v1/pel/run \ -d '{"program_ref":"","input_refs":[""],"params_ref":""}' ``` +Run a v2 PEL execute request with inline artifact ingest: + +```sh +curl --unix-socket amduatd.sock -X POST http://localhost/v2/pel/execute \ + -H 'Content-Type: application/json' \ + -d '{ + "scheme_ref":"dag", + "program_ref":"", + "inputs":{ + "refs":[""], + "inline_artifacts":[{"body_hex":"48656c6c6f2c20763221","type_tag":"0x00000000"}] + }, + "receipt":{ + "input_manifest_ref":"", + "environment_ref":"", + "evaluator_id":"local-amduatd", + "executor_ref":"", + "started_at":1731000000, + "completed_at":1731000001 + } + }' +``` + +The v2 execute response returns `run_ref` (and `result_ref` alias), +`receipt_ref`, `stored_input_refs[]`, `output_refs[]`, and `status`. + +Simplified async v2 operations (PEL-backed under the hood): + +```sh +# put (returns job_id) +curl --unix-socket amduatd.sock -X POST http://localhost/v2/ops/put \ + -H 'Content-Type: application/json' \ + -d '{"body_hex":"48656c6c6f","type_tag":"0x00000000"}' + +# concat (returns job_id) +curl --unix-socket amduatd.sock -X POST http://localhost/v2/ops/concat \ + -H 'Content-Type: application/json' \ + -d '{"left_ref":"","right_ref":""}' + +# slice (returns job_id) +curl --unix-socket amduatd.sock -X POST http://localhost/v2/ops/slice \ + -H 'Content-Type: application/json' \ + -d '{"ref":"","offset":1,"length":3}' + +# poll job +curl --unix-socket amduatd.sock http://localhost/v2/jobs/1 + +# get bytes +curl --unix-socket amduatd.sock http://localhost/v2/get/ +``` + When derivation indexing is enabled, successful PEL runs record derivations under `/index/derivations/by_artifact/` keyed by output refs (plus result/trace/receipt refs). @@ -560,6 +618,49 @@ When the daemon uses the `fs` store backend, index-only checks are reported as - request: `{program_ref, input_refs[], params_ref?, scheme_ref?}` (`program_ref`/`input_refs`/`params_ref` accept hex refs or concept names; omit `scheme_ref` to use `dag`) - request receipt (optional): `{receipt:{input_manifest_ref, environment_ref, evaluator_id, executor_ref, started_at, completed_at, sbom_ref?, parity_digest_hex?, executor_fingerprint_ref?, run_id_hex?, limits?, logs?, determinism_level?, rng_seed_hex?, signature_hex?}}` - response: `{result_ref, trace_ref?, receipt_ref?, output_refs[], status}` +- `POST /v2/pel/execute` (first v2 slice) + - request: `{program_ref, scheme_ref?, params_ref?, inputs:{refs[], inline_artifacts:[{body_hex, type_tag?}]}, receipt:{input_manifest_ref, environment_ref, evaluator_id, executor_ref, started_at, completed_at}}` + - response: `{run_ref, result_ref, trace_ref?, receipt_ref, stored_input_refs[], output_refs[], status}` +- `POST /v2/ops/put` → async enqueue `{job_id, status}` + - request: `{body_hex, type_tag?}` +- `POST /v2/ops/concat` → async enqueue `{job_id, status}` + - request: `{left_ref, right_ref}` (refs or concept names) +- `POST /v2/ops/slice` → async enqueue `{job_id, status}` + - request: `{ref, offset, length}` (`ref` accepts ref or concept name) +- `GET /v2/jobs/{id}` → `{job_id, kind, status, created_at_ms, started_at_ms, completed_at_ms, result_ref, error}` +- `GET /v2/get/{ref}` → artifact bytes (alias to `/v1/artifacts/{ref}`) +- `GET /v2/healthz` → liveness probe `{ok, status, time_ms}` +- `GET /v2/readyz` → readiness probe `{ok, status, components:{graph_index, federation}}` (`503` when not ready) +- `GET /v2/metrics` → Prometheus text metrics (`text/plain; version=0.0.4`) +- `POST /v2/graph/nodes` → create/seed graph node via concept `{name, ref?}` +- `POST /v2/graph/nodes/{name}/versions` → publish node version `{ref, metadata_ref?|provenance?}` (`metadata_ref` and `provenance` are mutually exclusive) +- `POST /v2/graph/nodes/{name}/versions/tombstone` → tombstone a previously published node version `{ref, metadata_ref?|provenance?}` (`metadata_ref` and `provenance` are mutually exclusive; append-only) +- `GET /v2/graph/nodes/{name}/versions?as_of=&include_tombstoned=` → get node versions (same shape as node read; defaults hide tombstoned versions from `latest_ref` and `versions[]`; set `include_tombstoned=true` to include them) +- `GET /v2/graph/nodes/{name}/neighbors?dir=&predicate=&limit=&cursor=&as_of=&provenance_ref=&include_tombstoned=&expand_names=&expand_artifacts=` → neighbor scan (paged, snapshot-bounded; tombstoned edges excluded by default unless `include_tombstoned=true`; optional provenance filter; add names/latest refs via expansion flags) +- `GET /v2/graph/search?name_prefix=&limit=&cursor=&as_of=` → search nodes by name prefix (paged, snapshot-bounded) +- `GET /v2/graph/paths?from=&to=&max_depth=&predicate=&as_of=&k=&expand_names=&expand_artifacts=&include_tombstoned=&provenance_ref=&max_fanout=&max_result_bytes=&include_stats=` → shortest directed path query (bounded BFS, snapshot-bounded, returns up to `k` paths with hop metadata and optional names/latest refs; tombstoned edges excluded by default unless `include_tombstoned=true`; optional provenance filter; supports fanout/response-size guards and optional stats) +- `GET /v2/graph/subgraph?roots[]=&max_depth=&max_fanout=&predicates[]=&dir=&as_of=&include_versions=&include_tombstoned=&limit_nodes=&limit_edges=&cursor=&provenance_ref=&max_result_bytes=&include_stats=` → bounded multi-hop subgraph retrieval from roots (snapshot-bounded, paged by opaque `next_cursor`; tombstoned edges excluded by default unless `include_tombstoned=true`; optional provenance filter; supports fanout/response-size guards and optional stats; returns `{nodes[], edges[], frontier?[], next_cursor, truncated}`) +- `POST /v2/graph/edges` → append graph edge `{subject, predicate, object, metadata_ref?|provenance?}` (`metadata_ref` and `provenance` are mutually exclusive) +- `POST /v2/graph/edges/tombstone` → append tombstone for an existing edge `{edge_ref, metadata_ref?|provenance?}` (append-only correction/retraction; `metadata_ref` and `provenance` are mutually exclusive) +- `POST /v2/graph/batch` → apply mixed graph mutations `{idempotency_key?, mode?, nodes?, versions?, edges?}` (versions/edges items support `metadata_ref?|provenance?`; mode: `fail_fast|continue_on_error`; deterministic replay for repeated `idempotency_key` + identical payload; returns `{ok, applied, results[]}` with per-item status/error) +- `POST /v2/graph/query` → unified graph query `{where?, predicates?, direction?, include_versions?, include_tombstoned?, include_stats?, max_result_bytes?, as_of?, limit?, cursor?}` returning `{nodes[], edges[], paging, stats?}` (`nodes[].versions[]` included when `include_versions=true`; `where.provenance_ref` filters edges linked via `ms.has_provenance`; tombstoned edges are excluded as-of snapshot unless `include_tombstoned=true`) +- `POST /v2/graph/retrieve` → agent-oriented bounded retrieval `{roots[], goal_predicates?[], max_depth?, as_of?, provenance_min_confidence?, include_versions?, include_tombstoned?, max_fanout?, limit_nodes?, limit_edges?, max_result_bytes?}` returning `{nodes[], edges[], explanations[], truncated, stats}` (multi-hop traversal from roots with optional predicate targeting and provenance-confidence gating; each explanation includes edge inclusion reasons and traversal depth) +- `POST /v2/graph/export` → paged graph envelope export `{as_of?, cursor?, limit?, predicates?[], roots?[], include_tombstoned?, max_result_bytes?}` returning `{items[], next_cursor, has_more, snapshot_as_of, stats}` (`items[]` preserve edge refs/order plus predicate, tombstone flag, and attached `metadata_ref` when present) +- `POST /v2/graph/import` → ordered graph envelope replay `{mode?, items[]}` (`mode=fail_fast|continue_on_error`) returning `{ok, applied, results[]}` with per-item `index/status/code/error/edge_ref` +- `GET /v2/graph/schema/predicates` → current graph governance policy `{mode, provenance_mode, predicates[]}` where `mode` is `strict|warn|off` and `provenance_mode` is `optional|required` +- `POST /v2/graph/schema/predicates` → update graph governance `{mode?, provenance_mode?, predicates?[]}` (`predicates[]` entries accept `{predicate_ref|predicate, domain?, range?}`; predicate validation mode is enforced for edge writes in `POST /v2/graph/edges`, batch edge items, and graph import edge items; `provenance_mode=required` enforces provenance attachment (`metadata_ref` or `provenance`) for version/edge/tombstone writes; policy is persisted per space root) +- `GET /v2/graph/stats` → graph/index health summary `{edges_total, aliases_total, index{...}, tombstones{edges, ratio}}` +- `GET /v2/graph/capabilities` → graph feature/version negotiation payload `{contract, graph{version, features, limits, modes}, runtime{...}}` +- `GET /v2/graph/changes?since_cursor=&since_as_of=&limit=&wait_ms=&event_types[]=&predicates[]=&roots[]=` → incremental appended graph events with strict cursor replay window (returns `410` when cursor falls outside retained window), resumable cursor tokens, optional long-poll (`wait_ms`), and event/predicate/root filters; response `{events[], next_cursor, has_more}` +- `GET /v2/graph/edges?subject=&predicate=&object=&dir=&limit=&cursor=&as_of=&expand_names=&expand_artifacts=&provenance_ref=&include_tombstoned=&max_result_bytes=&include_stats=` → filtered edge scan with paging (`dir=any|outgoing|incoming`, snapshot-bounded; tombstoned edges excluded by default unless `include_tombstoned=true`; optional provenance filter; add names/latest refs via expansion flags; supports response-size guards and optional stats) + - graph paging cursors are opaque tokens (`g1_*`); legacy numeric cursors are still accepted +- `GET /v2/graph/nodes/{name}?as_of=&include_tombstoned=` → `{name, concept_ref, latest_ref, versions[], outgoing[], incoming[]}` (`latest_ref` is resolved from visible versions at `as_of`; tombstoned versions excluded by default) +- `GET /v2/graph/history/{name}?as_of=&include_tombstoned=` → event timeline for node versions/edges (snapshot-bounded; `latest_ref` and tombstoned fact events both respect same visibility rules) + +Graph client helper examples for agent flows are in `scripts/graph_client_helpers.sh`: +- `batch-ingest` (idempotent batch write helper) +- `sync-once` (incremental changes cursor step) +- `subgraph` (bounded retrieval helper) - `POST /v1/pel/programs` - request: authoring JSON for `PEL/PROGRAM-DAG/1` (kernel ops only; `params_hex` is raw hex bytes) - response: `{program_ref}` diff --git a/docs/amduatd-api-v2-design.md b/docs/amduatd-api-v2-design.md new file mode 100644 index 0000000..6d11e14 --- /dev/null +++ b/docs/amduatd-api-v2-design.md @@ -0,0 +1,196 @@ +# amduatd API v2 Design (PEL-only writes) + +## Status + +Draft proposal for discussion. +First server slice implemented: `POST /v2/pel/execute` with `inputs.refs`, +`inputs.inline_artifacts[{body_hex,type_tag?}]`, and required `receipt`. + +## Goals + +- Keep `amduatd` thin: transport, parsing, auth, and mapping to core calls. +- Make **PEL the only write primitive** in HTTP v2. +- Enforce artifact provenance for writes: adding new artifacts must go through: + 1. store inputs, + 2. run a PEL program, + 3. persist and return receipt. +- Keep `/v1/*` behavior unchanged while v2 is introduced. + +## Non-goals + +- Re-implementing PEL semantics in `amduat-api`. +- Creating new substrate semantics in daemon code. +- Removing v1 immediately. + +## High-level model + +v2 separates read and write concerns: + +- Read paths remain direct and deterministic (`GET /v2/meta`, `GET /v2/artifacts/{ref}`, federation/space read endpoints). +- All state-changing operations are executed as PEL runs. + +In v2, `POST /v1/artifacts` equivalent is replaced by a mutation endpoint that always drives a PEL run. + +For simplified clients, v2 also exposes async operation endpoints: + +- `POST /v2/ops/put` +- `POST /v2/ops/concat` +- `POST /v2/ops/slice` +- `GET /v2/jobs/{id}` + +These enqueue work and execute PEL in the daemon background loop. + +Graph-oriented endpoints can be added as thin wrappers over concepts/relations: + +- `POST /v2/graph/nodes` (maps to concept create/publish) +- `POST /v2/graph/edges` (maps to relation edge append) +- `GET /v2/graph/nodes/{name}` (node + versions + incoming/outgoing edges) +- `GET /v2/graph/history/{name}` (version + edge timeline) + +## Versioning and contract id + +- Base path: `/v2` +- Contract id: `AMDUATD/API/2` +- Keep the same store-seeded contract model used in v1. + +## Endpoint shape + +### Read-only endpoints (v2) + +- `GET /v2/meta` +- `GET /v2/contract` +- `GET /v2/artifacts/{ref}` +- `HEAD /v2/artifacts/{ref}` +- `GET /v2/artifacts/{ref}?format=info` +- Space/federation read endpoints can be mirrored under `/v2/*` unchanged where possible. + +### Write endpoints (v2) + +- `POST /v2/pel/execute` + +No direct artifact write endpoint in v2 (`POST /v2/artifacts` is intentionally absent). + +## `POST /v2/pel/execute` + +Single mutation primitive for v2. + +### Request + +```json +{ + "program_ref": "", + "scheme_ref": "dag", + "params_ref": "", + "inputs": { + "refs": [""], + "inline_artifacts": [ + { + "content_type": "application/octet-stream", + "type_tag": "0x00000000", + "body_hex": "48656c6c6f" + } + ] + }, + "receipt": { + "input_manifest_ref": "", + "environment_ref": "", + "evaluator_id": "local-amduatd", + "executor_ref": "", + "started_at": 1731000000, + "completed_at": 1731000005 + }, + "effects": { + "publish_outputs": true, + "append_fed_log": true + } +} +``` + +### Processing contract + +Server behavior is deterministic and ordered: + +1. Resolve `program_ref`, `scheme_ref`, `params_ref`, and `inputs.refs` to refs. +2. Decode and store `inputs.inline_artifacts` into ASL store. +3. Build effective `input_refs = refs + stored_inline_refs`. +4. Execute PEL using core `pel_surf_run_with_result`. +5. Build FER receipt from run result + receipt fields. +6. Store receipt artifact and return `receipt_ref`. +7. Apply requested side effects (`publish_outputs`, `append_fed_log`) after successful run. + +If any step fails: + +- return non-2xx, +- do not publish outputs, +- do not append federation publish records. + +## Response + +```json +{ + "run_ref": "", + "trace_ref": "", + "receipt_ref": "", + "stored_input_refs": [""], + "output_refs": [""], + "status": "OK" +} +``` + +Notes: + +- `stored_input_refs` are refs created from `inline_artifacts`. +- `receipt_ref` is required for successful writes in v2. + +## Error model + +- `400` invalid request or schema violation. +- `404` referenced input/program/params not found. +- `409` conflict in post-run side effects (e.g. cursor/update conflicts). +- `422` run completed but business policy failed (reserved for policy checks). +- `500` internal failure. + +Error payload: + +```json +{ + "error": { + "code": "invalid_input_ref", + "message": "input ref not found", + "retryable": false + } +} +``` + +## Behavioral differences from v1 + +- Removed write path: direct `POST /artifacts`. +- New requirement: every write returns a receipt ref tied to a PEL run. +- `pel/run` semantics become the write gateway rather than an optional compute endpoint. + +## Compatibility and rollout + +1. Add `/v2/pel/execute` while keeping `/v1/*` unchanged. +2. Mirror core read endpoints under `/v2/*`. +3. Mark `POST /v1/artifacts` as deprecated in docs. +4. Migrate clients to v2 execute flow. +5. Remove v1 write endpoints in a later major release. + +## Open questions + +- Should v2 require exactly one output for write operations, or allow multiple outputs with one receipt? +- Should inline artifact body support both `body_hex` and `body_base64`, or move to multipart for large payloads? +- Should `publish_outputs` default to `true` or be explicit-only? +- Do we need async execution (`202 + run status endpoint`) for long-running programs in v2? + +## Minimal implementation plan + +1. Add v2 contract bytes (`registry/amduatd-api-contract.v2.json`). +2. Add `/v2/pel/execute` handler by adapting existing `POST /v1/pel/run` + artifact ingest path. +3. Factor shared receipt parsing/building from current v1 pel handler. +4. Add tests: + - inline artifact -> stored -> run -> receipt success, + - failure before run, + - failure during run, + - side effect gating (no publish on failure). +5. Document deprecation notice for `POST /v1/artifacts`. diff --git a/docs/v2-app-developer-guide.md b/docs/v2-app-developer-guide.md new file mode 100644 index 0000000..ebbf9a7 --- /dev/null +++ b/docs/v2-app-developer-guide.md @@ -0,0 +1,237 @@ +# Amduat v2 App Developer Guide + +This is the compact handoff guide for building an application against `amduatd` v2. + +For machine-readable contracts, see `registry/amduatd-api-contract.v2.json`. + +## 1) Runtime Model + +- One daemon instance serves one ASL store root. +- Transport is local Unix socket HTTP. +- Auth is currently filesystem/socket permission based. +- All graph APIs are under `/v2/graph/*`. + +Minimal local run: + +```sh +./vendor/amduat/build/amduat-asl init --root .amduat-asl +./build/amduatd --root .amduat-asl --sock amduatd.sock --store-backend index +``` + +## 2) Request Conventions + +- Use `X-Amduat-Space: ` for app isolation. +- Treat graph cursors as opaque tokens (`g1_*`), do not parse. +- Use `as_of` for snapshot-consistent reads. +- Use `include_tombstoned=true` only when you explicitly want retracted facts. + +## 3) Core App Flows + +### A. High-throughput ingest + +Use `POST /v2/graph/batch` with: + +- `idempotency_key` for deterministic retries +- `mode=continue_on_error` for partial-apply behavior +- per-item `metadata_ref` or `provenance` for trust/debug + +Expect response: + +- `ok` (overall) +- `applied` aggregate counts +- `results[]` with `{kind,index,status,code,error}` + +### B. Multi-hop retrieval for agents + +Primary endpoints: + +- `GET /v2/graph/subgraph` +- `POST /v2/graph/retrieve` +- `POST /v2/graph/query` for declarative filtering + +Use: + +- `as_of` for stable reasoning snapshots +- `max_depth`, `max_fanout`, `limit_nodes`, `limit_edges`, `max_result_bytes` +- provenance filters where needed (`provenance_ref` / `provenance_min_confidence`) + +### C. Incremental sync loop + +Use `GET /v2/graph/changes`: + +- Start with `since_cursor` (or bootstrap with `since_as_of`) +- Persist returned `next_cursor` after successful processing +- Handle `410` as replay-window expiry (full resync required) +- Optional long poll: `wait_ms` + +### D. Fact correction + +- Edge retraction: `POST /v2/graph/edges/tombstone` +- Node-version retraction: `POST /v2/graph/nodes/{name}/versions/tombstone` + +Reads default to exclude tombstoned facts on retrieval surfaces unless `include_tombstoned=true`. + +## 4) Endpoint Map (what to use when) + +- Write node: `POST /v2/graph/nodes` +- Write version: `POST /v2/graph/nodes/{name}/versions` +- Write edge: `POST /v2/graph/edges` +- Batch write: `POST /v2/graph/batch` +- Point-ish read: `GET /v2/graph/nodes/{name}` +- Edge scan: `GET /v2/graph/edges` +- Neighbor scan: `GET /v2/graph/nodes/{name}/neighbors` +- Path lookup: `GET /v2/graph/paths` +- Subgraph: `GET /v2/graph/subgraph` +- Declarative query: `POST /v2/graph/query` +- Agent retrieval: `POST /v2/graph/retrieve` +- Changes feed: `GET /v2/graph/changes` +- Export: `POST /v2/graph/export` +- Import: `POST /v2/graph/import` +- Predicate policy: `GET/POST /v2/graph/schema/predicates` +- Health/readiness/metrics: `GET /v2/healthz`, `GET /v2/readyz`, `GET /v2/metrics` +- Graph runtime/capability: `GET /v2/graph/stats`, `GET /v2/graph/capabilities` + +## 5) Provenance and Policy + +Provenance object fields for writes: + +- required: `source_uri`, `extractor`, `observed_at`, `ingested_at`, `trace_id` +- optional: `confidence`, `license` + +Policy endpoint: + +- `POST /v2/graph/schema/predicates` + +Key modes: + +- predicate validation: `strict|warn|off` +- provenance enforcement: `optional|required` + +## 6) Error Handling and Retry Rules + +- Retry-safe writes: only retries with same `idempotency_key` and identical payload. +- Validation failures: `400` or `422` (do not blind-retry). +- Not found for references/nodes: `404`. +- Cursor window expired: `410` on `/changes` (rebootstrap sync state). +- Result guard triggered: `422` (`max_result_bytes` or traversal/search limits). +- Internal errors: `500` (retry with backoff). + +## 7) Performance and Safety Defaults + +Recommended client defaults: + +- Set explicit `limit` on scans. +- Always pass `max_result_bytes` on large retrieval requests. +- Keep `max_depth` conservative (start with 2-4). +- Enable `include_stats=true` in development to monitor scanned/returned counts and selected plan. +- Call `/v2/graph/capabilities` once at startup for feature/limit negotiation. + +## 8) Minimal Startup Checklist (for external app) + +1. Probe `GET /v2/readyz`. +2. Read `GET /v2/graph/capabilities`. +3. Configure schema/provenance policy (`POST /v2/graph/schema/predicates`) if your app owns policy. +4. Start ingest path (`/v2/graph/batch` idempotent). +5. Start change-consumer loop (`/v2/graph/changes`). +6. Serve retrieval via `/v2/graph/retrieve` and `/v2/graph/subgraph`. +7. Monitor `/v2/metrics` and `/v2/graph/stats`. + +## 9) Useful Local Helpers + +- `scripts/graph_client_helpers.sh` contains practical shell helpers for: + - idempotent batch ingest + - one-step changes sync + - subgraph retrieval + +For integration tests/examples: + +- `scripts/test_graph_queries.sh` +- `scripts/test_graph_contract.sh` + +## 10) Copy/Paste Integration Skeleton + +Set local defaults: + +```sh +SOCK="amduatd.sock" +SPACE="app1" +BASE="http://localhost" +``` + +Startup probes: + +```sh +curl --unix-socket "${SOCK}" -sS "${BASE}/v2/readyz" +curl --unix-socket "${SOCK}" -sS "${BASE}/v2/graph/capabilities" +``` + +Idempotent batch ingest: + +```sh +curl --unix-socket "${SOCK}" -sS -X POST "${BASE}/v2/graph/batch" \ + -H "Content-Type: application/json" \ + -H "X-Amduat-Space: ${SPACE}" \ + -d '{ + "idempotency_key":"app1-batch-0001", + "mode":"continue_on_error", + "nodes":[{"name":"doc:1"}], + "edges":[ + {"subject":"doc:1","predicate":"ms.within_domain","object":"topic:alpha", + "provenance":{"source_uri":"urn:app:seed","extractor":"app-loader","observed_at":1,"ingested_at":2,"trace_id":"trace-1"}} + ] + }' +``` + +Incremental changes loop (bash skeleton): + +```sh +cursor="" +while true; do + if [ -n "${cursor}" ]; then + path="/v2/graph/changes?since_cursor=${cursor}&limit=200&wait_ms=15000" + else + path="/v2/graph/changes?limit=200&wait_ms=15000" + fi + + resp="$(curl --unix-socket "${SOCK}" -sS "${BASE}${path}" -H "X-Amduat-Space: ${SPACE}")" || break + + # TODO: parse and process resp.events[] in your app. + next="$(printf '%s\n' "${resp}" | sed -n 's/.*"next_cursor":"\([^"]*\)".*/\1/p')" + [ -n "${next}" ] && cursor="${next}" +done +``` + +Agent retrieval call: + +```sh +curl --unix-socket "${SOCK}" -sS -X POST "${BASE}/v2/graph/retrieve" \ + -H "Content-Type: application/json" \ + -H "X-Amduat-Space: ${SPACE}" \ + -d '{ + "roots":["doc:1"], + "goal_predicates":["ms.within_domain"], + "max_depth":2, + "max_fanout":1024, + "limit_nodes":200, + "limit_edges":400, + "max_result_bytes":1048576 + }' +``` + +Subgraph snapshot read: + +```sh +curl --unix-socket "${SOCK}" -sS \ + "${BASE}/v2/graph/subgraph?roots[]=doc:1&max_depth=2&dir=outgoing&limit_nodes=200&limit_edges=400&include_stats=true&max_result_bytes=1048576" \ + -H "X-Amduat-Space: ${SPACE}" +``` + +Edge correction (tombstone): + +```sh +EDGE_REF="" +curl --unix-socket "${SOCK}" -sS -X POST "${BASE}/v2/graph/edges/tombstone" \ + -H "Content-Type: application/json" \ + -H "X-Amduat-Space: ${SPACE}" \ + -d "{\"edge_ref\":\"${EDGE_REF}\"}" +``` diff --git a/registry/README.md b/registry/README.md index 76b0b20..0344a02 100644 --- a/registry/README.md +++ b/registry/README.md @@ -17,6 +17,7 @@ acts as the version identifier. - `api-contract.schema.md` — JSONL manifest schema for API contracts. - `api-contract.jsonl` — manifest of published contracts. - `amduatd-api-contract.v1.json` — contract bytes (v1). +- `amduatd-api-contract.v2.json` — draft contract bytes (v2, PEL-only writes). Receipt note: - `/v1/pel/run` accepts optional receipt v1.1 fields (executor fingerprint, run id, diff --git a/registry/amduatd-api-contract.v2.json b/registry/amduatd-api-contract.v2.json new file mode 100644 index 0000000..f99f9b3 --- /dev/null +++ b/registry/amduatd-api-contract.v2.json @@ -0,0 +1,820 @@ +{ + "contract": "AMDUATD/API/2", + "base_path": "/v2", + "notes": "Draft v2: PEL-only write surface. Direct artifact write endpoint removed.", + "endpoints": [ + {"method": "GET", "path": "/v2/meta"}, + {"method": "HEAD", "path": "/v2/meta"}, + {"method": "GET", "path": "/v2/contract"}, + {"method": "GET", "path": "/v2/healthz"}, + {"method": "GET", "path": "/v2/readyz"}, + {"method": "GET", "path": "/v2/metrics"}, + {"method": "GET", "path": "/v2/artifacts/{ref}"}, + {"method": "HEAD", "path": "/v2/artifacts/{ref}"}, + {"method": "GET", "path": "/v2/artifacts/{ref}?format=info"}, + {"method": "POST", "path": "/v2/pel/execute"}, + {"method": "POST", "path": "/v2/ops/put"}, + {"method": "POST", "path": "/v2/ops/concat"}, + {"method": "POST", "path": "/v2/ops/slice"}, + {"method": "GET", "path": "/v2/jobs/{id}"}, + {"method": "GET", "path": "/v2/get/{ref}"}, + {"method": "POST", "path": "/v2/graph/nodes"}, + {"method": "POST", "path": "/v2/graph/nodes/{name}/versions"}, + {"method": "POST", "path": "/v2/graph/nodes/{name}/versions/tombstone"}, + {"method": "GET", "path": "/v2/graph/nodes/{name}/versions"}, + {"method": "GET", "path": "/v2/graph/nodes/{name}/neighbors"}, + {"method": "GET", "path": "/v2/graph/search"}, + {"method": "GET", "path": "/v2/graph/paths"}, + {"method": "GET", "path": "/v2/graph/subgraph"}, + {"method": "POST", "path": "/v2/graph/edges"}, + {"method": "POST", "path": "/v2/graph/edges/tombstone"}, + {"method": "POST", "path": "/v2/graph/batch"}, + {"method": "POST", "path": "/v2/graph/query"}, + {"method": "POST", "path": "/v2/graph/retrieve"}, + {"method": "POST", "path": "/v2/graph/export"}, + {"method": "POST", "path": "/v2/graph/import"}, + {"method": "GET", "path": "/v2/graph/schema/predicates"}, + {"method": "POST", "path": "/v2/graph/schema/predicates"}, + {"method": "GET", "path": "/v2/graph/stats"}, + {"method": "GET", "path": "/v2/graph/capabilities"}, + {"method": "GET", "path": "/v2/graph/changes"}, + {"method": "GET", "path": "/v2/graph/edges"}, + {"method": "GET", "path": "/v2/graph/nodes/{name}"}, + {"method": "GET", "path": "/v2/graph/history/{name}"} + ], + "schemas": { + "job_enqueue_response": { + "type": "object", + "required": ["job_id", "status"], + "properties": { + "job_id": {"type": "integer"}, + "status": {"type": "string"} + } + }, + "job_status_response": { + "type": "object", + "required": ["job_id", "kind", "status", "created_at_ms"], + "properties": { + "job_id": {"type": "integer"}, + "kind": {"type": "string"}, + "status": {"type": "string"}, + "created_at_ms": {"type": "integer"}, + "started_at_ms": {"type": ["integer", "null"]}, + "completed_at_ms": {"type": ["integer", "null"]}, + "result_ref": {"type": ["string", "null"]}, + "error": {"type": ["string", "null"]} + } + }, + "healthz_response": { + "type": "object", + "required": ["ok", "status", "time_ms"], + "properties": { + "ok": {"type": "boolean"}, + "status": {"type": "string"}, + "time_ms": {"type": "integer"} + } + }, + "readyz_response": { + "type": "object", + "required": ["ok", "status", "components"], + "properties": { + "ok": {"type": "boolean"}, + "status": {"type": "string"}, + "components": { + "type": "object", + "required": ["graph_index", "federation"], + "properties": { + "graph_index": {"type": "boolean"}, + "federation": {"type": "boolean"} + } + } + } + }, + "put_request": { + "type": "object", + "required": ["body_hex"], + "properties": { + "body_hex": {"type": "string"}, + "type_tag": {"type": "string"} + } + }, + "concat_request": { + "type": "object", + "required": ["left_ref", "right_ref"], + "properties": { + "left_ref": {"type": "string"}, + "right_ref": {"type": "string"} + } + }, + "slice_request": { + "type": "object", + "required": ["ref", "offset", "length"], + "properties": { + "ref": {"type": "string"}, + "offset": {"type": "integer"}, + "length": {"type": "integer"} + } + }, + "graph_node_create_request": { + "type": "object", + "required": ["name"], + "properties": { + "name": {"type": "string"}, + "ref": {"type": "string", "description": "optional initial published ref"} + } + }, + "graph_node_create_response": { + "type": "object", + "required": ["name", "concept_ref"], + "properties": { + "name": {"type": "string"}, + "concept_ref": {"type": "string"} + } + }, + "graph_provenance": { + "type": "object", + "required": ["source_uri", "extractor", "observed_at", "ingested_at", "trace_id"], + "properties": { + "source_uri": {"type": "string"}, + "extractor": {"type": "string"}, + "confidence": {"type": ["string", "number", "integer"]}, + "observed_at": {"type": "integer"}, + "ingested_at": {"type": "integer"}, + "license": {"type": "string"}, + "trace_id": {"type": "string"} + } + }, + "graph_edge_create_request": { + "type": "object", + "required": ["subject", "predicate", "object"], + "properties": { + "subject": {"type": "string", "description": "concept name or hex ref"}, + "predicate": {"type": "string", "description": "relation alias/name or hex ref"}, + "object": {"type": "string", "description": "concept name or hex ref"}, + "metadata_ref": {"type": "string", "description": "optional artifact ref"}, + "provenance": {"$ref": "#/schemas/graph_provenance"} + } + }, + "graph_edge_create_response": { + "type": "object", + "required": ["subject_ref", "predicate_ref", "object_ref", "edge_ref"], + "properties": { + "subject_ref": {"type": "string"}, + "predicate_ref": {"type": "string"}, + "object_ref": {"type": "string"}, + "edge_ref": {"type": "string"}, + "metadata_ref": {"type": "string"} + } + }, + "graph_edge_tombstone_request": { + "type": "object", + "required": ["edge_ref"], + "properties": { + "edge_ref": {"type": "string"}, + "metadata_ref": {"type": "string"}, + "provenance": {"$ref": "#/schemas/graph_provenance"} + } + }, + "graph_edge_tombstone_response": { + "type": "object", + "required": ["ok", "target_edge_ref", "tombstone_edge_ref"], + "properties": { + "ok": {"type": "boolean"}, + "target_edge_ref": {"type": "string"}, + "tombstone_edge_ref": {"type": "string"}, + "metadata_ref": {"type": "string"} + } + }, + "graph_node_version_tombstone_request": { + "type": "object", + "required": ["ref"], + "properties": { + "ref": {"type": "string"}, + "metadata_ref": {"type": "string"}, + "provenance": {"$ref": "#/schemas/graph_provenance"} + } + }, + "graph_node_version_tombstone_response": { + "type": "object", + "required": ["ok", "name", "ref", "target_edge_ref", "tombstone_edge_ref"], + "properties": { + "ok": {"type": "boolean"}, + "name": {"type": "string"}, + "ref": {"type": "string"}, + "target_edge_ref": {"type": "string"}, + "tombstone_edge_ref": {"type": "string"}, + "metadata_ref": {"type": "string"} + } + }, + "graph_batch_request": { + "type": "object", + "properties": { + "idempotency_key": {"type": "string"}, + "mode": {"type": "string", "enum": ["fail_fast", "continue_on_error"]}, + "nodes": { + "type": "array", + "items": { + "type": "object", + "required": ["name"], + "properties": { + "name": {"type": "string"}, + "ref": {"type": "string"} + } + } + }, + "versions": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "ref"], + "properties": { + "name": {"type": "string"}, + "ref": {"type": "string"}, + "metadata_ref": {"type": "string"}, + "provenance": {"$ref": "#/schemas/graph_provenance"} + } + } + }, + "edges": { + "type": "array", + "items": { + "type": "object", + "required": ["subject", "predicate", "object"], + "properties": { + "subject": {"type": "string"}, + "predicate": {"type": "string"}, + "object": {"type": "string"}, + "metadata_ref": {"type": "string"}, + "provenance": {"$ref": "#/schemas/graph_provenance"} + } + } + } + } + }, + "graph_batch_response": { + "type": "object", + "required": ["ok", "applied", "results"], + "properties": { + "ok": {"type": "boolean"}, + "idempotency_key": {"type": "string"}, + "mode": {"type": "string", "enum": ["fail_fast", "continue_on_error"]}, + "applied": { + "type": "object", + "required": ["nodes", "versions", "edges"], + "properties": { + "nodes": {"type": "integer"}, + "versions": {"type": "integer"}, + "edges": {"type": "integer"} + } + }, + "results": { + "type": "array", + "items": { + "type": "object", + "required": ["kind", "index", "status", "code", "error"], + "properties": { + "kind": {"type": "string", "enum": ["node", "version", "edge"]}, + "index": {"type": "integer"}, + "status": {"type": "string", "enum": ["applied", "error"]}, + "code": {"type": "integer"}, + "error": {"type": ["string", "null"]} + } + } + } + } + }, + "graph_query_request": { + "type": "object", + "properties": { + "where": { + "type": "object", + "properties": { + "subject": {"type": "string"}, + "object": {"type": "string"}, + "node": {"type": "string"}, + "provenance_ref": {"type": "string"} + } + }, + "predicates": {"type": "array", "items": {"type": "string"}}, + "direction": {"type": "string", "enum": ["any", "outgoing", "incoming"]}, + "include_versions": {"type": "boolean"}, + "include_tombstoned": {"type": "boolean"}, + "include_stats": {"type": "boolean"}, + "max_result_bytes": {"type": "integer"}, + "as_of": {"type": ["string", "integer"]}, + "limit": {"type": "integer"}, + "cursor": {"type": ["string", "integer"]} + } + }, + "graph_query_response": { + "type": "object", + "required": ["nodes", "edges", "paging"], + "properties": { + "nodes": { + "type": "array", + "items": { + "type": "object", + "required": ["concept_ref", "name", "latest_ref"], + "properties": { + "concept_ref": {"type": "string"}, + "name": {"type": ["string", "null"]}, + "latest_ref": {"type": ["string", "null"]}, + "versions": { + "type": "array", + "items": { + "type": "object", + "required": ["edge_ref", "ref"], + "properties": { + "edge_ref": {"type": "string"}, + "ref": {"type": "string"} + } + } + } + } + } + }, + "edges": { + "type": "array", + "items": { + "type": "object", + "required": ["subject_ref", "predicate_ref", "object_ref", "edge_ref"], + "properties": { + "subject_ref": {"type": "string"}, + "predicate_ref": {"type": "string"}, + "object_ref": {"type": "string"}, + "edge_ref": {"type": "string"} + } + } + }, + "paging": { + "type": "object", + "required": ["next_cursor", "has_more"], + "properties": { + "next_cursor": {"type": ["string", "null"]}, + "has_more": {"type": "boolean"} + } + }, + "stats": { + "type": "object", + "properties": { + "scanned_edges": {"type": "integer"}, + "returned_edges": {"type": "integer"} + } + } + } + }, + "graph_retrieve_request": { + "type": "object", + "required": ["roots"], + "properties": { + "roots": {"type": "array", "items": {"type": "string"}}, + "goal_predicates": {"type": "array", "items": {"type": "string"}}, + "max_depth": {"type": "integer"}, + "max_fanout": {"type": "integer"}, + "include_versions": {"type": "boolean"}, + "include_tombstoned": {"type": "boolean"}, + "as_of": {"type": ["string", "integer"]}, + "provenance_min_confidence": {"type": ["string", "number", "integer"]}, + "limit_nodes": {"type": "integer"}, + "limit_edges": {"type": "integer"}, + "max_result_bytes": {"type": "integer"} + } + }, + "graph_retrieve_response": { + "type": "object", + "required": ["nodes", "edges", "explanations", "truncated", "stats"], + "properties": { + "nodes": { + "type": "array", + "items": { + "type": "object", + "required": ["concept_ref", "name", "latest_ref"], + "properties": { + "concept_ref": {"type": "string"}, + "name": {"type": ["string", "null"]}, + "latest_ref": {"type": ["string", "null"]}, + "versions": { + "type": "array", + "items": { + "type": "object", + "required": ["edge_ref", "ref"], + "properties": { + "edge_ref": {"type": "string"}, + "ref": {"type": "string"} + } + } + } + } + } + }, + "edges": { + "type": "array", + "items": { + "type": "object", + "required": ["subject_ref", "predicate_ref", "object_ref", "edge_ref"], + "properties": { + "subject_ref": {"type": "string"}, + "predicate_ref": {"type": "string"}, + "object_ref": {"type": "string"}, + "edge_ref": {"type": "string"} + } + } + }, + "explanations": { + "type": "array", + "items": { + "type": "object", + "required": ["edge_ref", "depth", "reasons"], + "properties": { + "edge_ref": {"type": "string"}, + "depth": {"type": "integer"}, + "reasons": {"type": "array", "items": {"type": "string"}}, + "confidence": {"type": ["number", "null"]} + } + } + }, + "truncated": {"type": "boolean"}, + "stats": { + "type": "object", + "properties": { + "scanned_edges": {"type": "integer"}, + "traversed_edges": {"type": "integer"}, + "returned_nodes": {"type": "integer"}, + "returned_edges": {"type": "integer"} + } + } + } + }, + "graph_export_request": { + "type": "object", + "properties": { + "as_of": {"type": ["string", "integer"]}, + "cursor": {"type": ["string", "integer"]}, + "limit": {"type": "integer"}, + "predicates": {"type": "array", "items": {"type": "string"}}, + "roots": {"type": "array", "items": {"type": "string"}}, + "include_tombstoned": {"type": "boolean"}, + "max_result_bytes": {"type": "integer"} + } + }, + "graph_export_response": { + "type": "object", + "required": ["items", "next_cursor", "has_more", "snapshot_as_of", "stats"], + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "required": ["seq", "edge_ref", "subject_ref", "predicate_ref", "predicate", "object_ref", "tombstoned"], + "properties": { + "seq": {"type": "integer"}, + "edge_ref": {"type": "string"}, + "subject_ref": {"type": "string"}, + "predicate_ref": {"type": "string"}, + "predicate": {"type": "string"}, + "object_ref": {"type": "string"}, + "tombstoned": {"type": "boolean"}, + "metadata_ref": {"type": ["string", "null"]} + } + } + }, + "next_cursor": {"type": ["string", "null"]}, + "has_more": {"type": "boolean"}, + "snapshot_as_of": {"type": "string"}, + "stats": { + "type": "object", + "properties": { + "scanned_edges": {"type": "integer"}, + "exported_items": {"type": "integer"} + } + } + } + }, + "graph_import_request": { + "type": "object", + "required": ["items"], + "properties": { + "mode": {"type": "string", "enum": ["fail_fast", "continue_on_error"]}, + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subject_ref": {"type": "string"}, + "subject": {"type": "string"}, + "predicate_ref": {"type": "string"}, + "predicate": {"type": "string"}, + "object_ref": {"type": "string"}, + "object": {"type": "string"}, + "metadata_ref": {"type": "string"} + } + } + } + } + }, + "graph_import_response": { + "type": "object", + "required": ["ok", "applied", "results"], + "properties": { + "ok": {"type": "boolean"}, + "applied": {"type": "integer"}, + "results": { + "type": "array", + "items": { + "type": "object", + "required": ["index", "status", "code", "error", "edge_ref"], + "properties": { + "index": {"type": "integer"}, + "status": {"type": "string", "enum": ["applied", "error"]}, + "code": {"type": "integer"}, + "error": {"type": ["string", "null"]}, + "edge_ref": {"type": ["string", "null"]} + } + } + } + } + }, + "graph_schema_predicates_request": { + "type": "object", + "properties": { + "mode": {"type": "string", "enum": ["strict", "warn", "off"]}, + "provenance_mode": {"type": "string", "enum": ["optional", "required"]}, + "predicates": { + "type": "array", + "items": { + "type": "object", + "properties": { + "predicate_ref": {"type": "string"}, + "predicate": {"type": "string"}, + "domain": {"type": "string"}, + "range": {"type": "string"} + } + } + } + } + }, + "graph_schema_predicates_response": { + "type": "object", + "required": ["mode", "provenance_mode", "predicates"], + "properties": { + "mode": {"type": "string", "enum": ["strict", "warn", "off"]}, + "provenance_mode": {"type": "string", "enum": ["optional", "required"]}, + "predicates": { + "type": "array", + "items": { + "type": "object", + "required": ["predicate_ref", "domain", "range"], + "properties": { + "predicate_ref": {"type": "string"}, + "domain": {"type": ["string", "null"]}, + "range": {"type": ["string", "null"]} + } + } + } + } + }, + "graph_stats_response": { + "type": "object", + "required": ["edges_total", "aliases_total", "index", "tombstones"], + "properties": { + "edges_total": {"type": "integer"}, + "aliases_total": {"type": "integer"}, + "index": { + "type": "object", + "required": ["built_for_edges", "src_buckets", "dst_buckets", "predicate_buckets", "src_predicate_buckets", "dst_predicate_buckets", "healthy"], + "properties": { + "built_for_edges": {"type": "integer"}, + "src_buckets": {"type": "integer"}, + "dst_buckets": {"type": "integer"}, + "predicate_buckets": {"type": "integer"}, + "src_predicate_buckets": {"type": "integer"}, + "dst_predicate_buckets": {"type": "integer"}, + "healthy": {"type": "boolean"} + } + }, + "tombstones": { + "type": "object", + "required": ["edges", "ratio"], + "properties": { + "edges": {"type": "integer"}, + "ratio": {"type": "number"} + } + } + } + }, + "graph_capabilities_response": { + "type": "object", + "required": ["contract", "graph", "runtime"], + "properties": { + "contract": {"type": "string"}, + "graph": { + "type": "object", + "required": ["version", "features", "limits", "modes"], + "properties": { + "version": {"type": "string"}, + "features": {"type": "array", "items": {"type": "string"}}, + "limits": {"type": "object"}, + "modes": {"type": "object"} + } + }, + "runtime": {"type": "object"} + } + }, + "graph_changes_response": { + "type": "object", + "required": ["events", "next_cursor", "has_more"], + "properties": { + "events": { + "type": "array", + "items": { + "type": "object", + "required": ["event", "cursor", "edge_ref", "subject_ref", "predicate_ref", "object_ref"], + "properties": { + "event": {"type": "string", "enum": ["edge_appended", "version_published", "tombstone_applied"]}, + "cursor": {"type": "string"}, + "edge_ref": {"type": "string"}, + "subject_ref": {"type": "string"}, + "predicate_ref": {"type": "string"}, + "object_ref": {"type": "string"}, + "concept_ref": {"type": "string"}, + "ref": {"type": "string"}, + "tombstoned_edge_ref": {"type": "string"} + } + } + }, + "next_cursor": {"type": ["string", "null"]}, + "has_more": {"type": "boolean"} + } + }, + "graph_node_response": { + "type": "object", + "required": ["name", "concept_ref", "latest_ref", "versions", "outgoing", "incoming"], + "properties": { + "name": {"type": "string"}, + "concept_ref": {"type": "string"}, + "latest_ref": {"type": ["string", "null"]}, + "versions": {"type": "array", "items": {"type": "string"}}, + "outgoing": { + "type": "array", + "items": { + "type": "object", + "required": ["predicate_ref", "object_ref"], + "properties": { + "predicate_ref": {"type": "string"}, + "object_ref": {"type": "string"}, + "edge_ref": {"type": "string"}, + "metadata_ref": {"type": ["string", "null"]} + } + } + }, + "incoming": { + "type": "array", + "items": { + "type": "object", + "required": ["predicate_ref", "subject_ref"], + "properties": { + "predicate_ref": {"type": "string"}, + "subject_ref": {"type": "string"}, + "edge_ref": {"type": "string"}, + "metadata_ref": {"type": ["string", "null"]} + } + } + } + } + }, + "graph_history_response": { + "type": "object", + "required": ["name", "events"], + "properties": { + "name": {"type": "string"}, + "events": { + "type": "array", + "items": { + "type": "object", + "required": ["event", "at_ms"], + "properties": { + "event": {"type": "string"}, + "at_ms": {"type": "integer"}, + "ref": {"type": ["string", "null"]}, + "edge_ref": {"type": ["string", "null"]}, + "details": {"type": "object"} + } + } + } + } + }, + "pel_execute_request": { + "type": "object", + "required": ["program_ref", "inputs", "receipt"], + "properties": { + "program_ref": {"type": "string", "description": "hex ref or concept name"}, + "scheme_ref": {"type": "string", "description": "hex ref or 'dag'"}, + "params_ref": {"type": "string", "description": "hex ref or concept name"}, + "inputs": { + "type": "object", + "properties": { + "refs": { + "type": "array", + "items": {"type": "string", "description": "hex ref or concept name"} + }, + "inline_artifacts": { + "type": "array", + "items": { + "type": "object", + "required": ["body_hex"], + "properties": { + "content_type": {"type": "string"}, + "type_tag": {"type": "string", "description": "hex tag id, optional"}, + "body_hex": {"type": "string"} + } + } + } + } + }, + "receipt": { + "type": "object", + "required": [ + "input_manifest_ref", + "environment_ref", + "evaluator_id", + "executor_ref", + "started_at", + "completed_at" + ], + "properties": { + "input_manifest_ref": {"type": "string", "description": "hex ref or concept name"}, + "environment_ref": {"type": "string", "description": "hex ref or concept name"}, + "evaluator_id": {"type": "string"}, + "executor_ref": {"type": "string", "description": "hex ref or concept name"}, + "sbom_ref": {"type": "string", "description": "hex ref or concept name"}, + "parity_digest_hex": {"type": "string"}, + "executor_fingerprint_ref": {"type": "string", "description": "hex ref or concept name"}, + "run_id_hex": {"type": "string"}, + "limits": { + "type": "object", + "required": ["cpu_ms", "wall_ms", "max_rss_kib", "io_reads", "io_writes"], + "properties": { + "cpu_ms": {"type": "integer"}, + "wall_ms": {"type": "integer"}, + "max_rss_kib": {"type": "integer"}, + "io_reads": {"type": "integer"}, + "io_writes": {"type": "integer"} + } + }, + "logs": { + "type": "array", + "items": { + "type": "object", + "required": ["kind", "log_ref", "sha256_hex"], + "properties": { + "kind": {"type": "integer"}, + "log_ref": {"type": "string", "description": "hex ref or concept name"}, + "sha256_hex": {"type": "string"} + } + } + }, + "determinism_level": {"type": "integer", "description": "0-255"}, + "rng_seed_hex": {"type": "string"}, + "signature_hex": {"type": "string"}, + "started_at": {"type": "integer"}, + "completed_at": {"type": "integer"} + } + }, + "effects": { + "type": "object", + "properties": { + "publish_outputs": {"type": "boolean"}, + "append_fed_log": {"type": "boolean"} + } + } + } + }, + "pel_execute_response": { + "type": "object", + "required": ["run_ref", "receipt_ref", "stored_input_refs", "output_refs", "status"], + "properties": { + "run_ref": {"type": "string", "description": "hex ref"}, + "trace_ref": {"type": "string", "description": "hex ref"}, + "receipt_ref": {"type": "string", "description": "hex ref"}, + "stored_input_refs": {"type": "array", "items": {"type": "string", "description": "hex ref"}}, + "output_refs": {"type": "array", "items": {"type": "string", "description": "hex ref"}}, + "status": {"type": "string"} + } + }, + "error_response": { + "type": "object", + "required": ["error"], + "properties": { + "error": { + "type": "object", + "required": ["code", "message", "retryable"], + "properties": { + "code": {"type": "string"}, + "message": {"type": "string"}, + "retryable": {"type": "boolean"} + } + } + } + } + } +} diff --git a/scripts/graph_client_helpers.sh b/scripts/graph_client_helpers.sh new file mode 100755 index 0000000..130527c --- /dev/null +++ b/scripts/graph_client_helpers.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Reusable HTTP and graph client helpers for local unix-socket amduatd usage. + +graph_helpers_init() { + if [[ $# -lt 1 ]]; then + echo "usage: graph_helpers_init ROOT_DIR" >&2 + return 1 + fi + GRAPH_HELPERS_ROOT_DIR="$1" + GRAPH_HELPERS_HTTP="${GRAPH_HELPERS_ROOT_DIR}/build/amduatd_http_unix" + GRAPH_HELPERS_USE_HTTP=0 + + if command -v curl >/dev/null 2>&1; then + if curl --help 2>/dev/null | grep -q -- '--unix-socket'; then + GRAPH_HELPERS_USE_HTTP=0 + else + GRAPH_HELPERS_USE_HTTP=1 + fi + else + GRAPH_HELPERS_USE_HTTP=1 + fi + + if [[ "${GRAPH_HELPERS_USE_HTTP}" -eq 1 && ! -x "${GRAPH_HELPERS_HTTP}" ]]; then + echo "missing http transport (need curl --unix-socket or build/amduatd_http_unix)" >&2 + return 1 + fi +} + +graph_http_get() { + local sock="$1" + local path="$2" + shift 2 + if [[ "${GRAPH_HELPERS_USE_HTTP}" -eq 1 ]]; then + "${GRAPH_HELPERS_HTTP}" --sock "${sock}" --method GET --path "${path}" "$@" + else + curl --silent --show-error --fail \ + --unix-socket "${sock}" \ + "$@" \ + "http://localhost${path}" + fi +} + +graph_http_get_allow() { + local sock="$1" + local path="$2" + shift 2 + if [[ "${GRAPH_HELPERS_USE_HTTP}" -eq 1 ]]; then + "${GRAPH_HELPERS_HTTP}" --sock "${sock}" --method GET --path "${path}" --allow-status "$@" + else + curl --silent --show-error \ + --unix-socket "${sock}" \ + "$@" \ + "http://localhost${path}" + fi +} + +graph_http_post() { + local sock="$1" + local path="$2" + local data="$3" + shift 3 + if [[ "${GRAPH_HELPERS_USE_HTTP}" -eq 1 ]]; then + "${GRAPH_HELPERS_HTTP}" --sock "${sock}" --method POST --path "${path}" --data "${data}" "$@" + else + curl --silent --show-error --fail \ + --unix-socket "${sock}" \ + "$@" \ + --data-binary "${data}" \ + "http://localhost${path}" + fi +} + +graph_http_post_allow() { + local sock="$1" + local path="$2" + local data="$3" + shift 3 + if [[ "${GRAPH_HELPERS_USE_HTTP}" -eq 1 ]]; then + "${GRAPH_HELPERS_HTTP}" --sock "${sock}" --method POST --path "${path}" --data "${data}" --allow-status "$@" + else + curl --silent --show-error \ + --unix-socket "${sock}" \ + "$@" \ + --data-binary "${data}" \ + "http://localhost${path}" + fi +} + +graph_wait_for_ready() { + local sock="$1" + local pid="$2" + local log_path="$3" + local i + for i in $(seq 1 120); do + if ! kill -0 "${pid}" >/dev/null 2>&1; then + if [[ -f "${log_path}" ]] && grep -q "bind: Operation not permitted" "${log_path}"; then + echo "skip: bind not permitted for unix socket" >&2 + return 77 + fi + if [[ -f "${log_path}" ]]; then + cat "${log_path}" >&2 + fi + return 1 + fi + if [[ -S "${sock}" ]] && graph_http_get "${sock}" "/v1/meta" >/dev/null 2>&1; then + return 0 + fi + sleep 0.1 + done + return 1 +} + +graph_batch_ingest() { + local sock="$1" + local space="$2" + local payload="$3" + graph_http_post "${sock}" "/v2/graph/batch" "${payload}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space}" +} + +graph_changes_sync_once() { + local sock="$1" + local space="$2" + local cursor="$3" + local limit="$4" + local path="/v2/graph/changes?limit=${limit}" + if [[ -n "${cursor}" ]]; then + path+="&since_cursor=${cursor}" + fi + graph_http_get "${sock}" "${path}" --header "X-Amduat-Space: ${space}" +} + +graph_subgraph_fetch() { + local sock="$1" + local space="$2" + local root="$3" + local max_depth="$4" + local predicates="${5:-}" + local path="/v2/graph/subgraph?roots[]=${root}&max_depth=${max_depth}&dir=outgoing&limit_nodes=256&limit_edges=256" + if [[ -n "${predicates}" ]]; then + path+="&predicates[]=${predicates}" + fi + graph_http_get "${sock}" "${path}" --header "X-Amduat-Space: ${space}" +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + if [[ $# -lt 1 ]]; then + echo "usage: $0 COMMAND ..." >&2 + echo "commands: batch-ingest, sync-once, subgraph" >&2 + exit 2 + fi + cmd="$1" + shift + : "${AMDUATD_ROOT:?set AMDUATD_ROOT to repo root}" + graph_helpers_init "${AMDUATD_ROOT}" + case "${cmd}" in + batch-ingest) + if [[ $# -ne 3 ]]; then + echo "usage: $0 batch-ingest SOCK SPACE PAYLOAD_JSON" >&2 + exit 2 + fi + graph_batch_ingest "$1" "$2" "$3" + ;; + sync-once) + if [[ $# -ne 4 ]]; then + echo "usage: $0 sync-once SOCK SPACE CURSOR LIMIT" >&2 + exit 2 + fi + graph_changes_sync_once "$1" "$2" "$3" "$4" + ;; + subgraph) + if [[ $# -lt 4 || $# -gt 5 ]]; then + echo "usage: $0 subgraph SOCK SPACE ROOT MAX_DEPTH [PREDICATE]" >&2 + exit 2 + fi + graph_subgraph_fetch "$1" "$2" "$3" "$4" "${5:-}" + ;; + *) + echo "unknown command: ${cmd}" >&2 + exit 2 + ;; + esac +fi diff --git a/scripts/test_graph_contract.sh b/scripts/test_graph_contract.sh new file mode 100755 index 0000000..c881a67 --- /dev/null +++ b/scripts/test_graph_contract.sh @@ -0,0 +1,379 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMPDIR="${TMPDIR:-/tmp}" +mkdir -p "${TMPDIR}" + +if ! command -v grep >/dev/null 2>&1; then + echo "skip: grep not found" >&2 + exit 77 +fi +if ! command -v awk >/dev/null 2>&1; then + echo "skip: awk not found" >&2 + exit 77 +fi + +# shellcheck source=/dev/null +source "${ROOT_DIR}/scripts/graph_client_helpers.sh" +graph_helpers_init "${ROOT_DIR}" + +AMDUATD_BIN="${ROOT_DIR}/build/amduatd" +ASL_BIN="${ROOT_DIR}/vendor/amduat/build/amduat-asl" +if [[ ! -x "${AMDUATD_BIN}" || ! -x "${ASL_BIN}" ]]; then + echo "missing binaries; build amduatd and amduat-asl first" >&2 + exit 1 +fi + +tmp_root="$(mktemp -d -p "${TMPDIR}" amduatd-graph-contract-XXXXXX)" +root="${tmp_root}/root" +sock="${tmp_root}/amduatd.sock" +space_id="graphcontract" +log_file="${tmp_root}/amduatd.log" + +cleanup() { + if [[ -n "${pid:-}" ]]; then + kill "${pid}" >/dev/null 2>&1 || true + wait "${pid}" >/dev/null 2>&1 || true + fi + rm -rf "${tmp_root}" +} +trap cleanup EXIT + +extract_json_string() { + local key="$1" + awk -v k="\"${key}\":\"" ' + { + pos = index($0, k); + if (pos > 0) { + rest = substr($0, pos + length(k)); + end = index(rest, "\""); + if (end > 0) { + print substr(rest, 1, end - 1); + exit 0; + } + } + } + ' +} + +cursor_to_num() { + local token="$1" + if [[ -z "${token}" ]]; then + echo 0 + return 0 + fi + if [[ "${token}" == g1_* ]]; then + echo "${token#g1_}" + return 0 + fi + echo "${token}" +} + +cursor_plus_one() { + local token="$1" + local n + if [[ -z "${token}" ]]; then + return 1 + fi + if [[ "${token}" == g1_* ]]; then + n="${token#g1_}" + printf 'g1_%s' "$((n + 1))" + return 0 + fi + printf '%s' "$((token + 1))" +} + +create_artifact() { + local payload="$1" + local resp + local ref + resp="$({ + graph_http_post "${sock}" "/v1/artifacts" "${payload}" \ + --header "Content-Type: application/octet-stream" \ + --header "X-Amduat-Space: ${space_id}" + })" + ref="$(printf '%s\n' "${resp}" | extract_json_string "ref")" + if [[ -z "${ref}" ]]; then + echo "failed to parse artifact ref: ${resp}" >&2 + exit 1 + fi + printf '%s' "${ref}" +} + +create_node() { + local name="$1" + local ref="$2" + graph_http_post "${sock}" "/v2/graph/nodes" "{\"name\":\"${name}\",\"ref\":\"${ref}\"}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" >/dev/null +} + +create_edge() { + local s="$1" + local p="$2" + local o="$3" + graph_http_post "${sock}" "/v2/graph/edges" "{\"subject\":\"${s}\",\"predicate\":\"${p}\",\"object\":\"${o}\"}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" >/dev/null +} + +mkdir -p "${root}" +"${ASL_BIN}" init --root "${root}" >/dev/null + +"${AMDUATD_BIN}" --root "${root}" --sock "${sock}" --store-backend index --space "${space_id}" \ + >"${log_file}" 2>&1 & +pid=$! + +ready_rc=0 +graph_wait_for_ready "${sock}" "${pid}" "${log_file}" || ready_rc=$? +if [[ ${ready_rc} -eq 77 ]]; then + exit 77 +fi +if [[ ${ready_rc} -ne 0 ]]; then + echo "daemon not ready" >&2 + exit 1 +fi + +ref_a="$(create_artifact "contract-a")" +ref_b="$(create_artifact "contract-b")" +ref_c="$(create_artifact "contract-c")" +ref_v1="$(create_artifact "contract-v1")" +ref_v2="$(create_artifact "contract-v2")" + +create_node "gc-a" "${ref_a}" +create_node "gc-b" "${ref_b}" +create_node "gc-c" "${ref_c}" +create_node "gc-v" "${ref_v1}" + +create_edge "gc-a" "ms.computed_by" "gc-b" +create_edge "gc-b" "ms.computed_by" "gc-c" + +graph_http_post "${sock}" "/v2/graph/nodes/gc-v/versions" "{\"ref\":\"${ref_v2}\"}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" >/dev/null + +versions_cutoff_raw="$( + graph_http_get "${sock}" "/v2/graph/changes?event_types[]=version_published&limit=100" \ + --header "X-Amduat-Space: ${space_id}" +)" +versions_cutoff="$(printf '%s\n' "${versions_cutoff_raw}" | extract_json_string "next_cursor")" +if [[ -z "${versions_cutoff}" ]]; then + echo "missing version cutoff cursor: ${versions_cutoff_raw}" >&2 + exit 1 +fi +versions_cutoff="$(cursor_plus_one "${versions_cutoff}")" + +graph_http_post "${sock}" "/v2/graph/nodes/gc-v/versions/tombstone" "{\"ref\":\"${ref_v2}\"}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" >/dev/null + +node_default="$( + graph_http_get "${sock}" "/v2/graph/nodes/gc-v" \ + --header "X-Amduat-Space: ${space_id}" +)" +node_default_latest="$(printf '%s\n' "${node_default}" | extract_json_string "latest_ref")" +if [[ "${node_default_latest}" != "${ref_v1}" ]]; then + echo "node default latest_ref mismatch (want ${ref_v1}): ${node_default}" >&2 + exit 1 +fi +if echo "${node_default}" | grep -q "\"ref\":\"${ref_v2}\""; then + echo "node default should hide tombstoned ${ref_v2}: ${node_default}" >&2 + exit 1 +fi + +node_all="$( + graph_http_get "${sock}" "/v2/graph/nodes/gc-v?include_tombstoned=true" \ + --header "X-Amduat-Space: ${space_id}" +)" +node_all_latest="$(printf '%s\n' "${node_all}" | extract_json_string "latest_ref")" +if [[ "${node_all_latest}" != "${ref_v2}" ]]; then + echo "node include_tombstoned latest_ref mismatch (want ${ref_v2}): ${node_all}" >&2 + exit 1 +fi +echo "${node_all}" | grep -q "\"ref\":\"${ref_v2}\"" || { + echo "node include_tombstoned should include ${ref_v2}: ${node_all}" >&2 + exit 1 +} + +node_asof="$( + graph_http_get "${sock}" "/v2/graph/nodes/gc-v?as_of=${versions_cutoff}" \ + --header "X-Amduat-Space: ${space_id}" +)" +node_asof_latest="$(printf '%s\n' "${node_asof}" | extract_json_string "latest_ref")" +if [[ "${node_asof_latest}" != "${ref_v2}" ]]; then + echo "node as_of before tombstone latest_ref mismatch (want ${ref_v2}): ${node_asof}" >&2 + exit 1 +fi +echo "${node_asof}" | grep -q "\"ref\":\"${ref_v2}\"" || { + echo "node as_of before tombstone should include ${ref_v2}: ${node_asof}" >&2 + exit 1 +} + +history_default="$( + graph_http_get "${sock}" "/v2/graph/history/gc-v" \ + --header "X-Amduat-Space: ${space_id}" +)" +history_default_latest="$(printf '%s\n' "${history_default}" | extract_json_string "latest_ref")" +if [[ "${history_default_latest}" != "${ref_v1}" ]]; then + echo "history default latest_ref mismatch (want ${ref_v1}): ${history_default}" >&2 + exit 1 +fi +if echo "${history_default}" | grep -q "\"ref\":\"${ref_v2}\""; then + echo "history default should hide tombstoned ${ref_v2}: ${history_default}" >&2 + exit 1 +fi + +history_all="$( + graph_http_get "${sock}" "/v2/graph/history/gc-v?include_tombstoned=true" \ + --header "X-Amduat-Space: ${space_id}" +)" +history_all_latest="$(printf '%s\n' "${history_all}" | extract_json_string "latest_ref")" +if [[ "${history_all_latest}" != "${ref_v2}" ]]; then + echo "history include_tombstoned latest_ref mismatch (want ${ref_v2}): ${history_all}" >&2 + exit 1 +fi +echo "${history_all}" | grep -q "\"ref\":\"${ref_v2}\"" || { + echo "history include_tombstoned should include ${ref_v2}: ${history_all}" >&2 + exit 1 +} + +# schema strict: block predicate not in allowed list. +strict_policy='{"mode":"strict","predicates":[{"predicate":"ms.computed_by"}]}' +graph_http_post "${sock}" "/v2/graph/schema/predicates" "${strict_policy}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" >/dev/null + +strict_reject="$({ + graph_http_post_allow "${sock}" "/v2/graph/edges" '{"subject":"gc-a","predicate":"ms.within_domain","object":"gc-c"}' \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" +})" +echo "${strict_reject}" | grep -q 'predicate rejected by schema policy' || { + echo "strict mode should reject disallowed predicate: ${strict_reject}" >&2 + exit 1 +} + +# schema warn: same write should pass. +warn_policy='{"mode":"warn","predicates":[{"predicate":"ms.computed_by"}]}' +graph_http_post "${sock}" "/v2/graph/schema/predicates" "${warn_policy}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" >/dev/null + +warn_accept="$({ + graph_http_post "${sock}" "/v2/graph/edges" '{"subject":"gc-a","predicate":"ms.within_domain","object":"gc-c"}' \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" +})" +echo "${warn_accept}" | grep -q '"edge_ref":"' || { + echo "warn mode should allow disallowed predicate: ${warn_accept}" >&2 + exit 1 +} + +# provenance required: writes without metadata/provenance must fail with 422. +required_policy='{"mode":"warn","provenance_mode":"required","predicates":[{"predicate":"ms.computed_by"}]}' +graph_http_post "${sock}" "/v2/graph/schema/predicates" "${required_policy}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" >/dev/null + +schema_required="$( + graph_http_get "${sock}" "/v2/graph/schema/predicates" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${schema_required}" | grep -q '"provenance_mode":"required"' || { + echo "schema provenance_mode did not persist: ${schema_required}" >&2 + exit 1 +} + +required_reject="$({ + graph_http_post_allow "${sock}" "/v2/graph/edges" '{"subject":"gc-a","predicate":"ms.computed_by","object":"gc-b"}' \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" +})" +echo "${required_reject}" | grep -q 'provenance required by schema policy' || { + echo "required provenance should reject missing attachment: ${required_reject}" >&2 + exit 1 +} + +required_version_reject="$({ + graph_http_post_allow "${sock}" "/v2/graph/nodes/gc-a/versions" "{\"ref\":\"${ref_v1}\"}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" +})" +echo "${required_version_reject}" | grep -q 'provenance required by schema policy' || { + echo "required provenance should reject version write without attachment: ${required_version_reject}" >&2 + exit 1 +} + +required_accept="$({ + graph_http_post "${sock}" "/v2/graph/edges" '{"subject":"gc-a","predicate":"ms.computed_by","object":"gc-b","provenance":{"source_uri":"urn:test","extractor":"contract-test","observed_at":1,"ingested_at":2,"trace_id":"trace-required-1"}}' \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" +})" +echo "${required_accept}" | grep -q '"edge_ref":"' || { + echo "required provenance should allow explicit provenance payload: ${required_accept}" >&2 + exit 1 +} + +# reset to optional so remaining tests can use minimal payloads. +optional_policy='{"mode":"warn","provenance_mode":"optional","predicates":[{"predicate":"ms.computed_by"}]}' +graph_http_post "${sock}" "/v2/graph/schema/predicates" "${optional_policy}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" >/dev/null + +# batch idempotency replay must be deterministic. +idem_payload='{"idempotency_key":"gc-idem-1","mode":"continue_on_error","edges":[{"subject":"gc-a","predicate":"ms.computed_by","object":"gc-c"},{"subject":"gc-missing","predicate":"ms.computed_by","object":"gc-c"}]}' +idem_first="$(graph_batch_ingest "${sock}" "${space_id}" "${idem_payload}")" +idem_second="$(graph_batch_ingest "${sock}" "${space_id}" "${idem_payload}")" +if [[ "${idem_first}" != "${idem_second}" ]]; then + echo "idempotent replay mismatch" >&2 + echo "first=${idem_first}" >&2 + echo "second=${idem_second}" >&2 + exit 1 +fi + +# payload mismatch on same idempotency key must fail. +idem_conflict="$({ + graph_http_post_allow "${sock}" "/v2/graph/batch" '{"idempotency_key":"gc-idem-1","mode":"continue_on_error","edges":[{"subject":"gc-a","predicate":"ms.computed_by","object":"gc-b"}]}' \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" +})" +echo "${idem_conflict}" | grep -q 'idempotency_key reuse with different payload' || { + echo "idempotency conflict missing expected error: ${idem_conflict}" >&2 + exit 1 +} + +# changes sync helper: cursor monotonic + resumable loop. +changes_1="$(graph_changes_sync_once "${sock}" "${space_id}" "" 2)" +cursor_1="$(printf '%s\n' "${changes_1}" | extract_json_string "next_cursor")" +if [[ -z "${cursor_1}" ]]; then + echo "changes first page missing next_cursor: ${changes_1}" >&2 + exit 1 +fi + +changes_2="$(graph_changes_sync_once "${sock}" "${space_id}" "${cursor_1}" 2)" +cursor_2="$(printf '%s\n' "${changes_2}" | extract_json_string "next_cursor")" +if [[ -z "${cursor_2}" ]]; then + echo "changes second page missing next_cursor: ${changes_2}" >&2 + exit 1 +fi + +num_1="$(cursor_to_num "${cursor_1}")" +num_2="$(cursor_to_num "${cursor_2}")" +if [[ "${num_2}" -lt "${num_1}" ]]; then + echo "changes cursor regressed: ${cursor_1} -> ${cursor_2}" >&2 + exit 1 +fi + +# subgraph helper should return connected nodes. +subgraph_resp="$(graph_subgraph_fetch "${sock}" "${space_id}" "gc-a" 2 "ms.computed_by")" +echo "${subgraph_resp}" | grep -q '"name":"gc-a"' || { + echo "subgraph missing gc-a: ${subgraph_resp}" >&2 + exit 1 +} +echo "${subgraph_resp}" | grep -q '"name":"gc-b"' || { + echo "subgraph missing gc-b: ${subgraph_resp}" >&2 + exit 1 +} + +echo "ok: v2 graph contract tests passed" diff --git a/scripts/test_graph_queries.sh b/scripts/test_graph_queries.sh new file mode 100755 index 0000000..13d56c8 --- /dev/null +++ b/scripts/test_graph_queries.sh @@ -0,0 +1,778 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +HTTP_HELPER="${ROOT_DIR}/build/amduatd_http_unix" +USE_HTTP_HELPER=0 +TMPDIR="${TMPDIR:-/tmp}" +mkdir -p "${TMPDIR}" + +if ! command -v grep >/dev/null 2>&1; then + echo "skip: grep not found" >&2 + exit 77 +fi +if ! command -v awk >/dev/null 2>&1; then + echo "skip: awk not found" >&2 + exit 77 +fi +if command -v curl >/dev/null 2>&1; then + if curl --help 2>/dev/null | grep -q -- '--unix-socket'; then + USE_HTTP_HELPER=0 + else + USE_HTTP_HELPER=1 + fi +else + USE_HTTP_HELPER=1 +fi +if [[ "${USE_HTTP_HELPER}" -eq 1 && ! -x "${HTTP_HELPER}" ]]; then + echo "skip: curl lacks --unix-socket support and helper missing" >&2 + exit 77 +fi + +AMDUATD_BIN="${ROOT_DIR}/build/amduatd" +ASL_BIN="${ROOT_DIR}/vendor/amduat/build/amduat-asl" + +if [[ ! -x "${AMDUATD_BIN}" || ! -x "${ASL_BIN}" ]]; then + echo "missing binaries; build amduatd and amduat-asl first" >&2 + exit 1 +fi + +tmp_root="$(mktemp -d -p "${TMPDIR}" amduatd-graph-queries-XXXXXX)" +root="${tmp_root}/root" +sock="${tmp_root}/amduatd.sock" +space_id="graphq" +log_file="${tmp_root}/amduatd.log" + +cleanup() { + if [[ -n "${pid:-}" ]]; then + kill "${pid}" >/dev/null 2>&1 || true + fi + rm -rf "${tmp_root}" +} +trap cleanup EXIT + +http_get() { + local path="$1" + shift + if [[ "${USE_HTTP_HELPER}" -eq 1 ]]; then + "${HTTP_HELPER}" --sock "${sock}" --method GET --path "${path}" "$@" + else + curl --silent --show-error --fail \ + --unix-socket "${sock}" \ + "$@" \ + "http://localhost${path}" + fi +} + +http_post() { + local path="$1" + local data="$2" + shift 2 + if [[ "${USE_HTTP_HELPER}" -eq 1 ]]; then + "${HTTP_HELPER}" --sock "${sock}" --method POST --path "${path}" \ + --data "${data}" \ + "$@" + else + curl --silent --show-error --fail \ + --unix-socket "${sock}" \ + "$@" \ + --data-binary "${data}" \ + "http://localhost${path}" + fi +} + +extract_json_string() { + local key="$1" + awk -v k="\"${key}\":\"" ' + { + pos = index($0, k); + if (pos > 0) { + rest = substr($0, pos + length(k)); + end = index(rest, "\""); + if (end > 0) { + print substr(rest, 1, end - 1); + exit 0; + } + } + } + ' +} + +cursor_plus_one() { + local token="$1" + local n + if [[ -z "${token}" ]]; then + return 1 + fi + if [[ "${token}" == g1_* ]]; then + n="${token#g1_}" + printf 'g1_%s' "$((n + 1))" + return 0 + fi + printf '%s' "$((token + 1))" +} + +wait_for_ready() { + local i + for i in $(seq 1 100); do + if ! kill -0 "${pid}" >/dev/null 2>&1; then + if [[ -f "${log_file}" ]] && grep -q "bind: Operation not permitted" "${log_file}"; then + echo "skip: bind not permitted for unix socket" >&2 + exit 77 + fi + [[ -f "${log_file}" ]] && cat "${log_file}" >&2 + return 1 + fi + if [[ -S "${sock}" ]] && http_get "/v1/meta" >/dev/null 2>&1; then + return 0 + fi + sleep 0.1 + done + return 1 +} + +create_artifact() { + local payload="$1" + local resp + local ref + resp="$( + http_post "/v1/artifacts" "${payload}" \ + --header "Content-Type: application/octet-stream" \ + --header "X-Amduat-Space: ${space_id}" + )" + ref="$(printf '%s\n' "${resp}" | extract_json_string "ref")" + if [[ -z "${ref}" ]]; then + echo "failed to parse artifact ref: ${resp}" >&2 + exit 1 + fi + printf '%s' "${ref}" +} + +create_node() { + local name="$1" + local ref="$2" + http_post "/v2/graph/nodes" "{\"name\":\"${name}\",\"ref\":\"${ref}\"}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" >/dev/null +} + +create_edge() { + local subject="$1" + local predicate="$2" + local object="$3" + http_post "/v2/graph/edges" \ + "{\"subject\":\"${subject}\",\"predicate\":\"${predicate}\",\"object\":\"${object}\"}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" >/dev/null +} + +create_edge_with_metadata() { + local subject="$1" + local predicate="$2" + local object="$3" + local metadata_ref="$4" + http_post "/v2/graph/edges" \ + "{\"subject\":\"${subject}\",\"predicate\":\"${predicate}\",\"object\":\"${object}\",\"metadata_ref\":\"${metadata_ref}\"}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" >/dev/null +} + +start_daemon() { + "${AMDUATD_BIN}" --root "${root}" --sock "${sock}" --store-backend index --space "${space_id}" \ + >"${log_file}" 2>&1 & + pid=$! + if ! wait_for_ready; then + echo "daemon not ready" >&2 + exit 1 + fi +} + +restart_daemon() { + if [[ -n "${pid:-}" ]]; then + kill "${pid}" >/dev/null 2>&1 || true + wait "${pid}" >/dev/null 2>&1 || true + fi + start_daemon +} + +mkdir -p "${root}" +"${ASL_BIN}" init --root "${root}" +start_daemon + +ref_a="$(create_artifact "payload-a")" +ref_b="$(create_artifact "payload-b")" +ref_c="$(create_artifact "payload-c")" +ref_d="$(create_artifact "payload-d")" +ref_e="$(create_artifact "payload-e")" +ref_e2="$(create_artifact "payload-e2")" +ref_v1="$(create_artifact "payload-v1")" +ref_v2="$(create_artifact "payload-v2")" +ref_p1="$(create_artifact "payload-provenance-1")" +ref_p2="$(create_artifact "payload-provenance-2")" + +create_node "gq-a" "${ref_a}" +create_node "gq-b" "${ref_b}" +create_node "gq-c" "${ref_c}" +create_node "gq-d" "${ref_d}" +create_node "gq-e" "${ref_e}" +create_node "gq-v" "${ref_v1}" +create_node "gq-prov1" "${ref_p1}" +create_node "gq-prov2" "${ref_p2}" + +# Seed path and neighbor data in a controlled edge order. +create_edge "gq-a" "ms.within_domain" "gq-b" +create_edge "gq-b" "ms.within_domain" "gq-c" +create_edge "gq-a" "ms.computed_by" "gq-b" + +cutoff_resp="$( + http_get "/v2/graph/changes?event_types[]=edge_appended&limit=100" \ + --header "X-Amduat-Space: ${space_id}" +)" +cutoff_cursor="$(printf '%s\n' "${cutoff_resp}" | extract_json_string "next_cursor")" +if [[ -z "${cutoff_cursor}" ]]; then + echo "failed to parse cutoff cursor: ${cutoff_resp}" >&2 + exit 1 +fi +cutoff_cursor="$(cursor_plus_one "${cutoff_cursor}")" + +create_edge "gq-a" "ms.computed_by" "gq-d" +create_edge "gq-a" "ms.within_domain" "gq-c" + +search_page1="$( + http_get "/v2/graph/search?name_prefix=gq-&limit=2" \ + --header "X-Amduat-Space: ${space_id}" +)" +search_cursor="$(printf '%s\n' "${search_page1}" | extract_json_string "next_cursor")" +if [[ -z "${search_cursor}" ]]; then + echo "missing search cursor in page1: ${search_page1}" >&2 + exit 1 +fi +search_page2="$( + http_get "/v2/graph/search?name_prefix=gq-&limit=10&cursor=${search_cursor}" \ + --header "X-Amduat-Space: ${space_id}" +)" +search_joined="${search_page1} ${search_page2}" +echo "${search_joined}" | grep -q '"name":"gq-a"' || { echo "search missing gq-a" >&2; exit 1; } +echo "${search_joined}" | grep -q '"name":"gq-b"' || { echo "search missing gq-b" >&2; exit 1; } +echo "${search_joined}" | grep -q '"name":"gq-c"' || { echo "search missing gq-c" >&2; exit 1; } +echo "${search_joined}" | grep -q '"name":"gq-d"' || { echo "search missing gq-d" >&2; exit 1; } + +neighbors_page1="$( + http_get "/v2/graph/nodes/gq-a/neighbors?dir=outgoing&predicate=ms.computed_by&limit=1&expand_names=true" \ + --header "X-Amduat-Space: ${space_id}" +)" +neighbors_cursor="$(printf '%s\n' "${neighbors_page1}" | extract_json_string "next_cursor")" +if [[ -z "${neighbors_cursor}" ]]; then + echo "missing neighbors cursor in page1: ${neighbors_page1}" >&2 + exit 1 +fi +neighbors_page2="$( + http_get "/v2/graph/nodes/gq-a/neighbors?dir=outgoing&predicate=ms.computed_by&limit=10&cursor=${neighbors_cursor}&expand_names=true" \ + --header "X-Amduat-Space: ${space_id}" +)" +neighbors_joined="${neighbors_page1} ${neighbors_page2}" +echo "${neighbors_joined}" | grep -q '"neighbor_name":"gq-b"' || { echo "neighbors missing gq-b" >&2; exit 1; } +echo "${neighbors_joined}" | grep -q '"neighbor_name":"gq-d"' || { echo "neighbors missing gq-d" >&2; exit 1; } + +neighbors_asof="$( + http_get "/v2/graph/nodes/gq-a/neighbors?dir=outgoing&predicate=ms.computed_by&limit=10&as_of=${cutoff_cursor}&expand_names=true" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${neighbors_asof}" | grep -q '"neighbor_name":"gq-b"' || { echo "neighbors as_of missing gq-b" >&2; exit 1; } +if echo "${neighbors_asof}" | grep -q '"neighbor_name":"gq-d"'; then + echo "neighbors as_of unexpectedly includes gq-d" >&2 + exit 1 +fi + +paths_latest="$( + http_get "/v2/graph/paths?from=gq-a&to=gq-c&predicate=ms.within_domain&max_depth=4&expand_names=true" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${paths_latest}" | grep -q '"depth":1' || { + echo "paths latest expected direct depth 1: ${paths_latest}" >&2 + exit 1 +} + +paths_asof="$( + http_get "/v2/graph/paths?from=gq-a&to=gq-c&predicate=ms.within_domain&max_depth=4&as_of=${cutoff_cursor}&expand_names=true" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${paths_asof}" | grep -q '"depth":2' || { + echo "paths as_of expected historical depth 2: ${paths_asof}" >&2 + exit 1 +} + +subgraph_page1="$( + http_get "/v2/graph/subgraph?roots[]=gq-a&max_depth=2&predicates[]=ms.computed_by&dir=outgoing&limit_edges=1&limit_nodes=10&include_versions=true" \ + --header "X-Amduat-Space: ${space_id}" +)" +subgraph_cursor="$(printf '%s\n' "${subgraph_page1}" | extract_json_string "next_cursor")" +if [[ -z "${subgraph_cursor}" ]]; then + echo "missing subgraph cursor in page1: ${subgraph_page1}" >&2 + exit 1 +fi +subgraph_page2="$( + http_get "/v2/graph/subgraph?roots[]=gq-a&max_depth=2&predicates[]=ms.computed_by&dir=outgoing&limit_edges=10&limit_nodes=10&cursor=${subgraph_cursor}" \ + --header "X-Amduat-Space: ${space_id}" +)" +subgraph_joined="${subgraph_page1} ${subgraph_page2}" +echo "${subgraph_joined}" | grep -q '"name":"gq-a"' || { echo "subgraph missing root node gq-a" >&2; exit 1; } +echo "${subgraph_joined}" | grep -q '"name":"gq-b"' || { echo "subgraph missing gq-b" >&2; exit 1; } +echo "${subgraph_joined}" | grep -q '"name":"gq-d"' || { echo "subgraph missing gq-d" >&2; exit 1; } +echo "${subgraph_joined}" | grep -q '"versions":\[' || { echo "subgraph include_versions missing versions" >&2; exit 1; } + +subgraph_asof="$( + http_get "/v2/graph/subgraph?roots[]=gq-a&max_depth=2&predicates[]=ms.computed_by&dir=outgoing&as_of=${cutoff_cursor}&limit_edges=10&limit_nodes=10" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${subgraph_asof}" | grep -q '"name":"gq-b"' || { echo "subgraph as_of missing gq-b" >&2; exit 1; } +if echo "${subgraph_asof}" | grep -q '"name":"gq-d"'; then + echo "subgraph as_of unexpectedly includes gq-d" >&2 + exit 1 +fi + +gqd_edge_resp="$( + http_get "/v2/graph/edges?subject=gq-a&predicate=ms.computed_by&object=gq-d&dir=outgoing&limit=1" \ + --header "X-Amduat-Space: ${space_id}" +)" +gqd_edge_ref="$(printf '%s\n' "${gqd_edge_resp}" | extract_json_string "edge_ref")" +if [[ -z "${gqd_edge_ref}" ]]; then + echo "failed to parse gq-a->gq-d edge ref: ${gqd_edge_resp}" >&2 + exit 1 +fi +http_post "/v2/graph/edges/tombstone" \ + "{\"edge_ref\":\"${gqd_edge_ref}\"}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" >/dev/null + +subgraph_after_tombstone="$( + http_get "/v2/graph/subgraph?roots[]=gq-a&max_depth=2&predicates[]=ms.computed_by&dir=outgoing&limit_edges=10&limit_nodes=10" \ + --header "X-Amduat-Space: ${space_id}" +)" +if echo "${subgraph_after_tombstone}" | grep -q '"name":"gq-d"'; then + echo "subgraph after tombstone unexpectedly includes gq-d: ${subgraph_after_tombstone}" >&2 + exit 1 +fi + +subgraph_include_tombstoned="$( + http_get "/v2/graph/subgraph?roots[]=gq-a&max_depth=2&predicates[]=ms.computed_by&dir=outgoing&include_tombstoned=true&limit_edges=10&limit_nodes=10" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${subgraph_include_tombstoned}" | grep -q '"name":"gq-d"' || { + echo "subgraph include_tombstoned missing gq-d: ${subgraph_include_tombstoned}" >&2 + exit 1 +} + +edges_include_tombstoned="$( + http_get "/v2/graph/edges?subject=gq-a&predicate=ms.computed_by&dir=outgoing&include_tombstoned=true&limit=10&expand_names=true" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${edges_include_tombstoned}" | grep -q '"object_name":"gq-d"' || { + echo "edges include_tombstoned missing gq-d: ${edges_include_tombstoned}" >&2 + exit 1 +} + +neighbors_after_tombstone="$( + http_get "/v2/graph/nodes/gq-a/neighbors?dir=outgoing&predicate=ms.computed_by&limit=10&expand_names=true" \ + --header "X-Amduat-Space: ${space_id}" +)" +if echo "${neighbors_after_tombstone}" | grep -q '"neighbor_name":"gq-d"'; then + echo "neighbors default should exclude tombstoned gq-d edge: ${neighbors_after_tombstone}" >&2 + exit 1 +fi + +neighbors_include_tombstoned="$( + http_get "/v2/graph/nodes/gq-a/neighbors?dir=outgoing&predicate=ms.computed_by&include_tombstoned=true&limit=10&expand_names=true" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${neighbors_include_tombstoned}" | grep -q '"neighbor_name":"gq-d"' || { + echo "neighbors include_tombstoned missing gq-d: ${neighbors_include_tombstoned}" >&2 + exit 1 +} + +paths_excluding_tombstoned="$( + http_get "/v2/graph/paths?from=gq-a&to=gq-d&predicate=ms.computed_by&max_depth=2" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${paths_excluding_tombstoned}" | grep -q '"paths":\[\]' || { + echo "paths default should exclude tombstoned edge: ${paths_excluding_tombstoned}" >&2 + exit 1 +} + +paths_include_tombstoned="$( + http_get "/v2/graph/paths?from=gq-a&to=gq-d&predicate=ms.computed_by&max_depth=2&include_tombstoned=true" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${paths_include_tombstoned}" | grep -q '"depth":1' || { + echo "paths include_tombstoned expected depth 1 path: ${paths_include_tombstoned}" >&2 + exit 1 +} + +create_edge_with_metadata "gq-b" "ms.computed_by" "gq-d" "gq-prov1" +create_edge_with_metadata "gq-b" "ms.computed_by" "gq-a" "gq-prov2" + +neighbors_provenance="$( + http_get "/v2/graph/nodes/gq-b/neighbors?dir=outgoing&predicate=ms.computed_by&provenance_ref=gq-prov1&limit=10&expand_names=true" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${neighbors_provenance}" | grep -q '"neighbor_name":"gq-d"' || { + echo "neighbors provenance filter missing gq-d: ${neighbors_provenance}" >&2 + exit 1 +} + +neighbors_provenance_missing="$( + http_get "/v2/graph/nodes/gq-b/neighbors?dir=outgoing&predicate=ms.computed_by&provenance_ref=gq-prov-missing&limit=10" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${neighbors_provenance_missing}" | grep -q '"neighbors":\[\]' || { + echo "neighbors unresolved provenance expected empty result: ${neighbors_provenance_missing}" >&2 + exit 1 +} + +paths_provenance_match="$( + http_get "/v2/graph/paths?from=gq-b&to=gq-d&predicate=ms.computed_by&provenance_ref=gq-prov1&max_depth=2&expand_names=true" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${paths_provenance_match}" | grep -q '"depth":1' || { + echo "paths provenance filter expected depth 1 path: ${paths_provenance_match}" >&2 + exit 1 +} +echo "${paths_provenance_match}" | grep -q '"object_name":"gq-d"' || { + echo "paths provenance filter missing gq-d: ${paths_provenance_match}" >&2 + exit 1 +} + +paths_provenance_excluded="$( + http_get "/v2/graph/paths?from=gq-b&to=gq-a&predicate=ms.computed_by&provenance_ref=gq-prov1&max_depth=2" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${paths_provenance_excluded}" | grep -q '"paths":\[\]' || { + echo "paths provenance filter unexpectedly includes gq-b->gq-a path: ${paths_provenance_excluded}" >&2 + exit 1 +} + +batch_resp="$( + http_post "/v2/graph/batch" \ + "{\"edges\":[{\"subject\":\"gq-c\",\"predicate\":\"ms.computed_by\",\"object\":\"gq-d\",\"metadata_ref\":\"gq-prov1\"}]}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${batch_resp}" | grep -q '"ok":true' || { + echo "batch edge with metadata_ref failed: ${batch_resp}" >&2 + exit 1 +} +echo "${batch_resp}" | grep -q '"results":\[' || { + echo "batch response missing results array: ${batch_resp}" >&2 + exit 1 +} +echo "${batch_resp}" | grep -q '"status":"applied"' || { + echo "batch response missing applied item status: ${batch_resp}" >&2 + exit 1 +} + +batch_version_provenance="$( + http_post "/v2/graph/batch" \ + "{\"versions\":[{\"name\":\"gq-e\",\"ref\":\"${ref_e2}\",\"provenance\":{\"source_uri\":\"urn:test:gq-e-v2\",\"extractor\":\"graph-test\",\"confidence\":\"0.91\",\"observed_at\":1730000000000,\"ingested_at\":1730000000100,\"license\":\"test-only\",\"trace_id\":\"trace-gq-e-v2\"}}]}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${batch_version_provenance}" | grep -q '"ok":true' || { + echo "batch version with provenance failed: ${batch_version_provenance}" >&2 + exit 1 +} +echo "${batch_version_provenance}" | grep -q '"status":"applied"' || { + echo "batch version with provenance missing applied status: ${batch_version_provenance}" >&2 + exit 1 +} + +batch_edge_provenance="$( + http_post "/v2/graph/batch" \ + "{\"edges\":[{\"subject\":\"gq-c\",\"predicate\":\"ms.computed_by\",\"object\":\"gq-a\",\"provenance\":{\"source_uri\":\"urn:test:gq-c-a\",\"extractor\":\"graph-test\",\"confidence\":\"0.87\",\"observed_at\":1730000000200,\"ingested_at\":1730000000300,\"trace_id\":\"trace-gq-c-a\"}}]}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${batch_edge_provenance}" | grep -q '"ok":true' || { + echo "batch edge with provenance failed: ${batch_edge_provenance}" >&2 + exit 1 +} +echo "${batch_edge_provenance}" | grep -q '"status":"applied"' || { + echo "batch edge with provenance missing applied status: ${batch_edge_provenance}" >&2 + exit 1 +} + +http_post "/v2/graph/nodes/gq-v/versions" \ + "{\"ref\":\"${ref_v2}\"}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" >/dev/null + +http_post "/v2/graph/nodes/gq-v/versions/tombstone" \ + "{\"ref\":\"${ref_v2}\"}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" >/dev/null + +gqv_after_tombstone="$( + http_get "/v2/graph/nodes/gq-v" \ + --header "X-Amduat-Space: ${space_id}" +)" +gqv_latest="$(printf '%s\n' "${gqv_after_tombstone}" | extract_json_string "latest_ref")" +if [[ "${gqv_latest}" != "${ref_v1}" ]]; then + echo "version tombstone expected latest_ref=${ref_v1}, got ${gqv_latest}: ${gqv_after_tombstone}" >&2 + exit 1 +fi +if echo "${gqv_after_tombstone}" | grep -q "\"ref\":\"${ref_v2}\""; then + echo "version tombstone expected ${ref_v2} hidden by default: ${gqv_after_tombstone}" >&2 + exit 1 +fi + +gqv_include_tombstoned="$( + http_get "/v2/graph/nodes/gq-v?include_tombstoned=true" \ + --header "X-Amduat-Space: ${space_id}" +)" +gqv_latest_all="$(printf '%s\n' "${gqv_include_tombstoned}" | extract_json_string "latest_ref")" +if [[ "${gqv_latest_all}" != "${ref_v2}" ]]; then + echo "include_tombstoned expected latest_ref=${ref_v2}, got ${gqv_latest_all}: ${gqv_include_tombstoned}" >&2 + exit 1 +fi +echo "${gqv_include_tombstoned}" | grep -q "\"ref\":\"${ref_v2}\"" || { + echo "include_tombstoned expected historical version ${ref_v2}: ${gqv_include_tombstoned}" >&2 + exit 1 +} + +history_default="$( + http_get "/v2/graph/history/gq-v" \ + --header "X-Amduat-Space: ${space_id}" +)" +history_default_latest="$(printf '%s\n' "${history_default}" | extract_json_string "latest_ref")" +if [[ "${history_default_latest}" != "${ref_v1}" ]]; then + echo "history default expected latest_ref=${ref_v1}, got ${history_default_latest}: ${history_default}" >&2 + exit 1 +fi +if echo "${history_default}" | grep -q "\"ref\":\"${ref_v2}\""; then + echo "history default expected tombstoned version ${ref_v2} hidden: ${history_default}" >&2 + exit 1 +fi + +history_include_tombstoned="$( + http_get "/v2/graph/history/gq-v?include_tombstoned=true" \ + --header "X-Amduat-Space: ${space_id}" +)" +history_all_latest="$(printf '%s\n' "${history_include_tombstoned}" | extract_json_string "latest_ref")" +if [[ "${history_all_latest}" != "${ref_v2}" ]]; then + echo "history include_tombstoned expected latest_ref=${ref_v2}, got ${history_all_latest}: ${history_include_tombstoned}" >&2 + exit 1 +fi +echo "${history_include_tombstoned}" | grep -q "\"ref\":\"${ref_v2}\"" || { + echo "history include_tombstoned expected tombstoned version ${ref_v2}: ${history_include_tombstoned}" >&2 + exit 1 +} + +batch_continue="$( + http_post "/v2/graph/batch" \ + "{\"mode\":\"continue_on_error\",\"edges\":[{\"subject\":\"gq-a\",\"predicate\":\"ms.computed_by\",\"object\":\"gq-b\"},{\"subject\":\"gq-missing\",\"predicate\":\"ms.computed_by\",\"object\":\"gq-b\"}]}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${batch_continue}" | grep -q '"ok":false' || { + echo "batch continue_on_error expected ok=false: ${batch_continue}" >&2 + exit 1 +} +echo "${batch_continue}" | grep -q '"mode":"continue_on_error"' || { + echo "batch continue_on_error mode echo missing: ${batch_continue}" >&2 + exit 1 +} +echo "${batch_continue}" | grep -q '"status":"applied"' || { + echo "batch continue_on_error missing applied result: ${batch_continue}" >&2 + exit 1 +} +echo "${batch_continue}" | grep -q '"status":"error"' || { + echo "batch continue_on_error missing error result: ${batch_continue}" >&2 + exit 1 +} + +idem_payload='{"idempotency_key":"gq-idem-1","mode":"continue_on_error","edges":[{"subject":"gq-b","predicate":"ms.computed_by","object":"gq-c"},{"subject":"gq-nope","predicate":"ms.computed_by","object":"gq-c"}]}' +idem_first="$( + http_post "/v2/graph/batch" \ + "${idem_payload}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" +)" +idem_second="$( + http_post "/v2/graph/batch" \ + "${idem_payload}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" +)" +if [[ "${idem_first}" != "${idem_second}" ]]; then + echo "idempotency replay response mismatch" >&2 + echo "first=${idem_first}" >&2 + echo "second=${idem_second}" >&2 + exit 1 +fi +restart_daemon +idem_third="$( + http_post "/v2/graph/batch" \ + "${idem_payload}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" +)" +if [[ "${idem_first}" != "${idem_third}" ]]; then + echo "idempotency replay after restart mismatch" >&2 + echo "first=${idem_first}" >&2 + echo "third=${idem_third}" >&2 + exit 1 +fi + +query_include_versions="$( + http_post "/v2/graph/query" \ + "{\"where\":{\"subject\":\"gq-a\"},\"predicates\":[\"ms.within_domain\"],\"direction\":\"outgoing\",\"include_versions\":true,\"limit\":10}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${query_include_versions}" | grep -q '"versions":\[' || { + echo "graph query include_versions missing versions: ${query_include_versions}" >&2 + exit 1 +} + +query_with_stats="$( + http_post "/v2/graph/query" \ + "{\"where\":{\"subject\":\"gq-a\"},\"predicates\":[\"ms.within_domain\"],\"direction\":\"outgoing\",\"include_stats\":true,\"max_result_bytes\":1048576,\"limit\":10}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${query_with_stats}" | grep -q '"stats":{' || { + echo "graph query include_stats missing stats block: ${query_with_stats}" >&2 + exit 1 +} +echo "${query_with_stats}" | grep -q '"plan":"' || { + echo "graph query include_stats missing plan: ${query_with_stats}" >&2 + exit 1 +} + +query_provenance="$( + http_post "/v2/graph/query" \ + "{\"where\":{\"subject\":\"gq-b\",\"provenance_ref\":\"gq-prov1\"},\"predicates\":[\"ms.computed_by\"],\"direction\":\"outgoing\",\"limit\":10}" \ + --header "Content-Type: application/json" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${query_provenance}" | grep -q '"name":"gq-d"' || { + echo "graph query provenance filter missing expected node gq-d: ${query_provenance}" >&2 + exit 1 +} +if echo "${query_provenance}" | grep -q '"name":"gq-a"'; then + echo "graph query provenance filter unexpectedly includes gq-a: ${query_provenance}" >&2 + exit 1 +fi + +query_provenance_count="$(printf '%s\n' "${query_provenance}" | grep -o '"edge_ref":"' | wc -l | awk '{print $1}')" +if [[ "${query_provenance_count}" != "1" ]]; then + echo "graph query provenance expected exactly one edge, got ${query_provenance_count}: ${query_provenance}" >&2 + exit 1 +fi + +edges_provenance="$( + http_get "/v2/graph/edges?subject=gq-b&predicate=ms.computed_by&dir=outgoing&provenance_ref=gq-prov1&limit=10&expand_names=true" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${edges_provenance}" | grep -q '"object_name":"gq-d"' || { + echo "graph edges provenance filter missing gq-d: ${edges_provenance}" >&2 + exit 1 +} +if echo "${edges_provenance}" | grep -q '"object_name":"gq-a"'; then + echo "graph edges provenance filter unexpectedly includes gq-a: ${edges_provenance}" >&2 + exit 1 +fi + +edges_with_stats="$( + http_get "/v2/graph/edges?subject=gq-b&predicate=ms.computed_by&dir=outgoing&include_stats=true&max_result_bytes=1048576&limit=10" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${edges_with_stats}" | grep -q '"stats":{' || { + echo "graph edges include_stats missing stats block: ${edges_with_stats}" >&2 + exit 1 +} +echo "${edges_with_stats}" | grep -q '"plan":"' || { + echo "graph edges include_stats missing plan: ${edges_with_stats}" >&2 + exit 1 +} + +subgraph_provenance="$( + http_get "/v2/graph/subgraph?roots[]=gq-b&max_depth=1&predicates[]=ms.computed_by&dir=outgoing&provenance_ref=gq-prov1&limit_edges=10&limit_nodes=10" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${subgraph_provenance}" | grep -q '"name":"gq-d"' || { + echo "subgraph provenance filter missing gq-d: ${subgraph_provenance}" >&2 + exit 1 +} +if echo "${subgraph_provenance}" | grep -q '"name":"gq-a"'; then + echo "subgraph provenance filter unexpectedly includes gq-a: ${subgraph_provenance}" >&2 + exit 1 +fi + +subgraph_with_stats="$( + http_get "/v2/graph/subgraph?roots[]=gq-a&max_depth=2&max_fanout=4096&include_stats=true&max_result_bytes=1048576&limit_edges=10&limit_nodes=10" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${subgraph_with_stats}" | grep -q '"stats":{' || { + echo "subgraph include_stats missing stats block: ${subgraph_with_stats}" >&2 + exit 1 +} +echo "${subgraph_with_stats}" | grep -q '"plan":"' || { + echo "subgraph include_stats missing plan: ${subgraph_with_stats}" >&2 + exit 1 +} + +paths_with_stats="$( + http_get "/v2/graph/paths?from=gq-a&to=gq-c&predicate=ms.within_domain&max_depth=4&include_stats=true&max_fanout=4096&max_result_bytes=1048576" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${paths_with_stats}" | grep -q '"stats":{' || { + echo "paths include_stats missing stats block: ${paths_with_stats}" >&2 + exit 1 +} +echo "${paths_with_stats}" | grep -q '"plan":"' || { + echo "paths include_stats missing plan: ${paths_with_stats}" >&2 + exit 1 +} + +gqb_node="$( + http_get "/v2/graph/nodes/gq-b" \ + --header "X-Amduat-Space: ${space_id}" +)" +gqb_ref="$(printf '%s\n' "${gqb_node}" | extract_json_string "concept_ref")" +if [[ -z "${gqb_ref}" ]]; then + echo "failed to resolve gq-b concept ref: ${gqb_node}" >&2 + exit 1 +fi + +changes_tombstone="$( + http_get "/v2/graph/changes?event_types[]=tombstone_applied&limit=100" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${changes_tombstone}" | grep -q '"event":"tombstone_applied"' || { + echo "changes tombstone filter missing tombstone event: ${changes_tombstone}" >&2 + exit 1 +} + +changes_filtered="$( + http_get "/v2/graph/changes?since_as_of=${cutoff_cursor}&predicates[]=ms.computed_by&roots[]=gq-b&limit=100" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${changes_filtered}" | grep -q "\"${gqb_ref}\"" || { + echo "changes root/predicate filter missing gq-b involvement: ${changes_filtered}" >&2 + exit 1 +} +if echo "${changes_filtered}" | grep -q '"event":"version_published"'; then + echo "changes predicate filter unexpectedly includes version events: ${changes_filtered}" >&2 + exit 1 +fi + +changes_wait_empty="$( + http_get "/v2/graph/changes?since_cursor=g1_999999&wait_ms=1&limit=1" \ + --header "X-Amduat-Space: ${space_id}" +)" +echo "${changes_wait_empty}" | grep -q '"events":\[\]' || { + echo "changes wait_ms empty poll expected no events: ${changes_wait_empty}" >&2 + exit 1 +} + +echo "ok: v2 graph query tests passed" diff --git a/src/amduatd.c b/src/amduatd.c index 7c722bc..03b76cd 100644 --- a/src/amduatd.c +++ b/src/amduatd.c @@ -82,6 +82,52 @@ typedef struct amduatd_strbuf { size_t cap; } amduatd_strbuf_t; +typedef enum { + AMDUATD_V2_JOB_KIND_NONE = 0, + AMDUATD_V2_JOB_KIND_PUT = 1, + AMDUATD_V2_JOB_KIND_CONCAT = 2, + AMDUATD_V2_JOB_KIND_SLICE = 3 +} amduatd_v2_job_kind_t; + +typedef enum { + AMDUATD_V2_JOB_PENDING = 0, + AMDUATD_V2_JOB_RUNNING = 1, + AMDUATD_V2_JOB_SUCCEEDED = 2, + AMDUATD_V2_JOB_FAILED = 3 +} amduatd_v2_job_status_t; + +typedef struct { + uint64_t id; + amduatd_v2_job_kind_t kind; + amduatd_v2_job_status_t status; + uint64_t created_at_ms; + uint64_t started_at_ms; + uint64_t completed_at_ms; + char error[128]; + + uint8_t *put_body; + size_t put_body_len; + bool put_has_type_tag; + amduat_type_tag_t put_type_tag; + + amduat_reference_t concat_left_ref; + amduat_reference_t concat_right_ref; + + amduat_reference_t slice_ref; + uint64_t slice_offset; + uint64_t slice_length; + + amduat_reference_t result_ref; + bool result_ref_set; +} amduatd_v2_job_t; + +typedef struct { + amduatd_v2_job_t *items; + size_t len; + size_t cap; + uint64_t next_id; +} amduatd_v2_job_queue_t; + amduatd_ref_status_t amduatd_decode_ref_or_name_latest( amduat_asl_store_t *store, const amduat_asl_store_fs_config_t *cfg, @@ -1348,6 +1394,142 @@ static bool amduatd_strbuf_append_char(amduatd_strbuf_t *b, char c) { return amduatd_strbuf_append(b, &c, 1u); } +static void amduatd_v2_job_free(amduatd_v2_job_t *job) { + if (job == NULL) { + return; + } + free(job->put_body); + job->put_body = NULL; + job->put_body_len = 0u; + amduat_reference_free(&job->concat_left_ref); + amduat_reference_free(&job->concat_right_ref); + amduat_reference_free(&job->slice_ref); + if (job->result_ref_set) { + amduat_reference_free(&job->result_ref); + job->result_ref_set = false; + } +} + +static void amduatd_v2_jobs_free(amduatd_v2_job_queue_t *q) { + size_t i; + if (q == NULL) { + return; + } + for (i = 0u; i < q->len; ++i) { + amduatd_v2_job_free(&q->items[i]); + } + free(q->items); + q->items = NULL; + q->len = 0u; + q->cap = 0u; +} + +static const char *amduatd_v2_job_kind_name(amduatd_v2_job_kind_t kind) { + switch (kind) { + case AMDUATD_V2_JOB_KIND_PUT: + return "put"; + case AMDUATD_V2_JOB_KIND_CONCAT: + return "concat"; + case AMDUATD_V2_JOB_KIND_SLICE: + return "slice"; + default: + return "unknown"; + } +} + +static const char *amduatd_v2_job_status_name(amduatd_v2_job_status_t status) { + switch (status) { + case AMDUATD_V2_JOB_PENDING: + return "pending"; + case AMDUATD_V2_JOB_RUNNING: + return "running"; + case AMDUATD_V2_JOB_SUCCEEDED: + return "succeeded"; + case AMDUATD_V2_JOB_FAILED: + return "failed"; + default: + return "unknown"; + } +} + +static bool amduatd_v2_jobs_has_pending(const amduatd_v2_job_queue_t *q) { + size_t i; + if (q == NULL) { + return false; + } + for (i = 0u; i < q->len; ++i) { + if (q->items[i].status == AMDUATD_V2_JOB_PENDING) { + return true; + } + } + return false; +} + +static amduatd_v2_job_t *amduatd_v2_jobs_find(amduatd_v2_job_queue_t *q, + uint64_t id) { + size_t i; + if (q == NULL) { + return NULL; + } + for (i = 0u; i < q->len; ++i) { + if (q->items[i].id == id) { + return &q->items[i]; + } + } + return NULL; +} + +static bool amduatd_v2_jobs_append(amduatd_v2_job_queue_t *q, + const amduatd_v2_job_t *job, + uint64_t *out_id) { + amduatd_v2_job_t copy; + amduatd_v2_job_t *next = NULL; + if (q == NULL || job == NULL || out_id == NULL) { + return false; + } + memset(©, 0, sizeof(copy)); + copy = *job; + copy.id = q->next_id++; + if (copy.id == 0u) { + copy.id = q->next_id++; + } + copy.status = AMDUATD_V2_JOB_PENDING; + copy.created_at_ms = amduatd_now_ms(); + if (job->put_body_len != 0u && job->put_body != NULL) { + copy.put_body = (uint8_t *)malloc(job->put_body_len); + if (copy.put_body == NULL) { + return false; + } + memcpy(copy.put_body, job->put_body, job->put_body_len); + copy.put_body_len = job->put_body_len; + } + if (job->kind == AMDUATD_V2_JOB_KIND_CONCAT) { + if (!amduat_reference_clone(job->concat_left_ref, ©.concat_left_ref) || + !amduat_reference_clone(job->concat_right_ref, ©.concat_right_ref)) { + amduatd_v2_job_free(©); + return false; + } + } else if (job->kind == AMDUATD_V2_JOB_KIND_SLICE) { + if (!amduat_reference_clone(job->slice_ref, ©.slice_ref)) { + amduatd_v2_job_free(©); + return false; + } + } + if (q->len == q->cap) { + size_t next_cap = q->cap != 0u ? q->cap * 2u : 16u; + next = (amduatd_v2_job_t *)realloc(q->items, next_cap * sizeof(*q->items)); + if (next == NULL) { + amduatd_v2_job_free(©); + return false; + } + q->items = next; + q->cap = next_cap; + } + q->items[q->len++] = copy; + *out_id = copy.id; + return true; +} + static bool amduatd_json_parse_u32(const char **p, const char *end, uint32_t *out) { @@ -6880,6 +7062,957 @@ pel_run_cleanup: return ok; } +typedef struct { + uint8_t *bytes; + size_t len; + bool has_type_tag; + amduat_type_tag_t type_tag; +} amduatd_v2_inline_artifact_t; + +static void amduatd_v2_inline_artifacts_free( + amduatd_v2_inline_artifact_t *items, + size_t len) { + size_t i; + if (items == NULL) { + return; + } + for (i = 0; i < len; ++i) { + free(items[i].bytes); + items[i].bytes = NULL; + items[i].len = 0u; + } + free(items); +} + +static bool amduatd_handle_post_v2_pel_execute( + int fd, + amduat_asl_store_t *store, + const amduat_asl_store_fs_config_t *cfg, + const amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const char *root_path, + const amduatd_http_req_t *req) { + uint8_t *body = NULL; + const char *p = NULL; + const char *end = NULL; + bool have_program_ref = false; + bool have_input_refs = false; + bool have_inline_artifacts = false; + bool have_inputs = false; + bool have_scheme_ref = false; + bool scheme_is_dag = false; + bool has_params_ref = false; + bool have_receipt = false; + bool receipt_have_input_manifest = false; + bool receipt_have_environment = false; + bool receipt_have_evaluator = false; + bool receipt_have_executor = false; + bool receipt_have_started = false; + bool receipt_have_completed = false; + amduat_reference_t scheme_ref; + amduat_reference_t program_ref; + amduat_reference_t params_ref; + amduat_reference_t *input_refs = NULL; + size_t input_refs_len = 0; + size_t input_refs_cap = 0; + amduatd_v2_inline_artifact_t *inline_artifacts = NULL; + size_t inline_artifacts_len = 0; + size_t inline_artifacts_cap = 0; + amduat_pel_run_result_t run_result; + amduat_artifact_t receipt_artifact; + amduat_reference_t receipt_ref; + char *receipt_evaluator_id = NULL; + uint64_t receipt_started_at = 0; + uint64_t receipt_completed_at = 0; + amduat_reference_t receipt_input_manifest_ref; + amduat_reference_t receipt_environment_ref; + amduat_reference_t receipt_executor_ref; + int status_code = 200; + const char *status_reason = "OK"; + bool ok = false; + size_t stored_input_start = 0u; + + memset(&scheme_ref, 0, sizeof(scheme_ref)); + memset(&program_ref, 0, sizeof(program_ref)); + memset(¶ms_ref, 0, sizeof(params_ref)); + memset(&run_result, 0, sizeof(run_result)); + memset(&receipt_artifact, 0, sizeof(receipt_artifact)); + memset(&receipt_ref, 0, sizeof(receipt_ref)); + memset(&receipt_input_manifest_ref, 0, sizeof(receipt_input_manifest_ref)); + memset(&receipt_environment_ref, 0, sizeof(receipt_environment_ref)); + memset(&receipt_executor_ref, 0, sizeof(receipt_executor_ref)); + + if (store == NULL || req == NULL || cfg == NULL || concepts == NULL || + dcfg == NULL || root_path == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", + "internal error"); + } + + if (req->content_length > (1u * 1024u * 1024u)) { + return amduatd_send_json_error(fd, 413, "Payload Too Large", + "payload too large"); + } + if (req->content_length == 0u) { + return amduatd_send_json_error(fd, 400, "Bad Request", "missing body"); + } + body = (uint8_t *)malloc(req->content_length); + if (body == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + if (!amduatd_read_exact(fd, body, req->content_length)) { + free(body); + return false; + } + + p = (const char *)body; + end = (const char *)body + req->content_length; + if (!amduatd_json_expect(&p, end, '{')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto v2_pel_cleanup; + } + + for (;;) { + const char *key = NULL; + size_t key_len = 0; + const char *sv = NULL; + size_t sv_len = 0; + const char *cur = NULL; + + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &key, &key_len) || + !amduatd_json_expect(&p, end, ':')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto v2_pel_cleanup; + } + + if (key_len == strlen("program_ref") && + memcmp(key, "program_ref", key_len) == 0) { + amduatd_ref_status_t st; + if (have_program_ref || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid program_ref"); + goto v2_pel_cleanup; + } + st = amduatd_decode_ref_or_name_latest(store, cfg, concepts, dcfg, sv, + sv_len, &program_ref); + if (st == AMDUATD_REF_ERR_NOT_FOUND) { + ok = amduatd_send_json_error(fd, 404, "Not Found", + "program_ref not found"); + goto v2_pel_cleanup; + } + if (st != AMDUATD_REF_OK) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid program_ref"); + goto v2_pel_cleanup; + } + have_program_ref = true; + } else if (key_len == strlen("scheme_ref") && + memcmp(key, "scheme_ref", key_len) == 0) { + if (have_scheme_ref || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid scheme_ref"); + goto v2_pel_cleanup; + } + if (sv_len == 3u && memcmp(sv, "dag", 3u) == 0) { + scheme_ref = amduat_pel_program_dag_scheme_ref(); + scheme_is_dag = true; + } else if (!amduatd_decode_ref_hex_str(sv, sv_len, &scheme_ref)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid scheme_ref"); + goto v2_pel_cleanup; + } + have_scheme_ref = true; + } else if (key_len == strlen("params_ref") && + memcmp(key, "params_ref", key_len) == 0) { + amduatd_ref_status_t st; + if (has_params_ref || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid params_ref"); + goto v2_pel_cleanup; + } + st = amduatd_decode_ref_or_name_latest(store, cfg, concepts, dcfg, sv, + sv_len, ¶ms_ref); + if (st == AMDUATD_REF_ERR_NOT_FOUND) { + ok = amduatd_send_json_error(fd, 404, "Not Found", + "params_ref not found"); + goto v2_pel_cleanup; + } + if (st != AMDUATD_REF_OK) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid params_ref"); + goto v2_pel_cleanup; + } + has_params_ref = true; + } else if (key_len == strlen("inputs") && + memcmp(key, "inputs", key_len) == 0) { + if (have_inputs || !amduatd_json_expect(&p, end, '{')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid inputs"); + goto v2_pel_cleanup; + } + have_inputs = true; + for (;;) { + const char *ikey = NULL; + size_t ikey_len = 0; + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &ikey, &ikey_len) || + !amduatd_json_expect(&p, end, ':')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid inputs"); + goto v2_pel_cleanup; + } + if (ikey_len == strlen("refs") && memcmp(ikey, "refs", ikey_len) == 0) { + if (have_input_refs || !amduatd_json_expect(&p, end, '[')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid refs"); + goto v2_pel_cleanup; + } + have_input_refs = true; + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ']') { + p = cur + 1; + } else { + for (;;) { + amduat_reference_t ref; + amduatd_ref_status_t st; + memset(&ref, 0, sizeof(ref)); + if (!amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid input_ref"); + goto v2_pel_cleanup; + } + st = amduatd_decode_ref_or_name_latest(store, cfg, concepts, dcfg, + sv, sv_len, &ref); + if (st == AMDUATD_REF_ERR_NOT_FOUND) { + ok = amduatd_send_json_error(fd, 404, "Not Found", + "input_ref not found"); + goto v2_pel_cleanup; + } + if (st != AMDUATD_REF_OK) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid input_ref"); + goto v2_pel_cleanup; + } + if (input_refs_len == input_refs_cap) { + size_t next_cap = input_refs_cap != 0u ? input_refs_cap * 2u : 4u; + amduat_reference_t *next = + (amduat_reference_t *)realloc(input_refs, + next_cap * sizeof(*input_refs)); + if (next == NULL) { + amduat_reference_free(&ref); + ok = amduatd_send_json_error(fd, 500, + "Internal Server Error", "oom"); + goto v2_pel_cleanup; + } + input_refs = next; + input_refs_cap = next_cap; + } + input_refs[input_refs_len++] = ref; + + cur = amduatd_json_skip_ws(p, end); + if (cur >= end) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid refs"); + goto v2_pel_cleanup; + } + if (*cur == ',') { + p = cur + 1; + continue; + } + if (*cur == ']') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid refs"); + goto v2_pel_cleanup; + } + } + } else if (ikey_len == strlen("inline_artifacts") && + memcmp(ikey, "inline_artifacts", ikey_len) == 0) { + if (have_inline_artifacts || !amduatd_json_expect(&p, end, '[')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid inline_artifacts"); + goto v2_pel_cleanup; + } + have_inline_artifacts = true; + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ']') { + p = cur + 1; + } else { + for (;;) { + amduatd_v2_inline_artifact_t art; + bool have_body_hex = false; + bool have_type_tag = false; + char *type_tag_text = NULL; + memset(&art, 0, sizeof(art)); + if (!amduatd_json_expect(&p, end, '{')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid inline_artifact"); + free(type_tag_text); + goto v2_pel_cleanup; + } + for (;;) { + const char *akey = NULL; + size_t akey_len = 0; + const char *cur2 = amduatd_json_skip_ws(p, end); + if (cur2 < end && *cur2 == '}') { + p = cur2 + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &akey, &akey_len) || + !amduatd_json_expect(&p, end, ':')) { + free(type_tag_text); + free(art.bytes); + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid inline_artifact"); + goto v2_pel_cleanup; + } + if (akey_len == strlen("body_hex") && + memcmp(akey, "body_hex", akey_len) == 0) { + char *tmp = NULL; + if (have_body_hex || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &tmp) || + !amduat_hex_decode_alloc(tmp, &art.bytes, &art.len)) { + free(type_tag_text); + free(art.bytes); + free(tmp); + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid body_hex"); + goto v2_pel_cleanup; + } + free(tmp); + have_body_hex = true; + } else if (akey_len == strlen("type_tag") && + memcmp(akey, "type_tag", akey_len) == 0) { + if (have_type_tag || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &type_tag_text)) { + free(type_tag_text); + free(art.bytes); + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid type_tag"); + goto v2_pel_cleanup; + } + if (!amduatd_parse_type_tag_hex(type_tag_text, + &art.has_type_tag, + &art.type_tag)) { + free(type_tag_text); + free(art.bytes); + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid type_tag"); + goto v2_pel_cleanup; + } + have_type_tag = true; + } else { + if (!amduatd_json_skip_value(&p, end, 0)) { + free(type_tag_text); + free(art.bytes); + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid inline_artifact"); + goto v2_pel_cleanup; + } + } + cur2 = amduatd_json_skip_ws(p, end); + if (cur2 >= end) { + free(type_tag_text); + free(art.bytes); + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid inline_artifact"); + goto v2_pel_cleanup; + } + if (*cur2 == ',') { + p = cur2 + 1; + continue; + } + if (*cur2 == '}') { + p = cur2 + 1; + break; + } + free(type_tag_text); + free(art.bytes); + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid inline_artifact"); + goto v2_pel_cleanup; + } + free(type_tag_text); + if (!have_body_hex) { + free(art.bytes); + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "missing body_hex"); + goto v2_pel_cleanup; + } + if (inline_artifacts_len == inline_artifacts_cap) { + size_t next_cap = inline_artifacts_cap != 0u + ? inline_artifacts_cap * 2u + : 4u; + amduatd_v2_inline_artifact_t *next = + (amduatd_v2_inline_artifact_t *)realloc( + inline_artifacts, next_cap * sizeof(*inline_artifacts)); + if (next == NULL) { + free(art.bytes); + ok = amduatd_send_json_error(fd, 500, + "Internal Server Error", "oom"); + goto v2_pel_cleanup; + } + inline_artifacts = next; + inline_artifacts_cap = next_cap; + } + inline_artifacts[inline_artifacts_len++] = art; + + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == ']') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid inline_artifacts"); + goto v2_pel_cleanup; + } + } + } else { + if (!amduatd_json_skip_value(&p, end, 0)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid inputs"); + goto v2_pel_cleanup; + } + } + cur = amduatd_json_skip_ws(p, end); + if (cur >= end) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid inputs"); + goto v2_pel_cleanup; + } + if (*cur == ',') { + p = cur + 1; + continue; + } + if (*cur == '}') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid inputs"); + goto v2_pel_cleanup; + } + } else if (key_len == strlen("receipt") && + memcmp(key, "receipt", key_len) == 0) { + const char *rkey = NULL; + size_t rkey_len = 0; + if (have_receipt || !amduatd_json_expect(&p, end, '{')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid receipt"); + goto v2_pel_cleanup; + } + have_receipt = true; + for (;;) { + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &rkey, &rkey_len) || + !amduatd_json_expect(&p, end, ':')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid receipt"); + goto v2_pel_cleanup; + } + if (rkey_len == strlen("input_manifest_ref") && + memcmp(rkey, "input_manifest_ref", rkey_len) == 0) { + amduatd_ref_status_t st; + if (receipt_have_input_manifest || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid input_manifest_ref"); + goto v2_pel_cleanup; + } + st = amduatd_decode_ref_or_name_latest(store, cfg, concepts, dcfg, + sv, sv_len, + &receipt_input_manifest_ref); + if (st == AMDUATD_REF_ERR_NOT_FOUND) { + ok = amduatd_send_json_error(fd, 404, "Not Found", + "input_manifest_ref not found"); + goto v2_pel_cleanup; + } + if (st != AMDUATD_REF_OK) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid input_manifest_ref"); + goto v2_pel_cleanup; + } + receipt_have_input_manifest = true; + } else if (rkey_len == strlen("environment_ref") && + memcmp(rkey, "environment_ref", rkey_len) == 0) { + amduatd_ref_status_t st; + if (receipt_have_environment || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid environment_ref"); + goto v2_pel_cleanup; + } + st = amduatd_decode_ref_or_name_latest(store, cfg, concepts, dcfg, + sv, sv_len, + &receipt_environment_ref); + if (st == AMDUATD_REF_ERR_NOT_FOUND) { + ok = amduatd_send_json_error(fd, 404, "Not Found", + "environment_ref not found"); + goto v2_pel_cleanup; + } + if (st != AMDUATD_REF_OK) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid environment_ref"); + goto v2_pel_cleanup; + } + receipt_have_environment = true; + } else if (rkey_len == strlen("evaluator_id") && + memcmp(rkey, "evaluator_id", rkey_len) == 0) { + if (receipt_have_evaluator || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &receipt_evaluator_id)) { + free(receipt_evaluator_id); + receipt_evaluator_id = NULL; + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid evaluator_id"); + goto v2_pel_cleanup; + } + receipt_have_evaluator = true; + } else if (rkey_len == strlen("executor_ref") && + memcmp(rkey, "executor_ref", rkey_len) == 0) { + amduatd_ref_status_t st; + if (receipt_have_executor || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid executor_ref"); + goto v2_pel_cleanup; + } + st = amduatd_decode_ref_or_name_latest(store, cfg, concepts, dcfg, + sv, sv_len, &receipt_executor_ref); + if (st == AMDUATD_REF_ERR_NOT_FOUND) { + ok = amduatd_send_json_error(fd, 404, "Not Found", + "executor_ref not found"); + goto v2_pel_cleanup; + } + if (st != AMDUATD_REF_OK) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid executor_ref"); + goto v2_pel_cleanup; + } + receipt_have_executor = true; + } else if (rkey_len == strlen("started_at") && + memcmp(rkey, "started_at", rkey_len) == 0) { + if (receipt_have_started || + !amduatd_json_parse_u64(&p, end, &receipt_started_at)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid started_at"); + goto v2_pel_cleanup; + } + receipt_have_started = true; + } else if (rkey_len == strlen("completed_at") && + memcmp(rkey, "completed_at", rkey_len) == 0) { + if (receipt_have_completed || + !amduatd_json_parse_u64(&p, end, &receipt_completed_at)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid completed_at"); + goto v2_pel_cleanup; + } + receipt_have_completed = true; + } else { + if (!amduatd_json_skip_value(&p, end, 0)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid receipt"); + goto v2_pel_cleanup; + } + } + cur = amduatd_json_skip_ws(p, end); + if (cur >= end) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid receipt"); + goto v2_pel_cleanup; + } + if (*cur == ',') { + p = cur + 1; + continue; + } + if (*cur == '}') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid receipt"); + goto v2_pel_cleanup; + } + } else { + if (!amduatd_json_skip_value(&p, end, 0)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto v2_pel_cleanup; + } + } + + cur = amduatd_json_skip_ws(p, end); + if (cur >= end) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto v2_pel_cleanup; + } + if (*cur == ',') { + p = cur + 1; + continue; + } + if (*cur == '}') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto v2_pel_cleanup; + } + + p = amduatd_json_skip_ws(p, end); + if (p != end) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto v2_pel_cleanup; + } + + if (!have_program_ref) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "missing program_ref"); + goto v2_pel_cleanup; + } + if (!have_inputs) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "missing inputs"); + goto v2_pel_cleanup; + } + if (!have_input_refs) { + input_refs = NULL; + input_refs_len = 0u; + input_refs_cap = 0u; + } + if (!have_scheme_ref) { + scheme_ref = amduat_pel_program_dag_scheme_ref(); + scheme_is_dag = true; + } + if (!have_receipt || + !receipt_have_input_manifest || + !receipt_have_environment || + !receipt_have_evaluator || + !receipt_have_executor || + !receipt_have_started || + !receipt_have_completed) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "missing receipt fields"); + goto v2_pel_cleanup; + } + + stored_input_start = input_refs_len; + { + size_t i; + for (i = 0; i < inline_artifacts_len; ++i) { + amduat_artifact_t artifact; + amduat_reference_t ref; + amduat_asl_store_error_t err; + memset(&artifact, 0, sizeof(artifact)); + memset(&ref, 0, sizeof(ref)); + if (!amduat_asl_artifact_from_bytes( + amduat_octets(inline_artifacts[i].bytes, inline_artifacts[i].len), + AMDUAT_ASL_IO_RAW, + inline_artifacts[i].has_type_tag, + inline_artifacts[i].type_tag, + &artifact)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "invalid inline_artifact"); + goto v2_pel_cleanup; + } + if (store->ops.put_indexed != NULL) { + amduat_asl_index_state_t state; + err = amduat_asl_store_put_indexed(store, artifact, &ref, &state); + } else { + err = amduat_asl_store_put(store, artifact, &ref); + } + amduat_asl_artifact_free(&artifact); + if (err != AMDUAT_ASL_STORE_OK) { + amduat_reference_free(&ref); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", + "inline artifact store failed"); + goto v2_pel_cleanup; + } + if (input_refs_len == input_refs_cap) { + size_t next_cap = input_refs_cap != 0u ? input_refs_cap * 2u : 4u; + amduat_reference_t *next = + (amduat_reference_t *)realloc(input_refs, + next_cap * sizeof(*input_refs)); + if (next == NULL) { + amduat_reference_free(&ref); + ok = amduatd_send_json_error(fd, 500, + "Internal Server Error", "oom"); + goto v2_pel_cleanup; + } + input_refs = next; + input_refs_cap = next_cap; + } + input_refs[input_refs_len++] = ref; + } + } + + if (!amduat_pel_surf_run_with_result(store, + scheme_ref, + program_ref, + input_refs, + input_refs_len, + has_params_ref, + params_ref, + &run_result)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", + "pel run failed"); + goto v2_pel_cleanup; + } + + if (run_result.has_result_value && run_result.result_value.has_store_failure) { + if (run_result.result_value.store_failure.error_code == + AMDUAT_PEL_STORE_ERROR_NOT_FOUND) { + status_code = 404; + status_reason = "Not Found"; + } else { + status_code = 500; + status_reason = "Internal Server Error"; + } + } + if (!run_result.has_result_value || run_result.result_value.output_refs_len != 1u) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "receipt requires single output"); + goto v2_pel_cleanup; + } + { + amduat_octets_t evaluator_id = amduat_octets( + receipt_evaluator_id, + receipt_evaluator_id != NULL ? strlen(receipt_evaluator_id) : 0u); + if (!amduat_fer1_receipt_from_pel_result( + &run_result.result_value, + receipt_input_manifest_ref, + receipt_environment_ref, + evaluator_id, + receipt_executor_ref, + false, + (amduat_reference_t){0}, + amduat_octets(NULL, 0u), + receipt_started_at, + receipt_completed_at, + &receipt_artifact)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", + "receipt build failed"); + goto v2_pel_cleanup; + } + } + if (amduat_asl_store_put(store, receipt_artifact, &receipt_ref) != + AMDUAT_ASL_STORE_OK) { + amduat_asl_artifact_free(&receipt_artifact); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", + "receipt store failed"); + goto v2_pel_cleanup; + } + amduat_asl_artifact_free(&receipt_artifact); + + if (dcfg->derivation_index_enabled) { + amduat_asl_store_error_t idx_err = AMDUAT_ASL_STORE_OK; + if (!amduatd_derivation_index_pel_run(root_path, + true, + program_ref, + input_refs, + input_refs_len, + has_params_ref, + params_ref, + &run_result, + true, + receipt_ref, + &idx_err)) { + if (dcfg->derivation_index_strict) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", + "derivation index failed"); + goto v2_pel_cleanup; + } + amduat_log(AMDUAT_LOG_WARN, + "v2 pel execute derivation index failed: %d", + (int)idx_err); + } + } + + { + amduatd_strbuf_t resp; + char *run_hex = NULL; + char *trace_hex = NULL; + char *receipt_hex = NULL; + const char *status = "UNKNOWN"; + size_t i; + + memset(&resp, 0, sizeof(resp)); + if (!amduat_asl_ref_encode_hex(run_result.result_ref, &run_hex) || + !amduat_asl_ref_encode_hex(receipt_ref, &receipt_hex)) { + free(run_hex); + free(receipt_hex); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", + "encode error"); + goto v2_pel_cleanup; + } + if (run_result.has_result_value) { + status = amduat_format_pel_status_name( + run_result.result_value.core_result.status); + if (run_result.result_value.has_trace_ref && + !amduat_asl_ref_encode_hex(run_result.result_value.trace_ref, + &trace_hex)) { + free(run_hex); + free(receipt_hex); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", + "encode error"); + goto v2_pel_cleanup; + } + } + + if (!amduatd_strbuf_append_cstr(&resp, "{\"run_ref\":\"") || + !amduatd_strbuf_append_cstr(&resp, run_hex) || + !amduatd_strbuf_append_cstr(&resp, "\",\"result_ref\":\"") || + !amduatd_strbuf_append_cstr(&resp, run_hex) || + !amduatd_strbuf_append_cstr(&resp, "\"")) { + free(run_hex); + free(trace_hex); + free(receipt_hex); + amduatd_strbuf_free(&resp); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto v2_pel_cleanup; + } + free(run_hex); + if (trace_hex != NULL) { + if (!amduatd_strbuf_append_cstr(&resp, ",\"trace_ref\":\"") || + !amduatd_strbuf_append_cstr(&resp, trace_hex) || + !amduatd_strbuf_append_cstr(&resp, "\"")) { + free(trace_hex); + free(receipt_hex); + amduatd_strbuf_free(&resp); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto v2_pel_cleanup; + } + free(trace_hex); + } + if (!amduatd_strbuf_append_cstr(&resp, ",\"receipt_ref\":\"") || + !amduatd_strbuf_append_cstr(&resp, receipt_hex) || + !amduatd_strbuf_append_cstr(&resp, "\",\"stored_input_refs\":[")) { + free(receipt_hex); + amduatd_strbuf_free(&resp); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto v2_pel_cleanup; + } + free(receipt_hex); + for (i = stored_input_start; i < input_refs_len; ++i) { + char *hex = NULL; + if (!amduat_asl_ref_encode_hex(input_refs[i], &hex)) { + amduatd_strbuf_free(&resp); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", + "encode error"); + goto v2_pel_cleanup; + } + if (i != stored_input_start) { + if (!amduatd_strbuf_append_cstr(&resp, ",")) { + free(hex); + amduatd_strbuf_free(&resp); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto v2_pel_cleanup; + } + } + if (!amduatd_strbuf_append_cstr(&resp, "\"") || + !amduatd_strbuf_append_cstr(&resp, hex) || + !amduatd_strbuf_append_cstr(&resp, "\"")) { + free(hex); + amduatd_strbuf_free(&resp); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto v2_pel_cleanup; + } + free(hex); + } + if (!amduatd_strbuf_append_cstr(&resp, "],\"output_refs\":[")) { + amduatd_strbuf_free(&resp); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto v2_pel_cleanup; + } + for (i = 0; i < run_result.output_refs_len; ++i) { + char *hex = NULL; + if (!amduat_asl_ref_encode_hex(run_result.output_refs[i], &hex)) { + amduatd_strbuf_free(&resp); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", + "encode error"); + goto v2_pel_cleanup; + } + if (i != 0u && !amduatd_strbuf_append_cstr(&resp, ",")) { + free(hex); + amduatd_strbuf_free(&resp); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto v2_pel_cleanup; + } + if (!amduatd_strbuf_append_cstr(&resp, "\"") || + !amduatd_strbuf_append_cstr(&resp, hex) || + !amduatd_strbuf_append_cstr(&resp, "\"")) { + free(hex); + amduatd_strbuf_free(&resp); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto v2_pel_cleanup; + } + free(hex); + } + if (!amduatd_strbuf_append_cstr(&resp, "],\"status\":\"") || + !amduatd_strbuf_append_cstr(&resp, status) || + !amduatd_strbuf_append_cstr(&resp, "\"}\n")) { + amduatd_strbuf_free(&resp); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto v2_pel_cleanup; + } + ok = amduatd_http_send_json(fd, status_code, status_reason, resp.data, false); + amduatd_strbuf_free(&resp); + } + +v2_pel_cleanup: + free(body); + if (run_result.has_result_value) { + amduat_enc_pel1_result_free(&run_result.result_value); + } + if (run_result.output_refs != NULL) { + amduat_pel_surf_free_refs(run_result.output_refs, run_result.output_refs_len); + } + amduat_pel_surf_free_ref(&run_result.result_ref); + if (input_refs != NULL) { + size_t i; + for (i = 0; i < input_refs_len; ++i) { + amduat_reference_free(&input_refs[i]); + } + free(input_refs); + } + amduatd_v2_inline_artifacts_free(inline_artifacts, inline_artifacts_len); + if (has_params_ref) { + amduat_reference_free(¶ms_ref); + } + if (have_program_ref) { + amduat_reference_free(&program_ref); + } + if (have_scheme_ref && !scheme_is_dag) { + amduat_reference_free(&scheme_ref); + } + if (have_receipt) { + if (receipt_have_input_manifest) { + amduat_reference_free(&receipt_input_manifest_ref); + } + if (receipt_have_environment) { + amduat_reference_free(&receipt_environment_ref); + } + if (receipt_have_executor) { + amduat_reference_free(&receipt_executor_ref); + } + } + if (receipt_ref.digest.data != NULL) { + amduat_reference_free(&receipt_ref); + } + free(receipt_evaluator_id); + return ok; +} + typedef struct { amduat_pel_node_t *nodes; size_t nodes_len; @@ -9670,6 +10803,1016 @@ cf_cleanup: return ok; } +static void amduatd_store_u64_be(uint8_t *out, uint64_t value) { + out[0] = (uint8_t)((value >> 56) & 0xffu); + out[1] = (uint8_t)((value >> 48) & 0xffu); + out[2] = (uint8_t)((value >> 40) & 0xffu); + out[3] = (uint8_t)((value >> 32) & 0xffu); + out[4] = (uint8_t)((value >> 24) & 0xffu); + out[5] = (uint8_t)((value >> 16) & 0xffu); + out[6] = (uint8_t)((value >> 8) & 0xffu); + out[7] = (uint8_t)(value & 0xffu); +} + +static bool amduatd_build_single_node_program_ref(amduat_asl_store_t *store, + const char *op_name, + uint32_t input_count, + amduat_octets_t params, + amduat_reference_t *out_ref) { + amduat_pel_program_t program; + amduat_pel_node_t node; + amduat_pel_root_ref_t root; + amduat_pel_dag_input_t *inputs = NULL; + amduat_octets_t program_bytes = amduat_octets(NULL, 0u); + amduat_artifact_t artifact; + bool ok = false; + uint32_t i; + + if (out_ref != NULL) { + memset(out_ref, 0, sizeof(*out_ref)); + } + if (store == NULL || op_name == NULL || out_ref == NULL) { + return false; + } + + memset(&program, 0, sizeof(program)); + memset(&node, 0, sizeof(node)); + memset(&root, 0, sizeof(root)); + memset(&artifact, 0, sizeof(artifact)); + + inputs = (amduat_pel_dag_input_t *)calloc(input_count, sizeof(*inputs)); + if (input_count != 0u && inputs == NULL) { + goto build_prog_cleanup; + } + for (i = 0u; i < input_count; ++i) { + inputs[i].kind = AMDUAT_PEL_DAG_INPUT_EXTERNAL; + inputs[i].value.external.input_index = i; + } + + node.id = 1u; + node.op.name = amduat_octets(op_name, strlen(op_name)); + node.op.version = 1u; + node.inputs = inputs; + node.inputs_len = input_count; + node.params = params; + + root.node_id = 1u; + root.output_index = 0u; + + program.nodes = &node; + program.nodes_len = 1u; + program.roots = &root; + program.roots_len = 1u; + + if (!amduat_pel_program_dag_validate(&program)) { + goto build_prog_cleanup; + } + if (!amduat_enc_pel_program_dag_encode_v1(&program, &program_bytes)) { + goto build_prog_cleanup; + } + artifact = amduat_artifact_with_type( + program_bytes, amduat_type_tag(AMDUAT_PEL_TYPE_TAG_PROGRAM_DAG_1)); + if (amduat_asl_store_put(store, artifact, out_ref) != AMDUAT_ASL_STORE_OK) { + goto build_prog_cleanup; + } + ok = true; + +build_prog_cleanup: + free(inputs); + free((void *)program_bytes.data); + return ok; +} + +static bool amduatd_v2_job_send_enqueued(int fd, uint64_t id) { + char json[128]; + int n = snprintf(json, sizeof(json), + "{\"job_id\":%llu,\"status\":\"pending\"}\n", + (unsigned long long)id); + if (n <= 0 || (size_t)n >= sizeof(json)) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "error"); + } + return amduatd_http_send_json(fd, 202, "Accepted", json, false); +} + +static bool amduatd_v2_job_send_status(int fd, const amduatd_v2_job_t *job) { + amduatd_strbuf_t b; + char *result_ref_hex = NULL; + char tmp[64]; + bool ok = false; + + if (job == NULL) { + return amduatd_send_json_error(fd, 404, "Not Found", "job not found"); + } + + memset(&b, 0, sizeof(b)); + if (!amduatd_strbuf_append_cstr(&b, "{")) { + goto job_status_cleanup; + } + snprintf(tmp, sizeof(tmp), "%llu", (unsigned long long)job->id); + if (!amduatd_strbuf_append_cstr(&b, "\"job_id\":") || + !amduatd_strbuf_append_cstr(&b, tmp) || + !amduatd_strbuf_append_cstr(&b, ",\"kind\":\"") || + !amduatd_strbuf_append_cstr(&b, amduatd_v2_job_kind_name(job->kind)) || + !amduatd_strbuf_append_cstr(&b, "\",\"status\":\"") || + !amduatd_strbuf_append_cstr(&b, amduatd_v2_job_status_name(job->status)) || + !amduatd_strbuf_append_cstr(&b, "\",")) { + goto job_status_cleanup; + } + snprintf(tmp, sizeof(tmp), "%llu", (unsigned long long)job->created_at_ms); + if (!amduatd_strbuf_append_cstr(&b, "\"created_at_ms\":") || + !amduatd_strbuf_append_cstr(&b, tmp) || + !amduatd_strbuf_append_cstr(&b, ",\"started_at_ms\":")) { + goto job_status_cleanup; + } + if (job->started_at_ms == 0u) { + if (!amduatd_strbuf_append_cstr(&b, "null")) { + goto job_status_cleanup; + } + } else { + snprintf(tmp, sizeof(tmp), "%llu", (unsigned long long)job->started_at_ms); + if (!amduatd_strbuf_append_cstr(&b, tmp)) { + goto job_status_cleanup; + } + } + if (!amduatd_strbuf_append_cstr(&b, ",\"completed_at_ms\":")) { + goto job_status_cleanup; + } + if (job->completed_at_ms == 0u) { + if (!amduatd_strbuf_append_cstr(&b, "null")) { + goto job_status_cleanup; + } + } else { + snprintf(tmp, sizeof(tmp), "%llu", (unsigned long long)job->completed_at_ms); + if (!amduatd_strbuf_append_cstr(&b, tmp)) { + goto job_status_cleanup; + } + } + if (!amduatd_strbuf_append_cstr(&b, ",\"result_ref\":")) { + goto job_status_cleanup; + } + if (job->result_ref_set && + amduat_asl_ref_encode_hex(job->result_ref, &result_ref_hex)) { + if (!amduatd_strbuf_append_cstr(&b, "\"") || + !amduatd_strbuf_append_cstr(&b, result_ref_hex) || + !amduatd_strbuf_append_cstr(&b, "\"")) { + goto job_status_cleanup; + } + } else { + if (!amduatd_strbuf_append_cstr(&b, "null")) { + goto job_status_cleanup; + } + } + if (!amduatd_strbuf_append_cstr(&b, ",\"error\":")) { + goto job_status_cleanup; + } + if (job->error[0] != '\0') { + if (!amduatd_strbuf_append_cstr(&b, "\"") || + !amduatd_strbuf_append_cstr(&b, job->error) || + !amduatd_strbuf_append_cstr(&b, "\"")) { + goto job_status_cleanup; + } + } else { + if (!amduatd_strbuf_append_cstr(&b, "null")) { + goto job_status_cleanup; + } + } + if (!amduatd_strbuf_append_cstr(&b, "}\n")) { + goto job_status_cleanup; + } + ok = amduatd_http_send_json(fd, 200, "OK", b.data, false); + +job_status_cleanup: + free(result_ref_hex); + amduatd_strbuf_free(&b); + if (!ok) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "error"); + } + return true; +} + +static void amduatd_v2_job_fail(amduatd_v2_job_t *job, const char *msg) { + if (job == NULL) { + return; + } + job->status = AMDUATD_V2_JOB_FAILED; + job->completed_at_ms = amduatd_now_ms(); + if (msg != NULL) { + strncpy(job->error, msg, sizeof(job->error) - 1u); + job->error[sizeof(job->error) - 1u] = '\0'; + } +} + +static bool amduatd_v2_run_job(amduatd_v2_job_t *job, amduat_asl_store_t *store) { + amduat_reference_t program_ref; + amduat_reference_t input_refs[2]; + size_t input_refs_len = 0u; + amduat_pel_run_result_t run_result; + amduat_octets_t params = amduat_octets(NULL, 0u); + uint8_t slice_params[16]; + bool have_program_ref = false; + bool ok = false; + + if (job == NULL || store == NULL) { + return false; + } + + memset(&program_ref, 0, sizeof(program_ref)); + memset(&input_refs, 0, sizeof(input_refs)); + memset(&run_result, 0, sizeof(run_result)); + + if (job->kind == AMDUATD_V2_JOB_KIND_PUT) { + amduat_artifact_t artifact; + memset(&artifact, 0, sizeof(artifact)); + if (!amduat_asl_artifact_from_bytes( + amduat_octets(job->put_body, job->put_body_len), + AMDUAT_ASL_IO_RAW, + job->put_has_type_tag, + job->put_type_tag, + &artifact)) { + amduatd_v2_job_fail(job, "invalid put body"); + return false; + } + if (amduat_asl_store_put(store, artifact, &input_refs[0]) != AMDUAT_ASL_STORE_OK) { + amduat_asl_artifact_free(&artifact); + amduatd_v2_job_fail(job, "store put failed"); + return false; + } + amduat_asl_artifact_free(&artifact); + input_refs_len = 1u; + if (!amduatd_build_single_node_program_ref(store, + AMDUAT_PEL_KERNEL_OP_CONCAT_NAME, + 1u, + amduat_octets(NULL, 0u), + &program_ref)) { + amduatd_v2_job_fail(job, "program build failed"); + goto run_job_cleanup; + } + have_program_ref = true; + } else if (job->kind == AMDUATD_V2_JOB_KIND_CONCAT) { + if (!amduat_reference_clone(job->concat_left_ref, &input_refs[0]) || + !amduat_reference_clone(job->concat_right_ref, &input_refs[1])) { + amduatd_v2_job_fail(job, "ref clone failed"); + goto run_job_cleanup; + } + input_refs_len = 2u; + if (!amduatd_build_single_node_program_ref(store, + AMDUAT_PEL_KERNEL_OP_CONCAT_NAME, + 2u, + amduat_octets(NULL, 0u), + &program_ref)) { + amduatd_v2_job_fail(job, "program build failed"); + goto run_job_cleanup; + } + have_program_ref = true; + } else if (job->kind == AMDUATD_V2_JOB_KIND_SLICE) { + if (!amduat_reference_clone(job->slice_ref, &input_refs[0])) { + amduatd_v2_job_fail(job, "ref clone failed"); + goto run_job_cleanup; + } + input_refs_len = 1u; + amduatd_store_u64_be(slice_params, job->slice_offset); + amduatd_store_u64_be(slice_params + 8u, job->slice_length); + params = amduat_octets(slice_params, sizeof(slice_params)); + if (!amduatd_build_single_node_program_ref(store, + AMDUAT_PEL_KERNEL_OP_SLICE_NAME, + 1u, + params, + &program_ref)) { + amduatd_v2_job_fail(job, "program build failed"); + goto run_job_cleanup; + } + have_program_ref = true; + } else { + amduatd_v2_job_fail(job, "unsupported job kind"); + return false; + } + + if (!amduat_pel_surf_run_with_result(store, + amduat_pel_program_dag_scheme_ref(), + program_ref, + input_refs, + input_refs_len, + false, + (amduat_reference_t){0}, + &run_result)) { + amduatd_v2_job_fail(job, "pel run failed"); + goto run_job_cleanup; + } + if (!run_result.has_result_value || run_result.output_refs_len == 0u) { + amduatd_v2_job_fail(job, "empty pel output"); + goto run_job_cleanup; + } + if (!amduat_reference_clone(run_result.output_refs[0], &job->result_ref)) { + amduatd_v2_job_fail(job, "result clone failed"); + goto run_job_cleanup; + } + job->result_ref_set = true; + job->status = AMDUATD_V2_JOB_SUCCEEDED; + job->completed_at_ms = amduatd_now_ms(); + ok = true; + +run_job_cleanup: + if (run_result.has_result_value) { + amduat_enc_pel1_result_free(&run_result.result_value); + } + if (run_result.output_refs != NULL) { + amduat_pel_surf_free_refs(run_result.output_refs, run_result.output_refs_len); + } + amduat_pel_surf_free_ref(&run_result.result_ref); + if (input_refs_len >= 1u) { + amduat_reference_free(&input_refs[0]); + } + if (input_refs_len >= 2u) { + amduat_reference_free(&input_refs[1]); + } + if (have_program_ref) { + amduat_reference_free(&program_ref); + } + return ok; +} + +static void amduatd_v2_jobs_process_next(amduatd_v2_job_queue_t *jobs, + amduat_asl_store_t *store) { + size_t i; + if (jobs == NULL || store == NULL) { + return; + } + for (i = 0u; i < jobs->len; ++i) { + amduatd_v2_job_t *job = &jobs->items[i]; + if (job->status != AMDUATD_V2_JOB_PENDING) { + continue; + } + job->status = AMDUATD_V2_JOB_RUNNING; + job->started_at_ms = amduatd_now_ms(); + job->error[0] = '\0'; + (void)amduatd_v2_run_job(job, store); + return; + } +} + +static bool amduatd_handle_post_v2_put(int fd, + const amduatd_http_req_t *req, + amduatd_v2_job_queue_t *jobs) { + uint8_t *body = NULL; + const char *p = NULL; + const char *end = NULL; + bool have_body_hex = false; + bool have_type_tag = false; + char *body_hex = NULL; + char *type_tag_text = NULL; + uint8_t *put_bytes = NULL; + size_t put_len = 0u; + amduatd_v2_job_t job; + uint64_t id = 0u; + bool ok = false; + + if (req == NULL || jobs == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "internal"); + } + if (req->content_length == 0u || req->content_length > (1u * 1024u * 1024u)) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid body"); + } + body = (uint8_t *)malloc(req->content_length); + if (body == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + if (!amduatd_read_exact(fd, body, req->content_length)) { + free(body); + return false; + } + + p = (const char *)body; + end = (const char *)body + req->content_length; + if (!amduatd_json_expect(&p, end, '{')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto v2_put_cleanup; + } + for (;;) { + const char *key = NULL; + size_t key_len = 0; + const char *sv = NULL; + size_t sv_len = 0; + const char *cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &key, &key_len) || + !amduatd_json_expect(&p, end, ':')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto v2_put_cleanup; + } + if (key_len == strlen("body_hex") && memcmp(key, "body_hex", key_len) == 0) { + if (have_body_hex || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &body_hex)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid body_hex"); + goto v2_put_cleanup; + } + have_body_hex = true; + } else if (key_len == strlen("type_tag") && + memcmp(key, "type_tag", key_len) == 0) { + if (have_type_tag || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &type_tag_text)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid type_tag"); + goto v2_put_cleanup; + } + have_type_tag = true; + } else { + if (!amduatd_json_skip_value(&p, end, 0)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto v2_put_cleanup; + } + } + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto v2_put_cleanup; + } + p = amduatd_json_skip_ws(p, end); + if (p != end || !have_body_hex || !amduat_hex_decode_alloc(body_hex, &put_bytes, &put_len)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid body_hex"); + goto v2_put_cleanup; + } + + memset(&job, 0, sizeof(job)); + job.kind = AMDUATD_V2_JOB_KIND_PUT; + job.put_body = put_bytes; + job.put_body_len = put_len; + job.put_has_type_tag = false; + if (have_type_tag && + !amduatd_parse_type_tag_hex(type_tag_text, &job.put_has_type_tag, + &job.put_type_tag)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid type_tag"); + goto v2_put_cleanup; + } + if (!amduatd_v2_jobs_append(jobs, &job, &id)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "enqueue failed"); + goto v2_put_cleanup; + } + ok = amduatd_v2_job_send_enqueued(fd, id); + +v2_put_cleanup: + free(body); + free(body_hex); + free(type_tag_text); + free(put_bytes); + return ok; +} + +static bool amduatd_handle_post_v2_concat(int fd, + amduat_asl_store_t *store, + const amduat_asl_store_fs_config_t *cfg, + const amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const amduatd_http_req_t *req, + amduatd_v2_job_queue_t *jobs) { + uint8_t *body = NULL; + const char *p = NULL; + const char *end = NULL; + bool have_left = false; + bool have_right = false; + amduatd_v2_job_t job; + uint64_t id = 0u; + bool ok = false; + + if (store == NULL || cfg == NULL || concepts == NULL || dcfg == NULL || + req == NULL || jobs == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "internal"); + } + if (req->content_length == 0u || req->content_length > (1u * 1024u * 1024u)) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid body"); + } + body = (uint8_t *)malloc(req->content_length); + if (body == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + if (!amduatd_read_exact(fd, body, req->content_length)) { + free(body); + return false; + } + memset(&job, 0, sizeof(job)); + job.kind = AMDUATD_V2_JOB_KIND_CONCAT; + + p = (const char *)body; + end = (const char *)body + req->content_length; + if (!amduatd_json_expect(&p, end, '{')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto v2_concat_cleanup; + } + for (;;) { + const char *key = NULL; + size_t key_len = 0; + const char *sv = NULL; + size_t sv_len = 0; + const char *cur = amduatd_json_skip_ws(p, end); + amduatd_ref_status_t st; + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &key, &key_len) || + !amduatd_json_expect(&p, end, ':') || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto v2_concat_cleanup; + } + if (key_len == strlen("left_ref") && memcmp(key, "left_ref", key_len) == 0) { + if (have_left) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "duplicate left_ref"); + goto v2_concat_cleanup; + } + st = amduatd_decode_ref_or_name_latest(store, cfg, concepts, dcfg, sv, sv_len, + &job.concat_left_ref); + if (st != AMDUATD_REF_OK) { + ok = amduatd_send_json_error(fd, st == AMDUATD_REF_ERR_NOT_FOUND ? 404 : 400, + st == AMDUATD_REF_ERR_NOT_FOUND ? "Not Found" : "Bad Request", + "invalid left_ref"); + goto v2_concat_cleanup; + } + have_left = true; + } else if (key_len == strlen("right_ref") && + memcmp(key, "right_ref", key_len) == 0) { + if (have_right) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "duplicate right_ref"); + goto v2_concat_cleanup; + } + st = amduatd_decode_ref_or_name_latest(store, cfg, concepts, dcfg, sv, sv_len, + &job.concat_right_ref); + if (st != AMDUATD_REF_OK) { + ok = amduatd_send_json_error(fd, st == AMDUATD_REF_ERR_NOT_FOUND ? 404 : 400, + st == AMDUATD_REF_ERR_NOT_FOUND ? "Not Found" : "Bad Request", + "invalid right_ref"); + goto v2_concat_cleanup; + } + have_right = true; + } else { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "unknown field"); + goto v2_concat_cleanup; + } + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto v2_concat_cleanup; + } + if (!have_left || !have_right) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "missing refs"); + goto v2_concat_cleanup; + } + if (!amduatd_v2_jobs_append(jobs, &job, &id)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "enqueue failed"); + goto v2_concat_cleanup; + } + ok = amduatd_v2_job_send_enqueued(fd, id); + +v2_concat_cleanup: + free(body); + amduatd_v2_job_free(&job); + return ok; +} + +static bool amduatd_handle_post_v2_slice(int fd, + amduat_asl_store_t *store, + const amduat_asl_store_fs_config_t *cfg, + const amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const amduatd_http_req_t *req, + amduatd_v2_job_queue_t *jobs) { + uint8_t *body = NULL; + const char *p = NULL; + const char *end = NULL; + bool have_ref = false; + bool have_offset = false; + bool have_length = false; + amduatd_v2_job_t job; + uint64_t id = 0u; + bool ok = false; + + if (store == NULL || cfg == NULL || concepts == NULL || dcfg == NULL || + req == NULL || jobs == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "internal"); + } + if (req->content_length == 0u || req->content_length > (1u * 1024u * 1024u)) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid body"); + } + body = (uint8_t *)malloc(req->content_length); + if (body == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + if (!amduatd_read_exact(fd, body, req->content_length)) { + free(body); + return false; + } + memset(&job, 0, sizeof(job)); + job.kind = AMDUATD_V2_JOB_KIND_SLICE; + + p = (const char *)body; + end = (const char *)body + req->content_length; + if (!amduatd_json_expect(&p, end, '{')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto v2_slice_cleanup; + } + for (;;) { + const char *key = NULL; + size_t key_len = 0; + const char *sv = NULL; + size_t sv_len = 0; + const char *cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &key, &key_len) || + !amduatd_json_expect(&p, end, ':')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto v2_slice_cleanup; + } + if (key_len == strlen("ref") && memcmp(key, "ref", key_len) == 0) { + amduatd_ref_status_t st; + if (have_ref || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid ref"); + goto v2_slice_cleanup; + } + st = amduatd_decode_ref_or_name_latest(store, cfg, concepts, dcfg, sv, sv_len, + &job.slice_ref); + if (st != AMDUATD_REF_OK) { + ok = amduatd_send_json_error(fd, st == AMDUATD_REF_ERR_NOT_FOUND ? 404 : 400, + st == AMDUATD_REF_ERR_NOT_FOUND ? "Not Found" : "Bad Request", + "invalid ref"); + goto v2_slice_cleanup; + } + have_ref = true; + } else if (key_len == strlen("offset") && memcmp(key, "offset", key_len) == 0) { + if (have_offset || !amduatd_json_parse_u64(&p, end, &job.slice_offset)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid offset"); + goto v2_slice_cleanup; + } + have_offset = true; + } else if (key_len == strlen("length") && memcmp(key, "length", key_len) == 0) { + if (have_length || !amduatd_json_parse_u64(&p, end, &job.slice_length)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid length"); + goto v2_slice_cleanup; + } + have_length = true; + } else { + if (!amduatd_json_skip_value(&p, end, 0)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto v2_slice_cleanup; + } + } + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto v2_slice_cleanup; + } + if (!have_ref || !have_offset || !have_length) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "missing fields"); + goto v2_slice_cleanup; + } + if (!amduatd_v2_jobs_append(jobs, &job, &id)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "enqueue failed"); + goto v2_slice_cleanup; + } + ok = amduatd_v2_job_send_enqueued(fd, id); + +v2_slice_cleanup: + free(body); + amduatd_v2_job_free(&job); + return ok; +} + +static bool amduatd_handle_get_v2_job(int fd, + const amduatd_http_req_t *req, + amduatd_v2_job_queue_t *jobs) { + char no_query[1024]; + const char *prefix = "/v2/jobs/"; + const char *id_text = NULL; + char *endp = NULL; + unsigned long long id = 0ULL; + amduatd_v2_job_t *job = NULL; + + if (req == NULL || jobs == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "internal"); + } + if (!amduatd_path_without_query(req->path, no_query, sizeof(no_query))) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid path"); + } + if (strncmp(no_query, prefix, strlen(prefix)) != 0) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid path"); + } + id_text = no_query + strlen(prefix); + if (*id_text == '\0') { + return amduatd_send_json_error(fd, 400, "Bad Request", "missing job id"); + } + errno = 0; + id = strtoull(id_text, &endp, 10); + if (errno != 0 || endp == NULL || *endp != '\0' || id == 0ULL) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid job id"); + } + job = amduatd_v2_jobs_find(jobs, (uint64_t)id); + return amduatd_v2_job_send_status(fd, job); +} + +static size_t amduatd_v2_jobs_count_status(const amduatd_v2_job_queue_t *jobs, + amduatd_v2_job_status_t status) { + size_t i; + size_t count = 0u; + if (jobs == NULL) { + return 0u; + } + for (i = 0u; i < jobs->len; ++i) { + if (jobs->items[i].status == status) { + count++; + } + } + return count; +} + +static bool amduatd_operational_ready(amduatd_concepts_t *concepts, + const amduatd_fed_cfg_t *fed_cfg, + const amduat_fed_coord_t *fed_coord, + bool *out_index_ready, + bool *out_fed_ready) { + bool index_ready = false; + bool fed_ready = false; + if (concepts != NULL && + amduatd_concepts_ensure_query_index_ready(concepts) && + concepts->qindex.built_for_edges_len == concepts->edges.len) { + index_ready = true; + } + fed_ready = (fed_cfg == NULL || !fed_cfg->enabled || fed_coord != NULL); + if (out_index_ready != NULL) { + *out_index_ready = index_ready; + } + if (out_fed_ready != NULL) { + *out_fed_ready = fed_ready; + } + return index_ready && fed_ready; +} + +static bool amduatd_handle_get_v2_healthz(int fd) { + char json[192]; + int n = snprintf(json, + sizeof(json), + "{\"ok\":true,\"status\":\"healthy\",\"time_ms\":%llu}\n", + (unsigned long long)amduatd_now_ms()); + if (n <= 0 || (size_t)n >= sizeof(json)) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + } + return amduatd_http_send_json(fd, 200, "OK", json, false); +} + +static bool amduatd_handle_get_v2_readyz(int fd, + amduatd_concepts_t *concepts, + const amduatd_fed_cfg_t *fed_cfg, + const amduat_fed_coord_t *fed_coord) { + bool index_ready = false; + bool fed_ready = false; + bool ready = amduatd_operational_ready(concepts, + fed_cfg, + fed_coord, + &index_ready, + &fed_ready); + char json[320]; + int n = snprintf(json, + sizeof(json), + "{\"ok\":%s,\"status\":\"%s\",\"components\":{\"graph_index\":%s,\"federation\":%s}}\n", + ready ? "true" : "false", + ready ? "ready" : "not_ready", + index_ready ? "true" : "false", + fed_ready ? "true" : "false"); + if (n <= 0 || (size_t)n >= sizeof(json)) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + } + if (ready) { + return amduatd_http_send_json(fd, 200, "OK", json, false); + } + return amduatd_http_send_json(fd, 503, "Service Unavailable", json, false); +} + +static bool amduatd_handle_get_v2_graph_stats(int fd, + amduatd_concepts_t *concepts) { + size_t i; + size_t tombstone_edges = 0u; + double tombstone_ratio = 0.0; + char ratio_buf[64]; + char json[1024]; + int n; + + if (concepts == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "internal error"); + } + if (!amduatd_concepts_ensure_query_index_ready(concepts)) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "index error"); + } + for (i = 0u; i < concepts->edges.len; ++i) { + const amduatd_edge_entry_t *entry = &concepts->edges.items[i]; + if (entry->rel != NULL && strcmp(entry->rel, "tombstones") == 0) { + tombstone_edges++; + } + } + if (concepts->edges.len != 0u) { + tombstone_ratio = (double)tombstone_edges / (double)concepts->edges.len; + } + n = snprintf(ratio_buf, sizeof(ratio_buf), "%.6f", tombstone_ratio); + if (n <= 0 || (size_t)n >= sizeof(ratio_buf)) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + } + n = snprintf(json, + sizeof(json), + "{\"edges_total\":%llu," + "\"aliases_total\":%llu," + "\"index\":{\"built_for_edges\":%llu," + "\"src_buckets\":%llu," + "\"dst_buckets\":%llu," + "\"predicate_buckets\":%llu," + "\"src_predicate_buckets\":%llu," + "\"dst_predicate_buckets\":%llu," + "\"healthy\":%s}," + "\"tombstones\":{\"edges\":%llu,\"ratio\":%s}}\n", + (unsigned long long)concepts->edges.len, + (unsigned long long)concepts->qindex.alias_len, + (unsigned long long)concepts->qindex.built_for_edges_len, + (unsigned long long)concepts->qindex.src_len, + (unsigned long long)concepts->qindex.dst_len, + (unsigned long long)concepts->qindex.predicate_len, + (unsigned long long)concepts->qindex.src_predicate_len, + (unsigned long long)concepts->qindex.dst_predicate_len, + concepts->qindex.built_for_edges_len == concepts->edges.len + ? "true" + : "false", + (unsigned long long)tombstone_edges, + ratio_buf); + if (n <= 0 || (size_t)n >= sizeof(json)) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + } + return amduatd_http_send_json(fd, 200, "OK", json, false); +} + +static bool amduatd_handle_get_v2_graph_capabilities(int fd, + const amduatd_cfg_t *dcfg, + const amduatd_fed_cfg_t *fed_cfg, + const amduatd_caps_t *caps) { + char json[1200]; + int n; + bool fed_enabled = fed_cfg != NULL && fed_cfg->enabled; + bool caps_enabled = caps != NULL && caps->enabled; + n = snprintf(json, + sizeof(json), + "{\"contract\":\"AMDUATD/API/2\"," + "\"graph\":{\"version\":\"v2\"," + "\"features\":[\"query\",\"subgraph\",\"paths\",\"retrieve\",\"changes\",\"batch\",\"tombstones\",\"provenance\",\"provenance_policy\"]," + "\"limits\":{\"max_depth\":32,\"max_fanout\":10000,\"max_limit\":2000,\"max_result_bytes\":8388608}," + "\"modes\":{\"single_store\":true,\"local_socket_auth\":true,\"provenance_mode_values\":[\"optional\",\"required\"]}}," + "\"runtime\":{\"derivation_index_enabled\":%s,\"derivation_index_strict\":%s,\"federation_enabled\":%s,\"cap_reads_enabled\":%s}}\n", + (dcfg != NULL && dcfg->derivation_index_enabled) ? "true" : "false", + (dcfg != NULL && dcfg->derivation_index_strict) ? "true" : "false", + fed_enabled ? "true" : "false", + caps_enabled && caps->enable_cap_reads ? "true" : "false"); + if (n <= 0 || (size_t)n >= sizeof(json)) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + } + return amduatd_http_send_json(fd, 200, "OK", json, false); +} + +static bool amduatd_handle_get_v2_metrics(int fd, + amduatd_concepts_t *concepts, + const amduatd_fed_cfg_t *fed_cfg, + const amduat_fed_coord_t *fed_coord, + amduatd_v2_job_queue_t *jobs) { + amduatd_strbuf_t b; + bool index_ready = false; + bool fed_ready = false; + bool ready; + size_t pending_jobs; + size_t running_jobs; + size_t succeeded_jobs; + size_t failed_jobs; + size_t i; + size_t tombstone_edges = 0u; + char line[256]; + int n; + + if (concepts == NULL) { + return amduatd_http_send_text(fd, 500, "Internal Server Error", "internal error\n", false); + } + memset(&b, 0, sizeof(b)); + ready = amduatd_operational_ready(concepts, fed_cfg, fed_coord, &index_ready, &fed_ready); + pending_jobs = amduatd_v2_jobs_count_status(jobs, AMDUATD_V2_JOB_PENDING); + running_jobs = amduatd_v2_jobs_count_status(jobs, AMDUATD_V2_JOB_RUNNING); + succeeded_jobs = amduatd_v2_jobs_count_status(jobs, AMDUATD_V2_JOB_SUCCEEDED); + failed_jobs = amduatd_v2_jobs_count_status(jobs, AMDUATD_V2_JOB_FAILED); + for (i = 0u; i < concepts->edges.len; ++i) { + const amduatd_edge_entry_t *entry = &concepts->edges.items[i]; + if (entry->rel != NULL && strcmp(entry->rel, "tombstones") == 0) { + tombstone_edges++; + } + } + + if (!amduatd_strbuf_append_cstr(&b, "# HELP amduatd_up daemon process health\n" + "# TYPE amduatd_up gauge\n" + "amduatd_up 1\n" + "# HELP amduatd_ready readiness for serving graph traffic\n" + "# TYPE amduatd_ready gauge\n")) { + amduatd_strbuf_free(&b); + return amduatd_http_send_text(fd, 500, "Internal Server Error", "oom\n", false); + } + n = snprintf(line, sizeof(line), "amduatd_ready %d\n", ready ? 1 : 0); + if (n <= 0 || (size_t)n >= sizeof(line) || !amduatd_strbuf_append_cstr(&b, line)) { + amduatd_strbuf_free(&b); + return amduatd_http_send_text(fd, 500, "Internal Server Error", "oom\n", false); + } + n = snprintf(line, + sizeof(line), + "amduatd_graph_edges_total %llu\n", + (unsigned long long)concepts->edges.len); + if (n <= 0 || (size_t)n >= sizeof(line) || !amduatd_strbuf_append_cstr(&b, line)) { + amduatd_strbuf_free(&b); + return amduatd_http_send_text(fd, 500, "Internal Server Error", "oom\n", false); + } + n = snprintf(line, + sizeof(line), + "amduatd_graph_alias_total %llu\n", + (unsigned long long)concepts->qindex.alias_len); + if (n <= 0 || (size_t)n >= sizeof(line) || !amduatd_strbuf_append_cstr(&b, line)) { + amduatd_strbuf_free(&b); + return amduatd_http_send_text(fd, 500, "Internal Server Error", "oom\n", false); + } + n = snprintf(line, + sizeof(line), + "amduatd_graph_tombstone_edges_total %llu\n", + (unsigned long long)tombstone_edges); + if (n <= 0 || (size_t)n >= sizeof(line) || !amduatd_strbuf_append_cstr(&b, line)) { + amduatd_strbuf_free(&b); + return amduatd_http_send_text(fd, 500, "Internal Server Error", "oom\n", false); + } + n = snprintf(line, sizeof(line), "amduatd_graph_index_ready %d\n", index_ready ? 1 : 0); + if (n <= 0 || (size_t)n >= sizeof(line) || !amduatd_strbuf_append_cstr(&b, line)) { + amduatd_strbuf_free(&b); + return amduatd_http_send_text(fd, 500, "Internal Server Error", "oom\n", false); + } + n = snprintf(line, sizeof(line), "amduatd_federation_ready %d\n", fed_ready ? 1 : 0); + if (n <= 0 || (size_t)n >= sizeof(line) || !amduatd_strbuf_append_cstr(&b, line)) { + amduatd_strbuf_free(&b); + return amduatd_http_send_text(fd, 500, "Internal Server Error", "oom\n", false); + } + n = snprintf(line, sizeof(line), "amduatd_v2_jobs_pending %llu\n", (unsigned long long)pending_jobs); + if (n <= 0 || (size_t)n >= sizeof(line) || !amduatd_strbuf_append_cstr(&b, line)) { + amduatd_strbuf_free(&b); + return amduatd_http_send_text(fd, 500, "Internal Server Error", "oom\n", false); + } + n = snprintf(line, sizeof(line), "amduatd_v2_jobs_running %llu\n", (unsigned long long)running_jobs); + if (n <= 0 || (size_t)n >= sizeof(line) || !amduatd_strbuf_append_cstr(&b, line)) { + amduatd_strbuf_free(&b); + return amduatd_http_send_text(fd, 500, "Internal Server Error", "oom\n", false); + } + n = snprintf(line, sizeof(line), "amduatd_v2_jobs_succeeded %llu\n", (unsigned long long)succeeded_jobs); + if (n <= 0 || (size_t)n >= sizeof(line) || !amduatd_strbuf_append_cstr(&b, line)) { + amduatd_strbuf_free(&b); + return amduatd_http_send_text(fd, 500, "Internal Server Error", "oom\n", false); + } + n = snprintf(line, sizeof(line), "amduatd_v2_jobs_failed %llu\n", (unsigned long long)failed_jobs); + if (n <= 0 || (size_t)n >= sizeof(line) || !amduatd_strbuf_append_cstr(&b, line)) { + amduatd_strbuf_free(&b); + return amduatd_http_send_text(fd, 500, "Internal Server Error", "oom\n", false); + } + + { + bool ok = amduatd_http_send_status(fd, + 200, + "OK", + "text/plain; version=0.0.4; charset=utf-8", + (const uint8_t *)b.data, + b.len, + false); + amduatd_strbuf_free(&b); + return ok; + } +} + static bool amduatd_handle_conn(int fd, amduat_asl_store_t *store, const amduat_asl_store_fs_config_t *cfg, @@ -9681,6 +11824,7 @@ static bool amduatd_handle_conn(int fd, const amduat_fed_coord_t *coord, const amduatd_allowlist_t *allowlist, amduatd_caps_t *caps, + amduatd_v2_job_queue_t *jobs, const char *root_path, amduatd_store_backend_t store_backend) { amduatd_http_req_t req; @@ -10004,6 +12148,69 @@ static bool amduatd_handle_conn(int fd, root_path, &req); goto conn_cleanup; } + if (strcmp(req.method, "POST") == 0 && + strcmp(no_query, "/v2/pel/execute") == 0) { + ok = amduatd_handle_post_v2_pel_execute(fd, + store, + cfg, + concepts, + effective_cfg, + root_path, + &req); + goto conn_cleanup; + } + if (strcmp(req.method, "POST") == 0 && + strcmp(no_query, "/v2/ops/put") == 0) { + ok = amduatd_handle_post_v2_put(fd, &req, jobs); + goto conn_cleanup; + } + if (strcmp(req.method, "POST") == 0 && + strcmp(no_query, "/v2/ops/concat") == 0) { + ok = amduatd_handle_post_v2_concat(fd, + store, + cfg, + concepts, + effective_cfg, + &req, + jobs); + goto conn_cleanup; + } + if (strcmp(req.method, "POST") == 0 && + strcmp(no_query, "/v2/ops/slice") == 0) { + ok = amduatd_handle_post_v2_slice(fd, + store, + cfg, + concepts, + effective_cfg, + &req, + jobs); + goto conn_cleanup; + } + if (strcmp(req.method, "GET") == 0 && + strcmp(no_query, "/v2/healthz") == 0) { + ok = amduatd_handle_get_v2_healthz(fd); + goto conn_cleanup; + } + if (strcmp(req.method, "GET") == 0 && + strcmp(no_query, "/v2/readyz") == 0) { + ok = amduatd_handle_get_v2_readyz(fd, concepts, fed_cfg, coord); + goto conn_cleanup; + } + if (strcmp(req.method, "GET") == 0 && + strcmp(no_query, "/v2/metrics") == 0) { + ok = amduatd_handle_get_v2_metrics(fd, concepts, fed_cfg, coord, jobs); + goto conn_cleanup; + } + if (strcmp(req.method, "GET") == 0 && + strcmp(no_query, "/v2/graph/stats") == 0) { + ok = amduatd_handle_get_v2_graph_stats(fd, concepts); + goto conn_cleanup; + } + if (strcmp(req.method, "GET") == 0 && + strcmp(no_query, "/v2/graph/capabilities") == 0) { + ok = amduatd_handle_get_v2_graph_capabilities(fd, effective_cfg, fed_cfg, caps); + goto conn_cleanup; + } if (strcmp(req.method, "POST") == 0 && strcmp(no_query, "/v1/pel/programs") == 0) { ok = amduatd_handle_post_pel_programs(fd, store, &req); @@ -10065,6 +12272,23 @@ static bool amduatd_handle_conn(int fd, ok = amduatd_handle_get_artifact(fd, store, &req, req.path, false); goto conn_cleanup; } + if (strcmp(req.method, "GET") == 0 && + strncmp(no_query, "/v2/get/", 8) == 0) { + char path[1024]; + const char *suffix = no_query + 7; + if (snprintf(path, sizeof(path), "/v1/artifacts%s", suffix) <= 0 || + strlen(path) >= sizeof(path)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid path"); + } else { + ok = amduatd_handle_get_artifact(fd, store, &req, path, false); + } + goto conn_cleanup; + } + if (strcmp(req.method, "GET") == 0 && + strncmp(no_query, "/v2/jobs/", 9) == 0) { + ok = amduatd_handle_get_v2_job(fd, &req, jobs); + goto conn_cleanup; + } if (strcmp(req.method, "HEAD") == 0 && strncmp(no_query, "/v1/artifacts/", 14) == 0) { ok = amduatd_handle_head_artifact(fd, store, &req, req.path); @@ -10121,6 +12345,7 @@ int main(int argc, char **argv) { amduat_fed_transport_unix_t fed_unix; amduat_fed_coord_cfg_t fed_coord_cfg; amduat_fed_coord_t *fed_coord = NULL; + amduatd_v2_job_queue_t v2_jobs; amduatd_allowlist_t allowlist; int i; int sfd = -1; @@ -10131,6 +12356,8 @@ int main(int argc, char **argv) { memset(&api_contract_ref, 0, sizeof(api_contract_ref)); memset(&ui_ref, 0, sizeof(ui_ref)); memset(&concepts, 0, sizeof(concepts)); + memset(&v2_jobs, 0, sizeof(v2_jobs)); + v2_jobs.next_id = 1u; memset(&allowlist, 0, sizeof(allowlist)); memset(&dcfg, 0, sizeof(dcfg)); memset(&caps, 0, sizeof(caps)); @@ -10391,6 +12618,10 @@ int main(int argc, char **argv) { tv.tv_sec = (time_t)(wait_ms / 1000u); tv.tv_usec = (suseconds_t)((wait_ms % 1000u) * 1000u); tvp = &tv; + } else if (amduatd_v2_jobs_has_pending(&v2_jobs)) { + tv.tv_sec = 0; + tv.tv_usec = 10000; + tvp = &tv; } FD_ZERO(&rfds); @@ -10412,6 +12643,9 @@ int main(int argc, char **argv) { last_tick_ms = now_ms; } } + if (amduatd_v2_jobs_has_pending(&v2_jobs)) { + amduatd_v2_jobs_process_next(&v2_jobs, &store); + } if (rc == 0) { continue; @@ -10441,6 +12675,7 @@ int main(int argc, char **argv) { fed_coord, &allowlist, &caps, + &v2_jobs, root, store_backend); (void)close(cfd); @@ -10470,6 +12705,7 @@ int main(int argc, char **argv) { if (fed_coord != NULL) { amduat_fed_coord_close(fed_coord); } + amduatd_v2_jobs_free(&v2_jobs); amduatd_allowlist_free(&allowlist); (void)unlink(sock_path); (void)close(sfd); diff --git a/src/amduatd_concepts.c b/src/amduatd_concepts.c index feb1d85..bdf0cab 100644 --- a/src/amduatd_concepts.c +++ b/src/amduatd_concepts.c @@ -48,6 +48,19 @@ static const uint32_t AMDUATD_EDGE_VIEW_BATCH = 1024u; static const uint32_t AMDUATD_EDGE_REFRESH_BATCH = 512u; static const uint16_t AMDUATD_EDGE_COLLECTION_KIND = 1u; +static bool amduatd_relation_entry_ref(const amduatd_concepts_t *concepts, + const char *rel_name, + amduat_reference_t *out_ref); +static void amduatd_graph_schema_reset_globals(void); +static bool amduatd_graph_schema_load_into_globals(amduat_asl_store_t *store, + amduatd_concepts_t *concepts, + const amduatd_space_t *space); +static bool amduatd_graph_schema_validate_provenance_write( + bool has_provenance_attachment, + int *out_status, + const char **out_error); +static bool amduatd_parse_bool_query(const char *s, bool *out); + static void amduatd_edge_entry_free(amduatd_edge_entry_t *entry) { if (entry == NULL) { return; @@ -72,6 +85,364 @@ static void amduatd_edge_list_clear(amduatd_edge_list_t *list) { list->cap = 0; } +static void amduatd_query_index_bucket_clear(amduatd_ref_edge_bucket_t *b) { + if (b == NULL) { + return; + } + amduat_reference_free(&b->ref); + free(b->edge_indices); + b->edge_indices = NULL; + b->len = 0u; + b->cap = 0u; +} + +static void amduatd_query_index_pair_bucket_clear( + amduatd_ref_pair_edge_bucket_t *b) { + if (b == NULL) { + return; + } + amduat_reference_free(&b->left_ref); + amduat_reference_free(&b->right_ref); + free(b->edge_indices); + b->edge_indices = NULL; + b->len = 0u; + b->cap = 0u; +} + +static void amduatd_query_index_clear(amduatd_query_index_t *idx) { + size_t i; + if (idx == NULL) { + return; + } + free(idx->alias_edge_indices); + idx->alias_edge_indices = NULL; + idx->alias_len = 0u; + idx->alias_cap = 0u; + for (i = 0; i < idx->src_len; ++i) { + amduatd_query_index_bucket_clear(&idx->src_buckets[i]); + } + free(idx->src_buckets); + idx->src_buckets = NULL; + idx->src_len = 0u; + idx->src_cap = 0u; + for (i = 0; i < idx->dst_len; ++i) { + amduatd_query_index_bucket_clear(&idx->dst_buckets[i]); + } + free(idx->dst_buckets); + idx->dst_buckets = NULL; + idx->dst_len = 0u; + idx->dst_cap = 0u; + for (i = 0; i < idx->predicate_len; ++i) { + amduatd_query_index_bucket_clear(&idx->predicate_buckets[i]); + } + free(idx->predicate_buckets); + idx->predicate_buckets = NULL; + idx->predicate_len = 0u; + idx->predicate_cap = 0u; + for (i = 0; i < idx->src_predicate_len; ++i) { + amduatd_query_index_pair_bucket_clear(&idx->src_predicate_buckets[i]); + } + free(idx->src_predicate_buckets); + idx->src_predicate_buckets = NULL; + idx->src_predicate_len = 0u; + idx->src_predicate_cap = 0u; + for (i = 0; i < idx->dst_predicate_len; ++i) { + amduatd_query_index_pair_bucket_clear(&idx->dst_predicate_buckets[i]); + } + free(idx->dst_predicate_buckets); + idx->dst_predicate_buckets = NULL; + idx->dst_predicate_len = 0u; + idx->dst_predicate_cap = 0u; + for (i = 0; i < idx->tombstoned_src_len; ++i) { + amduatd_query_index_bucket_clear(&idx->tombstoned_src_buckets[i]); + } + free(idx->tombstoned_src_buckets); + idx->tombstoned_src_buckets = NULL; + idx->tombstoned_src_len = 0u; + idx->tombstoned_src_cap = 0u; + idx->built_for_edges_len = SIZE_MAX; +} + +static bool amduatd_query_index_append_size_t(size_t **items, + size_t *len, + size_t *cap, + size_t value) { + size_t *next; + size_t next_cap; + if (items == NULL || len == NULL || cap == NULL) { + return false; + } + if (*len == *cap) { + next_cap = *cap != 0u ? *cap * 2u : 64u; + next = (size_t *)realloc(*items, next_cap * sizeof(*next)); + if (next == NULL) { + return false; + } + *items = next; + *cap = next_cap; + } + (*items)[(*len)++] = value; + return true; +} + +static amduatd_ref_edge_bucket_t *amduatd_query_index_find_bucket( + amduatd_ref_edge_bucket_t *buckets, + size_t buckets_len, + amduat_reference_t ref) { + size_t i; + for (i = 0; i < buckets_len; ++i) { + if (amduat_reference_eq(buckets[i].ref, ref)) { + return &buckets[i]; + } + } + return NULL; +} + +static const amduatd_ref_edge_bucket_t *amduatd_query_index_find_bucket_const( + const amduatd_ref_edge_bucket_t *buckets, + size_t buckets_len, + amduat_reference_t ref) { + size_t i; + for (i = 0; i < buckets_len; ++i) { + if (amduat_reference_eq(buckets[i].ref, ref)) { + return &buckets[i]; + } + } + return NULL; +} + +static amduatd_ref_pair_edge_bucket_t *amduatd_query_index_find_pair_bucket( + amduatd_ref_pair_edge_bucket_t *buckets, + size_t buckets_len, + amduat_reference_t left_ref, + amduat_reference_t right_ref) { + size_t i; + for (i = 0; i < buckets_len; ++i) { + if (amduat_reference_eq(buckets[i].left_ref, left_ref) && + amduat_reference_eq(buckets[i].right_ref, right_ref)) { + return &buckets[i]; + } + } + return NULL; +} + +static const amduatd_ref_pair_edge_bucket_t * +amduatd_query_index_find_pair_bucket_const( + const amduatd_ref_pair_edge_bucket_t *buckets, + size_t buckets_len, + amduat_reference_t left_ref, + amduat_reference_t right_ref) { + size_t i; + for (i = 0; i < buckets_len; ++i) { + if (amduat_reference_eq(buckets[i].left_ref, left_ref) && + amduat_reference_eq(buckets[i].right_ref, right_ref)) { + return &buckets[i]; + } + } + return NULL; +} + +static amduatd_ref_edge_bucket_t *amduatd_query_index_get_bucket( + amduatd_ref_edge_bucket_t **buckets, + size_t *buckets_len, + size_t *buckets_cap, + amduat_reference_t ref) { + amduatd_ref_edge_bucket_t *found; + amduatd_ref_edge_bucket_t *next; + size_t next_cap; + if (buckets == NULL || buckets_len == NULL || buckets_cap == NULL) { + return NULL; + } + found = amduatd_query_index_find_bucket(*buckets, *buckets_len, ref); + if (found != NULL) { + return found; + } + if (*buckets_len == *buckets_cap) { + next_cap = *buckets_cap != 0u ? *buckets_cap * 2u : 64u; + next = (amduatd_ref_edge_bucket_t *)realloc(*buckets, + next_cap * sizeof(*next)); + if (next == NULL) { + return NULL; + } + *buckets = next; + *buckets_cap = next_cap; + } + found = &(*buckets)[(*buckets_len)++]; + memset(found, 0, sizeof(*found)); + if (!amduat_reference_clone(ref, &found->ref)) { + (*buckets_len)--; + return NULL; + } + return found; +} + +static amduatd_ref_pair_edge_bucket_t *amduatd_query_index_get_pair_bucket( + amduatd_ref_pair_edge_bucket_t **buckets, + size_t *buckets_len, + size_t *buckets_cap, + amduat_reference_t left_ref, + amduat_reference_t right_ref) { + amduatd_ref_pair_edge_bucket_t *found; + amduatd_ref_pair_edge_bucket_t *next; + size_t next_cap; + if (buckets == NULL || buckets_len == NULL || buckets_cap == NULL) { + return NULL; + } + found = amduatd_query_index_find_pair_bucket(*buckets, + *buckets_len, + left_ref, + right_ref); + if (found != NULL) { + return found; + } + if (*buckets_len == *buckets_cap) { + next_cap = *buckets_cap != 0u ? *buckets_cap * 2u : 64u; + next = (amduatd_ref_pair_edge_bucket_t *)realloc(*buckets, + next_cap * sizeof(*next)); + if (next == NULL) { + return NULL; + } + *buckets = next; + *buckets_cap = next_cap; + } + found = &(*buckets)[(*buckets_len)++]; + memset(found, 0, sizeof(*found)); + if (!amduat_reference_clone(left_ref, &found->left_ref) || + !amduat_reference_clone(right_ref, &found->right_ref)) { + amduat_reference_free(&found->left_ref); + amduat_reference_free(&found->right_ref); + (*buckets_len)--; + return NULL; + } + return found; +} + +static bool amduatd_concepts_rebuild_query_index(amduatd_concepts_t *c) { + size_t i; + if (c == NULL) { + return false; + } + amduatd_query_index_clear(&c->qindex); + for (i = 0; i < c->edges.len; ++i) { + const amduatd_edge_entry_t *entry = &c->edges.items[i]; + amduatd_ref_edge_bucket_t *src_bucket; + amduatd_ref_edge_bucket_t *dst_bucket; + amduatd_ref_edge_bucket_t *predicate_bucket; + amduatd_ref_edge_bucket_t *tombstone_src_bucket; + amduat_reference_t predicate_ref; + amduatd_ref_pair_edge_bucket_t *src_predicate_bucket; + amduatd_ref_pair_edge_bucket_t *dst_predicate_bucket; + memset(&predicate_ref, 0, sizeof(predicate_ref)); + if (entry->rel != NULL && strcmp(entry->rel, "alias") == 0) { + if (!amduatd_query_index_append_size_t(&c->qindex.alias_edge_indices, + &c->qindex.alias_len, + &c->qindex.alias_cap, + i)) { + amduatd_query_index_clear(&c->qindex); + return false; + } + } + if (entry->rel != NULL && strcmp(entry->rel, "tombstones") == 0) { + tombstone_src_bucket = amduatd_query_index_get_bucket( + &c->qindex.tombstoned_src_buckets, + &c->qindex.tombstoned_src_len, + &c->qindex.tombstoned_src_cap, + entry->src_ref); + if (tombstone_src_bucket == NULL || + !amduatd_query_index_append_size_t(&tombstone_src_bucket->edge_indices, + &tombstone_src_bucket->len, + &tombstone_src_bucket->cap, + i)) { + amduatd_query_index_clear(&c->qindex); + return false; + } + } + src_bucket = amduatd_query_index_get_bucket(&c->qindex.src_buckets, + &c->qindex.src_len, + &c->qindex.src_cap, + entry->src_ref); + if (src_bucket == NULL || + !amduatd_query_index_append_size_t(&src_bucket->edge_indices, + &src_bucket->len, + &src_bucket->cap, + i)) { + amduatd_query_index_clear(&c->qindex); + return false; + } + dst_bucket = amduatd_query_index_get_bucket(&c->qindex.dst_buckets, + &c->qindex.dst_len, + &c->qindex.dst_cap, + entry->dst_ref); + if (dst_bucket == NULL || + !amduatd_query_index_append_size_t(&dst_bucket->edge_indices, + &dst_bucket->len, + &dst_bucket->cap, + i)) { + amduatd_query_index_clear(&c->qindex); + return false; + } + if (!amduatd_relation_entry_ref(c, entry->rel, &predicate_ref)) { + continue; + } + predicate_bucket = amduatd_query_index_get_bucket(&c->qindex.predicate_buckets, + &c->qindex.predicate_len, + &c->qindex.predicate_cap, + predicate_ref); + if (predicate_bucket == NULL || + !amduatd_query_index_append_size_t(&predicate_bucket->edge_indices, + &predicate_bucket->len, + &predicate_bucket->cap, + i)) { + amduat_reference_free(&predicate_ref); + amduatd_query_index_clear(&c->qindex); + return false; + } + src_predicate_bucket = amduatd_query_index_get_pair_bucket( + &c->qindex.src_predicate_buckets, + &c->qindex.src_predicate_len, + &c->qindex.src_predicate_cap, + entry->src_ref, + predicate_ref); + if (src_predicate_bucket == NULL || + !amduatd_query_index_append_size_t(&src_predicate_bucket->edge_indices, + &src_predicate_bucket->len, + &src_predicate_bucket->cap, + i)) { + amduat_reference_free(&predicate_ref); + amduatd_query_index_clear(&c->qindex); + return false; + } + dst_predicate_bucket = amduatd_query_index_get_pair_bucket( + &c->qindex.dst_predicate_buckets, + &c->qindex.dst_predicate_len, + &c->qindex.dst_predicate_cap, + entry->dst_ref, + predicate_ref); + if (dst_predicate_bucket == NULL || + !amduatd_query_index_append_size_t(&dst_predicate_bucket->edge_indices, + &dst_predicate_bucket->len, + &dst_predicate_bucket->cap, + i)) { + amduat_reference_free(&predicate_ref); + amduatd_query_index_clear(&c->qindex); + return false; + } + amduat_reference_free(&predicate_ref); + } + c->qindex.built_for_edges_len = c->edges.len; + return true; +} + +static bool amduatd_concepts_ensure_query_index(amduatd_concepts_t *c) { + if (c == NULL) { + return false; + } + if (c->qindex.built_for_edges_len == c->edges.len) { + return true; + } + return amduatd_concepts_rebuild_query_index(c); +} + static bool amduatd_edge_list_push(amduatd_edge_list_t *list, const amduatd_edge_entry_t *entry) { amduatd_edge_entry_t cloned; @@ -113,6 +484,7 @@ void amduatd_concepts_free(amduatd_concepts_t *c) { if (c == NULL) { return; } + amduatd_graph_schema_reset_globals(); amduat_reference_free(&c->rel_aliases_ref); amduat_reference_free(&c->rel_materializes_ref); amduat_reference_free(&c->rel_represents_ref); @@ -120,7 +492,9 @@ void amduatd_concepts_free(amduatd_concepts_t *c) { amduat_reference_free(&c->rel_within_domain_ref); amduat_reference_free(&c->rel_computed_by_ref); amduat_reference_free(&c->rel_has_provenance_ref); + amduat_reference_free(&c->rel_tombstones_ref); amduatd_edge_list_clear(&c->edges); + amduatd_query_index_clear(&c->qindex); free(c->edge_collection_name); c->edge_collection_name = NULL; } @@ -164,6 +538,11 @@ static const char *const AMDUATD_REL_REQUIRES_KEY = "requires_key"; static const char *const AMDUATD_REL_WITHIN_DOMAIN = "within_domain"; static const char *const AMDUATD_REL_COMPUTED_BY = "computed_by"; static const char *const AMDUATD_REL_HAS_PROVENANCE = "has_provenance"; +static const char *const AMDUATD_REL_TOMBSTONES = "tombstones"; +static const char *const AMDUATD_BATCH_IDEMPOTENCY_SCHEMA = + "tgk/batch_idempotency_state"; +static const char *const AMDUATD_GRAPH_SCHEMA_POLICY_SCHEMA = + "tgk/graph_schema_policy_state"; static bool amduatd_concepts_put_name_artifact(amduat_asl_store_t *store, const char *name, @@ -827,6 +1206,9 @@ static const char *amduatd_relation_name_for_ref( if (amduat_reference_eq(ref, c->rel_has_provenance_ref)) { return AMDUATD_REL_HAS_PROVENANCE; } + if (amduat_reference_eq(ref, c->rel_tombstones_ref)) { + return AMDUATD_REL_TOMBSTONES; + } return NULL; } @@ -1437,6 +1819,7 @@ static bool amduatd_concepts_load_edges(amduatd_concepts_t *c, } amduatd_edge_list_clear(&c->edges); + c->qindex.built_for_edges_len = SIZE_MAX; while (true) { memset(&view, 0, sizeof(view)); @@ -1493,6 +1876,7 @@ static bool amduatd_concepts_load_edges(amduatd_concepts_t *c, amduatd_edge_entry_free(&entry); goto load_cleanup; } + c->qindex.built_for_edges_len = SIZE_MAX; amduatd_edge_entry_free(&entry); } @@ -1629,6 +2013,7 @@ static bool amduatd_concepts_append_edge_record(amduatd_concepts_t *c, amduat_reference_free(&record_ref); return false; } + c->qindex.built_for_edges_len = SIZE_MAX; if (offset != 0u && (offset % 256u) == 0u) { amduat_reference_t snapshot_ref; @@ -2208,6 +2593,8 @@ bool amduatd_concepts_init(amduatd_concepts_t *c, return false; } memset(c, 0, sizeof(*c)); + amduatd_graph_schema_reset_globals(); + c->qindex.built_for_edges_len = SIZE_MAX; memset(&index_head_ref, 0, sizeof(index_head_ref)); c->root_path = root_path; (void)snprintf(c->edges_path, sizeof(c->edges_path), "%s/%s", root_path, @@ -2301,6 +2688,17 @@ bool amduatd_concepts_init(amduatd_concepts_t *c, "concepts init: ensure ms.has_provenance failed"); return false; } + if (!amduatd_concepts_seed_relation(store, space, "tombstones", + &c->rel_tombstones_ref)) { + amduat_log(AMDUAT_LOG_ERROR, "concepts init: seed tombstones failed"); + return false; + } + if (!amduatd_concepts_ensure_alias(c, store, space, "ms.tombstones", + c->rel_tombstones_ref)) { + amduat_log(AMDUAT_LOG_ERROR, + "concepts init: ensure ms.tombstones failed"); + return false; + } if (amduatd_concepts_edge_index_pointer_name(space, &index_head_name)) { (void)amduat_asl_pointer_get(&c->edge_collection.pointer_store, @@ -2386,6 +2784,12 @@ bool amduatd_concepts_init(amduatd_concepts_t *c, } amduat_reference_free(&index_head_ref); + (void)amduatd_concepts_rebuild_query_index(c); + if (!amduatd_graph_schema_load_into_globals(store, c, space)) { + amduat_log(AMDUAT_LOG_ERROR, + "concepts init: graph schema policy load failed"); + return false; + } return true; } @@ -2406,6 +2810,10 @@ bool amduatd_concepts_refresh_edges(amduatd_ctx_t *ctx, max_new_entries); } +bool amduatd_concepts_ensure_query_index_ready(amduatd_concepts_t *c) { + return amduatd_concepts_ensure_query_index(c); +} + static bool amduatd_concepts_derive_name_ref( amduat_asl_store_t *store, const char *name, @@ -2534,9 +2942,9 @@ static bool amduatd_concepts_put_edge(amduat_asl_store_t *store, } static bool amduatd_concepts_lookup_alias(amduat_asl_store_t *store, - const amduatd_concepts_t *c, - const char *name, - amduat_reference_t *out_concept_ref) { + const amduatd_concepts_t *c, + const char *name, + amduat_reference_t *out_concept_ref) { amduat_reference_t name_ref; size_t i; @@ -2569,11 +2977,36 @@ static bool amduatd_concepts_lookup_alias(amduat_asl_store_t *store, return false; } +static bool amduatd_graph_edge_is_tombstoned(const amduatd_concepts_t *concepts, + amduat_reference_t edge_record_ref, + size_t scan_end); +static bool amduatd_concepts_resolve_latest_at(amduat_asl_store_t *store, + const amduatd_concepts_t *c, + amduat_reference_t concept_ref, + size_t scan_end, + bool include_tombstoned, + amduat_reference_t *out_ref); + static bool amduatd_concepts_resolve_latest(amduat_asl_store_t *store, const amduatd_concepts_t *c, amduat_reference_t concept_ref, amduat_reference_t *out_ref) { + return amduatd_concepts_resolve_latest_at(store, + c, + concept_ref, + c != NULL ? c->edges.len : 0u, + false, + out_ref); +} + +static bool amduatd_concepts_resolve_latest_at(amduat_asl_store_t *store, + const amduatd_concepts_t *c, + amduat_reference_t concept_ref, + size_t scan_end, + bool include_tombstoned, + amduat_reference_t *out_ref) { size_t i; + const amduatd_ref_edge_bucket_t *src_bucket = NULL; if (out_ref != NULL) { *out_ref = amduat_reference(0u, amduat_octets(NULL, 0u)); @@ -2581,8 +3014,34 @@ static bool amduatd_concepts_resolve_latest(amduat_asl_store_t *store, if (store == NULL || c == NULL || out_ref == NULL) { return false; } + if (scan_end > c->edges.len) { + scan_end = c->edges.len; + } - for (i = c->edges.len; i > 0; --i) { + if (scan_end == c->edges.len && + c->qindex.built_for_edges_len == c->edges.len) { + src_bucket = amduatd_query_index_find_bucket_const(c->qindex.src_buckets, + c->qindex.src_len, + concept_ref); + for (i = src_bucket != NULL ? src_bucket->len : 0u; i > 0; --i) { + size_t edge_i = src_bucket->edge_indices[i - 1u]; + const amduatd_edge_entry_t *entry = &c->edges.items[edge_i]; + if (entry->rel == NULL || strcmp(entry->rel, AMDUATD_REL_MATERIALIZES) != 0) { + continue; + } + if (!include_tombstoned && + amduatd_graph_edge_is_tombstoned(c, entry->record_ref, scan_end)) { + continue; + } + if (amduat_reference_eq(entry->src_ref, concept_ref)) { + amduat_reference_clone(entry->dst_ref, out_ref); + return true; + } + } + return false; + } + + for (i = scan_end; i > 0; --i) { const amduatd_edge_entry_t *entry = &c->edges.items[i - 1u]; if (entry->rel == NULL) { continue; @@ -2590,6 +3049,10 @@ static bool amduatd_concepts_resolve_latest(amduat_asl_store_t *store, if (strcmp(entry->rel, AMDUATD_REL_MATERIALIZES) != 0) { continue; } + if (!include_tombstoned && + amduatd_graph_edge_is_tombstoned(c, entry->record_ref, scan_end)) { + continue; + } if (amduat_reference_eq(entry->src_ref, concept_ref)) { amduat_reference_clone(entry->dst_ref, out_ref); return true; @@ -2664,6 +3127,10 @@ static bool amduatd_strbuf_append(amduatd_strbuf_t *b, static void amduatd_strbuf_free(amduatd_strbuf_t *b); static bool amduatd_strbuf_append_cstr(amduatd_strbuf_t *b, const char *s); static bool amduatd_strbuf_append_char(amduatd_strbuf_t *b, char c); +static bool amduatd_graph_cursor_decode(const char *s, size_t *out_cursor); +static bool amduatd_graph_edge_is_tombstoned(const amduatd_concepts_t *concepts, + amduat_reference_t edge_record_ref, + size_t scan_end); static bool amduatd_handle_get_concepts(int fd, amduat_asl_store_t *store, @@ -2784,7 +3251,9 @@ static bool amduatd_handle_get_relations(int fd, !amduatd_append_relation_entry(&b, &first, "ms.computed_by", concepts->rel_computed_by_ref) || !amduatd_append_relation_entry(&b, &first, "ms.has_provenance", - concepts->rel_has_provenance_ref)) { + concepts->rel_has_provenance_ref) || + !amduatd_append_relation_entry(&b, &first, "ms.tombstones", + concepts->rel_tombstones_ref)) { amduatd_strbuf_free(&b); return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); } @@ -2801,7 +3270,8 @@ static bool amduatd_handle_get_concept(int fd, amduat_asl_store_t *store, const amduatd_concepts_t *concepts, const amduatd_cfg_t *dcfg, - const char *name) { + const char *name, + const amduatd_http_req_t *req) { amduat_reference_t concept_ref; amduat_reference_t latest_ref; amduatd_strbuf_t b; @@ -2813,15 +3283,39 @@ static bool amduatd_handle_get_concept(int fd, bool have_latest = false; size_t i; size_t version_count = 0; + size_t scan_end = 0u; + bool include_tombstoned = false; + char as_of_buf[32]; + char include_tombstoned_buf[16]; memset(&concept_ref, 0, sizeof(concept_ref)); memset(&latest_ref, 0, sizeof(latest_ref)); memset(&b, 0, sizeof(b)); - if (store == NULL || concepts == NULL || name == NULL) { + if (store == NULL || concepts == NULL || name == NULL || req == NULL) { return amduatd_send_json_error(fd, 500, "Internal Server Error", "internal error"); } + if (!amduatd_concepts_ensure_query_index((amduatd_concepts_t *)concepts)) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "index error"); + } + scan_end = concepts->edges.len; + if (amduatd_query_param(req->path, "as_of", as_of_buf, sizeof(as_of_buf)) != NULL) { + if (!amduatd_graph_cursor_decode(as_of_buf, &scan_end)) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid as_of"); + } + if (scan_end > concepts->edges.len) { + scan_end = concepts->edges.len; + } + } + if (amduatd_query_param(req->path, + "include_tombstoned", + include_tombstoned_buf, + sizeof(include_tombstoned_buf)) != NULL) { + if (!amduatd_parse_bool_query(include_tombstoned_buf, &include_tombstoned)) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid include_tombstoned"); + } + } if (!amduatd_space_scope_name(space, name, &scoped_bytes)) { return amduatd_send_json_error(fd, 400, "Bad Request", "invalid name"); } @@ -2834,8 +3328,12 @@ static bool amduatd_handle_get_concept(int fd, } free(scoped_name); - if (amduatd_concepts_resolve_latest(store, concepts, concept_ref, - &latest_ref)) { + if (amduatd_concepts_resolve_latest_at(store, + concepts, + concept_ref, + scan_end, + include_tombstoned, + &latest_ref)) { have_latest = true; } @@ -2876,10 +3374,11 @@ static bool amduatd_handle_get_concept(int fd, } (void)amduatd_strbuf_append_cstr(&b, ",\"versions\":["); - for (i = 0; i < concepts->edges.len; ++i) { + for (i = 0; i < scan_end; ++i) { const amduatd_edge_entry_t *entry = &concepts->edges.items[i]; char *edge_hex = NULL; char *ref_hex = NULL; + bool tombstoned = false; if (entry->rel == NULL || strcmp(entry->rel, AMDUATD_REL_MATERIALIZES) != 0) { continue; @@ -2887,6 +3386,12 @@ static bool amduatd_handle_get_concept(int fd, if (!amduat_reference_eq(entry->src_ref, concept_ref)) { continue; } + tombstoned = amduatd_graph_edge_is_tombstoned(concepts, + entry->record_ref, + scan_end); + if (tombstoned && !include_tombstoned) { + continue; + } if (!amduat_asl_ref_encode_hex(entry->record_ref, &edge_hex) || !amduat_asl_ref_encode_hex(entry->dst_ref, &ref_hex)) { free(edge_hex); @@ -2903,7 +3408,9 @@ static bool amduatd_handle_get_concept(int fd, (void)amduatd_strbuf_append_cstr(&b, edge_hex); (void)amduatd_strbuf_append_cstr(&b, "\",\"ref\":\""); (void)amduatd_strbuf_append_cstr(&b, ref_hex); - (void)amduatd_strbuf_append_cstr(&b, "\"}"); + (void)amduatd_strbuf_append_cstr(&b, "\",\"tombstoned\":"); + (void)amduatd_strbuf_append_cstr(&b, tombstoned ? "true" : "false"); + (void)amduatd_strbuf_append_cstr(&b, "}"); free(edge_hex); free(ref_hex); @@ -2926,6 +3433,210 @@ static bool amduatd_handle_get_concept(int fd, } } +static bool amduatd_handle_get_graph_history(int fd, + amduat_asl_store_t *store, + const amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const char *name, + const amduatd_http_req_t *req) { + amduat_reference_t concept_ref; + amduat_reference_t latest_ref; + amduatd_strbuf_t b; + char *concept_hex = NULL; + char *latest_hex = NULL; + char *scoped_name = NULL; + amduat_octets_t scoped_bytes = amduat_octets(NULL, 0u); + const amduatd_space_t *space = dcfg != NULL ? &dcfg->space : NULL; + bool have_latest = false; + size_t i; + size_t event_count = 0u; + size_t scan_end = 0u; + char as_of_buf[32]; + char include_tombstoned_buf[16]; + bool include_tombstoned = false; + + memset(&concept_ref, 0, sizeof(concept_ref)); + memset(&latest_ref, 0, sizeof(latest_ref)); + memset(&b, 0, sizeof(b)); + + if (store == NULL || concepts == NULL || name == NULL || req == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", + "internal error"); + } + if (!amduatd_concepts_ensure_query_index((amduatd_concepts_t *)concepts)) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "index error"); + } + scan_end = concepts->edges.len; + if (amduatd_query_param(req->path, "as_of", as_of_buf, sizeof(as_of_buf)) != NULL) { + if (!amduatd_graph_cursor_decode(as_of_buf, &scan_end)) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid as_of"); + } + if (scan_end > concepts->edges.len) { + scan_end = concepts->edges.len; + } + } + if (amduatd_query_param(req->path, + "include_tombstoned", + include_tombstoned_buf, + sizeof(include_tombstoned_buf)) != NULL) { + if (!amduatd_parse_bool_query(include_tombstoned_buf, &include_tombstoned)) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid include_tombstoned"); + } + } + if (!amduatd_space_scope_name(space, name, &scoped_bytes)) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid name"); + } + scoped_name = (char *)scoped_bytes.data; + amduatd_space_log_mapping(space, name, scoped_name); + if (!amduatd_concepts_lookup_alias(store, concepts, scoped_name, + &concept_ref)) { + free(scoped_name); + return amduatd_send_json_error(fd, 404, "Not Found", "unknown concept"); + } + free(scoped_name); + + if (amduatd_concepts_resolve_latest_at(store, + concepts, + concept_ref, + scan_end, + include_tombstoned, + &latest_ref)) { + have_latest = true; + } + + if (!amduat_asl_ref_encode_hex(concept_ref, &concept_hex)) { + amduat_reference_free(&concept_ref); + amduat_reference_free(&latest_ref); + return amduatd_send_json_error(fd, 500, "Internal Server Error", + "encode error"); + } + if (have_latest) { + if (!amduat_asl_ref_encode_hex(latest_ref, &latest_hex)) { + free(concept_hex); + amduat_reference_free(&concept_ref); + amduat_reference_free(&latest_ref); + return amduatd_send_json_error(fd, 500, "Internal Server Error", + "encode error"); + } + } + + if (!amduatd_strbuf_append_cstr(&b, "{")) { + free(concept_hex); + free(latest_hex); + amduat_reference_free(&concept_ref); + amduat_reference_free(&latest_ref); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + (void)amduatd_strbuf_append_cstr(&b, "\"name\":\""); + (void)amduatd_strbuf_append_cstr(&b, name); + (void)amduatd_strbuf_append_cstr(&b, "\",\"concept_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, concept_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"latest_ref\":"); + if (latest_hex != NULL) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, latest_hex); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + (void)amduatd_strbuf_append_cstr(&b, ",\"events\":["); + + for (i = 0; i < scan_end; ++i) { + const amduatd_edge_entry_t *entry = &concepts->edges.items[i]; + bool outgoing = false; + bool incoming = false; + bool edge_tombstoned = false; + const char *event_name = NULL; + const char *direction = NULL; + amduat_reference_t event_ref; + char *event_ref_hex = NULL; + char *edge_hex = NULL; + outgoing = amduat_reference_eq(entry->src_ref, concept_ref); + incoming = amduat_reference_eq(entry->dst_ref, concept_ref); + if (!outgoing && !incoming) { + continue; + } + edge_tombstoned = amduatd_graph_edge_is_tombstoned(concepts, + entry->record_ref, + scan_end); + if (edge_tombstoned && !include_tombstoned) { + continue; + } + + memset(&event_ref, 0, sizeof(event_ref)); + if (outgoing && + entry->rel != NULL && + strcmp(entry->rel, AMDUATD_REL_MATERIALIZES) == 0) { + event_name = "version_published"; + direction = "outgoing"; + if (!amduat_reference_clone(entry->dst_ref, &event_ref)) { + continue; + } + } else if (outgoing) { + event_name = "edge_outgoing"; + direction = "outgoing"; + if (!amduat_reference_clone(entry->dst_ref, &event_ref)) { + continue; + } + } else { + event_name = "edge_incoming"; + direction = "incoming"; + if (!amduat_reference_clone(entry->src_ref, &event_ref)) { + continue; + } + } + + if (!amduat_asl_ref_encode_hex(event_ref, &event_ref_hex) || + !amduat_asl_ref_encode_hex(entry->record_ref, &edge_hex)) { + amduat_reference_free(&event_ref); + free(event_ref_hex); + free(edge_hex); + continue; + } + amduat_reference_free(&event_ref); + + if (event_count != 0u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + event_count++; + (void)amduatd_strbuf_append_cstr(&b, "{\"event\":\""); + (void)amduatd_strbuf_append_cstr(&b, event_name); + (void)amduatd_strbuf_append_cstr(&b, "\",\"at_ms\":"); + { + char at_buf[32]; + int at_len = snprintf(at_buf, sizeof(at_buf), "%llu", + (unsigned long long)event_count); + if (at_len > 0 && (size_t)at_len < sizeof(at_buf)) { + (void)amduatd_strbuf_append(&b, at_buf, (size_t)at_len); + } + } + (void)amduatd_strbuf_append_cstr(&b, ",\"ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, event_ref_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"edge_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, edge_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"details\":{\"direction\":\""); + (void)amduatd_strbuf_append_cstr(&b, direction); + (void)amduatd_strbuf_append_cstr(&b, "\",\"predicate\":\""); + (void)amduatd_strbuf_append_cstr(&b, entry->rel != NULL ? entry->rel : ""); + (void)amduatd_strbuf_append_cstr(&b, "\"}}"); + + free(event_ref_hex); + free(edge_hex); + } + + (void)amduatd_strbuf_append_cstr(&b, "]}\n"); + free(concept_hex); + free(latest_hex); + amduat_reference_free(&concept_ref); + amduat_reference_free(&latest_ref); + + { + bool ok = amduatd_http_send_json(fd, 200, "OK", b.data, false); + amduatd_strbuf_free(&b); + return ok; + } +} + static bool amduatd_decode_ref_hex_str(const char *s, size_t len, amduat_reference_t *out_ref) { @@ -3661,6 +4372,262 @@ seed_cleanup: return ok; } +typedef struct { + bool present; + char *source_uri; + char *extractor; + char *confidence; + uint64_t observed_at; + bool have_observed_at; + uint64_t ingested_at; + bool have_ingested_at; + char *license; + char *trace_id; +} amduatd_provenance_input_t; + +static void amduatd_provenance_input_free(amduatd_provenance_input_t *p) { + if (p == NULL) { + return; + } + free(p->source_uri); + free(p->extractor); + free(p->confidence); + free(p->license); + free(p->trace_id); + memset(p, 0, sizeof(*p)); +} + +static bool amduatd_json_parse_provenance_input(const char **p, + const char *end, + amduatd_provenance_input_t *out, + const char **out_error) { + if (out_error != NULL) { + *out_error = "invalid provenance"; + } + if (p == NULL || *p == NULL || out == NULL) { + return false; + } + if (!amduatd_json_expect(p, end, '{')) { + if (out_error != NULL) { + *out_error = "invalid provenance"; + } + return false; + } + for (;;) { + const char *k = NULL; + size_t k_len = 0u; + const char *sv = NULL; + size_t sv_len = 0u; + const char *cur = amduatd_json_skip_ws(*p, end); + if (cur < end && *cur == '}') { + *p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(p, end, &k, &k_len) || + !amduatd_json_expect(p, end, ':')) { + if (out_error != NULL) { + *out_error = "invalid provenance"; + } + return false; + } + if (k_len == strlen("source_uri") && + memcmp(k, "source_uri", k_len) == 0) { + if (out->source_uri != NULL || + !amduatd_json_parse_string_noesc(p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &out->source_uri)) { + if (out_error != NULL) { + *out_error = "invalid provenance.source_uri"; + } + return false; + } + } else if (k_len == strlen("extractor") && + memcmp(k, "extractor", k_len) == 0) { + if (out->extractor != NULL || + !amduatd_json_parse_string_noesc(p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &out->extractor)) { + if (out_error != NULL) { + *out_error = "invalid provenance.extractor"; + } + return false; + } + } else if (k_len == strlen("confidence") && + memcmp(k, "confidence", k_len) == 0) { + uint64_t conf_u64 = 0u; + if (out->confidence != NULL) { + if (out_error != NULL) { + *out_error = "invalid provenance.confidence"; + } + return false; + } + cur = amduatd_json_skip_ws(*p, end); + if (cur < end && *cur == '"') { + if (!amduatd_json_parse_string_noesc(p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &out->confidence)) { + if (out_error != NULL) { + *out_error = "invalid provenance.confidence"; + } + return false; + } + } else if (amduatd_json_parse_u64(p, end, &conf_u64)) { + char tmp[32]; + int n = snprintf(tmp, sizeof(tmp), "%llu", (unsigned long long)conf_u64); + if (n <= 0 || (size_t)n >= sizeof(tmp) || + (out->confidence = strdup(tmp)) == NULL) { + if (out_error != NULL) { + *out_error = "invalid provenance.confidence"; + } + return false; + } + } else { + if (out_error != NULL) { + *out_error = "invalid provenance.confidence"; + } + return false; + } + } else if (k_len == strlen("observed_at") && + memcmp(k, "observed_at", k_len) == 0) { + if (out->have_observed_at || + !amduatd_json_parse_u64(p, end, &out->observed_at)) { + if (out_error != NULL) { + *out_error = "invalid provenance.observed_at"; + } + return false; + } + out->have_observed_at = true; + } else if (k_len == strlen("ingested_at") && + memcmp(k, "ingested_at", k_len) == 0) { + if (out->have_ingested_at || + !amduatd_json_parse_u64(p, end, &out->ingested_at)) { + if (out_error != NULL) { + *out_error = "invalid provenance.ingested_at"; + } + return false; + } + out->have_ingested_at = true; + } else if (k_len == strlen("license") && + memcmp(k, "license", k_len) == 0) { + if (out->license != NULL || + !amduatd_json_parse_string_noesc(p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &out->license)) { + if (out_error != NULL) { + *out_error = "invalid provenance.license"; + } + return false; + } + } else if (k_len == strlen("trace_id") && + memcmp(k, "trace_id", k_len) == 0) { + if (out->trace_id != NULL || + !amduatd_json_parse_string_noesc(p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &out->trace_id)) { + if (out_error != NULL) { + *out_error = "invalid provenance.trace_id"; + } + return false; + } + } else if (!amduatd_json_skip_value(p, end, 0)) { + if (out_error != NULL) { + *out_error = "invalid provenance"; + } + return false; + } + cur = amduatd_json_skip_ws(*p, end); + if (cur < end && *cur == ',') { + *p = cur + 1; + continue; + } + if (cur < end && *cur == '}') { + *p = cur + 1; + break; + } + if (out_error != NULL) { + *out_error = "invalid provenance"; + } + return false; + } + out->present = true; + if (out->source_uri == NULL || out->extractor == NULL || + !out->have_observed_at || !out->have_ingested_at || + out->trace_id == NULL) { + if (out_error != NULL) { + *out_error = "missing provenance required fields"; + } + return false; + } + return true; +} + +static bool amduatd_provenance_store(amduat_asl_store_t *store, + const amduatd_provenance_input_t *prov, + amduat_reference_t *out_ref) { + amduatd_strbuf_t b; + amduat_artifact_t artifact; + char num_buf[64]; + int n; + if (out_ref != NULL) { + *out_ref = amduat_reference(0u, amduat_octets(NULL, 0u)); + } + if (store == NULL || prov == NULL || out_ref == NULL || !prov->present) { + return false; + } + memset(&b, 0, sizeof(b)); + if (!amduatd_strbuf_append_cstr(&b, "{\"source_uri\":\"") || + !amduatd_strbuf_append_cstr(&b, prov->source_uri) || + !amduatd_strbuf_append_cstr(&b, "\",\"extractor\":\"") || + !amduatd_strbuf_append_cstr(&b, prov->extractor) || + !amduatd_strbuf_append_cstr(&b, "\",\"observed_at\":")) { + amduatd_strbuf_free(&b); + return false; + } + n = snprintf(num_buf, sizeof(num_buf), "%llu", (unsigned long long)prov->observed_at); + if (n <= 0 || (size_t)n >= sizeof(num_buf) || + !amduatd_strbuf_append(&b, num_buf, (size_t)n) || + !amduatd_strbuf_append_cstr(&b, ",\"ingested_at\":")) { + amduatd_strbuf_free(&b); + return false; + } + n = snprintf(num_buf, sizeof(num_buf), "%llu", (unsigned long long)prov->ingested_at); + if (n <= 0 || (size_t)n >= sizeof(num_buf) || + !amduatd_strbuf_append(&b, num_buf, (size_t)n)) { + amduatd_strbuf_free(&b); + return false; + } + if (prov->confidence != NULL) { + if (!amduatd_strbuf_append_cstr(&b, ",\"confidence\":\"") || + !amduatd_strbuf_append_cstr(&b, prov->confidence) || + !amduatd_strbuf_append_cstr(&b, "\"")) { + amduatd_strbuf_free(&b); + return false; + } + } + if (prov->license != NULL) { + if (!amduatd_strbuf_append_cstr(&b, ",\"license\":\"") || + !amduatd_strbuf_append_cstr(&b, prov->license) || + !amduatd_strbuf_append_cstr(&b, "\"")) { + amduatd_strbuf_free(&b); + return false; + } + } + if (!amduatd_strbuf_append_cstr(&b, ",\"trace_id\":\"") || + !amduatd_strbuf_append_cstr(&b, prov->trace_id) || + !amduatd_strbuf_append_cstr(&b, "\"}")) { + amduatd_strbuf_free(&b); + return false; + } + artifact = amduat_artifact(amduat_octets((uint8_t *)b.data, b.len)); + if (amduat_asl_store_put(store, artifact, out_ref) != AMDUAT_ASL_STORE_OK) { + amduatd_strbuf_free(&b); + return false; + } + amduatd_strbuf_free(&b); + return true; +} + +static bool amduatd_resolve_graph_ref(amduat_asl_store_t *store, + const amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const char *text, + amduat_reference_t *out_ref); + static bool amduatd_handle_post_concepts(int fd, amduat_asl_store_t *store, amduatd_concepts_t *concepts, @@ -3879,16 +4846,26 @@ static bool amduatd_handle_post_concept_publish(int fd, amduat_reference_t target_ref; amduat_reference_t concept_ref; amduat_reference_t edge_ref; + amduat_reference_t metadata_target_ref; + amduat_reference_t provenance_edge_ref; bool have_ref = false; + bool have_metadata_ref = false; + bool have_provenance = false; bool ok = false; amduat_octets_t actor = amduat_octets(NULL, 0u); char *scoped_name = NULL; + char *metadata_ref = NULL; + const char *prov_err = NULL; + amduatd_provenance_input_t provenance; amduat_octets_t scoped_bytes = amduat_octets(NULL, 0u); const amduatd_space_t *space = dcfg != NULL ? &dcfg->space : NULL; memset(&target_ref, 0, sizeof(target_ref)); memset(&concept_ref, 0, sizeof(concept_ref)); memset(&edge_ref, 0, sizeof(edge_ref)); + memset(&metadata_target_ref, 0, sizeof(metadata_target_ref)); + memset(&provenance_edge_ref, 0, sizeof(provenance_edge_ref)); + memset(&provenance, 0, sizeof(provenance)); if (store == NULL || concepts == NULL || name == NULL || req == NULL) { return amduatd_send_json_error(fd, 500, "Internal Server Error", @@ -3962,6 +4939,26 @@ static bool amduatd_handle_post_concept_publish(int fd, goto publish_cleanup; } have_ref = true; + } else if (key_len == strlen("metadata_ref") && + memcmp(key, "metadata_ref", key_len) == 0) { + if (have_metadata_ref || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &metadata_ref)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid metadata_ref"); + goto publish_cleanup; + } + have_metadata_ref = true; + } else if (key_len == strlen("provenance") && + memcmp(key, "provenance", key_len) == 0) { + if (have_provenance || + !amduatd_json_parse_provenance_input(&p, end, &provenance, &prov_err)) { + ok = amduatd_send_json_error(fd, + 400, + "Bad Request", + prov_err != NULL ? prov_err : "invalid provenance"); + goto publish_cleanup; + } + have_provenance = true; } else { if (!amduatd_json_skip_value(&p, end, 0)) { ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); @@ -3984,6 +4981,25 @@ static bool amduatd_handle_post_concept_publish(int fd, ok = amduatd_send_json_error(fd, 400, "Bad Request", "missing ref"); goto publish_cleanup; } + if (have_metadata_ref && have_provenance) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "metadata_ref and provenance are mutually exclusive"); + goto publish_cleanup; + } + { + int validation_status = 200; + const char *validation_error = NULL; + if (!amduatd_graph_schema_validate_provenance_write(have_metadata_ref || have_provenance, + &validation_status, + &validation_error)) { + ok = amduatd_send_json_error( + fd, + validation_status, + validation_status == 422 ? "Unprocessable Entity" : "Bad Request", + validation_error != NULL ? validation_error : "invalid provenance"); + goto publish_cleanup; + } + } if (!amduatd_concepts_put_edge(store, concepts, @@ -3996,14 +5012,46 @@ static bool amduatd_handle_post_concept_publish(int fd, "store error"); goto publish_cleanup; } + if (have_provenance) { + if (!amduatd_provenance_store(store, &provenance, &metadata_target_ref)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "store error"); + goto publish_cleanup; + } + have_metadata_ref = true; + } else if (have_metadata_ref) { + if (!amduatd_resolve_graph_ref(store, + concepts, + dcfg, + metadata_ref, + &metadata_target_ref)) { + ok = amduatd_send_json_error(fd, 404, "Not Found", "metadata_ref not found"); + goto publish_cleanup; + } + } + if (have_metadata_ref) { + if (!amduatd_concepts_put_edge(store, + concepts, + edge_ref, + metadata_target_ref, + concepts->rel_has_provenance_ref, + actor, + &provenance_edge_ref)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "store error"); + goto publish_cleanup; + } + } ok = amduatd_http_send_json(fd, 200, "OK", "{\"ok\":true}\n", false); publish_cleanup: free(body); + free(metadata_ref); + amduatd_provenance_input_free(&provenance); amduat_reference_free(&target_ref); amduat_reference_free(&concept_ref); amduat_reference_free(&edge_ref); + amduat_reference_free(&metadata_target_ref); + amduat_reference_free(&provenance_edge_ref); return ok; } @@ -4060,6 +5108,10759 @@ static bool amduatd_handle_get_resolve(int fd, return amduatd_http_send_json(fd, 200, "OK", json, false); } +static bool amduatd_resolve_relation_ref(const amduatd_concepts_t *concepts, + const char *text, + amduat_reference_t *out_ref) { + if (concepts == NULL || text == NULL || out_ref == NULL) { + return false; + } + memset(out_ref, 0, sizeof(*out_ref)); + if (strcmp(text, AMDUATD_REL_ALIAS) == 0 || strcmp(text, "aliases") == 0 || + strcmp(text, "ms.aliases") == 0) { + return amduat_reference_clone(concepts->rel_aliases_ref, out_ref); + } + if (strcmp(text, AMDUATD_REL_MATERIALIZES) == 0 || + strcmp(text, "materializesAs") == 0 || + strcmp(text, "ms.materializes_as") == 0) { + return amduat_reference_clone(concepts->rel_materializes_ref, out_ref); + } + if (strcmp(text, AMDUATD_REL_REPRESENTS) == 0 || + strcmp(text, "ms.represents") == 0) { + return amduat_reference_clone(concepts->rel_represents_ref, out_ref); + } + if (strcmp(text, AMDUATD_REL_REQUIRES_KEY) == 0 || + strcmp(text, "requiresKey") == 0 || + strcmp(text, "ms.requires_key") == 0) { + return amduat_reference_clone(concepts->rel_requires_key_ref, out_ref); + } + if (strcmp(text, AMDUATD_REL_WITHIN_DOMAIN) == 0 || + strcmp(text, "withinDomain") == 0 || + strcmp(text, "ms.within_domain") == 0) { + return amduat_reference_clone(concepts->rel_within_domain_ref, out_ref); + } + if (strcmp(text, AMDUATD_REL_COMPUTED_BY) == 0 || + strcmp(text, "computedBy") == 0 || + strcmp(text, "ms.computed_by") == 0) { + return amduat_reference_clone(concepts->rel_computed_by_ref, out_ref); + } + if (strcmp(text, AMDUATD_REL_HAS_PROVENANCE) == 0 || + strcmp(text, "hasProvenance") == 0 || + strcmp(text, "ms.has_provenance") == 0) { + return amduat_reference_clone(concepts->rel_has_provenance_ref, out_ref); + } + if (strcmp(text, AMDUATD_REL_TOMBSTONES) == 0 || + strcmp(text, "ms.tombstones") == 0) { + return amduat_reference_clone(concepts->rel_tombstones_ref, out_ref); + } + if (amduat_asl_ref_decode_hex(text, out_ref)) { + return true; + } + return false; +} + +typedef enum { + AMDUATD_GRAPH_VALIDATION_OFF = 0, + AMDUATD_GRAPH_VALIDATION_WARN = 1, + AMDUATD_GRAPH_VALIDATION_STRICT = 2 +} amduatd_graph_validation_mode_t; + +typedef enum { + AMDUATD_GRAPH_PROVENANCE_OPTIONAL = 0, + AMDUATD_GRAPH_PROVENANCE_REQUIRED = 1 +} amduatd_graph_provenance_mode_t; + +typedef struct { + amduat_reference_t predicate_ref; + char *domain; + char *range; +} amduatd_graph_schema_predicate_rule_t; + +static amduatd_graph_validation_mode_t g_amduatd_graph_validation_mode = + AMDUATD_GRAPH_VALIDATION_OFF; +static amduatd_graph_provenance_mode_t g_amduatd_graph_provenance_mode = + AMDUATD_GRAPH_PROVENANCE_OPTIONAL; +static amduatd_graph_schema_predicate_rule_t *g_amduatd_graph_schema_rules = NULL; +static size_t g_amduatd_graph_schema_rules_len = 0u; +static size_t g_amduatd_graph_schema_rules_cap = 0u; + +static const char *amduatd_graph_validation_mode_text( + amduatd_graph_validation_mode_t mode) { + switch (mode) { + case AMDUATD_GRAPH_VALIDATION_WARN: + return "warn"; + case AMDUATD_GRAPH_VALIDATION_STRICT: + return "strict"; + case AMDUATD_GRAPH_VALIDATION_OFF: + default: + return "off"; + } +} + +static bool amduatd_graph_validation_mode_parse(const char *text, + amduatd_graph_validation_mode_t *out_mode) { + if (text == NULL || out_mode == NULL) { + return false; + } + if (strcmp(text, "off") == 0) { + *out_mode = AMDUATD_GRAPH_VALIDATION_OFF; + return true; + } + if (strcmp(text, "warn") == 0) { + *out_mode = AMDUATD_GRAPH_VALIDATION_WARN; + return true; + } + if (strcmp(text, "strict") == 0) { + *out_mode = AMDUATD_GRAPH_VALIDATION_STRICT; + return true; + } + return false; +} + +static const char *amduatd_graph_provenance_mode_text( + amduatd_graph_provenance_mode_t mode) { + switch (mode) { + case AMDUATD_GRAPH_PROVENANCE_REQUIRED: + return "required"; + case AMDUATD_GRAPH_PROVENANCE_OPTIONAL: + default: + return "optional"; + } +} + +static bool amduatd_graph_provenance_mode_parse(const char *text, + amduatd_graph_provenance_mode_t *out_mode) { + if (text == NULL || out_mode == NULL) { + return false; + } + if (strcmp(text, "optional") == 0) { + *out_mode = AMDUATD_GRAPH_PROVENANCE_OPTIONAL; + return true; + } + if (strcmp(text, "required") == 0) { + *out_mode = AMDUATD_GRAPH_PROVENANCE_REQUIRED; + return true; + } + return false; +} + +static void amduatd_graph_schema_rules_free( + amduatd_graph_schema_predicate_rule_t *rules, + size_t rules_len) { + size_t i; + if (rules == NULL) { + return; + } + for (i = 0u; i < rules_len; ++i) { + amduat_reference_free(&rules[i].predicate_ref); + free(rules[i].domain); + free(rules[i].range); + } + free(rules); +} + +static void amduatd_graph_schema_reset_globals(void) { + amduatd_graph_schema_rules_free(g_amduatd_graph_schema_rules, + g_amduatd_graph_schema_rules_len); + g_amduatd_graph_schema_rules = NULL; + g_amduatd_graph_schema_rules_len = 0u; + g_amduatd_graph_schema_rules_cap = 0u; + g_amduatd_graph_validation_mode = AMDUATD_GRAPH_VALIDATION_OFF; + g_amduatd_graph_provenance_mode = AMDUATD_GRAPH_PROVENANCE_OPTIONAL; +} + +static bool amduatd_graph_schema_rule_exists( + const amduatd_graph_schema_predicate_rule_t *rules, + size_t rules_len, + amduat_reference_t predicate_ref) { + size_t i; + for (i = 0u; i < rules_len; ++i) { + if (amduat_reference_eq(rules[i].predicate_ref, predicate_ref)) { + return true; + } + } + return false; +} + +static bool amduatd_graph_schema_rule_append( + amduatd_graph_schema_predicate_rule_t **rules, + size_t *rules_len, + size_t *rules_cap, + amduat_reference_t predicate_ref, + const char *domain, + const char *range) { + amduatd_graph_schema_predicate_rule_t *next; + size_t next_cap; + if (rules == NULL || rules_len == NULL || rules_cap == NULL) { + return false; + } + if (amduatd_graph_schema_rule_exists(*rules, *rules_len, predicate_ref)) { + return true; + } + if (*rules_len == *rules_cap) { + next_cap = *rules_cap != 0u ? (*rules_cap * 2u) : 32u; + next = (amduatd_graph_schema_predicate_rule_t *)realloc(*rules, + next_cap * sizeof(*next)); + if (next == NULL) { + return false; + } + *rules = next; + *rules_cap = next_cap; + } + memset(&(*rules)[*rules_len], 0, sizeof((*rules)[*rules_len])); + if (!amduat_reference_clone(predicate_ref, &(*rules)[*rules_len].predicate_ref)) { + return false; + } + if (domain != NULL) { + (*rules)[*rules_len].domain = strdup(domain); + if ((*rules)[*rules_len].domain == NULL) { + amduat_reference_free(&(*rules)[*rules_len].predicate_ref); + return false; + } + } + if (range != NULL) { + (*rules)[*rules_len].range = strdup(range); + if ((*rules)[*rules_len].range == NULL) { + amduat_reference_free(&(*rules)[*rules_len].predicate_ref); + free((*rules)[*rules_len].domain); + (*rules)[*rules_len].domain = NULL; + return false; + } + } + (*rules_len)++; + return true; +} + +static void amduatd_graph_schema_set_rules( + amduatd_graph_validation_mode_t mode, + amduatd_graph_provenance_mode_t provenance_mode, + amduatd_graph_schema_predicate_rule_t *rules, + size_t rules_len, + size_t rules_cap) { + amduatd_graph_schema_rules_free(g_amduatd_graph_schema_rules, + g_amduatd_graph_schema_rules_len); + g_amduatd_graph_schema_rules = rules; + g_amduatd_graph_schema_rules_len = rules_len; + g_amduatd_graph_schema_rules_cap = rules_cap; + g_amduatd_graph_validation_mode = mode; + g_amduatd_graph_provenance_mode = provenance_mode; +} + +static bool amduatd_graph_schema_predicate_allowed( + amduat_reference_t predicate_ref) { + size_t i; + if (g_amduatd_graph_validation_mode == AMDUATD_GRAPH_VALIDATION_OFF) { + return true; + } + if (g_amduatd_graph_schema_rules_len == 0u) { + return true; + } + for (i = 0u; i < g_amduatd_graph_schema_rules_len; ++i) { + if (amduat_reference_eq(g_amduatd_graph_schema_rules[i].predicate_ref, + predicate_ref)) { + return true; + } + } + return false; +} + +static bool amduatd_graph_schema_validate_predicate_write( + amduat_reference_t predicate_ref, + int *out_status, + const char **out_error) { + if (out_status != NULL) { + *out_status = 200; + } + if (out_error != NULL) { + *out_error = NULL; + } + if (amduatd_graph_schema_predicate_allowed(predicate_ref)) { + return true; + } + if (g_amduatd_graph_validation_mode == AMDUATD_GRAPH_VALIDATION_WARN) { + amduat_log(AMDUAT_LOG_WARN, + "graph schema warn: predicate rejected by policy but allowed"); + return true; + } + if (out_status != NULL) { + *out_status = 422; + } + if (out_error != NULL) { + *out_error = "predicate rejected by schema policy"; + } + return false; +} + +static bool amduatd_graph_schema_validate_provenance_write( + bool has_provenance_attachment, + int *out_status, + const char **out_error) { + if (out_status != NULL) { + *out_status = 200; + } + if (out_error != NULL) { + *out_error = NULL; + } + if (g_amduatd_graph_provenance_mode == AMDUATD_GRAPH_PROVENANCE_OPTIONAL) { + return true; + } + if (has_provenance_attachment) { + return true; + } + if (out_status != NULL) { + *out_status = 422; + } + if (out_error != NULL) { + *out_error = "provenance required by schema policy"; + } + return false; +} + +enum { + AMDUATD_GRAPH_SCHEMA_POLICY_MAGIC_LEN = 8, + AMDUATD_GRAPH_SCHEMA_POLICY_VERSION = 2 +}; + +static const uint8_t + k_amduatd_graph_schema_policy_magic[AMDUATD_GRAPH_SCHEMA_POLICY_MAGIC_LEN] = + {'A', 'S', 'L', 'S', 'C', 'H', '1', '\0'}; + +static bool amduatd_graph_schema_pointer_name(const amduatd_space_t *space, + amduat_octets_t *out_name) { + if (out_name == NULL) { + return false; + } + return amduatd_space_scope_name(space, "daemon/graph/schema-policy", out_name); +} + +static bool amduatd_graph_schema_policy_encode( + amduatd_graph_validation_mode_t mode, + amduatd_graph_provenance_mode_t provenance_mode, + const amduatd_graph_schema_predicate_rule_t *rules, + size_t rules_len, + amduat_octets_t *out_payload) { + size_t total_len = 0u; + size_t offset = 0u; + uint8_t *payload = NULL; + size_t i; + + if (out_payload != NULL) { + *out_payload = amduat_octets(NULL, 0u); + } + if (out_payload == NULL || rules_len > UINT32_MAX) { + return false; + } + total_len = AMDUATD_GRAPH_SCHEMA_POLICY_MAGIC_LEN + 4u + 4u + 4u + 4u; + for (i = 0u; i < rules_len; ++i) { + char *predicate_hex = NULL; + size_t predicate_hex_len = 0u; + size_t domain_len = 0u; + size_t range_len = 0u; + if (!amduat_asl_ref_encode_hex(rules[i].predicate_ref, &predicate_hex)) { + return false; + } + predicate_hex_len = strlen(predicate_hex); + if (predicate_hex_len > UINT32_MAX) { + free(predicate_hex); + return false; + } + if (rules[i].domain != NULL) { + domain_len = strlen(rules[i].domain); + if (domain_len > UINT32_MAX) { + free(predicate_hex); + return false; + } + } + if (rules[i].range != NULL) { + range_len = strlen(rules[i].range); + if (range_len > UINT32_MAX) { + free(predicate_hex); + return false; + } + } + if (!amduatd_add_size(&total_len, 4u + predicate_hex_len) || + !amduatd_add_size(&total_len, 4u + domain_len) || + !amduatd_add_size(&total_len, 4u + range_len)) { + free(predicate_hex); + return false; + } + free(predicate_hex); + } + payload = (uint8_t *)malloc(total_len); + if (payload == NULL) { + return false; + } + memcpy(payload + offset, + k_amduatd_graph_schema_policy_magic, + AMDUATD_GRAPH_SCHEMA_POLICY_MAGIC_LEN); + offset += AMDUATD_GRAPH_SCHEMA_POLICY_MAGIC_LEN; + amduatd_store_u32_le(payload + offset, AMDUATD_GRAPH_SCHEMA_POLICY_VERSION); + offset += 4u; + amduatd_store_u32_le(payload + offset, (uint32_t)mode); + offset += 4u; + amduatd_store_u32_le(payload + offset, (uint32_t)provenance_mode); + offset += 4u; + amduatd_store_u32_le(payload + offset, (uint32_t)rules_len); + offset += 4u; + for (i = 0u; i < rules_len; ++i) { + char *predicate_hex = NULL; + size_t predicate_hex_len = 0u; + size_t domain_len = 0u; + size_t range_len = 0u; + uint32_t domain_field_len = UINT32_MAX; + uint32_t range_field_len = UINT32_MAX; + if (!amduat_asl_ref_encode_hex(rules[i].predicate_ref, &predicate_hex)) { + free(payload); + return false; + } + predicate_hex_len = strlen(predicate_hex); + if (rules[i].domain != NULL) { + domain_len = strlen(rules[i].domain); + domain_field_len = (uint32_t)domain_len; + } + if (rules[i].range != NULL) { + range_len = strlen(rules[i].range); + range_field_len = (uint32_t)range_len; + } + amduatd_store_u32_le(payload + offset, (uint32_t)predicate_hex_len); + offset += 4u; + if (predicate_hex_len != 0u) { + memcpy(payload + offset, predicate_hex, predicate_hex_len); + offset += predicate_hex_len; + } + amduatd_store_u32_le(payload + offset, domain_field_len); + offset += 4u; + if (domain_len != 0u) { + memcpy(payload + offset, rules[i].domain, domain_len); + offset += domain_len; + } + amduatd_store_u32_le(payload + offset, range_field_len); + offset += 4u; + if (range_len != 0u) { + memcpy(payload + offset, rules[i].range, range_len); + offset += range_len; + } + free(predicate_hex); + } + if (offset != total_len) { + free(payload); + return false; + } + *out_payload = amduat_octets(payload, total_len); + return true; +} + +static bool amduatd_graph_schema_policy_decode( + amduat_octets_t payload, + amduatd_graph_validation_mode_t *out_mode, + amduatd_graph_provenance_mode_t *out_provenance_mode, + amduatd_graph_schema_predicate_rule_t **out_rules, + size_t *out_rules_len, + size_t *out_rules_cap) { + size_t offset = 0u; + uint32_t version = 0u; + uint32_t mode_u32 = 0u; + uint32_t provenance_mode_u32 = (uint32_t)AMDUATD_GRAPH_PROVENANCE_OPTIONAL; + uint32_t rules_count = 0u; + amduatd_graph_validation_mode_t mode = AMDUATD_GRAPH_VALIDATION_OFF; + amduatd_graph_provenance_mode_t provenance_mode = + AMDUATD_GRAPH_PROVENANCE_OPTIONAL; + amduatd_graph_schema_predicate_rule_t *rules = NULL; + size_t rules_len = 0u; + size_t rules_cap = 0u; + uint32_t i; + if (out_mode != NULL) { + *out_mode = AMDUATD_GRAPH_VALIDATION_OFF; + } + if (out_rules != NULL) { + *out_rules = NULL; + } + if (out_rules_len != NULL) { + *out_rules_len = 0u; + } + if (out_rules_cap != NULL) { + *out_rules_cap = 0u; + } + if (out_provenance_mode != NULL) { + *out_provenance_mode = AMDUATD_GRAPH_PROVENANCE_OPTIONAL; + } + if (payload.len < AMDUATD_GRAPH_SCHEMA_POLICY_MAGIC_LEN + 4u + 4u + 4u || + out_provenance_mode == NULL || + out_mode == NULL || out_rules == NULL || out_rules_len == NULL || + out_rules_cap == NULL) { + return false; + } + if (memcmp(payload.data, + k_amduatd_graph_schema_policy_magic, + AMDUATD_GRAPH_SCHEMA_POLICY_MAGIC_LEN) != 0) { + return false; + } + offset += AMDUATD_GRAPH_SCHEMA_POLICY_MAGIC_LEN; + if (!amduatd_read_u32_le(payload.data, payload.len, &offset, &version) || + !amduatd_read_u32_le(payload.data, payload.len, &offset, &mode_u32)) { + return false; + } + if (version == 1u) { + provenance_mode = AMDUATD_GRAPH_PROVENANCE_OPTIONAL; + if (!amduatd_read_u32_le(payload.data, payload.len, &offset, &rules_count)) { + return false; + } + } else if (version == AMDUATD_GRAPH_SCHEMA_POLICY_VERSION) { + if (!amduatd_read_u32_le(payload.data, payload.len, &offset, &provenance_mode_u32) || + !amduatd_read_u32_le(payload.data, payload.len, &offset, &rules_count)) { + return false; + } + if (provenance_mode_u32 == (uint32_t)AMDUATD_GRAPH_PROVENANCE_OPTIONAL) { + provenance_mode = AMDUATD_GRAPH_PROVENANCE_OPTIONAL; + } else if (provenance_mode_u32 == (uint32_t)AMDUATD_GRAPH_PROVENANCE_REQUIRED) { + provenance_mode = AMDUATD_GRAPH_PROVENANCE_REQUIRED; + } else { + return false; + } + } else { + return false; + } + if (mode_u32 == (uint32_t)AMDUATD_GRAPH_VALIDATION_OFF) { + mode = AMDUATD_GRAPH_VALIDATION_OFF; + } else if (mode_u32 == (uint32_t)AMDUATD_GRAPH_VALIDATION_WARN) { + mode = AMDUATD_GRAPH_VALIDATION_WARN; + } else if (mode_u32 == (uint32_t)AMDUATD_GRAPH_VALIDATION_STRICT) { + mode = AMDUATD_GRAPH_VALIDATION_STRICT; + } else { + return false; + } + for (i = 0u; i < rules_count; ++i) { + uint32_t predicate_hex_len = 0u; + uint32_t domain_len = 0u; + uint32_t range_len = 0u; + char *predicate_hex = NULL; + char *domain = NULL; + char *range = NULL; + amduat_reference_t predicate_ref; + memset(&predicate_ref, 0, sizeof(predicate_ref)); + if (!amduatd_read_u32_le(payload.data, payload.len, &offset, + &predicate_hex_len) || + payload.len - offset < (size_t)predicate_hex_len) { + goto decode_fail; + } + predicate_hex = (char *)malloc((size_t)predicate_hex_len + 1u); + if (predicate_hex == NULL) { + goto decode_fail; + } + if (predicate_hex_len != 0u) { + memcpy(predicate_hex, payload.data + offset, (size_t)predicate_hex_len); + offset += (size_t)predicate_hex_len; + } + predicate_hex[predicate_hex_len] = '\0'; + if (!amduat_asl_ref_decode_hex(predicate_hex, &predicate_ref)) { + free(predicate_hex); + goto decode_fail; + } + free(predicate_hex); + if (!amduatd_read_u32_le(payload.data, payload.len, &offset, &domain_len)) { + amduat_reference_free(&predicate_ref); + goto decode_fail; + } + if (domain_len != UINT32_MAX) { + if (payload.len - offset < (size_t)domain_len) { + amduat_reference_free(&predicate_ref); + goto decode_fail; + } + domain = (char *)malloc((size_t)domain_len + 1u); + if (domain == NULL) { + amduat_reference_free(&predicate_ref); + goto decode_fail; + } + if (domain_len != 0u) { + memcpy(domain, payload.data + offset, (size_t)domain_len); + offset += (size_t)domain_len; + } + domain[domain_len] = '\0'; + } + if (!amduatd_read_u32_le(payload.data, payload.len, &offset, &range_len)) { + free(domain); + amduat_reference_free(&predicate_ref); + goto decode_fail; + } + if (range_len != UINT32_MAX) { + if (payload.len - offset < (size_t)range_len) { + free(domain); + amduat_reference_free(&predicate_ref); + goto decode_fail; + } + range = (char *)malloc((size_t)range_len + 1u); + if (range == NULL) { + free(domain); + amduat_reference_free(&predicate_ref); + goto decode_fail; + } + if (range_len != 0u) { + memcpy(range, payload.data + offset, (size_t)range_len); + offset += (size_t)range_len; + } + range[range_len] = '\0'; + } + if (!amduatd_graph_schema_rule_append(&rules, + &rules_len, + &rules_cap, + predicate_ref, + domain, + range)) { + free(domain); + free(range); + amduat_reference_free(&predicate_ref); + goto decode_fail; + } + free(domain); + free(range); + amduat_reference_free(&predicate_ref); + } + if (offset != payload.len) { + goto decode_fail; + } + *out_mode = mode; + *out_provenance_mode = provenance_mode; + *out_rules = rules; + *out_rules_len = rules_len; + *out_rules_cap = rules_cap; + return true; + +decode_fail: + amduatd_graph_schema_rules_free(rules, rules_len); + return false; +} + +static bool amduatd_graph_schema_load_persistent( + amduat_asl_store_t *store, + amduatd_concepts_t *concepts, + const amduatd_space_t *space, + amduatd_graph_validation_mode_t *out_mode, + amduatd_graph_provenance_mode_t *out_provenance_mode, + amduatd_graph_schema_predicate_rule_t **out_rules, + size_t *out_rules_len, + size_t *out_rules_cap, + bool *out_exists) { + amduat_octets_t pointer_name = amduat_octets(NULL, 0u); + amduat_reference_t pointer_ref; + amduat_asl_record_t record; + bool exists = false; + bool ok = false; + if (out_exists != NULL) { + *out_exists = false; + } + if (out_mode != NULL) { + *out_mode = AMDUATD_GRAPH_VALIDATION_OFF; + } + if (out_provenance_mode != NULL) { + *out_provenance_mode = AMDUATD_GRAPH_PROVENANCE_OPTIONAL; + } + if (out_rules != NULL) { + *out_rules = NULL; + } + if (out_rules_len != NULL) { + *out_rules_len = 0u; + } + if (out_rules_cap != NULL) { + *out_rules_cap = 0u; + } + if (store == NULL || concepts == NULL || out_mode == NULL || out_rules == NULL || + out_rules_len == NULL || out_rules_cap == NULL || out_exists == NULL) { + return false; + } + if (!amduatd_graph_schema_pointer_name(space, &pointer_name)) { + return false; + } + memset(&pointer_ref, 0, sizeof(pointer_ref)); + if (amduat_asl_pointer_get(&concepts->edge_collection.pointer_store, + (const char *)pointer_name.data, + &exists, + &pointer_ref) != AMDUAT_ASL_POINTER_OK) { + amduat_octets_free(&pointer_name); + return false; + } + amduat_octets_free(&pointer_name); + if (!exists) { + return true; + } + memset(&record, 0, sizeof(record)); + if (amduat_asl_record_store_get(store, pointer_ref, &record) != + AMDUAT_ASL_STORE_OK) { + amduat_reference_free(&pointer_ref); + return false; + } + amduat_reference_free(&pointer_ref); + if (record.schema.len != strlen(AMDUATD_GRAPH_SCHEMA_POLICY_SCHEMA) || + memcmp(record.schema.data, + AMDUATD_GRAPH_SCHEMA_POLICY_SCHEMA, + record.schema.len) != 0) { + amduat_asl_record_free(&record); + return false; + } + ok = amduatd_graph_schema_policy_decode(record.payload, + out_mode, + out_provenance_mode, + out_rules, + out_rules_len, + out_rules_cap); + amduat_asl_record_free(&record); + if (!ok) { + return false; + } + *out_exists = true; + return true; +} + +static bool amduatd_graph_schema_store_persistent( + amduat_asl_store_t *store, + amduatd_concepts_t *concepts, + const amduatd_space_t *space, + amduatd_graph_validation_mode_t mode, + amduatd_graph_provenance_mode_t provenance_mode, + const amduatd_graph_schema_predicate_rule_t *rules, + size_t rules_len) { + amduat_octets_t payload = amduat_octets(NULL, 0u); + amduat_octets_t pointer_name = amduat_octets(NULL, 0u); + amduat_reference_t record_ref; + amduat_reference_t expected_ref; + bool expected_exists = false; + bool swapped = false; + if (store == NULL || concepts == NULL) { + return false; + } + memset(&record_ref, 0, sizeof(record_ref)); + memset(&expected_ref, 0, sizeof(expected_ref)); + if (!amduatd_graph_schema_policy_encode(mode, provenance_mode, rules, rules_len, &payload)) { + return false; + } + if (amduat_asl_record_store_put( + store, + amduat_octets(AMDUATD_GRAPH_SCHEMA_POLICY_SCHEMA, + strlen(AMDUATD_GRAPH_SCHEMA_POLICY_SCHEMA)), + payload, + &record_ref) != AMDUAT_ASL_STORE_OK) { + free((void *)payload.data); + return false; + } + free((void *)payload.data); + if (!amduatd_graph_schema_pointer_name(space, &pointer_name)) { + amduat_reference_free(&record_ref); + return false; + } + if (amduat_asl_pointer_get(&concepts->edge_collection.pointer_store, + (const char *)pointer_name.data, + &expected_exists, + &expected_ref) != AMDUAT_ASL_POINTER_OK) { + amduat_octets_free(&pointer_name); + amduat_reference_free(&record_ref); + return false; + } + if (amduat_asl_pointer_cas(&concepts->edge_collection.pointer_store, + (const char *)pointer_name.data, + expected_exists, + expected_exists ? &expected_ref : NULL, + &record_ref, + &swapped) != AMDUAT_ASL_POINTER_OK) { + amduat_octets_free(&pointer_name); + amduat_reference_free(&expected_ref); + amduat_reference_free(&record_ref); + return false; + } + amduat_octets_free(&pointer_name); + amduat_reference_free(&expected_ref); + amduat_reference_free(&record_ref); + return swapped; +} + +static bool amduatd_graph_schema_load_into_globals(amduat_asl_store_t *store, + amduatd_concepts_t *concepts, + const amduatd_space_t *space) { + amduatd_graph_validation_mode_t loaded_mode = AMDUATD_GRAPH_VALIDATION_OFF; + amduatd_graph_provenance_mode_t loaded_provenance_mode = + AMDUATD_GRAPH_PROVENANCE_OPTIONAL; + amduatd_graph_schema_predicate_rule_t *loaded_rules = NULL; + size_t loaded_rules_len = 0u; + size_t loaded_rules_cap = 0u; + bool loaded_exists = false; + if (!amduatd_graph_schema_load_persistent(store, + concepts, + space, + &loaded_mode, + &loaded_provenance_mode, + &loaded_rules, + &loaded_rules_len, + &loaded_rules_cap, + &loaded_exists)) { + amduatd_graph_schema_rules_free(loaded_rules, loaded_rules_len); + return false; + } + if (loaded_exists) { + amduatd_graph_schema_set_rules(loaded_mode, + loaded_provenance_mode, + loaded_rules, + loaded_rules_len, + loaded_rules_cap); + } else { + amduatd_graph_schema_rules_free(loaded_rules, loaded_rules_len); + } + return true; +} + +static bool amduatd_handle_get_graph_schema_predicates(int fd, + const amduatd_concepts_t *concepts) { + amduatd_strbuf_t b; + size_t i; + bool ok = false; + (void)concepts; + memset(&b, 0, sizeof(b)); + if (!amduatd_strbuf_append_cstr(&b, "{\"mode\":\"") || + !amduatd_strbuf_append_cstr(&b, amduatd_graph_validation_mode_text( + g_amduatd_graph_validation_mode)) || + !amduatd_strbuf_append_cstr(&b, "\",\"provenance_mode\":\"") || + !amduatd_strbuf_append_cstr(&b, amduatd_graph_provenance_mode_text( + g_amduatd_graph_provenance_mode)) || + !amduatd_strbuf_append_cstr(&b, "\",\"predicates\":[")) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + for (i = 0u; i < g_amduatd_graph_schema_rules_len; ++i) { + char *predicate_hex = NULL; + if (!amduat_asl_ref_encode_hex(g_amduatd_graph_schema_rules[i].predicate_ref, + &predicate_hex)) { + amduatd_strbuf_free(&b); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + } + if (i != 0u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + (void)amduatd_strbuf_append_cstr(&b, "{\"predicate_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, predicate_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"domain\":"); + if (g_amduatd_graph_schema_rules[i].domain != NULL) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, g_amduatd_graph_schema_rules[i].domain); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + (void)amduatd_strbuf_append_cstr(&b, ",\"range\":"); + if (g_amduatd_graph_schema_rules[i].range != NULL) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, g_amduatd_graph_schema_rules[i].range); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + (void)amduatd_strbuf_append_cstr(&b, "}"); + free(predicate_hex); + } + (void)amduatd_strbuf_append_cstr(&b, "]}\n"); + ok = amduatd_http_send_json(fd, 200, "OK", b.data, false); + amduatd_strbuf_free(&b); + return ok; +} + +static bool amduatd_handle_post_graph_schema_predicates(int fd, + amduat_asl_store_t *store, + amduatd_concepts_t *concepts, + const amduatd_http_req_t *req) { + uint8_t *body = NULL; + const char *p = NULL; + const char *end = NULL; + amduatd_graph_validation_mode_t new_mode = g_amduatd_graph_validation_mode; + amduatd_graph_provenance_mode_t new_provenance_mode = + g_amduatd_graph_provenance_mode; + amduatd_graph_schema_predicate_rule_t *new_rules = NULL; + size_t new_rules_len = 0u; + size_t new_rules_cap = 0u; + bool have_predicates = false; + bool ok = false; + + if (store == NULL || concepts == NULL || req == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "internal error"); + } + if (req->content_length == 0u) { + return amduatd_send_json_error(fd, 400, "Bad Request", "missing body"); + } + if (req->content_length > (256u * 1024u)) { + return amduatd_send_json_error(fd, 413, "Payload Too Large", "payload too large"); + } + body = (uint8_t *)malloc(req->content_length); + if (body == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + if (!amduatd_read_exact(fd, body, req->content_length)) { + free(body); + return false; + } + p = (const char *)body; + end = (const char *)body + req->content_length; + if (!amduatd_json_expect(&p, end, '{')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + for (;;) { + const char *key = NULL; + size_t key_len = 0u; + const char *cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &key, &key_len) || + !amduatd_json_expect(&p, end, ':')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + if (key_len == strlen("mode") && memcmp(key, "mode", key_len) == 0) { + const char *sv = NULL; + size_t sv_len = 0u; + char *mode_text = NULL; + if (!amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &mode_text) || + !amduatd_graph_validation_mode_parse(mode_text, &new_mode)) { + free(mode_text); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid mode"); + goto cleanup; + } + free(mode_text); + } else if (key_len == strlen("provenance_mode") && + memcmp(key, "provenance_mode", key_len) == 0) { + const char *sv = NULL; + size_t sv_len = 0u; + char *mode_text = NULL; + if (!amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &mode_text) || + !amduatd_graph_provenance_mode_parse(mode_text, &new_provenance_mode)) { + free(mode_text); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid provenance_mode"); + goto cleanup; + } + free(mode_text); + } else if (key_len == strlen("predicates") && memcmp(key, "predicates", key_len) == 0) { + have_predicates = true; + if (!amduatd_json_expect(&p, end, '[')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicates"); + goto cleanup; + } + for (;;) { + amduat_reference_t predicate_ref; + char *domain = NULL; + char *range = NULL; + const char *sv = NULL; + size_t sv_len = 0u; + memset(&predicate_ref, 0, sizeof(predicate_ref)); + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ']') { + p = cur + 1; + break; + } + if (!amduatd_json_expect(&p, end, '{')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicates"); + free(domain); + free(range); + goto cleanup; + } + for (;;) { + const char *ik = NULL; + size_t ik_len = 0u; + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &ik, &ik_len) || + !amduatd_json_expect(&p, end, ':')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicates"); + free(domain); + free(range); + amduat_reference_free(&predicate_ref); + goto cleanup; + } + if (ik_len == strlen("predicate_ref") && + memcmp(ik, "predicate_ref", ik_len) == 0) { + char *value = NULL; + if (!amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &value) || + !amduat_asl_ref_decode_hex(value, &predicate_ref)) { + free(value); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicate_ref"); + free(domain); + free(range); + amduat_reference_free(&predicate_ref); + goto cleanup; + } + free(value); + } else if (ik_len == strlen("predicate") && + memcmp(ik, "predicate", ik_len) == 0) { + char *value = NULL; + if (!amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &value) || + !amduatd_resolve_relation_ref(concepts, value, &predicate_ref)) { + free(value); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicate"); + free(domain); + free(range); + amduat_reference_free(&predicate_ref); + goto cleanup; + } + free(value); + } else if (ik_len == strlen("domain") && + memcmp(ik, "domain", ik_len) == 0) { + if (domain != NULL || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &domain)) { + free(domain); + free(range); + amduat_reference_free(&predicate_ref); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid domain"); + goto cleanup; + } + } else if (ik_len == strlen("range") && + memcmp(ik, "range", ik_len) == 0) { + if (range != NULL || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &range)) { + free(domain); + free(range); + amduat_reference_free(&predicate_ref); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid range"); + goto cleanup; + } + } else { + if (!amduatd_json_skip_value(&p, end, 0u)) { + free(domain); + free(range); + amduat_reference_free(&predicate_ref); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicates"); + goto cleanup; + } + } + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + free(domain); + free(range); + amduat_reference_free(&predicate_ref); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicates"); + goto cleanup; + } + if (predicate_ref.hash_id == 0 || predicate_ref.digest.data == NULL) { + free(domain); + free(range); + amduat_reference_free(&predicate_ref); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "missing predicate"); + goto cleanup; + } + if (!amduatd_graph_schema_rule_append(&new_rules, + &new_rules_len, + &new_rules_cap, + predicate_ref, + domain, + range)) { + free(domain); + free(range); + amduat_reference_free(&predicate_ref); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + free(domain); + free(range); + amduat_reference_free(&predicate_ref); + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == ']') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicates"); + goto cleanup; + } + } else { + if (!amduatd_json_skip_value(&p, end, 0u)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + } + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + p = amduatd_json_skip_ws(p, end); + if (p != end) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + + if (!have_predicates) { + new_rules = NULL; + new_rules_len = 0u; + new_rules_cap = 0u; + } + if (!amduatd_graph_schema_store_persistent(store, + concepts, + req->effective_space, + new_mode, + new_provenance_mode, + new_rules, + new_rules_len)) { + ok = amduatd_send_json_error(fd, + 500, + "Internal Server Error", + "schema policy persist failed"); + goto cleanup; + } + amduatd_graph_schema_set_rules(new_mode, + new_provenance_mode, + new_rules, + new_rules_len, + new_rules_cap); + new_rules = NULL; + new_rules_len = 0u; + new_rules_cap = 0u; + + ok = amduatd_handle_get_graph_schema_predicates(fd, concepts); + +cleanup: + free(body); + amduatd_graph_schema_rules_free(new_rules, new_rules_len); + return ok; +} + +static bool amduatd_resolve_graph_ref(amduat_asl_store_t *store, + const amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const char *text, + amduat_reference_t *out_ref) { + amduat_octets_t scoped = amduat_octets(NULL, 0u); + bool ok = false; + if (store == NULL || concepts == NULL || text == NULL || out_ref == NULL) { + return false; + } + memset(out_ref, 0, sizeof(*out_ref)); + if (amduat_asl_ref_decode_hex(text, out_ref)) { + return true; + } + if (!amduatd_space_scope_name(dcfg != NULL ? &dcfg->space : NULL, + text, + &scoped)) { + return false; + } + ok = amduatd_concepts_lookup_alias(store, concepts, (const char *)scoped.data, out_ref); + amduat_octets_free(&scoped); + return ok; +} + +static bool amduatd_relation_entry_ref(const amduatd_concepts_t *concepts, + const char *rel_name, + amduat_reference_t *out_ref) { + if (concepts == NULL || rel_name == NULL || out_ref == NULL) { + return false; + } + memset(out_ref, 0, sizeof(*out_ref)); + if (strcmp(rel_name, AMDUATD_REL_ALIAS) == 0) { + return amduat_reference_clone(concepts->rel_aliases_ref, out_ref); + } + if (strcmp(rel_name, AMDUATD_REL_MATERIALIZES) == 0) { + return amduat_reference_clone(concepts->rel_materializes_ref, out_ref); + } + if (strcmp(rel_name, AMDUATD_REL_REPRESENTS) == 0) { + return amduat_reference_clone(concepts->rel_represents_ref, out_ref); + } + if (strcmp(rel_name, AMDUATD_REL_REQUIRES_KEY) == 0) { + return amduat_reference_clone(concepts->rel_requires_key_ref, out_ref); + } + if (strcmp(rel_name, AMDUATD_REL_WITHIN_DOMAIN) == 0) { + return amduat_reference_clone(concepts->rel_within_domain_ref, out_ref); + } + if (strcmp(rel_name, AMDUATD_REL_COMPUTED_BY) == 0) { + return amduat_reference_clone(concepts->rel_computed_by_ref, out_ref); + } + if (strcmp(rel_name, AMDUATD_REL_HAS_PROVENANCE) == 0) { + return amduat_reference_clone(concepts->rel_has_provenance_ref, out_ref); + } + if (strcmp(rel_name, AMDUATD_REL_TOMBSTONES) == 0) { + return amduat_reference_clone(concepts->rel_tombstones_ref, out_ref); + } + return false; +} + +static bool amduatd_parse_u64_query(const char *s, uint64_t *out) { + unsigned long long v = 0ull; + char *end = NULL; + if (s == NULL || s[0] == '\0' || out == NULL) { + return false; + } + errno = 0; + v = strtoull(s, &end, 10); + if (errno != 0 || end == NULL || *end != '\0') { + return false; + } + *out = (uint64_t)v; + return true; +} + +static bool amduatd_parse_bool_query(const char *s, bool *out) { + if (s == NULL || out == NULL) { + return false; + } + if (strcmp(s, "1") == 0 || strcmp(s, "true") == 0 || + strcmp(s, "yes") == 0 || strcmp(s, "on") == 0) { + *out = true; + return true; + } + if (strcmp(s, "0") == 0 || strcmp(s, "false") == 0 || + strcmp(s, "no") == 0 || strcmp(s, "off") == 0) { + *out = false; + return true; + } + return false; +} + +static bool amduatd_graph_batch_apply_node(amduat_asl_store_t *store, + amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const char *name, + const char *ref_text, + bool has_ref, + amduat_octets_t actor, + int *out_status, + const char **out_error) { + amduat_reference_t target_ref; + amduat_reference_t name_ref; + amduat_reference_t concept_ref; + amduat_reference_t edge_ref; + amduat_reference_t existing_ref; + amduat_octets_t scoped_bytes = amduat_octets(NULL, 0u); + char *scoped_name = NULL; + const amduatd_space_t *space = dcfg != NULL ? &dcfg->space : NULL; + bool ok = false; + + memset(&target_ref, 0, sizeof(target_ref)); + memset(&name_ref, 0, sizeof(name_ref)); + memset(&concept_ref, 0, sizeof(concept_ref)); + memset(&edge_ref, 0, sizeof(edge_ref)); + memset(&existing_ref, 0, sizeof(existing_ref)); + if (out_status != NULL) { + *out_status = 500; + } + if (out_error != NULL) { + *out_error = "store error"; + } + if (store == NULL || concepts == NULL || name == NULL || + (has_ref && ref_text == NULL)) { + if (out_error != NULL) { + *out_error = "internal error"; + } + return false; + } + + if (!amduatd_space_scope_name(space, name, &scoped_bytes)) { + if (out_status != NULL) { + *out_status = 400; + } + if (out_error != NULL) { + *out_error = "invalid name"; + } + return false; + } + scoped_name = (char *)scoped_bytes.data; + amduatd_space_log_mapping(space, name, scoped_name); + + if (has_ref && !amduat_asl_ref_decode_hex(ref_text, &target_ref)) { + if (out_status != NULL) { + *out_status = 400; + } + if (out_error != NULL) { + *out_error = "invalid ref"; + } + goto cleanup; + } + if (amduatd_concepts_lookup_alias(store, concepts, scoped_name, &existing_ref)) { + if (out_status != NULL) { + *out_status = 409; + } + if (out_error != NULL) { + *out_error = "name exists"; + } + goto cleanup; + } + if (!amduatd_concepts_put_name_artifact(store, scoped_name, &name_ref)) { + goto cleanup; + } + if (!amduatd_concepts_put_concept_id(store, &concept_ref)) { + goto cleanup; + } + if (!amduatd_concepts_put_edge(store, + concepts, + name_ref, + concept_ref, + concepts->rel_aliases_ref, + actor, + &edge_ref)) { + goto cleanup; + } + if (has_ref) { + if (!amduatd_concepts_put_edge(store, + concepts, + concept_ref, + target_ref, + concepts->rel_materializes_ref, + actor, + &edge_ref)) { + goto cleanup; + } + } + + ok = true; + +cleanup: + free(scoped_name); + amduat_reference_free(&target_ref); + amduat_reference_free(&name_ref); + amduat_reference_free(&concept_ref); + amduat_reference_free(&edge_ref); + amduat_reference_free(&existing_ref); + return ok; +} + +static bool amduatd_graph_batch_apply_version(amduat_asl_store_t *store, + amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const char *name, + const char *ref_text, + const char *metadata_ref, + bool has_metadata_ref, + amduat_octets_t actor, + int *out_status, + const char **out_error) { + amduat_reference_t target_ref; + amduat_reference_t concept_ref; + amduat_reference_t edge_ref; + amduat_reference_t metadata_target_ref; + amduat_reference_t provenance_edge_ref; + amduat_octets_t scoped_bytes = amduat_octets(NULL, 0u); + char *scoped_name = NULL; + const amduatd_space_t *space = dcfg != NULL ? &dcfg->space : NULL; + bool ok = false; + + memset(&target_ref, 0, sizeof(target_ref)); + memset(&concept_ref, 0, sizeof(concept_ref)); + memset(&edge_ref, 0, sizeof(edge_ref)); + memset(&metadata_target_ref, 0, sizeof(metadata_target_ref)); + memset(&provenance_edge_ref, 0, sizeof(provenance_edge_ref)); + if (out_status != NULL) { + *out_status = 500; + } + if (out_error != NULL) { + *out_error = "store error"; + } + if (store == NULL || concepts == NULL || name == NULL || ref_text == NULL) { + if (out_error != NULL) { + *out_error = "internal error"; + } + return false; + } + if (has_metadata_ref && metadata_ref == NULL) { + if (out_status != NULL) { + *out_status = 500; + } + if (out_error != NULL) { + *out_error = "internal error"; + } + return false; + } + if (!amduat_asl_ref_decode_hex(ref_text, &target_ref)) { + if (out_status != NULL) { + *out_status = 400; + } + if (out_error != NULL) { + *out_error = "invalid ref"; + } + return false; + } + if (!amduatd_space_scope_name(space, name, &scoped_bytes)) { + if (out_status != NULL) { + *out_status = 400; + } + if (out_error != NULL) { + *out_error = "invalid name"; + } + goto cleanup; + } + scoped_name = (char *)scoped_bytes.data; + amduatd_space_log_mapping(space, name, scoped_name); + if (!amduatd_concepts_lookup_alias(store, concepts, scoped_name, &concept_ref)) { + if (out_status != NULL) { + *out_status = 404; + } + if (out_error != NULL) { + *out_error = "unknown concept"; + } + goto cleanup; + } + if (!amduatd_concepts_put_edge(store, + concepts, + concept_ref, + target_ref, + concepts->rel_materializes_ref, + actor, + &edge_ref)) { + goto cleanup; + } + if (has_metadata_ref) { + if (!amduatd_resolve_graph_ref(store, + concepts, + dcfg, + metadata_ref, + &metadata_target_ref)) { + if (out_status != NULL) { + *out_status = 404; + } + if (out_error != NULL) { + *out_error = "metadata_ref not found"; + } + goto cleanup; + } + if (!amduatd_concepts_put_edge(store, + concepts, + edge_ref, + metadata_target_ref, + concepts->rel_has_provenance_ref, + actor, + &provenance_edge_ref)) { + goto cleanup; + } + } + + ok = true; + +cleanup: + free(scoped_name); + amduat_reference_free(&target_ref); + amduat_reference_free(&concept_ref); + amduat_reference_free(&edge_ref); + amduat_reference_free(&metadata_target_ref); + amduat_reference_free(&provenance_edge_ref); + return ok; +} + +static bool amduatd_graph_batch_apply_edge(amduat_asl_store_t *store, + amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const char *subject, + const char *predicate, + const char *object, + const char *metadata_ref, + bool has_metadata_ref, + amduat_octets_t actor, + int *out_status, + const char **out_error) { + amduat_reference_t subject_ref; + amduat_reference_t predicate_ref; + amduat_reference_t object_ref; + amduat_reference_t edge_ref; + amduat_reference_t metadata_target_ref; + amduat_reference_t prov_edge_ref; + bool ok = false; + + memset(&subject_ref, 0, sizeof(subject_ref)); + memset(&predicate_ref, 0, sizeof(predicate_ref)); + memset(&object_ref, 0, sizeof(object_ref)); + memset(&edge_ref, 0, sizeof(edge_ref)); + memset(&metadata_target_ref, 0, sizeof(metadata_target_ref)); + memset(&prov_edge_ref, 0, sizeof(prov_edge_ref)); + if (out_status != NULL) { + *out_status = 500; + } + if (out_error != NULL) { + *out_error = "store error"; + } + if (store == NULL || concepts == NULL || subject == NULL || predicate == NULL || + object == NULL || (has_metadata_ref && metadata_ref == NULL)) { + if (out_error != NULL) { + *out_error = "internal error"; + } + return false; + } + if (!amduatd_resolve_graph_ref(store, concepts, dcfg, subject, &subject_ref)) { + if (out_status != NULL) { + *out_status = 404; + } + if (out_error != NULL) { + *out_error = "subject not found"; + } + goto cleanup; + } + if (!amduatd_resolve_relation_ref(concepts, predicate, &predicate_ref)) { + if (out_status != NULL) { + *out_status = 400; + } + if (out_error != NULL) { + *out_error = "invalid predicate"; + } + goto cleanup; + } + if (!amduatd_graph_schema_validate_predicate_write(predicate_ref, + out_status, + out_error)) { + goto cleanup; + } + if (!amduatd_resolve_graph_ref(store, concepts, dcfg, object, &object_ref)) { + if (out_status != NULL) { + *out_status = 404; + } + if (out_error != NULL) { + *out_error = "object not found"; + } + goto cleanup; + } + if (!amduatd_concepts_put_edge(store, + concepts, + subject_ref, + object_ref, + predicate_ref, + actor, + &edge_ref)) { + goto cleanup; + } + if (has_metadata_ref) { + if (!amduatd_resolve_graph_ref(store, + concepts, + dcfg, + metadata_ref, + &metadata_target_ref)) { + if (out_status != NULL) { + *out_status = 404; + } + if (out_error != NULL) { + *out_error = "metadata_ref not found"; + } + goto cleanup; + } + if (!amduatd_concepts_put_edge(store, + concepts, + edge_ref, + metadata_target_ref, + concepts->rel_has_provenance_ref, + actor, + &prov_edge_ref)) { + goto cleanup; + } + } + + ok = true; + +cleanup: + amduat_reference_free(&subject_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&object_ref); + amduat_reference_free(&edge_ref); + amduat_reference_free(&metadata_target_ref); + amduat_reference_free(&prov_edge_ref); + return ok; +} + +typedef struct { + bool used; + char *key; + uint64_t body_hash; + uint8_t *body; + size_t body_len; + int status_code; + char *response_json; +} amduatd_batch_idempotency_entry_t; + +enum { AMDUATD_BATCH_IDEMPOTENCY_CACHE_MAX = 256 }; +enum { + AMDUATD_BATCH_IDEMPOTENCY_MAGIC_LEN = 8, + AMDUATD_BATCH_IDEMPOTENCY_VERSION = 1 +}; + +static const uint8_t + k_amduatd_batch_idempotency_magic[AMDUATD_BATCH_IDEMPOTENCY_MAGIC_LEN] = { + 'A', 'S', 'L', 'I', 'D', 'E', 'M', '\0'}; + +static amduatd_batch_idempotency_entry_t + g_amduatd_batch_idempotency_cache[AMDUATD_BATCH_IDEMPOTENCY_CACHE_MAX]; +static size_t g_amduatd_batch_idempotency_next = 0u; + +static uint64_t amduatd_batch_hash_bytes(const uint8_t *data, size_t len) { + size_t i; + uint64_t h = 1469598103934665603ull; + if (data == NULL) { + return h; + } + for (i = 0u; i < len; ++i) { + h ^= (uint64_t)data[i]; + h *= 1099511628211ull; + } + return h; +} + +static amduatd_batch_idempotency_entry_t * +amduatd_batch_idempotency_find(const char *key) { + size_t i; + if (key == NULL) { + return NULL; + } + for (i = 0u; i < AMDUATD_BATCH_IDEMPOTENCY_CACHE_MAX; ++i) { + if (!g_amduatd_batch_idempotency_cache[i].used || + g_amduatd_batch_idempotency_cache[i].key == NULL) { + continue; + } + if (strcmp(g_amduatd_batch_idempotency_cache[i].key, key) == 0) { + return &g_amduatd_batch_idempotency_cache[i]; + } + } + return NULL; +} + +static void amduatd_batch_idempotency_entry_free( + amduatd_batch_idempotency_entry_t *entry) { + if (entry == NULL) { + return; + } + free(entry->key); + free(entry->body); + free(entry->response_json); + memset(entry, 0, sizeof(*entry)); +} + +static bool amduatd_batch_idempotency_pointer_name(const amduatd_space_t *space, + const char *key, + amduat_octets_t *out_name) { + char scoped_name[96]; + uint64_t key_hash; + int n; + if (key == NULL || out_name == NULL) { + return false; + } + key_hash = amduatd_batch_hash_bytes((const uint8_t *)key, strlen(key)); + n = snprintf(scoped_name, + sizeof(scoped_name), + "daemon/graph/batch-idem/%016llx", + (unsigned long long)key_hash); + if (n <= 0 || (size_t)n >= sizeof(scoped_name)) { + return false; + } + return amduatd_space_scope_name(space, scoped_name, out_name); +} + +static bool amduatd_batch_idempotency_encode( + const amduatd_batch_idempotency_entry_t *entry, + amduat_octets_t *out_payload) { + uint8_t *payload = NULL; + size_t payload_len = 0u; + size_t key_len = 0u; + size_t resp_len = 0u; + size_t offset = 0u; + if (out_payload != NULL) { + *out_payload = amduat_octets(NULL, 0u); + } + if (entry == NULL || out_payload == NULL || entry->key == NULL || + entry->body == NULL || entry->body_len == 0u || + entry->response_json == NULL) { + return false; + } + key_len = strlen(entry->key); + resp_len = strlen(entry->response_json); + if (key_len > UINT32_MAX || entry->body_len > UINT32_MAX || + resp_len > UINT32_MAX || entry->status_code < 0) { + return false; + } + payload_len = AMDUATD_BATCH_IDEMPOTENCY_MAGIC_LEN + 4u + 4u + 8u + 4u + 4u + 4u; + if (!amduatd_add_size(&payload_len, key_len) || + !amduatd_add_size(&payload_len, entry->body_len) || + !amduatd_add_size(&payload_len, resp_len)) { + return false; + } + payload = (uint8_t *)malloc(payload_len); + if (payload == NULL) { + return false; + } + memcpy(payload + offset, + k_amduatd_batch_idempotency_magic, + AMDUATD_BATCH_IDEMPOTENCY_MAGIC_LEN); + offset += AMDUATD_BATCH_IDEMPOTENCY_MAGIC_LEN; + amduatd_store_u32_le(payload + offset, AMDUATD_BATCH_IDEMPOTENCY_VERSION); + offset += 4u; + amduatd_store_u32_le(payload + offset, (uint32_t)entry->status_code); + offset += 4u; + amduatd_store_u64_le(payload + offset, entry->body_hash); + offset += 8u; + amduatd_store_u32_le(payload + offset, (uint32_t)key_len); + offset += 4u; + amduatd_store_u32_le(payload + offset, (uint32_t)entry->body_len); + offset += 4u; + amduatd_store_u32_le(payload + offset, (uint32_t)resp_len); + offset += 4u; + if (key_len != 0u) { + memcpy(payload + offset, entry->key, key_len); + offset += key_len; + } + memcpy(payload + offset, entry->body, entry->body_len); + offset += entry->body_len; + if (resp_len != 0u) { + memcpy(payload + offset, entry->response_json, resp_len); + offset += resp_len; + } + if (offset != payload_len) { + free(payload); + return false; + } + *out_payload = amduat_octets(payload, payload_len); + return true; +} + +static bool amduatd_batch_idempotency_decode( + amduat_octets_t payload, + amduatd_batch_idempotency_entry_t *out_entry) { + size_t offset = 0u; + uint32_t version = 0u; + uint32_t status_code = 0u; + uint64_t body_hash = 0u; + uint32_t key_len = 0u; + uint32_t body_len = 0u; + uint32_t resp_len = 0u; + char *key = NULL; + uint8_t *body = NULL; + char *resp = NULL; + if (out_entry == NULL) { + return false; + } + memset(out_entry, 0, sizeof(*out_entry)); + if (payload.len < AMDUATD_BATCH_IDEMPOTENCY_MAGIC_LEN + 4u + 4u + 8u + 4u + 4u + + 4u || + memcmp(payload.data, + k_amduatd_batch_idempotency_magic, + AMDUATD_BATCH_IDEMPOTENCY_MAGIC_LEN) != 0) { + return false; + } + offset += AMDUATD_BATCH_IDEMPOTENCY_MAGIC_LEN; + if (!amduatd_read_u32_le(payload.data, payload.len, &offset, &version) || + !amduatd_read_u32_le(payload.data, payload.len, &offset, &status_code) || + !amduatd_read_u64_le(payload.data, payload.len, &offset, &body_hash) || + !amduatd_read_u32_le(payload.data, payload.len, &offset, &key_len) || + !amduatd_read_u32_le(payload.data, payload.len, &offset, &body_len) || + !amduatd_read_u32_le(payload.data, payload.len, &offset, &resp_len)) { + return false; + } + if (version != AMDUATD_BATCH_IDEMPOTENCY_VERSION || + payload.len - offset != (size_t)key_len + (size_t)body_len + (size_t)resp_len) { + return false; + } + key = (char *)malloc((size_t)key_len + 1u); + body = (uint8_t *)malloc((size_t)body_len); + resp = (char *)malloc((size_t)resp_len + 1u); + if (key == NULL || body == NULL || resp == NULL) { + free(key); + free(body); + free(resp); + return false; + } + if (key_len != 0u) { + memcpy(key, payload.data + offset, (size_t)key_len); + } + key[key_len] = '\0'; + offset += (size_t)key_len; + if (body_len == 0u) { + free(key); + free(body); + free(resp); + return false; + } + memcpy(body, payload.data + offset, (size_t)body_len); + offset += (size_t)body_len; + if (resp_len != 0u) { + memcpy(resp, payload.data + offset, (size_t)resp_len); + } + resp[resp_len] = '\0'; + + out_entry->used = true; + out_entry->key = key; + out_entry->body_hash = body_hash; + out_entry->body = body; + out_entry->body_len = (size_t)body_len; + out_entry->status_code = (int)status_code; + out_entry->response_json = resp; + return true; +} + +static bool amduatd_batch_idempotency_load_persistent( + amduat_asl_store_t *store, + amduatd_concepts_t *concepts, + const amduatd_space_t *space, + const char *key, + amduatd_batch_idempotency_entry_t *out_entry, + bool *out_exists) { + amduat_octets_t pointer_name = amduat_octets(NULL, 0u); + amduat_reference_t pointer_ref; + amduat_asl_record_t record; + bool exists = false; + bool ok = false; + if (out_exists != NULL) { + *out_exists = false; + } + if (out_entry != NULL) { + memset(out_entry, 0, sizeof(*out_entry)); + } + if (store == NULL || concepts == NULL || key == NULL || out_entry == NULL || + out_exists == NULL) { + return false; + } + if (!amduatd_batch_idempotency_pointer_name(space, key, &pointer_name)) { + return false; + } + memset(&pointer_ref, 0, sizeof(pointer_ref)); + if (amduat_asl_pointer_get(&concepts->edge_collection.pointer_store, + (const char *)pointer_name.data, + &exists, + &pointer_ref) != AMDUAT_ASL_POINTER_OK) { + amduat_octets_free(&pointer_name); + return false; + } + amduat_octets_free(&pointer_name); + if (!exists) { + return true; + } + memset(&record, 0, sizeof(record)); + if (amduat_asl_record_store_get(store, pointer_ref, &record) != AMDUAT_ASL_STORE_OK) { + amduat_reference_free(&pointer_ref); + return false; + } + amduat_reference_free(&pointer_ref); + if (record.schema.len != strlen(AMDUATD_BATCH_IDEMPOTENCY_SCHEMA) || + memcmp(record.schema.data, + AMDUATD_BATCH_IDEMPOTENCY_SCHEMA, + record.schema.len) != 0) { + amduat_asl_record_free(&record); + return false; + } + ok = amduatd_batch_idempotency_decode(record.payload, out_entry); + amduat_asl_record_free(&record); + if (!ok) { + return false; + } + if (out_entry->key == NULL || strcmp(out_entry->key, key) != 0) { + amduatd_batch_idempotency_entry_free(out_entry); + return true; + } + *out_exists = true; + return true; +} + +static bool amduatd_batch_idempotency_store_persistent( + amduat_asl_store_t *store, + amduatd_concepts_t *concepts, + const amduatd_space_t *space, + const char *key, + const uint8_t *body, + size_t body_len, + int status_code, + const char *response_json) { + amduatd_batch_idempotency_entry_t entry; + amduat_octets_t payload = amduat_octets(NULL, 0u); + amduat_octets_t pointer_name = amduat_octets(NULL, 0u); + amduat_reference_t record_ref; + amduat_reference_t expected_ref; + bool expected_exists = false; + bool swapped = false; + bool ok = false; + + memset(&entry, 0, sizeof(entry)); + memset(&record_ref, 0, sizeof(record_ref)); + memset(&expected_ref, 0, sizeof(expected_ref)); + if (store == NULL || concepts == NULL || key == NULL || body == NULL || + body_len == 0u || response_json == NULL) { + return false; + } + entry.used = true; + entry.key = (char *)key; + entry.body_hash = amduatd_batch_hash_bytes(body, body_len); + entry.body = (uint8_t *)body; + entry.body_len = body_len; + entry.status_code = status_code; + entry.response_json = (char *)response_json; + if (!amduatd_batch_idempotency_encode(&entry, &payload)) { + return false; + } + if (amduat_asl_record_store_put( + store, + amduat_octets(AMDUATD_BATCH_IDEMPOTENCY_SCHEMA, + strlen(AMDUATD_BATCH_IDEMPOTENCY_SCHEMA)), + payload, + &record_ref) != AMDUAT_ASL_STORE_OK) { + free((void *)payload.data); + return false; + } + free((void *)payload.data); + if (!amduatd_batch_idempotency_pointer_name(space, key, &pointer_name)) { + amduat_reference_free(&record_ref); + return false; + } + if (amduat_asl_pointer_get(&concepts->edge_collection.pointer_store, + (const char *)pointer_name.data, + &expected_exists, + &expected_ref) != AMDUAT_ASL_POINTER_OK) { + amduat_octets_free(&pointer_name); + amduat_reference_free(&record_ref); + return false; + } + if (amduat_asl_pointer_cas(&concepts->edge_collection.pointer_store, + (const char *)pointer_name.data, + expected_exists, + expected_exists ? &expected_ref : NULL, + &record_ref, + &swapped) != AMDUAT_ASL_POINTER_OK) { + amduat_octets_free(&pointer_name); + amduat_reference_free(&expected_ref); + amduat_reference_free(&record_ref); + return false; + } + ok = swapped; + amduat_octets_free(&pointer_name); + amduat_reference_free(&expected_ref); + amduat_reference_free(&record_ref); + return ok; +} + +static bool amduatd_batch_idempotency_store(const char *key, + const uint8_t *body, + size_t body_len, + int status_code, + const char *response_json) { + amduatd_batch_idempotency_entry_t *slot; + uint8_t *body_copy = NULL; + char *key_copy = NULL; + char *json_copy = NULL; + uint64_t body_hash; + if (key == NULL || body == NULL || body_len == 0u || response_json == NULL) { + return false; + } + body_hash = amduatd_batch_hash_bytes(body, body_len); + slot = amduatd_batch_idempotency_find(key); + if (slot == NULL) { + slot = &g_amduatd_batch_idempotency_cache[g_amduatd_batch_idempotency_next]; + g_amduatd_batch_idempotency_next = + (g_amduatd_batch_idempotency_next + 1u) % + AMDUATD_BATCH_IDEMPOTENCY_CACHE_MAX; + } + key_copy = strdup(key); + if (key_copy == NULL) { + return false; + } + body_copy = (uint8_t *)malloc(body_len); + if (body_copy == NULL) { + free(key_copy); + return false; + } + memcpy(body_copy, body, body_len); + json_copy = strdup(response_json); + if (json_copy == NULL) { + free(key_copy); + free(body_copy); + return false; + } + if (slot->used) { + free(slot->key); + free(slot->body); + free(slot->response_json); + } + memset(slot, 0, sizeof(*slot)); + slot->used = true; + slot->key = key_copy; + slot->body = body_copy; + slot->body_len = body_len; + slot->body_hash = body_hash; + slot->status_code = status_code; + slot->response_json = json_copy; + return true; +} + +static const char *amduatd_http_reason_phrase(int status_code) { + switch (status_code) { + case 200: + return "OK"; + case 400: + return "Bad Request"; + case 409: + return "Conflict"; + default: + return "OK"; + } +} + +static bool amduatd_batch_results_append(amduatd_strbuf_t *b, + bool *first, + const char *kind, + size_t index, + const char *status, + int code, + const char *error) { + char num[32]; + int n; + if (b == NULL || first == NULL || kind == NULL || status == NULL) { + return false; + } + n = snprintf(num, sizeof(num), "%llu", (unsigned long long)index); + if (n <= 0 || (size_t)n >= sizeof(num)) { + return false; + } + if (!*first) { + if (!amduatd_strbuf_append_char(b, ',')) { + return false; + } + } + *first = false; + if (!amduatd_strbuf_append_cstr(b, "{\"kind\":\"") || + !amduatd_strbuf_append_cstr(b, kind) || + !amduatd_strbuf_append_cstr(b, "\",\"index\":") || + !amduatd_strbuf_append(b, num, (size_t)n) || + !amduatd_strbuf_append_cstr(b, ",\"status\":\"") || + !amduatd_strbuf_append_cstr(b, status) || + !amduatd_strbuf_append_cstr(b, "\",\"code\":")) { + return false; + } + n = snprintf(num, sizeof(num), "%d", code); + if (n <= 0 || (size_t)n >= sizeof(num) || + !amduatd_strbuf_append(b, num, (size_t)n)) { + return false; + } + if (error != NULL) { + if (!amduatd_strbuf_append_cstr(b, ",\"error\":\"") || + !amduatd_strbuf_append_cstr(b, error) || + !amduatd_strbuf_append_cstr(b, "\"")) { + return false; + } + } else if (!amduatd_strbuf_append_cstr(b, ",\"error\":null")) { + return false; + } + return amduatd_strbuf_append_cstr(b, "}"); +} + +static bool amduatd_handle_post_graph_batch(int fd, + amduat_asl_store_t *store, + amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const amduatd_http_req_t *req) { + uint8_t *body = NULL; + const char *p = NULL; + const char *end = NULL; + bool ok = false; + amduat_octets_t actor = amduat_octets(NULL, 0u); + char *idempotency_key = NULL; + char mode_buf[32]; + bool mode_continue_on_error = false; + size_t nodes_applied = 0u; + size_t versions_applied = 0u; + size_t edges_applied = 0u; + size_t failures = 0u; + int err_status = 400; + const char *err_text = "invalid json"; + size_t item_index = 0u; + bool stop_on_error = false; + amduatd_strbuf_t results; + bool results_first = true; + int response_code = 200; + const char *response_reason = "OK"; + amduatd_batch_idempotency_entry_t *cached = NULL; + amduatd_batch_idempotency_entry_t persisted; + bool persisted_exists = false; + const amduatd_space_t *space = dcfg != NULL ? &dcfg->space : NULL; + uint64_t body_hash = 0u; + const char *pre_p = NULL; + const char *pre_end = NULL; + + memset(&results, 0, sizeof(results)); + memset(&persisted, 0, sizeof(persisted)); + mode_buf[0] = '\0'; + + if (store == NULL || concepts == NULL || req == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "internal error"); + } + if (req->has_actor) { + actor = req->actor; + } + if (req->content_length == 0u) { + return amduatd_send_json_error(fd, 400, "Bad Request", "missing body"); + } + if (req->content_length > (1024u * 1024u)) { + return amduatd_send_json_error(fd, 413, "Payload Too Large", "payload too large"); + } + body = (uint8_t *)malloc(req->content_length); + if (body == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + if (!amduatd_read_exact(fd, body, req->content_length)) { + free(body); + return false; + } + body_hash = amduatd_batch_hash_bytes(body, req->content_length); + + if (!amduatd_strbuf_append_cstr(&results, "[")) { + free(body); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + + /* Prepass: discover mode/idempotency regardless of key order. */ + pre_p = (const char *)body; + pre_end = (const char *)body + req->content_length; + if (!amduatd_json_expect(&pre_p, pre_end, '{')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + for (;;) { + const char *key = NULL; + size_t key_len = 0u; + const char *sv = NULL; + size_t sv_len = 0u; + const char *cur = amduatd_json_skip_ws(pre_p, pre_end); + if (cur < pre_end && *cur == '}') { + pre_p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&pre_p, pre_end, &key, &key_len) || + !amduatd_json_expect(&pre_p, pre_end, ':')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + if (key_len == strlen("idempotency_key") && + memcmp(key, "idempotency_key", key_len) == 0) { + char *tmp = NULL; + if (!amduatd_json_parse_string_noesc(&pre_p, pre_end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &tmp)) { + free(tmp); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid idempotency_key"); + goto cleanup; + } + if (idempotency_key != NULL) { + free(tmp); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid idempotency_key"); + goto cleanup; + } + idempotency_key = tmp; + } else if (key_len == strlen("mode") && memcmp(key, "mode", key_len) == 0) { + if (!amduatd_json_parse_string_noesc(&pre_p, pre_end, &sv, &sv_len) || + sv_len >= sizeof(mode_buf)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid mode"); + goto cleanup; + } + memcpy(mode_buf, sv, sv_len); + mode_buf[sv_len] = '\0'; + if (strcmp(mode_buf, "fail_fast") == 0 || mode_buf[0] == '\0') { + mode_continue_on_error = false; + } else if (strcmp(mode_buf, "continue_on_error") == 0) { + mode_continue_on_error = true; + } else { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid mode"); + goto cleanup; + } + } else if (!amduatd_json_skip_value(&pre_p, pre_end, 0)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + cur = amduatd_json_skip_ws(pre_p, pre_end); + if (cur < pre_end && *cur == ',') { + pre_p = cur + 1; + continue; + } + if (cur < pre_end && *cur == '}') { + pre_p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + pre_p = amduatd_json_skip_ws(pre_p, pre_end); + if (pre_p != pre_end) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + if (idempotency_key != NULL) { + cached = amduatd_batch_idempotency_find(idempotency_key); + if (cached != NULL) { + if (cached->body_hash != body_hash || + cached->body_len != req->content_length || + memcmp(cached->body, body, req->content_length) != 0) { + ok = amduatd_send_json_error(fd, + 409, + "Conflict", + "idempotency_key reuse with different payload"); + goto cleanup; + } + ok = amduatd_http_send_json(fd, + cached->status_code, + amduatd_http_reason_phrase(cached->status_code), + cached->response_json, + false); + goto cleanup; + } + if (!amduatd_batch_idempotency_load_persistent(store, + concepts, + space, + idempotency_key, + &persisted, + &persisted_exists)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "idempotency lookup failed"); + goto cleanup; + } + if (persisted_exists) { + if (persisted.body_hash != body_hash || + persisted.body_len != req->content_length || + memcmp(persisted.body, body, req->content_length) != 0) { + ok = amduatd_send_json_error(fd, + 409, + "Conflict", + "idempotency_key reuse with different payload"); + goto cleanup; + } + (void)amduatd_batch_idempotency_store(persisted.key, + persisted.body, + persisted.body_len, + persisted.status_code, + persisted.response_json); + ok = amduatd_http_send_json(fd, + persisted.status_code, + amduatd_http_reason_phrase(persisted.status_code), + persisted.response_json, + false); + goto cleanup; + } + } + + p = (const char *)body; + end = (const char *)body + req->content_length; + if (!amduatd_json_expect(&p, end, '{')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + for (;;) { + const char *key = NULL; + size_t key_len = 0u; + const char *sv = NULL; + size_t sv_len = 0u; + const char *cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &key, &key_len) || + !amduatd_json_expect(&p, end, ':')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + + if (key_len == strlen("idempotency_key") && + memcmp(key, "idempotency_key", key_len) == 0) { + char *tmp = NULL; + if (!amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &tmp)) { + free(tmp); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid idempotency_key"); + goto cleanup; + } + if (idempotency_key == NULL || strcmp(idempotency_key, tmp) != 0) { + free(tmp); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid idempotency_key"); + goto cleanup; + } + free(tmp); + } else if (key_len == strlen("mode") && memcmp(key, "mode", key_len) == 0) { + if (!amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + sv_len >= sizeof(mode_buf)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid mode"); + goto cleanup; + } + memcpy(mode_buf, sv, sv_len); + mode_buf[sv_len] = '\0'; + if (strcmp(mode_buf, "fail_fast") == 0 || mode_buf[0] == '\0') { + mode_continue_on_error = false; + } else if (strcmp(mode_buf, "continue_on_error") == 0) { + mode_continue_on_error = true; + } else { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid mode"); + goto cleanup; + } + } else if (key_len == strlen("nodes") && memcmp(key, "nodes", key_len) == 0) { + item_index = 0u; + if (!amduatd_json_expect(&p, end, '[')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid nodes"); + goto cleanup; + } + for (;;) { + char *name = NULL; + char *ref_text = NULL; + bool have_name = false; + bool have_ref = false; + const char *arr_cur = amduatd_json_skip_ws(p, end); + if (arr_cur < end && *arr_cur == ']') { + p = arr_cur + 1; + break; + } + if (!amduatd_json_expect(&p, end, '{')) { + free(name); + free(ref_text); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid nodes"); + goto cleanup; + } + for (;;) { + const char *item_key = NULL; + size_t item_key_len = 0u; + const char *item_sv = NULL; + size_t item_sv_len = 0u; + const char *obj_cur = amduatd_json_skip_ws(p, end); + if (obj_cur < end && *obj_cur == '}') { + p = obj_cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &item_key, &item_key_len) || + !amduatd_json_expect(&p, end, ':')) { + free(name); + free(ref_text); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid nodes"); + goto cleanup; + } + if (item_key_len == strlen("name") && + memcmp(item_key, "name", item_key_len) == 0) { + if (have_name || + !amduatd_json_parse_string_noesc(&p, end, &item_sv, &item_sv_len) || + !amduatd_copy_json_str(item_sv, item_sv_len, &name)) { + free(name); + free(ref_text); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid node name"); + goto cleanup; + } + have_name = true; + } else if (item_key_len == strlen("ref") && + memcmp(item_key, "ref", item_key_len) == 0) { + if (have_ref || + !amduatd_json_parse_string_noesc(&p, end, &item_sv, &item_sv_len) || + !amduatd_copy_json_str(item_sv, item_sv_len, &ref_text)) { + free(name); + free(ref_text); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid node ref"); + goto cleanup; + } + have_ref = true; + } else { + if (!amduatd_json_skip_value(&p, end, 0)) { + free(name); + free(ref_text); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid nodes"); + goto cleanup; + } + } + obj_cur = amduatd_json_skip_ws(p, end); + if (obj_cur < end && *obj_cur == ',') { + p = obj_cur + 1; + continue; + } + if (obj_cur < end && *obj_cur == '}') { + p = obj_cur + 1; + break; + } + free(name); + free(ref_text); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid nodes"); + goto cleanup; + } + if (!have_name) { + free(name); + free(ref_text); + failures++; + if (!amduatd_batch_results_append(&results, + &results_first, + "node", + item_index, + "error", + 400, + "missing node name")) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + if (!mode_continue_on_error) { + stop_on_error = true; + } + } else if (!amduatd_graph_batch_apply_node(store, + concepts, + dcfg, + name, + ref_text, + have_ref, + actor, + &err_status, + &err_text)) { + failures++; + if (!amduatd_batch_results_append(&results, + &results_first, + "node", + item_index, + "error", + err_status, + err_text)) { + free(name); + free(ref_text); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + if (!mode_continue_on_error) { + stop_on_error = true; + } + } else { + nodes_applied++; + if (!amduatd_batch_results_append(&results, + &results_first, + "node", + item_index, + "applied", + 200, + NULL)) { + free(name); + free(ref_text); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + } + free(name); + free(ref_text); + item_index++; + if (stop_on_error) { + /* consume remaining array payload to keep parser consistent */ + arr_cur = amduatd_json_skip_ws(p, end); + if (arr_cur < end && *arr_cur == ',') { + p = arr_cur + 1; + for (;;) { + const char *tmp_cur = amduatd_json_skip_ws(p, end); + if (tmp_cur < end && *tmp_cur == ']') { + p = tmp_cur + 1; + break; + } + if (!amduatd_json_skip_value(&p, end, 0)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid nodes"); + goto cleanup; + } + tmp_cur = amduatd_json_skip_ws(p, end); + if (tmp_cur < end && *tmp_cur == ',') { + p = tmp_cur + 1; + continue; + } + if (tmp_cur < end && *tmp_cur == ']') { + p = tmp_cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid nodes"); + goto cleanup; + } + } else if (arr_cur < end && *arr_cur == ']') { + p = arr_cur + 1; + } + goto cleanup; + } + arr_cur = amduatd_json_skip_ws(p, end); + if (arr_cur < end && *arr_cur == ',') { + p = arr_cur + 1; + continue; + } + if (arr_cur < end && *arr_cur == ']') { + p = arr_cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid nodes"); + goto cleanup; + } + } else if (key_len == strlen("versions") && memcmp(key, "versions", key_len) == 0) { + item_index = 0u; + if (!amduatd_json_expect(&p, end, '[')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid versions"); + goto cleanup; + } + for (;;) { + char *name = NULL; + char *ref_text = NULL; + char *metadata_ref = NULL; + char *provenance_metadata_ref = NULL; + bool have_name = false; + bool have_ref = false; + bool have_metadata_ref = false; + bool have_provenance = false; + const char *prov_err = NULL; + amduatd_provenance_input_t provenance; + const char *arr_cur = amduatd_json_skip_ws(p, end); + memset(&provenance, 0, sizeof(provenance)); + if (arr_cur < end && *arr_cur == ']') { + p = arr_cur + 1; + break; + } + if (!amduatd_json_expect(&p, end, '{')) { + free(name); + free(ref_text); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid versions"); + goto cleanup; + } + for (;;) { + const char *item_key = NULL; + size_t item_key_len = 0u; + const char *item_sv = NULL; + size_t item_sv_len = 0u; + const char *obj_cur = amduatd_json_skip_ws(p, end); + if (obj_cur < end && *obj_cur == '}') { + p = obj_cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &item_key, &item_key_len) || + !amduatd_json_expect(&p, end, ':')) { + free(name); + free(ref_text); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid versions"); + goto cleanup; + } + if (item_key_len == strlen("name") && + memcmp(item_key, "name", item_key_len) == 0) { + if (have_name || + !amduatd_json_parse_string_noesc(&p, end, &item_sv, &item_sv_len) || + !amduatd_copy_json_str(item_sv, item_sv_len, &name)) { + free(name); + free(ref_text); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid version name"); + goto cleanup; + } + have_name = true; + } else if (item_key_len == strlen("ref") && + memcmp(item_key, "ref", item_key_len) == 0) { + if (have_ref || + !amduatd_json_parse_string_noesc(&p, end, &item_sv, &item_sv_len) || + !amduatd_copy_json_str(item_sv, item_sv_len, &ref_text)) { + free(name); + free(ref_text); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid version ref"); + goto cleanup; + } + have_ref = true; + } else if (item_key_len == strlen("metadata_ref") && + memcmp(item_key, "metadata_ref", item_key_len) == 0) { + if (have_metadata_ref || + !amduatd_json_parse_string_noesc(&p, end, &item_sv, &item_sv_len) || + !amduatd_copy_json_str(item_sv, item_sv_len, &metadata_ref)) { + free(name); + free(ref_text); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid version metadata_ref"); + goto cleanup; + } + have_metadata_ref = true; + } else if (item_key_len == strlen("provenance") && + memcmp(item_key, "provenance", item_key_len) == 0) { + if (have_provenance || + !amduatd_json_parse_provenance_input(&p, end, &provenance, &prov_err)) { + free(name); + free(ref_text); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + ok = amduatd_send_json_error(fd, + 400, + "Bad Request", + prov_err != NULL ? prov_err : "invalid version provenance"); + goto cleanup; + } + have_provenance = true; + } else { + if (!amduatd_json_skip_value(&p, end, 0)) { + free(name); + free(ref_text); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid versions"); + goto cleanup; + } + } + obj_cur = amduatd_json_skip_ws(p, end); + if (obj_cur < end && *obj_cur == ',') { + p = obj_cur + 1; + continue; + } + if (obj_cur < end && *obj_cur == '}') { + p = obj_cur + 1; + break; + } + free(name); + free(ref_text); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid versions"); + goto finalize_batch_response; + } + if (!have_name || !have_ref) { + failures++; + if (!amduatd_batch_results_append(&results, + &results_first, + "version", + item_index, + "error", + 400, + "missing version name/ref")) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + if (!mode_continue_on_error) { + stop_on_error = true; + } + } else if (have_metadata_ref && have_provenance) { + failures++; + if (!amduatd_batch_results_append(&results, + &results_first, + "version", + item_index, + "error", + 400, + "metadata_ref and provenance are mutually exclusive")) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + if (!mode_continue_on_error) { + stop_on_error = true; + } + } else if (!amduatd_graph_schema_validate_provenance_write( + have_metadata_ref || have_provenance, + &err_status, + &err_text)) { + failures++; + if (!amduatd_batch_results_append(&results, + &results_first, + "version", + item_index, + "error", + err_status, + err_text)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + if (!mode_continue_on_error) { + stop_on_error = true; + } + } else { + bool item_failed = false; + if (have_provenance) { + amduat_reference_t prov_ref; + memset(&prov_ref, 0, sizeof(prov_ref)); + if (!amduatd_provenance_store(store, &provenance, &prov_ref) || + !amduat_asl_ref_encode_hex(prov_ref, &provenance_metadata_ref)) { + amduat_reference_free(&prov_ref); + item_failed = true; + failures++; + if (!amduatd_batch_results_append(&results, + &results_first, + "version", + item_index, + "error", + 500, + "store error")) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + if (!mode_continue_on_error) { + stop_on_error = true; + } + } + amduat_reference_free(&prov_ref); + } + if (!item_failed && !stop_on_error && + !amduatd_graph_batch_apply_version(store, + concepts, + dcfg, + name, + ref_text, + have_provenance ? provenance_metadata_ref : metadata_ref, + have_metadata_ref || have_provenance, + actor, + &err_status, + &err_text)) { + failures++; + if (!amduatd_batch_results_append(&results, + &results_first, + "version", + item_index, + "error", + err_status, + err_text)) { + free(name); + free(ref_text); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + if (!mode_continue_on_error) { + stop_on_error = true; + } + } else if (!stop_on_error) { + versions_applied++; + if (!amduatd_batch_results_append(&results, + &results_first, + "version", + item_index, + "applied", + 200, + NULL)) { + free(name); + free(ref_text); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + } + } + free(name); + free(ref_text); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + item_index++; + if (stop_on_error) { + arr_cur = amduatd_json_skip_ws(p, end); + if (arr_cur < end && *arr_cur == ',') { + p = arr_cur + 1; + for (;;) { + const char *tmp_cur = amduatd_json_skip_ws(p, end); + if (tmp_cur < end && *tmp_cur == ']') { + p = tmp_cur + 1; + break; + } + if (!amduatd_json_skip_value(&p, end, 0)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid versions"); + goto cleanup; + } + tmp_cur = amduatd_json_skip_ws(p, end); + if (tmp_cur < end && *tmp_cur == ',') { + p = tmp_cur + 1; + continue; + } + if (tmp_cur < end && *tmp_cur == ']') { + p = tmp_cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid versions"); + goto cleanup; + } + } else if (arr_cur < end && *arr_cur == ']') { + p = arr_cur + 1; + } + goto finalize_batch_response; + } + arr_cur = amduatd_json_skip_ws(p, end); + if (arr_cur < end && *arr_cur == ',') { + p = arr_cur + 1; + continue; + } + if (arr_cur < end && *arr_cur == ']') { + p = arr_cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid versions"); + goto cleanup; + } + } else if (key_len == strlen("edges") && memcmp(key, "edges", key_len) == 0) { + item_index = 0u; + if (!amduatd_json_expect(&p, end, '[')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid edges"); + goto cleanup; + } + for (;;) { + char *subject = NULL; + char *predicate = NULL; + char *object = NULL; + char *metadata_ref = NULL; + char *provenance_metadata_ref = NULL; + bool have_subject = false; + bool have_predicate = false; + bool have_object = false; + bool have_metadata_ref = false; + bool have_provenance = false; + const char *prov_err = NULL; + amduatd_provenance_input_t provenance; + const char *arr_cur = amduatd_json_skip_ws(p, end); + memset(&provenance, 0, sizeof(provenance)); + if (arr_cur < end && *arr_cur == ']') { + p = arr_cur + 1; + break; + } + if (!amduatd_json_expect(&p, end, '{')) { + free(subject); + free(predicate); + free(object); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid edges"); + goto cleanup; + } + for (;;) { + const char *item_key = NULL; + size_t item_key_len = 0u; + const char *item_sv = NULL; + size_t item_sv_len = 0u; + const char *obj_cur = amduatd_json_skip_ws(p, end); + if (obj_cur < end && *obj_cur == '}') { + p = obj_cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &item_key, &item_key_len) || + !amduatd_json_expect(&p, end, ':')) { + free(subject); + free(predicate); + free(object); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid edges"); + goto cleanup; + } + if (item_key_len == strlen("subject") && + memcmp(item_key, "subject", item_key_len) == 0) { + if (have_subject || + !amduatd_json_parse_string_noesc(&p, end, &item_sv, &item_sv_len) || + !amduatd_copy_json_str(item_sv, item_sv_len, &subject)) { + free(subject); + free(predicate); + free(object); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid edge subject"); + goto cleanup; + } + have_subject = true; + } else if (item_key_len == strlen("predicate") && + memcmp(item_key, "predicate", item_key_len) == 0) { + if (have_predicate || + !amduatd_json_parse_string_noesc(&p, end, &item_sv, &item_sv_len) || + !amduatd_copy_json_str(item_sv, item_sv_len, &predicate)) { + free(subject); + free(predicate); + free(object); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid edge predicate"); + goto cleanup; + } + have_predicate = true; + } else if (item_key_len == strlen("object") && + memcmp(item_key, "object", item_key_len) == 0) { + if (have_object || + !amduatd_json_parse_string_noesc(&p, end, &item_sv, &item_sv_len) || + !amduatd_copy_json_str(item_sv, item_sv_len, &object)) { + free(subject); + free(predicate); + free(object); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid edge object"); + goto cleanup; + } + have_object = true; + } else if (item_key_len == strlen("metadata_ref") && + memcmp(item_key, "metadata_ref", item_key_len) == 0) { + if (have_metadata_ref || + !amduatd_json_parse_string_noesc(&p, end, &item_sv, &item_sv_len) || + !amduatd_copy_json_str(item_sv, item_sv_len, &metadata_ref)) { + free(subject); + free(predicate); + free(object); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid edge metadata_ref"); + goto cleanup; + } + have_metadata_ref = true; + } else if (item_key_len == strlen("provenance") && + memcmp(item_key, "provenance", item_key_len) == 0) { + if (have_provenance || + !amduatd_json_parse_provenance_input(&p, end, &provenance, &prov_err)) { + free(subject); + free(predicate); + free(object); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + ok = amduatd_send_json_error(fd, + 400, + "Bad Request", + prov_err != NULL ? prov_err : "invalid edge provenance"); + goto cleanup; + } + have_provenance = true; + } else { + if (!amduatd_json_skip_value(&p, end, 0)) { + free(subject); + free(predicate); + free(object); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid edges"); + goto cleanup; + } + } + obj_cur = amduatd_json_skip_ws(p, end); + if (obj_cur < end && *obj_cur == ',') { + p = obj_cur + 1; + continue; + } + if (obj_cur < end && *obj_cur == '}') { + p = obj_cur + 1; + break; + } + free(subject); + free(predicate); + free(object); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid edges"); + goto cleanup; + } + if (!have_subject || !have_predicate || !have_object) { + failures++; + if (!amduatd_batch_results_append(&results, + &results_first, + "edge", + item_index, + "error", + 400, + "missing edge subject/predicate/object")) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + if (!mode_continue_on_error) { + stop_on_error = true; + } + } else if (have_metadata_ref && have_provenance) { + failures++; + if (!amduatd_batch_results_append(&results, + &results_first, + "edge", + item_index, + "error", + 400, + "metadata_ref and provenance are mutually exclusive")) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + if (!mode_continue_on_error) { + stop_on_error = true; + } + } else if (!amduatd_graph_schema_validate_provenance_write( + have_metadata_ref || have_provenance, + &err_status, + &err_text)) { + failures++; + if (!amduatd_batch_results_append(&results, + &results_first, + "edge", + item_index, + "error", + err_status, + err_text)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + if (!mode_continue_on_error) { + stop_on_error = true; + } + } else { + bool item_failed = false; + if (have_provenance) { + amduat_reference_t prov_ref; + memset(&prov_ref, 0, sizeof(prov_ref)); + if (!amduatd_provenance_store(store, &provenance, &prov_ref) || + !amduat_asl_ref_encode_hex(prov_ref, &provenance_metadata_ref)) { + amduat_reference_free(&prov_ref); + item_failed = true; + failures++; + if (!amduatd_batch_results_append(&results, + &results_first, + "edge", + item_index, + "error", + 500, + "store error")) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + if (!mode_continue_on_error) { + stop_on_error = true; + } + } + amduat_reference_free(&prov_ref); + } + if (!item_failed && !stop_on_error && + !amduatd_graph_batch_apply_edge(store, + concepts, + dcfg, + subject, + predicate, + object, + have_provenance ? provenance_metadata_ref : metadata_ref, + have_metadata_ref || have_provenance, + actor, + &err_status, + &err_text)) { + failures++; + if (!amduatd_batch_results_append(&results, + &results_first, + "edge", + item_index, + "error", + err_status, + err_text)) { + free(subject); + free(predicate); + free(object); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + if (!mode_continue_on_error) { + stop_on_error = true; + } + } else if (!stop_on_error) { + edges_applied++; + if (!amduatd_batch_results_append(&results, + &results_first, + "edge", + item_index, + "applied", + 200, + NULL)) { + free(subject); + free(predicate); + free(object); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + } + } + free(subject); + free(predicate); + free(object); + free(metadata_ref); + free(provenance_metadata_ref); + amduatd_provenance_input_free(&provenance); + item_index++; + if (stop_on_error) { + arr_cur = amduatd_json_skip_ws(p, end); + if (arr_cur < end && *arr_cur == ',') { + p = arr_cur + 1; + for (;;) { + const char *tmp_cur = amduatd_json_skip_ws(p, end); + if (tmp_cur < end && *tmp_cur == ']') { + p = tmp_cur + 1; + break; + } + if (!amduatd_json_skip_value(&p, end, 0)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid edges"); + goto cleanup; + } + tmp_cur = amduatd_json_skip_ws(p, end); + if (tmp_cur < end && *tmp_cur == ',') { + p = tmp_cur + 1; + continue; + } + if (tmp_cur < end && *tmp_cur == ']') { + p = tmp_cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid edges"); + goto cleanup; + } + } else if (arr_cur < end && *arr_cur == ']') { + p = arr_cur + 1; + } + goto cleanup; + } + arr_cur = amduatd_json_skip_ws(p, end); + if (arr_cur < end && *arr_cur == ',') { + p = arr_cur + 1; + continue; + } + if (arr_cur < end && *arr_cur == ']') { + p = arr_cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid edges"); + goto cleanup; + } + } else { + if (!amduatd_json_skip_value(&p, end, 0)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + } + + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + p = amduatd_json_skip_ws(p, end); + if (p != end) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + +finalize_batch_response: + if (!amduatd_strbuf_append_cstr(&results, "]")) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + + { + amduatd_strbuf_t out; + memset(&out, 0, sizeof(out)); + if (!amduatd_strbuf_append_cstr(&out, "{\"ok\":") || + !amduatd_strbuf_append_cstr(&out, failures == 0u ? "true" : "false")) { + amduatd_strbuf_free(&out); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + if (idempotency_key != NULL) { + if (!amduatd_strbuf_append_cstr(&out, ",\"idempotency_key\":\"") || + !amduatd_strbuf_append_cstr(&out, idempotency_key) || + !amduatd_strbuf_append_cstr(&out, "\"")) { + amduatd_strbuf_free(&out); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + } + if (!amduatd_strbuf_append_cstr(&out, ",\"mode\":\"") || + !amduatd_strbuf_append_cstr(&out, mode_continue_on_error ? "continue_on_error" : "fail_fast") || + !amduatd_strbuf_append_cstr(&out, "\"") || + !amduatd_strbuf_append_cstr(&out, ",\"applied\":{\"nodes\":") ) { + amduatd_strbuf_free(&out); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + { + char nbuf[32]; + int n = snprintf(nbuf, sizeof(nbuf), "%llu", + (unsigned long long)nodes_applied); + if (n <= 0 || (size_t)n >= sizeof(nbuf) || + !amduatd_strbuf_append(&out, nbuf, (size_t)n) || + !amduatd_strbuf_append_cstr(&out, ",\"versions\":")) { + amduatd_strbuf_free(&out); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + n = snprintf(nbuf, sizeof(nbuf), "%llu", (unsigned long long)versions_applied); + if (n <= 0 || (size_t)n >= sizeof(nbuf) || + !amduatd_strbuf_append(&out, nbuf, (size_t)n) || + !amduatd_strbuf_append_cstr(&out, ",\"edges\":")) { + amduatd_strbuf_free(&out); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + n = snprintf(nbuf, sizeof(nbuf), "%llu", (unsigned long long)edges_applied); + if (n <= 0 || (size_t)n >= sizeof(nbuf) || + !amduatd_strbuf_append(&out, nbuf, (size_t)n) || + !amduatd_strbuf_append_cstr(&out, "},\"results\":") || + !amduatd_strbuf_append_cstr(&out, results.data != NULL ? results.data : "[]") || + !amduatd_strbuf_append_cstr(&out, "}\n")) { + amduatd_strbuf_free(&out); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + } + if (failures != 0u) { + response_code = mode_continue_on_error ? 200 : 400; + response_reason = mode_continue_on_error ? "OK" : "Bad Request"; + } else { + response_code = 200; + response_reason = "OK"; + } + ok = amduatd_http_send_json(fd, response_code, response_reason, out.data, false); + if (ok && idempotency_key != NULL) { + (void)amduatd_batch_idempotency_store(idempotency_key, + body, + req->content_length, + response_code, + out.data); + (void)amduatd_batch_idempotency_store_persistent(store, + concepts, + space, + idempotency_key, + body, + req->content_length, + response_code, + out.data); + } + amduatd_strbuf_free(&out); + } + +cleanup: + amduatd_batch_idempotency_entry_free(&persisted); + free(body); + free(idempotency_key); + amduatd_strbuf_free(&results); + return ok; +} + +static bool amduatd_concepts_name_for_ref(amduat_asl_store_t *store, + const amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + amduat_reference_t concept_ref, + char *out_name, + size_t out_cap) { + size_t i; + if (store == NULL || concepts == NULL || out_name == NULL || out_cap == 0u) { + return false; + } + out_name[0] = '\0'; + for (i = 0; i < concepts->edges.len; ++i) { + const amduatd_edge_entry_t *entry = &concepts->edges.items[i]; + amduat_artifact_t artifact; + amduat_asl_store_error_t err; + if (entry->rel == NULL || strcmp(entry->rel, AMDUATD_REL_ALIAS) != 0) { + continue; + } + if (!amduat_reference_eq(entry->dst_ref, concept_ref)) { + continue; + } + memset(&artifact, 0, sizeof(artifact)); + err = amduat_asl_store_get(store, entry->src_ref, &artifact); + if (err != AMDUAT_ASL_STORE_OK || + !amduatd_parse_name_artifact(artifact, dcfg, out_name, out_cap)) { + amduat_asl_artifact_free(&artifact); + continue; + } + amduat_asl_artifact_free(&artifact); + return true; + } + return false; +} + +typedef struct { + bool used; + amduat_reference_t ref; + bool name_checked; + bool have_name; + char *name; + bool latest_checked; + char *latest_hex; +} amduatd_expand_cache_entry_t; + +enum { AMDUATD_EXPAND_CACHE_MAX = 512 }; + +static amduatd_expand_cache_entry_t g_amduatd_expand_cache[AMDUATD_EXPAND_CACHE_MAX]; +static size_t g_amduatd_expand_cache_next = 0u; +static size_t g_amduatd_expand_cache_edges_len = SIZE_MAX; + +static void amduatd_expand_cache_clear(void) { + size_t i; + for (i = 0; i < AMDUATD_EXPAND_CACHE_MAX; ++i) { + if (!g_amduatd_expand_cache[i].used) { + continue; + } + amduat_reference_free(&g_amduatd_expand_cache[i].ref); + free(g_amduatd_expand_cache[i].name); + free(g_amduatd_expand_cache[i].latest_hex); + memset(&g_amduatd_expand_cache[i], 0, sizeof(g_amduatd_expand_cache[i])); + } + g_amduatd_expand_cache_next = 0u; +} + +static amduatd_expand_cache_entry_t *amduatd_expand_cache_get_entry( + const amduatd_concepts_t *concepts, + amduat_reference_t ref) { + size_t i; + amduatd_expand_cache_entry_t *slot; + if (concepts == NULL) { + return NULL; + } + if (g_amduatd_expand_cache_edges_len != concepts->edges.len) { + amduatd_expand_cache_clear(); + g_amduatd_expand_cache_edges_len = concepts->edges.len; + } + for (i = 0; i < AMDUATD_EXPAND_CACHE_MAX; ++i) { + if (!g_amduatd_expand_cache[i].used) { + continue; + } + if (amduat_reference_eq(g_amduatd_expand_cache[i].ref, ref)) { + return &g_amduatd_expand_cache[i]; + } + } + slot = &g_amduatd_expand_cache[g_amduatd_expand_cache_next]; + g_amduatd_expand_cache_next = + (g_amduatd_expand_cache_next + 1u) % AMDUATD_EXPAND_CACHE_MAX; + if (slot->used) { + amduat_reference_free(&slot->ref); + free(slot->name); + free(slot->latest_hex); + memset(slot, 0, sizeof(*slot)); + } + if (!amduat_reference_clone(ref, &slot->ref)) { + return NULL; + } + slot->used = true; + return slot; +} + +static bool amduatd_concepts_name_for_ref_cached(amduat_asl_store_t *store, + const amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + amduat_reference_t concept_ref, + char *out_name, + size_t out_cap) { + amduatd_expand_cache_entry_t *entry; + if (out_name == NULL || out_cap == 0u) { + return false; + } + out_name[0] = '\0'; + entry = amduatd_expand_cache_get_entry(concepts, concept_ref); + if (entry == NULL) { + return amduatd_concepts_name_for_ref(store, concepts, dcfg, concept_ref, out_name, out_cap); + } + if (!entry->name_checked) { + char tmp[AMDUAT_ASL_POINTER_NAME_MAX + 1u]; + entry->have_name = amduatd_concepts_name_for_ref(store, + concepts, + dcfg, + concept_ref, + tmp, + sizeof(tmp)); + if (entry->have_name) { + entry->name = strdup(tmp); + if (entry->name == NULL) { + entry->have_name = false; + } + } + entry->name_checked = true; + } + if (!entry->have_name || entry->name == NULL) { + return false; + } + if (strlen(entry->name) >= out_cap) { + return false; + } + strcpy(out_name, entry->name); + return true; +} + +static bool amduatd_concepts_latest_hex_for_ref(amduat_asl_store_t *store, + const amduatd_concepts_t *concepts, + amduat_reference_t concept_ref, + char **out_hex) { + amduatd_expand_cache_entry_t *entry; + amduat_reference_t latest_ref; + if (store == NULL || concepts == NULL || out_hex == NULL) { + return false; + } + *out_hex = NULL; + entry = amduatd_expand_cache_get_entry(concepts, concept_ref); + if (entry != NULL && entry->latest_checked) { + if (entry->latest_hex != NULL) { + *out_hex = strdup(entry->latest_hex); + return *out_hex != NULL; + } + return true; + } + memset(&latest_ref, 0, sizeof(latest_ref)); + if (!amduatd_concepts_resolve_latest(store, concepts, concept_ref, &latest_ref)) { + if (entry != NULL) { + entry->latest_checked = true; + free(entry->latest_hex); + entry->latest_hex = NULL; + } + return true; + } + if (!amduat_asl_ref_encode_hex(latest_ref, out_hex)) { + amduat_reference_free(&latest_ref); + return false; + } + amduat_reference_free(&latest_ref); + if (entry != NULL) { + entry->latest_checked = true; + free(entry->latest_hex); + entry->latest_hex = strdup(*out_hex); + } + return true; +} + +static bool amduatd_graph_cursor_decode(const char *s, size_t *out_cursor) { + uint64_t cursor_u64 = 0u; + if (s == NULL || out_cursor == NULL) { + return false; + } + if (strncmp(s, "g1_", 3) == 0) { + s += 3; + } + if (!amduatd_parse_u64_query(s, &cursor_u64) || + cursor_u64 > (uint64_t)SIZE_MAX) { + return false; + } + *out_cursor = (size_t)cursor_u64; + return true; +} + +static bool amduatd_graph_cursor_encode(size_t cursor, + char *out, + size_t out_cap) { + int n; + if (out == NULL || out_cap == 0u) { + return false; + } + n = snprintf(out, out_cap, "g1_%llu", (unsigned long long)cursor); + return n > 0 && (size_t)n < out_cap; +} + +static bool amduatd_graph_cursor_parse_json(const char **p, + const char *end, + size_t *out_cursor) { + const char *cur; + const char *sv = NULL; + size_t sv_len = 0u; + uint64_t raw = 0u; + char *text = NULL; + bool ok = false; + if (p == NULL || *p == NULL || out_cursor == NULL) { + return false; + } + cur = amduatd_json_skip_ws(*p, end); + if (cur < end && *cur == '"') { + if (!amduatd_json_parse_string_noesc(&cur, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &text)) { + return false; + } + ok = amduatd_graph_cursor_decode(text, out_cursor); + free(text); + if (!ok) { + return false; + } + *p = cur; + return true; + } + if (!amduatd_json_parse_u64(&cur, end, &raw) || raw > (uint64_t)SIZE_MAX) { + return false; + } + *out_cursor = (size_t)raw; + *p = cur; + return true; +} + +static bool amduatd_json_parse_bool_value(const char **p, + const char *end, + bool *out_value) { + const char *cur; + if (p == NULL || *p == NULL || out_value == NULL) { + return false; + } + cur = amduatd_json_skip_ws(*p, end); + if ((size_t)(end - cur) >= 4u && memcmp(cur, "true", 4u) == 0) { + *out_value = true; + *p = cur + 4u; + return true; + } + if ((size_t)(end - cur) >= 5u && memcmp(cur, "false", 5u) == 0) { + *out_value = false; + *p = cur + 5u; + return true; + } + return false; +} + +static bool amduatd_graph_ref_array_contains(const amduat_reference_t *items, + size_t items_len, + amduat_reference_t ref) { + size_t i; + for (i = 0u; i < items_len; ++i) { + if (amduat_reference_eq(items[i], ref)) { + return true; + } + } + return false; +} + +static bool amduatd_graph_ref_array_append_unique(amduat_reference_t **items, + size_t *items_len, + size_t *items_cap, + amduat_reference_t ref) { + amduat_reference_t *next; + size_t next_cap; + if (items == NULL || items_len == NULL || items_cap == NULL) { + return false; + } + if (amduatd_graph_ref_array_contains(*items, *items_len, ref)) { + return true; + } + if (*items_len == *items_cap) { + next_cap = *items_cap != 0u ? (*items_cap * 2u) : 16u; + next = (amduat_reference_t *)realloc(*items, next_cap * sizeof(*next)); + if (next == NULL) { + return false; + } + *items = next; + *items_cap = next_cap; + } + memset(&(*items)[*items_len], 0, sizeof((*items)[*items_len])); + if (!amduat_reference_clone(ref, &(*items)[*items_len])) { + return false; + } + (*items_len)++; + return true; +} + +static void amduatd_graph_ref_array_free(amduat_reference_t *items, + size_t items_len) { + size_t i; + if (items == NULL) { + return; + } + for (i = 0u; i < items_len; ++i) { + amduat_reference_free(&items[i]); + } + free(items); +} + +static void amduatd_scan_select_best(const size_t **scan_indices, + size_t *scan_indices_len, + const size_t *candidate_indices, + size_t candidate_len) { + if (scan_indices == NULL || scan_indices_len == NULL || + candidate_indices == NULL) { + return; + } + if (*scan_indices == NULL || candidate_len < *scan_indices_len) { + *scan_indices = candidate_indices; + *scan_indices_len = candidate_len; + } +} + +static void amduatd_scan_select_best_with_plan(const size_t **scan_indices, + size_t *scan_indices_len, + const size_t *candidate_indices, + size_t candidate_len, + const char **out_plan, + const char *candidate_plan) { + if (scan_indices == NULL || scan_indices_len == NULL || + candidate_indices == NULL) { + return; + } + if (*scan_indices == NULL || candidate_len < *scan_indices_len) { + *scan_indices = candidate_indices; + *scan_indices_len = candidate_len; + if (out_plan != NULL && candidate_plan != NULL) { + *out_plan = candidate_plan; + } + } +} + +static void amduatd_scan_select_best_bucket(const size_t **scan_indices, + size_t *scan_indices_len, + const amduatd_ref_edge_bucket_t *bucket) { + if (bucket == NULL) { + return; + } + amduatd_scan_select_best(scan_indices, + scan_indices_len, + bucket->edge_indices, + bucket->len); +} + +static void amduatd_scan_select_best_bucket_with_plan( + const size_t **scan_indices, + size_t *scan_indices_len, + const amduatd_ref_edge_bucket_t *bucket, + const char **out_plan, + const char *candidate_plan) { + if (bucket == NULL) { + return; + } + amduatd_scan_select_best_with_plan(scan_indices, + scan_indices_len, + bucket->edge_indices, + bucket->len, + out_plan, + candidate_plan); +} + +static void amduatd_scan_select_best_pair_bucket( + const size_t **scan_indices, + size_t *scan_indices_len, + const amduatd_ref_pair_edge_bucket_t *bucket) { + if (bucket == NULL) { + return; + } + amduatd_scan_select_best(scan_indices, + scan_indices_len, + bucket->edge_indices, + bucket->len); +} + +static void amduatd_scan_select_best_pair_bucket_with_plan( + const size_t **scan_indices, + size_t *scan_indices_len, + const amduatd_ref_pair_edge_bucket_t *bucket, + const char **out_plan, + const char *candidate_plan) { + if (bucket == NULL) { + return; + } + amduatd_scan_select_best_with_plan(scan_indices, + scan_indices_len, + bucket->edge_indices, + bucket->len, + out_plan, + candidate_plan); +} + +static bool amduatd_graph_edge_is_tombstoned(const amduatd_concepts_t *concepts, + amduat_reference_t edge_record_ref, + size_t scan_end) { + const amduatd_ref_edge_bucket_t *bucket = NULL; + size_t bi; + if (concepts == NULL) { + return false; + } + if (scan_end > concepts->edges.len) { + scan_end = concepts->edges.len; + } + if (concepts->qindex.built_for_edges_len == concepts->edges.len) { + bucket = amduatd_query_index_find_bucket_const( + concepts->qindex.tombstoned_src_buckets, + concepts->qindex.tombstoned_src_len, + edge_record_ref); + for (bi = 0u; bucket != NULL && bi < bucket->len; ++bi) { + size_t edge_i = bucket->edge_indices[bi]; + if (edge_i < scan_end) { + return true; + } + } + return false; + } + for (bi = 0u; bi < scan_end; ++bi) { + const amduatd_edge_entry_t *entry = &concepts->edges.items[bi]; + if (entry->rel == NULL || strcmp(entry->rel, AMDUATD_REL_TOMBSTONES) != 0) { + continue; + } + if (amduat_reference_eq(entry->src_ref, edge_record_ref)) { + return true; + } + } + return false; +} + +static bool amduatd_graph_append_versions_json(amduatd_strbuf_t *b, + const amduatd_concepts_t *concepts, + amduat_reference_t concept_ref, + size_t scan_end) { + const amduatd_ref_edge_bucket_t *bucket; + size_t bi; + size_t version_count = 0u; + if (b == NULL || concepts == NULL) { + return false; + } + if (!amduatd_strbuf_append_cstr(b, ",\"versions\":[")) { + return false; + } + bucket = amduatd_query_index_find_bucket_const(concepts->qindex.src_buckets, + concepts->qindex.src_len, + concept_ref); + for (bi = 0u; bucket != NULL && bi < bucket->len; ++bi) { + size_t edge_i = bucket->edge_indices[bi]; + const amduatd_edge_entry_t *entry; + char *edge_hex = NULL; + char *ref_hex = NULL; + if (edge_i >= scan_end) { + continue; + } + entry = &concepts->edges.items[edge_i]; + if (entry->rel == NULL || strcmp(entry->rel, AMDUATD_REL_MATERIALIZES) != 0) { + continue; + } + if (amduatd_graph_edge_is_tombstoned(concepts, entry->record_ref, scan_end)) { + continue; + } + if (!amduat_asl_ref_encode_hex(entry->record_ref, &edge_hex) || + !amduat_asl_ref_encode_hex(entry->dst_ref, &ref_hex)) { + free(edge_hex); + free(ref_hex); + continue; + } + if (version_count != 0u) { + if (!amduatd_strbuf_append_char(b, ',')) { + free(edge_hex); + free(ref_hex); + return false; + } + } + if (!amduatd_strbuf_append_cstr(b, "{\"edge_ref\":\"") || + !amduatd_strbuf_append_cstr(b, edge_hex) || + !amduatd_strbuf_append_cstr(b, "\",\"ref\":\"") || + !amduatd_strbuf_append_cstr(b, ref_hex) || + !amduatd_strbuf_append_cstr(b, "\"}")) { + free(edge_hex); + free(ref_hex); + return false; + } + version_count++; + free(edge_hex); + free(ref_hex); + if (version_count >= 64u) { + break; + } + } + return amduatd_strbuf_append_cstr(b, "]"); +} + +static bool amduatd_graph_edge_matches_provenance(const amduatd_concepts_t *concepts, + amduat_reference_t edge_record_ref, + amduat_reference_t provenance_ref, + size_t scan_end) { + const amduatd_ref_edge_bucket_t *bucket; + size_t bi; + if (concepts == NULL) { + return false; + } + bucket = amduatd_query_index_find_bucket_const(concepts->qindex.src_buckets, + concepts->qindex.src_len, + edge_record_ref); + for (bi = 0u; bucket != NULL && bi < bucket->len; ++bi) { + size_t edge_i = bucket->edge_indices[bi]; + const amduatd_edge_entry_t *entry; + if (edge_i >= scan_end) { + continue; + } + entry = &concepts->edges.items[edge_i]; + if (entry->rel == NULL || + strcmp(entry->rel, AMDUATD_REL_HAS_PROVENANCE) != 0) { + continue; + } + if (amduat_reference_eq(entry->src_ref, edge_record_ref) && + amduat_reference_eq(entry->dst_ref, provenance_ref)) { + return true; + } + } + return false; +} + +static bool amduatd_graph_edge_first_provenance_ref(const amduatd_concepts_t *concepts, + amduat_reference_t edge_record_ref, + size_t scan_end, + amduat_reference_t *out_ref) { + const amduatd_ref_edge_bucket_t *bucket; + size_t bi; + if (out_ref != NULL) { + memset(out_ref, 0, sizeof(*out_ref)); + } + if (concepts == NULL || out_ref == NULL) { + return false; + } + bucket = amduatd_query_index_find_bucket_const(concepts->qindex.src_buckets, + concepts->qindex.src_len, + edge_record_ref); + for (bi = 0u; bucket != NULL && bi < bucket->len; ++bi) { + size_t edge_i = bucket->edge_indices[bi]; + const amduatd_edge_entry_t *entry; + if (edge_i >= scan_end) { + continue; + } + entry = &concepts->edges.items[edge_i]; + if (entry->rel == NULL || + strcmp(entry->rel, AMDUATD_REL_HAS_PROVENANCE) != 0) { + continue; + } + if (amduat_reference_eq(entry->src_ref, edge_record_ref) && + amduat_reference_clone(entry->dst_ref, out_ref)) { + return true; + } + } + return false; +} + +typedef struct { + char *subject; + char *predicate; + char *predicate_ref; + char *object; + char *metadata_ref; +} amduatd_graph_import_item_t; + +static void amduatd_graph_import_items_free(amduatd_graph_import_item_t *items, + size_t items_len) { + size_t i; + if (items == NULL) { + return; + } + for (i = 0u; i < items_len; ++i) { + free(items[i].subject); + free(items[i].predicate); + free(items[i].predicate_ref); + free(items[i].object); + free(items[i].metadata_ref); + } + free(items); +} + +static bool amduatd_handle_post_graph_export(int fd, + amduat_asl_store_t *store, + const amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const amduatd_http_req_t *req) { + uint8_t *body = NULL; + const char *p = NULL; + const char *end = NULL; + bool include_tombstoned = true; + bool have_as_of = false; + uint64_t limit_u64 = 256u; + uint64_t max_result_bytes_u64 = 1048576u; + size_t cursor = 0u; + size_t scan_end = 0u; + size_t limit = 256u; + amduat_reference_t *predicate_refs = NULL; + size_t predicate_refs_len = 0u; + size_t predicate_refs_cap = 0u; + amduat_reference_t *root_refs = NULL; + size_t root_refs_len = 0u; + size_t root_refs_cap = 0u; + amduatd_strbuf_t b; + size_t i; + size_t returned = 0u; + size_t scanned = 0u; + size_t next_cursor = 0u; + bool has_more = false; + bool ok = false; + + memset(&b, 0, sizeof(b)); + if (store == NULL || concepts == NULL || req == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "internal error"); + } + if (!amduatd_concepts_ensure_query_index((amduatd_concepts_t *)concepts)) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "index error"); + } + if (req->content_length == 0u) { + return amduatd_send_json_error(fd, 400, "Bad Request", "missing body"); + } + if (req->content_length > (1024u * 1024u)) { + return amduatd_send_json_error(fd, 413, "Payload Too Large", "payload too large"); + } + body = (uint8_t *)malloc(req->content_length); + if (body == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + if (!amduatd_read_exact(fd, body, req->content_length)) { + free(body); + return false; + } + + p = (const char *)body; + end = (const char *)body + req->content_length; + if (!amduatd_json_expect(&p, end, '{')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + for (;;) { + const char *key = NULL; + size_t key_len = 0u; + const char *cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &key, &key_len) || + !amduatd_json_expect(&p, end, ':')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + if (key_len == strlen("as_of") && memcmp(key, "as_of", key_len) == 0) { + if (!amduatd_graph_cursor_parse_json(&p, end, &scan_end)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid as_of"); + goto cleanup; + } + have_as_of = true; + } else if (key_len == strlen("cursor") && memcmp(key, "cursor", key_len) == 0) { + if (!amduatd_graph_cursor_parse_json(&p, end, &cursor)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid cursor"); + goto cleanup; + } + } else if (key_len == strlen("limit") && memcmp(key, "limit", key_len) == 0) { + if (!amduatd_json_parse_u64(&p, end, &limit_u64) || + limit_u64 == 0u || limit_u64 > 2000u) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid limit"); + goto cleanup; + } + } else if (key_len == strlen("include_tombstoned") && + memcmp(key, "include_tombstoned", key_len) == 0) { + if (!amduatd_json_parse_bool_value(&p, end, &include_tombstoned)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid include_tombstoned"); + goto cleanup; + } + } else if (key_len == strlen("max_result_bytes") && + memcmp(key, "max_result_bytes", key_len) == 0) { + if (!amduatd_json_parse_u64(&p, end, &max_result_bytes_u64) || + max_result_bytes_u64 < 1024u || + max_result_bytes_u64 > (8u * 1024u * 1024u)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid max_result_bytes"); + goto cleanup; + } + } else if (key_len == strlen("predicates") && memcmp(key, "predicates", key_len) == 0) { + if (!amduatd_json_expect(&p, end, '[')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicates"); + goto cleanup; + } + for (;;) { + const char *sv = NULL; + size_t sv_len = 0u; + char *value = NULL; + amduat_reference_t predicate_ref; + memset(&predicate_ref, 0, sizeof(predicate_ref)); + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ']') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &value)) { + free(value); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicates"); + goto cleanup; + } + if (!amduatd_resolve_relation_ref(concepts, value, &predicate_ref)) { + free(value); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicate"); + goto cleanup; + } + free(value); + if (!amduatd_graph_ref_array_append_unique(&predicate_refs, + &predicate_refs_len, + &predicate_refs_cap, + predicate_ref)) { + amduat_reference_free(&predicate_ref); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + amduat_reference_free(&predicate_ref); + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == ']') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicates"); + goto cleanup; + } + } else if (key_len == strlen("roots") && memcmp(key, "roots", key_len) == 0) { + if (!amduatd_json_expect(&p, end, '[')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid roots"); + goto cleanup; + } + for (;;) { + const char *sv = NULL; + size_t sv_len = 0u; + char *value = NULL; + amduat_reference_t root_ref; + memset(&root_ref, 0, sizeof(root_ref)); + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ']') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &value)) { + free(value); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid roots"); + goto cleanup; + } + if (!amduatd_resolve_graph_ref(store, concepts, dcfg, value, &root_ref)) { + free(value); + ok = amduatd_send_json_error(fd, 404, "Not Found", "root not found"); + goto cleanup; + } + free(value); + if (!amduatd_graph_ref_array_append_unique(&root_refs, + &root_refs_len, + &root_refs_cap, + root_ref)) { + amduat_reference_free(&root_ref); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + amduat_reference_free(&root_ref); + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == ']') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid roots"); + goto cleanup; + } + } else { + if (!amduatd_json_skip_value(&p, end, 0u)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + } + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + p = amduatd_json_skip_ws(p, end); + if (p != end) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + + if (!have_as_of) { + scan_end = concepts->edges.len; + } else if (scan_end > concepts->edges.len) { + scan_end = concepts->edges.len; + } + if (cursor > scan_end) { + cursor = scan_end; + } + if (limit_u64 > (uint64_t)SIZE_MAX) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid limit"); + goto cleanup; + } + limit = (size_t)limit_u64; + + if (!amduatd_strbuf_append_cstr(&b, "{\"items\":[")) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + + for (i = cursor; i < scan_end; ++i) { + const amduatd_edge_entry_t *entry = &concepts->edges.items[i]; + amduat_reference_t predicate_ref; + amduat_reference_t metadata_ref; + bool tombstoned = false; + bool match = true; + char *subject_hex = NULL; + char *predicate_hex = NULL; + char *object_hex = NULL; + char *edge_hex = NULL; + char *metadata_hex = NULL; + memset(&predicate_ref, 0, sizeof(predicate_ref)); + memset(&metadata_ref, 0, sizeof(metadata_ref)); + + scanned++; + if (!amduatd_relation_entry_ref(concepts, entry->rel, &predicate_ref)) { + continue; + } + if (predicate_refs_len != 0u && + !amduatd_graph_ref_array_contains(predicate_refs, predicate_refs_len, predicate_ref)) { + match = false; + } + if (match && root_refs_len != 0u && + !amduatd_graph_ref_array_contains(root_refs, root_refs_len, entry->src_ref) && + !amduatd_graph_ref_array_contains(root_refs, root_refs_len, entry->dst_ref)) { + match = false; + } + tombstoned = amduatd_graph_edge_is_tombstoned(concepts, entry->record_ref, scan_end); + if (match && !include_tombstoned && tombstoned) { + match = false; + } + if (!match) { + amduat_reference_free(&predicate_ref); + continue; + } + if (!amduat_asl_ref_encode_hex(entry->src_ref, &subject_hex) || + !amduat_asl_ref_encode_hex(predicate_ref, &predicate_hex) || + !amduat_asl_ref_encode_hex(entry->dst_ref, &object_hex) || + !amduat_asl_ref_encode_hex(entry->record_ref, &edge_hex)) { + amduat_reference_free(&predicate_ref); + free(subject_hex); + free(predicate_hex); + free(object_hex); + free(edge_hex); + continue; + } + if (amduatd_graph_edge_first_provenance_ref(concepts, entry->record_ref, scan_end, &metadata_ref)) { + (void)amduat_asl_ref_encode_hex(metadata_ref, &metadata_hex); + } + if (returned != 0u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + returned++; + (void)amduatd_strbuf_append_cstr(&b, "{\"seq\":"); + { + char num[64]; + int n = snprintf(num, sizeof(num), "%llu", (unsigned long long)i); + if (n > 0 && (size_t)n < sizeof(num)) { + (void)amduatd_strbuf_append(&b, num, (size_t)n); + } else { + (void)amduatd_strbuf_append_cstr(&b, "0"); + } + } + (void)amduatd_strbuf_append_cstr(&b, ",\"edge_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, edge_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"subject_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, subject_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"predicate_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, predicate_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"predicate\":\""); + (void)amduatd_strbuf_append_cstr(&b, entry->rel != NULL ? entry->rel : ""); + (void)amduatd_strbuf_append_cstr(&b, "\",\"object_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, object_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"tombstoned\":"); + (void)amduatd_strbuf_append_cstr(&b, tombstoned ? "true" : "false"); + (void)amduatd_strbuf_append_cstr(&b, ",\"metadata_ref\":"); + if (metadata_hex != NULL) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, metadata_hex); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + (void)amduatd_strbuf_append_cstr(&b, "}"); + + amduat_reference_free(&predicate_ref); + amduat_reference_free(&metadata_ref); + free(subject_hex); + free(predicate_hex); + free(object_hex); + free(edge_hex); + free(metadata_hex); + if (returned >= limit) { + next_cursor = i + 1u; + break; + } + } + + if (returned >= limit && next_cursor < scan_end) { + size_t j; + for (j = next_cursor; j < scan_end; ++j) { + const amduatd_edge_entry_t *entry = &concepts->edges.items[j]; + amduat_reference_t predicate_ref; + bool tombstoned = false; + bool match = true; + memset(&predicate_ref, 0, sizeof(predicate_ref)); + if (!amduatd_relation_entry_ref(concepts, entry->rel, &predicate_ref)) { + continue; + } + if (predicate_refs_len != 0u && + !amduatd_graph_ref_array_contains(predicate_refs, predicate_refs_len, predicate_ref)) { + match = false; + } + if (match && root_refs_len != 0u && + !amduatd_graph_ref_array_contains(root_refs, root_refs_len, entry->src_ref) && + !amduatd_graph_ref_array_contains(root_refs, root_refs_len, entry->dst_ref)) { + match = false; + } + tombstoned = amduatd_graph_edge_is_tombstoned(concepts, entry->record_ref, scan_end); + if (match && !include_tombstoned && tombstoned) { + match = false; + } + amduat_reference_free(&predicate_ref); + if (match) { + has_more = true; + break; + } + } + } + + (void)amduatd_strbuf_append_cstr(&b, "],\"next_cursor\":"); + if (has_more) { + char token[64]; + if (!amduatd_graph_cursor_encode(next_cursor, token, sizeof(token))) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto cleanup; + } + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, token); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + (void)amduatd_strbuf_append_cstr(&b, ",\"has_more\":"); + (void)amduatd_strbuf_append_cstr(&b, has_more ? "true" : "false"); + (void)amduatd_strbuf_append_cstr(&b, ",\"snapshot_as_of\":\""); + { + char token[64]; + if (!amduatd_graph_cursor_encode(scan_end, token, sizeof(token))) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto cleanup; + } + (void)amduatd_strbuf_append_cstr(&b, token); + } + (void)amduatd_strbuf_append_cstr(&b, "\",\"stats\":{"); + { + char num[64]; + int n = snprintf(num, sizeof(num), "%llu", (unsigned long long)scanned); + if (n > 0 && (size_t)n < sizeof(num)) { + (void)amduatd_strbuf_append_cstr(&b, "\"scanned_edges\":"); + (void)amduatd_strbuf_append(&b, num, (size_t)n); + } + n = snprintf(num, sizeof(num), "%llu", (unsigned long long)returned); + if (n > 0 && (size_t)n < sizeof(num)) { + (void)amduatd_strbuf_append_cstr(&b, ",\"exported_items\":"); + (void)amduatd_strbuf_append(&b, num, (size_t)n); + } + } + (void)amduatd_strbuf_append_cstr(&b, "}}\n"); + if (b.len > (size_t)max_result_bytes_u64) { + ok = amduatd_send_json_error(fd, 422, "Unprocessable Entity", "result too large"); + goto cleanup; + } + ok = amduatd_http_send_json(fd, 200, "OK", b.data, false); + +cleanup: + free(body); + amduatd_graph_ref_array_free(predicate_refs, predicate_refs_len); + amduatd_graph_ref_array_free(root_refs, root_refs_len); + amduatd_strbuf_free(&b); + return ok; +} + +static bool amduatd_handle_post_graph_import(int fd, + amduat_asl_store_t *store, + amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const amduatd_http_req_t *req) { + enum { AMDUATD_IMPORT_MAX_ITEMS = 5000 }; + uint8_t *body = NULL; + const char *p = NULL; + const char *end = NULL; + amduatd_graph_import_item_t *items = NULL; + size_t items_len = 0u; + size_t items_cap = 0u; + bool fail_fast = true; + amduat_octets_t actor = amduat_octets(NULL, 0u); + amduatd_strbuf_t results; + size_t i; + size_t applied = 0u; + bool overall_ok = true; + bool ok = false; + + memset(&results, 0, sizeof(results)); + if (store == NULL || concepts == NULL || req == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "internal error"); + } + if (!amduatd_concepts_ensure_query_index(concepts)) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "index error"); + } + if (req->has_actor) { + actor = req->actor; + } + if (req->content_length == 0u) { + return amduatd_send_json_error(fd, 400, "Bad Request", "missing body"); + } + if (req->content_length > (2u * 1024u * 1024u)) { + return amduatd_send_json_error(fd, 413, "Payload Too Large", "payload too large"); + } + body = (uint8_t *)malloc(req->content_length); + if (body == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + if (!amduatd_read_exact(fd, body, req->content_length)) { + free(body); + return false; + } + + p = (const char *)body; + end = (const char *)body + req->content_length; + if (!amduatd_json_expect(&p, end, '{')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + for (;;) { + const char *key = NULL; + size_t key_len = 0u; + const char *cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &key, &key_len) || + !amduatd_json_expect(&p, end, ':')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + if (key_len == strlen("mode") && memcmp(key, "mode", key_len) == 0) { + const char *sv = NULL; + size_t sv_len = 0u; + char *mode = NULL; + if (!amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &mode)) { + free(mode); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid mode"); + goto cleanup; + } + if (strcmp(mode, "fail_fast") == 0) { + fail_fast = true; + } else if (strcmp(mode, "continue_on_error") == 0) { + fail_fast = false; + } else { + free(mode); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid mode"); + goto cleanup; + } + free(mode); + } else if (key_len == strlen("items") && memcmp(key, "items", key_len) == 0) { + if (!amduatd_json_expect(&p, end, '[')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid items"); + goto cleanup; + } + for (;;) { + amduatd_graph_import_item_t item; + memset(&item, 0, sizeof(item)); + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ']') { + p = cur + 1; + break; + } + if (!amduatd_json_expect(&p, end, '{')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid items"); + goto cleanup; + } + for (;;) { + const char *ik = NULL; + size_t ik_len = 0u; + const char *sv = NULL; + size_t sv_len = 0u; + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &ik, &ik_len) || + !amduatd_json_expect(&p, end, ':')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid items"); + free(item.subject); + free(item.predicate); + free(item.predicate_ref); + free(item.object); + free(item.metadata_ref); + goto cleanup; + } + if ((ik_len == strlen("subject_ref") && memcmp(ik, "subject_ref", ik_len) == 0) || + (ik_len == strlen("subject") && memcmp(ik, "subject", ik_len) == 0)) { + if (item.subject != NULL || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &item.subject)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid item subject"); + free(item.subject); + free(item.predicate); + free(item.predicate_ref); + free(item.object); + free(item.metadata_ref); + goto cleanup; + } + } else if ((ik_len == strlen("predicate_ref") && memcmp(ik, "predicate_ref", ik_len) == 0)) { + if (item.predicate_ref != NULL || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &item.predicate_ref)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid item predicate_ref"); + free(item.subject); + free(item.predicate); + free(item.predicate_ref); + free(item.object); + free(item.metadata_ref); + goto cleanup; + } + } else if ((ik_len == strlen("predicate") && memcmp(ik, "predicate", ik_len) == 0)) { + if (item.predicate != NULL || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &item.predicate)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid item predicate"); + free(item.subject); + free(item.predicate); + free(item.predicate_ref); + free(item.object); + free(item.metadata_ref); + goto cleanup; + } + } else if ((ik_len == strlen("object_ref") && memcmp(ik, "object_ref", ik_len) == 0) || + (ik_len == strlen("object") && memcmp(ik, "object", ik_len) == 0)) { + if (item.object != NULL || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &item.object)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid item object"); + free(item.subject); + free(item.predicate); + free(item.predicate_ref); + free(item.object); + free(item.metadata_ref); + goto cleanup; + } + } else if (ik_len == strlen("metadata_ref") && memcmp(ik, "metadata_ref", ik_len) == 0) { + if (item.metadata_ref != NULL || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &item.metadata_ref)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid item metadata_ref"); + free(item.subject); + free(item.predicate); + free(item.predicate_ref); + free(item.object); + free(item.metadata_ref); + goto cleanup; + } + } else { + if (!amduatd_json_skip_value(&p, end, 0u)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid items"); + free(item.subject); + free(item.predicate); + free(item.predicate_ref); + free(item.object); + free(item.metadata_ref); + goto cleanup; + } + } + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid items"); + free(item.subject); + free(item.predicate); + free(item.predicate_ref); + free(item.object); + free(item.metadata_ref); + goto cleanup; + } + if (items_len >= AMDUATD_IMPORT_MAX_ITEMS) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "too many items"); + free(item.subject); + free(item.predicate); + free(item.predicate_ref); + free(item.object); + free(item.metadata_ref); + goto cleanup; + } + if (items_len == items_cap) { + size_t next_cap = items_cap != 0u ? (items_cap * 2u) : 128u; + amduatd_graph_import_item_t *next = + (amduatd_graph_import_item_t *)realloc(items, next_cap * sizeof(*next)); + if (next == NULL) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + free(item.subject); + free(item.predicate); + free(item.predicate_ref); + free(item.object); + free(item.metadata_ref); + goto cleanup; + } + items = next; + items_cap = next_cap; + } + items[items_len++] = item; + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == ']') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid items"); + goto cleanup; + } + } else { + if (!amduatd_json_skip_value(&p, end, 0u)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + } + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + p = amduatd_json_skip_ws(p, end); + if (p != end) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + if (items_len == 0u) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "missing items"); + goto cleanup; + } + + if (!amduatd_strbuf_append_cstr(&results, "[")) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + for (i = 0u; i < items_len; ++i) { + amduat_reference_t subject_ref; + amduat_reference_t predicate_ref; + amduat_reference_t object_ref; + amduat_reference_t edge_ref; + amduat_reference_t metadata_target_ref; + amduat_reference_t prov_edge_ref; + int status_code = 200; + const char *error = NULL; + bool applied_item = false; + char *edge_hex = NULL; + memset(&subject_ref, 0, sizeof(subject_ref)); + memset(&predicate_ref, 0, sizeof(predicate_ref)); + memset(&object_ref, 0, sizeof(object_ref)); + memset(&edge_ref, 0, sizeof(edge_ref)); + memset(&metadata_target_ref, 0, sizeof(metadata_target_ref)); + memset(&prov_edge_ref, 0, sizeof(prov_edge_ref)); + + if (items[i].subject == NULL || items[i].object == NULL || + (items[i].predicate == NULL && items[i].predicate_ref == NULL)) { + status_code = 400; + error = "missing required fields"; + } else if (!amduatd_graph_schema_validate_provenance_write(items[i].metadata_ref != NULL, + &status_code, + &error)) { + } else if (!amduatd_resolve_graph_ref(store, concepts, dcfg, items[i].subject, &subject_ref)) { + status_code = 404; + error = "subject not found"; + } else if (items[i].predicate_ref != NULL && + !amduat_asl_ref_decode_hex(items[i].predicate_ref, &predicate_ref)) { + status_code = 400; + error = "invalid predicate_ref"; + } else if (items[i].predicate_ref == NULL && + !amduatd_resolve_relation_ref(concepts, items[i].predicate, &predicate_ref)) { + status_code = 400; + error = "invalid predicate"; + } else if (!amduatd_graph_schema_validate_predicate_write(predicate_ref, + &status_code, + &error)) { + } else if (!amduatd_resolve_graph_ref(store, concepts, dcfg, items[i].object, &object_ref)) { + status_code = 404; + error = "object not found"; + } else if (!amduatd_concepts_put_edge(store, + concepts, + subject_ref, + object_ref, + predicate_ref, + actor, + &edge_ref)) { + status_code = 500; + error = "store error"; + } else if (items[i].metadata_ref != NULL && + !amduatd_resolve_graph_ref(store, + concepts, + dcfg, + items[i].metadata_ref, + &metadata_target_ref)) { + status_code = 404; + error = "metadata_ref not found"; + } else if (items[i].metadata_ref != NULL && + !amduatd_concepts_put_edge(store, + concepts, + edge_ref, + metadata_target_ref, + concepts->rel_has_provenance_ref, + actor, + &prov_edge_ref)) { + status_code = 500; + error = "store error"; + } else { + applied_item = true; + applied++; + if (!amduat_asl_ref_encode_hex(edge_ref, &edge_hex)) { + status_code = 500; + error = "encode error"; + applied_item = false; + applied--; + } + } + + if (i != 0u) { + (void)amduatd_strbuf_append_char(&results, ','); + } + (void)amduatd_strbuf_append_cstr(&results, "{\"index\":"); + { + char num[64]; + int n = snprintf(num, sizeof(num), "%llu", (unsigned long long)i); + if (n > 0 && (size_t)n < sizeof(num)) { + (void)amduatd_strbuf_append(&results, num, (size_t)n); + } else { + (void)amduatd_strbuf_append_cstr(&results, "0"); + } + } + (void)amduatd_strbuf_append_cstr(&results, ",\"status\":\""); + (void)amduatd_strbuf_append_cstr(&results, applied_item ? "applied" : "error"); + (void)amduatd_strbuf_append_cstr(&results, "\",\"code\":"); + { + char num[64]; + int n = snprintf(num, sizeof(num), "%d", status_code); + if (n > 0 && (size_t)n < sizeof(num)) { + (void)amduatd_strbuf_append(&results, num, (size_t)n); + } else { + (void)amduatd_strbuf_append_cstr(&results, "500"); + } + } + (void)amduatd_strbuf_append_cstr(&results, ",\"error\":"); + if (error != NULL) { + (void)amduatd_strbuf_append_cstr(&results, "\""); + (void)amduatd_strbuf_append_cstr(&results, error); + (void)amduatd_strbuf_append_cstr(&results, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&results, "null"); + } + (void)amduatd_strbuf_append_cstr(&results, ",\"edge_ref\":"); + if (edge_hex != NULL) { + (void)amduatd_strbuf_append_cstr(&results, "\""); + (void)amduatd_strbuf_append_cstr(&results, edge_hex); + (void)amduatd_strbuf_append_cstr(&results, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&results, "null"); + } + (void)amduatd_strbuf_append_cstr(&results, "}"); + + if (!applied_item) { + overall_ok = false; + if (fail_fast) { + amduat_reference_free(&subject_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&object_ref); + amduat_reference_free(&edge_ref); + amduat_reference_free(&metadata_target_ref); + amduat_reference_free(&prov_edge_ref); + free(edge_hex); + break; + } + } + amduat_reference_free(&subject_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&object_ref); + amduat_reference_free(&edge_ref); + amduat_reference_free(&metadata_target_ref); + amduat_reference_free(&prov_edge_ref); + free(edge_hex); + } + + (void)amduatd_strbuf_append_cstr(&results, "]"); + { + char prefix[128]; + int n = snprintf(prefix, + sizeof(prefix), + "{\"ok\":%s,\"applied\":%llu,\"results\":", + overall_ok ? "true" : "false", + (unsigned long long)applied); + amduatd_strbuf_t out; + memset(&out, 0, sizeof(out)); + if (n <= 0 || (size_t)n >= sizeof(prefix) || + !amduatd_strbuf_append_cstr(&out, prefix) || + !amduatd_strbuf_append_cstr(&out, results.data) || + !amduatd_strbuf_append_cstr(&out, "}\n")) { + amduatd_strbuf_free(&out); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + ok = amduatd_http_send_json(fd, overall_ok ? 200 : 207, overall_ok ? "OK" : "Multi-Status", out.data, false); + amduatd_strbuf_free(&out); + } + +cleanup: + free(body); + amduatd_graph_import_items_free(items, items_len); + amduatd_strbuf_free(&results); + return ok; +} + +static bool amduatd_json_parse_double_token(const char **p, + const char *end, + double *out_value) { + const char *cur = amduatd_json_skip_ws(*p, end); + const char *start = cur; + char *num_text = NULL; + char *endptr = NULL; + double value = 0.0; + if (p == NULL || *p == NULL || out_value == NULL) { + return false; + } + if (cur >= end) { + return false; + } + if (*cur == '-' || *cur == '+') { + cur++; + } + while (cur < end && + ((*cur >= '0' && *cur <= '9') || + *cur == '.' || *cur == 'e' || *cur == 'E' || + *cur == '-' || *cur == '+')) { + cur++; + } + if (cur <= start) { + return false; + } + num_text = (char *)malloc((size_t)(cur - start) + 1u); + if (num_text == NULL) { + return false; + } + memcpy(num_text, start, (size_t)(cur - start)); + num_text[cur - start] = '\0'; + value = strtod(num_text, &endptr); + if (endptr == NULL || *endptr != '\0') { + free(num_text); + return false; + } + free(num_text); + *out_value = value; + *p = cur; + return true; +} + +static bool amduatd_graph_parse_provenance_confidence_artifact(amduat_artifact_t artifact, + double *out_confidence, + bool *out_have_confidence) { + char *text = NULL; + const char *p = NULL; + const char *end = NULL; + bool ok = false; + + if (out_confidence != NULL) { + *out_confidence = 0.0; + } + if (out_have_confidence != NULL) { + *out_have_confidence = false; + } + if (out_confidence == NULL || out_have_confidence == NULL) { + return false; + } + if (artifact.bytes.len == 0u || artifact.bytes.data == NULL) { + return false; + } + + text = (char *)malloc(artifact.bytes.len + 1u); + if (text == NULL) { + return false; + } + memcpy(text, artifact.bytes.data, artifact.bytes.len); + text[artifact.bytes.len] = '\0'; + p = text; + end = text + artifact.bytes.len; + + if (!amduatd_json_expect(&p, end, '{')) { + goto cleanup; + } + for (;;) { + const char *key = NULL; + size_t key_len = 0u; + const char *cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == '}') { + ok = true; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &key, &key_len) || + !amduatd_json_expect(&p, end, ':')) { + goto cleanup; + } + if (key_len == strlen("confidence") && + memcmp(key, "confidence", key_len) == 0) { + const char *sv = NULL; + size_t sv_len = 0u; + char *sv_text = NULL; + uint64_t conf_u64 = 0u; + double conf = 0.0; + if (amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len)) { + sv_text = (char *)malloc(sv_len + 1u); + if (sv_text == NULL) { + goto cleanup; + } + memcpy(sv_text, sv, sv_len); + sv_text[sv_len] = '\0'; + conf = strtod(sv_text, NULL); + free(sv_text); + *out_confidence = conf; + *out_have_confidence = true; + } else if (amduatd_json_parse_u64(&p, end, &conf_u64)) { + *out_confidence = (double)conf_u64; + *out_have_confidence = true; + } else if (amduatd_json_parse_double_token(&p, end, &conf)) { + *out_confidence = conf; + *out_have_confidence = true; + } else { + if (!amduatd_json_skip_value(&p, end, 0u)) { + goto cleanup; + } + } + } else { + if (!amduatd_json_skip_value(&p, end, 0u)) { + goto cleanup; + } + } + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == '}') { + ok = true; + break; + } + goto cleanup; + } + +cleanup: + free(text); + return ok; +} + +static bool amduatd_graph_edge_min_confidence_ok(amduat_asl_store_t *store, + const amduatd_concepts_t *concepts, + amduat_reference_t edge_record_ref, + size_t scan_end, + double min_confidence, + bool *out_have_confidence, + double *out_confidence) { + const amduatd_ref_edge_bucket_t *bucket; + size_t bi; + bool have_confidence = false; + double confidence = 0.0; + + if (out_have_confidence != NULL) { + *out_have_confidence = false; + } + if (out_confidence != NULL) { + *out_confidence = 0.0; + } + if (store == NULL || concepts == NULL) { + return false; + } + + bucket = amduatd_query_index_find_bucket_const(concepts->qindex.src_buckets, + concepts->qindex.src_len, + edge_record_ref); + for (bi = 0u; bucket != NULL && bi < bucket->len; ++bi) { + size_t edge_i = bucket->edge_indices[bi]; + const amduatd_edge_entry_t *entry; + amduat_artifact_t artifact; + amduat_asl_store_error_t err; + if (edge_i >= scan_end) { + continue; + } + entry = &concepts->edges.items[edge_i]; + if (entry->rel == NULL || + strcmp(entry->rel, AMDUATD_REL_HAS_PROVENANCE) != 0) { + continue; + } + if (!amduat_reference_eq(entry->src_ref, edge_record_ref)) { + continue; + } + memset(&artifact, 0, sizeof(artifact)); + err = amduat_asl_store_get(store, entry->dst_ref, &artifact); + if (err != AMDUAT_ASL_STORE_OK) { + continue; + } + (void)amduatd_graph_parse_provenance_confidence_artifact(artifact, + &confidence, + &have_confidence); + amduat_asl_artifact_free(&artifact); + if (have_confidence) { + if (out_have_confidence != NULL) { + *out_have_confidence = true; + } + if (out_confidence != NULL) { + *out_confidence = confidence; + } + return confidence >= min_confidence; + } + } + return false; +} + +typedef struct { + size_t edge_index; + uint64_t depth; + bool goal_match; + bool have_confidence; + double confidence; +} amduatd_retrieve_edge_t; + +typedef struct { + amduat_reference_t node_ref; + uint64_t depth; +} amduatd_retrieve_node_t; + +static void amduatd_retrieve_nodes_free(amduatd_retrieve_node_t *nodes, + size_t nodes_len) { + size_t i; + if (nodes == NULL) { + return; + } + for (i = 0u; i < nodes_len; ++i) { + amduat_reference_free(&nodes[i].node_ref); + } + free(nodes); +} + +static bool amduatd_retrieve_nodes_append_unique(amduatd_retrieve_node_t **nodes, + size_t *nodes_len, + size_t *nodes_cap, + amduat_reference_t ref, + uint64_t depth, + size_t *out_index, + bool *out_inserted) { + size_t i; + amduatd_retrieve_node_t *next; + size_t next_cap; + if (nodes == NULL || nodes_len == NULL || nodes_cap == NULL) { + return false; + } + for (i = 0u; i < *nodes_len; ++i) { + if (amduat_reference_eq((*nodes)[i].node_ref, ref)) { + if (out_index != NULL) { + *out_index = i; + } + if (out_inserted != NULL) { + *out_inserted = false; + } + return true; + } + } + if (*nodes_len == *nodes_cap) { + next_cap = (*nodes_cap != 0u) ? (*nodes_cap * 2u) : 64u; + next = (amduatd_retrieve_node_t *)realloc(*nodes, next_cap * sizeof(*next)); + if (next == NULL) { + return false; + } + *nodes = next; + *nodes_cap = next_cap; + } + memset(&(*nodes)[*nodes_len], 0, sizeof((*nodes)[*nodes_len])); + if (!amduat_reference_clone(ref, &(*nodes)[*nodes_len].node_ref)) { + return false; + } + (*nodes)[*nodes_len].depth = depth; + if (out_index != NULL) { + *out_index = *nodes_len; + } + if (out_inserted != NULL) { + *out_inserted = true; + } + (*nodes_len)++; + return true; +} + +static bool amduatd_retrieve_edges_append(amduatd_retrieve_edge_t **edges, + size_t *edges_len, + size_t *edges_cap, + size_t edge_index, + uint64_t depth, + bool goal_match, + bool have_confidence, + double confidence) { + amduatd_retrieve_edge_t *next; + size_t next_cap; + if (edges == NULL || edges_len == NULL || edges_cap == NULL) { + return false; + } + if (*edges_len == *edges_cap) { + next_cap = (*edges_cap != 0u) ? (*edges_cap * 2u) : 256u; + next = (amduatd_retrieve_edge_t *)realloc(*edges, next_cap * sizeof(*next)); + if (next == NULL) { + return false; + } + *edges = next; + *edges_cap = next_cap; + } + (*edges)[*edges_len].edge_index = edge_index; + (*edges)[*edges_len].depth = depth; + (*edges)[*edges_len].goal_match = goal_match; + (*edges)[*edges_len].have_confidence = have_confidence; + (*edges)[*edges_len].confidence = confidence; + (*edges_len)++; + return true; +} + +static bool amduatd_handle_post_graph_retrieve(int fd, + amduat_asl_store_t *store, + const amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const amduatd_http_req_t *req) { + uint8_t *body = NULL; + const char *p = NULL; + const char *end = NULL; + const size_t traversal_edge_cap = 50000u; + const size_t traversal_node_cap = 20000u; + bool include_versions = false; + bool include_tombstoned = false; + bool have_as_of = false; + bool have_provenance_min_confidence = false; + bool always_empty = false; + uint64_t max_depth_u64 = 2u; + uint64_t max_fanout_u64 = 2048u; + uint64_t max_result_bytes_u64 = 1048576u; + uint64_t limit_nodes_u64 = 256u; + uint64_t limit_edges_u64 = 256u; + size_t max_depth = 2u; + size_t max_fanout = 2048u; + size_t limit_nodes = 256u; + size_t limit_edges = 256u; + size_t scan_end = 0u; + double provenance_min_confidence = 0.0; + amduat_reference_t *root_refs = NULL; + size_t root_refs_len = 0u; + size_t root_refs_cap = 0u; + amduat_reference_t *goal_predicate_refs = NULL; + size_t goal_predicate_refs_len = 0u; + size_t goal_predicate_refs_cap = 0u; + amduatd_retrieve_node_t *seen_nodes = NULL; + size_t seen_nodes_len = 0u; + size_t seen_nodes_cap = 0u; + size_t *queue = NULL; + size_t queue_len = 0u; + size_t queue_cap = 0u; + size_t q_pos = 0u; + amduatd_retrieve_edge_t *traversed_edges = NULL; + size_t traversed_edges_len = 0u; + size_t traversed_edges_cap = 0u; + uint8_t *seen_edges = NULL; + bool traversal_truncated = false; + size_t *page_edge_positions = NULL; + size_t page_edge_positions_len = 0u; + size_t page_edge_positions_cap = 0u; + amduat_reference_t *response_nodes = NULL; + size_t response_nodes_len = 0u; + size_t response_nodes_cap = 0u; + bool truncated = false; + bool ok = false; + size_t scanned_edges = 0u; + amduatd_strbuf_t b; + size_t i; + + memset(&b, 0, sizeof(b)); + + if (store == NULL || concepts == NULL || req == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "internal error"); + } + if (!amduatd_concepts_ensure_query_index((amduatd_concepts_t *)concepts)) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "index error"); + } + if (req->content_length == 0u) { + return amduatd_send_json_error(fd, 400, "Bad Request", "missing body"); + } + if (req->content_length > (1024u * 1024u)) { + return amduatd_send_json_error(fd, 413, "Payload Too Large", "payload too large"); + } + body = (uint8_t *)malloc(req->content_length); + if (body == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + if (!amduatd_read_exact(fd, body, req->content_length)) { + free(body); + return false; + } + + p = (const char *)body; + end = (const char *)body + req->content_length; + if (!amduatd_json_expect(&p, end, '{')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + for (;;) { + const char *key = NULL; + size_t key_len = 0u; + const char *cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &key, &key_len) || + !amduatd_json_expect(&p, end, ':')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + if (key_len == strlen("roots") && memcmp(key, "roots", key_len) == 0) { + if (root_refs_len != 0u || !amduatd_json_expect(&p, end, '[')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid roots"); + goto cleanup; + } + for (;;) { + const char *sv = NULL; + size_t sv_len = 0u; + char *value = NULL; + amduat_reference_t root_ref; + memset(&root_ref, 0, sizeof(root_ref)); + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ']') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &value)) { + free(value); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid roots"); + goto cleanup; + } + if (!amduatd_resolve_graph_ref(store, concepts, dcfg, value, &root_ref)) { + free(value); + ok = amduatd_send_json_error(fd, 404, "Not Found", "root not found"); + goto cleanup; + } + free(value); + if (!amduatd_graph_ref_array_append_unique(&root_refs, + &root_refs_len, + &root_refs_cap, + root_ref)) { + amduat_reference_free(&root_ref); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + amduat_reference_free(&root_ref); + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == ']') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid roots"); + goto cleanup; + } + } else if (key_len == strlen("goal_predicates") && + memcmp(key, "goal_predicates", key_len) == 0) { + if (!amduatd_json_expect(&p, end, '[')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid goal_predicates"); + goto cleanup; + } + for (;;) { + const char *sv = NULL; + size_t sv_len = 0u; + char *value = NULL; + amduat_reference_t predicate_ref; + memset(&predicate_ref, 0, sizeof(predicate_ref)); + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ']') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &value)) { + free(value); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid goal_predicates"); + goto cleanup; + } + if (!amduatd_resolve_relation_ref(concepts, value, &predicate_ref)) { + free(value); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid goal_predicate"); + goto cleanup; + } + free(value); + if (!amduatd_graph_ref_array_append_unique(&goal_predicate_refs, + &goal_predicate_refs_len, + &goal_predicate_refs_cap, + predicate_ref)) { + amduat_reference_free(&predicate_ref); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + amduat_reference_free(&predicate_ref); + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == ']') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid goal_predicates"); + goto cleanup; + } + } else if (key_len == strlen("max_depth") && memcmp(key, "max_depth", key_len) == 0) { + if (!amduatd_json_parse_u64(&p, end, &max_depth_u64) || max_depth_u64 > 32u) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid max_depth"); + goto cleanup; + } + } else if (key_len == strlen("max_fanout") && memcmp(key, "max_fanout", key_len) == 0) { + if (!amduatd_json_parse_u64(&p, end, &max_fanout_u64) || + max_fanout_u64 == 0u || max_fanout_u64 > 10000u) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid max_fanout"); + goto cleanup; + } + } else if (key_len == strlen("include_versions") && + memcmp(key, "include_versions", key_len) == 0) { + if (!amduatd_json_parse_bool_value(&p, end, &include_versions)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid include_versions"); + goto cleanup; + } + } else if (key_len == strlen("include_tombstoned") && + memcmp(key, "include_tombstoned", key_len) == 0) { + if (!amduatd_json_parse_bool_value(&p, end, &include_tombstoned)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid include_tombstoned"); + goto cleanup; + } + } else if (key_len == strlen("as_of") && memcmp(key, "as_of", key_len) == 0) { + if (!amduatd_graph_cursor_parse_json(&p, end, &scan_end)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid as_of"); + goto cleanup; + } + have_as_of = true; + } else if (key_len == strlen("provenance_min_confidence") && + memcmp(key, "provenance_min_confidence", key_len) == 0) { + uint64_t conf_u64 = 0u; + const char *sv = NULL; + size_t sv_len = 0u; + char *conf_text = NULL; + double conf = 0.0; + if (amduatd_json_parse_u64(&p, end, &conf_u64)) { + conf = (double)conf_u64; + } else if (amduatd_json_parse_double_token(&p, end, &conf)) { + } else if (amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len)) { + conf_text = (char *)malloc(sv_len + 1u); + if (conf_text == NULL) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + memcpy(conf_text, sv, sv_len); + conf_text[sv_len] = '\0'; + conf = strtod(conf_text, NULL); + free(conf_text); + } else { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid provenance_min_confidence"); + goto cleanup; + } + if (conf < 0.0) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid provenance_min_confidence"); + goto cleanup; + } + provenance_min_confidence = conf; + have_provenance_min_confidence = true; + } else if (key_len == strlen("limit_nodes") && + memcmp(key, "limit_nodes", key_len) == 0) { + if (!amduatd_json_parse_u64(&p, end, &limit_nodes_u64) || + limit_nodes_u64 == 0u || limit_nodes_u64 > 2000u) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid limit_nodes"); + goto cleanup; + } + } else if (key_len == strlen("limit_edges") && + memcmp(key, "limit_edges", key_len) == 0) { + if (!amduatd_json_parse_u64(&p, end, &limit_edges_u64) || + limit_edges_u64 == 0u || limit_edges_u64 > 2000u) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid limit_edges"); + goto cleanup; + } + } else if (key_len == strlen("max_result_bytes") && + memcmp(key, "max_result_bytes", key_len) == 0) { + if (!amduatd_json_parse_u64(&p, end, &max_result_bytes_u64) || + max_result_bytes_u64 < 1024u || + max_result_bytes_u64 > (8u * 1024u * 1024u)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid max_result_bytes"); + goto cleanup; + } + } else { + if (!amduatd_json_skip_value(&p, end, 0u)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + } + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + p = amduatd_json_skip_ws(p, end); + if (p != end) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + if (root_refs_len == 0u) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "missing roots"); + goto cleanup; + } + + if (!have_as_of) { + scan_end = concepts->edges.len; + } else if (scan_end > concepts->edges.len) { + scan_end = concepts->edges.len; + } + max_depth = (size_t)max_depth_u64; + max_fanout = (size_t)max_fanout_u64; + limit_nodes = (size_t)limit_nodes_u64; + limit_edges = (size_t)limit_edges_u64; + if (scan_end > 0u) { + seen_edges = (uint8_t *)calloc(scan_end, sizeof(*seen_edges)); + if (seen_edges == NULL) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + } + + for (i = 0u; i < root_refs_len; ++i) { + size_t node_index = 0u; + bool inserted = false; + if (!amduatd_retrieve_nodes_append_unique(&seen_nodes, + &seen_nodes_len, + &seen_nodes_cap, + root_refs[i], + 0u, + &node_index, + &inserted)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + if (inserted && + !amduatd_query_index_append_size_t(&queue, &queue_len, &queue_cap, node_index)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + } + + while (!always_empty && q_pos < queue_len) { + const amduatd_retrieve_node_t *cur = &seen_nodes[queue[q_pos++]]; + size_t bi = 0u; + int pass = 0; + if (cur->depth >= max_depth) { + continue; + } + for (pass = 0; pass < 2; ++pass) { + bool outgoing_pass = (pass == 0); + const size_t *edge_indices = NULL; + size_t edge_indices_len = 0u; + size_t fanout_seen = 0u; + size_t pi = 0u; + size_t predicate_iters = 1u; + if (goal_predicate_refs_len > 1u) { + predicate_iters = goal_predicate_refs_len; + } + for (pi = 0u; pi < predicate_iters; ++pi) { + if (goal_predicate_refs_len == 0u) { + const amduatd_ref_edge_bucket_t *bucket = + outgoing_pass + ? amduatd_query_index_find_bucket_const(concepts->qindex.src_buckets, + concepts->qindex.src_len, + cur->node_ref) + : amduatd_query_index_find_bucket_const(concepts->qindex.dst_buckets, + concepts->qindex.dst_len, + cur->node_ref); + if (bucket != NULL) { + edge_indices = bucket->edge_indices; + edge_indices_len = bucket->len; + } else { + edge_indices = NULL; + edge_indices_len = 0u; + } + } else { + const amduatd_ref_pair_edge_bucket_t *pair_bucket = + outgoing_pass + ? amduatd_query_index_find_pair_bucket_const( + concepts->qindex.src_predicate_buckets, + concepts->qindex.src_predicate_len, + cur->node_ref, + goal_predicate_refs[pi]) + : amduatd_query_index_find_pair_bucket_const( + concepts->qindex.dst_predicate_buckets, + concepts->qindex.dst_predicate_len, + cur->node_ref, + goal_predicate_refs[pi]); + if (pair_bucket != NULL) { + edge_indices = pair_bucket->edge_indices; + edge_indices_len = pair_bucket->len; + } else { + edge_indices = NULL; + edge_indices_len = 0u; + } + } + for (bi = 0u; bi < edge_indices_len; ++bi) { + size_t edge_i = edge_indices[bi]; + const amduatd_edge_entry_t *entry; + amduat_reference_t rel_ref; + amduat_reference_t next_ref; + size_t next_index = 0u; + bool inserted = false; + bool goal_match = false; + bool have_confidence = false; + double confidence = 0.0; + if (edge_i >= scan_end) { + continue; + } + scanned_edges++; + fanout_seen++; + if (fanout_seen > max_fanout) { + traversal_truncated = true; + goto traversal_done; + } + if (seen_edges != NULL && seen_edges[edge_i] != 0u) { + continue; + } + entry = &concepts->edges.items[edge_i]; + memset(&rel_ref, 0, sizeof(rel_ref)); + if (!amduatd_relation_entry_ref(concepts, entry->rel, &rel_ref)) { + continue; + } + goal_match = (goal_predicate_refs_len == 0u) || + amduatd_graph_ref_array_contains(goal_predicate_refs, + goal_predicate_refs_len, + rel_ref); + amduat_reference_free(&rel_ref); + if (!goal_match) { + continue; + } + if (!include_tombstoned && + amduatd_graph_edge_is_tombstoned(concepts, entry->record_ref, scan_end)) { + continue; + } + if (have_provenance_min_confidence && + !amduatd_graph_edge_min_confidence_ok(store, + concepts, + entry->record_ref, + scan_end, + provenance_min_confidence, + &have_confidence, + &confidence)) { + continue; + } + if (traversed_edges_len >= traversal_edge_cap) { + traversal_truncated = true; + goto traversal_done; + } + if (seen_edges != NULL) { + seen_edges[edge_i] = 1u; + } + if (!amduatd_retrieve_edges_append(&traversed_edges, + &traversed_edges_len, + &traversed_edges_cap, + edge_i, + cur->depth + 1u, + goal_match, + have_confidence, + confidence)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + next_ref = outgoing_pass ? entry->dst_ref : entry->src_ref; + if (!amduatd_retrieve_nodes_append_unique(&seen_nodes, + &seen_nodes_len, + &seen_nodes_cap, + next_ref, + cur->depth + 1u, + &next_index, + &inserted)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + if (inserted) { + if (seen_nodes_len > traversal_node_cap) { + traversal_truncated = true; + goto traversal_done; + } + if (!amduatd_query_index_append_size_t(&queue, + &queue_len, + &queue_cap, + next_index)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + } + } + } + } + } + +traversal_done: + for (i = 0u; i < root_refs_len; ++i) { + if (!amduatd_graph_ref_array_append_unique(&response_nodes, + &response_nodes_len, + &response_nodes_cap, + root_refs[i])) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + if (response_nodes_len >= limit_nodes) { + truncated = true; + break; + } + } + + { + size_t pos = 0u; + bool node_limit_hit = false; + for (; pos < traversed_edges_len; ++pos) { + const amduatd_retrieve_edge_t *te = &traversed_edges[pos]; + const amduatd_edge_entry_t *entry = &concepts->edges.items[te->edge_index]; + size_t needed = 0u; + if (page_edge_positions_len >= limit_edges) { + break; + } + if (!amduatd_graph_ref_array_contains(response_nodes, response_nodes_len, entry->src_ref)) { + needed++; + } + if (!amduatd_graph_ref_array_contains(response_nodes, response_nodes_len, entry->dst_ref) && + !amduat_reference_eq(entry->src_ref, entry->dst_ref)) { + needed++; + } + if (response_nodes_len + needed > limit_nodes) { + node_limit_hit = true; + break; + } + if (!amduatd_graph_ref_array_append_unique(&response_nodes, + &response_nodes_len, + &response_nodes_cap, + entry->src_ref) || + !amduatd_graph_ref_array_append_unique(&response_nodes, + &response_nodes_len, + &response_nodes_cap, + entry->dst_ref) || + !amduatd_query_index_append_size_t(&page_edge_positions, + &page_edge_positions_len, + &page_edge_positions_cap, + pos)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + } + if (pos < traversed_edges_len || node_limit_hit) { + truncated = true; + } + } + if (traversal_truncated) { + truncated = true; + } + + if (!amduatd_strbuf_append_cstr(&b, "{\"nodes\":[")) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + for (i = 0u; i < response_nodes_len; ++i) { + char *ref_hex = NULL; + char name[AMDUAT_ASL_POINTER_NAME_MAX + 1u]; + bool have_name = false; + char *latest_hex = NULL; + if (!amduat_asl_ref_encode_hex(response_nodes[i], &ref_hex)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto cleanup; + } + have_name = amduatd_concepts_name_for_ref_cached(store, + concepts, + dcfg, + response_nodes[i], + name, + sizeof(name)); + if (!amduatd_concepts_latest_hex_for_ref(store, concepts, response_nodes[i], &latest_hex)) { + free(ref_hex); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto cleanup; + } + if (i != 0u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + (void)amduatd_strbuf_append_cstr(&b, "{\"concept_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, ref_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"name\":"); + if (have_name) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, name); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + (void)amduatd_strbuf_append_cstr(&b, ",\"latest_ref\":"); + if (latest_hex != NULL) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, latest_hex); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + if (include_versions && + !amduatd_graph_append_versions_json(&b, concepts, response_nodes[i], scan_end)) { + free(ref_hex); + free(latest_hex); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + (void)amduatd_strbuf_append_cstr(&b, "}"); + free(ref_hex); + free(latest_hex); + } + (void)amduatd_strbuf_append_cstr(&b, "],\"edges\":["); + for (i = 0u; i < page_edge_positions_len; ++i) { + const amduatd_retrieve_edge_t *te = &traversed_edges[page_edge_positions[i]]; + const amduatd_edge_entry_t *entry = &concepts->edges.items[te->edge_index]; + amduat_reference_t predicate_ref; + char *subject_hex = NULL; + char *predicate_hex = NULL; + char *object_hex = NULL; + char *edge_hex = NULL; + memset(&predicate_ref, 0, sizeof(predicate_ref)); + if (!amduatd_relation_entry_ref(concepts, entry->rel, &predicate_ref) || + !amduat_asl_ref_encode_hex(entry->src_ref, &subject_hex) || + !amduat_asl_ref_encode_hex(predicate_ref, &predicate_hex) || + !amduat_asl_ref_encode_hex(entry->dst_ref, &object_hex) || + !amduat_asl_ref_encode_hex(entry->record_ref, &edge_hex)) { + amduat_reference_free(&predicate_ref); + free(subject_hex); + free(predicate_hex); + free(object_hex); + free(edge_hex); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto cleanup; + } + amduat_reference_free(&predicate_ref); + if (i != 0u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + (void)amduatd_strbuf_append_cstr(&b, "{\"subject_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, subject_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"predicate_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, predicate_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"object_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, object_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"edge_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, edge_hex); + (void)amduatd_strbuf_append_cstr(&b, "\"}"); + free(subject_hex); + free(predicate_hex); + free(object_hex); + free(edge_hex); + } + (void)amduatd_strbuf_append_cstr(&b, "],\"explanations\":["); + for (i = 0u; i < page_edge_positions_len; ++i) { + const amduatd_retrieve_edge_t *te = &traversed_edges[page_edge_positions[i]]; + const amduatd_edge_entry_t *entry = &concepts->edges.items[te->edge_index]; + char *edge_hex = NULL; + char depth_num[32]; + int n = 0; + if (!amduat_asl_ref_encode_hex(entry->record_ref, &edge_hex)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto cleanup; + } + if (i != 0u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + (void)amduatd_strbuf_append_cstr(&b, "{\"edge_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, edge_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"depth\":"); + n = snprintf(depth_num, sizeof(depth_num), "%llu", (unsigned long long)te->depth); + if (n > 0 && (size_t)n < sizeof(depth_num)) { + (void)amduatd_strbuf_append(&b, depth_num, (size_t)n); + } else { + (void)amduatd_strbuf_append_cstr(&b, "0"); + } + (void)amduatd_strbuf_append_cstr(&b, ",\"reasons\":[\"reachable_from_roots\""); + if (te->goal_match) { + (void)amduatd_strbuf_append_cstr(&b, ",\"goal_predicate_match\""); + } + if (have_provenance_min_confidence && te->have_confidence) { + (void)amduatd_strbuf_append_cstr(&b, ",\"provenance_confidence\""); + } + (void)amduatd_strbuf_append_cstr(&b, "]"); + if (have_provenance_min_confidence) { + if (te->have_confidence) { + char conf_num[64]; + int cn = snprintf(conf_num, sizeof(conf_num), "%.6f", te->confidence); + (void)amduatd_strbuf_append_cstr(&b, ",\"confidence\":"); + if (cn > 0 && (size_t)cn < sizeof(conf_num)) { + (void)amduatd_strbuf_append(&b, conf_num, (size_t)cn); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + } else { + (void)amduatd_strbuf_append_cstr(&b, ",\"confidence\":null"); + } + } + (void)amduatd_strbuf_append_cstr(&b, "}"); + free(edge_hex); + } + (void)amduatd_strbuf_append_cstr(&b, "],\"truncated\":"); + (void)amduatd_strbuf_append_cstr(&b, truncated ? "true" : "false"); + (void)amduatd_strbuf_append_cstr(&b, ",\"stats\":{"); + { + char num[64]; + int n = snprintf(num, sizeof(num), "%llu", (unsigned long long)scanned_edges); + if (n > 0 && (size_t)n < sizeof(num)) { + (void)amduatd_strbuf_append_cstr(&b, "\"scanned_edges\":"); + (void)amduatd_strbuf_append(&b, num, (size_t)n); + } + n = snprintf(num, sizeof(num), "%llu", (unsigned long long)traversed_edges_len); + if (n > 0 && (size_t)n < sizeof(num)) { + (void)amduatd_strbuf_append_cstr(&b, ",\"traversed_edges\":"); + (void)amduatd_strbuf_append(&b, num, (size_t)n); + } + n = snprintf(num, sizeof(num), "%llu", (unsigned long long)response_nodes_len); + if (n > 0 && (size_t)n < sizeof(num)) { + (void)amduatd_strbuf_append_cstr(&b, ",\"returned_nodes\":"); + (void)amduatd_strbuf_append(&b, num, (size_t)n); + } + n = snprintf(num, sizeof(num), "%llu", (unsigned long long)page_edge_positions_len); + if (n > 0 && (size_t)n < sizeof(num)) { + (void)amduatd_strbuf_append_cstr(&b, ",\"returned_edges\":"); + (void)amduatd_strbuf_append(&b, num, (size_t)n); + } + } + (void)amduatd_strbuf_append_cstr(&b, "}}\n"); + + if (b.len > (size_t)max_result_bytes_u64) { + ok = amduatd_send_json_error(fd, 422, "Unprocessable Entity", "result too large"); + goto cleanup; + } + ok = amduatd_http_send_json(fd, 200, "OK", b.data, false); + +cleanup: + free(body); + amduatd_graph_ref_array_free(root_refs, root_refs_len); + amduatd_graph_ref_array_free(goal_predicate_refs, goal_predicate_refs_len); + amduatd_retrieve_nodes_free(seen_nodes, seen_nodes_len); + free(queue); + free(traversed_edges); + free(seen_edges); + free(page_edge_positions); + amduatd_graph_ref_array_free(response_nodes, response_nodes_len); + amduatd_strbuf_free(&b); + return ok; +} + +static bool amduatd_handle_post_graph_query(int fd, + amduat_asl_store_t *store, + const amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const amduatd_http_req_t *req) { + uint8_t *body = NULL; + const char *p = NULL; + const char *end = NULL; + bool have_subject = false; + bool have_object = false; + bool have_node = false; + bool have_provenance_ref = false; + bool always_empty = false; + bool include_versions = false; + bool include_tombstoned = false; + bool include_stats = false; + int direction = 0; /* 0=any, 1=outgoing, 2=incoming */ + bool have_as_of = false; + size_t cursor = 0u; + size_t scan_end = 0u; + uint64_t limit_u64 = 100u; + uint64_t max_result_bytes_u64 = 1048576u; + size_t limit = 100u; + amduat_reference_t subject_ref; + amduat_reference_t object_ref; + amduat_reference_t node_ref; + amduat_reference_t provenance_ref; + amduat_reference_t *predicate_refs = NULL; + size_t predicate_refs_len = 0u; + size_t predicate_refs_cap = 0u; + amduat_reference_t *node_results = NULL; + size_t node_results_len = 0u; + size_t node_results_cap = 0u; + amduatd_strbuf_t b; + size_t i; + size_t returned = 0u; + size_t next_cursor = 0u; + size_t scanned_edges = 0u; + const size_t *scan_indices = NULL; + size_t scan_indices_len = 0u; + const char *scan_plan = "full_scan"; + bool has_more = false; + bool ok = false; + + memset(&subject_ref, 0, sizeof(subject_ref)); + memset(&object_ref, 0, sizeof(object_ref)); + memset(&node_ref, 0, sizeof(node_ref)); + memset(&provenance_ref, 0, sizeof(provenance_ref)); + memset(&b, 0, sizeof(b)); + + if (store == NULL || concepts == NULL || req == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "internal error"); + } + if (!amduatd_concepts_ensure_query_index((amduatd_concepts_t *)concepts)) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "index error"); + } + if (req->content_length == 0u) { + return amduatd_send_json_error(fd, 400, "Bad Request", "missing body"); + } + if (req->content_length > (1024u * 1024u)) { + return amduatd_send_json_error(fd, 413, "Payload Too Large", "payload too large"); + } + body = (uint8_t *)malloc(req->content_length); + if (body == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + if (!amduatd_read_exact(fd, body, req->content_length)) { + free(body); + return false; + } + + p = (const char *)body; + end = (const char *)body + req->content_length; + if (!amduatd_json_expect(&p, end, '{')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + for (;;) { + const char *key = NULL; + size_t key_len = 0u; + const char *cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &key, &key_len) || + !amduatd_json_expect(&p, end, ':')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + if (key_len == strlen("where") && memcmp(key, "where", key_len) == 0) { + if (!amduatd_json_expect(&p, end, '{')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid where"); + goto cleanup; + } + for (;;) { + const char *wk = NULL; + size_t wk_len = 0u; + const char *sv = NULL; + size_t sv_len = 0u; + char *value = NULL; + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &wk, &wk_len) || + !amduatd_json_expect(&p, end, ':')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid where"); + goto cleanup; + } + if (wk_len == strlen("subject") && memcmp(wk, "subject", wk_len) == 0) { + if (have_subject || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &value)) { + free(value); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid where.subject"); + goto cleanup; + } + if (!amduatd_resolve_graph_ref(store, concepts, dcfg, value, &subject_ref)) { + always_empty = true; + } else { + have_subject = true; + } + free(value); + } else if (wk_len == strlen("object") && memcmp(wk, "object", wk_len) == 0) { + if (have_object || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &value)) { + free(value); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid where.object"); + goto cleanup; + } + if (!amduatd_resolve_graph_ref(store, concepts, dcfg, value, &object_ref)) { + always_empty = true; + } else { + have_object = true; + } + free(value); + } else if (wk_len == strlen("node") && memcmp(wk, "node", wk_len) == 0) { + if (have_node || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &value)) { + free(value); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid where.node"); + goto cleanup; + } + if (!amduatd_resolve_graph_ref(store, concepts, dcfg, value, &node_ref)) { + always_empty = true; + } else { + have_node = true; + } + free(value); + } else if (wk_len == strlen("provenance_ref") && + memcmp(wk, "provenance_ref", wk_len) == 0) { + if (have_provenance_ref || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &value)) { + free(value); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid where.provenance_ref"); + goto cleanup; + } + if (!amduatd_resolve_graph_ref(store, concepts, dcfg, value, &provenance_ref)) { + always_empty = true; + } else { + have_provenance_ref = true; + } + free(value); + } else { + if (!amduatd_json_skip_value(&p, end, 0)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid where"); + goto cleanup; + } + } + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid where"); + goto cleanup; + } + } else if (key_len == strlen("predicates") && memcmp(key, "predicates", key_len) == 0) { + if (!amduatd_json_expect(&p, end, '[')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicates"); + goto cleanup; + } + for (;;) { + const char *sv = NULL; + size_t sv_len = 0u; + char *predicate = NULL; + amduat_reference_t predicate_ref; + memset(&predicate_ref, 0, sizeof(predicate_ref)); + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ']') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &predicate)) { + free(predicate); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicates"); + goto cleanup; + } + if (!amduatd_resolve_relation_ref(concepts, predicate, &predicate_ref)) { + free(predicate); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicate"); + goto cleanup; + } + free(predicate); + if (!amduatd_graph_ref_array_append_unique(&predicate_refs, + &predicate_refs_len, + &predicate_refs_cap, + predicate_ref)) { + amduat_reference_free(&predicate_ref); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + amduat_reference_free(&predicate_ref); + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == ']') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicates"); + goto cleanup; + } + } else if (key_len == strlen("direction") && memcmp(key, "direction", key_len) == 0) { + const char *sv = NULL; + size_t sv_len = 0u; + char *value = NULL; + if (!amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &value)) { + free(value); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid direction"); + goto cleanup; + } + if (strcmp(value, "any") == 0 || value[0] == '\0') { + direction = 0; + } else if (strcmp(value, "outgoing") == 0) { + direction = 1; + } else if (strcmp(value, "incoming") == 0) { + direction = 2; + } else { + free(value); + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid direction"); + goto cleanup; + } + free(value); + } else if (key_len == strlen("as_of") && memcmp(key, "as_of", key_len) == 0) { + if (!amduatd_graph_cursor_parse_json(&p, end, &scan_end)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid as_of"); + goto cleanup; + } + have_as_of = true; + } else if (key_len == strlen("limit") && memcmp(key, "limit", key_len) == 0) { + if (!amduatd_json_parse_u64(&p, end, &limit_u64) || + limit_u64 == 0u || limit_u64 > 1000u) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid limit"); + goto cleanup; + } + } else if (key_len == strlen("cursor") && memcmp(key, "cursor", key_len) == 0) { + if (!amduatd_graph_cursor_parse_json(&p, end, &cursor)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid cursor"); + goto cleanup; + } + } else if (key_len == strlen("include_versions") && + memcmp(key, "include_versions", key_len) == 0) { + if (!amduatd_json_parse_bool_value(&p, end, &include_versions)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid include_versions"); + goto cleanup; + } + } else if (key_len == strlen("include_tombstoned") && + memcmp(key, "include_tombstoned", key_len) == 0) { + if (!amduatd_json_parse_bool_value(&p, end, &include_tombstoned)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid include_tombstoned"); + goto cleanup; + } + } else if (key_len == strlen("include_stats") && + memcmp(key, "include_stats", key_len) == 0) { + if (!amduatd_json_parse_bool_value(&p, end, &include_stats)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid include_stats"); + goto cleanup; + } + } else if (key_len == strlen("max_result_bytes") && + memcmp(key, "max_result_bytes", key_len) == 0) { + if (!amduatd_json_parse_u64(&p, end, &max_result_bytes_u64) || + max_result_bytes_u64 < 1024u || + max_result_bytes_u64 > (8u * 1024u * 1024u)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid max_result_bytes"); + goto cleanup; + } + } else { + if (!amduatd_json_skip_value(&p, end, 0)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + } + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + p = amduatd_json_skip_ws(p, end); + if (p != end) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + + if (!have_as_of) { + scan_end = concepts->edges.len; + } else if (scan_end > concepts->edges.len) { + scan_end = concepts->edges.len; + } + if (cursor > scan_end) { + cursor = scan_end; + } + if (limit_u64 > (uint64_t)SIZE_MAX) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid limit"); + goto cleanup; + } + limit = (size_t)limit_u64; + if (!always_empty) { + if (predicate_refs_len == 1u && have_subject) { + const amduatd_ref_pair_edge_bucket_t *bucket = + amduatd_query_index_find_pair_bucket_const( + concepts->qindex.src_predicate_buckets, + concepts->qindex.src_predicate_len, + subject_ref, + predicate_refs[0]); + amduatd_scan_select_best_pair_bucket_with_plan(&scan_indices, + &scan_indices_len, + bucket, + &scan_plan, + "src_predicate"); + } + if (predicate_refs_len == 1u && have_object) { + const amduatd_ref_pair_edge_bucket_t *bucket = + amduatd_query_index_find_pair_bucket_const( + concepts->qindex.dst_predicate_buckets, + concepts->qindex.dst_predicate_len, + object_ref, + predicate_refs[0]); + amduatd_scan_select_best_pair_bucket_with_plan(&scan_indices, + &scan_indices_len, + bucket, + &scan_plan, + "dst_predicate"); + } + if (predicate_refs_len == 1u && have_node && direction == 1) { + const amduatd_ref_pair_edge_bucket_t *bucket = + amduatd_query_index_find_pair_bucket_const( + concepts->qindex.src_predicate_buckets, + concepts->qindex.src_predicate_len, + node_ref, + predicate_refs[0]); + amduatd_scan_select_best_pair_bucket_with_plan(&scan_indices, + &scan_indices_len, + bucket, + &scan_plan, + "node_src_predicate"); + } + if (predicate_refs_len == 1u && have_node && direction == 2) { + const amduatd_ref_pair_edge_bucket_t *bucket = + amduatd_query_index_find_pair_bucket_const( + concepts->qindex.dst_predicate_buckets, + concepts->qindex.dst_predicate_len, + node_ref, + predicate_refs[0]); + amduatd_scan_select_best_pair_bucket_with_plan(&scan_indices, + &scan_indices_len, + bucket, + &scan_plan, + "node_dst_predicate"); + } + if (scan_indices == NULL && predicate_refs_len == 1u) { + const amduatd_ref_edge_bucket_t *bucket = + amduatd_query_index_find_bucket_const(concepts->qindex.predicate_buckets, + concepts->qindex.predicate_len, + predicate_refs[0]); + amduatd_scan_select_best_bucket_with_plan(&scan_indices, + &scan_indices_len, + bucket, + &scan_plan, + "predicate"); + } + if (scan_indices == NULL && have_subject) { + const amduatd_ref_edge_bucket_t *bucket = + amduatd_query_index_find_bucket_const(concepts->qindex.src_buckets, + concepts->qindex.src_len, + subject_ref); + amduatd_scan_select_best_bucket_with_plan(&scan_indices, + &scan_indices_len, + bucket, + &scan_plan, + "src"); + } + if (scan_indices == NULL && have_object) { + const amduatd_ref_edge_bucket_t *bucket = + amduatd_query_index_find_bucket_const(concepts->qindex.dst_buckets, + concepts->qindex.dst_len, + object_ref); + amduatd_scan_select_best_bucket_with_plan(&scan_indices, + &scan_indices_len, + bucket, + &scan_plan, + "dst"); + } + if (scan_indices == NULL && have_node && direction == 1) { + const amduatd_ref_edge_bucket_t *bucket = + amduatd_query_index_find_bucket_const(concepts->qindex.src_buckets, + concepts->qindex.src_len, + node_ref); + amduatd_scan_select_best_bucket_with_plan(&scan_indices, + &scan_indices_len, + bucket, + &scan_plan, + "node_src"); + } + if (scan_indices == NULL && have_node && direction == 2) { + const amduatd_ref_edge_bucket_t *bucket = + amduatd_query_index_find_bucket_const(concepts->qindex.dst_buckets, + concepts->qindex.dst_len, + node_ref); + amduatd_scan_select_best_bucket_with_plan(&scan_indices, + &scan_indices_len, + bucket, + &scan_plan, + "node_dst"); + } + } + + if (!amduatd_strbuf_append_cstr(&b, "{\"nodes\":[")) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + + for (i = 0u; !always_empty && + ((scan_indices != NULL) ? (i < scan_indices_len) : (i < scan_end)); + ++i) { + size_t edge_i = (scan_indices != NULL) ? scan_indices[i] : i; + const amduatd_edge_entry_t *entry; + amduat_reference_t entry_predicate_ref; + bool match = true; + if (scan_indices != NULL) { + if (edge_i < cursor || edge_i >= scan_end) { + continue; + } + } else if (edge_i < cursor) { + continue; + } + entry = &concepts->edges.items[edge_i]; + memset(&entry_predicate_ref, 0, sizeof(entry_predicate_ref)); + if (!amduatd_relation_entry_ref(concepts, entry->rel, &entry_predicate_ref)) { + continue; + } + scanned_edges++; + if (predicate_refs_len != 0u && + !amduatd_graph_ref_array_contains(predicate_refs, + predicate_refs_len, + entry_predicate_ref)) { + match = false; + } + if (match && have_subject && !amduat_reference_eq(entry->src_ref, subject_ref)) { + match = false; + } + if (match && have_object && !amduat_reference_eq(entry->dst_ref, object_ref)) { + match = false; + } + if (match && have_node) { + if (direction == 1) { + if (!amduat_reference_eq(entry->src_ref, node_ref)) { + match = false; + } + } else if (direction == 2) { + if (!amduat_reference_eq(entry->dst_ref, node_ref)) { + match = false; + } + } else if (!amduat_reference_eq(entry->src_ref, node_ref) && + !amduat_reference_eq(entry->dst_ref, node_ref)) { + match = false; + } + } + if (match && have_provenance_ref && + !amduatd_graph_edge_matches_provenance(concepts, + entry->record_ref, + provenance_ref, + scan_end)) { + match = false; + } + if (match && !include_tombstoned && + amduatd_graph_edge_is_tombstoned(concepts, entry->record_ref, scan_end)) { + match = false; + } + amduat_reference_free(&entry_predicate_ref); + if (!match) { + continue; + } + if (!amduatd_graph_ref_array_append_unique(&node_results, + &node_results_len, + &node_results_cap, + entry->src_ref) || + !amduatd_graph_ref_array_append_unique(&node_results, + &node_results_len, + &node_results_cap, + entry->dst_ref)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + returned++; + if (returned >= limit) { + next_cursor = edge_i + 1u; + break; + } + } + + if (returned >= limit && next_cursor < scan_end && !always_empty) { + size_t j; + for (j = 0u; + (scan_indices != NULL) ? (j < scan_indices_len) : (j < scan_end); + ++j) { + size_t edge_j = (scan_indices != NULL) ? scan_indices[j] : j; + const amduatd_edge_entry_t *entry; + amduat_reference_t entry_predicate_ref; + bool match = true; + if (edge_j < next_cursor || edge_j >= scan_end) { + continue; + } + entry = &concepts->edges.items[edge_j]; + memset(&entry_predicate_ref, 0, sizeof(entry_predicate_ref)); + if (!amduatd_relation_entry_ref(concepts, entry->rel, &entry_predicate_ref)) { + continue; + } + scanned_edges++; + if (predicate_refs_len != 0u && + !amduatd_graph_ref_array_contains(predicate_refs, + predicate_refs_len, + entry_predicate_ref)) { + match = false; + } + if (match && have_subject && !amduat_reference_eq(entry->src_ref, subject_ref)) { + match = false; + } + if (match && have_object && !amduat_reference_eq(entry->dst_ref, object_ref)) { + match = false; + } + if (match && have_node) { + if (direction == 1) { + if (!amduat_reference_eq(entry->src_ref, node_ref)) { + match = false; + } + } else if (direction == 2) { + if (!amduat_reference_eq(entry->dst_ref, node_ref)) { + match = false; + } + } else if (!amduat_reference_eq(entry->src_ref, node_ref) && + !amduat_reference_eq(entry->dst_ref, node_ref)) { + match = false; + } + } + if (match && have_provenance_ref && + !amduatd_graph_edge_matches_provenance(concepts, + entry->record_ref, + provenance_ref, + scan_end)) { + match = false; + } + if (match && !include_tombstoned && + amduatd_graph_edge_is_tombstoned(concepts, entry->record_ref, scan_end)) { + match = false; + } + amduat_reference_free(&entry_predicate_ref); + if (match) { + has_more = true; + break; + } + } + } + + for (i = 0u; i < node_results_len; ++i) { + char *ref_hex = NULL; + char name[AMDUAT_ASL_POINTER_NAME_MAX + 1u]; + bool have_name = false; + char *latest_hex = NULL; + if (!amduat_asl_ref_encode_hex(node_results[i], &ref_hex)) { + continue; + } + have_name = amduatd_concepts_name_for_ref_cached(store, + concepts, + dcfg, + node_results[i], + name, + sizeof(name)); + if (!amduatd_concepts_latest_hex_for_ref(store, concepts, node_results[i], &latest_hex)) { + free(ref_hex); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto cleanup; + } + if (i != 0u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + (void)amduatd_strbuf_append_cstr(&b, "{\"concept_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, ref_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"name\":"); + if (have_name) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, name); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + (void)amduatd_strbuf_append_cstr(&b, ",\"latest_ref\":"); + if (latest_hex != NULL) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, latest_hex); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + if (include_versions && + !amduatd_graph_append_versions_json(&b, concepts, node_results[i], scan_end)) { + free(ref_hex); + free(latest_hex); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + (void)amduatd_strbuf_append_cstr(&b, "}"); + free(ref_hex); + free(latest_hex); + } + + (void)amduatd_strbuf_append_cstr(&b, "],\"edges\":["); + returned = 0u; + for (i = 0u; !always_empty && + ((scan_indices != NULL) ? (i < scan_indices_len) : (i < scan_end)); + ++i) { + size_t edge_i = (scan_indices != NULL) ? scan_indices[i] : i; + const amduatd_edge_entry_t *entry; + amduat_reference_t entry_predicate_ref; + bool match = true; + char *subject_hex = NULL; + char *predicate_hex = NULL; + char *object_hex = NULL; + char *edge_hex = NULL; + + if (scan_indices != NULL) { + if (edge_i < cursor || edge_i >= scan_end) { + continue; + } + } else if (edge_i < cursor) { + continue; + } + entry = &concepts->edges.items[edge_i]; + memset(&entry_predicate_ref, 0, sizeof(entry_predicate_ref)); + if (!amduatd_relation_entry_ref(concepts, entry->rel, &entry_predicate_ref)) { + continue; + } + scanned_edges++; + if (predicate_refs_len != 0u && + !amduatd_graph_ref_array_contains(predicate_refs, + predicate_refs_len, + entry_predicate_ref)) { + match = false; + } + if (match && have_subject && !amduat_reference_eq(entry->src_ref, subject_ref)) { + match = false; + } + if (match && have_object && !amduat_reference_eq(entry->dst_ref, object_ref)) { + match = false; + } + if (match && have_node) { + if (direction == 1) { + if (!amduat_reference_eq(entry->src_ref, node_ref)) { + match = false; + } + } else if (direction == 2) { + if (!amduat_reference_eq(entry->dst_ref, node_ref)) { + match = false; + } + } else if (!amduat_reference_eq(entry->src_ref, node_ref) && + !amduat_reference_eq(entry->dst_ref, node_ref)) { + match = false; + } + } + if (match && have_provenance_ref && + !amduatd_graph_edge_matches_provenance(concepts, + entry->record_ref, + provenance_ref, + scan_end)) { + match = false; + } + if (match && !include_tombstoned && + amduatd_graph_edge_is_tombstoned(concepts, entry->record_ref, scan_end)) { + match = false; + } + if (!match) { + amduat_reference_free(&entry_predicate_ref); + continue; + } + if (!amduat_asl_ref_encode_hex(entry->src_ref, &subject_hex) || + !amduat_asl_ref_encode_hex(entry_predicate_ref, &predicate_hex) || + !amduat_asl_ref_encode_hex(entry->dst_ref, &object_hex) || + !amduat_asl_ref_encode_hex(entry->record_ref, &edge_hex)) { + amduat_reference_free(&entry_predicate_ref); + free(subject_hex); + free(predicate_hex); + free(object_hex); + free(edge_hex); + continue; + } + amduat_reference_free(&entry_predicate_ref); + if (returned != 0u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + returned++; + (void)amduatd_strbuf_append_cstr(&b, "{\"subject_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, subject_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"predicate_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, predicate_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"object_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, object_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"edge_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, edge_hex); + (void)amduatd_strbuf_append_cstr(&b, "\"}"); + free(subject_hex); + free(predicate_hex); + free(object_hex); + free(edge_hex); + if (returned >= limit) { + next_cursor = edge_i + 1u; + break; + } + } + if (include_stats) { + char num[64]; + int n; + (void)amduatd_strbuf_append_cstr(&b, "],\"stats\":{"); + (void)amduatd_strbuf_append_cstr(&b, "\"plan\":\""); + (void)amduatd_strbuf_append_cstr(&b, scan_plan); + (void)amduatd_strbuf_append_cstr(&b, "\""); + n = snprintf(num, sizeof(num), "%llu", (unsigned long long)scanned_edges); + if (n > 0 && (size_t)n < sizeof(num)) { + (void)amduatd_strbuf_append_cstr(&b, ",\"scanned_edges\":"); + (void)amduatd_strbuf_append(&b, num, (size_t)n); + } + n = snprintf(num, sizeof(num), "%llu", (unsigned long long)returned); + if (n > 0 && (size_t)n < sizeof(num)) { + (void)amduatd_strbuf_append_cstr(&b, ",\"returned_edges\":"); + (void)amduatd_strbuf_append(&b, num, (size_t)n); + } + (void)amduatd_strbuf_append_cstr(&b, "},\"paging\":{"); + } else { + (void)amduatd_strbuf_append_cstr(&b, "],\"paging\":{"); + } + if (has_more) { + char token[64]; + if (!amduatd_graph_cursor_encode(next_cursor, token, sizeof(token))) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto cleanup; + } + (void)amduatd_strbuf_append_cstr(&b, "\"next_cursor\":\""); + (void)amduatd_strbuf_append_cstr(&b, token); + (void)amduatd_strbuf_append_cstr(&b, "\",\"has_more\":true"); + } else { + (void)amduatd_strbuf_append_cstr(&b, "\"next_cursor\":null,\"has_more\":false"); + } + (void)amduatd_strbuf_append_cstr(&b, "}}\n"); + if (b.len > (size_t)max_result_bytes_u64) { + ok = amduatd_send_json_error(fd, 422, "Unprocessable Entity", "result too large"); + goto cleanup; + } + + ok = amduatd_http_send_json(fd, 200, "OK", b.data, false); + +cleanup: + free(body); + amduat_reference_free(&subject_ref); + amduat_reference_free(&object_ref); + amduat_reference_free(&node_ref); + amduat_reference_free(&provenance_ref); + amduatd_graph_ref_array_free(predicate_refs, predicate_refs_len); + amduatd_graph_ref_array_free(node_results, node_results_len); + amduatd_strbuf_free(&b); + return ok; +} + +static bool amduatd_handle_get_graph_search(int fd, + amduat_asl_store_t *store, + const amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const amduatd_http_req_t *req) { + char prefix_buf[AMDUAT_ASL_POINTER_NAME_MAX + 1u]; + char limit_buf[32]; + char cursor_buf[32]; + char as_of_buf[32]; + const char *prefix = NULL; + size_t prefix_len = 0u; + uint64_t limit_u64 = 100u; + size_t limit = 100u; + size_t cursor = 0u; + size_t scan_end = 0u; + size_t i; + size_t returned = 0u; + size_t next_cursor = 0u; + bool has_more = false; + amduatd_strbuf_t b; + + if (store == NULL || concepts == NULL || req == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", + "internal error"); + } + memset(&b, 0, sizeof(b)); + if (!amduatd_concepts_ensure_query_index((amduatd_concepts_t *)concepts)) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "index error"); + } + + if (amduatd_query_param(req->path, + "name_prefix", + prefix_buf, + sizeof(prefix_buf)) != NULL) { + prefix = prefix_buf; + prefix_len = strlen(prefix); + } + if (amduatd_query_param(req->path, "limit", limit_buf, sizeof(limit_buf)) != NULL) { + if (!amduatd_parse_u64_query(limit_buf, &limit_u64) || + limit_u64 == 0u || + limit_u64 > 1000u) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid limit"); + } + } + if (amduatd_query_param(req->path, "cursor", cursor_buf, sizeof(cursor_buf)) != NULL) { + if (!amduatd_graph_cursor_decode(cursor_buf, &cursor)) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid cursor"); + } + } + scan_end = concepts->edges.len; + if (amduatd_query_param(req->path, "as_of", as_of_buf, sizeof(as_of_buf)) != NULL) { + if (!amduatd_graph_cursor_decode(as_of_buf, &scan_end)) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid as_of"); + } + if (scan_end > concepts->edges.len) { + scan_end = concepts->edges.len; + } + } + if (limit_u64 > (uint64_t)SIZE_MAX) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid paging"); + } + limit = (size_t)limit_u64; + if (cursor > scan_end) { + cursor = scan_end; + } + + if (!amduatd_strbuf_append_cstr(&b, "{\"nodes\":[")) { + amduatd_strbuf_free(&b); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + + for (i = 0; i < concepts->qindex.alias_len; ++i) { + size_t edge_i = concepts->qindex.alias_edge_indices[i]; + const amduatd_edge_entry_t *entry; + amduat_artifact_t artifact; + amduat_asl_store_error_t err; + char name[AMDUAT_ASL_POINTER_NAME_MAX + 1u]; + char *concept_hex = NULL; + + if (edge_i < cursor || edge_i >= scan_end) { + continue; + } + entry = &concepts->edges.items[edge_i]; + + memset(&artifact, 0, sizeof(artifact)); + err = amduat_asl_store_get(store, entry->src_ref, &artifact); + if (err != AMDUAT_ASL_STORE_OK || + !amduatd_parse_name_artifact(artifact, dcfg, name, sizeof(name))) { + amduat_asl_artifact_free(&artifact); + continue; + } + amduat_asl_artifact_free(&artifact); + + if (prefix != NULL) { + if (strncmp(name, prefix, prefix_len) != 0) { + continue; + } + } + + if (!amduat_asl_ref_encode_hex(entry->dst_ref, &concept_hex)) { + continue; + } + + if (returned != 0u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + returned++; + (void)amduatd_strbuf_append_cstr(&b, "{\"name\":\""); + (void)amduatd_strbuf_append_cstr(&b, name); + (void)amduatd_strbuf_append_cstr(&b, "\",\"concept_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, concept_hex); + (void)amduatd_strbuf_append_cstr(&b, "\"}"); + free(concept_hex); + + if (returned >= limit) { + next_cursor = edge_i + 1u; + break; + } + } + + if (returned >= limit && next_cursor < scan_end) { + size_t j; + for (j = 0; j < concepts->qindex.alias_len; ++j) { + size_t edge_j = concepts->qindex.alias_edge_indices[j]; + const amduatd_edge_entry_t *entry; + amduat_artifact_t artifact; + amduat_asl_store_error_t err; + char name[AMDUAT_ASL_POINTER_NAME_MAX + 1u]; + if (edge_j < next_cursor || edge_j >= scan_end) { + continue; + } + entry = &concepts->edges.items[edge_j]; + memset(&artifact, 0, sizeof(artifact)); + err = amduat_asl_store_get(store, entry->src_ref, &artifact); + if (err != AMDUAT_ASL_STORE_OK || + !amduatd_parse_name_artifact(artifact, dcfg, name, sizeof(name))) { + amduat_asl_artifact_free(&artifact); + continue; + } + amduat_asl_artifact_free(&artifact); + if (prefix != NULL && strncmp(name, prefix, prefix_len) != 0) { + continue; + } + has_more = true; + break; + } + } + + (void)amduatd_strbuf_append_cstr(&b, "],\"paging\":{"); + if (has_more) { + char token[64]; + if (!amduatd_graph_cursor_encode(next_cursor, token, sizeof(token))) { + amduatd_strbuf_free(&b); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + } + (void)amduatd_strbuf_append_cstr(&b, "\"next_cursor\":\""); + (void)amduatd_strbuf_append_cstr(&b, token); + (void)amduatd_strbuf_append_cstr(&b, "\",\"has_more\":true"); + } else { + (void)amduatd_strbuf_append_cstr(&b, "\"next_cursor\":null,\"has_more\":false"); + } + (void)amduatd_strbuf_append_cstr(&b, "}}\n"); + + { + bool ok = amduatd_http_send_json(fd, 200, "OK", b.data, false); + amduatd_strbuf_free(&b); + return ok; + } +} + +typedef struct { + amduat_reference_t node_ref; + ssize_t parent_index; + size_t via_edge_index; + uint64_t depth; +} amduatd_bfs_state_t; + +typedef struct { + amduat_reference_t node_ref; + uint64_t depth; +} amduatd_subgraph_node_t; + +static ssize_t amduatd_bfs_find_state(const amduatd_bfs_state_t *states, + size_t states_len, + amduat_reference_t ref) { + size_t i; + if (states == NULL) { + return -1; + } + for (i = 0; i < states_len; ++i) { + if (amduat_reference_eq(states[i].node_ref, ref)) { + return (ssize_t)i; + } + } + return -1; +} + +static void amduatd_bfs_states_free(amduatd_bfs_state_t *states, + size_t states_len) { + size_t i; + if (states == NULL) { + return; + } + for (i = 0; i < states_len; ++i) { + amduat_reference_free(&states[i].node_ref); + } + free(states); +} + +static void amduatd_subgraph_nodes_free(amduatd_subgraph_node_t *nodes, + size_t nodes_len) { + size_t i; + if (nodes == NULL) { + return; + } + for (i = 0u; i < nodes_len; ++i) { + amduat_reference_free(&nodes[i].node_ref); + } + free(nodes); +} + +static bool amduatd_subgraph_nodes_append_unique(amduatd_subgraph_node_t **nodes, + size_t *nodes_len, + size_t *nodes_cap, + amduat_reference_t ref, + uint64_t depth, + size_t *out_index, + bool *out_inserted) { + size_t i; + amduatd_subgraph_node_t *next; + size_t next_cap; + if (nodes == NULL || nodes_len == NULL || nodes_cap == NULL) { + return false; + } + for (i = 0u; i < *nodes_len; ++i) { + if (amduat_reference_eq((*nodes)[i].node_ref, ref)) { + if (out_index != NULL) { + *out_index = i; + } + if (out_inserted != NULL) { + *out_inserted = false; + } + return true; + } + } + if (*nodes_len == *nodes_cap) { + next_cap = (*nodes_cap != 0u) ? (*nodes_cap * 2u) : 64u; + next = (amduatd_subgraph_node_t *)realloc(*nodes, next_cap * sizeof(*next)); + if (next == NULL) { + return false; + } + *nodes = next; + *nodes_cap = next_cap; + } + memset(&(*nodes)[*nodes_len], 0, sizeof((*nodes)[*nodes_len])); + if (!amduat_reference_clone(ref, &(*nodes)[*nodes_len].node_ref)) { + return false; + } + (*nodes)[*nodes_len].depth = depth; + if (out_index != NULL) { + *out_index = *nodes_len; + } + if (out_inserted != NULL) { + *out_inserted = true; + } + (*nodes_len)++; + return true; +} + +static size_t amduatd_graph_split_csv(char *text, + char **items, + size_t items_cap) { + char *p; + size_t count = 0u; + if (text == NULL || items == NULL || items_cap == 0u) { + return 0u; + } + p = text; + while (*p != '\0' && count < items_cap) { + char *start = p; + char *end = NULL; + while (*start == ' ' || *start == '\t') { + start++; + } + end = start; + while (*end != '\0' && *end != ',') { + end++; + } + if (*end == ',') { + *end = '\0'; + } + { + char *trim = end; + while (trim > start && (trim[-1] == ' ' || trim[-1] == '\t')) { + trim--; + } + *trim = '\0'; + } + if (start[0] != '\0') { + items[count++] = start; + } + if (*end == '\0') { + break; + } + p = end + 1; + } + return count; +} + +static bool amduatd_handle_get_graph_subgraph(int fd, + amduat_asl_store_t *store, + const amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const amduatd_http_req_t *req) { + enum { AMDUATD_SUBGRAPH_MAX_CSV_ITEMS = 256 }; + char roots_buf[4096]; + char predicates_buf[4096]; + char dir_buf[32]; + char depth_buf[32]; + char as_of_buf[32]; + char include_versions_buf[16]; + char include_stats_buf[16]; + char include_tombstoned_buf[16]; + char max_fanout_buf[32]; + char max_result_bytes_buf[32]; + char limit_nodes_buf[32]; + char limit_edges_buf[32]; + char cursor_buf[32]; + char provenance_buf[512]; + char *csv_items[AMDUATD_SUBGRAPH_MAX_CSV_ITEMS]; + const size_t traversal_edge_cap = 50000u; + const size_t traversal_node_cap = 20000u; + int direction = 0; /* 0=any, 1=outgoing, 2=incoming */ + bool include_versions = false; + bool include_stats = false; + bool include_tombstoned = false; + bool have_as_of = false; + bool have_provenance_ref = false; + bool always_empty = false; + uint64_t max_depth_u64 = 2u; + uint64_t max_fanout_u64 = 2048u; + uint64_t max_result_bytes_u64 = 1048576u; + uint64_t limit_nodes_u64 = 256u; + uint64_t limit_edges_u64 = 256u; + size_t max_depth = 2u; + size_t max_fanout = 2048u; + size_t limit_nodes = 256u; + size_t limit_edges = 256u; + size_t scan_end = 0u; + size_t cursor = 0u; + amduat_reference_t *root_refs = NULL; + size_t root_refs_len = 0u; + size_t root_refs_cap = 0u; + amduat_reference_t *predicate_refs = NULL; + size_t predicate_refs_len = 0u; + size_t predicate_refs_cap = 0u; + amduatd_subgraph_node_t *seen_nodes = NULL; + size_t seen_nodes_len = 0u; + size_t seen_nodes_cap = 0u; + size_t *queue = NULL; + size_t queue_len = 0u; + size_t queue_cap = 0u; + size_t q_pos = 0u; + size_t *traversed_edges = NULL; + size_t traversed_edges_len = 0u; + size_t traversed_edges_cap = 0u; + uint8_t *seen_edges = NULL; + bool traversal_truncated = false; + size_t *page_edges = NULL; + size_t page_edges_len = 0u; + size_t page_edges_cap = 0u; + amduat_reference_t *response_nodes = NULL; + size_t response_nodes_len = 0u; + size_t response_nodes_cap = 0u; + amduat_reference_t *frontier_refs = NULL; + size_t frontier_refs_len = 0u; + size_t frontier_refs_cap = 0u; + size_t next_cursor = 0u; + bool has_more = false; + bool truncated = false; + bool ok = false; + size_t scanned_edges = 0u; + amduatd_strbuf_t b; + size_t i; + amduat_reference_t provenance_ref; + const char *scan_plan = "src+dst_bucket"; + + memset(&b, 0, sizeof(b)); + memset(csv_items, 0, sizeof(csv_items)); + memset(&provenance_ref, 0, sizeof(provenance_ref)); + roots_buf[0] = '\0'; + predicates_buf[0] = '\0'; + + if (store == NULL || concepts == NULL || req == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "internal error"); + } + if (!amduatd_concepts_ensure_query_index((amduatd_concepts_t *)concepts)) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "index error"); + } + + if (amduatd_query_param(req->path, "roots[]", roots_buf, sizeof(roots_buf)) == NULL && + amduatd_query_param(req->path, "roots", roots_buf, sizeof(roots_buf)) == NULL) { + return amduatd_send_json_error(fd, 400, "Bad Request", "missing roots"); + } + { + size_t roots_count = amduatd_graph_split_csv(roots_buf, csv_items, AMDUATD_SUBGRAPH_MAX_CSV_ITEMS); + if (roots_count == 0u) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid roots"); + } + for (i = 0u; i < roots_count; ++i) { + amduat_reference_t root_ref; + memset(&root_ref, 0, sizeof(root_ref)); + if (!amduatd_resolve_graph_ref(store, concepts, dcfg, csv_items[i], &root_ref)) { + amduatd_graph_ref_array_free(root_refs, root_refs_len); + return amduatd_send_json_error(fd, 404, "Not Found", "root not found"); + } + if (!amduatd_graph_ref_array_append_unique(&root_refs, + &root_refs_len, + &root_refs_cap, + root_ref)) { + amduat_reference_free(&root_ref); + amduatd_graph_ref_array_free(root_refs, root_refs_len); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + amduat_reference_free(&root_ref); + } + } + + if (amduatd_query_param(req->path, + "predicates[]", + predicates_buf, + sizeof(predicates_buf)) == NULL && + amduatd_query_param(req->path, + "predicates", + predicates_buf, + sizeof(predicates_buf)) == NULL && + amduatd_query_param(req->path, + "predicate", + predicates_buf, + sizeof(predicates_buf)) == NULL) { + predicates_buf[0] = '\0'; + } + if (predicates_buf[0] != '\0') { + size_t predicates_count = + amduatd_graph_split_csv(predicates_buf, csv_items, AMDUATD_SUBGRAPH_MAX_CSV_ITEMS); + for (i = 0u; i < predicates_count; ++i) { + amduat_reference_t predicate_ref; + memset(&predicate_ref, 0, sizeof(predicate_ref)); + if (!amduatd_resolve_relation_ref(concepts, csv_items[i], &predicate_ref)) { + amduatd_graph_ref_array_free(root_refs, root_refs_len); + amduatd_graph_ref_array_free(predicate_refs, predicate_refs_len); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicate"); + } + if (!amduatd_graph_ref_array_append_unique(&predicate_refs, + &predicate_refs_len, + &predicate_refs_cap, + predicate_ref)) { + amduat_reference_free(&predicate_ref); + amduatd_graph_ref_array_free(root_refs, root_refs_len); + amduatd_graph_ref_array_free(predicate_refs, predicate_refs_len); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + amduat_reference_free(&predicate_ref); + } + } + + if (amduatd_query_param(req->path, "dir", dir_buf, sizeof(dir_buf)) != NULL) { + if (strcmp(dir_buf, "any") == 0 || dir_buf[0] == '\0') { + direction = 0; + } else if (strcmp(dir_buf, "outgoing") == 0) { + direction = 1; + } else if (strcmp(dir_buf, "incoming") == 0) { + direction = 2; + } else { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid dir"); + goto cleanup; + } + } + if (amduatd_query_param(req->path, "max_depth", depth_buf, sizeof(depth_buf)) != NULL) { + if (!amduatd_parse_u64_query(depth_buf, &max_depth_u64) || max_depth_u64 > 32u) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid max_depth"); + goto cleanup; + } + } + if (amduatd_query_param(req->path, + "include_versions", + include_versions_buf, + sizeof(include_versions_buf)) != NULL) { + if (!amduatd_parse_bool_query(include_versions_buf, &include_versions)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid include_versions"); + goto cleanup; + } + } + if (amduatd_query_param(req->path, + "include_stats", + include_stats_buf, + sizeof(include_stats_buf)) != NULL) { + if (!amduatd_parse_bool_query(include_stats_buf, &include_stats)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid include_stats"); + goto cleanup; + } + } + if (amduatd_query_param(req->path, + "include_tombstoned", + include_tombstoned_buf, + sizeof(include_tombstoned_buf)) != NULL) { + if (!amduatd_parse_bool_query(include_tombstoned_buf, &include_tombstoned)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid include_tombstoned"); + goto cleanup; + } + } + if (amduatd_query_param(req->path, "max_fanout", max_fanout_buf, sizeof(max_fanout_buf)) != NULL) { + if (!amduatd_parse_u64_query(max_fanout_buf, &max_fanout_u64) || + max_fanout_u64 == 0u || max_fanout_u64 > 10000u) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid max_fanout"); + goto cleanup; + } + } + if (amduatd_query_param(req->path, + "max_result_bytes", + max_result_bytes_buf, + sizeof(max_result_bytes_buf)) != NULL) { + if (!amduatd_parse_u64_query(max_result_bytes_buf, &max_result_bytes_u64) || + max_result_bytes_u64 < 1024u || max_result_bytes_u64 > (8u * 1024u * 1024u)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid max_result_bytes"); + goto cleanup; + } + } + if (amduatd_query_param(req->path, "limit_nodes", limit_nodes_buf, sizeof(limit_nodes_buf)) != NULL) { + if (!amduatd_parse_u64_query(limit_nodes_buf, &limit_nodes_u64) || + limit_nodes_u64 == 0u || limit_nodes_u64 > 2000u) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid limit_nodes"); + goto cleanup; + } + } + if (amduatd_query_param(req->path, "limit_edges", limit_edges_buf, sizeof(limit_edges_buf)) != NULL) { + if (!amduatd_parse_u64_query(limit_edges_buf, &limit_edges_u64) || + limit_edges_u64 == 0u || limit_edges_u64 > 2000u) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid limit_edges"); + goto cleanup; + } + } + if (amduatd_query_param(req->path, "cursor", cursor_buf, sizeof(cursor_buf)) != NULL) { + if (!amduatd_graph_cursor_decode(cursor_buf, &cursor)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid cursor"); + goto cleanup; + } + } + if (amduatd_query_param(req->path, + "provenance_ref", + provenance_buf, + sizeof(provenance_buf)) != NULL) { + if (!amduatd_resolve_graph_ref(store, + concepts, + dcfg, + provenance_buf, + &provenance_ref)) { + always_empty = true; + } else { + have_provenance_ref = true; + } + } + + scan_end = concepts->edges.len; + if (amduatd_query_param(req->path, "as_of", as_of_buf, sizeof(as_of_buf)) != NULL) { + if (!amduatd_graph_cursor_decode(as_of_buf, &scan_end)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid as_of"); + goto cleanup; + } + have_as_of = true; + } + if (have_as_of && scan_end > concepts->edges.len) { + scan_end = concepts->edges.len; + } + max_depth = (size_t)max_depth_u64; + max_fanout = (size_t)max_fanout_u64; + limit_nodes = (size_t)limit_nodes_u64; + limit_edges = (size_t)limit_edges_u64; + if (predicate_refs_len > 0u) { + if (direction == 1) { + scan_plan = "src_predicate_bucket"; + } else if (direction == 2) { + scan_plan = "dst_predicate_bucket"; + } else { + scan_plan = "src_predicate+dst_predicate_bucket"; + } + } else if (direction == 1) { + scan_plan = "src_bucket"; + } else if (direction == 2) { + scan_plan = "dst_bucket"; + } + + if (scan_end > 0u) { + seen_edges = (uint8_t *)calloc(scan_end, sizeof(*seen_edges)); + if (seen_edges == NULL) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + } + + for (i = 0u; i < root_refs_len; ++i) { + size_t node_index = 0u; + bool inserted = false; + if (!amduatd_subgraph_nodes_append_unique(&seen_nodes, + &seen_nodes_len, + &seen_nodes_cap, + root_refs[i], + 0u, + &node_index, + &inserted)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + if (inserted && + !amduatd_query_index_append_size_t(&queue, &queue_len, &queue_cap, node_index)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + } + + while (!always_empty && q_pos < queue_len) { + const amduatd_subgraph_node_t *cur = &seen_nodes[queue[q_pos++]]; + const amduatd_ref_edge_bucket_t *bucket = NULL; + size_t bi = 0u; + int pass = 0; + if (cur->depth >= max_depth) { + continue; + } + for (pass = 0; pass < 2; ++pass) { + bool outgoing_pass = (pass == 0); + const size_t *edge_indices = NULL; + size_t edge_indices_len = 0u; + size_t fanout_seen = 0u; + size_t pi = 0u; + size_t predicate_iters = 1u; + if ((outgoing_pass && direction == 2) || + (!outgoing_pass && direction == 1)) { + continue; + } + if (predicate_refs_len > 1u) { + predicate_iters = predicate_refs_len; + } + for (pi = 0u; pi < predicate_iters; ++pi) { + if (predicate_refs_len == 0u) { + bucket = outgoing_pass + ? amduatd_query_index_find_bucket_const(concepts->qindex.src_buckets, + concepts->qindex.src_len, + cur->node_ref) + : amduatd_query_index_find_bucket_const(concepts->qindex.dst_buckets, + concepts->qindex.dst_len, + cur->node_ref); + if (bucket != NULL) { + edge_indices = bucket->edge_indices; + edge_indices_len = bucket->len; + } else { + edge_indices = NULL; + edge_indices_len = 0u; + } + } else { + const amduatd_ref_pair_edge_bucket_t *pair_bucket = + outgoing_pass + ? amduatd_query_index_find_pair_bucket_const( + concepts->qindex.src_predicate_buckets, + concepts->qindex.src_predicate_len, + cur->node_ref, + predicate_refs[pi]) + : amduatd_query_index_find_pair_bucket_const( + concepts->qindex.dst_predicate_buckets, + concepts->qindex.dst_predicate_len, + cur->node_ref, + predicate_refs[pi]); + if (pair_bucket != NULL) { + edge_indices = pair_bucket->edge_indices; + edge_indices_len = pair_bucket->len; + } else { + edge_indices = NULL; + edge_indices_len = 0u; + } + } + for (bi = 0u; bi < edge_indices_len; ++bi) { + size_t edge_i = edge_indices[bi]; + const amduatd_edge_entry_t *entry; + amduat_reference_t rel_ref; + amduat_reference_t next_ref; + size_t next_index = 0u; + bool inserted = false; + if (edge_i >= scan_end) { + continue; + } + scanned_edges++; + fanout_seen++; + if (fanout_seen > max_fanout) { + traversal_truncated = true; + goto traversal_done; + } + if (seen_edges != NULL && seen_edges[edge_i] != 0u) { + continue; + } + entry = &concepts->edges.items[edge_i]; + if (predicate_refs_len == 0u) { + memset(&rel_ref, 0, sizeof(rel_ref)); + if (!amduatd_relation_entry_ref(concepts, entry->rel, &rel_ref)) { + continue; + } + amduat_reference_free(&rel_ref); + } + if (!include_tombstoned && + amduatd_graph_edge_is_tombstoned(concepts, entry->record_ref, scan_end)) { + continue; + } + if (have_provenance_ref && + !amduatd_graph_edge_matches_provenance(concepts, + entry->record_ref, + provenance_ref, + scan_end)) { + continue; + } + if (traversed_edges_len >= traversal_edge_cap) { + traversal_truncated = true; + goto traversal_done; + } + if (seen_edges != NULL) { + seen_edges[edge_i] = 1u; + } + if (!amduatd_query_index_append_size_t(&traversed_edges, + &traversed_edges_len, + &traversed_edges_cap, + edge_i)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + next_ref = outgoing_pass ? entry->dst_ref : entry->src_ref; + if (!amduatd_subgraph_nodes_append_unique(&seen_nodes, + &seen_nodes_len, + &seen_nodes_cap, + next_ref, + cur->depth + 1u, + &next_index, + &inserted)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + if (inserted) { + if (seen_nodes_len > traversal_node_cap) { + traversal_truncated = true; + goto traversal_done; + } + if (!amduatd_query_index_append_size_t(&queue, + &queue_len, + &queue_cap, + next_index)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + } + } + } + } + } + +traversal_done: + if (cursor > traversed_edges_len) { + cursor = traversed_edges_len; + } + + for (i = 0u; i < root_refs_len; ++i) { + if (!amduatd_graph_ref_array_append_unique(&response_nodes, + &response_nodes_len, + &response_nodes_cap, + root_refs[i])) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + if (response_nodes_len >= limit_nodes) { + truncated = true; + break; + } + } + + { + size_t pos = cursor; + bool node_limit_hit = false; + for (; pos < traversed_edges_len; ++pos) { + const amduatd_edge_entry_t *entry = &concepts->edges.items[traversed_edges[pos]]; + size_t needed = 0u; + if (page_edges_len >= limit_edges) { + break; + } + if (!amduatd_graph_ref_array_contains(response_nodes, response_nodes_len, entry->src_ref)) { + needed++; + } + if (!amduatd_graph_ref_array_contains(response_nodes, response_nodes_len, entry->dst_ref) && + !amduat_reference_eq(entry->src_ref, entry->dst_ref)) { + needed++; + } + if (response_nodes_len + needed > limit_nodes) { + node_limit_hit = true; + break; + } + if (!amduatd_graph_ref_array_append_unique(&response_nodes, + &response_nodes_len, + &response_nodes_cap, + entry->src_ref) || + !amduatd_graph_ref_array_append_unique(&response_nodes, + &response_nodes_len, + &response_nodes_cap, + entry->dst_ref) || + !amduatd_query_index_append_size_t(&page_edges, + &page_edges_len, + &page_edges_cap, + traversed_edges[pos])) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + } + next_cursor = pos; + has_more = (next_cursor < traversed_edges_len); + if (page_edges_len >= limit_edges && has_more) { + truncated = true; + } + if (node_limit_hit && has_more) { + truncated = true; + } + } + + if (traversal_truncated) { + for (i = q_pos; i < queue_len; ++i) { + size_t node_index = queue[i]; + if (node_index >= seen_nodes_len) { + continue; + } + if (!amduatd_graph_ref_array_append_unique(&frontier_refs, + &frontier_refs_len, + &frontier_refs_cap, + seen_nodes[node_index].node_ref)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + if (frontier_refs_len >= 256u) { + break; + } + } + truncated = true; + } + + if (!amduatd_strbuf_append_cstr(&b, "{\"nodes\":[")) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + for (i = 0u; i < response_nodes_len; ++i) { + char *ref_hex = NULL; + char name[AMDUAT_ASL_POINTER_NAME_MAX + 1u]; + bool have_name = false; + char *latest_hex = NULL; + if (!amduat_asl_ref_encode_hex(response_nodes[i], &ref_hex)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto cleanup; + } + have_name = amduatd_concepts_name_for_ref_cached(store, + concepts, + dcfg, + response_nodes[i], + name, + sizeof(name)); + if (!amduatd_concepts_latest_hex_for_ref(store, concepts, response_nodes[i], &latest_hex)) { + free(ref_hex); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto cleanup; + } + if (i != 0u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + (void)amduatd_strbuf_append_cstr(&b, "{\"concept_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, ref_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"name\":"); + if (have_name) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, name); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + (void)amduatd_strbuf_append_cstr(&b, ",\"latest_ref\":"); + if (latest_hex != NULL) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, latest_hex); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + if (include_versions && + !amduatd_graph_append_versions_json(&b, concepts, response_nodes[i], scan_end)) { + free(ref_hex); + free(latest_hex); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + (void)amduatd_strbuf_append_cstr(&b, "}"); + free(ref_hex); + free(latest_hex); + } + (void)amduatd_strbuf_append_cstr(&b, "],\"edges\":["); + for (i = 0u; i < page_edges_len; ++i) { + const amduatd_edge_entry_t *entry = &concepts->edges.items[page_edges[i]]; + amduat_reference_t predicate_ref; + char *subject_hex = NULL; + char *predicate_hex = NULL; + char *object_hex = NULL; + char *edge_hex = NULL; + memset(&predicate_ref, 0, sizeof(predicate_ref)); + if (!amduatd_relation_entry_ref(concepts, entry->rel, &predicate_ref) || + !amduat_asl_ref_encode_hex(entry->src_ref, &subject_hex) || + !amduat_asl_ref_encode_hex(predicate_ref, &predicate_hex) || + !amduat_asl_ref_encode_hex(entry->dst_ref, &object_hex) || + !amduat_asl_ref_encode_hex(entry->record_ref, &edge_hex)) { + amduat_reference_free(&predicate_ref); + free(subject_hex); + free(predicate_hex); + free(object_hex); + free(edge_hex); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto cleanup; + } + amduat_reference_free(&predicate_ref); + if (i != 0u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + (void)amduatd_strbuf_append_cstr(&b, "{\"subject_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, subject_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"predicate_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, predicate_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"object_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, object_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"edge_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, edge_hex); + (void)amduatd_strbuf_append_cstr(&b, "\"}"); + free(subject_hex); + free(predicate_hex); + free(object_hex); + free(edge_hex); + } + (void)amduatd_strbuf_append_cstr(&b, "]"); + if (traversal_truncated && frontier_refs_len > 0u) { + (void)amduatd_strbuf_append_cstr(&b, ",\"frontier\":["); + for (i = 0u; i < frontier_refs_len; ++i) { + char *frontier_hex = NULL; + if (!amduat_asl_ref_encode_hex(frontier_refs[i], &frontier_hex)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto cleanup; + } + if (i != 0u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, frontier_hex); + (void)amduatd_strbuf_append_cstr(&b, "\""); + free(frontier_hex); + } + (void)amduatd_strbuf_append_cstr(&b, "]"); + } + if (has_more) { + char token[64]; + if (!amduatd_graph_cursor_encode(next_cursor, token, sizeof(token))) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto cleanup; + } + (void)amduatd_strbuf_append_cstr(&b, ",\"next_cursor\":\""); + (void)amduatd_strbuf_append_cstr(&b, token); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, ",\"next_cursor\":null"); + } + (void)amduatd_strbuf_append_cstr(&b, ",\"truncated\":"); + (void)amduatd_strbuf_append_cstr(&b, (truncated || has_more) ? "true" : "false"); + if (include_stats) { + char num[64]; + int n; + (void)amduatd_strbuf_append_cstr(&b, ",\"stats\":{"); + n = snprintf(num, sizeof(num), "%llu", (unsigned long long)scanned_edges); + if (n > 0 && (size_t)n < sizeof(num)) { + (void)amduatd_strbuf_append_cstr(&b, "\"scanned_edges\":"); + (void)amduatd_strbuf_append(&b, num, (size_t)n); + } + n = snprintf(num, sizeof(num), "%llu", (unsigned long long)traversed_edges_len); + if (n > 0 && (size_t)n < sizeof(num)) { + (void)amduatd_strbuf_append_cstr(&b, ",\"traversed_edges\":"); + (void)amduatd_strbuf_append(&b, num, (size_t)n); + } + n = snprintf(num, sizeof(num), "%llu", (unsigned long long)response_nodes_len); + if (n > 0 && (size_t)n < sizeof(num)) { + (void)amduatd_strbuf_append_cstr(&b, ",\"returned_nodes\":"); + (void)amduatd_strbuf_append(&b, num, (size_t)n); + } + n = snprintf(num, sizeof(num), "%llu", (unsigned long long)page_edges_len); + if (n > 0 && (size_t)n < sizeof(num)) { + (void)amduatd_strbuf_append_cstr(&b, ",\"returned_edges\":"); + (void)amduatd_strbuf_append(&b, num, (size_t)n); + } + (void)amduatd_strbuf_append_cstr(&b, ",\"truncated_traversal\":"); + (void)amduatd_strbuf_append_cstr(&b, traversal_truncated ? "true" : "false"); + (void)amduatd_strbuf_append_cstr(&b, ",\"truncated_page\":"); + (void)amduatd_strbuf_append_cstr(&b, (truncated || has_more) ? "true" : "false"); + (void)amduatd_strbuf_append_cstr(&b, ",\"plan\":\""); + (void)amduatd_strbuf_append_cstr(&b, scan_plan); + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, "}"); + } + (void)amduatd_strbuf_append_cstr(&b, "}\n"); + + if (b.len > (size_t)max_result_bytes_u64) { + ok = amduatd_send_json_error(fd, 422, "Unprocessable Entity", "result too large"); + goto cleanup; + } + + ok = amduatd_http_send_json(fd, 200, "OK", b.data, false); + +cleanup: + amduatd_graph_ref_array_free(root_refs, root_refs_len); + amduatd_graph_ref_array_free(predicate_refs, predicate_refs_len); + amduatd_subgraph_nodes_free(seen_nodes, seen_nodes_len); + free(queue); + free(traversed_edges); + free(seen_edges); + free(page_edges); + amduatd_graph_ref_array_free(response_nodes, response_nodes_len); + amduatd_graph_ref_array_free(frontier_refs, frontier_refs_len); + amduat_reference_free(&provenance_ref); + amduatd_strbuf_free(&b); + return ok; +} + +static bool amduatd_handle_get_graph_paths(int fd, + amduat_asl_store_t *store, + const amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const amduatd_http_req_t *req) { + char from_buf[512]; + char to_buf[512]; + char pred_buf[512]; + char depth_buf[32]; + char k_buf[32]; + char as_of_buf[32]; + char expand_buf[16]; + char expand_artifacts_buf[16]; + char include_tombstoned_buf[16]; + char max_fanout_buf[32]; + char max_result_bytes_buf[32]; + char include_stats_buf[16]; + char provenance_buf[512]; + const size_t max_states = 20000u; + bool have_predicate = false; + bool have_provenance_ref = false; + bool expand_names = false; + bool expand_artifacts = false; + bool include_tombstoned = false; + bool include_stats = false; + bool always_empty = false; + uint64_t max_depth = 4u; + uint64_t k_u64 = 1u; + uint64_t max_fanout_u64 = 2048u; + uint64_t max_result_bytes_u64 = 1048576u; + size_t k = 1u; + size_t max_fanout = 2048u; + size_t scan_end = 0u; + amduat_reference_t from_ref; + amduat_reference_t to_ref; + amduat_reference_t predicate_ref; + amduat_reference_t provenance_ref; + amduatd_bfs_state_t *states = NULL; + size_t states_len = 0u; + size_t states_cap = 0u; + size_t *queue = NULL; + size_t q_pos = 0u; + size_t q_len = 0u; + size_t q_cap = 0u; + size_t *results = NULL; + size_t results_len = 0u; + size_t results_cap = 0u; + amduatd_strbuf_t b; + bool ok = false; + size_t i; + size_t scanned_edges = 0u; + const char *scan_plan = "src_bucket"; + + memset(&from_ref, 0, sizeof(from_ref)); + memset(&to_ref, 0, sizeof(to_ref)); + memset(&predicate_ref, 0, sizeof(predicate_ref)); + memset(&provenance_ref, 0, sizeof(provenance_ref)); + memset(&b, 0, sizeof(b)); + + if (store == NULL || concepts == NULL || req == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "internal error"); + } + if (!amduatd_concepts_ensure_query_index((amduatd_concepts_t *)concepts)) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "index error"); + } + if (amduatd_query_param(req->path, "from", from_buf, sizeof(from_buf)) == NULL || + amduatd_query_param(req->path, "to", to_buf, sizeof(to_buf)) == NULL) { + return amduatd_send_json_error(fd, 400, "Bad Request", "missing from/to"); + } + if (!amduatd_resolve_graph_ref(store, concepts, dcfg, from_buf, &from_ref)) { + return amduatd_send_json_error(fd, 404, "Not Found", "from not found"); + } + if (!amduatd_resolve_graph_ref(store, concepts, dcfg, to_buf, &to_ref)) { + amduat_reference_free(&from_ref); + return amduatd_send_json_error(fd, 404, "Not Found", "to not found"); + } + if (amduatd_query_param(req->path, "predicate", pred_buf, sizeof(pred_buf)) != NULL) { + if (!amduatd_resolve_relation_ref(concepts, pred_buf, &predicate_ref)) { + amduat_reference_free(&from_ref); + amduat_reference_free(&to_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicate"); + } + have_predicate = true; + } + if (amduatd_query_param(req->path, "max_depth", depth_buf, sizeof(depth_buf)) != NULL) { + if (!amduatd_parse_u64_query(depth_buf, &max_depth) || max_depth == 0u || max_depth > 32u) { + amduat_reference_free(&from_ref); + amduat_reference_free(&to_ref); + amduat_reference_free(&predicate_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid max_depth"); + } + } + if (amduatd_query_param(req->path, "k", k_buf, sizeof(k_buf)) != NULL) { + if (!amduatd_parse_u64_query(k_buf, &k_u64) || k_u64 == 0u || k_u64 > 16u) { + amduat_reference_free(&from_ref); + amduat_reference_free(&to_ref); + amduat_reference_free(&predicate_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid k"); + } + } + k = (size_t)k_u64; + if (amduatd_query_param(req->path, "expand_names", expand_buf, sizeof(expand_buf)) != NULL) { + if (!amduatd_parse_bool_query(expand_buf, &expand_names)) { + amduat_reference_free(&from_ref); + amduat_reference_free(&to_ref); + amduat_reference_free(&predicate_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid expand_names"); + } + } + if (amduatd_query_param(req->path, + "expand_artifacts", + expand_artifacts_buf, + sizeof(expand_artifacts_buf)) != NULL) { + if (!amduatd_parse_bool_query(expand_artifacts_buf, &expand_artifacts)) { + amduat_reference_free(&from_ref); + amduat_reference_free(&to_ref); + amduat_reference_free(&predicate_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid expand_artifacts"); + } + } + if (amduatd_query_param(req->path, + "include_tombstoned", + include_tombstoned_buf, + sizeof(include_tombstoned_buf)) != NULL) { + if (!amduatd_parse_bool_query(include_tombstoned_buf, &include_tombstoned)) { + amduat_reference_free(&from_ref); + amduat_reference_free(&to_ref); + amduat_reference_free(&predicate_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid include_tombstoned"); + } + } + if (amduatd_query_param(req->path, + "include_stats", + include_stats_buf, + sizeof(include_stats_buf)) != NULL) { + if (!amduatd_parse_bool_query(include_stats_buf, &include_stats)) { + amduat_reference_free(&from_ref); + amduat_reference_free(&to_ref); + amduat_reference_free(&predicate_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid include_stats"); + } + } + if (amduatd_query_param(req->path, + "provenance_ref", + provenance_buf, + sizeof(provenance_buf)) != NULL) { + if (!amduatd_resolve_graph_ref(store, + concepts, + dcfg, + provenance_buf, + &provenance_ref)) { + always_empty = true; + } else { + have_provenance_ref = true; + } + } + if (amduatd_query_param(req->path, + "max_fanout", + max_fanout_buf, + sizeof(max_fanout_buf)) != NULL) { + if (!amduatd_parse_u64_query(max_fanout_buf, &max_fanout_u64) || + max_fanout_u64 == 0u || max_fanout_u64 > 10000u) { + amduat_reference_free(&from_ref); + amduat_reference_free(&to_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid max_fanout"); + } + } + if (amduatd_query_param(req->path, + "max_result_bytes", + max_result_bytes_buf, + sizeof(max_result_bytes_buf)) != NULL) { + if (!amduatd_parse_u64_query(max_result_bytes_buf, &max_result_bytes_u64) || + max_result_bytes_u64 < 1024u || max_result_bytes_u64 > (8u * 1024u * 1024u)) { + amduat_reference_free(&from_ref); + amduat_reference_free(&to_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid max_result_bytes"); + } + } + scan_end = concepts->edges.len; + if (amduatd_query_param(req->path, "as_of", as_of_buf, sizeof(as_of_buf)) != NULL) { + if (!amduatd_graph_cursor_decode(as_of_buf, &scan_end)) { + amduat_reference_free(&from_ref); + amduat_reference_free(&to_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid as_of"); + } + if (scan_end > concepts->edges.len) { + scan_end = concepts->edges.len; + } + } + max_fanout = (size_t)max_fanout_u64; + + if (amduat_reference_eq(from_ref, to_ref)) { + char *from_hex = NULL; + char json[2048]; + int n; + if (!amduat_asl_ref_encode_hex(from_ref, &from_hex)) { + amduat_reference_free(&from_ref); + amduat_reference_free(&to_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + } + n = snprintf(json, + sizeof(json), + "{\"paths\":[{\"nodes\":[\"%s\"],\"edges\":[],\"hops\":[],\"depth\":0}]}\n", + from_hex); + if (n <= 0 || (size_t)n >= sizeof(json)) { + free(from_hex); + amduat_reference_free(&from_ref); + amduat_reference_free(&to_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "error"); + } + ok = amduatd_http_send_json(fd, 200, "OK", json, false); + free(from_hex); + amduat_reference_free(&from_ref); + amduat_reference_free(&to_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&provenance_ref); + return ok; + } + + states_cap = 128u; + states = (amduatd_bfs_state_t *)calloc(states_cap, sizeof(*states)); + q_cap = 128u; + queue = (size_t *)calloc(q_cap, sizeof(*queue)); + results_cap = k > 0u ? k : 1u; + results = (size_t *)calloc(results_cap, sizeof(*results)); + if (states == NULL || queue == NULL || results == NULL) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto paths_cleanup; + } + + if (!amduat_reference_clone(from_ref, &states[0].node_ref)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto paths_cleanup; + } + states[0].parent_index = -1; + states[0].via_edge_index = 0u; + states[0].depth = 0u; + states_len = 1u; + queue[q_len++] = 0u; + + while (!always_empty && q_pos < q_len && results_len < k) { + size_t cur_index = queue[q_pos++]; + const amduatd_bfs_state_t *cur = &states[cur_index]; + const size_t *edge_indices = NULL; + size_t edge_indices_len = 0u; + size_t bi; + if (cur->depth >= max_depth) { + continue; + } + if (have_predicate) { + const amduatd_ref_pair_edge_bucket_t *src_predicate_bucket = + amduatd_query_index_find_pair_bucket_const(concepts->qindex.src_predicate_buckets, + concepts->qindex.src_predicate_len, + cur->node_ref, + predicate_ref); + scan_plan = "src_predicate_bucket"; + if (src_predicate_bucket == NULL) { + continue; + } + edge_indices = src_predicate_bucket->edge_indices; + edge_indices_len = src_predicate_bucket->len; + } else { + const amduatd_ref_edge_bucket_t *src_bucket = + amduatd_query_index_find_bucket_const(concepts->qindex.src_buckets, + concepts->qindex.src_len, + cur->node_ref); + if (src_bucket == NULL) { + continue; + } + edge_indices = src_bucket->edge_indices; + edge_indices_len = src_bucket->len; + } + for (bi = 0; bi < edge_indices_len; ++bi) { + size_t edge_idx = edge_indices[bi]; + const amduatd_edge_entry_t *entry; + amduat_reference_t next_ref; + ssize_t walk; + bool cycle = false; + size_t new_index; + if (edge_idx >= scan_end) { + continue; + } + scanned_edges++; + if ((bi + 1u) > max_fanout) { + break; + } + entry = &concepts->edges.items[edge_idx]; + if (!include_tombstoned && + amduatd_graph_edge_is_tombstoned(concepts, entry->record_ref, scan_end)) { + continue; + } + if (have_provenance_ref && + !amduatd_graph_edge_matches_provenance(concepts, + entry->record_ref, + provenance_ref, + scan_end)) { + continue; + } + next_ref = entry->dst_ref; + walk = (ssize_t)cur_index; + while (walk >= 0) { + if (amduat_reference_eq(states[walk].node_ref, next_ref)) { + cycle = true; + break; + } + walk = states[walk].parent_index; + } + if (cycle) { + continue; + } + + if (states_len >= max_states) { + ok = amduatd_send_json_error(fd, 422, "Unprocessable Entity", "path search limit exceeded"); + goto paths_cleanup; + } + if (states_len == states_cap) { + size_t next_cap = states_cap * 2u; + amduatd_bfs_state_t *next_states; + if (next_cap > max_states) { + next_cap = max_states; + } + next_states = (amduatd_bfs_state_t *)realloc(states, next_cap * sizeof(*states)); + if (next_states == NULL) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto paths_cleanup; + } + states = next_states; + states_cap = next_cap; + } + if (q_len == q_cap) { + size_t next_q_cap = q_cap * 2u; + size_t *next_queue = (size_t *)realloc(queue, next_q_cap * sizeof(*queue)); + if (next_queue == NULL) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto paths_cleanup; + } + queue = next_queue; + q_cap = next_q_cap; + } + new_index = states_len; + memset(&states[new_index], 0, sizeof(states[new_index])); + if (!amduat_reference_clone(next_ref, &states[new_index].node_ref)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto paths_cleanup; + } + states[new_index].parent_index = (ssize_t)cur_index; + states[new_index].via_edge_index = edge_idx; + states[new_index].depth = cur->depth + 1u; + states_len++; + + if (amduat_reference_eq(next_ref, to_ref)) { + if (results_len == results_cap) { + size_t next_cap = results_cap * 2u; + size_t *next_results = (size_t *)realloc(results, next_cap * sizeof(*results)); + if (next_results == NULL) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto paths_cleanup; + } + results = next_results; + results_cap = next_cap; + } + results[results_len++] = new_index; + if (results_len >= k) { + break; + } + } else { + queue[q_len++] = new_index; + } + } + } + + if (!amduatd_strbuf_append_cstr(&b, "{\"paths\":[")) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto paths_cleanup; + } + for (i = 0; i < results_len; ++i) { + ssize_t walk = (ssize_t)results[i]; + size_t count = 0u; + ssize_t *chain = NULL; + size_t j; + + while (walk >= 0) { + count++; + walk = states[walk].parent_index; + } + chain = (ssize_t *)malloc(count * sizeof(*chain)); + if (chain == NULL) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto paths_cleanup; + } + walk = (ssize_t)results[i]; + for (j = count; j > 0; --j) { + chain[j - 1u] = walk; + walk = states[walk].parent_index; + } + if (i != 0u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + (void)amduatd_strbuf_append_cstr(&b, "{\"nodes\":["); + for (j = 0; j < count; ++j) { + char *node_hex = NULL; + if (!amduat_asl_ref_encode_hex(states[chain[j]].node_ref, &node_hex)) { + free(chain); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto paths_cleanup; + } + if (j != 0u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, node_hex); + (void)amduatd_strbuf_append_cstr(&b, "\""); + free(node_hex); + } + if (expand_names) { + (void)amduatd_strbuf_append_cstr(&b, "],\"node_names\":["); + for (j = 0; j < count; ++j) { + char node_name[AMDUAT_ASL_POINTER_NAME_MAX + 1u]; + bool have_node_name = amduatd_concepts_name_for_ref_cached(store, + concepts, + dcfg, + states[chain[j]].node_ref, + node_name, + sizeof(node_name)); + if (j != 0u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + if (have_node_name) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, node_name); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + } + (void)amduatd_strbuf_append_cstr(&b, "],\"edges\":["); + } else { + (void)amduatd_strbuf_append_cstr(&b, "],\"edges\":["); + } + if (expand_artifacts) { + (void)amduatd_strbuf_append_cstr(&b, "],\"node_latest_refs\":["); + for (j = 0; j < count; ++j) { + char *latest_hex = NULL; + if (!amduatd_concepts_latest_hex_for_ref(store, + concepts, + states[chain[j]].node_ref, + &latest_hex)) { + free(chain); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto paths_cleanup; + } + if (j != 0u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + if (latest_hex != NULL) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, latest_hex); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + free(latest_hex); + } + (void)amduatd_strbuf_append_cstr(&b, "],\"edges\":["); + for (j = 1u; j < count; ++j) { + char *edge_hex = NULL; + size_t edge_index = states[chain[j]].via_edge_index; + if (edge_index >= scan_end || + !amduat_asl_ref_encode_hex(concepts->edges.items[edge_index].record_ref, &edge_hex)) { + free(chain); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto paths_cleanup; + } + if (j != 1u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, edge_hex); + (void)amduatd_strbuf_append_cstr(&b, "\""); + free(edge_hex); + } + } + if (!expand_artifacts) for (j = 1u; j < count; ++j) { + char *edge_hex = NULL; + size_t edge_index = states[chain[j]].via_edge_index; + if (edge_index >= scan_end || + !amduat_asl_ref_encode_hex(concepts->edges.items[edge_index].record_ref, &edge_hex)) { + free(chain); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto paths_cleanup; + } + if (j != 1u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, edge_hex); + (void)amduatd_strbuf_append_cstr(&b, "\""); + free(edge_hex); + } + (void)amduatd_strbuf_append_cstr(&b, "],\"hops\":["); + for (j = 1u; j < count; ++j) { + size_t edge_index = states[chain[j]].via_edge_index; + const amduatd_edge_entry_t *entry = NULL; + amduat_reference_t rel_ref; + char *subject_hex = NULL; + char *object_hex = NULL; + char *predicate_hex = NULL; + char *edge_hex = NULL; + char subject_name[AMDUAT_ASL_POINTER_NAME_MAX + 1u]; + char object_name[AMDUAT_ASL_POINTER_NAME_MAX + 1u]; + bool have_subject_name = false; + bool have_object_name = false; + char *subject_latest_hex = NULL; + char *object_latest_hex = NULL; + if (edge_index >= scan_end) { + continue; + } + entry = &concepts->edges.items[edge_index]; + memset(&rel_ref, 0, sizeof(rel_ref)); + if (!amduatd_relation_entry_ref(concepts, entry->rel, &rel_ref) || + !amduat_asl_ref_encode_hex(entry->src_ref, &subject_hex) || + !amduat_asl_ref_encode_hex(rel_ref, &predicate_hex) || + !amduat_asl_ref_encode_hex(entry->dst_ref, &object_hex) || + !amduat_asl_ref_encode_hex(entry->record_ref, &edge_hex)) { + amduat_reference_free(&rel_ref); + free(subject_hex); + free(predicate_hex); + free(object_hex); + free(edge_hex); + continue; + } + amduat_reference_free(&rel_ref); + if (expand_names) { + have_subject_name = amduatd_concepts_name_for_ref_cached(store, + concepts, + dcfg, + entry->src_ref, + subject_name, + sizeof(subject_name)); + have_object_name = amduatd_concepts_name_for_ref_cached(store, + concepts, + dcfg, + entry->dst_ref, + object_name, + sizeof(object_name)); + } + if (expand_artifacts) { + if (!amduatd_concepts_latest_hex_for_ref(store, concepts, entry->src_ref, &subject_latest_hex) || + !amduatd_concepts_latest_hex_for_ref(store, concepts, entry->dst_ref, &object_latest_hex)) { + free(subject_hex); + free(predicate_hex); + free(object_hex); + free(edge_hex); + free(subject_latest_hex); + free(object_latest_hex); + free(chain); + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto paths_cleanup; + } + } + if (j != 1u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + (void)amduatd_strbuf_append_cstr(&b, "{\"subject_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, subject_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"predicate_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, predicate_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"object_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, object_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"edge_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, edge_hex); + (void)amduatd_strbuf_append_cstr(&b, "\""); + if (expand_names) { + (void)amduatd_strbuf_append_cstr(&b, ",\"subject_name\":"); + if (have_subject_name) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, subject_name); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + (void)amduatd_strbuf_append_cstr(&b, ",\"object_name\":"); + if (have_object_name) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, object_name); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + } + if (expand_artifacts) { + (void)amduatd_strbuf_append_cstr(&b, ",\"subject_latest_ref\":"); + if (subject_latest_hex != NULL) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, subject_latest_hex); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + (void)amduatd_strbuf_append_cstr(&b, ",\"object_latest_ref\":"); + if (object_latest_hex != NULL) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, object_latest_hex); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + } + (void)amduatd_strbuf_append_cstr(&b, "}"); + free(subject_hex); + free(predicate_hex); + free(object_hex); + free(edge_hex); + free(subject_latest_hex); + free(object_latest_hex); + } + (void)amduatd_strbuf_append_cstr(&b, "],\"depth\":"); + { + char depth_num[32]; + int n = snprintf(depth_num, sizeof(depth_num), "%llu", + (unsigned long long)(count > 0u ? (count - 1u) : 0u)); + if (n > 0 && (size_t)n < sizeof(depth_num)) { + (void)amduatd_strbuf_append(&b, depth_num, (size_t)n); + } + } + (void)amduatd_strbuf_append_cstr(&b, "}"); + free(chain); + } + (void)amduatd_strbuf_append_cstr(&b, "]"); + if (include_stats) { + char num[64]; + int n; + (void)amduatd_strbuf_append_cstr(&b, ",\"stats\":{"); + n = snprintf(num, sizeof(num), "%llu", (unsigned long long)scanned_edges); + if (n > 0 && (size_t)n < sizeof(num)) { + (void)amduatd_strbuf_append_cstr(&b, "\"scanned_edges\":"); + (void)amduatd_strbuf_append(&b, num, (size_t)n); + } + n = snprintf(num, sizeof(num), "%llu", (unsigned long long)results_len); + if (n > 0 && (size_t)n < sizeof(num)) { + (void)amduatd_strbuf_append_cstr(&b, ",\"returned_paths\":"); + (void)amduatd_strbuf_append(&b, num, (size_t)n); + } + (void)amduatd_strbuf_append_cstr(&b, ",\"plan\":\""); + (void)amduatd_strbuf_append_cstr(&b, scan_plan); + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, "}"); + } + (void)amduatd_strbuf_append_cstr(&b, "}\n"); + if (b.len > (size_t)max_result_bytes_u64) { + ok = amduatd_send_json_error(fd, 422, "Unprocessable Entity", "result too large"); + goto paths_cleanup; + } + ok = amduatd_http_send_json(fd, 200, "OK", b.data, false); + +paths_cleanup: + amduat_reference_free(&from_ref); + amduat_reference_free(&to_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&provenance_ref); + amduatd_bfs_states_free(states, states_len); + free(queue); + free(results); + amduatd_strbuf_free(&b); + return ok; +} + +static bool amduatd_handle_get_graph_edges(int fd, + amduat_asl_store_t *store, + const amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const amduatd_http_req_t *req) { + char subject_buf[512]; + char predicate_buf[512]; + char object_buf[512]; + char dir_buf[32]; + char limit_buf[32]; + char cursor_buf[32]; + char as_of_buf[32]; + char expand_buf[16]; + char expand_artifacts_buf[16]; + char provenance_buf[512]; + char include_tombstoned_buf[16]; + char include_stats_buf[16]; + char max_result_bytes_buf[32]; + bool have_subject = false; + bool have_predicate = false; + bool have_object = false; + bool always_empty = false; + bool expand_names = false; + bool expand_artifacts = false; + bool have_provenance_ref = false; + bool include_tombstoned = false; + bool include_stats = false; + int direction = 0; /* 0=any, 1=outgoing, 2=incoming */ + uint64_t limit_u64 = 100u; + size_t limit = 100u; + size_t cursor = 0u; + size_t scan_end = 0u; + size_t i; + size_t returned = 0u; + size_t next_cursor = 0u; + size_t scanned_edges = 0u; + const size_t *scan_indices = NULL; + size_t scan_indices_len = 0u; + const char *scan_plan = "full_scan"; + bool has_more = false; + uint64_t max_result_bytes_u64 = 1048576u; + amduat_reference_t subject_ref; + amduat_reference_t predicate_ref; + amduat_reference_t object_ref; + amduat_reference_t provenance_ref; + amduatd_strbuf_t b; + + memset(&subject_ref, 0, sizeof(subject_ref)); + memset(&predicate_ref, 0, sizeof(predicate_ref)); + memset(&object_ref, 0, sizeof(object_ref)); + memset(&provenance_ref, 0, sizeof(provenance_ref)); + memset(&b, 0, sizeof(b)); + + if (store == NULL || concepts == NULL || req == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", + "internal error"); + } + if (!amduatd_concepts_ensure_query_index((amduatd_concepts_t *)concepts)) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "index error"); + } + + if (amduatd_query_param(req->path, + "subject", + subject_buf, + sizeof(subject_buf)) != NULL) { + if (!amduatd_resolve_graph_ref(store, concepts, dcfg, subject_buf, &subject_ref)) { + always_empty = true; + } else { + have_subject = true; + } + } + if (amduatd_query_param(req->path, + "predicate", + predicate_buf, + sizeof(predicate_buf)) != NULL) { + if (!amduatd_resolve_relation_ref(concepts, predicate_buf, &predicate_ref)) { + amduat_reference_free(&subject_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicate"); + } + have_predicate = true; + } + if (amduatd_query_param(req->path, + "object", + object_buf, + sizeof(object_buf)) != NULL) { + if (!amduatd_resolve_graph_ref(store, concepts, dcfg, object_buf, &object_ref)) { + always_empty = true; + } else { + have_object = true; + } + } + if (amduatd_query_param(req->path, "dir", dir_buf, sizeof(dir_buf)) != NULL) { + if (strcmp(dir_buf, "any") == 0 || dir_buf[0] == '\0') { + direction = 0; + } else if (strcmp(dir_buf, "outgoing") == 0) { + direction = 1; + } else if (strcmp(dir_buf, "incoming") == 0) { + direction = 2; + } else { + amduat_reference_free(&subject_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&object_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid dir"); + } + } + if (amduatd_query_param(req->path, "limit", limit_buf, sizeof(limit_buf)) != NULL) { + if (!amduatd_parse_u64_query(limit_buf, &limit_u64) || + limit_u64 == 0u || + limit_u64 > 1000u) { + amduat_reference_free(&subject_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&object_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid limit"); + } + } + if (amduatd_query_param(req->path, "cursor", cursor_buf, sizeof(cursor_buf)) != NULL) { + if (!amduatd_graph_cursor_decode(cursor_buf, &cursor)) { + amduat_reference_free(&subject_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&object_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid cursor"); + } + } + if (amduatd_query_param(req->path, + "provenance_ref", + provenance_buf, + sizeof(provenance_buf)) != NULL) { + if (!amduatd_resolve_graph_ref(store, + concepts, + dcfg, + provenance_buf, + &provenance_ref)) { + always_empty = true; + } else { + have_provenance_ref = true; + } + } + if (amduatd_query_param(req->path, + "include_tombstoned", + include_tombstoned_buf, + sizeof(include_tombstoned_buf)) != NULL) { + if (!amduatd_parse_bool_query(include_tombstoned_buf, &include_tombstoned)) { + amduat_reference_free(&subject_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&object_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid include_tombstoned"); + } + } + if (amduatd_query_param(req->path, + "include_stats", + include_stats_buf, + sizeof(include_stats_buf)) != NULL) { + if (!amduatd_parse_bool_query(include_stats_buf, &include_stats)) { + amduat_reference_free(&subject_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&object_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid include_stats"); + } + } + if (amduatd_query_param(req->path, + "max_result_bytes", + max_result_bytes_buf, + sizeof(max_result_bytes_buf)) != NULL) { + if (!amduatd_parse_u64_query(max_result_bytes_buf, &max_result_bytes_u64) || + max_result_bytes_u64 < 1024u || max_result_bytes_u64 > (8u * 1024u * 1024u)) { + amduat_reference_free(&subject_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&object_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid max_result_bytes"); + } + } + if (amduatd_query_param(req->path, "expand_names", expand_buf, sizeof(expand_buf)) != NULL) { + if (!amduatd_parse_bool_query(expand_buf, &expand_names)) { + amduat_reference_free(&subject_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&object_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid expand_names"); + } + } + if (amduatd_query_param(req->path, + "expand_artifacts", + expand_artifacts_buf, + sizeof(expand_artifacts_buf)) != NULL) { + if (!amduatd_parse_bool_query(expand_artifacts_buf, &expand_artifacts)) { + amduat_reference_free(&subject_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&object_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid expand_artifacts"); + } + } + scan_end = concepts->edges.len; + if (amduatd_query_param(req->path, "as_of", as_of_buf, sizeof(as_of_buf)) != NULL) { + if (!amduatd_graph_cursor_decode(as_of_buf, &scan_end)) { + amduat_reference_free(&subject_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&object_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid as_of"); + } + if (scan_end > concepts->edges.len) { + scan_end = concepts->edges.len; + } + } + if (limit_u64 > (uint64_t)SIZE_MAX) { + amduat_reference_free(&subject_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&object_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid paging"); + } + limit = (size_t)limit_u64; + if (cursor > scan_end) { + cursor = scan_end; + } + if (!always_empty) { + if (have_predicate && have_subject) { + const amduatd_ref_pair_edge_bucket_t *bucket = + amduatd_query_index_find_pair_bucket_const( + (direction == 2) ? concepts->qindex.dst_predicate_buckets + : concepts->qindex.src_predicate_buckets, + (direction == 2) ? concepts->qindex.dst_predicate_len + : concepts->qindex.src_predicate_len, + subject_ref, + predicate_ref); + amduatd_scan_select_best_pair_bucket_with_plan(&scan_indices, + &scan_indices_len, + bucket, + &scan_plan, + "subject_predicate"); + } + if (have_predicate && have_object) { + const amduatd_ref_pair_edge_bucket_t *bucket = + amduatd_query_index_find_pair_bucket_const( + (direction == 2) ? concepts->qindex.src_predicate_buckets + : concepts->qindex.dst_predicate_buckets, + (direction == 2) ? concepts->qindex.src_predicate_len + : concepts->qindex.dst_predicate_len, + object_ref, + predicate_ref); + amduatd_scan_select_best_pair_bucket_with_plan(&scan_indices, + &scan_indices_len, + bucket, + &scan_plan, + "object_predicate"); + } + if (scan_indices == NULL && have_predicate) { + const amduatd_ref_edge_bucket_t *bucket = + amduatd_query_index_find_bucket_const(concepts->qindex.predicate_buckets, + concepts->qindex.predicate_len, + predicate_ref); + amduatd_scan_select_best_bucket_with_plan(&scan_indices, + &scan_indices_len, + bucket, + &scan_plan, + "predicate"); + } + if (scan_indices == NULL && have_subject) { + const amduatd_ref_edge_bucket_t *bucket = + amduatd_query_index_find_bucket_const( + (direction == 2) ? concepts->qindex.dst_buckets : concepts->qindex.src_buckets, + (direction == 2) ? concepts->qindex.dst_len : concepts->qindex.src_len, + subject_ref); + amduatd_scan_select_best_bucket_with_plan(&scan_indices, + &scan_indices_len, + bucket, + &scan_plan, + "subject"); + } + if (scan_indices == NULL && have_object) { + const amduatd_ref_edge_bucket_t *bucket = + amduatd_query_index_find_bucket_const( + (direction == 2) ? concepts->qindex.src_buckets : concepts->qindex.dst_buckets, + (direction == 2) ? concepts->qindex.src_len : concepts->qindex.dst_len, + object_ref); + amduatd_scan_select_best_bucket_with_plan(&scan_indices, + &scan_indices_len, + bucket, + &scan_plan, + "object"); + } + } + + if (!amduatd_strbuf_append_cstr(&b, "{\"edges\":[")) { + amduat_reference_free(&subject_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&object_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + + for (i = 0u; !always_empty && + ((scan_indices != NULL) ? (i < scan_indices_len) : (i < scan_end)); + ++i) { + size_t edge_i = (scan_indices != NULL) ? scan_indices[i] : i; + const amduatd_edge_entry_t *entry; + amduat_reference_t lhs_ref; + amduat_reference_t rhs_ref; + amduat_reference_t entry_predicate_ref; + char *lhs_hex = NULL; + char *rhs_hex = NULL; + char *predicate_hex = NULL; + char *edge_hex = NULL; + char lhs_name[AMDUAT_ASL_POINTER_NAME_MAX + 1u]; + char rhs_name[AMDUAT_ASL_POINTER_NAME_MAX + 1u]; + bool have_lhs_name = false; + bool have_rhs_name = false; + char *lhs_latest_hex = NULL; + char *rhs_latest_hex = NULL; + bool match = true; + + if (scan_indices != NULL) { + if (edge_i < cursor || edge_i >= scan_end) { + continue; + } + } else if (edge_i < cursor) { + continue; + } + entry = &concepts->edges.items[edge_i]; + memset(&lhs_ref, 0, sizeof(lhs_ref)); + memset(&rhs_ref, 0, sizeof(rhs_ref)); + memset(&entry_predicate_ref, 0, sizeof(entry_predicate_ref)); + scanned_edges++; + + if (!amduatd_relation_entry_ref(concepts, entry->rel, &entry_predicate_ref)) { + continue; + } + if (direction == 2) { + lhs_ref = entry->dst_ref; + rhs_ref = entry->src_ref; + } else { + lhs_ref = entry->src_ref; + rhs_ref = entry->dst_ref; + } + if (have_subject && !amduat_reference_eq(lhs_ref, subject_ref)) { + match = false; + } + if (match && have_object && !amduat_reference_eq(rhs_ref, object_ref)) { + match = false; + } + if (match && have_predicate && + !amduat_reference_eq(entry_predicate_ref, predicate_ref)) { + match = false; + } + if (match && !include_tombstoned && + amduatd_graph_edge_is_tombstoned(concepts, entry->record_ref, scan_end)) { + match = false; + } + if (match && have_provenance_ref && + !amduatd_graph_edge_matches_provenance(concepts, + entry->record_ref, + provenance_ref, + scan_end)) { + match = false; + } + amduat_reference_free(&entry_predicate_ref); + if (!match) { + continue; + } + + if (!amduat_asl_ref_encode_hex(lhs_ref, &lhs_hex) || + !amduat_asl_ref_encode_hex(rhs_ref, &rhs_hex) || + !amduatd_relation_entry_ref(concepts, entry->rel, &entry_predicate_ref) || + !amduat_asl_ref_encode_hex(entry_predicate_ref, &predicate_hex) || + !amduat_asl_ref_encode_hex(entry->record_ref, &edge_hex)) { + amduat_reference_free(&entry_predicate_ref); + free(lhs_hex); + free(rhs_hex); + free(predicate_hex); + free(edge_hex); + continue; + } + amduat_reference_free(&entry_predicate_ref); + if (expand_names) { + have_lhs_name = amduatd_concepts_name_for_ref_cached(store, + concepts, + dcfg, + lhs_ref, + lhs_name, + sizeof(lhs_name)); + have_rhs_name = amduatd_concepts_name_for_ref_cached(store, + concepts, + dcfg, + rhs_ref, + rhs_name, + sizeof(rhs_name)); + } + if (expand_artifacts) { + if (!amduatd_concepts_latest_hex_for_ref(store, concepts, lhs_ref, &lhs_latest_hex) || + !amduatd_concepts_latest_hex_for_ref(store, concepts, rhs_ref, &rhs_latest_hex)) { + free(lhs_hex); + free(rhs_hex); + free(predicate_hex); + free(edge_hex); + free(lhs_latest_hex); + free(rhs_latest_hex); + amduat_reference_free(&subject_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&object_ref); + amduatd_strbuf_free(&b); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + } + } + + if (returned != 0u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + returned++; + (void)amduatd_strbuf_append_cstr(&b, "{\"subject_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, lhs_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"predicate_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, predicate_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"object_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, rhs_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"edge_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, edge_hex); + (void)amduatd_strbuf_append_cstr(&b, "\""); + if (expand_names) { + (void)amduatd_strbuf_append_cstr(&b, ",\"subject_name\":"); + if (have_lhs_name) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, lhs_name); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + (void)amduatd_strbuf_append_cstr(&b, ",\"object_name\":"); + if (have_rhs_name) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, rhs_name); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + } + if (expand_artifacts) { + (void)amduatd_strbuf_append_cstr(&b, ",\"subject_latest_ref\":"); + if (lhs_latest_hex != NULL) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, lhs_latest_hex); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + (void)amduatd_strbuf_append_cstr(&b, ",\"object_latest_ref\":"); + if (rhs_latest_hex != NULL) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, rhs_latest_hex); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + } + (void)amduatd_strbuf_append_cstr(&b, "}"); + + free(lhs_hex); + free(rhs_hex); + free(predicate_hex); + free(edge_hex); + free(lhs_latest_hex); + free(rhs_latest_hex); + + if (returned >= limit) { + next_cursor = edge_i + 1u; + break; + } + } + + if (returned >= limit && next_cursor < scan_end) { + size_t j; + for (j = 0u; + (scan_indices != NULL) ? (j < scan_indices_len) : (j < scan_end); + ++j) { + size_t edge_j = (scan_indices != NULL) ? scan_indices[j] : j; + const amduatd_edge_entry_t *entry; + amduat_reference_t lhs_ref; + amduat_reference_t rhs_ref; + amduat_reference_t entry_predicate_ref; + bool match = true; + if (edge_j < next_cursor || edge_j >= scan_end) { + continue; + } + entry = &concepts->edges.items[edge_j]; + memset(&lhs_ref, 0, sizeof(lhs_ref)); + memset(&rhs_ref, 0, sizeof(rhs_ref)); + memset(&entry_predicate_ref, 0, sizeof(entry_predicate_ref)); + if (!amduatd_relation_entry_ref(concepts, entry->rel, &entry_predicate_ref)) { + continue; + } + if (direction == 2) { + lhs_ref = entry->dst_ref; + rhs_ref = entry->src_ref; + } else { + lhs_ref = entry->src_ref; + rhs_ref = entry->dst_ref; + } + if (have_subject && !amduat_reference_eq(lhs_ref, subject_ref)) { + match = false; + } + if (match && have_object && !amduat_reference_eq(rhs_ref, object_ref)) { + match = false; + } + if (match && have_predicate && + !amduat_reference_eq(entry_predicate_ref, predicate_ref)) { + match = false; + } + if (match && !include_tombstoned && + amduatd_graph_edge_is_tombstoned(concepts, entry->record_ref, scan_end)) { + match = false; + } + if (match && have_provenance_ref && + !amduatd_graph_edge_matches_provenance(concepts, + entry->record_ref, + provenance_ref, + scan_end)) { + match = false; + } + amduat_reference_free(&entry_predicate_ref); + if (match) { + has_more = true; + break; + } + } + } + + if (include_stats) { + char num[64]; + int n; + (void)amduatd_strbuf_append_cstr(&b, "],\"stats\":{"); + (void)amduatd_strbuf_append_cstr(&b, "\"plan\":\""); + (void)amduatd_strbuf_append_cstr(&b, scan_plan); + (void)amduatd_strbuf_append_cstr(&b, "\""); + n = snprintf(num, sizeof(num), "%llu", (unsigned long long)scanned_edges); + if (n > 0 && (size_t)n < sizeof(num)) { + (void)amduatd_strbuf_append_cstr(&b, ",\"scanned_edges\":"); + (void)amduatd_strbuf_append(&b, num, (size_t)n); + } + n = snprintf(num, sizeof(num), "%llu", (unsigned long long)returned); + if (n > 0 && (size_t)n < sizeof(num)) { + (void)amduatd_strbuf_append_cstr(&b, ",\"returned_edges\":"); + (void)amduatd_strbuf_append(&b, num, (size_t)n); + } + (void)amduatd_strbuf_append_cstr(&b, "},\"paging\":{"); + } else { + (void)amduatd_strbuf_append_cstr(&b, "],\"paging\":{"); + } + if (has_more) { + char token[64]; + if (!amduatd_graph_cursor_encode(next_cursor, token, sizeof(token))) { + amduat_reference_free(&subject_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&object_ref); + amduat_reference_free(&provenance_ref); + amduatd_strbuf_free(&b); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + } + (void)amduatd_strbuf_append_cstr(&b, "\"next_cursor\":\""); + (void)amduatd_strbuf_append_cstr(&b, token); + (void)amduatd_strbuf_append_cstr(&b, "\",\"has_more\":true"); + } else { + (void)amduatd_strbuf_append_cstr(&b, "\"next_cursor\":null,\"has_more\":false"); + } + (void)amduatd_strbuf_append_cstr(&b, "}}\n"); + + if (b.len > (size_t)max_result_bytes_u64) { + amduat_reference_free(&subject_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&object_ref); + amduat_reference_free(&provenance_ref); + amduatd_strbuf_free(&b); + return amduatd_send_json_error(fd, 422, "Unprocessable Entity", "result too large"); + } + + amduat_reference_free(&subject_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&object_ref); + amduat_reference_free(&provenance_ref); + { + bool ok = amduatd_http_send_json(fd, 200, "OK", b.data, false); + amduatd_strbuf_free(&b); + return ok; + } +} + +static bool amduatd_handle_get_graph_neighbors(int fd, + amduat_asl_store_t *store, + const amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const char *name, + const amduatd_http_req_t *req) { + char predicate_buf[512]; + char provenance_buf[512]; + char dir_buf[32]; + char limit_buf[32]; + char cursor_buf[32]; + char as_of_buf[32]; + char include_tombstoned_buf[16]; + char expand_buf[16]; + char expand_artifacts_buf[16]; + bool have_predicate = false; + bool have_provenance_ref = false; + bool always_empty = false; + bool include_tombstoned = false; + bool expand_names = false; + bool expand_artifacts = false; + int direction = 0; /* 0=any, 1=outgoing, 2=incoming */ + uint64_t limit_u64 = 100u; + size_t limit = 100u; + size_t cursor = 0u; + size_t scan_end = 0u; + size_t i; + size_t returned = 0u; + size_t next_cursor = 0u; + bool has_more = false; + amduat_reference_t node_ref; + amduat_reference_t predicate_ref; + amduat_reference_t provenance_ref; + amduatd_strbuf_t b; + amduat_octets_t scoped = amduat_octets(NULL, 0u); + const amduatd_space_t *space = dcfg != NULL ? &dcfg->space : NULL; + + memset(&node_ref, 0, sizeof(node_ref)); + memset(&predicate_ref, 0, sizeof(predicate_ref)); + memset(&provenance_ref, 0, sizeof(provenance_ref)); + memset(&b, 0, sizeof(b)); + + if (store == NULL || concepts == NULL || name == NULL || req == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", + "internal error"); + } + if (!amduatd_concepts_ensure_query_index((amduatd_concepts_t *)concepts)) { + amduat_octets_free(&scoped); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "index error"); + } + if (!amduatd_space_scope_name(space, name, &scoped)) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid name"); + } + if (!amduatd_concepts_lookup_alias(store, concepts, (const char *)scoped.data, &node_ref)) { + amduat_octets_free(&scoped); + return amduatd_send_json_error(fd, 404, "Not Found", "unknown concept"); + } + amduat_octets_free(&scoped); + + if (amduatd_query_param(req->path, + "predicate", + predicate_buf, + sizeof(predicate_buf)) != NULL) { + if (!amduatd_resolve_relation_ref(concepts, predicate_buf, &predicate_ref)) { + amduat_reference_free(&node_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicate"); + } + have_predicate = true; + } + if (amduatd_query_param(req->path, + "provenance_ref", + provenance_buf, + sizeof(provenance_buf)) != NULL) { + if (!amduatd_resolve_graph_ref(store, + concepts, + dcfg, + provenance_buf, + &provenance_ref)) { + always_empty = true; + } else { + have_provenance_ref = true; + } + } + if (amduatd_query_param(req->path, "dir", dir_buf, sizeof(dir_buf)) != NULL) { + if (strcmp(dir_buf, "any") == 0 || dir_buf[0] == '\0') { + direction = 0; + } else if (strcmp(dir_buf, "outgoing") == 0) { + direction = 1; + } else if (strcmp(dir_buf, "incoming") == 0) { + direction = 2; + } else { + amduat_reference_free(&node_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid dir"); + } + } + if (amduatd_query_param(req->path, "limit", limit_buf, sizeof(limit_buf)) != NULL) { + if (!amduatd_parse_u64_query(limit_buf, &limit_u64) || + limit_u64 == 0u || + limit_u64 > 1000u) { + amduat_reference_free(&node_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid limit"); + } + } + if (amduatd_query_param(req->path, "cursor", cursor_buf, sizeof(cursor_buf)) != NULL) { + if (!amduatd_graph_cursor_decode(cursor_buf, &cursor)) { + amduat_reference_free(&node_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid cursor"); + } + } + if (amduatd_query_param(req->path, + "include_tombstoned", + include_tombstoned_buf, + sizeof(include_tombstoned_buf)) != NULL) { + if (!amduatd_parse_bool_query(include_tombstoned_buf, &include_tombstoned)) { + amduat_reference_free(&node_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid include_tombstoned"); + } + } + if (amduatd_query_param(req->path, "expand_names", expand_buf, sizeof(expand_buf)) != NULL) { + if (!amduatd_parse_bool_query(expand_buf, &expand_names)) { + amduat_reference_free(&node_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid expand_names"); + } + } + if (amduatd_query_param(req->path, + "expand_artifacts", + expand_artifacts_buf, + sizeof(expand_artifacts_buf)) != NULL) { + if (!amduatd_parse_bool_query(expand_artifacts_buf, &expand_artifacts)) { + amduat_reference_free(&node_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid expand_artifacts"); + } + } + scan_end = concepts->edges.len; + if (amduatd_query_param(req->path, "as_of", as_of_buf, sizeof(as_of_buf)) != NULL) { + if (!amduatd_graph_cursor_decode(as_of_buf, &scan_end)) { + amduat_reference_free(&node_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid as_of"); + } + if (scan_end > concepts->edges.len) { + scan_end = concepts->edges.len; + } + } + if (limit_u64 > (uint64_t)SIZE_MAX) { + amduat_reference_free(&node_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid paging"); + } + limit = (size_t)limit_u64; + if (cursor > scan_end) { + cursor = scan_end; + } + + if (!amduatd_strbuf_append_cstr(&b, "{\"neighbors\":[")) { + amduat_reference_free(&node_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&provenance_ref); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + + if (!always_empty && (direction == 1 || direction == 2)) { + const amduatd_ref_edge_bucket_t *bucket = + direction == 1 + ? amduatd_query_index_find_bucket_const(concepts->qindex.src_buckets, + concepts->qindex.src_len, + node_ref) + : amduatd_query_index_find_bucket_const(concepts->qindex.dst_buckets, + concepts->qindex.dst_len, + node_ref); + size_t bi; + if (bucket == NULL) { + scan_end = 0u; + } + for (bi = 0u; bucket != NULL && bi < bucket->len; ++bi) { + size_t edge_i = bucket->edge_indices[bi]; + const amduatd_edge_entry_t *entry; + if (edge_i < cursor || edge_i >= scan_end) { + continue; + } + i = edge_i; + entry = &concepts->edges.items[i]; + { + amduat_reference_t entry_predicate_ref; + amduat_reference_t neighbor_ref; + const char *dir_name = NULL; + bool match = false; + char *neighbor_hex = NULL; + char *predicate_hex = NULL; + char *edge_hex = NULL; + char neighbor_name[AMDUAT_ASL_POINTER_NAME_MAX + 1u]; + bool have_neighbor_name = false; + char *neighbor_latest_hex = NULL; + + memset(&entry_predicate_ref, 0, sizeof(entry_predicate_ref)); + memset(&neighbor_ref, 0, sizeof(neighbor_ref)); + + if (!amduatd_relation_entry_ref(concepts, entry->rel, &entry_predicate_ref)) { + continue; + } + if (have_predicate && !amduat_reference_eq(entry_predicate_ref, predicate_ref)) { + amduat_reference_free(&entry_predicate_ref); + continue; + } + if (direction == 1 && amduat_reference_eq(entry->src_ref, node_ref)) { + if (!amduat_reference_clone(entry->dst_ref, &neighbor_ref)) { + amduat_reference_free(&entry_predicate_ref); + continue; + } + dir_name = "outgoing"; + match = true; + } else if (direction == 2 && amduat_reference_eq(entry->dst_ref, node_ref)) { + if (!amduat_reference_clone(entry->src_ref, &neighbor_ref)) { + amduat_reference_free(&entry_predicate_ref); + continue; + } + dir_name = "incoming"; + match = true; + } + if (!match) { + amduat_reference_free(&entry_predicate_ref); + continue; + } + if (!include_tombstoned && + amduatd_graph_edge_is_tombstoned(concepts, entry->record_ref, scan_end)) { + amduat_reference_free(&entry_predicate_ref); + amduat_reference_free(&neighbor_ref); + continue; + } + if (have_provenance_ref && + !amduatd_graph_edge_matches_provenance(concepts, + entry->record_ref, + provenance_ref, + scan_end)) { + amduat_reference_free(&entry_predicate_ref); + amduat_reference_free(&neighbor_ref); + continue; + } + + if (!amduat_asl_ref_encode_hex(neighbor_ref, &neighbor_hex) || + !amduat_asl_ref_encode_hex(entry_predicate_ref, &predicate_hex) || + !amduat_asl_ref_encode_hex(entry->record_ref, &edge_hex)) { + amduat_reference_free(&entry_predicate_ref); + amduat_reference_free(&neighbor_ref); + free(neighbor_hex); + free(predicate_hex); + free(edge_hex); + continue; + } + amduat_reference_free(&entry_predicate_ref); + amduat_reference_free(&neighbor_ref); + if (expand_names) { + have_neighbor_name = amduatd_concepts_name_for_ref_cached(store, + concepts, + dcfg, + match && dir_name != NULL && + strcmp(dir_name, "incoming") == 0 + ? entry->src_ref + : entry->dst_ref, + neighbor_name, + sizeof(neighbor_name)); + } + if (expand_artifacts) { + if (!amduatd_concepts_latest_hex_for_ref(store, + concepts, + match && dir_name != NULL && + strcmp(dir_name, "incoming") == 0 + ? entry->src_ref + : entry->dst_ref, + &neighbor_latest_hex)) { + free(neighbor_hex); + free(predicate_hex); + free(edge_hex); + amduat_reference_free(&node_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&provenance_ref); + amduatd_strbuf_free(&b); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + } + } + if (returned != 0u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + returned++; + (void)amduatd_strbuf_append_cstr(&b, "{\"neighbor_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, neighbor_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"predicate_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, predicate_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"edge_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, edge_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"direction\":\""); + (void)amduatd_strbuf_append_cstr(&b, dir_name != NULL ? dir_name : "any"); + (void)amduatd_strbuf_append_cstr(&b, "\""); + if (expand_names) { + (void)amduatd_strbuf_append_cstr(&b, ",\"neighbor_name\":"); + if (have_neighbor_name) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, neighbor_name); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + } + if (expand_artifacts) { + (void)amduatd_strbuf_append_cstr(&b, ",\"neighbor_latest_ref\":"); + if (neighbor_latest_hex != NULL) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, neighbor_latest_hex); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + } + (void)amduatd_strbuf_append_cstr(&b, "}"); + free(neighbor_hex); + free(predicate_hex); + free(edge_hex); + free(neighbor_latest_hex); + if (returned >= limit) { + next_cursor = i + 1u; + break; + } + } + } + } else for (i = cursor; !always_empty && i < scan_end; ++i) { + const amduatd_edge_entry_t *entry = &concepts->edges.items[i]; + amduat_reference_t entry_predicate_ref; + amduat_reference_t neighbor_ref; + const char *dir_name = NULL; + bool match = false; + char *neighbor_hex = NULL; + char *predicate_hex = NULL; + char *edge_hex = NULL; + char neighbor_name[AMDUAT_ASL_POINTER_NAME_MAX + 1u]; + bool have_neighbor_name = false; + char *neighbor_latest_hex = NULL; + + memset(&entry_predicate_ref, 0, sizeof(entry_predicate_ref)); + memset(&neighbor_ref, 0, sizeof(neighbor_ref)); + + if (!amduatd_relation_entry_ref(concepts, entry->rel, &entry_predicate_ref)) { + continue; + } + if (have_predicate && !amduat_reference_eq(entry_predicate_ref, predicate_ref)) { + amduat_reference_free(&entry_predicate_ref); + continue; + } + if (direction == 0 || direction == 1) { + if (amduat_reference_eq(entry->src_ref, node_ref)) { + if (!amduat_reference_clone(entry->dst_ref, &neighbor_ref)) { + amduat_reference_free(&entry_predicate_ref); + continue; + } + dir_name = "outgoing"; + match = true; + } + } + if (!match && (direction == 0 || direction == 2)) { + if (amduat_reference_eq(entry->dst_ref, node_ref)) { + if (!amduat_reference_clone(entry->src_ref, &neighbor_ref)) { + amduat_reference_free(&entry_predicate_ref); + continue; + } + dir_name = "incoming"; + match = true; + } + } + if (!match) { + amduat_reference_free(&entry_predicate_ref); + continue; + } + if (!include_tombstoned && + amduatd_graph_edge_is_tombstoned(concepts, entry->record_ref, scan_end)) { + amduat_reference_free(&entry_predicate_ref); + amduat_reference_free(&neighbor_ref); + continue; + } + if (have_provenance_ref && + !amduatd_graph_edge_matches_provenance(concepts, + entry->record_ref, + provenance_ref, + scan_end)) { + amduat_reference_free(&entry_predicate_ref); + amduat_reference_free(&neighbor_ref); + continue; + } + + if (!amduat_asl_ref_encode_hex(neighbor_ref, &neighbor_hex) || + !amduat_asl_ref_encode_hex(entry_predicate_ref, &predicate_hex) || + !amduat_asl_ref_encode_hex(entry->record_ref, &edge_hex)) { + amduat_reference_free(&entry_predicate_ref); + amduat_reference_free(&neighbor_ref); + free(neighbor_hex); + free(predicate_hex); + free(edge_hex); + continue; + } + amduat_reference_free(&entry_predicate_ref); + amduat_reference_free(&neighbor_ref); + if (expand_names) { + have_neighbor_name = amduatd_concepts_name_for_ref_cached(store, + concepts, + dcfg, + match && dir_name != NULL && + strcmp(dir_name, "incoming") == 0 + ? entry->src_ref + : entry->dst_ref, + neighbor_name, + sizeof(neighbor_name)); + } + if (expand_artifacts) { + if (!amduatd_concepts_latest_hex_for_ref(store, + concepts, + match && dir_name != NULL && + strcmp(dir_name, "incoming") == 0 + ? entry->src_ref + : entry->dst_ref, + &neighbor_latest_hex)) { + free(neighbor_hex); + free(predicate_hex); + free(edge_hex); + amduat_reference_free(&node_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&provenance_ref); + amduatd_strbuf_free(&b); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + } + } + + if (returned != 0u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + returned++; + (void)amduatd_strbuf_append_cstr(&b, "{\"neighbor_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, neighbor_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"predicate_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, predicate_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"edge_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, edge_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"direction\":\""); + (void)amduatd_strbuf_append_cstr(&b, dir_name != NULL ? dir_name : "any"); + (void)amduatd_strbuf_append_cstr(&b, "\""); + if (expand_names) { + (void)amduatd_strbuf_append_cstr(&b, ",\"neighbor_name\":"); + if (have_neighbor_name) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, neighbor_name); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + } + if (expand_artifacts) { + (void)amduatd_strbuf_append_cstr(&b, ",\"neighbor_latest_ref\":"); + if (neighbor_latest_hex != NULL) { + (void)amduatd_strbuf_append_cstr(&b, "\""); + (void)amduatd_strbuf_append_cstr(&b, neighbor_latest_hex); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else { + (void)amduatd_strbuf_append_cstr(&b, "null"); + } + } + (void)amduatd_strbuf_append_cstr(&b, "}"); + + free(neighbor_hex); + free(predicate_hex); + free(edge_hex); + free(neighbor_latest_hex); + + if (returned >= limit) { + next_cursor = i + 1u; + break; + } + } + + if (returned >= limit && next_cursor < scan_end) { + size_t j; + for (j = next_cursor; !always_empty && j < scan_end; ++j) { + const amduatd_edge_entry_t *entry = &concepts->edges.items[j]; + amduat_reference_t entry_predicate_ref; + bool match = false; + memset(&entry_predicate_ref, 0, sizeof(entry_predicate_ref)); + if (!amduatd_relation_entry_ref(concepts, entry->rel, &entry_predicate_ref)) { + continue; + } + if (have_predicate && !amduat_reference_eq(entry_predicate_ref, predicate_ref)) { + amduat_reference_free(&entry_predicate_ref); + continue; + } + if (direction == 0 || direction == 1) { + if (amduat_reference_eq(entry->src_ref, node_ref)) { + match = true; + } + } + if (!match && (direction == 0 || direction == 2)) { + if (amduat_reference_eq(entry->dst_ref, node_ref)) { + match = true; + } + } + if (match && !include_tombstoned && + amduatd_graph_edge_is_tombstoned(concepts, entry->record_ref, scan_end)) { + match = false; + } + if (match && have_provenance_ref && + !amduatd_graph_edge_matches_provenance(concepts, + entry->record_ref, + provenance_ref, + scan_end)) { + match = false; + } + amduat_reference_free(&entry_predicate_ref); + if (match) { + has_more = true; + break; + } + } + } + + (void)amduatd_strbuf_append_cstr(&b, "],\"paging\":{"); + if (has_more) { + char token[64]; + if (!amduatd_graph_cursor_encode(next_cursor, token, sizeof(token))) { + amduat_reference_free(&node_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&provenance_ref); + amduatd_strbuf_free(&b); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + } + (void)amduatd_strbuf_append_cstr(&b, "\"next_cursor\":\""); + (void)amduatd_strbuf_append_cstr(&b, token); + (void)amduatd_strbuf_append_cstr(&b, "\",\"has_more\":true"); + } else { + (void)amduatd_strbuf_append_cstr(&b, "\"next_cursor\":null,\"has_more\":false"); + } + (void)amduatd_strbuf_append_cstr(&b, "}}\n"); + + amduat_reference_free(&node_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&provenance_ref); + { + bool ok = amduatd_http_send_json(fd, 200, "OK", b.data, false); + amduatd_strbuf_free(&b); + return ok; + } +} + +static bool amduatd_handle_post_graph_edges(int fd, + amduat_asl_store_t *store, + amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const amduatd_http_req_t *req) { + uint8_t *body = NULL; + const char *p = NULL; + const char *end = NULL; + bool have_subject = false; + bool have_predicate = false; + bool have_object = false; + bool have_metadata_ref = false; + bool have_provenance = false; + char *subject = NULL; + char *predicate = NULL; + char *object = NULL; + char *metadata_ref = NULL; + const char *prov_err = NULL; + amduatd_provenance_input_t provenance; + amduat_reference_t subject_ref; + amduat_reference_t predicate_ref; + amduat_reference_t object_ref; + amduat_reference_t edge_ref; + amduat_reference_t metadata_target_ref; + amduat_reference_t provenance_edge_ref; + char *subject_hex = NULL; + char *predicate_hex = NULL; + char *object_hex = NULL; + char *edge_hex = NULL; + char *metadata_hex = NULL; + amduat_octets_t actor = amduat_octets(NULL, 0u); + char json[2048]; + bool ok = false; + + memset(&provenance, 0, sizeof(provenance)); + memset(&subject_ref, 0, sizeof(subject_ref)); + memset(&predicate_ref, 0, sizeof(predicate_ref)); + memset(&object_ref, 0, sizeof(object_ref)); + memset(&edge_ref, 0, sizeof(edge_ref)); + memset(&metadata_target_ref, 0, sizeof(metadata_target_ref)); + memset(&provenance_edge_ref, 0, sizeof(provenance_edge_ref)); + + if (store == NULL || concepts == NULL || req == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "internal error"); + } + if (req->has_actor) { + actor = req->actor; + } + if (req->content_length == 0u || req->content_length > (256u * 1024u)) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid body"); + } + body = (uint8_t *)malloc(req->content_length); + if (body == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + if (!amduatd_read_exact(fd, body, req->content_length)) { + free(body); + return false; + } + + p = (const char *)body; + end = (const char *)body + req->content_length; + if (!amduatd_json_expect(&p, end, '{')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto graph_edges_cleanup; + } + for (;;) { + const char *key = NULL; + size_t key_len = 0; + const char *sv = NULL; + size_t sv_len = 0; + const char *cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &key, &key_len) || + !amduatd_json_expect(&p, end, ':')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto graph_edges_cleanup; + } + if (key_len == strlen("subject") && memcmp(key, "subject", key_len) == 0) { + if (have_subject || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &subject)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid subject"); + goto graph_edges_cleanup; + } + have_subject = true; + } else if (key_len == strlen("predicate") && + memcmp(key, "predicate", key_len) == 0) { + if (have_predicate || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &predicate)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicate"); + goto graph_edges_cleanup; + } + have_predicate = true; + } else if (key_len == strlen("object") && + memcmp(key, "object", key_len) == 0) { + if (have_object || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &object)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid object"); + goto graph_edges_cleanup; + } + have_object = true; + } else if (key_len == strlen("metadata_ref") && + memcmp(key, "metadata_ref", key_len) == 0) { + if (have_metadata_ref || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &metadata_ref)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid metadata_ref"); + goto graph_edges_cleanup; + } + have_metadata_ref = true; + } else if (key_len == strlen("provenance") && + memcmp(key, "provenance", key_len) == 0) { + if (have_provenance || + !amduatd_json_parse_provenance_input(&p, end, &provenance, &prov_err)) { + ok = amduatd_send_json_error(fd, + 400, + "Bad Request", + prov_err != NULL ? prov_err : "invalid provenance"); + goto graph_edges_cleanup; + } + have_provenance = true; + } else { + if (!amduatd_json_skip_value(&p, end, 0)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto graph_edges_cleanup; + } + } + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto graph_edges_cleanup; + } + p = amduatd_json_skip_ws(p, end); + if (p != end) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto graph_edges_cleanup; + } + if (!have_subject || !have_predicate || !have_object) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "missing subject/predicate/object"); + goto graph_edges_cleanup; + } + if (have_metadata_ref && have_provenance) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "metadata_ref and provenance are mutually exclusive"); + goto graph_edges_cleanup; + } + { + int validation_status = 200; + const char *validation_error = NULL; + if (!amduatd_graph_schema_validate_provenance_write(have_metadata_ref || have_provenance, + &validation_status, + &validation_error)) { + ok = amduatd_send_json_error( + fd, + validation_status, + validation_status == 422 ? "Unprocessable Entity" : "Bad Request", + validation_error != NULL ? validation_error : "invalid provenance"); + goto graph_edges_cleanup; + } + } + + if (!amduatd_resolve_graph_ref(store, concepts, dcfg, subject, &subject_ref)) { + ok = amduatd_send_json_error(fd, 404, "Not Found", "subject not found"); + goto graph_edges_cleanup; + } + if (!amduatd_resolve_relation_ref(concepts, predicate, &predicate_ref)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicate"); + goto graph_edges_cleanup; + } + { + int validation_status = 200; + const char *validation_error = NULL; + if (!amduatd_graph_schema_validate_predicate_write(predicate_ref, + &validation_status, + &validation_error)) { + ok = amduatd_send_json_error( + fd, + validation_status, + validation_status == 422 ? "Unprocessable Entity" : "Bad Request", + validation_error != NULL ? validation_error : "invalid predicate"); + goto graph_edges_cleanup; + } + } + if (!amduatd_resolve_graph_ref(store, concepts, dcfg, object, &object_ref)) { + ok = amduatd_send_json_error(fd, 404, "Not Found", "object not found"); + goto graph_edges_cleanup; + } + if (!amduatd_concepts_put_edge(store, + concepts, + subject_ref, + object_ref, + predicate_ref, + actor, + &edge_ref)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "store error"); + goto graph_edges_cleanup; + } + if (have_provenance) { + if (!amduatd_provenance_store(store, &provenance, &metadata_target_ref)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "store error"); + goto graph_edges_cleanup; + } + have_metadata_ref = true; + } else if (have_metadata_ref) { + if (!amduatd_resolve_graph_ref(store, + concepts, + dcfg, + metadata_ref, + &metadata_target_ref)) { + ok = amduatd_send_json_error(fd, 404, "Not Found", "metadata_ref not found"); + goto graph_edges_cleanup; + } + if (!amduatd_concepts_put_edge(store, + concepts, + edge_ref, + metadata_target_ref, + concepts->rel_has_provenance_ref, + actor, + &provenance_edge_ref)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "store error"); + goto graph_edges_cleanup; + } + } + + if (!amduat_asl_ref_encode_hex(subject_ref, &subject_hex) || + !amduat_asl_ref_encode_hex(predicate_ref, &predicate_hex) || + !amduat_asl_ref_encode_hex(object_ref, &object_hex) || + !amduat_asl_ref_encode_hex(edge_ref, &edge_hex)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto graph_edges_cleanup; + } + if (have_metadata_ref && + !amduat_asl_ref_encode_hex(metadata_target_ref, &metadata_hex)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto graph_edges_cleanup; + } + { + int n; + if (metadata_hex != NULL) { + n = snprintf(json, + sizeof(json), + "{" + "\"subject_ref\":\"%s\"," + "\"predicate_ref\":\"%s\"," + "\"object_ref\":\"%s\"," + "\"edge_ref\":\"%s\"," + "\"metadata_ref\":\"%s\"" + "}\n", + subject_hex, + predicate_hex, + object_hex, + edge_hex, + metadata_hex); + } else { + n = snprintf(json, + sizeof(json), + "{" + "\"subject_ref\":\"%s\"," + "\"predicate_ref\":\"%s\"," + "\"object_ref\":\"%s\"," + "\"edge_ref\":\"%s\"" + "}\n", + subject_hex, + predicate_hex, + object_hex, + edge_hex); + } + if (n <= 0 || (size_t)n >= sizeof(json)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "error"); + goto graph_edges_cleanup; + } + } + ok = amduatd_http_send_json(fd, 200, "OK", json, false); + +graph_edges_cleanup: + free(body); + free(subject); + free(predicate); + free(object); + free(metadata_ref); + amduatd_provenance_input_free(&provenance); + free(subject_hex); + free(predicate_hex); + free(object_hex); + free(edge_hex); + free(metadata_hex); + amduat_reference_free(&subject_ref); + amduat_reference_free(&predicate_ref); + amduat_reference_free(&object_ref); + amduat_reference_free(&edge_ref); + amduat_reference_free(&metadata_target_ref); + amduat_reference_free(&provenance_edge_ref); + return ok; +} + +static bool amduatd_graph_edge_exists(const amduatd_concepts_t *concepts, + amduat_reference_t edge_record_ref) { + size_t i; + if (concepts == NULL) { + return false; + } + for (i = 0u; i < concepts->edges.len; ++i) { + if (amduat_reference_eq(concepts->edges.items[i].record_ref, edge_record_ref)) { + return true; + } + } + return false; +} + +static bool amduatd_graph_find_materialization_edge( + const amduatd_concepts_t *concepts, + amduat_reference_t concept_ref, + amduat_reference_t version_ref, + bool include_tombstoned, + amduat_reference_t *out_edge_ref) { + size_t i; + size_t scan_end; + if (out_edge_ref != NULL) { + *out_edge_ref = amduat_reference(0u, amduat_octets(NULL, 0u)); + } + if (concepts == NULL || out_edge_ref == NULL) { + return false; + } + scan_end = concepts->edges.len; + for (i = concepts->edges.len; i > 0; --i) { + const amduatd_edge_entry_t *entry = &concepts->edges.items[i - 1u]; + if (entry->rel == NULL || strcmp(entry->rel, AMDUATD_REL_MATERIALIZES) != 0) { + continue; + } + if (!include_tombstoned && + amduatd_graph_edge_is_tombstoned(concepts, entry->record_ref, scan_end)) { + continue; + } + if (!amduat_reference_eq(entry->src_ref, concept_ref) || + !amduat_reference_eq(entry->dst_ref, version_ref)) { + continue; + } + return amduat_reference_clone(entry->record_ref, out_edge_ref); + } + return false; +} + +static bool amduatd_handle_post_graph_node_versions_tombstone( + int fd, + amduat_asl_store_t *store, + amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const char *name, + const amduatd_http_req_t *req) { + uint8_t *body = NULL; + const char *p = NULL; + const char *end = NULL; + bool have_ref = false; + bool have_metadata_ref = false; + bool have_provenance = false; + char *version_ref_text = NULL; + char *metadata_ref = NULL; + const char *prov_err = NULL; + amduatd_provenance_input_t provenance; + amduat_reference_t concept_ref; + amduat_reference_t version_ref; + amduat_reference_t target_edge_ref; + amduat_reference_t metadata_target_ref; + amduat_reference_t none_ref; + amduat_reference_t tombstone_edge_ref; + amduat_reference_t provenance_edge_ref; + amduat_octets_t actor = amduat_octets(NULL, 0u); + amduat_octets_t scoped_name = amduat_octets(NULL, 0u); + const amduatd_space_t *space = dcfg != NULL ? &dcfg->space : NULL; + char *target_hex = NULL; + char *tombstone_hex = NULL; + char *metadata_hex = NULL; + char *version_hex = NULL; + char json[2304]; + bool ok = false; + + memset(&provenance, 0, sizeof(provenance)); + memset(&concept_ref, 0, sizeof(concept_ref)); + memset(&version_ref, 0, sizeof(version_ref)); + memset(&target_edge_ref, 0, sizeof(target_edge_ref)); + memset(&metadata_target_ref, 0, sizeof(metadata_target_ref)); + memset(&none_ref, 0, sizeof(none_ref)); + memset(&tombstone_edge_ref, 0, sizeof(tombstone_edge_ref)); + memset(&provenance_edge_ref, 0, sizeof(provenance_edge_ref)); + + if (store == NULL || concepts == NULL || req == NULL || name == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "internal error"); + } + if (req->has_actor) { + actor = req->actor; + } + if (req->content_length == 0u || req->content_length > (256u * 1024u)) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid body"); + } + if (!amduatd_space_scope_name(space, name, &scoped_name)) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid name"); + } + if (!amduatd_concepts_lookup_alias(store, + concepts, + (const char *)scoped_name.data, + &concept_ref)) { + amduat_octets_free(&scoped_name); + return amduatd_send_json_error(fd, 404, "Not Found", "unknown concept"); + } + amduat_octets_free(&scoped_name); + + body = (uint8_t *)malloc(req->content_length); + if (body == NULL) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + goto cleanup; + } + if (!amduatd_read_exact(fd, body, req->content_length)) { + free(body); + body = NULL; + goto cleanup; + } + p = (const char *)body; + end = (const char *)body + req->content_length; + if (!amduatd_json_expect(&p, end, '{')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + for (;;) { + const char *key = NULL; + size_t key_len = 0u; + const char *sv = NULL; + size_t sv_len = 0u; + const char *cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &key, &key_len) || + !amduatd_json_expect(&p, end, ':')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + if (key_len == strlen("ref") && memcmp(key, "ref", key_len) == 0) { + if (have_ref || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &version_ref_text)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid ref"); + goto cleanup; + } + have_ref = true; + } else if (key_len == strlen("metadata_ref") && + memcmp(key, "metadata_ref", key_len) == 0) { + if (have_metadata_ref || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &metadata_ref)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid metadata_ref"); + goto cleanup; + } + have_metadata_ref = true; + } else if (key_len == strlen("provenance") && + memcmp(key, "provenance", key_len) == 0) { + if (have_provenance || + !amduatd_json_parse_provenance_input(&p, end, &provenance, &prov_err)) { + ok = amduatd_send_json_error(fd, + 400, + "Bad Request", + prov_err != NULL ? prov_err : "invalid provenance"); + goto cleanup; + } + have_provenance = true; + } else { + if (!amduatd_json_skip_value(&p, end, 0)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + } + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + p = amduatd_json_skip_ws(p, end); + if (p != end) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + if (!have_ref) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "missing ref"); + goto cleanup; + } + if (have_metadata_ref && have_provenance) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "metadata_ref and provenance are mutually exclusive"); + goto cleanup; + } + { + int validation_status = 200; + const char *validation_error = NULL; + if (!amduatd_graph_schema_validate_provenance_write(have_metadata_ref || have_provenance, + &validation_status, + &validation_error)) { + ok = amduatd_send_json_error( + fd, + validation_status, + validation_status == 422 ? "Unprocessable Entity" : "Bad Request", + validation_error != NULL ? validation_error : "invalid provenance"); + goto cleanup; + } + } + if (!amduatd_resolve_graph_ref(store, concepts, dcfg, version_ref_text, &version_ref)) { + ok = amduatd_send_json_error(fd, 404, "Not Found", "ref not found"); + goto cleanup; + } + if (!amduatd_graph_find_materialization_edge(concepts, + concept_ref, + version_ref, + true, + &target_edge_ref)) { + ok = amduatd_send_json_error(fd, 404, "Not Found", "version not found"); + goto cleanup; + } + if (!amduatd_get_none_ref(store, &none_ref)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "store error"); + goto cleanup; + } + if (!amduatd_concepts_put_edge(store, + concepts, + target_edge_ref, + none_ref, + concepts->rel_tombstones_ref, + actor, + &tombstone_edge_ref)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "store error"); + goto cleanup; + } + if (have_provenance) { + if (!amduatd_provenance_store(store, &provenance, &metadata_target_ref)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "store error"); + goto cleanup; + } + have_metadata_ref = true; + } else if (have_metadata_ref) { + if (!amduatd_resolve_graph_ref(store, + concepts, + dcfg, + metadata_ref, + &metadata_target_ref)) { + ok = amduatd_send_json_error(fd, 404, "Not Found", "metadata_ref not found"); + goto cleanup; + } + } + if (have_metadata_ref) { + if (!amduatd_concepts_put_edge(store, + concepts, + tombstone_edge_ref, + metadata_target_ref, + concepts->rel_has_provenance_ref, + actor, + &provenance_edge_ref)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "store error"); + goto cleanup; + } + } + if (!amduat_asl_ref_encode_hex(version_ref, &version_hex) || + !amduat_asl_ref_encode_hex(target_edge_ref, &target_hex) || + !amduat_asl_ref_encode_hex(tombstone_edge_ref, &tombstone_hex)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto cleanup; + } + if (have_metadata_ref && + !amduat_asl_ref_encode_hex(metadata_target_ref, &metadata_hex)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto cleanup; + } + { + int n; + if (metadata_hex != NULL) { + n = snprintf(json, + sizeof(json), + "{" + "\"ok\":true," + "\"name\":\"%s\"," + "\"ref\":\"%s\"," + "\"target_edge_ref\":\"%s\"," + "\"tombstone_edge_ref\":\"%s\"," + "\"metadata_ref\":\"%s\"" + "}\n", + name, + version_hex, + target_hex, + tombstone_hex, + metadata_hex); + } else { + n = snprintf(json, + sizeof(json), + "{" + "\"ok\":true," + "\"name\":\"%s\"," + "\"ref\":\"%s\"," + "\"target_edge_ref\":\"%s\"," + "\"tombstone_edge_ref\":\"%s\"" + "}\n", + name, + version_hex, + target_hex, + tombstone_hex); + } + if (n <= 0 || (size_t)n >= sizeof(json)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "error"); + goto cleanup; + } + } + ok = amduatd_http_send_json(fd, 200, "OK", json, false); + +cleanup: + free(body); + free(version_ref_text); + free(metadata_ref); + amduatd_provenance_input_free(&provenance); + free(target_hex); + free(tombstone_hex); + free(metadata_hex); + free(version_hex); + amduat_reference_free(&concept_ref); + amduat_reference_free(&version_ref); + amduat_reference_free(&target_edge_ref); + amduat_reference_free(&metadata_target_ref); + amduat_reference_free(&none_ref); + amduat_reference_free(&tombstone_edge_ref); + amduat_reference_free(&provenance_edge_ref); + return ok; +} + +static bool amduatd_handle_post_graph_edges_tombstone(int fd, + amduat_asl_store_t *store, + amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const amduatd_http_req_t *req) { + uint8_t *body = NULL; + const char *p = NULL; + const char *end = NULL; + bool have_edge_ref = false; + bool have_metadata_ref = false; + bool have_provenance = false; + char *edge_ref_text = NULL; + char *metadata_ref = NULL; + const char *prov_err = NULL; + amduatd_provenance_input_t provenance; + amduat_reference_t target_edge_ref; + amduat_reference_t metadata_target_ref; + amduat_reference_t none_ref; + amduat_reference_t tombstone_edge_ref; + amduat_reference_t provenance_edge_ref; + char *target_hex = NULL; + char *tombstone_hex = NULL; + char *metadata_hex = NULL; + amduat_octets_t actor = amduat_octets(NULL, 0u); + char json[2048]; + bool ok = false; + + memset(&provenance, 0, sizeof(provenance)); + memset(&target_edge_ref, 0, sizeof(target_edge_ref)); + memset(&metadata_target_ref, 0, sizeof(metadata_target_ref)); + memset(&none_ref, 0, sizeof(none_ref)); + memset(&tombstone_edge_ref, 0, sizeof(tombstone_edge_ref)); + memset(&provenance_edge_ref, 0, sizeof(provenance_edge_ref)); + + if (store == NULL || concepts == NULL || req == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "internal error"); + } + if (req->has_actor) { + actor = req->actor; + } + if (req->content_length == 0u || req->content_length > (256u * 1024u)) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid body"); + } + body = (uint8_t *)malloc(req->content_length); + if (body == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + if (!amduatd_read_exact(fd, body, req->content_length)) { + free(body); + return false; + } + + p = (const char *)body; + end = (const char *)body + req->content_length; + if (!amduatd_json_expect(&p, end, '{')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + for (;;) { + const char *key = NULL; + size_t key_len = 0u; + const char *sv = NULL; + size_t sv_len = 0u; + const char *cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + if (!amduatd_json_parse_string_noesc(&p, end, &key, &key_len) || + !amduatd_json_expect(&p, end, ':')) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + if (key_len == strlen("edge_ref") && + memcmp(key, "edge_ref", key_len) == 0) { + if (have_edge_ref || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &edge_ref_text)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid edge_ref"); + goto cleanup; + } + have_edge_ref = true; + } else if (key_len == strlen("metadata_ref") && + memcmp(key, "metadata_ref", key_len) == 0) { + if (have_metadata_ref || + !amduatd_json_parse_string_noesc(&p, end, &sv, &sv_len) || + !amduatd_copy_json_str(sv, sv_len, &metadata_ref)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid metadata_ref"); + goto cleanup; + } + have_metadata_ref = true; + } else if (key_len == strlen("provenance") && + memcmp(key, "provenance", key_len) == 0) { + if (have_provenance || + !amduatd_json_parse_provenance_input(&p, end, &provenance, &prov_err)) { + ok = amduatd_send_json_error(fd, + 400, + "Bad Request", + prov_err != NULL ? prov_err : "invalid provenance"); + goto cleanup; + } + have_provenance = true; + } else { + if (!amduatd_json_skip_value(&p, end, 0)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + } + cur = amduatd_json_skip_ws(p, end); + if (cur < end && *cur == ',') { + p = cur + 1; + continue; + } + if (cur < end && *cur == '}') { + p = cur + 1; + break; + } + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + p = amduatd_json_skip_ws(p, end); + if (p != end) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid json"); + goto cleanup; + } + if (!have_edge_ref) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "missing edge_ref"); + goto cleanup; + } + if (have_metadata_ref && have_provenance) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", + "metadata_ref and provenance are mutually exclusive"); + goto cleanup; + } + { + int validation_status = 200; + const char *validation_error = NULL; + if (!amduatd_graph_schema_validate_provenance_write(have_metadata_ref || have_provenance, + &validation_status, + &validation_error)) { + ok = amduatd_send_json_error( + fd, + validation_status, + validation_status == 422 ? "Unprocessable Entity" : "Bad Request", + validation_error != NULL ? validation_error : "invalid provenance"); + goto cleanup; + } + } + if (!amduat_asl_ref_decode_hex(edge_ref_text, &target_edge_ref)) { + ok = amduatd_send_json_error(fd, 400, "Bad Request", "invalid edge_ref"); + goto cleanup; + } + if (!amduatd_graph_edge_exists(concepts, target_edge_ref)) { + ok = amduatd_send_json_error(fd, 404, "Not Found", "edge not found"); + goto cleanup; + } + if (!amduatd_get_none_ref(store, &none_ref)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "store error"); + goto cleanup; + } + if (!amduatd_concepts_put_edge(store, + concepts, + target_edge_ref, + none_ref, + concepts->rel_tombstones_ref, + actor, + &tombstone_edge_ref)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "store error"); + goto cleanup; + } + if (have_provenance) { + if (!amduatd_provenance_store(store, &provenance, &metadata_target_ref)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "store error"); + goto cleanup; + } + have_metadata_ref = true; + } else if (have_metadata_ref) { + if (!amduatd_resolve_graph_ref(store, + concepts, + dcfg, + metadata_ref, + &metadata_target_ref)) { + ok = amduatd_send_json_error(fd, 404, "Not Found", "metadata_ref not found"); + goto cleanup; + } + } + if (have_metadata_ref) { + if (!amduatd_concepts_put_edge(store, + concepts, + tombstone_edge_ref, + metadata_target_ref, + concepts->rel_has_provenance_ref, + actor, + &provenance_edge_ref)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "store error"); + goto cleanup; + } + } + if (!amduat_asl_ref_encode_hex(target_edge_ref, &target_hex) || + !amduat_asl_ref_encode_hex(tombstone_edge_ref, &tombstone_hex)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto cleanup; + } + if (have_metadata_ref && + !amduat_asl_ref_encode_hex(metadata_target_ref, &metadata_hex)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + goto cleanup; + } + { + int n; + if (metadata_hex != NULL) { + n = snprintf(json, + sizeof(json), + "{" + "\"ok\":true," + "\"target_edge_ref\":\"%s\"," + "\"tombstone_edge_ref\":\"%s\"," + "\"metadata_ref\":\"%s\"" + "}\n", + target_hex, + tombstone_hex, + metadata_hex); + } else { + n = snprintf(json, + sizeof(json), + "{" + "\"ok\":true," + "\"target_edge_ref\":\"%s\"," + "\"tombstone_edge_ref\":\"%s\"" + "}\n", + target_hex, + tombstone_hex); + } + if (n <= 0 || (size_t)n >= sizeof(json)) { + ok = amduatd_send_json_error(fd, 500, "Internal Server Error", "error"); + goto cleanup; + } + } + ok = amduatd_http_send_json(fd, 200, "OK", json, false); + +cleanup: + free(body); + free(edge_ref_text); + free(metadata_ref); + amduatd_provenance_input_free(&provenance); + free(target_hex); + free(tombstone_hex); + free(metadata_hex); + amduat_reference_free(&target_edge_ref); + amduat_reference_free(&metadata_target_ref); + amduat_reference_free(&none_ref); + amduat_reference_free(&tombstone_edge_ref); + amduat_reference_free(&provenance_edge_ref); + return ok; +} + +static bool amduatd_handle_get_graph_changes(int fd, + amduat_asl_store_t *store, + const amduatd_concepts_t *concepts, + const amduatd_cfg_t *dcfg, + const amduatd_http_req_t *req) { + enum { + AMDUATD_CHANGES_MAX_CSV_ITEMS = 256, + AMDUATD_CHANGES_REPLAY_WINDOW = 50000 + }; + char since_buf[32]; + char since_as_of_buf[32]; + char limit_buf[32]; + char wait_ms_buf[32]; + char event_types_buf[2048]; + char predicates_buf[4096]; + char roots_buf[4096]; + char *csv_items[AMDUATD_CHANGES_MAX_CSV_ITEMS]; + bool have_since = false; + bool have_since_as_of = false; + size_t since_cursor = 0u; + size_t since_as_of = 0u; + uint64_t limit_u64 = 100u; + uint64_t wait_ms_u64 = 0u; + size_t limit = 100u; + size_t start_i = 0u; + size_t scan_len = 0u; + size_t i; + size_t returned = 0u; + size_t last_emitted_i = 0u; + bool has_more = false; + bool allow_edge_appended = true; + bool allow_version_published = true; + bool allow_tombstone_applied = true; + amduat_reference_t *predicate_refs = NULL; + size_t predicate_refs_len = 0u; + size_t predicate_refs_cap = 0u; + amduat_reference_t *root_refs = NULL; + size_t root_refs_len = 0u; + size_t root_refs_cap = 0u; + amduatd_strbuf_t b; + int attempt; + + memset(csv_items, 0, sizeof(csv_items)); + event_types_buf[0] = '\0'; + predicates_buf[0] = '\0'; + roots_buf[0] = '\0'; + if (store == NULL || concepts == NULL || req == NULL) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "internal error"); + } + if (!amduatd_concepts_ensure_query_index((amduatd_concepts_t *)concepts)) { + return amduatd_send_json_error(fd, 500, "Internal Server Error", "index error"); + } + if (amduatd_query_param(req->path, + "since_cursor", + since_buf, + sizeof(since_buf)) != NULL) { + if (!amduatd_graph_cursor_decode(since_buf, &since_cursor)) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid since_cursor"); + } + have_since = true; + } + if (amduatd_query_param(req->path, + "since_as_of", + since_as_of_buf, + sizeof(since_as_of_buf)) != NULL) { + if (!amduatd_graph_cursor_decode(since_as_of_buf, &since_as_of)) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid since_as_of"); + } + have_since_as_of = true; + } + if (amduatd_query_param(req->path, "limit", limit_buf, sizeof(limit_buf)) != NULL) { + if (!amduatd_parse_u64_query(limit_buf, &limit_u64) || + limit_u64 == 0u || + limit_u64 > 1000u) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid limit"); + } + } + if (limit_u64 > (uint64_t)SIZE_MAX) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid limit"); + } + if (amduatd_query_param(req->path, "wait_ms", wait_ms_buf, sizeof(wait_ms_buf)) != NULL) { + if (!amduatd_parse_u64_query(wait_ms_buf, &wait_ms_u64) || wait_ms_u64 > 30000u) { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid wait_ms"); + } + } + if (amduatd_query_param(req->path, + "event_types[]", + event_types_buf, + sizeof(event_types_buf)) == NULL && + amduatd_query_param(req->path, + "event_types", + event_types_buf, + sizeof(event_types_buf)) == NULL) { + event_types_buf[0] = '\0'; + } + if (event_types_buf[0] != '\0') { + size_t et_count = + amduatd_graph_split_csv(event_types_buf, csv_items, AMDUATD_CHANGES_MAX_CSV_ITEMS); + allow_edge_appended = false; + allow_version_published = false; + allow_tombstone_applied = false; + for (i = 0u; i < et_count; ++i) { + if (strcmp(csv_items[i], "edge_appended") == 0) { + allow_edge_appended = true; + } else if (strcmp(csv_items[i], "version_published") == 0) { + allow_version_published = true; + } else if (strcmp(csv_items[i], "tombstone_applied") == 0) { + allow_tombstone_applied = true; + } else { + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid event_types"); + } + } + } + if (amduatd_query_param(req->path, + "predicates[]", + predicates_buf, + sizeof(predicates_buf)) == NULL && + amduatd_query_param(req->path, + "predicates", + predicates_buf, + sizeof(predicates_buf)) == NULL && + amduatd_query_param(req->path, + "predicate", + predicates_buf, + sizeof(predicates_buf)) == NULL) { + predicates_buf[0] = '\0'; + } + if (predicates_buf[0] != '\0') { + size_t pred_count = + amduatd_graph_split_csv(predicates_buf, csv_items, AMDUATD_CHANGES_MAX_CSV_ITEMS); + for (i = 0u; i < pred_count; ++i) { + amduat_reference_t pred_ref; + memset(&pred_ref, 0, sizeof(pred_ref)); + if (!amduatd_resolve_relation_ref(concepts, csv_items[i], &pred_ref)) { + amduatd_graph_ref_array_free(predicate_refs, predicate_refs_len); + return amduatd_send_json_error(fd, 400, "Bad Request", "invalid predicate"); + } + if (!amduatd_graph_ref_array_append_unique(&predicate_refs, + &predicate_refs_len, + &predicate_refs_cap, + pred_ref)) { + amduat_reference_free(&pred_ref); + amduatd_graph_ref_array_free(predicate_refs, predicate_refs_len); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + amduat_reference_free(&pred_ref); + } + } + if (amduatd_query_param(req->path, "roots[]", roots_buf, sizeof(roots_buf)) == NULL && + amduatd_query_param(req->path, "roots", roots_buf, sizeof(roots_buf)) == NULL) { + roots_buf[0] = '\0'; + } + if (roots_buf[0] != '\0') { + size_t root_count = + amduatd_graph_split_csv(roots_buf, csv_items, AMDUATD_CHANGES_MAX_CSV_ITEMS); + for (i = 0u; i < root_count; ++i) { + amduat_reference_t root_ref; + memset(&root_ref, 0, sizeof(root_ref)); + if (!amduatd_resolve_graph_ref(store, concepts, dcfg, csv_items[i], &root_ref)) { + amduatd_graph_ref_array_free(predicate_refs, predicate_refs_len); + amduatd_graph_ref_array_free(root_refs, root_refs_len); + return amduatd_send_json_error(fd, 404, "Not Found", "root not found"); + } + if (!amduatd_graph_ref_array_append_unique(&root_refs, + &root_refs_len, + &root_refs_cap, + root_ref)) { + amduat_reference_free(&root_ref); + amduatd_graph_ref_array_free(predicate_refs, predicate_refs_len); + amduatd_graph_ref_array_free(root_refs, root_refs_len); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + amduat_reference_free(&root_ref); + } + } + limit = (size_t)limit_u64; + scan_len = concepts->edges.len; + + { + size_t replay_min = 0u; + if (scan_len > AMDUATD_CHANGES_REPLAY_WINDOW) { + replay_min = scan_len - AMDUATD_CHANGES_REPLAY_WINDOW; + } + if (have_since) { + size_t since_next = (scan_len == 0u || since_cursor >= (scan_len - 1u)) + ? scan_len + : (since_cursor + 1u); + if (since_next < replay_min) { + amduatd_graph_ref_array_free(predicate_refs, predicate_refs_len); + amduatd_graph_ref_array_free(root_refs, root_refs_len); + return amduatd_send_json_error(fd, + 410, + "Gone", + "since_cursor outside replay window"); + } + start_i = since_next; + } + if (have_since_as_of) { + size_t as_of_next = since_as_of >= scan_len ? scan_len : (since_as_of + 1u); + if (as_of_next < replay_min) { + amduatd_graph_ref_array_free(predicate_refs, predicate_refs_len); + amduatd_graph_ref_array_free(root_refs, root_refs_len); + return amduatd_send_json_error(fd, + 410, + "Gone", + "since_as_of outside replay window"); + } + if (as_of_next > start_i) { + start_i = as_of_next; + } + } + } + + memset(&b, 0, sizeof(b)); + if (!amduatd_strbuf_append_cstr(&b, "{\"events\":[")) { + amduatd_strbuf_free(&b); + amduatd_graph_ref_array_free(predicate_refs, predicate_refs_len); + amduatd_graph_ref_array_free(root_refs, root_refs_len); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + + for (attempt = 0; attempt < ((wait_ms_u64 > 0u) ? 2 : 1); ++attempt) { + returned = 0u; + last_emitted_i = 0u; + has_more = false; + if (b.len != strlen("{\"events\":[") || b.data == NULL) { + amduatd_strbuf_free(&b); + memset(&b, 0, sizeof(b)); + if (!amduatd_strbuf_append_cstr(&b, "{\"events\":[")) { + amduatd_graph_ref_array_free(predicate_refs, predicate_refs_len); + amduatd_graph_ref_array_free(root_refs, root_refs_len); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "oom"); + } + } else { + b.len = strlen("{\"events\":["); + b.data[b.len] = '\0'; + } + scan_len = concepts->edges.len; + if (start_i > scan_len) { + start_i = scan_len; + } + for (i = start_i; i < scan_len; ++i) { + const amduatd_edge_entry_t *entry = &concepts->edges.items[i]; + const char *event_name = "edge_appended"; + amduat_reference_t predicate_ref; + char *subject_hex = NULL; + char *predicate_hex = NULL; + char *object_hex = NULL; + char *edge_hex = NULL; + char cursor_token[64]; + bool is_version = false; + bool is_tombstone = false; + bool match = true; + + memset(&predicate_ref, 0, sizeof(predicate_ref)); + if (!amduatd_relation_entry_ref(concepts, entry->rel, &predicate_ref)) { + continue; + } + if (entry->rel != NULL && strcmp(entry->rel, AMDUATD_REL_MATERIALIZES) == 0) { + event_name = "version_published"; + is_version = true; + } else if (entry->rel != NULL && strcmp(entry->rel, AMDUATD_REL_TOMBSTONES) == 0) { + event_name = "tombstone_applied"; + is_tombstone = true; + } + if (is_version && !allow_version_published) { + match = false; + } else if (is_tombstone && !allow_tombstone_applied) { + match = false; + } else if (!is_version && !is_tombstone && !allow_edge_appended) { + match = false; + } + if (match && predicate_refs_len != 0u && + !amduatd_graph_ref_array_contains(predicate_refs, predicate_refs_len, predicate_ref)) { + match = false; + } + if (match && root_refs_len != 0u && + !amduatd_graph_ref_array_contains(root_refs, root_refs_len, entry->src_ref) && + !amduatd_graph_ref_array_contains(root_refs, root_refs_len, entry->dst_ref)) { + match = false; + } + if (!match) { + amduat_reference_free(&predicate_ref); + continue; + } + + if (!amduat_asl_ref_encode_hex(entry->src_ref, &subject_hex) || + !amduat_asl_ref_encode_hex(predicate_ref, &predicate_hex) || + !amduat_asl_ref_encode_hex(entry->dst_ref, &object_hex) || + !amduat_asl_ref_encode_hex(entry->record_ref, &edge_hex) || + !amduatd_graph_cursor_encode(i, cursor_token, sizeof(cursor_token))) { + amduat_reference_free(&predicate_ref); + free(subject_hex); + free(predicate_hex); + free(object_hex); + free(edge_hex); + continue; + } + amduat_reference_free(&predicate_ref); + + if (returned != 0u) { + (void)amduatd_strbuf_append_char(&b, ','); + } + returned++; + last_emitted_i = i; + (void)amduatd_strbuf_append_cstr(&b, "{\"event\":\""); + (void)amduatd_strbuf_append_cstr(&b, event_name); + (void)amduatd_strbuf_append_cstr(&b, "\",\"cursor\":\""); + (void)amduatd_strbuf_append_cstr(&b, cursor_token); + (void)amduatd_strbuf_append_cstr(&b, "\",\"edge_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, edge_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"subject_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, subject_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"predicate_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, predicate_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"object_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, object_hex); + (void)amduatd_strbuf_append_cstr(&b, "\""); + if (is_version) { + (void)amduatd_strbuf_append_cstr(&b, ",\"concept_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, subject_hex); + (void)amduatd_strbuf_append_cstr(&b, "\",\"ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, object_hex); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } else if (is_tombstone) { + (void)amduatd_strbuf_append_cstr(&b, ",\"tombstoned_edge_ref\":\""); + (void)amduatd_strbuf_append_cstr(&b, subject_hex); + (void)amduatd_strbuf_append_cstr(&b, "\""); + } + (void)amduatd_strbuf_append_cstr(&b, "}"); + + free(subject_hex); + free(predicate_hex); + free(object_hex); + free(edge_hex); + + if (returned >= limit) { + break; + } + } + if (returned > 0u || attempt == 1 || wait_ms_u64 == 0u) { + break; + } + usleep((useconds_t)(wait_ms_u64 * 1000u)); + } + + if (returned >= limit && (last_emitted_i + 1u) < scan_len) { + for (i = last_emitted_i + 1u; i < scan_len; ++i) { + const amduatd_edge_entry_t *entry = &concepts->edges.items[i]; + amduat_reference_t predicate_ref; + bool is_version = false; + bool is_tombstone = false; + bool match = true; + memset(&predicate_ref, 0, sizeof(predicate_ref)); + if (!amduatd_relation_entry_ref(concepts, entry->rel, &predicate_ref)) { + continue; + } + if (entry->rel != NULL && strcmp(entry->rel, AMDUATD_REL_MATERIALIZES) == 0) { + is_version = true; + } else if (entry->rel != NULL && strcmp(entry->rel, AMDUATD_REL_TOMBSTONES) == 0) { + is_tombstone = true; + } + if (is_version && !allow_version_published) { + match = false; + } else if (is_tombstone && !allow_tombstone_applied) { + match = false; + } else if (!is_version && !is_tombstone && !allow_edge_appended) { + match = false; + } + if (match && predicate_refs_len != 0u && + !amduatd_graph_ref_array_contains(predicate_refs, predicate_refs_len, predicate_ref)) { + match = false; + } + if (match && root_refs_len != 0u && + !amduatd_graph_ref_array_contains(root_refs, root_refs_len, entry->src_ref) && + !amduatd_graph_ref_array_contains(root_refs, root_refs_len, entry->dst_ref)) { + match = false; + } + amduat_reference_free(&predicate_ref); + if (match) { + has_more = true; + break; + } + } + } + + (void)amduatd_strbuf_append_cstr(&b, "],"); + if (returned != 0u) { + char token[64]; + if (!amduatd_graph_cursor_encode(last_emitted_i, token, sizeof(token))) { + amduatd_strbuf_free(&b); + amduatd_graph_ref_array_free(predicate_refs, predicate_refs_len); + amduatd_graph_ref_array_free(root_refs, root_refs_len); + return amduatd_send_json_error(fd, 500, "Internal Server Error", "encode error"); + } + (void)amduatd_strbuf_append_cstr(&b, "\"next_cursor\":\""); + (void)amduatd_strbuf_append_cstr(&b, token); + (void)amduatd_strbuf_append_cstr(&b, "\",\"has_more\":"); + } else { + (void)amduatd_strbuf_append_cstr(&b, "\"next_cursor\":null,\"has_more\":"); + } + (void)amduatd_strbuf_append_cstr(&b, has_more ? "true" : "false"); + (void)amduatd_strbuf_append_cstr(&b, "}\n"); + + { + bool ok = amduatd_http_send_json(fd, 200, "OK", b.data, false); + amduatd_strbuf_free(&b); + amduatd_graph_ref_array_free(predicate_refs, predicate_refs_len); + amduatd_graph_ref_array_free(root_refs, root_refs_len); + return ok; + } +} + bool amduatd_concepts_can_handle(const amduatd_http_req_t *req) { char no_query[1024]; @@ -4093,6 +15894,94 @@ bool amduatd_concepts_can_handle(const amduatd_http_req_t *req) { strncmp(no_query, "/v1/resolve/", 12) == 0) { return true; } + if (strcmp(req->method, "POST") == 0 && + strcmp(no_query, "/v2/graph/nodes") == 0) { + return true; + } + if (strcmp(req->method, "POST") == 0 && + strcmp(no_query, "/v2/graph/edges") == 0) { + return true; + } + if (strcmp(req->method, "POST") == 0 && + strcmp(no_query, "/v2/graph/edges/tombstone") == 0) { + return true; + } + if (strcmp(req->method, "POST") == 0 && + strcmp(no_query, "/v2/graph/batch") == 0) { + return true; + } + if (strcmp(req->method, "POST") == 0 && + strcmp(no_query, "/v2/graph/query") == 0) { + return true; + } + if (strcmp(req->method, "POST") == 0 && + strcmp(no_query, "/v2/graph/retrieve") == 0) { + return true; + } + if (strcmp(req->method, "POST") == 0 && + strcmp(no_query, "/v2/graph/export") == 0) { + return true; + } + if (strcmp(req->method, "POST") == 0 && + strcmp(no_query, "/v2/graph/import") == 0) { + return true; + } + if (strcmp(req->method, "GET") == 0 && + strcmp(no_query, "/v2/graph/schema/predicates") == 0) { + return true; + } + if (strcmp(req->method, "POST") == 0 && + strcmp(no_query, "/v2/graph/schema/predicates") == 0) { + return true; + } + if (strcmp(req->method, "GET") == 0 && + strcmp(no_query, "/v2/graph/changes") == 0) { + return true; + } + if (strcmp(req->method, "GET") == 0 && + strcmp(no_query, "/v2/graph/edges") == 0) { + return true; + } + if (strcmp(req->method, "GET") == 0 && + strcmp(no_query, "/v2/graph/search") == 0) { + return true; + } + if (strcmp(req->method, "GET") == 0 && + strcmp(no_query, "/v2/graph/paths") == 0) { + return true; + } + if (strcmp(req->method, "GET") == 0 && + strcmp(no_query, "/v2/graph/subgraph") == 0) { + return true; + } + if (strcmp(req->method, "GET") == 0 && + strncmp(no_query, "/v2/graph/nodes/", 16) == 0 && + strstr(no_query, "/versions") != NULL) { + return true; + } + if (strcmp(req->method, "GET") == 0 && + strncmp(no_query, "/v2/graph/nodes/", 16) == 0 && + strstr(no_query, "/neighbors") != NULL) { + return true; + } + if (strcmp(req->method, "GET") == 0 && + strncmp(no_query, "/v2/graph/nodes/", 16) == 0) { + return true; + } + if (strcmp(req->method, "POST") == 0 && + strncmp(no_query, "/v2/graph/nodes/", 16) == 0 && + strstr(no_query, "/versions/tombstone") != NULL) { + return true; + } + if (strcmp(req->method, "POST") == 0 && + strncmp(no_query, "/v2/graph/nodes/", 16) == 0 && + strstr(no_query, "/versions") != NULL) { + return true; + } + if (strcmp(req->method, "GET") == 0 && + strncmp(no_query, "/v2/graph/history/", 18) == 0) { + return true; + } return false; } @@ -4138,7 +16027,8 @@ bool amduatd_concepts_handle(amduatd_ctx_t *ctx, ctx->store, ctx->concepts, ctx->daemon_cfg, - name); + name, + req); return true; } if (strcmp(req->method, "POST") == 0 && @@ -4181,6 +16071,273 @@ bool amduatd_concepts_handle(amduatd_ctx_t *ctx, name); return true; } + if (strcmp(req->method, "POST") == 0 && + strcmp(no_query, "/v2/graph/nodes") == 0) { + resp->ok = amduatd_handle_post_concepts(resp->fd, + ctx->store, + ctx->concepts, + ctx->daemon_cfg, + req); + return true; + } + if (strcmp(req->method, "POST") == 0 && + strcmp(no_query, "/v2/graph/edges") == 0) { + resp->ok = amduatd_handle_post_graph_edges(resp->fd, + ctx->store, + ctx->concepts, + ctx->daemon_cfg, + req); + return true; + } + if (strcmp(req->method, "POST") == 0 && + strcmp(no_query, "/v2/graph/edges/tombstone") == 0) { + resp->ok = amduatd_handle_post_graph_edges_tombstone(resp->fd, + ctx->store, + ctx->concepts, + ctx->daemon_cfg, + req); + return true; + } + if (strcmp(req->method, "POST") == 0 && + strcmp(no_query, "/v2/graph/batch") == 0) { + resp->ok = amduatd_handle_post_graph_batch(resp->fd, + ctx->store, + ctx->concepts, + ctx->daemon_cfg, + req); + return true; + } + if (strcmp(req->method, "POST") == 0 && + strcmp(no_query, "/v2/graph/query") == 0) { + resp->ok = amduatd_handle_post_graph_query(resp->fd, + ctx->store, + ctx->concepts, + ctx->daemon_cfg, + req); + return true; + } + if (strcmp(req->method, "POST") == 0 && + strcmp(no_query, "/v2/graph/retrieve") == 0) { + resp->ok = amduatd_handle_post_graph_retrieve(resp->fd, + ctx->store, + ctx->concepts, + ctx->daemon_cfg, + req); + return true; + } + if (strcmp(req->method, "POST") == 0 && + strcmp(no_query, "/v2/graph/export") == 0) { + resp->ok = amduatd_handle_post_graph_export(resp->fd, + ctx->store, + ctx->concepts, + ctx->daemon_cfg, + req); + return true; + } + if (strcmp(req->method, "POST") == 0 && + strcmp(no_query, "/v2/graph/import") == 0) { + resp->ok = amduatd_handle_post_graph_import(resp->fd, + ctx->store, + ctx->concepts, + ctx->daemon_cfg, + req); + return true; + } + if (strcmp(req->method, "GET") == 0 && + strcmp(no_query, "/v2/graph/schema/predicates") == 0) { + resp->ok = amduatd_handle_get_graph_schema_predicates(resp->fd, + ctx->concepts); + return true; + } + if (strcmp(req->method, "POST") == 0 && + strcmp(no_query, "/v2/graph/schema/predicates") == 0) { + resp->ok = amduatd_handle_post_graph_schema_predicates(resp->fd, + ctx->store, + ctx->concepts, + req); + return true; + } + if (strcmp(req->method, "GET") == 0 && + strcmp(no_query, "/v2/graph/changes") == 0) { + resp->ok = amduatd_handle_get_graph_changes(resp->fd, + ctx->store, + ctx->concepts, + ctx->daemon_cfg, + req); + return true; + } + if (strcmp(req->method, "GET") == 0 && + strcmp(no_query, "/v2/graph/edges") == 0) { + resp->ok = amduatd_handle_get_graph_edges(resp->fd, + ctx->store, + ctx->concepts, + ctx->daemon_cfg, + req); + return true; + } + if (strcmp(req->method, "GET") == 0 && + strcmp(no_query, "/v2/graph/search") == 0) { + resp->ok = amduatd_handle_get_graph_search(resp->fd, + ctx->store, + ctx->concepts, + ctx->daemon_cfg, + req); + return true; + } + if (strcmp(req->method, "GET") == 0 && + strcmp(no_query, "/v2/graph/paths") == 0) { + resp->ok = amduatd_handle_get_graph_paths(resp->fd, + ctx->store, + ctx->concepts, + ctx->daemon_cfg, + req); + return true; + } + if (strcmp(req->method, "GET") == 0 && + strcmp(no_query, "/v2/graph/subgraph") == 0) { + resp->ok = amduatd_handle_get_graph_subgraph(resp->fd, + ctx->store, + ctx->concepts, + ctx->daemon_cfg, + req); + return true; + } + if (strcmp(req->method, "GET") == 0 && + strncmp(no_query, "/v2/graph/nodes/", 16) == 0 && + strstr(no_query, "/versions") != NULL) { + char name[256]; + char *slash; + if (!amduatd_path_extract_name(no_query, "/v2/graph/nodes/", name, + sizeof(name))) { + resp->ok = amduatd_send_json_error(resp->fd, 400, "Bad Request", + "invalid path"); + return true; + } + slash = strstr(name, "/versions"); + if (slash == NULL || strcmp(slash, "/versions") != 0) { + resp->ok = amduatd_send_json_error(resp->fd, 400, "Bad Request", + "invalid path"); + return true; + } + *slash = '\0'; + resp->ok = amduatd_handle_get_concept(resp->fd, + ctx->store, + ctx->concepts, + ctx->daemon_cfg, + name, + req); + return true; + } + if (strcmp(req->method, "GET") == 0 && + strncmp(no_query, "/v2/graph/nodes/", 16) == 0 && + strstr(no_query, "/neighbors") != NULL) { + char name[256]; + char *slash; + if (!amduatd_path_extract_name(no_query, "/v2/graph/nodes/", name, + sizeof(name))) { + resp->ok = amduatd_send_json_error(resp->fd, 400, "Bad Request", + "invalid path"); + return true; + } + slash = strstr(name, "/neighbors"); + if (slash == NULL || strcmp(slash, "/neighbors") != 0) { + resp->ok = amduatd_send_json_error(resp->fd, 400, "Bad Request", + "invalid path"); + return true; + } + *slash = '\0'; + resp->ok = amduatd_handle_get_graph_neighbors(resp->fd, + ctx->store, + ctx->concepts, + ctx->daemon_cfg, + name, + req); + return true; + } + if (strcmp(req->method, "GET") == 0 && + strncmp(no_query, "/v2/graph/nodes/", 16) == 0) { + const char *name = no_query + 16; + if (name[0] == '\0') { + resp->ok = amduatd_send_json_error(resp->fd, 400, "Bad Request", + "missing name"); + return true; + } + resp->ok = amduatd_handle_get_concept(resp->fd, + ctx->store, + ctx->concepts, + ctx->daemon_cfg, + name, + req); + return true; + } + if (strcmp(req->method, "POST") == 0 && + strncmp(no_query, "/v2/graph/nodes/", 16) == 0 && + strstr(no_query, "/versions/tombstone") != NULL) { + char name[256]; + char *slash; + if (!amduatd_path_extract_name(no_query, "/v2/graph/nodes/", name, + sizeof(name))) { + resp->ok = amduatd_send_json_error(resp->fd, 400, "Bad Request", + "invalid path"); + return true; + } + slash = strstr(name, "/versions/tombstone"); + if (slash == NULL || strcmp(slash, "/versions/tombstone") != 0) { + resp->ok = amduatd_send_json_error(resp->fd, 400, "Bad Request", + "invalid path"); + return true; + } + *slash = '\0'; + resp->ok = amduatd_handle_post_graph_node_versions_tombstone(resp->fd, + ctx->store, + ctx->concepts, + ctx->daemon_cfg, + name, + req); + return true; + } + if (strcmp(req->method, "POST") == 0 && + strncmp(no_query, "/v2/graph/nodes/", 16) == 0 && + strstr(no_query, "/versions") != NULL) { + char name[256]; + char *slash; + if (!amduatd_path_extract_name(no_query, "/v2/graph/nodes/", name, + sizeof(name))) { + resp->ok = amduatd_send_json_error(resp->fd, 400, "Bad Request", + "invalid path"); + return true; + } + slash = strstr(name, "/versions"); + if (slash == NULL || strcmp(slash, "/versions") != 0) { + resp->ok = amduatd_send_json_error(resp->fd, 400, "Bad Request", + "invalid path"); + return true; + } + *slash = '\0'; + resp->ok = amduatd_handle_post_concept_publish(resp->fd, + ctx->store, + ctx->concepts, + ctx->daemon_cfg, + name, + req); + return true; + } + if (strcmp(req->method, "GET") == 0 && + strncmp(no_query, "/v2/graph/history/", 18) == 0) { + const char *name = no_query + 18; + if (name[0] == '\0') { + resp->ok = amduatd_send_json_error(resp->fd, 400, "Bad Request", + "missing name"); + return true; + } + resp->ok = amduatd_handle_get_graph_history(resp->fd, + ctx->store, + ctx->concepts, + ctx->daemon_cfg, + name, + req); + return true; + } return false; } diff --git a/src/amduatd_concepts.h b/src/amduatd_concepts.h index 5e8bbc8..3295229 100644 --- a/src/amduatd_concepts.h +++ b/src/amduatd_concepts.h @@ -25,6 +25,46 @@ typedef struct { size_t cap; } amduatd_edge_list_t; +typedef struct { + amduat_reference_t ref; + size_t *edge_indices; + size_t len; + size_t cap; +} amduatd_ref_edge_bucket_t; + +typedef struct { + amduat_reference_t left_ref; + amduat_reference_t right_ref; + size_t *edge_indices; + size_t len; + size_t cap; +} amduatd_ref_pair_edge_bucket_t; + +typedef struct { + size_t built_for_edges_len; + size_t *alias_edge_indices; + size_t alias_len; + size_t alias_cap; + amduatd_ref_edge_bucket_t *src_buckets; + size_t src_len; + size_t src_cap; + amduatd_ref_edge_bucket_t *dst_buckets; + size_t dst_len; + size_t dst_cap; + amduatd_ref_edge_bucket_t *predicate_buckets; + size_t predicate_len; + size_t predicate_cap; + amduatd_ref_pair_edge_bucket_t *src_predicate_buckets; + size_t src_predicate_len; + size_t src_predicate_cap; + amduatd_ref_pair_edge_bucket_t *dst_predicate_buckets; + size_t dst_predicate_len; + size_t dst_predicate_cap; + amduatd_ref_edge_bucket_t *tombstoned_src_buckets; + size_t tombstoned_src_len; + size_t tombstoned_src_cap; +} amduatd_query_index_t; + typedef struct amduatd_concepts_t { const char *root_path; char edges_path[1024]; @@ -36,8 +76,10 @@ typedef struct amduatd_concepts_t { amduat_reference_t rel_within_domain_ref; amduat_reference_t rel_computed_by_ref; amduat_reference_t rel_has_provenance_ref; + amduat_reference_t rel_tombstones_ref; amduat_asl_collection_store_t edge_collection; amduatd_edge_list_t edges; + amduatd_query_index_t qindex; } amduatd_concepts_t; bool amduatd_concepts_init(amduatd_concepts_t *c, @@ -51,6 +93,8 @@ void amduatd_concepts_free(amduatd_concepts_t *c); bool amduatd_concepts_refresh_edges(amduatd_ctx_t *ctx, size_t max_new_entries); +bool amduatd_concepts_ensure_query_index_ready(amduatd_concepts_t *c); + bool amduatd_concepts_can_handle(const amduatd_http_req_t *req); bool amduatd_concepts_handle(amduatd_ctx_t *ctx,