From 281f775ebb7a2b77b626100f67919212e31c653a Mon Sep 17 00:00:00 2001
From: Carl Niklas Rydberg <carlniklasrydberg@gmail.com>
Date: Sat, 7 Feb 2026 20:45:39 +0100
Subject: [PATCH] Add index backend health probe with fs fallback

---
 README.md                   |   4 ++
 config/env.example          |   4 ++
 scripts/dev_start_daemon.sh | 136 +++++++++++++++++++++++++++++++-----
 3 files changed, 126 insertions(+), 18 deletions(-)

diff --git a/README.md b/README.md
index 4da307a..81c110f 100644
--- a/README.md
+++ b/README.md
@@ -21,6 +21,10 @@ cp config/env.example config/env.local
 ```sh
 ./scripts/dev_start_daemon.sh
 ```
+`dev_start_daemon.sh` initializes the store for the selected backend and, when
+`STORE_BACKEND=index`, runs a quick startup write probe. If index writes are
+unhealthy, it automatically falls back to `fs` (configurable via
+`INDEX_BACKEND_PROBE` and `INDEX_BACKEND_FALLBACK` in `config/env.local`).
 
 3. Run startup checks against the daemon socket:
 
diff --git a/config/env.example b/config/env.example
index 087ad20..9ba2b92 100644
--- a/config/env.example
+++ b/config/env.example
@@ -6,6 +6,10 @@ SPACE="app1"
 # Optional daemon startup defaults (used by scripts/dev_start_daemon.sh)
 STORE_ROOT=".amduat-asl"
 STORE_BACKEND="index"
+# For index backend, run a startup write probe and fallback to fs if broken.
+INDEX_BACKEND_PROBE="1"
+INDEX_BACKEND_FALLBACK="fs"
+FS_FALLBACK_STORE_ROOT=".amduat-asl-fs"
 # AMDUATD_BIN="/path/to/amduatd"
 # ASL_BIN="/path/to/amduat-asl"
 
diff --git a/scripts/dev_start_daemon.sh b/scripts/dev_start_daemon.sh
index 7300724..fc3d098 100755
--- a/scripts/dev_start_daemon.sh
+++ b/scripts/dev_start_daemon.sh
@@ -2,6 +2,13 @@
 set -euo pipefail
 
 ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+override_store_root="${STORE_ROOT:-}"
+override_store_backend="${STORE_BACKEND:-}"
+override_space="${SPACE:-}"
+override_sock="${SOCK:-}"
+override_amduatd_bin="${AMDUATD_BIN:-}"
+override_asl_bin="${ASL_BIN:-}"
+
 ENV_FILE="${ROOT_DIR}/config/env.local"
 if [[ ! -f "${ENV_FILE}" ]]; then
   ENV_FILE="${ROOT_DIR}/config/env.example"
@@ -9,16 +16,24 @@ fi
 # shellcheck source=/dev/null
 source "${ENV_FILE}"
 
+if [[ -n "${override_store_root}" ]]; then STORE_ROOT="${override_store_root}"; fi
+if [[ -n "${override_store_backend}" ]]; then STORE_BACKEND="${override_store_backend}"; fi
+if [[ -n "${override_space}" ]]; then SPACE="${override_space}"; fi
+if [[ -n "${override_sock}" ]]; then SOCK="${override_sock}"; fi
+if [[ -n "${override_amduatd_bin}" ]]; then AMDUATD_BIN="${override_amduatd_bin}"; fi
+if [[ -n "${override_asl_bin}" ]]; then ASL_BIN="${override_asl_bin}"; fi
+
 STORE_ROOT="${STORE_ROOT:-${ROOT_DIR}/.amduat-asl}"
 STORE_BACKEND="${STORE_BACKEND:-index}"
 SPACE="${SPACE:-app1}"
 SOCK="${SOCK:-${ROOT_DIR}/amduatd.sock}"
-if [[ "${STORE_ROOT}" != /* ]]; then
-  STORE_ROOT="${ROOT_DIR}/${STORE_ROOT}"
-fi
-if [[ "${SOCK}" != /* ]]; then
-  SOCK="${ROOT_DIR}/${SOCK}"
-fi
+INDEX_BACKEND_PROBE="${INDEX_BACKEND_PROBE:-1}"
+INDEX_BACKEND_FALLBACK="${INDEX_BACKEND_FALLBACK:-fs}"
+FS_FALLBACK_STORE_ROOT="${FS_FALLBACK_STORE_ROOT:-${STORE_ROOT}-fs}"
+
+if [[ "${STORE_ROOT}" != /* ]]; then STORE_ROOT="${ROOT_DIR}/${STORE_ROOT}"; fi
+if [[ "${SOCK}" != /* ]]; then SOCK="${ROOT_DIR}/${SOCK}"; fi
+if [[ "${FS_FALLBACK_STORE_ROOT}" != /* ]]; then FS_FALLBACK_STORE_ROOT="${ROOT_DIR}/${FS_FALLBACK_STORE_ROOT}"; fi
 
 # Try common local build paths first, then PATH.
 AMDUATD_BIN="${AMDUATD_BIN:-}"
@@ -60,19 +75,104 @@ if [[ -z "${ASL_BIN}" || ! -x "${ASL_BIN}" ]]; then
   exit 1
 fi
 
-mkdir -p "${STORE_ROOT}"
+init_store() {
+  local backend="$1"
+  local root="$2"
+  mkdir -p "${root}"
+  if [[ "${backend}" == "index" ]]; then
+    if ! "${ASL_BIN}" index state --root "${root}" >/dev/null 2>&1; then
+      echo "initializing index-backed ASL store at ${root}" >&2
+      "${ASL_BIN}" index init --root "${root}"
+    fi
+  else
+    if ! "${ASL_BIN}" log inspect --root "${root}" >/dev/null 2>&1; then
+      echo "initializing ASL store at ${root}" >&2
+      "${ASL_BIN}" init --root "${root}"
+    fi
+  fi
+}
 
-if [[ "${STORE_BACKEND}" == "index" ]]; then
-  if ! "${ASL_BIN}" index state --root "${STORE_ROOT}" >/dev/null 2>&1; then
-    echo "initializing index-backed ASL store at ${STORE_ROOT}" >&2
-    "${ASL_BIN}" index init --root "${STORE_ROOT}"
-  fi
-else
-  if ! "${ASL_BIN}" log inspect --root "${STORE_ROOT}" >/dev/null 2>&1; then
-    echo "initializing ASL store at ${STORE_ROOT}" >&2
-    "${ASL_BIN}" init --root "${STORE_ROOT}"
+wait_ready() {
+  local sock="$1"
+  for _ in $(seq 1 80); do
+    if [[ -S "${sock}" ]] && curl --globoff --silent --show-error --unix-socket "${sock}" "http://localhost/v2/readyz" >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 0.1
+  done
+  return 1
+}
+
+probe_index_write_path() {
+  local sock="$1"
+  local space="$2"
+  local run_id
+  run_id="$(date +%s)"
+  local doc="probe-doc-${run_id}"
+  local topic="probe-topic-${run_id}"
+  local payload
+  payload="$(cat <<JSON
+{"idempotency_key":"probe-${run_id}","mode":"continue_on_error","nodes":[{"name":"${doc}"},{"name":"${topic}"}],"edges":[{"subject":"${doc}","predicate":"ms.within_domain","object":"${topic}","provenance":{"source_uri":"urn:probe","extractor":"dev-start","observed_at":1,"ingested_at":2,"trace_id":"probe-${run_id}"}}]}
+JSON
+)"
+  local out
+  out="$(curl --globoff --silent --show-error --unix-socket "${sock}" \
+    -H "Content-Type: application/json" \
+    -H "X-Amduat-Space: ${space}" \
+    -X POST --data-binary "${payload}" \
+    "http://localhost/v2/graph/batch")" || return 1
+  if [[ "${out}" == *'"ok":true'* ]]; then
+    return 0
   fi
+  echo "index probe response: ${out}" >&2
+  return 1
+}
+
+run_daemon_foreground() {
+  local backend="$1"
+  local root="$2"
+  echo "starting amduatd: root=${root} sock=${SOCK} backend=${backend} space=${SPACE}" >&2
+  exec "${AMDUATD_BIN}" --root "${root}" --sock "${SOCK}" --store-backend "${backend}" --space "${SPACE}"
+}
+
+init_store "${STORE_BACKEND}" "${STORE_ROOT}"
+
+if [[ "${STORE_BACKEND}" != "index" || "${INDEX_BACKEND_PROBE}" != "1" ]]; then
+  run_daemon_foreground "${STORE_BACKEND}" "${STORE_ROOT}"
 fi
 
-echo "starting amduatd: root=${STORE_ROOT} sock=${SOCK} backend=${STORE_BACKEND} space=${SPACE}" >&2
-exec "${AMDUATD_BIN}" --root "${STORE_ROOT}" --sock "${SOCK}" --store-backend "${STORE_BACKEND}" --space "${SPACE}"
+echo "starting amduatd (probe mode): root=${STORE_ROOT} sock=${SOCK} backend=${STORE_BACKEND} space=${SPACE}" >&2
+"${AMDUATD_BIN}" --root "${STORE_ROOT}" --sock "${SOCK}" --store-backend "${STORE_BACKEND}" --space "${SPACE}" &
+daemon_pid=$!
+cleanup_probe() {
+  kill "${daemon_pid}" >/dev/null 2>&1 || true
+}
+trap cleanup_probe EXIT
+
+fallback_to_fs=0
+if ! wait_ready "${SOCK}"; then
+  echo "index backend startup/readiness failed" >&2
+  fallback_to_fs=1
+elif ! probe_index_write_path "${SOCK}" "${SPACE}"; then
+  echo "index backend write probe failed" >&2
+  fallback_to_fs=1
+fi
+
+if [[ "${fallback_to_fs}" == "0" ]]; then
+  trap - EXIT
+  wait "${daemon_pid}"
+  exit $?
+fi
+
+kill "${daemon_pid}" >/dev/null 2>&1 || true
+wait "${daemon_pid}" >/dev/null 2>&1 || true
+trap - EXIT
+
+if [[ "${INDEX_BACKEND_FALLBACK}" != "fs" ]]; then
+  echo "set INDEX_BACKEND_FALLBACK=fs to auto-fallback, or INDEX_BACKEND_PROBE=0 to disable probe" >&2
+  exit 1
+fi
+
+echo "falling back to fs backend: root=${FS_FALLBACK_STORE_ROOT}" >&2
+init_store "fs" "${FS_FALLBACK_STORE_ROOT}"
+run_daemon_foreground "fs" "${FS_FALLBACK_STORE_ROOT}"