Add index backend health probe with fs fallback
This commit is contained in:
Carl Niklas Rydberg
parent
c10e7e04e6
commit
281f775ebb
|
|
@ -21,6 +21,10 @@ cp config/env.example config/env.local
|
|||
```sh
|
||||
./scripts/dev_start_daemon.sh
|
||||
```
|
||||
`dev_start_daemon.sh` initializes the store for the selected backend and, when
|
||||
`STORE_BACKEND=index`, runs a quick startup write probe. If index writes are
|
||||
unhealthy, it automatically falls back to `fs` (configurable via
|
||||
`INDEX_BACKEND_PROBE` and `INDEX_BACKEND_FALLBACK` in `config/env.local`).
|
||||
|
||||
3. Run startup checks against the daemon socket:
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,10 @@ SPACE="app1"
|
|||
# Optional daemon startup defaults (used by scripts/dev_start_daemon.sh)
|
||||
STORE_ROOT=".amduat-asl"
|
||||
STORE_BACKEND="index"
|
||||
# For index backend, run a startup write probe and fallback to fs if broken.
|
||||
INDEX_BACKEND_PROBE="1"
|
||||
INDEX_BACKEND_FALLBACK="fs"
|
||||
FS_FALLBACK_STORE_ROOT=".amduat-asl-fs"
|
||||
# AMDUATD_BIN="/path/to/amduatd"
|
||||
# ASL_BIN="/path/to/amduat-asl"
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,13 @@
|
|||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
override_store_root="${STORE_ROOT:-}"
|
||||
override_store_backend="${STORE_BACKEND:-}"
|
||||
override_space="${SPACE:-}"
|
||||
override_sock="${SOCK:-}"
|
||||
override_amduatd_bin="${AMDUATD_BIN:-}"
|
||||
override_asl_bin="${ASL_BIN:-}"
|
||||
|
||||
ENV_FILE="${ROOT_DIR}/config/env.local"
|
||||
if [[ ! -f "${ENV_FILE}" ]]; then
|
||||
ENV_FILE="${ROOT_DIR}/config/env.example"
|
||||
|
|
@ -9,16 +16,24 @@ fi
|
|||
# shellcheck source=/dev/null
|
||||
source "${ENV_FILE}"
|
||||
|
||||
if [[ -n "${override_store_root}" ]]; then STORE_ROOT="${override_store_root}"; fi
|
||||
if [[ -n "${override_store_backend}" ]]; then STORE_BACKEND="${override_store_backend}"; fi
|
||||
if [[ -n "${override_space}" ]]; then SPACE="${override_space}"; fi
|
||||
if [[ -n "${override_sock}" ]]; then SOCK="${override_sock}"; fi
|
||||
if [[ -n "${override_amduatd_bin}" ]]; then AMDUATD_BIN="${override_amduatd_bin}"; fi
|
||||
if [[ -n "${override_asl_bin}" ]]; then ASL_BIN="${override_asl_bin}"; fi
|
||||
|
||||
STORE_ROOT="${STORE_ROOT:-${ROOT_DIR}/.amduat-asl}"
|
||||
STORE_BACKEND="${STORE_BACKEND:-index}"
|
||||
SPACE="${SPACE:-app1}"
|
||||
SOCK="${SOCK:-${ROOT_DIR}/amduatd.sock}"
|
||||
if [[ "${STORE_ROOT}" != /* ]]; then
|
||||
STORE_ROOT="${ROOT_DIR}/${STORE_ROOT}"
|
||||
fi
|
||||
if [[ "${SOCK}" != /* ]]; then
|
||||
SOCK="${ROOT_DIR}/${SOCK}"
|
||||
fi
|
||||
INDEX_BACKEND_PROBE="${INDEX_BACKEND_PROBE:-1}"
|
||||
INDEX_BACKEND_FALLBACK="${INDEX_BACKEND_FALLBACK:-fs}"
|
||||
FS_FALLBACK_STORE_ROOT="${FS_FALLBACK_STORE_ROOT:-${STORE_ROOT}-fs}"
|
||||
|
||||
if [[ "${STORE_ROOT}" != /* ]]; then STORE_ROOT="${ROOT_DIR}/${STORE_ROOT}"; fi
|
||||
if [[ "${SOCK}" != /* ]]; then SOCK="${ROOT_DIR}/${SOCK}"; fi
|
||||
if [[ "${FS_FALLBACK_STORE_ROOT}" != /* ]]; then FS_FALLBACK_STORE_ROOT="${ROOT_DIR}/${FS_FALLBACK_STORE_ROOT}"; fi
|
||||
|
||||
# Try common local build paths first, then PATH.
|
||||
AMDUATD_BIN="${AMDUATD_BIN:-}"
|
||||
|
|
@ -60,19 +75,104 @@ if [[ -z "${ASL_BIN}" || ! -x "${ASL_BIN}" ]]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "${STORE_ROOT}"
|
||||
init_store() {
|
||||
local backend="$1"
|
||||
local root="$2"
|
||||
mkdir -p "${root}"
|
||||
if [[ "${backend}" == "index" ]]; then
|
||||
if ! "${ASL_BIN}" index state --root "${root}" >/dev/null 2>&1; then
|
||||
echo "initializing index-backed ASL store at ${root}" >&2
|
||||
"${ASL_BIN}" index init --root "${root}"
|
||||
fi
|
||||
else
|
||||
if ! "${ASL_BIN}" log inspect --root "${root}" >/dev/null 2>&1; then
|
||||
echo "initializing ASL store at ${root}" >&2
|
||||
"${ASL_BIN}" init --root "${root}"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ "${STORE_BACKEND}" == "index" ]]; then
|
||||
if ! "${ASL_BIN}" index state --root "${STORE_ROOT}" >/dev/null 2>&1; then
|
||||
echo "initializing index-backed ASL store at ${STORE_ROOT}" >&2
|
||||
"${ASL_BIN}" index init --root "${STORE_ROOT}"
|
||||
fi
|
||||
else
|
||||
if ! "${ASL_BIN}" log inspect --root "${STORE_ROOT}" >/dev/null 2>&1; then
|
||||
echo "initializing ASL store at ${STORE_ROOT}" >&2
|
||||
"${ASL_BIN}" init --root "${STORE_ROOT}"
|
||||
wait_ready() {
|
||||
local sock="$1"
|
||||
for _ in $(seq 1 80); do
|
||||
if [[ -S "${sock}" ]] && curl --globoff --silent --show-error --unix-socket "${sock}" "http://localhost/v2/readyz" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
sleep 0.1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
probe_index_write_path() {
|
||||
local sock="$1"
|
||||
local space="$2"
|
||||
local run_id
|
||||
run_id="$(date +%s)"
|
||||
local doc="probe-doc-${run_id}"
|
||||
local topic="probe-topic-${run_id}"
|
||||
local payload
|
||||
payload="$(cat <<JSON
|
||||
{"idempotency_key":"probe-${run_id}","mode":"continue_on_error","nodes":[{"name":"${doc}"},{"name":"${topic}"}],"edges":[{"subject":"${doc}","predicate":"ms.within_domain","object":"${topic}","provenance":{"source_uri":"urn:probe","extractor":"dev-start","observed_at":1,"ingested_at":2,"trace_id":"probe-${run_id}"}}]}
|
||||
JSON
|
||||
)"
|
||||
local out
|
||||
out="$(curl --globoff --silent --show-error --unix-socket "${sock}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "X-Amduat-Space: ${space}" \
|
||||
-X POST --data-binary "${payload}" \
|
||||
"http://localhost/v2/graph/batch")" || return 1
|
||||
if [[ "${out}" == *'"ok":true'* ]]; then
|
||||
return 0
|
||||
fi
|
||||
echo "index probe response: ${out}" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
run_daemon_foreground() {
|
||||
local backend="$1"
|
||||
local root="$2"
|
||||
echo "starting amduatd: root=${root} sock=${SOCK} backend=${backend} space=${SPACE}" >&2
|
||||
exec "${AMDUATD_BIN}" --root "${root}" --sock "${SOCK}" --store-backend "${backend}" --space "${SPACE}"
|
||||
}
|
||||
|
||||
init_store "${STORE_BACKEND}" "${STORE_ROOT}"
|
||||
|
||||
if [[ "${STORE_BACKEND}" != "index" || "${INDEX_BACKEND_PROBE}" != "1" ]]; then
|
||||
run_daemon_foreground "${STORE_BACKEND}" "${STORE_ROOT}"
|
||||
fi
|
||||
|
||||
echo "starting amduatd: root=${STORE_ROOT} sock=${SOCK} backend=${STORE_BACKEND} space=${SPACE}" >&2
|
||||
exec "${AMDUATD_BIN}" --root "${STORE_ROOT}" --sock "${SOCK}" --store-backend "${STORE_BACKEND}" --space "${SPACE}"
|
||||
echo "starting amduatd (probe mode): root=${STORE_ROOT} sock=${SOCK} backend=${STORE_BACKEND} space=${SPACE}" >&2
|
||||
"${AMDUATD_BIN}" --root "${STORE_ROOT}" --sock "${SOCK}" --store-backend "${STORE_BACKEND}" --space "${SPACE}" &
|
||||
daemon_pid=$!
|
||||
cleanup_probe() {
|
||||
kill "${daemon_pid}" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup_probe EXIT
|
||||
|
||||
fallback_to_fs=0
|
||||
if ! wait_ready "${SOCK}"; then
|
||||
echo "index backend startup/readiness failed" >&2
|
||||
fallback_to_fs=1
|
||||
elif ! probe_index_write_path "${SOCK}" "${SPACE}"; then
|
||||
echo "index backend write probe failed" >&2
|
||||
fallback_to_fs=1
|
||||
fi
|
||||
|
||||
if [[ "${fallback_to_fs}" == "0" ]]; then
|
||||
trap - EXIT
|
||||
wait "${daemon_pid}"
|
||||
exit $?
|
||||
fi
|
||||
|
||||
kill "${daemon_pid}" >/dev/null 2>&1 || true
|
||||
wait "${daemon_pid}" >/dev/null 2>&1 || true
|
||||
trap - EXIT
|
||||
|
||||
if [[ "${INDEX_BACKEND_FALLBACK}" != "fs" ]]; then
|
||||
echo "set INDEX_BACKEND_FALLBACK=fs to auto-fallback, or INDEX_BACKEND_PROBE=0 to disable probe" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "falling back to fs backend: root=${FS_FALLBACK_STORE_ROOT}" >&2
|
||||
init_store "fs" "${FS_FALLBACK_STORE_ROOT}"
|
||||
run_daemon_foreground "fs" "${FS_FALLBACK_STORE_ROOT}"
|
||||
|
|
|
|||
Loading…
Reference in a new issue