142 lines
4 KiB
Bash
142 lines
4 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
set -euo pipefail
|
||
|
|
|
||
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||
|
|
# shellcheck source=/dev/null
|
||
|
|
source "${ROOT_DIR}/src/app_v2.sh"
|
||
|
|
|
||
|
|
usage() {
|
||
|
|
cat <<USAGE
|
||
|
|
usage: $0 [--skip-evals] [--auto-start-daemon]
|
||
|
|
|
||
|
|
Runs the AI vertical slice:
|
||
|
|
1) startup checks
|
||
|
|
2) seed graph ingest from ai/fixtures/seed_batch.json
|
||
|
|
3) retrieve grounding context
|
||
|
|
4) generate grounded answer (require evidence)
|
||
|
|
5) optional eval scripts
|
||
|
|
|
||
|
|
Environment overrides:
|
||
|
|
AI_SLICE_FIXTURE_PATH (default: ai/fixtures/seed_batch.json)
|
||
|
|
AI_SLICE_ROOTS_CSV (default: doc-ai-1)
|
||
|
|
AI_SLICE_GOALS_CSV (default: ms.within_domain)
|
||
|
|
AI_SLICE_QUESTION (default: What domain is doc-ai-1 in?)
|
||
|
|
AI_SLICE_SKIP_EVALS (default: 0)
|
||
|
|
AI_SLICE_AUTO_START_DAEMON (default: 0)
|
||
|
|
USAGE
|
||
|
|
}
|
||
|
|
|
||
|
|
require_jq() {
|
||
|
|
if ! command -v jq >/dev/null 2>&1; then
|
||
|
|
echo "ai_vertical_slice.sh: jq is required" >&2
|
||
|
|
exit 2
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
skip_evals="${AI_SLICE_SKIP_EVALS:-0}"
|
||
|
|
auto_start_daemon="${AI_SLICE_AUTO_START_DAEMON:-0}"
|
||
|
|
while [[ $# -gt 0 ]]; do
|
||
|
|
case "$1" in
|
||
|
|
--skip-evals)
|
||
|
|
skip_evals=1
|
||
|
|
shift
|
||
|
|
;;
|
||
|
|
--auto-start-daemon)
|
||
|
|
auto_start_daemon=1
|
||
|
|
shift
|
||
|
|
;;
|
||
|
|
-h|--help)
|
||
|
|
usage
|
||
|
|
exit 0
|
||
|
|
;;
|
||
|
|
*)
|
||
|
|
usage >&2
|
||
|
|
exit 2
|
||
|
|
;;
|
||
|
|
esac
|
||
|
|
done
|
||
|
|
|
||
|
|
fixture_path="${AI_SLICE_FIXTURE_PATH:-${ROOT_DIR}/ai/fixtures/seed_batch.json}"
|
||
|
|
roots_csv="${AI_SLICE_ROOTS_CSV:-doc-ai-1}"
|
||
|
|
goals_csv="${AI_SLICE_GOALS_CSV:-ms.within_domain}"
|
||
|
|
question="${AI_SLICE_QUESTION:-What domain is doc-ai-1 in?}"
|
||
|
|
|
||
|
|
[[ -f "${fixture_path}" ]] || {
|
||
|
|
echo "ai_vertical_slice.sh: fixture not found: ${fixture_path}" >&2
|
||
|
|
exit 2
|
||
|
|
}
|
||
|
|
|
||
|
|
require_jq
|
||
|
|
app_init
|
||
|
|
|
||
|
|
ensure_daemon_ready() {
|
||
|
|
if app_startup_checks >/dev/null 2>&1; then
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [[ "${auto_start_daemon}" == "1" ]]; then
|
||
|
|
local daemon_backend="${AI_DAEMON_STORE_BACKEND:-fs}"
|
||
|
|
local daemon_root="${AI_DAEMON_STORE_ROOT:-/tmp/amduat-asl-ai-slice}"
|
||
|
|
local daemon_log="${AI_DAEMON_LOG_PATH:-/tmp/ai-vertical-slice-daemon.log}"
|
||
|
|
echo "daemon not reachable; attempting startup via scripts/dev_start_daemon.sh" >&2
|
||
|
|
STORE_BACKEND="${daemon_backend}" STORE_ROOT="${daemon_root}" SOCK="${SOCK}" SPACE="${SPACE}" \
|
||
|
|
nohup "${ROOT_DIR}/scripts/dev_start_daemon.sh" >"${daemon_log}" 2>&1 &
|
||
|
|
local daemon_boot_pid="$!"
|
||
|
|
disown "${daemon_boot_pid}" 2>/dev/null || true
|
||
|
|
local i
|
||
|
|
for i in $(seq 1 80); do
|
||
|
|
if app_startup_checks >/dev/null 2>&1; then
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
sleep 0.1
|
||
|
|
done
|
||
|
|
app_startup_checks >/dev/null 2>&1 || {
|
||
|
|
echo "ai_vertical_slice.sh: daemon still unreachable after startup attempt" >&2
|
||
|
|
echo "see ${daemon_log} for startup logs" >&2
|
||
|
|
return 1
|
||
|
|
}
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo "ai_vertical_slice.sh: daemon unreachable on SOCK=${SOCK}" >&2
|
||
|
|
echo "hint: run ./scripts/dev_start_daemon.sh or pass --auto-start-daemon" >&2
|
||
|
|
return 1
|
||
|
|
}
|
||
|
|
|
||
|
|
ensure_daemon_ready
|
||
|
|
|
||
|
|
echo "== startup-check =="
|
||
|
|
app_startup_checks | jq .
|
||
|
|
|
||
|
|
echo "== ingest fixture =="
|
||
|
|
idempotency_key="ai-slice-$(date +%s)"
|
||
|
|
payload="$(jq -c --arg k "${idempotency_key}" '.idempotency_key = $k' "${fixture_path}")"
|
||
|
|
ingest_out="$(app_ingest_batch "${payload}")"
|
||
|
|
printf '%s\n' "${ingest_out}" | jq .
|
||
|
|
|
||
|
|
echo "== retrieve context =="
|
||
|
|
retrieve_out="$(app_retrieve_with_fallback "${roots_csv}" "${goals_csv}")"
|
||
|
|
printf '%s\n' "${retrieve_out}" | jq .
|
||
|
|
printf '%s' "${retrieve_out}" | jq -e '((.nodes // []) | length) > 0 and ((.edges // []) | length) > 0' >/dev/null || {
|
||
|
|
echo "ai_vertical_slice.sh: retrieve produced no graph context" >&2
|
||
|
|
exit 1
|
||
|
|
}
|
||
|
|
|
||
|
|
echo "== grounded answer =="
|
||
|
|
answer_out="$(app_ai_answer_json "${roots_csv}" "${question}" "${goals_csv}" "1")"
|
||
|
|
printf '%s\n' "${answer_out}" | jq .
|
||
|
|
printf '%s' "${answer_out}" | jq -e '.grounding.has_evidence == true and (.response | type == "string" and length > 0)' >/dev/null || {
|
||
|
|
echo "ai_vertical_slice.sh: answer was not grounded with evidence" >&2
|
||
|
|
exit 1
|
||
|
|
}
|
||
|
|
|
||
|
|
if [[ "${skip_evals}" != "1" ]]; then
|
||
|
|
echo "== evals =="
|
||
|
|
"${ROOT_DIR}/tests/ai_eval.sh"
|
||
|
|
"${ROOT_DIR}/tests/ai_answer_eval.sh"
|
||
|
|
else
|
||
|
|
echo "== evals skipped =="
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo "ai_vertical_slice.sh: PASS"
|