64 lines
2.1 KiB
Bash
64 lines
2.1 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
set -euo pipefail
|
||
|
|
|
||
|
|
MANIFEST_FILE="${1:-/tmp/rel_2026-02-14_docs-v1.json}"
|
||
|
|
CONCEPT_TABLE="${2:-lake.db1.docs}"
|
||
|
|
MODE="${3:-}"
|
||
|
|
TARGETS="${4:-both}"
|
||
|
|
RELEASE_NAME="${5:-${RELEASE_NAME:-}}"
|
||
|
|
|
||
|
|
CONTAINER_NAME="${SPARK_CONTAINER_NAME:-spark}"
|
||
|
|
SPARK_PROPS="${SPARK_PROPS:-/opt/lakehouse/spark-conf/lakehouse-spark-defaults.conf}"
|
||
|
|
PACKAGES="${SPARK_PACKAGES:-org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.10.1,org.apache.iceberg:iceberg-aws-bundle:1.10.1,org.projectnessie.nessie-integrations:nessie-spark-extensions-3.5_2.12:0.104.5}"
|
||
|
|
|
||
|
|
SCRIPT_LOCAL="${SCRIPT_LOCAL:-./release_projector.py}"
|
||
|
|
SCRIPT_REMOTE="/tmp/release_projector.py"
|
||
|
|
MANIFEST_REMOTE="/tmp/$(basename "$MANIFEST_FILE")"
|
||
|
|
|
||
|
|
if [[ ! -f "$SCRIPT_LOCAL" ]]; then
|
||
|
|
echo "release_projector.py not found at: $SCRIPT_LOCAL" >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [[ -z "$RELEASE_NAME" && ! -f "$MANIFEST_FILE" ]]; then
|
||
|
|
echo "manifest file not found: $MANIFEST_FILE (or provide release name arg5)" >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
docker cp "$SCRIPT_LOCAL" "$CONTAINER_NAME":"$SCRIPT_REMOTE"
|
||
|
|
if [[ -f "$MANIFEST_FILE" ]]; then
|
||
|
|
docker cp "$MANIFEST_FILE" "$CONTAINER_NAME":"$MANIFEST_REMOTE"
|
||
|
|
fi
|
||
|
|
|
||
|
|
ARGS=(
|
||
|
|
"$SCRIPT_REMOTE"
|
||
|
|
"--concept-table" "$CONCEPT_TABLE"
|
||
|
|
"--targets" "$TARGETS"
|
||
|
|
)
|
||
|
|
|
||
|
|
if [[ -n "$RELEASE_NAME" ]]; then
|
||
|
|
ARGS+=("--release-name" "$RELEASE_NAME")
|
||
|
|
else
|
||
|
|
ARGS+=("--manifest-file" "$MANIFEST_REMOTE")
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [[ -n "$MODE" ]]; then
|
||
|
|
ARGS+=("$MODE")
|
||
|
|
fi
|
||
|
|
|
||
|
|
docker exec -e AWS_REGION="${AWS_REGION:-us-east-1}" \
|
||
|
|
-e AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}" \
|
||
|
|
-e NESSIE_URI="${NESSIE_URI:-http://lakehouse-core:19120/api/v2}" \
|
||
|
|
-e NESSIE_WAREHOUSE="${NESSIE_WAREHOUSE:-s3a://lakehouse/warehouse}" \
|
||
|
|
-e S3_ENDPOINT="${S3_ENDPOINT:-http://lakehouse-core:9000}" \
|
||
|
|
-e AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-minioadmin}" \
|
||
|
|
-e AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-minioadmin}" \
|
||
|
|
-e GREMLIN_URL="${GREMLIN_URL:-ws://janus.rakeroots.lan:8182/gremlin}" \
|
||
|
|
-e ES_URL="${ES_URL:-http://janus.rakeroots.lan:9200}" \
|
||
|
|
-e ES_INDEX="${ES_INDEX:-concepts}" \
|
||
|
|
"$CONTAINER_NAME" \
|
||
|
|
/opt/spark/bin/spark-submit \
|
||
|
|
--properties-file "$SPARK_PROPS" \
|
||
|
|
--packages "$PACKAGES" \
|
||
|
|
"${ARGS[@]}"
|