48 lines
1.7 KiB
Bash
48 lines
1.7 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
set -euo pipefail
|
||
|
|
|
||
|
|
RELEASE_NAME="${1:-rel_$(date -u +%Y-%m-%d)_messages-v1}"
|
||
|
|
TABLE="${MESSAGES_TABLE:-lake.db1.messages}"
|
||
|
|
MANIFEST_LOCAL="${2:-./manifests/${RELEASE_NAME}.json}"
|
||
|
|
DESCRIPTION="${RELEASE_DESCRIPTION:-Messages release for ${TABLE}}"
|
||
|
|
CREATED_BY="${RELEASE_CREATED_BY:-${USER:-unknown}}"
|
||
|
|
NESSIE_URI="${NESSIE_URI:-http://nessie:19120/api/v2}"
|
||
|
|
RELEASES_TABLE="${RELEASES_TABLE:-lake.db1.releases_v2}"
|
||
|
|
|
||
|
|
CONTAINER_NAME="${SPARK_CONTAINER_NAME:-spark}"
|
||
|
|
SPARK_PROPS="${SPARK_PROPS:-/opt/lakehouse/spark-conf/lakehouse-spark-defaults.conf}"
|
||
|
|
PACKAGES="${SPARK_PACKAGES:-org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.10.1,org.apache.iceberg:iceberg-aws-bundle:1.10.1,org.projectnessie.nessie-integrations:nessie-spark-extensions-3.5_2.12:0.104.5}"
|
||
|
|
|
||
|
|
SCRIPT_LOCAL="${SCRIPT_LOCAL:-./create_release_manifest.py}"
|
||
|
|
SCRIPT_REMOTE="/tmp/create_release_manifest.py"
|
||
|
|
MANIFEST_REMOTE="/tmp/${RELEASE_NAME}.json"
|
||
|
|
|
||
|
|
if [[ ! -f "$SCRIPT_LOCAL" ]]; then
|
||
|
|
echo "create_release_manifest.py not found at: $SCRIPT_LOCAL" >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
mkdir -p "$(dirname "$MANIFEST_LOCAL")"
|
||
|
|
|
||
|
|
docker cp "$SCRIPT_LOCAL" "$CONTAINER_NAME":"$SCRIPT_REMOTE"
|
||
|
|
|
||
|
|
docker exec \
|
||
|
|
-e AWS_REGION="${AWS_REGION:-us-east-1}" \
|
||
|
|
-e AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}" \
|
||
|
|
"$CONTAINER_NAME" \
|
||
|
|
/opt/spark/bin/spark-submit \
|
||
|
|
--properties-file "$SPARK_PROPS" \
|
||
|
|
--packages "$PACKAGES" \
|
||
|
|
"$SCRIPT_REMOTE" \
|
||
|
|
--release-name "$RELEASE_NAME" \
|
||
|
|
--table "$TABLE" \
|
||
|
|
--nessie-uri "$NESSIE_URI" \
|
||
|
|
--manifest-out "$MANIFEST_REMOTE" \
|
||
|
|
--description "$DESCRIPTION" \
|
||
|
|
--created-by "$CREATED_BY" \
|
||
|
|
--releases-table "$RELEASES_TABLE"
|
||
|
|
|
||
|
|
docker cp "$CONTAINER_NAME":"$MANIFEST_REMOTE" "$MANIFEST_LOCAL"
|
||
|
|
|
||
|
|
echo "[DONE] Saved manifest: $MANIFEST_LOCAL"
|