diff --git a/bitnami/spark/3.4/debian-11/Dockerfile b/bitnami/spark/3.4/debian-11/Dockerfile index dae0d73c10e4..c81b3a309e16 100644 --- a/bitnami/spark/3.4/debian-11/Dockerfile +++ b/bitnami/spark/3.4/debian-11/Dockerfile @@ -8,10 +8,10 @@ ARG TARGETARCH LABEL com.vmware.cp.artifact.flavor="sha256:1e1b4657a77f0d47e9220f0c37b9bf7802581b93214fff7d1bd2364c8bf22e8e" \ org.opencontainers.image.base.name="docker.io/bitnami/minideb:bullseye" \ - org.opencontainers.image.created="2023-10-10T07:11:24Z" \ + org.opencontainers.image.created="2023-10-25T17:33:26Z" \ org.opencontainers.image.description="Application packaged by VMware, Inc" \ org.opencontainers.image.licenses="Apache-2.0" \ - org.opencontainers.image.ref.name="3.4.1-debian-11-r82" \ + org.opencontainers.image.ref.name="3.4.1-debian-11-r83" \ org.opencontainers.image.title="spark" \ org.opencontainers.image.vendor="VMware, Inc." \ org.opencontainers.image.version="3.4.1" @@ -28,9 +28,9 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN install_packages ca-certificates curl libbz2-1.0 libcom-err2 libcrypt1 libffi7 libgcc-s1 libgssapi-krb5-2 libk5crypto3 libkeyutils1 libkrb5-3 libkrb5support0 liblzma5 libncursesw6 libnsl2 libreadline8 libsqlite3-0 libssl1.1 libstdc++6 libtinfo6 libtirpc3 procps zlib1g RUN mkdir -p /tmp/bitnami/pkg/cache/ && cd /tmp/bitnami/pkg/cache/ && \ COMPONENTS=( \ - "python-3.9.18-2-linux-${OS_ARCH}-debian-11" \ - "java-17.0.8-7-5-linux-${OS_ARCH}-debian-11" \ - "spark-3.4.1-4-linux-${OS_ARCH}-debian-11" \ + "python-3.11.6-6-linux-${OS_ARCH}-debian-11" \ + "java-17.0.9-11-1-linux-${OS_ARCH}-debian-11" \ + "spark-3.4.1-5-linux-${OS_ARCH}-debian-11" \ ) && \ for COMPONENT in "${COMPONENTS[@]}"; do \ if [ ! -f "${COMPONENT}.tar.gz" ]; then \ diff --git a/bitnami/spark/3.4/debian-11/prebuildfs/opt/bitnami/.bitnami_components.json b/bitnami/spark/3.4/debian-11/prebuildfs/opt/bitnami/.bitnami_components.json index f6c86c345d49..18e257442e23 100644 --- a/bitnami/spark/3.4/debian-11/prebuildfs/opt/bitnami/.bitnami_components.json +++ b/bitnami/spark/3.4/debian-11/prebuildfs/opt/bitnami/.bitnami_components.json @@ -3,18 +3,18 @@ "arch": "amd64", "distro": "debian-11", "type": "NAMI", - "version": "17.0.8-7-5" + "version": "17.0.9-11-1" }, "python": { "arch": "amd64", "distro": "debian-11", "type": "NAMI", - "version": "3.9.18-2" + "version": "3.11.6-6" }, "spark": { "arch": "amd64", "distro": "debian-11", "type": "NAMI", - "version": "3.4.1-4" + "version": "3.4.1-5" } } \ No newline at end of file diff --git a/bitnami/spark/3.4/debian-11/rootfs/opt/bitnami/scripts/spark/entrypoint.sh b/bitnami/spark/3.4/debian-11/rootfs/opt/bitnami/scripts/spark/entrypoint.sh index 59eecd50dd53..ec07b0552d2c 100755 --- a/bitnami/spark/3.4/debian-11/rootfs/opt/bitnami/scripts/spark/entrypoint.sh +++ b/bitnami/spark/3.4/debian-11/rootfs/opt/bitnami/scripts/spark/entrypoint.sh @@ -30,14 +30,15 @@ if [[ "$1" = "/opt/bitnami/scripts/spark/run.sh" ]]; then info "** Spark setup finished! **" fi -# Spark has an special 'driver' command which is an alias for spark-submit -# https://github.com/apache/spark/blob/master/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh +# ref: https://spark.apache.org/docs/latest/running-on-kubernetes.html +# inspired by https://github.com/apache/spark/blob/master/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh case "$1" in driver) shift 1 CMD=( "/opt/bitnami/spark/bin/spark-submit" - --master "${SPARK_MASTER_URL}" + --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS" + --conf "spark.executorEnv.SPARK_DRIVER_POD_IP=$SPARK_DRIVER_BIND_ADDRESS" --conf "spark.jars.ivy=/tmp/.ivy" --deploy-mode client "$@"