From 56fe4018638f383b529cbe516274473139700969 Mon Sep 17 00:00:00 2001 From: Bitnami Bot Date: Mon, 22 Jan 2024 13:02:06 +0100 Subject: [PATCH] [bitnami/kafka] Release 3.3.2-debian-11-r244 (#55245) Signed-off-by: Bitnami Containers --- bitnami/kafka/3.3/debian-11/Dockerfile | 11 +-- .../opt/bitnami/.bitnami_components.json | 6 +- .../debian-11/prebuildfs/usr/sbin/run-script | 2 +- .../rootfs/opt/bitnami/scripts/kafka-env.sh | 3 + .../rootfs/opt/bitnami/scripts/libkafka.sh | 2 + bitnami/kafka/README.md | 83 ++++++++++--------- 6 files changed, 60 insertions(+), 47 deletions(-) diff --git a/bitnami/kafka/3.3/debian-11/Dockerfile b/bitnami/kafka/3.3/debian-11/Dockerfile index 8d61b874609b..0503e55b47d5 100644 --- a/bitnami/kafka/3.3/debian-11/Dockerfile +++ b/bitnami/kafka/3.3/debian-11/Dockerfile @@ -8,10 +8,10 @@ ARG TARGETARCH LABEL com.vmware.cp.artifact.flavor="sha256:1e1b4657a77f0d47e9220f0c37b9bf7802581b93214fff7d1bd2364c8bf22e8e" \ org.opencontainers.image.base.name="docker.io/bitnami/minideb:bullseye" \ - org.opencontainers.image.created="2023-12-24T07:38:33Z" \ + org.opencontainers.image.created="2024-01-22T10:32:08Z" \ org.opencontainers.image.description="Application packaged by VMware, Inc" \ org.opencontainers.image.licenses="Apache-2.0" \ - org.opencontainers.image.ref.name="3.3.2-debian-11-r243" \ + org.opencontainers.image.ref.name="3.3.2-debian-11-r244" \ org.opencontainers.image.title="kafka" \ org.opencontainers.image.vendor="VMware, Inc." \ org.opencontainers.image.version="3.3.2" @@ -27,9 +27,9 @@ SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-c"] RUN install_packages ca-certificates curl procps zlib1g RUN mkdir -p /tmp/bitnami/pkg/cache/ ; cd /tmp/bitnami/pkg/cache/ ; \ COMPONENTS=( \ - "wait-for-port-1.0.7-4-linux-${OS_ARCH}-debian-11" \ - "java-17.0.9-11-6-linux-${OS_ARCH}-debian-11" \ - "render-template-1.0.6-4-linux-${OS_ARCH}-debian-11" \ + "wait-for-port-1.0.7-5-linux-${OS_ARCH}-debian-11" \ + "render-template-1.0.6-5-linux-${OS_ARCH}-debian-11" \ + "java-17.0.10-13-0-linux-${OS_ARCH}-debian-11" \ "kafka-3.3.2-9-linux-${OS_ARCH}-debian-11" \ ) ; \ for COMPONENT in "${COMPONENTS[@]}"; do \ @@ -44,6 +44,7 @@ RUN mkdir -p /tmp/bitnami/pkg/cache/ ; cd /tmp/bitnami/pkg/cache/ ; \ RUN apt-get update && apt-get upgrade -y && \ apt-get clean && rm -rf /var/lib/apt/lists /var/cache/apt/archives RUN chmod g+rwX /opt/bitnami +RUN find / -perm /6000 -type f -exec chmod a-s {} \; || true RUN ln -s /opt/bitnami/scripts/kafka/entrypoint.sh /entrypoint.sh RUN ln -s /opt/bitnami/scripts/kafka/run.sh /run.sh diff --git a/bitnami/kafka/3.3/debian-11/prebuildfs/opt/bitnami/.bitnami_components.json b/bitnami/kafka/3.3/debian-11/prebuildfs/opt/bitnami/.bitnami_components.json index cff70bc24ab5..87bb131b3feb 100644 --- a/bitnami/kafka/3.3/debian-11/prebuildfs/opt/bitnami/.bitnami_components.json +++ b/bitnami/kafka/3.3/debian-11/prebuildfs/opt/bitnami/.bitnami_components.json @@ -3,7 +3,7 @@ "arch": "amd64", "distro": "debian-11", "type": "NAMI", - "version": "17.0.9-11-6" + "version": "17.0.10-13-0" }, "kafka": { "arch": "amd64", @@ -15,12 +15,12 @@ "arch": "amd64", "distro": "debian-11", "type": "NAMI", - "version": "1.0.6-4" + "version": "1.0.6-5" }, "wait-for-port": { "arch": "amd64", "distro": "debian-11", "type": "NAMI", - "version": "1.0.7-4" + "version": "1.0.7-5" } } \ No newline at end of file diff --git a/bitnami/kafka/3.3/debian-11/prebuildfs/usr/sbin/run-script b/bitnami/kafka/3.3/debian-11/prebuildfs/usr/sbin/run-script index b7a5bf1e50bf..4ca0f897277e 100755 --- a/bitnami/kafka/3.3/debian-11/prebuildfs/usr/sbin/run-script +++ b/bitnami/kafka/3.3/debian-11/prebuildfs/usr/sbin/run-script @@ -10,7 +10,7 @@ fi script=$1 exit_code="${2:-96}" -fail_if_not_present="${3:-y}" +fail_if_not_present="${3:-n}" if test -f "$script"; then sh $script diff --git a/bitnami/kafka/3.3/debian-11/rootfs/opt/bitnami/scripts/kafka-env.sh b/bitnami/kafka/3.3/debian-11/rootfs/opt/bitnami/scripts/kafka-env.sh index 0a87f31c4b58..ff4684a556fd 100644 --- a/bitnami/kafka/3.3/debian-11/rootfs/opt/bitnami/scripts/kafka-env.sh +++ b/bitnami/kafka/3.3/debian-11/rootfs/opt/bitnami/scripts/kafka-env.sh @@ -37,6 +37,9 @@ kafka_env_vars=( KAFKA_CFG_SASL_ENABLED_MECHANISMS KAFKA_KRAFT_CLUSTER_ID KAFKA_SKIP_KRAFT_STORAGE_INIT + KAFKA_CFG_MAX_REQUEST_SIZE + KAFKA_CFG_MESSAGE_MAX_BYTES + KAFKA_CFG_MAX_PARTITION_FETCH_BYTES KAFKA_CLIENT_LISTENER_NAME KAFKA_ZOOKEEPER_PROTOCOL KAFKA_ZOOKEEPER_PASSWORD diff --git a/bitnami/kafka/3.3/debian-11/rootfs/opt/bitnami/scripts/libkafka.sh b/bitnami/kafka/3.3/debian-11/rootfs/opt/bitnami/scripts/libkafka.sh index c463866bb9f5..3c147196920a 100644 --- a/bitnami/kafka/3.3/debian-11/rootfs/opt/bitnami/scripts/libkafka.sh +++ b/bitnami/kafka/3.3/debian-11/rootfs/opt/bitnami/scripts/libkafka.sh @@ -399,6 +399,8 @@ kafka_validate() { if [[ "$protocol" = "SASL_PLAINTEXT" ]] || [[ "$protocol" = "SASL_SSL" ]]; then if is_empty_value "${KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL:-}"; then print_validation_error "When using SASL for controller comunication the mechanism should be provided at KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL" + elif [[ "$KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL" =~ SCRAM ]]; then + warn "KRaft controller listener may not support SCRAM-SHA-256/SCRAM-SHA-512 mechanisms. If facing any issues, we recommend switching to PLAIN mechanism. More information at: https://issues.apache.org/jira/browse/KAFKA-15513" fi if is_empty_value "${KAFKA_CONTROLLER_USER:-}" || is_empty_value "${KAFKA_CONTROLLER_PASSWORD:-}"; then print_validation_error "In order to configure SASL authentication for Kafka control plane communications, you must provide the SASL credentials. Set the environment variables KAFKA_CONTROLLER_USER and KAFKA_CONTROLLER_PASSWORD to configure the credentials for SASL authentication with between controllers." diff --git a/bitnami/kafka/README.md b/bitnami/kafka/README.md index b975811de64b..0d4f19e1698d 100644 --- a/bitnami/kafka/README.md +++ b/bitnami/kafka/README.md @@ -9,13 +9,8 @@ Trademarks: This software listing is packaged by Bitnami. The respective tradema ## TL;DR -## Run the application using Docker Compose - -The main folder of this repository contains a functional [`docker-compose.yml`](https://github.com/bitnami/containers/blob/main/bitnami/kafka/docker-compose.yml) file. Run the application using it as shown below: - ```console -curl -sSL https://raw.githubusercontent.com/bitnami/containers/main/bitnami/kafka/docker-compose.yml > docker-compose.yml -docker-compose up -d +docker run --name kafka bitnami/kafka:latest ``` ## Why use Bitnami Images? @@ -177,39 +172,45 @@ docker-compose up -d ## Configuration -The configuration can easily be setup with the Bitnami Apache Kafka Docker image using the following environment variables: +### Environment variables -* `KAFKA_CERTIFICATE_PASSWORD`: Password for certificates. No defaults. -* `KAFKA_HEAP_OPTS`: Apache Kafka's Java Heap size. Default: **-Xmx1024m -Xms1024m**. -* `KAFKA_ZOOKEEPER_PROTOCOL`: Authentication protocol for Zookeeper connections. Allowed protocols: **PLAINTEXT**, **SASL**, **SSL**, and **SASL_SSL**. Defaults: **PLAINTEXT**. -* `KAFKA_ZOOKEEPER_USER`: Apache Kafka Zookeeper user for SASL authentication. No defaults. -* `KAFKA_ZOOKEEPER_PASSWORD`: Apache Kafka Zookeeper user password for SASL authentication. No defaults. -* `KAFKA_ZOOKEEPER_TLS_KEYSTORE_PASSWORD`: Apache Kafka Zookeeper keystore file password and key password. No defaults. -* `KAFKA_ZOOKEEPER_TLS_TRUSTSTORE_PASSWORD`: Apache Kafka Zookeeper truststore file password. No defaults. -* `KAFKA_ZOOKEEPER_TLS_VERIFY_HOSTNAME`: Verify Zookeeper hostname on TLS certificates. Defaults: **true**. -* `KAFKA_ZOOKEEPER_TLS_TYPE`: Choose the TLS certificate format to use. Allowed values: `JKS`, `PEM`. Defaults: **JKS**. -* `KAFKA_CFG_LISTENERS`: Kafka `listeners` configuration override. No defaults. -* `KAFKA_CFG_ADVERTISED_LISTENERS`: Kafka `advertised.listeners` configuration override. No defaults. -* `KAFKA_CFG_SASL_ENABLED_MECHANISMS`: Allowed mechanism when using SASL either for clients, inter broker, or zookeeper comunications. Allowed values: `PLAIN`, `SCRAM-SHA-256`, `SCRAM-SHA-512` or a comma separated combination of those values. Default: **PLAIN,SCRAM-SHA-256,SCRAM-SHA-512**. NOTE: KRaft <= 3.4 does not yet support SCRAM mechanisms, so this list will be automatically reduced to only `PLAIN` SASL mechanism. -* `KAFKA_TLS_CLIENT_AUTH`: Sets the value for `ssl.client.auth`. Allowed values: `required`, `requested`, `none`. Defaults: **required**. -* `KAFKA_TLS__CLIENT_AUTH`: Sets the value for `listener.name..ssl.client.auth` used to configure mTLS with SASL. Allowed values: `required`, `requested`, `none`. -* `KAFKA_TLS_TYPE`: Choose the TLS certificate format to use. Allowed values: `JKS`, `PEM`. Defaults: **JKS**. -* `KAFKA_CLIENT_USERS`: Users that will be created into Zookeeper when using SASL for client communications. Separated by commas. Default: **user** -* `KAFKA_CLIENT_PASSWORDS`: Passwords for the users specified at`KAFKA_CLIENT_USERS`. Separated by commas. Default: **bitnami** -* `KAFKA_CFG_MAX_PARTITION_FETCH_BYTES`: The maximum amount of data per-partition the server will return. No defaults. -* `KAFKA_CFG_MAX_REQUEST_SIZE`: The maximum size of a request in bytes. No defaults. -* `KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL`: SASL mechanism to use for inter broker communications. No defaults. NOTE: KRaft <= 3.4 does not yet support SCRAM mechanisms, so the only supported SASL mechanism in KRaft mode would be `PLAIN`. -* `KAFKA_INTER_BROKER_USER`: Apache Kafka inter broker communication user. Default: **user**. -* `KAFKA_INTER_BROKER_PASSWORD`: Apache Kafka inter broker communication password. Default: **bitnami**. -* `KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL`: SASL mechanism to use for controllers communications. No defaults. NOTE: KRaft <= 3.4 does not yet support SCRAM mechanisms, so the only supported SASL mechanism in KRaft mode would be `PLAIN`. -* `KAFKA_CONTROLLER_USER`: Apache Kafka controllers communication user. Default: **controller_user**. -* `KAFKA_CONTROLLER_PASSWORD`: Apache Kafka controllers communication password. Default: **bitnami**. -* `KAFKA_CFG_PROCESS_ROLES`: Node roles when running in KRaft mode. No defaults. -* `KAFKA_CFG_NODE_ID`: Unique node id, required when running in KRaft mode. No defaults. -* `KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP`: Maps each listener with a Apache Kafka security protocol. If node is set with `controller` role, this setting is required in order to assign a security protocol for the `CONTROLLER LISTENER`. E.g.: `PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT`. No defaults. -* `KAFKA_CFG_CONTROLLER_QUORUM_VOTERS`: Map of id/endpoint information for the set of controller quorum voters in a comma-separated list of {id}@{host}:{port} entries. No defaults. -* `KAFKA_CFG_CONTROLLER_LISTENER_NAMES`: Comma-separated list of the names of the listeners used by the controller. This is required if running in KRaft mode. No defaults. -* `KAFKA_KRAFT_CLUSTER_ID`: Kafka cluster ID when using Kafka Raft (KRaft). No defaults. +#### Customizable environment variables + +| Name | Description | Default Value | +|---------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------| +| `KAFKA_MOUNTED_CONF_DIR` | Kafka directory for mounted configuration files. | `${KAFKA_VOLUME_DIR}/config` | +| `KAFKA_INTER_BROKER_USER` | Kafka inter broker communication user. | `user` | +| `KAFKA_INTER_BROKER_PASSWORD` | Kafka inter broker communication password. | `bitnami` | +| `KAFKA_CONTROLLER_USER` | Kafka control plane communication user. | `controller_user` | +| `KAFKA_CONTROLLER_PASSWORD` | Kafka control plane communication password. | `bitnami` | +| `KAFKA_TLS_TYPE` | Choose the TLS certificate format to use. | `JKS` | +| `KAFKA_TLS_CLIENT_AUTH` | Configures kafka broker to request client authentication. | `required` | +| `KAFKA_CFG_SASL_ENABLED_MECHANISMS` | Kafka `sasl.enabled.mechanisms` configuration override. | `PLAIN,SCRAM-SHA-256,SCRAM-SHA-512` | +| `KAFKA_SKIP_KRAFT_STORAGE_INIT` | If set to true, skip Kraft storage initialization when process.roles are configured. | `false` | +| `KAFKA_ZOOKEEPER_PROTOCOL` | Authentication protocol for Zookeeper connections. Allowed protocols: `PLAINTEXT`, `SASL, SSL`, and `SASL_SSL`. | `PLAINTEXT` | +| `KAFKA_ZOOKEEPER_TLS_VERIFY_HOSTNAME` | Verify Zookeeper hostname on TLS certificates. | `true` | +| `KAFKA_ZOOKEEPER_TLS_TYPE` | Choose the TLS certificate format to use. Allowed values: `JKS`, `PEM`. | `JKS` | +| `KAFKA_CLIENT_USERS` | List of additional users to `KAFKA_CLIENT_USER` that will be created into Zookeeper when using `SASL_SCRAM` for client communications. Separated by commas, semicolons or whitespaces. | `user` | +| `KAFKA_CLIENT_PASSWORDS` | Passwords for the users specified at `KAFKA_CLIENT_USERS`. Separated by commas, semicolons or whitespaces. | `bitnami` | +| `KAFKA_HEAP_OPTS` | Kafka heap options for Java. | `-Xmx1024m -Xms1024m` | + +#### Read-only environment variables + +| Name | Description | Value | +|-------------------------|---------------------------------------------------|---------------------------------------| +| `KAFKA_BASE_DIR` | Kafka installation directory. | `${BITNAMI_ROOT_DIR}/kafka` | +| `KAFKA_VOLUME_DIR` | Kafka persistence directory. | `/bitnami/kafka` | +| `KAFKA_DATA_DIR` | Kafka directory where data is stored. | `${KAFKA_VOLUME_DIR}/data` | +| `KAFKA_CONF_DIR` | Kafka configuration directory. | `${KAFKA_BASE_DIR}/config` | +| `KAFKA_CONF_FILE` | Kafka configuration file. | `${KAFKA_CONF_DIR}/server.properties` | +| `KAFKA_CERTS_DIR` | Kafka directory for certificate files. | `${KAFKA_CONF_DIR}/certs` | +| `KAFKA_INITSCRIPTS_DIR` | Kafka directory for init scripts. | `/docker-entrypoint-initdb.d` | +| `KAFKA_LOG_DIR` | Directory where Kafka logs are stored. | `${KAFKA_BASE_DIR}/logs` | +| `KAFKA_TMP_DIR` | Directory where Kafka temporary files are stored. | `${KAFKA_BASE_DIR}/tmp` | +| `KAFKA_PID_FILE` | Path to the PID file for Kafka. | `${KAFKA_TMP_DIR}/kafka.pid` | +| `KAFKA_HOME` | Kafka home directory. | `$KAFKA_BASE_DIR` | +| `KAFKA_DAEMON_USER` | Kafka system user. | `kafka` | +| `KAFKA_DAEMON_GROUP` | Kafka system group. | `kafka` | Additionally, any environment variable beginning with `KAFKA_CFG_` will be mapped to its corresponding Apache Kafka key. For example, use `KAFKA_CFG_BACKGROUND_THREADS` in order to set `background.threads` or `KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE` in order to configure `auto.create.topics.enable`. @@ -1024,6 +1025,12 @@ Configuration changes. Most environment variables now start with `KAFKA_CFG_`, a * New Bitnami release +## Using `docker-compose.yaml` + +Please be aware this file has not undergone internal testing. Consequently, we advise its use exclusively for development or testing purposes. For production-ready deployments, we highly recommend utilizing its associated [Bitnami Helm chart](https://github.com/bitnami/charts/tree/main/bitnami/kafka). + +If you detect any issue in the `docker-compose.yaml` file, feel free to report it or contribute with a fix by following our [Contributing Guidelines](https://github.com/bitnami/containers/blob/main/CONTRIBUTING.md). + ## Contributing We'd love for you to contribute to this Docker image. You can request new features by creating an [issue](https://github.com/bitnami/containers/issues) or submitting a [pull request](https://github.com/bitnami/containers/pulls) with your contribution.