TEZ-4682: [Cloud] Tez AM docker image

.gitignore

@@ -10,3 +10,4 @@
 .settings
 pom.xml.versionsBackup
 target
+tez-dist/src/docker/cache/

tez-dist/pom.xml

@@ -118,6 +118,35 @@
         </dependency>
       </dependencies>
     </profile>
+    <profile>
+      <id>docker</id>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>exec-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>build-docker-image</id>
+                <phase>package</phase>
+                <goals>
+                  <goal>exec</goal>
+                </goals>
+                <configuration>
+                  <executable>/bin/bash</executable>
+                  <arguments>
+                    <argument>${project.basedir}/src/docker/build-docker.sh</argument>
+                    <argument>-hadoop ${hadoop.version}</argument>
+                    <argument>-tez ${project.version}</argument>
+                    <argument>-repo apache</argument>
+                  </arguments>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
   </profiles>
 
   <build>

tez-dist/src/docker/Dockerfile

@@ -0,0 +1,91 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+ARG BUILD_ENV=unarchive
+
+# hadolint ignore=DL3006
+FROM ubuntu AS unarchive
+# hadolint ignore=DL3010
+ONBUILD COPY hadoop-*.tar.gz /opt
+# hadolint ignore=DL3010
+ONBUILD COPY tez-*.tar.gz /opt
+
+# hadolint ignore=DL3006
+FROM ${BUILD_ENV} AS env
+ARG HADOOP_VERSION
+ARG TEZ_VERSION
+
+RUN mkdir -p /opt/hadoop \
+    && tar -xzv \
+        --exclude="hadoop-$HADOOP_VERSION/share/doc" \
+        --exclude="*/jdiff" \
+        --exclude="*/sources" \
+        --exclude="*tests.jar" \
+        --exclude="*/webapps" \
+        -f /opt/hadoop-$HADOOP_VERSION.tar.gz \
+        -C /opt/hadoop --strip-components 1 \
+    && mkdir -p /opt/tez \
+    && tar -xzv \
+        -f /opt/tez-$TEZ_VERSION.tar.gz \
+        -C /opt/tez \
+    && rm -rf /opt/hadoop-$HADOOP_VERSION.tar.gz /opt/tez-$TEZ_VERSION.tar.gz
+
+FROM eclipse-temurin:21-jdk-ubi9-minimal AS run
+
+ARG UID=1000
+ARG HADOOP_VERSION
+ARG TEZ_VERSION
+
+# Install dependencies
+# hadolint ignore=DL3041
+RUN set -ex; \
+    microdnf update -y; \
+    microdnf -y install procps gettext findutils hostname; \
+    microdnf clean all; \
+    useradd --no-create-home -s /sbin/nologin -c "" --uid $UID tez
+
+# Set necessary environment variables
+ENV HADOOP_HOME=/opt/hadoop \
+    TEZ_HOME=/opt/tez \
+    TEZ_CONF_DIR=/opt/tez/conf \
+    HADOOP_CONF_DIR=/opt/tez/conf
+
+ENV TEZ_CLIENT_VERSION=$TEZ_VERSION
+
+ENV PATH=$TEZ_HOME/bin:$HADOOP_HOME/bin:$PATH
+
+COPY --from=env --chown=tez /opt/hadoop $HADOOP_HOME
+# UPDATED: Copy from the normalized directory name created in 'env' stage
+COPY --from=env --chown=tez /opt/tez $TEZ_HOME
+
+RUN mkdir -p $TEZ_CONF_DIR && chown tez:tez $TEZ_CONF_DIR
+
+COPY --chown=tez tez-am-entrypoint.sh /
+COPY --chown=tez conf $TEZ_CONF_DIR
+
+# Create Extension Point Directory
+RUN mkdir -p /opt/tez/plugins && chown tez:tez /opt/tez/plugins && chmod 755 /opt/tez/plugins
+
+RUN chmod +x /tez-am-entrypoint.sh
+
+USER tez
+WORKDIR $TEZ_HOME
+
+# Expose AM ports via -p flag in docker command
+# EXPOSE 10001 10002 10003 8042
+
+ENTRYPOINT ["/tez-am-entrypoint.sh"]

tez-dist/src/docker/README.md

@@ -0,0 +1,95 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+# Tez AM Docker
+
+1. Building the docker image:
+
+    ```bash
+    mvn clean install -DskipTests -Pdocker
+    ```
+
+2. Install zookeeper in mac by:
+
+    ```bash
+    brew install zookeeper
+    zkServer start
+    ```
+
+3. Running the Tez AM container:
+
+    ```bash
+    export TEZ_VERSION=1.0.0-SNAPSHOT
+   
+    docker run --rm \
+        -p 10001:10001 \
+        --env-file tez-dist/src/docker/tez.env \
+        --name tez-am \
+        --hostname localhost \
+        apache/tez-am:$TEZ_VERSION
+    ```
+
+   * `TEZ_VERSION` corresponds to the Maven `${project.version}`.
+       Set this environment variable in your shell before running the commands.
+   * Expose ports using the `-p` flag based on the
+     `tez.am.client.am.port-range` property in `tez-site.xml`.
+   * The `--hostname` flag configures the container's hostname, allowing
+      services on the host (e.g., macOS) to connect to it.
+   * Ensure the `--env-file` flag is included, or at a minimum, pass
+     `-e TEZ_FRAMEWORK_MODE=STANDALONE_ZOOKEEPER` to the `docker run` command.
+
+4. Debugging the Tez AM container:
+Uncomment the `JAVA_TOOL_OPTIONS` in `tez.env` and expose 5005 port using `-p` flag
+
+    ```bash
+    docker run --rm \
+        -p 10001:10001 -p 5005:5005 \
+        --env-file tez-dist/src/docker/tez.env \
+        --name tez-am \
+        --hostname localhost \
+        apache/tez-am:$TEZ_VERSION
+    ```
+
+5. To override the tez-site.xml in docker image use:
+   * Set the `TEZ_CUSTOM_CONF_DIR` environment variable in `tez.env`
+      or via the `docker run` command (e.g., `/opt/tez/custom-conf`).
+
+    ```bash
+    export TEZ_SITE_PATH=$(pwd)/tez-dist/src/docker/conf/tez-site.xml
+
+    docker run --rm \
+        -p 10001:10001 \
+        --env-file tez-dist/src/docker/tez.env \
+        -v "$TEZ_SITE_PATH:/opt/tez/custom-conf/tez-site.xml" \
+        --name tez-am \
+        --hostname localhost \
+        apache/tez-am:$TEZ_VERSION
+    ```
+
+6. To add plugin jars in docker image use:
+   * The plugin directory path inside the Docker container is fixed at `/opt/tez/plugins`.
+
+    ```bash
+    docker run --rm \
+        -p 10001:10001 \
+        --env-file tez-dist/src/docker/tez.env \
+        -v "/path/to/your/local/plugins:/opt/tez/plugins" \
+        --name tez-am \
+        --hostname localhost \
+        apache/tez-am:$TEZ_VERSION
+    ```

tez-dist/src/docker/build-docker.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -xeou pipefail
+
+HADOOP_VERSION=
+TEZ_VERSION=
+REPO=
+
+usage() {
+  cat <<EOF 1>&2
+Usage: $0 [-h] [-hadoop <Hadoop version>] [-tez <Tez version>] [-repo <Docker repo>]
+Build the Apache Tez AM Docker image
+-help                Display help
+-hadoop              Build image with the specified Hadoop version
+-tez                 Build image with the specified Tez version
+-repo                Docker repository
+EOF
+}
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+  -h)
+    usage
+    exit 0
+    ;;
+  -hadoop)
+    shift
+    HADOOP_VERSION=$1
+    shift
+    ;;
+  -tez)
+    shift
+    TEZ_VERSION=$1
+    shift
+    ;;
+  -repo)
+    shift
+    REPO=$1
+    shift
+    ;;
+  *)
+    shift
+    ;;
+  esac
+done
+
+SCRIPT_DIR=$(
+  cd "$(dirname "$0")"
+  pwd
+)
+
+DIST_DIR=${DIST_DIR:-"$SCRIPT_DIR/../.."}
+PROJECT_ROOT=${PROJECT_ROOT:-"$SCRIPT_DIR/../../.."}
+
+REPO=${REPO:-apache}
+WORK_DIR="$(mktemp -d)"
+CACHE_DIR="$SCRIPT_DIR/cache"
+mkdir -p "$CACHE_DIR"
+
+# Defaults Hadoop and Tez versions from pom.xml if not provided
+HADOOP_VERSION=${HADOOP_VERSION:-$(mvn -f "$PROJECT_ROOT/pom.xml" -q help:evaluate -Dexpression=hadoop.version -DforceStdout)}
+TEZ_VERSION=${TEZ_VERSION:-$(mvn -f "$PROJECT_ROOT/pom.xml" -q help:evaluate -Dexpression=project.version -DforceStdout)}
+
+######################
+# HADOOP FETCH LOGIC #
+######################
+HADOOP_FILE_NAME="hadoop-$HADOOP_VERSION.tar.gz"
+HADOOP_URL=${HADOOP_URL:-"https://dlcdn.apache.org/hadoop/common/hadoop-$HADOOP_VERSION/$HADOOP_FILE_NAME"}
+if [ ! -f "$CACHE_DIR/$HADOOP_FILE_NAME" ]; then
+  echo "Downloading Hadoop from $HADOOP_URL..."
+  if ! curl --fail -L "$HADOOP_URL" -o "$CACHE_DIR/$HADOOP_FILE_NAME.tmp"; then
+    echo "Fail to download Hadoop, exiting...."
+    exit 1
+  fi
+  mv "$CACHE_DIR/$HADOOP_FILE_NAME.tmp" "$CACHE_DIR/$HADOOP_FILE_NAME"
+fi
+
+#####################################
+# Pick tez tarball from local build #
+#####################################
+TEZ_FILE_NAME="tez-$TEZ_VERSION.tar.gz"
+LOCAL_DIST_PATH="$DIST_DIR/target/$TEZ_FILE_NAME"
+
+if [ -f "$LOCAL_DIST_PATH" ]; then
+  echo "--> Found local Tez build artifact at: $LOCAL_DIST_PATH"
+  cp "$LOCAL_DIST_PATH" "$WORK_DIR/"
+else
+  echo "--> Error: Local Tez artifact not found at $LOCAL_DIST_PATH"
+  echo "--> Please build the project first (e.g., mvn clean install -DskipTests)."
+  exit 1
+fi
+
+# -------------------------------------------------------------------------
+# BUILD CONTEXT PREPARATION
+# -------------------------------------------------------------------------
+cp "$CACHE_DIR/$HADOOP_FILE_NAME" "$WORK_DIR/"
+cp -R "$SCRIPT_DIR/conf" "$WORK_DIR/" 2>/dev/null || mkdir -p "$WORK_DIR/conf"
+cp "$SCRIPT_DIR/tez-am-entrypoint.sh" "$WORK_DIR/"
+cp "$SCRIPT_DIR/Dockerfile" "$WORK_DIR/"
+
+echo "Building Docker image..."
+docker build \
+  "$WORK_DIR" \
+  -f "$WORK_DIR/Dockerfile" \
+  -t "$REPO/tez-am:$TEZ_VERSION" \
+  --build-arg "BUILD_ENV=unarchive" \
+  --build-arg "HADOOP_VERSION=$HADOOP_VERSION" \
+  --build-arg "TEZ_VERSION=$TEZ_VERSION"
+
+rm -r "${WORK_DIR}"
+echo "Docker image $REPO/tez-am:$TEZ_VERSION built successfully."

tez-dist/src/docker/conf/log4j2.properties

@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n
+
+rootLogger.level = INFO
+rootLogger.appenderRef.console.ref = console