razvan · Oct 28, 2026 · Oct 26, 2026 · Oct 26, 2026 · Oct 26, 2026 · Oct 26, 2026
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -33,6 +33,7 @@ All notable changes to this project will be documented in this file.
 - opa: Add 1.20.2 ([#1697]).
 - patchable: Use long hashes, and myers diff algorithm for portability ([#1705]).
 - spark: Add 4.1.3 and 4.2.0 ([#1710]).
+- spark: Add HBase connector images for Spark 4.x. There is no connector release that supports Spark 4.x yet, so this is built from an upstream commit (`07f5791`) on the master branch ([#1723])
 
 ### Changed
 
@@ -184,6 +185,7 @@ All notable changes to this project will be documented in this file.
 [#1718]: https://github.com/stackabletech/docker-images/pull/1718
 [#1720]: https://github.com/stackabletech/docker-images/pull/1720
 [#1721]: https://github.com/stackabletech/docker-images/pull/1721
+[#1723]: https://github.com/stackabletech/docker-images/pull/1723
 [#1725]: https://github.com/stackabletech/docker-images/pull/1725
 
 ## [26.7.0] - 2026-07-21

diff --git a/spark-k8s/Dockerfile.4 b/spark-k8s/Dockerfile.4
@@ -4,6 +4,9 @@
 # hadoop-builder: Provides Hadoop libraries
 FROM local-image/hadoop/hadoop AS hadoop-builder
 
+# hbase-builder: Provides HBase libraries
+FROM local-image/hbase AS hbase-builder
+
 # spark-source-builder: Clone the Spark source code and apply patches
 FROM local-image/java-devel AS spark-source-builder
 
@@ -28,6 +31,10 @@ tar -czf /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-src.tar
 chmod g=u /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-src.tar.gz
 EOF
 
+# hbase-connectors-builder: Build the Spark HBase connector and copy
+# required JARs into /stackable/spark/jars
+FROM local-image/spark-k8s/hbase-connectors AS hbase-connectors-builder
+
 # spark-builder: Build Spark into /stackable/spark-${PRODUCT_VERSION}/dist,
 # download additional JARs and perform checks
 FROM local-image/java-devel AS spark-builder
@@ -36,6 +43,7 @@ ARG PRODUCT_VERSION
 ARG HADOOP_HADOOP_VERSION
 # Reassign the arg to `HADOOP_VERSION` for better readability.
 ENV HADOOP_VERSION=${HADOOP_HADOOP_VERSION}
+ARG HBASE_VERSION
 ARG JACKSON_DATAFORMAT_XML_VERSION
 ARG STAX2_API_VERSION
 ARG WOODSTOX_CORE_VERSION
@@ -136,6 +144,22 @@ WORKDIR /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/jar
 # Copy cloud libraries for s3a:// and abfs:// support
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/hadoop-cloud-libraries/*.jar ./
 
+# Copy the HBase connector including required modules
+COPY --from=hbase-connectors-builder --chown=${STACKABLE_USER_UID}:0 \
+    /stackable/spark/jars/* \
+    ./
+
+# Copy modules required to access HBase
+COPY --from=hbase-builder --chown=${STACKABLE_USER_UID}:0 \
+    /stackable/hbase/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${HBASE_VERSION}-stackable${RELEASE_VERSION}.jar \
+    /stackable/hbase/lib/shaded-clients/hbase-shaded-mapreduce-${HBASE_VERSION}-stackable${RELEASE_VERSION}.jar \
+    ./
+COPY --from=hbase-builder --chown=${STACKABLE_USER_UID}:0 \
+    /stackable/hbase/lib/client-facing-thirdparty/opentelemetry-api-*.jar \
+    /stackable/hbase/lib/client-facing-thirdparty/opentelemetry-context-*.jar \
+    /stackable/hbase/lib/client-facing-thirdparty/opentelemetry-semconv-*-alpha.jar \
+    ./
+
 COPY spark-k8s/stackable/jmx /stackable/jmx
 
 WORKDIR /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/extra-jars
@@ -191,6 +215,7 @@ ARG PRODUCT_VERSION
 ARG PYTHON_VERSION
 ARG RELEASE_VERSION
 ARG JMX_EXPORTER_VERSION
+ARG HBASE_CONNECTOR_VERSION
 ARG STACKABLE_USER_UID
 
 LABEL name="Apache Spark" \
@@ -213,6 +238,7 @@ ENV PYTHONPATH=$SPARK_HOME/python
 
 COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-builder /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist /stackable/spark
 COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-source-builder /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-src.tar.gz /stackable
+COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-connectors-builder /stackable/hbase-connector-${HBASE_CONNECTOR_VERSION}-stackable${RELEASE_VERSION}-src.tar.gz /stackable
 COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-builder /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/assembly/target/bom.json /stackable/spark/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}.cdx.json
 COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-builder /stackable/spark-web-ui-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}.cdx.json /stackable/spark/spark-web-ui-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}.cdx.json
 COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-builder /stackable/jmx /stackable/jmx

diff --git a/spark-k8s/boil-config.toml b/spark-k8s/boil-config.toml
@@ -8,6 +8,8 @@ containerfile = "Dockerfile.4"
 "hadoop/hadoop" = "3.4.3"
 java-base = "21"
 java-devel = "21"
+hbase = "2.6.7"
+"spark-k8s/hbase-connectors" = "1.1.0-SNAPSHOT-07f5791_4.1.2"
 
 [versions."4.1.2".build-arguments]
 python-version = "3.12"
@@ -16,6 +18,7 @@ stax2-api-version = "4.2.2"               # Needs to match https://mvnrepository
 woodstox-core-version = "7.1.1"           # Needs to match https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.21.2
 jmx-exporter-version = "1.3.0"
 tini-version = "0.19.0"
+hbase-connector-version = "1.1.0-SNAPSHOT-07f5791_4.1.2"
 openlineage-spark-version = "1.54.0"      # Scala 2.13 build for Spark 4.x; see https://mvnrepository.com/artifact/io.openlineage/openlineage-spark_2.13
 
 # LTS since 26.11.0
@@ -26,6 +29,8 @@ containerfile = "Dockerfile.4"
 "hadoop/hadoop" = "3.4.3"
 java-base = "21"
 java-devel = "21"
+hbase = "2.6.7"
+"spark-k8s/hbase-connectors" = "1.1.0-SNAPSHOT-07f5791_4.1.3"
 
 [versions."4.1.3".build-arguments]
 python-version = "3.12"
@@ -34,6 +39,7 @@ stax2-api-version = "4.2.2"               # Needs to match https://mvnrepository
 woodstox-core-version = "7.1.1"           # Needs to match https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.21.2
 jmx-exporter-version = "1.3.0"
 tini-version = "0.19.0"
+hbase-connector-version = "1.1.0-SNAPSHOT-07f5791_4.1.3"
 openlineage-spark-version = "1.54.0"      # Scala 2.13 build for Spark 4.x; see https://mvnrepository.com/artifact/io.openlineage/openlineage-spark_2.13
 
 [versions."4.2.0"]
@@ -43,6 +49,8 @@ containerfile = "Dockerfile.4"
 "hadoop/hadoop" = "3.5.0"
 java-base = "25"
 java-devel = "25"
+hbase = "2.6.7"
+"spark-k8s/hbase-connectors" = "1.1.0-SNAPSHOT-07f5791_4.2.0"
 
 [versions."4.2.0".build-arguments]
 python-version = "3.14"
@@ -51,4 +59,5 @@ stax2-api-version = "4.2.2"               # Needs to match https://mvnrepository
 woodstox-core-version = "7.1.1"           # Needs to match https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.21.2
 jmx-exporter-version = "1.3.0"
 tini-version = "0.19.0"
+hbase-connector-version = "1.1.0-SNAPSHOT-07f5791_4.2.0"
 openlineage-spark-version = "1.54.0"      # Scala 2.13 build for Spark 4.x; see https://mvnrepository.com/artifact/io.openlineage/openlineage-spark_2.13
diff --git a/spark-k8s/hbase-connectors/Dockerfile b/spark-k8s/hbase-connectors/Dockerfile
@@ -55,8 +55,7 @@ ARG RELEASE_VERSION
 ARG HADOOP_VERSION
 ARG HBASE_VERSION
 ARG HBASE_THIRDPARTY_VERSION
-ARG PROTOBUF_VERSION
-ARG PROTOBUF_PLUGIN_VERSION
+ARG THIRDPARTY_PROTOBUF_VERSION
 ARG STACKABLE_USER_UID
 
 # Patch the hbase-connectors source code
@@ -74,27 +73,21 @@ COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-source-builder \
     spark/env
 
 RUN <<EOF
-cd "$(/stackable/patchable --images-repo-root=src checkout spark-k8s/hbase-connectors ${PRODUCT_VERSION})"/spark
+cd "$(/stackable/patchable --images-repo-root=src checkout spark-k8s/hbase-connectors ${PRODUCT_VERSION})"
 
 NEW_VERSION="${PRODUCT_VERSION}-stackable${RELEASE_VERSION}"
 
-mvn "org.codehaus.mojo:versions-maven-plugin:${VERSIONS_MAVEN_PLUGIN_VERSION}:set" -DnewVersion=$NEW_VERSION
-
 # Create snapshot of the source code including custom patches
 tar -czf /stackable/hbase-connector-${NEW_VERSION}-src.tar.gz .
 
-# Building the hbase-connectors with JDK 17 is not yet supported, see
-# https://github.com/apache/hbase-connectors/pull/132.
-# As there are no JDK profiles, access to the non-public elements must
-# be enabled with --add-opens, see https://openjdk.org/jeps/403 and
-# https://openjdk.org/jeps/261#Breaking-encapsulation.
-export JDK_JAVA_OPTIONS="\
-    --add-opens java.base/java.lang=ALL-UNNAMED \
-    --add-opens java.base/java.util=ALL-UNNAMED"
-
 . /stackable/spark/env
 
-# Build the Spark HBase connector
+# Build the Spark 4 HBase connector.
+# The `spark4` modules are not part of the default reactor, hence the profile.
+# The Scala binary version is deliberately not overridden: it must stay 2.13 for the spark4
+# modules and 2.12 for the spark ones, otherwise both reactors produce the same
+# hbase-spark-pushdown artifact and Maven rejects the build as a duplicate.
+# `revision` is the project version (CI friendly version).
 # Skip the tests because the MiniHBaseCluster does not get ready for
 # whatever reason:
 #   Caused by: java.lang.RuntimeException: Master not active after 30000ms
@@ -105,23 +98,25 @@ export JDK_JAVA_OPTIONS="\
 mvn \
     --batch-mode \
     --no-transfer-progress \
-    --define spark.version="${SPARK_VERSION}" \
+    --activate-profiles spark4 \
+    --projects spark4/hbase-spark4 \
+    --also-make \
+    --define revision="${NEW_VERSION}" \
+    --define spark4.version="${SPARK_VERSION}" \
     --define scala.version="${SCALA_VERSION}" \
-    --define scala.binary.version="${SCALA_BINARY_VERSION}" \
     --define hadoop-three.version="${HADOOP_VERSION}" \
     --define hbase.version="${HBASE_VERSION}" \
     --define hbase-thirdparty.version="${HBASE_THIRDPARTY_VERSION}" \
-    --define external.protobuf.version="${PROTOBUF_VERSION}" \
-    --define protobuf.plugin.version="${PROTOBUF_PLUGIN_VERSION}" \
+    --define thirdparty.protobuf.version="${THIRDPARTY_PROTOBUF_VERSION}" \
     --define skipTests \
     --define maven.test.skip=true \
     clean package
 
-
-cp "$(pwd)/hbase-spark/target/hbase-spark-${NEW_VERSION}.jar" /stackable/spark/jars/hbase-spark-${NEW_VERSION}.jar
-# The hbase-spark JAR is a plain module JAR and does not bundle its
-# dependencies. The generated protobuf classes live here.
-cp "$(pwd)/hbase-spark-protocol-shaded/target/hbase-spark-protocol-shaded-${NEW_VERSION}.jar" /stackable/spark/jars/hbase-spark-protocol-shaded-${NEW_VERSION}.jar
+cp "$(pwd)/spark4/hbase-spark4/target/hbase-spark4-${NEW_VERSION}.jar" /stackable/spark/jars/
+# hbase-spark4 is a plain module JAR and does not bundle its dependencies, so ship the
+# modules it needs at runtime: the shared SQL pushdown filters and the generated protobuf classes.
+cp "$(pwd)/spark4/hbase-spark-pushdown/target/hbase-spark-pushdown_${SCALA_BINARY_VERSION}-${NEW_VERSION}.jar" /stackable/spark/jars/
+cp "$(pwd)/spark/hbase-spark-protocol-shaded/target/hbase-spark-protocol-shaded-${NEW_VERSION}.jar" /stackable/spark/jars/
 
 chmod g=u /stackable/hbase-connector-${NEW_VERSION}-src.tar.gz .
 EOF
diff --git a/spark-k8s/hbase-connectors/boil-config.toml b/spark-k8s/hbase-connectors/boil-config.toml
@@ -1,16 +1,41 @@
-[versions."1.0.1_3.5.8".local-images]
-java-devel = "17"
+# Unreleased upstream code: there is no hbase-connectors release that supports Spark 4.x yet.
+# The `spark4` modules (HBASE-30179 and follow-ups) are only on the master branch, so each
+# version is pinned to an upstream commit (see the `base` of the patchable.toml of that version).
+# The version is `<upstream revision>-<short commit sha>_<Spark version>`.
 
-[versions."1.0.1_3.5.8".build-arguments]
-spark-version = "3.5.8"
-# TODO: Bump to 3.4.3. Note that this causes compilation errors that need to be fixed.
-hadoop-version = "3.4.2"
-hbase-version = "2.6.6"
-# The following three must match the hbase-thirdparty (and its bundled protobuf)
+[versions."1.1.0-SNAPSHOT-07f5791_4.1.2".local-images]
+java-devel = "21"
+
+[versions."1.1.0-SNAPSHOT-07f5791_4.1.2".build-arguments]
+spark-version = "4.1.2"
+hadoop-version = "3.4.3"
+hbase-version = "2.6.7"
+# The following two must match the hbase-thirdparty (and its bundled protobuf)
 # shipped by the hbase-shaded-client in the final Spark image. That client comes
-# from the `hbase` local-image in spark-k8s/boil-config.toml (currently 2.6.6),
-# whose hbase-thirdparty is 4.1.13 -> protobuf 4.34.0. Keep these in sync if the
+# from the `hbase` local-image in spark-k8s/boil-config.toml (currently 2.6.7),
+# whose hbase-thirdparty is 4.1.14 -> protobuf 4.34.0. Keep these in sync if the
 # Spark image's HBASE_VERSION changes.
-hbase-thirdparty-version = "4.1.13"
-protobuf-version = "4.34.0"
-protobuf-plugin-version = "0.6.1"
+hbase-thirdparty-version = "4.1.14"
+thirdparty-protobuf-version = "4.34.0"
+
+[versions."1.1.0-SNAPSHOT-07f5791_4.1.3".local-images]
+java-devel = "21"
+
+[versions."1.1.0-SNAPSHOT-07f5791_4.1.3".build-arguments]
+spark-version = "4.1.3"
+hadoop-version = "3.4.3"
+hbase-version = "2.6.7"
+# See the comment above.
+hbase-thirdparty-version = "4.1.14"
+thirdparty-protobuf-version = "4.34.0"
+
+[versions."1.1.0-SNAPSHOT-07f5791_4.2.0".local-images]
+java-devel = "25"
+
+[versions."1.1.0-SNAPSHOT-07f5791_4.2.0".build-arguments]
+spark-version = "4.2.0"
+hadoop-version = "3.5.0"
+hbase-version = "2.6.7"
+# See the comment above.
+hbase-thirdparty-version = "4.1.14"
+thirdparty-protobuf-version = "4.34.0"
diff --git a/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.8/0001-Fix-protobuf-on-aarch64.patch b/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.8/0001-Fix-protobuf-on-aarch64.patch
diff --git a/...ckable/patches/1.0.1_3.5.8/patchable.toml → ...1.0-SNAPSHOT-07f5791_4.1.2/patchable.toml b/...ckable/patches/1.0.1_3.5.8/patchable.toml → ...1.0-SNAPSHOT-07f5791_4.1.2/patchable.toml
@@ -1,2 +1,2 @@
-base = "e5217d13ed729703580ff2d1b02378ada2d94f4d"
 mirror = "https://github.com/stackabletech/hbase-connectors.git"
+base = "07f5791c4921e471d328c8ce5f760ac72f0090cb"
diff --git a/spark-k8s/hbase-connectors/stackable/patches/1.1.0-SNAPSHOT-07f5791_4.1.3/patchable.toml b/spark-k8s/hbase-connectors/stackable/patches/1.1.0-SNAPSHOT-07f5791_4.1.3/patchable.toml
@@ -0,0 +1,2 @@
+mirror = "https://github.com/stackabletech/hbase-connectors.git"
+base = "07f5791c4921e471d328c8ce5f760ac72f0090cb"
diff --git a/spark-k8s/hbase-connectors/stackable/patches/1.1.0-SNAPSHOT-07f5791_4.2.0/patchable.toml b/spark-k8s/hbase-connectors/stackable/patches/1.1.0-SNAPSHOT-07f5791_4.2.0/patchable.toml
@@ -0,0 +1,2 @@
+mirror = "https://github.com/stackabletech/hbase-connectors.git"
+base = "07f5791c4921e471d328c8ce5f760ac72f0090cb"
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		mirror = "https://github.com/stackabletech/hbase-connectors.git"
		base = "07f5791c4921e471d328c8ce5f760ac72f0090cb"