Update rel-2300 in order to build Dremio arrow 9.0 jars.

.github/workflows/java_jni.yml

@@ -47,7 +47,7 @@ env:
 jobs:
 
  docker:
- name: AMD64 Debian 9 Java JNI (Gandiva, Plasma, ORC, Dataset)
+ name: AMD64 manylinux2014 Java JNI
  runs-on: ubuntu-latest
  if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
  timeout-minutes: 90
@@ -61,23 +61,23 @@ jobs:
  uses: actions/cache@v2
  with:
  path: .docker
- key: maven-${{ hashFiles('java/**') }}
- restore-keys: maven-
+ key: java-jni-manylinux-2014-${{ hashFiles('cpp/**', 'java/**') }}
+ restore-keys: java-jni-manylinux-2014-
  - name: Setup Python
  uses: actions/setup-python@v4
  with:
  python-version: 3.8
  - name: Setup Archery
  run: pip install -e dev/archery[docker]
  - name: Execute Docker Build
- run: archery docker run debian-java-jni
+ run: archery docker run java-jni-manylinux-2014
  - name: Docker Push
  if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
  continue-on-error: true
- run: archery docker push debian-java-jni
+ run: archery docker push java-jni-manylinux-2014
 
  docker_integration_python:
- name: AMD64 Debian 9 Java C Data Interface Integration
+ name: AMD64 Conda Java C Data Interface Integration
  runs-on: ubuntu-latest
  if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
  timeout-minutes: 90

ci/docker/java-jni-manylinux-201x.dockerfile

@@ -33,5 +33,13 @@ RUN vcpkg install \
 
 # Install Java
 ARG java=1.8.0
-RUN yum install -y java-$java-openjdk-devel && yum clean all
+RUN yum install -y java-$java-openjdk-devel rh-maven35 && yum clean all
 ENV JAVA_HOME=/usr/lib/jvm/java-$java-openjdk/
+
+# For ci/scripts/java_*.sh
+ENV ARROW_GANDIVA_JAVA=ON \
+ ARROW_HOME=/tmp/local \
+ ARROW_JAVA_CDATA=ON \
+ ARROW_JNI=ON \
+ ARROW_PLASMA=ON \
+ ARROW_USE_CCACHE=ON

ci/docker/java-jni-manylinux-aarch64-201x.dockerfile

@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG base
+FROM ${base}
+
+# Install the libaries required by the Gandiva to run
+# Use enable llvm[enable-rtti] in the vcpkg.json to avoid link problems in Gandiva
+RUN vcpkg install \
+ --clean-after-build \
+ --x-install-root=${VCPKG_ROOT}/installed \
+ --x-manifest-root=/arrow/ci/vcpkg \
+ --x-feature=flight \
+ --x-feature=gcs \
+ --x-feature=json \
+ --x-feature=parquet \
+ --x-feature=gandiva \
+ --x-feature=s3
+
+# Install Java
+ARG java=1.8.0
+RUN yum install -y java-$java-openjdk-devel && yum clean all
+ENV JAVA_HOME=/usr/lib/jvm/java-$java-openjdk/

ci/scripts/java_build.sh

@@ -18,10 +18,13 @@
 
 set -ex
 
+if [[ "${ARROW_JAVA_BUILD:-ON}" != "ON" ]]; then
+ exit
+fi
+
 arrow_dir=${1}
 source_dir=${1}/java
 build_dir=${2}
-cpp_build_dir=${build_dir}/cpp/${ARROW_BUILD_TYPE:-debug}
 java_jni_dist_dir=${3}
 
 : ${BUILD_DOCS_JAVA:=OFF}
@@ -80,7 +83,7 @@ if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then
 fi
 
 if [ "${ARROW_GANDIVA_JAVA}" = "ON" ]; then
- ${mvn} -Darrow.cpp.build.dir=${cpp_build_dir} -Parrow-jni install
+ ${mvn} -Darrow.cpp.build.dir=${java_jni_dist_dir} -Parrow-jni install
 fi
 
 if [ "${ARROW_PLASMA}" = "ON" ]; then

ci/scripts/java_gandiva_build.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+arrow_dir=${1}
+dist_dir=${2}
+
+export ARROW_TEST_DATA=${arrow_dir}/testing/data
+
+pushd ${arrow_dir}/java
+
+# Ensure that there is no old jar
+# inside the maven repository
+maven_repo=~/.m2/repository/org/apache/arrow
+if [ -d $maven_repo ]; then
+ find $maven_repo \
+ "(" -name "*.jar" -o -name "*.zip" -o -name "*.pom" ")" \
+ -exec echo {} ";" \
+ -exec rm -rf {} ";"
+fi
+
+# generate dummy GPG key for -Papache-release.
+# -Papache-release generates signs (*.asc) of artifacts.
+# We don't use these signs in our release process.
+(echo "Key-Type: RSA"; \
+ echo "Key-Length: 4096"; \
+ echo "Name-Real: Build"; \
+ echo "Name-Email: [email protected]"; \
+ echo "%no-protection") | \
+ gpg --gen-key --batch
+
+# build the entire project
+mvn clean \
+ install \
+ assembly:single \
+ source:jar \
+ javadoc:jar \
+ -DskipTests \
+ -Papache-release \
+ -Parrow-c-data \
+ -Parrow-jni \
+ -Darrow.cpp.build.dir=$dist_dir \
+ -Darrow.c.jni.dist.dir=$dist_dir \
+ -DdescriptorId=source-release \
+ --projects gandiva \
+ --also-make
+
+# copy all jar, zip and pom files to the distribution folder
+find . \
+ "(" -name "*-javadoc.jar" -o -name "*-sources.jar" ")" \
+ -exec echo {} ";" \
+ -exec cp {} $dist_dir ";"
+find ~/.m2/repository/org/apache/arrow \
+ "(" -name "*.jar" -o -name "*.zip" -o -name "*.pom" ")" \
+ -exec echo {} ";" \
+ -exec cp {} $dist_dir ";"
+
+popd

ci/scripts/java_jni_build.sh

@@ -24,6 +24,8 @@ build_dir=${2}/java_jni
 # The directory where the final binaries will be stored when scripts finish
 dist_dir=${3}
 
+prefix_dir="${build_dir}/java-jni" 
+
 echo "=== Clear output directories and leftovers ==="
 # Clear output directories and leftovers
 rm -rf ${build_dir}
@@ -40,3 +42,11 @@ cmake \
  ${arrow_dir}/java
 cmake --build . --target install --config ${ARROW_BUILD_TYPE:-release}
 popd
+
+mkdir -p ${dist_dir} 
+# For Windows. *.dll are installed into bin/ on Windows. 
+if [ -d "${prefix_dir}/bin" ]; then 
+ mv ${prefix_dir}/bin/* ${dist_dir}/ 
+else 
+ mv ${prefix_dir}/lib/* ${dist_dir}/ 
+fi

ci/scripts/java_jni_macos_build.sh

@@ -21,14 +21,24 @@ set -ex
 
 arrow_dir=${1}
 build_dir=${2}
+normalized_arch=$(arch)
+case ${normalized_arch} in
+ arm64)
+ normalized_arch=aarch_64
+ ;;
+ i386)
+ normalized_arch=x86_64
+ ;;
+esac
 # The directory where the final binaries will be stored when scripts finish
-dist_dir=${3}
+dist_dir=${3}/${normalized_arch}
 
 echo "=== Clear output directories and leftovers ==="
 # Clear output directories and leftovers
 rm -rf ${build_dir}
 
 echo "=== Building Arrow C++ libraries ==="
+install_dir=${build_dir}/cpp-install
 : ${ARROW_BUILD_TESTS:=OFF}
 : ${ARROW_DATASET:=ON}
 : ${ARROW_FILESYSTEM:=ON}
@@ -40,9 +50,15 @@ echo "=== Building Arrow C++ libraries ==="
 : ${ARROW_PLASMA:=ON}
 : ${ARROW_PYTHON:=OFF}
 : ${ARROW_S3:=ON}
+: ${ARROW_USE_CCACHE:=OFF}
 : ${CMAKE_BUILD_TYPE:=Release}
 : ${CMAKE_UNITY_BUILD:=ON}
 
+if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
+ echo "=== ccache statistics before build ==="
+ ccache -s
+fi
+
 export ARROW_TEST_DATA="${arrow_dir}/testing/data"
 export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data"
 export AWS_EC2_METADATA_DISABLED=TRUE
@@ -75,12 +91,13 @@ cmake \
  -DARROW_S3=${ARROW_S3} \
  -DARROW_SNAPPY_USE_SHARED=OFF \
  -DARROW_THRIFT_USE_SHARED=OFF \
+ -DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \
  -DARROW_UTF8PROC_USE_SHARED=OFF \
  -DARROW_ZSTD_USE_SHARED=OFF \
  -DAWSSDK_SOURCE=BUNDLED \
  -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
  -DCMAKE_INSTALL_LIBDIR=lib \
- -DCMAKE_INSTALL_PREFIX=${build_dir}/cpp \
+ -DCMAKE_INSTALL_PREFIX=${install_dir} \
  -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
  -DPARQUET_BUILD_EXAMPLES=OFF \
  -DPARQUET_BUILD_EXECUTABLES=OFF \
@@ -102,12 +119,17 @@ ${arrow_dir}/ci/scripts/java_jni_build.sh \
  ${build_dir} \
  ${dist_dir}
 
+if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
+ echo "=== ccache statistics after build ==="
+ ccache -s
+fi
 
 echo "=== Copying libraries to the distribution folder ==="
 mkdir -p "${dist_dir}"
-cp -L ${build_dir}/cpp/lib/libgandiva_jni.dylib ${dist_dir}
-cp -L ${build_dir}/cpp/lib/libarrow_dataset_jni.dylib ${dist_dir}
-cp -L ${build_dir}/cpp/lib/libarrow_orc_jni.dylib ${dist_dir}
+cp -L ${install_dir}/lib/libarrow_dataset_jni.dylib ${dist_dir}
+cp -L ${install_dir}/lib/libarrow_orc_jni.dylib ${dist_dir}
+cp -L ${install_dir}/lib/libgandiva_jni.dylib ${dist_dir}
+cp -L ${build_dir}/cpp/*/libplasma_java.dylib ${dist_dir}
 
 echo "=== Checking shared dependencies for libraries ==="
 
@@ -122,9 +144,11 @@ archery linking check-dependencies \
  --allow libcurl \
  --allow libgandiva_jni \
  --allow libncurses \
+ --allow libplasma_java \
  --allow libz \
  libarrow_cdata_jni.dylib \
  libarrow_dataset_jni.dylib \
  libarrow_orc_jni.dylib \
- libgandiva_jni.dylib
+ libgandiva_jni.dylib \
+ libplasma_java.dylib
 popd